diff --git a/.gitattributes b/.gitattributes index d6e22db5a9a134bc1aab0e0b33df2e966cbd0008..6f0b63b90e4ff191dbb91ddaf09986f1a0ca179a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log filter=lfs diff=lfs merge=lfs -text tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text +tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/train.log filter=lfs diff=lfs merge=lfs -text +tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text diff --git a/tierb_study_20260826_144644/echo/tierbprobe/manifest.txt b/tierb_study_20260826_144644/echo/tierbprobe/manifest.txt index 8ba7405c0bf2400736da38cccb33256aa4b11b05..29cfbb727d6ef4c822e7771167bbbc1237221cb1 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/manifest.txt +++ b/tierb_study_20260826_144644/echo/tierbprobe/manifest.txt @@ -1,9 +1,9 @@ -timestamp: 2026-08-26T16:16:58+00:00 +timestamp: 2026-08-26T16:26:06+00:00 host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 gpu: NVIDIA B200 x8 driver: 580.95.05 study_scripts_branch: granularity-echo -study_scripts_commit: b99bce1 +study_scripts_commit: 6e4b1c8 campaign: echo backend: echo phase: tierbprobe diff --git a/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt b/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt index 4a672cd661596398d5d2133f77b52b1e48ffbdda..8cd33d3ef4c004e0cb511d96c7252ad7d384774a 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt +++ b/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt @@ -27,3 +27,64 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_ 7, 120 MHz, 1965 MHz, 1000.00 W, 28 NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 + python : 3.12.3 + torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 + torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] + transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] + deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] + ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] + moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] + megatron : 0.14.0rc5 [None] + flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] + numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] + huggingface_hub: 1.28.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] + +---- repositories -------------------------------------------- + Megatron-MoE-Echo token-drop-analysis b099fd8494b0 + Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21 + UltraEP main 94cab099b44f + DeepEP HEAD 34152ae28f80 (+3 untracked: build output) + MoonEP megatron-integration c27d35da15f2 + blackwell_run_scripts granularity-echo 6e4b1c8e7fae + +---- environment --------------------------------------------- + CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 + CUDA_DRIVER_VERSION=575.57.08 + CUDA_HOME=/usr/local/cuda + CUDA_MODULE_LOADING=LAZY + CUDA_VERSION=12.9.1.010 + HF_REMOTE_ROOT=tierb_study_20260826_144644 + NCCL_VERSION=2.27.3 + NVSHMEM_VERSION=3.2.5 + PYTORCH_BUILD_NUMBER=0 + PYTORCH_BUILD_VERSION=2.8.0a0+5228986 + PYTORCH_HOME=/opt/pytorch/pytorch + PYTORCH_VERSION=2.8.0a0+5228986 + TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 + TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX + TORCH_NCCL_USE_COMM_NONBLOCKING=0 + +---- shared library resolution ------------------------------- + libnvshmem_host.so: not in ldconfig + libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 + lrwxrwxrwx 1 root root 20 Aug 26 14:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 + -rw-r--r-- 1 root root 179045744 Aug 26 14:09 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..f446f5f24a03b479bf8d92c5fe17ebfc4269da3d --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,48 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 27 +0, 819 +1, 817 +2, 799 +3, 799 +4, 801 +5, 801 +6, 803 +7, 805 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/run.env b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..8566fc2e008c8ad1b0f462b3d117ee19bcb31b3d --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d26e1c239d02d4ff0c5d2ebf4a51f59830de129f --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/train.log @@ -0,0 +1,620 @@ +[16:35:35] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + DSV2_LITE_L24 / echo (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0826 16:35:36.740000 237450 torch/distributed/run.py:766] +W0826 16:35:36.740000 237450 torch/distributed/run.py:766] ***************************************** +W0826 16:35:36.740000 237450 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:35:36.740000 237450 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5 is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : [Gloo] Rank 000 + is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0[Gloo] Rank is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W826 16:35:54.040729199 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 16:35:54.040744910 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:35:54.042537371 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 16:35:55.045097105 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 16:35:55.045219285 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 16:35:55.045217490 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:35:55.045741026 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 16:35:55.205831838 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv index 164c4d83a6d285f25d4be27d7e47e2e0c09d0a60..2314d4a706f8aa1df216eea153d884c86d57be49 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 2 +0, 1 1, 1 2, 1 3, 1 @@ -36,24 +36,24 @@ 3, 4 4, 4 5, 4 -6, 1 -7, 1 -0, 4 -1, 45 -2, 4 -3, 4 -4, 4 -5, 4 -6, 27 +6, 4 7, 4 -0, 821 -1, 821 -2, 821 -3, 821 -4, 821 -5, 821 -6, 821 -7, 821 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 47 +7, 47 +0, 975 +1, 959 +2, 949 +3, 979 +4, 955 +5, 951 +6, 953 +7, 957 0, 1955 1, 1955 2, 1955 @@ -70,22 +70,22 @@ 5, 26223 6, 26355 7, 26377 -0, 26887 -1, 27039 -2, 26773 -3, 26885 -4, 26885 -5, 26885 -6, 26953 -7, 26927 -0, 36511 -1, 36595 -2, 36445 -3, 36443 -4, 36443 -5, 36441 -6, 36577 -7, 36595 +0, 32143 +1, 32229 +2, 32079 +3, 32077 +4, 32077 +5, 32075 +6, 32209 +7, 32229 +0, 36899 +1, 36983 +2, 36833 +3, 36831 +4, 36831 +5, 36829 +6, 36965 +7, 36983 0, 37783 1, 37867 2, 37717 @@ -102,14 +102,14 @@ 5, 37801 6, 37937 7, 37955 -0, 38003 -1, 38123 -2, 37971 -3, 37933 -4, 37931 -5, 37955 -6, 38077 -7, 38117 +0, 38131 +1, 38211 +2, 38063 +3, 38063 +4, 38037 +5, 38047 +6, 38197 +7, 38203 0, 38819 1, 38903 2, 38753 @@ -126,14 +126,6 @@ 5, 38751 6, 38885 7, 38905 -0, 38825 -1, 38909 -2, 38759 -3, 38759 -4, 38759 -5, 38755 -6, 38891 -7, 38909 0, 38827 1, 38913 2, 38763 @@ -142,14 +134,22 @@ 5, 38759 6, 38893 7, 38913 -0, 121349 -1, 121649 -2, 121471 -3, 121465 -4, 121453 -5, 121559 -6, 121627 -7, 121653 +0, 38827 +1, 38913 +2, 38763 +3, 38761 +4, 38761 +5, 38759 +6, 38893 +7, 38913 +0, 121341 +1, 121633 +2, 121465 +3, 121469 +4, 121457 +5, 121539 +6, 121593 +7, 121615 0, 121857 1, 122153 2, 121983 @@ -184,12 +184,12 @@ 7, 122153 0, 122541 1, 122837 -2, 122627 -3, 122621 +2, 122667 +3, 122667 4, 122667 -5, 122745 -6, 122801 -7, 122769 +5, 122747 +6, 122817 +7, 122837 0, 123719 1, 126063 2, 125893 @@ -198,14 +198,14 @@ 5, 125973 6, 126043 7, 126063 -0, 124389 -1, 126747 -2, 126543 -3, 126587 -4, 126547 -5, 126593 -6, 126697 -7, 126687 +0, 124305 +1, 126599 +2, 126513 +3, 126471 +4, 126481 +5, 126525 +6, 126613 +7, 126617 0, 139007 1, 138657 2, 138497 @@ -238,19 +238,19 @@ 5, 138567 6, 138707 7, 140961 -0, 138381 -1, 137775 +0, 138707 +1, 138357 2, 138197 -3, 137867 -4, 137547 -5, 137621 -6, 137825 +3, 138129 +4, 137867 +5, 137685 +6, 138087 7, 140661 -0, 2055 +0, 775 1, 778 2, 778 3, 778 4, 778 -5, 778 -6, 1034 -7, 140013 +5, 1034 +6, 137759 +7, 2058 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log index 0d4d963e481d9abc7581dab00dc4f622c99e97da..843cce799cf2d4f2adf34251dd571e35ad1c10c8 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log @@ -1,4 +1,4 @@ -[15:38:11] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[16:34:13] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) ============================================================== DSV2_LITE_L24 / echo (slots/rank=8 policy=matched) E=64 K=6 H'=1408 m=8 @@ -7,10 +7,14 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ b099fd84 ============================================================== -W0826 15:38:12.487000 113834 torch/distributed/run.py:766] -W0826 15:38:12.487000 113834 torch/distributed/run.py:766] ***************************************** -W0826 15:38:12.487000 113834 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 15:38:12.487000 113834 torch/distributed/run.py:766] ***************************************** +W0826 16:34:15.052000 233895 torch/distributed/run.py:766] +W0826 16:34:15.052000 233895 torch/distributed/run.py:766] ***************************************** +W0826 16:34:15.052000 233895 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:34:15.052000 233895 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled @@ -571,26 +575,22 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7[Gloo] Rank peer ranks. Expected number of connected peer ranks is : 37 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 75 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 7[Gloo] Rank 5 - is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 @@ -598,18 +598,18 @@ WARNING: one_logger package is required to enable e2e metrics tracking. please g > setting random seeds to 1234 ... [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank2]:[W826 15:38:30.717922993 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 15:38:30.718009626 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 15:38:30.718015763 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 15:38:30.718112037 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 15:38:30.718203065 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 15:38:30.719727706 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 15:38:30.720969289 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[rank1]:[W826 16:34:33.504616121 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 16:34:33.509182543 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 16:34:33.512397516 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 16:34:33.512419454 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:34:33.512461120 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 16:34:33.512474494 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:34:33.512493594 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. @@ -617,10 +617,10 @@ make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' >>> done with dataset index builder. Compilation time: 0.034 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 15:38:30.877984914 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.879 seconds -time to initialize megatron (seconds): 7.182 -[after megatron is initialized] datetime: 2026-08-26 15:38:36 +[rank0]:[W826 16:34:33.659888371 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.478 seconds +time to initialize megatron (seconds): 6.430 +[after megatron is initialized] datetime: 2026-08-26 16:34:39 building GPT model ... /workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1454: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. warnings.warn( @@ -660,20 +660,22 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (103022592 elements, 103022592 padded size): module.output_layer.weight Params for bucket 2 (48503296 elements, 48503296 padded size): - module.decoder.final_layernorm.weight - module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.23.mlp.router.weight - module.decoder.layers.23.pre_mlp_layernorm.weight - module.decoder.layers.23.self_attention.linear_q_proj.weight - module.decoder.layers.23.self_attention.linear_proj.weight - module.decoder.layers.23.input_layernorm.weight - module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight module.decoder.layers.22.self_attention.linear_kv_down_proj.weight module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.22.self_attention.linear_proj.weight @@ -681,9 +683,7 @@ Params for bucket 3 (40901632 elements, 40901632 padded size): module.decoder.layers.21.mlp.router.weight module.decoder.layers.21.pre_mlp_layernorm.weight module.decoder.layers.21.self_attention.linear_q_proj.weight - module.decoder.layers.22.mlp.router.weight module.decoder.layers.22.pre_mlp_layernorm.weight - module.decoder.layers.22.self_attention.linear_kv_up_proj.weight module.decoder.layers.22.self_attention.linear_q_proj.weight module.decoder.layers.22.input_layernorm.weight module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight @@ -696,72 +696,72 @@ Params for bucket 4 (41163264 elements, 41163264 padded size): module.decoder.layers.20.self_attention.linear_kv_up_proj.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.20.self_attention.linear_kv_down_proj.weight - module.decoder.layers.20.self_attention.linear_q_proj.weight module.decoder.layers.20.self_attention.linear_proj.weight - module.decoder.layers.20.input_layernorm.weight - module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight module.decoder.layers.20.mlp.router.weight module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight module.decoder.layers.20.pre_mlp_layernorm.weight - module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.input_layernorm.weight Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight module.decoder.layers.19.self_attention.linear_kv_up_proj.weight module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.19.self_attention.linear_kv_down_proj.weight - module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.19.input_layernorm.weight module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.19.pre_mlp_layernorm.weight - module.decoder.layers.19.mlp.router.weight - module.decoder.layers.19.self_attention.linear_q_proj.weight Params for bucket 6 (40901632 elements, 40901632 padded size): - module.decoder.layers.18.self_attention.linear_kv_down_proj.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.18.self_attention.linear_proj.weight - module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.17.mlp.router.weight - module.decoder.layers.17.pre_mlp_layernorm.weight module.decoder.layers.18.mlp.router.weight module.decoder.layers.18.pre_mlp_layernorm.weight module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight module.decoder.layers.18.input_layernorm.weight module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.17.self_attention.linear_kv_down_proj.weight module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight Params for bucket 7 (41163264 elements, 41163264 padded size): - module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.17.input_layernorm.weight module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.pre_mlp_layernorm.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.16.self_attention.linear_kv_down_proj.weight module.decoder.layers.16.self_attention.linear_proj.weight - module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.17.input_layernorm.weight module.decoder.layers.16.mlp.router.weight - module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight module.decoder.layers.16.self_attention.linear_q_proj.weight module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight Params for bucket 8 (42734080 elements, 42734080 padded size): - module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight module.decoder.layers.15.pre_mlp_layernorm.weight module.decoder.layers.15.self_attention.linear_q_proj.weight - module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.mlp.router.weight module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.15.input_layernorm.weight Params for bucket 9 (40901632 elements, 40901632 padded size): - module.decoder.layers.14.self_attention.linear_kv_up_proj.weight module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.14.self_attention.linear_q_proj.weight module.decoder.layers.14.self_attention.linear_proj.weight @@ -777,117 +777,117 @@ Params for bucket 9 (40901632 elements, 40901632 padded size): module.decoder.layers.13.pre_mlp_layernorm.weight module.decoder.layers.13.self_attention.linear_q_proj.weight Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight module.decoder.layers.13.input_layernorm.weight module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.12.self_attention.linear_kv_down_proj.weight - module.decoder.layers.12.self_attention.linear_q_proj.weight - module.decoder.layers.12.self_attention.linear_proj.weight module.decoder.layers.12.input_layernorm.weight - module.decoder.layers.13.self_attention.linear_proj.weight - module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.12.pre_mlp_layernorm.weight module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight Params for bucket 11 (42734080 elements, 42734080 padded size): module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.11.self_attention.linear_kv_up_proj.weight module.decoder.layers.11.pre_mlp_layernorm.weight module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight module.decoder.layers.11.self_attention.linear_proj.weight module.decoder.layers.11.input_layernorm.weight module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.11.mlp.router.weight - module.decoder.layers.11.self_attention.linear_kv_down_proj.weight Params for bucket 12 (40901632 elements, 40901632 padded size): - module.decoder.layers.10.self_attention.linear_kv_up_proj.weight - module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.10.self_attention.linear_kv_down_proj.weight - module.decoder.layers.10.self_attention.linear_q_proj.weight - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.linear_q_proj.weight module.decoder.layers.10.mlp.router.weight module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.input_layernorm.weight module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight module.decoder.layers.9.self_attention.linear_kv_up_proj.weight module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight Params for bucket 13 (41163264 elements, 41163264 padded size): - module.decoder.layers.9.self_attention.linear_proj.weight - module.decoder.layers.9.input_layernorm.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.input_layernorm.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.8.self_attention.linear_kv_down_proj.weight module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.linear_q_proj.weight - module.decoder.layers.8.input_layernorm.weight Params for bucket 14 (42734080 elements, 42734080 padded size): module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.7.input_layernorm.weight - module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.linear_kv_up_proj.weight - module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.7.self_attention.linear_kv_down_proj.weight - module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight Params for bucket 15 (40901632 elements, 40901632 padded size): - module.decoder.layers.6.self_attention.linear_kv_up_proj.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.6.self_attention.linear_kv_down_proj.weight - module.decoder.layers.6.self_attention.linear_q_proj.weight - module.decoder.layers.6.self_attention.linear_proj.weight - module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.self_attention.linear_q_proj.weight module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.6.input_layernorm.weight module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.5.self_attention.linear_proj.weight - module.decoder.layers.5.input_layernorm.weight module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.input_layernorm.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.4.self_attention.linear_kv_down_proj.weight - module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.4.input_layernorm.weight module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.self_attention.linear_q_proj.weight Params for bucket 17 (42734080 elements, 42734080 padded size): - module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.pre_mlp_layernorm.weight - module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.3.input_layernorm.weight - module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.3.self_attention.linear_kv_down_proj.weight module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight Params for bucket 18 (40901632 elements, 40901632 padded size): module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.2.self_attention.linear_kv_down_proj.weight @@ -895,79 +895,79 @@ Params for bucket 18 (40901632 elements, 40901632 padded size): module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.2.input_layernorm.weight module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_kv_up_proj.weight module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.2.self_attention.linear_q_proj.weight - module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.self_attention.linear_kv_down_proj.weight module.decoder.layers.1.self_attention.linear_q_proj.weight Params for bucket 19 (71436288 elements, 71436288 padded size): module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc2.weight module.decoder.layers.1.input_layernorm.weight module.decoder.layers.0.mlp.linear_fc1.weight - module.decoder.layers.0.mlp.linear_fc2.weight Params for bucket 20 (116789760 elements, 116789760 padded size): - module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight module.decoder.layers.0.self_attention.linear_kv_up_proj.weight - module.decoder.layers.0.input_layernorm.weight - module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.0.self_attention.linear_q_proj.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight module.decoder.layers.0.self_attention.linear_kv_down_proj.weight - module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.input_layernorm.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 Params for bucket 1 (40370176 elements, 40370176 padded size): module.decoder.layers.23.mlp.experts.linear_fc2.weight7 - module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight3 - module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight0 - module.decoder.layers.23.mlp.experts.linear_fc1.weight6 - module.decoder.layers.23.mlp.experts.linear_fc2.weight6 - module.decoder.layers.23.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc2.weight1 module.decoder.layers.23.mlp.experts.linear_fc1.weight7 module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc1.weight4 module.decoder.layers.23.mlp.experts.linear_fc1.weight3 module.decoder.layers.23.mlp.experts.linear_fc1.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight6 module.decoder.layers.22.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight1 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 - module.decoder.layers.22.mlp.experts.linear_fc2.weight7 - module.decoder.layers.22.mlp.experts.linear_fc2.weight5 Params for bucket 3 (40370176 elements, 40370176 padded size): - module.decoder.layers.22.mlp.experts.linear_fc1.weight7 - module.decoder.layers.22.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc1.weight5 module.decoder.layers.22.mlp.experts.linear_fc2.weight3 - module.decoder.layers.22.mlp.experts.linear_fc2.weight2 module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 module.decoder.layers.22.mlp.experts.linear_fc1.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 Params for bucket 4 (40370176 elements, 40370176 padded size): - module.decoder.layers.22.mlp.experts.linear_fc1.weight2 - module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 module.decoder.layers.21.mlp.experts.linear_fc2.weight7 module.decoder.layers.21.mlp.experts.linear_fc2.weight5 module.decoder.layers.21.mlp.experts.linear_fc2.weight3 module.decoder.layers.21.mlp.experts.linear_fc2.weight2 module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 module.decoder.layers.21.mlp.experts.linear_fc2.weight6 - module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 module.decoder.layers.21.mlp.experts.linear_fc2.weight4 module.decoder.layers.21.mlp.experts.linear_fc2.weight1 Params for bucket 5 (40370176 elements, 40370176 padded size): - module.decoder.layers.21.mlp.experts.linear_fc1.weight4 - module.decoder.layers.21.mlp.experts.linear_fc1.weight3 module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 module.decoder.layers.21.mlp.experts.linear_fc1.weight7 module.decoder.layers.21.mlp.experts.linear_fc1.weight5 module.decoder.layers.21.mlp.experts.linear_fc1.weight2 @@ -978,40 +978,40 @@ Params for bucket 6 (40370176 elements, 40370176 padded size): module.decoder.layers.20.mlp.experts.linear_fc2.weight4 module.decoder.layers.20.mlp.experts.linear_fc2.weight1 module.decoder.layers.20.mlp.experts.linear_fc1.weight7 - module.decoder.layers.20.mlp.experts.linear_fc2.weight5 module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 module.decoder.layers.20.mlp.experts.linear_fc2.weight2 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 module.decoder.layers.20.mlp.experts.linear_fc1.weight6 Params for bucket 7 (40370176 elements, 40370176 padded size): module.decoder.layers.20.mlp.experts.linear_fc1.weight5 - module.decoder.layers.20.mlp.experts.linear_fc1.weight2 module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 module.decoder.layers.19.mlp.experts.linear_fc2.weight7 - module.decoder.layers.20.mlp.experts.linear_fc1.weight3 module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 module.decoder.layers.20.mlp.experts.linear_fc1.weight0 module.decoder.layers.19.mlp.experts.linear_fc2.weight6 Params for bucket 8 (40370176 elements, 40370176 padded size): module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 module.decoder.layers.19.mlp.experts.linear_fc2.weight3 module.decoder.layers.19.mlp.experts.linear_fc2.weight0 - module.decoder.layers.19.mlp.experts.linear_fc1.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 - module.decoder.layers.19.mlp.experts.linear_fc2.weight1 - module.decoder.layers.19.mlp.experts.linear_fc2.weight2 - module.decoder.layers.19.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc1.weight6 module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 Params for bucket 9 (40370176 elements, 40370176 padded size): - module.decoder.layers.19.mlp.experts.linear_fc1.weight2 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 module.decoder.layers.18.mlp.experts.linear_fc2.weight6 module.decoder.layers.18.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 module.decoder.layers.18.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 module.decoder.layers.18.mlp.experts.linear_fc2.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight2 @@ -1026,24 +1026,24 @@ Params for bucket 10 (40370176 elements, 40370176 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight3 Params for bucket 11 (40370176 elements, 40370176 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 module.decoder.layers.17.mlp.experts.linear_fc2.weight5 module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight3 module.decoder.layers.17.mlp.experts.linear_fc2.weight2 module.decoder.layers.17.mlp.experts.linear_fc2.weight0 - module.decoder.layers.18.mlp.experts.linear_fc1.weight0 - module.decoder.layers.17.mlp.experts.linear_fc2.weight6 - module.decoder.layers.17.mlp.experts.linear_fc2.weight4 - module.decoder.layers.17.mlp.experts.linear_fc2.weight1 - module.decoder.layers.17.mlp.experts.linear_fc1.weight7 Params for bucket 12 (40370176 elements, 40370176 padded size): - module.decoder.layers.17.mlp.experts.linear_fc1.weight6 - module.decoder.layers.17.mlp.experts.linear_fc1.weight3 - module.decoder.layers.17.mlp.experts.linear_fc1.weight4 - module.decoder.layers.17.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc1.weight2 module.decoder.layers.17.mlp.experts.linear_fc1.weight5 module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 Params for bucket 13 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc2.weight7 module.decoder.layers.16.mlp.experts.linear_fc2.weight5 @@ -1057,53 +1057,53 @@ Params for bucket 13 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc2.weight1 module.decoder.layers.16.mlp.experts.linear_fc1.weight6 Params for bucket 14 (40370176 elements, 40370176 padded size): - module.decoder.layers.16.mlp.experts.linear_fc1.weight1 - module.decoder.layers.16.mlp.experts.linear_fc1.weight2 - module.decoder.layers.15.mlp.experts.linear_fc2.weight7 - module.decoder.layers.15.mlp.experts.linear_fc2.weight5 - module.decoder.layers.16.mlp.experts.linear_fc1.weight4 - module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 module.decoder.layers.15.mlp.experts.linear_fc2.weight6 module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 module.decoder.layers.15.mlp.experts.linear_fc2.weight3 module.decoder.layers.15.mlp.experts.linear_fc2.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight2 module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight4 module.decoder.layers.15.mlp.experts.linear_fc1.weight3 - module.decoder.layers.15.mlp.experts.linear_fc1.weight7 - module.decoder.layers.15.mlp.experts.linear_fc2.weight1 - module.decoder.layers.15.mlp.experts.linear_fc1.weight5 Params for bucket 16 (40370176 elements, 40370176 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight0 - module.decoder.layers.14.mlp.experts.linear_fc2.weight6 - module.decoder.layers.14.mlp.experts.linear_fc2.weight4 - module.decoder.layers.14.mlp.experts.linear_fc2.weight1 module.decoder.layers.15.mlp.experts.linear_fc1.weight2 - module.decoder.layers.14.mlp.experts.linear_fc2.weight7 module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 module.decoder.layers.14.mlp.experts.linear_fc2.weight5 module.decoder.layers.14.mlp.experts.linear_fc2.weight3 - module.decoder.layers.14.mlp.experts.linear_fc2.weight2 module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 module.decoder.layers.14.mlp.experts.linear_fc1.weight5 - module.decoder.layers.14.mlp.experts.linear_fc1.weight6 module.decoder.layers.14.mlp.experts.linear_fc1.weight2 module.decoder.layers.14.mlp.experts.linear_fc1.weight1 - module.decoder.layers.14.mlp.experts.linear_fc1.weight7 - module.decoder.layers.14.mlp.experts.linear_fc1.weight3 module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 Params for bucket 18 (40370176 elements, 40370176 padded size): module.decoder.layers.13.mlp.experts.linear_fc2.weight4 module.decoder.layers.13.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight1 module.decoder.layers.13.mlp.experts.linear_fc1.weight7 module.decoder.layers.14.mlp.experts.linear_fc1.weight0 - module.decoder.layers.13.mlp.experts.linear_fc2.weight5 module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 module.decoder.layers.13.mlp.experts.linear_fc2.weight3 module.decoder.layers.13.mlp.experts.linear_fc2.weight2 module.decoder.layers.13.mlp.experts.linear_fc2.weight0 @@ -1112,20 +1112,20 @@ Params for bucket 19 (40370176 elements, 40370176 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight5 module.decoder.layers.13.mlp.experts.linear_fc1.weight2 module.decoder.layers.13.mlp.experts.linear_fc1.weight1 - module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc1.weight4 module.decoder.layers.13.mlp.experts.linear_fc1.weight3 module.decoder.layers.13.mlp.experts.linear_fc1.weight0 - module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 Params for bucket 20 (40370176 elements, 40370176 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight5 - module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 module.decoder.layers.12.mlp.experts.linear_fc2.weight0 module.decoder.layers.12.mlp.experts.linear_fc1.weight6 module.decoder.layers.12.mlp.experts.linear_fc1.weight4 - module.decoder.layers.12.mlp.experts.linear_fc2.weight4 - module.decoder.layers.12.mlp.experts.linear_fc2.weight2 - module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 module.decoder.layers.12.mlp.experts.linear_fc1.weight7 module.decoder.layers.12.mlp.experts.linear_fc1.weight5 Params for bucket 21 (40370176 elements, 40370176 padded size): @@ -1134,76 +1134,76 @@ Params for bucket 21 (40370176 elements, 40370176 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight6 module.decoder.layers.11.mlp.experts.linear_fc2.weight4 module.decoder.layers.12.mlp.experts.linear_fc1.weight1 - module.decoder.layers.12.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight5 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight2 Params for bucket 22 (40370176 elements, 40370176 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight1 - module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 Params for bucket 23 (40370176 elements, 40370176 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight1 - module.decoder.layers.10.mlp.experts.linear_fc2.weight6 - module.decoder.layers.10.mlp.experts.linear_fc2.weight4 - module.decoder.layers.10.mlp.experts.linear_fc2.weight2 - module.decoder.layers.10.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc1.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight5 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 Params for bucket 24 (40370176 elements, 40370176 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight5 - module.decoder.layers.10.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc1.weight6 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 Params for bucket 25 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 - module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 Params for bucket 26 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight5 - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 Params for bucket 27 (40370176 elements, 40370176 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 Params for bucket 28 (40370176 elements, 40370176 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 @@ -1213,8 +1213,8 @@ Params for bucket 28 (40370176 elements, 40370176 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight4 module.decoder.layers.7.mlp.experts.linear_fc2.weight1 Params for bucket 29 (40370176 elements, 40370176 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 @@ -1222,24 +1222,24 @@ Params for bucket 29 (40370176 elements, 40370176 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight1 Params for bucket 30 (40370176 elements, 40370176 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 - module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 31 (40370176 elements, 40370176 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight5 - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 - module.decoder.layers.6.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 Params for bucket 32 (40370176 elements, 40370176 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight4 @@ -1255,82 +1255,82 @@ Params for bucket 32 (40370176 elements, 40370176 padded size): Params for bucket 33 (40370176 elements, 40370176 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight3 module.decoder.layers.5.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight6 - module.decoder.layers.4.mlp.experts.linear_fc1.weight4 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 Params for bucket 35 (40370176 elements, 40370176 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 - module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 - module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 Params for bucket 36 (40370176 elements, 40370176 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight0 Params for bucket 37 (40370176 elements, 40370176 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 - module.decoder.layers.2.mlp.experts.linear_fc2.weight6 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc1.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 Params for bucket 38 (40370176 elements, 40370176 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight4 - module.decoder.layers.2.mlp.experts.linear_fc1.weight3 - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 - module.decoder.layers.1.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc1.weight7 - module.decoder.layers.1.mlp.experts.linear_fc1.weight5 Params for bucket 40 (17301504 elements, 17301504 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:38:37 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:34:39 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1338,7 +1338,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1351,34 +1351,30 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 15:38:37 +[after dataloaders are built] datetime: 2026-08-26 16:34:39 done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (613.22, 627.63) + train/valid/test-data-iterators-setup ..........: (61.17, 66.88) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (627.09, 670.48) - train/valid/test-data-iterators-setup ..........: (59.82, 66.33) -[before the start of training step] datetime: 2026-08-26 15:38:37 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1008, 980, 480, 787, 1021, 627, 1422, 531, 447, 1331, 1035, 557, 255, 1588, 698, 1124, 613, 744, 668, 696, 515, 444, 1012, 839, 732, 896, 495, 1026, 865, 625, 921, 953, 625, 553, 603, 562, 485, 408, 1035, 1137, 696, 665, 505, 606, 1408, 884, 694, 459, 682, 511, 959, 848, 1512, 896, 351, 679, 662, 542, 1304, 850, 619, 405, 754, 318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +[before the start of training step] datetime: 2026-08-26 16:34:39 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1124, 897, 526, 734, 821, 523, 1439, 540, 551, 1080, 1080, 439, 272, 1690, 666, 1058, 619, 646, 736, 804, 541, 471, 991, 820, 721, 795, 515, 973, 860, 658, 906, 990, 484, 663, 690, 629, 554, 401, 948, 1234, 639, 695, 447, 537, 1381, 959, 688, 433, 726, 571, 951, 932, 1468, 883, 324, 662, 696, 540, 1335, 941, 616, 440, 805, 394] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1086, 878, 503, 728, 694, 613, 1438, 549, 566, 1102, 864, 620, 276, 1613, 630, 990, 697, 655, 793, 721, 542, 452, 1006, 868, 744, 1048, 528, 1083, 906, 771, 907, 1038, 539, 556, 671, 610, 471, 390, 1052, 1227, 714, 723, 458, 681, 1424, 853, 682, 457, 838, 495, 879, 891, 1317, 797, 367, 638, 748, 533, 1163, 909, 714, 346, 765, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1152, 1076, 467, 783, 655, 617, 1506, 614, 593, 1015, 1041, 649, 226, 1745, 575, 996, 666, 617, 772, 759, 572, 392, 1031, 930, 644, 838, 537, 1019, 882, 603, 978, 1053, 622, 917, 658, 586, 539, 343, 837, 1149, 698, 686, 443, 661, 1395, 896, 570, 452, 704, 557, 949, 753, 1397, 828, 329, 609, 736, 559, 1295, 955, 603, 346, 747, 330] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1111, 1011, 510, 748, 707, 617, 1460, 549, 543, 1122, 956, 616, 234, 1644, 711, 1065, 634, 668, 714, 732, 547, 398, 915, 857, 703, 1209, 525, 963, 1003, 688, 811, 1045, 474, 741, 608, 628, 502, 426, 971, 1228, 792, 720, 513, 499, 1437, 837, 690, 403, 705, 538, 1037, 852, 997, 925, 344, 598, 687, 518, 1237, 991, 701, 369, 742, 426] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1136, 806, 598, 801, 1057, 568, 1425, 551, 493, 1125, 847, 618, 233, 1609, 616, 1132, 762, 670, 669, 798, 511, 444, 1000, 765, 684, 990, 497, 949, 788, 750, 780, 1019, 499, 640, 538, 572, 447, 407, 968, 1245, 693, 755, 504, 528, 1388, 964, 869, 440, 760, 548, 821, 911, 1227, 906, 363, 573, 687, 588, 1239, 1058, 648, 389, 763, 523] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1069, 744, 547, 742, 950, 510, 1462, 546, 490, 1225, 1037, 548, 259, 1619, 553, 1003, 672, 646, 672, 758, 488, 461, 940, 820, 772, 849, 548, 922, 847, 702, 926, 1006, 742, 755, 563, 591, 477, 373, 911, 1248, 717, 692, 544, 485, 1420, 882, 707, 455, 794, 527, 786, 867, 1430, 1018, 314, 669, 750, 450, 1336, 918, 738, 446, 750, 464] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1124, 897, 526, 734, 821, 523, 1439, 540, 551, 1080, 1080, 439, 272, 1690, 666, 1058, 619, 646, 736, 804, 541, 471, 991, 820, 721, 795, 515, 973, 860, 658, 906, 990, 484, 663, 690, 629, 554, 401, 948, 1234, 639, 695, 447, 537, 1381, 959, 688, 433, 726, 571, 951, 932, 1468, 883, 324, 662, 696, 540, 1335, 941, 616, 440, 805, 394] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1152, 1076, 467, 783, 655, 617, 1506, 614, 593, 1015, 1041, 649, 226, 1745, 575, 996, 666, 617, 772, 759, 572, 392, 1031, 930, 644, 838, 537, 1019, 882, 603, 978, 1053, 622, 917, 658, 586, 539, 343, 837, 1149, 698, 686, 443, 661, 1395, 896, 570, 452, 704, 557, 949, 753, 1397, 828, 329, 609, 736, 559, 1295, 955, 603, 346, 747, 330] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1110, 698, 519, 864, 620, 586, 1478, 595, 527, 1414, 847, 607, 240, 1699, 627, 976, 686, 621, 761, 779, 499, 443, 934, 901, 760, 752, 519, 978, 928, 788, 889, 1055, 501, 903, 648, 599, 486, 341, 1025, 1248, 767, 658, 451, 591, 1401, 897, 677, 373, 882, 517, 662, 936, 1459, 801, 356, 633, 721, 532, 1234, 868, 625, 359, 827, 474] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1152, 1076, 467, 783, 655, 617, 942, 614, 0, 0, 0, 0, 0, 0, 0, 0, 593, 1015, 1041, 649, 226, 1026, 575, 996, 0, 0, 0, 0, 0, 0, 0, 0, 666, 617, 772, 759, 572, 392, 1031, 930, 462, 103, 0, 0, 0, 0, 0, 0, 644, 838, 537, 1019, 882, 603, 978, 538, 0, 0, 0, 0, 0, 0, 0, 0, 622, 917, 658, 586, 539, 343, 837, 1149, 616, 0, 0, 0, 0, 0, 0, 0, 698, 686, 443, 661, 1395, 896, 570, 452, 129, 117, 0, 0, 0, 0, 0, 0, 704, 557, 949, 753, 1280, 828, 329, 609, 0, 0, 0, 0, 0, 0, 0, 0, 736, 559, 1295, 955, 603, 346, 747, 330, 386, 102, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6306, 6121, 6304, 6039, 6267, 6047, 6009, 6059] avg=6144.0000 max=6306 max_rank=0 min=6009 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1008, 980, 480, 787, 1021, 627, 890, 531, 0, 0, 0, 0, 0, 0, 0, 0, 447, 1331, 1035, 557, 255, 933, 698, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 613, 744, 668, 696, 515, 444, 1012, 839, 436, 94, 0, 0, 0, 0, 0, 0, 732, 896, 495, 1026, 865, 625, 921, 487, 0, 0, 0, 0, 0, 0, 0, 0, 625, 553, 603, 562, 485, 408, 1035, 1137, 561, 0, 0, 0, 0, 0, 0, 0, 696, 665, 505, 606, 1408, 884, 694, 459, 117, 126, 0, 0, 0, 0, 0, 0, 682, 511, 959, 848, 1386, 896, 351, 679, 0, 0, 0, 0, 0, 0, 0, 0, 662, 542, 1304, 850, 619, 405, 754, 318, 349, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6324, 6380, 6061, 6047, 5969, 6160, 6312, 5899] avg=6144.0000 max=6380 max_rank=1 min=5899 min_rank=7 imbalance_ratio=1.038411 max_load_reduction=42772 max_load_reduction_pct=0.870199 moved_to_echo=24340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1069, 744, 547, 742, 950, 510, 1462, 546, 490, 1225, 1037, 548, 259, 1619, 553, 1003, 672, 646, 672, 758, 488, 461, 940, 820, 772, 849, 548, 922, 847, 702, 926, 1006, 742, 755, 563, 591, 477, 373, 911, 1248, 717, 692, 544, 485, 1420, 882, 707, 455, 794, 527, 786, 867, 1430, 1018, 314, 669, 750, 450, 1336, 918, 738, 446, 750, 464] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1008, 980, 480, 787, 1021, 627, 1422, 531, 447, 1331, 1035, 557, 255, 1588, 698, 1124, 613, 744, 668, 696, 515, 444, 1012, 839, 732, 896, 495, 1026, 865, 625, 921, 953, 625, 553, 603, 562, 485, 408, 1035, 1137, 696, 665, 505, 606, 1408, 884, 694, 459, 682, 511, 959, 848, 1512, 896, 351, 679, 662, 542, 1304, 850, 619, 405, 754, 318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1086, 878, 503, 728, 694, 613, 899, 549, 0, 0, 0, 0, 0, 0, 0, 0, 566, 1102, 864, 620, 276, 948, 630, 990, 0, 0, 0, 0, 0, 0, 0, 0, 697, 655, 793, 721, 542, 452, 1006, 868, 441, 95, 0, 0, 0, 0, 0, 0, 744, 1048, 528, 1083, 906, 771, 907, 529, 0, 0, 0, 0, 0, 0, 0, 0, 539, 556, 671, 610, 471, 390, 1052, 1227, 570, 0, 0, 0, 0, 0, 0, 0, 714, 723, 458, 681, 1424, 853, 682, 457, 128, 110, 0, 0, 0, 0, 0, 0, 838, 495, 879, 891, 1207, 797, 367, 638, 0, 0, 0, 0, 0, 0, 0, 0, 748, 533, 1163, 909, 714, 346, 765, 335, 381, 98, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5950, 5996, 6270, 6516, 6086, 6230, 6112, 5992] avg=6144.0000 max=6516 max_rank=3 min=5950 min_rank=0 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1152, 1076, 467, 783, 655, 617, 942, 614, 0, 0, 0, 0, 0, 0, 0, 0, 593, 1015, 1041, 649, 226, 1026, 575, 996, 0, 0, 0, 0, 0, 0, 0, 0, 666, 617, 772, 759, 572, 392, 1031, 930, 462, 103, 0, 0, 0, 0, 0, 0, 644, 838, 537, 1019, 882, 603, 978, 538, 0, 0, 0, 0, 0, 0, 0, 0, 622, 917, 658, 586, 539, 343, 837, 1149, 616, 0, 0, 0, 0, 0, 0, 0, 698, 686, 443, 661, 1395, 896, 570, 452, 129, 117, 0, 0, 0, 0, 0, 0, 704, 557, 949, 753, 1280, 828, 329, 609, 0, 0, 0, 0, 0, 0, 0, 0, 736, 559, 1295, 955, 603, 346, 747, 330, 386, 102, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6306, 6121, 6304, 6039, 6267, 6047, 6009, 6059] avg=6144.0000 max=6306 max_rank=0 min=6009 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1110, 698, 519, 864, 620, 586, 925, 595, 0, 0, 0, 0, 0, 0, 0, 0, 527, 1414, 847, 607, 240, 998, 627, 976, 0, 0, 0, 0, 0, 0, 0, 0, 686, 621, 761, 779, 499, 443, 934, 901, 453, 101, 0, 0, 0, 0, 0, 0, 760, 752, 519, 978, 928, 788, 889, 538, 0, 0, 0, 0, 0, 0, 0, 0, 501, 903, 648, 599, 486, 341, 1025, 1248, 600, 0, 0, 0, 0, 0, 0, 0, 767, 658, 451, 591, 1401, 897, 677, 373, 130, 122, 0, 0, 0, 0, 0, 0, 882, 517, 662, 936, 1337, 801, 356, 633, 0, 0, 0, 0, 0, 0, 0, 0, 721, 532, 1234, 868, 625, 359, 827, 474, 387, 100, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6236, 6178, 6152, 6351, 6067, 6124, 6127] avg=6144.0000 max=6351 max_rank=4 min=5917 min_rank=0 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1124, 897, 526, 734, 821, 523, 900, 540, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1080, 1080, 439, 272, 993, 666, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 619, 646, 736, 804, 541, 471, 991, 820, 441, 100, 0, 0, 0, 0, 0, 0, 721, 795, 515, 973, 860, 658, 906, 505, 0, 0, 0, 0, 0, 0, 0, 0, 484, 663, 690, 629, 554, 401, 948, 1234, 597, 0, 0, 0, 0, 0, 0, 0, 639, 695, 447, 537, 1381, 959, 688, 433, 122, 123, 0, 0, 0, 0, 0, 0, 726, 571, 951, 932, 1345, 883, 324, 662, 0, 0, 0, 0, 0, 0, 0, 0, 696, 540, 1335, 941, 616, 440, 805, 394, 363, 98, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6065, 6139, 6169, 5933, 6200, 6024, 6394, 6228] avg=6144.0000 max=6394 max_rank=6 min=5933 min_rank=3 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1111, 1011, 510, 748, 707, 617, 913, 549, 0, 0, 0, 0, 0, 0, 0, 0, 543, 1122, 956, 616, 234, 966, 711, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 634, 668, 714, 732, 547, 398, 915, 857, 448, 97, 0, 0, 0, 0, 0, 0, 703, 1209, 525, 963, 1003, 688, 811, 534, 0, 0, 0, 0, 0, 0, 0, 0, 474, 741, 608, 628, 502, 426, 971, 1228, 581, 0, 0, 0, 0, 0, 0, 0, 792, 720, 513, 499, 1437, 837, 690, 403, 128, 83, 0, 0, 0, 0, 0, 0, 705, 538, 1037, 852, 914, 925, 344, 598, 0, 0, 0, 0, 0, 0, 0, 0, 687, 518, 1237, 991, 701, 369, 742, 426, 383, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6213, 6010, 6436, 6159, 6102, 5913, 6153] avg=6144.0000 max=6436 max_rank=3 min=5913 min_rank=6 imbalance_ratio=1.047526 max_load_reduction=42716 max_load_reduction_pct=0.869059 moved_to_echo=24327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1008, 980, 480, 787, 1021, 627, 890, 531, 0, 0, 0, 0, 0, 0, 0, 0, 447, 1331, 1035, 557, 255, 933, 698, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 613, 744, 668, 696, 515, 444, 1012, 839, 436, 94, 0, 0, 0, 0, 0, 0, 732, 896, 495, 1026, 865, 625, 921, 487, 0, 0, 0, 0, 0, 0, 0, 0, 625, 553, 603, 562, 485, 408, 1035, 1137, 561, 0, 0, 0, 0, 0, 0, 0, 696, 665, 505, 606, 1408, 884, 694, 459, 117, 126, 0, 0, 0, 0, 0, 0, 682, 511, 959, 848, 1386, 896, 351, 679, 0, 0, 0, 0, 0, 0, 0, 0, 662, 542, 1304, 850, 619, 405, 754, 318, 349, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6324, 6380, 6061, 6047, 5969, 6160, 6312, 5899] avg=6144.0000 max=6380 max_rank=1 min=5899 min_rank=7 imbalance_ratio=1.038411 max_load_reduction=42772 max_load_reduction_pct=0.870199 moved_to_echo=24340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1069, 744, 547, 742, 950, 510, 915, 546, 0, 0, 0, 0, 0, 0, 0, 0, 490, 1225, 1037, 548, 259, 951, 553, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 672, 646, 672, 758, 488, 461, 940, 820, 448, 96, 0, 0, 0, 0, 0, 0, 772, 849, 548, 922, 847, 702, 926, 513, 0, 0, 0, 0, 0, 0, 0, 0, 742, 755, 563, 591, 477, 373, 911, 1248, 572, 0, 0, 0, 0, 0, 0, 0, 717, 692, 544, 485, 1420, 882, 707, 455, 124, 120, 0, 0, 0, 0, 0, 0, 794, 527, 786, 867, 1310, 1018, 314, 669, 0, 0, 0, 0, 0, 0, 0, 0, 750, 450, 1336, 918, 738, 446, 750, 464, 369, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 6066, 6001, 6079, 6232, 6146, 6285, 6320] avg=6144.0000 max=6320 max_rank=7 min=6001 min_rank=2 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1136, 806, 598, 801, 1057, 568, 884, 551, 0, 0, 0, 0, 0, 0, 0, 0, 493, 1125, 847, 618, 233, 937, 616, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 762, 670, 669, 798, 511, 444, 1000, 765, 440, 100, 0, 0, 0, 0, 0, 0, 684, 990, 497, 949, 788, 750, 780, 512, 0, 0, 0, 0, 0, 0, 0, 0, 499, 640, 538, 572, 447, 407, 968, 1245, 572, 0, 0, 0, 0, 0, 0, 0, 693, 755, 504, 528, 1388, 964, 869, 440, 129, 106, 0, 0, 0, 0, 0, 0, 760, 548, 821, 911, 1121, 906, 363, 573, 0, 0, 0, 0, 0, 0, 0, 0, 687, 588, 1239, 1058, 648, 389, 763, 523, 378, 101, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6401, 6001, 6159, 5950, 5888, 6376, 6003, 6374] avg=6144.0000 max=6401 max_rank=0 min=5888 min_rank=4 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1110, 698, 519, 864, 620, 586, 925, 595, 0, 0, 0, 0, 0, 0, 0, 0, 527, 1414, 847, 607, 240, 998, 627, 976, 0, 0, 0, 0, 0, 0, 0, 0, 686, 621, 761, 779, 499, 443, 934, 901, 453, 101, 0, 0, 0, 0, 0, 0, 760, 752, 519, 978, 928, 788, 889, 538, 0, 0, 0, 0, 0, 0, 0, 0, 501, 903, 648, 599, 486, 341, 1025, 1248, 600, 0, 0, 0, 0, 0, 0, 0, 767, 658, 451, 591, 1401, 897, 677, 373, 130, 122, 0, 0, 0, 0, 0, 0, 882, 517, 662, 936, 1337, 801, 356, 633, 0, 0, 0, 0, 0, 0, 0, 0, 721, 532, 1234, 868, 625, 359, 827, 474, 387, 100, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6236, 6178, 6152, 6351, 6067, 6124, 6127] avg=6144.0000 max=6351 max_rank=4 min=5917 min_rank=0 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1401,36 +1397,40 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[5760, 4352, 10240, 7552, 5376, 3200, 6272, 3328, 3072, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5760, 7424, 4224, 7936, 7168, 5632, 7168, 4224, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 5632, 3968, 4608, 11264, 7296, 5632, 3584, 1024, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5376, 5376, 5888, 6144, 4224, 3584, 7936, 6912, 3584, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 5760, 4992, 4864, 3968, 3200, 7808, 9728, 4736, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8832, 7168, 4224, 6272, 6528, 4736, 7296, 4480, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6144, 4352, 7168, 7040, 9984, 7168, 2816, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8832, 7168, 4224, 6272, 6528, 4736, 7296, 4480, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 9472, 7808, 4736, 2048, 7808, 5120, 8448, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5760, 7424, 4224, 7936, 7168, 5632, 7168, 4224, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5376, 5376, 5888, 6144, 4224, 3584, 7936, 6912, 3584, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 5760, 4992, 4864, 3968, 3200, 7808, 9728, 4736, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[5760, 4352, 10240, 7552, 5376, 3200, 6272, 3328, 3072, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1448,20 +1448,20 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[506, 566, 687, 613, 927, 713, 649, 802, 417, 850, 1357, 592, 1199, 924, 1030, 1008, 671, 716, 696, 631, 1088, 871, 1065, 699, 847, 818, 579, 464, 770, 621, 1312, 430, 1236, 377, 642, 500, 681, 384, 460, 1045, 1033, 389, 456, 697, 683, 880, 320, 347, 1988, 1344, 1001, 441, 572, 818, 619, 894, 948, 621, 770, 606, 806, 860, 931, 685] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[462, 370, 844, 668, 817, 869, 742, 715, 330, 845, 1407, 575, 1301, 1140, 926, 1042, 731, 555, 759, 488, 900, 840, 1057, 506, 841, 913, 594, 593, 631, 724, 1526, 495, 1166, 537, 656, 338, 720, 524, 391, 1069, 1276, 421, 330, 820, 561, 867, 294, 425, 2259, 957, 879, 561, 571, 759, 748, 1082, 865, 519, 773, 598, 725, 873, 774, 608] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[596, 615, 717, 744, 1206, 610, 599, 762, 466, 922, 1314, 532, 1008, 823, 1038, 1024, 699, 447, 750, 818, 1093, 874, 1205, 528, 835, 1008, 648, 483, 752, 664, 1403, 317, 1126, 526, 590, 468, 785, 419, 338, 972, 1077, 411, 382, 759, 474, 864, 244, 346, 2091, 1075, 984, 383, 605, 784, 661, 967, 882, 555, 867, 616, 883, 854, 928, 736] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[470, 523, 603, 587, 938, 629, 697, 741, 407, 939, 1397, 549, 1252, 885, 960, 1234, 703, 613, 652, 848, 1001, 826, 1098, 599, 862, 792, 659, 553, 892, 648, 1416, 393, 1058, 522, 632, 607, 780, 381, 382, 1047, 1285, 434, 519, 823, 592, 881, 287, 482, 2023, 1024, 888, 299, 486, 884, 637, 1066, 859, 486, 749, 713, 709, 739, 961, 551] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[449, 563, 694, 949, 814, 753, 630, 764, 375, 877, 1387, 634, 1079, 853, 1180, 1009, 751, 531, 731, 771, 995, 782, 1165, 639, 791, 680, 600, 548, 582, 667, 1306, 429, 1276, 461, 698, 496, 742, 480, 416, 997, 919, 432, 443, 788, 531, 915, 306, 324, 2016, 1176, 847, 359, 790, 759, 655, 1160, 1167, 556, 802, 602, 743, 924, 720, 674] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[359, 597, 622, 761, 941, 650, 575, 969, 436, 921, 1336, 533, 961, 830, 1091, 1147, 862, 427, 683, 553, 1150, 902, 942, 918, 805, 750, 610, 456, 746, 656, 1327, 369, 1143, 518, 735, 428, 607, 485, 517, 886, 1062, 509, 260, 658, 615, 939, 410, 306, 2189, 1342, 976, 344, 548, 870, 702, 913, 976, 602, 811, 728, 627, 932, 915, 714] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[408, 488, 720, 671, 895, 708, 668, 776, 405, 941, 1409, 635, 1246, 856, 1000, 1041, 715, 511, 746, 617, 1059, 980, 1036, 639, 944, 811, 710, 495, 747, 647, 1404, 397, 1063, 482, 668, 436, 767, 504, 354, 926, 1197, 478, 496, 695, 512, 809, 343, 399, 2038, 1074, 930, 308, 638, 874, 603, 977, 924, 582, 733, 650, 628, 962, 1019, 758] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[449, 563, 694, 949, 814, 753, 630, 764, 375, 877, 1387, 634, 1079, 853, 1180, 1009, 751, 531, 731, 771, 995, 782, 1165, 639, 791, 680, 600, 548, 582, 667, 1306, 429, 1276, 461, 698, 496, 742, 480, 416, 997, 919, 432, 443, 788, 531, 915, 306, 324, 2016, 1176, 847, 359, 790, 759, 655, 1160, 1167, 556, 802, 602, 743, 924, 720, 674] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[462, 370, 844, 668, 817, 869, 742, 715, 330, 845, 1407, 575, 1301, 1140, 926, 1042, 731, 555, 759, 488, 900, 840, 1057, 506, 841, 913, 594, 593, 631, 724, 1526, 495, 1166, 537, 656, 338, 720, 524, 391, 1069, 1276, 421, 330, 820, 561, 867, 294, 425, 2259, 957, 879, 561, 571, 759, 748, 1082, 865, 519, 773, 598, 725, 873, 774, 608] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[523, 658, 723, 635, 808, 728, 741, 782, 359, 964, 1419, 523, 1182, 852, 950, 1099, 633, 574, 684, 788, 1050, 700, 1160, 745, 789, 744, 573, 532, 749, 718, 1486, 464, 1222, 435, 679, 475, 823, 453, 391, 997, 1335, 475, 439, 727, 615, 847, 294, 424, 1933, 1194, 830, 435, 536, 829, 653, 1007, 866, 503, 818, 699, 640, 801, 895, 547] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[449, 563, 694, 949, 814, 753, 630, 764, 646, 0, 0, 0, 0, 0, 0, 0, 375, 877, 122, 634, 1079, 853, 1180, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 751, 531, 731, 771, 995, 782, 991, 639, 0, 0, 0, 0, 0, 0, 0, 0, 791, 680, 600, 548, 582, 667, 1306, 429, 97, 51, 0, 0, 0, 0, 0, 0, 1276, 461, 698, 496, 742, 480, 416, 997, 568, 208, 0, 0, 0, 0, 0, 0, 919, 432, 443, 788, 531, 915, 306, 324, 1217, 0, 0, 0, 0, 0, 0, 0, 591, 1176, 847, 359, 790, 759, 655, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 556, 802, 602, 743, 924, 720, 674, 77, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6262, 6129, 6191, 5751, 6342, 5875, 6337, 6265] avg=6144.0000 max=6342 max_rank=4 min=5751 min_rank=3 imbalance_ratio=1.032227 max_load_reduction=42810 max_load_reduction_pct=0.870972 moved_to_echo=24819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[470, 523, 603, 587, 938, 629, 697, 741, 651, 0, 0, 0, 0, 0, 0, 0, 407, 939, 123, 549, 1252, 885, 960, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 703, 613, 652, 848, 1001, 826, 934, 599, 0, 0, 0, 0, 0, 0, 0, 0, 862, 792, 659, 553, 892, 648, 1416, 393, 91, 51, 0, 0, 0, 0, 0, 0, 1058, 522, 632, 607, 780, 381, 382, 1047, 572, 209, 0, 0, 0, 0, 0, 0, 1285, 434, 519, 823, 592, 881, 287, 482, 1222, 0, 0, 0, 0, 0, 0, 0, 592, 1024, 888, 299, 486, 884, 637, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 859, 486, 749, 713, 709, 739, 961, 551, 73, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5839, 6349, 6176, 6357, 6190, 6525, 5876, 5840] avg=6144.0000 max=6525 max_rank=5 min=5839 min_rank=0 imbalance_ratio=1.062012 max_load_reduction=42627 max_load_reduction_pct=0.867249 moved_to_echo=24431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[408, 488, 720, 671, 895, 708, 668, 776, 657, 0, 0, 0, 0, 0, 0, 0, 405, 941, 123, 635, 1246, 856, 1000, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 715, 511, 746, 617, 1059, 980, 881, 639, 0, 0, 0, 0, 0, 0, 0, 0, 944, 811, 710, 495, 747, 647, 1404, 397, 86, 52, 0, 0, 0, 0, 0, 0, 1063, 482, 668, 436, 767, 504, 354, 926, 577, 210, 0, 0, 0, 0, 0, 0, 1197, 478, 496, 695, 512, 809, 343, 399, 1231, 0, 0, 0, 0, 0, 0, 0, 597, 1074, 930, 308, 638, 874, 603, 977, 0, 0, 0, 0, 0, 0, 0, 0, 924, 582, 733, 650, 628, 962, 1019, 758, 69, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5991, 6247, 6148, 6293, 5987, 6160, 6001, 6325] avg=6144.0000 max=6325 max_rank=7 min=5987 min_rank=4 imbalance_ratio=1.029460 max_load_reduction=42827 max_load_reduction_pct=0.871318 moved_to_echo=24473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[596, 615, 717, 744, 1206, 610, 599, 762, 612, 0, 0, 0, 0, 0, 0, 0, 466, 922, 116, 532, 1008, 823, 1038, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 699, 447, 750, 818, 1093, 874, 1025, 528, 0, 0, 0, 0, 0, 0, 0, 0, 835, 1008, 648, 483, 752, 664, 1403, 317, 100, 48, 0, 0, 0, 0, 0, 0, 1126, 526, 590, 468, 785, 419, 338, 972, 538, 216, 0, 0, 0, 0, 0, 0, 1077, 411, 382, 759, 474, 864, 244, 346, 1263, 0, 0, 0, 0, 0, 0, 0, 612, 1075, 984, 383, 605, 784, 661, 967, 0, 0, 0, 0, 0, 0, 0, 0, 882, 555, 867, 616, 883, 854, 928, 736, 80, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6461, 5929, 6234, 6258, 5978, 5820, 6071, 6401] avg=6144.0000 max=6461 max_rank=0 min=5820 min_rank=5 imbalance_ratio=1.051595 max_load_reduction=42691 max_load_reduction_pct=0.868551 moved_to_echo=24270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 566, 687, 613, 927, 713, 649, 802, 637, 0, 0, 0, 0, 0, 0, 0, 417, 850, 105, 592, 1199, 924, 1030, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 671, 716, 696, 631, 1088, 871, 897, 699, 0, 0, 0, 0, 0, 0, 0, 0, 847, 818, 579, 464, 770, 621, 1312, 430, 93, 53, 0, 0, 0, 0, 0, 0, 1236, 377, 642, 500, 681, 384, 460, 1045, 562, 210, 0, 0, 0, 0, 0, 0, 1033, 389, 456, 697, 683, 880, 320, 347, 1205, 0, 0, 0, 0, 0, 0, 0, 573, 1344, 1001, 441, 572, 818, 619, 894, 0, 0, 0, 0, 0, 0, 0, 0, 948, 621, 770, 606, 806, 860, 931, 685, 75, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6125, 6269, 5987, 6097, 6010, 6262, 6302] avg=6144.0000 max=6302 max_rank=7 min=5987 min_rank=3 imbalance_ratio=1.025716 max_load_reduction=42850 max_load_reduction_pct=0.871785 moved_to_echo=24671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[359, 597, 622, 761, 941, 650, 575, 969, 623, 0, 0, 0, 0, 0, 0, 0, 436, 921, 117, 533, 961, 830, 1091, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 862, 427, 683, 553, 1150, 902, 801, 918, 0, 0, 0, 0, 0, 0, 0, 0, 805, 750, 610, 456, 746, 656, 1327, 369, 78, 49, 0, 0, 0, 0, 0, 0, 1143, 518, 735, 428, 607, 485, 517, 886, 547, 226, 0, 0, 0, 0, 0, 0, 1062, 509, 260, 658, 615, 939, 410, 306, 1322, 0, 0, 0, 0, 0, 0, 0, 641, 1342, 976, 344, 548, 870, 702, 913, 0, 0, 0, 0, 0, 0, 0, 0, 976, 602, 811, 728, 627, 932, 915, 714, 63, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6036, 6296, 5846, 6092, 6081, 6336, 6368] avg=6144.0000 max=6368 max_rank=7 min=5846 min_rank=3 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[470, 523, 603, 587, 938, 629, 697, 741, 651, 0, 0, 0, 0, 0, 0, 0, 407, 939, 123, 549, 1252, 885, 960, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 703, 613, 652, 848, 1001, 826, 934, 599, 0, 0, 0, 0, 0, 0, 0, 0, 862, 792, 659, 553, 892, 648, 1416, 393, 91, 51, 0, 0, 0, 0, 0, 0, 1058, 522, 632, 607, 780, 381, 382, 1047, 572, 209, 0, 0, 0, 0, 0, 0, 1285, 434, 519, 823, 592, 881, 287, 482, 1222, 0, 0, 0, 0, 0, 0, 0, 592, 1024, 888, 299, 486, 884, 637, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 859, 486, 749, 713, 709, 739, 961, 551, 73, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5839, 6349, 6176, 6357, 6190, 6525, 5876, 5840] avg=6144.0000 max=6525 max_rank=5 min=5839 min_rank=0 imbalance_ratio=1.062012 max_load_reduction=42627 max_load_reduction_pct=0.867249 moved_to_echo=24431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 566, 687, 613, 927, 713, 649, 802, 637, 0, 0, 0, 0, 0, 0, 0, 417, 850, 105, 592, 1199, 924, 1030, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 671, 716, 696, 631, 1088, 871, 897, 699, 0, 0, 0, 0, 0, 0, 0, 0, 847, 818, 579, 464, 770, 621, 1312, 430, 93, 53, 0, 0, 0, 0, 0, 0, 1236, 377, 642, 500, 681, 384, 460, 1045, 562, 210, 0, 0, 0, 0, 0, 0, 1033, 389, 456, 697, 683, 880, 320, 347, 1205, 0, 0, 0, 0, 0, 0, 0, 573, 1344, 1001, 441, 572, 818, 619, 894, 0, 0, 0, 0, 0, 0, 0, 0, 948, 621, 770, 606, 806, 860, 931, 685, 75, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6125, 6269, 5987, 6097, 6010, 6262, 6302] avg=6144.0000 max=6302 max_rank=7 min=5987 min_rank=3 imbalance_ratio=1.025716 max_load_reduction=42850 max_load_reduction_pct=0.871785 moved_to_echo=24671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[449, 563, 694, 949, 814, 753, 630, 764, 646, 0, 0, 0, 0, 0, 0, 0, 375, 877, 122, 634, 1079, 853, 1180, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 751, 531, 731, 771, 995, 782, 991, 639, 0, 0, 0, 0, 0, 0, 0, 0, 791, 680, 600, 548, 582, 667, 1306, 429, 97, 51, 0, 0, 0, 0, 0, 0, 1276, 461, 698, 496, 742, 480, 416, 997, 568, 208, 0, 0, 0, 0, 0, 0, 919, 432, 443, 788, 531, 915, 306, 324, 1217, 0, 0, 0, 0, 0, 0, 0, 591, 1176, 847, 359, 790, 759, 655, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 556, 802, 602, 743, 924, 720, 674, 77, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6262, 6129, 6191, 5751, 6342, 5875, 6337, 6265] avg=6144.0000 max=6342 max_rank=4 min=5751 min_rank=3 imbalance_ratio=1.032227 max_load_reduction=42810 max_load_reduction_pct=0.870972 moved_to_echo=24819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 658, 723, 635, 808, 728, 741, 782, 661, 0, 0, 0, 0, 0, 0, 0, 359, 964, 125, 523, 1182, 852, 950, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 633, 574, 684, 788, 1050, 700, 986, 745, 0, 0, 0, 0, 0, 0, 0, 0, 789, 744, 573, 532, 749, 718, 1486, 464, 97, 52, 0, 0, 0, 0, 0, 0, 1222, 435, 679, 475, 823, 453, 391, 997, 581, 200, 0, 0, 0, 0, 0, 0, 1335, 475, 439, 727, 615, 847, 294, 424, 1167, 0, 0, 0, 0, 0, 0, 0, 566, 1194, 830, 435, 536, 829, 653, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 866, 503, 818, 699, 640, 801, 895, 547, 77, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6259, 6054, 6160, 6204, 6256, 6323, 6050, 5846] avg=6144.0000 max=6323 max_rank=5 min=5846 min_rank=7 imbalance_ratio=1.029134 max_load_reduction=42829 max_load_reduction_pct=0.871358 moved_to_echo=24475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[408, 488, 720, 671, 895, 708, 668, 776, 657, 0, 0, 0, 0, 0, 0, 0, 405, 941, 123, 635, 1246, 856, 1000, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 715, 511, 746, 617, 1059, 980, 881, 639, 0, 0, 0, 0, 0, 0, 0, 0, 944, 811, 710, 495, 747, 647, 1404, 397, 86, 52, 0, 0, 0, 0, 0, 0, 1063, 482, 668, 436, 767, 504, 354, 926, 577, 210, 0, 0, 0, 0, 0, 0, 1197, 478, 496, 695, 512, 809, 343, 399, 1231, 0, 0, 0, 0, 0, 0, 0, 597, 1074, 930, 308, 638, 874, 603, 977, 0, 0, 0, 0, 0, 0, 0, 0, 924, 582, 733, 650, 628, 962, 1019, 758, 69, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5991, 6247, 6148, 6293, 5987, 6160, 6001, 6325] avg=6144.0000 max=6325 max_rank=7 min=5987 min_rank=4 imbalance_ratio=1.029460 max_load_reduction=42827 max_load_reduction_pct=0.871318 moved_to_echo=24473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[462, 370, 844, 668, 817, 869, 742, 715, 656, 0, 0, 0, 0, 0, 0, 0, 330, 845, 124, 575, 1301, 1140, 926, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 731, 555, 759, 488, 900, 840, 899, 506, 0, 0, 0, 0, 0, 0, 0, 0, 841, 913, 594, 593, 631, 724, 1526, 495, 88, 51, 0, 0, 0, 0, 0, 0, 1166, 537, 656, 338, 720, 524, 391, 1069, 576, 233, 0, 0, 0, 0, 0, 0, 1276, 421, 330, 820, 561, 867, 294, 425, 1364, 0, 0, 0, 0, 0, 0, 0, 662, 957, 879, 561, 571, 759, 748, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 865, 519, 773, 598, 725, 873, 774, 608, 70, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6283, 5678, 6456, 6210, 6358, 6219, 5805] avg=6144.0000 max=6456 max_rank=3 min=5678 min_rank=2 imbalance_ratio=1.050781 max_load_reduction=42696 max_load_reduction_pct=0.868652 moved_to_echo=24592 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") @@ -1497,12 +1497,12 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 4480, 5632, 5632, 7424, 5760, 5376, 6400, 5248, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 9216, 7424, 3200, 4864, 6656, 5376, 8192, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 6528, 4992, 4224, 5888, 5376, 11264, 3328, 768, 512, 0, 0, 0, 0, 0, 0] @@ -1536,34 +1536,34 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[592, 883, 436, 547, 467, 501, 401, 679, 478, 1210, 383, 1621, 980, 429, 1046, 718, 1255, 608, 998, 906, 748, 420, 1029, 810, 670, 760, 1237, 1094, 844, 279, 800, 132, 1354, 810, 1334, 495, 1328, 830, 499, 787, 896, 454, 309, 1248, 1069, 1237, 555, 537, 378, 1011, 1353, 584, 793, 678, 397, 657, 1314, 814, 247, 390, 473, 905, 735, 720] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[535, 856, 467, 449, 537, 457, 353, 815, 454, 982, 340, 1476, 1021, 871, 1005, 679, 1213, 634, 874, 1114, 882, 490, 901, 808, 622, 597, 1568, 727, 788, 328, 750, 206, 1277, 822, 1183, 934, 1082, 770, 784, 768, 875, 529, 278, 1197, 1095, 1010, 706, 554, 420, 1105, 1486, 545, 809, 608, 301, 648, 1143, 842, 208, 535, 540, 967, 644, 688] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[550, 958, 316, 467, 575, 360, 433, 517, 513, 1060, 298, 1484, 962, 1076, 1056, 680, 1195, 661, 977, 1033, 726, 354, 909, 774, 561, 668, 1227, 789, 732, 410, 661, 195, 1471, 797, 1339, 755, 1313, 878, 777, 726, 837, 515, 363, 1267, 1041, 1207, 480, 788, 383, 973, 1160, 710, 898, 623, 388, 604, 1029, 911, 314, 507, 504, 870, 750, 797] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[647, 855, 372, 525, 551, 573, 600, 831, 483, 947, 341, 1486, 1008, 1044, 982, 564, 1159, 565, 1074, 1130, 664, 345, 1073, 893, 543, 717, 1194, 781, 973, 255, 708, 310, 1399, 862, 1145, 860, 955, 819, 608, 948, 902, 461, 249, 1109, 939, 912, 495, 692, 468, 1107, 1555, 576, 904, 515, 351, 535, 847, 830, 370, 663, 483, 1006, 713, 681] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[591, 991, 296, 428, 595, 433, 577, 478, 560, 1066, 278, 1567, 965, 556, 948, 637, 1334, 609, 978, 1121, 837, 443, 950, 843, 620, 642, 1367, 1015, 880, 246, 532, 289, 1392, 772, 1494, 430, 1178, 718, 652, 1024, 807, 446, 350, 1597, 746, 825, 743, 908, 363, 866, 1487, 802, 967, 548, 375, 536, 1025, 837, 383, 411, 565, 832, 727, 674] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[637, 849, 305, 566, 655, 316, 390, 702, 558, 973, 438, 1507, 1006, 353, 970, 734, 1115, 654, 635, 951, 954, 413, 1366, 809, 772, 752, 1019, 1152, 616, 271, 1100, 136, 1398, 714, 1087, 774, 1052, 897, 708, 777, 801, 480, 431, 1003, 828, 1098, 639, 566, 415, 1303, 1505, 648, 832, 719, 371, 591, 1293, 919, 244, 478, 425, 885, 780, 817] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[535, 856, 467, 449, 537, 457, 353, 815, 454, 982, 340, 1476, 1021, 871, 1005, 679, 1213, 634, 874, 1114, 882, 490, 901, 808, 622, 597, 1568, 727, 788, 328, 750, 206, 1277, 822, 1183, 934, 1082, 770, 784, 768, 875, 529, 278, 1197, 1095, 1010, 706, 554, 420, 1105, 1486, 545, 809, 608, 301, 648, 1143, 842, 208, 535, 540, 967, 644, 688] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[694, 958, 444, 505, 606, 397, 472, 692, 580, 876, 296, 1504, 1210, 823, 973, 862, 993, 563, 952, 1248, 798, 500, 729, 951, 554, 604, 1168, 608, 671, 312, 753, 291, 1404, 788, 1176, 608, 1080, 665, 688, 719, 838, 475, 350, 1282, 912, 1272, 577, 700, 362, 979, 1558, 707, 818, 721, 332, 545, 1174, 918, 587, 485, 429, 902, 737, 777] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[637, 849, 305, 566, 655, 316, 390, 702, 558, 973, 438, 1507, 1006, 353, 970, 734, 1115, 654, 635, 951, 954, 413, 1366, 809, 772, 752, 1019, 1152, 616, 271, 1100, 136, 1398, 714, 1087, 774, 1052, 897, 708, 777, 801, 480, 431, 1003, 828, 1098, 639, 566, 415, 1303, 1505, 648, 832, 719, 371, 591, 1293, 919, 244, 478, 425, 885, 780, 817] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[654, 876, 367, 385, 453, 370, 534, 642, 476, 1039, 410, 1513, 1072, 923, 864, 712, 1049, 650, 1077, 866, 751, 588, 860, 926, 607, 775, 1185, 594, 752, 298, 643, 377, 1406, 1008, 1350, 776, 1227, 672, 691, 852, 706, 532, 354, 1213, 940, 848, 473, 954, 527, 919, 1525, 620, 927, 588, 438, 503, 899, 831, 348, 656, 547, 1008, 825, 701] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[647, 855, 372, 525, 551, 573, 600, 831, 483, 947, 341, 1486, 1008, 1044, 982, 564, 1159, 565, 1074, 1130, 664, 345, 1073, 893, 543, 717, 1194, 781, 973, 255, 708, 310, 1399, 862, 1145, 860, 955, 819, 608, 948, 902, 461, 249, 1109, 939, 912, 495, 692, 468, 1107, 1555, 576, 904, 515, 351, 535, 847, 830, 370, 663, 483, 1006, 713, 681] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[535, 856, 467, 449, 537, 457, 353, 815, 1277, 253, 0, 0, 0, 0, 0, 0, 454, 982, 340, 776, 1021, 871, 1005, 679, 0, 0, 0, 0, 0, 0, 0, 0, 488, 634, 874, 1114, 882, 490, 901, 808, 0, 0, 0, 0, 0, 0, 0, 0, 622, 597, 1568, 727, 788, 328, 750, 206, 447, 226, 0, 0, 0, 0, 0, 0, 0, 822, 1109, 934, 1082, 770, 784, 768, 0, 0, 0, 0, 0, 0, 0, 0, 875, 529, 278, 1155, 1095, 1010, 706, 554, 0, 0, 0, 0, 0, 0, 0, 0, 420, 1105, 1486, 545, 809, 608, 301, 648, 74, 42, 32, 0, 0, 0, 0, 0, 1143, 842, 208, 535, 540, 967, 644, 688, 467, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 6128, 6191, 6259, 6269, 6202, 6070, 6034] avg=6144.0000 max=6269 max_rank=4 min=5999 min_rank=0 imbalance_ratio=1.020345 max_load_reduction=42883 max_load_reduction_pct=0.872457 moved_to_echo=24575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[592, 883, 436, 547, 467, 501, 401, 679, 1354, 278, 0, 0, 0, 0, 0, 0, 478, 1210, 383, 852, 980, 429, 1046, 718, 0, 0, 0, 0, 0, 0, 0, 0, 504, 608, 998, 906, 748, 420, 1029, 810, 0, 0, 0, 0, 0, 0, 0, 0, 670, 760, 1237, 1094, 844, 279, 800, 132, 491, 234, 0, 0, 0, 0, 0, 0, 0, 810, 1251, 495, 1328, 830, 499, 787, 0, 0, 0, 0, 0, 0, 0, 0, 896, 454, 309, 1205, 1069, 1237, 555, 537, 0, 0, 0, 0, 0, 0, 0, 0, 378, 1011, 1353, 584, 793, 678, 397, 657, 83, 43, 33, 0, 0, 0, 0, 0, 1314, 814, 247, 390, 473, 905, 735, 720, 484, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6096, 6023, 6541, 6000, 6262, 6010, 6082] avg=6144.0000 max=6541 max_rank=3 min=6000 min_rank=4 imbalance_ratio=1.064616 max_load_reduction=42611 max_load_reduction_pct=0.866923 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[591, 991, 296, 428, 595, 433, 577, 478, 1392, 269, 0, 0, 0, 0, 0, 0, 560, 1066, 278, 823, 965, 556, 948, 637, 0, 0, 0, 0, 0, 0, 0, 0, 536, 609, 978, 1121, 837, 443, 950, 843, 0, 0, 0, 0, 0, 0, 0, 0, 620, 642, 1367, 1015, 880, 246, 532, 289, 475, 249, 0, 0, 0, 0, 0, 0, 0, 772, 1401, 430, 1178, 718, 652, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 807, 446, 350, 1541, 746, 825, 743, 908, 0, 0, 0, 0, 0, 0, 0, 0, 363, 866, 1487, 802, 967, 548, 375, 536, 93, 56, 35, 0, 0, 0, 0, 0, 1025, 837, 383, 411, 565, 832, 727, 674, 514, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6050, 5833, 6317, 6315, 6175, 6366, 6128, 5968] avg=6144.0000 max=6366 max_rank=5 min=5833 min_rank=1 imbalance_ratio=1.036133 max_load_reduction=42786 max_load_reduction_pct=0.870483 moved_to_echo=24637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[592, 883, 436, 547, 467, 501, 401, 679, 1354, 278, 0, 0, 0, 0, 0, 0, 478, 1210, 383, 852, 980, 429, 1046, 718, 0, 0, 0, 0, 0, 0, 0, 0, 504, 608, 998, 906, 748, 420, 1029, 810, 0, 0, 0, 0, 0, 0, 0, 0, 670, 760, 1237, 1094, 844, 279, 800, 132, 491, 234, 0, 0, 0, 0, 0, 0, 0, 810, 1251, 495, 1328, 830, 499, 787, 0, 0, 0, 0, 0, 0, 0, 0, 896, 454, 309, 1205, 1069, 1237, 555, 537, 0, 0, 0, 0, 0, 0, 0, 0, 378, 1011, 1353, 584, 793, 678, 397, 657, 83, 43, 33, 0, 0, 0, 0, 0, 1314, 814, 247, 390, 473, 905, 735, 720, 484, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6096, 6023, 6541, 6000, 6262, 6010, 6082] avg=6144.0000 max=6541 max_rank=3 min=6000 min_rank=4 imbalance_ratio=1.064616 max_load_reduction=42611 max_load_reduction_pct=0.866923 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[637, 849, 305, 566, 655, 316, 390, 702, 1398, 258, 0, 0, 0, 0, 0, 0, 558, 973, 438, 793, 1006, 353, 970, 734, 0, 0, 0, 0, 0, 0, 0, 0, 447, 654, 635, 951, 954, 413, 1366, 809, 0, 0, 0, 0, 0, 0, 0, 0, 772, 752, 1019, 1152, 616, 271, 1100, 136, 456, 208, 0, 0, 0, 0, 0, 0, 0, 714, 1019, 774, 1052, 897, 708, 777, 0, 0, 0, 0, 0, 0, 0, 0, 801, 480, 431, 968, 828, 1098, 639, 566, 0, 0, 0, 0, 0, 0, 0, 0, 415, 1303, 1505, 648, 832, 719, 371, 591, 68, 35, 30, 0, 0, 0, 0, 0, 1293, 919, 244, 478, 425, 885, 780, 817, 430, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 5825, 6229, 6482, 5941, 5811, 6517, 6271] avg=6144.0000 max=6517 max_rank=6 min=5811 min_rank=5 imbalance_ratio=1.060710 max_load_reduction=42635 max_load_reduction_pct=0.867411 moved_to_echo=24540 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[550, 958, 316, 467, 575, 360, 433, 517, 1471, 258, 0, 0, 0, 0, 0, 0, 513, 1060, 298, 772, 962, 1076, 1056, 680, 0, 0, 0, 0, 0, 0, 0, 0, 469, 661, 977, 1033, 726, 354, 909, 774, 0, 0, 0, 0, 0, 0, 0, 0, 561, 668, 1227, 789, 732, 410, 661, 195, 454, 226, 0, 0, 0, 0, 0, 0, 0, 797, 1252, 755, 1313, 878, 777, 726, 0, 0, 0, 0, 0, 0, 0, 0, 837, 515, 363, 1219, 1041, 1207, 480, 788, 0, 0, 0, 0, 0, 0, 0, 0, 383, 973, 1160, 710, 898, 623, 388, 604, 87, 48, 36, 0, 0, 0, 0, 0, 1029, 911, 314, 507, 504, 870, 750, 797, 464, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5905, 6417, 5903, 5923, 6498, 6450, 5910, 6146] avg=6144.0000 max=6498 max_rank=4 min=5903 min_rank=2 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=25004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[654, 876, 367, 385, 453, 370, 534, 642, 1406, 259, 0, 0, 0, 0, 0, 0, 476, 1039, 410, 796, 1072, 923, 864, 712, 0, 0, 0, 0, 0, 0, 0, 0, 422, 650, 1077, 866, 751, 588, 860, 926, 0, 0, 0, 0, 0, 0, 0, 0, 607, 775, 1185, 594, 752, 298, 643, 377, 458, 195, 0, 0, 0, 0, 0, 0, 0, 1008, 1266, 776, 1227, 672, 691, 852, 0, 0, 0, 0, 0, 0, 0, 0, 706, 532, 354, 1171, 940, 848, 473, 954, 0, 0, 0, 0, 0, 0, 0, 0, 527, 919, 1525, 620, 927, 588, 438, 503, 84, 42, 28, 0, 0, 0, 0, 0, 899, 831, 348, 656, 547, 1008, 825, 701, 404, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5946, 6292, 6140, 5884, 6492, 5978, 6201, 6219] avg=6144.0000 max=6492 max_rank=4 min=5884 min_rank=3 imbalance_ratio=1.056641 max_load_reduction=42660 max_load_reduction_pct=0.867920 moved_to_echo=24890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[647, 855, 372, 525, 551, 573, 600, 831, 1399, 255, 0, 0, 0, 0, 0, 0, 483, 947, 341, 781, 1008, 1044, 982, 564, 0, 0, 0, 0, 0, 0, 0, 0, 465, 565, 1074, 1130, 664, 345, 1073, 893, 0, 0, 0, 0, 0, 0, 0, 0, 543, 717, 1194, 781, 973, 255, 708, 310, 450, 216, 0, 0, 0, 0, 0, 0, 0, 862, 1074, 860, 955, 819, 608, 948, 0, 0, 0, 0, 0, 0, 0, 0, 902, 461, 249, 1071, 939, 912, 495, 692, 0, 0, 0, 0, 0, 0, 0, 0, 468, 1107, 1555, 576, 904, 515, 351, 535, 71, 38, 31, 0, 0, 0, 0, 0, 847, 830, 370, 663, 483, 1006, 713, 681, 447, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6608, 6150, 6209, 6147, 6126, 5721, 6151, 6040] avg=6144.0000 max=6608 max_rank=0 min=5721 min_rank=5 imbalance_ratio=1.075521 max_load_reduction=42544 max_load_reduction_pct=0.865560 moved_to_echo=24038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[535, 856, 467, 449, 537, 457, 353, 815, 1277, 253, 0, 0, 0, 0, 0, 0, 454, 982, 340, 776, 1021, 871, 1005, 679, 0, 0, 0, 0, 0, 0, 0, 0, 488, 634, 874, 1114, 882, 490, 901, 808, 0, 0, 0, 0, 0, 0, 0, 0, 622, 597, 1568, 727, 788, 328, 750, 206, 447, 226, 0, 0, 0, 0, 0, 0, 0, 822, 1109, 934, 1082, 770, 784, 768, 0, 0, 0, 0, 0, 0, 0, 0, 875, 529, 278, 1155, 1095, 1010, 706, 554, 0, 0, 0, 0, 0, 0, 0, 0, 420, 1105, 1486, 545, 809, 608, 301, 648, 74, 42, 32, 0, 0, 0, 0, 0, 1143, 842, 208, 535, 540, 967, 644, 688, 467, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 6128, 6191, 6259, 6269, 6202, 6070, 6034] avg=6144.0000 max=6269 max_rank=4 min=5999 min_rank=0 imbalance_ratio=1.020345 max_load_reduction=42883 max_load_reduction_pct=0.872457 moved_to_echo=24575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[694, 958, 444, 505, 606, 397, 472, 692, 1404, 258, 0, 0, 0, 0, 0, 0, 580, 876, 296, 791, 1210, 823, 973, 862, 0, 0, 0, 0, 0, 0, 0, 0, 399, 563, 952, 1248, 798, 500, 729, 951, 0, 0, 0, 0, 0, 0, 0, 0, 554, 604, 1168, 608, 671, 312, 753, 291, 455, 185, 0, 0, 0, 0, 0, 0, 0, 788, 1103, 608, 1080, 665, 688, 719, 0, 0, 0, 0, 0, 0, 0, 0, 838, 475, 350, 1238, 912, 1272, 577, 700, 0, 0, 0, 0, 0, 0, 0, 0, 362, 979, 1558, 707, 818, 721, 332, 545, 73, 44, 26, 0, 0, 0, 0, 0, 1174, 918, 587, 485, 429, 902, 737, 777, 383, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6430, 6411, 6140, 5601, 5651, 6362, 6165, 6392] avg=6144.0000 max=6430 max_rank=0 min=5601 min_rank=3 imbalance_ratio=1.046549 max_load_reduction=42722 max_load_reduction_pct=0.869181 moved_to_echo=24570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[647, 855, 372, 525, 551, 573, 600, 831, 1399, 255, 0, 0, 0, 0, 0, 0, 483, 947, 341, 781, 1008, 1044, 982, 564, 0, 0, 0, 0, 0, 0, 0, 0, 465, 565, 1074, 1130, 664, 345, 1073, 893, 0, 0, 0, 0, 0, 0, 0, 0, 543, 717, 1194, 781, 973, 255, 708, 310, 450, 216, 0, 0, 0, 0, 0, 0, 0, 862, 1074, 860, 955, 819, 608, 948, 0, 0, 0, 0, 0, 0, 0, 0, 902, 461, 249, 1071, 939, 912, 495, 692, 0, 0, 0, 0, 0, 0, 0, 0, 468, 1107, 1555, 576, 904, 515, 351, 535, 71, 38, 31, 0, 0, 0, 0, 0, 847, 830, 370, 663, 483, 1006, 713, 681, 447, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6608, 6150, 6209, 6147, 6126, 5721, 6151, 6040] avg=6144.0000 max=6608 max_rank=0 min=5721 min_rank=5 imbalance_ratio=1.075521 max_load_reduction=42544 max_load_reduction_pct=0.865560 moved_to_echo=24038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[654, 876, 367, 385, 453, 370, 534, 642, 1406, 259, 0, 0, 0, 0, 0, 0, 476, 1039, 410, 796, 1072, 923, 864, 712, 0, 0, 0, 0, 0, 0, 0, 0, 422, 650, 1077, 866, 751, 588, 860, 926, 0, 0, 0, 0, 0, 0, 0, 0, 607, 775, 1185, 594, 752, 298, 643, 377, 458, 195, 0, 0, 0, 0, 0, 0, 0, 1008, 1266, 776, 1227, 672, 691, 852, 0, 0, 0, 0, 0, 0, 0, 0, 706, 532, 354, 1171, 940, 848, 473, 954, 0, 0, 0, 0, 0, 0, 0, 0, 527, 919, 1525, 620, 927, 588, 438, 503, 84, 42, 28, 0, 0, 0, 0, 0, 899, 831, 348, 656, 547, 1008, 825, 701, 404, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5946, 6292, 6140, 5884, 6492, 5978, 6201, 6219] avg=6144.0000 max=6492 max_rank=4 min=5884 min_rank=3 imbalance_ratio=1.056641 max_load_reduction=42660 max_load_reduction_pct=0.867920 moved_to_echo=24890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[550, 958, 316, 467, 575, 360, 433, 517, 1471, 258, 0, 0, 0, 0, 0, 0, 513, 1060, 298, 772, 962, 1076, 1056, 680, 0, 0, 0, 0, 0, 0, 0, 0, 469, 661, 977, 1033, 726, 354, 909, 774, 0, 0, 0, 0, 0, 0, 0, 0, 561, 668, 1227, 789, 732, 410, 661, 195, 454, 226, 0, 0, 0, 0, 0, 0, 0, 797, 1252, 755, 1313, 878, 777, 726, 0, 0, 0, 0, 0, 0, 0, 0, 837, 515, 363, 1219, 1041, 1207, 480, 788, 0, 0, 0, 0, 0, 0, 0, 0, 383, 973, 1160, 710, 898, 623, 388, 604, 87, 48, 36, 0, 0, 0, 0, 0, 1029, 911, 314, 507, 504, 870, 750, 797, 464, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5905, 6417, 5903, 5923, 6498, 6450, 5910, 6146] avg=6144.0000 max=6498 max_rank=4 min=5903 min_rank=2 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=25004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 7296, 3072, 3968, 4480, 3456, 3840, 5376, 11136, 2176, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4992, 5632, 9984, 6784, 6272, 2432, 6016, 2048, 3712, 1792, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 8320, 11648, 5248, 7040, 5120, 3072, 4736, 640, 384, 256, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 3968, 2688, 9600, 7680, 8448, 4736, 5760, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 6656, 9600, 5632, 9216, 6272, 5504, 6656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 3968, 2688, 9600, 7680, 8448, 4736, 5760, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8832, 6912, 2816, 4224, 3968, 7424, 6016, 5888, 3712, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 4992, 7680, 8448, 6400, 3584, 7936, 6912, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 8192, 2816, 6400, 8320, 6144, 7936, 5632, 0, 0, 0, 0, 0, 0, 0, 0] @@ -1575,109 +1575,109 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[690, 712, 1004, 447, 1242, 676, 1031, 1329, 791, 402, 368, 265, 876, 769, 637, 485, 1262, 1096, 821, 941, 586, 932, 655, 829, 559, 510, 1130, 990, 679, 1478, 749, 427, 320, 659, 477, 1070, 645, 629, 427, 375, 590, 738, 970, 701, 840, 532, 777, 910, 1420, 850, 1614, 605, 253, 440, 771, 822, 1170, 404, 510, 332, 1079, 1588, 744, 522] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[669, 592, 787, 461, 967, 530, 1131, 1580, 783, 383, 428, 317, 856, 914, 753, 639, 1466, 827, 699, 669, 523, 929, 431, 702, 524, 453, 1120, 1052, 722, 1316, 877, 579, 305, 932, 435, 1533, 485, 401, 519, 539, 444, 805, 586, 818, 777, 771, 808, 659, 897, 1106, 2598, 615, 257, 652, 750, 671, 1015, 272, 508, 317, 790, 1768, 870, 570] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[582, 607, 748, 312, 933, 637, 1068, 1484, 896, 530, 339, 220, 916, 862, 514, 498, 954, 1138, 779, 618, 591, 819, 526, 549, 523, 395, 1081, 1225, 1018, 1327, 810, 621, 576, 843, 492, 1278, 509, 401, 470, 402, 428, 1150, 770, 766, 870, 783, 795, 816, 663, 1058, 2230, 439, 207, 727, 1061, 801, 1108, 327, 670, 439, 535, 1787, 999, 632] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[735, 687, 587, 548, 1425, 690, 1158, 1235, 901, 297, 257, 364, 790, 813, 650, 739, 1186, 1316, 720, 838, 556, 1085, 587, 759, 818, 288, 839, 1195, 545, 1407, 955, 378, 361, 828, 502, 1238, 771, 479, 511, 361, 402, 722, 902, 517, 717, 669, 884, 692, 1081, 960, 2456, 720, 116, 552, 750, 597, 1346, 413, 438, 299, 736, 1502, 845, 437] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[779, 511, 886, 543, 1176, 639, 1127, 1422, 819, 495, 326, 643, 918, 738, 492, 583, 1009, 1008, 926, 816, 551, 1004, 708, 1039, 609, 297, 1155, 1085, 605, 1639, 854, 761, 633, 832, 504, 1228, 430, 458, 397, 422, 473, 1061, 860, 490, 712, 739, 904, 654, 935, 909, 1473, 592, 219, 704, 741, 557, 1150, 395, 466, 348, 937, 1442, 890, 434] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[690, 712, 1004, 447, 1242, 676, 1031, 1329, 791, 402, 368, 265, 876, 769, 637, 485, 1262, 1096, 821, 941, 586, 932, 655, 829, 559, 510, 1130, 990, 679, 1478, 749, 427, 320, 659, 477, 1070, 645, 629, 427, 375, 590, 738, 970, 701, 840, 532, 777, 910, 1420, 850, 1614, 605, 253, 440, 771, 822, 1170, 404, 510, 332, 1079, 1588, 744, 522] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[684, 653, 734, 522, 1447, 319, 1121, 1310, 507, 429, 316, 297, 929, 848, 987, 488, 1039, 873, 1141, 617, 674, 918, 623, 382, 796, 452, 1102, 1004, 559, 1524, 793, 852, 515, 845, 487, 955, 735, 491, 496, 354, 507, 883, 777, 639, 940, 728, 925, 771, 1220, 851, 1963, 713, 245, 473, 638, 800, 973, 437, 546, 183, 1071, 1536, 1043, 472] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[730, 705, 911, 453, 1050, 537, 1115, 1366, 925, 376, 285, 321, 909, 841, 460, 573, 982, 1001, 651, 914, 707, 919, 764, 1160, 675, 469, 968, 1042, 669, 1350, 809, 360, 364, 974, 504, 1212, 371, 500, 419, 386, 437, 964, 1160, 570, 962, 672, 767, 638, 879, 1005, 2291, 478, 214, 621, 646, 620, 1053, 387, 594, 390, 871, 1860, 813, 533] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[777, 597, 1189, 426, 1118, 479, 1162, 1673, 652, 395, 295, 209, 954, 762, 619, 537, 1154, 1099, 781, 689, 811, 707, 635, 1008, 475, 461, 1244, 1125, 958, 1439, 878, 452, 287, 794, 427, 1385, 654, 705, 612, 287, 359, 708, 909, 665, 842, 586, 827, 715, 981, 951, 2133, 448, 211, 663, 807, 893, 1057, 274, 592, 271, 727, 1575, 531, 516] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[582, 607, 748, 312, 933, 637, 1068, 1484, 896, 530, 339, 220, 916, 862, 514, 498, 954, 1138, 779, 618, 591, 819, 526, 549, 523, 395, 1081, 1225, 1018, 1327, 810, 621, 576, 843, 492, 1278, 509, 401, 470, 402, 428, 1150, 770, 766, 870, 783, 795, 816, 663, 1058, 2230, 439, 207, 727, 1061, 801, 1108, 327, 670, 439, 535, 1787, 999, 632] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[669, 592, 787, 461, 967, 530, 1131, 1580, 783, 383, 428, 317, 856, 914, 753, 639, 1466, 827, 699, 669, 523, 929, 431, 702, 524, 453, 1120, 1052, 722, 1316, 877, 579, 305, 932, 435, 1533, 485, 401, 519, 539, 444, 805, 586, 818, 777, 771, 808, 659, 897, 1106, 2598, 615, 257, 652, 750, 671, 1015, 272, 508, 317, 790, 1768, 870, 570] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[690, 712, 1004, 447, 1242, 676, 1031, 597, 0, 0, 0, 0, 0, 0, 0, 0, 791, 402, 368, 265, 876, 769, 637, 485, 622, 525, 0, 0, 0, 0, 0, 0, 629, 1096, 821, 941, 586, 932, 655, 829, 0, 0, 0, 0, 0, 0, 0, 0, 559, 510, 1130, 990, 679, 848, 749, 427, 0, 0, 0, 0, 0, 0, 0, 0, 320, 659, 477, 1070, 645, 629, 427, 375, 630, 423, 207, 0, 0, 0, 0, 0, 590, 738, 970, 701, 840, 532, 777, 910, 210, 22, 0, 0, 0, 0, 0, 0, 1420, 850, 992, 605, 253, 440, 771, 822, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 404, 510, 332, 1079, 1566, 744, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6399, 5740, 6489, 5892, 5862, 6290, 6153, 6327] avg=6144.0000 max=6489 max_rank=2 min=5740 min_rank=1 imbalance_ratio=1.056152 max_load_reduction=42663 max_load_reduction_pct=0.867981 moved_to_echo=24632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[669, 592, 787, 461, 967, 530, 1131, 708, 0, 0, 0, 0, 0, 0, 0, 0, 783, 383, 428, 317, 856, 914, 753, 639, 1001, 625, 0, 0, 0, 0, 0, 0, 731, 827, 699, 669, 523, 929, 431, 702, 0, 0, 0, 0, 0, 0, 0, 0, 524, 453, 1120, 1052, 722, 755, 877, 579, 0, 0, 0, 0, 0, 0, 0, 0, 305, 932, 435, 1533, 485, 401, 519, 539, 561, 491, 247, 0, 0, 0, 0, 0, 444, 805, 586, 818, 777, 771, 808, 659, 244, 25, 0, 0, 0, 0, 0, 0, 897, 1106, 1597, 615, 257, 652, 750, 671, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 272, 508, 317, 790, 1743, 870, 570, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5845, 6699, 5511, 6082, 6448, 5937, 6545, 6085] avg=6144.0000 max=6699 max_rank=1 min=5511 min_rank=2 imbalance_ratio=1.090332 max_load_reduction=42453 max_load_reduction_pct=0.863708 moved_to_echo=25015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[735, 687, 587, 548, 1425, 690, 1158, 554, 0, 0, 0, 0, 0, 0, 0, 0, 901, 297, 257, 364, 790, 813, 650, 739, 946, 488, 0, 0, 0, 0, 0, 0, 591, 1316, 720, 838, 556, 1085, 587, 759, 0, 0, 0, 0, 0, 0, 0, 0, 818, 288, 839, 1195, 545, 807, 955, 378, 0, 0, 0, 0, 0, 0, 0, 0, 361, 828, 502, 1238, 771, 479, 511, 361, 600, 397, 193, 0, 0, 0, 0, 0, 402, 722, 902, 517, 717, 669, 884, 692, 198, 21, 0, 0, 0, 0, 0, 0, 1081, 960, 1510, 720, 116, 552, 750, 597, 0, 0, 0, 0, 0, 0, 0, 0, 1346, 413, 438, 299, 736, 1481, 845, 437, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6384, 6245, 6452, 5825, 6241, 5724, 6286, 5995] avg=6144.0000 max=6452 max_rank=2 min=5724 min_rank=5 imbalance_ratio=1.050130 max_load_reduction=42700 max_load_reduction_pct=0.868734 moved_to_echo=24246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[690, 712, 1004, 447, 1242, 676, 1031, 597, 0, 0, 0, 0, 0, 0, 0, 0, 791, 402, 368, 265, 876, 769, 637, 485, 622, 525, 0, 0, 0, 0, 0, 0, 629, 1096, 821, 941, 586, 932, 655, 829, 0, 0, 0, 0, 0, 0, 0, 0, 559, 510, 1130, 990, 679, 848, 749, 427, 0, 0, 0, 0, 0, 0, 0, 0, 320, 659, 477, 1070, 645, 629, 427, 375, 630, 423, 207, 0, 0, 0, 0, 0, 590, 738, 970, 701, 840, 532, 777, 910, 210, 22, 0, 0, 0, 0, 0, 0, 1420, 850, 992, 605, 253, 440, 771, 822, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 404, 510, 332, 1079, 1566, 744, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6399, 5740, 6489, 5892, 5862, 6290, 6153, 6327] avg=6144.0000 max=6489 max_rank=2 min=5740 min_rank=1 imbalance_ratio=1.056152 max_load_reduction=42663 max_load_reduction_pct=0.867981 moved_to_echo=24632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[684, 653, 734, 522, 1447, 319, 1121, 588, 0, 0, 0, 0, 0, 0, 0, 0, 507, 429, 316, 297, 929, 848, 987, 488, 756, 518, 0, 0, 0, 0, 0, 0, 518, 873, 1141, 617, 674, 918, 623, 382, 0, 0, 0, 0, 0, 0, 0, 0, 796, 452, 1102, 1004, 559, 874, 793, 852, 0, 0, 0, 0, 0, 0, 0, 0, 515, 845, 487, 955, 735, 491, 496, 354, 650, 348, 204, 0, 0, 0, 0, 0, 507, 883, 777, 639, 940, 728, 925, 771, 173, 21, 0, 0, 0, 0, 0, 0, 1220, 851, 1207, 713, 245, 473, 638, 800, 0, 0, 0, 0, 0, 0, 0, 0, 973, 437, 546, 183, 1071, 1515, 1043, 472, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6068, 6075, 5746, 6432, 6080, 6364, 6147, 6240] avg=6144.0000 max=6432 max_rank=3 min=5746 min_rank=2 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[669, 592, 787, 461, 967, 530, 1131, 708, 0, 0, 0, 0, 0, 0, 0, 0, 783, 383, 428, 317, 856, 914, 753, 639, 1001, 625, 0, 0, 0, 0, 0, 0, 731, 827, 699, 669, 523, 929, 431, 702, 0, 0, 0, 0, 0, 0, 0, 0, 524, 453, 1120, 1052, 722, 755, 877, 579, 0, 0, 0, 0, 0, 0, 0, 0, 305, 932, 435, 1533, 485, 401, 519, 539, 561, 491, 247, 0, 0, 0, 0, 0, 444, 805, 586, 818, 777, 771, 808, 659, 244, 25, 0, 0, 0, 0, 0, 0, 897, 1106, 1597, 615, 257, 652, 750, 671, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 272, 508, 317, 790, 1743, 870, 570, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5845, 6699, 5511, 6082, 6448, 5937, 6545, 6085] avg=6144.0000 max=6699 max_rank=1 min=5511 min_rank=2 imbalance_ratio=1.090332 max_load_reduction=42453 max_load_reduction_pct=0.863708 moved_to_echo=25015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[582, 607, 748, 312, 933, 637, 1068, 666, 0, 0, 0, 0, 0, 0, 0, 0, 896, 530, 339, 220, 916, 862, 514, 498, 859, 587, 0, 0, 0, 0, 0, 0, 476, 1138, 779, 618, 591, 819, 526, 549, 0, 0, 0, 0, 0, 0, 0, 0, 523, 395, 1081, 1225, 1018, 761, 810, 621, 0, 0, 0, 0, 0, 0, 0, 0, 576, 843, 492, 1278, 509, 401, 470, 402, 566, 319, 231, 0, 0, 0, 0, 0, 428, 1150, 770, 766, 870, 783, 795, 816, 159, 25, 0, 0, 0, 0, 0, 0, 663, 1058, 1371, 439, 207, 727, 1061, 801, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 327, 670, 439, 535, 1762, 999, 632, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5553, 6221, 5496, 6434, 6087, 6562, 6327, 6472] avg=6144.0000 max=6562 max_rank=5 min=5496 min_rank=2 imbalance_ratio=1.068034 max_load_reduction=42590 max_load_reduction_pct=0.866496 moved_to_echo=25448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[730, 705, 911, 453, 1050, 537, 1115, 613, 0, 0, 0, 0, 0, 0, 0, 0, 925, 376, 285, 321, 909, 841, 460, 573, 883, 540, 0, 0, 0, 0, 0, 0, 489, 1001, 651, 914, 707, 919, 764, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 675, 469, 968, 1042, 669, 774, 809, 360, 0, 0, 0, 0, 0, 0, 0, 0, 364, 974, 504, 1212, 371, 500, 419, 386, 576, 329, 213, 0, 0, 0, 0, 0, 437, 964, 1160, 570, 962, 672, 767, 638, 164, 26, 0, 0, 0, 0, 0, 0, 879, 1005, 1408, 478, 214, 621, 646, 620, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 387, 594, 390, 871, 1834, 813, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6113, 6605, 5766, 5848, 6360, 5871, 6475] avg=6144.0000 max=6605 max_rank=2 min=5766 min_rank=3 imbalance_ratio=1.075033 max_load_reduction=42547 max_load_reduction_pct=0.865621 moved_to_echo=24554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[777, 597, 1189, 426, 1118, 479, 1162, 750, 0, 0, 0, 0, 0, 0, 0, 0, 652, 395, 295, 209, 954, 762, 619, 537, 822, 662, 0, 0, 0, 0, 0, 0, 576, 1099, 781, 689, 811, 707, 635, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 475, 461, 1244, 1125, 958, 825, 878, 452, 0, 0, 0, 0, 0, 0, 0, 0, 287, 794, 427, 1385, 654, 705, 612, 287, 614, 386, 261, 0, 0, 0, 0, 0, 359, 708, 909, 665, 842, 586, 827, 715, 192, 22, 0, 0, 0, 0, 0, 0, 981, 951, 1311, 448, 211, 663, 807, 893, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 274, 592, 271, 727, 1553, 531, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6498, 5907, 6306, 6418, 6412, 5825, 6265, 5521] avg=6144.0000 max=6498 max_rank=0 min=5521 min_rank=7 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=24023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[779, 511, 886, 543, 1176, 639, 1127, 630, 0, 0, 0, 0, 0, 0, 0, 0, 819, 495, 326, 643, 918, 738, 492, 583, 572, 566, 0, 0, 0, 0, 0, 0, 495, 1008, 926, 816, 551, 1004, 708, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 609, 297, 1155, 1085, 605, 937, 854, 761, 0, 0, 0, 0, 0, 0, 0, 0, 633, 832, 504, 1228, 430, 458, 397, 422, 702, 342, 226, 0, 0, 0, 0, 0, 473, 1061, 860, 490, 712, 739, 904, 654, 172, 25, 0, 0, 0, 0, 0, 0, 935, 909, 901, 592, 219, 704, 741, 557, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 395, 466, 348, 937, 1417, 890, 434, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6291, 6152, 6547, 6303, 6174, 6090, 5558, 6037] avg=6144.0000 max=6547 max_rank=2 min=5558 min_rank=6 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=23859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[777, 597, 1189, 426, 1118, 479, 1162, 750, 0, 0, 0, 0, 0, 0, 0, 0, 652, 395, 295, 209, 954, 762, 619, 537, 822, 662, 0, 0, 0, 0, 0, 0, 576, 1099, 781, 689, 811, 707, 635, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 475, 461, 1244, 1125, 958, 825, 878, 452, 0, 0, 0, 0, 0, 0, 0, 0, 287, 794, 427, 1385, 654, 705, 612, 287, 614, 386, 261, 0, 0, 0, 0, 0, 359, 708, 909, 665, 842, 586, 827, 715, 192, 22, 0, 0, 0, 0, 0, 0, 981, 951, 1311, 448, 211, 663, 807, 893, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 274, 592, 271, 727, 1553, 531, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6498, 5907, 6306, 6418, 6412, 5825, 6265, 5521] avg=6144.0000 max=6498 max_rank=0 min=5521 min_rank=7 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=24023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 6784, 3840, 9984, 4608, 4096, 3968, 3200, 4992, 3072, 1792, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8192, 7808, 10368, 4736, 1792, 4864, 6272, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 7040, 7040, 5248, 6784, 5504, 6784, 5888, 1536, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4992, 3328, 8704, 8832, 5760, 6656, 6784, 4480, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8960, 2944, 4352, 2688, 6784, 12928, 6784, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 5120, 6912, 3712, 9472, 4608, 8960, 5120, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 3328, 2688, 2688, 7168, 6656, 5120, 4608, 6528, 4608, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4992, 3328, 8704, 8832, 5760, 6656, 6784, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 7040, 7040, 5248, 6784, 5504, 6784, 5888, 1536, 256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 8448, 6528, 6144, 5120, 7424, 4992, 6528, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 5120, 6912, 3712, 9472, 4608, 8960, 5120, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1301, 1146, 924, 503, 797, 1690, 978, 552, 740, 626, 829, 622, 647, 411, 472, 336, 394, 1358, 304, 1755, 1030, 629, 568, 731, 673, 377, 964, 977, 591, 1916, 699, 826, 339, 519, 430, 1135, 712, 491, 356, 575, 745, 713, 168, 531, 602, 463, 802, 697, 311, 1560, 1199, 316, 658, 1341, 1581, 709, 409, 1602, 511, 995, 658, 581, 768, 309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1213, 993, 810, 383, 491, 2001, 1106, 848, 739, 660, 662, 634, 502, 683, 607, 468, 525, 1264, 373, 1593, 1069, 621, 790, 506, 503, 405, 1030, 1056, 470, 2214, 576, 692, 635, 300, 539, 1098, 944, 504, 555, 693, 541, 795, 224, 669, 597, 665, 719, 307, 324, 1249, 1326, 352, 480, 1198, 1074, 724, 467, 1131, 512, 1573, 533, 672, 642, 623] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1005, 972, 772, 473, 795, 1632, 873, 615, 690, 685, 566, 695, 535, 494, 335, 558, 445, 1179, 171, 1829, 669, 382, 945, 725, 505, 484, 418, 1051, 754, 1964, 492, 626, 547, 264, 420, 1800, 1169, 324, 425, 981, 754, 1050, 146, 509, 524, 635, 1071, 480, 366, 1441, 1677, 344, 442, 1426, 1394, 700, 348, 1117, 476, 1037, 531, 1242, 682, 496] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[926, 1081, 786, 301, 632, 1952, 1096, 890, 811, 637, 713, 502, 612, 588, 488, 371, 406, 1108, 346, 1358, 680, 801, 524, 1129, 680, 258, 654, 1165, 562, 1723, 808, 731, 605, 325, 987, 733, 1011, 337, 455, 795, 783, 631, 373, 1025, 1070, 476, 376, 536, 317, 1565, 1507, 326, 574, 1230, 1183, 682, 456, 1706, 458, 984, 489, 717, 785, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1212, 1284, 1010, 436, 692, 1968, 988, 597, 672, 568, 1045, 694, 669, 531, 378, 414, 363, 1204, 332, 1669, 528, 601, 673, 717, 507, 368, 672, 1450, 715, 2000, 423, 796, 459, 467, 392, 1609, 707, 444, 420, 928, 654, 877, 356, 736, 672, 533, 936, 537, 377, 1100, 1389, 261, 742, 929, 1679, 713, 331, 1074, 452, 915, 769, 507, 586, 425] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1301, 1146, 924, 503, 797, 1690, 978, 552, 740, 626, 829, 622, 647, 411, 472, 336, 394, 1358, 304, 1755, 1030, 629, 568, 731, 673, 377, 964, 977, 591, 1916, 699, 826, 339, 519, 430, 1135, 712, 491, 356, 575, 745, 713, 168, 531, 602, 463, 802, 697, 311, 1560, 1199, 316, 658, 1341, 1581, 709, 409, 1602, 511, 995, 658, 581, 768, 309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1182, 1219, 759, 462, 542, 1779, 868, 661, 624, 693, 599, 679, 600, 639, 493, 642, 577, 1229, 303, 1607, 832, 703, 860, 864, 659, 375, 975, 1072, 653, 2073, 452, 724, 327, 428, 694, 1321, 941, 325, 504, 783, 640, 564, 162, 661, 503, 473, 783, 424, 382, 1616, 1556, 354, 496, 1685, 1119, 707, 240, 1072, 520, 1277, 472, 961, 455, 308] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1005, 972, 772, 473, 795, 1632, 873, 615, 690, 685, 566, 695, 535, 494, 335, 558, 445, 1179, 171, 1829, 669, 382, 945, 725, 505, 484, 418, 1051, 754, 1964, 492, 626, 547, 264, 420, 1800, 1169, 324, 425, 981, 754, 1050, 146, 509, 524, 635, 1071, 480, 366, 1441, 1677, 344, 442, 1426, 1394, 700, 348, 1117, 476, 1037, 531, 1242, 682, 496] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1314, 1238, 905, 345, 561, 1746, 1134, 465, 391, 912, 1130, 638, 619, 532, 302, 611, 401, 1161, 302, 1829, 607, 634, 701, 639, 500, 298, 1007, 1690, 420, 1582, 441, 780, 367, 520, 432, 1375, 577, 454, 566, 643, 466, 657, 308, 654, 690, 556, 1020, 313, 426, 1291, 1352, 331, 745, 1317, 1087, 766, 444, 1465, 495, 1461, 609, 658, 557, 715] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[926, 1081, 786, 301, 632, 1952, 1096, 890, 811, 637, 713, 502, 612, 588, 488, 371, 406, 1108, 346, 1358, 680, 801, 524, 1129, 680, 258, 654, 1165, 562, 1723, 808, 731, 605, 325, 987, 733, 1011, 337, 455, 795, 783, 631, 373, 1025, 1070, 476, 376, 536, 317, 1565, 1507, 326, 574, 1230, 1183, 682, 456, 1706, 458, 984, 489, 717, 785, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1263, 1117, 1148, 541, 440, 1662, 937, 655, 615, 652, 525, 984, 691, 579, 481, 315, 466, 1405, 275, 1981, 887, 907, 895, 849, 578, 380, 1171, 1263, 331, 1903, 484, 525, 272, 377, 500, 1602, 837, 296, 366, 527, 574, 535, 141, 593, 456, 509, 969, 250, 412, 1130, 1635, 263, 893, 1421, 1125, 720, 405, 1430, 411, 1524, 509, 482, 665, 418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1213, 993, 810, 383, 491, 2001, 1106, 848, 739, 660, 662, 634, 502, 683, 607, 468, 525, 1264, 373, 1593, 1069, 621, 790, 506, 503, 405, 1030, 1056, 470, 2214, 576, 692, 635, 300, 539, 1098, 944, 504, 555, 693, 541, 795, 224, 669, 597, 665, 719, 307, 324, 1249, 1326, 352, 480, 1198, 1074, 724, 467, 1131, 512, 1573, 533, 672, 642, 623] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1301, 1146, 924, 503, 797, 225, 978, 552, 0, 0, 0, 0, 0, 0, 0, 0, 740, 626, 829, 622, 647, 411, 472, 336, 934, 145, 0, 0, 0, 0, 0, 0, 394, 1358, 304, 1233, 1030, 629, 568, 731, 0, 0, 0, 0, 0, 0, 0, 0, 673, 377, 964, 977, 591, 1299, 699, 826, 0, 0, 0, 0, 0, 0, 0, 0, 339, 519, 430, 1135, 712, 491, 356, 575, 617, 140, 175, 0, 0, 0, 0, 0, 745, 713, 168, 531, 602, 463, 802, 697, 1320, 0, 0, 0, 0, 0, 0, 0, 311, 1560, 125, 316, 658, 1341, 1581, 709, 0, 0, 0, 0, 0, 0, 0, 0, 409, 1602, 511, 995, 658, 581, 768, 309, 347, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6426, 5762, 6247, 6406, 5489, 6041, 6601, 6180] avg=6144.0000 max=6601 max_rank=6 min=5489 min_rank=4 imbalance_ratio=1.074382 max_load_reduction=42551 max_load_reduction_pct=0.865702 moved_to_echo=24311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1005, 972, 772, 473, 795, 218, 873, 615, 0, 0, 0, 0, 0, 0, 0, 0, 690, 685, 566, 695, 535, 494, 335, 558, 1307, 140, 0, 0, 0, 0, 0, 0, 445, 1179, 171, 1286, 669, 382, 945, 725, 0, 0, 0, 0, 0, 0, 0, 0, 505, 484, 418, 1051, 754, 1331, 492, 626, 0, 0, 0, 0, 0, 0, 0, 0, 547, 264, 420, 1800, 1169, 324, 425, 981, 633, 197, 182, 0, 0, 0, 0, 0, 754, 1050, 146, 509, 524, 635, 1071, 480, 1274, 0, 0, 0, 0, 0, 0, 0, 366, 1441, 173, 344, 442, 1426, 1394, 700, 0, 0, 0, 0, 0, 0, 0, 0, 348, 1117, 476, 1037, 531, 1242, 682, 496, 361, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5723, 6005, 5802, 5661, 6942, 6443, 6286, 6290] avg=6144.0000 max=6942 max_rank=4 min=5661 min_rank=3 imbalance_ratio=1.129883 max_load_reduction=42210 max_load_reduction_pct=0.858765 moved_to_echo=25961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1213, 993, 810, 383, 491, 266, 1106, 848, 0, 0, 0, 0, 0, 0, 0, 0, 739, 660, 662, 634, 502, 683, 607, 468, 1033, 172, 0, 0, 0, 0, 0, 0, 525, 1264, 373, 1119, 1069, 621, 790, 506, 0, 0, 0, 0, 0, 0, 0, 0, 503, 405, 1030, 1056, 470, 1500, 576, 692, 0, 0, 0, 0, 0, 0, 0, 0, 635, 300, 539, 1098, 944, 504, 555, 693, 714, 155, 159, 0, 0, 0, 0, 0, 541, 795, 224, 669, 597, 665, 719, 307, 1563, 0, 0, 0, 0, 0, 0, 0, 324, 1249, 138, 352, 480, 1198, 1074, 724, 0, 0, 0, 0, 0, 0, 0, 0, 467, 1131, 512, 1573, 533, 672, 642, 623, 315, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 6160, 6267, 6232, 6296, 6080, 5539, 6468] avg=6144.0000 max=6468 max_rank=7 min=5539 min_rank=6 imbalance_ratio=1.052734 max_load_reduction=42684 max_load_reduction_pct=0.868408 moved_to_echo=24383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[926, 1081, 786, 301, 632, 260, 1096, 890, 0, 0, 0, 0, 0, 0, 0, 0, 811, 637, 713, 502, 612, 588, 488, 371, 1174, 168, 0, 0, 0, 0, 0, 0, 406, 1108, 346, 955, 680, 801, 524, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 680, 258, 654, 1165, 562, 1168, 808, 731, 0, 0, 0, 0, 0, 0, 0, 0, 605, 325, 987, 733, 1011, 337, 455, 795, 555, 177, 135, 0, 0, 0, 0, 0, 783, 631, 373, 1025, 1070, 476, 376, 536, 1524, 0, 0, 0, 0, 0, 0, 0, 317, 1565, 156, 326, 574, 1230, 1183, 682, 0, 0, 0, 0, 0, 0, 0, 0, 456, 1706, 458, 984, 489, 717, 785, 336, 268, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5972, 6064, 5949, 6026, 6115, 6794, 6033, 6199] avg=6144.0000 max=6794 max_rank=5 min=5949 min_rank=2 imbalance_ratio=1.105794 max_load_reduction=42358 max_load_reduction_pct=0.861776 moved_to_echo=25141 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1314, 1238, 905, 345, 561, 232, 1134, 465, 0, 0, 0, 0, 0, 0, 0, 0, 391, 912, 1130, 638, 619, 532, 302, 611, 1053, 150, 0, 0, 0, 0, 0, 0, 401, 1161, 302, 1286, 607, 634, 701, 639, 0, 0, 0, 0, 0, 0, 0, 0, 500, 298, 1007, 1690, 420, 1072, 441, 780, 0, 0, 0, 0, 0, 0, 0, 0, 367, 520, 432, 1375, 577, 454, 566, 643, 510, 158, 182, 0, 0, 0, 0, 0, 466, 657, 308, 654, 690, 556, 1020, 313, 1364, 0, 0, 0, 0, 0, 0, 0, 426, 1291, 141, 331, 745, 1317, 1087, 766, 0, 0, 0, 0, 0, 0, 0, 0, 444, 1465, 495, 1461, 609, 658, 557, 715, 361, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6338, 5731, 6208, 5784, 6028, 6104, 6765] avg=6144.0000 max=6765 max_rank=7 min=5731 min_rank=2 imbalance_ratio=1.101074 max_load_reduction=42387 max_load_reduction_pct=0.862366 moved_to_echo=24681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1182, 1219, 759, 462, 542, 237, 868, 661, 0, 0, 0, 0, 0, 0, 0, 0, 624, 693, 599, 679, 600, 639, 493, 642, 1213, 153, 0, 0, 0, 0, 0, 0, 577, 1229, 303, 1130, 832, 703, 860, 864, 0, 0, 0, 0, 0, 0, 0, 0, 659, 375, 975, 1072, 653, 1405, 452, 724, 0, 0, 0, 0, 0, 0, 0, 0, 327, 428, 694, 1321, 941, 325, 504, 783, 668, 182, 160, 0, 0, 0, 0, 0, 640, 564, 162, 661, 503, 473, 783, 424, 1389, 0, 0, 0, 0, 0, 0, 0, 382, 1616, 161, 354, 496, 1685, 1119, 707, 0, 0, 0, 0, 0, 0, 0, 0, 240, 1072, 520, 1277, 472, 961, 455, 308, 317, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 6335, 6498, 6315, 6333, 5599, 6520, 5622] avg=6144.0000 max=6520 max_rank=6 min=5599 min_rank=5 imbalance_ratio=1.061198 max_load_reduction=42632 max_load_reduction_pct=0.867350 moved_to_echo=24074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1212, 1284, 1010, 436, 692, 256, 988, 597, 0, 0, 0, 0, 0, 0, 0, 0, 672, 568, 1045, 694, 669, 531, 378, 414, 1087, 171, 0, 0, 0, 0, 0, 0, 363, 1204, 332, 1164, 528, 601, 673, 717, 0, 0, 0, 0, 0, 0, 0, 0, 507, 368, 672, 1450, 715, 1351, 423, 796, 0, 0, 0, 0, 0, 0, 0, 0, 459, 467, 392, 1609, 707, 444, 420, 928, 649, 167, 171, 0, 0, 0, 0, 0, 654, 877, 356, 736, 672, 533, 936, 537, 1541, 0, 0, 0, 0, 0, 0, 0, 377, 1100, 135, 261, 742, 929, 1679, 713, 0, 0, 0, 0, 0, 0, 0, 0, 331, 1074, 452, 915, 769, 507, 586, 425, 334, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6475, 6229, 5582, 6282, 6413, 6842, 5936, 5393] avg=6144.0000 max=6842 max_rank=5 min=5393 min_rank=7 imbalance_ratio=1.113607 max_load_reduction=42310 max_load_reduction_pct=0.860799 moved_to_echo=24584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1301, 1146, 924, 503, 797, 225, 978, 552, 0, 0, 0, 0, 0, 0, 0, 0, 740, 626, 829, 622, 647, 411, 472, 336, 934, 145, 0, 0, 0, 0, 0, 0, 394, 1358, 304, 1233, 1030, 629, 568, 731, 0, 0, 0, 0, 0, 0, 0, 0, 673, 377, 964, 977, 591, 1299, 699, 826, 0, 0, 0, 0, 0, 0, 0, 0, 339, 519, 430, 1135, 712, 491, 356, 575, 617, 140, 175, 0, 0, 0, 0, 0, 745, 713, 168, 531, 602, 463, 802, 697, 1320, 0, 0, 0, 0, 0, 0, 0, 311, 1560, 125, 316, 658, 1341, 1581, 709, 0, 0, 0, 0, 0, 0, 0, 0, 409, 1602, 511, 995, 658, 581, 768, 309, 347, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6426, 5762, 6247, 6406, 5489, 6041, 6601, 6180] avg=6144.0000 max=6601 max_rank=6 min=5489 min_rank=4 imbalance_ratio=1.074382 max_load_reduction=42551 max_load_reduction_pct=0.865702 moved_to_echo=24311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1314, 1238, 905, 345, 561, 232, 1134, 465, 0, 0, 0, 0, 0, 0, 0, 0, 391, 912, 1130, 638, 619, 532, 302, 611, 1053, 150, 0, 0, 0, 0, 0, 0, 401, 1161, 302, 1286, 607, 634, 701, 639, 0, 0, 0, 0, 0, 0, 0, 0, 500, 298, 1007, 1690, 420, 1072, 441, 780, 0, 0, 0, 0, 0, 0, 0, 0, 367, 520, 432, 1375, 577, 454, 566, 643, 510, 158, 182, 0, 0, 0, 0, 0, 466, 657, 308, 654, 690, 556, 1020, 313, 1364, 0, 0, 0, 0, 0, 0, 0, 426, 1291, 141, 331, 745, 1317, 1087, 766, 0, 0, 0, 0, 0, 0, 0, 0, 444, 1465, 495, 1461, 609, 658, 557, 715, 361, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6338, 5731, 6208, 5784, 6028, 6104, 6765] avg=6144.0000 max=6765 max_rank=7 min=5731 min_rank=2 imbalance_ratio=1.101074 max_load_reduction=42387 max_load_reduction_pct=0.862366 moved_to_echo=24681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1263, 1117, 1148, 541, 440, 221, 937, 655, 0, 0, 0, 0, 0, 0, 0, 0, 615, 652, 525, 984, 691, 579, 481, 315, 1274, 143, 0, 0, 0, 0, 0, 0, 466, 1405, 275, 1392, 887, 907, 895, 849, 0, 0, 0, 0, 0, 0, 0, 0, 578, 380, 1171, 1263, 331, 1290, 484, 525, 0, 0, 0, 0, 0, 0, 0, 0, 272, 377, 500, 1602, 837, 296, 366, 527, 613, 192, 198, 0, 0, 0, 0, 0, 574, 535, 141, 593, 456, 509, 969, 250, 1298, 0, 0, 0, 0, 0, 0, 0, 412, 1130, 169, 263, 893, 1421, 1125, 720, 0, 0, 0, 0, 0, 0, 0, 0, 405, 1430, 411, 1524, 509, 482, 665, 418, 391, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6322, 6259, 7076, 6022, 5780, 5325, 6133, 6235] avg=6144.0000 max=7076 max_rank=2 min=5325 min_rank=5 imbalance_ratio=1.151693 max_load_reduction=42076 max_load_reduction_pct=0.856038 moved_to_echo=23473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1213, 993, 810, 383, 491, 266, 1106, 848, 0, 0, 0, 0, 0, 0, 0, 0, 739, 660, 662, 634, 502, 683, 607, 468, 1033, 172, 0, 0, 0, 0, 0, 0, 525, 1264, 373, 1119, 1069, 621, 790, 506, 0, 0, 0, 0, 0, 0, 0, 0, 503, 405, 1030, 1056, 470, 1500, 576, 692, 0, 0, 0, 0, 0, 0, 0, 0, 635, 300, 539, 1098, 944, 504, 555, 693, 714, 155, 159, 0, 0, 0, 0, 0, 541, 795, 224, 669, 597, 665, 719, 307, 1563, 0, 0, 0, 0, 0, 0, 0, 324, 1249, 138, 352, 480, 1198, 1074, 724, 0, 0, 0, 0, 0, 0, 0, 0, 467, 1131, 512, 1573, 533, 672, 642, 623, 315, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 6160, 6267, 6232, 6296, 6080, 5539, 6468] avg=6144.0000 max=6468 max_rank=7 min=5539 min_rank=6 imbalance_ratio=1.052734 max_load_reduction=42684 max_load_reduction_pct=0.868408 moved_to_echo=24383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 11008, 1280, 2560, 5120, 10624, 10368, 5760, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5376, 5504, 6144, 5504, 4992, 4480, 3584, 3840, 9088, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 10624, 3840, 9856, 4608, 5888, 5248, 3712, 2816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 9984, 2432, 9600, 6400, 5376, 6016, 6272, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 3200, 4480, 10752, 6912, 3200, 3712, 6016, 4992, 1408, 1408, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 3072, 6912, 9728, 4608, 10496, 4480, 5760, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 9984, 2432, 9600, 6400, 5376, 6016, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 10624, 3840, 9856, 4608, 5888, 5248, 3712, 2816, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 5888, 1920, 5504, 5120, 4352, 6784, 3584, 11392, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9472, 9088, 7168, 3456, 4992, 1920, 8064, 5376, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[400, 1053, 307, 232, 210, 1564, 1330, 689, 1051, 376, 592, 314, 225, 458, 801, 242, 2106, 938, 1035, 790, 643, 2398, 754, 429, 728, 671, 843, 407, 667, 602, 348, 347, 663, 373, 934, 571, 660, 1287, 1223, 1024, 1013, 678, 913, 1168, 437, 562, 1585, 830, 613, 1397, 888, 848, 479, 433, 136, 625, 1114, 1428, 1102, 633, 637, 224, 783, 341] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[597, 1117, 236, 295, 403, 1185, 1227, 401, 703, 346, 420, 597, 225, 356, 622, 89, 1855, 1108, 835, 1011, 706, 2416, 1055, 546, 904, 681, 660, 494, 740, 1182, 348, 368, 491, 152, 558, 651, 561, 1504, 1424, 1375, 1017, 408, 567, 1439, 242, 506, 1745, 865, 633, 1134, 1057, 1060, 1145, 304, 195, 420, 1097, 1196, 531, 339, 871, 248, 1343, 346] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[749, 974, 208, 221, 417, 931, 2067, 1107, 1038, 352, 804, 546, 300, 519, 745, 199, 1776, 1088, 850, 468, 937, 2037, 688, 272, 868, 588, 626, 308, 550, 546, 424, 352, 493, 263, 832, 849, 634, 1441, 1127, 944, 790, 411, 1065, 1572, 315, 547, 2203, 587, 776, 1576, 798, 682, 810, 442, 199, 699, 1081, 1146, 541, 440, 720, 254, 885, 475] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[656, 935, 197, 294, 402, 1145, 1729, 511, 564, 337, 908, 488, 398, 430, 928, 376, 2005, 879, 916, 584, 649, 1980, 1010, 388, 905, 589, 842, 384, 756, 548, 353, 465, 423, 169, 630, 691, 628, 1425, 1151, 819, 1227, 308, 774, 1284, 632, 509, 2387, 902, 571, 979, 1073, 690, 1103, 793, 208, 755, 752, 1123, 652, 555, 659, 557, 677, 495] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[400, 1053, 307, 232, 210, 1564, 1330, 689, 1051, 376, 592, 314, 225, 458, 801, 242, 2106, 938, 1035, 790, 643, 2398, 754, 429, 728, 671, 843, 407, 667, 602, 348, 347, 663, 373, 934, 571, 660, 1287, 1223, 1024, 1013, 678, 913, 1168, 437, 562, 1585, 830, 613, 1397, 888, 848, 479, 433, 136, 625, 1114, 1428, 1102, 633, 637, 224, 783, 341] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[580, 1093, 283, 204, 523, 1072, 1152, 586, 439, 380, 783, 409, 362, 609, 930, 291, 1816, 1124, 954, 532, 682, 2021, 672, 345, 1103, 946, 863, 512, 651, 803, 465, 281, 463, 180, 1162, 623, 746, 1214, 1523, 1064, 970, 769, 1103, 1115, 356, 504, 2246, 597, 761, 698, 1115, 644, 917, 490, 169, 496, 972, 1516, 616, 440, 915, 278, 677, 347] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[692, 938, 208, 188, 306, 1407, 1138, 674, 593, 281, 982, 251, 359, 423, 666, 246, 1812, 1708, 1189, 657, 964, 1995, 883, 396, 588, 833, 684, 422, 698, 1193, 509, 372, 740, 254, 1564, 875, 551, 1148, 1091, 1286, 729, 507, 870, 1470, 303, 602, 1064, 575, 647, 1176, 1100, 968, 746, 491, 283, 391, 864, 1232, 572, 478, 388, 348, 1138, 446] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1050, 617, 373, 182, 250, 1514, 1301, 459, 789, 510, 596, 258, 341, 632, 591, 406, 2320, 459, 1080, 819, 372, 2072, 774, 276, 508, 629, 920, 721, 794, 470, 400, 293, 602, 628, 634, 998, 390, 1500, 537, 713, 1227, 748, 1124, 1439, 559, 431, 1311, 681, 573, 579, 1303, 600, 1120, 502, 195, 763, 1686, 1341, 1242, 552, 910, 332, 919, 237] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[580, 1093, 283, 204, 523, 1072, 1152, 586, 439, 380, 783, 409, 362, 609, 930, 291, 1816, 1124, 954, 532, 682, 2021, 672, 345, 1103, 946, 863, 512, 651, 803, 465, 281, 463, 180, 1162, 623, 746, 1214, 1523, 1064, 970, 769, 1103, 1115, 356, 504, 2246, 597, 761, 698, 1115, 644, 917, 490, 169, 496, 972, 1516, 616, 440, 915, 278, 677, 347] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[656, 935, 197, 294, 402, 1145, 1729, 511, 564, 337, 908, 488, 398, 430, 928, 376, 2005, 879, 916, 584, 649, 1980, 1010, 388, 905, 589, 842, 384, 756, 548, 353, 465, 423, 169, 630, 691, 628, 1425, 1151, 819, 1227, 308, 774, 1284, 632, 509, 2387, 902, 571, 979, 1073, 690, 1103, 793, 208, 755, 752, 1123, 652, 555, 659, 557, 677, 495] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[925, 1172, 315, 191, 363, 1410, 1226, 678, 819, 270, 704, 338, 210, 330, 911, 179, 2160, 1205, 1014, 598, 1151, 2046, 799, 961, 1097, 1194, 675, 377, 620, 803, 403, 538, 422, 371, 655, 531, 662, 1113, 1105, 996, 726, 320, 751, 1064, 452, 577, 1799, 1119, 625, 869, 1149, 691, 625, 580, 143, 472, 769, 1066, 713, 562, 804, 247, 906, 586] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[597, 1117, 236, 295, 403, 1185, 1227, 401, 703, 346, 420, 597, 225, 356, 622, 89, 1855, 1108, 835, 1011, 706, 2416, 1055, 546, 904, 681, 660, 494, 740, 1182, 348, 368, 491, 152, 558, 651, 561, 1504, 1424, 1375, 1017, 408, 567, 1439, 242, 506, 1745, 865, 633, 1134, 1057, 1060, 1145, 304, 195, 420, 1097, 1196, 531, 339, 871, 248, 1343, 346] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[580, 1093, 283, 204, 523, 1072, 1152, 586, 154, 107, 0, 0, 0, 0, 0, 0, 439, 380, 783, 409, 362, 609, 930, 291, 2015, 0, 0, 0, 0, 0, 0, 0, 1258, 1124, 954, 532, 682, 0, 672, 345, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 946, 863, 512, 651, 803, 465, 281, 1322, 22, 6, 0, 0, 0, 0, 0, 463, 180, 1162, 623, 746, 854, 1523, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 970, 769, 1103, 1115, 356, 504, 924, 597, 0, 0, 0, 0, 0, 0, 0, 0, 761, 698, 1115, 644, 917, 490, 169, 496, 429, 0, 0, 0, 0, 0, 0, 0, 972, 1516, 616, 440, 915, 278, 677, 347, 206, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5754, 6218, 5567, 6974, 6615, 6338, 5719, 5967] avg=6144.0000 max=6974 max_rank=3 min=5567 min_rank=2 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=24639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[749, 974, 208, 221, 417, 931, 2067, 1107, 183, 105, 0, 0, 0, 0, 0, 0, 1038, 352, 804, 546, 300, 519, 745, 199, 2030, 0, 0, 0, 0, 0, 0, 0, 1229, 1088, 850, 468, 937, 0, 688, 272, 0, 0, 0, 0, 0, 0, 0, 0, 868, 588, 626, 308, 550, 546, 424, 352, 1297, 22, 7, 0, 0, 0, 0, 0, 493, 263, 832, 849, 634, 1013, 1127, 944, 0, 0, 0, 0, 0, 0, 0, 0, 790, 411, 1065, 1572, 315, 547, 906, 587, 0, 0, 0, 0, 0, 0, 0, 0, 776, 1576, 798, 682, 810, 442, 199, 699, 420, 0, 0, 0, 0, 0, 0, 0, 1081, 1146, 541, 440, 720, 254, 885, 475, 245, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6962, 6533, 5532, 5588, 6155, 6193, 6402, 5787] avg=6144.0000 max=6962 max_rank=0 min=5532 min_rank=2 imbalance_ratio=1.133138 max_load_reduction=42190 max_load_reduction_pct=0.858358 moved_to_echo=24537 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[400, 1053, 307, 232, 210, 1564, 1330, 689, 163, 125, 0, 0, 0, 0, 0, 0, 1051, 376, 592, 314, 225, 458, 801, 242, 2390, 0, 0, 0, 0, 0, 0, 0, 1457, 938, 1035, 790, 643, 0, 754, 429, 0, 0, 0, 0, 0, 0, 0, 0, 728, 671, 843, 407, 667, 602, 348, 347, 933, 26, 8, 0, 0, 0, 0, 0, 663, 373, 934, 571, 660, 906, 1223, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 678, 913, 1168, 437, 562, 652, 830, 0, 0, 0, 0, 0, 0, 0, 0, 613, 1397, 888, 848, 479, 433, 136, 625, 498, 0, 0, 0, 0, 0, 0, 0, 1114, 1428, 1102, 633, 637, 224, 783, 341, 218, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6449, 6046, 5580, 6354, 6253, 5917, 6480] avg=6144.0000 max=6480 max_rank=7 min=5580 min_rank=3 imbalance_ratio=1.054688 max_load_reduction=42672 max_load_reduction_pct=0.868164 moved_to_echo=25004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1050, 617, 373, 182, 250, 1514, 1301, 459, 190, 137, 0, 0, 0, 0, 0, 0, 789, 510, 596, 258, 341, 632, 591, 406, 2066, 0, 0, 0, 0, 0, 0, 0, 1606, 459, 1080, 819, 372, 0, 774, 276, 0, 0, 0, 0, 0, 0, 0, 0, 508, 629, 920, 721, 794, 470, 400, 293, 771, 29, 6, 0, 0, 0, 0, 0, 602, 628, 634, 998, 390, 1055, 537, 713, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 748, 1124, 1439, 559, 431, 540, 681, 0, 0, 0, 0, 0, 0, 0, 0, 573, 579, 1303, 600, 1120, 502, 195, 763, 548, 0, 0, 0, 0, 0, 0, 0, 1686, 1341, 1242, 552, 910, 332, 919, 237, 255, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6189, 5386, 5541, 5557, 6749, 6183, 7474] avg=6144.0000 max=7474 max_rank=7 min=5386 min_rank=2 imbalance_ratio=1.216471 max_load_reduction=41678 max_load_reduction_pct=0.847941 moved_to_echo=25963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[692, 938, 208, 188, 306, 1407, 1138, 674, 145, 107, 0, 0, 0, 0, 0, 0, 593, 281, 982, 251, 359, 423, 666, 246, 1990, 0, 0, 0, 0, 0, 0, 0, 1255, 1708, 1189, 657, 964, 0, 883, 396, 0, 0, 0, 0, 0, 0, 0, 0, 588, 833, 684, 422, 698, 1193, 509, 372, 626, 22, 5, 0, 0, 0, 0, 0, 740, 254, 1564, 875, 551, 808, 1091, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 729, 507, 870, 1470, 303, 602, 438, 575, 0, 0, 0, 0, 0, 0, 0, 0, 647, 1176, 1100, 968, 746, 491, 283, 391, 428, 0, 0, 0, 0, 0, 0, 0, 864, 1232, 572, 478, 388, 348, 1138, 446, 195, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5803, 5791, 7052, 5952, 7169, 5494, 6230, 5661] avg=6144.0000 max=7169 max_rank=4 min=5494 min_rank=5 imbalance_ratio=1.166829 max_load_reduction=41983 max_load_reduction_pct=0.854146 moved_to_echo=24554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[749, 974, 208, 221, 417, 931, 2067, 1107, 183, 105, 0, 0, 0, 0, 0, 0, 1038, 352, 804, 546, 300, 519, 745, 199, 2030, 0, 0, 0, 0, 0, 0, 0, 1229, 1088, 850, 468, 937, 0, 688, 272, 0, 0, 0, 0, 0, 0, 0, 0, 868, 588, 626, 308, 550, 546, 424, 352, 1297, 22, 7, 0, 0, 0, 0, 0, 493, 263, 832, 849, 634, 1013, 1127, 944, 0, 0, 0, 0, 0, 0, 0, 0, 790, 411, 1065, 1572, 315, 547, 906, 587, 0, 0, 0, 0, 0, 0, 0, 0, 776, 1576, 798, 682, 810, 442, 199, 699, 420, 0, 0, 0, 0, 0, 0, 0, 1081, 1146, 541, 440, 720, 254, 885, 475, 245, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6962, 6533, 5532, 5588, 6155, 6193, 6402, 5787] avg=6144.0000 max=6962 max_rank=0 min=5532 min_rank=2 imbalance_ratio=1.133138 max_load_reduction=42190 max_load_reduction_pct=0.858358 moved_to_echo=24537 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[656, 935, 197, 294, 402, 1145, 1729, 511, 184, 123, 0, 0, 0, 0, 0, 0, 564, 337, 908, 488, 398, 430, 928, 376, 1975, 0, 0, 0, 0, 0, 0, 0, 1376, 879, 916, 584, 649, 0, 1010, 388, 0, 0, 0, 0, 0, 0, 0, 0, 905, 589, 842, 384, 756, 548, 353, 465, 1409, 28, 5, 0, 0, 0, 0, 0, 423, 169, 630, 691, 628, 996, 1151, 819, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 308, 774, 1284, 632, 509, 978, 902, 0, 0, 0, 0, 0, 0, 0, 0, 571, 979, 1073, 690, 1103, 793, 208, 755, 478, 0, 0, 0, 0, 0, 0, 0, 752, 1123, 652, 555, 659, 557, 677, 495, 245, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 6404, 5802, 6284, 5507, 6614, 6650, 5715] avg=6144.0000 max=6650 max_rank=6 min=5507 min_rank=4 imbalance_ratio=1.082357 max_load_reduction=42502 max_load_reduction_pct=0.864705 moved_to_echo=24486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1050, 617, 373, 182, 250, 1514, 1301, 459, 190, 137, 0, 0, 0, 0, 0, 0, 789, 510, 596, 258, 341, 632, 591, 406, 2066, 0, 0, 0, 0, 0, 0, 0, 1606, 459, 1080, 819, 372, 0, 774, 276, 0, 0, 0, 0, 0, 0, 0, 0, 508, 629, 920, 721, 794, 470, 400, 293, 771, 29, 6, 0, 0, 0, 0, 0, 602, 628, 634, 998, 390, 1055, 537, 713, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 748, 1124, 1439, 559, 431, 540, 681, 0, 0, 0, 0, 0, 0, 0, 0, 573, 579, 1303, 600, 1120, 502, 195, 763, 548, 0, 0, 0, 0, 0, 0, 0, 1686, 1341, 1242, 552, 910, 332, 919, 237, 255, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6189, 5386, 5541, 5557, 6749, 6183, 7474] avg=6144.0000 max=7474 max_rank=7 min=5386 min_rank=2 imbalance_ratio=1.216471 max_load_reduction=41678 max_load_reduction_pct=0.847941 moved_to_echo=25963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[925, 1172, 315, 191, 363, 1410, 1226, 678, 141, 128, 0, 0, 0, 0, 0, 0, 819, 270, 704, 338, 210, 330, 911, 179, 2040, 0, 0, 0, 0, 0, 0, 0, 1494, 1205, 1014, 598, 1151, 0, 799, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 1194, 675, 377, 620, 803, 403, 538, 1059, 27, 6, 0, 0, 0, 0, 0, 422, 371, 655, 531, 662, 783, 1105, 996, 0, 0, 0, 0, 0, 0, 0, 0, 726, 320, 751, 1064, 452, 577, 740, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 625, 869, 1149, 691, 625, 580, 143, 472, 511, 0, 0, 0, 0, 0, 0, 0, 769, 1066, 713, 562, 804, 247, 906, 586, 189, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6549, 5801, 7222, 6799, 5525, 5749, 5665, 5842] avg=6144.0000 max=7222 max_rank=2 min=5525 min_rank=4 imbalance_ratio=1.175456 max_load_reduction=41930 max_load_reduction_pct=0.853068 moved_to_echo=22781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[597, 1117, 236, 295, 403, 1185, 1227, 401, 191, 110, 0, 0, 0, 0, 0, 0, 703, 346, 420, 597, 225, 356, 622, 89, 2409, 0, 0, 0, 0, 0, 0, 0, 1284, 1108, 835, 1011, 706, 0, 1055, 546, 0, 0, 0, 0, 0, 0, 0, 0, 904, 681, 660, 494, 740, 1182, 348, 368, 1027, 23, 7, 0, 0, 0, 0, 0, 491, 152, 558, 651, 561, 1058, 1424, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 408, 567, 1439, 242, 506, 718, 865, 0, 0, 0, 0, 0, 0, 0, 0, 633, 1134, 1057, 1060, 1145, 304, 195, 420, 438, 0, 0, 0, 0, 0, 0, 0, 1097, 1196, 531, 339, 871, 248, 1343, 346, 255, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5762, 5767, 6545, 6434, 6270, 5762, 6386, 6226] avg=6144.0000 max=6545 max_rank=2 min=5762 min_rank=0 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=24644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[580, 1093, 283, 204, 523, 1072, 1152, 586, 154, 107, 0, 0, 0, 0, 0, 0, 439, 380, 783, 409, 362, 609, 930, 291, 2015, 0, 0, 0, 0, 0, 0, 0, 1258, 1124, 954, 532, 682, 0, 672, 345, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 946, 863, 512, 651, 803, 465, 281, 1322, 22, 6, 0, 0, 0, 0, 0, 463, 180, 1162, 623, 746, 854, 1523, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 970, 769, 1103, 1115, 356, 504, 924, 597, 0, 0, 0, 0, 0, 0, 0, 0, 761, 698, 1115, 644, 917, 490, 169, 496, 429, 0, 0, 0, 0, 0, 0, 0, 972, 1516, 616, 440, 915, 278, 677, 347, 206, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5754, 6218, 5567, 6974, 6615, 6338, 5719, 5967] avg=6144.0000 max=6974 max_rank=3 min=5567 min_rank=2 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=24639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[656, 935, 197, 294, 402, 1145, 1729, 511, 184, 123, 0, 0, 0, 0, 0, 0, 564, 337, 908, 488, 398, 430, 928, 376, 1975, 0, 0, 0, 0, 0, 0, 0, 1376, 879, 916, 584, 649, 0, 1010, 388, 0, 0, 0, 0, 0, 0, 0, 0, 905, 589, 842, 384, 756, 548, 353, 465, 1409, 28, 5, 0, 0, 0, 0, 0, 423, 169, 630, 691, 628, 996, 1151, 819, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 308, 774, 1284, 632, 509, 978, 902, 0, 0, 0, 0, 0, 0, 0, 0, 571, 979, 1073, 690, 1103, 793, 208, 755, 478, 0, 0, 0, 0, 0, 0, 0, 752, 1123, 652, 555, 659, 557, 677, 495, 245, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 6404, 5802, 6284, 5507, 6614, 6650, 5715] avg=6144.0000 max=6650 max_rank=6 min=5507 min_rank=4 imbalance_ratio=1.082357 max_load_reduction=42502 max_load_reduction_pct=0.864705 moved_to_echo=24486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 2944, 5888, 3328, 2432, 3840, 6272, 2048, 17024, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7808, 4224, 7168, 10624, 3328, 4352, 6016, 6272, 0, 0, 0, 0, 0, 0, 0, 0] @@ -1687,692 +1687,692 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 7936, 2176, 1920, 2944, 10240, 11264, 5120, 1408, 1024, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 2432, 7040, 5888, 4864, 7552, 9216, 8320, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6784, 6144, 6144, 3712, 5504, 6272, 3328, 3072, 8448, 256, 128, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1287, 477, 2069, 608, 371, 831, 1031, 1390, 1083, 701, 811, 1088, 1048, 340, 618, 775, 988, 317, 934, 949, 560, 1097, 1076, 1219, 451, 387, 578, 429, 639, 452, 1140, 229, 537, 684, 877, 240, 476, 793, 590, 904, 228, 857, 650, 248, 938, 445, 496, 689, 701, 484, 887, 1163, 1280, 1351, 510, 581, 737, 980, 266, 775, 482, 1454, 1273, 603] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1784, 543, 2085, 588, 263, 737, 772, 1366, 831, 845, 707, 1143, 907, 480, 629, 483, 1588, 337, 1316, 347, 767, 1795, 985, 1678, 482, 184, 652, 416, 776, 403, 1284, 151, 447, 636, 1260, 162, 614, 691, 588, 743, 896, 1139, 271, 146, 660, 132, 300, 988, 936, 210, 385, 903, 1366, 849, 520, 349, 592, 635, 445, 591, 748, 1326, 1896, 404] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2113, 473, 2218, 326, 308, 806, 308, 1348, 1223, 927, 829, 1225, 740, 490, 305, 380, 1182, 1161, 1190, 707, 363, 2461, 1617, 1949, 439, 321, 822, 256, 778, 372, 1216, 283, 604, 386, 1087, 390, 697, 481, 626, 435, 407, 928, 231, 165, 518, 278, 401, 864, 737, 223, 550, 468, 1563, 311, 360, 323, 840, 767, 389, 571, 1453, 1065, 1475, 423] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1766, 312, 2267, 678, 375, 855, 570, 1400, 934, 907, 866, 1000, 635, 502, 304, 475, 1118, 1017, 1326, 669, 395, 1724, 1169, 1781, 357, 329, 532, 403, 1024, 750, 1324, 246, 832, 774, 1296, 244, 1218, 933, 625, 671, 591, 1666, 164, 259, 390, 491, 217, 619, 620, 207, 490, 452, 795, 1013, 300, 257, 465, 533, 190, 472, 855, 857, 2105, 541] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1293, 645, 2066, 712, 215, 1181, 327, 1662, 1071, 693, 885, 1589, 654, 357, 787, 444, 734, 872, 1380, 457, 686, 987, 1108, 1085, 495, 229, 836, 197, 922, 501, 1249, 120, 441, 846, 742, 319, 783, 687, 613, 694, 528, 1624, 642, 200, 671, 829, 355, 411, 652, 221, 477, 1153, 1444, 758, 369, 219, 1139, 625, 357, 877, 1180, 1141, 1353, 363] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1287, 477, 2069, 608, 371, 831, 1031, 1390, 1083, 701, 811, 1088, 1048, 340, 618, 775, 988, 317, 934, 949, 560, 1097, 1076, 1219, 451, 387, 578, 429, 639, 452, 1140, 229, 537, 684, 877, 240, 476, 793, 590, 904, 228, 857, 650, 248, 938, 445, 496, 689, 701, 484, 887, 1163, 1280, 1351, 510, 581, 737, 980, 266, 775, 482, 1454, 1273, 603] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1443, 508, 1849, 1029, 336, 989, 435, 1446, 1154, 790, 1174, 729, 948, 421, 423, 508, 820, 384, 1066, 748, 501, 913, 953, 1201, 353, 889, 878, 323, 748, 407, 1245, 232, 1093, 601, 938, 315, 844, 1035, 706, 1027, 498, 1317, 541, 300, 495, 444, 570, 577, 624, 444, 526, 1030, 904, 911, 381, 298, 713, 1469, 214, 682, 966, 1333, 1056, 457] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1766, 312, 2267, 678, 375, 855, 570, 1400, 934, 907, 866, 1000, 635, 502, 304, 475, 1118, 1017, 1326, 669, 395, 1724, 1169, 1781, 357, 329, 532, 403, 1024, 750, 1324, 246, 832, 774, 1296, 244, 1218, 933, 625, 671, 591, 1666, 164, 259, 390, 491, 217, 619, 620, 207, 490, 452, 795, 1013, 300, 257, 465, 533, 190, 472, 855, 857, 2105, 541] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1367, 308, 2513, 401, 358, 825, 696, 1439, 617, 1555, 716, 685, 965, 383, 885, 364, 969, 709, 1547, 1058, 236, 1359, 981, 1583, 428, 295, 782, 382, 728, 489, 1840, 269, 589, 765, 893, 653, 787, 881, 383, 655, 234, 1399, 182, 177, 649, 375, 467, 402, 743, 48, 541, 921, 1296, 923, 303, 260, 310, 1681, 595, 263, 618, 1252, 1402, 773] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[2070, 507, 1893, 766, 336, 922, 837, 1694, 1198, 987, 670, 1019, 704, 469, 748, 221, 828, 678, 1081, 663, 687, 827, 1195, 1202, 216, 278, 334, 458, 788, 384, 1993, 280, 547, 522, 864, 309, 528, 836, 624, 1009, 712, 1850, 388, 149, 588, 462, 271, 503, 1146, 402, 791, 1008, 1349, 695, 727, 259, 482, 657, 494, 899, 743, 739, 1246, 420] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2113, 473, 2218, 326, 308, 806, 308, 1348, 1223, 927, 829, 1225, 740, 490, 305, 380, 1182, 1161, 1190, 707, 363, 2461, 1617, 1949, 439, 321, 822, 256, 778, 372, 1216, 283, 604, 386, 1087, 390, 697, 481, 626, 435, 407, 928, 231, 165, 518, 278, 401, 864, 737, 223, 550, 468, 1563, 311, 360, 323, 840, 767, 389, 571, 1453, 1065, 1475, 423] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1367, 308, 2513, 401, 358, 825, 696, 1439, 617, 1555, 716, 685, 965, 383, 885, 364, 969, 709, 1547, 1058, 236, 1359, 981, 1583, 428, 295, 782, 382, 728, 489, 1840, 269, 589, 765, 893, 653, 787, 881, 383, 655, 234, 1399, 182, 177, 649, 375, 467, 402, 743, 48, 541, 921, 1296, 923, 303, 260, 310, 1681, 595, 263, 618, 1252, 1402, 773] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1766, 312, 97, 678, 375, 855, 570, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 934, 907, 866, 1000, 635, 502, 304, 475, 19, 0, 0, 0, 0, 0, 0, 0, 1118, 1017, 1326, 669, 395, 1041, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 329, 532, 403, 1024, 750, 1324, 246, 1264, 423, 0, 0, 0, 0, 0, 0, 832, 774, 1296, 244, 1218, 933, 625, 671, 614, 252, 0, 0, 0, 0, 0, 0, 591, 1666, 164, 259, 390, 491, 217, 619, 1747, 0, 0, 0, 0, 0, 0, 0, 620, 207, 490, 452, 795, 1013, 300, 257, 517, 431, 0, 0, 0, 0, 0, 0, 465, 533, 190, 472, 855, 857, 1472, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6053, 5642, 6735, 6652, 7459, 6144, 5082, 5385] avg=6144.0000 max=7459 max_rank=4 min=5082 min_rank=6 imbalance_ratio=1.214030 max_load_reduction=41693 max_load_reduction_pct=0.848246 moved_to_echo=24070 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1367, 308, 108, 401, 358, 825, 696, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 617, 1555, 716, 685, 965, 383, 885, 364, 12, 0, 0, 0, 0, 0, 0, 0, 969, 709, 1547, 1058, 236, 820, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 295, 782, 382, 728, 489, 1840, 269, 1125, 469, 0, 0, 0, 0, 0, 0, 589, 765, 893, 653, 787, 881, 383, 655, 409, 199, 0, 0, 0, 0, 0, 0, 234, 1399, 182, 177, 649, 375, 467, 402, 1936, 0, 0, 0, 0, 0, 0, 0, 743, 48, 541, 921, 1296, 923, 303, 260, 458, 340, 0, 0, 0, 0, 0, 0, 310, 1681, 595, 263, 618, 1252, 981, 773, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5502, 6182, 6320, 6807, 6214, 5821, 5833, 6473] avg=6144.0000 max=6807 max_rank=3 min=5502 min_rank=0 imbalance_ratio=1.107910 max_load_reduction=42345 max_load_reduction_pct=0.861511 moved_to_echo=24341 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2113, 473, 95, 326, 308, 806, 308, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 927, 829, 1225, 740, 490, 305, 380, 13, 0, 0, 0, 0, 0, 0, 0, 1182, 1161, 1190, 707, 363, 1485, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 321, 822, 256, 778, 372, 1216, 283, 1385, 414, 0, 0, 0, 0, 0, 0, 604, 386, 1087, 390, 697, 481, 626, 435, 430, 360, 0, 0, 0, 0, 0, 0, 407, 928, 231, 165, 518, 278, 401, 864, 1709, 0, 0, 0, 0, 0, 0, 0, 737, 223, 550, 468, 1563, 311, 360, 323, 564, 616, 0, 0, 0, 0, 0, 0, 840, 767, 389, 571, 1453, 1065, 1032, 423, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 6132, 7705, 6286, 5496, 5501, 5715, 6540] avg=6144.0000 max=7705 max_rank=2 min=5496 min_rank=4 imbalance_ratio=1.254069 max_load_reduction=41447 max_load_reduction_pct=0.843241 moved_to_echo=23252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1784, 543, 90, 588, 263, 737, 772, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 831, 845, 707, 1143, 907, 480, 629, 483, 17, 0, 0, 0, 0, 0, 0, 0, 1588, 337, 1316, 347, 767, 1084, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 184, 652, 416, 776, 403, 1284, 151, 1192, 389, 0, 0, 0, 0, 0, 0, 447, 636, 1260, 162, 614, 691, 588, 743, 553, 262, 0, 0, 0, 0, 0, 0, 896, 1139, 271, 146, 660, 132, 300, 988, 1606, 0, 0, 0, 0, 0, 0, 0, 936, 210, 385, 903, 1366, 849, 520, 349, 486, 449, 0, 0, 0, 0, 0, 0, 592, 635, 445, 591, 748, 1326, 1326, 404, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6042, 6424, 5929, 5956, 6138, 6453, 6067] avg=6144.0000 max=6453 max_rank=6 min=5929 min_rank=3 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1443, 508, 70, 1029, 336, 989, 435, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 790, 1174, 729, 948, 421, 423, 508, 13, 0, 0, 0, 0, 0, 0, 0, 820, 384, 1066, 748, 501, 545, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 889, 878, 323, 748, 407, 1245, 232, 853, 351, 0, 0, 0, 0, 0, 0, 1093, 601, 938, 315, 844, 1035, 706, 1027, 312, 137, 0, 0, 0, 0, 0, 0, 498, 1317, 541, 300, 495, 444, 570, 577, 1428, 0, 0, 0, 0, 0, 0, 0, 624, 444, 526, 1030, 904, 911, 381, 298, 348, 231, 0, 0, 0, 0, 0, 0, 713, 1469, 214, 682, 966, 1333, 731, 457, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6256, 6160, 5017, 6279, 7008, 6170, 5697, 6565] avg=6144.0000 max=7008 max_rank=4 min=5017 min_rank=2 imbalance_ratio=1.140625 max_load_reduction=42144 max_load_reduction_pct=0.857422 moved_to_echo=25440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2070, 507, 82, 766, 336, 922, 837, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 1198, 987, 670, 1019, 704, 469, 748, 221, 11, 0, 0, 0, 0, 0, 0, 0, 828, 678, 1081, 663, 687, 499, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 278, 334, 458, 788, 384, 1993, 280, 853, 353, 0, 0, 0, 0, 0, 0, 547, 522, 864, 309, 528, 836, 624, 1009, 363, 121, 0, 0, 0, 0, 0, 0, 712, 1850, 388, 149, 588, 462, 271, 503, 1458, 0, 0, 0, 0, 0, 0, 0, 1146, 402, 791, 1008, 1349, 695, 727, 259, 349, 207, 0, 0, 0, 0, 0, 0, 482, 657, 494, 899, 743, 739, 872, 420, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7214, 6027, 5631, 5937, 5723, 6381, 6933, 5306] avg=6144.0000 max=7214 max_rank=0 min=5306 min_rank=7 imbalance_ratio=1.174154 max_load_reduction=41938 max_load_reduction_pct=0.853231 moved_to_echo=24343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1287, 477, 89, 608, 371, 831, 1031, 1390, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 701, 811, 1088, 1048, 340, 618, 775, 11, 0, 0, 0, 0, 0, 0, 0, 988, 317, 934, 949, 560, 663, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 387, 578, 429, 639, 452, 1140, 229, 865, 386, 0, 0, 0, 0, 0, 0, 537, 684, 877, 240, 476, 793, 590, 904, 371, 160, 0, 0, 0, 0, 0, 0, 228, 857, 650, 248, 938, 445, 496, 689, 1594, 0, 0, 0, 0, 0, 0, 0, 701, 484, 887, 1163, 1280, 1351, 510, 581, 354, 274, 0, 0, 0, 0, 0, 0, 737, 980, 266, 775, 482, 1454, 891, 603, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6084, 6475, 5487, 5556, 5632, 6145, 7585, 6188] avg=6144.0000 max=7585 max_rank=6 min=5487 min_rank=2 imbalance_ratio=1.234538 max_load_reduction=41567 max_load_reduction_pct=0.845683 moved_to_echo=25550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1784, 543, 90, 588, 263, 737, 772, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 831, 845, 707, 1143, 907, 480, 629, 483, 17, 0, 0, 0, 0, 0, 0, 0, 1588, 337, 1316, 347, 767, 1084, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 184, 652, 416, 776, 403, 1284, 151, 1192, 389, 0, 0, 0, 0, 0, 0, 447, 636, 1260, 162, 614, 691, 588, 743, 553, 262, 0, 0, 0, 0, 0, 0, 896, 1139, 271, 146, 660, 132, 300, 988, 1606, 0, 0, 0, 0, 0, 0, 0, 936, 210, 385, 903, 1366, 849, 520, 349, 486, 449, 0, 0, 0, 0, 0, 0, 592, 635, 445, 591, 748, 1326, 1326, 404, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6042, 6424, 5929, 5956, 6138, 6453, 6067] avg=6144.0000 max=6453 max_rank=6 min=5929 min_rank=3 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2113, 473, 95, 326, 308, 806, 308, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 927, 829, 1225, 740, 490, 305, 380, 13, 0, 0, 0, 0, 0, 0, 0, 1182, 1161, 1190, 707, 363, 1485, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 321, 822, 256, 778, 372, 1216, 283, 1385, 414, 0, 0, 0, 0, 0, 0, 604, 386, 1087, 390, 697, 481, 626, 435, 430, 360, 0, 0, 0, 0, 0, 0, 407, 928, 231, 165, 518, 278, 401, 864, 1709, 0, 0, 0, 0, 0, 0, 0, 737, 223, 550, 468, 1563, 311, 360, 323, 564, 616, 0, 0, 0, 0, 0, 0, 840, 767, 389, 571, 1453, 1065, 1032, 423, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 6132, 7705, 6286, 5496, 5501, 5715, 6540] avg=6144.0000 max=7705 max_rank=2 min=5496 min_rank=4 imbalance_ratio=1.254069 max_load_reduction=41447 max_load_reduction_pct=0.843241 moved_to_echo=23252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1367, 308, 108, 401, 358, 825, 696, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 617, 1555, 716, 685, 965, 383, 885, 364, 12, 0, 0, 0, 0, 0, 0, 0, 969, 709, 1547, 1058, 236, 820, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 295, 782, 382, 728, 489, 1840, 269, 1125, 469, 0, 0, 0, 0, 0, 0, 589, 765, 893, 653, 787, 881, 383, 655, 409, 199, 0, 0, 0, 0, 0, 0, 234, 1399, 182, 177, 649, 375, 467, 402, 1936, 0, 0, 0, 0, 0, 0, 0, 743, 48, 541, 921, 1296, 923, 303, 260, 458, 340, 0, 0, 0, 0, 0, 0, 310, 1681, 595, 263, 618, 1252, 981, 773, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5502, 6182, 6320, 6807, 6214, 5821, 5833, 6473] avg=6144.0000 max=6807 max_rank=3 min=5502 min_rank=0 imbalance_ratio=1.107910 max_load_reduction=42345 max_load_reduction_pct=0.861511 moved_to_echo=24341 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1293, 645, 88, 712, 215, 1181, 327, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 693, 885, 1589, 654, 357, 787, 444, 12, 0, 0, 0, 0, 0, 0, 0, 734, 872, 1380, 457, 686, 596, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 229, 836, 197, 922, 501, 1249, 120, 771, 386, 0, 0, 0, 0, 0, 0, 441, 846, 742, 319, 783, 687, 613, 694, 395, 144, 0, 0, 0, 0, 0, 0, 528, 1624, 642, 200, 671, 829, 355, 411, 1592, 0, 0, 0, 0, 0, 0, 0, 652, 221, 477, 1153, 1444, 758, 369, 219, 314, 247, 0, 0, 0, 0, 0, 0, 1139, 625, 357, 877, 1180, 1141, 946, 363, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6492, 5833, 5706, 5664, 6852, 5854, 6628] avg=6144.0000 max=6852 max_rank=5 min=5664 min_rank=4 imbalance_ratio=1.115234 max_load_reduction=42300 max_load_reduction_pct=0.860596 moved_to_echo=24998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2070, 507, 82, 766, 336, 922, 837, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 1198, 987, 670, 1019, 704, 469, 748, 221, 11, 0, 0, 0, 0, 0, 0, 0, 828, 678, 1081, 663, 687, 499, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 278, 334, 458, 788, 384, 1993, 280, 853, 353, 0, 0, 0, 0, 0, 0, 547, 522, 864, 309, 528, 836, 624, 1009, 363, 121, 0, 0, 0, 0, 0, 0, 712, 1850, 388, 149, 588, 462, 271, 503, 1458, 0, 0, 0, 0, 0, 0, 0, 1146, 402, 791, 1008, 1349, 695, 727, 259, 349, 207, 0, 0, 0, 0, 0, 0, 482, 657, 494, 899, 743, 739, 872, 420, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7214, 6027, 5631, 5937, 5723, 6381, 6933, 5306] avg=6144.0000 max=7214 max_rank=0 min=5306 min_rank=7 imbalance_ratio=1.174154 max_load_reduction=41938 max_load_reduction_pct=0.853231 moved_to_echo=24343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1443, 508, 70, 1029, 336, 989, 435, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 790, 1174, 729, 948, 421, 423, 508, 13, 0, 0, 0, 0, 0, 0, 0, 820, 384, 1066, 748, 501, 545, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 889, 878, 323, 748, 407, 1245, 232, 853, 351, 0, 0, 0, 0, 0, 0, 1093, 601, 938, 315, 844, 1035, 706, 1027, 312, 137, 0, 0, 0, 0, 0, 0, 498, 1317, 541, 300, 495, 444, 570, 577, 1428, 0, 0, 0, 0, 0, 0, 0, 624, 444, 526, 1030, 904, 911, 381, 298, 348, 231, 0, 0, 0, 0, 0, 0, 713, 1469, 214, 682, 966, 1333, 731, 457, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6256, 6160, 5017, 6279, 7008, 6170, 5697, 6565] avg=6144.0000 max=7008 max_rank=4 min=5017 min_rank=2 imbalance_ratio=1.140625 max_load_reduction=42144 max_load_reduction_pct=0.857422 moved_to_echo=25440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3328, 2944, 5504, 2944, 6528, 3840, 11392, 1920, 8320, 3200, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5376, 7424, 3072, 5248, 7168, 9216, 8320, 4096, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5120, 5248, 8064, 2688, 6016, 6400, 4864, 6144, 3456, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[13184, 3840, 768, 5120, 2688, 7168, 4992, 11776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4096, 10880, 3072, 1664, 4992, 3456, 3200, 5120, 13184, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8320, 5504, 9856, 5632, 4224, 6784, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3328, 2944, 5504, 2944, 6528, 3840, 11392, 1920, 8320, 3200, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8192, 7424, 6784, 8576, 6656, 3456, 4736, 3712, 128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4096, 10880, 3072, 1664, 4992, 3456, 3200, 5120, 13184, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[13184, 3840, 768, 5120, 2688, 7168, 4992, 11776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5376, 7424, 3072, 5248, 7168, 9216, 8320, 4096, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6272, 2304, 4736, 7168, 10112, 6912, 3584, 2560, 3456, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[731, 1181, 310, 489, 429, 270, 303, 1237, 370, 227, 876, 346, 1388, 1182, 1316, 438, 674, 343, 757, 490, 907, 969, 362, 682, 498, 629, 1042, 857, 961, 1905, 1568, 513, 921, 654, 923, 255, 697, 1624, 650, 918, 676, 1775, 596, 647, 570, 399, 907, 384, 236, 881, 126, 1083, 1251, 447, 2059, 255, 538, 191, 1957, 418, 632, 996, 362, 874] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[546, 712, 164, 553, 232, 241, 138, 464, 506, 237, 1550, 455, 883, 1661, 1463, 214, 264, 254, 414, 472, 1224, 1334, 308, 511, 520, 181, 1550, 1189, 1113, 1644, 537, 320, 1792, 238, 2205, 365, 695, 1613, 346, 1599, 286, 1386, 505, 917, 577, 519, 1155, 511, 186, 598, 69, 1258, 1818, 851, 2126, 123, 623, 127, 1785, 616, 981, 793, 151, 484] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[827, 1647, 286, 658, 389, 409, 162, 1286, 429, 186, 1436, 485, 496, 1115, 1051, 930, 263, 286, 269, 500, 1111, 1156, 454, 721, 399, 606, 1092, 1243, 945, 1421, 885, 452, 1561, 496, 1007, 312, 879, 2096, 360, 1250, 404, 657, 541, 1415, 392, 612, 652, 955, 265, 1083, 60, 1106, 955, 814, 1337, 634, 613, 663, 1766, 706, 668, 763, 170, 365] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[920, 781, 289, 765, 321, 402, 227, 576, 647, 138, 1518, 398, 686, 1296, 1110, 867, 356, 275, 217, 651, 1187, 1034, 278, 1026, 767, 453, 1150, 1101, 1081, 827, 1750, 411, 831, 613, 295, 94, 805, 1507, 745, 1476, 246, 1553, 540, 1119, 751, 343, 619, 377, 255, 904, 112, 1188, 1356, 1446, 1464, 513, 671, 154, 2209, 623, 940, 741, 292, 865] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[605, 481, 390, 725, 378, 166, 288, 1584, 770, 94, 1607, 227, 604, 963, 1573, 389, 324, 328, 384, 388, 775, 1589, 354, 608, 429, 670, 1699, 1397, 942, 1088, 1130, 517, 1002, 496, 1189, 189, 1179, 1869, 1942, 1052, 284, 1460, 427, 1170, 619, 422, 1019, 591, 621, 652, 142, 447, 806, 860, 802, 636, 765, 177, 1513, 617, 1169, 985, 160, 424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[546, 712, 164, 553, 232, 241, 138, 464, 506, 237, 1550, 455, 883, 1661, 1463, 214, 264, 254, 414, 472, 1224, 1334, 308, 511, 520, 181, 1550, 1189, 1113, 1644, 537, 320, 1792, 238, 2205, 365, 695, 1613, 346, 1599, 286, 1386, 505, 917, 577, 519, 1155, 511, 186, 598, 69, 1258, 1818, 851, 2126, 123, 623, 127, 1785, 616, 981, 793, 151, 484] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[731, 1181, 310, 489, 429, 270, 303, 1237, 370, 227, 876, 346, 1388, 1182, 1316, 438, 674, 343, 757, 490, 907, 969, 362, 682, 498, 629, 1042, 857, 961, 1905, 1568, 513, 921, 654, 923, 255, 697, 1624, 650, 918, 676, 1775, 596, 647, 570, 399, 907, 384, 236, 881, 126, 1083, 1251, 447, 2059, 255, 538, 191, 1957, 418, 632, 996, 362, 874] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[920, 781, 289, 765, 321, 402, 227, 576, 647, 138, 1518, 398, 686, 1296, 1110, 867, 356, 275, 217, 651, 1187, 1034, 278, 1026, 767, 453, 1150, 1101, 1081, 827, 1750, 411, 831, 613, 295, 94, 805, 1507, 745, 1476, 246, 1553, 540, 1119, 751, 343, 619, 377, 255, 904, 112, 1188, 1356, 1446, 1464, 513, 671, 154, 2209, 623, 940, 741, 292, 865] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1156, 1475, 723, 613, 292, 334, 868, 1027, 1042, 98, 1201, 328, 662, 1650, 2076, 369, 358, 554, 597, 187, 1052, 880, 415, 1084, 548, 470, 849, 582, 1029, 1219, 1112, 694, 657, 227, 801, 294, 864, 1657, 461, 1018, 240, 970, 539, 1754, 307, 757, 813, 578, 280, 814, 46, 1364, 879, 574, 1803, 448, 793, 277, 1196, 838, 454, 795, 639, 471] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[772, 842, 652, 540, 443, 316, 122, 792, 481, 134, 819, 294, 441, 1046, 1618, 495, 404, 399, 442, 331, 1870, 1101, 430, 320, 433, 680, 1210, 1101, 1020, 1752, 1380, 504, 1374, 1160, 1510, 341, 707, 1785, 589, 1139, 407, 1038, 471, 1003, 505, 932, 997, 292, 320, 887, 60, 990, 461, 504, 1690, 1032, 692, 274, 1412, 804, 855, 722, 379, 636] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[675, 927, 862, 696, 374, 340, 95, 1135, 498, 254, 1524, 467, 361, 991, 1131, 402, 213, 371, 246, 426, 1559, 1064, 132, 391, 447, 739, 1903, 1377, 1066, 1043, 1004, 407, 1714, 408, 1085, 275, 1551, 2124, 522, 1895, 190, 346, 351, 2286, 645, 1039, 705, 476, 233, 487, 42, 1279, 1174, 737, 1266, 827, 410, 303, 1575, 505, 359, 443, 155, 625] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1156, 1475, 723, 613, 292, 334, 868, 1027, 1042, 98, 1201, 328, 662, 1650, 2076, 369, 358, 554, 597, 187, 1052, 880, 415, 1084, 548, 470, 849, 582, 1029, 1219, 1112, 694, 657, 227, 801, 294, 864, 1657, 461, 1018, 240, 970, 539, 1754, 307, 757, 813, 578, 280, 814, 46, 1364, 879, 574, 1803, 448, 793, 277, 1196, 838, 454, 795, 639, 471] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[546, 712, 164, 553, 232, 241, 138, 464, 1188, 0, 0, 0, 0, 0, 0, 0, 506, 237, 1550, 455, 883, 1661, 1288, 214, 0, 0, 0, 0, 0, 0, 0, 0, 264, 254, 414, 472, 1224, 1334, 308, 511, 995, 364, 0, 0, 0, 0, 0, 0, 520, 181, 1550, 1189, 1113, 0, 537, 320, 0, 0, 0, 0, 0, 0, 0, 0, 1792, 238, 2205, 365, 695, 61, 346, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 286, 1386, 505, 917, 577, 519, 1155, 511, 175, 147, 46, 0, 0, 0, 0, 0, 186, 598, 69, 1258, 1818, 851, 1979, 123, 0, 0, 0, 0, 0, 0, 0, 0, 623, 127, 1785, 616, 981, 793, 151, 484, 603, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4238, 6794, 6140, 5410, 7301, 6224, 6882, 6163] avg=6144.0000 max=7301 max_rank=4 min=4238 min_rank=0 imbalance_ratio=1.188314 max_load_reduction=41851 max_load_reduction_pct=0.851461 moved_to_echo=26570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[827, 1647, 286, 658, 389, 409, 162, 1286, 1544, 0, 0, 0, 0, 0, 0, 0, 429, 186, 1436, 485, 496, 1115, 926, 930, 0, 0, 0, 0, 0, 0, 0, 0, 263, 286, 269, 500, 1111, 1156, 454, 721, 860, 473, 0, 0, 0, 0, 0, 0, 399, 606, 1092, 1243, 945, 0, 885, 452, 0, 0, 0, 0, 0, 0, 0, 0, 1561, 496, 1007, 312, 879, 79, 360, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 404, 657, 541, 1415, 392, 612, 652, 955, 125, 92, 39, 0, 0, 0, 0, 0, 265, 1083, 60, 1106, 955, 814, 1245, 634, 0, 0, 0, 0, 0, 0, 0, 0, 613, 663, 1766, 706, 668, 763, 170, 365, 522, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7208, 6003, 6093, 5622, 5944, 5884, 6162, 6236] avg=6144.0000 max=7208 max_rank=0 min=5622 min_rank=3 imbalance_ratio=1.173177 max_load_reduction=41944 max_load_reduction_pct=0.853353 moved_to_echo=24226 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[605, 481, 390, 725, 378, 166, 288, 1584, 1377, 0, 0, 0, 0, 0, 0, 0, 770, 94, 1607, 227, 604, 963, 1385, 389, 0, 0, 0, 0, 0, 0, 0, 0, 324, 328, 384, 388, 775, 1589, 354, 608, 660, 422, 0, 0, 0, 0, 0, 0, 429, 670, 1699, 1397, 942, 0, 1130, 517, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 496, 1189, 189, 1179, 70, 1942, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 284, 1460, 427, 1170, 619, 422, 1019, 591, 188, 55, 30, 0, 0, 0, 0, 0, 621, 652, 142, 447, 806, 860, 747, 636, 0, 0, 0, 0, 0, 0, 0, 0, 765, 177, 1513, 617, 1169, 985, 160, 424, 398, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5994, 6039, 5832, 6784, 7119, 6265, 4911, 6208] avg=6144.0000 max=7119 max_rank=4 min=4911 min_rank=6 imbalance_ratio=1.158691 max_load_reduction=42033 max_load_reduction_pct=0.855164 moved_to_echo=24503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[546, 712, 164, 553, 232, 241, 138, 464, 1188, 0, 0, 0, 0, 0, 0, 0, 506, 237, 1550, 455, 883, 1661, 1288, 214, 0, 0, 0, 0, 0, 0, 0, 0, 264, 254, 414, 472, 1224, 1334, 308, 511, 995, 364, 0, 0, 0, 0, 0, 0, 520, 181, 1550, 1189, 1113, 0, 537, 320, 0, 0, 0, 0, 0, 0, 0, 0, 1792, 238, 2205, 365, 695, 61, 346, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 286, 1386, 505, 917, 577, 519, 1155, 511, 175, 147, 46, 0, 0, 0, 0, 0, 186, 598, 69, 1258, 1818, 851, 1979, 123, 0, 0, 0, 0, 0, 0, 0, 0, 623, 127, 1785, 616, 981, 793, 151, 484, 603, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4238, 6794, 6140, 5410, 7301, 6224, 6882, 6163] avg=6144.0000 max=7301 max_rank=4 min=4238 min_rank=0 imbalance_ratio=1.188314 max_load_reduction=41851 max_load_reduction_pct=0.851461 moved_to_echo=26570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[772, 842, 652, 540, 443, 316, 122, 792, 1315, 0, 0, 0, 0, 0, 0, 0, 481, 134, 819, 294, 441, 1046, 1425, 495, 0, 0, 0, 0, 0, 0, 0, 0, 404, 399, 442, 331, 1870, 1101, 430, 320, 1060, 403, 0, 0, 0, 0, 0, 0, 433, 680, 1210, 1101, 1020, 0, 1380, 504, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 1160, 1510, 341, 707, 67, 589, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 407, 1038, 471, 1003, 505, 932, 997, 292, 193, 117, 51, 0, 0, 0, 0, 0, 320, 887, 60, 990, 461, 504, 1573, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 692, 274, 1412, 804, 855, 722, 379, 636, 641, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5794, 5135, 6760, 6328, 6887, 6006, 5827, 6415] avg=6144.0000 max=6887 max_rank=4 min=5135 min_rank=1 imbalance_ratio=1.120931 max_load_reduction=42265 max_load_reduction_pct=0.859884 moved_to_echo=25135 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[731, 1181, 310, 489, 429, 270, 303, 1237, 1199, 0, 0, 0, 0, 0, 0, 0, 370, 227, 876, 346, 1388, 1182, 1155, 438, 0, 0, 0, 0, 0, 0, 0, 0, 674, 343, 757, 490, 907, 969, 362, 682, 1155, 371, 0, 0, 0, 0, 0, 0, 498, 629, 1042, 857, 961, 0, 1568, 513, 0, 0, 0, 0, 0, 0, 0, 0, 921, 654, 923, 255, 697, 54, 650, 918, 0, 0, 0, 0, 0, 0, 0, 0, 676, 1775, 596, 647, 570, 399, 907, 384, 161, 146, 53, 0, 0, 0, 0, 0, 236, 881, 126, 1083, 1251, 447, 1913, 255, 0, 0, 0, 0, 0, 0, 0, 0, 538, 191, 1957, 418, 632, 996, 362, 874, 697, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 5982, 6710, 6068, 5072, 6314, 6192, 6665] avg=6144.0000 max=6710 max_rank=2 min=5072 min_rank=4 imbalance_ratio=1.092122 max_load_reduction=42442 max_load_reduction_pct=0.863485 moved_to_echo=24243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[675, 927, 862, 696, 374, 340, 95, 1135, 1565, 0, 0, 0, 0, 0, 0, 0, 498, 254, 1524, 467, 361, 991, 996, 402, 0, 0, 0, 0, 0, 0, 0, 0, 213, 371, 246, 426, 1559, 1064, 132, 391, 631, 480, 0, 0, 0, 0, 0, 0, 447, 739, 1903, 1377, 1066, 0, 1004, 407, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 408, 1085, 275, 1551, 79, 522, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 190, 346, 351, 2286, 645, 1039, 705, 476, 135, 87, 30, 0, 0, 0, 0, 0, 233, 487, 42, 1279, 1174, 737, 1179, 827, 0, 0, 0, 0, 0, 0, 0, 0, 410, 303, 1575, 505, 359, 443, 155, 625, 382, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6669, 5493, 5513, 6943, 7529, 6290, 5958, 4757] avg=6144.0000 max=7529 max_rank=4 min=4757 min_rank=7 imbalance_ratio=1.225423 max_load_reduction=41623 max_load_reduction_pct=0.846822 moved_to_echo=24534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[920, 781, 289, 765, 321, 402, 227, 576, 1110, 0, 0, 0, 0, 0, 0, 0, 647, 138, 1518, 398, 686, 1296, 978, 867, 0, 0, 0, 0, 0, 0, 0, 0, 356, 275, 217, 651, 1187, 1034, 278, 1026, 500, 340, 0, 0, 0, 0, 0, 0, 767, 453, 1150, 1101, 1081, 0, 1750, 411, 0, 0, 0, 0, 0, 0, 0, 0, 831, 613, 295, 94, 805, 57, 745, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1553, 540, 1119, 751, 343, 619, 377, 132, 101, 23, 0, 0, 0, 0, 0, 255, 904, 112, 1188, 1356, 1446, 1363, 513, 0, 0, 0, 0, 0, 0, 0, 0, 671, 154, 2209, 623, 940, 741, 292, 865, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5391, 6528, 5864, 6713, 4916, 5804, 7137, 6799] avg=6144.0000 max=7137 max_rank=6 min=4916 min_rank=4 imbalance_ratio=1.161621 max_load_reduction=42015 max_load_reduction_pct=0.854797 moved_to_echo=24656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1156, 1475, 723, 613, 292, 334, 868, 1027, 1221, 0, 0, 0, 0, 0, 0, 0, 1042, 98, 1201, 328, 662, 1650, 1828, 369, 0, 0, 0, 0, 0, 0, 0, 0, 358, 554, 597, 187, 1052, 880, 415, 1084, 739, 374, 0, 0, 0, 0, 0, 0, 548, 470, 849, 582, 1029, 0, 1112, 694, 0, 0, 0, 0, 0, 0, 0, 0, 657, 227, 801, 294, 864, 62, 461, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 240, 970, 539, 1754, 307, 757, 813, 578, 248, 125, 34, 0, 0, 0, 0, 0, 280, 814, 46, 1364, 879, 574, 1678, 448, 0, 0, 0, 0, 0, 0, 0, 0, 793, 277, 1196, 838, 454, 795, 639, 471, 446, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7709, 7178, 6240, 5284, 4384, 6365, 6083, 5909] avg=6144.0000 max=7709 max_rank=0 min=4384 min_rank=4 imbalance_ratio=1.254720 max_load_reduction=41443 max_load_reduction_pct=0.843160 moved_to_echo=22741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[731, 1181, 310, 489, 429, 270, 303, 1237, 1199, 0, 0, 0, 0, 0, 0, 0, 370, 227, 876, 346, 1388, 1182, 1155, 438, 0, 0, 0, 0, 0, 0, 0, 0, 674, 343, 757, 490, 907, 969, 362, 682, 1155, 371, 0, 0, 0, 0, 0, 0, 498, 629, 1042, 857, 961, 0, 1568, 513, 0, 0, 0, 0, 0, 0, 0, 0, 921, 654, 923, 255, 697, 54, 650, 918, 0, 0, 0, 0, 0, 0, 0, 0, 676, 1775, 596, 647, 570, 399, 907, 384, 161, 146, 53, 0, 0, 0, 0, 0, 236, 881, 126, 1083, 1251, 447, 1913, 255, 0, 0, 0, 0, 0, 0, 0, 0, 538, 191, 1957, 418, 632, 996, 362, 874, 697, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 5982, 6710, 6068, 5072, 6314, 6192, 6665] avg=6144.0000 max=6710 max_rank=2 min=5072 min_rank=4 imbalance_ratio=1.092122 max_load_reduction=42442 max_load_reduction_pct=0.863485 moved_to_echo=24243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[675, 927, 862, 696, 374, 340, 95, 1135, 1565, 0, 0, 0, 0, 0, 0, 0, 498, 254, 1524, 467, 361, 991, 996, 402, 0, 0, 0, 0, 0, 0, 0, 0, 213, 371, 246, 426, 1559, 1064, 132, 391, 631, 480, 0, 0, 0, 0, 0, 0, 447, 739, 1903, 1377, 1066, 0, 1004, 407, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 408, 1085, 275, 1551, 79, 522, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 190, 346, 351, 2286, 645, 1039, 705, 476, 135, 87, 30, 0, 0, 0, 0, 0, 233, 487, 42, 1279, 1174, 737, 1179, 827, 0, 0, 0, 0, 0, 0, 0, 0, 410, 303, 1575, 505, 359, 443, 155, 625, 382, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6669, 5493, 5513, 6943, 7529, 6290, 5958, 4757] avg=6144.0000 max=7529 max_rank=4 min=4757 min_rank=7 imbalance_ratio=1.225423 max_load_reduction=41623 max_load_reduction_pct=0.846822 moved_to_echo=24534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4096, 4480, 10496, 8960, 8192, 0, 9472, 3840, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9856, 4352, 9088, 2176, 7424, 640, 5632, 10368, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 9216, 4096, 10368, 4480, 5120, 6912, 4224, 1408, 896, 384, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 1408, 10624, 3072, 5632, 9984, 9984, 4224, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 2816, 3328, 3456, 9728, 9216, 2816, 5376, 6656, 3328, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 2176, 13440, 5248, 6144, 6272, 2432, 4864, 4096, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 1408, 10624, 3072, 5632, 9984, 9984, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 9216, 4096, 10368, 4480, 5120, 6912, 4224, 1408, 896, 384, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6272, 8064, 3712, 5120, 2944, 2560, 2304, 8192, 10624, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 2176, 13440, 5248, 6144, 6272, 2432, 4864, 4096, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 6400, 768, 8832, 8704, 6272, 11776, 4480, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[922, 742, 2241, 547, 329, 111, 756, 232, 272, 416, 350, 325, 1134, 655, 543, 682, 734, 1948, 59, 174, 1518, 379, 1668, 737, 670, 346, 147, 715, 970, 1509, 261, 1276, 465, 365, 632, 267, 790, 446, 382, 691, 1089, 1931, 1403, 1196, 611, 803, 432, 834, 460, 587, 596, 792, 1281, 1667, 228, 831, 1028, 1120, 637, 325, 471, 584, 1006, 1834] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[685, 410, 1262, 342, 1498, 138, 1088, 585, 498, 519, 389, 340, 477, 223, 675, 1372, 868, 2369, 107, 333, 1915, 390, 1895, 851, 345, 342, 286, 678, 1320, 925, 281, 1943, 500, 540, 659, 308, 811, 798, 319, 381, 991, 808, 1341, 840, 452, 409, 761, 528, 415, 677, 531, 939, 1483, 1562, 416, 798, 765, 775, 675, 215, 534, 730, 1015, 1827] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1385, 398, 1036, 449, 1997, 154, 1551, 578, 593, 592, 330, 333, 1053, 400, 585, 1053, 941, 1567, 113, 345, 1776, 551, 1380, 476, 626, 335, 210, 614, 575, 1237, 278, 1030, 206, 558, 683, 556, 350, 655, 448, 582, 927, 1255, 1755, 731, 504, 510, 682, 786, 252, 562, 1285, 915, 1019, 1648, 595, 518, 1763, 641, 369, 263, 582, 141, 1371, 1499] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[486, 612, 1942, 440, 2199, 151, 1751, 507, 313, 667, 245, 436, 193, 599, 835, 1163, 715, 2081, 145, 735, 1741, 423, 891, 556, 477, 379, 138, 980, 876, 1307, 343, 1775, 246, 691, 424, 321, 681, 896, 648, 328, 912, 1329, 1013, 1256, 509, 273, 709, 760, 399, 581, 601, 370, 1580, 1734, 200, 856, 821, 370, 666, 228, 292, 843, 808, 1706] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1140, 620, 1237, 429, 2229, 229, 1576, 350, 384, 789, 393, 407, 655, 900, 603, 600, 633, 2057, 151, 387, 1533, 612, 1324, 582, 271, 249, 326, 613, 616, 1358, 255, 1482, 342, 557, 688, 492, 1242, 584, 701, 684, 676, 1255, 1179, 1392, 380, 557, 382, 983, 460, 611, 571, 486, 1383, 1165, 573, 879, 676, 449, 548, 469, 798, 191, 701, 2108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[429, 368, 665, 272, 1600, 185, 920, 578, 434, 548, 133, 275, 648, 850, 789, 744, 819, 1731, 495, 251, 1864, 239, 923, 552, 398, 323, 61, 542, 1503, 1502, 336, 2610, 325, 557, 342, 310, 1025, 703, 670, 645, 487, 2412, 1866, 1101, 612, 201, 450, 711, 380, 168, 855, 508, 1428, 1713, 367, 557, 488, 355, 462, 689, 638, 1270, 1055, 2215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[241, 513, 882, 890, 1427, 103, 994, 738, 310, 795, 257, 333, 663, 139, 1761, 692, 1292, 1795, 84, 470, 2363, 272, 1048, 692, 440, 307, 95, 1026, 494, 901, 311, 1374, 352, 746, 860, 178, 855, 744, 1219, 588, 1186, 1271, 1094, 1444, 1034, 513, 620, 583, 337, 186, 210, 1145, 1782, 1386, 657, 539, 414, 454, 448, 309, 645, 729, 999, 1923] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1385, 398, 1036, 449, 1997, 154, 1551, 578, 593, 592, 330, 333, 1053, 400, 585, 1053, 941, 1567, 113, 345, 1776, 551, 1380, 476, 626, 335, 210, 614, 575, 1237, 278, 1030, 206, 558, 683, 556, 350, 655, 448, 582, 927, 1255, 1755, 731, 504, 510, 682, 786, 252, 562, 1285, 915, 1019, 1648, 595, 518, 1763, 641, 369, 263, 582, 141, 1371, 1499] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1888, 662, 1094, 788, 591, 160, 857, 494, 224, 582, 270, 335, 2024, 127, 1770, 1029, 1419, 1328, 81, 380, 1260, 320, 941, 924, 330, 467, 319, 340, 958, 751, 412, 1160, 227, 334, 486, 585, 836, 508, 199, 1751, 286, 2098, 866, 847, 439, 742, 356, 1126, 359, 773, 1251, 610, 1063, 1455, 245, 819, 1358, 900, 1073, 146, 392, 153, 635, 1949] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[685, 410, 1262, 342, 1498, 138, 1088, 585, 498, 519, 389, 340, 477, 223, 675, 1372, 868, 2369, 107, 333, 1915, 390, 1895, 851, 345, 342, 286, 678, 1320, 925, 281, 1943, 500, 540, 659, 308, 811, 798, 319, 381, 991, 808, 1341, 840, 452, 409, 761, 528, 415, 677, 531, 939, 1483, 1562, 416, 798, 765, 775, 675, 215, 534, 730, 1015, 1827] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[241, 513, 882, 890, 1427, 103, 994, 738, 310, 795, 257, 333, 663, 139, 1761, 692, 1292, 1795, 84, 470, 2363, 272, 1048, 692, 440, 307, 95, 1026, 494, 901, 311, 1374, 352, 746, 860, 178, 855, 744, 1219, 588, 1186, 1271, 1094, 1444, 1034, 513, 620, 583, 337, 186, 210, 1145, 1782, 1386, 657, 539, 414, 454, 448, 309, 645, 729, 999, 1923] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[922, 742, 2241, 547, 329, 111, 756, 232, 272, 416, 350, 325, 1134, 655, 543, 682, 734, 1948, 59, 174, 1518, 379, 1668, 737, 670, 346, 147, 715, 970, 1509, 261, 1276, 465, 365, 632, 267, 790, 446, 382, 691, 1089, 1931, 1403, 1196, 611, 803, 432, 834, 460, 587, 596, 792, 1281, 1667, 228, 831, 1028, 1120, 637, 325, 471, 584, 1006, 1834] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[429, 368, 665, 272, 1600, 185, 920, 578, 434, 548, 133, 275, 648, 850, 789, 744, 819, 1731, 495, 251, 1864, 239, 923, 552, 398, 323, 61, 542, 1503, 1502, 336, 2610, 325, 557, 342, 310, 1025, 703, 670, 645, 487, 2412, 1866, 1101, 612, 201, 450, 711, 380, 168, 855, 508, 1428, 1713, 367, 557, 488, 355, 462, 689, 638, 1270, 1055, 2215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[486, 612, 1942, 440, 2199, 151, 1751, 507, 313, 667, 245, 436, 193, 599, 835, 1163, 715, 2081, 145, 735, 1741, 423, 891, 556, 477, 379, 138, 980, 876, 1307, 343, 1775, 246, 691, 424, 321, 681, 896, 648, 328, 912, 1329, 1013, 1256, 509, 273, 709, 760, 399, 581, 601, 370, 1580, 1734, 200, 856, 821, 370, 666, 228, 292, 843, 808, 1706] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[922, 742, 2241, 547, 232, 111, 756, 232, 0, 0, 0, 0, 0, 0, 0, 0, 272, 416, 350, 325, 1134, 655, 543, 682, 968, 97, 86, 0, 0, 0, 0, 0, 734, 632, 59, 174, 1518, 379, 1668, 737, 0, 0, 0, 0, 0, 0, 0, 0, 670, 346, 147, 715, 970, 1509, 261, 1276, 290, 204, 0, 0, 0, 0, 0, 0, 465, 365, 632, 267, 790, 446, 382, 691, 1316, 334, 0, 0, 0, 0, 0, 0, 1089, 629, 1403, 1196, 611, 803, 432, 834, 0, 0, 0, 0, 0, 0, 0, 0, 460, 587, 596, 792, 1281, 1377, 228, 831, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1120, 637, 325, 471, 584, 1006, 1544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5783, 5528, 5901, 6388, 5688, 6997, 6152, 6715] avg=6144.0000 max=6997 max_rank=5 min=5528 min_rank=1 imbalance_ratio=1.138835 max_load_reduction=42155 max_load_reduction_pct=0.857646 moved_to_echo=25552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1140, 620, 1237, 429, 1568, 229, 1576, 350, 0, 0, 0, 0, 0, 0, 0, 0, 384, 789, 393, 407, 655, 900, 603, 600, 629, 661, 99, 0, 0, 0, 0, 0, 633, 668, 151, 387, 1533, 612, 1324, 582, 0, 0, 0, 0, 0, 0, 0, 0, 271, 249, 326, 613, 616, 1358, 255, 1482, 202, 235, 0, 0, 0, 0, 0, 0, 342, 557, 688, 492, 1242, 584, 701, 684, 1389, 217, 0, 0, 0, 0, 0, 0, 676, 409, 1179, 1392, 380, 557, 382, 983, 0, 0, 0, 0, 0, 0, 0, 0, 460, 611, 571, 486, 1383, 963, 573, 879, 0, 0, 0, 0, 0, 0, 0, 0, 676, 449, 548, 469, 798, 191, 701, 1774, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7149, 6120, 5890, 5607, 6896, 5958, 5926, 5606] avg=6144.0000 max=7149 max_rank=0 min=5606 min_rank=7 imbalance_ratio=1.163574 max_load_reduction=42003 max_load_reduction_pct=0.854553 moved_to_echo=24386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[685, 410, 1262, 342, 1054, 138, 1088, 585, 0, 0, 0, 0, 0, 0, 0, 0, 498, 519, 389, 340, 477, 223, 675, 1372, 405, 444, 86, 0, 0, 0, 0, 0, 868, 769, 107, 333, 1915, 390, 1895, 851, 0, 0, 0, 0, 0, 0, 0, 0, 345, 342, 286, 678, 1320, 925, 281, 1943, 271, 203, 0, 0, 0, 0, 0, 0, 500, 540, 659, 308, 811, 798, 319, 381, 1600, 139, 0, 0, 0, 0, 0, 0, 991, 264, 1341, 840, 452, 409, 761, 528, 0, 0, 0, 0, 0, 0, 0, 0, 415, 677, 531, 939, 1483, 1291, 416, 798, 0, 0, 0, 0, 0, 0, 0, 0, 765, 775, 675, 215, 534, 730, 1015, 1538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5564, 5428, 7128, 6594, 6055, 5586, 6550, 6247] avg=6144.0000 max=7128 max_rank=2 min=5428 min_rank=1 imbalance_ratio=1.160156 max_load_reduction=42024 max_load_reduction_pct=0.854980 moved_to_echo=24438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[429, 368, 665, 272, 1126, 185, 920, 578, 0, 0, 0, 0, 0, 0, 0, 0, 434, 548, 133, 275, 648, 850, 789, 744, 1209, 474, 104, 0, 0, 0, 0, 0, 819, 562, 495, 251, 1864, 239, 923, 552, 0, 0, 0, 0, 0, 0, 0, 0, 398, 323, 61, 542, 1503, 1502, 336, 2610, 298, 247, 0, 0, 0, 0, 0, 0, 325, 557, 342, 310, 1025, 703, 670, 645, 1169, 417, 0, 0, 0, 0, 0, 0, 487, 786, 1866, 1101, 612, 201, 450, 711, 0, 0, 0, 0, 0, 0, 0, 0, 380, 168, 855, 508, 1428, 1415, 367, 557, 0, 0, 0, 0, 0, 0, 0, 0, 488, 355, 462, 689, 638, 1270, 1055, 1864, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4543, 6208, 5705, 7820, 6163, 6214, 5678, 6821] avg=6144.0000 max=7820 max_rank=3 min=4543 min_rank=0 imbalance_ratio=1.272786 max_load_reduction=41332 max_load_reduction_pct=0.840902 moved_to_echo=24876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[486, 612, 1942, 440, 1547, 151, 1751, 507, 0, 0, 0, 0, 0, 0, 0, 0, 313, 667, 245, 436, 193, 599, 835, 1163, 666, 652, 80, 0, 0, 0, 0, 0, 715, 675, 145, 735, 1741, 423, 891, 556, 0, 0, 0, 0, 0, 0, 0, 0, 477, 379, 138, 980, 876, 1307, 343, 1775, 301, 190, 0, 0, 0, 0, 0, 0, 246, 691, 424, 321, 681, 896, 648, 328, 1406, 230, 0, 0, 0, 0, 0, 0, 912, 433, 1013, 1256, 509, 273, 709, 760, 0, 0, 0, 0, 0, 0, 0, 0, 399, 581, 601, 370, 1580, 1433, 200, 856, 0, 0, 0, 0, 0, 0, 0, 0, 821, 370, 666, 228, 292, 843, 808, 1436, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7436, 5849, 5881, 6766, 5871, 5865, 6020, 5464] avg=6144.0000 max=7436 max_rank=0 min=5464 min_rank=7 imbalance_ratio=1.210286 max_load_reduction=41716 max_load_reduction_pct=0.848714 moved_to_echo=23220 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1385, 398, 1036, 449, 1402, 154, 1551, 578, 0, 0, 0, 0, 0, 0, 0, 0, 593, 592, 330, 333, 1053, 400, 585, 1053, 632, 595, 74, 0, 0, 0, 0, 0, 941, 505, 113, 345, 1776, 551, 1380, 476, 0, 0, 0, 0, 0, 0, 0, 0, 626, 335, 210, 614, 575, 1237, 278, 1030, 291, 171, 0, 0, 0, 0, 0, 0, 206, 558, 683, 556, 350, 655, 448, 582, 1062, 220, 0, 0, 0, 0, 0, 0, 927, 403, 1755, 731, 504, 510, 682, 786, 0, 0, 0, 0, 0, 0, 0, 0, 252, 562, 1285, 915, 1019, 1357, 595, 518, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 641, 369, 263, 582, 141, 1371, 1254, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6953, 6240, 6087, 5367, 5320, 6298, 6503, 6384] avg=6144.0000 max=6953 max_rank=0 min=5320 min_rank=4 imbalance_ratio=1.131673 max_load_reduction=42199 max_load_reduction_pct=0.858541 moved_to_echo=24505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1140, 620, 1237, 429, 1568, 229, 1576, 350, 0, 0, 0, 0, 0, 0, 0, 0, 384, 789, 393, 407, 655, 900, 603, 600, 629, 661, 99, 0, 0, 0, 0, 0, 633, 668, 151, 387, 1533, 612, 1324, 582, 0, 0, 0, 0, 0, 0, 0, 0, 271, 249, 326, 613, 616, 1358, 255, 1482, 202, 235, 0, 0, 0, 0, 0, 0, 342, 557, 688, 492, 1242, 584, 701, 684, 1389, 217, 0, 0, 0, 0, 0, 0, 676, 409, 1179, 1392, 380, 557, 382, 983, 0, 0, 0, 0, 0, 0, 0, 0, 460, 611, 571, 486, 1383, 963, 573, 879, 0, 0, 0, 0, 0, 0, 0, 0, 676, 449, 548, 469, 798, 191, 701, 1774, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7149, 6120, 5890, 5607, 6896, 5958, 5926, 5606] avg=6144.0000 max=7149 max_rank=0 min=5606 min_rank=7 imbalance_ratio=1.163574 max_load_reduction=42003 max_load_reduction_pct=0.854553 moved_to_echo=24386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[241, 513, 882, 890, 1004, 103, 994, 738, 0, 0, 0, 0, 0, 0, 0, 0, 310, 795, 257, 333, 663, 139, 1761, 692, 637, 423, 90, 0, 0, 0, 0, 0, 1292, 583, 84, 470, 2363, 272, 1048, 692, 0, 0, 0, 0, 0, 0, 0, 0, 440, 307, 95, 1026, 494, 901, 311, 1374, 241, 214, 0, 0, 0, 0, 0, 0, 352, 746, 860, 178, 855, 744, 1219, 588, 1212, 219, 0, 0, 0, 0, 0, 0, 1186, 415, 1094, 1444, 1034, 513, 620, 583, 0, 0, 0, 0, 0, 0, 0, 0, 337, 186, 210, 1145, 1782, 1145, 657, 539, 0, 0, 0, 0, 0, 0, 0, 0, 414, 454, 448, 309, 645, 729, 999, 1619, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5365, 6100, 6804, 5403, 6973, 6889, 6001, 5617] avg=6144.0000 max=6973 max_rank=4 min=5365 min_rank=0 imbalance_ratio=1.134928 max_load_reduction=42179 max_load_reduction_pct=0.858134 moved_to_echo=25480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1888, 662, 1094, 788, 416, 160, 857, 494, 0, 0, 0, 0, 0, 0, 0, 0, 224, 582, 270, 335, 2024, 127, 1770, 1029, 1052, 175, 91, 0, 0, 0, 0, 0, 1419, 431, 81, 380, 1260, 320, 941, 924, 0, 0, 0, 0, 0, 0, 0, 0, 330, 467, 319, 340, 958, 751, 412, 1160, 253, 217, 0, 0, 0, 0, 0, 0, 227, 334, 486, 585, 836, 508, 199, 1751, 897, 363, 0, 0, 0, 0, 0, 0, 286, 683, 866, 847, 439, 742, 356, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 359, 773, 1251, 610, 1063, 1202, 245, 819, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 900, 1073, 146, 392, 153, 635, 1641, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6359, 7679, 5756, 5207, 6186, 5345, 6322, 6298] avg=6144.0000 max=7679 max_rank=1 min=5207 min_rank=3 imbalance_ratio=1.249837 max_load_reduction=41473 max_load_reduction_pct=0.843770 moved_to_echo=24151 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[241, 513, 882, 890, 1004, 103, 994, 738, 0, 0, 0, 0, 0, 0, 0, 0, 310, 795, 257, 333, 663, 139, 1761, 692, 637, 423, 90, 0, 0, 0, 0, 0, 1292, 583, 84, 470, 2363, 272, 1048, 692, 0, 0, 0, 0, 0, 0, 0, 0, 440, 307, 95, 1026, 494, 901, 311, 1374, 241, 214, 0, 0, 0, 0, 0, 0, 352, 746, 860, 178, 855, 744, 1219, 588, 1212, 219, 0, 0, 0, 0, 0, 0, 1186, 415, 1094, 1444, 1034, 513, 620, 583, 0, 0, 0, 0, 0, 0, 0, 0, 337, 186, 210, 1145, 1782, 1145, 657, 539, 0, 0, 0, 0, 0, 0, 0, 0, 414, 454, 448, 309, 645, 729, 999, 1619, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5365, 6100, 6804, 5403, 6973, 6889, 6001, 5617] avg=6144.0000 max=6973 max_rank=4 min=5365 min_rank=0 imbalance_ratio=1.134928 max_load_reduction=42179 max_load_reduction_pct=0.858134 moved_to_echo=25480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1385, 398, 1036, 449, 1402, 154, 1551, 578, 0, 0, 0, 0, 0, 0, 0, 0, 593, 592, 330, 333, 1053, 400, 585, 1053, 632, 595, 74, 0, 0, 0, 0, 0, 941, 505, 113, 345, 1776, 551, 1380, 476, 0, 0, 0, 0, 0, 0, 0, 0, 626, 335, 210, 614, 575, 1237, 278, 1030, 291, 171, 0, 0, 0, 0, 0, 0, 206, 558, 683, 556, 350, 655, 448, 582, 1062, 220, 0, 0, 0, 0, 0, 0, 927, 403, 1755, 731, 504, 510, 682, 786, 0, 0, 0, 0, 0, 0, 0, 0, 252, 562, 1285, 915, 1019, 1357, 595, 518, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 641, 369, 263, 582, 141, 1371, 1254, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6953, 6240, 6087, 5367, 5320, 6298, 6503, 6384] avg=6144.0000 max=6953 max_rank=0 min=5320 min_rank=4 imbalance_ratio=1.131673 max_load_reduction=42199 max_load_reduction_pct=0.858541 moved_to_echo=24505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[486, 612, 1942, 440, 1547, 151, 1751, 507, 0, 0, 0, 0, 0, 0, 0, 0, 313, 667, 245, 436, 193, 599, 835, 1163, 666, 652, 80, 0, 0, 0, 0, 0, 715, 675, 145, 735, 1741, 423, 891, 556, 0, 0, 0, 0, 0, 0, 0, 0, 477, 379, 138, 980, 876, 1307, 343, 1775, 301, 190, 0, 0, 0, 0, 0, 0, 246, 691, 424, 321, 681, 896, 648, 328, 1406, 230, 0, 0, 0, 0, 0, 0, 912, 433, 1013, 1256, 509, 273, 709, 760, 0, 0, 0, 0, 0, 0, 0, 0, 399, 581, 601, 370, 1580, 1433, 200, 856, 0, 0, 0, 0, 0, 0, 0, 0, 821, 370, 666, 228, 292, 843, 808, 1436, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7436, 5849, 5881, 6766, 5871, 5865, 6020, 5464] avg=6144.0000 max=7436 max_rank=0 min=5464 min_rank=7 imbalance_ratio=1.210286 max_load_reduction=41716 max_load_reduction_pct=0.848714 moved_to_echo=23220 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7424, 5120, 4992, 2688, 4352, 4736, 7680, 12672, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 2816, 1664, 5632, 7424, 9600, 2560, 12672, 2176, 1792, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6656, 4096, 10624, 8832, 4608, 4096, 4480, 6400, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3072, 4992, 2432, 2816, 6912, 3968, 7680, 7424, 6272, 3584, 768, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7424, 4864, 1280, 3200, 14080, 3200, 10112, 5376, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 4352, 4864, 3072, 6656, 5376, 4608, 5760, 10112, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 2816, 1664, 5632, 7424, 9600, 2560, 12672, 2176, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7296, 4352, 10368, 4224, 8448, 1280, 9600, 4096, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 4224, 6016, 5888, 11136, 10240, 3328, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7296, 4352, 10368, 4224, 8448, 1280, 9600, 4096, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[353, 1886, 1265, 608, 468, 981, 1365, 179, 1367, 343, 769, 296, 576, 343, 273, 498, 179, 1047, 256, 1382, 726, 61, 306, 2382, 626, 304, 782, 503, 2296, 278, 561, 548, 797, 1186, 1092, 1714, 1425, 1541, 690, 385, 1254, 858, 509, 377, 329, 80, 271, 485, 847, 253, 219, 725, 1051, 331, 1694, 399, 468, 1332, 1197, 876, 620, 1208, 893, 239] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1203, 2182, 843, 415, 231, 78, 1637, 127, 1114, 307, 103, 308, 606, 303, 409, 230, 58, 509, 285, 806, 977, 88, 352, 2377, 668, 297, 799, 424, 1748, 836, 523, 619, 1698, 2460, 1371, 757, 1312, 1356, 1833, 790, 1706, 486, 314, 207, 347, 256, 76, 660, 988, 336, 349, 921, 503, 1251, 735, 319, 901, 613, 1265, 667, 681, 1196, 745, 591] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[294, 1224, 1065, 1503, 298, 599, 2386, 265, 853, 266, 899, 475, 649, 637, 444, 207, 168, 893, 298, 819, 537, 268, 871, 1404, 565, 1298, 984, 575, 861, 229, 492, 713, 620, 2519, 517, 452, 1349, 1053, 1112, 1383, 1123, 753, 503, 276, 518, 104, 54, 587, 1249, 373, 264, 278, 450, 2125, 882, 1069, 287, 1843, 587, 562, 783, 1568, 250, 620] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[162, 1684, 883, 836, 323, 273, 1249, 224, 975, 700, 281, 372, 1467, 340, 975, 862, 85, 580, 148, 1427, 678, 291, 439, 460, 539, 719, 615, 148, 1336, 689, 199, 534, 1387, 2797, 1061, 1053, 828, 1122, 1347, 773, 1123, 545, 390, 280, 639, 198, 82, 521, 1072, 216, 286, 878, 644, 2082, 577, 312, 908, 1329, 506, 387, 1494, 2767, 395, 660] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[767, 860, 1299, 305, 507, 310, 1043, 115, 749, 269, 387, 194, 674, 1403, 584, 248, 104, 1047, 156, 876, 570, 178, 981, 1958, 905, 343, 878, 503, 1346, 460, 623, 699, 491, 2904, 606, 1050, 1378, 1721, 1175, 1044, 836, 876, 739, 292, 595, 202, 189, 701, 1309, 677, 223, 789, 1111, 1151, 500, 289, 683, 1508, 1181, 494, 831, 1438, 441, 387] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[100, 1576, 1710, 283, 498, 90, 1127, 198, 1949, 741, 982, 255, 756, 815, 415, 489, 69, 1931, 504, 618, 390, 62, 1021, 1136, 567, 549, 1148, 620, 2138, 41, 552, 467, 1238, 2279, 874, 1403, 785, 1554, 1330, 271, 1151, 233, 591, 201, 294, 662, 178, 566, 655, 342, 342, 1430, 442, 267, 502, 505, 440, 1127, 924, 1471, 862, 1392, 122, 922] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[431, 1229, 1182, 417, 470, 51, 2013, 219, 954, 428, 274, 340, 561, 1260, 307, 250, 273, 1875, 440, 614, 1106, 106, 408, 1659, 358, 216, 1215, 501, 2079, 143, 918, 809, 1110, 1693, 604, 1537, 1133, 1050, 1619, 208, 1549, 526, 609, 131, 564, 742, 158, 635, 1044, 563, 331, 1163, 217, 572, 1150, 531, 440, 454, 931, 1057, 852, 2078, 482, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[353, 1886, 1265, 608, 468, 981, 1365, 179, 1367, 343, 769, 296, 576, 343, 273, 498, 179, 1047, 256, 1382, 726, 61, 306, 2382, 626, 304, 782, 503, 2296, 278, 561, 548, 797, 1186, 1092, 1714, 1425, 1541, 690, 385, 1254, 858, 509, 377, 329, 80, 271, 485, 847, 253, 219, 725, 1051, 331, 1694, 399, 468, 1332, 1197, 876, 620, 1208, 893, 239] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[861, 1844, 559, 789, 443, 239, 2072, 250, 1362, 600, 1165, 265, 399, 967, 410, 292, 122, 661, 373, 880, 491, 376, 277, 1361, 145, 529, 484, 1184, 802, 248, 635, 469, 464, 1935, 840, 916, 1359, 1250, 1101, 799, 1183, 319, 629, 324, 789, 254, 408, 1293, 364, 152, 303, 951, 489, 1716, 710, 1047, 372, 2199, 1091, 569, 1160, 1611, 435, 566] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1203, 2182, 843, 415, 231, 78, 1410, 127, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 307, 103, 308, 606, 303, 409, 230, 918, 408, 0, 0, 0, 0, 0, 0, 58, 509, 285, 806, 977, 88, 352, 2377, 428, 180, 0, 0, 0, 0, 0, 0, 668, 297, 799, 424, 1748, 836, 523, 619, 385, 0, 0, 0, 0, 0, 0, 0, 1698, 0, 1371, 757, 1312, 10, 1833, 790, 0, 0, 0, 0, 0, 0, 0, 0, 1706, 486, 314, 207, 347, 256, 76, 660, 2052, 0, 0, 0, 0, 0, 0, 0, 988, 336, 349, 921, 503, 1251, 735, 319, 166, 227, 0, 0, 0, 0, 0, 0, 901, 613, 1265, 667, 681, 465, 745, 591, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6489, 4706, 6060, 6299, 7771, 6104, 5795, 5928] avg=6144.0000 max=7771 max_rank=4 min=4706 min_rank=1 imbalance_ratio=1.264811 max_load_reduction=41381 max_load_reduction_pct=0.841899 moved_to_echo=25598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[767, 860, 1299, 305, 507, 310, 898, 115, 0, 0, 0, 0, 0, 0, 0, 0, 749, 269, 387, 194, 674, 1403, 584, 248, 1165, 482, 0, 0, 0, 0, 0, 0, 104, 1047, 156, 876, 570, 178, 981, 1958, 544, 217, 0, 0, 0, 0, 0, 0, 905, 343, 878, 503, 1346, 460, 623, 699, 463, 0, 0, 0, 0, 0, 0, 0, 491, 0, 606, 1050, 1378, 12, 1175, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 836, 876, 739, 292, 595, 202, 189, 701, 2422, 0, 0, 0, 0, 0, 0, 0, 1309, 677, 223, 789, 1111, 1151, 500, 289, 199, 145, 0, 0, 0, 0, 0, 0, 683, 1508, 1181, 494, 831, 559, 441, 387, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5061, 6155, 6631, 6220, 5756, 6852, 6393, 6084] avg=6144.0000 max=6852 max_rank=5 min=5061 min_rank=0 imbalance_ratio=1.115234 max_load_reduction=42300 max_load_reduction_pct=0.860596 moved_to_echo=25085 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[431, 1229, 1182, 417, 470, 51, 1733, 219, 0, 0, 0, 0, 0, 0, 0, 0, 954, 428, 274, 340, 561, 1260, 307, 250, 711, 281, 0, 0, 0, 0, 0, 0, 273, 1875, 440, 614, 1106, 106, 408, 1659, 331, 314, 0, 0, 0, 0, 0, 0, 358, 216, 1215, 501, 2079, 143, 918, 809, 669, 0, 0, 0, 0, 0, 0, 0, 1110, 0, 604, 1537, 1133, 8, 1619, 208, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 526, 609, 131, 564, 742, 158, 635, 1412, 0, 0, 0, 0, 0, 0, 0, 1044, 563, 331, 1163, 217, 572, 1150, 531, 288, 280, 0, 0, 0, 0, 0, 0, 440, 454, 931, 1057, 852, 807, 482, 313, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5732, 5366, 7126, 6908, 6219, 6326, 6139, 5336] avg=6144.0000 max=7126 max_rank=2 min=5336 min_rank=7 imbalance_ratio=1.159831 max_load_reduction=42026 max_load_reduction_pct=0.855021 moved_to_echo=24020 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[294, 1224, 1065, 1503, 298, 599, 2386, 265, 853, 266, 899, 475, 649, 637, 444, 207, 168, 893, 298, 819, 537, 268, 871, 1404, 565, 1298, 984, 575, 861, 229, 492, 713, 620, 2519, 517, 452, 1349, 1053, 1112, 1383, 1123, 753, 503, 276, 518, 104, 54, 587, 1249, 373, 264, 278, 450, 2125, 882, 1069, 287, 1843, 587, 562, 783, 1568, 250, 620] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[162, 1684, 883, 836, 323, 273, 1249, 224, 975, 700, 281, 372, 1467, 340, 975, 862, 85, 580, 148, 1427, 678, 291, 439, 460, 539, 719, 615, 148, 1336, 689, 199, 534, 1387, 2797, 1061, 1053, 828, 1122, 1347, 773, 1123, 545, 390, 280, 639, 198, 82, 521, 1072, 216, 286, 878, 644, 2082, 577, 312, 908, 1329, 506, 387, 1494, 2767, 395, 660] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[353, 1886, 1265, 608, 468, 981, 1176, 179, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 343, 769, 296, 576, 343, 273, 498, 1043, 196, 0, 0, 0, 0, 0, 0, 179, 1047, 256, 1382, 726, 61, 306, 2382, 487, 182, 0, 0, 0, 0, 0, 0, 626, 304, 782, 503, 2296, 278, 561, 548, 389, 0, 0, 0, 0, 0, 0, 0, 797, 0, 1092, 1714, 1425, 11, 690, 385, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 858, 509, 377, 329, 80, 271, 485, 990, 0, 0, 0, 0, 0, 0, 0, 847, 253, 219, 725, 1051, 331, 1694, 399, 167, 189, 0, 0, 0, 0, 0, 0, 468, 1332, 1197, 876, 620, 470, 893, 239, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6916, 5704, 7008, 6287, 6114, 5153, 5875, 6095] avg=6144.0000 max=7008 max_rank=2 min=5153 min_rank=5 imbalance_ratio=1.140625 max_load_reduction=42144 max_load_reduction_pct=0.857422 moved_to_echo=23237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[767, 860, 1299, 305, 507, 310, 898, 115, 0, 0, 0, 0, 0, 0, 0, 0, 749, 269, 387, 194, 674, 1403, 584, 248, 1165, 482, 0, 0, 0, 0, 0, 0, 104, 1047, 156, 876, 570, 178, 981, 1958, 544, 217, 0, 0, 0, 0, 0, 0, 905, 343, 878, 503, 1346, 460, 623, 699, 463, 0, 0, 0, 0, 0, 0, 0, 491, 0, 606, 1050, 1378, 12, 1175, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 836, 876, 739, 292, 595, 202, 189, 701, 2422, 0, 0, 0, 0, 0, 0, 0, 1309, 677, 223, 789, 1111, 1151, 500, 289, 199, 145, 0, 0, 0, 0, 0, 0, 683, 1508, 1181, 494, 831, 559, 441, 387, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5061, 6155, 6631, 6220, 5756, 6852, 6393, 6084] avg=6144.0000 max=6852 max_rank=5 min=5061 min_rank=0 imbalance_ratio=1.115234 max_load_reduction=42300 max_load_reduction_pct=0.860596 moved_to_echo=25085 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[100, 1576, 1710, 283, 498, 90, 971, 198, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 741, 982, 255, 756, 815, 415, 489, 1052, 379, 0, 0, 0, 0, 0, 0, 69, 1931, 504, 618, 390, 62, 1021, 1136, 491, 210, 0, 0, 0, 0, 0, 0, 567, 549, 1148, 620, 2138, 41, 552, 467, 448, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 874, 1403, 785, 11, 1330, 271, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 233, 591, 201, 294, 662, 178, 566, 1900, 0, 0, 0, 0, 0, 0, 0, 655, 342, 342, 1430, 442, 267, 502, 505, 193, 156, 0, 0, 0, 0, 0, 0, 440, 1127, 924, 1471, 862, 541, 122, 922, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5426, 7833, 6432, 6530, 5912, 5776, 4834, 6409] avg=6144.0000 max=7833 max_rank=1 min=4834 min_rank=6 imbalance_ratio=1.274902 max_load_reduction=41319 max_load_reduction_pct=0.840637 moved_to_echo=22931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[294, 1224, 1065, 1503, 298, 599, 2050, 265, 0, 0, 0, 0, 0, 0, 0, 0, 853, 266, 899, 475, 649, 637, 444, 207, 716, 419, 0, 0, 0, 0, 0, 0, 168, 893, 298, 819, 537, 268, 871, 1404, 336, 242, 0, 0, 0, 0, 0, 0, 565, 1298, 984, 575, 861, 229, 492, 713, 507, 0, 0, 0, 0, 0, 0, 0, 620, 0, 517, 452, 1349, 1, 1112, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 753, 503, 276, 518, 104, 54, 587, 2100, 0, 0, 0, 0, 0, 0, 0, 1249, 373, 264, 278, 450, 2125, 882, 1069, 223, 336, 0, 0, 0, 0, 0, 0, 287, 1843, 587, 562, 783, 596, 250, 620, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7298, 5565, 5836, 6224, 5434, 6018, 7249, 5528] avg=6144.0000 max=7298 max_rank=0 min=5434 min_rank=4 imbalance_ratio=1.187826 max_load_reduction=41854 max_load_reduction_pct=0.851522 moved_to_echo=24229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[431, 1229, 1182, 417, 470, 51, 1733, 219, 0, 0, 0, 0, 0, 0, 0, 0, 954, 428, 274, 340, 561, 1260, 307, 250, 711, 281, 0, 0, 0, 0, 0, 0, 273, 1875, 440, 614, 1106, 106, 408, 1659, 331, 314, 0, 0, 0, 0, 0, 0, 358, 216, 1215, 501, 2079, 143, 918, 809, 669, 0, 0, 0, 0, 0, 0, 0, 1110, 0, 604, 1537, 1133, 8, 1619, 208, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 526, 609, 131, 564, 742, 158, 635, 1412, 0, 0, 0, 0, 0, 0, 0, 1044, 563, 331, 1163, 217, 572, 1150, 531, 288, 280, 0, 0, 0, 0, 0, 0, 440, 454, 931, 1057, 852, 807, 482, 313, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5732, 5366, 7126, 6908, 6219, 6326, 6139, 5336] avg=6144.0000 max=7126 max_rank=2 min=5336 min_rank=7 imbalance_ratio=1.159831 max_load_reduction=42026 max_load_reduction_pct=0.855021 moved_to_echo=24020 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1203, 2182, 843, 415, 231, 78, 1410, 127, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 307, 103, 308, 606, 303, 409, 230, 918, 408, 0, 0, 0, 0, 0, 0, 58, 509, 285, 806, 977, 88, 352, 2377, 428, 180, 0, 0, 0, 0, 0, 0, 668, 297, 799, 424, 1748, 836, 523, 619, 385, 0, 0, 0, 0, 0, 0, 0, 1698, 0, 1371, 757, 1312, 10, 1833, 790, 0, 0, 0, 0, 0, 0, 0, 0, 1706, 486, 314, 207, 347, 256, 76, 660, 2052, 0, 0, 0, 0, 0, 0, 0, 988, 336, 349, 921, 503, 1251, 735, 319, 166, 227, 0, 0, 0, 0, 0, 0, 901, 613, 1265, 667, 681, 465, 745, 591, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6489, 4706, 6060, 6299, 7771, 6104, 5795, 5928] avg=6144.0000 max=7771 max_rank=4 min=4706 min_rank=1 imbalance_ratio=1.264811 max_load_reduction=41381 max_load_reduction_pct=0.841899 moved_to_echo=25598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[162, 1684, 883, 836, 323, 273, 1076, 224, 0, 0, 0, 0, 0, 0, 0, 0, 975, 700, 281, 372, 1467, 340, 975, 862, 759, 465, 0, 0, 0, 0, 0, 0, 85, 580, 148, 1427, 678, 291, 439, 460, 354, 418, 0, 0, 0, 0, 0, 0, 539, 719, 615, 148, 1336, 689, 199, 534, 891, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 1061, 1053, 828, 9, 1347, 773, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 545, 390, 280, 639, 198, 82, 521, 2332, 0, 0, 0, 0, 0, 0, 0, 1072, 216, 286, 878, 644, 2082, 577, 312, 384, 173, 0, 0, 0, 0, 0, 0, 908, 1329, 506, 387, 1494, 1074, 395, 660, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5461, 7196, 4880, 5670, 6458, 6110, 6624, 6753] avg=6144.0000 max=7196 max_rank=1 min=4880 min_rank=2 imbalance_ratio=1.171224 max_load_reduction=41956 max_load_reduction_pct=0.853597 moved_to_echo=25945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[861, 1844, 559, 789, 443, 239, 1784, 250, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 600, 1165, 265, 399, 967, 410, 292, 846, 321, 0, 0, 0, 0, 0, 0, 122, 661, 373, 880, 491, 376, 277, 1361, 395, 243, 0, 0, 0, 0, 0, 0, 145, 529, 484, 1184, 802, 248, 635, 469, 518, 0, 0, 0, 0, 0, 0, 0, 464, 0, 840, 916, 1359, 9, 1101, 799, 0, 0, 0, 0, 0, 0, 0, 0, 1183, 319, 629, 324, 789, 254, 408, 1293, 1614, 0, 0, 0, 0, 0, 0, 0, 364, 152, 303, 951, 489, 1716, 710, 1047, 223, 288, 0, 0, 0, 0, 0, 0, 372, 2199, 1091, 569, 1160, 627, 435, 566, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6769, 6627, 5179, 5014, 5488, 6813, 6243, 7019] avg=6144.0000 max=7019 max_rank=7 min=5014 min_rank=3 imbalance_ratio=1.142415 max_load_reduction=42133 max_load_reduction_pct=0.857198 moved_to_echo=25563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[294, 1224, 1065, 1503, 298, 599, 2050, 265, 0, 0, 0, 0, 0, 0, 0, 0, 853, 266, 899, 475, 649, 637, 444, 207, 716, 419, 0, 0, 0, 0, 0, 0, 168, 893, 298, 819, 537, 268, 871, 1404, 336, 242, 0, 0, 0, 0, 0, 0, 565, 1298, 984, 575, 861, 229, 492, 713, 507, 0, 0, 0, 0, 0, 0, 0, 620, 0, 517, 452, 1349, 1, 1112, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 753, 503, 276, 518, 104, 54, 587, 2100, 0, 0, 0, 0, 0, 0, 0, 1249, 373, 264, 278, 450, 2125, 882, 1069, 223, 336, 0, 0, 0, 0, 0, 0, 287, 1843, 587, 562, 783, 596, 250, 620, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7298, 5565, 5836, 6224, 5434, 6018, 7249, 5528] avg=6144.0000 max=7298 max_rank=0 min=5434 min_rank=4 imbalance_ratio=1.187826 max_load_reduction=41854 max_load_reduction_pct=0.851522 moved_to_echo=24229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 10496, 7808, 6144, 7296, 5248, 3840, 4352, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9344, 3712, 4864, 2560, 5760, 6144, 3840, 3200, 7296, 3072, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7552, 2944, 2432, 7168, 4992, 9600, 6784, 4480, 1920, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9984, 4608, 4352, 2176, 4096, 2560, 1536, 5504, 14848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9344, 3712, 4864, 2560, 5760, 6144, 3840, 3200, 7296, 3072, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 12544, 8832, 5248, 3328, 2688, 11136, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 8576, 2560, 7424, 5504, 1536, 4736, 12800, 3456, 2048, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7808, 0, 7040, 8960, 9600, 128, 10240, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 8576, 2560, 7424, 5504, 1536, 4736, 12800, 3456, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9984, 4608, 4352, 2176, 4096, 2560, 1536, 5504, 14848, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 4352, 6912, 4480, 12672, 2944, 4608, 4864, 4352, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[653, 526, 265, 965, 1507, 586, 348, 1644, 903, 513, 600, 441, 1742, 890, 1386, 2407, 507, 510, 427, 347, 509, 382, 1415, 586, 94, 571, 960, 522, 350, 181, 1138, 92, 1117, 830, 334, 931, 2328, 519, 252, 400, 516, 1341, 761, 1923, 785, 1741, 244, 612, 1147, 188, 141, 477, 770, 1261, 819, 749, 466, 1487, 950, 154, 1342, 404, 102, 94] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[215, 340, 422, 1422, 1103, 622, 313, 1261, 833, 204, 187, 1337, 1251, 667, 1062, 1224, 667, 372, 277, 388, 361, 183, 1082, 2490, 258, 951, 1253, 905, 419, 139, 955, 124, 2605, 1029, 1152, 1521, 1931, 668, 180, 514, 1786, 1535, 535, 643, 528, 1739, 437, 788, 464, 226, 160, 842, 73, 998, 568, 377, 417, 372, 1479, 134, 1105, 703, 237, 119] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[332, 359, 168, 1261, 384, 420, 169, 441, 930, 291, 520, 304, 1836, 711, 986, 2805, 449, 284, 644, 538, 1043, 522, 795, 546, 141, 648, 2766, 398, 146, 206, 1429, 79, 1380, 791, 1422, 1086, 2378, 1430, 224, 1660, 1185, 1687, 1311, 426, 1014, 919, 63, 532, 933, 72, 391, 210, 483, 673, 486, 990, 306, 776, 613, 262, 1509, 389, 968, 32] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[371, 66, 258, 2232, 1083, 501, 288, 730, 414, 374, 386, 1349, 1317, 1009, 1240, 705, 820, 1197, 768, 287, 496, 505, 742, 649, 131, 515, 1669, 1002, 213, 978, 2392, 223, 457, 698, 1180, 1064, 3093, 536, 106, 468, 906, 884, 879, 726, 823, 1874, 89, 533, 1047, 113, 67, 569, 366, 1210, 329, 661, 567, 807, 1604, 297, 879, 1145, 183, 82] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[276, 393, 339, 2148, 662, 516, 523, 787, 814, 457, 616, 457, 1451, 968, 1135, 840, 626, 592, 632, 691, 974, 522, 618, 657, 350, 758, 1005, 1070, 210, 712, 1070, 766, 598, 1104, 1715, 1105, 2194, 1226, 176, 2383, 895, 1546, 606, 374, 419, 2575, 111, 422, 202, 616, 608, 257, 81, 964, 557, 606, 191, 748, 1122, 260, 516, 1023, 109, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[129, 124, 573, 2707, 362, 526, 130, 112, 549, 396, 724, 1286, 1809, 405, 601, 1708, 1606, 88, 484, 326, 405, 234, 1320, 777, 834, 657, 1221, 608, 82, 422, 1876, 416, 912, 688, 1440, 1101, 3858, 614, 271, 2174, 1298, 1517, 510, 272, 1044, 915, 145, 76, 541, 225, 478, 265, 297, 954, 562, 1080, 842, 346, 405, 117, 1594, 333, 669, 112] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[371, 66, 258, 2232, 1083, 501, 288, 730, 414, 374, 386, 1349, 1317, 1009, 1240, 705, 820, 1197, 768, 287, 496, 505, 742, 649, 131, 515, 1669, 1002, 213, 978, 2392, 223, 457, 698, 1180, 1064, 3093, 536, 106, 468, 906, 884, 879, 726, 823, 1874, 89, 533, 1047, 113, 67, 569, 366, 1210, 329, 661, 567, 807, 1604, 297, 879, 1145, 183, 82] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[295, 590, 255, 1549, 1627, 377, 511, 722, 1100, 282, 393, 237, 2271, 486, 429, 385, 901, 322, 1260, 678, 635, 597, 732, 529, 379, 1046, 410, 1578, 48, 726, 405, 971, 576, 856, 640, 918, 2761, 356, 73, 1892, 499, 1403, 380, 2874, 480, 1202, 377, 502, 216, 355, 31, 655, 240, 810, 1833, 1055, 67, 1200, 619, 599, 540, 2233, 61, 123] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[215, 340, 422, 1422, 1103, 622, 313, 1261, 833, 204, 187, 1337, 1251, 667, 1062, 1224, 667, 372, 277, 388, 361, 183, 1082, 2490, 258, 951, 1253, 905, 419, 139, 955, 124, 2605, 1029, 1152, 1521, 1931, 668, 180, 514, 1786, 1535, 535, 643, 528, 1739, 437, 788, 464, 226, 160, 842, 73, 998, 568, 377, 417, 372, 1479, 134, 1105, 703, 237, 119] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[653, 526, 265, 965, 1507, 586, 348, 1644, 903, 513, 600, 441, 1742, 890, 1386, 2407, 507, 510, 427, 347, 509, 382, 1415, 586, 94, 571, 960, 522, 350, 181, 1138, 92, 1117, 830, 334, 931, 2328, 519, 252, 400, 516, 1341, 761, 1923, 785, 1741, 244, 612, 1147, 188, 141, 477, 770, 1261, 819, 749, 466, 1487, 950, 154, 1342, 404, 102, 94] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[276, 393, 339, 2148, 662, 516, 523, 787, 814, 457, 616, 457, 1451, 968, 1135, 840, 626, 592, 632, 691, 974, 522, 618, 657, 350, 758, 1005, 1070, 210, 712, 1070, 766, 598, 1104, 1715, 1105, 2194, 1226, 176, 2383, 895, 1546, 606, 374, 419, 2575, 111, 422, 202, 616, 608, 257, 81, 964, 557, 606, 191, 748, 1122, 260, 516, 1023, 109, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[310, 212, 394, 2086, 820, 508, 113, 755, 828, 190, 334, 1201, 1740, 254, 1108, 764, 870, 1457, 424, 726, 299, 437, 989, 730, 144, 749, 1265, 1726, 440, 304, 1666, 917, 883, 852, 1383, 1371, 2384, 527, 791, 1461, 1205, 1449, 1025, 263, 350, 1288, 233, 553, 372, 538, 522, 615, 57, 1259, 370, 291, 77, 373, 1869, 447, 580, 707, 222, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[332, 359, 168, 1261, 384, 420, 169, 441, 582, 165, 0, 0, 0, 0, 0, 0, 930, 291, 520, 304, 757, 711, 986, 2805, 0, 0, 0, 0, 0, 0, 0, 0, 449, 284, 644, 538, 1043, 522, 795, 546, 390, 153, 0, 0, 0, 0, 0, 0, 141, 648, 2766, 398, 146, 206, 1429, 79, 374, 0, 0, 0, 0, 0, 0, 0, 1380, 791, 1422, 1086, 0, 1430, 224, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 1687, 1311, 426, 1014, 364, 63, 532, 0, 0, 0, 0, 0, 0, 0, 0, 933, 72, 391, 210, 483, 673, 486, 990, 1565, 0, 0, 0, 0, 0, 0, 0, 306, 776, 613, 262, 1509, 389, 968, 32, 813, 497, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4281, 7304, 5364, 6187, 7466, 6582, 5803, 6165] avg=6144.0000 max=7466 max_rank=4 min=4281 min_rank=0 imbalance_ratio=1.215169 max_load_reduction=41686 max_load_reduction_pct=0.848104 moved_to_echo=26016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[276, 393, 339, 2148, 662, 516, 523, 787, 460, 464, 0, 0, 0, 0, 0, 0, 814, 457, 616, 457, 598, 968, 1135, 840, 0, 0, 0, 0, 0, 0, 0, 0, 626, 592, 632, 691, 974, 522, 618, 657, 1095, 220, 0, 0, 0, 0, 0, 0, 350, 758, 1005, 1070, 210, 712, 1070, 766, 538, 0, 0, 0, 0, 0, 0, 0, 598, 1104, 1715, 1105, 0, 1226, 176, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 895, 1546, 606, 374, 419, 1016, 111, 422, 0, 0, 0, 0, 0, 0, 0, 0, 202, 616, 608, 257, 81, 964, 557, 606, 1444, 0, 0, 0, 0, 0, 0, 0, 191, 748, 1122, 260, 516, 1023, 109, 208, 750, 393, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6568, 5885, 6627, 6479, 7549, 5389, 5335, 5320] avg=6144.0000 max=7549 max_rank=4 min=5320 min_rank=7 imbalance_ratio=1.228678 max_load_reduction=41603 max_load_reduction_pct=0.846415 moved_to_echo=23593 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[129, 124, 573, 2707, 362, 526, 130, 112, 573, 165, 0, 0, 0, 0, 0, 0, 549, 396, 724, 1286, 747, 405, 601, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 88, 484, 326, 405, 234, 1320, 777, 389, 201, 0, 0, 0, 0, 0, 0, 834, 657, 1221, 608, 82, 422, 1876, 416, 490, 0, 0, 0, 0, 0, 0, 0, 912, 688, 1440, 1101, 0, 614, 271, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 1298, 1517, 510, 272, 1044, 361, 145, 76, 0, 0, 0, 0, 0, 0, 0, 0, 541, 225, 478, 265, 297, 954, 562, 1080, 2541, 0, 0, 0, 0, 0, 0, 0, 842, 346, 405, 117, 1594, 333, 669, 112, 1317, 489, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5401, 6416, 5830, 6606, 6509, 5223, 6943, 6224] avg=6144.0000 max=6943 max_rank=6 min=5223 min_rank=5 imbalance_ratio=1.130046 max_load_reduction=42209 max_load_reduction_pct=0.858744 moved_to_echo=24899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[215, 340, 422, 1422, 1103, 622, 313, 1261, 396, 313, 0, 0, 0, 0, 0, 0, 833, 204, 187, 1337, 517, 667, 1062, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 667, 372, 277, 388, 361, 183, 1082, 2490, 739, 47, 0, 0, 0, 0, 0, 0, 258, 951, 1253, 905, 419, 139, 955, 124, 116, 0, 0, 0, 0, 0, 0, 0, 2605, 1029, 1152, 1521, 0, 668, 180, 351, 0, 0, 0, 0, 0, 0, 0, 0, 1786, 1535, 535, 643, 528, 687, 437, 788, 0, 0, 0, 0, 0, 0, 0, 0, 464, 226, 160, 842, 73, 998, 568, 377, 1271, 0, 0, 0, 0, 0, 0, 0, 417, 372, 1479, 134, 1105, 703, 237, 119, 660, 338, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6407, 6031, 6606, 5120, 7506, 6939, 4979, 5564] avg=6144.0000 max=7506 max_rank=4 min=4979 min_rank=6 imbalance_ratio=1.221680 max_load_reduction=41646 max_load_reduction_pct=0.847290 moved_to_echo=24988 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[332, 359, 168, 1261, 384, 420, 169, 441, 582, 165, 0, 0, 0, 0, 0, 0, 930, 291, 520, 304, 757, 711, 986, 2805, 0, 0, 0, 0, 0, 0, 0, 0, 449, 284, 644, 538, 1043, 522, 795, 546, 390, 153, 0, 0, 0, 0, 0, 0, 141, 648, 2766, 398, 146, 206, 1429, 79, 374, 0, 0, 0, 0, 0, 0, 0, 1380, 791, 1422, 1086, 0, 1430, 224, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 1687, 1311, 426, 1014, 364, 63, 532, 0, 0, 0, 0, 0, 0, 0, 0, 933, 72, 391, 210, 483, 673, 486, 990, 1565, 0, 0, 0, 0, 0, 0, 0, 306, 776, 613, 262, 1509, 389, 968, 32, 813, 497, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4281, 7304, 5364, 6187, 7466, 6582, 5803, 6165] avg=6144.0000 max=7466 max_rank=4 min=4281 min_rank=0 imbalance_ratio=1.215169 max_load_reduction=41686 max_load_reduction_pct=0.848104 moved_to_echo=26016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[371, 66, 258, 2232, 1083, 501, 288, 730, 417, 338, 0, 0, 0, 0, 0, 0, 414, 374, 386, 1349, 544, 1009, 1240, 705, 0, 0, 0, 0, 0, 0, 0, 0, 820, 1197, 768, 287, 496, 505, 742, 649, 796, 43, 0, 0, 0, 0, 0, 0, 131, 515, 1669, 1002, 213, 978, 2392, 223, 105, 0, 0, 0, 0, 0, 0, 0, 457, 698, 1180, 1064, 0, 536, 106, 320, 0, 0, 0, 0, 0, 0, 0, 0, 906, 884, 879, 726, 823, 740, 89, 533, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 113, 67, 569, 366, 1210, 329, 661, 2037, 0, 0, 0, 0, 0, 0, 0, 567, 807, 1604, 297, 879, 1145, 183, 82, 1056, 356, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6021, 6303, 7228, 4361, 5580, 6399, 6976] avg=6144.0000 max=7228 max_rank=3 min=4361 min_rank=4 imbalance_ratio=1.176432 max_load_reduction=41924 max_load_reduction_pct=0.852946 moved_to_echo=23316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[295, 590, 255, 1549, 1627, 377, 511, 722, 720, 216, 0, 0, 0, 0, 0, 0, 1100, 282, 393, 237, 936, 486, 429, 385, 0, 0, 0, 0, 0, 0, 0, 0, 901, 322, 1260, 678, 635, 597, 732, 529, 511, 175, 0, 0, 0, 0, 0, 0, 379, 1046, 410, 1578, 48, 726, 405, 971, 427, 0, 0, 0, 0, 0, 0, 0, 576, 856, 640, 918, 0, 356, 73, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 499, 1403, 380, 2874, 480, 475, 377, 502, 0, 0, 0, 0, 0, 0, 0, 0, 216, 355, 31, 655, 240, 810, 1833, 1055, 1819, 0, 0, 0, 0, 0, 0, 0, 67, 1200, 619, 599, 540, 2233, 61, 123, 942, 615, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6862, 4248, 6340, 5990, 4709, 6990, 7014, 6999] avg=6144.0000 max=7014 max_rank=6 min=4248 min_rank=1 imbalance_ratio=1.141602 max_load_reduction=42138 max_load_reduction_pct=0.857300 moved_to_echo=25712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[276, 393, 339, 2148, 662, 516, 523, 787, 460, 464, 0, 0, 0, 0, 0, 0, 814, 457, 616, 457, 598, 968, 1135, 840, 0, 0, 0, 0, 0, 0, 0, 0, 626, 592, 632, 691, 974, 522, 618, 657, 1095, 220, 0, 0, 0, 0, 0, 0, 350, 758, 1005, 1070, 210, 712, 1070, 766, 538, 0, 0, 0, 0, 0, 0, 0, 598, 1104, 1715, 1105, 0, 1226, 176, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 895, 1546, 606, 374, 419, 1016, 111, 422, 0, 0, 0, 0, 0, 0, 0, 0, 202, 616, 608, 257, 81, 964, 557, 606, 1444, 0, 0, 0, 0, 0, 0, 0, 191, 748, 1122, 260, 516, 1023, 109, 208, 750, 393, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6568, 5885, 6627, 6479, 7549, 5389, 5335, 5320] avg=6144.0000 max=7549 max_rank=4 min=5320 min_rank=7 imbalance_ratio=1.228678 max_load_reduction=41603 max_load_reduction_pct=0.846415 moved_to_echo=23593 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[653, 526, 265, 965, 1507, 586, 348, 1644, 557, 317, 0, 0, 0, 0, 0, 0, 903, 513, 600, 441, 710, 890, 1386, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 427, 347, 509, 382, 1415, 586, 744, 41, 0, 0, 0, 0, 0, 0, 94, 571, 960, 522, 350, 181, 1138, 92, 94, 0, 0, 0, 0, 0, 0, 0, 1117, 830, 334, 931, 0, 519, 252, 265, 0, 0, 0, 0, 0, 0, 0, 0, 516, 1341, 761, 1923, 785, 680, 244, 612, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 188, 141, 477, 770, 1261, 819, 749, 1533, 0, 0, 0, 0, 0, 0, 0, 466, 1487, 950, 154, 1342, 404, 102, 94, 795, 475, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7368, 7850, 5468, 4002, 4248, 6862, 7085, 6269] avg=6144.0000 max=7850 max_rank=1 min=4002 min_rank=3 imbalance_ratio=1.277669 max_load_reduction=41302 max_load_reduction_pct=0.840291 moved_to_echo=24464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[371, 66, 258, 2232, 1083, 501, 288, 730, 417, 338, 0, 0, 0, 0, 0, 0, 414, 374, 386, 1349, 544, 1009, 1240, 705, 0, 0, 0, 0, 0, 0, 0, 0, 820, 1197, 768, 287, 496, 505, 742, 649, 796, 43, 0, 0, 0, 0, 0, 0, 131, 515, 1669, 1002, 213, 978, 2392, 223, 105, 0, 0, 0, 0, 0, 0, 0, 457, 698, 1180, 1064, 0, 536, 106, 320, 0, 0, 0, 0, 0, 0, 0, 0, 906, 884, 879, 726, 823, 740, 89, 533, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 113, 67, 569, 366, 1210, 329, 661, 2037, 0, 0, 0, 0, 0, 0, 0, 567, 807, 1604, 297, 879, 1145, 183, 82, 1056, 356, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6021, 6303, 7228, 4361, 5580, 6399, 6976] avg=6144.0000 max=7228 max_rank=3 min=4361 min_rank=4 imbalance_ratio=1.176432 max_load_reduction=41924 max_load_reduction_pct=0.852946 moved_to_echo=23316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[215, 340, 422, 1422, 1103, 622, 313, 1261, 396, 313, 0, 0, 0, 0, 0, 0, 833, 204, 187, 1337, 517, 667, 1062, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 667, 372, 277, 388, 361, 183, 1082, 2490, 739, 47, 0, 0, 0, 0, 0, 0, 258, 951, 1253, 905, 419, 139, 955, 124, 116, 0, 0, 0, 0, 0, 0, 0, 2605, 1029, 1152, 1521, 0, 668, 180, 351, 0, 0, 0, 0, 0, 0, 0, 0, 1786, 1535, 535, 643, 528, 687, 437, 788, 0, 0, 0, 0, 0, 0, 0, 0, 464, 226, 160, 842, 73, 998, 568, 377, 1271, 0, 0, 0, 0, 0, 0, 0, 417, 372, 1479, 134, 1105, 703, 237, 119, 660, 338, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6407, 6031, 6606, 5120, 7506, 6939, 4979, 5564] avg=6144.0000 max=7506 max_rank=4 min=4979 min_rank=6 imbalance_ratio=1.221680 max_load_reduction=41646 max_load_reduction_pct=0.847290 moved_to_echo=24988 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[310, 212, 394, 2086, 820, 508, 113, 755, 551, 232, 0, 0, 0, 0, 0, 0, 828, 190, 334, 1201, 718, 254, 1108, 764, 0, 0, 0, 0, 0, 0, 0, 0, 870, 1457, 424, 726, 299, 437, 989, 730, 547, 135, 0, 0, 0, 0, 0, 0, 144, 749, 1265, 1726, 440, 304, 1666, 917, 329, 0, 0, 0, 0, 0, 0, 0, 883, 852, 1383, 1371, 0, 527, 791, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1205, 1449, 1025, 263, 350, 509, 233, 553, 0, 0, 0, 0, 0, 0, 0, 0, 372, 538, 522, 615, 57, 1259, 370, 291, 1570, 0, 0, 0, 0, 0, 0, 0, 77, 373, 1869, 447, 580, 707, 222, 75, 814, 471, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5981, 5397, 6614, 7540, 6804, 5587, 5594, 5635] avg=6144.0000 max=7540 max_rank=3 min=5397 min_rank=1 imbalance_ratio=1.227214 max_load_reduction=41612 max_load_reduction_pct=0.846598 moved_to_echo=23620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 2816, 3840, 6656, 5632, 5504, 8064, 10880, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8320, 11392, 6016, 7552, 5504, 4864, 1792, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 2688, 2688, 14464, 7552, 4096, 2432, 6528, 4352, 2304, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 2432, 2432, 3968, 2432, 8192, 5632, 5888, 13824, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8320, 11392, 6016, 7552, 5504, 4864, 1792, 4096, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6528, 4864, 4992, 4096, 4736, 3456, 7808, 7040, 5248, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 2688, 2688, 14464, 7552, 4096, 2432, 6528, 4352, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 2816, 3840, 6656, 5632, 5504, 8064, 10880, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 6144, 8704, 2304, 8192, 7040, 2560, 896, 7168, 3712, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8576, 6912, 9344, 9216, 0, 5888, 2176, 7552, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2432, 6016, 10624, 7936, 1920, 3712, 11008, 3712, 2560, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[270, 779, 574, 642, 822, 464, 100, 273, 530, 665, 1044, 979, 211, 651, 867, 198, 914, 87, 1369, 1412, 1116, 320, 379, 968, 355, 712, 325, 1243, 663, 423, 119, 2479, 209, 215, 2248, 585, 201, 291, 578, 560, 767, 523, 278, 1556, 385, 712, 867, 2141, 569, 1136, 172, 332, 581, 486, 1276, 1380, 980, 832, 2030, 2129, 1624, 1152, 189, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[88, 572, 883, 1003, 686, 651, 181, 787, 735, 1102, 134, 713, 616, 822, 827, 143, 413, 238, 1038, 3299, 413, 947, 243, 468, 451, 75, 180, 1261, 1005, 176, 379, 1545, 144, 1040, 2091, 536, 150, 639, 326, 1406, 1069, 556, 1238, 3039, 261, 174, 13, 1345, 834, 1212, 250, 525, 891, 340, 1641, 173, 359, 863, 899, 1806, 117, 701, 1147, 1293] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[113, 642, 1083, 716, 934, 669, 437, 348, 789, 1140, 112, 861, 618, 534, 688, 209, 933, 74, 1212, 2056, 1899, 695, 639, 1231, 488, 557, 431, 1476, 826, 237, 169, 1327, 204, 148, 1776, 485, 117, 135, 259, 1025, 1009, 896, 564, 2109, 99, 228, 168, 1512, 956, 837, 169, 725, 1113, 298, 1666, 159, 483, 986, 2609, 1607, 693, 1011, 855, 108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[112, 1010, 1762, 1143, 471, 733, 183, 775, 660, 772, 424, 511, 217, 803, 1009, 209, 1052, 134, 479, 952, 608, 287, 875, 915, 313, 428, 335, 1706, 424, 576, 520, 1604, 455, 212, 2119, 584, 442, 388, 626, 1312, 1368, 1930, 968, 1190, 333, 426, 501, 1592, 480, 1140, 230, 381, 216, 694, 1648, 467, 379, 651, 629, 2177, 1165, 1298, 753, 396] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[331, 524, 629, 222, 3034, 1020, 41, 690, 911, 620, 180, 345, 376, 1075, 625, 168, 372, 170, 601, 709, 585, 497, 130, 1111, 504, 629, 83, 1143, 999, 147, 162, 2272, 344, 858, 2112, 646, 439, 285, 203, 647, 471, 852, 833, 3059, 336, 190, 200, 1115, 620, 725, 330, 257, 698, 507, 2781, 223, 455, 1511, 2584, 550, 867, 1841, 559, 1149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[536, 862, 1291, 606, 111, 576, 530, 635, 746, 1281, 282, 954, 400, 846, 620, 263, 564, 137, 1078, 1980, 460, 556, 243, 2100, 152, 615, 193, 1143, 832, 556, 294, 1510, 353, 314, 2329, 576, 436, 243, 266, 1243, 1120, 547, 1545, 1178, 426, 463, 272, 826, 1149, 347, 127, 492, 334, 323, 2368, 229, 295, 568, 1221, 2606, 1349, 1357, 959, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[270, 779, 574, 642, 822, 464, 100, 273, 530, 665, 1044, 979, 211, 651, 867, 198, 914, 87, 1369, 1412, 1116, 320, 379, 968, 355, 712, 325, 1243, 663, 423, 119, 2479, 209, 215, 2248, 585, 201, 291, 578, 560, 767, 523, 278, 1556, 385, 712, 867, 2141, 569, 1136, 172, 332, 581, 486, 1276, 1380, 980, 832, 2030, 2129, 1624, 1152, 189, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[137, 369, 374, 706, 459, 1015, 137, 1464, 662, 1849, 319, 927, 623, 104, 1218, 157, 129, 290, 2455, 1752, 301, 425, 884, 4114, 148, 152, 209, 974, 223, 802, 285, 753, 253, 241, 2032, 489, 752, 197, 467, 694, 2397, 67, 510, 1010, 110, 159, 135, 1105, 907, 513, 130, 319, 487, 257, 1176, 107, 753, 870, 2832, 1797, 1869, 683, 567, 851] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[112, 1010, 1762, 1143, 471, 733, 183, 775, 660, 772, 424, 511, 217, 803, 1009, 209, 1052, 134, 479, 952, 608, 287, 875, 915, 313, 428, 335, 1706, 424, 576, 520, 1604, 455, 212, 2119, 584, 442, 388, 626, 1312, 1368, 1930, 968, 1190, 333, 426, 501, 1592, 480, 1140, 230, 381, 216, 694, 1648, 467, 379, 651, 629, 2177, 1165, 1298, 753, 396] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[113, 642, 1083, 716, 934, 669, 437, 348, 789, 1140, 112, 861, 618, 534, 688, 209, 933, 74, 1212, 2056, 1899, 695, 639, 1231, 488, 557, 431, 1476, 826, 237, 169, 1327, 204, 148, 1776, 485, 117, 135, 259, 1025, 1009, 896, 564, 2109, 99, 228, 168, 1512, 956, 837, 169, 725, 1113, 298, 1666, 159, 483, 986, 2609, 1607, 693, 1011, 855, 108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[258, 899, 615, 507, 707, 728, 153, 990, 531, 601, 1742, 519, 286, 1210, 1312, 202, 122, 60, 637, 1749, 692, 507, 518, 2000, 287, 525, 534, 1818, 1021, 456, 737, 1665, 270, 372, 796, 612, 713, 379, 276, 1496, 423, 949, 840, 2407, 206, 62, 585, 1211, 1077, 303, 285, 410, 407, 913, 2174, 176, 547, 1245, 1502, 1394, 438, 1055, 539, 502] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[331, 524, 629, 222, 3034, 1020, 41, 690, 911, 620, 180, 345, 376, 1075, 625, 168, 372, 170, 601, 709, 585, 497, 130, 1111, 504, 629, 83, 1143, 999, 147, 162, 2272, 344, 858, 2112, 646, 439, 285, 203, 647, 471, 852, 833, 3059, 336, 190, 200, 1115, 620, 725, 330, 257, 698, 507, 2781, 223, 455, 1511, 2584, 550, 867, 1841, 559, 1149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[113, 642, 1083, 716, 934, 669, 437, 348, 1475, 0, 0, 0, 0, 0, 0, 0, 789, 1140, 112, 861, 618, 534, 688, 209, 1134, 175, 0, 0, 0, 0, 0, 0, 933, 74, 1212, 1105, 1899, 695, 639, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 488, 557, 431, 1476, 826, 237, 169, 1327, 494, 25, 0, 0, 0, 0, 0, 0, 204, 148, 1776, 485, 117, 135, 259, 1025, 776, 143, 0, 0, 0, 0, 0, 0, 1009, 896, 564, 1260, 99, 228, 168, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 956, 837, 169, 725, 1113, 298, 1666, 159, 681, 0, 0, 0, 0, 0, 0, 0, 483, 986, 0, 1113, 693, 1011, 855, 108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 6260, 7788, 6030, 5068, 5736, 6604, 5249] avg=6144.0000 max=7788 max_rank=2 min=5068 min_rank=4 imbalance_ratio=1.267578 max_load_reduction=41364 max_load_reduction_pct=0.841553 moved_to_echo=22657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[270, 779, 574, 642, 822, 464, 100, 273, 1148, 0, 0, 0, 0, 0, 0, 0, 530, 665, 1044, 979, 211, 651, 867, 198, 882, 124, 0, 0, 0, 0, 0, 0, 914, 87, 1369, 751, 1116, 320, 379, 968, 0, 0, 0, 0, 0, 0, 0, 0, 355, 712, 325, 1243, 663, 423, 119, 2479, 659, 22, 0, 0, 0, 0, 0, 0, 209, 215, 2248, 585, 201, 291, 578, 560, 537, 110, 0, 0, 0, 0, 0, 0, 767, 523, 278, 917, 385, 712, 867, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 569, 1136, 172, 332, 581, 486, 1276, 1380, 507, 0, 0, 0, 0, 0, 0, 0, 980, 832, 0, 1470, 1624, 1152, 189, 215, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5072, 6151, 5904, 7000, 5534, 6590, 6439, 6462] avg=6144.0000 max=7000 max_rank=3 min=5072 min_rank=0 imbalance_ratio=1.139323 max_load_reduction=42152 max_load_reduction_pct=0.857585 moved_to_echo=25025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[88, 572, 883, 1003, 686, 651, 181, 787, 735, 1102, 134, 713, 616, 822, 827, 143, 413, 238, 1038, 3299, 413, 947, 243, 468, 451, 75, 180, 1261, 1005, 176, 379, 1545, 144, 1040, 2091, 536, 150, 639, 326, 1406, 1069, 556, 1238, 3039, 261, 174, 13, 1345, 834, 1212, 250, 525, 891, 340, 1641, 173, 359, 863, 899, 1806, 117, 701, 1147, 1293] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[112, 1010, 1762, 1143, 471, 733, 183, 775, 355, 0, 0, 0, 0, 0, 0, 0, 660, 772, 424, 511, 217, 803, 1009, 209, 274, 81, 0, 0, 0, 0, 0, 0, 1052, 134, 479, 512, 608, 287, 875, 915, 0, 0, 0, 0, 0, 0, 0, 0, 313, 428, 335, 1706, 424, 576, 520, 1604, 670, 14, 0, 0, 0, 0, 0, 0, 455, 212, 2119, 584, 442, 388, 626, 1312, 359, 81, 0, 0, 0, 0, 0, 0, 1368, 1930, 968, 711, 333, 426, 501, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 480, 1140, 230, 381, 216, 694, 1648, 467, 384, 0, 0, 0, 0, 0, 0, 0, 379, 651, 0, 1507, 1165, 1298, 753, 396, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6544, 4960, 4862, 6590, 6578, 7829, 5640, 6149] avg=6144.0000 max=7829 max_rank=5 min=4862 min_rank=2 imbalance_ratio=1.274251 max_load_reduction=41323 max_load_reduction_pct=0.840719 moved_to_echo=26196 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[258, 899, 615, 507, 707, 728, 153, 990, 849, 0, 0, 0, 0, 0, 0, 0, 531, 601, 1742, 519, 286, 1210, 1312, 202, 653, 149, 0, 0, 0, 0, 0, 0, 122, 60, 637, 940, 692, 507, 518, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 287, 525, 534, 1818, 1021, 456, 737, 1665, 429, 29, 0, 0, 0, 0, 0, 0, 270, 372, 796, 612, 713, 379, 276, 1496, 660, 164, 0, 0, 0, 0, 0, 0, 423, 949, 840, 1437, 206, 62, 585, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 303, 285, 410, 407, 913, 2174, 176, 777, 0, 0, 0, 0, 0, 0, 0, 547, 1245, 0, 965, 438, 1055, 539, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5706, 7205, 5476, 7501, 5738, 5713, 6522, 5291] avg=6144.0000 max=7501 max_rank=3 min=5291 min_rank=7 imbalance_ratio=1.220866 max_load_reduction=41651 max_load_reduction_pct=0.847392 moved_to_echo=23264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[536, 862, 1291, 606, 111, 576, 530, 635, 690, 0, 0, 0, 0, 0, 0, 0, 746, 1281, 282, 954, 400, 846, 620, 263, 531, 169, 0, 0, 0, 0, 0, 0, 564, 137, 1078, 1064, 460, 556, 243, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 152, 615, 193, 1143, 832, 556, 294, 1510, 802, 14, 0, 0, 0, 0, 0, 0, 353, 314, 2329, 576, 436, 243, 266, 1243, 747, 80, 0, 0, 0, 0, 0, 0, 1120, 547, 1545, 704, 426, 463, 272, 826, 0, 0, 0, 0, 0, 0, 0, 0, 1149, 347, 127, 492, 334, 323, 2368, 229, 380, 0, 0, 0, 0, 0, 0, 0, 295, 568, 0, 1804, 1349, 1357, 959, 339, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5837, 6092, 6202, 6111, 6587, 5903, 5749, 6671] avg=6144.0000 max=6671 max_rank=7 min=5749 min_rank=6 imbalance_ratio=1.085775 max_load_reduction=42481 max_load_reduction_pct=0.864278 moved_to_echo=24910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[113, 642, 1083, 716, 934, 669, 437, 348, 1475, 0, 0, 0, 0, 0, 0, 0, 789, 1140, 112, 861, 618, 534, 688, 209, 1134, 175, 0, 0, 0, 0, 0, 0, 933, 74, 1212, 1105, 1899, 695, 639, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 488, 557, 431, 1476, 826, 237, 169, 1327, 494, 25, 0, 0, 0, 0, 0, 0, 204, 148, 1776, 485, 117, 135, 259, 1025, 776, 143, 0, 0, 0, 0, 0, 0, 1009, 896, 564, 1260, 99, 228, 168, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 956, 837, 169, 725, 1113, 298, 1666, 159, 681, 0, 0, 0, 0, 0, 0, 0, 483, 986, 0, 1113, 693, 1011, 855, 108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 6260, 7788, 6030, 5068, 5736, 6604, 5249] avg=6144.0000 max=7788 max_rank=2 min=5068 min_rank=4 imbalance_ratio=1.267578 max_load_reduction=41364 max_load_reduction_pct=0.841553 moved_to_echo=22657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[137, 369, 374, 706, 459, 1015, 137, 1464, 1602, 0, 0, 0, 0, 0, 0, 0, 662, 1849, 319, 927, 623, 104, 1218, 157, 1230, 149, 0, 0, 0, 0, 0, 0, 129, 290, 2455, 942, 301, 425, 884, 4114, 0, 0, 0, 0, 0, 0, 0, 0, 148, 152, 209, 974, 223, 802, 285, 753, 553, 12, 0, 0, 0, 0, 0, 0, 253, 241, 2032, 489, 752, 197, 467, 694, 661, 68, 0, 0, 0, 0, 0, 0, 2397, 67, 510, 604, 110, 159, 135, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 907, 513, 130, 319, 487, 257, 1176, 107, 326, 0, 0, 0, 0, 0, 0, 0, 753, 870, 0, 1244, 1869, 683, 567, 851, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 7238, 9540, 4111, 5854, 5087, 4222, 6837] avg=6144.0000 max=9540 max_rank=2 min=4111 min_rank=3 imbalance_ratio=1.552734 max_load_reduction=39612 max_load_reduction_pct=0.805908 moved_to_echo=22000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[88, 572, 883, 1003, 686, 651, 181, 787, 509, 0, 0, 0, 0, 0, 0, 0, 735, 1102, 134, 713, 616, 822, 827, 143, 390, 282, 0, 0, 0, 0, 0, 0, 413, 238, 1038, 1772, 413, 947, 243, 468, 0, 0, 0, 0, 0, 0, 0, 0, 451, 75, 180, 1261, 1005, 176, 379, 1545, 556, 36, 0, 0, 0, 0, 0, 0, 144, 1040, 2091, 536, 150, 639, 326, 1406, 1245, 207, 0, 0, 0, 0, 0, 0, 1069, 556, 1238, 1815, 261, 174, 13, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 834, 1212, 250, 525, 891, 340, 1641, 173, 981, 0, 0, 0, 0, 0, 0, 0, 359, 863, 0, 1250, 117, 701, 1147, 1293, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5360, 5764, 5532, 5664, 7784, 6471, 6847, 5730] avg=6144.0000 max=7784 max_rank=4 min=5360 min_rank=0 imbalance_ratio=1.266927 max_load_reduction=41368 max_load_reduction_pct=0.841634 moved_to_echo=26832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[331, 524, 629, 222, 3034, 1020, 41, 690, 1462, 0, 0, 0, 0, 0, 0, 0, 911, 620, 180, 345, 376, 1075, 625, 168, 1122, 60, 0, 0, 0, 0, 0, 0, 372, 170, 601, 382, 585, 497, 130, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 504, 629, 83, 1143, 999, 147, 162, 2272, 169, 37, 0, 0, 0, 0, 0, 0, 344, 858, 2112, 646, 439, 285, 203, 647, 267, 208, 0, 0, 0, 0, 0, 0, 471, 852, 833, 1826, 336, 190, 200, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 620, 725, 330, 257, 698, 507, 2781, 223, 988, 0, 0, 0, 0, 0, 0, 0, 455, 1511, 0, 381, 867, 1841, 559, 1149, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7953, 5482, 3848, 6145, 6009, 5823, 7129, 6763] avg=6144.0000 max=7953 max_rank=0 min=3848 min_rank=2 imbalance_ratio=1.294434 max_load_reduction=41199 max_load_reduction_pct=0.838196 moved_to_echo=25724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[112, 1010, 1762, 1143, 471, 733, 183, 775, 355, 0, 0, 0, 0, 0, 0, 0, 660, 772, 424, 511, 217, 803, 1009, 209, 274, 81, 0, 0, 0, 0, 0, 0, 1052, 134, 479, 512, 608, 287, 875, 915, 0, 0, 0, 0, 0, 0, 0, 0, 313, 428, 335, 1706, 424, 576, 520, 1604, 670, 14, 0, 0, 0, 0, 0, 0, 455, 212, 2119, 584, 442, 388, 626, 1312, 359, 81, 0, 0, 0, 0, 0, 0, 1368, 1930, 968, 711, 333, 426, 501, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 480, 1140, 230, 381, 216, 694, 1648, 467, 384, 0, 0, 0, 0, 0, 0, 0, 379, 651, 0, 1507, 1165, 1298, 753, 396, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6544, 4960, 4862, 6590, 6578, 7829, 5640, 6149] avg=6144.0000 max=7829 max_rank=5 min=4862 min_rank=2 imbalance_ratio=1.274251 max_load_reduction=41323 max_load_reduction_pct=0.840719 moved_to_echo=26196 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[536, 862, 1291, 606, 111, 576, 530, 635, 690, 0, 0, 0, 0, 0, 0, 0, 746, 1281, 282, 954, 400, 846, 620, 263, 531, 169, 0, 0, 0, 0, 0, 0, 564, 137, 1078, 1064, 460, 556, 243, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 152, 615, 193, 1143, 832, 556, 294, 1510, 802, 14, 0, 0, 0, 0, 0, 0, 353, 314, 2329, 576, 436, 243, 266, 1243, 747, 80, 0, 0, 0, 0, 0, 0, 1120, 547, 1545, 704, 426, 463, 272, 826, 0, 0, 0, 0, 0, 0, 0, 0, 1149, 347, 127, 492, 334, 323, 2368, 229, 380, 0, 0, 0, 0, 0, 0, 0, 295, 568, 0, 1804, 1349, 1357, 959, 339, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5837, 6092, 6202, 6111, 6587, 5903, 5749, 6671] avg=6144.0000 max=6671 max_rank=7 min=5749 min_rank=6 imbalance_ratio=1.085775 max_load_reduction=42481 max_load_reduction_pct=0.864278 moved_to_echo=24910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[270, 779, 574, 642, 822, 464, 100, 273, 1148, 0, 0, 0, 0, 0, 0, 0, 530, 665, 1044, 979, 211, 651, 867, 198, 882, 124, 0, 0, 0, 0, 0, 0, 914, 87, 1369, 751, 1116, 320, 379, 968, 0, 0, 0, 0, 0, 0, 0, 0, 355, 712, 325, 1243, 663, 423, 119, 2479, 659, 22, 0, 0, 0, 0, 0, 0, 209, 215, 2248, 585, 201, 291, 578, 560, 537, 110, 0, 0, 0, 0, 0, 0, 767, 523, 278, 917, 385, 712, 867, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 569, 1136, 172, 332, 581, 486, 1276, 1380, 507, 0, 0, 0, 0, 0, 0, 0, 980, 832, 0, 1470, 1624, 1152, 189, 215, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5072, 6151, 5904, 7000, 5534, 6590, 6439, 6462] avg=6144.0000 max=7000 max_rank=3 min=5072 min_rank=0 imbalance_ratio=1.139323 max_load_reduction=42152 max_load_reduction_pct=0.857585 moved_to_echo=25025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[88, 572, 883, 1003, 686, 651, 181, 787, 509, 0, 0, 0, 0, 0, 0, 0, 735, 1102, 134, 713, 616, 822, 827, 143, 390, 282, 0, 0, 0, 0, 0, 0, 413, 238, 1038, 1772, 413, 947, 243, 468, 0, 0, 0, 0, 0, 0, 0, 0, 451, 75, 180, 1261, 1005, 176, 379, 1545, 556, 36, 0, 0, 0, 0, 0, 0, 144, 1040, 2091, 536, 150, 639, 326, 1406, 1245, 207, 0, 0, 0, 0, 0, 0, 1069, 556, 1238, 1815, 261, 174, 13, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 834, 1212, 250, 525, 891, 340, 1641, 173, 981, 0, 0, 0, 0, 0, 0, 0, 359, 863, 0, 1250, 117, 701, 1147, 1293, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5360, 5764, 5532, 5664, 7784, 6471, 6847, 5730] avg=6144.0000 max=7784 max_rank=4 min=5360 min_rank=0 imbalance_ratio=1.266927 max_load_reduction=41368 max_load_reduction_pct=0.841634 moved_to_echo=26832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5632, 8064, 4352, 5888, 3456, 6144, 7168, 1664, 6272, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 1280, 8960, 7552, 6144, 4352, 3968, 12928, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 7552, 0, 9856, 8192, 9216, 5632, 4864, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6656, 6272, 1792, 3456, 4736, 3840, 14848, 2944, 5120, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 5760, 7296, 5632, 7296, 5888, 1792, 6016, 8192, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 7552, 0, 9856, 8192, 9216, 5632, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 1280, 8960, 7552, 6144, 4352, 3968, 12928, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8704, 6400, 6784, 9344, 2176, 2432, 2816, 10880, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 3712, 2304, 10880, 6016, 3456, 2688, 13184, 4352, 256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2304, 3456, 15616, 4608, 3328, 2560, 3072, 8448, 5376, 1152, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[701, 26, 799, 322, 309, 765, 31, 853, 69, 1184, 1011, 936, 792, 399, 1039, 869, 506, 500, 539, 1991, 247, 641, 513, 168, 323, 348, 256, 189, 79, 1227, 401, 746, 672, 794, 636, 767, 1715, 605, 336, 1602, 3184, 1085, 1577, 216, 1334, 343, 273, 67, 780, 1021, 336, 3246, 412, 1767, 204, 809, 217, 46, 770, 561, 930, 1175, 2300, 593] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[712, 15, 729, 845, 353, 455, 14, 601, 28, 463, 763, 1524, 977, 592, 1343, 1007, 305, 1587, 293, 736, 1731, 307, 1519, 365, 475, 88, 334, 96, 204, 1826, 345, 541, 1040, 611, 297, 652, 1968, 629, 377, 939, 2164, 177, 993, 175, 2202, 1162, 1119, 596, 1283, 397, 235, 3024, 150, 1337, 580, 567, 151, 200, 1291, 816, 749, 731, 1080, 287] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1909, 18, 55, 1351, 79, 353, 16, 304, 33, 1213, 525, 716, 724, 1299, 1190, 1286, 670, 947, 163, 994, 289, 342, 392, 434, 470, 436, 212, 88, 169, 1390, 694, 429, 1117, 308, 1426, 1523, 1896, 768, 1455, 540, 1656, 356, 1450, 74, 1877, 1120, 272, 397, 410, 491, 257, 3807, 120, 2463, 225, 606, 140, 312, 587, 525, 1025, 1290, 799, 670] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[604, 59, 347, 1273, 54, 494, 84, 940, 93, 1285, 585, 1153, 950, 126, 2342, 762, 834, 1439, 552, 1240, 193, 908, 442, 755, 615, 240, 751, 345, 113, 423, 342, 645, 784, 2030, 503, 226, 1211, 478, 2086, 355, 563, 1922, 923, 191, 687, 2456, 126, 129, 1308, 919, 786, 2039, 269, 1464, 138, 34, 593, 364, 770, 1424, 1611, 928, 650, 197] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[167, 24, 825, 888, 228, 499, 219, 133, 571, 3907, 501, 1150, 1145, 866, 1671, 1683, 1249, 931, 563, 605, 703, 403, 192, 320, 641, 80, 267, 65, 643, 575, 1059, 555, 1694, 455, 848, 797, 627, 540, 432, 2017, 1253, 706, 1160, 69, 2563, 1195, 394, 61, 404, 937, 554, 2546, 266, 1346, 636, 394, 89, 175, 807, 1054, 270, 232, 1234, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1026, 38, 519, 649, 165, 290, 44, 568, 454, 1126, 765, 1371, 734, 489, 268, 842, 1410, 486, 420, 700, 705, 1426, 961, 799, 377, 308, 246, 223, 256, 1485, 431, 321, 1230, 286, 965, 1598, 1553, 563, 1192, 818, 1734, 742, 906, 608, 1196, 884, 280, 476, 814, 732, 713, 2996, 381, 1710, 496, 553, 419, 200, 744, 1168, 507, 733, 1398, 655] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[630, 20, 125, 829, 73, 410, 35, 316, 17, 869, 404, 1034, 552, 235, 2347, 944, 954, 1916, 1281, 484, 189, 135, 410, 277, 594, 109, 556, 8, 330, 790, 941, 125, 933, 723, 632, 896, 2678, 284, 1991, 820, 2867, 930, 833, 162, 628, 740, 269, 148, 768, 679, 363, 2478, 415, 3232, 273, 892, 736, 155, 523, 596, 2400, 906, 1039, 224] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[701, 26, 799, 322, 309, 765, 31, 853, 69, 1184, 1011, 936, 792, 399, 1039, 869, 506, 500, 539, 1991, 247, 641, 513, 168, 323, 348, 256, 189, 79, 1227, 401, 746, 672, 794, 636, 767, 1715, 605, 336, 1602, 3184, 1085, 1577, 216, 1334, 343, 273, 67, 780, 1021, 336, 3246, 412, 1767, 204, 809, 217, 46, 770, 561, 930, 1175, 2300, 593] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[604, 59, 347, 1273, 54, 494, 84, 940, 93, 1285, 585, 1153, 950, 126, 2342, 762, 834, 1439, 552, 1240, 193, 908, 442, 755, 615, 240, 751, 345, 113, 423, 342, 645, 784, 2030, 503, 226, 1211, 478, 2086, 355, 563, 1922, 923, 191, 687, 2456, 126, 129, 1308, 919, 786, 2039, 269, 1464, 138, 34, 593, 364, 770, 1424, 1611, 928, 650, 197] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[226, 15, 1220, 389, 185, 961, 68, 373, 22, 634, 388, 710, 824, 335, 1894, 1456, 1812, 991, 579, 1353, 203, 359, 285, 323, 168, 326, 224, 177, 95, 1608, 1454, 61, 572, 512, 1537, 949, 1703, 290, 1311, 2610, 2174, 942, 601, 381, 1206, 1260, 72, 160, 540, 1802, 617, 1898, 94, 1375, 189, 811, 303, 274, 835, 1382, 1348, 1001, 619, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[167, 24, 825, 888, 228, 499, 219, 133, 571, 3907, 501, 1150, 1145, 866, 1671, 1683, 1249, 931, 563, 605, 703, 403, 192, 320, 641, 80, 267, 65, 643, 575, 1059, 555, 1694, 455, 848, 797, 627, 540, 432, 2017, 1253, 706, 1160, 69, 2563, 1195, 394, 61, 404, 937, 554, 2546, 266, 1346, 636, 394, 89, 175, 807, 1054, 270, 232, 1234, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[630, 20, 125, 829, 73, 410, 35, 316, 1601, 1462, 0, 0, 0, 0, 0, 0, 17, 869, 404, 1034, 552, 235, 732, 944, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1916, 1281, 484, 189, 135, 410, 277, 160, 249, 0, 0, 0, 0, 0, 0, 594, 109, 556, 8, 330, 790, 941, 125, 1712, 1216, 744, 0, 0, 0, 0, 0, 933, 723, 632, 896, 0, 284, 1991, 660, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 930, 833, 162, 628, 740, 269, 148, 0, 0, 0, 0, 0, 0, 0, 0, 768, 679, 363, 877, 415, 3232, 273, 892, 0, 0, 0, 0, 0, 0, 0, 0, 736, 155, 523, 596, 2400, 906, 1039, 224, 622, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5501, 4787, 6055, 7125, 6119, 4865, 7499, 7201] avg=6144.0000 max=7499 max_rank=6 min=4787 min_rank=1 imbalance_ratio=1.220540 max_load_reduction=41653 max_load_reduction_pct=0.847432 moved_to_echo=25684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[630, 20, 125, 829, 73, 410, 35, 316, 17, 869, 404, 1034, 552, 235, 2347, 944, 954, 1916, 1281, 484, 189, 135, 410, 277, 594, 109, 556, 8, 330, 790, 941, 125, 933, 723, 632, 896, 2678, 284, 1991, 820, 2867, 930, 833, 162, 628, 740, 269, 148, 768, 679, 363, 2478, 415, 3232, 273, 892, 736, 155, 523, 596, 2400, 906, 1039, 224] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[701, 26, 799, 322, 309, 765, 31, 853, 2097, 936, 0, 0, 0, 0, 0, 0, 69, 1184, 1011, 936, 792, 399, 325, 869, 0, 0, 0, 0, 0, 0, 0, 0, 506, 500, 539, 1991, 247, 641, 513, 168, 313, 110, 0, 0, 0, 0, 0, 0, 323, 348, 256, 189, 79, 1227, 401, 746, 1901, 779, 329, 0, 0, 0, 0, 0, 672, 794, 636, 767, 0, 605, 336, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 1085, 1577, 216, 1334, 343, 273, 67, 0, 0, 0, 0, 0, 0, 0, 0, 780, 1021, 336, 1149, 412, 1767, 204, 809, 0, 0, 0, 0, 0, 0, 0, 0, 217, 46, 770, 561, 930, 1175, 2300, 593, 275, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6839, 5585, 5528, 6578, 5099, 6178, 6478, 6867] avg=6144.0000 max=6867 max_rank=7 min=5099 min_rank=4 imbalance_ratio=1.117676 max_load_reduction=42285 max_load_reduction_pct=0.860291 moved_to_echo=24622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[712, 15, 729, 845, 353, 455, 14, 601, 1953, 1074, 0, 0, 0, 0, 0, 0, 28, 463, 763, 1524, 977, 592, 419, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 305, 1587, 293, 736, 1731, 307, 1519, 365, 183, 142, 0, 0, 0, 0, 0, 0, 475, 88, 334, 96, 204, 1826, 345, 541, 1292, 894, 426, 0, 0, 0, 0, 0, 1040, 611, 297, 652, 0, 629, 377, 756, 0, 0, 0, 0, 0, 0, 0, 0, 872, 177, 993, 175, 2202, 1162, 1119, 596, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 397, 235, 1071, 150, 1337, 580, 567, 0, 0, 0, 0, 0, 0, 0, 0, 151, 200, 1291, 816, 749, 731, 1080, 287, 356, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6751, 5773, 7168, 6521, 4362, 7296, 5620, 5661] avg=6144.0000 max=7296 max_rank=5 min=4362 min_rank=4 imbalance_ratio=1.187500 max_load_reduction=41856 max_load_reduction_pct=0.851562 moved_to_echo=22939 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 38, 519, 649, 165, 290, 44, 568, 1939, 848, 0, 0, 0, 0, 0, 0, 454, 1126, 765, 1371, 734, 489, 72, 842, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 486, 420, 700, 705, 1426, 961, 799, 165, 33, 0, 0, 0, 0, 0, 0, 377, 308, 246, 223, 256, 1485, 431, 321, 1038, 705, 88, 0, 0, 0, 0, 0, 1230, 286, 965, 1598, 0, 563, 1192, 653, 0, 0, 0, 0, 0, 0, 0, 0, 696, 742, 906, 608, 1196, 884, 280, 476, 0, 0, 0, 0, 0, 0, 0, 0, 814, 732, 713, 1057, 381, 1710, 496, 553, 0, 0, 0, 0, 0, 0, 0, 0, 419, 200, 744, 1168, 507, 733, 1398, 655, 75, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6086, 5853, 7105, 5478, 6487, 5788, 6456, 5899] avg=6144.0000 max=7105 max_rank=2 min=5478 min_rank=3 imbalance_ratio=1.156413 max_load_reduction=42047 max_load_reduction_pct=0.855448 moved_to_echo=24630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1909, 18, 55, 1351, 79, 353, 16, 304, 2459, 1035, 0, 0, 0, 0, 0, 0, 33, 1213, 525, 716, 724, 1299, 372, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 670, 947, 163, 994, 289, 342, 392, 434, 105, 126, 0, 0, 0, 0, 0, 0, 470, 436, 212, 88, 169, 1390, 694, 429, 989, 861, 377, 0, 0, 0, 0, 0, 1117, 308, 1426, 1523, 0, 768, 1455, 435, 0, 0, 0, 0, 0, 0, 0, 0, 667, 356, 1450, 74, 1877, 1120, 272, 397, 0, 0, 0, 0, 0, 0, 0, 0, 410, 491, 257, 1348, 120, 2463, 225, 606, 0, 0, 0, 0, 0, 0, 0, 0, 140, 312, 587, 525, 1025, 1290, 799, 670, 315, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7579, 6168, 4462, 6115, 7032, 6213, 5920, 5663] avg=6144.0000 max=7579 max_rank=0 min=4462 min_rank=2 imbalance_ratio=1.233561 max_load_reduction=41573 max_load_reduction_pct=0.845805 moved_to_echo=24828 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[167, 24, 825, 888, 228, 499, 219, 133, 1644, 343, 0, 0, 0, 0, 0, 0, 571, 3907, 501, 1150, 1145, 866, 521, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 931, 563, 605, 703, 403, 192, 320, 394, 177, 0, 0, 0, 0, 0, 0, 641, 80, 267, 65, 643, 575, 1059, 555, 748, 284, 530, 0, 0, 0, 0, 0, 1694, 455, 848, 797, 0, 540, 432, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 505, 706, 1160, 69, 2563, 1195, 394, 61, 0, 0, 0, 0, 0, 0, 0, 0, 404, 937, 554, 902, 266, 1346, 636, 394, 0, 0, 0, 0, 0, 0, 0, 0, 89, 175, 807, 1054, 270, 232, 1234, 69, 443, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4970, 10344, 5537, 5447, 6389, 6653, 5439, 4373] avg=6144.0000 max=10344 max_rank=1 min=4373 min_rank=7 imbalance_ratio=1.683594 max_load_reduction=38808 max_load_reduction_pct=0.789551 moved_to_echo=22854 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[604, 59, 347, 1273, 54, 494, 84, 940, 1317, 662, 0, 0, 0, 0, 0, 0, 93, 1285, 585, 1153, 950, 126, 729, 762, 0, 0, 0, 0, 0, 0, 0, 0, 834, 1439, 552, 1240, 193, 908, 442, 755, 69, 249, 0, 0, 0, 0, 0, 0, 615, 240, 751, 345, 113, 423, 342, 645, 336, 549, 743, 0, 0, 0, 0, 0, 784, 2030, 503, 226, 0, 478, 2086, 286, 0, 0, 0, 0, 0, 0, 0, 0, 227, 1922, 923, 191, 687, 2456, 126, 129, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 919, 786, 722, 269, 1464, 138, 34, 0, 0, 0, 0, 0, 0, 0, 0, 593, 364, 770, 1424, 1611, 928, 650, 197, 621, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5834, 5683, 6681, 5102, 6393, 6661, 5640, 7158] avg=6144.0000 max=7158 max_rank=7 min=5102 min_rank=3 imbalance_ratio=1.165039 max_load_reduction=41994 max_load_reduction_pct=0.854370 moved_to_echo=25852 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 38, 519, 649, 165, 290, 44, 568, 1939, 848, 0, 0, 0, 0, 0, 0, 454, 1126, 765, 1371, 734, 489, 72, 842, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 486, 420, 700, 705, 1426, 961, 799, 165, 33, 0, 0, 0, 0, 0, 0, 377, 308, 246, 223, 256, 1485, 431, 321, 1038, 705, 88, 0, 0, 0, 0, 0, 1230, 286, 965, 1598, 0, 563, 1192, 653, 0, 0, 0, 0, 0, 0, 0, 0, 696, 742, 906, 608, 1196, 884, 280, 476, 0, 0, 0, 0, 0, 0, 0, 0, 814, 732, 713, 1057, 381, 1710, 496, 553, 0, 0, 0, 0, 0, 0, 0, 0, 419, 200, 744, 1168, 507, 733, 1398, 655, 75, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6086, 5853, 7105, 5478, 6487, 5788, 6456, 5899] avg=6144.0000 max=7105 max_rank=2 min=5478 min_rank=3 imbalance_ratio=1.156413 max_load_reduction=42047 max_load_reduction_pct=0.855448 moved_to_echo=24630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[226, 15, 1220, 389, 185, 961, 68, 373, 1226, 929, 0, 0, 0, 0, 0, 0, 22, 634, 388, 710, 824, 335, 590, 1456, 0, 0, 0, 0, 0, 0, 0, 0, 1812, 991, 579, 1353, 203, 359, 285, 323, 510, 201, 0, 0, 0, 0, 0, 0, 168, 326, 224, 177, 95, 1608, 1454, 61, 1298, 774, 601, 0, 0, 0, 0, 0, 572, 512, 1537, 949, 0, 290, 1311, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 876, 942, 601, 381, 1206, 1260, 72, 160, 0, 0, 0, 0, 0, 0, 0, 0, 540, 1802, 617, 672, 94, 1375, 189, 811, 0, 0, 0, 0, 0, 0, 0, 0, 303, 274, 835, 1382, 1348, 1001, 619, 66, 502, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5592, 4959, 6616, 6786, 7271, 5498, 6100, 6330] avg=6144.0000 max=7271 max_rank=4 min=4959 min_rank=1 imbalance_ratio=1.183431 max_load_reduction=41881 max_load_reduction_pct=0.852071 moved_to_echo=25199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1909, 18, 55, 1351, 79, 353, 16, 304, 2459, 1035, 0, 0, 0, 0, 0, 0, 33, 1213, 525, 716, 724, 1299, 372, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 670, 947, 163, 994, 289, 342, 392, 434, 105, 126, 0, 0, 0, 0, 0, 0, 470, 436, 212, 88, 169, 1390, 694, 429, 989, 861, 377, 0, 0, 0, 0, 0, 1117, 308, 1426, 1523, 0, 768, 1455, 435, 0, 0, 0, 0, 0, 0, 0, 0, 667, 356, 1450, 74, 1877, 1120, 272, 397, 0, 0, 0, 0, 0, 0, 0, 0, 410, 491, 257, 1348, 120, 2463, 225, 606, 0, 0, 0, 0, 0, 0, 0, 0, 140, 312, 587, 525, 1025, 1290, 799, 670, 315, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7579, 6168, 4462, 6115, 7032, 6213, 5920, 5663] avg=6144.0000 max=7579 max_rank=0 min=4462 min_rank=2 imbalance_ratio=1.233561 max_load_reduction=41573 max_load_reduction_pct=0.845805 moved_to_echo=24828 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[701, 26, 799, 322, 309, 765, 31, 853, 2097, 936, 0, 0, 0, 0, 0, 0, 69, 1184, 1011, 936, 792, 399, 325, 869, 0, 0, 0, 0, 0, 0, 0, 0, 506, 500, 539, 1991, 247, 641, 513, 168, 313, 110, 0, 0, 0, 0, 0, 0, 323, 348, 256, 189, 79, 1227, 401, 746, 1901, 779, 329, 0, 0, 0, 0, 0, 672, 794, 636, 767, 0, 605, 336, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 1085, 1577, 216, 1334, 343, 273, 67, 0, 0, 0, 0, 0, 0, 0, 0, 780, 1021, 336, 1149, 412, 1767, 204, 809, 0, 0, 0, 0, 0, 0, 0, 0, 217, 46, 770, 561, 930, 1175, 2300, 593, 275, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6839, 5585, 5528, 6578, 5099, 6178, 6478, 6867] avg=6144.0000 max=6867 max_rank=7 min=5099 min_rank=4 imbalance_ratio=1.117676 max_load_reduction=42285 max_load_reduction_pct=0.860291 moved_to_echo=24622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[630, 20, 125, 829, 73, 410, 35, 316, 1601, 1462, 0, 0, 0, 0, 0, 0, 17, 869, 404, 1034, 552, 235, 732, 944, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1916, 1281, 484, 189, 135, 410, 277, 160, 249, 0, 0, 0, 0, 0, 0, 594, 109, 556, 8, 330, 790, 941, 125, 1712, 1216, 744, 0, 0, 0, 0, 0, 933, 723, 632, 896, 0, 284, 1991, 660, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 930, 833, 162, 628, 740, 269, 148, 0, 0, 0, 0, 0, 0, 0, 0, 768, 679, 363, 877, 415, 3232, 273, 892, 0, 0, 0, 0, 0, 0, 0, 0, 736, 155, 523, 596, 2400, 906, 1039, 224, 622, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5501, 4787, 6055, 7125, 6119, 4865, 7499, 7201] avg=6144.0000 max=7499 max_rank=6 min=4787 min_rank=1 imbalance_ratio=1.220540 max_load_reduction=41653 max_load_reduction_pct=0.847432 moved_to_echo=25684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8064, 5760, 6912, 7424, 0, 4224, 9216, 7808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7808, 8832, 4480, 8192, 4352, 4608, 4736, 3456, 1920, 1408, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 256, 4736, 6656, 1536, 4352, 512, 4096, 14336, 7296, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 2048, 2944, 1280, 1920, 9344, 5760, 3456, 9344, 6144, 3840, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7808, 8832, 4480, 8192, 4352, 4608, 4736, 3456, 1920, 1408, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 7040, 3968, 7808, 2176, 14720, 2816, 4736, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6400, 6912, 8448, 1920, 11776, 9216, 2816, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 2048, 2944, 1280, 1920, 9344, 5760, 3456, 9344, 6144, 3840, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 1792, 6400, 7552, 8960, 7040, 9216, 2816, 3328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6400, 6912, 8448, 1920, 11776, 9216, 2816, 2048, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 10752, 4992, 8704, 6784, 4352, 3840, 8960, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[286, 63, 240, 846, 369, 455, 972, 416, 569, 766, 963, 449, 949, 844, 948, 253, 550, 930, 200, 993, 1337, 739, 206, 886, 961, 130, 329, 955, 1134, 657, 465, 167, 193, 425, 177, 356, 883, 1002, 827, 662, 3261, 438, 1563, 733, 81, 1169, 141, 548, 515, 799, 182, 475, 3122, 2531, 944, 1029, 606, 2103, 782, 406, 210, 933, 997, 1032] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[359, 572, 140, 1022, 503, 712, 51, 327, 306, 1278, 1675, 537, 501, 370, 659, 298, 1049, 912, 83, 539, 1490, 564, 415, 2021, 792, 131, 232, 1635, 1407, 533, 1068, 607, 35, 84, 529, 737, 549, 210, 446, 300, 6033, 2130, 2282, 507, 847, 341, 443, 712, 13, 1190, 395, 492, 624, 2197, 404, 525, 1757, 611, 348, 181, 121, 633, 405, 283] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[394, 631, 162, 1167, 569, 257, 103, 309, 366, 407, 1744, 416, 460, 1119, 385, 529, 1453, 252, 303, 743, 714, 459, 1033, 1248, 519, 47, 597, 1885, 242, 164, 843, 73, 18, 287, 768, 141, 497, 141, 842, 1013, 2608, 2075, 1832, 232, 210, 683, 181, 469, 100, 1177, 871, 276, 2611, 2771, 516, 640, 1094, 2973, 1520, 485, 799, 1407, 61, 261] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[244, 47, 159, 1113, 248, 643, 161, 710, 358, 542, 1193, 767, 925, 1490, 1058, 362, 1383, 465, 66, 781, 717, 1415, 839, 1267, 810, 150, 186, 1855, 850, 119, 1192, 53, 10, 33, 2154, 571, 1657, 82, 518, 311, 4571, 1158, 1808, 621, 162, 474, 114, 92, 29, 859, 1551, 65, 3308, 2081, 206, 469, 897, 1201, 577, 502, 204, 447, 60, 192] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[118, 419, 147, 650, 435, 274, 251, 266, 304, 518, 1523, 683, 351, 508, 712, 743, 949, 722, 447, 797, 555, 938, 110, 700, 226, 114, 554, 1383, 2312, 684, 157, 95, 198, 515, 1835, 626, 1110, 260, 804, 467, 3523, 1229, 983, 528, 232, 1022, 422, 111, 96, 495, 253, 558, 3082, 2430, 1549, 363, 726, 3595, 813, 424, 609, 833, 594, 222] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[413, 234, 235, 802, 297, 136, 204, 228, 796, 1053, 800, 509, 617, 316, 386, 512, 2639, 737, 65, 1994, 943, 2232, 106, 1750, 1002, 20, 408, 1370, 1094, 197, 562, 106, 27, 61, 911, 515, 1415, 894, 387, 500, 3757, 613, 1156, 552, 224, 1191, 76, 241, 449, 1416, 166, 411, 1477, 1493, 788, 726, 1406, 1419, 318, 93, 1175, 621, 1304, 607] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[394, 631, 162, 1167, 569, 257, 103, 309, 366, 407, 1744, 416, 460, 1119, 385, 529, 1453, 252, 303, 743, 714, 459, 1033, 1248, 519, 47, 597, 1885, 242, 164, 843, 73, 18, 287, 768, 141, 497, 141, 842, 1013, 2608, 2075, 1832, 232, 210, 683, 181, 469, 100, 1177, 871, 276, 2611, 2771, 516, 640, 1094, 2973, 1520, 485, 799, 1407, 61, 261] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[286, 63, 240, 846, 369, 455, 972, 416, 569, 766, 963, 449, 949, 844, 948, 253, 550, 930, 200, 993, 1337, 739, 206, 886, 961, 130, 329, 955, 1134, 657, 465, 167, 193, 425, 177, 356, 883, 1002, 827, 662, 3261, 438, 1563, 733, 81, 1169, 141, 548, 515, 799, 182, 475, 3122, 2531, 944, 1029, 606, 2103, 782, 406, 210, 933, 997, 1032] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[323, 804, 273, 676, 445, 452, 357, 368, 242, 475, 2557, 420, 1090, 484, 1002, 371, 689, 635, 203, 781, 1064, 2126, 374, 948, 147, 258, 708, 1148, 869, 358, 532, 248, 293, 574, 550, 459, 1202, 175, 1641, 553, 4051, 990, 1074, 502, 131, 384, 221, 907, 317, 892, 425, 176, 2885, 2176, 730, 807, 1482, 763, 527, 230, 1406, 1013, 112, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[118, 419, 147, 650, 435, 274, 251, 266, 304, 518, 1523, 683, 351, 508, 712, 743, 949, 722, 447, 797, 555, 938, 110, 700, 226, 114, 554, 1383, 2312, 684, 157, 95, 198, 515, 1835, 626, 1110, 260, 804, 467, 3523, 1229, 983, 528, 232, 1022, 422, 111, 96, 495, 253, 558, 3082, 2430, 1549, 363, 726, 3595, 813, 424, 609, 833, 594, 222] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[109, 538, 186, 954, 470, 25, 226, 235, 576, 408, 1478, 396, 712, 607, 588, 1548, 2087, 332, 556, 1472, 1008, 764, 230, 116, 294, 46, 72, 973, 867, 307, 571, 439, 59, 19, 907, 114, 313, 376, 163, 421, 4767, 721, 1270, 224, 82, 1794, 933, 72, 470, 952, 307, 291, 1552, 3173, 672, 1234, 1901, 2402, 175, 95, 1269, 1054, 2030, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[244, 47, 159, 1113, 248, 643, 161, 710, 358, 542, 1193, 767, 925, 1490, 1058, 362, 1383, 465, 66, 781, 717, 1415, 839, 1267, 810, 150, 186, 1855, 850, 119, 1192, 53, 10, 33, 2154, 571, 1657, 82, 518, 311, 4571, 1158, 1808, 621, 162, 474, 114, 92, 29, 859, 1551, 65, 3308, 2081, 206, 469, 897, 1201, 577, 502, 204, 447, 60, 192] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[359, 572, 140, 1022, 503, 712, 51, 327, 4325, 0, 0, 0, 0, 0, 0, 0, 306, 1278, 1675, 537, 501, 370, 659, 298, 96, 0, 0, 0, 0, 0, 0, 0, 470, 912, 83, 539, 1490, 564, 415, 2021, 0, 0, 0, 0, 0, 0, 0, 0, 792, 131, 232, 1635, 1407, 533, 1068, 607, 579, 265, 81, 0, 0, 0, 0, 0, 35, 84, 529, 737, 549, 210, 446, 300, 1672, 0, 0, 0, 0, 0, 0, 0, 1708, 2130, 2282, 507, 847, 341, 443, 712, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1190, 395, 492, 624, 260, 404, 525, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 434, 348, 181, 121, 633, 405, 283, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8011, 5720, 6494, 7330, 4562, 8970, 3903, 4162] avg=6144.0000 max=8970 max_rank=5 min=3903 min_rank=6 imbalance_ratio=1.459961 max_load_reduction=40182 max_load_reduction_pct=0.817505 moved_to_echo=21597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[394, 631, 162, 1167, 569, 257, 103, 309, 1869, 0, 0, 0, 0, 0, 0, 0, 366, 407, 1744, 416, 460, 1119, 385, 529, 469, 0, 0, 0, 0, 0, 0, 0, 650, 252, 303, 743, 714, 459, 1033, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 519, 47, 597, 1885, 242, 164, 843, 73, 803, 334, 394, 0, 0, 0, 0, 0, 18, 287, 768, 141, 497, 141, 842, 1013, 2109, 0, 0, 0, 0, 0, 0, 0, 739, 2075, 1832, 232, 210, 683, 181, 469, 0, 0, 0, 0, 0, 0, 0, 0, 100, 1177, 871, 276, 2611, 328, 516, 640, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 2110, 1520, 485, 799, 1407, 61, 261, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5461, 5895, 5402, 5901, 5816, 6421, 6519, 7737] avg=6144.0000 max=7737 max_rank=7 min=5402 min_rank=2 imbalance_ratio=1.259277 max_load_reduction=41415 max_load_reduction_pct=0.842590 moved_to_echo=26493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[413, 234, 235, 802, 297, 136, 204, 228, 2693, 0, 0, 0, 0, 0, 0, 0, 796, 1053, 800, 509, 617, 316, 386, 512, 224, 0, 0, 0, 0, 0, 0, 0, 1181, 737, 65, 1994, 943, 2232, 106, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 20, 408, 1370, 1094, 197, 562, 106, 1458, 180, 188, 0, 0, 0, 0, 0, 27, 61, 911, 515, 1415, 894, 387, 500, 1136, 0, 0, 0, 0, 0, 0, 0, 1064, 613, 1156, 552, 224, 1191, 76, 241, 0, 0, 0, 0, 0, 0, 0, 0, 449, 1416, 166, 411, 1477, 177, 788, 726, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 1007, 318, 93, 1175, 621, 1304, 607, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5242, 5213, 9008, 6585, 5846, 5117, 5610, 6531] avg=6144.0000 max=9008 max_rank=2 min=5117 min_rank=5 imbalance_ratio=1.466146 max_load_reduction=40144 max_load_reduction_pct=0.816732 moved_to_echo=23104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[118, 419, 147, 650, 435, 274, 251, 266, 2525, 0, 0, 0, 0, 0, 0, 0, 304, 518, 1523, 683, 351, 508, 712, 743, 568, 0, 0, 0, 0, 0, 0, 0, 425, 722, 447, 797, 555, 938, 110, 700, 0, 0, 0, 0, 0, 0, 0, 0, 226, 114, 554, 1383, 2312, 684, 157, 95, 524, 293, 477, 0, 0, 0, 0, 0, 198, 515, 1835, 626, 1110, 260, 804, 467, 1849, 0, 0, 0, 0, 0, 0, 0, 998, 1229, 983, 528, 232, 1022, 422, 111, 0, 0, 0, 0, 0, 0, 0, 0, 96, 495, 253, 558, 3082, 288, 1549, 363, 0, 0, 0, 0, 0, 0, 0, 0, 726, 2550, 813, 424, 609, 833, 594, 222, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5085, 5910, 4694, 6819, 7664, 5525, 6684, 6771] avg=6144.0000 max=7664 max_rank=4 min=4694 min_rank=2 imbalance_ratio=1.247396 max_load_reduction=41488 max_load_reduction_pct=0.844076 moved_to_echo=26644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[323, 804, 273, 676, 445, 452, 357, 368, 2904, 0, 0, 0, 0, 0, 0, 0, 242, 475, 2557, 420, 1090, 484, 1002, 371, 120, 0, 0, 0, 0, 0, 0, 0, 309, 635, 203, 781, 1064, 2126, 374, 948, 0, 0, 0, 0, 0, 0, 0, 0, 147, 258, 708, 1148, 869, 358, 532, 248, 380, 262, 101, 0, 0, 0, 0, 0, 293, 574, 550, 459, 1202, 175, 1641, 553, 1656, 0, 0, 0, 0, 0, 0, 0, 1147, 990, 1074, 502, 131, 384, 221, 907, 0, 0, 0, 0, 0, 0, 0, 0, 317, 892, 425, 176, 2885, 258, 730, 807, 0, 0, 0, 0, 0, 0, 0, 0, 1482, 542, 527, 230, 1406, 1013, 112, 77, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6602, 6761, 6440, 5011, 7103, 5356, 6490, 5389] avg=6144.0000 max=7103 max_rank=4 min=5011 min_rank=3 imbalance_ratio=1.156087 max_load_reduction=42049 max_load_reduction_pct=0.855489 moved_to_echo=24338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[286, 63, 240, 846, 369, 455, 972, 416, 2341, 2, 0, 0, 0, 0, 0, 0, 569, 766, 963, 449, 949, 844, 948, 253, 336, 0, 0, 0, 0, 0, 0, 0, 242, 930, 200, 993, 1337, 739, 206, 886, 0, 0, 0, 0, 0, 0, 0, 0, 961, 130, 329, 955, 1134, 657, 465, 167, 308, 307, 282, 0, 0, 0, 0, 0, 193, 425, 177, 356, 883, 1002, 827, 662, 1929, 0, 0, 0, 0, 0, 0, 0, 920, 438, 1563, 733, 81, 1169, 141, 548, 0, 0, 0, 0, 0, 0, 0, 0, 515, 799, 182, 475, 3122, 293, 944, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 606, 1485, 782, 406, 210, 933, 997, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5990, 6077, 5533, 5695, 6454, 5593, 7359, 6451] avg=6144.0000 max=7359 max_rank=6 min=5533 min_rank=2 imbalance_ratio=1.197754 max_load_reduction=41793 max_load_reduction_pct=0.850281 moved_to_echo=25857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[244, 47, 159, 1113, 248, 643, 161, 710, 3276, 0, 0, 0, 0, 0, 0, 0, 358, 542, 1193, 767, 925, 1490, 1058, 362, 189, 0, 0, 0, 0, 0, 0, 0, 619, 465, 66, 781, 717, 1415, 839, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 810, 150, 186, 1855, 850, 119, 1192, 53, 764, 251, 159, 0, 0, 0, 0, 0, 10, 33, 2154, 571, 1657, 82, 518, 311, 1584, 0, 0, 0, 0, 0, 0, 0, 1295, 1158, 1808, 621, 162, 474, 114, 92, 0, 0, 0, 0, 0, 0, 0, 0, 29, 859, 1551, 65, 3308, 246, 206, 469, 0, 0, 0, 0, 0, 0, 0, 0, 897, 853, 577, 502, 204, 447, 60, 192, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6601, 6884, 6169, 6389, 6920, 5724, 6733, 3732] avg=6144.0000 max=6920 max_rank=4 min=3732 min_rank=7 imbalance_ratio=1.126302 max_load_reduction=42232 max_load_reduction_pct=0.859212 moved_to_echo=23109 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[394, 631, 162, 1167, 569, 257, 103, 309, 1869, 0, 0, 0, 0, 0, 0, 0, 366, 407, 1744, 416, 460, 1119, 385, 529, 469, 0, 0, 0, 0, 0, 0, 0, 650, 252, 303, 743, 714, 459, 1033, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 519, 47, 597, 1885, 242, 164, 843, 73, 803, 334, 394, 0, 0, 0, 0, 0, 18, 287, 768, 141, 497, 141, 842, 1013, 2109, 0, 0, 0, 0, 0, 0, 0, 739, 2075, 1832, 232, 210, 683, 181, 469, 0, 0, 0, 0, 0, 0, 0, 0, 100, 1177, 871, 276, 2611, 328, 516, 640, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 2110, 1520, 485, 799, 1407, 61, 261, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5461, 5895, 5402, 5901, 5816, 6421, 6519, 7737] avg=6144.0000 max=7737 max_rank=7 min=5402 min_rank=2 imbalance_ratio=1.259277 max_load_reduction=41415 max_load_reduction_pct=0.842590 moved_to_echo=26493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[359, 572, 140, 1022, 503, 712, 51, 327, 4325, 0, 0, 0, 0, 0, 0, 0, 306, 1278, 1675, 537, 501, 370, 659, 298, 96, 0, 0, 0, 0, 0, 0, 0, 470, 912, 83, 539, 1490, 564, 415, 2021, 0, 0, 0, 0, 0, 0, 0, 0, 792, 131, 232, 1635, 1407, 533, 1068, 607, 579, 265, 81, 0, 0, 0, 0, 0, 35, 84, 529, 737, 549, 210, 446, 300, 1672, 0, 0, 0, 0, 0, 0, 0, 1708, 2130, 2282, 507, 847, 341, 443, 712, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1190, 395, 492, 624, 260, 404, 525, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 434, 348, 181, 121, 633, 405, 283, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8011, 5720, 6494, 7330, 4562, 8970, 3903, 4162] avg=6144.0000 max=8970 max_rank=5 min=3903 min_rank=6 imbalance_ratio=1.459961 max_load_reduction=40182 max_load_reduction_pct=0.817505 moved_to_echo=21597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[109, 538, 186, 954, 470, 25, 226, 235, 576, 408, 1478, 396, 712, 607, 588, 1548, 2087, 332, 556, 1472, 1008, 764, 230, 116, 294, 46, 72, 973, 867, 307, 571, 439, 59, 19, 907, 114, 313, 376, 163, 421, 4767, 721, 1270, 224, 82, 1794, 933, 72, 470, 952, 307, 291, 1552, 3173, 672, 1234, 1901, 2402, 175, 95, 1269, 1054, 2030, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[109, 538, 186, 954, 470, 25, 226, 235, 3417, 0, 0, 0, 0, 0, 0, 0, 576, 408, 1478, 396, 712, 607, 588, 1548, 379, 0, 0, 0, 0, 0, 0, 0, 934, 332, 556, 1472, 1008, 764, 230, 116, 0, 0, 0, 0, 0, 0, 0, 0, 294, 46, 72, 973, 867, 307, 571, 439, 1153, 382, 318, 0, 0, 0, 0, 0, 59, 19, 907, 114, 313, 376, 163, 421, 2415, 0, 0, 0, 0, 0, 0, 0, 1350, 721, 1270, 224, 82, 1794, 933, 72, 0, 0, 0, 0, 0, 0, 0, 0, 470, 952, 307, 291, 1552, 376, 672, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 1901, 1705, 175, 95, 1269, 1054, 2030, 150, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6692, 5412, 5422, 4787, 6446, 5854, 8379] avg=6144.0000 max=8379 max_rank=7 min=4787 min_rank=4 imbalance_ratio=1.363770 max_load_reduction=40773 max_load_reduction_pct=0.829529 moved_to_echo=25466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[286, 63, 240, 846, 369, 455, 972, 416, 2341, 2, 0, 0, 0, 0, 0, 0, 569, 766, 963, 449, 949, 844, 948, 253, 336, 0, 0, 0, 0, 0, 0, 0, 242, 930, 200, 993, 1337, 739, 206, 886, 0, 0, 0, 0, 0, 0, 0, 0, 961, 130, 329, 955, 1134, 657, 465, 167, 308, 307, 282, 0, 0, 0, 0, 0, 193, 425, 177, 356, 883, 1002, 827, 662, 1929, 0, 0, 0, 0, 0, 0, 0, 920, 438, 1563, 733, 81, 1169, 141, 548, 0, 0, 0, 0, 0, 0, 0, 0, 515, 799, 182, 475, 3122, 293, 944, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 606, 1485, 782, 406, 210, 933, 997, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5990, 6077, 5533, 5695, 6454, 5593, 7359, 6451] avg=6144.0000 max=7359 max_rank=6 min=5533 min_rank=2 imbalance_ratio=1.197754 max_load_reduction=41793 max_load_reduction_pct=0.850281 moved_to_echo=25857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[323, 804, 273, 676, 445, 452, 357, 368, 2904, 0, 0, 0, 0, 0, 0, 0, 242, 475, 2557, 420, 1090, 484, 1002, 371, 120, 0, 0, 0, 0, 0, 0, 0, 309, 635, 203, 781, 1064, 2126, 374, 948, 0, 0, 0, 0, 0, 0, 0, 0, 147, 258, 708, 1148, 869, 358, 532, 248, 380, 262, 101, 0, 0, 0, 0, 0, 293, 574, 550, 459, 1202, 175, 1641, 553, 1656, 0, 0, 0, 0, 0, 0, 0, 1147, 990, 1074, 502, 131, 384, 221, 907, 0, 0, 0, 0, 0, 0, 0, 0, 317, 892, 425, 176, 2885, 258, 730, 807, 0, 0, 0, 0, 0, 0, 0, 0, 1482, 542, 527, 230, 1406, 1013, 112, 77, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6602, 6761, 6440, 5011, 7103, 5356, 6490, 5389] avg=6144.0000 max=7103 max_rank=4 min=5011 min_rank=3 imbalance_ratio=1.156087 max_load_reduction=42049 max_load_reduction_pct=0.855489 moved_to_echo=24338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[118, 419, 147, 650, 435, 274, 251, 266, 2525, 0, 0, 0, 0, 0, 0, 0, 304, 518, 1523, 683, 351, 508, 712, 743, 568, 0, 0, 0, 0, 0, 0, 0, 425, 722, 447, 797, 555, 938, 110, 700, 0, 0, 0, 0, 0, 0, 0, 0, 226, 114, 554, 1383, 2312, 684, 157, 95, 524, 293, 477, 0, 0, 0, 0, 0, 198, 515, 1835, 626, 1110, 260, 804, 467, 1849, 0, 0, 0, 0, 0, 0, 0, 998, 1229, 983, 528, 232, 1022, 422, 111, 0, 0, 0, 0, 0, 0, 0, 0, 96, 495, 253, 558, 3082, 288, 1549, 363, 0, 0, 0, 0, 0, 0, 0, 0, 726, 2550, 813, 424, 609, 833, 594, 222, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5085, 5910, 4694, 6819, 7664, 5525, 6684, 6771] avg=6144.0000 max=7664 max_rank=4 min=4694 min_rank=2 imbalance_ratio=1.247396 max_load_reduction=41488 max_load_reduction_pct=0.844076 moved_to_echo=26644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 7808, 4224, 2816, 18688, 2304, 5888, 5888, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4864, 4992, 2048, 8192, 7936, 9344, 3328, 8960, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 896, 3200, 11264, 8832, 3072, 5504, 1792, 6016, 2304, 2048, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2304, 3328, 1664, 7296, 3456, 3072, 2432, 2944, 23424, 128, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 2048, 7936, 3584, 7680, 3200, 5632, 4352, 14464, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 7808, 4224, 2816, 18688, 2304, 5888, 5888, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9984, 10752, 5120, 2432, 5888, 7040, 5632, 2944, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9344, 9472, 12032, 3968, 2048, 7168, 2560, 3200, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 5504, 12032, 4224, 5632, 5760, 5760, 4736, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[520, 1015, 418, 900, 462, 210, 3138, 1082, 1260, 570, 439, 460, 1106, 417, 83, 27, 1139, 900, 663, 502, 202, 1453, 59, 1677, 22, 3467, 725, 81, 492, 320, 800, 202, 373, 87, 682, 265, 475, 197, 737, 2281, 501, 234, 655, 1673, 372, 1586, 1758, 61, 954, 199, 1683, 566, 85, 139, 640, 2852, 62, 160, 817, 692, 1380, 304, 1648, 223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[364, 2338, 327, 28, 626, 171, 1407, 790, 1451, 649, 93, 414, 1041, 583, 202, 52, 898, 1273, 1213, 94, 552, 748, 294, 1463, 106, 1484, 713, 317, 395, 1365, 1010, 230, 1405, 6, 766, 638, 593, 399, 1926, 1101, 235, 455, 533, 2980, 83, 1361, 2802, 16, 311, 1410, 1869, 480, 480, 159, 173, 1649, 580, 700, 1146, 530, 268, 358, 930, 119] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[279, 1423, 308, 689, 347, 220, 1402, 1399, 1725, 556, 347, 720, 423, 1662, 237, 567, 549, 1155, 378, 246, 125, 457, 367, 853, 74, 1865, 1321, 101, 386, 372, 263, 483, 85, 190, 1376, 1074, 1017, 55, 1105, 1397, 426, 178, 1046, 1395, 631, 1456, 923, 69, 1055, 1902, 769, 514, 1344, 55, 425, 2385, 83, 286, 2292, 813, 1310, 1589, 467, 141] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[378, 1021, 290, 336, 528, 134, 2087, 1154, 2124, 263, 821, 780, 2125, 365, 267, 39, 1267, 1589, 260, 432, 278, 1194, 14, 402, 65, 1465, 665, 192, 403, 891, 1284, 13, 297, 58, 125, 403, 3076, 60, 832, 827, 448, 223, 1166, 2542, 1780, 2061, 2304, 99, 930, 338, 414, 507, 264, 27, 133, 2927, 390, 305, 826, 684, 790, 305, 542, 343] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1363, 1468, 93, 360, 830, 448, 1188, 948, 1176, 1165, 295, 641, 182, 437, 159, 351, 1617, 892, 1429, 170, 417, 980, 346, 1056, 204, 944, 1220, 97, 129, 102, 862, 1049, 411, 115, 1028, 774, 906, 218, 891, 1357, 685, 638, 854, 2713, 154, 1064, 3509, 78, 311, 465, 1014, 544, 357, 60, 547, 1665, 64, 626, 926, 1046, 520, 348, 2326, 320] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[542, 2306, 258, 923, 476, 119, 1126, 1330, 1090, 637, 1502, 850, 1066, 856, 247, 1181, 2199, 821, 796, 470, 557, 1322, 42, 1794, 85, 1630, 642, 144, 211, 876, 620, 69, 60, 119, 288, 609, 1245, 52, 426, 255, 661, 199, 1101, 1843, 282, 1826, 1857, 56, 241, 558, 391, 310, 681, 395, 109, 2295, 294, 761, 1158, 1179, 1145, 862, 862, 245] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[605, 1765, 62, 801, 518, 141, 1573, 1956, 970, 445, 944, 420, 2944, 967, 119, 166, 1411, 219, 349, 276, 214, 938, 63, 519, 50, 908, 1195, 337, 231, 355, 3537, 75, 812, 66, 521, 574, 698, 73, 2249, 862, 1130, 205, 1401, 1415, 276, 196, 2255, 57, 199, 1077, 882, 442, 391, 21, 985, 1729, 86, 443, 2306, 316, 682, 506, 1167, 57] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[471, 544, 859, 208, 560, 344, 2174, 1525, 1704, 307, 1656, 286, 781, 250, 33, 293, 1010, 1137, 926, 436, 556, 1214, 743, 1530, 184, 758, 850, 114, 209, 245, 1367, 8, 367, 331, 824, 776, 1814, 217, 2118, 1913, 742, 469, 298, 2143, 742, 1079, 1346, 251, 87, 320, 1618, 607, 181, 328, 54, 1473, 189, 705, 1535, 349, 509, 278, 1959, 248] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[364, 2338, 327, 28, 626, 171, 1407, 790, 1451, 649, 93, 414, 1041, 583, 202, 52, 898, 1273, 1213, 94, 552, 748, 294, 1463, 106, 1484, 713, 317, 395, 1365, 1010, 230, 1405, 6, 766, 638, 593, 399, 1926, 1101, 235, 455, 533, 2980, 83, 1361, 2802, 16, 311, 1410, 1869, 480, 480, 159, 173, 1649, 580, 700, 1146, 530, 268, 358, 930, 119] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[520, 1015, 418, 900, 462, 210, 3138, 1082, 1260, 570, 439, 460, 1106, 417, 83, 27, 1139, 900, 663, 502, 202, 1453, 59, 1677, 22, 3467, 725, 81, 492, 320, 800, 202, 373, 87, 682, 265, 475, 197, 737, 2281, 501, 234, 655, 1673, 372, 1586, 1758, 61, 954, 199, 1683, 566, 85, 139, 640, 2852, 62, 160, 817, 692, 1380, 304, 1648, 223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1363, 1468, 93, 360, 830, 448, 1188, 948, 1176, 1165, 295, 641, 182, 437, 159, 351, 1617, 892, 1429, 170, 417, 980, 346, 1056, 204, 944, 1220, 97, 129, 102, 862, 1049, 411, 115, 1028, 774, 906, 218, 891, 1357, 685, 638, 854, 2713, 154, 1064, 3509, 78, 311, 465, 1014, 544, 357, 60, 547, 1665, 64, 626, 926, 1046, 520, 348, 2326, 320] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[378, 1021, 290, 336, 528, 134, 2087, 1154, 2124, 263, 821, 780, 2125, 365, 267, 39, 1267, 1589, 260, 432, 278, 1194, 14, 402, 65, 1465, 665, 192, 403, 891, 1284, 13, 297, 58, 125, 403, 3076, 60, 832, 827, 448, 223, 1166, 2542, 1780, 2061, 2304, 99, 930, 338, 414, 507, 264, 27, 133, 2927, 390, 305, 826, 684, 790, 305, 542, 343] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[279, 1423, 308, 689, 347, 220, 1402, 1399, 1725, 556, 347, 720, 423, 1662, 237, 567, 549, 1155, 378, 246, 125, 457, 367, 853, 74, 1865, 1321, 101, 386, 372, 263, 483, 85, 190, 1376, 1074, 1017, 55, 1105, 1397, 426, 178, 1046, 1395, 631, 1456, 923, 69, 1055, 1902, 769, 514, 1344, 55, 425, 2385, 83, 286, 2292, 813, 1310, 1589, 467, 141] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[605, 1765, 62, 801, 518, 141, 1071, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 970, 445, 944, 420, 2944, 967, 119, 166, 282, 118, 0, 0, 0, 0, 0, 0, 1411, 219, 349, 276, 214, 938, 63, 519, 0, 0, 0, 0, 0, 0, 0, 0, 50, 908, 1195, 337, 231, 355, 3537, 75, 1113, 0, 0, 0, 0, 0, 0, 0, 812, 66, 521, 574, 698, 73, 2249, 862, 253, 0, 0, 0, 0, 0, 0, 0, 1130, 205, 1401, 1415, 276, 196, 315, 57, 0, 0, 0, 0, 0, 0, 0, 0, 199, 1077, 882, 442, 391, 21, 985, 1729, 131, 0, 0, 0, 0, 0, 0, 0, 86, 443, 2306, 316, 682, 506, 1167, 57, 545, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6919, 7375, 3989, 7801, 6108, 4995, 5857, 6108] avg=6144.0000 max=7801 max_rank=3 min=3989 min_rank=2 imbalance_ratio=1.269694 max_load_reduction=41351 max_load_reduction_pct=0.841288 moved_to_echo=23068 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[542, 2306, 258, 923, 476, 119, 767, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 637, 1502, 850, 1066, 856, 247, 1181, 232, 84, 0, 0, 0, 0, 0, 0, 2199, 821, 796, 470, 557, 1322, 42, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 85, 1630, 642, 144, 211, 876, 620, 69, 916, 0, 0, 0, 0, 0, 0, 0, 60, 119, 288, 609, 1245, 52, 426, 255, 181, 0, 0, 0, 0, 0, 0, 0, 661, 199, 1101, 1843, 282, 1826, 260, 56, 0, 0, 0, 0, 0, 0, 0, 0, 241, 558, 391, 310, 681, 395, 109, 2295, 94, 0, 0, 0, 0, 0, 0, 0, 294, 761, 1158, 1179, 1145, 862, 862, 245, 449, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6721, 7745, 8001, 5193, 3235, 6228, 5074, 6955] avg=6144.0000 max=8001 max_rank=2 min=3235 min_rank=4 imbalance_ratio=1.302246 max_load_reduction=41151 max_load_reduction_pct=0.837219 moved_to_echo=21492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[364, 2338, 327, 28, 626, 171, 957, 790, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 649, 93, 414, 1041, 583, 202, 52, 351, 106, 0, 0, 0, 0, 0, 0, 898, 1273, 1213, 94, 552, 748, 294, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 106, 1484, 713, 317, 395, 1365, 1010, 230, 1383, 0, 0, 0, 0, 0, 0, 0, 1405, 6, 766, 638, 593, 399, 1926, 1101, 227, 0, 0, 0, 0, 0, 0, 0, 235, 455, 533, 2980, 83, 1361, 391, 16, 0, 0, 0, 0, 0, 0, 0, 0, 311, 1410, 1869, 480, 480, 159, 173, 1649, 117, 0, 0, 0, 0, 0, 0, 0, 580, 700, 1146, 530, 268, 358, 930, 119, 677, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5601, 4942, 6535, 7003, 7061, 6054, 6648, 5308] avg=6144.0000 max=7061 max_rank=4 min=4942 min_rank=1 imbalance_ratio=1.149251 max_load_reduction=42091 max_load_reduction_pct=0.856344 moved_to_echo=25071 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[520, 1015, 418, 900, 462, 210, 2133, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 570, 439, 460, 1106, 417, 83, 27, 220, 236, 0, 0, 0, 0, 0, 0, 1139, 900, 663, 502, 202, 1453, 59, 1677, 1, 0, 0, 0, 0, 0, 0, 0, 22, 3467, 725, 81, 492, 320, 800, 202, 868, 0, 0, 0, 0, 0, 0, 0, 373, 87, 682, 265, 475, 197, 737, 2281, 506, 0, 0, 0, 0, 0, 0, 0, 501, 234, 655, 1673, 372, 1586, 245, 61, 0, 0, 0, 0, 0, 0, 0, 0, 954, 199, 1683, 566, 85, 139, 640, 2852, 262, 0, 0, 0, 0, 0, 0, 0, 62, 160, 817, 692, 1380, 304, 1648, 223, 425, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6740, 4818, 6596, 6977, 5603, 5327, 7380, 5711] avg=6144.0000 max=7380 max_rank=6 min=4818 min_rank=1 imbalance_ratio=1.201172 max_load_reduction=41772 max_load_reduction_pct=0.849854 moved_to_echo=24021 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[471, 544, 859, 208, 560, 344, 1478, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 1704, 307, 1656, 286, 781, 250, 33, 293, 168, 164, 0, 0, 0, 0, 0, 0, 1010, 1137, 926, 436, 556, 1214, 743, 1530, 1, 0, 0, 0, 0, 0, 0, 0, 184, 758, 850, 114, 209, 245, 1367, 8, 664, 0, 0, 0, 0, 0, 0, 0, 367, 331, 824, 776, 1814, 217, 2118, 1913, 350, 0, 0, 0, 0, 0, 0, 0, 742, 469, 298, 2143, 742, 1079, 189, 251, 0, 0, 0, 0, 0, 0, 0, 0, 87, 320, 1618, 607, 181, 328, 54, 1473, 181, 0, 0, 0, 0, 0, 0, 0, 189, 705, 1535, 349, 509, 278, 1959, 248, 325, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 5642, 7553, 4399, 8710, 5913, 4849, 6097] avg=6144.0000 max=8710 max_rank=4 min=4399 min_rank=3 imbalance_ratio=1.417643 max_load_reduction=40442 max_load_reduction_pct=0.822795 moved_to_echo=25569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[279, 1423, 308, 689, 347, 220, 954, 1399, 0, 0, 0, 0, 0, 0, 0, 0, 1725, 556, 347, 720, 423, 1662, 237, 567, 115, 105, 0, 0, 0, 0, 0, 0, 549, 1155, 378, 246, 125, 457, 367, 853, 0, 0, 0, 0, 0, 0, 0, 0, 74, 1865, 1321, 101, 386, 372, 263, 483, 455, 0, 0, 0, 0, 0, 0, 0, 85, 190, 1376, 1074, 1017, 55, 1105, 1397, 226, 0, 0, 0, 0, 0, 0, 0, 426, 178, 1046, 1395, 631, 1456, 130, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1902, 769, 514, 1344, 55, 425, 2385, 117, 0, 0, 0, 0, 0, 0, 0, 83, 286, 2292, 813, 1310, 1589, 467, 141, 223, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5619, 6457, 4130, 5320, 6525, 5331, 8566, 7204] avg=6144.0000 max=8566 max_rank=6 min=4130 min_rank=2 imbalance_ratio=1.394206 max_load_reduction=40586 max_load_reduction_pct=0.825724 moved_to_echo=27626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[378, 1021, 290, 336, 528, 134, 1419, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 2124, 263, 821, 780, 2125, 365, 267, 39, 288, 157, 0, 0, 0, 0, 0, 0, 1267, 1589, 260, 432, 278, 1194, 14, 402, 1, 0, 0, 0, 0, 0, 0, 0, 65, 1465, 665, 192, 403, 891, 1284, 13, 1137, 0, 0, 0, 0, 0, 0, 0, 297, 58, 125, 403, 3076, 60, 832, 827, 336, 0, 0, 0, 0, 0, 0, 0, 448, 223, 1166, 2542, 1780, 2061, 322, 99, 0, 0, 0, 0, 0, 0, 0, 0, 930, 338, 414, 507, 264, 27, 133, 2927, 174, 0, 0, 0, 0, 0, 0, 0, 390, 305, 826, 684, 790, 305, 542, 343, 557, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5260, 7229, 5437, 6115, 6014, 8641, 5714, 4742] avg=6144.0000 max=8641 max_rank=5 min=4742 min_rank=7 imbalance_ratio=1.406413 max_load_reduction=40511 max_load_reduction_pct=0.824198 moved_to_echo=25111 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[605, 1765, 62, 801, 518, 141, 1071, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 970, 445, 944, 420, 2944, 967, 119, 166, 282, 118, 0, 0, 0, 0, 0, 0, 1411, 219, 349, 276, 214, 938, 63, 519, 0, 0, 0, 0, 0, 0, 0, 0, 50, 908, 1195, 337, 231, 355, 3537, 75, 1113, 0, 0, 0, 0, 0, 0, 0, 812, 66, 521, 574, 698, 73, 2249, 862, 253, 0, 0, 0, 0, 0, 0, 0, 1130, 205, 1401, 1415, 276, 196, 315, 57, 0, 0, 0, 0, 0, 0, 0, 0, 199, 1077, 882, 442, 391, 21, 985, 1729, 131, 0, 0, 0, 0, 0, 0, 0, 86, 443, 2306, 316, 682, 506, 1167, 57, 545, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6919, 7375, 3989, 7801, 6108, 4995, 5857, 6108] avg=6144.0000 max=7801 max_rank=3 min=3989 min_rank=2 imbalance_ratio=1.269694 max_load_reduction=41351 max_load_reduction_pct=0.841288 moved_to_echo=23068 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1363, 1468, 93, 360, 830, 448, 793, 948, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 1165, 295, 641, 182, 437, 159, 351, 444, 94, 0, 0, 0, 0, 0, 0, 1617, 892, 1429, 170, 417, 980, 346, 1056, 4, 0, 0, 0, 0, 0, 0, 0, 204, 944, 1220, 97, 129, 102, 862, 1049, 1737, 0, 0, 0, 0, 0, 0, 0, 411, 115, 1028, 774, 906, 218, 891, 1357, 196, 0, 0, 0, 0, 0, 0, 0, 685, 638, 854, 2713, 154, 1064, 477, 78, 0, 0, 0, 0, 0, 0, 0, 0, 311, 465, 1014, 544, 357, 60, 547, 1665, 101, 0, 0, 0, 0, 0, 0, 0, 64, 626, 926, 1046, 520, 348, 2326, 320, 851, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6303, 4944, 6911, 6344, 5896, 6663, 5064, 7027] avg=6144.0000 max=7027 max_rank=7 min=4944 min_rank=1 imbalance_ratio=1.143717 max_load_reduction=42125 max_load_reduction_pct=0.857035 moved_to_echo=24650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[542, 2306, 258, 923, 476, 119, 767, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 637, 1502, 850, 1066, 856, 247, 1181, 232, 84, 0, 0, 0, 0, 0, 0, 2199, 821, 796, 470, 557, 1322, 42, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 85, 1630, 642, 144, 211, 876, 620, 69, 916, 0, 0, 0, 0, 0, 0, 0, 60, 119, 288, 609, 1245, 52, 426, 255, 181, 0, 0, 0, 0, 0, 0, 0, 661, 199, 1101, 1843, 282, 1826, 260, 56, 0, 0, 0, 0, 0, 0, 0, 0, 241, 558, 391, 310, 681, 395, 109, 2295, 94, 0, 0, 0, 0, 0, 0, 0, 294, 761, 1158, 1179, 1145, 862, 862, 245, 449, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6721, 7745, 8001, 5193, 3235, 6228, 5074, 6955] avg=6144.0000 max=8001 max_rank=2 min=3235 min_rank=4 imbalance_ratio=1.302246 max_load_reduction=41151 max_load_reduction_pct=0.837219 moved_to_echo=21492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[471, 544, 859, 208, 560, 344, 1478, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 1704, 307, 1656, 286, 781, 250, 33, 293, 168, 164, 0, 0, 0, 0, 0, 0, 1010, 1137, 926, 436, 556, 1214, 743, 1530, 1, 0, 0, 0, 0, 0, 0, 0, 184, 758, 850, 114, 209, 245, 1367, 8, 664, 0, 0, 0, 0, 0, 0, 0, 367, 331, 824, 776, 1814, 217, 2118, 1913, 350, 0, 0, 0, 0, 0, 0, 0, 742, 469, 298, 2143, 742, 1079, 189, 251, 0, 0, 0, 0, 0, 0, 0, 0, 87, 320, 1618, 607, 181, 328, 54, 1473, 181, 0, 0, 0, 0, 0, 0, 0, 189, 705, 1535, 349, 509, 278, 1959, 248, 325, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 5642, 7553, 4399, 8710, 5913, 4849, 6097] avg=6144.0000 max=8710 max_rank=4 min=4399 min_rank=3 imbalance_ratio=1.417643 max_load_reduction=40442 max_load_reduction_pct=0.822795 moved_to_echo=25569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[364, 2338, 327, 28, 626, 171, 957, 790, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 649, 93, 414, 1041, 583, 202, 52, 351, 106, 0, 0, 0, 0, 0, 0, 898, 1273, 1213, 94, 552, 748, 294, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 106, 1484, 713, 317, 395, 1365, 1010, 230, 1383, 0, 0, 0, 0, 0, 0, 0, 1405, 6, 766, 638, 593, 399, 1926, 1101, 227, 0, 0, 0, 0, 0, 0, 0, 235, 455, 533, 2980, 83, 1361, 391, 16, 0, 0, 0, 0, 0, 0, 0, 0, 311, 1410, 1869, 480, 480, 159, 173, 1649, 117, 0, 0, 0, 0, 0, 0, 0, 580, 700, 1146, 530, 268, 358, 930, 119, 677, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5601, 4942, 6535, 7003, 7061, 6054, 6648, 5308] avg=6144.0000 max=7061 max_rank=4 min=4942 min_rank=1 imbalance_ratio=1.149251 max_load_reduction=42091 max_load_reduction_pct=0.856344 moved_to_echo=25071 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3840, 1024, 5632, 5120, 9856, 1280, 10368, 10112, 2304, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4096, 6272, 8704, 4096, 3840, 1280, 3072, 17024, 1280, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11520, 4608, 6144, 4608, 9728, 5632, 1408, 2688, 2176, 1152, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 12544, 7424, 1408, 2560, 4608, 9856, 2176, 8320, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 4096, 11008, 5632, 6656, 4608, 9984, 1792, 4096, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4608, 11904, 2688, 4352, 4352, 1792, 9600, 10240, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 2688, 7168, 16768, 4352, 10752, 2432, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10112, 8064, 6016, 2688, 2944, 8320, 2048, 9344, 128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11520, 4608, 6144, 4608, 9728, 5632, 1408, 2688, 2176, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 4096, 11008, 5632, 6656, 4608, 9984, 1792, 4096, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 2688, 7168, 16768, 4352, 10752, 2432, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[18, 221, 118, 501, 320, 87, 410, 343, 622, 1561, 531, 255, 366, 1724, 1518, 2373, 1648, 261, 417, 24, 375, 394, 400, 762, 1853, 224, 1392, 755, 601, 633, 2513, 253, 1660, 57, 1573, 1758, 523, 742, 176, 89, 1010, 905, 260, 166, 3850, 458, 807, 764, 269, 925, 831, 551, 555, 946, 654, 445, 447, 524, 310, 516, 1965, 460, 1208, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[4, 214, 170, 291, 1867, 523, 545, 378, 365, 2823, 180, 712, 346, 1995, 53, 1112, 801, 184, 1201, 259, 393, 165, 931, 1439, 471, 535, 162, 1153, 184, 133, 2267, 709, 765, 241, 1018, 1348, 117, 624, 686, 41, 748, 2202, 392, 263, 2206, 2327, 1074, 317, 598, 858, 188, 2083, 591, 669, 321, 790, 522, 430, 788, 207, 1170, 526, 2279, 198] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[45, 136, 251, 290, 872, 909, 46, 783, 423, 2633, 520, 416, 1068, 1700, 570, 1003, 1583, 273, 727, 470, 613, 427, 793, 2115, 1782, 935, 1403, 370, 272, 686, 1330, 137, 522, 289, 1072, 801, 175, 818, 682, 105, 418, 1185, 429, 332, 2282, 1898, 919, 1304, 134, 641, 425, 1256, 149, 184, 536, 475, 1374, 248, 397, 440, 1364, 1134, 1072, 511] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[19, 246, 43, 103, 3878, 1018, 338, 391, 891, 564, 294, 807, 179, 1726, 589, 1243, 1674, 1377, 439, 408, 852, 126, 398, 67, 1515, 358, 555, 361, 92, 606, 1677, 710, 870, 696, 1081, 1197, 440, 1254, 1765, 10, 227, 1637, 211, 256, 2438, 1045, 342, 1049, 639, 207, 146, 850, 1523, 190, 68, 616, 110, 102, 201, 543, 2125, 312, 2263, 1195] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[18, 221, 118, 501, 320, 87, 410, 343, 622, 1561, 531, 255, 366, 1724, 1518, 2373, 1648, 261, 417, 24, 375, 394, 400, 762, 1853, 224, 1392, 755, 601, 633, 2513, 253, 1660, 57, 1573, 1758, 523, 742, 176, 89, 1010, 905, 260, 166, 3850, 458, 807, 764, 269, 925, 831, 551, 555, 946, 654, 445, 447, 524, 310, 516, 1965, 460, 1208, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[19, 66, 120, 448, 1456, 617, 756, 1074, 654, 1280, 785, 1851, 329, 2904, 814, 1387, 2601, 1141, 422, 68, 370, 518, 479, 272, 1517, 261, 736, 502, 812, 512, 1777, 120, 671, 111, 153, 497, 228, 628, 1089, 127, 117, 1030, 159, 1537, 2368, 983, 1434, 313, 822, 575, 55, 2560, 125, 155, 339, 951, 92, 422, 433, 1706, 432, 708, 1222, 442] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[28, 167, 125, 319, 1390, 417, 738, 903, 1463, 168, 648, 1609, 279, 270, 57, 1307, 2206, 669, 342, 117, 735, 349, 1372, 1789, 906, 588, 408, 196, 51, 333, 2055, 161, 1262, 341, 1162, 1376, 118, 2351, 723, 10, 253, 2617, 83, 160, 2604, 1792, 1485, 391, 202, 95, 154, 2901, 246, 96, 226, 1685, 88, 821, 31, 1120, 1347, 265, 833, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[3, 290, 88, 249, 1523, 1045, 1404, 614, 487, 1580, 909, 2133, 146, 1322, 92, 613, 1456, 1151, 420, 217, 250, 168, 1237, 418, 752, 1146, 1305, 528, 178, 140, 1729, 91, 743, 525, 703, 1766, 650, 815, 803, 31, 233, 1561, 368, 977, 1582, 2156, 1526, 911, 625, 128, 186, 2012, 288, 210, 135, 708, 132, 335, 397, 1187, 1880, 656, 578, 661] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[19, 66, 120, 448, 1456, 617, 756, 1074, 654, 1280, 785, 1851, 329, 2904, 814, 1387, 2601, 1141, 422, 68, 370, 518, 479, 272, 1517, 261, 736, 502, 812, 512, 1777, 120, 671, 111, 153, 497, 228, 628, 1089, 127, 117, 1030, 159, 1537, 2368, 983, 1434, 313, 822, 575, 55, 2560, 125, 155, 339, 951, 92, 422, 433, 1706, 432, 708, 1222, 442] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[196, 707, 534, 505, 899, 1721, 655, 1316, 756, 1058, 612, 701, 222, 1967, 289, 738, 2151, 567, 190, 535, 167, 1022, 869, 898, 1594, 235, 356, 812, 78, 651, 1810, 199, 200, 198, 2302, 285, 535, 232, 434, 213, 314, 2147, 1170, 363, 1239, 2377, 2547, 368, 69, 454, 145, 537, 39, 586, 452, 454, 257, 851, 164, 1264, 1933, 1186, 610, 217] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[45, 136, 251, 290, 872, 909, 46, 783, 423, 2633, 520, 416, 1068, 1700, 570, 1003, 1583, 273, 727, 470, 613, 427, 793, 2115, 1782, 935, 1403, 370, 272, 686, 1330, 137, 522, 289, 1072, 801, 175, 818, 682, 105, 418, 1185, 429, 332, 2282, 1898, 919, 1304, 134, 641, 425, 1256, 149, 184, 536, 475, 1374, 248, 397, 440, 1364, 1134, 1072, 511] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[19, 246, 43, 103, 3878, 1018, 338, 391, 1756, 0, 0, 0, 0, 0, 0, 0, 891, 564, 294, 807, 179, 277, 589, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 1377, 439, 408, 852, 126, 398, 67, 204, 41, 0, 0, 0, 0, 0, 0, 1515, 358, 555, 361, 92, 606, 1676, 710, 0, 0, 0, 0, 0, 0, 0, 0, 870, 696, 1081, 1197, 440, 1254, 1765, 10, 592, 0, 0, 0, 0, 0, 0, 0, 227, 1248, 211, 256, 0, 1045, 342, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 639, 207, 146, 850, 1523, 190, 68, 616, 816, 682, 0, 0, 0, 0, 0, 0, 110, 102, 201, 543, 2125, 312, 2263, 1195, 185, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7792, 4844, 5586, 5873, 7905, 4378, 5737, 7037] avg=6144.0000 max=7905 max_rank=4 min=4378 min_rank=5 imbalance_ratio=1.286621 max_load_reduction=41247 max_load_reduction_pct=0.839172 moved_to_echo=25057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[4, 214, 170, 291, 1867, 523, 545, 378, 1589, 0, 0, 0, 0, 0, 0, 0, 365, 2823, 180, 712, 346, 319, 53, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 801, 184, 1201, 259, 393, 165, 931, 1439, 275, 48, 0, 0, 0, 0, 0, 0, 471, 535, 162, 1153, 184, 133, 2265, 709, 0, 0, 0, 0, 0, 0, 0, 0, 765, 241, 1018, 1348, 117, 624, 686, 41, 685, 0, 0, 0, 0, 0, 0, 0, 748, 1678, 392, 263, 0, 2327, 1074, 317, 0, 0, 0, 0, 0, 0, 0, 0, 598, 858, 188, 2083, 591, 669, 321, 790, 943, 617, 0, 0, 0, 0, 0, 0, 522, 430, 788, 207, 1170, 526, 2279, 198, 249, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5581, 5910, 5696, 5612, 5525, 6799, 7658, 6371] avg=6144.0000 max=7658 max_rank=6 min=5525 min_rank=4 imbalance_ratio=1.246419 max_load_reduction=41494 max_load_reduction_pct=0.844198 moved_to_echo=26353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[18, 221, 118, 501, 320, 87, 410, 343, 2775, 0, 0, 0, 0, 0, 0, 0, 622, 1561, 531, 255, 366, 276, 1518, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 1648, 261, 417, 24, 375, 394, 400, 762, 113, 41, 0, 0, 0, 0, 0, 0, 1853, 224, 1392, 755, 601, 633, 2511, 253, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 57, 1573, 1758, 523, 742, 176, 89, 592, 0, 0, 0, 0, 0, 0, 0, 1010, 690, 260, 166, 0, 458, 807, 764, 0, 0, 0, 0, 0, 0, 0, 0, 269, 925, 831, 551, 555, 946, 654, 445, 815, 1075, 0, 0, 0, 0, 0, 0, 447, 524, 310, 516, 1965, 460, 1208, 275, 102, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4793, 7502, 4435, 8222, 7170, 4155, 7066, 5809] avg=6144.0000 max=8222 max_rank=3 min=4155 min_rank=5 imbalance_ratio=1.338216 max_load_reduction=40930 max_load_reduction_pct=0.832723 moved_to_echo=24200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3, 290, 88, 249, 1523, 1045, 1404, 614, 1141, 0, 0, 0, 0, 0, 0, 0, 487, 1580, 909, 2133, 146, 212, 92, 613, 0, 0, 0, 0, 0, 0, 0, 0, 1456, 1151, 420, 217, 250, 168, 1237, 418, 195, 31, 0, 0, 0, 0, 0, 0, 752, 1146, 1305, 528, 178, 140, 1728, 91, 0, 0, 0, 0, 0, 0, 0, 0, 743, 525, 703, 1766, 650, 815, 803, 31, 454, 0, 0, 0, 0, 0, 0, 0, 233, 1190, 368, 977, 0, 2156, 1526, 911, 0, 0, 0, 0, 0, 0, 0, 0, 625, 128, 186, 2012, 288, 210, 135, 708, 625, 441, 0, 0, 0, 0, 0, 0, 132, 335, 397, 1187, 1880, 656, 578, 661, 176, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6357, 6172, 5543, 5868, 6490, 7361, 5358, 6003] avg=6144.0000 max=7361 max_rank=5 min=5358 min_rank=6 imbalance_ratio=1.198079 max_load_reduction=41791 max_load_reduction_pct=0.850240 moved_to_echo=25212 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[19, 66, 120, 448, 1456, 617, 756, 1074, 1706, 0, 0, 0, 0, 0, 0, 0, 654, 1280, 785, 1851, 329, 465, 814, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 2601, 1141, 422, 68, 370, 518, 479, 272, 128, 69, 0, 0, 0, 0, 0, 0, 1517, 261, 736, 502, 812, 512, 1776, 120, 0, 0, 0, 0, 0, 0, 0, 0, 671, 111, 153, 497, 228, 628, 1089, 127, 997, 0, 0, 0, 0, 0, 0, 0, 117, 786, 159, 1537, 0, 983, 1434, 313, 0, 0, 0, 0, 0, 0, 0, 0, 822, 575, 55, 2560, 125, 155, 339, 951, 1373, 662, 0, 0, 0, 0, 0, 0, 92, 422, 433, 1706, 432, 708, 1222, 442, 116, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6262, 7565, 6068, 6236, 4501, 5329, 7617, 5574] avg=6144.0000 max=7617 max_rank=6 min=4501 min_rank=4 imbalance_ratio=1.239746 max_load_reduction=41535 max_load_reduction_pct=0.845032 moved_to_echo=23021 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[28, 167, 125, 319, 1390, 417, 738, 903, 1877, 0, 0, 0, 0, 0, 0, 0, 1463, 168, 648, 1609, 279, 45, 57, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 669, 342, 117, 735, 349, 1372, 1789, 327, 6, 0, 0, 0, 0, 0, 0, 906, 588, 408, 196, 51, 333, 2054, 161, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 341, 1162, 1376, 118, 2351, 723, 10, 92, 0, 0, 0, 0, 0, 0, 0, 253, 1994, 83, 160, 0, 1792, 1485, 391, 0, 0, 0, 0, 0, 0, 0, 0, 202, 95, 154, 2901, 246, 96, 226, 1685, 127, 727, 0, 0, 0, 0, 0, 0, 88, 821, 31, 1120, 1347, 265, 833, 149, 296, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5964, 5576, 7912, 4697, 7435, 6158, 6459, 4951] avg=6144.0000 max=7912 max_rank=2 min=4697 min_rank=3 imbalance_ratio=1.287760 max_load_reduction=41240 max_load_reduction_pct=0.839030 moved_to_echo=25003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[45, 136, 251, 290, 872, 909, 46, 783, 1645, 0, 0, 0, 0, 0, 0, 0, 423, 2633, 520, 416, 1068, 259, 570, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1583, 273, 727, 470, 613, 427, 793, 2115, 152, 45, 0, 0, 0, 0, 0, 0, 1782, 935, 1403, 370, 272, 686, 1325, 137, 0, 0, 0, 0, 0, 0, 0, 0, 522, 289, 1072, 801, 175, 818, 682, 105, 588, 0, 0, 0, 0, 0, 0, 0, 418, 896, 429, 332, 0, 1898, 919, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 134, 641, 425, 1256, 149, 184, 536, 475, 808, 637, 0, 0, 0, 0, 0, 0, 1374, 248, 397, 440, 1364, 1134, 1072, 511, 137, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4977, 6892, 7198, 6910, 5052, 6196, 5245, 6682] avg=6144.0000 max=7198 max_rank=2 min=4977 min_rank=0 imbalance_ratio=1.171549 max_load_reduction=41954 max_load_reduction_pct=0.853556 moved_to_echo=23175 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[19, 246, 43, 103, 3878, 1018, 338, 391, 1756, 0, 0, 0, 0, 0, 0, 0, 891, 564, 294, 807, 179, 277, 589, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 1377, 439, 408, 852, 126, 398, 67, 204, 41, 0, 0, 0, 0, 0, 0, 1515, 358, 555, 361, 92, 606, 1676, 710, 0, 0, 0, 0, 0, 0, 0, 0, 870, 696, 1081, 1197, 440, 1254, 1765, 10, 592, 0, 0, 0, 0, 0, 0, 0, 227, 1248, 211, 256, 0, 1045, 342, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 639, 207, 146, 850, 1523, 190, 68, 616, 816, 682, 0, 0, 0, 0, 0, 0, 110, 102, 201, 543, 2125, 312, 2263, 1195, 185, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7792, 4844, 5586, 5873, 7905, 4378, 5737, 7037] avg=6144.0000 max=7905 max_rank=4 min=4378 min_rank=5 imbalance_ratio=1.286621 max_load_reduction=41247 max_load_reduction_pct=0.839172 moved_to_echo=25057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[196, 707, 534, 505, 899, 1721, 655, 1316, 893, 0, 0, 0, 0, 0, 0, 0, 756, 1058, 612, 701, 222, 315, 289, 738, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 567, 190, 535, 167, 1022, 869, 898, 268, 47, 0, 0, 0, 0, 0, 0, 1594, 235, 356, 812, 78, 651, 1809, 199, 0, 0, 0, 0, 0, 0, 0, 0, 200, 198, 2302, 285, 535, 232, 434, 213, 675, 0, 0, 0, 0, 0, 0, 0, 314, 1637, 1170, 363, 0, 2377, 2547, 368, 0, 0, 0, 0, 0, 0, 0, 0, 69, 454, 145, 537, 39, 586, 452, 454, 930, 346, 0, 0, 0, 0, 0, 0, 257, 851, 164, 1264, 1933, 1186, 610, 217, 242, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7426, 4691, 6714, 5734, 5074, 8776, 4012, 6725] avg=6144.0000 max=8776 max_rank=5 min=4012 min_rank=6 imbalance_ratio=1.428385 max_load_reduction=40376 max_load_reduction_pct=0.821452 moved_to_echo=24587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[19, 66, 120, 448, 1456, 617, 756, 1074, 1706, 0, 0, 0, 0, 0, 0, 0, 654, 1280, 785, 1851, 329, 465, 814, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 2601, 1141, 422, 68, 370, 518, 479, 272, 128, 69, 0, 0, 0, 0, 0, 0, 1517, 261, 736, 502, 812, 512, 1776, 120, 0, 0, 0, 0, 0, 0, 0, 0, 671, 111, 153, 497, 228, 628, 1089, 127, 997, 0, 0, 0, 0, 0, 0, 0, 117, 786, 159, 1537, 0, 983, 1434, 313, 0, 0, 0, 0, 0, 0, 0, 0, 822, 575, 55, 2560, 125, 155, 339, 951, 1373, 662, 0, 0, 0, 0, 0, 0, 92, 422, 433, 1706, 432, 708, 1222, 442, 116, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6262, 7565, 6068, 6236, 4501, 5329, 7617, 5574] avg=6144.0000 max=7617 max_rank=6 min=4501 min_rank=4 imbalance_ratio=1.239746 max_load_reduction=41535 max_load_reduction_pct=0.845032 moved_to_echo=23021 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[45, 136, 251, 290, 872, 909, 46, 783, 1645, 0, 0, 0, 0, 0, 0, 0, 423, 2633, 520, 416, 1068, 259, 570, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1583, 273, 727, 470, 613, 427, 793, 2115, 152, 45, 0, 0, 0, 0, 0, 0, 1782, 935, 1403, 370, 272, 686, 1325, 137, 0, 0, 0, 0, 0, 0, 0, 0, 522, 289, 1072, 801, 175, 818, 682, 105, 588, 0, 0, 0, 0, 0, 0, 0, 418, 896, 429, 332, 0, 1898, 919, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 134, 641, 425, 1256, 149, 184, 536, 475, 808, 637, 0, 0, 0, 0, 0, 0, 1374, 248, 397, 440, 1364, 1134, 1072, 511, 137, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4977, 6892, 7198, 6910, 5052, 6196, 5245, 6682] avg=6144.0000 max=7198 max_rank=2 min=4977 min_rank=0 imbalance_ratio=1.171549 max_load_reduction=41954 max_load_reduction_pct=0.853556 moved_to_echo=23175 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3, 290, 88, 249, 1523, 1045, 1404, 614, 1141, 0, 0, 0, 0, 0, 0, 0, 487, 1580, 909, 2133, 146, 212, 92, 613, 0, 0, 0, 0, 0, 0, 0, 0, 1456, 1151, 420, 217, 250, 168, 1237, 418, 195, 31, 0, 0, 0, 0, 0, 0, 752, 1146, 1305, 528, 178, 140, 1728, 91, 0, 0, 0, 0, 0, 0, 0, 0, 743, 525, 703, 1766, 650, 815, 803, 31, 454, 0, 0, 0, 0, 0, 0, 0, 233, 1190, 368, 977, 0, 2156, 1526, 911, 0, 0, 0, 0, 0, 0, 0, 0, 625, 128, 186, 2012, 288, 210, 135, 708, 625, 441, 0, 0, 0, 0, 0, 0, 132, 335, 397, 1187, 1880, 656, 578, 661, 176, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6357, 6172, 5543, 5868, 6490, 7361, 5358, 6003] avg=6144.0000 max=7361 max_rank=5 min=5358 min_rank=6 imbalance_ratio=1.198079 max_load_reduction=41791 max_load_reduction_pct=0.850240 moved_to_echo=25212 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 11776, 4480, 8576, 2944, 2176, 4096, 9856, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10496, 4352, 6400, 4736, 2304, 3712, 15232, 2432, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3328, 10240, 3072, 4096, 0, 13056, 10240, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 2048, 1536, 2816, 12288, 6400, 4992, 5888, 13440, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 2560, 9088, 9088, 2816, 7552, 6400, 640, 4736, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 11776, 4480, 8576, 2944, 2176, 4096, 9856, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 3968, 2176, 12800, 3584, 3072, 2816, 6144, 6528, 5248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 2048, 1536, 2816, 12288, 6400, 4992, 5888, 13440, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 3840, 2816, 7040, 12288, 5248, 10112, 3712, 1536, 128, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[14208, 5632, 4224, 2176, 3840, 3200, 6528, 7808, 1664, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[897, 836, 1547, 1178, 1184, 280, 2509, 963, 206, 2031, 1307, 45, 276, 228, 1084, 808, 216, 485, 2224, 15, 2224, 1122, 1387, 145, 322, 293, 177, 642, 306, 437, 120, 332, 911, 200, 387, 604, 516, 557, 103, 887, 90, 176, 353, 13, 1382, 46, 1413, 672, 564, 1184, 233, 497, 500, 23, 2158, 561, 1075, 1727, 110, 3601, 896, 621, 1127, 139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[774, 399, 1942, 2779, 763, 150, 804, 2587, 109, 1850, 183, 164, 1799, 190, 1568, 1474, 35, 391, 1714, 260, 678, 902, 318, 245, 627, 38, 220, 1539, 343, 751, 390, 523, 2281, 264, 160, 156, 577, 867, 9, 2060, 542, 625, 284, 66, 1036, 218, 755, 547, 356, 1070, 87, 537, 666, 13, 2014, 1016, 101, 358, 104, 1966, 62, 94, 1650, 2102] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[1190, 527, 3546, 1032, 292, 546, 2702, 1249, 186, 2121, 751, 194, 262, 516, 273, 184, 33, 518, 775, 48, 593, 1192, 241, 776, 450, 79, 967, 1216, 883, 286, 296, 279, 679, 1281, 63, 315, 851, 995, 212, 4183, 549, 766, 1685, 164, 904, 215, 578, 200, 30, 813, 176, 1073, 807, 474, 1679, 547, 32, 192, 987, 1609, 983, 757, 1007, 143] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[788, 2534, 4631, 1148, 847, 338, 763, 447, 365, 1263, 680, 496, 423, 279, 119, 538, 21, 646, 674, 972, 1402, 1107, 933, 405, 249, 39, 69, 3161, 325, 233, 538, 880, 1123, 2541, 50, 149, 52, 1084, 57, 1625, 486, 439, 642, 12, 951, 149, 1649, 189, 205, 1213, 42, 1599, 89, 32, 1744, 606, 125, 403, 146, 2652, 613, 934, 171, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[463, 470, 2682, 911, 224, 1392, 2190, 861, 89, 1843, 1114, 115, 318, 841, 856, 1321, 331, 308, 359, 267, 1330, 2640, 1607, 240, 155, 53, 55, 1503, 323, 390, 108, 512, 1814, 1739, 16, 932, 166, 988, 97, 1697, 191, 404, 669, 9, 1203, 668, 811, 649, 273, 858, 95, 1435, 997, 18, 1427, 1480, 58, 93, 292, 3218, 83, 310, 84, 507] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[305, 2006, 1080, 838, 902, 635, 1370, 2339, 612, 2109, 788, 54, 333, 341, 710, 1481, 1021, 271, 1411, 146, 478, 685, 577, 238, 449, 169, 234, 962, 370, 1162, 313, 179, 978, 2311, 796, 443, 373, 433, 690, 1198, 1105, 272, 194, 73, 749, 32, 1583, 604, 1992, 774, 510, 188, 580, 9, 810, 752, 472, 1106, 230, 1897, 894, 2117, 122, 297] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[619, 631, 3329, 816, 273, 1020, 922, 1363, 739, 2213, 387, 71, 687, 192, 689, 778, 226, 274, 543, 144, 2239, 1991, 2019, 300, 329, 49, 356, 2712, 673, 642, 585, 1160, 1662, 151, 3, 5, 231, 1004, 49, 3171, 179, 258, 442, 35, 1744, 103, 678, 239, 283, 442, 164, 221, 338, 5, 1891, 730, 29, 441, 785, 3316, 224, 429, 418, 511] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[1292, 1398, 4306, 1064, 15, 1545, 1516, 1097, 44, 2354, 869, 7, 241, 776, 237, 516, 73, 1246, 879, 434, 747, 639, 2149, 980, 157, 5, 726, 1871, 1226, 189, 137, 744, 1013, 443, 44, 31, 24, 344, 16, 2852, 35, 96, 715, 306, 663, 56, 2269, 102, 404, 1732, 30, 1606, 140, 34, 2163, 654, 49, 42, 810, 1635, 137, 593, 208, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[305, 2006, 1080, 838, 902, 635, 1370, 2339, 612, 2109, 788, 54, 333, 341, 710, 1481, 1021, 271, 1411, 146, 478, 685, 577, 238, 449, 169, 234, 962, 370, 1162, 313, 179, 978, 2311, 796, 443, 373, 433, 690, 1198, 1105, 272, 194, 73, 749, 32, 1583, 604, 1992, 774, 510, 188, 580, 9, 810, 752, 472, 1106, 230, 1897, 894, 2117, 122, 297] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[774, 399, 1942, 2779, 763, 150, 804, 2587, 109, 1850, 183, 164, 1799, 190, 1568, 1474, 35, 391, 1714, 260, 678, 902, 318, 245, 627, 38, 220, 1539, 343, 751, 390, 523, 2281, 264, 160, 156, 577, 867, 9, 2060, 542, 625, 284, 66, 1036, 218, 755, 547, 356, 1070, 87, 537, 666, 13, 2014, 1016, 101, 358, 104, 1966, 62, 94, 1650, 2102] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[463, 470, 2682, 911, 224, 1392, 2190, 861, 89, 1843, 1114, 115, 318, 841, 856, 1321, 331, 308, 359, 267, 1330, 2640, 1607, 240, 155, 53, 55, 1503, 323, 390, 108, 512, 1814, 1739, 16, 932, 166, 988, 97, 1697, 191, 404, 669, 9, 1203, 668, 811, 649, 273, 858, 95, 1435, 997, 18, 1427, 1480, 58, 93, 292, 3218, 83, 310, 84, 507] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[788, 2534, 4631, 1148, 847, 338, 763, 447, 365, 1263, 680, 496, 423, 279, 119, 538, 21, 646, 674, 972, 1402, 1107, 933, 405, 249, 39, 69, 3161, 325, 233, 538, 880, 1123, 2541, 50, 149, 52, 1084, 57, 1625, 486, 439, 642, 12, 951, 149, 1649, 189, 205, 1213, 42, 1599, 89, 32, 1744, 606, 125, 403, 146, 2652, 613, 934, 171, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[619, 631, 0, 816, 273, 1020, 214, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 739, 2213, 387, 71, 687, 192, 689, 778, 347, 109, 0, 0, 0, 0, 0, 0, 226, 274, 543, 144, 2239, 1926, 2019, 300, 0, 0, 0, 0, 0, 0, 0, 0, 329, 49, 356, 2712, 673, 642, 585, 1160, 1276, 346, 0, 0, 0, 0, 0, 0, 1662, 151, 3, 5, 231, 1004, 49, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 179, 258, 442, 35, 1744, 103, 678, 239, 2053, 0, 0, 0, 0, 0, 0, 0, 283, 442, 164, 221, 338, 5, 1891, 730, 253, 0, 0, 0, 0, 0, 0, 0, 29, 441, 785, 3316, 224, 429, 418, 511, 76, 65, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4936, 6212, 7671, 8128, 5853, 5731, 4327, 6294] avg=6144.0000 max=8128 max_rank=3 min=4327 min_rank=6 imbalance_ratio=1.322917 max_load_reduction=41024 max_load_reduction_pct=0.834635 moved_to_echo=22205 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[897, 836, 0, 1178, 1184, 280, 580, 963, 0, 0, 0, 0, 0, 0, 0, 0, 206, 2031, 1307, 45, 276, 228, 1084, 808, 97, 298, 0, 0, 0, 0, 0, 0, 216, 485, 2224, 15, 2224, 1086, 1387, 145, 0, 0, 0, 0, 0, 0, 0, 0, 322, 293, 177, 642, 306, 437, 120, 332, 593, 942, 0, 0, 0, 0, 0, 0, 911, 200, 387, 604, 516, 557, 103, 769, 0, 0, 0, 0, 0, 0, 0, 0, 90, 176, 353, 13, 1382, 46, 1413, 672, 954, 0, 0, 0, 0, 0, 0, 0, 564, 1184, 233, 497, 500, 23, 2158, 561, 689, 0, 0, 0, 0, 0, 0, 0, 1075, 1727, 110, 3601, 896, 621, 1127, 139, 21, 36, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5918, 6380, 7782, 4164, 4047, 5099, 6409, 9353] avg=6144.0000 max=9353 max_rank=7 min=4047 min_rank=4 imbalance_ratio=1.522298 max_load_reduction=39799 max_load_reduction_pct=0.809713 moved_to_echo=24908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1292, 1398, 0, 1064, 15, 1545, 351, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2354, 869, 7, 241, 776, 237, 516, 312, 180, 0, 0, 0, 0, 0, 0, 73, 1246, 879, 434, 747, 619, 2149, 980, 0, 0, 0, 0, 0, 0, 0, 0, 157, 5, 726, 1871, 1226, 189, 137, 744, 1651, 569, 0, 0, 0, 0, 0, 0, 1013, 443, 44, 31, 24, 344, 16, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 35, 96, 715, 306, 663, 56, 2269, 102, 2655, 0, 0, 0, 0, 0, 0, 0, 404, 1732, 30, 1606, 140, 34, 2163, 654, 416, 0, 0, 0, 0, 0, 0, 0, 49, 42, 810, 1635, 137, 593, 208, 427, 68, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6762, 5536, 7127, 7275, 4387, 6897, 7179, 3989] avg=6144.0000 max=7275 max_rank=3 min=3989 min_rank=7 imbalance_ratio=1.184082 max_load_reduction=41877 max_load_reduction_pct=0.851990 moved_to_echo=22452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[774, 399, 0, 2779, 763, 150, 187, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1850, 183, 164, 1799, 190, 1568, 1474, 225, 95, 0, 0, 0, 0, 0, 0, 35, 391, 1714, 260, 678, 873, 318, 245, 0, 0, 0, 0, 0, 0, 0, 0, 627, 38, 220, 1539, 343, 751, 390, 523, 745, 302, 0, 0, 0, 0, 0, 0, 2281, 264, 160, 156, 577, 867, 9, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 542, 625, 284, 66, 1036, 218, 755, 547, 1197, 0, 0, 0, 0, 0, 0, 0, 356, 1070, 87, 537, 666, 13, 2014, 1016, 220, 0, 0, 0, 0, 0, 0, 0, 101, 358, 104, 1966, 62, 94, 1650, 2102, 49, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7639, 7657, 4514, 5478, 6100, 5270, 5979, 6515] avg=6144.0000 max=7657 max_rank=1 min=4514 min_rank=2 imbalance_ratio=1.246257 max_load_reduction=41495 max_load_reduction_pct=0.844218 moved_to_echo=23864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1190, 527, 0, 1032, 292, 546, 624, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 186, 2121, 751, 194, 262, 516, 273, 184, 457, 321, 0, 0, 0, 0, 0, 0, 33, 518, 775, 48, 593, 1153, 241, 776, 0, 0, 0, 0, 0, 0, 0, 0, 450, 79, 967, 1216, 883, 286, 296, 279, 1359, 1015, 0, 0, 0, 0, 0, 0, 679, 1281, 63, 315, 851, 995, 212, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 549, 766, 1685, 164, 904, 215, 578, 200, 2187, 0, 0, 0, 0, 0, 0, 0, 30, 813, 176, 1073, 807, 474, 1679, 547, 742, 0, 0, 0, 0, 0, 0, 0, 32, 192, 987, 1609, 983, 757, 1007, 143, 101, 39, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5460, 5265, 4137, 6830, 8021, 7248, 6341, 5850] avg=6144.0000 max=8021 max_rank=4 min=4137 min_rank=2 imbalance_ratio=1.305501 max_load_reduction=41131 max_load_reduction_pct=0.836812 moved_to_echo=27460 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[305, 2006, 0, 838, 902, 635, 304, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 612, 2109, 788, 54, 333, 341, 710, 1481, 134, 168, 0, 0, 0, 0, 0, 0, 1021, 271, 1411, 146, 478, 659, 577, 238, 0, 0, 0, 0, 0, 0, 0, 0, 449, 169, 234, 962, 370, 1162, 313, 179, 415, 519, 0, 0, 0, 0, 0, 0, 978, 2311, 796, 443, 373, 433, 690, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 272, 194, 73, 749, 32, 1583, 604, 665, 0, 0, 0, 0, 0, 0, 0, 1992, 774, 510, 188, 580, 9, 810, 752, 379, 0, 0, 0, 0, 0, 0, 0, 472, 1106, 230, 1897, 894, 2117, 122, 297, 32, 26, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7329, 6730, 4801, 4772, 7056, 5277, 5994, 7193] avg=6144.0000 max=7329 max_rank=0 min=4772 min_rank=3 imbalance_ratio=1.192871 max_load_reduction=41823 max_load_reduction_pct=0.850891 moved_to_echo=25520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[774, 399, 0, 2779, 763, 150, 187, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1850, 183, 164, 1799, 190, 1568, 1474, 225, 95, 0, 0, 0, 0, 0, 0, 35, 391, 1714, 260, 678, 873, 318, 245, 0, 0, 0, 0, 0, 0, 0, 0, 627, 38, 220, 1539, 343, 751, 390, 523, 745, 302, 0, 0, 0, 0, 0, 0, 2281, 264, 160, 156, 577, 867, 9, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 542, 625, 284, 66, 1036, 218, 755, 547, 1197, 0, 0, 0, 0, 0, 0, 0, 356, 1070, 87, 537, 666, 13, 2014, 1016, 220, 0, 0, 0, 0, 0, 0, 0, 101, 358, 104, 1966, 62, 94, 1650, 2102, 49, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7639, 7657, 4514, 5478, 6100, 5270, 5979, 6515] avg=6144.0000 max=7657 max_rank=1 min=4514 min_rank=2 imbalance_ratio=1.246257 max_load_reduction=41495 max_load_reduction_pct=0.844218 moved_to_echo=23864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[463, 470, 0, 911, 224, 1392, 507, 861, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1843, 1114, 115, 318, 841, 856, 1321, 185, 260, 0, 0, 0, 0, 0, 0, 331, 308, 359, 267, 1330, 2554, 1607, 240, 0, 0, 0, 0, 0, 0, 0, 0, 155, 53, 55, 1503, 323, 390, 108, 512, 1028, 822, 0, 0, 0, 0, 0, 0, 1814, 1739, 16, 932, 166, 988, 97, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 191, 404, 669, 9, 1203, 668, 811, 649, 1654, 0, 0, 0, 0, 0, 0, 0, 273, 858, 95, 1435, 997, 18, 1427, 1480, 601, 0, 0, 0, 0, 0, 0, 0, 58, 93, 292, 3218, 83, 310, 84, 507, 41, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4828, 6942, 6996, 4949, 7223, 6258, 7184, 4772] avg=6144.0000 max=7223 max_rank=4 min=4772 min_rank=7 imbalance_ratio=1.175618 max_load_reduction=41929 max_load_reduction_pct=0.853048 moved_to_echo=25437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[788, 2534, 0, 1148, 847, 338, 178, 447, 0, 0, 0, 0, 0, 0, 0, 0, 365, 1263, 680, 496, 423, 279, 119, 538, 177, 90, 0, 0, 0, 0, 0, 0, 21, 646, 674, 972, 1402, 1071, 933, 405, 0, 0, 0, 0, 0, 0, 0, 0, 249, 39, 69, 3161, 325, 233, 538, 880, 1776, 286, 0, 0, 0, 0, 0, 0, 1123, 2541, 50, 149, 52, 1084, 57, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 486, 439, 642, 12, 951, 149, 1649, 189, 2855, 0, 0, 0, 0, 0, 0, 0, 205, 1213, 42, 1599, 89, 32, 1744, 606, 209, 0, 0, 0, 0, 0, 0, 0, 125, 403, 146, 2652, 613, 934, 171, 67, 39, 36, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6280, 4430, 6124, 7556, 6465, 7372, 5739, 5186] avg=6144.0000 max=7556 max_rank=3 min=4430 min_rank=1 imbalance_ratio=1.229818 max_load_reduction=41596 max_load_reduction_pct=0.846273 moved_to_echo=24762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[619, 631, 0, 816, 273, 1020, 214, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 739, 2213, 387, 71, 687, 192, 689, 778, 347, 109, 0, 0, 0, 0, 0, 0, 226, 274, 543, 144, 2239, 1926, 2019, 300, 0, 0, 0, 0, 0, 0, 0, 0, 329, 49, 356, 2712, 673, 642, 585, 1160, 1276, 346, 0, 0, 0, 0, 0, 0, 1662, 151, 3, 5, 231, 1004, 49, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 179, 258, 442, 35, 1744, 103, 678, 239, 2053, 0, 0, 0, 0, 0, 0, 0, 283, 442, 164, 221, 338, 5, 1891, 730, 253, 0, 0, 0, 0, 0, 0, 0, 29, 441, 785, 3316, 224, 429, 418, 511, 76, 65, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4936, 6212, 7671, 8128, 5853, 5731, 4327, 6294] avg=6144.0000 max=8128 max_rank=3 min=4327 min_rank=6 imbalance_ratio=1.322917 max_load_reduction=41024 max_load_reduction_pct=0.834635 moved_to_echo=22205 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[305, 2006, 0, 838, 902, 635, 304, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 612, 2109, 788, 54, 333, 341, 710, 1481, 134, 168, 0, 0, 0, 0, 0, 0, 1021, 271, 1411, 146, 478, 659, 577, 238, 0, 0, 0, 0, 0, 0, 0, 0, 449, 169, 234, 962, 370, 1162, 313, 179, 415, 519, 0, 0, 0, 0, 0, 0, 978, 2311, 796, 443, 373, 433, 690, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 272, 194, 73, 749, 32, 1583, 604, 665, 0, 0, 0, 0, 0, 0, 0, 1992, 774, 510, 188, 580, 9, 810, 752, 379, 0, 0, 0, 0, 0, 0, 0, 472, 1106, 230, 1897, 894, 2117, 122, 297, 32, 26, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7329, 6730, 4801, 4772, 7056, 5277, 5994, 7193] avg=6144.0000 max=7329 max_rank=0 min=4772 min_rank=3 imbalance_ratio=1.192871 max_load_reduction=41823 max_load_reduction_pct=0.850891 moved_to_echo=25520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1292, 1398, 0, 1064, 15, 1545, 351, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2354, 869, 7, 241, 776, 237, 516, 312, 180, 0, 0, 0, 0, 0, 0, 73, 1246, 879, 434, 747, 619, 2149, 980, 0, 0, 0, 0, 0, 0, 0, 0, 157, 5, 726, 1871, 1226, 189, 137, 744, 1651, 569, 0, 0, 0, 0, 0, 0, 1013, 443, 44, 31, 24, 344, 16, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 35, 96, 715, 306, 663, 56, 2269, 102, 2655, 0, 0, 0, 0, 0, 0, 0, 404, 1732, 30, 1606, 140, 34, 2163, 654, 416, 0, 0, 0, 0, 0, 0, 0, 49, 42, 810, 1635, 137, 593, 208, 427, 68, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6762, 5536, 7127, 7275, 4387, 6897, 7179, 3989] avg=6144.0000 max=7275 max_rank=3 min=3989 min_rank=7 imbalance_ratio=1.184082 max_load_reduction=41877 max_load_reduction_pct=0.851990 moved_to_echo=22452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 8192, 1408, 7168, 4224, 640, 13952, 6400, 3584, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 768, 2816, 13696, 4480, 4096, 2560, 4736, 8960, 4864, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2432, 15872, 6144, 1152, 4352, 3456, 5632, 7168, 2048, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 8832, 0, 9856, 4608, 6016, 3072, 11008, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 8192, 1408, 7168, 4224, 640, 13952, 6400, 3584, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2048, 4480, 3584, 19968, 3968, 5888, 4864, 4224, 512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 8832, 0, 9856, 4608, 6016, 3072, 11008, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2432, 15872, 6144, 1152, 4352, 3456, 5632, 7168, 2048, 1536, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[10496, 8960, 1536, 2688, 2816, 6272, 1280, 15360, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 3072, 4992, 768, 8704, 1536, 9856, 3328, 14336, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 4224, 8704, 2304, 9728, 9984, 9344, 3456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[864, 1466, 189, 950, 41, 38, 1822, 2309, 418, 1590, 1019, 174, 860, 1303, 1298, 175, 80, 2815, 72, 105, 1923, 854, 740, 2223, 2419, 525, 608, 229, 951, 288, 165, 273, 759, 346, 457, 1212, 847, 230, 2343, 401, 164, 611, 192, 272, 2644, 894, 687, 2224, 230, 54, 635, 313, 123, 158, 8, 773, 1435, 753, 195, 298, 12, 46, 47, 1003] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[449, 803, 72, 266, 43, 23, 878, 870, 179, 2839, 1939, 1631, 1304, 2403, 989, 1019, 160, 3960, 71, 607, 1073, 175, 321, 472, 719, 1154, 529, 157, 227, 355, 220, 313, 977, 551, 821, 784, 181, 705, 3642, 1296, 378, 37, 331, 265, 934, 976, 595, 1318, 1564, 39, 2054, 1733, 67, 153, 17, 1546, 342, 678, 564, 130, 12, 24, 208, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[626, 719, 273, 122, 61, 63, 1159, 1771, 562, 415, 2707, 592, 1890, 2541, 1566, 1502, 261, 3199, 47, 1078, 2551, 164, 411, 1466, 655, 33, 1230, 352, 582, 706, 583, 121, 629, 195, 808, 336, 60, 605, 775, 815, 1193, 729, 197, 94, 2025, 1317, 72, 2464, 1319, 114, 173, 165, 30, 124, 98, 613, 1335, 1168, 148, 232, 81, 71, 500, 659] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[428, 873, 118, 723, 435, 172, 2110, 1749, 576, 903, 2075, 1257, 437, 2702, 3189, 329, 396, 1280, 143, 96, 1785, 227, 707, 2257, 101, 212, 1032, 204, 41, 276, 19, 466, 1632, 267, 779, 1009, 10, 385, 650, 373, 222, 291, 462, 183, 880, 665, 571, 1140, 1500, 120, 1614, 2047, 979, 723, 412, 1384, 1095, 609, 487, 437, 38, 59, 626, 185] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[1104, 3380, 124, 351, 239, 140, 194, 714, 742, 1021, 662, 600, 2061, 2759, 2504, 3718, 30, 1615, 373, 135, 1188, 90, 325, 1475, 1120, 595, 1238, 488, 444, 735, 320, 170, 464, 386, 1354, 929, 482, 444, 1835, 1860, 360, 496, 477, 28, 2183, 492, 159, 112, 266, 164, 482, 293, 94, 221, 239, 140, 875, 463, 317, 700, 117, 356, 419, 1291] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[428, 610, 132, 73, 1049, 82, 195, 2150, 534, 1369, 875, 252, 508, 1735, 1558, 715, 110, 907, 259, 647, 1625, 902, 964, 1559, 689, 219, 1924, 393, 481, 57, 137, 615, 192, 480, 482, 1226, 710, 1263, 3434, 1128, 480, 284, 536, 678, 2396, 418, 41, 1549, 916, 27, 780, 800, 473, 115, 147, 1196, 2838, 588, 1151, 111, 87, 291, 304, 278] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[449, 803, 72, 266, 43, 23, 878, 870, 179, 2839, 1939, 1631, 1304, 2403, 989, 1019, 160, 3960, 71, 607, 1073, 175, 321, 472, 719, 1154, 529, 157, 227, 355, 220, 313, 977, 551, 821, 784, 181, 705, 3642, 1296, 378, 37, 331, 265, 934, 976, 595, 1318, 1564, 39, 2054, 1733, 67, 153, 17, 1546, 342, 678, 564, 130, 12, 24, 208, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[696, 509, 262, 133, 893, 199, 450, 359, 88, 3902, 314, 289, 1234, 391, 1513, 2538, 202, 2930, 415, 490, 2300, 113, 620, 2167, 1222, 983, 1930, 317, 246, 147, 67, 254, 829, 423, 1302, 1373, 301, 1063, 463, 1597, 479, 42, 106, 716, 1126, 1145, 94, 1159, 835, 9, 2604, 768, 417, 955, 15, 748, 368, 836, 207, 67, 49, 97, 266, 520] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[428, 873, 118, 723, 435, 172, 2110, 1749, 576, 903, 2075, 1257, 437, 2702, 3189, 329, 396, 1280, 143, 96, 1785, 227, 707, 2257, 101, 212, 1032, 204, 41, 276, 19, 466, 1632, 267, 779, 1009, 10, 385, 650, 373, 222, 291, 462, 183, 880, 665, 571, 1140, 1500, 120, 1614, 2047, 979, 723, 412, 1384, 1095, 609, 487, 437, 38, 59, 626, 185] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[864, 1466, 189, 950, 41, 38, 1822, 2309, 418, 1590, 1019, 174, 860, 1303, 1298, 175, 80, 2815, 72, 105, 1923, 854, 740, 2223, 2419, 525, 608, 229, 951, 288, 165, 273, 759, 346, 457, 1212, 847, 230, 2343, 401, 164, 611, 192, 272, 2644, 894, 687, 2224, 230, 54, 635, 313, 123, 158, 8, 773, 1435, 753, 195, 298, 12, 46, 47, 1003] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[498, 2405, 168, 213, 527, 197, 1980, 1841, 366, 1443, 430, 1068, 1154, 2587, 1551, 1553, 53, 974, 157, 524, 943, 221, 610, 289, 454, 650, 3069, 200, 846, 68, 2263, 272, 1279, 9, 343, 1365, 138, 1143, 499, 279, 38, 463, 401, 197, 709, 320, 1064, 270, 1702, 494, 1141, 494, 149, 106, 73, 1724, 1253, 1431, 1306, 196, 13, 360, 317, 302] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[1104, 3380, 124, 351, 239, 140, 194, 714, 742, 1021, 662, 600, 2061, 2759, 2504, 3718, 30, 1615, 373, 135, 1188, 90, 325, 1475, 1120, 595, 1238, 488, 444, 735, 320, 170, 464, 386, 1354, 929, 482, 444, 1835, 1860, 360, 496, 477, 28, 2183, 492, 159, 112, 266, 164, 482, 293, 94, 221, 239, 140, 875, 463, 317, 700, 117, 356, 419, 1291] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[626, 719, 273, 122, 61, 63, 1159, 1771, 134, 0, 0, 0, 0, 0, 0, 0, 562, 272, 2707, 592, 1890, 0, 0, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 261, 1792, 47, 1078, 2551, 164, 411, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 655, 33, 1230, 352, 582, 706, 583, 121, 1193, 0, 0, 0, 0, 0, 0, 0, 629, 195, 808, 336, 60, 605, 503, 815, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 729, 197, 94, 2025, 1317, 72, 2464, 272, 9, 0, 0, 0, 0, 0, 0, 1319, 114, 173, 165, 30, 124, 98, 613, 1407, 211, 0, 0, 0, 0, 0, 0, 1335, 1168, 148, 232, 81, 71, 500, 659, 2541, 162, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4928, 7525, 7770, 5455, 3951, 8372, 4254, 6897] avg=6144.0000 max=8372 max_rank=5 min=3951 min_rank=4 imbalance_ratio=1.362630 max_load_reduction=40780 max_load_reduction_pct=0.829671 moved_to_echo=23474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1104, 3380, 124, 351, 239, 140, 194, 714, 331, 0, 0, 0, 0, 0, 0, 0, 742, 667, 662, 600, 2061, 0, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 30, 905, 373, 135, 1188, 90, 325, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 595, 1238, 488, 444, 735, 320, 170, 1907, 0, 0, 0, 0, 0, 0, 0, 464, 386, 1354, 929, 482, 444, 1189, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 360, 496, 477, 28, 2183, 492, 159, 112, 646, 23, 0, 0, 0, 0, 0, 0, 266, 164, 482, 293, 94, 221, 239, 140, 710, 336, 0, 0, 0, 0, 0, 0, 875, 463, 317, 700, 117, 356, 419, 1291, 2759, 261, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6577, 8450, 4521, 7017, 7108, 4976, 2945, 7558] avg=6144.0000 max=8450 max_rank=1 min=2945 min_rank=6 imbalance_ratio=1.375326 max_load_reduction=40702 max_load_reduction_pct=0.828084 moved_to_echo=22587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[449, 803, 72, 266, 43, 23, 878, 870, 920, 0, 0, 0, 0, 0, 0, 0, 179, 1855, 1939, 1631, 1304, 0, 0, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 160, 2218, 71, 607, 1073, 175, 321, 472, 0, 0, 0, 0, 0, 0, 0, 0, 719, 1154, 529, 157, 227, 355, 220, 313, 753, 0, 0, 0, 0, 0, 0, 0, 977, 551, 821, 784, 181, 705, 2360, 1296, 0, 0, 0, 0, 0, 0, 0, 0, 378, 37, 331, 265, 934, 976, 595, 1318, 1282, 64, 0, 0, 0, 0, 0, 0, 1564, 39, 2054, 1733, 67, 153, 17, 1546, 1742, 132, 0, 0, 0, 0, 0, 0, 342, 678, 564, 130, 12, 24, 208, 10, 2403, 104, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4324, 7927, 5097, 4427, 7675, 6180, 9047, 4475] avg=6144.0000 max=9047 max_rank=6 min=4324 min_rank=0 imbalance_ratio=1.472493 max_load_reduction=40105 max_load_reduction_pct=0.815938 moved_to_echo=27377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[428, 610, 132, 73, 1049, 82, 195, 2150, 443, 0, 0, 0, 0, 0, 0, 0, 534, 896, 875, 252, 508, 0, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 110, 508, 259, 647, 1625, 902, 964, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 689, 219, 1924, 393, 481, 57, 137, 615, 1186, 0, 0, 0, 0, 0, 0, 0, 192, 480, 482, 1226, 710, 1263, 2225, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 480, 284, 536, 678, 2396, 418, 41, 1549, 1209, 30, 0, 0, 0, 0, 0, 0, 916, 27, 780, 800, 473, 115, 147, 1196, 399, 209, 0, 0, 0, 0, 0, 0, 2838, 588, 1151, 111, 87, 291, 304, 278, 1735, 163, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5162, 3780, 6574, 5701, 7706, 7621, 5062, 7546] avg=6144.0000 max=7706 max_rank=4 min=3780 min_rank=1 imbalance_ratio=1.254232 max_load_reduction=41446 max_load_reduction_pct=0.843221 moved_to_echo=27935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[428, 873, 118, 723, 435, 172, 2110, 1749, 292, 0, 0, 0, 0, 0, 0, 0, 576, 591, 2075, 1257, 437, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 396, 717, 143, 96, 1785, 227, 707, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 101, 212, 1032, 204, 41, 276, 19, 466, 2431, 0, 0, 0, 0, 0, 0, 0, 1632, 267, 779, 1009, 10, 385, 422, 373, 0, 0, 0, 0, 0, 0, 0, 0, 222, 291, 462, 183, 880, 665, 571, 1140, 228, 20, 0, 0, 0, 0, 0, 0, 1500, 120, 1614, 2047, 979, 723, 412, 1384, 563, 428, 0, 0, 0, 0, 0, 0, 1095, 609, 487, 437, 38, 59, 626, 185, 2702, 330, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6900, 5265, 6328, 4782, 4877, 4662, 9770, 6568] avg=6144.0000 max=9770 max_rank=6 min=4662 min_rank=5 imbalance_ratio=1.590169 max_load_reduction=39382 max_load_reduction_pct=0.801229 moved_to_echo=25877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[864, 1466, 189, 950, 41, 38, 1822, 2309, 515, 0, 0, 0, 0, 0, 0, 0, 418, 1040, 1019, 174, 860, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 80, 1577, 72, 105, 1923, 854, 740, 2223, 0, 0, 0, 0, 0, 0, 0, 0, 2419, 525, 608, 229, 951, 288, 165, 273, 988, 0, 0, 0, 0, 0, 0, 0, 759, 346, 457, 1212, 847, 230, 1519, 401, 0, 0, 0, 0, 0, 0, 0, 0, 164, 611, 192, 272, 2644, 894, 687, 2224, 824, 35, 0, 0, 0, 0, 0, 0, 230, 54, 635, 313, 123, 158, 8, 773, 1238, 176, 0, 0, 0, 0, 0, 0, 1435, 753, 195, 298, 12, 46, 47, 1003, 1303, 134, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 3686, 7574, 6446, 5771, 8547, 3708, 5226] avg=6144.0000 max=8547 max_rank=5 min=3686 min_rank=1 imbalance_ratio=1.391113 max_load_reduction=40605 max_load_reduction_pct=0.826111 moved_to_echo=23252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[626, 719, 273, 122, 61, 63, 1159, 1771, 134, 0, 0, 0, 0, 0, 0, 0, 562, 272, 2707, 592, 1890, 0, 0, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 261, 1792, 47, 1078, 2551, 164, 411, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 655, 33, 1230, 352, 582, 706, 583, 121, 1193, 0, 0, 0, 0, 0, 0, 0, 629, 195, 808, 336, 60, 605, 503, 815, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 729, 197, 94, 2025, 1317, 72, 2464, 272, 9, 0, 0, 0, 0, 0, 0, 1319, 114, 173, 165, 30, 124, 98, 613, 1407, 211, 0, 0, 0, 0, 0, 0, 1335, 1168, 148, 232, 81, 71, 500, 659, 2541, 162, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4928, 7525, 7770, 5455, 3951, 8372, 4254, 6897] avg=6144.0000 max=8372 max_rank=5 min=3951 min_rank=4 imbalance_ratio=1.362630 max_load_reduction=40780 max_load_reduction_pct=0.829671 moved_to_echo=23474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[498, 2405, 168, 213, 527, 197, 1980, 1841, 472, 0, 0, 0, 0, 0, 0, 0, 366, 935, 430, 1068, 1154, 0, 0, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 53, 543, 157, 524, 943, 221, 610, 289, 0, 0, 0, 0, 0, 0, 0, 0, 454, 650, 3069, 200, 846, 68, 2263, 272, 1183, 0, 0, 0, 0, 0, 0, 0, 1279, 9, 343, 1365, 138, 1143, 320, 279, 0, 0, 0, 0, 0, 0, 0, 0, 38, 463, 401, 197, 709, 320, 1064, 270, 179, 36, 0, 0, 0, 0, 0, 0, 1702, 494, 1141, 494, 149, 106, 73, 1724, 431, 208, 0, 0, 0, 0, 0, 0, 1253, 1431, 1306, 196, 13, 360, 317, 302, 2587, 160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 5506, 3340, 9005, 4876, 3677, 6522, 7925] avg=6144.0000 max=9005 max_rank=3 min=3340 min_rank=2 imbalance_ratio=1.465658 max_load_reduction=40147 max_load_reduction_pct=0.816793 moved_to_echo=23000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[696, 509, 262, 133, 893, 199, 450, 359, 1265, 0, 0, 0, 0, 0, 0, 0, 88, 2550, 314, 289, 1234, 0, 0, 2538, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1641, 415, 490, 2300, 113, 620, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 983, 1930, 317, 246, 147, 67, 254, 1153, 0, 0, 0, 0, 0, 0, 0, 829, 423, 1302, 1373, 301, 1063, 300, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 479, 42, 106, 716, 1126, 1145, 94, 1159, 163, 87, 0, 0, 0, 0, 0, 0, 835, 9, 2604, 768, 417, 955, 15, 748, 1289, 204, 0, 0, 0, 0, 0, 0, 368, 836, 207, 67, 49, 97, 266, 520, 391, 156, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4766, 7013, 7948, 6319, 7188, 5117, 7844, 2957] avg=6144.0000 max=7948 max_rank=2 min=2957 min_rank=7 imbalance_ratio=1.293620 max_load_reduction=41204 max_load_reduction_pct=0.838298 moved_to_echo=23106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1104, 3380, 124, 351, 239, 140, 194, 714, 331, 0, 0, 0, 0, 0, 0, 0, 742, 667, 662, 600, 2061, 0, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 30, 905, 373, 135, 1188, 90, 325, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 595, 1238, 488, 444, 735, 320, 170, 1907, 0, 0, 0, 0, 0, 0, 0, 464, 386, 1354, 929, 482, 444, 1189, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 360, 496, 477, 28, 2183, 492, 159, 112, 646, 23, 0, 0, 0, 0, 0, 0, 266, 164, 482, 293, 94, 221, 239, 140, 710, 336, 0, 0, 0, 0, 0, 0, 875, 463, 317, 700, 117, 356, 419, 1291, 2759, 261, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6577, 8450, 4521, 7017, 7108, 4976, 2945, 7558] avg=6144.0000 max=8450 max_rank=1 min=2945 min_rank=6 imbalance_ratio=1.375326 max_load_reduction=40702 max_load_reduction_pct=0.828084 moved_to_echo=22587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[498, 2405, 168, 213, 527, 197, 1980, 1841, 472, 0, 0, 0, 0, 0, 0, 0, 366, 935, 430, 1068, 1154, 0, 0, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 53, 543, 157, 524, 943, 221, 610, 289, 0, 0, 0, 0, 0, 0, 0, 0, 454, 650, 3069, 200, 846, 68, 2263, 272, 1183, 0, 0, 0, 0, 0, 0, 0, 1279, 9, 343, 1365, 138, 1143, 320, 279, 0, 0, 0, 0, 0, 0, 0, 0, 38, 463, 401, 197, 709, 320, 1064, 270, 179, 36, 0, 0, 0, 0, 0, 0, 1702, 494, 1141, 494, 149, 106, 73, 1724, 431, 208, 0, 0, 0, 0, 0, 0, 1253, 1431, 1306, 196, 13, 360, 317, 302, 2587, 160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 5506, 3340, 9005, 4876, 3677, 6522, 7925] avg=6144.0000 max=9005 max_rank=3 min=3340 min_rank=2 imbalance_ratio=1.465658 max_load_reduction=40147 max_load_reduction_pct=0.816793 moved_to_echo=23000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[864, 1466, 189, 950, 41, 38, 1822, 2309, 515, 0, 0, 0, 0, 0, 0, 0, 418, 1040, 1019, 174, 860, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 80, 1577, 72, 105, 1923, 854, 740, 2223, 0, 0, 0, 0, 0, 0, 0, 0, 2419, 525, 608, 229, 951, 288, 165, 273, 988, 0, 0, 0, 0, 0, 0, 0, 759, 346, 457, 1212, 847, 230, 1519, 401, 0, 0, 0, 0, 0, 0, 0, 0, 164, 611, 192, 272, 2644, 894, 687, 2224, 824, 35, 0, 0, 0, 0, 0, 0, 230, 54, 635, 313, 123, 158, 8, 773, 1238, 176, 0, 0, 0, 0, 0, 0, 1435, 753, 195, 298, 12, 46, 47, 1003, 1303, 134, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 3686, 7574, 6446, 5771, 8547, 3708, 5226] avg=6144.0000 max=8547 max_rank=5 min=3686 min_rank=1 imbalance_ratio=1.391113 max_load_reduction=40605 max_load_reduction_pct=0.826111 moved_to_echo=23252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8448, 1024, 9600, 6656, 2432, 2560, 1024, 8192, 7808, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 3072, 2816, 2560, 12928, 6272, 3328, 10240, 4864, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 2688, 6400, 8320, 2816, 5888, 8960, 7808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3584, 8832, 10112, 5888, 9472, 0, 0, 11648, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 10880, 1408, 2944, 3328, 1024, 8832, 11776, 4480, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8448, 1024, 9600, 6656, 2432, 2560, 1024, 8192, 7808, 1920, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9600, 6528, 4480, 2176, 512, 1408, 2688, 4352, 16512, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 4480, 11648, 2432, 3840, 2688, 3840, 2560, 10880, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 10880, 1408, 2944, 3328, 1024, 8832, 11776, 4480, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 2688, 6400, 8320, 2816, 5888, 8960, 7808, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 9984, 1664, 3712, 13440, 2816, 4736, 12032, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 4480, 11648, 2432, 3840, 2688, 3840, 2560, 10880, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3584, 8832, 10112, 5888, 9472, 0, 0, 11648, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[3343, 741, 7, 1122, 1987, 2297, 239, 110, 373, 269, 201, 343, 2186, 62, 22, 2797, 470, 991, 734, 259, 443, 21, 273, 801, 890, 2107, 1382, 1494, 28, 442, 206, 277, 418, 122, 244, 339, 82, 384, 2318, 2858, 457, 602, 86, 6, 1163, 259, 19, 46, 873, 54, 3138, 489, 314, 1177, 729, 1352, 25, 542, 2015, 338, 61, 272, 1311, 142] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[2074, 483, 687, 1096, 1564, 1182, 314, 128, 109, 874, 287, 561, 1249, 528, 308, 1401, 852, 164, 857, 447, 433, 74, 287, 194, 45, 2239, 283, 1457, 347, 281, 126, 470, 650, 1460, 261, 273, 333, 295, 669, 901, 123, 425, 137, 785, 1329, 1517, 235, 214, 1757, 191, 1591, 1554, 451, 757, 1031, 631, 1325, 1461, 2253, 1321, 341, 1053, 1946, 481] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[2069, 28, 16, 1050, 1613, 531, 441, 48, 653, 95, 179, 596, 371, 101, 236, 3594, 716, 223, 1751, 101, 566, 155, 2173, 290, 203, 1539, 940, 1682, 755, 857, 21, 335, 560, 440, 894, 91, 108, 273, 718, 1086, 383, 709, 207, 560, 383, 1162, 72, 291, 953, 117, 929, 1300, 390, 1235, 1815, 2106, 870, 21, 4511, 371, 202, 426, 1704, 337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[781, 97, 49, 962, 2854, 1143, 194, 72, 598, 212, 30, 815, 1156, 287, 3134, 3325, 729, 188, 399, 173, 1409, 335, 370, 1931, 408, 1779, 920, 1679, 265, 452, 195, 879, 306, 221, 20, 334, 98, 285, 1447, 46, 5, 2731, 479, 109, 1255, 225, 424, 1397, 1003, 1013, 1339, 1291, 744, 1746, 233, 1303, 80, 403, 1209, 249, 74, 196, 413, 654] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[537, 263, 2, 423, 3395, 89, 65, 47, 118, 27, 29, 573, 1793, 44, 191, 1224, 221, 854, 532, 619, 1380, 461, 589, 3893, 395, 2345, 1566, 1199, 182, 144, 657, 1786, 562, 123, 248, 115, 519, 667, 1967, 479, 118, 1318, 350, 160, 614, 395, 365, 538, 1994, 26, 1243, 1866, 897, 577, 441, 1568, 297, 491, 1180, 992, 258, 1496, 1324, 321] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[2074, 483, 687, 1096, 1564, 1182, 314, 128, 109, 874, 287, 561, 1249, 528, 308, 1401, 852, 164, 857, 447, 433, 74, 287, 194, 45, 2239, 283, 1457, 347, 281, 126, 470, 650, 1460, 261, 273, 333, 295, 669, 901, 123, 425, 137, 785, 1329, 1517, 235, 214, 1757, 191, 1591, 1554, 451, 757, 1031, 631, 1325, 1461, 2253, 1321, 341, 1053, 1946, 481] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[1834, 414, 97, 565, 484, 2421, 963, 280, 50, 243, 104, 330, 623, 291, 133, 2020, 142, 990, 3680, 274, 398, 56, 108, 205, 355, 319, 1511, 1166, 338, 169, 45, 234, 1464, 262, 380, 383, 416, 1012, 658, 2662, 115, 720, 361, 17, 540, 122, 723, 946, 666, 371, 2089, 252, 221, 3065, 2983, 1175, 251, 1655, 1546, 41, 1538, 587, 196, 893] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[2414, 228, 82, 71, 2595, 1650, 369, 215, 258, 14, 299, 397, 44, 15, 139, 712, 1252, 297, 433, 290, 1631, 254, 275, 796, 1429, 1124, 934, 318, 886, 427, 111, 906, 786, 144, 30, 183, 76, 438, 2310, 1135, 246, 351, 125, 390, 1018, 15, 183, 1132, 702, 65, 3230, 609, 674, 389, 4366, 2166, 73, 932, 1630, 1147, 447, 848, 2246, 201] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[488, 497, 242, 713, 2004, 312, 187, 199, 646, 30, 51, 697, 767, 152, 1380, 1157, 955, 340, 263, 266, 3349, 647, 446, 1087, 76, 1662, 2854, 2959, 1796, 5, 164, 32, 764, 48, 12, 549, 603, 2224, 335, 542, 47, 685, 299, 419, 480, 217, 226, 1065, 2092, 419, 1341, 1049, 608, 1488, 2103, 1282, 1171, 93, 563, 274, 372, 656, 549, 154] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2074, 483, 687, 1096, 1155, 1182, 314, 128, 0, 0, 0, 0, 0, 0, 0, 0, 109, 874, 287, 561, 1249, 528, 308, 1401, 409, 416, 67, 0, 0, 0, 0, 0, 852, 164, 857, 447, 433, 74, 287, 194, 283, 253, 0, 0, 0, 0, 0, 0, 45, 1570, 283, 1457, 347, 281, 126, 470, 0, 0, 0, 0, 0, 0, 0, 0, 650, 1460, 261, 273, 333, 295, 669, 901, 641, 0, 0, 0, 0, 0, 0, 0, 123, 425, 137, 785, 1329, 1517, 235, 214, 1591, 162, 0, 0, 0, 0, 0, 0, 1757, 191, 0, 1554, 451, 757, 161, 631, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 1461, 1970, 1321, 341, 1053, 1946, 481, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7119, 6209, 3844, 4579, 5483, 6518, 5502, 9898] avg=6144.0000 max=9898 max_rank=7 min=3844 min_rank=2 imbalance_ratio=1.611003 max_load_reduction=39254 max_load_reduction_pct=0.798625 moved_to_echo=27401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[3343, 741, 7, 1122, 1468, 2297, 239, 110, 0, 0, 0, 0, 0, 0, 0, 0, 373, 269, 201, 343, 2186, 62, 22, 2797, 519, 392, 47, 0, 0, 0, 0, 0, 470, 991, 734, 259, 443, 21, 273, 801, 253, 238, 0, 0, 0, 0, 0, 0, 890, 1477, 1382, 1494, 28, 442, 206, 277, 0, 0, 0, 0, 0, 0, 0, 0, 418, 122, 244, 339, 82, 384, 2318, 2858, 453, 0, 0, 0, 0, 0, 0, 0, 457, 602, 86, 6, 1163, 259, 19, 46, 3138, 114, 0, 0, 0, 0, 0, 0, 873, 54, 0, 489, 314, 1177, 115, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 25, 542, 1762, 338, 61, 272, 1311, 142, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9327, 7211, 4483, 6196, 7218, 5890, 4374, 4453] avg=6144.0000 max=9327 max_rank=0 min=4374 min_rank=6 imbalance_ratio=1.518066 max_load_reduction=39825 max_load_reduction_pct=0.810242 moved_to_echo=21935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[781, 97, 49, 962, 2108, 1143, 194, 72, 0, 0, 0, 0, 0, 0, 0, 0, 598, 212, 30, 815, 1156, 287, 3134, 3325, 746, 331, 15, 0, 0, 0, 0, 0, 729, 188, 399, 173, 1409, 335, 370, 1931, 152, 201, 0, 0, 0, 0, 0, 0, 408, 1247, 920, 1679, 265, 452, 195, 879, 0, 0, 0, 0, 0, 0, 0, 0, 306, 221, 20, 334, 98, 285, 1447, 46, 144, 0, 0, 0, 0, 0, 0, 0, 5, 2731, 479, 109, 1255, 225, 424, 1397, 1339, 36, 0, 0, 0, 0, 0, 0, 1003, 1013, 0, 1291, 744, 1746, 38, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 80, 403, 1057, 249, 74, 196, 413, 654, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5406, 10649, 5887, 6045, 2901, 8000, 7138, 3126] avg=6144.0000 max=10649 max_rank=1 min=2901 min_rank=4 imbalance_ratio=1.733236 max_load_reduction=38503 max_load_reduction_pct=0.783346 moved_to_echo=21165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[2414, 228, 82, 71, 2595, 1650, 369, 215, 258, 14, 299, 397, 44, 15, 139, 712, 1252, 297, 433, 290, 1631, 254, 275, 796, 1429, 1124, 934, 318, 886, 427, 111, 906, 786, 144, 30, 183, 76, 438, 2310, 1135, 246, 351, 125, 390, 1018, 15, 183, 1132, 702, 65, 3230, 609, 674, 389, 4366, 2166, 73, 932, 1630, 1147, 447, 848, 2246, 201] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[781, 97, 49, 962, 2854, 1143, 194, 72, 598, 212, 30, 815, 1156, 287, 3134, 3325, 729, 188, 399, 173, 1409, 335, 370, 1931, 408, 1779, 920, 1679, 265, 452, 195, 879, 306, 221, 20, 334, 98, 285, 1447, 46, 5, 2731, 479, 109, 1255, 225, 424, 1397, 1003, 1013, 1339, 1291, 744, 1746, 233, 1303, 80, 403, 1209, 249, 74, 196, 413, 654] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[3343, 741, 7, 1122, 1987, 2297, 239, 110, 373, 269, 201, 343, 2186, 62, 22, 2797, 470, 991, 734, 259, 443, 21, 273, 801, 890, 2107, 1382, 1494, 28, 442, 206, 277, 418, 122, 244, 339, 82, 384, 2318, 2858, 457, 602, 86, 6, 1163, 259, 19, 46, 873, 54, 3138, 489, 314, 1177, 729, 1352, 25, 542, 2015, 338, 61, 272, 1311, 142] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2069, 28, 16, 1050, 1192, 531, 441, 48, 0, 0, 0, 0, 0, 0, 0, 0, 653, 95, 179, 596, 371, 101, 236, 3594, 421, 286, 118, 0, 0, 0, 0, 0, 716, 223, 1751, 101, 566, 155, 2173, 290, 567, 174, 0, 0, 0, 0, 0, 0, 203, 1079, 940, 1682, 755, 857, 21, 335, 0, 0, 0, 0, 0, 0, 0, 0, 560, 440, 894, 91, 108, 273, 718, 1086, 1128, 0, 0, 0, 0, 0, 0, 0, 383, 709, 207, 560, 383, 1162, 72, 291, 929, 285, 0, 0, 0, 0, 0, 0, 953, 117, 0, 1300, 390, 1235, 284, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 870, 21, 3944, 371, 202, 426, 1704, 337, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5375, 6650, 6716, 5872, 5298, 4981, 6385, 7875] avg=6144.0000 max=7875 max_rank=7 min=4981 min_rank=5 imbalance_ratio=1.281738 max_load_reduction=41277 max_load_reduction_pct=0.839783 moved_to_echo=24539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[537, 263, 2, 423, 2507, 89, 65, 47, 0, 0, 0, 0, 0, 0, 0, 0, 118, 27, 29, 573, 1793, 44, 191, 1224, 888, 436, 28, 0, 0, 0, 0, 0, 221, 854, 532, 619, 1380, 461, 589, 3893, 148, 265, 0, 0, 0, 0, 0, 0, 395, 1644, 1566, 1199, 182, 144, 657, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 562, 123, 248, 115, 519, 667, 1967, 479, 274, 0, 0, 0, 0, 0, 0, 0, 118, 1318, 350, 160, 614, 395, 365, 538, 1243, 69, 0, 0, 0, 0, 0, 0, 1994, 26, 0, 1866, 897, 577, 70, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 297, 491, 1032, 992, 258, 1496, 1324, 321, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3933, 5351, 8962, 7573, 4954, 5170, 6998, 6211] avg=6144.0000 max=8962 max_rank=2 min=3933 min_rank=0 imbalance_ratio=1.458659 max_load_reduction=40190 max_load_reduction_pct=0.817668 moved_to_echo=23333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[488, 497, 242, 713, 1476, 312, 187, 199, 0, 0, 0, 0, 0, 0, 0, 0, 646, 30, 51, 697, 767, 152, 1380, 1157, 528, 313, 139, 0, 0, 0, 0, 0, 955, 340, 263, 266, 3349, 647, 446, 1087, 75, 189, 0, 0, 0, 0, 0, 0, 76, 1160, 2854, 2959, 1796, 5, 164, 32, 0, 0, 0, 0, 0, 0, 0, 0, 764, 48, 12, 549, 603, 2224, 335, 542, 1312, 0, 0, 0, 0, 0, 0, 0, 47, 685, 299, 419, 480, 217, 226, 1065, 1341, 335, 0, 0, 0, 0, 0, 0, 2092, 419, 0, 1049, 608, 1488, 317, 1282, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 93, 488, 274, 372, 656, 549, 154, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 5860, 7617, 9046, 6389, 5114, 7255, 3757] avg=6144.0000 max=9046 max_rank=3 min=3757 min_rank=7 imbalance_ratio=1.472331 max_load_reduction=40106 max_load_reduction_pct=0.815959 moved_to_echo=22515 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1834, 414, 97, 565, 358, 2421, 963, 280, 0, 0, 0, 0, 0, 0, 0, 0, 50, 243, 104, 330, 623, 291, 133, 2020, 126, 59, 194, 0, 0, 0, 0, 0, 142, 990, 3680, 274, 398, 56, 108, 205, 194, 36, 0, 0, 0, 0, 0, 0, 355, 224, 1511, 1166, 338, 169, 45, 234, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 262, 380, 383, 416, 1012, 658, 2662, 1855, 0, 0, 0, 0, 0, 0, 0, 115, 720, 361, 17, 540, 122, 723, 946, 2089, 469, 0, 0, 0, 0, 0, 0, 666, 371, 0, 252, 221, 3065, 465, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1655, 1352, 41, 1538, 587, 196, 893, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6932, 4173, 6083, 4042, 9092, 6102, 6215, 6513] avg=6144.0000 max=9092 max_rank=4 min=4042 min_rank=3 imbalance_ratio=1.479818 max_load_reduction=40060 max_load_reduction_pct=0.815023 moved_to_echo=27922 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[781, 97, 49, 962, 2108, 1143, 194, 72, 0, 0, 0, 0, 0, 0, 0, 0, 598, 212, 30, 815, 1156, 287, 3134, 3325, 746, 331, 15, 0, 0, 0, 0, 0, 729, 188, 399, 173, 1409, 335, 370, 1931, 152, 201, 0, 0, 0, 0, 0, 0, 408, 1247, 920, 1679, 265, 452, 195, 879, 0, 0, 0, 0, 0, 0, 0, 0, 306, 221, 20, 334, 98, 285, 1447, 46, 144, 0, 0, 0, 0, 0, 0, 0, 5, 2731, 479, 109, 1255, 225, 424, 1397, 1339, 36, 0, 0, 0, 0, 0, 0, 1003, 1013, 0, 1291, 744, 1746, 38, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 80, 403, 1057, 249, 74, 196, 413, 654, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5406, 10649, 5887, 6045, 2901, 8000, 7138, 3126] avg=6144.0000 max=10649 max_rank=1 min=2901 min_rank=4 imbalance_ratio=1.733236 max_load_reduction=38503 max_load_reduction_pct=0.783346 moved_to_echo=21165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2074, 483, 687, 1096, 1155, 1182, 314, 128, 0, 0, 0, 0, 0, 0, 0, 0, 109, 874, 287, 561, 1249, 528, 308, 1401, 409, 416, 67, 0, 0, 0, 0, 0, 852, 164, 857, 447, 433, 74, 287, 194, 283, 253, 0, 0, 0, 0, 0, 0, 45, 1570, 283, 1457, 347, 281, 126, 470, 0, 0, 0, 0, 0, 0, 0, 0, 650, 1460, 261, 273, 333, 295, 669, 901, 641, 0, 0, 0, 0, 0, 0, 0, 123, 425, 137, 785, 1329, 1517, 235, 214, 1591, 162, 0, 0, 0, 0, 0, 0, 1757, 191, 0, 1554, 451, 757, 161, 631, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 1461, 1970, 1321, 341, 1053, 1946, 481, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7119, 6209, 3844, 4579, 5483, 6518, 5502, 9898] avg=6144.0000 max=9898 max_rank=7 min=3844 min_rank=2 imbalance_ratio=1.611003 max_load_reduction=39254 max_load_reduction_pct=0.798625 moved_to_echo=27401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2414, 228, 82, 71, 1917, 1650, 369, 215, 0, 0, 0, 0, 0, 0, 0, 0, 258, 14, 299, 397, 44, 15, 139, 712, 678, 209, 284, 0, 0, 0, 0, 0, 1252, 297, 433, 290, 1631, 254, 275, 796, 205, 127, 0, 0, 0, 0, 0, 0, 1429, 788, 934, 318, 886, 427, 111, 906, 0, 0, 0, 0, 0, 0, 0, 0, 786, 144, 30, 183, 76, 438, 2310, 1135, 2715, 0, 0, 0, 0, 0, 0, 0, 246, 351, 125, 390, 1018, 15, 183, 1132, 3230, 687, 0, 0, 0, 0, 0, 0, 702, 65, 0, 609, 674, 389, 680, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 73, 932, 1425, 1147, 447, 848, 2246, 201, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6946, 3049, 5560, 5799, 7817, 7377, 5285, 7319] avg=6144.0000 max=7817 max_rank=4 min=3049 min_rank=1 imbalance_ratio=1.272298 max_load_reduction=41335 max_load_reduction_pct=0.840963 moved_to_echo=27798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1834, 414, 97, 565, 358, 2421, 963, 280, 0, 0, 0, 0, 0, 0, 0, 0, 50, 243, 104, 330, 623, 291, 133, 2020, 126, 59, 194, 0, 0, 0, 0, 0, 142, 990, 3680, 274, 398, 56, 108, 205, 194, 36, 0, 0, 0, 0, 0, 0, 355, 224, 1511, 1166, 338, 169, 45, 234, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 262, 380, 383, 416, 1012, 658, 2662, 1855, 0, 0, 0, 0, 0, 0, 0, 115, 720, 361, 17, 540, 122, 723, 946, 2089, 469, 0, 0, 0, 0, 0, 0, 666, 371, 0, 252, 221, 3065, 465, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1655, 1352, 41, 1538, 587, 196, 893, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6932, 4173, 6083, 4042, 9092, 6102, 6215, 6513] avg=6144.0000 max=9092 max_rank=4 min=4042 min_rank=3 imbalance_ratio=1.479818 max_load_reduction=40060 max_load_reduction_pct=0.815023 moved_to_echo=27922 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[488, 497, 242, 713, 1476, 312, 187, 199, 0, 0, 0, 0, 0, 0, 0, 0, 646, 30, 51, 697, 767, 152, 1380, 1157, 528, 313, 139, 0, 0, 0, 0, 0, 955, 340, 263, 266, 3349, 647, 446, 1087, 75, 189, 0, 0, 0, 0, 0, 0, 76, 1160, 2854, 2959, 1796, 5, 164, 32, 0, 0, 0, 0, 0, 0, 0, 0, 764, 48, 12, 549, 603, 2224, 335, 542, 1312, 0, 0, 0, 0, 0, 0, 0, 47, 685, 299, 419, 480, 217, 226, 1065, 1341, 335, 0, 0, 0, 0, 0, 0, 2092, 419, 0, 1049, 608, 1488, 317, 1282, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 93, 488, 274, 372, 656, 549, 154, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 5860, 7617, 9046, 6389, 5114, 7255, 3757] avg=6144.0000 max=9046 max_rank=3 min=3757 min_rank=7 imbalance_ratio=1.472331 max_load_reduction=40106 max_load_reduction_pct=0.815959 moved_to_echo=22515 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[3343, 741, 7, 1122, 1468, 2297, 239, 110, 0, 0, 0, 0, 0, 0, 0, 0, 373, 269, 201, 343, 2186, 62, 22, 2797, 519, 392, 47, 0, 0, 0, 0, 0, 470, 991, 734, 259, 443, 21, 273, 801, 253, 238, 0, 0, 0, 0, 0, 0, 890, 1477, 1382, 1494, 28, 442, 206, 277, 0, 0, 0, 0, 0, 0, 0, 0, 418, 122, 244, 339, 82, 384, 2318, 2858, 453, 0, 0, 0, 0, 0, 0, 0, 457, 602, 86, 6, 1163, 259, 19, 46, 3138, 114, 0, 0, 0, 0, 0, 0, 873, 54, 0, 489, 314, 1177, 115, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 25, 542, 1762, 338, 61, 272, 1311, 142, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9327, 7211, 4483, 6196, 7218, 5890, 4374, 4453] avg=6144.0000 max=9327 max_rank=0 min=4374 min_rank=6 imbalance_ratio=1.518066 max_load_reduction=39825 max_load_reduction_pct=0.810242 moved_to_echo=21935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 1792, 1280, 4352, 8192, 1536, 5632, 16256, 4352, 2560, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 4096, 8704, 2432, 9728, 2048, 4608, 9216, 1920, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3840, 9216, 10496, 12032, 4608, 2816, 1536, 4992, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13568, 2816, 1280, 6016, 12288, 9728, 2816, 1152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4096, 5632, 13056, 4736, 3328, 5632, 9728, 3200, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 4096, 8704, 2432, 9728, 2048, 4608, 9216, 1920, 1536, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 2944, 2176, 2304, 2304, 5632, 10496, 9728, 8576, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3840, 9216, 10496, 12032, 4608, 2816, 1536, 4992, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 7552, 2048, 2560, 6784, 3968, 2304, 5632, 14976, 2176, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10112, 2304, 0, 8448, 4352, 10496, 2176, 11648, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[42, 661, 1115, 688, 262, 458, 787, 3795, 65, 868, 1894, 109, 27, 465, 47, 834, 1337, 30, 269, 31, 2047, 1066, 169, 772, 309, 1080, 498, 185, 6768, 159, 29, 1019, 21, 337, 98, 52, 682, 59, 1377, 113, 1720, 815, 94, 37, 934, 76, 1413, 505, 664, 304, 2936, 153, 348, 2033, 384, 28, 281, 202, 2018, 79, 715, 203, 2426, 160] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[354, 204, 413, 94, 438, 256, 1375, 1507, 329, 497, 3001, 689, 91, 456, 14, 180, 1220, 306, 182, 90, 1206, 974, 550, 422, 11, 3359, 793, 297, 3237, 309, 22, 97, 66, 1761, 28, 29, 535, 163, 1088, 706, 3698, 1789, 482, 21, 388, 102, 2098, 1736, 1782, 329, 1496, 389, 166, 800, 894, 428, 1236, 204, 1101, 7, 239, 986, 983, 449] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[236, 308, 78, 950, 102, 167, 696, 864, 410, 2912, 5039, 291, 10, 384, 174, 628, 422, 39, 248, 65, 4267, 744, 537, 798, 322, 1425, 143, 696, 1922, 135, 57, 516, 5, 169, 1382, 17, 848, 107, 891, 76, 2653, 225, 779, 426, 402, 266, 1095, 3625, 86, 849, 1224, 246, 767, 2910, 751, 40, 466, 835, 894, 323, 49, 431, 154, 576] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[567, 45, 1741, 232, 267, 101, 195, 546, 336, 2327, 1052, 151, 60, 1278, 57, 500, 2143, 234, 693, 489, 2555, 2088, 241, 526, 149, 924, 127, 102, 3622, 804, 107, 259, 13, 2882, 323, 67, 769, 427, 1026, 605, 2735, 961, 238, 122, 2101, 293, 1013, 1077, 3080, 227, 823, 148, 613, 1034, 254, 28, 59, 22, 460, 84, 570, 494, 1425, 661] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[169, 328, 638, 756, 81, 34, 643, 331, 628, 854, 3026, 433, 125, 871, 159, 479, 1063, 30, 600, 811, 1949, 385, 888, 616, 712, 2781, 989, 81, 4117, 205, 46, 461, 28, 673, 192, 322, 865, 361, 261, 334, 3258, 1983, 83, 572, 351, 141, 1874, 164, 1917, 476, 1235, 595, 334, 1795, 480, 20, 245, 71, 2439, 134, 96, 683, 1752, 129] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[1697, 538, 45, 2209, 594, 97, 172, 1286, 535, 237, 939, 65, 18, 1438, 37, 527, 122, 157, 305, 547, 3841, 2137, 487, 355, 455, 2376, 206, 161, 6311, 275, 9, 2527, 7, 1680, 12, 94, 17, 15, 844, 90, 411, 204, 1021, 363, 1936, 597, 542, 496, 1410, 974, 415, 340, 1675, 742, 139, 59, 1216, 198, 794, 200, 302, 84, 253, 1317] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[317, 515, 53, 780, 32, 50, 511, 1806, 512, 1038, 3057, 1010, 39, 174, 359, 600, 776, 49, 620, 299, 2205, 847, 152, 973, 105, 743, 549, 270, 5273, 98, 59, 149, 23, 615, 320, 11, 1233, 250, 1297, 116, 4291, 1560, 977, 582, 406, 390, 1143, 2443, 835, 301, 777, 136, 97, 1141, 185, 16, 215, 163, 2191, 908, 264, 1064, 1174, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[2605, 649, 514, 351, 102, 142, 199, 2093, 24, 323, 580, 554, 214, 309, 30, 82, 1885, 123, 491, 1230, 4280, 863, 661, 1130, 32, 921, 1427, 786, 1472, 360, 28, 10, 23, 234, 88, 513, 460, 29, 384, 993, 2139, 1244, 582, 236, 688, 268, 630, 923, 2162, 2647, 2658, 825, 94, 1553, 53, 86, 920, 575, 909, 75, 4, 286, 2220, 181] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[567, 45, 1741, 232, 267, 101, 195, 546, 802, 249, 0, 0, 0, 0, 0, 0, 336, 2327, 1052, 151, 60, 1278, 57, 500, 278, 167, 0, 0, 0, 0, 0, 0, 2143, 234, 693, 489, 1586, 2088, 241, 526, 0, 0, 0, 0, 0, 0, 0, 0, 149, 924, 127, 102, 2038, 804, 107, 259, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2882, 323, 67, 769, 427, 1026, 605, 2258, 307, 0, 0, 0, 0, 0, 0, 477, 961, 238, 122, 2101, 293, 1013, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 227, 823, 148, 613, 756, 254, 28, 0, 0, 0, 0, 0, 0, 0, 0, 59, 22, 460, 84, 570, 494, 1425, 661, 1028, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4745, 6206, 8000, 4510, 8677, 6282, 5929, 4803] avg=6144.0000 max=8677 max_rank=4 min=4510 min_rank=3 imbalance_ratio=1.412272 max_load_reduction=40475 max_load_reduction_pct=0.823466 moved_to_echo=25691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[42, 661, 1115, 688, 262, 458, 787, 3795, 642, 465, 0, 0, 0, 0, 0, 0, 65, 868, 1894, 109, 27, 465, 47, 834, 547, 134, 0, 0, 0, 0, 0, 0, 1337, 30, 269, 31, 1271, 1066, 169, 772, 0, 0, 0, 0, 0, 0, 0, 0, 309, 1080, 498, 185, 3807, 159, 29, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 21, 337, 98, 52, 682, 59, 1377, 113, 1420, 575, 0, 0, 0, 0, 0, 0, 300, 815, 94, 37, 934, 76, 1413, 505, 0, 0, 0, 0, 0, 0, 0, 0, 664, 304, 2936, 153, 348, 1486, 384, 28, 0, 0, 0, 0, 0, 0, 0, 0, 281, 202, 2018, 79, 715, 203, 2426, 160, 1921, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8915, 4990, 4945, 7086, 4734, 4174, 6303, 8005] avg=6144.0000 max=8915 max_rank=0 min=4174 min_rank=5 imbalance_ratio=1.451009 max_load_reduction=40237 max_load_reduction_pct=0.818624 moved_to_echo=23216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[354, 204, 413, 94, 438, 256, 1375, 1507, 378, 222, 0, 0, 0, 0, 0, 0, 329, 497, 3001, 689, 91, 456, 14, 180, 215, 79, 0, 0, 0, 0, 0, 0, 1220, 306, 182, 90, 749, 974, 550, 422, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3359, 793, 297, 1821, 309, 22, 97, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1761, 28, 29, 535, 163, 1088, 706, 3054, 275, 0, 0, 0, 0, 0, 0, 644, 1789, 482, 21, 388, 102, 2098, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 329, 1496, 389, 166, 585, 894, 428, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 204, 1101, 7, 239, 986, 983, 449, 919, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5241, 5551, 4493, 6709, 7705, 7260, 6069, 6124] avg=6144.0000 max=7705 max_rank=4 min=4493 min_rank=2 imbalance_ratio=1.254069 max_load_reduction=41447 max_load_reduction_pct=0.843241 moved_to_echo=27158 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[236, 308, 78, 950, 102, 167, 696, 864, 410, 2912, 5039, 291, 10, 384, 174, 628, 422, 39, 248, 65, 4267, 744, 537, 798, 322, 1425, 143, 696, 1922, 135, 57, 516, 5, 169, 1382, 17, 848, 107, 891, 76, 2653, 225, 779, 426, 402, 266, 1095, 3625, 86, 849, 1224, 246, 767, 2910, 751, 40, 466, 835, 894, 323, 49, 431, 154, 576] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[42, 661, 1115, 688, 262, 458, 787, 3795, 65, 868, 1894, 109, 27, 465, 47, 834, 1337, 30, 269, 31, 2047, 1066, 169, 772, 309, 1080, 498, 185, 6768, 159, 29, 1019, 21, 337, 98, 52, 682, 59, 1377, 113, 1720, 815, 94, 37, 934, 76, 1413, 505, 664, 304, 2936, 153, 348, 2033, 384, 28, 281, 202, 2018, 79, 715, 203, 2426, 160] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[567, 45, 1741, 232, 267, 101, 195, 546, 336, 2327, 1052, 151, 60, 1278, 57, 500, 2143, 234, 693, 489, 2555, 2088, 241, 526, 149, 924, 127, 102, 3622, 804, 107, 259, 13, 2882, 323, 67, 769, 427, 1026, 605, 2735, 961, 238, 122, 2101, 293, 1013, 1077, 3080, 227, 823, 148, 613, 1034, 254, 28, 59, 22, 460, 84, 570, 494, 1425, 661] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[1697, 538, 45, 2209, 594, 97, 172, 1286, 535, 237, 939, 65, 18, 1438, 37, 527, 122, 157, 305, 547, 3841, 2137, 487, 355, 455, 2376, 206, 161, 6311, 275, 9, 2527, 7, 1680, 12, 94, 17, 15, 844, 90, 411, 204, 1021, 363, 1936, 597, 542, 496, 1410, 974, 415, 340, 1675, 742, 139, 59, 1216, 198, 794, 200, 302, 84, 253, 1317] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[354, 204, 413, 94, 438, 256, 1375, 1507, 329, 497, 3001, 689, 91, 456, 14, 180, 1220, 306, 182, 90, 1206, 974, 550, 422, 11, 3359, 793, 297, 3237, 309, 22, 97, 66, 1761, 28, 29, 535, 163, 1088, 706, 3698, 1789, 482, 21, 388, 102, 2098, 1736, 1782, 329, 1496, 389, 166, 800, 894, 428, 1236, 204, 1101, 7, 239, 986, 983, 449] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[317, 515, 53, 780, 32, 50, 511, 1806, 692, 363, 0, 0, 0, 0, 0, 0, 512, 1038, 3057, 1010, 39, 174, 359, 600, 307, 144, 0, 0, 0, 0, 0, 0, 776, 49, 620, 299, 1369, 847, 152, 973, 0, 0, 0, 0, 0, 0, 0, 0, 105, 743, 549, 270, 2965, 98, 59, 149, 0, 0, 0, 0, 0, 0, 0, 0, 23, 615, 320, 11, 1233, 250, 1297, 116, 3543, 448, 0, 0, 0, 0, 0, 0, 748, 1560, 977, 582, 406, 390, 1143, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 835, 301, 777, 136, 97, 834, 185, 16, 0, 0, 0, 0, 0, 0, 0, 0, 215, 163, 2191, 908, 264, 1064, 1174, 8, 1497, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5119, 7240, 5085, 4938, 7856, 8249, 3181, 7484] avg=6144.0000 max=8249 max_rank=5 min=3181 min_rank=6 imbalance_ratio=1.342611 max_load_reduction=40903 max_load_reduction_pct=0.832174 moved_to_echo=26770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[169, 328, 638, 756, 81, 34, 643, 331, 611, 283, 0, 0, 0, 0, 0, 0, 628, 854, 3026, 433, 125, 871, 159, 479, 483, 127, 0, 0, 0, 0, 0, 0, 1063, 30, 600, 811, 1211, 385, 888, 616, 0, 0, 0, 0, 0, 0, 0, 0, 712, 2781, 989, 81, 2317, 205, 46, 461, 0, 0, 0, 0, 0, 0, 0, 0, 28, 673, 192, 322, 865, 361, 261, 334, 2690, 349, 0, 0, 0, 0, 0, 0, 568, 1983, 83, 572, 351, 141, 1874, 164, 0, 0, 0, 0, 0, 0, 0, 0, 1917, 476, 1235, 595, 334, 1312, 480, 20, 0, 0, 0, 0, 0, 0, 0, 0, 245, 71, 2439, 134, 96, 683, 1752, 129, 1168, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3874, 7185, 5604, 7592, 6075, 5736, 6369, 6717] avg=6144.0000 max=7592 max_rank=3 min=3874 min_rank=0 imbalance_ratio=1.235677 max_load_reduction=41560 max_load_reduction_pct=0.845540 moved_to_echo=24897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[236, 308, 78, 950, 102, 167, 696, 864, 1344, 136, 0, 0, 0, 0, 0, 0, 410, 2912, 5039, 291, 10, 384, 174, 628, 787, 284, 0, 0, 0, 0, 0, 0, 422, 39, 248, 65, 2639, 744, 537, 798, 0, 0, 0, 0, 0, 0, 0, 0, 322, 1425, 143, 696, 1071, 135, 57, 516, 0, 0, 0, 0, 0, 0, 0, 0, 5, 169, 1382, 17, 848, 107, 891, 76, 2195, 166, 0, 0, 0, 0, 0, 0, 458, 225, 779, 426, 402, 266, 1095, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 86, 849, 1224, 246, 767, 2123, 751, 40, 0, 0, 0, 0, 0, 0, 0, 0, 466, 835, 894, 323, 49, 431, 154, 576, 549, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4881, 10919, 5492, 4365, 5856, 7276, 6086, 4277] avg=6144.0000 max=10919 max_rank=1 min=4277 min_rank=7 imbalance_ratio=1.777181 max_load_reduction=38233 max_load_reduction_pct=0.777852 moved_to_echo=23495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1697, 538, 45, 2209, 594, 97, 172, 1286, 1206, 434, 0, 0, 0, 0, 0, 0, 535, 237, 939, 65, 18, 1438, 37, 527, 199, 252, 0, 0, 0, 0, 0, 0, 122, 157, 305, 547, 2383, 2137, 487, 355, 0, 0, 0, 0, 0, 0, 0, 0, 455, 2376, 206, 161, 3550, 275, 9, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1680, 12, 94, 17, 15, 844, 90, 339, 536, 0, 0, 0, 0, 0, 0, 72, 204, 1021, 363, 1936, 597, 542, 496, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 974, 415, 340, 1675, 543, 139, 59, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 198, 794, 200, 302, 84, 253, 1317, 1791, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 4247, 6493, 9559, 3634, 5231, 5555, 6155] avg=6144.0000 max=9559 max_rank=3 min=3634 min_rank=4 imbalance_ratio=1.555827 max_load_reduction=39593 max_load_reduction_pct=0.805522 moved_to_echo=20575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[42, 661, 1115, 688, 262, 458, 787, 3795, 642, 465, 0, 0, 0, 0, 0, 0, 65, 868, 1894, 109, 27, 465, 47, 834, 547, 134, 0, 0, 0, 0, 0, 0, 1337, 30, 269, 31, 1271, 1066, 169, 772, 0, 0, 0, 0, 0, 0, 0, 0, 309, 1080, 498, 185, 3807, 159, 29, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 21, 337, 98, 52, 682, 59, 1377, 113, 1420, 575, 0, 0, 0, 0, 0, 0, 300, 815, 94, 37, 934, 76, 1413, 505, 0, 0, 0, 0, 0, 0, 0, 0, 664, 304, 2936, 153, 348, 1486, 384, 28, 0, 0, 0, 0, 0, 0, 0, 0, 281, 202, 2018, 79, 715, 203, 2426, 160, 1921, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8915, 4990, 4945, 7086, 4734, 4174, 6303, 8005] avg=6144.0000 max=8915 max_rank=0 min=4174 min_rank=5 imbalance_ratio=1.451009 max_load_reduction=40237 max_load_reduction_pct=0.818624 moved_to_echo=23216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[2605, 649, 514, 351, 102, 142, 199, 2093, 1343, 101, 0, 0, 0, 0, 0, 0, 24, 323, 580, 554, 214, 309, 30, 82, 418, 280, 0, 0, 0, 0, 0, 0, 1885, 123, 491, 1230, 2657, 863, 661, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 32, 921, 1427, 786, 829, 360, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 23, 234, 88, 513, 460, 29, 384, 993, 1766, 125, 0, 0, 0, 0, 0, 0, 373, 1244, 582, 236, 688, 268, 630, 923, 0, 0, 0, 0, 0, 0, 0, 0, 2162, 2647, 2658, 825, 94, 1135, 53, 86, 0, 0, 0, 0, 0, 0, 0, 0, 920, 575, 909, 75, 4, 286, 2220, 181, 417, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 2814, 9040, 4393, 4615, 4944, 9660, 5587] avg=6144.0000 max=9660 max_rank=6 min=2814 min_rank=1 imbalance_ratio=1.572266 max_load_reduction=39492 max_load_reduction_pct=0.803467 moved_to_echo=24806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[317, 515, 53, 780, 32, 50, 511, 1806, 692, 363, 0, 0, 0, 0, 0, 0, 512, 1038, 3057, 1010, 39, 174, 359, 600, 307, 144, 0, 0, 0, 0, 0, 0, 776, 49, 620, 299, 1369, 847, 152, 973, 0, 0, 0, 0, 0, 0, 0, 0, 105, 743, 549, 270, 2965, 98, 59, 149, 0, 0, 0, 0, 0, 0, 0, 0, 23, 615, 320, 11, 1233, 250, 1297, 116, 3543, 448, 0, 0, 0, 0, 0, 0, 748, 1560, 977, 582, 406, 390, 1143, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 835, 301, 777, 136, 97, 834, 185, 16, 0, 0, 0, 0, 0, 0, 0, 0, 215, 163, 2191, 908, 264, 1064, 1174, 8, 1497, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5119, 7240, 5085, 4938, 7856, 8249, 3181, 7484] avg=6144.0000 max=8249 max_rank=5 min=3181 min_rank=6 imbalance_ratio=1.342611 max_load_reduction=40903 max_load_reduction_pct=0.832174 moved_to_echo=26770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[567, 45, 1741, 232, 267, 101, 195, 546, 802, 249, 0, 0, 0, 0, 0, 0, 336, 2327, 1052, 151, 60, 1278, 57, 500, 278, 167, 0, 0, 0, 0, 0, 0, 2143, 234, 693, 489, 1586, 2088, 241, 526, 0, 0, 0, 0, 0, 0, 0, 0, 149, 924, 127, 102, 2038, 804, 107, 259, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2882, 323, 67, 769, 427, 1026, 605, 2258, 307, 0, 0, 0, 0, 0, 0, 477, 961, 238, 122, 2101, 293, 1013, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 227, 823, 148, 613, 756, 254, 28, 0, 0, 0, 0, 0, 0, 0, 0, 59, 22, 460, 84, 570, 494, 1425, 661, 1028, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4745, 6206, 8000, 4510, 8677, 6282, 5929, 4803] avg=6144.0000 max=8677 max_rank=4 min=4510 min_rank=3 imbalance_ratio=1.412272 max_load_reduction=40475 max_load_reduction_pct=0.823466 moved_to_echo=25691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1697, 538, 45, 2209, 594, 97, 172, 1286, 1206, 434, 0, 0, 0, 0, 0, 0, 535, 237, 939, 65, 18, 1438, 37, 527, 199, 252, 0, 0, 0, 0, 0, 0, 122, 157, 305, 547, 2383, 2137, 487, 355, 0, 0, 0, 0, 0, 0, 0, 0, 455, 2376, 206, 161, 3550, 275, 9, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1680, 12, 94, 17, 15, 844, 90, 339, 536, 0, 0, 0, 0, 0, 0, 72, 204, 1021, 363, 1936, 597, 542, 496, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 974, 415, 340, 1675, 543, 139, 59, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 198, 794, 200, 302, 84, 253, 1317, 1791, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 4247, 6493, 9559, 3634, 5231, 5555, 6155] avg=6144.0000 max=9559 max_rank=3 min=3634 min_rank=4 imbalance_ratio=1.555827 max_load_reduction=39593 max_load_reduction_pct=0.805522 moved_to_echo=20575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[236, 308, 78, 950, 102, 167, 696, 864, 1344, 136, 0, 0, 0, 0, 0, 0, 410, 2912, 5039, 291, 10, 384, 174, 628, 787, 284, 0, 0, 0, 0, 0, 0, 422, 39, 248, 65, 2639, 744, 537, 798, 0, 0, 0, 0, 0, 0, 0, 0, 322, 1425, 143, 696, 1071, 135, 57, 516, 0, 0, 0, 0, 0, 0, 0, 0, 5, 169, 1382, 17, 848, 107, 891, 76, 2195, 166, 0, 0, 0, 0, 0, 0, 458, 225, 779, 426, 402, 266, 1095, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 86, 849, 1224, 246, 767, 2123, 751, 40, 0, 0, 0, 0, 0, 0, 0, 0, 466, 835, 894, 323, 49, 431, 154, 576, 549, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4881, 10919, 5492, 4365, 5856, 7276, 6086, 4277] avg=6144.0000 max=10919 max_rank=1 min=4277 min_rank=7 imbalance_ratio=1.777181 max_load_reduction=38233 max_load_reduction_pct=0.777852 moved_to_echo=23495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[354, 204, 413, 94, 438, 256, 1375, 1507, 378, 222, 0, 0, 0, 0, 0, 0, 329, 497, 3001, 689, 91, 456, 14, 180, 215, 79, 0, 0, 0, 0, 0, 0, 1220, 306, 182, 90, 749, 974, 550, 422, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3359, 793, 297, 1821, 309, 22, 97, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1761, 28, 29, 535, 163, 1088, 706, 3054, 275, 0, 0, 0, 0, 0, 0, 644, 1789, 482, 21, 388, 102, 2098, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 329, 1496, 389, 166, 585, 894, 428, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 204, 1101, 7, 239, 986, 983, 449, 919, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5241, 5551, 4493, 6709, 7705, 7260, 6069, 6124] avg=6144.0000 max=7705 max_rank=4 min=4493 min_rank=2 imbalance_ratio=1.254069 max_load_reduction=41447 max_load_reduction_pct=0.843241 moved_to_echo=27158 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3712, 8832, 4352, 2432, 7296, 2176, 9856, 11008, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4736, 2304, 10880, 1920, 2304, 4352, 10496, 3584, 9344, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 9088, 18688, 3328, 640, 5376, 896, 3840, 3328, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 8448, 2560, 1152, 5504, 1536, 7168, 3072, 17280, 2816, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9088, 1024, 3456, 3584, 13952, 9216, 3712, 5632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3712, 8832, 4352, 2432, 7296, 2176, 9856, 11008, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4736, 2304, 10880, 1920, 2304, 4352, 10496, 3584, 9344, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 13696, 4736, 2688, 18432, 2432, 384, 5120, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 3328, 4608, 6144, 1920, 1408, 4608, 12288, 7040, 2304, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 8448, 2560, 1152, 5504, 1536, 7168, 3072, 17280, 2816, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12032, 6144, 11648, 2944, 4096, 8832, 3200, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[558, 81, 38, 596, 40, 19, 1177, 2150, 3426, 267, 25, 161, 26, 510, 1125, 994, 1177, 4735, 1405, 1273, 111, 4360, 4711, 249, 970, 1206, 0, 126, 1172, 766, 11, 167, 159, 295, 1357, 122, 631, 583, 2992, 224, 1048, 1354, 80, 242, 11, 276, 73, 560, 190, 905, 292, 342, 29, 277, 556, 19, 245, 34, 230, 210, 51, 1540, 95, 498] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[440, 332, 14, 133, 821, 56, 1924, 165, 1845, 69, 2276, 132, 79, 505, 17, 1443, 544, 2416, 3536, 98, 25, 592, 2713, 1604, 123, 875, 64, 29, 868, 1576, 5, 376, 107, 3575, 1757, 1022, 164, 496, 3840, 1266, 137, 252, 21, 31, 10, 113, 42, 199, 343, 1362, 39, 977, 44, 1192, 197, 69, 138, 1175, 222, 1567, 63, 1408, 311, 1318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[228, 109, 584, 536, 401, 487, 1613, 643, 1891, 147, 77, 234, 999, 2417, 801, 558, 695, 1124, 857, 480, 126, 1097, 2524, 1416, 58, 1565, 305, 175, 1992, 1189, 79, 533, 115, 2537, 1198, 398, 176, 1150, 3301, 868, 1003, 1622, 57, 255, 33, 965, 388, 638, 87, 808, 434, 288, 348, 804, 446, 824, 1807, 605, 209, 206, 489, 374, 228, 551] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[359, 315, 120, 631, 455, 46, 1024, 604, 3937, 35, 444, 578, 36, 358, 1828, 2205, 36, 554, 548, 1032, 490, 1781, 4492, 2266, 441, 1682, 96, 664, 2140, 1424, 413, 215, 96, 3321, 116, 523, 178, 449, 622, 148, 336, 743, 424, 437, 49, 233, 67, 505, 956, 506, 251, 356, 21, 1341, 333, 3, 1636, 146, 161, 1790, 766, 869, 275, 246] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[131, 82, 291, 1167, 199, 28, 669, 75, 4657, 15, 705, 360, 202, 434, 381, 581, 854, 94, 1083, 536, 64, 1826, 3464, 653, 409, 2003, 11, 188, 385, 2706, 12, 696, 109, 3075, 1675, 553, 417, 1155, 1482, 614, 535, 172, 250, 1083, 19, 182, 549, 1141, 437, 1239, 729, 1048, 41, 1313, 879, 6, 263, 155, 346, 402, 692, 1483, 173, 1974] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[106, 53, 381, 686, 549, 139, 2408, 440, 969, 127, 1426, 490, 1688, 615, 668, 586, 568, 367, 917, 488, 235, 1217, 2866, 1496, 416, 169, 306, 50, 1243, 1695, 7, 556, 30, 6703, 769, 146, 79, 273, 962, 1563, 1030, 1912, 63, 106, 201, 81, 756, 261, 147, 1147, 490, 624, 197, 829, 37, 102, 962, 1693, 219, 292, 1047, 571, 1297, 636] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[440, 332, 14, 133, 821, 56, 1924, 165, 1845, 69, 2276, 132, 79, 505, 17, 1443, 544, 2416, 3536, 98, 25, 592, 2713, 1604, 123, 875, 64, 29, 868, 1576, 5, 376, 107, 3575, 1757, 1022, 164, 496, 3840, 1266, 137, 252, 21, 31, 10, 113, 42, 199, 343, 1362, 39, 977, 44, 1192, 197, 69, 138, 1175, 222, 1567, 63, 1408, 311, 1318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[297, 231, 509, 406, 1335, 1636, 2667, 1577, 3863, 0, 195, 570, 1120, 611, 801, 1419, 1521, 2114, 308, 1142, 107, 1242, 901, 1173, 250, 842, 4, 756, 128, 1739, 6, 605, 35, 4181, 759, 227, 198, 1935, 316, 483, 640, 369, 108, 19, 418, 2236, 168, 327, 169, 259, 47, 1154, 97, 1459, 42, 13, 582, 437, 790, 280, 37, 1181, 19, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[359, 315, 120, 631, 455, 46, 1024, 604, 3937, 35, 444, 578, 36, 358, 1828, 2205, 36, 554, 548, 1032, 490, 1781, 4492, 2266, 441, 1682, 96, 664, 2140, 1424, 413, 215, 96, 3321, 116, 523, 178, 449, 622, 148, 336, 743, 424, 437, 49, 233, 67, 505, 956, 506, 251, 356, 21, 1341, 333, 3, 1636, 146, 161, 1790, 766, 869, 275, 246] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[228, 109, 584, 536, 401, 487, 1613, 643, 1891, 147, 77, 234, 999, 2417, 801, 558, 695, 1124, 857, 480, 126, 1097, 2524, 1416, 58, 1565, 305, 175, 1992, 1189, 79, 533, 115, 2537, 1198, 398, 176, 1150, 3301, 868, 1003, 1622, 57, 255, 33, 965, 388, 638, 87, 808, 434, 288, 348, 804, 446, 824, 1807, 605, 209, 206, 489, 374, 228, 551] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[232, 136, 394, 787, 1588, 129, 1398, 105, 2598, 414, 1291, 504, 363, 511, 2466, 372, 213, 278, 385, 311, 76, 2192, 4554, 24, 570, 1796, 7, 338, 2034, 1018, 53, 122, 1, 1811, 189, 407, 500, 736, 245, 154, 654, 463, 432, 326, 1484, 390, 1320, 283, 1233, 1187, 534, 550, 204, 3437, 554, 34, 509, 230, 365, 478, 1121, 517, 127, 1418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[131, 82, 291, 1167, 199, 28, 669, 75, 4657, 15, 705, 360, 202, 434, 381, 581, 854, 94, 1083, 536, 64, 1826, 3464, 653, 409, 2003, 11, 188, 385, 2706, 12, 696, 109, 3075, 1675, 553, 417, 1155, 1482, 614, 535, 172, 250, 1083, 19, 182, 549, 1141, 437, 1239, 729, 1048, 41, 1313, 879, 6, 263, 155, 346, 402, 692, 1483, 173, 1974] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[558, 81, 38, 596, 40, 19, 1177, 2150, 108, 328, 0, 0, 0, 0, 0, 0, 1758, 267, 25, 161, 26, 510, 1125, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 4735, 1405, 1273, 111, 2240, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1206, 0, 126, 1172, 766, 11, 167, 2050, 0, 0, 0, 0, 0, 0, 0, 159, 106, 1357, 122, 631, 583, 2992, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1354, 80, 242, 11, 276, 73, 560, 3408, 0, 0, 0, 0, 0, 0, 0, 190, 905, 292, 342, 29, 277, 556, 19, 1303, 81, 0, 0, 0, 0, 0, 0, 245, 34, 230, 210, 51, 1540, 95, 498, 1340, 70, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5095, 4866, 11190, 6468, 6174, 7052, 3994, 4313] avg=6144.0000 max=11190 max_rank=2 min=3994 min_rank=6 imbalance_ratio=1.821289 max_load_reduction=37962 max_load_reduction_pct=0.772339 moved_to_echo=21533 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[106, 53, 381, 686, 549, 139, 2408, 440, 2470, 93, 0, 0, 0, 0, 0, 0, 497, 127, 1426, 490, 1688, 615, 668, 586, 0, 0, 0, 0, 0, 0, 0, 0, 568, 367, 917, 488, 235, 626, 0, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 416, 169, 306, 50, 1243, 1695, 7, 556, 572, 0, 0, 0, 0, 0, 0, 0, 30, 2388, 769, 146, 79, 273, 962, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1912, 63, 106, 201, 81, 756, 261, 2073, 0, 0, 0, 0, 0, 0, 0, 147, 1147, 490, 624, 197, 829, 37, 102, 793, 1845, 0, 0, 0, 0, 0, 0, 962, 1693, 219, 292, 1047, 571, 1297, 636, 379, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7325, 6097, 4697, 5014, 6210, 6483, 6211, 7115] avg=6144.0000 max=7325 max_rank=0 min=4697 min_rank=2 imbalance_ratio=1.192220 max_load_reduction=41827 max_load_reduction_pct=0.850972 moved_to_echo=26019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[359, 315, 120, 631, 455, 46, 1024, 604, 1223, 377, 0, 0, 0, 0, 0, 0, 2021, 35, 444, 578, 36, 358, 1828, 2205, 0, 0, 0, 0, 0, 0, 0, 0, 36, 554, 548, 1032, 490, 916, 0, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 441, 1682, 96, 664, 2140, 1424, 413, 215, 837, 0, 0, 0, 0, 0, 0, 0, 96, 1184, 116, 523, 178, 449, 622, 148, 0, 0, 0, 0, 0, 0, 0, 0, 336, 743, 424, 437, 49, 233, 67, 505, 3249, 0, 0, 0, 0, 0, 0, 0, 956, 506, 251, 356, 21, 1341, 333, 3, 1243, 914, 0, 0, 0, 0, 0, 0, 1636, 146, 161, 1790, 766, 869, 275, 246, 1539, 28, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5154, 7505, 5842, 7912, 3316, 6043, 5924, 7456] avg=6144.0000 max=7912 max_rank=3 min=3316 min_rank=4 imbalance_ratio=1.287760 max_load_reduction=41240 max_load_reduction_pct=0.839030 moved_to_echo=22739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[131, 82, 291, 1167, 199, 28, 669, 75, 1133, 447, 0, 0, 0, 0, 0, 0, 2389, 15, 705, 360, 202, 434, 381, 581, 0, 0, 0, 0, 0, 0, 0, 0, 854, 94, 1083, 536, 64, 939, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 409, 2003, 11, 188, 385, 2706, 12, 696, 858, 0, 0, 0, 0, 0, 0, 0, 109, 1096, 1675, 553, 417, 1155, 1482, 614, 0, 0, 0, 0, 0, 0, 0, 0, 535, 172, 250, 1083, 19, 182, 549, 1141, 2505, 0, 0, 0, 0, 0, 0, 0, 437, 1239, 729, 1048, 41, 1313, 879, 6, 959, 846, 0, 0, 0, 0, 0, 0, 263, 155, 346, 402, 692, 1483, 173, 1974, 1821, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4222, 5067, 4223, 7268, 7101, 6436, 7497, 7338] avg=6144.0000 max=7497 max_rank=6 min=4222 min_rank=0 imbalance_ratio=1.220215 max_load_reduction=41655 max_load_reduction_pct=0.847473 moved_to_echo=28372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[228, 109, 584, 536, 401, 487, 1613, 643, 934, 181, 0, 0, 0, 0, 0, 0, 971, 147, 77, 234, 999, 2417, 801, 558, 0, 0, 0, 0, 0, 0, 0, 0, 695, 1124, 857, 480, 126, 565, 0, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1565, 305, 175, 1992, 1189, 79, 533, 515, 0, 0, 0, 0, 0, 0, 0, 115, 905, 1198, 398, 176, 1150, 3301, 868, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1622, 57, 255, 33, 965, 388, 638, 1825, 0, 0, 0, 0, 0, 0, 0, 87, 808, 434, 288, 348, 804, 446, 824, 699, 698, 0, 0, 0, 0, 0, 0, 1807, 605, 209, 206, 489, 374, 228, 551, 739, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5716, 6204, 5263, 6411, 8111, 6786, 5436, 5225] avg=6144.0000 max=8111 max_rank=4 min=5225 min_rank=7 imbalance_ratio=1.320150 max_load_reduction=41041 max_load_reduction_pct=0.834981 moved_to_echo=25558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[297, 231, 509, 406, 1335, 1636, 2667, 1577, 1540, 370, 0, 0, 0, 0, 0, 0, 1983, 0, 195, 570, 1120, 611, 801, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 2114, 308, 1142, 107, 639, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 250, 842, 4, 756, 128, 1739, 6, 605, 583, 0, 0, 0, 0, 0, 0, 0, 35, 1490, 759, 227, 198, 1935, 316, 483, 0, 0, 0, 0, 0, 0, 0, 0, 640, 369, 108, 19, 418, 2236, 168, 327, 652, 0, 0, 0, 0, 0, 0, 0, 169, 259, 47, 1154, 97, 1459, 42, 13, 249, 1151, 0, 0, 0, 0, 0, 0, 582, 437, 790, 280, 37, 1181, 19, 92, 1510, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10568, 6699, 7004, 4913, 5443, 4937, 4640, 4948] avg=6144.0000 max=10568 max_rank=0 min=4640 min_rank=6 imbalance_ratio=1.720052 max_load_reduction=38584 max_load_reduction_pct=0.784993 moved_to_echo=19968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[440, 332, 14, 133, 821, 56, 1924, 165, 1317, 177, 0, 0, 0, 0, 0, 0, 947, 69, 2276, 132, 79, 505, 17, 1443, 0, 0, 0, 0, 0, 0, 0, 0, 544, 2416, 3536, 98, 25, 305, 0, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 123, 875, 64, 29, 868, 1576, 5, 376, 278, 0, 0, 0, 0, 0, 0, 0, 107, 1274, 1757, 1022, 164, 496, 3840, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 137, 252, 21, 31, 10, 113, 42, 199, 1963, 0, 0, 0, 0, 0, 0, 0, 343, 1362, 39, 977, 44, 1192, 197, 69, 750, 984, 0, 0, 0, 0, 0, 0, 138, 1175, 222, 1567, 63, 1408, 311, 1318, 721, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5379, 5468, 8528, 4194, 9926, 2768, 5957, 6932] avg=6144.0000 max=9926 max_rank=4 min=2768 min_rank=5 imbalance_ratio=1.615560 max_load_reduction=39226 max_load_reduction_pct=0.798055 moved_to_echo=25583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[359, 315, 120, 631, 455, 46, 1024, 604, 1223, 377, 0, 0, 0, 0, 0, 0, 2021, 35, 444, 578, 36, 358, 1828, 2205, 0, 0, 0, 0, 0, 0, 0, 0, 36, 554, 548, 1032, 490, 916, 0, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 441, 1682, 96, 664, 2140, 1424, 413, 215, 837, 0, 0, 0, 0, 0, 0, 0, 96, 1184, 116, 523, 178, 449, 622, 148, 0, 0, 0, 0, 0, 0, 0, 0, 336, 743, 424, 437, 49, 233, 67, 505, 3249, 0, 0, 0, 0, 0, 0, 0, 956, 506, 251, 356, 21, 1341, 333, 3, 1243, 914, 0, 0, 0, 0, 0, 0, 1636, 146, 161, 1790, 766, 869, 275, 246, 1539, 28, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5154, 7505, 5842, 7912, 3316, 6043, 5924, 7456] avg=6144.0000 max=7912 max_rank=3 min=3316 min_rank=4 imbalance_ratio=1.287760 max_load_reduction=41240 max_load_reduction_pct=0.839030 moved_to_echo=22739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[106, 53, 381, 686, 549, 139, 2408, 440, 2470, 93, 0, 0, 0, 0, 0, 0, 497, 127, 1426, 490, 1688, 615, 668, 586, 0, 0, 0, 0, 0, 0, 0, 0, 568, 367, 917, 488, 235, 626, 0, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 416, 169, 306, 50, 1243, 1695, 7, 556, 572, 0, 0, 0, 0, 0, 0, 0, 30, 2388, 769, 146, 79, 273, 962, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1912, 63, 106, 201, 81, 756, 261, 2073, 0, 0, 0, 0, 0, 0, 0, 147, 1147, 490, 624, 197, 829, 37, 102, 793, 1845, 0, 0, 0, 0, 0, 0, 962, 1693, 219, 292, 1047, 571, 1297, 636, 379, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7325, 6097, 4697, 5014, 6210, 6483, 6211, 7115] avg=6144.0000 max=7325 max_rank=0 min=4697 min_rank=2 imbalance_ratio=1.192220 max_load_reduction=41827 max_load_reduction_pct=0.850972 moved_to_echo=26019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[232, 136, 394, 787, 1588, 129, 1398, 105, 671, 253, 0, 0, 0, 0, 0, 0, 1325, 414, 1291, 504, 363, 511, 2466, 372, 0, 0, 0, 0, 0, 0, 0, 0, 213, 278, 385, 311, 76, 1118, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 570, 1796, 7, 338, 2034, 1018, 53, 122, 1034, 0, 0, 0, 0, 0, 0, 0, 1, 639, 189, 407, 500, 736, 245, 154, 0, 0, 0, 0, 0, 0, 0, 0, 654, 463, 432, 326, 1484, 390, 1320, 283, 3295, 0, 0, 0, 0, 0, 0, 0, 1233, 1187, 534, 550, 204, 3437, 554, 34, 1259, 501, 0, 0, 0, 0, 0, 0, 509, 230, 365, 478, 1121, 517, 127, 1418, 1020, 40, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5693, 7246, 2405, 6972, 2871, 8647, 9493, 5825] avg=6144.0000 max=9493 max_rank=6 min=2405 min_rank=2 imbalance_ratio=1.545085 max_load_reduction=39659 max_load_reduction_pct=0.806864 moved_to_echo=26836 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[297, 231, 509, 406, 1335, 1636, 2667, 1577, 1540, 370, 0, 0, 0, 0, 0, 0, 1983, 0, 195, 570, 1120, 611, 801, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 2114, 308, 1142, 107, 639, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 250, 842, 4, 756, 128, 1739, 6, 605, 583, 0, 0, 0, 0, 0, 0, 0, 35, 1490, 759, 227, 198, 1935, 316, 483, 0, 0, 0, 0, 0, 0, 0, 0, 640, 369, 108, 19, 418, 2236, 168, 327, 652, 0, 0, 0, 0, 0, 0, 0, 169, 259, 47, 1154, 97, 1459, 42, 13, 249, 1151, 0, 0, 0, 0, 0, 0, 582, 437, 790, 280, 37, 1181, 19, 92, 1510, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10568, 6699, 7004, 4913, 5443, 4937, 4640, 4948] avg=6144.0000 max=10568 max_rank=0 min=4640 min_rank=6 imbalance_ratio=1.720052 max_load_reduction=38584 max_load_reduction_pct=0.784993 moved_to_echo=19968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[131, 82, 291, 1167, 199, 28, 669, 75, 1133, 447, 0, 0, 0, 0, 0, 0, 2389, 15, 705, 360, 202, 434, 381, 581, 0, 0, 0, 0, 0, 0, 0, 0, 854, 94, 1083, 536, 64, 939, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 409, 2003, 11, 188, 385, 2706, 12, 696, 858, 0, 0, 0, 0, 0, 0, 0, 109, 1096, 1675, 553, 417, 1155, 1482, 614, 0, 0, 0, 0, 0, 0, 0, 0, 535, 172, 250, 1083, 19, 182, 549, 1141, 2505, 0, 0, 0, 0, 0, 0, 0, 437, 1239, 729, 1048, 41, 1313, 879, 6, 959, 846, 0, 0, 0, 0, 0, 0, 263, 155, 346, 402, 692, 1483, 173, 1974, 1821, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4222, 5067, 4223, 7268, 7101, 6436, 7497, 7338] avg=6144.0000 max=7497 max_rank=6 min=4222 min_rank=0 imbalance_ratio=1.220215 max_load_reduction=41655 max_load_reduction_pct=0.847473 moved_to_echo=28372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6144, 4480, 2560, 5248, 4352, 8064, 2560, 6784, 9088, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11904, 1152, 6528, 3072, 4608, 6016, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5632, 11776, 9088, 5376, 1280, 7424, 0, 8960, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5504, 6912, 1536, 2560, 2304, 4480, 3456, 3968, 19072, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 9088, 7936, 3456, 2432, 6784, 13824, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5632, 11776, 9088, 5376, 1280, 7424, 0, 8960, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 10240, 896, 2432, 9984, 12160, 640, 3328, 6784, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6144, 4480, 2560, 5248, 4352, 8064, 2560, 6784, 9088, 256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2432, 1408, 2432, 4992, 5504, 2560, 12928, 5760, 9472, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 10240, 896, 2432, 9984, 12160, 640, 3328, 6784, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3584, 7424, 2816, 5376, 1024, 10752, 3072, 1152, 7296, 7040, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[272, 15, 204, 1029, 1686, 184, 178, 247, 157, 688, 141, 1660, 902, 175, 61, 415, 1208, 367, 680, 287, 420, 42, 37, 1528, 1266, 1875, 31, 496, 45, 2202, 363, 2808, 77, 39, 56, 1401, 56, 517, 211, 674, 1001, 1060, 305, 230, 1907, 209, 2628, 1570, 269, 725, 78, 2404, 9, 2736, 1875, 2406, 269, 162, 41, 309, 585, 662, 333, 2679] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[153, 84, 3422, 892, 1914, 36, 100, 2189, 60, 550, 27, 319, 2226, 1602, 70, 438, 729, 1087, 4533, 2120, 632, 74, 505, 544, 641, 778, 593, 647, 195, 399, 1749, 2846, 90, 448, 645, 603, 47, 2003, 245, 27, 83, 95, 1477, 365, 476, 37, 120, 125, 15, 636, 97, 1621, 423, 642, 1076, 1065, 334, 979, 3, 741, 33, 152, 1132, 1163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[1276, 236, 734, 969, 1443, 75, 174, 744, 213, 647, 83, 1438, 1676, 2098, 20, 646, 63, 1500, 1721, 463, 890, 56, 159, 1292, 1150, 3088, 10, 1954, 56, 783, 136, 1056, 24, 411, 144, 688, 979, 638, 55, 50, 715, 111, 564, 1033, 1164, 740, 1867, 962, 107, 213, 633, 676, 750, 2548, 774, 1655, 133, 83, 39, 765, 48, 738, 2150, 846] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[1632, 42, 863, 1089, 847, 36, 78, 346, 22, 573, 72, 1043, 2324, 2092, 169, 193, 1024, 1632, 588, 863, 2818, 226, 4, 254, 1572, 874, 25, 252, 13, 2574, 495, 398, 982, 35, 56, 1260, 228, 735, 896, 264, 92, 519, 1024, 371, 812, 225, 1383, 620, 80, 772, 63, 275, 192, 1009, 1463, 2879, 200, 564, 10, 3898, 534, 551, 1211, 916] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[1091, 256, 412, 2243, 1073, 315, 351, 1048, 101, 78, 15, 1284, 357, 1959, 2, 1466, 2625, 2226, 616, 1153, 347, 213, 78, 991, 146, 1295, 109, 228, 16, 1748, 1305, 1418, 361, 1268, 108, 901, 106, 844, 549, 63, 26, 103, 256, 1971, 1550, 21, 483, 159, 68, 371, 67, 2193, 2, 2522, 773, 1100, 424, 983, 918, 111, 37, 1666, 2243, 340] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[2384, 209, 2204, 484, 1765, 376, 255, 554, 147, 911, 522, 528, 1691, 1546, 104, 175, 379, 954, 1729, 866, 62, 139, 695, 518, 1250, 1494, 37, 1411, 283, 847, 511, 583, 984, 570, 1994, 926, 189, 246, 251, 174, 45, 78, 1946, 281, 2221, 432, 1102, 406, 10, 46, 31, 2856, 485, 1201, 2323, 1621, 77, 392, 95, 108, 729, 504, 721, 495] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[1767, 713, 1359, 419, 2341, 202, 57, 1415, 170, 1801, 41, 724, 1227, 1076, 752, 142, 764, 1683, 551, 546, 793, 65, 73, 4125, 958, 786, 440, 273, 86, 2112, 196, 1876, 196, 31, 687, 459, 0, 1777, 638, 1228, 1187, 189, 961, 86, 294, 513, 811, 582, 191, 749, 57, 371, 51, 1688, 207, 2952, 551, 348, 181, 293, 101, 403, 1423, 414] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[272, 15, 204, 1029, 1686, 184, 178, 247, 157, 688, 141, 1660, 902, 175, 61, 415, 1208, 367, 680, 287, 420, 42, 37, 1528, 1266, 1875, 31, 496, 45, 2202, 363, 2808, 77, 39, 56, 1401, 56, 517, 211, 674, 1001, 1060, 305, 230, 1907, 209, 2628, 1570, 269, 725, 78, 2404, 9, 2736, 1875, 2406, 269, 162, 41, 309, 585, 662, 333, 2679] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[153, 84, 3422, 892, 1914, 36, 100, 2189, 60, 550, 27, 319, 2226, 1602, 70, 438, 729, 1087, 4533, 2120, 632, 74, 505, 544, 641, 778, 593, 647, 195, 399, 1749, 2846, 90, 448, 645, 603, 47, 2003, 245, 27, 83, 95, 1477, 365, 476, 37, 120, 125, 15, 636, 97, 1621, 423, 642, 1076, 1065, 334, 979, 3, 741, 33, 152, 1132, 1163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[924, 281, 665, 2179, 639, 505, 203, 168, 2, 857, 41, 974, 1171, 676, 16, 658, 290, 1714, 4141, 503, 219, 425, 2, 2589, 907, 2159, 633, 290, 5, 1814, 609, 1030, 24, 67, 37, 155, 553, 1584, 114, 1056, 708, 971, 91, 596, 454, 164, 489, 50, 43, 553, 310, 778, 26, 705, 849, 3464, 202, 872, 13, 873, 247, 1412, 3562, 841] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[1276, 236, 734, 969, 1443, 75, 174, 744, 213, 647, 83, 1438, 1676, 2098, 20, 646, 63, 1500, 1721, 463, 890, 56, 159, 1292, 1150, 3088, 10, 1954, 56, 783, 136, 1056, 24, 411, 144, 688, 979, 638, 55, 50, 715, 111, 564, 1033, 1164, 740, 1867, 962, 107, 213, 633, 676, 750, 2548, 774, 1655, 133, 83, 39, 765, 48, 738, 2150, 846] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[2384, 209, 2204, 484, 1765, 376, 255, 554, 147, 911, 522, 528, 1691, 1546, 104, 175, 379, 954, 1729, 866, 62, 139, 695, 518, 1250, 1494, 37, 1411, 283, 847, 511, 583, 984, 570, 1994, 926, 189, 246, 251, 174, 45, 78, 1946, 281, 2221, 432, 1102, 406, 10, 46, 31, 2856, 485, 1201, 2323, 1621, 77, 392, 95, 108, 729, 504, 721, 495] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[1632, 42, 863, 1089, 847, 36, 78, 346, 22, 573, 72, 1043, 2324, 2092, 169, 193, 1024, 1632, 588, 863, 2818, 226, 4, 254, 1572, 874, 25, 252, 13, 2574, 495, 398, 982, 35, 56, 1260, 228, 735, 896, 264, 92, 519, 1024, 371, 812, 225, 1383, 620, 80, 772, 63, 275, 192, 1009, 1463, 2879, 200, 564, 10, 3898, 534, 551, 1211, 916] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[272, 15, 204, 1029, 1270, 184, 178, 247, 0, 0, 0, 0, 0, 0, 0, 0, 157, 688, 141, 1660, 902, 175, 61, 415, 416, 395, 0, 0, 0, 0, 0, 0, 1208, 367, 154, 287, 420, 42, 37, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 1875, 31, 496, 45, 603, 363, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 77, 39, 56, 1401, 56, 517, 211, 674, 526, 724, 0, 0, 0, 0, 0, 0, 1001, 1060, 305, 230, 1907, 209, 2628, 1570, 633, 268, 0, 0, 0, 0, 0, 0, 269, 725, 78, 2404, 9, 2736, 1875, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 269, 162, 41, 309, 585, 662, 333, 2679, 936, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3399, 5010, 4043, 7487, 4281, 9811, 9145, 5976] avg=6144.0000 max=9811 max_rank=5 min=3399 min_rank=0 imbalance_ratio=1.596842 max_load_reduction=39341 max_load_reduction_pct=0.800395 moved_to_echo=29213 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[153, 84, 3422, 892, 1441, 36, 100, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 60, 550, 27, 319, 2226, 1602, 70, 438, 473, 71, 0, 0, 0, 0, 0, 0, 729, 1087, 1025, 2120, 632, 74, 505, 544, 0, 0, 0, 0, 0, 0, 0, 0, 641, 778, 593, 647, 195, 111, 1749, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 90, 448, 645, 603, 47, 2003, 245, 27, 3508, 320, 0, 0, 0, 0, 0, 0, 83, 95, 1477, 365, 476, 37, 120, 125, 280, 48, 0, 0, 0, 0, 0, 0, 15, 636, 97, 1621, 423, 642, 1076, 465, 0, 0, 0, 0, 0, 0, 0, 0, 334, 979, 3, 741, 33, 152, 1132, 1163, 169, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 5836, 6716, 7560, 7936, 3106, 4975, 4706] avg=6144.0000 max=8317 max_rank=0 min=3106 min_rank=5 imbalance_ratio=1.353678 max_load_reduction=40835 max_load_reduction_pct=0.830790 moved_to_echo=20723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1767, 713, 1359, 419, 1763, 202, 57, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 170, 1801, 41, 724, 1227, 1076, 752, 142, 578, 379, 0, 0, 0, 0, 0, 0, 764, 1683, 125, 546, 793, 65, 73, 4125, 0, 0, 0, 0, 0, 0, 0, 0, 958, 786, 440, 273, 86, 578, 196, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 196, 31, 687, 459, 0, 1777, 638, 1228, 426, 888, 0, 0, 0, 0, 0, 0, 1187, 189, 961, 86, 294, 513, 811, 582, 777, 257, 0, 0, 0, 0, 0, 0, 191, 749, 57, 371, 51, 1688, 207, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 551, 348, 181, 293, 101, 403, 1423, 414, 898, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7695, 6890, 8174, 5193, 6330, 5657, 4601, 4612] avg=6144.0000 max=8174 max_rank=2 min=4601 min_rank=6 imbalance_ratio=1.330404 max_load_reduction=40978 max_load_reduction_pct=0.833700 moved_to_echo=21200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1091, 256, 412, 2243, 808, 315, 351, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 101, 78, 15, 1284, 357, 1959, 2, 1466, 265, 314, 0, 0, 0, 0, 0, 0, 2625, 2226, 140, 1153, 347, 213, 78, 991, 0, 0, 0, 0, 0, 0, 0, 0, 146, 1295, 109, 228, 16, 479, 1305, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 361, 1268, 108, 901, 106, 844, 549, 63, 476, 331, 0, 0, 0, 0, 0, 0, 26, 103, 256, 1971, 1550, 21, 483, 159, 289, 212, 0, 0, 0, 0, 0, 0, 68, 371, 67, 2193, 2, 2522, 773, 480, 0, 0, 0, 0, 0, 0, 0, 0, 424, 983, 918, 111, 37, 1666, 2243, 340, 743, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6524, 5841, 7773, 4996, 5007, 5070, 6476, 7465] avg=6144.0000 max=7773 max_rank=2 min=4996 min_rank=3 imbalance_ratio=1.265137 max_load_reduction=41379 max_load_reduction_pct=0.841858 moved_to_echo=24018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[924, 281, 665, 2179, 482, 505, 203, 168, 0, 0, 0, 0, 0, 0, 0, 0, 2, 857, 41, 974, 1171, 676, 16, 658, 157, 326, 0, 0, 0, 0, 0, 0, 290, 1714, 937, 503, 219, 425, 2, 2589, 0, 0, 0, 0, 0, 0, 0, 0, 907, 2159, 633, 290, 5, 497, 609, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 24, 67, 37, 155, 553, 1584, 114, 1056, 3204, 1042, 0, 0, 0, 0, 0, 0, 708, 971, 91, 596, 454, 164, 489, 50, 912, 220, 0, 0, 0, 0, 0, 0, 43, 553, 310, 778, 26, 705, 849, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 202, 872, 13, 873, 247, 1412, 3562, 841, 771, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5407, 4878, 6679, 6130, 7836, 4655, 4774, 8793] avg=6144.0000 max=8793 max_rank=7 min=4655 min_rank=5 imbalance_ratio=1.431152 max_load_reduction=40359 max_load_reduction_pct=0.821106 moved_to_echo=26058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2384, 209, 2204, 484, 1325, 376, 255, 554, 0, 0, 0, 0, 0, 0, 0, 0, 147, 911, 522, 528, 1691, 1546, 104, 175, 440, 157, 0, 0, 0, 0, 0, 0, 379, 954, 387, 866, 62, 139, 695, 518, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 1494, 37, 1411, 283, 219, 511, 583, 0, 0, 0, 0, 0, 0, 0, 0, 984, 570, 1994, 926, 189, 246, 251, 174, 1342, 491, 0, 0, 0, 0, 0, 0, 45, 78, 1946, 281, 2221, 432, 1102, 406, 432, 106, 0, 0, 0, 0, 0, 0, 10, 46, 31, 2856, 485, 1201, 2323, 698, 0, 0, 0, 0, 0, 0, 0, 0, 77, 392, 95, 108, 729, 504, 721, 495, 365, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7791, 6221, 4000, 5788, 7167, 7049, 7650, 3486] avg=6144.0000 max=7791 max_rank=0 min=3486 min_rank=7 imbalance_ratio=1.268066 max_load_reduction=41361 max_load_reduction_pct=0.841492 moved_to_echo=25352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1632, 42, 863, 1089, 638, 36, 78, 346, 0, 0, 0, 0, 0, 0, 0, 0, 22, 573, 72, 1043, 2324, 2092, 169, 193, 209, 462, 0, 0, 0, 0, 0, 0, 1024, 1632, 133, 863, 2818, 226, 4, 254, 0, 0, 0, 0, 0, 0, 0, 0, 1572, 874, 25, 252, 13, 704, 495, 398, 0, 0, 0, 0, 0, 0, 0, 0, 982, 35, 56, 1260, 228, 735, 896, 264, 455, 866, 0, 0, 0, 0, 0, 0, 92, 519, 1024, 371, 812, 225, 1383, 620, 758, 313, 0, 0, 0, 0, 0, 0, 80, 772, 63, 275, 192, 1009, 1463, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 200, 564, 10, 3898, 534, 551, 1211, 916, 1095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4724, 7159, 6954, 4333, 5777, 6117, 5109, 8979] avg=6144.0000 max=8979 max_rank=7 min=4333 min_rank=3 imbalance_ratio=1.461426 max_load_reduction=40173 max_load_reduction_pct=0.817322 moved_to_echo=25982 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1276, 236, 734, 969, 1087, 75, 174, 744, 0, 0, 0, 0, 0, 0, 0, 0, 213, 647, 83, 1438, 1676, 2098, 20, 646, 356, 140, 0, 0, 0, 0, 0, 0, 63, 1500, 390, 463, 890, 56, 159, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 3088, 10, 1954, 56, 215, 136, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 24, 411, 144, 688, 979, 638, 55, 50, 1331, 498, 0, 0, 0, 0, 0, 0, 715, 111, 564, 1033, 1164, 740, 1867, 962, 436, 95, 0, 0, 0, 0, 0, 0, 107, 213, 633, 676, 750, 2548, 774, 721, 0, 0, 0, 0, 0, 0, 0, 0, 133, 83, 39, 765, 48, 738, 2150, 846, 333, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5295, 7317, 4813, 7665, 4818, 7687, 6422, 5135] avg=6144.0000 max=7687 max_rank=5 min=4813 min_rank=2 imbalance_ratio=1.251139 max_load_reduction=41465 max_load_reduction_pct=0.843608 moved_to_echo=24062 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1091, 256, 412, 2243, 808, 315, 351, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 101, 78, 15, 1284, 357, 1959, 2, 1466, 265, 314, 0, 0, 0, 0, 0, 0, 2625, 2226, 140, 1153, 347, 213, 78, 991, 0, 0, 0, 0, 0, 0, 0, 0, 146, 1295, 109, 228, 16, 479, 1305, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 361, 1268, 108, 901, 106, 844, 549, 63, 476, 331, 0, 0, 0, 0, 0, 0, 26, 103, 256, 1971, 1550, 21, 483, 159, 289, 212, 0, 0, 0, 0, 0, 0, 68, 371, 67, 2193, 2, 2522, 773, 480, 0, 0, 0, 0, 0, 0, 0, 0, 424, 983, 918, 111, 37, 1666, 2243, 340, 743, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6524, 5841, 7773, 4996, 5007, 5070, 6476, 7465] avg=6144.0000 max=7773 max_rank=2 min=4996 min_rank=3 imbalance_ratio=1.265137 max_load_reduction=41379 max_load_reduction_pct=0.841858 moved_to_echo=24018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[272, 15, 204, 1029, 1270, 184, 178, 247, 0, 0, 0, 0, 0, 0, 0, 0, 157, 688, 141, 1660, 902, 175, 61, 415, 416, 395, 0, 0, 0, 0, 0, 0, 1208, 367, 154, 287, 420, 42, 37, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 1875, 31, 496, 45, 603, 363, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 77, 39, 56, 1401, 56, 517, 211, 674, 526, 724, 0, 0, 0, 0, 0, 0, 1001, 1060, 305, 230, 1907, 209, 2628, 1570, 633, 268, 0, 0, 0, 0, 0, 0, 269, 725, 78, 2404, 9, 2736, 1875, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 269, 162, 41, 309, 585, 662, 333, 2679, 936, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3399, 5010, 4043, 7487, 4281, 9811, 9145, 5976] avg=6144.0000 max=9811 max_rank=5 min=3399 min_rank=0 imbalance_ratio=1.596842 max_load_reduction=39341 max_load_reduction_pct=0.800395 moved_to_echo=29213 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[153, 84, 3422, 892, 1441, 36, 100, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 60, 550, 27, 319, 2226, 1602, 70, 438, 473, 71, 0, 0, 0, 0, 0, 0, 729, 1087, 1025, 2120, 632, 74, 505, 544, 0, 0, 0, 0, 0, 0, 0, 0, 641, 778, 593, 647, 195, 111, 1749, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 90, 448, 645, 603, 47, 2003, 245, 27, 3508, 320, 0, 0, 0, 0, 0, 0, 83, 95, 1477, 365, 476, 37, 120, 125, 280, 48, 0, 0, 0, 0, 0, 0, 15, 636, 97, 1621, 423, 642, 1076, 465, 0, 0, 0, 0, 0, 0, 0, 0, 334, 979, 3, 741, 33, 152, 1132, 1163, 169, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 5836, 6716, 7560, 7936, 3106, 4975, 4706] avg=6144.0000 max=8317 max_rank=0 min=3106 min_rank=5 imbalance_ratio=1.353678 max_load_reduction=40835 max_load_reduction_pct=0.830790 moved_to_echo=20723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2384, 209, 2204, 484, 1325, 376, 255, 554, 0, 0, 0, 0, 0, 0, 0, 0, 147, 911, 522, 528, 1691, 1546, 104, 175, 440, 157, 0, 0, 0, 0, 0, 0, 379, 954, 387, 866, 62, 139, 695, 518, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 1494, 37, 1411, 283, 219, 511, 583, 0, 0, 0, 0, 0, 0, 0, 0, 984, 570, 1994, 926, 189, 246, 251, 174, 1342, 491, 0, 0, 0, 0, 0, 0, 45, 78, 1946, 281, 2221, 432, 1102, 406, 432, 106, 0, 0, 0, 0, 0, 0, 10, 46, 31, 2856, 485, 1201, 2323, 698, 0, 0, 0, 0, 0, 0, 0, 0, 77, 392, 95, 108, 729, 504, 721, 495, 365, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7791, 6221, 4000, 5788, 7167, 7049, 7650, 3486] avg=6144.0000 max=7791 max_rank=0 min=3486 min_rank=7 imbalance_ratio=1.268066 max_load_reduction=41361 max_load_reduction_pct=0.841492 moved_to_echo=25352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[924, 281, 665, 2179, 482, 505, 203, 168, 0, 0, 0, 0, 0, 0, 0, 0, 2, 857, 41, 974, 1171, 676, 16, 658, 157, 326, 0, 0, 0, 0, 0, 0, 290, 1714, 937, 503, 219, 425, 2, 2589, 0, 0, 0, 0, 0, 0, 0, 0, 907, 2159, 633, 290, 5, 497, 609, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 24, 67, 37, 155, 553, 1584, 114, 1056, 3204, 1042, 0, 0, 0, 0, 0, 0, 708, 971, 91, 596, 454, 164, 489, 50, 912, 220, 0, 0, 0, 0, 0, 0, 43, 553, 310, 778, 26, 705, 849, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 202, 872, 13, 873, 247, 1412, 3562, 841, 771, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5407, 4878, 6679, 6130, 7836, 4655, 4774, 8793] avg=6144.0000 max=8793 max_rank=7 min=4655 min_rank=5 imbalance_ratio=1.431152 max_load_reduction=40359 max_load_reduction_pct=0.821106 moved_to_echo=26058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2304, 4480, 1408, 7168, 2432, 6144, 12800, 7808, 5376, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 2944, 3840, 6400, 2176, 8448, 3072, 3584, 11392, 5248, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3968, 3200, 6656, 4992, 8960, 2432, 8960, 4480, 4608, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7936, 12416, 1920, 5632, 768, 3456, 5376, 12032, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[896, 4096, 1408, 11264, 2048, 13056, 9344, 7552, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 6144, 1024, 8064, 11648, 11264, 1280, 4224, 2944, 2304, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2304, 4480, 1408, 7168, 2432, 6144, 12800, 7808, 5376, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7168, 11264, 3328, 6912, 6272, 1280, 1664, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 6144, 1024, 8064, 11648, 11264, 1280, 4224, 2944, 2304, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9600, 1920, 9984, 9344, 8832, 1792, 1408, 6784, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7936, 12416, 1920, 5632, 768, 3456, 5376, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[896, 4096, 1408, 11264, 2048, 13056, 9344, 7552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[322, 230, 511, 19, 1198, 107, 136, 1439, 1069, 1226, 207, 1431, 723, 601, 121, 1045, 937, 1594, 766, 192, 164, 2698, 782, 249, 28, 2079, 521, 362, 1006, 1274, 1620, 891, 59, 856, 60, 969, 2633, 66, 286, 1888, 2486, 895, 24, 1679, 63, 2350, 1347, 103, 186, 1026, 1249, 278, 150, 395, 1048, 168, 14, 377, 560, 506, 1186, 0, 372, 325] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[1020, 241, 270, 12, 65, 11, 527, 976, 519, 151, 1639, 1378, 1391, 3106, 1526, 654, 1875, 2030, 474, 492, 2193, 1038, 276, 805, 169, 923, 28, 343, 3, 840, 1564, 173, 117, 614, 996, 127, 1064, 26, 730, 3001, 224, 1742, 4, 921, 1812, 1160, 17, 48, 54, 2380, 981, 161, 81, 491, 76, 4, 275, 1412, 113, 215, 2549, 19, 44, 982] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[59, 102, 870, 15, 2290, 24, 416, 211, 674, 324, 1114, 1748, 203, 120, 652, 349, 370, 403, 561, 102, 1672, 156, 2030, 315, 611, 366, 277, 353, 576, 1143, 2322, 2637, 0, 290, 21, 2124, 3896, 32, 1826, 209, 2174, 188, 25, 768, 198, 1301, 999, 474, 706, 938, 877, 65, 2583, 1395, 1248, 95, 93, 89, 749, 846, 87, 85, 1470, 236] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[171, 181, 741, 1047, 2152, 503, 406, 183, 650, 1497, 189, 681, 1133, 716, 473, 859, 6, 2289, 3452, 400, 1781, 91, 1194, 3, 140, 883, 593, 476, 2568, 1906, 2195, 733, 84, 244, 497, 926, 2067, 11, 316, 2632, 315, 359, 40, 112, 545, 948, 2805, 236, 526, 289, 119, 149, 415, 178, 473, 1739, 462, 165, 476, 450, 505, 29, 404, 344] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[147, 1790, 1657, 148, 877, 189, 369, 133, 652, 386, 357, 2044, 991, 2488, 145, 1613, 255, 1647, 186, 2187, 724, 716, 1385, 11, 1095, 744, 602, 591, 2446, 736, 3458, 2858, 51, 654, 242, 218, 2418, 244, 126, 823, 1590, 194, 4, 38, 90, 2370, 270, 15, 907, 993, 350, 440, 78, 271, 306, 98, 1181, 391, 35, 136, 524, 132, 91, 245] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[1832, 290, 474, 313, 1911, 249, 232, 242, 101, 74, 217, 1570, 116, 313, 382, 907, 3573, 3364, 364, 3577, 810, 712, 1259, 267, 669, 415, 113, 287, 83, 32, 1501, 3316, 24, 348, 329, 136, 2197, 178, 128, 1280, 70, 436, 58, 620, 395, 3301, 622, 15, 1722, 441, 467, 318, 192, 1636, 1421, 655, 442, 7, 85, 811, 964, 122, 28, 139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[434, 174, 184, 8, 1538, 49, 136, 160, 1634, 299, 2771, 1851, 535, 668, 297, 405, 405, 1972, 436, 245, 394, 1508, 254, 326, 274, 1916, 625, 404, 117, 1073, 1262, 1659, 65, 79, 182, 2114, 3291, 10, 526, 2469, 1181, 900, 19, 268, 295, 3519, 1775, 63, 574, 1627, 772, 439, 614, 162, 506, 15, 109, 6, 289, 801, 455, 11, 112, 1891] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[322, 230, 511, 19, 1198, 107, 136, 1439, 1069, 1226, 207, 1431, 723, 601, 121, 1045, 937, 1594, 766, 192, 164, 2698, 782, 249, 28, 2079, 521, 362, 1006, 1274, 1620, 891, 59, 856, 60, 969, 2633, 66, 286, 1888, 2486, 895, 24, 1679, 63, 2350, 1347, 103, 186, 1026, 1249, 278, 150, 395, 1048, 168, 14, 377, 560, 506, 1186, 0, 372, 325] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[516, 56, 824, 11, 935, 46, 248, 888, 2282, 39, 1388, 900, 110, 23, 1266, 425, 450, 134, 1002, 504, 415, 575, 373, 6, 150, 1468, 961, 431, 8, 473, 2733, 1973, 10, 205, 2, 393, 4757, 157, 14, 1056, 893, 1555, 40, 1502, 488, 3191, 2629, 1912, 1212, 53, 1449, 261, 217, 953, 938, 24, 147, 144, 365, 1739, 735, 17, 238, 243] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[322, 230, 511, 19, 1198, 107, 136, 1439, 975, 726, 245, 0, 0, 0, 0, 0, 1069, 1226, 207, 659, 723, 601, 121, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 937, 138, 766, 192, 164, 2698, 782, 249, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2079, 521, 362, 1006, 1274, 169, 891, 0, 0, 0, 0, 0, 0, 0, 0, 59, 856, 60, 969, 2133, 66, 286, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 895, 24, 1679, 63, 1375, 1347, 103, 0, 0, 0, 0, 0, 0, 0, 0, 186, 1026, 1249, 278, 150, 395, 1048, 168, 527, 500, 0, 0, 0, 0, 0, 0, 14, 377, 560, 506, 1186, 0, 372, 325, 1451, 730, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5908, 5651, 5926, 6330, 6317, 7972, 5527, 5521] avg=6144.0000 max=7972 max_rank=5 min=5521 min_rank=7 imbalance_ratio=1.297526 max_load_reduction=41180 max_load_reduction_pct=0.837809 moved_to_echo=25337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[59, 102, 870, 15, 2290, 24, 416, 211, 674, 324, 1114, 1748, 203, 120, 652, 349, 370, 403, 561, 102, 1672, 156, 2030, 315, 611, 366, 277, 353, 576, 1143, 2322, 2637, 0, 290, 21, 2124, 3896, 32, 1826, 209, 2174, 188, 25, 768, 198, 1301, 999, 474, 706, 938, 877, 65, 2583, 1395, 1248, 95, 93, 89, 749, 846, 87, 85, 1470, 236] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[1020, 241, 270, 12, 65, 11, 527, 976, 519, 151, 1639, 1378, 1391, 3106, 1526, 654, 1875, 2030, 474, 492, 2193, 1038, 276, 805, 169, 923, 28, 343, 3, 840, 1564, 173, 117, 614, 996, 127, 1064, 26, 730, 3001, 224, 1742, 4, 921, 1812, 1160, 17, 48, 54, 2380, 981, 161, 81, 491, 76, 4, 275, 1412, 113, 215, 2549, 19, 44, 982] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[434, 174, 184, 8, 1538, 49, 136, 160, 1634, 299, 2771, 1851, 535, 668, 297, 405, 405, 1972, 436, 245, 394, 1508, 254, 326, 274, 1916, 625, 404, 117, 1073, 1262, 1659, 65, 79, 182, 2114, 3291, 10, 526, 2469, 1181, 900, 19, 268, 295, 3519, 1775, 63, 574, 1627, 772, 439, 614, 162, 506, 15, 109, 6, 289, 801, 455, 11, 112, 1891] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[1832, 290, 474, 313, 1911, 249, 232, 242, 101, 74, 217, 1570, 116, 313, 382, 907, 3573, 3364, 364, 3577, 810, 712, 1259, 267, 669, 415, 113, 287, 83, 32, 1501, 3316, 24, 348, 329, 136, 2197, 178, 128, 1280, 70, 436, 58, 620, 395, 3301, 622, 15, 1722, 441, 467, 318, 192, 1636, 1421, 655, 442, 7, 85, 811, 964, 122, 28, 139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[147, 1790, 1657, 148, 877, 189, 369, 133, 983, 750, 350, 0, 0, 0, 0, 0, 652, 386, 357, 941, 991, 2488, 145, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 255, 143, 186, 2187, 724, 716, 1385, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 744, 602, 591, 2446, 736, 360, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 51, 654, 242, 218, 1959, 244, 126, 823, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 194, 4, 38, 90, 1387, 270, 15, 0, 0, 0, 0, 0, 0, 0, 0, 907, 993, 350, 440, 78, 271, 306, 98, 753, 459, 0, 0, 0, 0, 0, 0, 1181, 391, 35, 136, 524, 132, 91, 245, 3098, 754, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7393, 7573, 5607, 9432, 4317, 3588, 4655, 6587] avg=6144.0000 max=9432 max_rank=3 min=3588 min_rank=5 imbalance_ratio=1.535156 max_load_reduction=39720 max_load_reduction_pct=0.808105 moved_to_echo=19147 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[1020, 241, 270, 12, 65, 11, 527, 976, 481, 924, 236, 0, 0, 0, 0, 0, 519, 151, 1639, 635, 1391, 3106, 1526, 654, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 177, 474, 492, 2193, 1038, 276, 805, 0, 0, 0, 0, 0, 0, 0, 0, 169, 923, 28, 343, 3, 840, 163, 173, 0, 0, 0, 0, 0, 0, 0, 0, 117, 614, 996, 127, 862, 26, 730, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1742, 4, 921, 1812, 679, 17, 48, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2380, 981, 161, 81, 491, 76, 4, 507, 202, 0, 0, 0, 0, 0, 0, 275, 1412, 113, 215, 2549, 19, 44, 982, 1401, 929, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4763, 9621, 7330, 2642, 6473, 5447, 4937, 7939] avg=6144.0000 max=9621 max_rank=1 min=2642 min_rank=3 imbalance_ratio=1.565918 max_load_reduction=39531 max_load_reduction_pct=0.804260 moved_to_echo=24796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[516, 56, 824, 11, 935, 46, 248, 888, 1324, 61, 154, 0, 0, 0, 0, 0, 2282, 39, 1388, 415, 110, 23, 1266, 425, 0, 0, 0, 0, 0, 0, 0, 0, 450, 12, 1002, 504, 415, 575, 373, 6, 0, 0, 0, 0, 0, 0, 0, 0, 150, 1468, 961, 431, 8, 473, 285, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 10, 205, 2, 393, 3854, 157, 14, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 893, 1555, 40, 1502, 488, 1867, 2629, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 53, 1449, 261, 217, 953, 938, 24, 331, 903, 0, 0, 0, 0, 0, 0, 147, 144, 365, 1739, 735, 17, 238, 243, 2448, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5063, 5948, 3337, 5749, 5691, 10886, 6341, 6137] avg=6144.0000 max=10886 max_rank=5 min=3337 min_rank=2 imbalance_ratio=1.771810 max_load_reduction=38266 max_load_reduction_pct=0.778524 moved_to_echo=29055 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[171, 181, 741, 1047, 2152, 503, 406, 183, 393, 1042, 116, 0, 0, 0, 0, 0, 650, 1497, 189, 314, 1133, 716, 473, 859, 0, 0, 0, 0, 0, 0, 0, 0, 6, 199, 3452, 400, 1781, 91, 1194, 3, 0, 0, 0, 0, 0, 0, 0, 0, 140, 883, 593, 476, 2568, 1906, 229, 733, 0, 0, 0, 0, 0, 0, 0, 0, 84, 244, 497, 926, 1675, 11, 316, 2632, 0, 0, 0, 0, 0, 0, 0, 0, 315, 359, 40, 112, 545, 555, 2805, 236, 0, 0, 0, 0, 0, 0, 0, 0, 526, 289, 119, 149, 415, 178, 473, 1739, 251, 392, 0, 0, 0, 0, 0, 0, 462, 165, 476, 450, 505, 29, 404, 344, 1966, 1048, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6935, 5831, 7126, 7528, 6385, 4967, 4531, 5849] avg=6144.0000 max=7528 max_rank=3 min=4531 min_rank=6 imbalance_ratio=1.225260 max_load_reduction=41624 max_load_reduction_pct=0.846842 moved_to_echo=21732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[59, 102, 870, 15, 2290, 24, 416, 211, 539, 183, 299, 0, 0, 0, 0, 0, 674, 324, 1114, 805, 203, 120, 652, 349, 0, 0, 0, 0, 0, 0, 0, 0, 370, 36, 561, 102, 1672, 156, 2030, 315, 0, 0, 0, 0, 0, 0, 0, 0, 611, 366, 277, 353, 576, 1143, 242, 2637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 21, 2124, 3156, 32, 1826, 209, 0, 0, 0, 0, 0, 0, 0, 0, 2174, 188, 25, 768, 198, 762, 999, 474, 0, 0, 0, 0, 0, 0, 0, 0, 706, 938, 877, 65, 2583, 1395, 1248, 95, 644, 740, 0, 0, 0, 0, 0, 0, 93, 89, 749, 846, 87, 85, 1470, 236, 2080, 184, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5008, 4241, 5242, 6205, 7658, 5588, 9291, 5919] avg=6144.0000 max=9291 max_rank=6 min=4241 min_rank=1 imbalance_ratio=1.512207 max_load_reduction=39861 max_load_reduction_pct=0.810974 moved_to_echo=28456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[434, 174, 184, 8, 1538, 49, 136, 160, 1460, 898, 317, 0, 0, 0, 0, 0, 1634, 299, 2771, 852, 535, 668, 297, 405, 0, 0, 0, 0, 0, 0, 0, 0, 405, 171, 436, 245, 394, 1508, 254, 326, 0, 0, 0, 0, 0, 0, 0, 0, 274, 1916, 625, 404, 117, 1073, 132, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 65, 79, 182, 2114, 2666, 10, 526, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 900, 19, 268, 295, 2059, 1775, 63, 0, 0, 0, 0, 0, 0, 0, 0, 574, 1627, 772, 439, 614, 162, 506, 15, 682, 625, 0, 0, 0, 0, 0, 0, 109, 6, 289, 801, 455, 11, 112, 1891, 1130, 903, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5358, 7461, 3739, 6200, 8111, 6560, 6016, 5707] avg=6144.0000 max=8111 max_rank=4 min=3739 min_rank=2 imbalance_ratio=1.320150 max_load_reduction=41041 max_load_reduction_pct=0.834981 moved_to_echo=26394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[1020, 241, 270, 12, 65, 11, 527, 976, 481, 924, 236, 0, 0, 0, 0, 0, 519, 151, 1639, 635, 1391, 3106, 1526, 654, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 177, 474, 492, 2193, 1038, 276, 805, 0, 0, 0, 0, 0, 0, 0, 0, 169, 923, 28, 343, 3, 840, 163, 173, 0, 0, 0, 0, 0, 0, 0, 0, 117, 614, 996, 127, 862, 26, 730, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1742, 4, 921, 1812, 679, 17, 48, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2380, 981, 161, 81, 491, 76, 4, 507, 202, 0, 0, 0, 0, 0, 0, 275, 1412, 113, 215, 2549, 19, 44, 982, 1401, 929, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4763, 9621, 7330, 2642, 6473, 5447, 4937, 7939] avg=6144.0000 max=9621 max_rank=1 min=2642 min_rank=3 imbalance_ratio=1.565918 max_load_reduction=39531 max_load_reduction_pct=0.804260 moved_to_echo=24796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[322, 230, 511, 19, 1198, 107, 136, 1439, 975, 726, 245, 0, 0, 0, 0, 0, 1069, 1226, 207, 659, 723, 601, 121, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 937, 138, 766, 192, 164, 2698, 782, 249, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2079, 521, 362, 1006, 1274, 169, 891, 0, 0, 0, 0, 0, 0, 0, 0, 59, 856, 60, 969, 2133, 66, 286, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 895, 24, 1679, 63, 1375, 1347, 103, 0, 0, 0, 0, 0, 0, 0, 0, 186, 1026, 1249, 278, 150, 395, 1048, 168, 527, 500, 0, 0, 0, 0, 0, 0, 14, 377, 560, 506, 1186, 0, 372, 325, 1451, 730, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5908, 5651, 5926, 6330, 6317, 7972, 5527, 5521] avg=6144.0000 max=7972 max_rank=5 min=5521 min_rank=7 imbalance_ratio=1.297526 max_load_reduction=41180 max_load_reduction_pct=0.837809 moved_to_echo=25337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[1832, 290, 474, 313, 1911, 249, 232, 242, 1373, 1536, 272, 0, 0, 0, 0, 0, 101, 74, 217, 716, 116, 313, 382, 907, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 283, 364, 3577, 810, 712, 1259, 267, 0, 0, 0, 0, 0, 0, 0, 0, 669, 415, 113, 287, 83, 32, 151, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 24, 348, 329, 136, 1777, 178, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 70, 436, 58, 620, 395, 1928, 622, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1722, 441, 467, 318, 192, 1636, 1421, 655, 582, 420, 0, 0, 0, 0, 0, 0, 442, 7, 85, 811, 964, 122, 28, 139, 1350, 1545, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8724, 2826, 10845, 5066, 4200, 4144, 7854, 5493] avg=6144.0000 max=10845 max_rank=2 min=2826 min_rank=1 imbalance_ratio=1.765137 max_load_reduction=38307 max_load_reduction_pct=0.779358 moved_to_echo=21691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[171, 181, 741, 1047, 2152, 503, 406, 183, 393, 1042, 116, 0, 0, 0, 0, 0, 650, 1497, 189, 314, 1133, 716, 473, 859, 0, 0, 0, 0, 0, 0, 0, 0, 6, 199, 3452, 400, 1781, 91, 1194, 3, 0, 0, 0, 0, 0, 0, 0, 0, 140, 883, 593, 476, 2568, 1906, 229, 733, 0, 0, 0, 0, 0, 0, 0, 0, 84, 244, 497, 926, 1675, 11, 316, 2632, 0, 0, 0, 0, 0, 0, 0, 0, 315, 359, 40, 112, 545, 555, 2805, 236, 0, 0, 0, 0, 0, 0, 0, 0, 526, 289, 119, 149, 415, 178, 473, 1739, 251, 392, 0, 0, 0, 0, 0, 0, 462, 165, 476, 450, 505, 29, 404, 344, 1966, 1048, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6935, 5831, 7126, 7528, 6385, 4967, 4531, 5849] avg=6144.0000 max=7528 max_rank=3 min=4531 min_rank=6 imbalance_ratio=1.225260 max_load_reduction=41624 max_load_reduction_pct=0.846842 moved_to_echo=21732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[516, 56, 824, 11, 935, 46, 248, 888, 1324, 61, 154, 0, 0, 0, 0, 0, 2282, 39, 1388, 415, 110, 23, 1266, 425, 0, 0, 0, 0, 0, 0, 0, 0, 450, 12, 1002, 504, 415, 575, 373, 6, 0, 0, 0, 0, 0, 0, 0, 0, 150, 1468, 961, 431, 8, 473, 285, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 10, 205, 2, 393, 3854, 157, 14, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 893, 1555, 40, 1502, 488, 1867, 2629, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 53, 1449, 261, 217, 953, 938, 24, 331, 903, 0, 0, 0, 0, 0, 0, 147, 144, 365, 1739, 735, 17, 238, 243, 2448, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5063, 5948, 3337, 5749, 5691, 10886, 6341, 6137] avg=6144.0000 max=10886 max_rank=5 min=3337 min_rank=2 imbalance_ratio=1.771810 max_load_reduction=38266 max_load_reduction_pct=0.778524 moved_to_echo=29055 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7680, 4096, 7936, 5376, 5248, 8064, 4864, 6272, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7936, 1280, 7296, 7808, 8192, 7552, 7680, 2048, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4608, 3072, 5632, 1664, 11008, 1280, 2560, 4352, 7552, 6144, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7936, 1280, 7296, 7808, 8192, 7552, 7680, 2048, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8960, 6272, 256, 6016, 3968, 10624, 10496, 2944, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5888, 7808, 6272, 2176, 4352, 5504, 6016, 2816, 4352, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7680, 4096, 7936, 5376, 5248, 8064, 4864, 6272, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 2688, 2688, 5504, 7040, 512, 2816, 4480, 14976, 6272, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 8832, 3840, 3328, 6912, 7552, 1792, 14336, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 3328, 2432, 7040, 18176, 768, 3968, 13440, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.rerun_state_machine:Result validation enabled @@ -2424,38 +2424,34 @@ WARNING:megatron.core.rerun_state_machine:Result validation enabled warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") - [2026-08-26 15:39:16] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38927.9 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092139E+01 | loss scale: 1.0 | grad norm: 30.620 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 13.53 - +Number of parameters in transformer block in billions: 13.53 Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 13.74 Number of parameters in most loaded shard in billions: 13.7356 Theoretical memory footprints: weight and optimizer=98244.96 MB + [2026-08-26 16:35:18] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 39045.9 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092139E+01 | loss scale: 1.0 | grad norm: 30.620 | number of skipped iterations: 0 | number of nan iterations: 0 | [Rank 0] (after 1 iterations) memory (MB) | allocated: 40462.88671875 | max allocated: 115427.2763671875 | reserved: 117354.0 | max reserved: 117354.0 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1128, 1018, 531, 821, 911, 587, 1428, 610, 506, 1038, 1063, 577, 228, 1915, 670, 1029, 670, 545, 719, 711, 526, 490, 944, 838, 791, 844, 534, 993, 859, 695, 954, 959, 552, 598, 696, 613, 582, 443, 975, 1202, 717, 621, 506, 538, 1337, 895, 581, 415, 714, 562, 805, 801, 1411, 1120, 322, 602, 752, 527, 1126, 831, 720, 428, 753, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1134, 902, 449, 782, 562, 622, 1492, 514, 530, 1127, 1047, 544, 256, 1684, 552, 1023, 626, 588, 615, 765, 555, 454, 956, 768, 775, 1040, 460, 912, 867, 647, 910, 1017, 552, 612, 610, 555, 510, 391, 881, 1308, 720, 710, 490, 554, 1458, 943, 722, 438, 710, 606, 1010, 807, 1437, 1078, 325, 767, 707, 604, 1370, 977, 708, 353, 770, 294] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1064, 1053, 564, 770, 623, 564, 1547, 571, 588, 962, 933, 535, 270, 1761, 730, 986, 641, 624, 624, 817, 519, 439, 948, 825, 726, 971, 478, 951, 855, 671, 947, 915, 577, 857, 678, 541, 523, 396, 1029, 1131, 700, 663, 493, 486, 1376, 868, 721, 428, 708, 523, 1096, 753, 1417, 935, 378, 677, 693, 563, 1305, 965, 704, 281, 862, 353] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1145, 932, 521, 814, 1011, 599, 1530, 574, 612, 1109, 836, 622, 241, 1742, 673, 1086, 743, 667, 703, 717, 544, 377, 963, 796, 699, 989, 441, 916, 838, 588, 777, 1019, 518, 593, 620, 566, 468, 335, 891, 1255, 716, 631, 524, 708, 1409, 894, 754, 430, 736, 632, 953, 893, 1382, 971, 368, 680, 744, 568, 1085, 951, 553, 403, 747, 350] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1197, 852, 487, 795, 646, 538, 1453, 605, 627, 1191, 1074, 638, 249, 1723, 583, 1056, 706, 548, 663, 869, 531, 473, 971, 775, 730, 889, 487, 977, 833, 746, 807, 1044, 540, 851, 609, 615, 467, 378, 908, 1214, 730, 627, 516, 383, 1447, 1038, 625, 473, 877, 593, 813, 824, 1380, 835, 305, 597, 746, 592, 1283, 878, 652, 340, 801, 452] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1194, 901, 462, 748, 746, 601, 1560, 573, 615, 1079, 972, 627, 266, 1569, 656, 1145, 630, 684, 627, 805, 509, 422, 1013, 799, 727, 1086, 484, 1083, 935, 585, 892, 967, 531, 914, 648, 581, 476, 426, 958, 1227, 728, 635, 473, 453, 1435, 897, 626, 420, 721, 507, 837, 827, 1408, 921, 340, 709, 725, 640, 1153, 924, 629, 363, 753, 305] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1178, 975, 539, 742, 911, 595, 1579, 602, 601, 1101, 914, 551, 254, 1767, 621, 998, 666, 703, 718, 653, 533, 440, 1034, 860, 743, 873, 473, 1061, 853, 621, 923, 1031, 661, 680, 645, 590, 543, 412, 1000, 1177, 680, 613, 426, 665, 1453, 835, 667, 358, 711, 519, 735, 850, 1367, 976, 375, 592, 716, 509, 1106, 911, 704, 432, 792, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1199, 984, 518, 796, 841, 597, 1402, 657, 512, 1097, 1013, 616, 245, 1618, 628, 1110, 607, 666, 600, 755, 472, 462, 927, 796, 762, 814, 488, 1188, 897, 662, 934, 972, 556, 778, 652, 563, 465, 402, 1031, 1138, 759, 756, 426, 532, 1279, 918, 594, 392, 738, 506, 799, 864, 1310, 1030, 387, 631, 747, 536, 1294, 927, 686, 342, 888, 391] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1128, 1018, 531, 821, 911, 587, 751, 610, 0, 0, 0, 0, 0, 0, 0, 0, 506, 1038, 1063, 577, 228, 1077, 670, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 670, 545, 719, 711, 526, 490, 944, 838, 766, 0, 0, 0, 0, 0, 0, 0, 791, 844, 534, 569, 859, 695, 954, 959, 0, 0, 0, 0, 0, 0, 0, 0, 552, 598, 696, 613, 582, 443, 975, 1202, 468, 72, 0, 0, 0, 0, 0, 0, 717, 621, 506, 538, 1337, 895, 581, 415, 242, 115, 0, 0, 0, 0, 0, 0, 714, 562, 805, 801, 1169, 1120, 322, 602, 0, 0, 0, 0, 0, 0, 0, 0, 752, 527, 1126, 831, 720, 428, 753, 275, 309, 209, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6357, 6188, 6209, 6205, 6201, 5967, 6095, 5930] avg=6144.0000 max=6357 max_rank=0 min=5930 min_rank=7 imbalance_ratio=1.034668 max_load_reduction=42795 max_load_reduction_pct=0.870667 moved_to_echo=24193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1178, 975, 539, 742, 911, 595, 1579, 602, 601, 1101, 914, 551, 254, 1767, 621, 998, 666, 703, 718, 653, 533, 440, 1034, 860, 743, 873, 473, 1061, 853, 621, 923, 1031, 661, 680, 645, 590, 543, 412, 1000, 1177, 680, 613, 426, 665, 1453, 835, 667, 358, 711, 519, 735, 850, 1367, 976, 375, 592, 716, 509, 1106, 911, 704, 432, 792, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1064, 1053, 564, 770, 623, 564, 1547, 571, 588, 962, 933, 535, 270, 1761, 730, 986, 641, 624, 624, 817, 519, 439, 948, 825, 726, 971, 478, 951, 855, 671, 947, 915, 577, 857, 678, 541, 523, 396, 1029, 1131, 700, 663, 493, 486, 1376, 868, 721, 428, 708, 523, 1096, 753, 1417, 935, 378, 677, 693, 563, 1305, 965, 704, 281, 862, 353] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1128, 1018, 531, 821, 911, 587, 1428, 610, 506, 1038, 1063, 577, 228, 1915, 670, 1029, 670, 545, 719, 711, 526, 490, 944, 838, 791, 844, 534, 993, 859, 695, 954, 959, 552, 598, 696, 613, 582, 443, 975, 1202, 717, 621, 506, 538, 1337, 895, 581, 415, 714, 562, 805, 801, 1411, 1120, 322, 602, 752, 527, 1126, 831, 720, 428, 753, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1134, 902, 449, 782, 562, 622, 1492, 514, 530, 1127, 1047, 544, 256, 1684, 552, 1023, 626, 588, 615, 765, 555, 454, 956, 768, 775, 1040, 460, 912, 867, 647, 910, 1017, 552, 612, 610, 555, 510, 391, 881, 1308, 720, 710, 490, 554, 1458, 943, 722, 438, 710, 606, 1010, 807, 1437, 1078, 325, 767, 707, 604, 1370, 977, 708, 353, 770, 294] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1145, 932, 521, 814, 1011, 599, 804, 574, 0, 0, 0, 0, 0, 0, 0, 0, 612, 1109, 836, 622, 241, 980, 673, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 743, 667, 703, 717, 544, 377, 963, 796, 697, 0, 0, 0, 0, 0, 0, 0, 699, 989, 441, 525, 838, 588, 777, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 518, 593, 620, 566, 468, 335, 891, 1255, 502, 65, 0, 0, 0, 0, 0, 0, 716, 631, 524, 708, 1409, 894, 754, 430, 237, 106, 0, 0, 0, 0, 0, 0, 736, 632, 953, 893, 1145, 971, 368, 680, 0, 0, 0, 0, 0, 0, 0, 0, 744, 568, 1085, 951, 553, 403, 747, 350, 285, 224, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6400, 6159, 6207, 5876, 5813, 6409, 6378, 5910] avg=6144.0000 max=6409 max_rank=5 min=5813 min_rank=4 imbalance_ratio=1.043132 max_load_reduction=42743 max_load_reduction_pct=0.869609 moved_to_echo=24510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1178, 975, 539, 742, 911, 595, 830, 602, 0, 0, 0, 0, 0, 0, 0, 0, 601, 1101, 914, 551, 254, 994, 621, 998, 0, 0, 0, 0, 0, 0, 0, 0, 666, 703, 718, 653, 533, 440, 1034, 860, 707, 0, 0, 0, 0, 0, 0, 0, 743, 873, 473, 609, 853, 621, 923, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 661, 680, 645, 590, 543, 412, 1000, 1177, 518, 66, 0, 0, 0, 0, 0, 0, 680, 613, 426, 665, 1453, 835, 667, 358, 235, 122, 0, 0, 0, 0, 0, 0, 711, 519, 735, 850, 1132, 976, 375, 592, 0, 0, 0, 0, 0, 0, 0, 0, 716, 509, 1106, 911, 704, 432, 792, 339, 330, 231, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6372, 6034, 6314, 6126, 6292, 6054, 5890, 6070] avg=6144.0000 max=6372 max_rank=0 min=5890 min_rank=6 imbalance_ratio=1.037109 max_load_reduction=42780 max_load_reduction_pct=0.870361 moved_to_echo=24306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1197, 852, 487, 795, 646, 538, 764, 605, 0, 0, 0, 0, 0, 0, 0, 0, 627, 1191, 1074, 638, 249, 968, 583, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 706, 548, 663, 869, 531, 473, 971, 775, 690, 0, 0, 0, 0, 0, 0, 0, 730, 889, 487, 560, 833, 746, 807, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 540, 851, 609, 615, 467, 378, 908, 1214, 477, 65, 0, 0, 0, 0, 0, 0, 730, 627, 516, 383, 1447, 1038, 625, 473, 237, 113, 0, 0, 0, 0, 0, 0, 877, 593, 813, 824, 1143, 835, 305, 597, 0, 0, 0, 0, 0, 0, 0, 0, 746, 592, 1283, 878, 652, 340, 801, 452, 304, 212, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5884, 6386, 6226, 6096, 6124, 6189, 5987, 6260] avg=6144.0000 max=6386 max_rank=1 min=5884 min_rank=0 imbalance_ratio=1.039388 max_load_reduction=42766 max_load_reduction_pct=0.870076 moved_to_echo=24560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1134, 902, 449, 782, 562, 622, 777, 514, 0, 0, 0, 0, 0, 0, 0, 0, 530, 1127, 1047, 544, 256, 939, 552, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 626, 588, 615, 765, 555, 454, 956, 768, 678, 0, 0, 0, 0, 0, 0, 0, 775, 1040, 460, 518, 867, 647, 910, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 552, 612, 610, 555, 510, 391, 881, 1308, 493, 67, 0, 0, 0, 0, 0, 0, 720, 710, 490, 554, 1458, 943, 722, 438, 250, 108, 0, 0, 0, 0, 0, 0, 710, 606, 1010, 807, 1187, 1078, 325, 767, 0, 0, 0, 0, 0, 0, 0, 0, 707, 604, 1370, 977, 708, 353, 770, 294, 286, 222, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5742, 6018, 6005, 6234, 5979, 6393, 6490, 6291] avg=6144.0000 max=6490 max_rank=6 min=5742 min_rank=0 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=25153 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1064, 1053, 564, 770, 623, 564, 814, 571, 0, 0, 0, 0, 0, 0, 0, 0, 588, 962, 933, 535, 270, 990, 730, 986, 0, 0, 0, 0, 0, 0, 0, 0, 641, 624, 624, 817, 519, 439, 948, 825, 705, 0, 0, 0, 0, 0, 0, 0, 726, 971, 478, 545, 855, 671, 947, 915, 0, 0, 0, 0, 0, 0, 0, 0, 577, 857, 678, 541, 523, 396, 1029, 1131, 507, 66, 0, 0, 0, 0, 0, 0, 700, 663, 493, 486, 1376, 868, 721, 428, 243, 110, 0, 0, 0, 0, 0, 0, 708, 523, 1096, 753, 1174, 935, 378, 677, 0, 0, 0, 0, 0, 0, 0, 0, 693, 563, 1305, 965, 704, 281, 862, 353, 296, 226, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 5994, 6142, 6108, 6305, 6088, 6244, 6248] avg=6144.0000 max=6305 max_rank=4 min=5994 min_rank=1 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1145, 932, 521, 814, 1011, 599, 804, 574, 0, 0, 0, 0, 0, 0, 0, 0, 612, 1109, 836, 622, 241, 980, 673, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 743, 667, 703, 717, 544, 377, 963, 796, 697, 0, 0, 0, 0, 0, 0, 0, 699, 989, 441, 525, 838, 588, 777, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 518, 593, 620, 566, 468, 335, 891, 1255, 502, 65, 0, 0, 0, 0, 0, 0, 716, 631, 524, 708, 1409, 894, 754, 430, 237, 106, 0, 0, 0, 0, 0, 0, 736, 632, 953, 893, 1145, 971, 368, 680, 0, 0, 0, 0, 0, 0, 0, 0, 744, 568, 1085, 951, 553, 403, 747, 350, 285, 224, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6400, 6159, 6207, 5876, 5813, 6409, 6378, 5910] avg=6144.0000 max=6409 max_rank=5 min=5813 min_rank=4 imbalance_ratio=1.043132 max_load_reduction=42743 max_load_reduction_pct=0.869609 moved_to_echo=24510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1194, 901, 462, 748, 746, 601, 820, 573, 0, 0, 0, 0, 0, 0, 0, 0, 615, 1079, 972, 627, 266, 882, 656, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 630, 684, 627, 805, 509, 422, 1013, 799, 628, 0, 0, 0, 0, 0, 0, 0, 727, 1086, 484, 621, 935, 585, 892, 967, 0, 0, 0, 0, 0, 0, 0, 0, 531, 914, 648, 581, 476, 426, 958, 1227, 512, 59, 0, 0, 0, 0, 0, 0, 728, 635, 473, 453, 1435, 897, 626, 420, 242, 125, 0, 0, 0, 0, 0, 0, 721, 507, 837, 827, 1166, 921, 340, 709, 0, 0, 0, 0, 0, 0, 0, 0, 725, 640, 1153, 924, 629, 363, 753, 305, 337, 228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6045, 6242, 6117, 6297, 6332, 6034, 6028, 6057] avg=6144.0000 max=6332 max_rank=4 min=6028 min_rank=6 imbalance_ratio=1.030599 max_load_reduction=42820 max_load_reduction_pct=0.871175 moved_to_echo=24451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1199, 984, 518, 796, 841, 597, 737, 657, 0, 0, 0, 0, 0, 0, 0, 0, 512, 1097, 1013, 616, 245, 910, 628, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 607, 666, 600, 755, 472, 462, 927, 796, 647, 0, 0, 0, 0, 0, 0, 0, 762, 814, 488, 681, 897, 662, 934, 972, 0, 0, 0, 0, 0, 0, 0, 0, 556, 778, 652, 563, 465, 402, 1031, 1138, 460, 61, 0, 0, 0, 0, 0, 0, 759, 756, 426, 532, 1279, 918, 594, 392, 225, 137, 0, 0, 0, 0, 0, 0, 738, 506, 799, 864, 1085, 1030, 387, 631, 0, 0, 0, 0, 0, 0, 0, 0, 747, 536, 1294, 927, 686, 342, 888, 391, 370, 205, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6329, 6131, 5932, 6210, 6106, 6018, 6040, 6386] avg=6144.0000 max=6386 max_rank=7 min=5932 min_rank=2 imbalance_ratio=1.039388 max_load_reduction=42766 max_load_reduction_pct=0.870076 moved_to_echo=24550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1128, 1018, 531, 821, 911, 587, 751, 610, 0, 0, 0, 0, 0, 0, 0, 0, 506, 1038, 1063, 577, 228, 1077, 670, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 670, 545, 719, 711, 526, 490, 944, 838, 766, 0, 0, 0, 0, 0, 0, 0, 791, 844, 534, 569, 859, 695, 954, 959, 0, 0, 0, 0, 0, 0, 0, 0, 552, 598, 696, 613, 582, 443, 975, 1202, 468, 72, 0, 0, 0, 0, 0, 0, 717, 621, 506, 538, 1337, 895, 581, 415, 242, 115, 0, 0, 0, 0, 0, 0, 714, 562, 805, 801, 1169, 1120, 322, 602, 0, 0, 0, 0, 0, 0, 0, 0, 752, 527, 1126, 831, 720, 428, 753, 275, 309, 209, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6357, 6188, 6209, 6205, 6201, 5967, 6095, 5930] avg=6144.0000 max=6357 max_rank=0 min=5930 min_rank=7 imbalance_ratio=1.034668 max_load_reduction=42795 max_load_reduction_pct=0.870667 moved_to_echo=24193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1178, 975, 539, 742, 911, 595, 830, 602, 0, 0, 0, 0, 0, 0, 0, 0, 601, 1101, 914, 551, 254, 994, 621, 998, 0, 0, 0, 0, 0, 0, 0, 0, 666, 703, 718, 653, 533, 440, 1034, 860, 707, 0, 0, 0, 0, 0, 0, 0, 743, 873, 473, 609, 853, 621, 923, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 661, 680, 645, 590, 543, 412, 1000, 1177, 518, 66, 0, 0, 0, 0, 0, 0, 680, 613, 426, 665, 1453, 835, 667, 358, 235, 122, 0, 0, 0, 0, 0, 0, 711, 519, 735, 850, 1132, 976, 375, 592, 0, 0, 0, 0, 0, 0, 0, 0, 716, 509, 1106, 911, 704, 432, 792, 339, 330, 231, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6372, 6034, 6314, 6126, 6292, 6054, 5890, 6070] avg=6144.0000 max=6372 max_rank=0 min=5890 min_rank=6 imbalance_ratio=1.037109 max_load_reduction=42780 max_load_reduction_pct=0.870361 moved_to_echo=24306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -2464,45 +2460,49 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1134, 902, 449, 782, 562, 622, 777, 514, 0, 0, 0, 0, 0, 0, 0, 0, 530, 1127, 1047, 544, 256, 939, 552, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 626, 588, 615, 765, 555, 454, 956, 768, 678, 0, 0, 0, 0, 0, 0, 0, 775, 1040, 460, 518, 867, 647, 910, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 552, 612, 610, 555, 510, 391, 881, 1308, 493, 67, 0, 0, 0, 0, 0, 0, 720, 710, 490, 554, 1458, 943, 722, 438, 250, 108, 0, 0, 0, 0, 0, 0, 710, 606, 1010, 807, 1187, 1078, 325, 767, 0, 0, 0, 0, 0, 0, 0, 0, 707, 604, 1370, 977, 708, 353, 770, 294, 286, 222, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5742, 6018, 6005, 6234, 5979, 6393, 6490, 6291] avg=6144.0000 max=6490 max_rank=6 min=5742 min_rank=0 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=25153 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1199, 984, 518, 796, 841, 597, 737, 657, 0, 0, 0, 0, 0, 0, 0, 0, 512, 1097, 1013, 616, 245, 910, 628, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 607, 666, 600, 755, 472, 462, 927, 796, 647, 0, 0, 0, 0, 0, 0, 0, 762, 814, 488, 681, 897, 662, 934, 972, 0, 0, 0, 0, 0, 0, 0, 0, 556, 778, 652, 563, 465, 402, 1031, 1138, 460, 61, 0, 0, 0, 0, 0, 0, 759, 756, 426, 532, 1279, 918, 594, 392, 225, 137, 0, 0, 0, 0, 0, 0, 738, 506, 799, 864, 1085, 1030, 387, 631, 0, 0, 0, 0, 0, 0, 0, 0, 747, 536, 1294, 927, 686, 342, 888, 391, 370, 205, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6329, 6131, 5932, 6210, 6106, 6018, 6040, 6386] avg=6144.0000 max=6386 max_rank=7 min=5932 min_rank=2 imbalance_ratio=1.039388 max_load_reduction=42766 max_load_reduction_pct=0.870076 moved_to_echo=24550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4608, 8704, 7936, 4736, 2048, 7808, 5120, 8448, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 7552, 3968, 4736, 7040, 5248, 7168, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 4480, 7168, 6656, 9216, 7936, 2816, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5376, 5120, 5376, 6144, 4224, 3584, 7808, 6528, 5632, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9344, 7680, 4096, 6272, 6272, 4736, 6400, 4736, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5888, 4608, 9728, 7424, 5376, 2944, 6400, 2816, 2560, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 5376, 3968, 4352, 11264, 7296, 5376, 3456, 1920, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5376, 5120, 5376, 6144, 4224, 3584, 7808, 6528, 5632, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 4480, 7168, 6656, 9216, 7936, 2816, 5376, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 5888, 5248, 4736, 4096, 3200, 7680, 9728, 3968, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 5376, 3968, 4352, 11264, 7296, 5376, 3456, 1920, 1024, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -2522,1668 +2522,1668 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[446, 564, 553, 613, 817, 798, 721, 835, 419, 919, 1267, 517, 1151, 750, 1005, 1104, 719, 590, 715, 766, 939, 979, 1168, 579, 810, 879, 593, 478, 802, 692, 1410, 426, 1170, 484, 665, 373, 826, 328, 427, 1048, 993, 349, 461, 750, 473, 874, 321, 420, 2175, 1273, 910, 398, 864, 928, 669, 1016, 923, 556, 746, 575, 692, 1015, 829, 597] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[409, 568, 700, 635, 842, 760, 616, 758, 375, 1010, 1309, 487, 1172, 844, 924, 1081, 776, 416, 638, 958, 1082, 995, 1086, 541, 798, 834, 651, 569, 696, 745, 1321, 455, 1092, 570, 655, 390, 740, 469, 353, 978, 1364, 458, 376, 775, 464, 911, 296, 387, 1857, 1150, 808, 586, 716, 795, 636, 1117, 908, 522, 765, 685, 722, 1004, 990, 562] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[522, 466, 587, 644, 907, 687, 689, 811, 408, 901, 1496, 642, 1044, 881, 1137, 1157, 819, 562, 691, 714, 1031, 928, 1114, 787, 780, 831, 626, 511, 664, 737, 1311, 338, 1171, 479, 599, 457, 621, 402, 488, 1052, 1310, 460, 415, 733, 476, 837, 209, 396, 2000, 1146, 1015, 422, 628, 846, 592, 921, 859, 598, 866, 677, 792, 862, 759, 641] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[518, 385, 699, 718, 886, 743, 675, 728, 407, 928, 1340, 630, 1163, 966, 1075, 1095, 660, 721, 656, 695, 940, 912, 1128, 535, 818, 815, 637, 609, 746, 611, 1359, 491, 1173, 471, 624, 418, 760, 508, 532, 1123, 1310, 470, 342, 834, 625, 767, 268, 388, 1974, 1120, 792, 449, 555, 799, 689, 940, 843, 553, 806, 667, 591, 971, 786, 715] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[502, 404, 781, 686, 920, 616, 676, 743, 396, 1066, 1303, 607, 1192, 806, 936, 1107, 680, 498, 690, 758, 981, 848, 1147, 614, 710, 891, 688, 577, 828, 654, 1403, 386, 1109, 525, 694, 395, 756, 458, 417, 1034, 1215, 454, 541, 792, 556, 809, 289, 492, 2156, 1219, 824, 366, 569, 790, 648, 1041, 789, 517, 756, 703, 676, 831, 940, 697] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[433, 443, 633, 712, 1031, 943, 757, 714, 436, 947, 1306, 394, 1401, 753, 1155, 993, 754, 461, 668, 784, 1144, 895, 1118, 459, 780, 788, 543, 513, 612, 602, 1359, 468, 1218, 400, 731, 459, 712, 405, 419, 1151, 1198, 461, 259, 792, 556, 743, 317, 382, 2198, 1060, 890, 574, 689, 838, 662, 1019, 880, 605, 792, 636, 708, 871, 845, 683] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[454, 480, 628, 680, 875, 804, 788, 757, 352, 978, 1364, 501, 1230, 777, 965, 999, 731, 738, 650, 802, 1151, 915, 1238, 640, 712, 1053, 633, 486, 654, 704, 1439, 376, 1204, 489, 634, 473, 729, 466, 383, 1048, 1233, 356, 374, 750, 448, 826, 260, 422, 2085, 1115, 943, 406, 602, 778, 735, 914, 763, 573, 797, 636, 816, 934, 739, 667] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[518, 385, 699, 718, 886, 743, 675, 728, 407, 928, 1340, 630, 1163, 966, 1075, 1095, 660, 721, 656, 695, 940, 912, 1128, 535, 818, 815, 637, 609, 746, 611, 1359, 491, 1173, 471, 624, 418, 760, 508, 532, 1123, 1310, 470, 342, 834, 625, 767, 268, 388, 1974, 1120, 792, 449, 555, 799, 689, 940, 843, 553, 806, 667, 591, 971, 786, 715] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[437, 435, 673, 728, 865, 718, 720, 775, 396, 1016, 1178, 526, 1192, 765, 1075, 1081, 737, 599, 636, 905, 1078, 965, 1040, 739, 866, 880, 561, 523, 642, 714, 1303, 437, 1097, 518, 683, 448, 757, 398, 390, 1110, 1236, 446, 237, 835, 523, 820, 269, 449, 1845, 1193, 885, 551, 675, 936, 573, 1055, 724, 679, 753, 587, 737, 1083, 830, 625] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[454, 480, 628, 680, 875, 804, 788, 757, 352, 978, 1364, 501, 1230, 777, 965, 999, 731, 738, 650, 802, 1151, 915, 1238, 640, 712, 1053, 633, 486, 654, 704, 1439, 376, 1204, 489, 634, 473, 729, 466, 383, 1048, 1233, 356, 374, 750, 448, 826, 260, 422, 2085, 1115, 943, 406, 602, 778, 735, 914, 763, 573, 797, 636, 816, 934, 739, 667] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[522, 466, 587, 644, 907, 687, 689, 811, 548, 266, 0, 0, 0, 0, 0, 0, 408, 901, 131, 642, 1044, 881, 1137, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 819, 562, 691, 714, 1031, 928, 776, 787, 0, 0, 0, 0, 0, 0, 0, 0, 780, 831, 626, 511, 664, 737, 1311, 338, 163, 0, 0, 0, 0, 0, 0, 0, 1171, 479, 599, 457, 621, 402, 488, 1052, 817, 27, 0, 0, 0, 0, 0, 0, 1310, 460, 415, 733, 476, 837, 209, 396, 1264, 0, 0, 0, 0, 0, 0, 0, 470, 1146, 1015, 422, 628, 846, 592, 921, 0, 0, 0, 0, 0, 0, 0, 0, 859, 598, 866, 677, 792, 862, 759, 641, 148, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6301, 6308, 5961, 6113, 6100, 6040, 6202] avg=6144.0000 max=6308 max_rank=2 min=5961 min_rank=3 imbalance_ratio=1.026693 max_load_reduction=42844 max_load_reduction_pct=0.871663 moved_to_echo=24455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[409, 568, 700, 635, 842, 760, 616, 758, 479, 247, 0, 0, 0, 0, 0, 0, 375, 1010, 115, 487, 1172, 844, 924, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 776, 416, 638, 958, 1082, 995, 757, 541, 0, 0, 0, 0, 0, 0, 0, 0, 798, 834, 651, 569, 696, 745, 1321, 455, 159, 0, 0, 0, 0, 0, 0, 0, 1092, 570, 655, 390, 740, 469, 353, 978, 715, 26, 0, 0, 0, 0, 0, 0, 1364, 458, 376, 775, 464, 911, 296, 387, 1174, 0, 0, 0, 0, 0, 0, 0, 436, 1150, 808, 586, 716, 795, 636, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 908, 522, 765, 685, 722, 1004, 990, 562, 144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 6008, 6163, 6228, 5988, 6205, 6244, 6302] avg=6144.0000 max=6302 max_rank=7 min=5988 min_rank=4 imbalance_ratio=1.025716 max_load_reduction=42850 max_load_reduction_pct=0.871785 moved_to_echo=24739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[446, 564, 553, 613, 817, 798, 721, 835, 464, 289, 0, 0, 0, 0, 0, 0, 419, 919, 111, 517, 1151, 750, 1005, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 719, 590, 715, 766, 939, 979, 814, 579, 0, 0, 0, 0, 0, 0, 0, 0, 810, 879, 593, 478, 802, 692, 1410, 426, 171, 0, 0, 0, 0, 0, 0, 0, 1170, 484, 665, 373, 826, 328, 427, 1048, 692, 28, 0, 0, 0, 0, 0, 0, 993, 349, 461, 750, 473, 874, 321, 420, 1375, 0, 0, 0, 0, 0, 0, 0, 511, 1273, 910, 398, 864, 928, 669, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 923, 556, 746, 575, 692, 1015, 829, 597, 155, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 5976, 6101, 6261, 6041, 6016, 6569, 6088] avg=6144.0000 max=6569 max_rank=6 min=5976 min_rank=1 imbalance_ratio=1.069173 max_load_reduction=42583 max_load_reduction_pct=0.866353 moved_to_echo=24714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[518, 385, 699, 718, 886, 743, 675, 728, 491, 262, 0, 0, 0, 0, 0, 0, 407, 928, 117, 630, 1163, 966, 1075, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 660, 721, 656, 695, 940, 912, 786, 535, 0, 0, 0, 0, 0, 0, 0, 0, 818, 815, 637, 609, 746, 611, 1359, 491, 165, 0, 0, 0, 0, 0, 0, 0, 1173, 471, 624, 418, 760, 508, 532, 1123, 732, 27, 0, 0, 0, 0, 0, 0, 1310, 470, 342, 834, 625, 767, 268, 388, 1248, 0, 0, 0, 0, 0, 0, 0, 464, 1120, 792, 449, 555, 799, 689, 940, 0, 0, 0, 0, 0, 0, 0, 0, 843, 553, 806, 667, 591, 971, 786, 715, 150, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 6381, 5905, 6251, 6368, 6252, 5808, 6082] avg=6144.0000 max=6381 max_rank=1 min=5808 min_rank=6 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[433, 443, 633, 712, 1031, 943, 757, 714, 483, 296, 0, 0, 0, 0, 0, 0, 436, 947, 107, 394, 1401, 753, 1155, 993, 0, 0, 0, 0, 0, 0, 0, 0, 754, 461, 668, 784, 1144, 895, 768, 459, 0, 0, 0, 0, 0, 0, 0, 0, 780, 788, 543, 513, 612, 602, 1359, 468, 165, 0, 0, 0, 0, 0, 0, 0, 1218, 400, 731, 459, 712, 405, 419, 1151, 716, 33, 0, 0, 0, 0, 0, 0, 1198, 461, 259, 792, 556, 743, 317, 382, 1392, 0, 0, 0, 0, 0, 0, 0, 510, 1060, 890, 574, 689, 838, 662, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 880, 605, 792, 636, 708, 871, 845, 683, 152, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6445, 6186, 5933, 5830, 6244, 6100, 6242, 6172] avg=6144.0000 max=6445 max_rank=0 min=5830 min_rank=3 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=24758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[502, 404, 781, 686, 920, 616, 676, 743, 477, 286, 0, 0, 0, 0, 0, 0, 396, 1066, 114, 607, 1192, 806, 936, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 680, 498, 690, 758, 981, 848, 799, 614, 0, 0, 0, 0, 0, 0, 0, 0, 710, 891, 688, 577, 828, 654, 1403, 386, 168, 0, 0, 0, 0, 0, 0, 0, 1109, 525, 694, 395, 756, 458, 417, 1034, 712, 28, 0, 0, 0, 0, 0, 0, 1215, 454, 541, 792, 556, 809, 289, 492, 1363, 0, 0, 0, 0, 0, 0, 0, 507, 1219, 824, 366, 569, 790, 648, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 789, 517, 756, 703, 676, 831, 940, 697, 152, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 6224, 5868, 6305, 6128, 6511, 5964, 6061] avg=6144.0000 max=6511 max_rank=5 min=5868 min_rank=2 imbalance_ratio=1.059733 max_load_reduction=42641 max_load_reduction_pct=0.867533 moved_to_echo=24664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[437, 435, 673, 728, 865, 718, 720, 775, 431, 245, 0, 0, 0, 0, 0, 0, 396, 1016, 104, 526, 1192, 765, 1075, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 737, 599, 636, 905, 1078, 965, 725, 739, 0, 0, 0, 0, 0, 0, 0, 0, 866, 880, 561, 523, 642, 714, 1303, 437, 152, 0, 0, 0, 0, 0, 0, 0, 1097, 518, 683, 448, 757, 398, 390, 1110, 643, 25, 0, 0, 0, 0, 0, 0, 1236, 446, 237, 835, 523, 820, 269, 449, 1166, 0, 0, 0, 0, 0, 0, 0, 434, 1193, 885, 551, 675, 936, 573, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 724, 679, 753, 587, 737, 1083, 830, 625, 138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6027, 6155, 6384, 6078, 6069, 5981, 6302, 6156] avg=6144.0000 max=6384 max_rank=2 min=5981 min_rank=5 imbalance_ratio=1.039062 max_load_reduction=42768 max_load_reduction_pct=0.870117 moved_to_echo=24508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[518, 385, 699, 718, 886, 743, 675, 728, 491, 262, 0, 0, 0, 0, 0, 0, 407, 928, 117, 630, 1163, 966, 1075, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 660, 721, 656, 695, 940, 912, 786, 535, 0, 0, 0, 0, 0, 0, 0, 0, 818, 815, 637, 609, 746, 611, 1359, 491, 165, 0, 0, 0, 0, 0, 0, 0, 1173, 471, 624, 418, 760, 508, 532, 1123, 732, 27, 0, 0, 0, 0, 0, 0, 1310, 470, 342, 834, 625, 767, 268, 388, 1248, 0, 0, 0, 0, 0, 0, 0, 464, 1120, 792, 449, 555, 799, 689, 940, 0, 0, 0, 0, 0, 0, 0, 0, 843, 553, 806, 667, 591, 971, 786, 715, 150, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 6381, 5905, 6251, 6368, 6252, 5808, 6082] avg=6144.0000 max=6381 max_rank=1 min=5808 min_rank=6 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[409, 568, 700, 635, 842, 760, 616, 758, 479, 247, 0, 0, 0, 0, 0, 0, 375, 1010, 115, 487, 1172, 844, 924, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 776, 416, 638, 958, 1082, 995, 757, 541, 0, 0, 0, 0, 0, 0, 0, 0, 798, 834, 651, 569, 696, 745, 1321, 455, 159, 0, 0, 0, 0, 0, 0, 0, 1092, 570, 655, 390, 740, 469, 353, 978, 715, 26, 0, 0, 0, 0, 0, 0, 1364, 458, 376, 775, 464, 911, 296, 387, 1174, 0, 0, 0, 0, 0, 0, 0, 436, 1150, 808, 586, 716, 795, 636, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 908, 522, 765, 685, 722, 1004, 990, 562, 144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 6008, 6163, 6228, 5988, 6205, 6244, 6302] avg=6144.0000 max=6302 max_rank=7 min=5988 min_rank=4 imbalance_ratio=1.025716 max_load_reduction=42850 max_load_reduction_pct=0.871785 moved_to_echo=24739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[454, 480, 628, 680, 875, 804, 788, 757, 500, 277, 0, 0, 0, 0, 0, 0, 352, 978, 119, 501, 1230, 777, 965, 999, 0, 0, 0, 0, 0, 0, 0, 0, 731, 738, 650, 802, 1151, 915, 863, 640, 0, 0, 0, 0, 0, 0, 0, 0, 712, 1053, 633, 486, 654, 704, 1439, 376, 181, 0, 0, 0, 0, 0, 0, 0, 1204, 489, 634, 473, 729, 466, 383, 1048, 745, 30, 0, 0, 0, 0, 0, 0, 1233, 356, 374, 750, 448, 826, 260, 422, 1318, 0, 0, 0, 0, 0, 0, 0, 490, 1115, 943, 406, 602, 778, 735, 914, 0, 0, 0, 0, 0, 0, 0, 0, 763, 573, 797, 636, 816, 934, 739, 667, 164, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6243, 5921, 6490, 6238, 6201, 5987, 5983, 6089] avg=6144.0000 max=6490 max_rank=2 min=5921 min_rank=1 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[437, 435, 673, 728, 865, 718, 720, 775, 431, 245, 0, 0, 0, 0, 0, 0, 396, 1016, 104, 526, 1192, 765, 1075, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 737, 599, 636, 905, 1078, 965, 725, 739, 0, 0, 0, 0, 0, 0, 0, 0, 866, 880, 561, 523, 642, 714, 1303, 437, 152, 0, 0, 0, 0, 0, 0, 0, 1097, 518, 683, 448, 757, 398, 390, 1110, 643, 25, 0, 0, 0, 0, 0, 0, 1236, 446, 237, 835, 523, 820, 269, 449, 1166, 0, 0, 0, 0, 0, 0, 0, 434, 1193, 885, 551, 675, 936, 573, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 724, 679, 753, 587, 737, 1083, 830, 625, 138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6027, 6155, 6384, 6078, 6069, 5981, 6302, 6156] avg=6144.0000 max=6384 max_rank=2 min=5981 min_rank=5 imbalance_ratio=1.039062 max_load_reduction=42768 max_load_reduction_pct=0.870117 moved_to_echo=24508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[433, 443, 633, 712, 1031, 943, 757, 714, 483, 296, 0, 0, 0, 0, 0, 0, 436, 947, 107, 394, 1401, 753, 1155, 993, 0, 0, 0, 0, 0, 0, 0, 0, 754, 461, 668, 784, 1144, 895, 768, 459, 0, 0, 0, 0, 0, 0, 0, 0, 780, 788, 543, 513, 612, 602, 1359, 468, 165, 0, 0, 0, 0, 0, 0, 0, 1218, 400, 731, 459, 712, 405, 419, 1151, 716, 33, 0, 0, 0, 0, 0, 0, 1198, 461, 259, 792, 556, 743, 317, 382, 1392, 0, 0, 0, 0, 0, 0, 0, 510, 1060, 890, 574, 689, 838, 662, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 880, 605, 792, 636, 708, 871, 845, 683, 152, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6445, 6186, 5933, 5830, 6244, 6100, 6242, 6172] avg=6144.0000 max=6445 max_rank=0 min=5830 min_rank=3 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=24758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3840, 3840, 5376, 5504, 7168, 6144, 5760, 6144, 3968, 2176, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6784, 4608, 6400, 5248, 5760, 7680, 6784, 5248, 1280, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6400, 7040, 4992, 4352, 5760, 5504, 11008, 3456, 1408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9984, 3456, 3072, 6272, 4224, 6656, 2304, 3456, 10368, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9344, 3968, 5376, 3456, 6016, 3456, 3456, 8576, 5888, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3840, 9344, 7168, 3840, 5376, 6784, 5248, 8064, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 7808, 1024, 4352, 9600, 6656, 8320, 8704, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5888, 4608, 5376, 6400, 8448, 7552, 6400, 4992, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9344, 3968, 5376, 3456, 6016, 3456, 3456, 8576, 5888, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9984, 3456, 3072, 6272, 4224, 6656, 2304, 3456, 10368, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3840, 9344, 7168, 3840, 5376, 6784, 5248, 8064, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[745, 973, 393, 483, 604, 409, 517, 700, 417, 887, 280, 1638, 1006, 724, 1000, 634, 1103, 591, 1040, 1006, 852, 424, 973, 779, 629, 787, 1283, 805, 788, 308, 607, 125, 1497, 847, 1397, 781, 1249, 836, 659, 708, 940, 461, 321, 1266, 823, 972, 580, 642, 318, 989, 1558, 575, 972, 739, 380, 473, 1056, 930, 225, 430, 467, 973, 827, 751] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[682, 980, 442, 417, 640, 438, 424, 527, 469, 1215, 319, 1662, 896, 943, 897, 608, 1230, 550, 869, 1081, 718, 383, 1144, 883, 561, 821, 1205, 658, 785, 322, 563, 218, 1452, 1016, 1183, 925, 913, 744, 855, 743, 853, 452, 275, 1332, 906, 1093, 606, 730, 357, 899, 1470, 621, 958, 728, 320, 468, 1275, 828, 218, 521, 405, 831, 824, 801] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[620, 989, 356, 422, 588, 333, 529, 815, 444, 1093, 354, 1649, 1081, 507, 939, 764, 1271, 499, 943, 1003, 729, 414, 877, 911, 458, 876, 1236, 773, 971, 259, 603, 240, 1428, 1082, 1320, 682, 1338, 687, 699, 913, 874, 558, 301, 1141, 886, 904, 546, 755, 170, 872, 1665, 670, 930, 685, 336, 522, 978, 892, 210, 534, 638, 1081, 620, 689] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[573, 740, 502, 487, 818, 293, 676, 590, 524, 1148, 305, 1434, 842, 524, 938, 687, 1269, 653, 889, 1188, 726, 418, 844, 911, 594, 652, 935, 795, 756, 284, 787, 217, 1451, 801, 1316, 867, 1114, 747, 656, 765, 954, 569, 318, 1051, 793, 1287, 572, 609, 394, 1050, 1591, 733, 942, 666, 328, 370, 1699, 960, 162, 415, 539, 1020, 632, 782] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[579, 1006, 370, 483, 565, 526, 437, 663, 508, 936, 316, 1581, 870, 610, 1084, 719, 1186, 712, 967, 900, 900, 384, 950, 848, 655, 803, 1277, 861, 780, 320, 634, 265, 1348, 785, 1361, 689, 1088, 748, 734, 835, 792, 446, 378, 1437, 777, 871, 530, 973, 439, 1012, 1303, 595, 980, 766, 374, 543, 884, 872, 342, 570, 590, 927, 707, 761] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[719, 913, 457, 422, 635, 494, 480, 738, 476, 805, 282, 1551, 769, 662, 959, 770, 1127, 571, 1209, 1072, 756, 436, 780, 902, 602, 772, 1195, 872, 783, 312, 647, 154, 1447, 966, 1622, 543, 1174, 731, 623, 1015, 815, 555, 320, 1295, 857, 972, 431, 837, 315, 957, 1527, 800, 785, 729, 370, 599, 1095, 877, 232, 442, 452, 967, 771, 709] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[643, 990, 336, 568, 615, 348, 532, 595, 536, 1090, 300, 1485, 868, 1049, 935, 594, 1339, 720, 879, 1023, 826, 341, 935, 894, 577, 652, 1243, 702, 752, 302, 654, 96, 1453, 810, 1254, 851, 1050, 855, 560, 1036, 871, 567, 417, 1375, 1054, 870, 516, 705, 307, 1008, 1448, 729, 1074, 697, 269, 447, 1054, 896, 279, 517, 387, 866, 735, 776] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[620, 989, 356, 422, 588, 333, 529, 815, 444, 1093, 354, 1649, 1081, 507, 939, 764, 1271, 499, 943, 1003, 729, 414, 877, 911, 458, 876, 1236, 773, 971, 259, 603, 240, 1428, 1082, 1320, 682, 1338, 687, 699, 913, 874, 558, 301, 1141, 886, 904, 546, 755, 170, 872, 1665, 670, 930, 685, 336, 522, 978, 892, 210, 534, 638, 1081, 620, 689] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[745, 973, 393, 483, 604, 409, 517, 700, 417, 887, 280, 1638, 1006, 724, 1000, 634, 1103, 591, 1040, 1006, 852, 424, 973, 779, 629, 787, 1283, 805, 788, 308, 607, 125, 1497, 847, 1397, 781, 1249, 836, 659, 708, 940, 461, 321, 1266, 823, 972, 580, 642, 318, 989, 1558, 575, 972, 739, 380, 473, 1056, 930, 225, 430, 467, 973, 827, 751] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[682, 980, 442, 417, 640, 438, 424, 527, 469, 1215, 319, 1662, 896, 943, 897, 608, 1230, 550, 869, 1081, 718, 383, 1144, 883, 561, 821, 1205, 658, 785, 322, 563, 218, 1452, 1016, 1183, 925, 913, 744, 855, 743, 853, 452, 275, 1332, 906, 1093, 606, 730, 357, 899, 1470, 621, 958, 728, 320, 468, 1275, 828, 218, 521, 405, 831, 824, 801] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[757, 1039, 442, 448, 656, 481, 487, 688, 413, 1050, 322, 1699, 1056, 536, 951, 679, 1128, 512, 917, 936, 756, 368, 1021, 808, 618, 647, 1129, 1123, 624, 269, 625, 178, 1511, 983, 1715, 529, 1100, 751, 771, 721, 866, 515, 336, 1306, 879, 1059, 505, 601, 313, 922, 1556, 691, 823, 729, 384, 524, 1109, 988, 254, 527, 410, 954, 722, 735] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[573, 740, 502, 487, 818, 293, 676, 590, 524, 1148, 305, 1434, 842, 524, 938, 687, 1269, 653, 889, 1188, 726, 418, 844, 911, 594, 652, 935, 795, 756, 284, 787, 217, 1451, 801, 1316, 867, 1114, 747, 656, 765, 954, 569, 318, 1051, 793, 1287, 572, 609, 394, 1050, 1591, 733, 942, 666, 328, 370, 1699, 960, 162, 415, 539, 1020, 632, 782] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[745, 973, 393, 483, 604, 409, 517, 700, 1464, 0, 0, 0, 0, 0, 0, 0, 417, 887, 280, 1105, 1006, 724, 1000, 634, 0, 0, 0, 0, 0, 0, 0, 0, 518, 591, 1040, 1006, 852, 424, 973, 779, 0, 0, 0, 0, 0, 0, 0, 0, 629, 787, 1283, 805, 788, 308, 607, 125, 585, 226, 33, 0, 0, 0, 0, 0, 0, 847, 1075, 781, 1249, 836, 659, 708, 0, 0, 0, 0, 0, 0, 0, 0, 940, 461, 321, 1266, 823, 972, 580, 642, 6, 0, 0, 0, 0, 0, 0, 0, 318, 989, 1558, 575, 972, 739, 380, 473, 173, 0, 0, 0, 0, 0, 0, 0, 1056, 930, 225, 430, 467, 973, 827, 751, 307, 143, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6288, 6053, 6183, 6176, 6155, 6011, 6177, 6109] avg=6144.0000 max=6288 max_rank=0 min=6011 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=42864 max_load_reduction_pct=0.872070 moved_to_echo=24452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[620, 989, 356, 422, 588, 333, 529, 815, 1396, 0, 0, 0, 0, 0, 0, 0, 444, 1093, 354, 1113, 1081, 507, 939, 764, 0, 0, 0, 0, 0, 0, 0, 0, 597, 499, 943, 1003, 729, 414, 877, 911, 0, 0, 0, 0, 0, 0, 0, 0, 458, 876, 1236, 773, 971, 259, 603, 240, 674, 227, 32, 0, 0, 0, 0, 0, 0, 1082, 1015, 682, 1338, 687, 699, 913, 0, 0, 0, 0, 0, 0, 0, 0, 874, 558, 301, 1141, 886, 904, 546, 755, 6, 0, 0, 0, 0, 0, 0, 0, 170, 872, 1665, 670, 930, 685, 336, 522, 164, 0, 0, 0, 0, 0, 0, 0, 978, 892, 210, 534, 638, 1081, 620, 689, 309, 135, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6048, 6295, 5973, 6349, 6416, 5971, 6014, 6086] avg=6144.0000 max=6416 max_rank=4 min=5971 min_rank=5 imbalance_ratio=1.044271 max_load_reduction=42736 max_load_reduction_pct=0.869466 moved_to_echo=24487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[643, 990, 336, 568, 615, 348, 532, 595, 1422, 0, 0, 0, 0, 0, 0, 0, 536, 1090, 300, 1002, 868, 1049, 935, 594, 0, 0, 0, 0, 0, 0, 0, 0, 629, 720, 879, 1023, 826, 341, 935, 894, 0, 0, 0, 0, 0, 0, 0, 0, 577, 652, 1243, 702, 752, 302, 654, 96, 710, 205, 31, 0, 0, 0, 0, 0, 0, 810, 964, 851, 1050, 855, 560, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 871, 567, 417, 1375, 1054, 870, 516, 705, 6, 0, 0, 0, 0, 0, 0, 0, 307, 1008, 1448, 729, 1074, 697, 269, 447, 155, 0, 0, 0, 0, 0, 0, 0, 1054, 896, 279, 517, 387, 866, 735, 776, 278, 129, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6049, 6374, 6247, 5924, 6126, 6381, 6134, 5917] avg=6144.0000 max=6381 max_rank=5 min=5917 min_rank=7 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[643, 990, 336, 568, 615, 348, 532, 595, 536, 1090, 300, 1485, 868, 1049, 935, 594, 1339, 720, 879, 1023, 826, 341, 935, 894, 577, 652, 1243, 702, 752, 302, 654, 96, 1453, 810, 1254, 851, 1050, 855, 560, 1036, 871, 567, 417, 1375, 1054, 870, 516, 705, 307, 1008, 1448, 729, 1074, 697, 269, 447, 1054, 896, 279, 517, 387, 866, 735, 776] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[573, 740, 502, 487, 818, 293, 676, 590, 1419, 0, 0, 0, 0, 0, 0, 0, 524, 1148, 305, 967, 842, 524, 938, 687, 0, 0, 0, 0, 0, 0, 0, 0, 596, 653, 889, 1188, 726, 418, 844, 911, 0, 0, 0, 0, 0, 0, 0, 0, 594, 652, 935, 795, 756, 284, 787, 217, 673, 198, 32, 0, 0, 0, 0, 0, 0, 801, 1012, 867, 1114, 747, 656, 765, 0, 0, 0, 0, 0, 0, 0, 0, 954, 569, 318, 1051, 793, 1287, 572, 609, 6, 0, 0, 0, 0, 0, 0, 0, 394, 1050, 1591, 733, 942, 666, 328, 370, 163, 0, 0, 0, 0, 0, 0, 0, 1699, 960, 162, 415, 539, 1020, 632, 782, 269, 135, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6098, 5935, 6225, 5923, 5962, 6159, 6237, 6613] avg=6144.0000 max=6613 max_rank=7 min=5923 min_rank=3 imbalance_ratio=1.076335 max_load_reduction=42539 max_load_reduction_pct=0.865458 moved_to_echo=24971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[579, 1006, 370, 483, 565, 526, 437, 663, 1318, 0, 0, 0, 0, 0, 0, 0, 508, 936, 316, 1067, 870, 610, 1084, 719, 0, 0, 0, 0, 0, 0, 0, 0, 557, 712, 967, 900, 900, 384, 950, 848, 0, 0, 0, 0, 0, 0, 0, 0, 655, 803, 1277, 861, 780, 320, 634, 265, 629, 218, 30, 0, 0, 0, 0, 0, 0, 785, 1046, 689, 1088, 748, 734, 835, 0, 0, 0, 0, 0, 0, 0, 0, 792, 446, 378, 1437, 777, 871, 530, 973, 6, 0, 0, 0, 0, 0, 0, 0, 439, 1012, 1303, 595, 980, 766, 374, 543, 169, 0, 0, 0, 0, 0, 0, 0, 884, 872, 342, 570, 590, 927, 707, 761, 296, 140, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5947, 6110, 6218, 6472, 5925, 6210, 6181, 6089] avg=6144.0000 max=6472 max_rank=3 min=5925 min_rank=4 imbalance_ratio=1.053385 max_load_reduction=42680 max_load_reduction_pct=0.868327 moved_to_echo=24405 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[719, 913, 457, 422, 635, 494, 480, 738, 1416, 0, 0, 0, 0, 0, 0, 0, 476, 805, 282, 1046, 769, 662, 959, 770, 0, 0, 0, 0, 0, 0, 0, 0, 529, 571, 1209, 1072, 756, 436, 780, 902, 0, 0, 0, 0, 0, 0, 0, 0, 602, 772, 1195, 872, 783, 312, 647, 154, 598, 214, 31, 0, 0, 0, 0, 0, 0, 966, 1248, 543, 1174, 731, 623, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 815, 555, 320, 1295, 857, 972, 431, 837, 7, 0, 0, 0, 0, 0, 0, 0, 315, 957, 1527, 800, 785, 729, 370, 599, 201, 0, 0, 0, 0, 0, 0, 0, 1095, 877, 232, 442, 452, 967, 771, 709, 291, 166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 5769, 6255, 6180, 6300, 6089, 6283, 6002] avg=6144.0000 max=6300 max_rank=4 min=5769 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=42852 max_load_reduction_pct=0.871826 moved_to_echo=24674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[682, 980, 442, 417, 640, 438, 424, 527, 1421, 0, 0, 0, 0, 0, 0, 0, 469, 1215, 319, 1115, 896, 943, 897, 608, 0, 0, 0, 0, 0, 0, 0, 0, 575, 550, 869, 1081, 718, 383, 1144, 883, 0, 0, 0, 0, 0, 0, 0, 0, 561, 821, 1205, 658, 785, 322, 563, 218, 655, 232, 31, 0, 0, 0, 0, 0, 0, 1016, 899, 925, 913, 744, 855, 743, 0, 0, 0, 0, 0, 0, 0, 0, 853, 452, 275, 1332, 906, 1093, 606, 730, 9, 0, 0, 0, 0, 0, 0, 0, 357, 899, 1470, 621, 958, 728, 320, 468, 150, 0, 0, 0, 0, 0, 0, 0, 1275, 828, 218, 521, 405, 831, 824, 801, 315, 125, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5971, 6462, 6203, 6051, 6095, 6256, 5971, 6143] avg=6144.0000 max=6462 max_rank=1 min=5971 min_rank=0 imbalance_ratio=1.051758 max_load_reduction=42690 max_load_reduction_pct=0.868530 moved_to_echo=24465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[573, 740, 502, 487, 818, 293, 676, 590, 1419, 0, 0, 0, 0, 0, 0, 0, 524, 1148, 305, 967, 842, 524, 938, 687, 0, 0, 0, 0, 0, 0, 0, 0, 596, 653, 889, 1188, 726, 418, 844, 911, 0, 0, 0, 0, 0, 0, 0, 0, 594, 652, 935, 795, 756, 284, 787, 217, 673, 198, 32, 0, 0, 0, 0, 0, 0, 801, 1012, 867, 1114, 747, 656, 765, 0, 0, 0, 0, 0, 0, 0, 0, 954, 569, 318, 1051, 793, 1287, 572, 609, 6, 0, 0, 0, 0, 0, 0, 0, 394, 1050, 1591, 733, 942, 666, 328, 370, 163, 0, 0, 0, 0, 0, 0, 0, 1699, 960, 162, 415, 539, 1020, 632, 782, 269, 135, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6098, 5935, 6225, 5923, 5962, 6159, 6237, 6613] avg=6144.0000 max=6613 max_rank=7 min=5923 min_rank=3 imbalance_ratio=1.076335 max_load_reduction=42539 max_load_reduction_pct=0.865458 moved_to_echo=24971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[745, 973, 393, 483, 604, 409, 517, 700, 1464, 0, 0, 0, 0, 0, 0, 0, 417, 887, 280, 1105, 1006, 724, 1000, 634, 0, 0, 0, 0, 0, 0, 0, 0, 518, 591, 1040, 1006, 852, 424, 973, 779, 0, 0, 0, 0, 0, 0, 0, 0, 629, 787, 1283, 805, 788, 308, 607, 125, 585, 226, 33, 0, 0, 0, 0, 0, 0, 847, 1075, 781, 1249, 836, 659, 708, 0, 0, 0, 0, 0, 0, 0, 0, 940, 461, 321, 1266, 823, 972, 580, 642, 6, 0, 0, 0, 0, 0, 0, 0, 318, 989, 1558, 575, 972, 739, 380, 473, 173, 0, 0, 0, 0, 0, 0, 0, 1056, 930, 225, 430, 467, 973, 827, 751, 307, 143, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6288, 6053, 6183, 6176, 6155, 6011, 6177, 6109] avg=6144.0000 max=6288 max_rank=0 min=6011 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=42864 max_load_reduction_pct=0.872070 moved_to_echo=24452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[757, 1039, 442, 448, 656, 481, 487, 688, 1479, 0, 0, 0, 0, 0, 0, 0, 413, 1050, 322, 1147, 1056, 536, 951, 679, 0, 0, 0, 0, 0, 0, 0, 0, 530, 512, 917, 936, 756, 368, 1021, 808, 0, 0, 0, 0, 0, 0, 0, 0, 618, 647, 1129, 1123, 624, 269, 625, 178, 598, 234, 32, 0, 0, 0, 0, 0, 0, 983, 1318, 529, 1100, 751, 771, 721, 0, 0, 0, 0, 0, 0, 0, 0, 866, 515, 336, 1306, 879, 1059, 505, 601, 8, 0, 0, 0, 0, 0, 0, 0, 313, 922, 1556, 691, 823, 729, 384, 524, 213, 0, 0, 0, 0, 0, 0, 0, 1109, 988, 254, 527, 410, 954, 722, 735, 318, 176, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6477, 6154, 5848, 6077, 6173, 6075, 6155, 6193] avg=6144.0000 max=6477 max_rank=0 min=5848 min_rank=2 imbalance_ratio=1.054199 max_load_reduction=42675 max_load_reduction_pct=0.868225 moved_to_echo=24596 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[643, 990, 336, 568, 615, 348, 532, 595, 1422, 0, 0, 0, 0, 0, 0, 0, 536, 1090, 300, 1002, 868, 1049, 935, 594, 0, 0, 0, 0, 0, 0, 0, 0, 629, 720, 879, 1023, 826, 341, 935, 894, 0, 0, 0, 0, 0, 0, 0, 0, 577, 652, 1243, 702, 752, 302, 654, 96, 710, 205, 31, 0, 0, 0, 0, 0, 0, 810, 964, 851, 1050, 855, 560, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 871, 567, 417, 1375, 1054, 870, 516, 705, 6, 0, 0, 0, 0, 0, 0, 0, 307, 1008, 1448, 729, 1074, 697, 269, 447, 155, 0, 0, 0, 0, 0, 0, 0, 1054, 896, 279, 517, 387, 866, 735, 776, 278, 129, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6049, 6374, 6247, 5924, 6126, 6381, 6134, 5917] avg=6144.0000 max=6381 max_rank=5 min=5917 min_rank=7 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[620, 989, 356, 422, 588, 333, 529, 815, 1396, 0, 0, 0, 0, 0, 0, 0, 444, 1093, 354, 1113, 1081, 507, 939, 764, 0, 0, 0, 0, 0, 0, 0, 0, 597, 499, 943, 1003, 729, 414, 877, 911, 0, 0, 0, 0, 0, 0, 0, 0, 458, 876, 1236, 773, 971, 259, 603, 240, 674, 227, 32, 0, 0, 0, 0, 0, 0, 1082, 1015, 682, 1338, 687, 699, 913, 0, 0, 0, 0, 0, 0, 0, 0, 874, 558, 301, 1141, 886, 904, 546, 755, 6, 0, 0, 0, 0, 0, 0, 0, 170, 872, 1665, 670, 930, 685, 336, 522, 164, 0, 0, 0, 0, 0, 0, 0, 978, 892, 210, 534, 638, 1081, 620, 689, 309, 135, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6048, 6295, 5973, 6349, 6416, 5971, 6014, 6086] avg=6144.0000 max=6416 max_rank=4 min=5971 min_rank=5 imbalance_ratio=1.044271 max_load_reduction=42736 max_load_reduction_pct=0.869466 moved_to_echo=24487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[682, 980, 442, 417, 640, 438, 424, 527, 1421, 0, 0, 0, 0, 0, 0, 0, 469, 1215, 319, 1115, 896, 943, 897, 608, 0, 0, 0, 0, 0, 0, 0, 0, 575, 550, 869, 1081, 718, 383, 1144, 883, 0, 0, 0, 0, 0, 0, 0, 0, 561, 821, 1205, 658, 785, 322, 563, 218, 655, 232, 31, 0, 0, 0, 0, 0, 0, 1016, 899, 925, 913, 744, 855, 743, 0, 0, 0, 0, 0, 0, 0, 0, 853, 452, 275, 1332, 906, 1093, 606, 730, 9, 0, 0, 0, 0, 0, 0, 0, 357, 899, 1470, 621, 958, 728, 320, 468, 150, 0, 0, 0, 0, 0, 0, 0, 1275, 828, 218, 521, 405, 831, 824, 801, 315, 125, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5971, 6462, 6203, 6051, 6095, 6256, 5971, 6143] avg=6144.0000 max=6462 max_rank=1 min=5971 min_rank=0 imbalance_ratio=1.051758 max_load_reduction=42690 max_load_reduction_pct=0.868530 moved_to_echo=24465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 7680, 3328, 3840, 5248, 3328, 4096, 5376, 11392, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 4864, 7808, 8320, 6272, 3200, 7552, 7040, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7040, 4224, 2688, 10240, 7040, 8064, 4352, 5888, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 4864, 7808, 8320, 6272, 3200, 7552, 7040, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 7296, 2048, 3968, 3968, 7680, 5888, 6016, 2432, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 7296, 8704, 5888, 9088, 6144, 5632, 6784, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4736, 6016, 9600, 6656, 6272, 2432, 5120, 1536, 5248, 1792, 256, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 7808, 12160, 5504, 7552, 5760, 2816, 3968, 1408, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3840, 8320, 2560, 8576, 7424, 5632, 7808, 5504, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 7296, 8704, 5888, 9088, 6144, 5632, 6784, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[634, 506, 1026, 406, 1186, 682, 1217, 1445, 808, 308, 334, 267, 926, 805, 630, 584, 1265, 1045, 580, 865, 737, 982, 538, 676, 641, 230, 1207, 1146, 734, 1424, 929, 558, 420, 1059, 468, 1078, 471, 443, 509, 316, 421, 830, 807, 661, 750, 771, 672, 680, 896, 1129, 2408, 652, 217, 532, 769, 740, 1200, 309, 472, 364, 704, 1571, 1078, 434] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[640, 716, 952, 550, 1438, 694, 1089, 1406, 841, 456, 303, 166, 885, 885, 727, 847, 1223, 905, 662, 754, 836, 839, 787, 1050, 630, 495, 1354, 865, 798, 1587, 817, 285, 361, 900, 427, 1188, 710, 574, 431, 258, 591, 544, 1025, 555, 606, 711, 837, 677, 944, 931, 1695, 520, 173, 634, 807, 850, 1148, 348, 657, 304, 872, 945, 953, 494] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[782, 651, 922, 397, 1059, 725, 1079, 1460, 839, 277, 310, 341, 862, 742, 809, 701, 1283, 986, 615, 767, 781, 656, 609, 1000, 744, 353, 932, 895, 976, 1328, 853, 358, 349, 932, 504, 1169, 561, 342, 441, 314, 787, 1106, 875, 791, 752, 679, 671, 725, 669, 1305, 1732, 500, 159, 549, 885, 787, 1084, 243, 658, 294, 851, 1752, 1094, 500] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[727, 649, 818, 433, 960, 556, 1170, 1534, 799, 577, 288, 331, 1012, 771, 734, 1000, 1231, 1010, 705, 807, 477, 505, 670, 943, 567, 361, 997, 938, 622, 1480, 861, 476, 309, 823, 477, 1171, 461, 708, 456, 299, 412, 741, 958, 563, 861, 824, 839, 861, 951, 1009, 2368, 444, 290, 612, 951, 642, 966, 373, 571, 337, 518, 1824, 954, 570] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[640, 716, 952, 550, 1438, 694, 1089, 1406, 841, 456, 303, 166, 885, 885, 727, 847, 1223, 905, 662, 754, 836, 839, 787, 1050, 630, 495, 1354, 865, 798, 1587, 817, 285, 361, 900, 427, 1188, 710, 574, 431, 258, 591, 544, 1025, 555, 606, 711, 837, 677, 944, 931, 1695, 520, 173, 634, 807, 850, 1148, 348, 657, 304, 872, 945, 953, 494] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[733, 672, 782, 573, 1153, 695, 1188, 1293, 774, 333, 376, 360, 883, 795, 574, 648, 1236, 918, 826, 859, 831, 702, 667, 998, 663, 309, 1082, 1010, 499, 1391, 738, 453, 390, 850, 464, 1089, 732, 427, 505, 364, 693, 868, 891, 622, 755, 642, 767, 818, 893, 995, 2119, 533, 319, 399, 946, 755, 1246, 459, 539, 273, 797, 1528, 1061, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[910, 590, 811, 454, 1303, 705, 1108, 1510, 636, 428, 343, 503, 862, 867, 457, 437, 1306, 1001, 788, 698, 604, 1002, 623, 1371, 547, 337, 1166, 1139, 697, 1484, 781, 501, 441, 899, 450, 1306, 599, 512, 470, 378, 681, 874, 933, 698, 797, 723, 853, 621, 735, 999, 1557, 504, 203, 466, 697, 531, 1298, 284, 480, 464, 863, 1635, 730, 502] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[714, 701, 787, 378, 1025, 599, 1078, 1469, 556, 364, 374, 335, 918, 847, 669, 565, 1347, 1171, 635, 696, 561, 1026, 550, 1172, 627, 321, 1196, 1140, 721, 1366, 825, 585, 360, 869, 441, 1187, 493, 540, 466, 570, 379, 813, 902, 611, 663, 699, 910, 888, 808, 1035, 2368, 642, 269, 657, 793, 623, 1105, 329, 561, 249, 677, 1798, 610, 519] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[834, 638, 799, 481, 1581, 495, 1192, 1370, 681, 274, 347, 386, 789, 710, 683, 400, 1490, 735, 1003, 792, 702, 593, 569, 656, 534, 504, 985, 920, 571, 1424, 864, 559, 412, 775, 424, 1140, 564, 381, 494, 361, 957, 902, 1228, 734, 761, 780, 897, 769, 871, 982, 2193, 545, 267, 441, 1046, 544, 1124, 324, 465, 252, 1038, 1676, 758, 486] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[782, 651, 922, 397, 1059, 725, 1079, 1460, 839, 277, 310, 341, 862, 742, 809, 701, 1283, 986, 615, 767, 781, 656, 609, 1000, 744, 353, 932, 895, 976, 1328, 853, 358, 349, 932, 504, 1169, 561, 342, 441, 314, 787, 1106, 875, 791, 752, 679, 671, 725, 669, 1305, 1732, 500, 159, 549, 885, 787, 1084, 243, 658, 294, 851, 1752, 1094, 500] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[733, 672, 782, 573, 1153, 695, 1188, 1293, 774, 333, 376, 360, 883, 795, 574, 648, 1236, 918, 826, 859, 831, 702, 667, 998, 663, 309, 1082, 1010, 499, 1391, 738, 453, 390, 850, 464, 1089, 732, 427, 505, 364, 693, 868, 891, 622, 755, 642, 767, 818, 893, 995, 2119, 533, 319, 399, 946, 755, 1246, 459, 539, 273, 797, 1528, 1061, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[714, 701, 787, 378, 1025, 599, 1078, 1469, 556, 364, 374, 335, 918, 847, 669, 565, 1347, 1171, 635, 696, 561, 1026, 550, 1172, 627, 321, 1196, 1140, 721, 1366, 825, 585, 360, 869, 441, 1187, 493, 540, 466, 570, 379, 813, 902, 611, 663, 699, 910, 888, 808, 1035, 2368, 642, 269, 657, 793, 623, 1105, 329, 561, 249, 677, 1798, 610, 519] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[634, 506, 1026, 406, 1186, 682, 1217, 1445, 808, 308, 334, 267, 926, 805, 630, 584, 1265, 1045, 580, 865, 737, 982, 538, 676, 641, 230, 1207, 1146, 734, 1424, 929, 558, 420, 1059, 468, 1078, 471, 443, 509, 316, 421, 830, 807, 661, 750, 771, 672, 680, 896, 1129, 2408, 652, 217, 532, 769, 740, 1200, 309, 472, 364, 704, 1571, 1078, 434] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[782, 651, 922, 397, 1059, 725, 1079, 446, 0, 0, 0, 0, 0, 0, 0, 0, 839, 277, 310, 341, 862, 742, 809, 701, 561, 355, 301, 0, 0, 0, 0, 0, 571, 986, 615, 767, 781, 656, 609, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 744, 353, 932, 895, 976, 957, 853, 358, 0, 0, 0, 0, 0, 0, 0, 0, 349, 932, 504, 1169, 561, 342, 441, 314, 1014, 357, 0, 0, 0, 0, 0, 0, 787, 1106, 875, 791, 752, 679, 671, 725, 50, 0, 0, 0, 0, 0, 0, 0, 669, 1305, 1171, 500, 159, 549, 885, 787, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 243, 658, 294, 851, 1752, 1094, 500, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6061, 6098, 5985, 6068, 5983, 6436, 6025, 6496] avg=6144.0000 max=6496 max_rank=7 min=5983 min_rank=4 imbalance_ratio=1.057292 max_load_reduction=42656 max_load_reduction_pct=0.867839 moved_to_echo=24940 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[640, 716, 952, 550, 1438, 694, 1089, 430, 0, 0, 0, 0, 0, 0, 0, 0, 841, 456, 303, 166, 885, 885, 727, 847, 549, 338, 360, 0, 0, 0, 0, 0, 545, 905, 662, 754, 836, 839, 787, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 630, 495, 1354, 865, 798, 1142, 817, 285, 0, 0, 0, 0, 0, 0, 0, 0, 361, 900, 427, 1188, 710, 574, 431, 258, 976, 340, 0, 0, 0, 0, 0, 0, 591, 544, 1025, 555, 606, 711, 837, 677, 60, 0, 0, 0, 0, 0, 0, 0, 944, 931, 1146, 520, 173, 634, 807, 850, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 348, 657, 304, 872, 945, 953, 494, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6509, 6357, 6378, 6386, 6165, 5606, 6005, 5746] avg=6144.0000 max=6509 max_rank=0 min=5606 min_rank=5 imbalance_ratio=1.059408 max_load_reduction=42643 max_load_reduction_pct=0.867574 moved_to_echo=23522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[634, 506, 1026, 406, 1186, 682, 1217, 442, 0, 0, 0, 0, 0, 0, 0, 0, 808, 308, 334, 267, 926, 805, 630, 584, 781, 350, 323, 0, 0, 0, 0, 0, 563, 1045, 580, 865, 737, 982, 538, 676, 0, 0, 0, 0, 0, 0, 0, 0, 641, 230, 1207, 1146, 734, 1025, 929, 558, 0, 0, 0, 0, 0, 0, 0, 0, 420, 1059, 468, 1078, 471, 443, 509, 316, 1003, 352, 0, 0, 0, 0, 0, 0, 421, 830, 807, 661, 750, 771, 672, 680, 54, 0, 0, 0, 0, 0, 0, 0, 896, 1129, 1627, 652, 217, 532, 769, 740, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 309, 472, 364, 704, 1571, 1078, 434, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6099, 6116, 5986, 6470, 6119, 5646, 6562, 6154] avg=6144.0000 max=6562 max_rank=6 min=5646 min_rank=5 imbalance_ratio=1.068034 max_load_reduction=42590 max_load_reduction_pct=0.866496 moved_to_echo=24481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[834, 638, 799, 481, 1581, 495, 1192, 419, 0, 0, 0, 0, 0, 0, 0, 0, 681, 274, 347, 386, 789, 710, 683, 400, 711, 412, 323, 0, 0, 0, 0, 0, 663, 735, 1003, 792, 702, 593, 569, 656, 0, 0, 0, 0, 0, 0, 0, 0, 534, 504, 985, 920, 571, 1025, 864, 559, 0, 0, 0, 0, 0, 0, 0, 0, 412, 775, 424, 1140, 564, 381, 494, 361, 951, 415, 0, 0, 0, 0, 0, 0, 957, 902, 1228, 734, 761, 780, 897, 769, 54, 0, 0, 0, 0, 0, 0, 0, 871, 982, 1482, 545, 267, 441, 1046, 544, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 324, 465, 252, 1038, 1676, 758, 486, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6439, 5716, 5713, 5962, 5917, 7082, 6178, 6145] avg=6144.0000 max=7082 max_rank=5 min=5713 min_rank=2 imbalance_ratio=1.152669 max_load_reduction=42070 max_load_reduction_pct=0.855916 moved_to_echo=25322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[910, 590, 811, 454, 1303, 705, 1108, 462, 0, 0, 0, 0, 0, 0, 0, 0, 636, 428, 343, 503, 862, 867, 457, 437, 505, 361, 336, 0, 0, 0, 0, 0, 582, 1001, 788, 698, 604, 1002, 623, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 547, 337, 1166, 1139, 697, 1069, 781, 501, 0, 0, 0, 0, 0, 0, 0, 0, 441, 899, 450, 1306, 599, 512, 470, 378, 1048, 363, 0, 0, 0, 0, 0, 0, 681, 874, 933, 698, 797, 723, 853, 621, 56, 0, 0, 0, 0, 0, 0, 0, 735, 999, 1052, 504, 203, 466, 697, 531, 0, 0, 0, 0, 0, 0, 0, 0, 1298, 284, 480, 464, 863, 1635, 730, 502, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6343, 5735, 6669, 6237, 6466, 6236, 5187, 6279] avg=6144.0000 max=6669 max_rank=2 min=5187 min_rank=6 imbalance_ratio=1.085449 max_load_reduction=42483 max_load_reduction_pct=0.864319 moved_to_echo=24168 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[714, 701, 787, 378, 1025, 599, 1078, 449, 0, 0, 0, 0, 0, 0, 0, 0, 556, 364, 374, 335, 918, 847, 669, 565, 768, 372, 310, 0, 0, 0, 0, 0, 600, 1171, 635, 696, 561, 1026, 550, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 627, 321, 1196, 1140, 721, 983, 825, 585, 0, 0, 0, 0, 0, 0, 0, 0, 360, 869, 441, 1187, 493, 540, 466, 570, 1020, 375, 0, 0, 0, 0, 0, 0, 379, 813, 902, 611, 663, 699, 910, 888, 52, 0, 0, 0, 0, 0, 0, 0, 808, 1035, 1600, 642, 269, 657, 793, 623, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 329, 561, 249, 677, 1798, 610, 519, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5731, 6078, 6411, 6398, 6321, 5917, 6427, 5869] avg=6144.0000 max=6427 max_rank=6 min=5731 min_rank=0 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[782, 651, 922, 397, 1059, 725, 1079, 446, 0, 0, 0, 0, 0, 0, 0, 0, 839, 277, 310, 341, 862, 742, 809, 701, 561, 355, 301, 0, 0, 0, 0, 0, 571, 986, 615, 767, 781, 656, 609, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 744, 353, 932, 895, 976, 957, 853, 358, 0, 0, 0, 0, 0, 0, 0, 0, 349, 932, 504, 1169, 561, 342, 441, 314, 1014, 357, 0, 0, 0, 0, 0, 0, 787, 1106, 875, 791, 752, 679, 671, 725, 50, 0, 0, 0, 0, 0, 0, 0, 669, 1305, 1171, 500, 159, 549, 885, 787, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 243, 658, 294, 851, 1752, 1094, 500, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6061, 6098, 5985, 6068, 5983, 6436, 6025, 6496] avg=6144.0000 max=6496 max_rank=7 min=5983 min_rank=4 imbalance_ratio=1.057292 max_load_reduction=42656 max_load_reduction_pct=0.867839 moved_to_echo=24940 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[727, 649, 818, 433, 960, 556, 1170, 466, 0, 0, 0, 0, 0, 0, 0, 0, 799, 577, 288, 331, 1012, 771, 734, 1000, 771, 343, 339, 0, 0, 0, 0, 0, 542, 1010, 705, 807, 477, 505, 670, 943, 0, 0, 0, 0, 0, 0, 0, 0, 567, 361, 997, 938, 622, 1053, 861, 476, 0, 0, 0, 0, 0, 0, 0, 0, 309, 823, 477, 1171, 461, 708, 456, 299, 1068, 346, 0, 0, 0, 0, 0, 0, 412, 741, 958, 563, 861, 824, 839, 861, 61, 0, 0, 0, 0, 0, 0, 0, 951, 1009, 1597, 444, 290, 612, 951, 642, 0, 0, 0, 0, 0, 0, 0, 0, 966, 373, 571, 337, 518, 1824, 954, 570, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5779, 6965, 5659, 5875, 6118, 6120, 6496, 6140] avg=6144.0000 max=6965 max_rank=1 min=5659 min_rank=2 imbalance_ratio=1.133626 max_load_reduction=42187 max_load_reduction_pct=0.858297 moved_to_echo=24874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[733, 672, 782, 573, 1153, 695, 1188, 395, 0, 0, 0, 0, 0, 0, 0, 0, 774, 333, 376, 360, 883, 795, 574, 648, 687, 342, 315, 0, 0, 0, 0, 0, 550, 918, 826, 859, 831, 702, 667, 998, 0, 0, 0, 0, 0, 0, 0, 0, 663, 309, 1082, 1010, 499, 1002, 738, 453, 0, 0, 0, 0, 0, 0, 0, 0, 390, 850, 464, 1089, 732, 427, 505, 364, 898, 344, 0, 0, 0, 0, 0, 0, 693, 868, 891, 622, 755, 642, 767, 818, 53, 0, 0, 0, 0, 0, 0, 0, 893, 995, 1432, 533, 319, 399, 946, 755, 0, 0, 0, 0, 0, 0, 0, 0, 1246, 459, 539, 273, 797, 1528, 1061, 399, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6191, 6087, 6351, 5756, 6063, 6109, 6272, 6323] avg=6144.0000 max=6351 max_rank=2 min=5756 min_rank=3 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[727, 649, 818, 433, 960, 556, 1170, 466, 0, 0, 0, 0, 0, 0, 0, 0, 799, 577, 288, 331, 1012, 771, 734, 1000, 771, 343, 339, 0, 0, 0, 0, 0, 542, 1010, 705, 807, 477, 505, 670, 943, 0, 0, 0, 0, 0, 0, 0, 0, 567, 361, 997, 938, 622, 1053, 861, 476, 0, 0, 0, 0, 0, 0, 0, 0, 309, 823, 477, 1171, 461, 708, 456, 299, 1068, 346, 0, 0, 0, 0, 0, 0, 412, 741, 958, 563, 861, 824, 839, 861, 61, 0, 0, 0, 0, 0, 0, 0, 951, 1009, 1597, 444, 290, 612, 951, 642, 0, 0, 0, 0, 0, 0, 0, 0, 966, 373, 571, 337, 518, 1824, 954, 570, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5779, 6965, 5659, 5875, 6118, 6120, 6496, 6140] avg=6144.0000 max=6965 max_rank=1 min=5659 min_rank=2 imbalance_ratio=1.133626 max_load_reduction=42187 max_load_reduction_pct=0.858297 moved_to_echo=24874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[714, 701, 787, 378, 1025, 599, 1078, 449, 0, 0, 0, 0, 0, 0, 0, 0, 556, 364, 374, 335, 918, 847, 669, 565, 768, 372, 310, 0, 0, 0, 0, 0, 600, 1171, 635, 696, 561, 1026, 550, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 627, 321, 1196, 1140, 721, 983, 825, 585, 0, 0, 0, 0, 0, 0, 0, 0, 360, 869, 441, 1187, 493, 540, 466, 570, 1020, 375, 0, 0, 0, 0, 0, 0, 379, 813, 902, 611, 663, 699, 910, 888, 52, 0, 0, 0, 0, 0, 0, 0, 808, 1035, 1600, 642, 269, 657, 793, 623, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 329, 561, 249, 677, 1798, 610, 519, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5731, 6078, 6411, 6398, 6321, 5917, 6427, 5869] avg=6144.0000 max=6427 max_rank=6 min=5731 min_rank=0 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[634, 506, 1026, 406, 1186, 682, 1217, 442, 0, 0, 0, 0, 0, 0, 0, 0, 808, 308, 334, 267, 926, 805, 630, 584, 781, 350, 323, 0, 0, 0, 0, 0, 563, 1045, 580, 865, 737, 982, 538, 676, 0, 0, 0, 0, 0, 0, 0, 0, 641, 230, 1207, 1146, 734, 1025, 929, 558, 0, 0, 0, 0, 0, 0, 0, 0, 420, 1059, 468, 1078, 471, 443, 509, 316, 1003, 352, 0, 0, 0, 0, 0, 0, 421, 830, 807, 661, 750, 771, 672, 680, 54, 0, 0, 0, 0, 0, 0, 0, 896, 1129, 1627, 652, 217, 532, 769, 740, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 309, 472, 364, 704, 1571, 1078, 434, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6099, 6116, 5986, 6470, 6119, 5646, 6562, 6154] avg=6144.0000 max=6562 max_rank=6 min=5646 min_rank=5 imbalance_ratio=1.068034 max_load_reduction=42590 max_load_reduction_pct=0.866496 moved_to_echo=24481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[834, 638, 799, 481, 1581, 495, 1192, 419, 0, 0, 0, 0, 0, 0, 0, 0, 681, 274, 347, 386, 789, 710, 683, 400, 711, 412, 323, 0, 0, 0, 0, 0, 663, 735, 1003, 792, 702, 593, 569, 656, 0, 0, 0, 0, 0, 0, 0, 0, 534, 504, 985, 920, 571, 1025, 864, 559, 0, 0, 0, 0, 0, 0, 0, 0, 412, 775, 424, 1140, 564, 381, 494, 361, 951, 415, 0, 0, 0, 0, 0, 0, 957, 902, 1228, 734, 761, 780, 897, 769, 54, 0, 0, 0, 0, 0, 0, 0, 871, 982, 1482, 545, 267, 441, 1046, 544, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 324, 465, 252, 1038, 1676, 758, 486, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6439, 5716, 5713, 5962, 5917, 7082, 6178, 6145] avg=6144.0000 max=7082 max_rank=5 min=5713 min_rank=2 imbalance_ratio=1.152669 max_load_reduction=42070 max_load_reduction_pct=0.855916 moved_to_echo=25322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 7168, 3712, 9344, 4608, 3968, 3840, 2944, 8064, 2944, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6016, 3072, 2688, 2816, 7168, 6528, 5376, 5248, 5376, 2944, 2688, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[6784, 8448, 11136, 4352, 1920, 4352, 6912, 5504, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4736, 7808, 5888, 6272, 5632, 6400, 5120, 7936, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 6784, 7680, 5248, 6016, 5888, 6528, 6144, 512, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 2688, 4480, 2560, 6400, 12800, 7296, 3968, 256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[6784, 8448, 11136, 4352, 1920, 4352, 6912, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 5248, 6912, 3712, 9728, 5248, 9216, 3584, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4992, 2944, 8960, 8064, 5632, 8320, 6784, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 2688, 4480, 2560, 6400, 12800, 7296, 3968, 256, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 5248, 6912, 3712, 9728, 5248, 9216, 3584, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1074, 853, 717, 546, 420, 2235, 916, 696, 633, 863, 1017, 687, 651, 580, 649, 454, 469, 1508, 223, 2006, 733, 537, 872, 1089, 442, 323, 797, 1068, 565, 1336, 497, 761, 326, 229, 401, 1499, 1008, 421, 286, 738, 304, 863, 198, 573, 645, 456, 1563, 280, 361, 1347, 1029, 366, 605, 1299, 1181, 858, 442, 1380, 435, 1285, 515, 943, 687, 412] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[914, 1212, 813, 472, 356, 2304, 987, 692, 586, 557, 560, 391, 663, 1167, 343, 709, 435, 1295, 206, 2092, 1016, 763, 556, 752, 602, 466, 772, 1180, 523, 1615, 421, 818, 545, 473, 466, 1299, 1025, 267, 511, 1073, 539, 824, 183, 791, 384, 339, 651, 656, 439, 1212, 1824, 185, 592, 1257, 1620, 511, 604, 994, 465, 1121, 616, 453, 675, 320] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1247, 1531, 952, 462, 435, 1841, 996, 830, 610, 666, 644, 732, 760, 676, 426, 333, 403, 1190, 320, 1709, 984, 515, 941, 1033, 415, 456, 611, 1499, 459, 1759, 673, 810, 461, 355, 501, 1368, 1162, 264, 467, 1059, 533, 961, 191, 576, 568, 491, 1081, 393, 491, 1239, 1307, 233, 401, 1431, 1042, 688, 397, 1248, 386, 1252, 472, 357, 520, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1000, 1235, 1039, 498, 480, 2096, 964, 1083, 609, 854, 746, 646, 576, 933, 557, 592, 430, 1264, 339, 1478, 835, 559, 743, 1232, 541, 341, 814, 1141, 622, 1908, 347, 750, 559, 321, 746, 1272, 1048, 325, 439, 683, 547, 529, 186, 906, 455, 538, 868, 349, 419, 1280, 1258, 212, 605, 814, 1406, 709, 662, 1255, 380, 1149, 566, 451, 559, 404] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[993, 873, 742, 464, 507, 2191, 1013, 770, 545, 1029, 710, 625, 624, 534, 448, 600, 529, 1397, 445, 1229, 1027, 590, 830, 1146, 490, 197, 702, 1066, 401, 1992, 619, 484, 433, 393, 638, 1032, 965, 298, 546, 740, 504, 752, 257, 709, 564, 689, 817, 366, 249, 1521, 1385, 467, 504, 1021, 1228, 770, 523, 1188, 463, 1601, 591, 863, 696, 567] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1119, 1199, 1065, 536, 296, 1651, 997, 866, 714, 639, 628, 599, 597, 822, 455, 1043, 549, 1137, 375, 1950, 859, 554, 861, 660, 717, 473, 1064, 1300, 338, 1505, 441, 1042, 479, 378, 559, 1474, 907, 502, 564, 596, 608, 729, 333, 506, 398, 478, 982, 392, 204, 1804, 1350, 185, 391, 1208, 1375, 682, 670, 792, 331, 1271, 478, 418, 608, 449] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1000, 1235, 1039, 498, 480, 2096, 964, 1083, 609, 854, 746, 646, 576, 933, 557, 592, 430, 1264, 339, 1478, 835, 559, 743, 1232, 541, 341, 814, 1141, 622, 1908, 347, 750, 559, 321, 746, 1272, 1048, 325, 439, 683, 547, 529, 186, 906, 455, 538, 868, 349, 419, 1280, 1258, 212, 605, 814, 1406, 709, 662, 1255, 380, 1149, 566, 451, 559, 404] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1074, 853, 717, 546, 420, 2235, 916, 696, 633, 863, 1017, 687, 651, 580, 649, 454, 469, 1508, 223, 2006, 733, 537, 872, 1089, 442, 323, 797, 1068, 565, 1336, 497, 761, 326, 229, 401, 1499, 1008, 421, 286, 738, 304, 863, 198, 573, 645, 456, 1563, 280, 361, 1347, 1029, 366, 605, 1299, 1181, 858, 442, 1380, 435, 1285, 515, 943, 687, 412] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1129, 1339, 743, 512, 552, 2005, 813, 744, 706, 856, 1101, 544, 664, 801, 511, 589, 346, 1328, 260, 1897, 762, 475, 880, 805, 546, 577, 826, 1207, 526, 1850, 582, 742, 356, 266, 584, 1232, 810, 435, 432, 786, 593, 799, 209, 857, 486, 404, 946, 420, 400, 1439, 1182, 232, 502, 1194, 1309, 634, 466, 1041, 449, 1282, 603, 592, 561, 433] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1325, 1221, 893, 588, 626, 2032, 975, 720, 636, 602, 595, 649, 823, 467, 405, 868, 399, 1408, 196, 1381, 758, 519, 973, 618, 517, 454, 1007, 1386, 388, 1957, 541, 913, 333, 333, 683, 1273, 906, 397, 414, 549, 548, 527, 180, 435, 549, 518, 1160, 554, 385, 1369, 1514, 194, 537, 1385, 988, 746, 354, 1404, 507, 1247, 592, 650, 536, 545] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1074, 853, 717, 546, 420, 289, 916, 696, 0, 0, 0, 0, 0, 0, 0, 0, 633, 863, 1017, 687, 651, 580, 649, 454, 716, 233, 0, 0, 0, 0, 0, 0, 469, 1508, 223, 1056, 733, 537, 872, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 442, 323, 797, 1068, 565, 1046, 497, 761, 0, 0, 0, 0, 0, 0, 0, 0, 326, 229, 401, 1499, 1008, 421, 286, 738, 658, 243, 0, 0, 0, 0, 0, 0, 304, 863, 198, 573, 645, 456, 1563, 280, 1703, 0, 0, 0, 0, 0, 0, 0, 361, 456, 1029, 366, 605, 1299, 1181, 858, 0, 0, 0, 0, 0, 0, 0, 0, 442, 1380, 435, 1285, 515, 943, 687, 412, 290, 234, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5511, 6483, 6487, 5499, 5809, 6585, 6155, 6623] avg=6144.0000 max=6623 max_rank=7 min=5499 min_rank=3 imbalance_ratio=1.077962 max_load_reduction=42529 max_load_reduction_pct=0.865255 moved_to_echo=25172 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1247, 1531, 952, 462, 435, 239, 996, 830, 0, 0, 0, 0, 0, 0, 0, 0, 610, 666, 644, 732, 760, 676, 426, 333, 610, 214, 0, 0, 0, 0, 0, 0, 403, 1190, 320, 900, 984, 515, 941, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 415, 456, 611, 1499, 459, 1377, 673, 810, 0, 0, 0, 0, 0, 0, 0, 0, 461, 355, 501, 1368, 1162, 264, 467, 1059, 605, 200, 0, 0, 0, 0, 0, 0, 533, 961, 191, 576, 568, 491, 1081, 393, 1402, 0, 0, 0, 0, 0, 0, 0, 491, 420, 1307, 233, 401, 1431, 1042, 688, 0, 0, 0, 0, 0, 0, 0, 0, 397, 1248, 386, 1252, 472, 357, 520, 339, 382, 199, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6692, 5671, 6286, 6300, 6442, 6196, 6013, 5552] avg=6144.0000 max=6692 max_rank=0 min=5552 min_rank=7 imbalance_ratio=1.089193 max_load_reduction=42460 max_load_reduction_pct=0.863851 moved_to_echo=24203 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1247, 1531, 952, 462, 435, 1841, 996, 830, 610, 666, 644, 732, 760, 676, 426, 333, 403, 1190, 320, 1709, 984, 515, 941, 1033, 415, 456, 611, 1499, 459, 1759, 673, 810, 461, 355, 501, 1368, 1162, 264, 467, 1059, 533, 961, 191, 576, 568, 491, 1081, 393, 491, 1239, 1307, 233, 401, 1431, 1042, 688, 397, 1248, 386, 1252, 472, 357, 520, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1119, 1199, 1065, 536, 296, 1651, 997, 866, 714, 639, 628, 599, 597, 822, 455, 1043, 549, 1137, 375, 1950, 859, 554, 861, 660, 717, 473, 1064, 1300, 338, 1505, 441, 1042, 479, 378, 559, 1474, 907, 502, 564, 596, 608, 729, 333, 506, 398, 478, 982, 392, 204, 1804, 1350, 185, 391, 1208, 1375, 682, 670, 792, 331, 1271, 478, 418, 608, 449] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[914, 1212, 813, 472, 356, 299, 987, 692, 0, 0, 0, 0, 0, 0, 0, 0, 586, 557, 560, 391, 663, 1167, 343, 709, 747, 209, 0, 0, 0, 0, 0, 0, 435, 1295, 206, 1101, 1016, 763, 556, 752, 0, 0, 0, 0, 0, 0, 0, 0, 602, 466, 772, 1180, 523, 1264, 421, 818, 0, 0, 0, 0, 0, 0, 0, 0, 545, 473, 466, 1299, 1025, 267, 511, 1073, 592, 250, 0, 0, 0, 0, 0, 0, 539, 824, 183, 791, 384, 339, 651, 656, 1755, 0, 0, 0, 0, 0, 0, 0, 439, 411, 1824, 185, 592, 1257, 1620, 511, 0, 0, 0, 0, 0, 0, 0, 0, 604, 994, 465, 1121, 616, 453, 675, 320, 351, 244, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5745, 5932, 6124, 6046, 6501, 6122, 6839, 5843] avg=6144.0000 max=6839 max_rank=6 min=5745 min_rank=0 imbalance_ratio=1.113118 max_load_reduction=42313 max_load_reduction_pct=0.860860 moved_to_echo=25305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1000, 1235, 1039, 498, 480, 271, 964, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 609, 854, 746, 646, 576, 933, 557, 592, 528, 221, 0, 0, 0, 0, 0, 0, 430, 1264, 339, 778, 835, 559, 743, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 541, 341, 814, 1141, 622, 1493, 347, 750, 0, 0, 0, 0, 0, 0, 0, 0, 559, 321, 746, 1272, 1048, 325, 439, 683, 625, 228, 0, 0, 0, 0, 0, 0, 547, 529, 186, 906, 455, 538, 868, 349, 1597, 0, 0, 0, 0, 0, 0, 0, 419, 434, 1258, 212, 605, 814, 1406, 709, 0, 0, 0, 0, 0, 0, 0, 0, 662, 1255, 380, 1149, 566, 451, 559, 404, 415, 172, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6570, 6262, 6180, 6049, 6246, 5975, 5857, 6013] avg=6144.0000 max=6570 max_rank=0 min=5857 min_rank=6 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=24091 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[993, 873, 742, 464, 507, 284, 1013, 770, 0, 0, 0, 0, 0, 0, 0, 0, 545, 1029, 710, 625, 624, 534, 448, 600, 439, 263, 0, 0, 0, 0, 0, 0, 529, 1397, 445, 647, 1027, 590, 830, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 490, 197, 702, 1066, 401, 1559, 619, 484, 0, 0, 0, 0, 0, 0, 0, 0, 433, 393, 638, 1032, 965, 298, 546, 740, 743, 238, 0, 0, 0, 0, 0, 0, 504, 752, 257, 709, 564, 689, 817, 366, 1669, 0, 0, 0, 0, 0, 0, 0, 249, 515, 1385, 467, 504, 1021, 1228, 770, 0, 0, 0, 0, 0, 0, 0, 0, 523, 1188, 463, 1601, 591, 863, 696, 567, 433, 143, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5646, 5817, 6611, 5518, 6026, 6327, 6139, 7068] avg=6144.0000 max=7068 max_rank=7 min=5518 min_rank=3 imbalance_ratio=1.150391 max_load_reduction=42084 max_load_reduction_pct=0.856201 moved_to_echo=25560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1000, 1235, 1039, 498, 480, 271, 964, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 609, 854, 746, 646, 576, 933, 557, 592, 528, 221, 0, 0, 0, 0, 0, 0, 430, 1264, 339, 778, 835, 559, 743, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 541, 341, 814, 1141, 622, 1493, 347, 750, 0, 0, 0, 0, 0, 0, 0, 0, 559, 321, 746, 1272, 1048, 325, 439, 683, 625, 228, 0, 0, 0, 0, 0, 0, 547, 529, 186, 906, 455, 538, 868, 349, 1597, 0, 0, 0, 0, 0, 0, 0, 419, 434, 1258, 212, 605, 814, 1406, 709, 0, 0, 0, 0, 0, 0, 0, 0, 662, 1255, 380, 1149, 566, 451, 559, 404, 415, 172, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6570, 6262, 6180, 6049, 6246, 5975, 5857, 6013] avg=6144.0000 max=6570 max_rank=0 min=5857 min_rank=6 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=24091 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1247, 1531, 952, 462, 435, 239, 996, 830, 0, 0, 0, 0, 0, 0, 0, 0, 610, 666, 644, 732, 760, 676, 426, 333, 610, 214, 0, 0, 0, 0, 0, 0, 403, 1190, 320, 900, 984, 515, 941, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 415, 456, 611, 1499, 459, 1377, 673, 810, 0, 0, 0, 0, 0, 0, 0, 0, 461, 355, 501, 1368, 1162, 264, 467, 1059, 605, 200, 0, 0, 0, 0, 0, 0, 533, 961, 191, 576, 568, 491, 1081, 393, 1402, 0, 0, 0, 0, 0, 0, 0, 491, 420, 1307, 233, 401, 1431, 1042, 688, 0, 0, 0, 0, 0, 0, 0, 0, 397, 1248, 386, 1252, 472, 357, 520, 339, 382, 199, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6692, 5671, 6286, 6300, 6442, 6196, 6013, 5552] avg=6144.0000 max=6692 max_rank=0 min=5552 min_rank=7 imbalance_ratio=1.089193 max_load_reduction=42460 max_load_reduction_pct=0.863851 moved_to_echo=24203 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1074, 853, 717, 546, 420, 289, 916, 696, 0, 0, 0, 0, 0, 0, 0, 0, 633, 863, 1017, 687, 651, 580, 649, 454, 716, 233, 0, 0, 0, 0, 0, 0, 469, 1508, 223, 1056, 733, 537, 872, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 442, 323, 797, 1068, 565, 1046, 497, 761, 0, 0, 0, 0, 0, 0, 0, 0, 326, 229, 401, 1499, 1008, 421, 286, 738, 658, 243, 0, 0, 0, 0, 0, 0, 304, 863, 198, 573, 645, 456, 1563, 280, 1703, 0, 0, 0, 0, 0, 0, 0, 361, 456, 1029, 366, 605, 1299, 1181, 858, 0, 0, 0, 0, 0, 0, 0, 0, 442, 1380, 435, 1285, 515, 943, 687, 412, 290, 234, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5511, 6483, 6487, 5499, 5809, 6585, 6155, 6623] avg=6144.0000 max=6623 max_rank=7 min=5499 min_rank=3 imbalance_ratio=1.077962 max_load_reduction=42529 max_load_reduction_pct=0.865255 moved_to_echo=25172 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1119, 1199, 1065, 536, 296, 214, 997, 866, 0, 0, 0, 0, 0, 0, 0, 0, 714, 639, 628, 599, 597, 822, 455, 1043, 696, 312, 0, 0, 0, 0, 0, 0, 549, 1137, 375, 1026, 859, 554, 861, 660, 0, 0, 0, 0, 0, 0, 0, 0, 717, 473, 1064, 1300, 338, 1178, 441, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 479, 378, 559, 1474, 907, 502, 564, 596, 881, 179, 0, 0, 0, 0, 0, 0, 608, 729, 333, 506, 398, 478, 982, 392, 1258, 0, 0, 0, 0, 0, 0, 0, 204, 611, 1350, 185, 391, 1208, 1375, 682, 0, 0, 0, 0, 0, 0, 0, 0, 670, 792, 331, 1271, 478, 418, 608, 449, 327, 228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6292, 6505, 6021, 6553, 6519, 5684, 6006, 5572] avg=6144.0000 max=6553 max_rank=3 min=5572 min_rank=7 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=23781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1129, 1339, 743, 512, 552, 260, 813, 744, 0, 0, 0, 0, 0, 0, 0, 0, 706, 856, 1101, 544, 664, 801, 511, 589, 677, 249, 0, 0, 0, 0, 0, 0, 346, 1328, 260, 999, 762, 475, 880, 805, 0, 0, 0, 0, 0, 0, 0, 0, 546, 577, 826, 1207, 526, 1448, 582, 742, 0, 0, 0, 0, 0, 0, 0, 0, 356, 266, 584, 1232, 810, 435, 432, 786, 703, 218, 0, 0, 0, 0, 0, 0, 593, 799, 209, 857, 486, 404, 946, 420, 1527, 0, 0, 0, 0, 0, 0, 0, 400, 487, 1182, 232, 502, 1194, 1309, 634, 0, 0, 0, 0, 0, 0, 0, 0, 466, 1041, 449, 1282, 603, 592, 561, 433, 402, 221, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6092, 6698, 5855, 6454, 5822, 6241, 5940, 6050] avg=6144.0000 max=6698 max_rank=1 min=5822 min_rank=4 imbalance_ratio=1.090169 max_load_reduction=42454 max_load_reduction_pct=0.863729 moved_to_echo=24053 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1325, 1221, 893, 588, 626, 256, 975, 720, 0, 0, 0, 0, 0, 0, 0, 0, 636, 602, 595, 649, 823, 467, 405, 868, 498, 241, 0, 0, 0, 0, 0, 0, 399, 1408, 196, 717, 758, 519, 973, 618, 0, 0, 0, 0, 0, 0, 0, 0, 517, 454, 1007, 1386, 388, 1527, 541, 913, 0, 0, 0, 0, 0, 0, 0, 0, 333, 333, 683, 1273, 906, 397, 414, 549, 674, 225, 0, 0, 0, 0, 0, 0, 548, 527, 180, 435, 549, 518, 1160, 554, 1551, 0, 0, 0, 0, 0, 0, 0, 385, 454, 1514, 194, 537, 1385, 988, 746, 0, 0, 0, 0, 0, 0, 0, 0, 354, 1404, 507, 1247, 592, 650, 536, 545, 430, 166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6604, 5784, 5588, 6733, 5787, 6022, 6203, 6431] avg=6144.0000 max=6733 max_rank=3 min=5588 min_rank=2 imbalance_ratio=1.095866 max_load_reduction=42419 max_load_reduction_pct=0.863017 moved_to_echo=24443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 3840, 10880, 2176, 4224, 9728, 10240, 5632, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 6144, 6016, 4992, 5376, 6016, 3840, 5248, 4992, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 2816, 4608, 10496, 7936, 2944, 3712, 6272, 5504, 1792, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 9344, 3456, 10240, 4480, 4736, 4864, 3584, 3072, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 6016, 1792, 5376, 4096, 3968, 8192, 3456, 12544, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 10624, 2432, 7296, 7040, 4608, 6656, 7424, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 6144, 6016, 4992, 5376, 6016, 3840, 5248, 4992, 2048, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 3328, 6656, 9856, 3840, 11008, 4224, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 2816, 4608, 10496, 7936, 2944, 3712, 6272, 5504, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 10624, 2432, 7296, 7040, 4608, 6656, 7424, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8832, 9472, 7040, 4096, 3712, 2176, 7680, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 6016, 1792, 5376, 4096, 3968, 8192, 3456, 12544, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[489, 517, 193, 381, 348, 1181, 1204, 859, 763, 331, 736, 451, 381, 461, 898, 289, 2306, 1035, 792, 1058, 901, 3171, 1042, 382, 1076, 949, 1088, 411, 549, 1114, 362, 271, 372, 699, 911, 544, 486, 1281, 1058, 1081, 639, 519, 920, 824, 326, 605, 1054, 1040, 797, 625, 1049, 1329, 452, 396, 163, 624, 969, 809, 553, 623, 779, 272, 923, 441] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[527, 796, 148, 237, 688, 1151, 1477, 523, 636, 232, 596, 410, 305, 698, 931, 140, 1272, 964, 1468, 604, 686, 2150, 1131, 431, 1168, 1065, 728, 444, 830, 623, 478, 540, 350, 315, 581, 536, 439, 1400, 1216, 1042, 1011, 679, 1265, 1342, 245, 536, 2175, 1044, 539, 549, 1034, 836, 486, 395, 179, 621, 779, 1753, 731, 809, 845, 263, 656, 424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[885, 625, 184, 334, 377, 1544, 1566, 497, 643, 282, 722, 538, 355, 517, 771, 216, 1632, 1468, 961, 709, 1032, 1855, 858, 535, 667, 692, 876, 381, 546, 1114, 310, 432, 401, 439, 970, 384, 617, 1102, 1397, 810, 1132, 1043, 822, 1129, 380, 486, 1665, 835, 667, 853, 935, 1137, 937, 304, 214, 573, 834, 852, 1002, 543, 491, 347, 1123, 604] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[852, 567, 263, 281, 420, 1576, 1279, 569, 963, 282, 589, 447, 392, 416, 580, 190, 1923, 1366, 988, 1224, 547, 2099, 818, 341, 558, 739, 735, 413, 621, 1255, 367, 428, 342, 262, 526, 553, 734, 1639, 828, 826, 1518, 622, 919, 1343, 348, 610, 1618, 898, 511, 1269, 840, 937, 689, 413, 204, 727, 880, 1301, 606, 546, 915, 402, 982, 256] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[885, 625, 184, 334, 377, 1544, 1566, 497, 643, 282, 722, 538, 355, 517, 771, 216, 1632, 1468, 961, 709, 1032, 1855, 858, 535, 667, 692, 876, 381, 546, 1114, 310, 432, 401, 439, 970, 384, 617, 1102, 1397, 810, 1132, 1043, 822, 1129, 380, 486, 1665, 835, 667, 853, 935, 1137, 937, 304, 214, 573, 834, 852, 1002, 543, 491, 347, 1123, 604] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[527, 796, 148, 237, 688, 1151, 1477, 523, 636, 232, 596, 410, 305, 698, 931, 140, 1272, 964, 1468, 604, 686, 2150, 1131, 431, 1168, 1065, 728, 444, 830, 623, 478, 540, 350, 315, 581, 536, 439, 1400, 1216, 1042, 1011, 679, 1265, 1342, 245, 536, 2175, 1044, 539, 549, 1034, 836, 486, 395, 179, 621, 779, 1753, 731, 809, 845, 263, 656, 424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[533, 780, 421, 227, 318, 738, 1300, 768, 463, 267, 800, 530, 370, 534, 860, 190, 2321, 1336, 1039, 938, 716, 1936, 1129, 382, 310, 988, 684, 381, 360, 1242, 504, 284, 396, 426, 425, 709, 436, 1463, 1665, 1187, 780, 583, 1370, 1459, 415, 564, 1787, 1307, 668, 646, 878, 765, 600, 303, 133, 603, 727, 1671, 576, 740, 1057, 245, 565, 354] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[438, 391, 104, 233, 233, 1207, 1643, 591, 1150, 262, 435, 611, 361, 369, 1043, 230, 2360, 850, 1032, 618, 932, 1838, 870, 300, 424, 599, 764, 543, 729, 565, 367, 358, 573, 280, 810, 862, 511, 1840, 1588, 1017, 1419, 548, 1131, 1629, 471, 391, 1234, 885, 601, 844, 1245, 1194, 953, 430, 178, 509, 1477, 871, 551, 643, 587, 192, 747, 491] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[741, 690, 318, 264, 290, 1396, 901, 769, 932, 442, 1207, 336, 479, 632, 718, 332, 1878, 978, 1029, 825, 674, 1666, 686, 381, 858, 515, 825, 565, 849, 488, 523, 383, 654, 265, 885, 794, 507, 1248, 1014, 884, 1017, 473, 914, 1464, 351, 396, 1790, 730, 525, 1351, 1157, 777, 792, 516, 340, 478, 831, 1509, 733, 597, 696, 319, 950, 625] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[586, 604, 187, 340, 239, 988, 1721, 517, 710, 439, 642, 207, 478, 482, 1167, 129, 2178, 931, 983, 564, 983, 2566, 762, 352, 537, 942, 728, 344, 488, 1194, 667, 309, 556, 552, 551, 530, 436, 1220, 1292, 1172, 906, 788, 1765, 1265, 477, 537, 1545, 743, 848, 433, 1249, 740, 593, 509, 152, 437, 1222, 1207, 855, 606, 864, 308, 363, 467] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[533, 780, 421, 227, 318, 738, 1300, 768, 463, 267, 800, 530, 370, 534, 860, 190, 2321, 1336, 1039, 938, 716, 1936, 1129, 382, 310, 988, 684, 381, 360, 1242, 504, 284, 396, 426, 425, 709, 436, 1463, 1665, 1187, 780, 583, 1370, 1459, 415, 564, 1787, 1307, 668, 646, 878, 765, 600, 303, 133, 603, 727, 1671, 576, 740, 1057, 245, 565, 354] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[489, 517, 193, 381, 348, 1181, 1204, 859, 891, 0, 0, 0, 0, 0, 0, 0, 763, 331, 736, 451, 381, 461, 898, 289, 2786, 0, 0, 0, 0, 0, 0, 0, 1240, 1035, 792, 1058, 901, 0, 1042, 382, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 949, 1088, 411, 549, 1114, 362, 271, 483, 385, 0, 0, 0, 0, 0, 0, 372, 699, 911, 544, 486, 1083, 1058, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 639, 519, 920, 824, 326, 605, 107, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 797, 625, 1049, 1329, 452, 396, 163, 624, 464, 109, 0, 0, 0, 0, 0, 0, 969, 809, 553, 623, 779, 272, 923, 441, 198, 66, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 7096, 6450, 6688, 6234, 4980, 6008, 5633] avg=6144.0000 max=7096 max_rank=1 min=4980 min_rank=5 imbalance_ratio=1.154948 max_load_reduction=42056 max_load_reduction_pct=0.855632 moved_to_echo=22855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[885, 625, 184, 334, 377, 1544, 1566, 497, 631, 0, 0, 0, 0, 0, 0, 0, 643, 282, 722, 538, 355, 517, 771, 216, 1630, 0, 0, 0, 0, 0, 0, 0, 878, 1468, 961, 709, 1032, 0, 858, 535, 0, 0, 0, 0, 0, 0, 0, 0, 667, 692, 876, 381, 546, 1114, 310, 432, 764, 225, 0, 0, 0, 0, 0, 0, 401, 439, 970, 384, 617, 931, 1397, 810, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 1043, 822, 1129, 380, 486, 168, 835, 0, 0, 0, 0, 0, 0, 0, 0, 667, 853, 935, 1137, 937, 304, 214, 573, 733, 77, 0, 0, 0, 0, 0, 0, 834, 852, 1002, 543, 491, 347, 1123, 604, 171, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6643, 5674, 6441, 6007, 5949, 5995, 6430, 6013] avg=6144.0000 max=6643 max_rank=0 min=5674 min_rank=1 imbalance_ratio=1.081217 max_load_reduction=42509 max_load_reduction_pct=0.864848 moved_to_echo=24387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[489, 517, 193, 381, 348, 1181, 1204, 859, 763, 331, 736, 451, 381, 461, 898, 289, 2306, 1035, 792, 1058, 901, 3171, 1042, 382, 1076, 949, 1088, 411, 549, 1114, 362, 271, 372, 699, 911, 544, 486, 1281, 1058, 1081, 639, 519, 920, 824, 326, 605, 1054, 1040, 797, 625, 1049, 1329, 452, 396, 163, 624, 969, 809, 553, 623, 779, 272, 923, 441] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[852, 567, 263, 281, 420, 1576, 1279, 569, 743, 0, 0, 0, 0, 0, 0, 0, 963, 282, 589, 447, 392, 416, 580, 190, 1844, 0, 0, 0, 0, 0, 0, 0, 1034, 1366, 988, 1224, 547, 0, 818, 341, 0, 0, 0, 0, 0, 0, 0, 0, 558, 739, 735, 413, 621, 1255, 367, 428, 742, 255, 0, 0, 0, 0, 0, 0, 342, 262, 526, 553, 734, 1385, 828, 826, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 622, 919, 1343, 348, 610, 164, 898, 0, 0, 0, 0, 0, 0, 0, 0, 511, 1269, 840, 937, 689, 413, 204, 727, 712, 91, 0, 0, 0, 0, 0, 0, 880, 1301, 606, 546, 915, 402, 982, 256, 254, 55, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6550, 5703, 6318, 6113, 5456, 6422, 6393, 6197] avg=6144.0000 max=6550 max_rank=0 min=5456 min_rank=4 imbalance_ratio=1.066081 max_load_reduction=42602 max_load_reduction_pct=0.866740 moved_to_echo=24468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[885, 625, 184, 334, 377, 1544, 1566, 497, 631, 0, 0, 0, 0, 0, 0, 0, 643, 282, 722, 538, 355, 517, 771, 216, 1630, 0, 0, 0, 0, 0, 0, 0, 878, 1468, 961, 709, 1032, 0, 858, 535, 0, 0, 0, 0, 0, 0, 0, 0, 667, 692, 876, 381, 546, 1114, 310, 432, 764, 225, 0, 0, 0, 0, 0, 0, 401, 439, 970, 384, 617, 931, 1397, 810, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 1043, 822, 1129, 380, 486, 168, 835, 0, 0, 0, 0, 0, 0, 0, 0, 667, 853, 935, 1137, 937, 304, 214, 573, 733, 77, 0, 0, 0, 0, 0, 0, 834, 852, 1002, 543, 491, 347, 1123, 604, 171, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6643, 5674, 6441, 6007, 5949, 5995, 6430, 6013] avg=6144.0000 max=6643 max_rank=0 min=5674 min_rank=1 imbalance_ratio=1.081217 max_load_reduction=42509 max_load_reduction_pct=0.864848 moved_to_echo=24387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[438, 391, 104, 233, 233, 1207, 1643, 591, 912, 0, 0, 0, 0, 0, 0, 0, 1150, 262, 435, 611, 361, 369, 1043, 230, 1614, 0, 0, 0, 0, 0, 0, 0, 1269, 850, 1032, 618, 932, 0, 870, 300, 0, 0, 0, 0, 0, 0, 0, 0, 424, 599, 764, 543, 729, 565, 367, 358, 566, 224, 0, 0, 0, 0, 0, 0, 573, 280, 810, 862, 511, 1555, 1588, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1419, 548, 1131, 1629, 471, 391, 125, 885, 0, 0, 0, 0, 0, 0, 0, 0, 601, 844, 1245, 1194, 953, 430, 178, 509, 543, 112, 0, 0, 0, 0, 0, 0, 1477, 871, 551, 643, 587, 192, 747, 491, 285, 67, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5752, 6075, 5871, 5139, 7196, 6599, 6609, 5911] avg=6144.0000 max=7196 max_rank=4 min=5139 min_rank=3 imbalance_ratio=1.171224 max_load_reduction=41956 max_load_reduction_pct=0.853597 moved_to_echo=26315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[527, 796, 148, 237, 688, 1151, 1477, 523, 496, 0, 0, 0, 0, 0, 0, 0, 636, 232, 596, 410, 305, 698, 931, 140, 1890, 0, 0, 0, 0, 0, 0, 0, 671, 964, 1468, 604, 686, 0, 1131, 431, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 1065, 728, 444, 830, 623, 478, 540, 1003, 260, 0, 0, 0, 0, 0, 0, 350, 315, 581, 536, 439, 1180, 1216, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 679, 1265, 1342, 245, 536, 212, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 539, 549, 1034, 836, 486, 395, 179, 621, 960, 64, 0, 0, 0, 0, 0, 0, 779, 1753, 731, 809, 845, 263, 656, 424, 220, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6043, 5838, 5955, 7139, 5659, 6334, 5663, 6521] avg=6144.0000 max=7139 max_rank=3 min=5659 min_rank=4 imbalance_ratio=1.161947 max_load_reduction=42013 max_load_reduction_pct=0.854757 moved_to_echo=24177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[741, 690, 318, 264, 290, 1396, 901, 769, 726, 0, 0, 0, 0, 0, 0, 0, 932, 442, 1207, 336, 479, 632, 718, 332, 1463, 0, 0, 0, 0, 0, 0, 0, 1010, 978, 1029, 825, 674, 0, 686, 381, 0, 0, 0, 0, 0, 0, 0, 0, 858, 515, 825, 565, 849, 488, 523, 383, 821, 203, 0, 0, 0, 0, 0, 0, 654, 265, 885, 794, 507, 1055, 1014, 884, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 473, 914, 1464, 351, 396, 181, 730, 0, 0, 0, 0, 0, 0, 0, 0, 525, 1351, 1157, 777, 792, 516, 340, 478, 788, 89, 0, 0, 0, 0, 0, 0, 831, 1509, 733, 597, 696, 319, 950, 625, 193, 53, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6541, 5583, 6030, 6058, 5526, 6813, 6506] avg=6144.0000 max=6813 max_rank=6 min=5526 min_rank=5 imbalance_ratio=1.108887 max_load_reduction=42339 max_load_reduction_pct=0.861389 moved_to_echo=24903 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[586, 604, 187, 340, 239, 988, 1721, 517, 842, 0, 0, 0, 0, 0, 0, 0, 710, 439, 642, 207, 478, 482, 1167, 129, 2255, 0, 0, 0, 0, 0, 0, 0, 1171, 931, 983, 564, 983, 0, 762, 352, 0, 0, 0, 0, 0, 0, 0, 0, 537, 942, 728, 344, 488, 1194, 667, 309, 709, 311, 0, 0, 0, 0, 0, 0, 556, 552, 551, 530, 436, 1031, 1292, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 906, 788, 1765, 1265, 477, 537, 156, 743, 0, 0, 0, 0, 0, 0, 0, 0, 848, 433, 1249, 740, 593, 509, 152, 437, 680, 103, 0, 0, 0, 0, 0, 0, 1222, 1207, 855, 606, 864, 308, 363, 467, 189, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6024, 6509, 5746, 6229, 6120, 6637, 5744, 6143] avg=6144.0000 max=6637 max_rank=5 min=5744 min_rank=6 imbalance_ratio=1.080241 max_load_reduction=42515 max_load_reduction_pct=0.864970 moved_to_echo=24644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[533, 780, 421, 227, 318, 738, 1300, 768, 897, 0, 0, 0, 0, 0, 0, 0, 463, 267, 800, 530, 370, 534, 860, 190, 1702, 0, 0, 0, 0, 0, 0, 0, 1248, 1336, 1039, 938, 716, 0, 1129, 382, 0, 0, 0, 0, 0, 0, 0, 0, 310, 988, 684, 381, 360, 1242, 504, 284, 820, 234, 0, 0, 0, 0, 0, 0, 396, 426, 425, 709, 436, 1236, 1665, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 780, 583, 1370, 1459, 415, 564, 181, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 668, 646, 878, 765, 600, 303, 133, 603, 786, 110, 0, 0, 0, 0, 0, 0, 727, 1671, 576, 740, 1057, 245, 565, 354, 227, 66, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5982, 5716, 6788, 5807, 6480, 6659, 5492, 6228] avg=6144.0000 max=6788 max_rank=2 min=5492 min_rank=6 imbalance_ratio=1.104818 max_load_reduction=42364 max_load_reduction_pct=0.861898 moved_to_echo=24859 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[489, 517, 193, 381, 348, 1181, 1204, 859, 891, 0, 0, 0, 0, 0, 0, 0, 763, 331, 736, 451, 381, 461, 898, 289, 2786, 0, 0, 0, 0, 0, 0, 0, 1240, 1035, 792, 1058, 901, 0, 1042, 382, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 949, 1088, 411, 549, 1114, 362, 271, 483, 385, 0, 0, 0, 0, 0, 0, 372, 699, 911, 544, 486, 1083, 1058, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 639, 519, 920, 824, 326, 605, 107, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 797, 625, 1049, 1329, 452, 396, 163, 624, 464, 109, 0, 0, 0, 0, 0, 0, 969, 809, 553, 623, 779, 272, 923, 441, 198, 66, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 7096, 6450, 6688, 6234, 4980, 6008, 5633] avg=6144.0000 max=7096 max_rank=1 min=4980 min_rank=5 imbalance_ratio=1.154948 max_load_reduction=42056 max_load_reduction_pct=0.855632 moved_to_echo=22855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[586, 604, 187, 340, 239, 988, 1721, 517, 842, 0, 0, 0, 0, 0, 0, 0, 710, 439, 642, 207, 478, 482, 1167, 129, 2255, 0, 0, 0, 0, 0, 0, 0, 1171, 931, 983, 564, 983, 0, 762, 352, 0, 0, 0, 0, 0, 0, 0, 0, 537, 942, 728, 344, 488, 1194, 667, 309, 709, 311, 0, 0, 0, 0, 0, 0, 556, 552, 551, 530, 436, 1031, 1292, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 906, 788, 1765, 1265, 477, 537, 156, 743, 0, 0, 0, 0, 0, 0, 0, 0, 848, 433, 1249, 740, 593, 509, 152, 437, 680, 103, 0, 0, 0, 0, 0, 0, 1222, 1207, 855, 606, 864, 308, 363, 467, 189, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6024, 6509, 5746, 6229, 6120, 6637, 5744, 6143] avg=6144.0000 max=6637 max_rank=5 min=5744 min_rank=6 imbalance_ratio=1.080241 max_load_reduction=42515 max_load_reduction_pct=0.864970 moved_to_echo=24644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[527, 796, 148, 237, 688, 1151, 1477, 523, 496, 0, 0, 0, 0, 0, 0, 0, 636, 232, 596, 410, 305, 698, 931, 140, 1890, 0, 0, 0, 0, 0, 0, 0, 671, 964, 1468, 604, 686, 0, 1131, 431, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 1065, 728, 444, 830, 623, 478, 540, 1003, 260, 0, 0, 0, 0, 0, 0, 350, 315, 581, 536, 439, 1180, 1216, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 679, 1265, 1342, 245, 536, 212, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 539, 549, 1034, 836, 486, 395, 179, 621, 960, 64, 0, 0, 0, 0, 0, 0, 779, 1753, 731, 809, 845, 263, 656, 424, 220, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6043, 5838, 5955, 7139, 5659, 6334, 5663, 6521] avg=6144.0000 max=7139 max_rank=3 min=5659 min_rank=4 imbalance_ratio=1.161947 max_load_reduction=42013 max_load_reduction_pct=0.854757 moved_to_echo=24177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 6656, 8448, 7808, 5504, 3328, 1664, 4608, 5760, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6272, 2560, 5760, 3584, 3200, 4224, 7040, 1792, 15232, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 6528, 6528, 3584, 4992, 7680, 3584, 3072, 6016, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7808, 9984, 5632, 5120, 6272, 2432, 6400, 3712, 1792, 512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8448, 5376, 9216, 10496, 3072, 4224, 1408, 7552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 6656, 8448, 7808, 5504, 3328, 1664, 4608, 5760, 768, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8576, 8960, 8320, 6656, 6528, 0, 7296, 3200, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5120, 4992, 1920, 2304, 2944, 9856, 11136, 5120, 6144, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3712, 3328, 5760, 4992, 4224, 9472, 10112, 8064, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 6528, 6528, 3584, 4992, 7680, 3584, 3072, 6016, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5120, 4992, 1920, 2304, 2944, 9856, 11136, 5120, 6144, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7808, 9984, 5632, 5120, 6272, 2432, 6400, 3712, 1792, 512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1826, 652, 1625, 748, 357, 1311, 515, 1850, 1071, 872, 973, 348, 1076, 508, 670, 505, 1101, 1278, 900, 1062, 654, 1034, 984, 1291, 321, 897, 286, 290, 893, 477, 1214, 147, 662, 629, 764, 386, 707, 1099, 555, 908, 233, 1430, 240, 239, 1014, 424, 328, 357, 688, 235, 325, 636, 403, 673, 527, 389, 432, 2033, 281, 578, 1101, 1265, 1078, 797] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1972, 560, 1669, 917, 360, 1297, 376, 1320, 1026, 991, 748, 622, 807, 473, 449, 424, 786, 1620, 1276, 766, 769, 733, 1375, 897, 279, 427, 280, 269, 1717, 395, 1497, 124, 521, 473, 735, 363, 624, 850, 903, 543, 365, 1030, 552, 260, 716, 411, 348, 480, 641, 272, 520, 935, 1004, 733, 480, 276, 813, 1384, 414, 934, 992, 1325, 1301, 733] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1922, 311, 2346, 691, 273, 1164, 463, 1594, 1006, 721, 858, 903, 1331, 667, 730, 319, 1551, 1086, 1655, 1129, 270, 1159, 1231, 1865, 219, 347, 652, 176, 473, 480, 899, 217, 767, 224, 1004, 318, 759, 1068, 533, 521, 302, 1580, 362, 246, 343, 322, 246, 342, 513, 199, 631, 707, 659, 1114, 378, 255, 511, 2148, 94, 894, 1117, 817, 957, 513] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1582, 406, 1703, 682, 336, 1035, 573, 1231, 815, 695, 1018, 1049, 905, 537, 566, 698, 1119, 905, 1554, 705, 689, 1288, 1143, 989, 928, 137, 583, 189, 938, 609, 1381, 198, 434, 299, 1343, 331, 905, 444, 695, 853, 1059, 1009, 403, 161, 408, 357, 335, 591, 757, 125, 504, 990, 1109, 1013, 503, 258, 673, 684, 228, 716, 1370, 1420, 1522, 467] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1595, 925, 1851, 412, 570, 1078, 494, 1486, 1031, 974, 741, 840, 713, 596, 1326, 224, 1158, 851, 1718, 757, 540, 1661, 1091, 1138, 482, 331, 709, 422, 552, 185, 1414, 196, 616, 361, 1079, 428, 505, 911, 639, 1030, 349, 1400, 357, 137, 333, 134, 158, 674, 1443, 268, 707, 707, 1055, 402, 491, 339, 777, 821, 543, 498, 724, 1227, 1321, 657] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1983, 662, 1939, 558, 358, 441, 429, 1629, 1339, 714, 1031, 618, 670, 373, 757, 544, 954, 1154, 1347, 896, 457, 772, 1175, 2364, 509, 172, 841, 404, 707, 444, 1624, 136, 633, 278, 684, 229, 614, 891, 784, 1128, 690, 696, 416, 278, 408, 266, 184, 829, 485, 187, 681, 698, 1089, 1215, 574, 403, 543, 620, 407, 859, 1582, 1188, 791, 821] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1826, 652, 1625, 748, 357, 1311, 515, 1850, 1071, 872, 973, 348, 1076, 508, 670, 505, 1101, 1278, 900, 1062, 654, 1034, 984, 1291, 321, 897, 286, 290, 893, 477, 1214, 147, 662, 629, 764, 386, 707, 1099, 555, 908, 233, 1430, 240, 239, 1014, 424, 328, 357, 688, 235, 325, 636, 403, 673, 527, 389, 432, 2033, 281, 578, 1101, 1265, 1078, 797] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1582, 406, 1703, 682, 336, 1035, 573, 1231, 815, 695, 1018, 1049, 905, 537, 566, 698, 1119, 905, 1554, 705, 689, 1288, 1143, 989, 928, 137, 583, 189, 938, 609, 1381, 198, 434, 299, 1343, 331, 905, 444, 695, 853, 1059, 1009, 403, 161, 408, 357, 335, 591, 757, 125, 504, 990, 1109, 1013, 503, 258, 673, 684, 228, 716, 1370, 1420, 1522, 467] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1866, 605, 1454, 999, 264, 772, 484, 1174, 809, 831, 810, 472, 771, 619, 830, 760, 1084, 575, 1186, 1200, 553, 620, 1253, 1672, 537, 485, 861, 508, 704, 253, 1507, 191, 632, 728, 928, 840, 655, 825, 881, 904, 508, 1497, 209, 217, 685, 166, 489, 391, 705, 173, 552, 1004, 858, 555, 239, 525, 547, 1173, 202, 419, 1084, 1572, 1533, 747] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1178, 648, 2087, 752, 231, 992, 797, 1385, 1104, 1132, 691, 809, 1152, 363, 627, 486, 477, 665, 625, 691, 557, 2043, 1244, 1301, 740, 436, 524, 709, 695, 371, 1685, 212, 494, 631, 863, 584, 689, 636, 562, 1056, 734, 1469, 105, 213, 1102, 264, 305, 371, 824, 334, 545, 919, 1445, 602, 297, 300, 688, 460, 444, 251, 542, 1088, 2356, 570] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1983, 662, 1939, 558, 358, 441, 429, 1629, 1339, 714, 1031, 618, 670, 373, 757, 544, 954, 1154, 1347, 896, 457, 772, 1175, 2364, 509, 172, 841, 404, 707, 444, 1624, 136, 633, 278, 684, 229, 614, 891, 784, 1128, 690, 696, 416, 278, 408, 266, 184, 829, 485, 187, 681, 698, 1089, 1215, 574, 403, 543, 620, 407, 859, 1582, 1188, 791, 821] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1922, 311, 2346, 691, 273, 1164, 463, 1594, 1006, 721, 858, 903, 1331, 667, 730, 319, 1551, 1086, 1655, 1129, 270, 1159, 1231, 1865, 219, 347, 652, 176, 473, 480, 899, 217, 767, 224, 1004, 318, 759, 1068, 533, 521, 302, 1580, 362, 246, 343, 322, 246, 342, 513, 199, 631, 707, 659, 1114, 378, 255, 511, 2148, 94, 894, 1117, 817, 957, 513] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1374, 925, 0, 412, 570, 1078, 494, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 974, 741, 840, 713, 596, 1326, 224, 1, 0, 0, 0, 0, 0, 0, 0, 1158, 851, 384, 757, 540, 1661, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 331, 709, 422, 552, 185, 1414, 196, 945, 351, 0, 0, 0, 0, 0, 0, 616, 361, 1079, 428, 505, 911, 639, 1030, 400, 220, 0, 0, 0, 0, 0, 0, 349, 1400, 357, 137, 333, 134, 158, 674, 1851, 193, 0, 0, 0, 0, 0, 0, 1443, 268, 707, 707, 1055, 402, 491, 339, 983, 511, 0, 0, 0, 0, 0, 0, 777, 821, 543, 498, 724, 1227, 410, 657, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6339, 6446, 6442, 5587, 6189, 5586, 6906, 5657] avg=6144.0000 max=6906 max_rank=6 min=5586 min_rank=5 imbalance_ratio=1.124023 max_load_reduction=42246 max_load_reduction_pct=0.859497 moved_to_echo=24338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1699, 560, 0, 917, 360, 1297, 376, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 991, 748, 622, 807, 473, 449, 424, 1, 0, 0, 0, 0, 0, 0, 0, 786, 1620, 285, 766, 769, 733, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 427, 280, 269, 1717, 395, 1497, 124, 745, 261, 0, 0, 0, 0, 0, 0, 521, 473, 735, 363, 624, 850, 903, 543, 394, 272, 0, 0, 0, 0, 0, 0, 365, 1030, 552, 260, 716, 411, 348, 480, 1669, 152, 0, 0, 0, 0, 0, 0, 641, 272, 520, 935, 1004, 733, 480, 276, 730, 503, 0, 0, 0, 0, 0, 0, 813, 1384, 414, 934, 992, 1325, 404, 733, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6529, 5541, 6334, 5994, 5678, 5983, 6094, 6999] avg=6144.0000 max=6999 max_rank=7 min=5541 min_rank=1 imbalance_ratio=1.139160 max_load_reduction=42153 max_load_reduction_pct=0.857605 moved_to_echo=24754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1708, 662, 0, 558, 358, 441, 429, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 714, 1031, 618, 670, 373, 757, 544, 1, 0, 0, 0, 0, 0, 0, 0, 954, 1154, 302, 896, 457, 772, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 172, 841, 404, 707, 444, 1624, 136, 1964, 275, 0, 0, 0, 0, 0, 0, 633, 278, 684, 229, 614, 891, 784, 1128, 239, 274, 0, 0, 0, 0, 0, 0, 690, 696, 416, 278, 408, 266, 184, 829, 1939, 400, 0, 0, 0, 0, 0, 0, 485, 187, 681, 698, 1089, 1215, 574, 403, 770, 306, 0, 0, 0, 0, 0, 0, 543, 620, 407, 859, 1582, 1188, 246, 821, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5785, 6047, 5710, 7076, 5754, 6106, 6408, 6266] avg=6144.0000 max=7076 max_rank=3 min=5710 min_rank=2 imbalance_ratio=1.151693 max_load_reduction=42076 max_load_reduction_pct=0.856038 moved_to_echo=24534 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1015, 648, 0, 752, 231, 992, 797, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 1132, 691, 809, 1152, 363, 627, 486, 1, 0, 0, 0, 0, 0, 0, 0, 477, 665, 141, 691, 557, 2043, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 436, 524, 709, 695, 371, 1685, 212, 1080, 127, 0, 0, 0, 0, 0, 0, 494, 631, 863, 584, 689, 636, 562, 1056, 713, 162, 0, 0, 0, 0, 0, 0, 734, 1469, 105, 213, 1102, 264, 305, 371, 2087, 221, 0, 0, 0, 0, 0, 0, 824, 334, 545, 919, 1445, 602, 297, 300, 357, 912, 0, 0, 0, 0, 0, 0, 688, 460, 444, 251, 542, 1088, 731, 570, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5820, 6365, 5818, 6579, 6390, 6871, 6535, 4774] avg=6144.0000 max=6871 max_rank=5 min=4774 min_rank=7 imbalance_ratio=1.118327 max_load_reduction=42281 max_load_reduction_pct=0.860209 moved_to_echo=24570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1608, 605, 0, 999, 264, 772, 484, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 809, 831, 810, 472, 771, 619, 830, 760, 1, 0, 0, 0, 0, 0, 0, 0, 1084, 575, 266, 1200, 553, 620, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 485, 861, 508, 704, 253, 1507, 191, 1389, 242, 0, 0, 0, 0, 0, 0, 632, 728, 928, 840, 655, 825, 881, 904, 464, 257, 0, 0, 0, 0, 0, 0, 508, 1497, 209, 217, 685, 166, 489, 391, 1454, 283, 0, 0, 0, 0, 0, 0, 705, 173, 552, 1004, 858, 555, 239, 525, 678, 593, 0, 0, 0, 0, 0, 0, 547, 1173, 202, 419, 1084, 1572, 476, 747, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5906, 5903, 5551, 6677, 7114, 5899, 5882, 6220] avg=6144.0000 max=7114 max_rank=4 min=5551 min_rank=2 imbalance_ratio=1.157878 max_load_reduction=42038 max_load_reduction_pct=0.855265 moved_to_echo=25115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1656, 311, 0, 691, 273, 1164, 463, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 721, 858, 903, 1331, 667, 730, 319, 1, 0, 0, 0, 0, 0, 0, 0, 1551, 1086, 370, 1129, 270, 1159, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 347, 652, 176, 473, 480, 899, 217, 1549, 338, 0, 0, 0, 0, 0, 0, 767, 224, 1004, 318, 759, 1068, 533, 521, 289, 265, 0, 0, 0, 0, 0, 0, 302, 1580, 362, 246, 343, 322, 246, 342, 2346, 316, 0, 0, 0, 0, 0, 0, 513, 199, 631, 707, 659, 1114, 378, 255, 947, 370, 0, 0, 0, 0, 0, 0, 511, 2148, 94, 894, 1117, 817, 298, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6536, 6796, 5350, 5748, 6405, 5773, 6392] avg=6144.0000 max=6796 max_rank=2 min=5350 min_rank=3 imbalance_ratio=1.106120 max_load_reduction=42356 max_load_reduction_pct=0.861735 moved_to_echo=24318 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1374, 925, 0, 412, 570, 1078, 494, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 974, 741, 840, 713, 596, 1326, 224, 1, 0, 0, 0, 0, 0, 0, 0, 1158, 851, 384, 757, 540, 1661, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 331, 709, 422, 552, 185, 1414, 196, 945, 351, 0, 0, 0, 0, 0, 0, 616, 361, 1079, 428, 505, 911, 639, 1030, 400, 220, 0, 0, 0, 0, 0, 0, 349, 1400, 357, 137, 333, 134, 158, 674, 1851, 193, 0, 0, 0, 0, 0, 0, 1443, 268, 707, 707, 1055, 402, 491, 339, 983, 511, 0, 0, 0, 0, 0, 0, 777, 821, 543, 498, 724, 1227, 410, 657, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6339, 6446, 6442, 5587, 6189, 5586, 6906, 5657] avg=6144.0000 max=6906 max_rank=6 min=5586 min_rank=5 imbalance_ratio=1.124023 max_load_reduction=42246 max_load_reduction_pct=0.859497 moved_to_echo=24338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1573, 652, 0, 748, 357, 1311, 515, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 872, 973, 348, 1076, 508, 670, 505, 1, 0, 0, 0, 0, 0, 0, 0, 1101, 1278, 201, 1062, 654, 1034, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 897, 286, 290, 893, 477, 1214, 147, 1072, 184, 0, 0, 0, 0, 0, 0, 662, 629, 764, 386, 707, 1099, 555, 908, 326, 252, 0, 0, 0, 0, 0, 0, 233, 1430, 240, 239, 1014, 424, 328, 357, 1625, 219, 0, 0, 0, 0, 0, 0, 688, 235, 325, 636, 403, 673, 527, 389, 515, 417, 0, 0, 0, 0, 0, 0, 432, 2033, 281, 578, 1101, 1265, 335, 797, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7006, 6024, 6314, 5781, 6288, 6109, 4808, 6822] avg=6144.0000 max=7006 max_rank=0 min=4808 min_rank=6 imbalance_ratio=1.140299 max_load_reduction=42146 max_load_reduction_pct=0.857463 moved_to_echo=24027 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1608, 605, 0, 999, 264, 772, 484, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 809, 831, 810, 472, 771, 619, 830, 760, 1, 0, 0, 0, 0, 0, 0, 0, 1084, 575, 266, 1200, 553, 620, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 485, 861, 508, 704, 253, 1507, 191, 1389, 242, 0, 0, 0, 0, 0, 0, 632, 728, 928, 840, 655, 825, 881, 904, 464, 257, 0, 0, 0, 0, 0, 0, 508, 1497, 209, 217, 685, 166, 489, 391, 1454, 283, 0, 0, 0, 0, 0, 0, 705, 173, 552, 1004, 858, 555, 239, 525, 678, 593, 0, 0, 0, 0, 0, 0, 547, 1173, 202, 419, 1084, 1572, 476, 747, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5906, 5903, 5551, 6677, 7114, 5899, 5882, 6220] avg=6144.0000 max=7114 max_rank=4 min=5551 min_rank=2 imbalance_ratio=1.157878 max_load_reduction=42038 max_load_reduction_pct=0.855265 moved_to_echo=25115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1352, 406, 0, 682, 336, 1035, 573, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 815, 695, 1018, 1049, 905, 537, 566, 698, 7, 0, 0, 0, 0, 0, 0, 0, 1119, 905, 338, 705, 689, 1288, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 137, 583, 189, 938, 609, 1381, 198, 822, 323, 0, 0, 0, 0, 0, 0, 434, 299, 1343, 331, 905, 444, 695, 853, 464, 223, 0, 0, 0, 0, 0, 0, 1059, 1009, 403, 161, 408, 357, 335, 591, 1703, 167, 0, 0, 0, 0, 0, 0, 757, 125, 504, 990, 1109, 1013, 503, 258, 893, 594, 0, 0, 0, 0, 0, 0, 673, 684, 228, 716, 1370, 1420, 464, 467, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5615, 6290, 6187, 6108, 5991, 6193, 6746, 6022] avg=6144.0000 max=6746 max_rank=6 min=5615 min_rank=0 imbalance_ratio=1.097982 max_load_reduction=42406 max_load_reduction_pct=0.862752 moved_to_echo=24952 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1015, 648, 0, 752, 231, 992, 797, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 1132, 691, 809, 1152, 363, 627, 486, 1, 0, 0, 0, 0, 0, 0, 0, 477, 665, 141, 691, 557, 2043, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 436, 524, 709, 695, 371, 1685, 212, 1080, 127, 0, 0, 0, 0, 0, 0, 494, 631, 863, 584, 689, 636, 562, 1056, 713, 162, 0, 0, 0, 0, 0, 0, 734, 1469, 105, 213, 1102, 264, 305, 371, 2087, 221, 0, 0, 0, 0, 0, 0, 824, 334, 545, 919, 1445, 602, 297, 300, 357, 912, 0, 0, 0, 0, 0, 0, 688, 460, 444, 251, 542, 1088, 731, 570, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5820, 6365, 5818, 6579, 6390, 6871, 6535, 4774] avg=6144.0000 max=6871 max_rank=5 min=4774 min_rank=7 imbalance_ratio=1.118327 max_load_reduction=42281 max_load_reduction_pct=0.860209 moved_to_echo=24570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1708, 662, 0, 558, 358, 441, 429, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 714, 1031, 618, 670, 373, 757, 544, 1, 0, 0, 0, 0, 0, 0, 0, 954, 1154, 302, 896, 457, 772, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 172, 841, 404, 707, 444, 1624, 136, 1964, 275, 0, 0, 0, 0, 0, 0, 633, 278, 684, 229, 614, 891, 784, 1128, 239, 274, 0, 0, 0, 0, 0, 0, 690, 696, 416, 278, 408, 266, 184, 829, 1939, 400, 0, 0, 0, 0, 0, 0, 485, 187, 681, 698, 1089, 1215, 574, 403, 770, 306, 0, 0, 0, 0, 0, 0, 543, 620, 407, 859, 1582, 1188, 246, 821, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5785, 6047, 5710, 7076, 5754, 6106, 6408, 6266] avg=6144.0000 max=7076 max_rank=3 min=5710 min_rank=2 imbalance_ratio=1.151693 max_load_reduction=42076 max_load_reduction_pct=0.856038 moved_to_echo=24534 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12032, 4864, 0, 5760, 2816, 8192, 4224, 11776, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4096, 3328, 4736, 3072, 6784, 3328, 11264, 1536, 9600, 2176, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6144, 1920, 4480, 6656, 7680, 6400, 3584, 2816, 5888, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4096, 3328, 4736, 3072, 6784, 3328, 11264, 1536, 9600, 2176, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4864, 3712, 7424, 3584, 5504, 6784, 5632, 7040, 3328, 2048, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8320, 7040, 6912, 5760, 7552, 4224, 6016, 3968, 128, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4992, 9344, 2688, 5248, 8576, 9984, 3456, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 10112, 2688, 1792, 5120, 2432, 2432, 4096, 14720, 2048, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8320, 8192, 2304, 7296, 4608, 9344, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 10112, 2688, 1792, 5120, 2432, 2432, 4096, 14720, 2048, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[457, 1357, 334, 830, 233, 288, 259, 319, 340, 113, 730, 407, 1054, 2015, 1087, 836, 626, 229, 285, 306, 1287, 1040, 130, 706, 555, 854, 1206, 1374, 1042, 1835, 1232, 725, 1644, 945, 1065, 270, 471, 1396, 692, 1542, 681, 472, 419, 759, 448, 589, 933, 832, 385, 541, 89, 738, 1121, 865, 1005, 539, 938, 304, 2237, 265, 84, 1462, 395, 935] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1463, 1361, 549, 554, 182, 166, 498, 381, 515, 255, 796, 336, 717, 1286, 1729, 840, 295, 533, 195, 560, 1041, 802, 399, 923, 514, 506, 920, 1040, 1154, 1543, 1471, 949, 938, 650, 1135, 373, 613, 2146, 847, 1261, 414, 1681, 476, 676, 336, 536, 635, 570, 517, 848, 193, 1173, 959, 622, 1164, 457, 648, 601, 1273, 365, 432, 890, 599, 651] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[812, 568, 699, 477, 210, 147, 90, 934, 575, 178, 1347, 545, 362, 1038, 1662, 331, 248, 441, 184, 288, 1422, 1765, 357, 601, 434, 984, 946, 1399, 1340, 1652, 1230, 752, 1663, 251, 1943, 776, 1087, 1735, 476, 1155, 314, 738, 274, 650, 1091, 386, 874, 707, 163, 640, 75, 923, 1606, 381, 1098, 519, 1111, 243, 1737, 524, 518, 480, 275, 721] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1225, 895, 345, 565, 268, 177, 299, 294, 566, 97, 760, 239, 1134, 712, 1106, 707, 451, 535, 373, 406, 1102, 1177, 321, 956, 421, 969, 1598, 1169, 1547, 1262, 1264, 430, 2083, 582, 595, 422, 437, 1346, 565, 1062, 536, 1469, 542, 911, 700, 444, 735, 379, 333, 465, 179, 1089, 1327, 799, 1767, 777, 597, 399, 2642, 433, 233, 908, 466, 560] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1236, 976, 390, 566, 370, 137, 179, 1527, 715, 148, 1328, 317, 546, 1103, 1575, 575, 296, 371, 650, 365, 919, 1388, 465, 492, 821, 450, 1862, 765, 840, 1514, 1012, 347, 1554, 506, 920, 288, 897, 1558, 1399, 1231, 904, 1757, 237, 991, 339, 728, 823, 578, 315, 824, 55, 976, 624, 527, 809, 598, 677, 113, 1940, 460, 838, 463, 470, 508] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[514, 1592, 368, 855, 383, 435, 173, 649, 682, 152, 1324, 487, 785, 928, 1194, 878, 389, 389, 540, 379, 1869, 1447, 208, 638, 807, 553, 1087, 892, 1133, 996, 708, 494, 989, 284, 1023, 154, 854, 1918, 315, 1564, 291, 744, 493, 1294, 629, 298, 557, 258, 205, 1031, 32, 1867, 1801, 571, 1759, 411, 765, 837, 1791, 753, 304, 459, 304, 669] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[549, 808, 571, 660, 281, 98, 385, 662, 386, 122, 879, 291, 899, 1367, 1612, 504, 343, 586, 560, 428, 1161, 716, 260, 407, 814, 1223, 1306, 1111, 1021, 1505, 1065, 605, 1887, 755, 1112, 353, 768, 1536, 1339, 1194, 587, 1470, 451, 662, 318, 861, 831, 663, 409, 1099, 61, 978, 741, 311, 854, 713, 586, 228, 1840, 545, 557, 861, 396, 1001] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[457, 1357, 334, 830, 233, 288, 259, 319, 340, 113, 730, 407, 1054, 2015, 1087, 836, 626, 229, 285, 306, 1287, 1040, 130, 706, 555, 854, 1206, 1374, 1042, 1835, 1232, 725, 1644, 945, 1065, 270, 471, 1396, 692, 1542, 681, 472, 419, 759, 448, 589, 933, 832, 385, 541, 89, 738, 1121, 865, 1005, 539, 938, 304, 2237, 265, 84, 1462, 395, 935] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1236, 976, 390, 566, 370, 137, 179, 1527, 715, 148, 1328, 317, 546, 1103, 1575, 575, 296, 371, 650, 365, 919, 1388, 465, 492, 821, 450, 1862, 765, 840, 1514, 1012, 347, 1554, 506, 920, 288, 897, 1558, 1399, 1231, 904, 1757, 237, 991, 339, 728, 823, 578, 315, 824, 55, 976, 624, 527, 809, 598, 677, 113, 1940, 460, 838, 463, 470, 508] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1225, 895, 345, 565, 268, 177, 299, 294, 566, 97, 760, 239, 1134, 712, 1106, 707, 451, 535, 373, 406, 1102, 1177, 321, 956, 421, 969, 1598, 1169, 1547, 1262, 1264, 430, 2083, 582, 595, 422, 437, 1346, 565, 1062, 536, 1469, 542, 911, 700, 444, 735, 379, 333, 465, 179, 1089, 1327, 799, 1767, 777, 597, 399, 2642, 433, 233, 908, 466, 560] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1010, 880, 267, 707, 379, 178, 293, 930, 695, 136, 1676, 286, 1197, 1340, 1387, 677, 479, 511, 568, 287, 1764, 915, 175, 790, 482, 914, 1585, 877, 1076, 1133, 1261, 687, 1095, 675, 397, 430, 876, 1842, 594, 841, 181, 1041, 567, 848, 599, 618, 631, 233, 292, 645, 54, 923, 1760, 256, 1861, 633, 772, 267, 1229, 966, 549, 599, 447, 889] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1463, 1361, 549, 554, 182, 166, 498, 381, 2055, 0, 0, 0, 0, 0, 0, 0, 515, 255, 796, 336, 717, 1286, 1509, 840, 0, 0, 0, 0, 0, 0, 0, 0, 295, 533, 195, 560, 1041, 802, 399, 923, 1057, 91, 0, 0, 0, 0, 0, 0, 514, 506, 513, 1040, 1154, 0, 1471, 949, 0, 0, 0, 0, 0, 0, 0, 0, 884, 650, 1135, 373, 613, 0, 847, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 414, 1681, 476, 676, 336, 536, 635, 570, 486, 250, 0, 0, 0, 0, 0, 0, 517, 848, 193, 1173, 959, 622, 1164, 457, 169, 54, 0, 0, 0, 0, 0, 0, 648, 601, 1273, 365, 432, 890, 599, 651, 157, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7209, 6254, 5896, 6147, 5763, 6060, 6156, 5667] avg=6144.0000 max=7209 max_rank=0 min=5667 min_rank=7 imbalance_ratio=1.173340 max_load_reduction=41943 max_load_reduction_pct=0.853333 moved_to_echo=23646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[549, 808, 571, 660, 281, 98, 385, 662, 1472, 0, 0, 0, 0, 0, 0, 0, 386, 122, 879, 291, 899, 1367, 1406, 504, 0, 0, 0, 0, 0, 0, 0, 0, 343, 586, 560, 428, 1161, 716, 260, 407, 1032, 64, 0, 0, 0, 0, 0, 0, 814, 1223, 726, 1111, 1021, 0, 1065, 605, 0, 0, 0, 0, 0, 0, 0, 0, 1778, 755, 1112, 353, 768, 0, 1339, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 587, 1470, 451, 662, 318, 861, 831, 663, 473, 356, 0, 0, 0, 0, 0, 0, 409, 1099, 61, 978, 741, 311, 854, 713, 158, 109, 0, 0, 0, 0, 0, 0, 586, 228, 1840, 545, 557, 861, 396, 1001, 224, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5486, 5854, 5557, 6565, 7299, 6672, 5433, 6286] avg=6144.0000 max=7299 max_rank=4 min=5433 min_rank=6 imbalance_ratio=1.187988 max_load_reduction=41853 max_load_reduction_pct=0.851501 moved_to_echo=25690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[812, 568, 699, 477, 210, 147, 90, 934, 1661, 0, 0, 0, 0, 0, 0, 0, 575, 178, 1347, 545, 362, 1038, 1450, 331, 0, 0, 0, 0, 0, 0, 0, 0, 248, 441, 184, 288, 1422, 1765, 357, 601, 1133, 74, 0, 0, 0, 0, 0, 0, 434, 984, 526, 1399, 1340, 0, 1230, 752, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 251, 1943, 776, 1087, 0, 476, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 314, 738, 274, 650, 1091, 386, 874, 707, 519, 258, 0, 0, 0, 0, 0, 0, 163, 640, 75, 923, 1606, 381, 1098, 519, 163, 96, 0, 0, 0, 0, 0, 0, 1111, 243, 1737, 524, 518, 480, 275, 721, 162, 49, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5598, 5826, 6513, 6665, 7255, 5811, 5664, 5820] avg=6144.0000 max=7255 max_rank=4 min=5598 min_rank=0 imbalance_ratio=1.180827 max_load_reduction=41897 max_load_reduction_pct=0.852397 moved_to_echo=24550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[514, 1592, 368, 855, 383, 435, 173, 649, 1838, 0, 0, 0, 0, 0, 0, 0, 682, 152, 1324, 487, 785, 928, 1035, 878, 0, 0, 0, 0, 0, 0, 0, 0, 389, 389, 540, 379, 1869, 1447, 208, 638, 683, 80, 0, 0, 0, 0, 0, 0, 807, 553, 597, 892, 1133, 0, 708, 494, 0, 0, 0, 0, 0, 0, 0, 0, 930, 284, 1023, 154, 854, 0, 315, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 291, 744, 493, 1294, 629, 298, 557, 258, 313, 301, 0, 0, 0, 0, 0, 0, 205, 1031, 32, 1867, 1801, 571, 1759, 411, 121, 59, 0, 0, 0, 0, 0, 0, 765, 837, 1791, 753, 304, 459, 304, 669, 189, 38, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6807, 6271, 6622, 5184, 5124, 5178, 7857, 6109] avg=6144.0000 max=7857 max_rank=6 min=5124 min_rank=4 imbalance_ratio=1.278809 max_load_reduction=41295 max_load_reduction_pct=0.840149 moved_to_echo=24268 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[812, 568, 699, 477, 210, 147, 90, 934, 575, 178, 1347, 545, 362, 1038, 1662, 331, 248, 441, 184, 288, 1422, 1765, 357, 601, 434, 984, 946, 1399, 1340, 1652, 1230, 752, 1663, 251, 1943, 776, 1087, 1735, 476, 1155, 314, 738, 274, 650, 1091, 386, 874, 707, 163, 640, 75, 923, 1606, 381, 1098, 519, 1111, 243, 1737, 524, 518, 480, 275, 721] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[457, 1357, 334, 830, 233, 288, 259, 319, 1337, 0, 0, 0, 0, 0, 0, 0, 340, 113, 730, 407, 1054, 2015, 949, 836, 0, 0, 0, 0, 0, 0, 0, 0, 626, 229, 285, 306, 1287, 1040, 130, 706, 1258, 59, 0, 0, 0, 0, 0, 0, 555, 854, 671, 1374, 1042, 0, 1232, 725, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 945, 1065, 270, 471, 0, 692, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 681, 472, 419, 759, 448, 589, 933, 832, 577, 328, 0, 0, 0, 0, 0, 0, 385, 541, 89, 738, 1121, 865, 1005, 539, 106, 95, 0, 0, 0, 0, 0, 0, 938, 304, 2237, 265, 84, 1462, 395, 935, 207, 32, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5414, 6444, 5926, 6453, 6534, 6038, 5484, 6859] avg=6144.0000 max=6859 max_rank=7 min=5414 min_rank=0 imbalance_ratio=1.116374 max_load_reduction=42293 max_load_reduction_pct=0.860453 moved_to_echo=24915 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1463, 1361, 549, 554, 182, 166, 498, 381, 2055, 0, 0, 0, 0, 0, 0, 0, 515, 255, 796, 336, 717, 1286, 1509, 840, 0, 0, 0, 0, 0, 0, 0, 0, 295, 533, 195, 560, 1041, 802, 399, 923, 1057, 91, 0, 0, 0, 0, 0, 0, 514, 506, 513, 1040, 1154, 0, 1471, 949, 0, 0, 0, 0, 0, 0, 0, 0, 884, 650, 1135, 373, 613, 0, 847, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 414, 1681, 476, 676, 336, 536, 635, 570, 486, 250, 0, 0, 0, 0, 0, 0, 517, 848, 193, 1173, 959, 622, 1164, 457, 169, 54, 0, 0, 0, 0, 0, 0, 648, 601, 1273, 365, 432, 890, 599, 651, 157, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7209, 6254, 5896, 6147, 5763, 6060, 6156, 5667] avg=6144.0000 max=7209 max_rank=0 min=5667 min_rank=7 imbalance_ratio=1.173340 max_load_reduction=41943 max_load_reduction_pct=0.853333 moved_to_echo=23646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1236, 976, 390, 566, 370, 137, 179, 1527, 1492, 0, 0, 0, 0, 0, 0, 0, 715, 148, 1328, 317, 546, 1103, 1374, 575, 0, 0, 0, 0, 0, 0, 0, 0, 296, 371, 650, 365, 919, 1388, 465, 492, 1038, 66, 0, 0, 0, 0, 0, 0, 821, 450, 1036, 765, 840, 0, 1012, 347, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 506, 920, 288, 897, 0, 1399, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 904, 1757, 237, 991, 339, 728, 823, 578, 476, 507, 0, 0, 0, 0, 0, 0, 315, 824, 55, 976, 624, 527, 809, 598, 154, 89, 0, 0, 0, 0, 0, 0, 677, 113, 1940, 460, 838, 463, 470, 508, 319, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6873, 6106, 6050, 5271, 6706, 7340, 4971, 5835] avg=6144.0000 max=7340 max_rank=5 min=4971 min_rank=6 imbalance_ratio=1.194661 max_load_reduction=41812 max_load_reduction_pct=0.850667 moved_to_echo=24852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[549, 808, 571, 660, 281, 98, 385, 662, 1472, 0, 0, 0, 0, 0, 0, 0, 386, 122, 879, 291, 899, 1367, 1406, 504, 0, 0, 0, 0, 0, 0, 0, 0, 343, 586, 560, 428, 1161, 716, 260, 407, 1032, 64, 0, 0, 0, 0, 0, 0, 814, 1223, 726, 1111, 1021, 0, 1065, 605, 0, 0, 0, 0, 0, 0, 0, 0, 1778, 755, 1112, 353, 768, 0, 1339, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 587, 1470, 451, 662, 318, 861, 831, 663, 473, 356, 0, 0, 0, 0, 0, 0, 409, 1099, 61, 978, 741, 311, 854, 713, 158, 109, 0, 0, 0, 0, 0, 0, 586, 228, 1840, 545, 557, 861, 396, 1001, 224, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5486, 5854, 5557, 6565, 7299, 6672, 5433, 6286] avg=6144.0000 max=7299 max_rank=4 min=5433 min_rank=6 imbalance_ratio=1.187988 max_load_reduction=41853 max_load_reduction_pct=0.851501 moved_to_echo=25690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1225, 895, 345, 565, 268, 177, 299, 294, 1289, 0, 0, 0, 0, 0, 0, 0, 566, 97, 760, 239, 1134, 712, 965, 707, 0, 0, 0, 0, 0, 0, 0, 0, 451, 535, 373, 406, 1102, 1177, 321, 956, 866, 57, 0, 0, 0, 0, 0, 0, 421, 969, 889, 1169, 1547, 0, 1264, 430, 0, 0, 0, 0, 0, 0, 0, 0, 1963, 582, 595, 422, 437, 0, 565, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 536, 1469, 542, 911, 700, 444, 735, 379, 396, 435, 0, 0, 0, 0, 0, 0, 333, 465, 179, 1089, 1327, 799, 1767, 777, 108, 120, 0, 0, 0, 0, 0, 0, 597, 399, 2642, 433, 233, 908, 466, 560, 274, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5357, 5180, 6244, 6689, 5626, 6547, 6964, 6545] avg=6144.0000 max=6964 max_rank=6 min=5180 min_rank=1 imbalance_ratio=1.133464 max_load_reduction=42188 max_load_reduction_pct=0.858317 moved_to_echo=25682 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[812, 568, 699, 477, 210, 147, 90, 934, 1661, 0, 0, 0, 0, 0, 0, 0, 575, 178, 1347, 545, 362, 1038, 1450, 331, 0, 0, 0, 0, 0, 0, 0, 0, 248, 441, 184, 288, 1422, 1765, 357, 601, 1133, 74, 0, 0, 0, 0, 0, 0, 434, 984, 526, 1399, 1340, 0, 1230, 752, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 251, 1943, 776, 1087, 0, 476, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 314, 738, 274, 650, 1091, 386, 874, 707, 519, 258, 0, 0, 0, 0, 0, 0, 163, 640, 75, 923, 1606, 381, 1098, 519, 163, 96, 0, 0, 0, 0, 0, 0, 1111, 243, 1737, 524, 518, 480, 275, 721, 162, 49, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5598, 5826, 6513, 6665, 7255, 5811, 5664, 5820] avg=6144.0000 max=7255 max_rank=4 min=5598 min_rank=0 imbalance_ratio=1.180827 max_load_reduction=41897 max_load_reduction_pct=0.852397 moved_to_echo=24550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1010, 880, 267, 707, 379, 178, 293, 930, 1764, 0, 0, 0, 0, 0, 0, 0, 695, 136, 1676, 286, 1197, 1340, 1210, 677, 0, 0, 0, 0, 0, 0, 0, 0, 479, 511, 568, 287, 1764, 915, 175, 790, 777, 78, 0, 0, 0, 0, 0, 0, 482, 914, 881, 877, 1076, 0, 1261, 687, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 675, 397, 430, 876, 0, 594, 841, 0, 0, 0, 0, 0, 0, 0, 0, 181, 1041, 567, 848, 599, 618, 631, 233, 356, 432, 0, 0, 0, 0, 0, 0, 292, 645, 54, 923, 1760, 256, 1861, 633, 136, 63, 0, 0, 0, 0, 0, 0, 772, 267, 1229, 966, 549, 599, 447, 889, 272, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6408, 7217, 6344, 6178, 4845, 5506, 6623, 6031] avg=6144.0000 max=7217 max_rank=1 min=4845 min_rank=4 imbalance_ratio=1.174642 max_load_reduction=41935 max_load_reduction_pct=0.853170 moved_to_echo=23005 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[514, 1592, 368, 855, 383, 435, 173, 649, 1838, 0, 0, 0, 0, 0, 0, 0, 682, 152, 1324, 487, 785, 928, 1035, 878, 0, 0, 0, 0, 0, 0, 0, 0, 389, 389, 540, 379, 1869, 1447, 208, 638, 683, 80, 0, 0, 0, 0, 0, 0, 807, 553, 597, 892, 1133, 0, 708, 494, 0, 0, 0, 0, 0, 0, 0, 0, 930, 284, 1023, 154, 854, 0, 315, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 291, 744, 493, 1294, 629, 298, 557, 258, 313, 301, 0, 0, 0, 0, 0, 0, 205, 1031, 32, 1867, 1801, 571, 1759, 411, 121, 59, 0, 0, 0, 0, 0, 0, 765, 837, 1791, 753, 304, 459, 304, 669, 189, 38, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6807, 6271, 6622, 5184, 5124, 5178, 7857, 6109] avg=6144.0000 max=7857 max_rank=6 min=5124 min_rank=4 imbalance_ratio=1.278809 max_load_reduction=41295 max_load_reduction_pct=0.840149 moved_to_echo=24268 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4864, 6528, 5888, 8704, 9216, 0, 9344, 4992, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[11264, 4736, 8192, 3072, 6016, 0, 6272, 9856, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 1280, 8960, 2944, 6784, 9856, 9984, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 6144, 768, 8704, 9984, 4352, 10368, 4736, 1152, 768, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 9472, 3584, 6912, 4480, 4480, 6144, 4224, 3712, 2944, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7296, 8448, 3584, 5248, 2432, 1664, 2176, 5760, 12928, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 6144, 768, 8704, 9984, 4352, 10368, 4736, 1152, 768, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6144, 3072, 14720, 4352, 3584, 6144, 3456, 6016, 1920, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 1280, 8960, 2944, 6784, 9856, 9984, 5376, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[3200, 3712, 3456, 3072, 10624, 9344, 2432, 5632, 7936, 640, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[531, 727, 752, 727, 1438, 92, 998, 377, 813, 1181, 565, 352, 617, 644, 820, 1604, 861, 1107, 150, 259, 1843, 484, 819, 1003, 768, 293, 186, 506, 738, 889, 271, 1449, 130, 580, 478, 577, 695, 752, 286, 704, 586, 1536, 1124, 522, 396, 434, 351, 1279, 501, 254, 938, 882, 1022, 2179, 273, 922, 1146, 1198, 728, 179, 662, 677, 960, 2337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[965, 321, 1382, 297, 928, 170, 1150, 839, 527, 582, 371, 285, 1132, 219, 534, 721, 704, 755, 115, 238, 2212, 568, 1207, 207, 729, 444, 372, 537, 390, 1617, 320, 1713, 936, 709, 870, 255, 720, 350, 1045, 658, 780, 2254, 724, 1027, 239, 769, 653, 626, 161, 705, 756, 699, 1493, 1445, 218, 825, 964, 892, 260, 673, 693, 841, 896, 2465] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1426, 365, 1116, 431, 1855, 99, 956, 610, 638, 675, 317, 301, 878, 425, 656, 1154, 728, 756, 204, 427, 2075, 820, 1546, 453, 875, 499, 275, 730, 1196, 1430, 216, 1332, 341, 593, 534, 485, 624, 541, 832, 1306, 544, 942, 748, 931, 271, 836, 463, 1231, 156, 289, 1283, 871, 944, 1619, 375, 824, 641, 531, 285, 254, 894, 367, 834, 2299] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1583, 422, 1053, 365, 1244, 271, 566, 77, 571, 433, 291, 422, 732, 829, 774, 999, 860, 935, 90, 189, 1751, 249, 2415, 646, 411, 539, 350, 164, 617, 1688, 318, 1677, 981, 464, 928, 578, 1138, 683, 420, 482, 750, 1460, 1011, 998, 413, 334, 418, 869, 537, 235, 1643, 764, 1011, 1432, 383, 803, 1100, 622, 259, 856, 276, 558, 913, 2302] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[730, 321, 711, 340, 2035, 193, 1830, 399, 292, 279, 238, 578, 563, 212, 593, 726, 942, 2689, 81, 253, 1916, 495, 981, 524, 487, 322, 215, 764, 1076, 961, 548, 2219, 484, 515, 1122, 165, 614, 800, 883, 1452, 681, 1754, 897, 1551, 461, 535, 448, 447, 272, 470, 312, 505, 1878, 1379, 247, 838, 753, 744, 422, 471, 442, 683, 729, 1685] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[814, 391, 1290, 828, 1434, 95, 398, 681, 382, 399, 469, 233, 446, 1087, 929, 178, 644, 2255, 183, 234, 1609, 531, 774, 689, 1085, 719, 70, 779, 898, 746, 317, 1742, 315, 988, 398, 399, 692, 536, 223, 444, 258, 1367, 2137, 1883, 461, 601, 462, 733, 258, 853, 485, 1071, 1097, 1008, 351, 944, 1210, 484, 1000, 255, 637, 970, 802, 2501] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1091, 631, 747, 310, 1154, 84, 1719, 476, 735, 215, 195, 204, 1093, 138, 1117, 665, 916, 1562, 113, 340, 2362, 210, 1151, 485, 597, 617, 174, 314, 732, 2457, 1220, 1468, 577, 581, 801, 393, 281, 294, 306, 795, 471, 1581, 922, 584, 347, 722, 439, 738, 333, 263, 902, 648, 1675, 1758, 673, 844, 1297, 1097, 683, 203, 379, 94, 870, 2309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[965, 321, 1382, 297, 928, 170, 1150, 839, 527, 582, 371, 285, 1132, 219, 534, 721, 704, 755, 115, 238, 2212, 568, 1207, 207, 729, 444, 372, 537, 390, 1617, 320, 1713, 936, 709, 870, 255, 720, 350, 1045, 658, 780, 2254, 724, 1027, 239, 769, 653, 626, 161, 705, 756, 699, 1493, 1445, 218, 825, 964, 892, 260, 673, 693, 841, 896, 2465] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1583, 422, 1053, 365, 1244, 271, 566, 77, 571, 433, 291, 422, 732, 829, 774, 999, 860, 935, 90, 189, 1751, 249, 2415, 646, 411, 539, 350, 164, 617, 1688, 318, 1677, 981, 464, 928, 578, 1138, 683, 420, 482, 750, 1460, 1011, 998, 413, 334, 418, 869, 537, 235, 1643, 764, 1011, 1432, 383, 803, 1100, 622, 259, 856, 276, 558, 913, 2302] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1426, 365, 1116, 431, 1855, 99, 956, 610, 638, 675, 317, 301, 878, 425, 656, 1154, 728, 756, 204, 427, 2075, 820, 1546, 453, 875, 499, 275, 730, 1196, 1430, 216, 1332, 341, 593, 534, 485, 624, 541, 832, 1306, 544, 942, 748, 931, 271, 836, 463, 1231, 156, 289, 1283, 871, 944, 1619, 375, 824, 641, 531, 285, 254, 894, 367, 834, 2299] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1078, 703, 713, 649, 1322, 193, 1258, 792, 366, 477, 258, 326, 618, 720, 1043, 1025, 247, 1082, 178, 248, 2077, 227, 2459, 568, 625, 345, 166, 1053, 672, 1081, 383, 1417, 373, 778, 408, 361, 940, 682, 507, 401, 568, 1482, 1382, 897, 250, 696, 755, 1382, 178, 336, 956, 933, 1564, 1466, 142, 653, 832, 797, 504, 377, 555, 749, 592, 2287] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[965, 321, 1382, 297, 897, 170, 1150, 839, 0, 0, 0, 0, 0, 0, 0, 0, 527, 582, 371, 285, 1132, 219, 534, 721, 924, 623, 0, 0, 0, 0, 0, 0, 704, 755, 115, 238, 1299, 568, 1207, 207, 0, 0, 0, 0, 0, 0, 0, 0, 729, 444, 372, 537, 390, 1617, 320, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 936, 709, 870, 255, 720, 350, 1045, 658, 775, 303, 290, 86, 31, 0, 0, 0, 780, 1479, 724, 1027, 239, 769, 653, 626, 0, 0, 0, 0, 0, 0, 0, 0, 161, 705, 756, 699, 1493, 1142, 218, 825, 0, 0, 0, 0, 0, 0, 0, 0, 964, 892, 260, 673, 693, 841, 896, 1541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6021, 5918, 5093, 6036, 7028, 6297, 5999, 6760] avg=6144.0000 max=7028 max_rank=4 min=5093 min_rank=2 imbalance_ratio=1.143880 max_load_reduction=42124 max_load_reduction_pct=0.857015 moved_to_echo=26084 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[730, 321, 711, 340, 2035, 193, 1830, 399, 292, 279, 238, 578, 563, 212, 593, 726, 942, 2689, 81, 253, 1916, 495, 981, 524, 487, 322, 215, 764, 1076, 961, 548, 2219, 484, 515, 1122, 165, 614, 800, 883, 1452, 681, 1754, 897, 1551, 461, 535, 448, 447, 272, 470, 312, 505, 1878, 1379, 247, 838, 753, 744, 422, 471, 442, 683, 729, 1685] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[531, 727, 752, 727, 1438, 92, 998, 377, 813, 1181, 565, 352, 617, 644, 820, 1604, 861, 1107, 150, 259, 1843, 484, 819, 1003, 768, 293, 186, 506, 738, 889, 271, 1449, 130, 580, 478, 577, 695, 752, 286, 704, 586, 1536, 1124, 522, 396, 434, 351, 1279, 501, 254, 938, 882, 1022, 2179, 273, 922, 1146, 1198, 728, 179, 662, 677, 960, 2337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[814, 391, 1290, 828, 1385, 95, 398, 681, 0, 0, 0, 0, 0, 0, 0, 0, 382, 399, 469, 233, 446, 1087, 929, 178, 938, 453, 0, 0, 0, 0, 0, 0, 644, 2255, 183, 234, 945, 531, 774, 689, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 719, 70, 779, 898, 746, 317, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 315, 988, 398, 399, 692, 536, 223, 444, 470, 211, 211, 87, 49, 0, 0, 0, 258, 897, 2137, 1883, 461, 601, 462, 733, 0, 0, 0, 0, 0, 0, 0, 0, 258, 853, 485, 1071, 1097, 797, 351, 944, 0, 0, 0, 0, 0, 0, 0, 0, 1210, 484, 1000, 255, 637, 970, 802, 1563, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5882, 5514, 6255, 6269, 5023, 7432, 5856, 6921] avg=6144.0000 max=7432 max_rank=5 min=5023 min_rank=4 imbalance_ratio=1.209635 max_load_reduction=41720 max_load_reduction_pct=0.848796 moved_to_echo=25232 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1426, 365, 1116, 431, 1792, 99, 956, 610, 0, 0, 0, 0, 0, 0, 0, 0, 638, 675, 317, 301, 878, 425, 656, 1154, 862, 585, 0, 0, 0, 0, 0, 0, 728, 756, 204, 427, 1218, 820, 1546, 453, 0, 0, 0, 0, 0, 0, 0, 0, 875, 499, 275, 730, 1196, 1430, 216, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 341, 593, 534, 485, 624, 541, 832, 1306, 324, 340, 272, 67, 63, 0, 0, 0, 544, 618, 748, 931, 271, 836, 463, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 156, 289, 1283, 871, 944, 1279, 375, 824, 0, 0, 0, 0, 0, 0, 0, 0, 641, 531, 285, 254, 894, 367, 834, 1437, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6795, 6491, 6152, 6486, 6322, 5642, 6021, 5243] avg=6144.0000 max=6795 max_rank=0 min=5243 min_rank=7 imbalance_ratio=1.105957 max_load_reduction=42357 max_load_reduction_pct=0.861755 moved_to_echo=23228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1091, 631, 747, 310, 1115, 84, 1719, 476, 0, 0, 0, 0, 0, 0, 0, 0, 735, 215, 195, 204, 1093, 138, 1117, 665, 866, 666, 0, 0, 0, 0, 0, 0, 916, 1562, 113, 340, 1386, 210, 1151, 485, 0, 0, 0, 0, 0, 0, 0, 0, 597, 617, 174, 314, 732, 2457, 1220, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 577, 581, 801, 393, 281, 294, 306, 795, 544, 369, 310, 73, 39, 0, 0, 0, 471, 1037, 922, 584, 347, 722, 439, 738, 0, 0, 0, 0, 0, 0, 0, 0, 333, 263, 902, 648, 1675, 1389, 673, 844, 0, 0, 0, 0, 0, 0, 0, 0, 1297, 1097, 683, 203, 379, 94, 870, 1443, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6173, 5894, 6163, 7506, 5363, 5260, 6727, 6066] avg=6144.0000 max=7506 max_rank=3 min=5260 min_rank=5 imbalance_ratio=1.221680 max_load_reduction=41646 max_load_reduction_pct=0.847290 moved_to_echo=23416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1426, 365, 1116, 431, 1792, 99, 956, 610, 0, 0, 0, 0, 0, 0, 0, 0, 638, 675, 317, 301, 878, 425, 656, 1154, 862, 585, 0, 0, 0, 0, 0, 0, 728, 756, 204, 427, 1218, 820, 1546, 453, 0, 0, 0, 0, 0, 0, 0, 0, 875, 499, 275, 730, 1196, 1430, 216, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 341, 593, 534, 485, 624, 541, 832, 1306, 324, 340, 272, 67, 63, 0, 0, 0, 544, 618, 748, 931, 271, 836, 463, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 156, 289, 1283, 871, 944, 1279, 375, 824, 0, 0, 0, 0, 0, 0, 0, 0, 641, 531, 285, 254, 894, 367, 834, 1437, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6795, 6491, 6152, 6486, 6322, 5642, 6021, 5243] avg=6144.0000 max=6795 max_rank=0 min=5243 min_rank=7 imbalance_ratio=1.105957 max_load_reduction=42357 max_load_reduction_pct=0.861755 moved_to_echo=23228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[531, 727, 752, 727, 1389, 92, 998, 377, 0, 0, 0, 0, 0, 0, 0, 0, 813, 1181, 565, 352, 617, 644, 820, 1604, 876, 519, 0, 0, 0, 0, 0, 0, 861, 1107, 150, 259, 1082, 484, 819, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 768, 293, 186, 506, 738, 889, 271, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 130, 580, 478, 577, 695, 752, 286, 704, 528, 457, 242, 73, 49, 0, 0, 0, 586, 1008, 1124, 522, 396, 434, 351, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 501, 254, 938, 882, 1022, 1722, 273, 922, 0, 0, 0, 0, 0, 0, 0, 0, 1146, 1198, 728, 179, 662, 677, 960, 1461, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5593, 7991, 5765, 5027, 5551, 5700, 6514, 7011] avg=6144.0000 max=7991 max_rank=1 min=5027 min_rank=3 imbalance_ratio=1.300618 max_load_reduction=41161 max_load_reduction_pct=0.837423 moved_to_echo=24776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[965, 321, 1382, 297, 897, 170, 1150, 839, 0, 0, 0, 0, 0, 0, 0, 0, 527, 582, 371, 285, 1132, 219, 534, 721, 924, 623, 0, 0, 0, 0, 0, 0, 704, 755, 115, 238, 1299, 568, 1207, 207, 0, 0, 0, 0, 0, 0, 0, 0, 729, 444, 372, 537, 390, 1617, 320, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 936, 709, 870, 255, 720, 350, 1045, 658, 775, 303, 290, 86, 31, 0, 0, 0, 780, 1479, 724, 1027, 239, 769, 653, 626, 0, 0, 0, 0, 0, 0, 0, 0, 161, 705, 756, 699, 1493, 1142, 218, 825, 0, 0, 0, 0, 0, 0, 0, 0, 964, 892, 260, 673, 693, 841, 896, 1541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6021, 5918, 5093, 6036, 7028, 6297, 5999, 6760] avg=6144.0000 max=7028 max_rank=4 min=5093 min_rank=2 imbalance_ratio=1.143880 max_load_reduction=42124 max_load_reduction_pct=0.857015 moved_to_echo=26084 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1078, 703, 713, 649, 1277, 193, 1258, 792, 0, 0, 0, 0, 0, 0, 0, 0, 366, 477, 258, 326, 618, 720, 1043, 1025, 858, 585, 0, 0, 0, 0, 0, 0, 247, 1082, 178, 248, 1220, 227, 2459, 568, 0, 0, 0, 0, 0, 0, 0, 0, 625, 345, 166, 1053, 672, 1081, 383, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 373, 778, 408, 361, 940, 682, 507, 401, 510, 308, 272, 71, 45, 0, 0, 0, 568, 972, 1382, 897, 250, 696, 755, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 178, 336, 956, 933, 1564, 1158, 142, 653, 0, 0, 0, 0, 0, 0, 0, 0, 832, 797, 504, 377, 555, 749, 592, 1429, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6663, 6276, 6229, 5671, 5656, 6902, 5920, 5835] avg=6144.0000 max=6902 max_rank=5 min=5656 min_rank=4 imbalance_ratio=1.123372 max_load_reduction=42250 max_load_reduction_pct=0.859578 moved_to_echo=24313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1583, 422, 1053, 365, 1202, 271, 566, 77, 0, 0, 0, 0, 0, 0, 0, 0, 571, 433, 291, 422, 732, 829, 774, 999, 863, 493, 0, 0, 0, 0, 0, 0, 860, 935, 90, 189, 1029, 249, 2415, 646, 0, 0, 0, 0, 0, 0, 0, 0, 411, 539, 350, 164, 617, 1688, 318, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 981, 464, 928, 578, 1138, 683, 420, 482, 502, 300, 229, 84, 42, 0, 0, 0, 750, 958, 1011, 998, 413, 334, 418, 869, 0, 0, 0, 0, 0, 0, 0, 0, 537, 235, 1643, 764, 1011, 1132, 383, 803, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 622, 259, 856, 276, 558, 913, 1439, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5539, 6407, 6413, 5680, 6831, 5751, 6508, 6023] avg=6144.0000 max=6831 max_rank=4 min=5539 min_rank=0 imbalance_ratio=1.111816 max_load_reduction=42321 max_load_reduction_pct=0.861023 moved_to_echo=25113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[730, 321, 711, 340, 1962, 193, 1830, 399, 0, 0, 0, 0, 0, 0, 0, 0, 292, 279, 238, 578, 563, 212, 593, 726, 636, 544, 0, 0, 0, 0, 0, 0, 942, 2689, 81, 253, 1117, 495, 981, 524, 0, 0, 0, 0, 0, 0, 0, 0, 487, 322, 215, 764, 1076, 961, 548, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 484, 515, 1122, 165, 614, 800, 883, 1452, 606, 294, 255, 115, 73, 0, 0, 0, 681, 1148, 897, 1551, 461, 535, 448, 447, 0, 0, 0, 0, 0, 0, 0, 0, 272, 470, 312, 505, 1878, 1085, 247, 838, 0, 0, 0, 0, 0, 0, 0, 0, 753, 744, 422, 471, 442, 683, 729, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6486, 4661, 7082, 6477, 7378, 6168, 5607, 5293] avg=6144.0000 max=7378 max_rank=4 min=4661 min_rank=1 imbalance_ratio=1.200846 max_load_reduction=41774 max_load_reduction_pct=0.849894 moved_to_echo=24446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1078, 703, 713, 649, 1277, 193, 1258, 792, 0, 0, 0, 0, 0, 0, 0, 0, 366, 477, 258, 326, 618, 720, 1043, 1025, 858, 585, 0, 0, 0, 0, 0, 0, 247, 1082, 178, 248, 1220, 227, 2459, 568, 0, 0, 0, 0, 0, 0, 0, 0, 625, 345, 166, 1053, 672, 1081, 383, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 373, 778, 408, 361, 940, 682, 507, 401, 510, 308, 272, 71, 45, 0, 0, 0, 568, 972, 1382, 897, 250, 696, 755, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 178, 336, 956, 933, 1564, 1158, 142, 653, 0, 0, 0, 0, 0, 0, 0, 0, 832, 797, 504, 377, 555, 749, 592, 1429, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6663, 6276, 6229, 5671, 5656, 6902, 5920, 5835] avg=6144.0000 max=6902 max_rank=5 min=5656 min_rank=4 imbalance_ratio=1.123372 max_load_reduction=42250 max_load_reduction_pct=0.859578 moved_to_echo=24313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4352, 4352, 2816, 2816, 6144, 4352, 6528, 7168, 6912, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8064, 6400, 4224, 3328, 4608, 4992, 6656, 11392, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50304 observed_loss=-1152 tokens_per_expert=[4224, 5248, 5632, 3328, 5760, 4736, 4608, 6272, 4352, 2688, 2176, 768, 512, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8320, 3968, 7808, 3968, 11136, 1280, 8960, 4352, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4736, 8192, 8960, 8448, 2944, 4992, 4096, 7424, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50304 observed_loss=-1152 tokens_per_expert=[4224, 5248, 5632, 3328, 5760, 4736, 4608, 6272, 4352, 2688, 2176, 768, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4352, 4352, 2816, 2816, 6144, 4352, 6528, 7168, 6912, 4480, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5632, 3840, 1920, 4864, 6400, 10880, 3712, 12416, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 11264, 1152, 2304, 9344, 3584, 11392, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8320, 3968, 7808, 3968, 11136, 1280, 8960, 4352, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[2432, 3456, 7296, 6400, 10752, 9728, 2688, 6656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50304 slack_slots=48000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50304 slack_slots=48000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[889, 1373, 1094, 1484, 792, 145, 1099, 168, 572, 224, 229, 472, 408, 533, 359, 127, 283, 1809, 179, 1268, 495, 66, 544, 3208, 368, 394, 873, 1032, 2157, 81, 532, 464, 223, 2927, 287, 593, 805, 697, 2421, 1400, 951, 317, 526, 46, 551, 228, 419, 1006, 1762, 351, 355, 826, 687, 763, 632, 434, 952, 1064, 915, 793, 199, 1597, 492, 212] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[668, 1534, 1932, 432, 185, 128, 1479, 580, 1265, 496, 824, 205, 667, 459, 213, 346, 557, 1098, 340, 917, 2046, 283, 613, 2501, 598, 377, 902, 911, 1382, 360, 379, 457, 606, 1349, 990, 1276, 1928, 1020, 1017, 370, 532, 826, 638, 594, 459, 358, 202, 619, 1448, 200, 218, 156, 335, 880, 531, 1179, 650, 631, 1321, 596, 639, 1241, 934, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[924, 1888, 2111, 1004, 465, 146, 1108, 88, 876, 800, 478, 243, 909, 521, 469, 220, 197, 710, 177, 1049, 796, 285, 354, 2056, 1216, 325, 1704, 674, 1100, 484, 492, 550, 326, 1148, 594, 1051, 514, 863, 1880, 1126, 1935, 523, 663, 499, 228, 332, 159, 490, 952, 249, 242, 340, 964, 772, 803, 314, 784, 1332, 918, 96, 1703, 2308, 273, 352] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[502, 2424, 1104, 1402, 231, 105, 1260, 116, 582, 235, 468, 217, 364, 400, 706, 436, 322, 617, 126, 1406, 1339, 260, 207, 2285, 383, 487, 692, 632, 1250, 646, 312, 628, 560, 2245, 787, 656, 979, 928, 2426, 859, 978, 932, 353, 105, 784, 58, 294, 471, 835, 159, 241, 718, 1275, 596, 1438, 182, 730, 2126, 702, 653, 1172, 1442, 904, 420] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1637, 894, 751, 1155, 806, 106, 2024, 211, 1802, 308, 475, 368, 253, 649, 143, 329, 314, 573, 215, 733, 415, 136, 760, 1689, 269, 476, 1290, 1431, 1856, 211, 180, 565, 1749, 1420, 477, 1328, 1417, 1708, 1364, 581, 1172, 519, 468, 237, 325, 334, 282, 565, 734, 334, 185, 949, 474, 1370, 930, 797, 722, 1090, 1301, 336, 1098, 841, 485, 536] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[668, 1534, 1932, 432, 185, 128, 1479, 580, 1265, 496, 824, 205, 667, 459, 213, 346, 557, 1098, 340, 917, 2046, 283, 613, 2501, 598, 377, 902, 911, 1382, 360, 379, 457, 606, 1349, 990, 1276, 1928, 1020, 1017, 370, 532, 826, 638, 594, 459, 358, 202, 619, 1448, 200, 218, 156, 335, 880, 531, 1179, 650, 631, 1321, 596, 639, 1241, 934, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[889, 1373, 1094, 1484, 792, 145, 1099, 168, 572, 224, 229, 472, 408, 533, 359, 127, 283, 1809, 179, 1268, 495, 66, 544, 3208, 368, 394, 873, 1032, 2157, 81, 532, 464, 223, 2927, 287, 593, 805, 697, 2421, 1400, 951, 317, 526, 46, 551, 228, 419, 1006, 1762, 351, 355, 826, 687, 763, 632, 434, 952, 1064, 915, 793, 199, 1597, 492, 212] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1308, 1568, 1453, 1456, 1432, 224, 1754, 169, 1056, 310, 518, 152, 381, 362, 116, 390, 199, 1403, 247, 445, 1430, 99, 804, 1521, 259, 369, 804, 672, 1445, 311, 904, 443, 539, 961, 1079, 1737, 1510, 959, 1062, 1157, 1051, 748, 361, 242, 410, 219, 613, 592, 1224, 195, 128, 337, 264, 646, 763, 750, 124, 1676, 1314, 1038, 1137, 1477, 310, 525] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1686, 2173, 1405, 1592, 460, 168, 1190, 126, 1583, 409, 555, 1048, 442, 519, 548, 298, 290, 581, 318, 932, 894, 611, 506, 2402, 267, 440, 1236, 389, 1215, 337, 938, 442, 416, 1219, 442, 925, 975, 914, 1903, 562, 1635, 424, 616, 312, 185, 188, 849, 668, 1631, 128, 512, 630, 224, 825, 701, 435, 622, 799, 1071, 307, 463, 1887, 432, 252] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[502, 2424, 1104, 1402, 231, 105, 1260, 116, 582, 235, 468, 217, 364, 400, 706, 436, 322, 617, 126, 1406, 1339, 260, 207, 2285, 383, 487, 692, 632, 1250, 646, 312, 628, 560, 2245, 787, 656, 979, 928, 2426, 859, 978, 932, 353, 105, 784, 58, 294, 471, 835, 159, 241, 718, 1275, 596, 1438, 182, 730, 2126, 702, 653, 1172, 1442, 904, 420] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[444, 1388, 1742, 513, 246, 275, 1103, 199, 1021, 1004, 354, 817, 866, 1381, 623, 618, 101, 693, 112, 701, 1008, 266, 1034, 841, 528, 551, 1186, 489, 2211, 672, 626, 672, 1194, 1425, 1059, 1168, 1253, 1001, 1769, 326, 1057, 670, 834, 189, 313, 191, 92, 498, 1032, 378, 318, 925, 591, 357, 409, 341, 395, 936, 520, 1282, 1250, 2128, 438, 528] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[924, 268, 2111, 1004, 465, 146, 1108, 88, 0, 0, 0, 0, 0, 0, 0, 0, 876, 800, 478, 243, 909, 521, 469, 220, 1424, 374, 0, 0, 0, 0, 0, 0, 197, 710, 177, 1049, 796, 285, 354, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 325, 1704, 674, 1100, 484, 492, 550, 188, 198, 0, 0, 0, 0, 0, 0, 326, 475, 594, 1051, 514, 863, 0, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 1935, 523, 663, 499, 228, 332, 159, 490, 1880, 196, 0, 0, 0, 0, 0, 0, 952, 249, 242, 340, 964, 772, 803, 314, 979, 299, 0, 0, 0, 0, 0, 0, 784, 1332, 918, 96, 1703, 1131, 273, 352, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6314, 5436, 6931, 4949, 6905, 5914, 6589] avg=6144.0000 max=6931 max_rank=3 min=4949 min_rank=4 imbalance_ratio=1.128092 max_load_reduction=42221 max_load_reduction_pct=0.858988 moved_to_echo=24357 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1686, 2173, 1405, 1592, 460, 168, 1190, 126, 1583, 409, 555, 1048, 442, 519, 548, 298, 290, 581, 318, 932, 894, 611, 506, 2402, 267, 440, 1236, 389, 1215, 337, 938, 442, 416, 1219, 442, 925, 975, 914, 1903, 562, 1635, 424, 616, 312, 185, 188, 849, 668, 1631, 128, 512, 630, 224, 825, 701, 435, 622, 799, 1071, 307, 463, 1887, 432, 252] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[668, 218, 1932, 432, 185, 128, 1479, 580, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 496, 824, 205, 667, 459, 213, 346, 1157, 439, 0, 0, 0, 0, 0, 0, 557, 1098, 340, 917, 2046, 283, 613, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 598, 377, 902, 911, 1382, 360, 379, 457, 229, 106, 0, 0, 0, 0, 0, 0, 606, 558, 990, 1276, 1928, 1020, 0, 370, 0, 0, 0, 0, 0, 0, 0, 0, 532, 826, 638, 594, 459, 358, 202, 619, 1017, 159, 0, 0, 0, 0, 0, 0, 1448, 200, 218, 156, 335, 880, 531, 1179, 526, 352, 0, 0, 0, 0, 0, 0, 650, 631, 1321, 596, 639, 609, 934, 275, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5622, 6071, 8126, 5701, 6748, 5404, 5825, 5655] avg=6144.0000 max=8126 max_rank=2 min=5404 min_rank=5 imbalance_ratio=1.322591 max_load_reduction=41026 max_load_reduction_pct=0.834676 moved_to_echo=23632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[889, 196, 1094, 1484, 792, 145, 1099, 168, 0, 0, 0, 0, 0, 0, 0, 0, 572, 224, 229, 472, 408, 533, 359, 127, 1035, 954, 0, 0, 0, 0, 0, 0, 283, 1809, 179, 1268, 495, 66, 544, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 368, 394, 873, 1032, 2157, 81, 532, 464, 294, 137, 0, 0, 0, 0, 0, 0, 223, 1209, 287, 593, 805, 697, 0, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 951, 317, 526, 46, 551, 228, 419, 1006, 2421, 142, 0, 0, 0, 0, 0, 0, 1762, 351, 355, 826, 687, 763, 632, 434, 677, 764, 0, 0, 0, 0, 0, 0, 952, 1064, 915, 793, 199, 783, 492, 212, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5867, 4913, 7558, 6332, 5214, 6607, 7251, 5410] avg=6144.0000 max=7558 max_rank=2 min=4913 min_rank=1 imbalance_ratio=1.230143 max_load_reduction=41594 max_load_reduction_pct=0.846232 moved_to_echo=24482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[924, 268, 2111, 1004, 465, 146, 1108, 88, 0, 0, 0, 0, 0, 0, 0, 0, 876, 800, 478, 243, 909, 521, 469, 220, 1424, 374, 0, 0, 0, 0, 0, 0, 197, 710, 177, 1049, 796, 285, 354, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 325, 1704, 674, 1100, 484, 492, 550, 188, 198, 0, 0, 0, 0, 0, 0, 326, 475, 594, 1051, 514, 863, 0, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 1935, 523, 663, 499, 228, 332, 159, 490, 1880, 196, 0, 0, 0, 0, 0, 0, 952, 249, 242, 340, 964, 772, 803, 314, 979, 299, 0, 0, 0, 0, 0, 0, 784, 1332, 918, 96, 1703, 1131, 273, 352, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6314, 5436, 6931, 4949, 6905, 5914, 6589] avg=6144.0000 max=6931 max_rank=3 min=4949 min_rank=4 imbalance_ratio=1.128092 max_load_reduction=42221 max_load_reduction_pct=0.858988 moved_to_echo=24357 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[502, 345, 1104, 1402, 231, 105, 1260, 116, 0, 0, 0, 0, 0, 0, 0, 0, 582, 235, 468, 217, 364, 400, 706, 436, 1828, 732, 0, 0, 0, 0, 0, 0, 322, 617, 126, 1406, 1339, 260, 207, 2076, 0, 0, 0, 0, 0, 0, 0, 0, 383, 487, 692, 632, 1250, 646, 312, 628, 209, 124, 0, 0, 0, 0, 0, 0, 560, 927, 787, 656, 979, 928, 0, 859, 0, 0, 0, 0, 0, 0, 0, 0, 978, 932, 353, 105, 784, 58, 294, 471, 2426, 251, 0, 0, 0, 0, 0, 0, 835, 159, 241, 718, 1275, 596, 1438, 182, 611, 586, 0, 0, 0, 0, 0, 0, 730, 2126, 702, 653, 1172, 707, 904, 420, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5065, 5968, 6353, 5363, 5696, 6652, 6641, 7414] avg=6144.0000 max=7414 max_rank=7 min=5065 min_rank=0 imbalance_ratio=1.206706 max_load_reduction=41738 max_load_reduction_pct=0.849162 moved_to_echo=26403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[444, 197, 1742, 513, 246, 275, 1103, 199, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1004, 354, 817, 866, 1381, 623, 618, 1047, 464, 0, 0, 0, 0, 0, 0, 101, 693, 112, 701, 1008, 266, 1034, 764, 0, 0, 0, 0, 0, 0, 0, 0, 528, 551, 1186, 489, 2211, 672, 626, 672, 77, 183, 0, 0, 0, 0, 0, 0, 1194, 589, 1059, 1168, 1253, 1001, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 670, 834, 189, 313, 191, 92, 498, 1769, 144, 0, 0, 0, 0, 0, 0, 1032, 378, 318, 925, 591, 357, 409, 341, 902, 372, 0, 0, 0, 0, 0, 0, 395, 936, 520, 1282, 1250, 1043, 438, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4719, 8195, 4679, 7195, 6590, 5757, 5625, 6392] avg=6144.0000 max=8195 max_rank=1 min=4679 min_rank=2 imbalance_ratio=1.333822 max_load_reduction=40957 max_load_reduction_pct=0.833272 moved_to_echo=24364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1308, 223, 1453, 1456, 1432, 224, 1754, 169, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 310, 518, 152, 381, 362, 116, 390, 1183, 313, 0, 0, 0, 0, 0, 0, 199, 1403, 247, 445, 1430, 99, 804, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 259, 369, 804, 672, 1445, 311, 904, 443, 139, 127, 0, 0, 0, 0, 0, 0, 539, 397, 1079, 1737, 1510, 959, 0, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 748, 361, 242, 410, 219, 613, 592, 1062, 162, 0, 0, 0, 0, 0, 0, 1224, 195, 128, 337, 264, 646, 763, 750, 626, 251, 0, 0, 0, 0, 0, 0, 124, 1676, 1314, 1038, 1137, 724, 310, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 4781, 6009, 5473, 7378, 5460, 5184, 6848] avg=6144.0000 max=8019 max_rank=0 min=4781 min_rank=1 imbalance_ratio=1.305176 max_load_reduction=41133 max_load_reduction_pct=0.836853 moved_to_echo=24870 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1637, 120, 751, 1155, 806, 106, 2024, 211, 0, 0, 0, 0, 0, 0, 0, 0, 1802, 308, 475, 368, 253, 649, 143, 329, 678, 467, 0, 0, 0, 0, 0, 0, 314, 573, 215, 733, 415, 136, 760, 1531, 0, 0, 0, 0, 0, 0, 0, 0, 269, 476, 1290, 1431, 1856, 211, 180, 565, 158, 75, 0, 0, 0, 0, 0, 0, 1749, 579, 477, 1328, 1417, 1708, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 519, 468, 237, 325, 334, 282, 565, 1364, 96, 0, 0, 0, 0, 0, 0, 734, 334, 185, 949, 474, 1370, 930, 797, 361, 374, 0, 0, 0, 0, 0, 0, 722, 1090, 1301, 336, 1098, 405, 485, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6810, 5472, 4677, 6511, 7839, 5362, 6508, 5973] avg=6144.0000 max=7839 max_rank=4 min=4677 min_rank=2 imbalance_ratio=1.275879 max_load_reduction=41313 max_load_reduction_pct=0.840515 moved_to_echo=25682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1686, 309, 1405, 1592, 460, 168, 1190, 126, 0, 0, 0, 0, 0, 0, 0, 0, 1583, 409, 555, 1048, 442, 519, 548, 298, 1639, 397, 0, 0, 0, 0, 0, 0, 290, 581, 318, 932, 894, 611, 506, 2182, 0, 0, 0, 0, 0, 0, 0, 0, 267, 440, 1236, 389, 1215, 337, 938, 442, 220, 162, 0, 0, 0, 0, 0, 0, 416, 504, 442, 925, 975, 914, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 1635, 424, 616, 312, 185, 188, 849, 668, 1903, 225, 0, 0, 0, 0, 0, 0, 1631, 128, 512, 630, 224, 825, 701, 435, 800, 318, 0, 0, 0, 0, 0, 0, 622, 799, 1071, 307, 463, 925, 432, 252, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6936, 7438, 6314, 5646, 4738, 7005, 6204, 4871] avg=6144.0000 max=7438 max_rank=1 min=4738 min_rank=4 imbalance_ratio=1.210612 max_load_reduction=41714 max_load_reduction_pct=0.848674 moved_to_echo=22818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[444, 197, 1742, 513, 246, 275, 1103, 199, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1004, 354, 817, 866, 1381, 623, 618, 1047, 464, 0, 0, 0, 0, 0, 0, 101, 693, 112, 701, 1008, 266, 1034, 764, 0, 0, 0, 0, 0, 0, 0, 0, 528, 551, 1186, 489, 2211, 672, 626, 672, 77, 183, 0, 0, 0, 0, 0, 0, 1194, 589, 1059, 1168, 1253, 1001, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 670, 834, 189, 313, 191, 92, 498, 1769, 144, 0, 0, 0, 0, 0, 0, 1032, 378, 318, 925, 591, 357, 409, 341, 902, 372, 0, 0, 0, 0, 0, 0, 395, 936, 520, 1282, 1250, 1043, 438, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4719, 8195, 4679, 7195, 6590, 5757, 5625, 6392] avg=6144.0000 max=8195 max_rank=1 min=4679 min_rank=2 imbalance_ratio=1.333822 max_load_reduction=40957 max_load_reduction_pct=0.833272 moved_to_echo=24364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[502, 345, 1104, 1402, 231, 105, 1260, 116, 0, 0, 0, 0, 0, 0, 0, 0, 582, 235, 468, 217, 364, 400, 706, 436, 1828, 732, 0, 0, 0, 0, 0, 0, 322, 617, 126, 1406, 1339, 260, 207, 2076, 0, 0, 0, 0, 0, 0, 0, 0, 383, 487, 692, 632, 1250, 646, 312, 628, 209, 124, 0, 0, 0, 0, 0, 0, 560, 927, 787, 656, 979, 928, 0, 859, 0, 0, 0, 0, 0, 0, 0, 0, 978, 932, 353, 105, 784, 58, 294, 471, 2426, 251, 0, 0, 0, 0, 0, 0, 835, 159, 241, 718, 1275, 596, 1438, 182, 611, 586, 0, 0, 0, 0, 0, 0, 730, 2126, 702, 653, 1172, 707, 904, 420, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5065, 5968, 6353, 5363, 5696, 6652, 6641, 7414] avg=6144.0000 max=7414 max_rank=7 min=5065 min_rank=0 imbalance_ratio=1.206706 max_load_reduction=41738 max_load_reduction_pct=0.849162 moved_to_echo=26403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3968, 3456, 8704, 6272, 12672, 3200, 4480, 4224, 1536, 1152, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5632, 5248, 5760, 8832, 9472, 8192, 0, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8064, 1920, 11648, 9088, 4736, 1408, 11136, 1664, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4992, 9728, 8064, 5120, 7680, 6400, 4352, 3200, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9728, 2048, 2304, 4992, 4864, 6272, 6272, 4480, 5504, 3328, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8832, 3840, 3968, 3584, 4352, 4864, 3200, 2816, 10112, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8064, 1920, 11648, 9088, 4736, 1408, 11136, 1664, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2304, 7552, 1792, 7552, 8448, 2048, 4864, 15104, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9344, 4992, 4480, 2304, 3328, 1920, 2944, 4992, 13952, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[622, 467, 318, 1068, 871, 548, 260, 1039, 943, 368, 336, 121, 1187, 667, 1072, 630, 1369, 464, 547, 399, 980, 418, 1587, 314, 153, 539, 1082, 2021, 426, 1044, 1261, 531, 1107, 773, 755, 1209, 1945, 370, 427, 2755, 1740, 869, 629, 1402, 1244, 799, 112, 671, 405, 432, 32, 611, 404, 1179, 833, 1038, 485, 667, 1190, 296, 509, 358, 121, 133] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[347, 437, 392, 1855, 2940, 464, 351, 579, 501, 446, 698, 843, 2857, 675, 1158, 1710, 178, 629, 812, 647, 167, 808, 631, 778, 981, 507, 569, 490, 112, 64, 1429, 157, 405, 1494, 276, 1449, 2904, 1233, 89, 1350, 585, 1817, 1459, 423, 115, 1281, 69, 1186, 280, 228, 326, 304, 15, 1063, 469, 493, 303, 690, 1346, 69, 1200, 874, 74, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[221, 638, 535, 2439, 538, 450, 332, 1251, 752, 195, 260, 647, 2400, 1866, 460, 1156, 822, 1179, 195, 291, 859, 866, 1130, 815, 489, 749, 801, 647, 232, 666, 483, 256, 734, 1245, 1672, 1860, 1913, 368, 208, 2200, 1042, 839, 491, 799, 584, 1545, 358, 1042, 401, 214, 135, 454, 61, 1453, 267, 484, 139, 291, 1166, 360, 583, 1045, 305, 274] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[314, 285, 419, 1839, 786, 379, 562, 686, 914, 601, 309, 894, 1202, 1159, 1301, 603, 1557, 1919, 970, 277, 519, 568, 857, 595, 138, 1089, 1095, 1096, 893, 479, 1368, 653, 257, 1151, 1084, 1233, 3409, 1255, 156, 751, 1123, 905, 675, 423, 603, 1407, 125, 421, 1035, 316, 255, 596, 154, 955, 421, 763, 513, 415, 972, 327, 497, 379, 121, 129] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[720, 282, 490, 2140, 824, 591, 523, 618, 1252, 402, 441, 806, 1306, 678, 1888, 957, 249, 1898, 491, 357, 512, 570, 745, 454, 397, 1044, 522, 805, 124, 239, 1274, 391, 878, 882, 481, 1582, 2576, 1242, 224, 1303, 1335, 978, 948, 375, 495, 1577, 196, 208, 326, 528, 340, 794, 23, 1401, 640, 602, 600, 572, 1637, 204, 530, 1017, 314, 354] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[622, 467, 318, 1068, 871, 548, 260, 1039, 943, 368, 336, 121, 1187, 667, 1072, 630, 1369, 464, 547, 399, 980, 418, 1587, 314, 153, 539, 1082, 2021, 426, 1044, 1261, 531, 1107, 773, 755, 1209, 1945, 370, 427, 2755, 1740, 869, 629, 1402, 1244, 799, 112, 671, 405, 432, 32, 611, 404, 1179, 833, 1038, 485, 667, 1190, 296, 509, 358, 121, 133] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[317, 259, 163, 2289, 2206, 456, 173, 1597, 618, 515, 625, 483, 2322, 746, 1012, 586, 1487, 442, 289, 359, 196, 295, 1044, 801, 142, 1082, 970, 1478, 64, 414, 1313, 178, 789, 1872, 166, 1283, 1511, 566, 327, 1502, 1201, 1125, 932, 1175, 391, 1416, 166, 1238, 481, 341, 70, 798, 649, 1168, 182, 764, 706, 515, 563, 724, 673, 184, 449, 304] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[421, 111, 532, 1090, 2529, 1370, 201, 870, 932, 101, 619, 495, 1118, 772, 666, 710, 1174, 706, 494, 396, 615, 395, 848, 1001, 560, 494, 1617, 1445, 199, 1356, 1379, 155, 483, 638, 830, 1422, 3256, 853, 604, 785, 1560, 1029, 1496, 751, 791, 888, 69, 718, 185, 270, 119, 234, 268, 1017, 233, 984, 504, 605, 872, 178, 1013, 811, 51, 264] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[270, 313, 322, 1576, 1123, 445, 555, 1575, 448, 393, 1239, 458, 1174, 1118, 909, 484, 631, 764, 683, 1060, 580, 282, 982, 609, 209, 1158, 759, 1361, 316, 335, 2366, 427, 458, 946, 646, 1508, 870, 551, 417, 988, 1387, 1646, 1883, 275, 695, 1262, 65, 605, 213, 942, 318, 542, 128, 989, 518, 1312, 163, 1022, 1146, 535, 960, 254, 938, 46] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[314, 285, 419, 1578, 786, 379, 562, 686, 0, 0, 0, 0, 0, 0, 0, 0, 914, 601, 309, 894, 667, 1159, 1301, 603, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 1919, 970, 277, 519, 568, 857, 595, 186, 62, 0, 0, 0, 0, 0, 0, 138, 1089, 1095, 1096, 893, 479, 1368, 653, 261, 3, 0, 0, 0, 0, 0, 0, 257, 1151, 1084, 1233, 0, 1255, 156, 562, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 905, 675, 423, 603, 508, 125, 421, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 316, 255, 596, 154, 955, 421, 763, 2910, 0, 0, 0, 0, 0, 0, 0, 513, 415, 972, 327, 497, 379, 121, 129, 899, 473, 499, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5009, 6448, 7510, 7075, 5698, 4783, 7405, 5224] avg=6144.0000 max=7510 max_rank=2 min=4783 min_rank=5 imbalance_ratio=1.222331 max_load_reduction=41642 max_load_reduction_pct=0.847209 moved_to_echo=23110 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[347, 437, 392, 1592, 2940, 464, 351, 579, 0, 0, 0, 0, 0, 0, 0, 0, 501, 446, 698, 843, 1583, 675, 1158, 1710, 0, 0, 0, 0, 0, 0, 0, 0, 178, 629, 812, 647, 167, 808, 631, 778, 335, 148, 0, 0, 0, 0, 0, 0, 981, 507, 569, 490, 112, 64, 1429, 157, 263, 5, 0, 0, 0, 0, 0, 0, 405, 1494, 276, 1449, 0, 1233, 89, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 585, 1817, 1459, 423, 115, 462, 69, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 280, 228, 326, 304, 15, 1063, 469, 493, 2479, 0, 0, 0, 0, 0, 0, 0, 303, 690, 1346, 69, 1200, 874, 74, 71, 819, 1126, 425, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7102, 7614, 5133, 4577, 5956, 6116, 5657, 6997] avg=6144.0000 max=7614 max_rank=1 min=4577 min_rank=3 imbalance_ratio=1.239258 max_load_reduction=41538 max_load_reduction_pct=0.845093 moved_to_echo=24726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[221, 638, 535, 2089, 538, 450, 332, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 752, 195, 260, 647, 1321, 1866, 460, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 822, 1179, 195, 291, 859, 866, 1130, 815, 551, 128, 0, 0, 0, 0, 0, 0, 489, 749, 801, 647, 232, 666, 483, 256, 350, 11, 0, 0, 0, 0, 0, 0, 734, 1245, 1672, 1860, 0, 368, 208, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 839, 491, 799, 584, 553, 358, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 401, 214, 135, 454, 61, 1453, 267, 484, 1634, 0, 0, 0, 0, 0, 0, 0, 139, 291, 1166, 360, 583, 1045, 305, 274, 992, 951, 279, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6054, 6657, 6836, 4684, 7725, 5708, 5103, 6385] avg=6144.0000 max=7725 max_rank=4 min=4684 min_rank=3 imbalance_ratio=1.257324 max_load_reduction=41427 max_load_reduction_pct=0.842834 moved_to_echo=24921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[421, 111, 532, 1090, 2529, 1370, 201, 870, 932, 101, 619, 495, 1118, 772, 666, 710, 1174, 706, 494, 396, 615, 395, 848, 1001, 560, 494, 1617, 1445, 199, 1356, 1379, 155, 483, 638, 830, 1422, 3256, 853, 604, 785, 1560, 1029, 1496, 751, 791, 888, 69, 718, 185, 270, 119, 234, 268, 1017, 233, 984, 504, 605, 872, 178, 1013, 811, 51, 264] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[221, 638, 535, 2439, 538, 450, 332, 1251, 752, 195, 260, 647, 2400, 1866, 460, 1156, 822, 1179, 195, 291, 859, 866, 1130, 815, 489, 749, 801, 647, 232, 666, 483, 256, 734, 1245, 1672, 1860, 1913, 368, 208, 2200, 1042, 839, 491, 799, 584, 1545, 358, 1042, 401, 214, 135, 454, 61, 1453, 267, 484, 139, 291, 1166, 360, 583, 1045, 305, 274] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[622, 467, 318, 917, 871, 548, 260, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 943, 368, 336, 121, 659, 667, 1072, 630, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 464, 547, 399, 980, 418, 1587, 314, 684, 61, 0, 0, 0, 0, 0, 0, 153, 539, 1082, 2021, 426, 1044, 1261, 531, 151, 11, 0, 0, 0, 0, 0, 0, 1107, 773, 755, 1209, 0, 370, 427, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 1740, 869, 629, 1402, 1244, 289, 112, 671, 0, 0, 0, 0, 0, 0, 0, 0, 405, 432, 32, 611, 404, 1179, 833, 1038, 1660, 0, 0, 0, 0, 0, 0, 0, 485, 667, 1190, 296, 509, 358, 121, 133, 510, 467, 285, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5042, 4796, 6823, 7219, 6701, 6956, 6594, 5021] avg=6144.0000 max=7219 max_rank=3 min=4796 min_rank=1 imbalance_ratio=1.174967 max_load_reduction=41933 max_load_reduction_pct=0.853129 moved_to_echo=25272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[270, 313, 322, 1353, 1123, 445, 555, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 448, 393, 1239, 458, 652, 1118, 909, 484, 0, 0, 0, 0, 0, 0, 0, 0, 631, 764, 683, 1060, 580, 282, 982, 609, 245, 60, 0, 0, 0, 0, 0, 0, 209, 1158, 759, 1361, 316, 335, 2366, 427, 223, 4, 0, 0, 0, 0, 0, 0, 458, 946, 646, 1508, 0, 551, 417, 739, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 1646, 1883, 275, 695, 455, 65, 605, 0, 0, 0, 0, 0, 0, 0, 0, 213, 942, 318, 542, 128, 989, 518, 1312, 742, 0, 0, 0, 0, 0, 0, 0, 163, 1022, 1146, 535, 960, 254, 938, 46, 807, 462, 128, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5956, 5701, 5896, 7158, 5265, 7011, 5704, 6461] avg=6144.0000 max=7158 max_rank=3 min=5265 min_rank=4 imbalance_ratio=1.165039 max_load_reduction=41994 max_load_reduction_pct=0.854370 moved_to_echo=24441 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[347, 437, 392, 1592, 2940, 464, 351, 579, 0, 0, 0, 0, 0, 0, 0, 0, 501, 446, 698, 843, 1583, 675, 1158, 1710, 0, 0, 0, 0, 0, 0, 0, 0, 178, 629, 812, 647, 167, 808, 631, 778, 335, 148, 0, 0, 0, 0, 0, 0, 981, 507, 569, 490, 112, 64, 1429, 157, 263, 5, 0, 0, 0, 0, 0, 0, 405, 1494, 276, 1449, 0, 1233, 89, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 585, 1817, 1459, 423, 115, 462, 69, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 280, 228, 326, 304, 15, 1063, 469, 493, 2479, 0, 0, 0, 0, 0, 0, 0, 303, 690, 1346, 69, 1200, 874, 74, 71, 819, 1126, 425, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7102, 7614, 5133, 4577, 5956, 6116, 5657, 6997] avg=6144.0000 max=7614 max_rank=1 min=4577 min_rank=3 imbalance_ratio=1.239258 max_load_reduction=41538 max_load_reduction_pct=0.845093 moved_to_echo=24726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[314, 285, 419, 1578, 786, 379, 562, 686, 0, 0, 0, 0, 0, 0, 0, 0, 914, 601, 309, 894, 667, 1159, 1301, 603, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 1919, 970, 277, 519, 568, 857, 595, 186, 62, 0, 0, 0, 0, 0, 0, 138, 1089, 1095, 1096, 893, 479, 1368, 653, 261, 3, 0, 0, 0, 0, 0, 0, 257, 1151, 1084, 1233, 0, 1255, 156, 562, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 905, 675, 423, 603, 508, 125, 421, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 316, 255, 596, 154, 955, 421, 763, 2910, 0, 0, 0, 0, 0, 0, 0, 513, 415, 972, 327, 497, 379, 121, 129, 899, 473, 499, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5009, 6448, 7510, 7075, 5698, 4783, 7405, 5224] avg=6144.0000 max=7510 max_rank=2 min=4783 min_rank=5 imbalance_ratio=1.222331 max_load_reduction=41642 max_load_reduction_pct=0.847209 moved_to_echo=23110 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[317, 259, 163, 1964, 2206, 456, 173, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 618, 515, 625, 483, 1287, 746, 1012, 586, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 442, 289, 359, 196, 295, 1044, 801, 373, 120, 0, 0, 0, 0, 0, 0, 142, 1082, 970, 1478, 64, 414, 1313, 178, 325, 6, 0, 0, 0, 0, 0, 0, 789, 1872, 166, 1283, 0, 566, 327, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 1125, 932, 1175, 391, 511, 166, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 481, 341, 70, 798, 649, 1168, 182, 764, 1291, 0, 0, 0, 0, 0, 0, 0, 706, 515, 563, 724, 673, 184, 449, 304, 905, 915, 220, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7135, 5872, 5406, 5972, 6126, 6739, 5744, 6158] avg=6144.0000 max=7135 max_rank=0 min=5406 min_rank=2 imbalance_ratio=1.161296 max_load_reduction=42017 max_load_reduction_pct=0.854838 moved_to_echo=24767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[622, 467, 318, 917, 871, 548, 260, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 943, 368, 336, 121, 659, 667, 1072, 630, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 464, 547, 399, 980, 418, 1587, 314, 684, 61, 0, 0, 0, 0, 0, 0, 153, 539, 1082, 2021, 426, 1044, 1261, 531, 151, 11, 0, 0, 0, 0, 0, 0, 1107, 773, 755, 1209, 0, 370, 427, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 1740, 869, 629, 1402, 1244, 289, 112, 671, 0, 0, 0, 0, 0, 0, 0, 0, 405, 432, 32, 611, 404, 1179, 833, 1038, 1660, 0, 0, 0, 0, 0, 0, 0, 485, 667, 1190, 296, 509, 358, 121, 133, 510, 467, 285, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5042, 4796, 6823, 7219, 6701, 6956, 6594, 5021] avg=6144.0000 max=7219 max_rank=3 min=4796 min_rank=1 imbalance_ratio=1.174967 max_load_reduction=41933 max_load_reduction_pct=0.853129 moved_to_echo=25272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[720, 282, 490, 1836, 824, 591, 523, 618, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 402, 441, 806, 725, 678, 1888, 957, 0, 0, 0, 0, 0, 0, 0, 0, 249, 1898, 491, 357, 512, 570, 745, 454, 323, 67, 0, 0, 0, 0, 0, 0, 397, 1044, 522, 805, 124, 239, 1274, 391, 304, 5, 0, 0, 0, 0, 0, 0, 878, 882, 481, 1582, 0, 1242, 224, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1335, 978, 948, 375, 495, 569, 196, 208, 0, 0, 0, 0, 0, 0, 0, 0, 326, 528, 340, 794, 23, 1401, 640, 602, 2200, 0, 0, 0, 0, 0, 0, 0, 600, 572, 1637, 204, 530, 1017, 314, 354, 1008, 514, 376, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5884, 7149, 5666, 5105, 6264, 5104, 6854, 7126] avg=6144.0000 max=7149 max_rank=1 min=5104 min_rank=5 imbalance_ratio=1.163574 max_load_reduction=42003 max_load_reduction_pct=0.854553 moved_to_echo=25348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[421, 111, 532, 936, 2529, 1370, 201, 870, 0, 0, 0, 0, 0, 0, 0, 0, 932, 101, 619, 495, 620, 772, 666, 710, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 706, 494, 396, 615, 395, 848, 1001, 195, 58, 0, 0, 0, 0, 0, 0, 560, 494, 1617, 1445, 199, 1356, 1379, 155, 154, 3, 0, 0, 0, 0, 0, 0, 483, 638, 830, 1422, 0, 853, 604, 587, 0, 0, 0, 0, 0, 0, 0, 0, 1560, 1029, 1496, 751, 791, 321, 69, 718, 0, 0, 0, 0, 0, 0, 0, 0, 185, 270, 119, 234, 268, 1017, 233, 984, 2781, 0, 0, 0, 0, 0, 0, 0, 504, 605, 872, 178, 1013, 811, 51, 264, 567, 440, 475, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6970, 4915, 5882, 7362, 5417, 6735, 6091, 5780] avg=6144.0000 max=7362 max_rank=3 min=4915 min_rank=1 imbalance_ratio=1.198242 max_load_reduction=41790 max_load_reduction_pct=0.850220 moved_to_echo=24023 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[720, 282, 490, 1836, 824, 591, 523, 618, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 402, 441, 806, 725, 678, 1888, 957, 0, 0, 0, 0, 0, 0, 0, 0, 249, 1898, 491, 357, 512, 570, 745, 454, 323, 67, 0, 0, 0, 0, 0, 0, 397, 1044, 522, 805, 124, 239, 1274, 391, 304, 5, 0, 0, 0, 0, 0, 0, 878, 882, 481, 1582, 0, 1242, 224, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1335, 978, 948, 375, 495, 569, 196, 208, 0, 0, 0, 0, 0, 0, 0, 0, 326, 528, 340, 794, 23, 1401, 640, 602, 2200, 0, 0, 0, 0, 0, 0, 0, 600, 572, 1637, 204, 530, 1017, 314, 354, 1008, 514, 376, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5884, 7149, 5666, 5105, 6264, 5104, 6854, 7126] avg=6144.0000 max=7149 max_rank=1 min=5104 min_rank=5 imbalance_ratio=1.163574 max_load_reduction=42003 max_load_reduction_pct=0.854553 moved_to_echo=25348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[221, 638, 535, 2089, 538, 450, 332, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 752, 195, 260, 647, 1321, 1866, 460, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 822, 1179, 195, 291, 859, 866, 1130, 815, 551, 128, 0, 0, 0, 0, 0, 0, 489, 749, 801, 647, 232, 666, 483, 256, 350, 11, 0, 0, 0, 0, 0, 0, 734, 1245, 1672, 1860, 0, 368, 208, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 839, 491, 799, 584, 553, 358, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 401, 214, 135, 454, 61, 1453, 267, 484, 1634, 0, 0, 0, 0, 0, 0, 0, 139, 291, 1166, 360, 583, 1045, 305, 274, 992, 951, 279, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6054, 6657, 6836, 4684, 7725, 5708, 5103, 6385] avg=6144.0000 max=7725 max_rank=4 min=4684 min_rank=3 imbalance_ratio=1.257324 max_load_reduction=41427 max_load_reduction_pct=0.842834 moved_to_echo=24921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5120, 9088, 6016, 11648, 0, 6528, 2560, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9984, 9216, 8576, 5632, 4992, 3712, 1280, 6144, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 2816, 3200, 12288, 11904, 4736, 3072, 8320, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 3072, 4608, 4864, 7552, 7808, 8576, 6912, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 3328, 1664, 4352, 1792, 9344, 3584, 6528, 15744, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 4864, 8960, 2816, 6016, 4992, 2432, 1664, 6528, 5376, 2688, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9984, 9216, 8576, 5632, 4992, 3712, 1280, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 3328, 1664, 4352, 1792, 9344, 3584, 6528, 15744, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7552, 8064, 4608, 3840, 4480, 4224, 7936, 5376, 2944, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5120, 9088, 6016, 11648, 0, 6528, 2560, 8704, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3072, 6784, 7424, 9344, 2432, 4608, 10880, 2816, 2048, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[529, 1057, 979, 321, 1300, 868, 245, 552, 974, 748, 560, 509, 381, 865, 774, 177, 643, 183, 1436, 2474, 541, 413, 688, 1747, 200, 374, 132, 964, 511, 148, 295, 2024, 150, 266, 2238, 115, 1083, 244, 509, 940, 239, 453, 234, 2325, 88, 459, 549, 1829, 2446, 491, 203, 186, 713, 279, 2114, 281, 596, 884, 1238, 583, 954, 1975, 394, 482] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[831, 387, 1281, 1319, 116, 111, 183, 556, 957, 1832, 1525, 1110, 225, 478, 1037, 75, 200, 152, 870, 1066, 642, 654, 486, 654, 140, 856, 49, 551, 1574, 312, 209, 1112, 80, 438, 1876, 334, 1081, 60, 190, 1639, 776, 1069, 340, 1367, 194, 391, 556, 1137, 1153, 384, 249, 648, 717, 326, 1034, 529, 1012, 1435, 2019, 3270, 555, 1872, 535, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[229, 906, 1207, 953, 171, 341, 316, 397, 822, 382, 660, 956, 724, 1259, 755, 480, 1188, 122, 1353, 2481, 597, 463, 273, 422, 210, 492, 268, 1619, 1151, 633, 309, 2255, 128, 972, 1582, 469, 561, 180, 233, 559, 396, 637, 1366, 1326, 100, 250, 773, 1544, 1385, 1077, 162, 378, 395, 760, 1671, 257, 425, 887, 521, 2101, 671, 1639, 969, 384] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1162, 1263, 1629, 323, 385, 177, 355, 1609, 705, 915, 130, 454, 273, 359, 1264, 285, 352, 107, 1724, 988, 331, 513, 450, 514, 237, 279, 439, 958, 1526, 535, 227, 1426, 206, 402, 2495, 147, 1298, 352, 376, 793, 772, 308, 765, 1350, 106, 530, 634, 1505, 767, 380, 466, 426, 852, 258, 2463, 869, 787, 1228, 1021, 1419, 1810, 1540, 324, 609] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[822, 840, 405, 480, 222, 700, 510, 636, 813, 307, 155, 825, 664, 605, 1339, 387, 214, 131, 1220, 2753, 542, 1067, 803, 93, 427, 158, 259, 1353, 1004, 582, 681, 1137, 297, 694, 3320, 632, 249, 32, 589, 1590, 1572, 275, 767, 2350, 239, 175, 238, 1313, 917, 1651, 77, 429, 736, 464, 1792, 640, 938, 600, 473, 1246, 398, 797, 638, 890] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[273, 337, 314, 470, 504, 698, 264, 541, 378, 406, 1216, 355, 213, 503, 571, 693, 92, 64, 1479, 1708, 1474, 382, 419, 551, 385, 113, 220, 1808, 1055, 357, 255, 1901, 105, 778, 3019, 837, 925, 306, 888, 904, 357, 638, 236, 2548, 304, 180, 336, 2217, 648, 841, 251, 226, 687, 544, 1769, 733, 2474, 1269, 2260, 769, 453, 801, 488, 362] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[229, 906, 1207, 953, 171, 341, 316, 397, 822, 382, 660, 956, 724, 1259, 755, 480, 1188, 122, 1353, 2481, 597, 463, 273, 422, 210, 492, 268, 1619, 1151, 633, 309, 2255, 128, 972, 1582, 469, 561, 180, 233, 559, 396, 637, 1366, 1326, 100, 250, 773, 1544, 1385, 1077, 162, 378, 395, 760, 1671, 257, 425, 887, 521, 2101, 671, 1639, 969, 384] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[521, 438, 455, 380, 1165, 636, 592, 1777, 658, 495, 192, 555, 367, 534, 965, 242, 439, 375, 412, 774, 1223, 536, 913, 1198, 208, 697, 277, 1923, 443, 205, 407, 1415, 384, 480, 1892, 410, 642, 344, 837, 1869, 1432, 292, 311, 2253, 265, 365, 204, 1903, 745, 339, 393, 225, 703, 566, 2041, 252, 739, 616, 1721, 2388, 945, 1409, 522, 248] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[831, 387, 1281, 1319, 116, 111, 183, 556, 957, 1832, 1525, 1110, 225, 478, 1037, 75, 200, 152, 870, 1066, 642, 654, 486, 654, 140, 856, 49, 551, 1574, 312, 209, 1112, 80, 438, 1876, 334, 1081, 60, 190, 1639, 776, 1069, 340, 1367, 194, 391, 556, 1137, 1153, 384, 249, 648, 717, 326, 1034, 529, 1012, 1435, 2019, 3270, 555, 1872, 535, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[822, 840, 405, 480, 222, 700, 510, 636, 813, 307, 155, 825, 664, 605, 1339, 387, 214, 131, 1220, 2753, 542, 1067, 803, 93, 427, 158, 259, 1353, 1004, 582, 681, 1137, 297, 694, 3320, 632, 249, 32, 589, 1590, 1572, 275, 767, 2350, 239, 175, 238, 1313, 917, 1651, 77, 429, 736, 464, 1792, 640, 938, 600, 473, 1246, 398, 797, 638, 890] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1225, 772, 2595, 516, 579, 925, 372, 606, 601, 720, 263, 1302, 242, 1338, 748, 179, 131, 198, 1912, 1168, 358, 570, 95, 544, 207, 620, 347, 1424, 1827, 457, 286, 2493, 93, 826, 1641, 641, 547, 574, 288, 499, 624, 488, 1751, 1759, 208, 669, 620, 1555, 1079, 290, 337, 297, 243, 387, 2232, 164, 293, 450, 1558, 1489, 165, 1188, 281, 296] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[529, 1057, 979, 321, 1300, 868, 245, 552, 242, 0, 0, 0, 0, 0, 0, 0, 974, 748, 560, 509, 381, 865, 774, 177, 337, 0, 0, 0, 0, 0, 0, 0, 643, 183, 1436, 2474, 541, 413, 688, 1747, 103, 0, 0, 0, 0, 0, 0, 0, 200, 374, 132, 964, 511, 148, 295, 2024, 325, 4, 0, 0, 0, 0, 0, 0, 150, 266, 2238, 115, 1083, 244, 509, 940, 4, 0, 0, 0, 0, 0, 0, 0, 239, 453, 234, 1769, 88, 459, 549, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 2446, 491, 203, 186, 713, 279, 2114, 281, 231, 107, 0, 0, 0, 0, 0, 0, 596, 884, 1238, 0, 954, 1761, 394, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6093, 5325, 8228, 4977, 5549, 5620, 7051, 6309] avg=6144.0000 max=8228 max_rank=2 min=4977 min_rank=3 imbalance_ratio=1.339193 max_load_reduction=40924 max_load_reduction_pct=0.832601 moved_to_echo=24529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[831, 387, 1281, 1319, 116, 111, 183, 556, 1361, 0, 0, 0, 0, 0, 0, 0, 957, 1832, 1525, 1110, 225, 478, 1037, 75, 1892, 0, 0, 0, 0, 0, 0, 0, 200, 152, 870, 1066, 642, 654, 486, 654, 97, 0, 0, 0, 0, 0, 0, 0, 140, 856, 49, 551, 1574, 312, 209, 1112, 191, 17, 0, 0, 0, 0, 0, 0, 80, 438, 1876, 334, 1081, 60, 190, 1639, 4, 0, 0, 0, 0, 0, 0, 0, 776, 1069, 340, 1040, 194, 391, 556, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 1153, 384, 249, 648, 717, 326, 1034, 529, 136, 101, 0, 0, 0, 0, 0, 0, 1012, 1435, 2019, 0, 555, 1670, 535, 336, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 9131, 4821, 5011, 5702, 5503, 5277, 7562] avg=6144.0000 max=9131 max_rank=1 min=4821 min_rank=2 imbalance_ratio=1.486165 max_load_reduction=40021 max_load_reduction_pct=0.814229 moved_to_echo=24044 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[529, 1057, 979, 321, 1300, 868, 245, 552, 974, 748, 560, 509, 381, 865, 774, 177, 643, 183, 1436, 2474, 541, 413, 688, 1747, 200, 374, 132, 964, 511, 148, 295, 2024, 150, 266, 2238, 115, 1083, 244, 509, 940, 239, 453, 234, 2325, 88, 459, 549, 1829, 2446, 491, 203, 186, 713, 279, 2114, 281, 596, 884, 1238, 583, 954, 1975, 394, 482] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1162, 1263, 1629, 323, 385, 177, 355, 1609, 705, 915, 130, 454, 273, 359, 1264, 285, 352, 107, 1724, 988, 331, 513, 450, 514, 237, 279, 439, 958, 1526, 535, 227, 1426, 206, 402, 2495, 147, 1298, 352, 376, 793, 772, 308, 765, 1350, 106, 530, 634, 1505, 767, 380, 466, 426, 852, 258, 2463, 869, 787, 1228, 1021, 1419, 1810, 1540, 324, 609] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[521, 438, 455, 380, 1165, 636, 592, 1777, 994, 0, 0, 0, 0, 0, 0, 0, 658, 495, 192, 555, 367, 534, 965, 242, 1380, 0, 0, 0, 0, 0, 0, 0, 439, 375, 412, 774, 1223, 536, 913, 1198, 73, 0, 0, 0, 0, 0, 0, 0, 208, 697, 277, 1923, 443, 205, 407, 1415, 314, 14, 0, 0, 0, 0, 0, 0, 384, 480, 1892, 410, 642, 344, 837, 1869, 3, 0, 0, 0, 0, 0, 0, 0, 1432, 292, 311, 1715, 265, 365, 204, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 745, 339, 393, 225, 703, 566, 2041, 252, 224, 76, 0, 0, 0, 0, 0, 0, 739, 616, 1721, 0, 945, 1257, 522, 248, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6958, 5388, 5943, 5903, 6861, 6487, 5564, 6048] avg=6144.0000 max=6958 max_rank=0 min=5388 min_rank=1 imbalance_ratio=1.132487 max_load_reduction=42194 max_load_reduction_pct=0.858439 moved_to_echo=24960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[822, 840, 405, 480, 222, 700, 510, 636, 518, 0, 0, 0, 0, 0, 0, 0, 813, 307, 155, 825, 664, 605, 1339, 387, 722, 0, 0, 0, 0, 0, 0, 0, 214, 131, 1220, 2753, 542, 1067, 803, 93, 41, 0, 0, 0, 0, 0, 0, 0, 427, 158, 259, 1353, 1004, 582, 681, 1137, 328, 6, 0, 0, 0, 0, 0, 0, 297, 694, 3320, 632, 249, 32, 589, 1590, 1, 0, 0, 0, 0, 0, 0, 0, 1572, 275, 767, 1788, 239, 175, 238, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 917, 1651, 77, 429, 736, 464, 1792, 640, 234, 43, 0, 0, 0, 0, 0, 0, 938, 600, 473, 0, 398, 712, 638, 890, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5133, 5817, 6864, 5935, 7404, 6367, 6983, 4649] avg=6144.0000 max=7404 max_rank=4 min=4649 min_rank=7 imbalance_ratio=1.205078 max_load_reduction=41748 max_load_reduction_pct=0.849365 moved_to_echo=25403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[831, 387, 1281, 1319, 116, 111, 183, 556, 1361, 0, 0, 0, 0, 0, 0, 0, 957, 1832, 1525, 1110, 225, 478, 1037, 75, 1892, 0, 0, 0, 0, 0, 0, 0, 200, 152, 870, 1066, 642, 654, 486, 654, 97, 0, 0, 0, 0, 0, 0, 0, 140, 856, 49, 551, 1574, 312, 209, 1112, 191, 17, 0, 0, 0, 0, 0, 0, 80, 438, 1876, 334, 1081, 60, 190, 1639, 4, 0, 0, 0, 0, 0, 0, 0, 776, 1069, 340, 1040, 194, 391, 556, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 1153, 384, 249, 648, 717, 326, 1034, 529, 136, 101, 0, 0, 0, 0, 0, 0, 1012, 1435, 2019, 0, 555, 1670, 535, 336, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 9131, 4821, 5011, 5702, 5503, 5277, 7562] avg=6144.0000 max=9131 max_rank=1 min=4821 min_rank=2 imbalance_ratio=1.486165 max_load_reduction=40021 max_load_reduction_pct=0.814229 moved_to_echo=24044 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[273, 337, 314, 470, 504, 698, 264, 541, 321, 0, 0, 0, 0, 0, 0, 0, 378, 406, 1216, 355, 213, 503, 571, 693, 444, 0, 0, 0, 0, 0, 0, 0, 92, 64, 1479, 1708, 1474, 382, 419, 551, 41, 0, 0, 0, 0, 0, 0, 0, 385, 113, 220, 1808, 1055, 357, 255, 1901, 356, 4, 0, 0, 0, 0, 0, 0, 105, 778, 3019, 837, 925, 306, 888, 904, 1, 0, 0, 0, 0, 0, 0, 0, 357, 638, 236, 1938, 304, 180, 336, 2217, 0, 0, 0, 0, 0, 0, 0, 0, 648, 841, 251, 226, 687, 544, 1769, 733, 254, 43, 0, 0, 0, 0, 0, 0, 2474, 1269, 2260, 0, 453, 716, 488, 362, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3722, 4779, 6210, 6454, 7763, 6206, 5996, 8022] avg=6144.0000 max=8022 max_rank=7 min=3722 min_rank=0 imbalance_ratio=1.305664 max_load_reduction=41130 max_load_reduction_pct=0.836792 moved_to_echo=27987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[529, 1057, 979, 321, 1300, 868, 245, 552, 242, 0, 0, 0, 0, 0, 0, 0, 974, 748, 560, 509, 381, 865, 774, 177, 337, 0, 0, 0, 0, 0, 0, 0, 643, 183, 1436, 2474, 541, 413, 688, 1747, 103, 0, 0, 0, 0, 0, 0, 0, 200, 374, 132, 964, 511, 148, 295, 2024, 325, 4, 0, 0, 0, 0, 0, 0, 150, 266, 2238, 115, 1083, 244, 509, 940, 4, 0, 0, 0, 0, 0, 0, 0, 239, 453, 234, 1769, 88, 459, 549, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 2446, 491, 203, 186, 713, 279, 2114, 281, 231, 107, 0, 0, 0, 0, 0, 0, 596, 884, 1238, 0, 954, 1761, 394, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6093, 5325, 8228, 4977, 5549, 5620, 7051, 6309] avg=6144.0000 max=8228 max_rank=2 min=4977 min_rank=3 imbalance_ratio=1.339193 max_load_reduction=40924 max_load_reduction_pct=0.832601 moved_to_echo=24529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1225, 772, 2595, 516, 579, 925, 372, 606, 621, 0, 0, 0, 0, 0, 0, 0, 601, 720, 263, 1302, 242, 1338, 748, 179, 861, 0, 0, 0, 0, 0, 0, 0, 131, 198, 1912, 1168, 358, 570, 95, 544, 62, 0, 0, 0, 0, 0, 0, 0, 207, 620, 347, 1424, 1827, 457, 286, 2493, 245, 7, 0, 0, 0, 0, 0, 0, 93, 826, 1641, 641, 547, 574, 288, 499, 2, 0, 0, 0, 0, 0, 0, 0, 624, 488, 1751, 1339, 208, 669, 620, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 290, 337, 297, 243, 387, 2232, 164, 175, 64, 0, 0, 0, 0, 0, 0, 293, 450, 1558, 0, 165, 1060, 281, 296, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 6254, 5038, 7913, 5111, 7254, 5268, 4103] avg=6144.0000 max=8211 max_rank=0 min=4103 min_rank=7 imbalance_ratio=1.336426 max_load_reduction=40941 max_load_reduction_pct=0.832947 moved_to_echo=21736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[229, 906, 1207, 953, 171, 341, 316, 397, 876, 0, 0, 0, 0, 0, 0, 0, 822, 382, 660, 956, 724, 1259, 755, 480, 1214, 0, 0, 0, 0, 0, 0, 0, 1188, 122, 1353, 2481, 597, 463, 273, 422, 90, 0, 0, 0, 0, 0, 0, 0, 210, 492, 268, 1619, 1151, 633, 309, 2255, 189, 11, 0, 0, 0, 0, 0, 0, 128, 972, 1582, 469, 561, 180, 233, 559, 6, 0, 0, 0, 0, 0, 0, 0, 396, 637, 1366, 1001, 100, 250, 773, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 1385, 1077, 162, 378, 395, 760, 1671, 257, 136, 94, 0, 0, 0, 0, 0, 0, 425, 887, 521, 0, 671, 1449, 969, 384, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5396, 7252, 6989, 7137, 4690, 6067, 6315, 5306] avg=6144.0000 max=7252 max_rank=1 min=4690 min_rank=4 imbalance_ratio=1.180339 max_load_reduction=41900 max_load_reduction_pct=0.852458 moved_to_echo=22378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1162, 1263, 1629, 323, 385, 177, 355, 1609, 591, 0, 0, 0, 0, 0, 0, 0, 705, 915, 130, 454, 273, 359, 1264, 285, 821, 0, 0, 0, 0, 0, 0, 0, 352, 107, 1724, 988, 331, 513, 450, 514, 80, 0, 0, 0, 0, 0, 0, 0, 237, 279, 439, 958, 1526, 535, 227, 1426, 188, 7, 0, 0, 0, 0, 0, 0, 206, 402, 2495, 147, 1298, 352, 376, 793, 3, 0, 0, 0, 0, 0, 0, 0, 772, 308, 765, 1028, 106, 530, 634, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 767, 380, 466, 426, 852, 258, 2463, 869, 134, 83, 0, 0, 0, 0, 0, 0, 787, 1228, 1021, 0, 1810, 1374, 324, 609, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7494, 5206, 5059, 5822, 6072, 5648, 6698, 7153] avg=6144.0000 max=7494 max_rank=0 min=5059 min_rank=2 imbalance_ratio=1.219727 max_load_reduction=41658 max_load_reduction_pct=0.847534 moved_to_echo=25571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[521, 438, 455, 380, 1165, 636, 592, 1777, 994, 0, 0, 0, 0, 0, 0, 0, 658, 495, 192, 555, 367, 534, 965, 242, 1380, 0, 0, 0, 0, 0, 0, 0, 439, 375, 412, 774, 1223, 536, 913, 1198, 73, 0, 0, 0, 0, 0, 0, 0, 208, 697, 277, 1923, 443, 205, 407, 1415, 314, 14, 0, 0, 0, 0, 0, 0, 384, 480, 1892, 410, 642, 344, 837, 1869, 3, 0, 0, 0, 0, 0, 0, 0, 1432, 292, 311, 1715, 265, 365, 204, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 745, 339, 393, 225, 703, 566, 2041, 252, 224, 76, 0, 0, 0, 0, 0, 0, 739, 616, 1721, 0, 945, 1257, 522, 248, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6958, 5388, 5943, 5903, 6861, 6487, 5564, 6048] avg=6144.0000 max=6958 max_rank=0 min=5388 min_rank=1 imbalance_ratio=1.132487 max_load_reduction=42194 max_load_reduction_pct=0.858439 moved_to_echo=24960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1225, 772, 2595, 516, 579, 925, 372, 606, 621, 0, 0, 0, 0, 0, 0, 0, 601, 720, 263, 1302, 242, 1338, 748, 179, 861, 0, 0, 0, 0, 0, 0, 0, 131, 198, 1912, 1168, 358, 570, 95, 544, 62, 0, 0, 0, 0, 0, 0, 0, 207, 620, 347, 1424, 1827, 457, 286, 2493, 245, 7, 0, 0, 0, 0, 0, 0, 93, 826, 1641, 641, 547, 574, 288, 499, 2, 0, 0, 0, 0, 0, 0, 0, 624, 488, 1751, 1339, 208, 669, 620, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 290, 337, 297, 243, 387, 2232, 164, 175, 64, 0, 0, 0, 0, 0, 0, 293, 450, 1558, 0, 165, 1060, 281, 296, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 6254, 5038, 7913, 5111, 7254, 5268, 4103] avg=6144.0000 max=8211 max_rank=0 min=4103 min_rank=7 imbalance_ratio=1.336426 max_load_reduction=40941 max_load_reduction_pct=0.832947 moved_to_echo=21736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 5888, 4736, 6144, 3200, 6016, 7552, 2560, 7680, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 3712, 2048, 10624, 9216, 3328, 2688, 13824, 2176, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 5888, 4736, 6144, 3200, 6016, 7552, 2560, 7680, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 1408, 10496, 13440, 5760, 4608, 4224, 5760, 640, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 6016, 8960, 4864, 4480, 4480, 2944, 6784, 5632, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 5504, 2176, 2816, 5120, 3584, 15232, 3840, 1536, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 1408, 10496, 13440, 5760, 4608, 4224, 5760, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7296, 7424, 10880, 0, 6016, 10112, 4224, 3712, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 4864, 18176, 3712, 6400, 2176, 3968, 8832, 128, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6272, 4224, 5888, 11648, 1536, 3072, 3968, 13056, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7296, 7424, 10880, 0, 6016, 10112, 4224, 3712, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[401, 210, 884, 1241, 190, 644, 23, 275, 85, 373, 595, 979, 996, 486, 1033, 795, 1312, 1282, 531, 814, 818, 247, 833, 940, 370, 856, 280, 252, 81, 528, 381, 1038, 975, 793, 346, 1239, 2779, 456, 1127, 663, 1173, 943, 1097, 512, 377, 1094, 118, 419, 1814, 1336, 799, 2592, 321, 1676, 114, 498, 479, 413, 702, 853, 1149, 538, 1603, 381] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[386, 8, 698, 196, 214, 564, 5, 782, 147, 342, 1362, 1039, 610, 605, 987, 1461, 919, 2507, 1576, 862, 407, 166, 473, 285, 853, 98, 522, 81, 385, 982, 740, 254, 485, 639, 935, 877, 763, 533, 1152, 1697, 1293, 753, 804, 132, 1040, 160, 824, 573, 1049, 459, 700, 1998, 135, 4347, 336, 1358, 967, 153, 182, 849, 1245, 782, 1059, 357] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[652, 92, 1306, 887, 528, 228, 101, 516, 164, 532, 850, 1717, 1173, 879, 1095, 468, 372, 2126, 153, 510, 1456, 75, 653, 729, 729, 155, 209, 174, 102, 1271, 986, 274, 1212, 722, 215, 301, 1973, 452, 339, 1442, 2235, 246, 491, 301, 981, 561, 753, 261, 588, 433, 1360, 2279, 83, 1721, 795, 1273, 810, 297, 944, 1601, 418, 756, 1670, 477] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[401, 210, 884, 1241, 190, 644, 23, 275, 85, 373, 595, 979, 996, 486, 1033, 795, 1312, 1282, 531, 814, 818, 247, 833, 940, 370, 856, 280, 252, 81, 528, 381, 1038, 975, 793, 346, 1239, 2779, 456, 1127, 663, 1173, 943, 1097, 512, 377, 1094, 118, 419, 1814, 1336, 799, 2592, 321, 1676, 114, 498, 479, 413, 702, 853, 1149, 538, 1603, 381] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[372, 27, 369, 664, 209, 445, 3, 666, 3, 1302, 1077, 838, 840, 912, 1195, 1691, 793, 1338, 555, 1618, 726, 625, 1259, 1067, 64, 192, 111, 64, 145, 2040, 551, 183, 1110, 433, 1339, 1099, 2163, 377, 1883, 1235, 3396, 618, 185, 464, 525, 706, 1026, 254, 881, 923, 152, 1293, 28, 1517, 328, 364, 154, 385, 316, 1800, 626, 507, 914, 207] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1085, 17, 1125, 516, 65, 1195, 9, 636, 36, 722, 1016, 834, 800, 241, 2731, 660, 413, 1156, 695, 1021, 587, 240, 1470, 227, 731, 69, 736, 57, 59, 1284, 127, 297, 687, 927, 1012, 657, 1985, 510, 1346, 600, 2040, 723, 733, 316, 1024, 2146, 242, 817, 1698, 1045, 865, 2143, 129, 838, 54, 61, 372, 138, 680, 1393, 604, 1136, 959, 415] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[658, 20, 763, 400, 173, 627, 78, 868, 365, 2127, 649, 1495, 1059, 173, 1243, 1878, 896, 814, 586, 1943, 1134, 554, 394, 930, 560, 64, 611, 420, 361, 1055, 194, 1154, 1295, 803, 274, 702, 1613, 940, 217, 775, 2862, 1076, 1071, 220, 674, 1241, 68, 210, 405, 649, 890, 1495, 272, 1473, 952, 534, 313, 144, 1007, 292, 406, 461, 1359, 213] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[245, 15, 1382, 289, 123, 645, 196, 882, 13, 1296, 699, 1027, 888, 197, 887, 1033, 1178, 672, 604, 1101, 340, 175, 1298, 857, 549, 1100, 37, 15, 306, 1356, 1462, 174, 481, 775, 1216, 917, 2723, 592, 1367, 1195, 3917, 428, 626, 762, 394, 337, 189, 556, 941, 783, 382, 2367, 386, 1054, 230, 399, 350, 743, 610, 321, 650, 1604, 526, 290] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[372, 27, 369, 664, 209, 445, 3, 666, 3, 1302, 1077, 838, 840, 912, 1195, 1691, 793, 1338, 555, 1618, 726, 625, 1259, 1067, 64, 192, 111, 64, 145, 2040, 551, 183, 1110, 433, 1339, 1099, 2163, 377, 1883, 1235, 3396, 618, 185, 464, 525, 706, 1026, 254, 881, 923, 152, 1293, 28, 1517, 328, 364, 154, 385, 316, 1800, 626, 507, 914, 207] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[319, 71, 571, 366, 298, 289, 117, 377, 44, 967, 895, 1329, 818, 273, 1688, 704, 806, 2206, 1114, 949, 187, 186, 366, 233, 521, 53, 38, 85, 139, 693, 1897, 588, 589, 694, 700, 766, 2966, 379, 1800, 1484, 1537, 525, 1044, 478, 1602, 943, 229, 81, 1092, 876, 436, 1725, 209, 3152, 492, 367, 787, 123, 247, 461, 2129, 1037, 858, 157] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[652, 92, 1306, 887, 528, 228, 101, 516, 164, 532, 850, 1717, 1173, 879, 1095, 468, 372, 2126, 153, 510, 1456, 75, 653, 729, 729, 155, 209, 174, 102, 1271, 986, 274, 1212, 722, 215, 301, 1973, 452, 339, 1442, 2235, 246, 491, 301, 981, 561, 753, 261, 588, 433, 1360, 2279, 83, 1721, 795, 1273, 810, 297, 944, 1601, 418, 756, 1670, 477] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[658, 20, 763, 400, 173, 627, 78, 868, 365, 2127, 649, 1495, 1059, 173, 1243, 1878, 896, 814, 586, 1943, 1134, 554, 394, 930, 560, 64, 611, 420, 361, 1055, 194, 1154, 1295, 803, 274, 702, 1613, 940, 217, 775, 2862, 1076, 1071, 220, 674, 1241, 68, 210, 405, 649, 890, 1495, 272, 1473, 952, 534, 313, 144, 1007, 292, 406, 461, 1359, 213] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[401, 210, 884, 1241, 190, 644, 23, 275, 2553, 873, 0, 0, 0, 0, 0, 0, 85, 373, 595, 979, 996, 486, 437, 795, 0, 0, 0, 0, 0, 0, 0, 0, 1312, 837, 531, 814, 818, 247, 833, 940, 0, 0, 0, 0, 0, 0, 0, 0, 370, 856, 280, 252, 81, 528, 381, 1038, 1212, 596, 235, 0, 0, 0, 0, 0, 975, 793, 346, 1239, 226, 456, 1127, 663, 0, 0, 0, 0, 0, 0, 0, 0, 903, 943, 1097, 512, 377, 1094, 118, 419, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 1336, 799, 507, 321, 1676, 114, 498, 0, 0, 0, 0, 0, 0, 0, 0, 479, 413, 702, 853, 1149, 538, 1603, 381, 445, 35, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7294, 4746, 6332, 5829, 5825, 5463, 7065, 6598] avg=6144.0000 max=7294 max_rank=0 min=4746 min_rank=1 imbalance_ratio=1.187174 max_load_reduction=41858 max_load_reduction_pct=0.851603 moved_to_echo=24951 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[386, 8, 698, 196, 214, 564, 5, 782, 701, 673, 0, 0, 0, 0, 0, 0, 147, 342, 1362, 1039, 610, 605, 418, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 919, 1636, 1576, 862, 407, 166, 473, 285, 0, 0, 0, 0, 0, 0, 0, 0, 853, 98, 522, 81, 385, 982, 740, 254, 934, 569, 259, 0, 0, 0, 0, 0, 485, 639, 935, 877, 62, 533, 1152, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 995, 753, 804, 132, 1040, 160, 824, 573, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 459, 700, 391, 135, 4347, 336, 1358, 0, 0, 0, 0, 0, 0, 0, 0, 967, 153, 182, 849, 1245, 782, 1059, 357, 871, 39, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4227, 5984, 6324, 5677, 6380, 5281, 8775, 6504] avg=6144.0000 max=8775 max_rank=6 min=4227 min_rank=0 imbalance_ratio=1.428223 max_load_reduction=40377 max_load_reduction_pct=0.821472 moved_to_echo=26940 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[652, 92, 1306, 887, 528, 228, 101, 516, 1812, 768, 0, 0, 0, 0, 0, 0, 164, 532, 850, 1717, 1173, 879, 464, 468, 0, 0, 0, 0, 0, 0, 0, 0, 372, 1387, 153, 510, 1456, 75, 653, 729, 0, 0, 0, 0, 0, 0, 0, 0, 729, 155, 209, 174, 102, 1271, 986, 274, 1066, 631, 449, 0, 0, 0, 0, 0, 1212, 722, 215, 301, 161, 452, 339, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 246, 491, 301, 981, 561, 753, 261, 0, 0, 0, 0, 0, 0, 0, 0, 588, 433, 1360, 445, 83, 1721, 795, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 810, 297, 944, 1601, 418, 756, 1670, 477, 739, 68, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6890, 6247, 5335, 6046, 4844, 5312, 6698, 7780] avg=6144.0000 max=7780 max_rank=7 min=4844 min_rank=4 imbalance_ratio=1.266276 max_load_reduction=41372 max_load_reduction_pct=0.841715 moved_to_echo=24634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[372, 27, 369, 664, 209, 445, 3, 666, 1987, 435, 0, 0, 0, 0, 0, 0, 3, 1302, 1077, 838, 840, 912, 506, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 793, 873, 555, 1618, 726, 625, 1259, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 64, 192, 111, 64, 145, 2040, 551, 183, 604, 689, 682, 0, 0, 0, 0, 0, 1110, 433, 1339, 1099, 176, 377, 1883, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 2610, 618, 185, 464, 525, 706, 1026, 254, 0, 0, 0, 0, 0, 0, 0, 0, 881, 923, 152, 254, 28, 1517, 328, 364, 0, 0, 0, 0, 0, 0, 0, 0, 154, 385, 316, 1800, 626, 507, 914, 207, 465, 104, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5177, 7169, 7516, 5325, 7652, 6388, 4447, 5478] avg=6144.0000 max=7652 max_rank=4 min=4447 min_rank=6 imbalance_ratio=1.245443 max_load_reduction=41500 max_load_reduction_pct=0.844320 moved_to_echo=23965 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[652, 92, 1306, 887, 528, 228, 101, 516, 1812, 768, 0, 0, 0, 0, 0, 0, 164, 532, 850, 1717, 1173, 879, 464, 468, 0, 0, 0, 0, 0, 0, 0, 0, 372, 1387, 153, 510, 1456, 75, 653, 729, 0, 0, 0, 0, 0, 0, 0, 0, 729, 155, 209, 174, 102, 1271, 986, 274, 1066, 631, 449, 0, 0, 0, 0, 0, 1212, 722, 215, 301, 161, 452, 339, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 246, 491, 301, 981, 561, 753, 261, 0, 0, 0, 0, 0, 0, 0, 0, 588, 433, 1360, 445, 83, 1721, 795, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 810, 297, 944, 1601, 418, 756, 1670, 477, 739, 68, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6890, 6247, 5335, 6046, 4844, 5312, 6698, 7780] avg=6144.0000 max=7780 max_rank=7 min=4844 min_rank=4 imbalance_ratio=1.266276 max_load_reduction=41372 max_load_reduction_pct=0.841715 moved_to_echo=24634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[401, 210, 884, 1241, 190, 644, 23, 275, 2553, 873, 0, 0, 0, 0, 0, 0, 85, 373, 595, 979, 996, 486, 437, 795, 0, 0, 0, 0, 0, 0, 0, 0, 1312, 837, 531, 814, 818, 247, 833, 940, 0, 0, 0, 0, 0, 0, 0, 0, 370, 856, 280, 252, 81, 528, 381, 1038, 1212, 596, 235, 0, 0, 0, 0, 0, 975, 793, 346, 1239, 226, 456, 1127, 663, 0, 0, 0, 0, 0, 0, 0, 0, 903, 943, 1097, 512, 377, 1094, 118, 419, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 1336, 799, 507, 321, 1676, 114, 498, 0, 0, 0, 0, 0, 0, 0, 0, 479, 413, 702, 853, 1149, 538, 1603, 381, 445, 35, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7294, 4746, 6332, 5829, 5825, 5463, 7065, 6598] avg=6144.0000 max=7294 max_rank=0 min=4746 min_rank=1 imbalance_ratio=1.187174 max_load_reduction=41858 max_load_reduction_pct=0.851603 moved_to_echo=24951 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[658, 20, 763, 400, 173, 627, 78, 868, 1482, 503, 0, 0, 0, 0, 0, 0, 365, 2127, 649, 1495, 1059, 173, 526, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 896, 532, 586, 1943, 1134, 554, 394, 930, 0, 0, 0, 0, 0, 0, 0, 0, 560, 64, 611, 420, 361, 1055, 194, 1154, 699, 717, 575, 0, 0, 0, 0, 0, 1295, 803, 274, 702, 131, 940, 217, 775, 0, 0, 0, 0, 0, 0, 0, 0, 2200, 1076, 1071, 220, 674, 1241, 68, 210, 0, 0, 0, 0, 0, 0, 0, 0, 405, 649, 890, 293, 272, 1473, 952, 534, 0, 0, 0, 0, 0, 0, 0, 0, 313, 144, 1007, 292, 406, 461, 1359, 213, 282, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5572, 8272, 6969, 6410, 5137, 6760, 5468, 4564] avg=6144.0000 max=8272 max_rank=1 min=4564 min_rank=7 imbalance_ratio=1.346354 max_load_reduction=40880 max_load_reduction_pct=0.831706 moved_to_echo=21929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1085, 17, 1125, 516, 65, 1195, 9, 636, 1824, 722, 0, 0, 0, 0, 0, 0, 36, 722, 1016, 834, 800, 241, 1156, 660, 0, 0, 0, 0, 0, 0, 0, 0, 413, 755, 695, 1021, 587, 240, 1470, 227, 0, 0, 0, 0, 0, 0, 0, 0, 731, 69, 736, 57, 59, 1284, 127, 297, 1002, 1575, 409, 0, 0, 0, 0, 0, 687, 927, 1012, 657, 161, 510, 1346, 600, 0, 0, 0, 0, 0, 0, 0, 0, 1569, 723, 733, 316, 1024, 2146, 242, 817, 0, 0, 0, 0, 0, 0, 0, 0, 1698, 1045, 865, 419, 129, 838, 54, 61, 0, 0, 0, 0, 0, 0, 0, 0, 372, 138, 680, 1393, 604, 1136, 959, 415, 401, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7194, 5465, 5408, 6346, 5900, 7570, 5109, 6160] avg=6144.0000 max=7570 max_rank=5 min=5109 min_rank=6 imbalance_ratio=1.232096 max_load_reduction=41582 max_load_reduction_pct=0.845988 moved_to_echo=24739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[245, 15, 1382, 289, 123, 645, 196, 882, 2502, 797, 0, 0, 0, 0, 0, 0, 13, 1296, 699, 1027, 888, 197, 376, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 439, 604, 1101, 340, 175, 1298, 857, 0, 0, 0, 0, 0, 0, 0, 0, 549, 1100, 37, 15, 306, 1356, 1462, 174, 1107, 511, 787, 0, 0, 0, 0, 0, 481, 775, 1216, 917, 221, 592, 1367, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 428, 626, 762, 394, 337, 189, 556, 0, 0, 0, 0, 0, 0, 0, 0, 941, 783, 382, 463, 386, 1054, 230, 399, 0, 0, 0, 0, 0, 0, 0, 0, 350, 743, 610, 321, 650, 1604, 526, 290, 233, 120, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7076, 5529, 5992, 7404, 6764, 6302, 4638, 5447] avg=6144.0000 max=7404 max_rank=3 min=4638 min_rank=6 imbalance_ratio=1.205078 max_load_reduction=41748 max_load_reduction_pct=0.849365 moved_to_echo=23151 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[319, 71, 571, 366, 298, 289, 117, 377, 2728, 586, 0, 0, 0, 0, 0, 0, 44, 967, 895, 1329, 818, 273, 710, 704, 0, 0, 0, 0, 0, 0, 0, 0, 806, 1435, 1114, 949, 187, 186, 366, 233, 0, 0, 0, 0, 0, 0, 0, 0, 521, 53, 38, 85, 139, 693, 1897, 588, 811, 978, 312, 0, 0, 0, 0, 0, 589, 694, 700, 766, 238, 379, 1800, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 525, 1044, 478, 1602, 943, 229, 81, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 876, 436, 328, 209, 3152, 492, 367, 0, 0, 0, 0, 0, 0, 0, 0, 787, 123, 247, 461, 2129, 1037, 858, 157, 771, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5722, 5740, 5276, 6115, 6650, 6076, 6952, 6621] avg=6144.0000 max=6952 max_rank=6 min=5276 min_rank=2 imbalance_ratio=1.131510 max_load_reduction=42200 max_load_reduction_pct=0.858561 moved_to_echo=26299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1085, 17, 1125, 516, 65, 1195, 9, 636, 1824, 722, 0, 0, 0, 0, 0, 0, 36, 722, 1016, 834, 800, 241, 1156, 660, 0, 0, 0, 0, 0, 0, 0, 0, 413, 755, 695, 1021, 587, 240, 1470, 227, 0, 0, 0, 0, 0, 0, 0, 0, 731, 69, 736, 57, 59, 1284, 127, 297, 1002, 1575, 409, 0, 0, 0, 0, 0, 687, 927, 1012, 657, 161, 510, 1346, 600, 0, 0, 0, 0, 0, 0, 0, 0, 1569, 723, 733, 316, 1024, 2146, 242, 817, 0, 0, 0, 0, 0, 0, 0, 0, 1698, 1045, 865, 419, 129, 838, 54, 61, 0, 0, 0, 0, 0, 0, 0, 0, 372, 138, 680, 1393, 604, 1136, 959, 415, 401, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7194, 5465, 5408, 6346, 5900, 7570, 5109, 6160] avg=6144.0000 max=7570 max_rank=5 min=5109 min_rank=6 imbalance_ratio=1.232096 max_load_reduction=41582 max_load_reduction_pct=0.845988 moved_to_echo=24739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[658, 20, 763, 400, 173, 627, 78, 868, 1482, 503, 0, 0, 0, 0, 0, 0, 365, 2127, 649, 1495, 1059, 173, 526, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 896, 532, 586, 1943, 1134, 554, 394, 930, 0, 0, 0, 0, 0, 0, 0, 0, 560, 64, 611, 420, 361, 1055, 194, 1154, 699, 717, 575, 0, 0, 0, 0, 0, 1295, 803, 274, 702, 131, 940, 217, 775, 0, 0, 0, 0, 0, 0, 0, 0, 2200, 1076, 1071, 220, 674, 1241, 68, 210, 0, 0, 0, 0, 0, 0, 0, 0, 405, 649, 890, 293, 272, 1473, 952, 534, 0, 0, 0, 0, 0, 0, 0, 0, 313, 144, 1007, 292, 406, 461, 1359, 213, 282, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5572, 8272, 6969, 6410, 5137, 6760, 5468, 4564] avg=6144.0000 max=8272 max_rank=1 min=4564 min_rank=7 imbalance_ratio=1.346354 max_load_reduction=40880 max_load_reduction_pct=0.831706 moved_to_echo=21929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 2432, 4736, 7680, 7296, 6912, 8960, 2560, 4224, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4224, 512, 7168, 4608, 1920, 4736, 640, 5120, 15616, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8576, 6528, 5632, 3200, 1664, 15872, 3328, 4864, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6912, 5888, 6144, 6656, 1408, 4352, 9344, 9216, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 2688, 2560, 1152, 1664, 9216, 6400, 3968, 7552, 6272, 3712, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[14208, 5376, 6144, 3200, 6656, 7296, 3456, 3200, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 7936, 5888, 8832, 5760, 2304, 6784, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4224, 512, 7168, 4608, 1920, 4736, 640, 5120, 15616, 5376, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 7680, 7168, 9344, 7296, 3840, 4608, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8576, 6528, 5632, 3200, 1664, 15872, 3328, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 7936, 5888, 8832, 5760, 2304, 6784, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 2688, 2560, 1152, 1664, 9216, 6400, 3968, 7552, 6272, 3712, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[53, 432, 140, 1170, 759, 133, 109, 870, 886, 1413, 320, 528, 854, 233, 796, 381, 1501, 1022, 200, 730, 1281, 1302, 464, 853, 247, 75, 283, 933, 544, 66, 1531, 187, 45, 69, 926, 220, 862, 738, 1428, 595, 4210, 1562, 1346, 1055, 119, 897, 48, 121, 560, 2458, 537, 145, 2565, 1249, 128, 690, 2559, 1574, 165, 416, 46, 458, 800, 265] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[269, 861, 308, 595, 321, 366, 324, 471, 459, 2056, 1501, 462, 621, 812, 813, 141, 996, 676, 159, 1065, 1870, 2883, 809, 1011, 175, 345, 453, 867, 613, 597, 317, 620, 89, 242, 716, 640, 622, 174, 1000, 919, 3091, 1835, 1758, 282, 58, 1018, 61, 747, 73, 801, 800, 228, 1683, 2351, 922, 774, 713, 746, 550, 475, 810, 912, 85, 141] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[449, 138, 43, 848, 474, 281, 312, 443, 681, 667, 352, 473, 275, 866, 392, 372, 1913, 595, 67, 285, 957, 1649, 216, 1654, 1741, 35, 98, 581, 387, 757, 439, 52, 15, 31, 130, 207, 269, 943, 1014, 269, 4922, 2320, 1521, 633, 145, 865, 568, 727, 282, 579, 271, 274, 3691, 2609, 148, 722, 1039, 2941, 239, 605, 482, 1549, 262, 388] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[23, 510, 280, 894, 365, 48, 235, 490, 334, 1369, 1623, 459, 1189, 283, 290, 557, 668, 413, 857, 972, 1678, 420, 126, 318, 1501, 133, 402, 1409, 507, 455, 437, 263, 110, 10, 741, 30, 1134, 903, 436, 178, 3445, 838, 1693, 180, 146, 1358, 162, 129, 410, 3281, 672, 189, 2084, 2968, 637, 1405, 2417, 1046, 294, 237, 511, 1238, 645, 117] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[81, 287, 131, 1482, 534, 302, 226, 210, 534, 473, 584, 388, 327, 874, 452, 367, 1624, 143, 36, 505, 503, 685, 569, 1278, 578, 85, 545, 1462, 1017, 412, 108, 330, 146, 210, 151, 182, 452, 944, 946, 932, 3589, 733, 1421, 425, 208, 1290, 102, 754, 496, 2135, 691, 1499, 2653, 1951, 773, 847, 2586, 1438, 278, 673, 909, 1617, 774, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[39, 242, 165, 1001, 499, 607, 198, 337, 148, 823, 542, 207, 686, 527, 1798, 317, 1032, 156, 294, 1614, 636, 844, 459, 892, 1157, 25, 538, 498, 714, 220, 223, 53, 246, 86, 639, 611, 1323, 1216, 947, 676, 5280, 1467, 1714, 638, 125, 165, 95, 108, 264, 755, 1264, 132, 3557, 3122, 588, 640, 1361, 1187, 554, 509, 235, 1773, 56, 328] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[297, 500, 152, 1330, 294, 1077, 262, 421, 234, 1274, 1712, 264, 1154, 600, 302, 503, 1260, 819, 317, 359, 543, 768, 280, 316, 563, 895, 170, 1026, 1090, 201, 670, 395, 20, 271, 314, 392, 717, 530, 441, 286, 5589, 2769, 1173, 488, 339, 1378, 278, 832, 234, 1170, 136, 292, 2205, 2091, 1106, 1298, 1076, 1163, 342, 178, 350, 362, 1062, 222] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[269, 861, 308, 595, 321, 366, 324, 471, 459, 2056, 1501, 462, 621, 812, 813, 141, 996, 676, 159, 1065, 1870, 2883, 809, 1011, 175, 345, 453, 867, 613, 597, 317, 620, 89, 242, 716, 640, 622, 174, 1000, 919, 3091, 1835, 1758, 282, 58, 1018, 61, 747, 73, 801, 800, 228, 1683, 2351, 922, 774, 713, 746, 550, 475, 810, 912, 85, 141] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[81, 287, 131, 1482, 534, 302, 226, 210, 534, 473, 584, 388, 327, 874, 452, 367, 1624, 143, 36, 505, 503, 685, 569, 1278, 578, 85, 545, 1462, 1017, 412, 108, 330, 146, 210, 151, 182, 452, 944, 946, 932, 3589, 733, 1421, 425, 208, 1290, 102, 754, 496, 2135, 691, 1499, 2653, 1951, 773, 847, 2586, 1438, 278, 673, 909, 1617, 774, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[449, 138, 43, 848, 474, 281, 312, 443, 681, 667, 352, 473, 275, 866, 392, 372, 1913, 595, 67, 285, 957, 1649, 216, 1654, 1741, 35, 98, 581, 387, 757, 439, 52, 15, 31, 130, 207, 269, 943, 1014, 269, 4922, 2320, 1521, 633, 145, 865, 568, 727, 282, 579, 271, 274, 3691, 2609, 148, 722, 1039, 2941, 239, 605, 482, 1549, 262, 388] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[215, 196, 89, 2132, 758, 609, 388, 155, 282, 481, 784, 274, 276, 1240, 573, 88, 796, 498, 361, 957, 348, 2790, 218, 1311, 212, 67, 206, 1670, 633, 674, 192, 31, 109, 119, 1233, 1163, 326, 163, 1112, 421, 4039, 1430, 1409, 829, 242, 1222, 903, 559, 145, 540, 1090, 190, 2770, 1790, 1142, 519, 1049, 1250, 701, 497, 1377, 472, 151, 686] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[269, 861, 308, 595, 321, 366, 324, 471, 1892, 0, 0, 0, 0, 0, 0, 0, 459, 2056, 1501, 462, 621, 812, 813, 141, 932, 380, 55, 0, 0, 0, 0, 0, 996, 676, 159, 1065, 1870, 1951, 809, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 175, 345, 453, 867, 613, 597, 317, 620, 1147, 126, 0, 0, 0, 0, 0, 0, 89, 242, 716, 640, 622, 174, 1000, 919, 848, 536, 0, 0, 0, 0, 0, 0, 351, 1835, 1758, 282, 58, 1018, 61, 747, 0, 0, 0, 0, 0, 0, 0, 0, 73, 801, 800, 228, 0, 1845, 922, 774, 0, 0, 0, 0, 0, 0, 0, 0, 658, 746, 550, 475, 810, 912, 85, 141, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5407, 8232, 8537, 5260, 5786, 6110, 5443, 4377] avg=6144.0000 max=8537 max_rank=2 min=4377 min_rank=7 imbalance_ratio=1.389486 max_load_reduction=40615 max_load_reduction_pct=0.826314 moved_to_echo=21716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[53, 432, 140, 1170, 759, 133, 109, 870, 2578, 0, 0, 0, 0, 0, 0, 0, 886, 1413, 320, 528, 854, 233, 796, 381, 421, 202, 200, 0, 0, 0, 0, 0, 1501, 1022, 200, 730, 1281, 881, 464, 853, 0, 0, 0, 0, 0, 0, 0, 0, 247, 75, 283, 933, 544, 66, 1531, 187, 1749, 67, 0, 0, 0, 0, 0, 0, 45, 69, 926, 220, 862, 738, 1428, 595, 1155, 816, 0, 0, 0, 0, 0, 0, 477, 1562, 1346, 1055, 119, 897, 48, 121, 0, 0, 0, 0, 0, 0, 0, 0, 560, 2458, 537, 145, 0, 980, 128, 690, 0, 0, 0, 0, 0, 0, 0, 0, 2359, 1574, 165, 416, 46, 458, 800, 265, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6244, 6234, 6932, 5682, 6854, 5625, 5498, 6083] avg=6144.0000 max=6932 max_rank=2 min=5498 min_rank=6 imbalance_ratio=1.128255 max_load_reduction=42220 max_load_reduction_pct=0.858968 moved_to_echo=24060 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[449, 138, 43, 848, 474, 281, 312, 443, 3014, 0, 0, 0, 0, 0, 0, 0, 681, 667, 352, 473, 275, 866, 392, 372, 533, 422, 81, 0, 0, 0, 0, 0, 1913, 595, 67, 285, 957, 1116, 216, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 1741, 35, 98, 581, 387, 757, 439, 52, 2517, 140, 0, 0, 0, 0, 0, 0, 15, 31, 130, 207, 269, 943, 1014, 269, 1351, 1174, 0, 0, 0, 0, 0, 0, 557, 2320, 1521, 633, 145, 865, 568, 727, 0, 0, 0, 0, 0, 0, 0, 0, 282, 579, 271, 274, 0, 2047, 148, 722, 0, 0, 0, 0, 0, 0, 0, 0, 958, 2941, 239, 605, 482, 1549, 262, 388, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6002, 5114, 6803, 6747, 5403, 7336, 4323, 7424] avg=6144.0000 max=7424 max_rank=7 min=4323 min_rank=6 imbalance_ratio=1.208333 max_load_reduction=41728 max_load_reduction_pct=0.848958 moved_to_echo=24486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[23, 510, 280, 894, 365, 48, 235, 490, 2109, 0, 0, 0, 0, 0, 0, 0, 334, 1369, 1623, 459, 1189, 283, 290, 557, 135, 480, 189, 0, 0, 0, 0, 0, 668, 413, 857, 972, 1678, 285, 126, 318, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 133, 402, 1409, 507, 455, 437, 263, 1421, 160, 0, 0, 0, 0, 0, 0, 110, 10, 741, 30, 1134, 903, 436, 178, 945, 663, 0, 0, 0, 0, 0, 0, 391, 838, 1693, 180, 146, 1358, 162, 129, 0, 0, 0, 0, 0, 0, 0, 0, 410, 3281, 672, 189, 0, 2328, 637, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 2228, 1046, 294, 237, 511, 1238, 645, 117, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4954, 6908, 5317, 6688, 5150, 4897, 8922, 6316] avg=6144.0000 max=8922 max_rank=6 min=4897 min_rank=5 imbalance_ratio=1.452148 max_load_reduction=40230 max_load_reduction_pct=0.818481 moved_to_echo=25285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[449, 138, 43, 848, 474, 281, 312, 443, 3014, 0, 0, 0, 0, 0, 0, 0, 681, 667, 352, 473, 275, 866, 392, 372, 533, 422, 81, 0, 0, 0, 0, 0, 1913, 595, 67, 285, 957, 1116, 216, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 1741, 35, 98, 581, 387, 757, 439, 52, 2517, 140, 0, 0, 0, 0, 0, 0, 15, 31, 130, 207, 269, 943, 1014, 269, 1351, 1174, 0, 0, 0, 0, 0, 0, 557, 2320, 1521, 633, 145, 865, 568, 727, 0, 0, 0, 0, 0, 0, 0, 0, 282, 579, 271, 274, 0, 2047, 148, 722, 0, 0, 0, 0, 0, 0, 0, 0, 958, 2941, 239, 605, 482, 1549, 262, 388, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6002, 5114, 6803, 6747, 5403, 7336, 4323, 7424] avg=6144.0000 max=7424 max_rank=7 min=4323 min_rank=6 imbalance_ratio=1.208333 max_load_reduction=41728 max_load_reduction_pct=0.848958 moved_to_echo=24486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[269, 861, 308, 595, 321, 366, 324, 471, 1892, 0, 0, 0, 0, 0, 0, 0, 459, 2056, 1501, 462, 621, 812, 813, 141, 932, 380, 55, 0, 0, 0, 0, 0, 996, 676, 159, 1065, 1870, 1951, 809, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 175, 345, 453, 867, 613, 597, 317, 620, 1147, 126, 0, 0, 0, 0, 0, 0, 89, 242, 716, 640, 622, 174, 1000, 919, 848, 536, 0, 0, 0, 0, 0, 0, 351, 1835, 1758, 282, 58, 1018, 61, 747, 0, 0, 0, 0, 0, 0, 0, 0, 73, 801, 800, 228, 0, 1845, 922, 774, 0, 0, 0, 0, 0, 0, 0, 0, 658, 746, 550, 475, 810, 912, 85, 141, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5407, 8232, 8537, 5260, 5786, 6110, 5443, 4377] avg=6144.0000 max=8537 max_rank=2 min=4377 min_rank=7 imbalance_ratio=1.389486 max_load_reduction=40615 max_load_reduction_pct=0.826314 moved_to_echo=21716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[39, 242, 165, 1001, 499, 607, 198, 337, 3233, 0, 0, 0, 0, 0, 0, 0, 148, 823, 542, 207, 686, 527, 1798, 317, 273, 505, 106, 0, 0, 0, 0, 0, 1032, 156, 294, 1614, 636, 571, 459, 892, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 25, 538, 498, 714, 220, 223, 53, 2426, 168, 0, 0, 0, 0, 0, 0, 246, 86, 639, 611, 1323, 1216, 947, 676, 1449, 1131, 0, 0, 0, 0, 0, 0, 598, 1467, 1714, 638, 125, 165, 95, 108, 0, 0, 0, 0, 0, 0, 0, 0, 264, 755, 1264, 132, 0, 2449, 588, 640, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 1187, 554, 509, 235, 1773, 56, 328, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6321, 5932, 5654, 6022, 8324, 4910, 6092, 5897] avg=6144.0000 max=8324 max_rank=4 min=4910 min_rank=5 imbalance_ratio=1.354818 max_load_reduction=40828 max_load_reduction_pct=0.830648 moved_to_echo=25223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[297, 500, 152, 1330, 294, 1077, 262, 421, 3422, 0, 0, 0, 0, 0, 0, 0, 234, 1274, 1712, 264, 1154, 600, 302, 503, 248, 338, 84, 0, 0, 0, 0, 0, 1260, 819, 317, 359, 543, 520, 280, 316, 0, 0, 0, 0, 0, 0, 0, 0, 563, 895, 170, 1026, 1090, 201, 670, 395, 1504, 112, 0, 0, 0, 0, 0, 0, 20, 271, 314, 392, 717, 530, 441, 286, 1534, 701, 0, 0, 0, 0, 0, 0, 633, 2769, 1173, 488, 339, 1378, 278, 832, 0, 0, 0, 0, 0, 0, 0, 0, 234, 1170, 136, 292, 0, 1641, 1106, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1163, 342, 178, 350, 362, 1062, 222, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7755, 6713, 4414, 6626, 5206, 7890, 5877, 4671] avg=6144.0000 max=7890 max_rank=5 min=4414 min_rank=2 imbalance_ratio=1.284180 max_load_reduction=41262 max_load_reduction_pct=0.839478 moved_to_echo=23644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[81, 287, 131, 1482, 534, 302, 226, 210, 2197, 0, 0, 0, 0, 0, 0, 0, 534, 473, 584, 388, 327, 874, 452, 367, 221, 315, 202, 0, 0, 0, 0, 0, 1624, 143, 36, 505, 503, 464, 569, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 578, 85, 545, 1462, 1017, 412, 108, 330, 1810, 105, 0, 0, 0, 0, 0, 0, 146, 210, 151, 182, 452, 944, 946, 932, 985, 843, 0, 0, 0, 0, 0, 0, 407, 733, 1421, 425, 208, 1290, 102, 754, 0, 0, 0, 0, 0, 0, 0, 0, 496, 2135, 691, 1499, 0, 1531, 773, 847, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 1438, 278, 673, 909, 1617, 774, 215, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5450, 4737, 5122, 6452, 5791, 5340, 7972, 8288] avg=6144.0000 max=8288 max_rank=7 min=4737 min_rank=1 imbalance_ratio=1.348958 max_load_reduction=40864 max_load_reduction_pct=0.831380 moved_to_echo=27391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[39, 242, 165, 1001, 499, 607, 198, 337, 3233, 0, 0, 0, 0, 0, 0, 0, 148, 823, 542, 207, 686, 527, 1798, 317, 273, 505, 106, 0, 0, 0, 0, 0, 1032, 156, 294, 1614, 636, 571, 459, 892, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 25, 538, 498, 714, 220, 223, 53, 2426, 168, 0, 0, 0, 0, 0, 0, 246, 86, 639, 611, 1323, 1216, 947, 676, 1449, 1131, 0, 0, 0, 0, 0, 0, 598, 1467, 1714, 638, 125, 165, 95, 108, 0, 0, 0, 0, 0, 0, 0, 0, 264, 755, 1264, 132, 0, 2449, 588, 640, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 1187, 554, 509, 235, 1773, 56, 328, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6321, 5932, 5654, 6022, 8324, 4910, 6092, 5897] avg=6144.0000 max=8324 max_rank=4 min=4910 min_rank=5 imbalance_ratio=1.354818 max_load_reduction=40828 max_load_reduction_pct=0.830648 moved_to_echo=25223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[215, 196, 89, 2132, 758, 609, 388, 155, 2477, 0, 0, 0, 0, 0, 0, 0, 282, 481, 784, 274, 276, 1240, 573, 88, 906, 291, 87, 0, 0, 0, 0, 0, 796, 498, 361, 957, 348, 1884, 218, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 212, 67, 206, 1670, 633, 674, 192, 31, 1890, 100, 0, 0, 0, 0, 0, 0, 109, 119, 1233, 1163, 326, 163, 1112, 421, 1112, 880, 0, 0, 0, 0, 0, 0, 450, 1430, 1409, 829, 242, 1222, 903, 559, 0, 0, 0, 0, 0, 0, 0, 0, 145, 540, 1090, 190, 0, 1399, 1142, 519, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1250, 701, 497, 1377, 472, 151, 686, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7019, 5282, 6373, 5675, 6638, 7044, 5025, 6096] avg=6144.0000 max=7044 max_rank=5 min=5025 min_rank=6 imbalance_ratio=1.146484 max_load_reduction=42108 max_load_reduction_pct=0.856689 moved_to_echo=24803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 11776, 5504, 3072, 0, 14336, 5504, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 8576, 7424, 3072, 5504, 5504, 5504, 2816, 3712, 2944, 1024, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9856, 4352, 2304, 6528, 7936, 7680, 3200, 7680, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 3200, 1408, 9472, 4096, 3456, 2176, 3456, 20992, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 1152, 4864, 3456, 5760, 5632, 7424, 4352, 9472, 6784, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6272, 1664, 2816, 8448, 5632, 3456, 3968, 2048, 14464, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 8576, 7424, 3072, 5504, 5504, 5504, 2816, 3712, 2944, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 3200, 1408, 9472, 4096, 3456, 2176, 3456, 20992, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11904, 11392, 3200, 3712, 4736, 8448, 3840, 2432, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 11776, 5504, 3072, 0, 14336, 5504, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6272, 1664, 2816, 8448, 5632, 3456, 3968, 2048, 14464, 1024, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 13056, 12160, 4608, 1408, 8320, 2304, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[188, 1164, 25, 827, 360, 262, 1872, 2315, 1007, 879, 365, 1413, 1544, 283, 135, 224, 1037, 187, 1060, 69, 410, 505, 612, 1242, 19, 1307, 1184, 113, 269, 438, 577, 19, 279, 35, 922, 957, 1006, 93, 3803, 1157, 433, 537, 494, 2505, 618, 648, 4456, 57, 661, 370, 1040, 127, 339, 46, 79, 1617, 236, 909, 847, 253, 492, 228, 705, 1292] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[398, 1169, 571, 911, 349, 825, 688, 613, 1590, 586, 54, 572, 401, 502, 46, 170, 1958, 726, 1546, 268, 83, 400, 499, 546, 261, 1869, 840, 67, 159, 278, 2718, 108, 260, 98, 1333, 166, 2098, 310, 1166, 1972, 550, 245, 1490, 896, 466, 437, 1667, 30, 622, 342, 1670, 322, 354, 134, 269, 3050, 107, 1191, 1642, 316, 737, 542, 1819, 1080] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[133, 1250, 330, 455, 330, 982, 1030, 1994, 1652, 711, 226, 937, 2337, 180, 72, 382, 1392, 779, 616, 178, 154, 352, 126, 836, 305, 1594, 466, 601, 245, 264, 1051, 134, 387, 638, 1135, 520, 1897, 585, 1587, 2030, 618, 1097, 608, 789, 96, 1604, 2371, 72, 255, 175, 1579, 455, 688, 87, 46, 1039, 627, 524, 1132, 294, 1705, 548, 834, 1036] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[418, 1341, 391, 641, 922, 200, 1648, 778, 1934, 878, 526, 1208, 227, 1133, 38, 292, 1815, 625, 730, 153, 207, 677, 170, 781, 265, 1936, 2955, 72, 167, 417, 398, 549, 106, 256, 1667, 535, 684, 153, 1921, 915, 1427, 279, 764, 882, 922, 719, 2042, 94, 540, 441, 1006, 220, 935, 893, 448, 1159, 48, 300, 1047, 1209, 1026, 608, 988, 426] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[391, 756, 169, 615, 576, 629, 1525, 1010, 1760, 53, 168, 483, 3216, 311, 37, 200, 1948, 777, 518, 58, 378, 544, 45, 361, 79, 2026, 1054, 56, 159, 404, 1602, 671, 149, 68, 942, 268, 1660, 227, 1929, 1182, 1014, 51, 1863, 1412, 320, 473, 4637, 88, 185, 328, 1313, 1841, 458, 4, 85, 2006, 116, 562, 281, 1216, 902, 301, 652, 40] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[711, 1238, 317, 378, 333, 151, 2275, 2307, 1655, 649, 478, 1232, 2207, 369, 33, 301, 1971, 770, 653, 455, 435, 517, 32, 768, 108, 1727, 667, 259, 221, 582, 1374, 95, 123, 44, 310, 311, 1148, 543, 808, 1413, 712, 247, 1782, 2480, 369, 843, 887, 31, 349, 461, 536, 383, 217, 189, 124, 2169, 302, 503, 1146, 551, 1629, 654, 2319, 301] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[418, 1341, 391, 641, 922, 200, 1648, 778, 1934, 878, 526, 1208, 227, 1133, 38, 292, 1815, 625, 730, 153, 207, 677, 170, 781, 265, 1936, 2955, 72, 167, 417, 398, 549, 106, 256, 1667, 535, 684, 153, 1921, 915, 1427, 279, 764, 882, 922, 719, 2042, 94, 540, 441, 1006, 220, 935, 893, 448, 1159, 48, 300, 1047, 1209, 1026, 608, 988, 426] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[398, 1169, 571, 911, 349, 825, 688, 613, 1590, 586, 54, 572, 401, 502, 46, 170, 1958, 726, 1546, 268, 83, 400, 499, 546, 261, 1869, 840, 67, 159, 278, 2718, 108, 260, 98, 1333, 166, 2098, 310, 1166, 1972, 550, 245, 1490, 896, 466, 437, 1667, 30, 622, 342, 1670, 322, 354, 134, 269, 3050, 107, 1191, 1642, 316, 737, 542, 1819, 1080] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[908, 539, 474, 1134, 472, 427, 881, 1343, 1995, 530, 80, 909, 898, 585, 20, 111, 1414, 614, 506, 306, 174, 2192, 144, 1319, 91, 1777, 324, 227, 60, 470, 575, 851, 29, 202, 1467, 648, 1871, 101, 1214, 1633, 823, 319, 541, 1503, 323, 178, 1518, 48, 374, 491, 1778, 1027, 1236, 247, 145, 2390, 55, 467, 3299, 477, 965, 378, 776, 279] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[337, 546, 214, 860, 393, 121, 3443, 1545, 1386, 809, 136, 976, 1316, 204, 45, 119, 2069, 217, 1002, 416, 335, 724, 2138, 865, 126, 1655, 900, 15, 219, 121, 1158, 116, 183, 125, 772, 531, 897, 263, 429, 1002, 514, 652, 1128, 1953, 227, 195, 2703, 20, 389, 302, 1424, 879, 142, 36, 155, 3645, 620, 1037, 1595, 488, 926, 36, 1045, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[711, 1238, 317, 378, 333, 151, 2275, 2307, 1655, 649, 478, 1232, 2207, 369, 33, 301, 1971, 770, 653, 455, 435, 517, 32, 768, 108, 1727, 667, 259, 221, 582, 1374, 95, 123, 44, 310, 311, 1148, 543, 808, 1413, 712, 247, 1782, 2480, 369, 843, 887, 31, 349, 461, 536, 383, 217, 189, 124, 2169, 302, 503, 1146, 551, 1629, 654, 2319, 301] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[133, 1250, 330, 455, 330, 982, 1030, 1994, 1652, 711, 226, 937, 2337, 180, 72, 382, 1392, 779, 616, 178, 154, 352, 126, 836, 305, 1594, 466, 601, 245, 264, 1051, 134, 387, 638, 1135, 520, 1897, 585, 1587, 2030, 618, 1097, 608, 789, 96, 1604, 2371, 72, 255, 175, 1579, 455, 688, 87, 46, 1039, 627, 524, 1132, 294, 1705, 548, 834, 1036] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[188, 1164, 25, 827, 360, 262, 1418, 2315, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 879, 365, 1413, 1544, 283, 135, 224, 928, 33, 0, 0, 0, 0, 0, 0, 1037, 187, 1060, 69, 410, 505, 612, 1242, 632, 257, 0, 0, 0, 0, 0, 0, 19, 1307, 1184, 113, 269, 438, 577, 19, 1700, 0, 0, 0, 0, 0, 0, 0, 279, 35, 922, 957, 1006, 93, 2618, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 433, 537, 494, 2505, 618, 648, 2124, 57, 0, 0, 0, 0, 0, 0, 0, 0, 661, 370, 1040, 127, 339, 46, 79, 1617, 397, 0, 0, 0, 0, 0, 0, 0, 236, 909, 847, 253, 492, 228, 705, 1292, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6559, 6811, 6011, 5626, 7067, 7416, 4676, 4986] avg=6144.0000 max=7416 max_rank=5 min=4676 min_rank=6 imbalance_ratio=1.207031 max_load_reduction=41736 max_load_reduction_pct=0.849121 moved_to_echo=24145 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[133, 1250, 330, 455, 330, 982, 781, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 711, 226, 937, 2337, 180, 72, 382, 387, 18, 0, 0, 0, 0, 0, 0, 1392, 779, 616, 178, 154, 352, 126, 836, 336, 107, 0, 0, 0, 0, 0, 0, 305, 1594, 466, 601, 245, 264, 1051, 134, 904, 0, 0, 0, 0, 0, 0, 0, 387, 638, 1135, 520, 1897, 585, 1093, 2030, 0, 0, 0, 0, 0, 0, 0, 0, 618, 1097, 608, 789, 96, 1604, 1131, 72, 0, 0, 0, 0, 0, 0, 0, 0, 255, 175, 1579, 455, 688, 87, 46, 1039, 218, 0, 0, 0, 0, 0, 0, 0, 627, 524, 1132, 294, 1705, 548, 834, 1036, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6255, 6902, 4876, 5564, 8285, 6015, 4542, 6713] avg=6144.0000 max=8285 max_rank=4 min=4542 min_rank=6 imbalance_ratio=1.348470 max_load_reduction=40867 max_load_reduction_pct=0.831441 moved_to_echo=25555 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[398, 1169, 571, 911, 349, 825, 522, 613, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 586, 54, 572, 401, 502, 46, 170, 284, 12, 0, 0, 0, 0, 0, 0, 1958, 726, 1546, 268, 83, 400, 499, 546, 236, 78, 0, 0, 0, 0, 0, 0, 261, 1869, 840, 67, 159, 278, 2718, 108, 636, 0, 0, 0, 0, 0, 0, 0, 260, 98, 1333, 166, 2098, 310, 804, 1972, 0, 0, 0, 0, 0, 0, 0, 0, 550, 245, 1490, 896, 466, 437, 795, 30, 0, 0, 0, 0, 0, 0, 0, 0, 622, 342, 1670, 322, 354, 134, 269, 3050, 146, 0, 0, 0, 0, 0, 0, 0, 107, 1191, 1642, 316, 737, 542, 1819, 1080, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5358, 4217, 6340, 6936, 7041, 4909, 6909, 7442] avg=6144.0000 max=7442 max_rank=7 min=4217 min_rank=1 imbalance_ratio=1.211263 max_load_reduction=41710 max_load_reduction_pct=0.848592 moved_to_echo=26301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[391, 756, 169, 615, 576, 629, 1156, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 53, 168, 483, 3216, 311, 37, 200, 471, 27, 0, 0, 0, 0, 0, 0, 1948, 777, 518, 58, 378, 544, 45, 361, 657, 130, 0, 0, 0, 0, 0, 0, 79, 2026, 1054, 56, 159, 404, 1602, 671, 1769, 0, 0, 0, 0, 0, 0, 0, 149, 68, 942, 268, 1660, 227, 1328, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 51, 1863, 1412, 320, 473, 2211, 88, 0, 0, 0, 0, 0, 0, 0, 0, 185, 328, 1313, 1841, 458, 4, 85, 2006, 323, 0, 0, 0, 0, 0, 0, 0, 116, 562, 281, 1216, 902, 301, 652, 40, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5302, 6726, 5416, 7820, 5824, 7432, 6543, 4089] avg=6144.0000 max=7820 max_rank=3 min=4089 min_rank=7 imbalance_ratio=1.272786 max_load_reduction=41332 max_load_reduction_pct=0.840902 moved_to_echo=23888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[418, 1341, 391, 641, 922, 200, 1249, 778, 0, 0, 0, 0, 0, 0, 0, 0, 1934, 878, 526, 1208, 227, 1133, 38, 292, 469, 29, 0, 0, 0, 0, 0, 0, 1815, 625, 730, 153, 207, 677, 170, 781, 289, 130, 0, 0, 0, 0, 0, 0, 265, 1936, 2955, 72, 167, 417, 398, 549, 779, 0, 0, 0, 0, 0, 0, 0, 106, 256, 1667, 535, 684, 153, 1322, 915, 0, 0, 0, 0, 0, 0, 0, 0, 1427, 279, 764, 882, 922, 719, 974, 94, 0, 0, 0, 0, 0, 0, 0, 0, 540, 441, 1006, 220, 935, 893, 448, 1159, 349, 0, 0, 0, 0, 0, 0, 0, 48, 300, 1047, 1209, 1026, 608, 988, 426, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5940, 6734, 5577, 7538, 5638, 6061, 5991, 5673] avg=6144.0000 max=7538 max_rank=3 min=5577 min_rank=2 imbalance_ratio=1.226888 max_load_reduction=41614 max_load_reduction_pct=0.846639 moved_to_echo=23363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[711, 1238, 317, 378, 333, 151, 1713, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 1655, 649, 478, 1232, 2207, 369, 33, 301, 201, 42, 0, 0, 0, 0, 0, 0, 1971, 770, 653, 455, 435, 517, 32, 768, 129, 58, 0, 0, 0, 0, 0, 0, 108, 1727, 667, 259, 221, 582, 1374, 95, 340, 0, 0, 0, 0, 0, 0, 0, 123, 44, 310, 311, 1148, 543, 549, 1413, 0, 0, 0, 0, 0, 0, 0, 0, 712, 247, 1782, 2480, 369, 843, 418, 31, 0, 0, 0, 0, 0, 0, 0, 0, 349, 461, 536, 383, 217, 189, 124, 2169, 487, 0, 0, 0, 0, 0, 0, 0, 302, 503, 1146, 551, 1629, 654, 2319, 301, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7148, 7167, 5788, 5373, 4441, 6882, 4915, 7438] avg=6144.0000 max=7438 max_rank=7 min=4441 min_rank=4 imbalance_ratio=1.210612 max_load_reduction=41714 max_load_reduction_pct=0.848674 moved_to_echo=23676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[908, 539, 474, 1134, 472, 427, 669, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 1995, 530, 80, 909, 898, 585, 20, 111, 296, 15, 0, 0, 0, 0, 0, 0, 1414, 614, 506, 306, 174, 2192, 144, 1319, 215, 82, 0, 0, 0, 0, 0, 0, 91, 1777, 324, 227, 60, 470, 575, 851, 579, 0, 0, 0, 0, 0, 0, 0, 29, 202, 1467, 648, 1871, 101, 836, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 823, 319, 541, 1503, 323, 178, 724, 48, 0, 0, 0, 0, 0, 0, 0, 0, 374, 491, 1778, 1027, 1236, 247, 145, 2390, 186, 0, 0, 0, 0, 0, 0, 0, 55, 467, 3299, 477, 965, 378, 776, 279, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5966, 5439, 6966, 4954, 6787, 4459, 7874, 6707] avg=6144.0000 max=7874 max_rank=6 min=4459 min_rank=5 imbalance_ratio=1.281576 max_load_reduction=41278 max_load_reduction_pct=0.839803 moved_to_echo=25827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[398, 1169, 571, 911, 349, 825, 522, 613, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 586, 54, 572, 401, 502, 46, 170, 284, 12, 0, 0, 0, 0, 0, 0, 1958, 726, 1546, 268, 83, 400, 499, 546, 236, 78, 0, 0, 0, 0, 0, 0, 261, 1869, 840, 67, 159, 278, 2718, 108, 636, 0, 0, 0, 0, 0, 0, 0, 260, 98, 1333, 166, 2098, 310, 804, 1972, 0, 0, 0, 0, 0, 0, 0, 0, 550, 245, 1490, 896, 466, 437, 795, 30, 0, 0, 0, 0, 0, 0, 0, 0, 622, 342, 1670, 322, 354, 134, 269, 3050, 146, 0, 0, 0, 0, 0, 0, 0, 107, 1191, 1642, 316, 737, 542, 1819, 1080, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5358, 4217, 6340, 6936, 7041, 4909, 6909, 7442] avg=6144.0000 max=7442 max_rank=7 min=4217 min_rank=1 imbalance_ratio=1.211263 max_load_reduction=41710 max_load_reduction_pct=0.848592 moved_to_echo=26301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[337, 546, 214, 860, 393, 121, 2608, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 1386, 809, 136, 976, 1316, 204, 45, 119, 104, 61, 0, 0, 0, 0, 0, 0, 2069, 217, 1002, 416, 335, 724, 2138, 865, 383, 29, 0, 0, 0, 0, 0, 0, 126, 1655, 900, 15, 219, 121, 1158, 116, 1031, 0, 0, 0, 0, 0, 0, 0, 183, 125, 772, 531, 897, 263, 296, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 514, 652, 1128, 1953, 227, 195, 1289, 20, 0, 0, 0, 0, 0, 0, 0, 0, 389, 302, 1424, 879, 142, 36, 155, 3645, 730, 0, 0, 0, 0, 0, 0, 0, 620, 1037, 1595, 488, 926, 36, 1045, 313, 44, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6624, 5156, 8178, 5341, 4069, 5978, 7702, 6104] avg=6144.0000 max=8178 max_rank=2 min=4069 min_rank=4 imbalance_ratio=1.331055 max_load_reduction=40974 max_load_reduction_pct=0.833618 moved_to_echo=23853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[908, 539, 474, 1134, 472, 427, 669, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 1995, 530, 80, 909, 898, 585, 20, 111, 296, 15, 0, 0, 0, 0, 0, 0, 1414, 614, 506, 306, 174, 2192, 144, 1319, 215, 82, 0, 0, 0, 0, 0, 0, 91, 1777, 324, 227, 60, 470, 575, 851, 579, 0, 0, 0, 0, 0, 0, 0, 29, 202, 1467, 648, 1871, 101, 836, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 823, 319, 541, 1503, 323, 178, 724, 48, 0, 0, 0, 0, 0, 0, 0, 0, 374, 491, 1778, 1027, 1236, 247, 145, 2390, 186, 0, 0, 0, 0, 0, 0, 0, 55, 467, 3299, 477, 965, 378, 776, 279, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5966, 5439, 6966, 4954, 6787, 4459, 7874, 6707] avg=6144.0000 max=7874 max_rank=6 min=4459 min_rank=5 imbalance_ratio=1.281576 max_load_reduction=41278 max_load_reduction_pct=0.839803 moved_to_echo=25827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[133, 1250, 330, 455, 330, 982, 781, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 711, 226, 937, 2337, 180, 72, 382, 387, 18, 0, 0, 0, 0, 0, 0, 1392, 779, 616, 178, 154, 352, 126, 836, 336, 107, 0, 0, 0, 0, 0, 0, 305, 1594, 466, 601, 245, 264, 1051, 134, 904, 0, 0, 0, 0, 0, 0, 0, 387, 638, 1135, 520, 1897, 585, 1093, 2030, 0, 0, 0, 0, 0, 0, 0, 0, 618, 1097, 608, 789, 96, 1604, 1131, 72, 0, 0, 0, 0, 0, 0, 0, 0, 255, 175, 1579, 455, 688, 87, 46, 1039, 218, 0, 0, 0, 0, 0, 0, 0, 627, 524, 1132, 294, 1705, 548, 834, 1036, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6255, 6902, 4876, 5564, 8285, 6015, 4542, 6713] avg=6144.0000 max=8285 max_rank=4 min=4542 min_rank=6 imbalance_ratio=1.348470 max_load_reduction=40867 max_load_reduction_pct=0.831441 moved_to_echo=25555 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[711, 1238, 317, 378, 333, 151, 1713, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 1655, 649, 478, 1232, 2207, 369, 33, 301, 201, 42, 0, 0, 0, 0, 0, 0, 1971, 770, 653, 455, 435, 517, 32, 768, 129, 58, 0, 0, 0, 0, 0, 0, 108, 1727, 667, 259, 221, 582, 1374, 95, 340, 0, 0, 0, 0, 0, 0, 0, 123, 44, 310, 311, 1148, 543, 549, 1413, 0, 0, 0, 0, 0, 0, 0, 0, 712, 247, 1782, 2480, 369, 843, 418, 31, 0, 0, 0, 0, 0, 0, 0, 0, 349, 461, 536, 383, 217, 189, 124, 2169, 487, 0, 0, 0, 0, 0, 0, 0, 302, 503, 1146, 551, 1629, 654, 2319, 301, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7148, 7167, 5788, 5373, 4441, 6882, 4915, 7438] avg=6144.0000 max=7438 max_rank=7 min=4441 min_rank=4 imbalance_ratio=1.210612 max_load_reduction=41714 max_load_reduction_pct=0.848674 moved_to_echo=23676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13056, 5120, 2048, 7808, 12160, 3584, 512, 1920, 3200, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6144, 3456, 8704, 12544, 3456, 5120, 9728, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 1536, 8576, 3968, 11264, 2304, 8960, 11392, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 5504, 11008, 4864, 8448, 3328, 9216, 4864, 256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13696, 4736, 6656, 1920, 2176, 6016, 3840, 6784, 2944, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6144, 3456, 8704, 12544, 3456, 5120, 9728, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 8064, 2560, 5888, 3840, 3712, 10240, 12032, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 2944, 10368, 5376, 4480, 1664, 1408, 17152, 2944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 8064, 2560, 5888, 3840, 3712, 10240, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13696, 4736, 6656, 1920, 2176, 6016, 3840, 6784, 2944, 896, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 13952, 8448, 1536, 1536, 3072, 9472, 2560, 7808, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[12, 490, 8, 53, 1098, 2258, 2110, 171, 380, 1424, 352, 236, 190, 1689, 582, 2303, 155, 147, 620, 401, 485, 20, 1329, 1432, 1195, 97, 288, 396, 181, 614, 976, 160, 816, 591, 1076, 1795, 774, 2009, 1925, 163, 201, 1235, 104, 432, 1973, 742, 545, 2417, 435, 887, 397, 554, 319, 247, 108, 1550, 1313, 348, 853, 321, 2455, 159, 398, 158] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[113, 167, 35, 623, 129, 910, 1168, 476, 90, 1182, 787, 626, 434, 1193, 540, 1012, 952, 569, 1148, 197, 606, 376, 957, 712, 1857, 557, 269, 2928, 432, 202, 1149, 335, 377, 722, 565, 1895, 1124, 541, 2272, 3, 126, 1336, 260, 225, 2171, 1709, 1327, 122, 666, 1469, 95, 1466, 297, 472, 111, 1323, 1200, 664, 587, 265, 1097, 535, 1136, 263] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[21, 60, 56, 471, 1730, 199, 705, 567, 217, 440, 612, 46, 723, 1918, 125, 2963, 1032, 154, 434, 462, 500, 284, 1531, 1396, 576, 1226, 552, 749, 165, 578, 1610, 361, 507, 1088, 1362, 951, 377, 947, 1280, 10, 697, 1552, 173, 95, 3549, 1251, 1252, 178, 739, 618, 307, 2246, 84, 75, 8, 1527, 96, 291, 995, 194, 856, 223, 2528, 633] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[331, 168, 575, 34, 445, 346, 426, 226, 448, 2205, 994, 244, 105, 2205, 298, 1105, 259, 1098, 426, 58, 460, 352, 917, 1362, 3765, 474, 37, 341, 160, 329, 1832, 306, 943, 69, 979, 2646, 1156, 824, 351, 145, 393, 933, 536, 320, 2785, 836, 692, 288, 464, 494, 214, 781, 210, 945, 225, 1660, 1229, 575, 1162, 459, 1176, 734, 1506, 1091] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[15, 257, 251, 767, 1322, 1048, 2388, 690, 1035, 975, 621, 1253, 277, 2457, 445, 1672, 1211, 514, 1116, 281, 349, 279, 742, 300, 271, 480, 1337, 246, 115, 612, 1583, 96, 437, 1047, 1113, 576, 725, 2053, 592, 77, 172, 1626, 235, 144, 1087, 2359, 631, 2188, 113, 158, 66, 1661, 171, 150, 144, 1416, 115, 631, 496, 827, 1570, 279, 979, 309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[113, 167, 35, 623, 129, 910, 1168, 476, 90, 1182, 787, 626, 434, 1193, 540, 1012, 952, 569, 1148, 197, 606, 376, 957, 712, 1857, 557, 269, 2928, 432, 202, 1149, 335, 377, 722, 565, 1895, 1124, 541, 2272, 3, 126, 1336, 260, 225, 2171, 1709, 1327, 122, 666, 1469, 95, 1466, 297, 472, 111, 1323, 1200, 664, 587, 265, 1097, 535, 1136, 263] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[72, 98, 69, 154, 1354, 879, 322, 315, 709, 2284, 1235, 373, 162, 2143, 429, 1824, 1728, 885, 399, 297, 129, 432, 863, 819, 1410, 389, 1405, 661, 25, 306, 2807, 68, 482, 107, 1631, 1429, 307, 438, 1034, 30, 85, 1259, 131, 243, 1533, 190, 1480, 575, 86, 1106, 706, 893, 299, 510, 133, 2783, 528, 678, 816, 908, 1496, 331, 1594, 286] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[331, 168, 575, 34, 445, 346, 426, 226, 448, 2205, 994, 244, 105, 2205, 298, 1105, 259, 1098, 426, 58, 460, 352, 917, 1362, 3765, 474, 37, 341, 160, 329, 1832, 306, 943, 69, 979, 2646, 1156, 824, 351, 145, 393, 933, 536, 320, 2785, 836, 692, 288, 464, 494, 214, 781, 210, 945, 225, 1660, 1229, 575, 1162, 459, 1176, 734, 1506, 1091] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[13, 52, 32, 224, 2157, 670, 516, 698, 624, 725, 539, 171, 651, 2275, 386, 1533, 453, 374, 985, 399, 1047, 612, 1200, 558, 986, 493, 117, 597, 118, 598, 1390, 176, 642, 1094, 459, 711, 186, 1606, 1555, 54, 87, 1699, 608, 186, 2273, 1371, 712, 160, 397, 447, 216, 4173, 242, 342, 98, 1002, 647, 283, 1167, 315, 2114, 551, 1790, 596] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[21, 60, 56, 471, 1730, 199, 705, 567, 217, 440, 612, 46, 723, 1918, 125, 2963, 1032, 154, 434, 462, 500, 284, 1531, 1396, 576, 1226, 552, 749, 165, 578, 1610, 361, 507, 1088, 1362, 951, 377, 947, 1280, 10, 697, 1552, 173, 95, 3549, 1251, 1252, 178, 739, 618, 307, 2246, 84, 75, 8, 1527, 96, 291, 995, 194, 856, 223, 2528, 633] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[68, 62, 11, 271, 565, 625, 107, 237, 240, 328, 846, 511, 1133, 1455, 287, 1528, 1526, 1405, 1458, 878, 468, 411, 1348, 313, 891, 312, 507, 147, 149, 306, 1786, 311, 367, 1187, 353, 1645, 155, 2059, 1645, 258, 153, 1907, 258, 248, 2556, 1310, 1478, 469, 171, 858, 284, 3119, 57, 249, 69, 1509, 141, 971, 506, 436, 1074, 390, 2014, 766] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[113, 167, 35, 623, 129, 910, 1168, 476, 1297, 462, 0, 0, 0, 0, 0, 0, 90, 1182, 787, 626, 434, 444, 540, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 952, 569, 1148, 197, 606, 376, 957, 712, 287, 137, 0, 0, 0, 0, 0, 0, 1857, 557, 269, 2928, 432, 202, 1149, 335, 185, 294, 0, 0, 0, 0, 0, 0, 377, 722, 565, 727, 1124, 541, 2272, 3, 0, 0, 0, 0, 0, 0, 0, 0, 126, 1336, 260, 225, 874, 1709, 1327, 122, 0, 0, 0, 0, 0, 0, 0, 0, 666, 1469, 95, 1466, 297, 472, 111, 1323, 737, 0, 0, 0, 0, 0, 0, 0, 1200, 664, 587, 265, 1097, 535, 951, 263, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5380, 5115, 5941, 8208, 6331, 5979, 6636, 5562] avg=6144.0000 max=8208 max_rank=3 min=5115 min_rank=1 imbalance_ratio=1.335938 max_load_reduction=40944 max_load_reduction_pct=0.833008 moved_to_echo=24508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[12, 490, 8, 53, 1098, 2258, 2110, 171, 1179, 654, 0, 0, 0, 0, 0, 0, 380, 1424, 352, 236, 190, 628, 582, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 155, 147, 620, 401, 485, 20, 1329, 1432, 407, 130, 0, 0, 0, 0, 0, 0, 1195, 97, 288, 396, 181, 614, 976, 160, 65, 278, 0, 0, 0, 0, 0, 0, 816, 591, 1076, 689, 774, 2009, 1925, 163, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1235, 104, 432, 794, 742, 545, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 435, 887, 397, 554, 319, 247, 108, 1550, 698, 0, 0, 0, 0, 0, 0, 0, 1313, 348, 853, 321, 2455, 159, 333, 158, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8033, 6095, 5126, 4250, 8043, 6470, 5195, 5940] avg=6144.0000 max=8043 max_rank=4 min=4250 min_rank=3 imbalance_ratio=1.309082 max_load_reduction=41109 max_load_reduction_pct=0.836365 moved_to_echo=25648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[72, 98, 69, 154, 1354, 879, 322, 315, 916, 830, 0, 0, 0, 0, 0, 0, 709, 2284, 1235, 373, 162, 797, 429, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 885, 399, 297, 129, 432, 863, 819, 516, 103, 0, 0, 0, 0, 0, 0, 1410, 389, 1405, 661, 25, 306, 2807, 68, 260, 221, 0, 0, 0, 0, 0, 0, 482, 107, 1631, 550, 307, 438, 1034, 30, 0, 0, 0, 0, 0, 0, 0, 0, 85, 1259, 131, 243, 617, 190, 1480, 575, 0, 0, 0, 0, 0, 0, 0, 0, 86, 1106, 706, 893, 299, 510, 133, 2783, 555, 0, 0, 0, 0, 0, 0, 0, 528, 678, 816, 908, 1496, 331, 1334, 286, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5009, 7813, 6171, 7552, 4579, 4580, 7071, 6377] avg=6144.0000 max=7813 max_rank=1 min=4579 min_rank=4 imbalance_ratio=1.271647 max_load_reduction=41339 max_load_reduction_pct=0.841044 moved_to_echo=22607 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[21, 60, 56, 471, 1730, 199, 705, 567, 2121, 743, 0, 0, 0, 0, 0, 0, 217, 440, 612, 46, 723, 713, 125, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 154, 434, 462, 500, 284, 1531, 1396, 462, 69, 0, 0, 0, 0, 0, 0, 576, 1226, 552, 749, 165, 578, 1610, 361, 413, 147, 0, 0, 0, 0, 0, 0, 507, 1088, 1362, 366, 377, 947, 1280, 10, 0, 0, 0, 0, 0, 0, 0, 0, 697, 1552, 173, 95, 1428, 1251, 1252, 178, 0, 0, 0, 0, 0, 0, 0, 0, 739, 618, 307, 2246, 84, 75, 8, 1527, 369, 0, 0, 0, 0, 0, 0, 0, 96, 291, 995, 194, 856, 223, 2115, 633, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6673, 5839, 6324, 6377, 5937, 6626, 5973, 5403] avg=6144.0000 max=6673 max_rank=0 min=5403 min_rank=7 imbalance_ratio=1.086100 max_load_reduction=42479 max_load_reduction_pct=0.864237 moved_to_echo=23939 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[15, 257, 251, 767, 1322, 1048, 2388, 690, 654, 957, 0, 0, 0, 0, 0, 0, 1035, 975, 621, 1253, 277, 905, 445, 1672, 0, 0, 0, 0, 0, 0, 0, 0, 1211, 514, 1116, 281, 349, 279, 742, 300, 595, 45, 0, 0, 0, 0, 0, 0, 271, 480, 1337, 246, 115, 612, 1583, 96, 162, 93, 0, 0, 0, 0, 0, 0, 437, 1047, 1113, 210, 725, 2053, 592, 77, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1626, 235, 144, 433, 2359, 631, 2188, 0, 0, 0, 0, 0, 0, 0, 0, 113, 158, 66, 1661, 171, 150, 144, 1416, 228, 0, 0, 0, 0, 0, 0, 0, 115, 631, 496, 827, 1570, 279, 817, 309, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 7183, 5432, 4995, 6254, 7788, 4107, 5044] avg=6144.0000 max=8349 max_rank=0 min=4107 min_rank=6 imbalance_ratio=1.358887 max_load_reduction=40803 max_load_reduction_pct=0.830139 moved_to_echo=23193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[331, 168, 575, 34, 445, 346, 426, 226, 1664, 854, 0, 0, 0, 0, 0, 0, 448, 2205, 994, 244, 105, 820, 298, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 259, 1098, 426, 58, 460, 352, 917, 1362, 531, 192, 0, 0, 0, 0, 0, 0, 3765, 474, 37, 341, 160, 329, 1832, 306, 246, 410, 0, 0, 0, 0, 0, 0, 943, 69, 979, 1015, 1156, 824, 351, 145, 0, 0, 0, 0, 0, 0, 0, 0, 393, 933, 536, 320, 1121, 836, 692, 288, 0, 0, 0, 0, 0, 0, 0, 0, 464, 494, 214, 781, 210, 945, 225, 1660, 1029, 0, 0, 0, 0, 0, 0, 0, 1229, 575, 1162, 459, 1176, 734, 1260, 1091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5069, 6219, 5655, 7900, 5482, 5119, 6022, 7686] avg=6144.0000 max=7900 max_rank=3 min=5069 min_rank=0 imbalance_ratio=1.285807 max_load_reduction=41252 max_load_reduction_pct=0.839274 moved_to_echo=24309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[13, 52, 32, 224, 2157, 670, 516, 698, 1358, 882, 0, 0, 0, 0, 0, 0, 624, 725, 539, 171, 651, 845, 386, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 453, 374, 985, 399, 1047, 612, 1200, 558, 548, 51, 0, 0, 0, 0, 0, 0, 986, 493, 117, 597, 118, 598, 1390, 176, 292, 110, 0, 0, 0, 0, 0, 0, 642, 1094, 459, 274, 186, 1606, 1555, 54, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1699, 608, 186, 915, 1371, 712, 160, 0, 0, 0, 0, 0, 0, 0, 0, 397, 447, 216, 4173, 242, 342, 98, 1002, 276, 0, 0, 0, 0, 0, 0, 0, 647, 283, 1167, 315, 2114, 551, 1498, 596, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6602, 5474, 6227, 4877, 5870, 5738, 7193, 7171] avg=6144.0000 max=7193 max_rank=6 min=4877 min_rank=3 imbalance_ratio=1.170736 max_load_reduction=41959 max_load_reduction_pct=0.853658 moved_to_echo=25972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[113, 167, 35, 623, 129, 910, 1168, 476, 1297, 462, 0, 0, 0, 0, 0, 0, 90, 1182, 787, 626, 434, 444, 540, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 952, 569, 1148, 197, 606, 376, 957, 712, 287, 137, 0, 0, 0, 0, 0, 0, 1857, 557, 269, 2928, 432, 202, 1149, 335, 185, 294, 0, 0, 0, 0, 0, 0, 377, 722, 565, 727, 1124, 541, 2272, 3, 0, 0, 0, 0, 0, 0, 0, 0, 126, 1336, 260, 225, 874, 1709, 1327, 122, 0, 0, 0, 0, 0, 0, 0, 0, 666, 1469, 95, 1466, 297, 472, 111, 1323, 737, 0, 0, 0, 0, 0, 0, 0, 1200, 664, 587, 265, 1097, 535, 951, 263, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5380, 5115, 5941, 8208, 6331, 5979, 6636, 5562] avg=6144.0000 max=8208 max_rank=3 min=5115 min_rank=1 imbalance_ratio=1.335938 max_load_reduction=40944 max_load_reduction_pct=0.833008 moved_to_echo=24508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[72, 98, 69, 154, 1354, 879, 322, 315, 916, 830, 0, 0, 0, 0, 0, 0, 709, 2284, 1235, 373, 162, 797, 429, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 885, 399, 297, 129, 432, 863, 819, 516, 103, 0, 0, 0, 0, 0, 0, 1410, 389, 1405, 661, 25, 306, 2807, 68, 260, 221, 0, 0, 0, 0, 0, 0, 482, 107, 1631, 550, 307, 438, 1034, 30, 0, 0, 0, 0, 0, 0, 0, 0, 85, 1259, 131, 243, 617, 190, 1480, 575, 0, 0, 0, 0, 0, 0, 0, 0, 86, 1106, 706, 893, 299, 510, 133, 2783, 555, 0, 0, 0, 0, 0, 0, 0, 528, 678, 816, 908, 1496, 331, 1334, 286, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5009, 7813, 6171, 7552, 4579, 4580, 7071, 6377] avg=6144.0000 max=7813 max_rank=1 min=4579 min_rank=4 imbalance_ratio=1.271647 max_load_reduction=41339 max_load_reduction_pct=0.841044 moved_to_echo=22607 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[15, 257, 251, 767, 1322, 1048, 2388, 690, 654, 957, 0, 0, 0, 0, 0, 0, 1035, 975, 621, 1253, 277, 905, 445, 1672, 0, 0, 0, 0, 0, 0, 0, 0, 1211, 514, 1116, 281, 349, 279, 742, 300, 595, 45, 0, 0, 0, 0, 0, 0, 271, 480, 1337, 246, 115, 612, 1583, 96, 162, 93, 0, 0, 0, 0, 0, 0, 437, 1047, 1113, 210, 725, 2053, 592, 77, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1626, 235, 144, 433, 2359, 631, 2188, 0, 0, 0, 0, 0, 0, 0, 0, 113, 158, 66, 1661, 171, 150, 144, 1416, 228, 0, 0, 0, 0, 0, 0, 0, 115, 631, 496, 827, 1570, 279, 817, 309, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 7183, 5432, 4995, 6254, 7788, 4107, 5044] avg=6144.0000 max=8349 max_rank=0 min=4107 min_rank=6 imbalance_ratio=1.358887 max_load_reduction=40803 max_load_reduction_pct=0.830139 moved_to_echo=23193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[68, 62, 11, 271, 565, 625, 107, 237, 1527, 564, 0, 0, 0, 0, 0, 0, 240, 328, 846, 511, 1133, 541, 287, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 1526, 1405, 1458, 878, 468, 411, 1348, 313, 350, 119, 0, 0, 0, 0, 0, 0, 891, 312, 507, 147, 149, 306, 1786, 311, 329, 255, 0, 0, 0, 0, 0, 0, 367, 1187, 353, 632, 155, 2059, 1645, 258, 0, 0, 0, 0, 0, 0, 0, 0, 153, 1907, 258, 248, 1029, 1310, 1478, 469, 0, 0, 0, 0, 0, 0, 0, 0, 171, 858, 284, 3119, 57, 249, 69, 1509, 639, 0, 0, 0, 0, 0, 0, 0, 141, 971, 506, 436, 1074, 390, 1685, 766, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 5414, 8276, 4993, 6656, 6852, 6955, 5969] avg=6144.0000 max=8276 max_rank=2 min=4037 min_rank=0 imbalance_ratio=1.347005 max_load_reduction=40876 max_load_reduction_pct=0.831624 moved_to_echo=26432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[13, 52, 32, 224, 2157, 670, 516, 698, 1358, 882, 0, 0, 0, 0, 0, 0, 624, 725, 539, 171, 651, 845, 386, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 453, 374, 985, 399, 1047, 612, 1200, 558, 548, 51, 0, 0, 0, 0, 0, 0, 986, 493, 117, 597, 118, 598, 1390, 176, 292, 110, 0, 0, 0, 0, 0, 0, 642, 1094, 459, 274, 186, 1606, 1555, 54, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1699, 608, 186, 915, 1371, 712, 160, 0, 0, 0, 0, 0, 0, 0, 0, 397, 447, 216, 4173, 242, 342, 98, 1002, 276, 0, 0, 0, 0, 0, 0, 0, 647, 283, 1167, 315, 2114, 551, 1498, 596, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6602, 5474, 6227, 4877, 5870, 5738, 7193, 7171] avg=6144.0000 max=7193 max_rank=6 min=4877 min_rank=3 imbalance_ratio=1.170736 max_load_reduction=41959 max_load_reduction_pct=0.853658 moved_to_echo=25972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[11008, 4096, 4608, 6144, 1408, 3584, 13184, 1920, 2048, 1920, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1920, 11648, 2432, 1920, 7296, 9856, 8192, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 1408, 1152, 2688, 8832, 7040, 7808, 3456, 10752, 6016, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 5248, 6656, 3072, 4096, 2816, 8960, 6912, 3712, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4608, 6016, 7552, 4480, 4864, 10496, 10752, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5376, 4480, 6656, 3840, 11904, 3328, 10112, 4224, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3840, 9600, 6016, 3584, 3712, 5760, 3200, 13952, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4608, 6016, 7552, 4480, 4864, 10496, 10752, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 1408, 1152, 2688, 8832, 7040, 7808, 3456, 10752, 6016, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 6144, 2304, 14976, 1792, 3072, 896, 12800, 4608, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5376, 4480, 6656, 3840, 11904, 3328, 10112, 4224, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[692, 1924, 2896, 2322, 1068, 1224, 2645, 2985, 528, 3528, 448, 176, 55, 229, 175, 431, 104, 350, 1040, 448, 995, 225, 1824, 165, 928, 17, 300, 1069, 555, 252, 147, 569, 1506, 1993, 178, 165, 39, 2126, 628, 1148, 135, 125, 1261, 20, 608, 660, 1112, 50, 675, 893, 366, 196, 194, 16, 947, 559, 59, 42, 409, 820, 181, 1203, 116, 408] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[703, 1806, 1004, 2599, 534, 994, 2518, 677, 46, 1332, 605, 190, 374, 720, 233, 610, 31, 353, 611, 729, 638, 1423, 2334, 44, 464, 113, 171, 2350, 283, 100, 64, 1379, 1112, 666, 93, 253, 97, 816, 261, 2538, 575, 447, 3244, 101, 731, 390, 830, 338, 554, 585, 20, 1282, 204, 19, 998, 413, 14, 99, 454, 3969, 243, 1422, 189, 163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[163, 617, 3007, 1967, 314, 723, 1158, 520, 32, 1296, 106, 89, 631, 730, 1155, 402, 766, 314, 1515, 375, 1293, 742, 1577, 56, 171, 90, 95, 1492, 1497, 322, 1489, 1191, 585, 746, 55, 168, 181, 559, 588, 1770, 216, 756, 2473, 60, 281, 60, 363, 111, 433, 812, 57, 925, 23, 203, 1705, 715, 95, 125, 897, 3822, 206, 3962, 84, 221] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[796, 2774, 3042, 2337, 1095, 368, 2314, 1389, 8, 4131, 544, 50, 386, 720, 1190, 572, 34, 142, 1044, 119, 587, 474, 831, 907, 590, 36, 207, 1066, 1270, 214, 330, 515, 161, 441, 45, 535, 81, 705, 102, 1280, 331, 202, 1108, 630, 488, 198, 855, 107, 959, 1039, 218, 647, 46, 93, 2073, 1020, 166, 77, 357, 514, 610, 2805, 1090, 87] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[736, 1193, 2388, 2089, 606, 175, 3149, 2192, 554, 1124, 525, 781, 399, 668, 1633, 132, 10, 581, 541, 123, 894, 967, 296, 93, 745, 35, 141, 634, 618, 662, 36, 395, 1358, 675, 165, 51, 698, 2684, 84, 1352, 276, 195, 208, 809, 341, 142, 1269, 482, 2229, 675, 112, 347, 369, 193, 2018, 450, 53, 672, 384, 3655, 359, 826, 477, 429] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[52, 632, 2136, 728, 269, 31, 641, 1509, 67, 2441, 770, 15, 126, 546, 853, 1094, 818, 126, 3148, 108, 261, 1324, 1603, 139, 739, 122, 60, 1812, 202, 1149, 355, 1849, 2270, 660, 75, 689, 259, 1243, 422, 2133, 115, 459, 1379, 1, 588, 74, 821, 63, 117, 2032, 47, 980, 257, 24, 1505, 3135, 431, 37, 1079, 825, 344, 1085, 151, 127] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[796, 2774, 3042, 2337, 1095, 368, 2314, 1389, 8, 4131, 544, 50, 386, 720, 1190, 572, 34, 142, 1044, 119, 587, 474, 831, 907, 590, 36, 207, 1066, 1270, 214, 330, 515, 161, 441, 45, 535, 81, 705, 102, 1280, 331, 202, 1108, 630, 488, 198, 855, 107, 959, 1039, 218, 647, 46, 93, 2073, 1020, 166, 77, 357, 514, 610, 2805, 1090, 87] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[574, 1433, 3912, 846, 698, 161, 3793, 660, 724, 3060, 611, 72, 485, 357, 888, 931, 279, 482, 321, 335, 1224, 1651, 606, 660, 289, 113, 208, 600, 1217, 488, 120, 1235, 1767, 1775, 52, 261, 124, 755, 1035, 941, 266, 90, 953, 266, 213, 257, 1086, 111, 1111, 928, 58, 592, 147, 96, 1082, 1293, 67, 27, 1738, 1569, 55, 848, 410, 146] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[703, 1806, 1004, 2599, 534, 994, 2518, 677, 46, 1332, 605, 190, 374, 720, 233, 610, 31, 353, 611, 729, 638, 1423, 2334, 44, 464, 113, 171, 2350, 283, 100, 64, 1379, 1112, 666, 93, 253, 97, 816, 261, 2538, 575, 447, 3244, 101, 731, 390, 830, 338, 554, 585, 20, 1282, 204, 19, 998, 413, 14, 99, 454, 3969, 243, 1422, 189, 163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[692, 1924, 2896, 2322, 1068, 1224, 2645, 2985, 528, 3528, 448, 176, 55, 229, 175, 431, 104, 350, 1040, 448, 995, 225, 1824, 165, 928, 17, 300, 1069, 555, 252, 147, 569, 1506, 1993, 178, 165, 39, 2126, 628, 1148, 135, 125, 1261, 20, 608, 660, 1112, 50, 675, 893, 366, 196, 194, 16, 947, 559, 59, 42, 409, 820, 181, 1203, 116, 408] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[52, 632, 2136, 728, 269, 31, 641, 1509, 67, 2441, 770, 15, 126, 546, 853, 1094, 818, 126, 3148, 108, 261, 1324, 1603, 139, 739, 122, 60, 1812, 202, 1149, 355, 1849, 2270, 660, 75, 689, 259, 1243, 422, 2133, 115, 459, 1379, 1, 588, 74, 821, 63, 117, 2032, 47, 980, 257, 24, 1505, 3135, 431, 37, 1079, 825, 344, 1085, 151, 127] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[163, 617, 3007, 1967, 314, 723, 1158, 520, 32, 1296, 106, 89, 631, 730, 1155, 402, 766, 314, 1515, 375, 1293, 742, 1577, 56, 171, 90, 95, 1492, 1497, 322, 1489, 1191, 585, 746, 55, 168, 181, 559, 588, 1770, 216, 756, 2473, 60, 281, 60, 363, 111, 433, 812, 57, 925, 23, 203, 1705, 715, 95, 125, 897, 3822, 206, 3962, 84, 221] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[710, 1287, 2512, 335, 1290, 400, 2017, 578, 12, 1484, 225, 137, 311, 734, 1043, 689, 38, 490, 756, 462, 2356, 2379, 2376, 123, 221, 20, 143, 1488, 208, 208, 957, 971, 1934, 955, 5, 534, 182, 1554, 57, 1137, 141, 216, 2033, 2, 717, 308, 2045, 63, 328, 538, 65, 663, 12, 71, 1440, 874, 32, 25, 619, 3667, 712, 306, 319, 638] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[703, 1806, 0, 2477, 534, 994, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1332, 605, 190, 374, 720, 233, 610, 543, 0, 0, 0, 0, 0, 0, 0, 31, 353, 611, 729, 638, 1423, 2334, 44, 122, 70, 73, 0, 0, 0, 0, 0, 464, 113, 171, 2350, 283, 100, 64, 1379, 317, 714, 0, 0, 0, 0, 0, 0, 1112, 666, 93, 253, 97, 816, 261, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 575, 447, 3244, 101, 731, 390, 830, 338, 687, 0, 0, 0, 0, 0, 0, 0, 554, 585, 20, 1282, 204, 19, 998, 413, 894, 0, 0, 0, 0, 0, 0, 0, 14, 99, 454, 3969, 243, 1422, 189, 163, 297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7191, 4653, 6428, 5955, 5763, 7343, 4969, 6850] avg=6144.0000 max=7343 max_rank=5 min=4653 min_rank=1 imbalance_ratio=1.195150 max_load_reduction=41809 max_load_reduction_pct=0.850606 moved_to_echo=24925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[736, 1193, 0, 1991, 606, 175, 0, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 554, 1124, 525, 781, 399, 668, 1633, 132, 680, 0, 0, 0, 0, 0, 0, 0, 10, 581, 541, 123, 894, 967, 296, 93, 98, 88, 39, 0, 0, 0, 0, 0, 745, 35, 141, 634, 618, 662, 36, 395, 754, 893, 0, 0, 0, 0, 0, 0, 1358, 675, 165, 51, 698, 2684, 84, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 276, 195, 208, 809, 341, 142, 1269, 482, 1634, 0, 0, 0, 0, 0, 0, 0, 2229, 675, 112, 347, 369, 193, 2018, 450, 1120, 0, 0, 0, 0, 0, 0, 0, 53, 672, 384, 3655, 359, 826, 477, 429, 368, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6893, 6496, 3730, 4913, 7028, 5356, 7513, 7223] avg=6144.0000 max=7513 max_rank=6 min=3730 min_rank=2 imbalance_ratio=1.222819 max_load_reduction=41639 max_load_reduction_pct=0.847148 moved_to_echo=27120 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[796, 2774, 0, 2227, 1095, 368, 0, 1389, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4131, 544, 50, 386, 720, 1190, 572, 499, 0, 0, 0, 0, 0, 0, 0, 34, 142, 1044, 119, 587, 474, 831, 907, 110, 65, 36, 0, 0, 0, 0, 0, 590, 36, 207, 1066, 1270, 214, 330, 515, 960, 659, 0, 0, 0, 0, 0, 0, 161, 441, 45, 535, 81, 705, 102, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 331, 202, 1108, 630, 488, 198, 855, 107, 2082, 0, 0, 0, 0, 0, 0, 0, 959, 1039, 218, 647, 46, 93, 2073, 1020, 821, 0, 0, 0, 0, 0, 0, 0, 166, 77, 357, 514, 610, 2805, 1090, 87, 270, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8649, 8100, 4349, 5847, 3314, 6001, 6916, 5976] avg=6144.0000 max=8649 max_rank=0 min=3314 min_rank=4 imbalance_ratio=1.407715 max_load_reduction=40503 max_load_reduction_pct=0.824036 moved_to_echo=22207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[692, 1924, 0, 2213, 1068, 1224, 0, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 528, 3528, 448, 176, 55, 229, 175, 431, 572, 0, 0, 0, 0, 0, 0, 0, 104, 350, 1040, 448, 995, 225, 1824, 165, 109, 75, 33, 0, 0, 0, 0, 0, 928, 17, 300, 1069, 555, 252, 147, 569, 915, 750, 0, 0, 0, 0, 0, 0, 1506, 1993, 178, 165, 39, 2126, 628, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 135, 125, 1261, 20, 608, 660, 1112, 50, 1981, 0, 0, 0, 0, 0, 0, 0, 675, 893, 366, 196, 194, 16, 947, 559, 939, 0, 0, 0, 0, 0, 0, 0, 59, 42, 409, 820, 181, 1203, 116, 408, 309, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10106, 6142, 5368, 5502, 7750, 5952, 4785, 3547] avg=6144.0000 max=10106 max_rank=0 min=3547 min_rank=7 imbalance_ratio=1.644857 max_load_reduction=39046 max_load_reduction_pct=0.794393 moved_to_echo=22034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[52, 632, 0, 694, 269, 31, 0, 1509, 0, 0, 0, 0, 0, 0, 0, 0, 67, 2441, 770, 15, 126, 546, 853, 1094, 138, 0, 0, 0, 0, 0, 0, 0, 818, 126, 3148, 108, 261, 1324, 1603, 139, 34, 18, 61, 0, 0, 0, 0, 0, 739, 122, 60, 1812, 202, 1149, 355, 1849, 674, 183, 0, 0, 0, 0, 0, 0, 2270, 660, 75, 689, 259, 1243, 422, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 115, 459, 1379, 1, 588, 74, 821, 63, 1462, 0, 0, 0, 0, 0, 0, 0, 117, 2032, 47, 980, 257, 24, 1505, 3135, 228, 0, 0, 0, 0, 0, 0, 0, 431, 37, 1079, 825, 344, 1085, 151, 127, 74, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3187, 6050, 7640, 7145, 7690, 4962, 8325, 4153] avg=6144.0000 max=8325 max_rank=6 min=3187 min_rank=0 imbalance_ratio=1.354980 max_load_reduction=40827 max_load_reduction_pct=0.830627 moved_to_echo=25130 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[163, 617, 0, 1875, 314, 723, 0, 520, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1296, 106, 89, 631, 730, 1155, 402, 250, 0, 0, 0, 0, 0, 0, 0, 766, 314, 1515, 375, 1293, 742, 1577, 56, 92, 33, 51, 0, 0, 0, 0, 0, 171, 90, 95, 1492, 1497, 322, 1489, 1191, 950, 329, 0, 0, 0, 0, 0, 0, 585, 746, 55, 168, 181, 559, 588, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 216, 756, 2473, 60, 281, 60, 363, 111, 2057, 0, 0, 0, 0, 0, 0, 0, 433, 812, 57, 925, 23, 203, 1705, 715, 411, 0, 0, 0, 0, 0, 0, 0, 95, 125, 897, 3822, 206, 3962, 84, 221, 135, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4691, 6814, 7626, 4601, 6377, 5284, 9547] avg=6144.0000 max=9547 max_rank=7 min=4212 min_rank=0 imbalance_ratio=1.553874 max_load_reduction=39605 max_load_reduction_pct=0.805766 moved_to_echo=25809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[736, 1193, 0, 1991, 606, 175, 0, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 554, 1124, 525, 781, 399, 668, 1633, 132, 680, 0, 0, 0, 0, 0, 0, 0, 10, 581, 541, 123, 894, 967, 296, 93, 98, 88, 39, 0, 0, 0, 0, 0, 745, 35, 141, 634, 618, 662, 36, 395, 754, 893, 0, 0, 0, 0, 0, 0, 1358, 675, 165, 51, 698, 2684, 84, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 276, 195, 208, 809, 341, 142, 1269, 482, 1634, 0, 0, 0, 0, 0, 0, 0, 2229, 675, 112, 347, 369, 193, 2018, 450, 1120, 0, 0, 0, 0, 0, 0, 0, 53, 672, 384, 3655, 359, 826, 477, 429, 368, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6893, 6496, 3730, 4913, 7028, 5356, 7513, 7223] avg=6144.0000 max=7513 max_rank=6 min=3730 min_rank=2 imbalance_ratio=1.222819 max_load_reduction=41639 max_load_reduction_pct=0.847148 moved_to_echo=27120 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[692, 1924, 0, 2213, 1068, 1224, 0, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 528, 3528, 448, 176, 55, 229, 175, 431, 572, 0, 0, 0, 0, 0, 0, 0, 104, 350, 1040, 448, 995, 225, 1824, 165, 109, 75, 33, 0, 0, 0, 0, 0, 928, 17, 300, 1069, 555, 252, 147, 569, 915, 750, 0, 0, 0, 0, 0, 0, 1506, 1993, 178, 165, 39, 2126, 628, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 135, 125, 1261, 20, 608, 660, 1112, 50, 1981, 0, 0, 0, 0, 0, 0, 0, 675, 893, 366, 196, 194, 16, 947, 559, 939, 0, 0, 0, 0, 0, 0, 0, 59, 42, 409, 820, 181, 1203, 116, 408, 309, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10106, 6142, 5368, 5502, 7750, 5952, 4785, 3547] avg=6144.0000 max=10106 max_rank=0 min=3547 min_rank=7 imbalance_ratio=1.644857 max_load_reduction=39046 max_load_reduction_pct=0.794393 moved_to_echo=22034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[574, 1433, 0, 807, 698, 161, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 724, 3060, 611, 72, 485, 357, 888, 931, 819, 0, 0, 0, 0, 0, 0, 0, 279, 482, 321, 335, 1224, 1651, 606, 660, 39, 109, 27, 0, 0, 0, 0, 0, 289, 113, 208, 600, 1217, 488, 120, 1235, 1236, 1076, 0, 0, 0, 0, 0, 0, 1767, 1775, 52, 261, 124, 755, 1035, 914, 0, 0, 0, 0, 0, 0, 0, 0, 266, 90, 953, 266, 213, 257, 1086, 111, 2676, 0, 0, 0, 0, 0, 0, 0, 1111, 928, 58, 592, 147, 96, 1082, 1293, 1346, 0, 0, 0, 0, 0, 0, 0, 67, 27, 1738, 1569, 55, 848, 410, 146, 443, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4333, 7947, 5733, 6582, 6683, 5918, 6653, 5303] avg=6144.0000 max=7947 max_rank=1 min=4333 min_rank=0 imbalance_ratio=1.293457 max_load_reduction=41205 max_load_reduction_pct=0.838318 moved_to_echo=24557 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[703, 1806, 0, 2477, 534, 994, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1332, 605, 190, 374, 720, 233, 610, 543, 0, 0, 0, 0, 0, 0, 0, 31, 353, 611, 729, 638, 1423, 2334, 44, 122, 70, 73, 0, 0, 0, 0, 0, 464, 113, 171, 2350, 283, 100, 64, 1379, 317, 714, 0, 0, 0, 0, 0, 0, 1112, 666, 93, 253, 97, 816, 261, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 575, 447, 3244, 101, 731, 390, 830, 338, 687, 0, 0, 0, 0, 0, 0, 0, 554, 585, 20, 1282, 204, 19, 998, 413, 894, 0, 0, 0, 0, 0, 0, 0, 14, 99, 454, 3969, 243, 1422, 189, 163, 297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7191, 4653, 6428, 5955, 5763, 7343, 4969, 6850] avg=6144.0000 max=7343 max_rank=5 min=4653 min_rank=1 imbalance_ratio=1.195150 max_load_reduction=41809 max_load_reduction_pct=0.850606 moved_to_echo=24925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[163, 617, 0, 1875, 314, 723, 0, 520, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1296, 106, 89, 631, 730, 1155, 402, 250, 0, 0, 0, 0, 0, 0, 0, 766, 314, 1515, 375, 1293, 742, 1577, 56, 92, 33, 51, 0, 0, 0, 0, 0, 171, 90, 95, 1492, 1497, 322, 1489, 1191, 950, 329, 0, 0, 0, 0, 0, 0, 585, 746, 55, 168, 181, 559, 588, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 216, 756, 2473, 60, 281, 60, 363, 111, 2057, 0, 0, 0, 0, 0, 0, 0, 433, 812, 57, 925, 23, 203, 1705, 715, 411, 0, 0, 0, 0, 0, 0, 0, 95, 125, 897, 3822, 206, 3962, 84, 221, 135, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4691, 6814, 7626, 4601, 6377, 5284, 9547] avg=6144.0000 max=9547 max_rank=7 min=4212 min_rank=0 imbalance_ratio=1.553874 max_load_reduction=39605 max_load_reduction_pct=0.805766 moved_to_echo=25809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[710, 1287, 0, 316, 1290, 400, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1484, 225, 137, 311, 734, 1043, 689, 438, 0, 0, 0, 0, 0, 0, 0, 38, 490, 756, 462, 2356, 2379, 2376, 123, 19, 56, 35, 0, 0, 0, 0, 0, 221, 20, 143, 1488, 208, 208, 957, 971, 794, 572, 0, 0, 0, 0, 0, 0, 1934, 955, 5, 534, 182, 1554, 57, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 141, 216, 2033, 2, 717, 308, 2045, 63, 1718, 0, 0, 0, 0, 0, 0, 0, 328, 538, 65, 663, 12, 71, 1440, 874, 716, 0, 0, 0, 0, 0, 0, 0, 32, 25, 619, 3667, 712, 306, 319, 638, 235, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4581, 5073, 9090, 5582, 6323, 7243, 4707, 6553] avg=6144.0000 max=9090 max_rank=2 min=4581 min_rank=0 imbalance_ratio=1.479492 max_load_reduction=40062 max_load_reduction_pct=0.815063 moved_to_echo=24826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2176, 2944, 9088, 2816, 8320, 9216, 11520, 2304, 640, 640, 384, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2048, 18432, 3840, 1536, 2816, 4736, 7296, 4864, 3968, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6528, 7552, 1024, 5632, 1280, 768, 11776, 8576, 6528, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10752, 7936, 768, 2688, 1664, 10496, 3200, 12032, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4224, 640, 1408, 10624, 5888, 3456, 3584, 8192, 6656, 5248, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4480, 11776, 0, 12672, 5888, 4096, 0, 10624, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1024, 1152, 6016, 18944, 2816, 12544, 2944, 2304, 2176, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 2560, 12672, 1920, 3968, 2176, 8448, 1408, 14336, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4224, 640, 1408, 10624, 5888, 3456, 3584, 8192, 6656, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10752, 7936, 768, 2688, 1664, 10496, 3200, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1024, 1152, 6016, 18944, 2816, 12544, 2944, 2304, 2176, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[1736, 63, 322, 87, 23, 57, 905, 443, 131, 508, 1428, 558, 2103, 1726, 1685, 1735, 122, 1583, 46, 166, 656, 688, 178, 1771, 1723, 189, 948, 19, 412, 724, 294, 650, 722, 302, 1594, 492, 1949, 639, 2101, 288, 225, 135, 52, 77, 1121, 527, 616, 5612, 172, 125, 1599, 1573, 293, 319, 236, 810, 273, 1123, 640, 939, 131, 191, 154, 443] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[1983, 731, 131, 891, 571, 12, 448, 377, 37, 1918, 1013, 56, 680, 3938, 1906, 518, 583, 1633, 54, 335, 2030, 822, 262, 831, 1459, 345, 666, 719, 1861, 236, 522, 265, 534, 74, 682, 1245, 640, 479, 1007, 678, 233, 970, 850, 64, 1536, 2058, 343, 1207, 320, 164, 1769, 532, 744, 455, 34, 700, 355, 926, 121, 157, 40, 644, 65, 1693] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[1679, 726, 729, 495, 223, 70, 311, 220, 116, 874, 1401, 189, 2479, 1368, 2026, 2019, 741, 804, 327, 1132, 2372, 392, 629, 2392, 829, 396, 1394, 706, 888, 432, 140, 922, 763, 501, 1085, 1062, 195, 957, 1353, 1363, 506, 582, 122, 502, 1502, 1968, 476, 211, 96, 728, 772, 701, 518, 630, 31, 344, 496, 293, 266, 154, 144, 248, 205, 957] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[875, 454, 288, 710, 270, 11, 618, 1833, 116, 640, 2240, 229, 2211, 3733, 449, 4019, 97, 747, 285, 243, 1458, 30, 297, 929, 311, 517, 1161, 1093, 440, 59, 378, 16, 609, 183, 553, 2103, 163, 273, 1438, 912, 9, 192, 159, 912, 868, 42, 1139, 1510, 1259, 28, 347, 781, 376, 235, 66, 2580, 554, 611, 2402, 121, 7, 300, 1448, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[1848, 449, 302, 75, 582, 67, 1511, 1935, 345, 2433, 1104, 251, 1456, 4609, 989, 502, 500, 850, 252, 107, 1227, 230, 409, 2163, 1016, 399, 1382, 88, 1174, 32, 28, 68, 1127, 154, 749, 1203, 609, 1389, 1037, 188, 477, 488, 442, 37, 2752, 224, 462, 579, 785, 77, 591, 1666, 444, 123, 487, 977, 731, 611, 422, 496, 339, 74, 34, 995] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[906, 182, 111, 90, 44, 115, 161, 2373, 493, 460, 1483, 8, 1009, 1779, 3004, 3124, 86, 427, 236, 1916, 1584, 17, 278, 2260, 201, 571, 2773, 857, 499, 64, 421, 102, 334, 415, 1702, 1981, 479, 1045, 1743, 1586, 7, 781, 91, 810, 1303, 342, 87, 621, 768, 71, 441, 181, 327, 84, 54, 671, 209, 982, 2680, 944, 8, 128, 271, 372] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[829, 2123, 897, 33, 72, 41, 1097, 1477, 514, 555, 525, 724, 2052, 1684, 1298, 375, 39, 2158, 25, 318, 863, 387, 362, 823, 1456, 360, 1197, 294, 848, 88, 376, 410, 618, 152, 497, 737, 940, 2834, 1375, 2883, 166, 323, 82, 317, 1786, 308, 224, 1312, 862, 40, 2230, 655, 468, 490, 16, 938, 719, 821, 634, 4, 438, 727, 398, 858] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[875, 454, 288, 710, 270, 11, 618, 1833, 116, 640, 2240, 229, 2211, 3733, 449, 4019, 97, 747, 285, 243, 1458, 30, 297, 929, 311, 517, 1161, 1093, 440, 59, 378, 16, 609, 183, 553, 2103, 163, 273, 1438, 912, 9, 192, 159, 912, 868, 42, 1139, 1510, 1259, 28, 347, 781, 376, 235, 66, 2580, 554, 611, 2402, 121, 7, 300, 1448, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[1720, 352, 25, 106, 287, 127, 1787, 3373, 144, 1252, 1896, 505, 670, 1602, 1598, 860, 132, 272, 178, 230, 1996, 319, 324, 1576, 770, 570, 1792, 37, 1004, 82, 103, 475, 2808, 329, 2518, 679, 679, 744, 685, 727, 29, 121, 361, 184, 1230, 1392, 23, 962, 372, 252, 794, 1666, 206, 94, 725, 1296, 447, 1406, 885, 433, 37, 142, 81, 681] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1679, 726, 729, 495, 223, 70, 311, 220, 726, 0, 0, 0, 0, 0, 0, 0, 116, 874, 1401, 189, 2479, 0, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 741, 804, 327, 1132, 2372, 392, 629, 2392, 152, 95, 0, 0, 0, 0, 0, 0, 829, 396, 1394, 706, 888, 432, 140, 922, 1197, 138, 0, 0, 0, 0, 0, 0, 763, 501, 1085, 840, 195, 957, 0, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 506, 582, 122, 502, 1502, 1968, 476, 211, 489, 70, 0, 0, 0, 0, 0, 0, 96, 728, 772, 701, 518, 630, 31, 344, 375, 822, 0, 0, 0, 0, 0, 0, 496, 293, 266, 154, 144, 248, 205, 957, 993, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5179, 6990, 9036, 7042, 5704, 6428, 5017, 3756] avg=6144.0000 max=9036 max_rank=2 min=3756 min_rank=7 imbalance_ratio=1.470703 max_load_reduction=40116 max_load_reduction_pct=0.816162 moved_to_echo=20905 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[829, 2123, 897, 33, 72, 41, 1097, 1477, 514, 555, 525, 724, 2052, 1684, 1298, 375, 39, 2158, 25, 318, 863, 387, 362, 823, 1456, 360, 1197, 294, 848, 88, 376, 410, 618, 152, 497, 737, 940, 2834, 1375, 2883, 166, 323, 82, 317, 1786, 308, 224, 1312, 862, 40, 2230, 655, 468, 490, 16, 938, 719, 821, 634, 4, 438, 727, 398, 858] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[1679, 726, 729, 495, 223, 70, 311, 220, 116, 874, 1401, 189, 2479, 1368, 2026, 2019, 741, 804, 327, 1132, 2372, 392, 629, 2392, 829, 396, 1394, 706, 888, 432, 140, 922, 763, 501, 1085, 1062, 195, 957, 1353, 1363, 506, 582, 122, 502, 1502, 1968, 476, 211, 96, 728, 772, 701, 518, 630, 31, 344, 496, 293, 266, 154, 144, 248, 205, 957] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[1848, 449, 302, 75, 582, 67, 1511, 1935, 345, 2433, 1104, 251, 1456, 4609, 989, 502, 500, 850, 252, 107, 1227, 230, 409, 2163, 1016, 399, 1382, 88, 1174, 32, 28, 68, 1127, 154, 749, 1203, 609, 1389, 1037, 188, 477, 488, 442, 37, 2752, 224, 462, 579, 785, 77, 591, 1666, 444, 123, 487, 977, 731, 611, 422, 496, 339, 74, 34, 995] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1736, 63, 322, 87, 23, 57, 905, 443, 1127, 0, 0, 0, 0, 0, 0, 0, 131, 508, 1428, 558, 2103, 0, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 1583, 46, 166, 656, 688, 178, 1771, 70, 79, 0, 0, 0, 0, 0, 0, 1723, 189, 948, 19, 412, 724, 294, 650, 1028, 217, 0, 0, 0, 0, 0, 0, 722, 302, 1594, 390, 1949, 639, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 225, 135, 52, 77, 1121, 527, 616, 5612, 757, 32, 0, 0, 0, 0, 0, 0, 172, 125, 1599, 1573, 293, 319, 236, 810, 473, 707, 0, 0, 0, 0, 0, 0, 273, 1123, 640, 939, 131, 191, 154, 443, 1253, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4763, 6334, 5359, 6204, 5884, 9154, 6307, 5147] avg=6144.0000 max=9154 max_rank=5 min=4763 min_rank=0 imbalance_ratio=1.489909 max_load_reduction=39998 max_load_reduction_pct=0.813761 moved_to_echo=26492 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1720, 352, 25, 106, 287, 127, 1787, 3373, 367, 0, 0, 0, 0, 0, 0, 0, 144, 1252, 1896, 505, 670, 0, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 272, 178, 230, 1996, 319, 324, 1576, 97, 75, 0, 0, 0, 0, 0, 0, 770, 570, 1792, 37, 1004, 82, 103, 475, 509, 70, 0, 0, 0, 0, 0, 0, 2808, 329, 2518, 537, 679, 744, 0, 727, 0, 0, 0, 0, 0, 0, 0, 0, 29, 121, 361, 184, 1230, 1392, 23, 962, 248, 45, 0, 0, 0, 0, 0, 0, 372, 252, 794, 1666, 206, 94, 725, 1296, 439, 351, 0, 0, 0, 0, 0, 0, 447, 1406, 885, 433, 37, 142, 81, 681, 1163, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 5990, 5199, 5412, 8342, 4595, 6195, 5275] avg=6144.0000 max=8342 max_rank=4 min=4595 min_rank=5 imbalance_ratio=1.357747 max_load_reduction=40810 max_load_reduction_pct=0.830282 moved_to_echo=24407 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[875, 454, 288, 710, 270, 11, 618, 1833, 771, 0, 0, 0, 0, 0, 0, 0, 116, 640, 2240, 229, 2211, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 747, 285, 243, 1458, 30, 297, 929, 301, 21, 0, 0, 0, 0, 0, 0, 311, 517, 1161, 1093, 440, 59, 378, 16, 2382, 147, 0, 0, 0, 0, 0, 0, 609, 183, 553, 1663, 163, 273, 0, 912, 0, 0, 0, 0, 0, 0, 0, 0, 9, 192, 159, 912, 868, 42, 1139, 1510, 520, 139, 0, 0, 0, 0, 0, 0, 1259, 28, 347, 781, 376, 235, 66, 2580, 1024, 1637, 0, 0, 0, 0, 0, 0, 554, 611, 2402, 121, 7, 300, 1448, 215, 2709, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5830, 5864, 4408, 6504, 4356, 5490, 8333, 8367] avg=6144.0000 max=8367 max_rank=7 min=4356 min_rank=4 imbalance_ratio=1.361816 max_load_reduction=40785 max_load_reduction_pct=0.829773 moved_to_echo=26546 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1848, 449, 302, 75, 582, 67, 1511, 1935, 558, 0, 0, 0, 0, 0, 0, 0, 345, 2433, 1104, 251, 1456, 0, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 850, 252, 107, 1227, 230, 409, 2163, 172, 46, 0, 0, 0, 0, 0, 0, 1016, 399, 1382, 88, 1174, 32, 28, 68, 298, 106, 0, 0, 0, 0, 0, 0, 1127, 154, 749, 952, 609, 1389, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 477, 488, 442, 37, 2752, 224, 462, 579, 373, 79, 0, 0, 0, 0, 0, 0, 785, 77, 591, 1666, 444, 123, 487, 977, 1266, 204, 0, 0, 0, 0, 0, 0, 731, 611, 422, 496, 339, 74, 34, 995, 3343, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7327, 6532, 5956, 4591, 5168, 5913, 6620, 7045] avg=6144.0000 max=7327 max_rank=0 min=4591 min_rank=3 imbalance_ratio=1.192546 max_load_reduction=41825 max_load_reduction_pct=0.850932 moved_to_echo=24746 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1983, 731, 131, 891, 571, 12, 448, 377, 541, 0, 0, 0, 0, 0, 0, 0, 37, 1918, 1013, 56, 680, 0, 1815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 1633, 54, 335, 2030, 822, 262, 831, 183, 91, 0, 0, 0, 0, 0, 0, 1459, 345, 666, 719, 1861, 236, 522, 265, 308, 103, 0, 0, 0, 0, 0, 0, 534, 74, 682, 976, 640, 479, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 233, 970, 850, 64, 1536, 2058, 343, 1207, 363, 86, 0, 0, 0, 0, 0, 0, 320, 164, 1769, 532, 744, 455, 34, 700, 1081, 210, 0, 0, 0, 0, 0, 0, 355, 926, 121, 157, 40, 644, 65, 1693, 2857, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5685, 5519, 6824, 6484, 4063, 7710, 6009, 6858] avg=6144.0000 max=7710 max_rank=5 min=4063 min_rank=4 imbalance_ratio=1.254883 max_load_reduction=41442 max_load_reduction_pct=0.843140 moved_to_echo=24640 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[906, 182, 111, 90, 44, 115, 161, 2373, 935, 0, 0, 0, 0, 0, 0, 0, 493, 460, 1483, 8, 1009, 0, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 427, 236, 1916, 1584, 17, 278, 2260, 284, 141, 0, 0, 0, 0, 0, 0, 201, 571, 2773, 857, 499, 64, 421, 102, 1853, 180, 0, 0, 0, 0, 0, 0, 334, 415, 1702, 1566, 479, 1045, 0, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 7, 781, 91, 810, 1303, 342, 87, 621, 628, 131, 0, 0, 0, 0, 0, 0, 768, 71, 441, 181, 327, 84, 54, 671, 489, 1271, 0, 0, 0, 0, 0, 0, 209, 982, 2680, 944, 8, 128, 271, 372, 1290, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4917, 6316, 7229, 7521, 7127, 4801, 4357, 6884] avg=6144.0000 max=7521 max_rank=3 min=4357 min_rank=6 imbalance_ratio=1.224121 max_load_reduction=41631 max_load_reduction_pct=0.846985 moved_to_echo=23169 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1720, 352, 25, 106, 287, 127, 1787, 3373, 367, 0, 0, 0, 0, 0, 0, 0, 144, 1252, 1896, 505, 670, 0, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 272, 178, 230, 1996, 319, 324, 1576, 97, 75, 0, 0, 0, 0, 0, 0, 770, 570, 1792, 37, 1004, 82, 103, 475, 509, 70, 0, 0, 0, 0, 0, 0, 2808, 329, 2518, 537, 679, 744, 0, 727, 0, 0, 0, 0, 0, 0, 0, 0, 29, 121, 361, 184, 1230, 1392, 23, 962, 248, 45, 0, 0, 0, 0, 0, 0, 372, 252, 794, 1666, 206, 94, 725, 1296, 439, 351, 0, 0, 0, 0, 0, 0, 447, 1406, 885, 433, 37, 142, 81, 681, 1163, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 5990, 5199, 5412, 8342, 4595, 6195, 5275] avg=6144.0000 max=8342 max_rank=4 min=4595 min_rank=5 imbalance_ratio=1.357747 max_load_reduction=40810 max_load_reduction_pct=0.830282 moved_to_echo=24407 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[829, 2123, 897, 33, 72, 41, 1097, 1477, 738, 0, 0, 0, 0, 0, 0, 0, 514, 555, 525, 724, 2052, 0, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 2158, 25, 318, 863, 387, 362, 823, 105, 61, 0, 0, 0, 0, 0, 0, 1456, 360, 1197, 294, 848, 88, 376, 410, 223, 142, 0, 0, 0, 0, 0, 0, 618, 152, 497, 584, 940, 2834, 0, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 166, 323, 82, 317, 1786, 308, 224, 1312, 495, 48, 0, 0, 0, 0, 0, 0, 862, 40, 2230, 655, 468, 490, 16, 938, 463, 152, 0, 0, 0, 0, 0, 0, 719, 821, 634, 4, 438, 727, 398, 858, 1221, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7307, 5607, 5141, 5394, 8508, 5061, 6314, 5820] avg=6144.0000 max=8508 max_rank=4 min=5061 min_rank=5 imbalance_ratio=1.384766 max_load_reduction=40644 max_load_reduction_pct=0.826904 moved_to_echo=25703 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1983, 731, 131, 891, 571, 12, 448, 377, 541, 0, 0, 0, 0, 0, 0, 0, 37, 1918, 1013, 56, 680, 0, 1815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 1633, 54, 335, 2030, 822, 262, 831, 183, 91, 0, 0, 0, 0, 0, 0, 1459, 345, 666, 719, 1861, 236, 522, 265, 308, 103, 0, 0, 0, 0, 0, 0, 534, 74, 682, 976, 640, 479, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 233, 970, 850, 64, 1536, 2058, 343, 1207, 363, 86, 0, 0, 0, 0, 0, 0, 320, 164, 1769, 532, 744, 455, 34, 700, 1081, 210, 0, 0, 0, 0, 0, 0, 355, 926, 121, 157, 40, 644, 65, 1693, 2857, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5685, 5519, 6824, 6484, 4063, 7710, 6009, 6858] avg=6144.0000 max=7710 max_rank=5 min=4063 min_rank=4 imbalance_ratio=1.254883 max_load_reduction=41442 max_load_reduction_pct=0.843140 moved_to_echo=24640 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1679, 726, 729, 495, 223, 70, 311, 220, 726, 0, 0, 0, 0, 0, 0, 0, 116, 874, 1401, 189, 2479, 0, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 741, 804, 327, 1132, 2372, 392, 629, 2392, 152, 95, 0, 0, 0, 0, 0, 0, 829, 396, 1394, 706, 888, 432, 140, 922, 1197, 138, 0, 0, 0, 0, 0, 0, 763, 501, 1085, 840, 195, 957, 0, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 506, 582, 122, 502, 1502, 1968, 476, 211, 489, 70, 0, 0, 0, 0, 0, 0, 96, 728, 772, 701, 518, 630, 31, 344, 375, 822, 0, 0, 0, 0, 0, 0, 496, 293, 266, 154, 144, 248, 205, 957, 993, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5179, 6990, 9036, 7042, 5704, 6428, 5017, 3756] avg=6144.0000 max=9036 max_rank=2 min=3756 min_rank=7 imbalance_ratio=1.470703 max_load_reduction=40116 max_load_reduction_pct=0.816162 moved_to_echo=20905 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1848, 449, 302, 75, 582, 67, 1511, 1935, 558, 0, 0, 0, 0, 0, 0, 0, 345, 2433, 1104, 251, 1456, 0, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 850, 252, 107, 1227, 230, 409, 2163, 172, 46, 0, 0, 0, 0, 0, 0, 1016, 399, 1382, 88, 1174, 32, 28, 68, 298, 106, 0, 0, 0, 0, 0, 0, 1127, 154, 749, 952, 609, 1389, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 477, 488, 442, 37, 2752, 224, 462, 579, 373, 79, 0, 0, 0, 0, 0, 0, 785, 77, 591, 1666, 444, 123, 487, 977, 1266, 204, 0, 0, 0, 0, 0, 0, 731, 611, 422, 496, 339, 74, 34, 995, 3343, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7327, 6532, 5956, 4591, 5168, 5913, 6620, 7045] avg=6144.0000 max=7327 max_rank=0 min=4591 min_rank=3 imbalance_ratio=1.192546 max_load_reduction=41825 max_load_reduction_pct=0.850932 moved_to_echo=24746 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7808, 3456, 11392, 3840, 7168, 1792, 2304, 2944, 7808, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2304, 8576, 1408, 4480, 12288, 2944, 2816, 12800, 1408, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 3712, 2176, 2944, 12160, 6912, 3456, 12032, 3968, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7552, 2176, 9472, 7552, 5760, 8448, 0, 8704, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3840, 6784, 8064, 3328, 1152, 2560, 2688, 6272, 14848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 3712, 2176, 2944, 12160, 6912, 3456, 12032, 3968, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7808, 3456, 11392, 3840, 7168, 1792, 2304, 2944, 7808, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2304, 8576, 1408, 4480, 12288, 2944, 2816, 12800, 1408, 640, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[1920, 8704, 11136, 2560, 12672, 0, 12416, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11648, 5120, 2816, 2560, 2176, 512, 6912, 12032, 5888, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4736, 1536, 8576, 7808, 3456, 2432, 1664, 8320, 5632, 5376, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11648, 5120, 2816, 2560, 2176, 512, 6912, 12032, 5888, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[806, 359, 56, 1455, 966, 826, 483, 145, 75, 502, 313, 459, 529, 161, 58, 2345, 939, 462, 2652, 483, 342, 210, 143, 275, 378, 567, 1420, 847, 1592, 120, 63, 776, 648, 591, 233, 89, 109, 320, 561, 1540, 150, 2750, 248, 303, 1791, 475, 784, 480, 1159, 1043, 887, 1068, 301, 795, 3300, 1343, 275, 1860, 1818, 768, 686, 734, 1010, 226] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[1179, 951, 653, 320, 141, 272, 344, 141, 50, 381, 100, 249, 658, 51, 498, 1393, 775, 630, 215, 588, 1364, 91, 253, 1279, 24, 1935, 824, 738, 259, 1159, 222, 157, 298, 592, 71, 663, 882, 194, 1489, 2736, 470, 313, 166, 972, 654, 38, 2610, 89, 853, 563, 1037, 1434, 593, 2151, 3428, 1604, 938, 557, 2714, 1117, 306, 1286, 215, 225] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[718, 386, 120, 371, 1218, 336, 466, 106, 699, 55, 490, 318, 1653, 43, 50, 2150, 355, 395, 979, 105, 1217, 583, 631, 1257, 27, 1395, 342, 2416, 2744, 668, 546, 358, 1636, 1123, 1852, 694, 1329, 768, 324, 566, 251, 1751, 214, 337, 651, 312, 160, 163, 295, 750, 1076, 1568, 1557, 602, 975, 1782, 166, 262, 1979, 877, 310, 640, 767, 218] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[550, 714, 733, 297, 2090, 196, 324, 319, 264, 216, 59, 237, 2622, 36, 240, 2549, 548, 888, 968, 54, 2340, 346, 103, 1792, 111, 1481, 1020, 1547, 884, 211, 12, 547, 602, 288, 370, 340, 511, 423, 1333, 386, 6, 963, 736, 49, 2373, 291, 417, 302, 1535, 310, 1009, 746, 1432, 2520, 1207, 1647, 457, 336, 778, 2229, 319, 521, 201, 217] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[1551, 77, 5, 320, 1445, 933, 80, 99, 946, 1210, 60, 42, 1311, 5, 1613, 1480, 662, 155, 623, 398, 1128, 437, 206, 400, 215, 3737, 583, 525, 1738, 140, 884, 1869, 950, 204, 2313, 297, 780, 881, 1780, 480, 203, 149, 551, 136, 1622, 26, 196, 598, 815, 485, 1813, 2186, 179, 330, 189, 1069, 807, 1157, 904, 986, 168, 286, 959, 776] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[738, 109, 593, 801, 2930, 633, 1017, 51, 424, 591, 240, 337, 1164, 36, 127, 1564, 260, 788, 3328, 221, 519, 98, 291, 28, 66, 2195, 533, 1703, 338, 945, 57, 34, 788, 1175, 2169, 1078, 237, 138, 1131, 1193, 5, 725, 753, 274, 1067, 34, 925, 334, 1012, 72, 1557, 418, 54, 2084, 3219, 641, 160, 754, 2164, 319, 656, 361, 849, 47] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[806, 359, 56, 1455, 966, 826, 483, 145, 75, 502, 313, 459, 529, 161, 58, 2345, 939, 462, 2652, 483, 342, 210, 143, 275, 378, 567, 1420, 847, 1592, 120, 63, 776, 648, 591, 233, 89, 109, 320, 561, 1540, 150, 2750, 248, 303, 1791, 475, 784, 480, 1159, 1043, 887, 1068, 301, 795, 3300, 1343, 275, 1860, 1818, 768, 686, 734, 1010, 226] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[2014, 200, 169, 568, 2433, 362, 207, 114, 116, 520, 119, 259, 707, 250, 825, 472, 451, 363, 280, 322, 2524, 84, 230, 807, 11, 553, 404, 794, 517, 748, 895, 50, 292, 575, 517, 450, 119, 237, 2102, 639, 800, 275, 1146, 1010, 1137, 274, 1549, 2484, 1844, 1085, 715, 754, 434, 1742, 1065, 1606, 59, 124, 3653, 681, 506, 1564, 1079, 267] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[550, 714, 733, 297, 2090, 196, 324, 319, 264, 216, 59, 237, 2622, 36, 240, 2549, 548, 888, 968, 54, 2340, 346, 103, 1792, 111, 1481, 1020, 1547, 884, 211, 12, 547, 602, 288, 370, 340, 511, 423, 1333, 386, 6, 963, 736, 49, 2373, 291, 417, 302, 1535, 310, 1009, 746, 1432, 2520, 1207, 1647, 457, 336, 778, 2229, 319, 521, 201, 217] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[2452, 1395, 94, 796, 1137, 2626, 707, 702, 228, 48, 41, 485, 69, 208, 185, 548, 434, 176, 684, 826, 1250, 55, 479, 139, 824, 2878, 2633, 1019, 197, 55, 819, 788, 473, 245, 396, 274, 143, 1257, 1020, 1409, 293, 97, 326, 66, 503, 61, 101, 173, 786, 252, 2477, 760, 199, 387, 1251, 2370, 77, 2261, 2806, 616, 587, 633, 1660, 216] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[806, 359, 56, 1455, 966, 826, 483, 145, 389, 8, 0, 0, 0, 0, 0, 0, 75, 502, 313, 459, 529, 161, 58, 2345, 3026, 0, 0, 0, 0, 0, 0, 0, 939, 462, 2652, 483, 342, 210, 143, 275, 191, 206, 43, 0, 0, 0, 0, 0, 378, 376, 1420, 847, 1592, 120, 63, 776, 0, 0, 0, 0, 0, 0, 0, 0, 648, 591, 233, 89, 109, 320, 553, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 150, 2750, 248, 303, 1791, 475, 784, 480, 977, 274, 0, 0, 0, 0, 0, 0, 1159, 1043, 887, 1068, 301, 795, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 275, 1860, 1386, 768, 686, 734, 1010, 226, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5493, 7468, 5946, 5572, 4083, 8232, 5413, 6945] avg=6144.0000 max=8232 max_rank=5 min=4083 min_rank=4 imbalance_ratio=1.339844 max_load_reduction=40920 max_load_reduction_pct=0.832520 moved_to_echo=24673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[738, 109, 593, 801, 2930, 633, 1017, 51, 424, 591, 240, 337, 1164, 36, 127, 1564, 260, 788, 3328, 221, 519, 98, 291, 28, 66, 2195, 533, 1703, 338, 945, 57, 34, 788, 1175, 2169, 1078, 237, 138, 1131, 1193, 5, 725, 753, 274, 1067, 34, 925, 334, 1012, 72, 1557, 418, 54, 2084, 3219, 641, 160, 754, 2164, 319, 656, 361, 849, 47] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[1179, 951, 653, 320, 141, 272, 344, 141, 50, 381, 100, 249, 658, 51, 498, 1393, 775, 630, 215, 588, 1364, 91, 253, 1279, 24, 1935, 824, 738, 259, 1159, 222, 157, 298, 592, 71, 663, 882, 194, 1489, 2736, 470, 313, 166, 972, 654, 38, 2610, 89, 853, 563, 1037, 1434, 593, 2151, 3428, 1604, 938, 557, 2714, 1117, 306, 1286, 215, 225] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1551, 77, 5, 320, 1445, 933, 80, 99, 193, 27, 0, 0, 0, 0, 0, 0, 946, 1210, 60, 42, 1311, 5, 1613, 1480, 173, 0, 0, 0, 0, 0, 0, 0, 662, 155, 623, 398, 1128, 437, 206, 400, 1263, 164, 21, 0, 0, 0, 0, 0, 215, 2474, 583, 525, 1738, 140, 884, 1869, 0, 0, 0, 0, 0, 0, 0, 0, 950, 204, 2313, 297, 780, 881, 1753, 480, 0, 0, 0, 0, 0, 0, 0, 0, 203, 149, 551, 136, 1622, 26, 196, 598, 777, 16, 0, 0, 0, 0, 0, 0, 815, 485, 1813, 2186, 179, 330, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 807, 1157, 690, 986, 168, 286, 959, 776, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4730, 6840, 5457, 8428, 7658, 4274, 5936, 5829] avg=6144.0000 max=8428 max_rank=3 min=4274 min_rank=5 imbalance_ratio=1.371745 max_load_reduction=40724 max_load_reduction_pct=0.828532 moved_to_echo=23697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1179, 951, 653, 320, 141, 272, 344, 141, 581, 23, 0, 0, 0, 0, 0, 0, 50, 381, 100, 249, 658, 51, 498, 1393, 3143, 0, 0, 0, 0, 0, 0, 0, 775, 630, 215, 588, 1364, 91, 253, 1279, 654, 247, 64, 0, 0, 0, 0, 0, 24, 1281, 824, 738, 259, 1159, 222, 157, 0, 0, 0, 0, 0, 0, 0, 0, 298, 592, 71, 663, 882, 194, 1466, 2736, 0, 0, 0, 0, 0, 0, 0, 0, 470, 313, 166, 972, 654, 38, 2610, 89, 1166, 285, 0, 0, 0, 0, 0, 0, 853, 563, 1037, 1434, 593, 2151, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 938, 557, 2069, 1117, 306, 1286, 215, 225, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4605, 6523, 6160, 4664, 6902, 6763, 6822, 6713] avg=6144.0000 max=6902 max_rank=4 min=4605 min_rank=0 imbalance_ratio=1.123372 max_load_reduction=42250 max_load_reduction_pct=0.859578 moved_to_echo=27200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[718, 386, 120, 371, 1218, 336, 466, 106, 423, 5, 0, 0, 0, 0, 0, 0, 699, 55, 490, 318, 1653, 43, 50, 2150, 895, 0, 0, 0, 0, 0, 0, 0, 355, 395, 979, 105, 1217, 583, 631, 1257, 471, 274, 47, 0, 0, 0, 0, 0, 27, 924, 342, 2416, 2744, 668, 546, 358, 0, 0, 0, 0, 0, 0, 0, 0, 1636, 1123, 1852, 694, 1329, 768, 319, 566, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1751, 214, 337, 651, 312, 160, 163, 1296, 80, 0, 0, 0, 0, 0, 0, 295, 750, 1076, 1568, 1557, 602, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 166, 262, 1509, 877, 310, 640, 767, 218, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 6353, 6314, 8025, 8287, 5215, 6060, 4749] avg=6144.0000 max=8287 max_rank=4 min=4149 min_rank=0 imbalance_ratio=1.348796 max_load_reduction=40865 max_load_reduction_pct=0.831401 moved_to_echo=24311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2014, 200, 169, 568, 2433, 362, 207, 114, 782, 32, 0, 0, 0, 0, 0, 0, 116, 520, 119, 259, 707, 250, 825, 472, 976, 0, 0, 0, 0, 0, 0, 0, 451, 363, 280, 322, 2524, 84, 230, 807, 187, 247, 86, 0, 0, 0, 0, 0, 11, 366, 404, 794, 517, 748, 895, 50, 0, 0, 0, 0, 0, 0, 0, 0, 292, 575, 517, 450, 119, 237, 2070, 639, 0, 0, 0, 0, 0, 0, 0, 0, 800, 275, 1146, 1010, 1137, 274, 1549, 2484, 1168, 89, 0, 0, 0, 0, 0, 0, 1844, 1085, 715, 754, 434, 1742, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 59, 124, 2785, 681, 506, 1564, 1079, 267, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6881, 4244, 5581, 3785, 4899, 9932, 6765, 7065] avg=6144.0000 max=9932 max_rank=5 min=3785 min_rank=3 imbalance_ratio=1.616536 max_load_reduction=39220 max_load_reduction_pct=0.797933 moved_to_echo=28661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[550, 714, 733, 297, 2090, 196, 324, 319, 166, 20, 0, 0, 0, 0, 0, 0, 264, 216, 59, 237, 2622, 36, 240, 2549, 1106, 0, 0, 0, 0, 0, 0, 0, 548, 888, 968, 54, 2340, 346, 103, 1792, 500, 253, 18, 0, 0, 0, 0, 0, 111, 981, 1020, 1547, 884, 211, 12, 547, 0, 0, 0, 0, 0, 0, 0, 0, 602, 288, 370, 340, 511, 423, 1313, 386, 0, 0, 0, 0, 0, 0, 0, 0, 6, 963, 736, 49, 2373, 291, 417, 302, 1198, 101, 0, 0, 0, 0, 0, 0, 1535, 310, 1009, 746, 1432, 2520, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 457, 336, 594, 2229, 319, 521, 201, 217, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5409, 7329, 7810, 5313, 4233, 6436, 7748, 4874] avg=6144.0000 max=7810 max_rank=2 min=4233 min_rank=4 imbalance_ratio=1.271159 max_load_reduction=41342 max_load_reduction_pct=0.841105 moved_to_echo=23291 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[806, 359, 56, 1455, 966, 826, 483, 145, 389, 8, 0, 0, 0, 0, 0, 0, 75, 502, 313, 459, 529, 161, 58, 2345, 3026, 0, 0, 0, 0, 0, 0, 0, 939, 462, 2652, 483, 342, 210, 143, 275, 191, 206, 43, 0, 0, 0, 0, 0, 378, 376, 1420, 847, 1592, 120, 63, 776, 0, 0, 0, 0, 0, 0, 0, 0, 648, 591, 233, 89, 109, 320, 553, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 150, 2750, 248, 303, 1791, 475, 784, 480, 977, 274, 0, 0, 0, 0, 0, 0, 1159, 1043, 887, 1068, 301, 795, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 275, 1860, 1386, 768, 686, 734, 1010, 226, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5493, 7468, 5946, 5572, 4083, 8232, 5413, 6945] avg=6144.0000 max=8232 max_rank=5 min=4083 min_rank=4 imbalance_ratio=1.339844 max_load_reduction=40920 max_load_reduction_pct=0.832520 moved_to_echo=24673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[738, 109, 593, 801, 2930, 633, 1017, 51, 463, 17, 0, 0, 0, 0, 0, 0, 424, 591, 240, 337, 1164, 36, 127, 1564, 2952, 0, 0, 0, 0, 0, 0, 0, 260, 788, 3328, 221, 519, 98, 291, 28, 742, 98, 51, 0, 0, 0, 0, 0, 66, 1453, 533, 1703, 338, 945, 57, 34, 0, 0, 0, 0, 0, 0, 0, 0, 788, 1175, 2169, 1078, 237, 138, 1114, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 5, 725, 753, 274, 1067, 34, 925, 334, 466, 267, 0, 0, 0, 0, 0, 0, 1012, 72, 1557, 418, 54, 2084, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 160, 754, 1650, 319, 656, 361, 849, 47, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7352, 7435, 6424, 5129, 7892, 4850, 5274, 4796] avg=6144.0000 max=7892 max_rank=4 min=4796 min_rank=7 imbalance_ratio=1.284505 max_load_reduction=41260 max_load_reduction_pct=0.839437 moved_to_echo=22812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2452, 1395, 94, 796, 1137, 2626, 707, 702, 605, 19, 0, 0, 0, 0, 0, 0, 228, 48, 41, 485, 69, 208, 185, 548, 1148, 0, 0, 0, 0, 0, 0, 0, 434, 176, 684, 826, 1250, 55, 479, 139, 977, 370, 70, 0, 0, 0, 0, 0, 824, 1901, 2633, 1019, 197, 55, 819, 788, 0, 0, 0, 0, 0, 0, 0, 0, 473, 245, 396, 274, 143, 1257, 1001, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 293, 97, 326, 66, 503, 61, 101, 173, 1727, 103, 0, 0, 0, 0, 0, 0, 786, 252, 2477, 760, 199, 387, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 77, 2261, 2131, 616, 587, 633, 1660, 216, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10533, 2960, 5460, 8236, 5198, 3450, 5134, 8181] avg=6144.0000 max=10533 max_rank=0 min=2960 min_rank=1 imbalance_ratio=1.714355 max_load_reduction=38619 max_load_reduction_pct=0.785706 moved_to_echo=21963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1179, 951, 653, 320, 141, 272, 344, 141, 581, 23, 0, 0, 0, 0, 0, 0, 50, 381, 100, 249, 658, 51, 498, 1393, 3143, 0, 0, 0, 0, 0, 0, 0, 775, 630, 215, 588, 1364, 91, 253, 1279, 654, 247, 64, 0, 0, 0, 0, 0, 24, 1281, 824, 738, 259, 1159, 222, 157, 0, 0, 0, 0, 0, 0, 0, 0, 298, 592, 71, 663, 882, 194, 1466, 2736, 0, 0, 0, 0, 0, 0, 0, 0, 470, 313, 166, 972, 654, 38, 2610, 89, 1166, 285, 0, 0, 0, 0, 0, 0, 853, 563, 1037, 1434, 593, 2151, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 938, 557, 2069, 1117, 306, 1286, 215, 225, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4605, 6523, 6160, 4664, 6902, 6763, 6822, 6713] avg=6144.0000 max=6902 max_rank=4 min=4605 min_rank=0 imbalance_ratio=1.123372 max_load_reduction=42250 max_load_reduction_pct=0.859578 moved_to_echo=27200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2944, 7424, 12928, 7680, 3584, 6144, 6784, 2304, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 3584, 1536, 2432, 8832, 896, 3712, 12544, 13440, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8320, 4608, 10624, 8960, 4864, 10624, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 3968, 9856, 3072, 10752, 1920, 2432, 6016, 4992, 1920, 512, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5760, 4864, 7936, 3968, 4224, 4224, 9600, 8960, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2304, 7040, 4224, 3200, 9856, 1536, 6784, 4736, 8832, 1280, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1664, 9856, 7808, 9600, 8320, 4096, 3584, 4608, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8320, 4608, 10624, 8960, 4864, 10624, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2304, 7040, 4224, 3200, 9856, 1536, 6784, 4736, 8832, 1280, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[10112, 4224, 2432, 4992, 12416, 6272, 3712, 1792, 3712, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 3968, 9856, 3072, 10752, 1920, 2432, 6016, 4992, 1920, 512, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2944, 7424, 12928, 7680, 3584, 6144, 6784, 2304, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[92, 534, 504, 98, 221, 159, 54, 1424, 507, 1943, 2349, 335, 238, 744, 428, 465, 1736, 4598, 666, 186, 1556, 891, 347, 100, 52, 807, 917, 65, 2662, 76, 54, 170, 93, 841, 71, 446, 429, 457, 1487, 94, 2063, 2847, 348, 90, 193, 30, 1110, 1205, 1722, 625, 1866, 197, 12, 727, 228, 212, 181, 231, 1833, 840, 2244, 771, 669, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[326, 146, 1413, 255, 34, 61, 1133, 225, 55, 3113, 3119, 512, 467, 1772, 91, 486, 2352, 235, 30, 40, 4907, 919, 507, 710, 197, 612, 823, 7, 3964, 557, 11, 47, 80, 2006, 31, 34, 1731, 352, 306, 337, 2565, 1003, 44, 8, 531, 610, 1284, 772, 269, 1443, 238, 340, 1309, 671, 87, 180, 21, 267, 304, 850, 482, 624, 1226, 21] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[57, 174, 1141, 139, 24, 233, 583, 1733, 296, 276, 282, 1007, 152, 341, 653, 482, 1094, 255, 471, 42, 4168, 1075, 141, 340, 237, 455, 1015, 1274, 1282, 73, 16, 7, 14, 427, 105, 1168, 1286, 13, 1372, 801, 4104, 2119, 517, 169, 915, 280, 1781, 3020, 528, 1318, 1432, 364, 213, 1967, 228, 52, 199, 80, 2424, 425, 294, 751, 967, 301] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[1377, 765, 305, 136, 239, 730, 215, 319, 143, 1863, 3393, 1428, 52, 495, 186, 638, 696, 183, 320, 83, 2178, 1025, 598, 790, 202, 1737, 1073, 587, 3310, 97, 1, 94, 37, 681, 1323, 46, 201, 30, 646, 795, 3154, 207, 395, 638, 2688, 157, 673, 1193, 745, 1536, 1120, 1038, 44, 591, 1468, 791, 35, 564, 907, 490, 41, 1319, 293, 48] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[658, 575, 15, 1180, 503, 243, 1886, 637, 16, 1294, 404, 153, 73, 2015, 257, 1988, 100, 56, 39, 1672, 3142, 1279, 654, 704, 531, 1985, 889, 211, 2849, 178, 15, 1268, 13, 844, 151, 547, 248, 36, 1346, 574, 1856, 1643, 309, 501, 2906, 37, 643, 587, 1110, 552, 1295, 121, 221, 463, 754, 637, 1656, 165, 556, 13, 36, 149, 576, 1138] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[173, 184, 365, 113, 80, 54, 523, 1332, 115, 3231, 2198, 119, 411, 956, 25, 64, 2606, 102, 16, 400, 859, 135, 246, 889, 16, 1233, 1001, 112, 4868, 13, 10, 205, 100, 974, 268, 135, 471, 145, 1075, 1046, 5615, 1991, 753, 282, 491, 100, 1216, 2421, 1361, 116, 1149, 115, 890, 642, 107, 135, 486, 181, 496, 141, 954, 1007, 1346, 289] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[509, 1313, 1344, 701, 30, 226, 830, 1671, 220, 5187, 1236, 3627, 189, 922, 208, 860, 1309, 689, 244, 618, 2019, 277, 83, 187, 66, 327, 1116, 59, 647, 192, 35, 92, 54, 604, 230, 335, 1836, 447, 876, 713, 2310, 1375, 21, 256, 2389, 930, 2571, 820, 199, 880, 241, 277, 380, 605, 338, 653, 19, 212, 160, 367, 488, 831, 693, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[1377, 765, 305, 136, 239, 730, 215, 319, 143, 1863, 3393, 1428, 52, 495, 186, 638, 696, 183, 320, 83, 2178, 1025, 598, 790, 202, 1737, 1073, 587, 3310, 97, 1, 94, 37, 681, 1323, 46, 201, 30, 646, 795, 3154, 207, 395, 638, 2688, 157, 673, 1193, 745, 1536, 1120, 1038, 44, 591, 1468, 791, 35, 564, 907, 490, 41, 1319, 293, 48] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[326, 146, 1413, 255, 34, 61, 1133, 225, 55, 3113, 3119, 512, 467, 1772, 91, 486, 2352, 235, 30, 40, 4907, 919, 507, 710, 197, 612, 823, 7, 3964, 557, 11, 47, 80, 2006, 31, 34, 1731, 352, 306, 337, 2565, 1003, 44, 8, 531, 610, 1284, 772, 269, 1443, 238, 340, 1309, 671, 87, 180, 21, 267, 304, 850, 482, 624, 1226, 21] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[57, 174, 1141, 139, 24, 233, 583, 1733, 296, 276, 282, 1007, 152, 341, 653, 482, 1094, 255, 471, 42, 4168, 1075, 141, 340, 237, 455, 1015, 1274, 1282, 73, 16, 7, 14, 427, 105, 1168, 1286, 13, 1372, 801, 4104, 2119, 517, 169, 915, 280, 1781, 3020, 528, 1318, 1432, 364, 213, 1967, 228, 52, 199, 80, 2424, 425, 294, 751, 967, 301] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[246, 258, 315, 239, 470, 29, 31, 213, 194, 1113, 1656, 1672, 26, 615, 731, 513, 2828, 327, 886, 1718, 2365, 694, 247, 420, 226, 1068, 490, 275, 1461, 609, 144, 314, 112, 847, 197, 322, 237, 23, 829, 462, 5101, 1578, 905, 155, 1303, 7, 2101, 852, 747, 197, 3729, 110, 16, 1174, 87, 363, 908, 310, 768, 541, 76, 1550, 241, 911] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[92, 534, 504, 98, 221, 159, 54, 1424, 507, 1943, 2349, 335, 238, 744, 428, 465, 1736, 4598, 666, 186, 1556, 891, 347, 100, 52, 807, 917, 65, 2662, 76, 54, 170, 93, 841, 71, 446, 429, 457, 1487, 94, 2063, 2847, 348, 90, 193, 30, 1110, 1205, 1722, 625, 1866, 197, 12, 727, 228, 212, 181, 231, 1833, 840, 2244, 771, 669, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[173, 184, 365, 113, 80, 54, 523, 1332, 115, 3231, 2198, 119, 411, 956, 25, 64, 2606, 102, 16, 400, 859, 135, 246, 889, 16, 1233, 1001, 112, 4868, 13, 10, 205, 100, 974, 268, 135, 471, 145, 1075, 1046, 5615, 1991, 753, 282, 491, 100, 1216, 2421, 1361, 116, 1149, 115, 890, 642, 107, 135, 486, 181, 496, 141, 954, 1007, 1346, 289] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[658, 575, 15, 1180, 503, 243, 1886, 637, 1203, 0, 0, 0, 0, 0, 0, 0, 16, 473, 404, 153, 73, 2015, 257, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 100, 56, 39, 1672, 1379, 1279, 654, 704, 0, 0, 0, 0, 0, 0, 0, 0, 531, 1985, 889, 211, 2849, 178, 15, 1268, 536, 0, 0, 0, 0, 0, 0, 0, 13, 844, 151, 547, 248, 36, 1346, 574, 653, 875, 0, 0, 0, 0, 0, 0, 0, 1024, 309, 501, 2906, 37, 643, 587, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 552, 1295, 121, 221, 463, 754, 637, 378, 83, 0, 0, 0, 0, 0, 0, 1656, 165, 556, 13, 36, 149, 576, 1138, 443, 888, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6900, 5379, 5883, 8462, 5287, 6007, 5614, 5620] avg=6144.0000 max=8462 max_rank=3 min=5287 min_rank=4 imbalance_ratio=1.377279 max_load_reduction=40690 max_load_reduction_pct=0.827840 moved_to_echo=22528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[326, 146, 1413, 255, 34, 61, 1133, 225, 1663, 0, 0, 0, 0, 0, 0, 0, 55, 1136, 3119, 512, 467, 1772, 91, 486, 0, 0, 0, 0, 0, 0, 0, 0, 2352, 235, 30, 40, 2154, 919, 507, 710, 0, 0, 0, 0, 0, 0, 0, 0, 197, 612, 823, 7, 3964, 557, 11, 47, 327, 0, 0, 0, 0, 0, 0, 0, 80, 2006, 31, 34, 1731, 352, 306, 337, 902, 1366, 0, 0, 0, 0, 0, 0, 0, 625, 44, 8, 531, 610, 1284, 772, 0, 0, 0, 0, 0, 0, 0, 0, 269, 1443, 238, 340, 1309, 671, 87, 180, 911, 51, 0, 0, 0, 0, 0, 0, 21, 267, 304, 850, 482, 624, 1226, 21, 1066, 1387, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5256, 7638, 6947, 6545, 7145, 3874, 5499, 6248] avg=6144.0000 max=7638 max_rank=1 min=3874 min_rank=5 imbalance_ratio=1.243164 max_load_reduction=41514 max_load_reduction_pct=0.844604 moved_to_echo=22766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[92, 534, 504, 98, 221, 159, 54, 1424, 1337, 0, 0, 0, 0, 0, 0, 0, 507, 710, 2349, 335, 238, 744, 428, 465, 0, 0, 0, 0, 0, 0, 0, 0, 1736, 4598, 666, 186, 684, 891, 347, 100, 0, 0, 0, 0, 0, 0, 0, 0, 52, 807, 917, 65, 2662, 76, 54, 170, 929, 0, 0, 0, 0, 0, 0, 0, 93, 841, 71, 446, 429, 457, 1487, 94, 726, 433, 0, 0, 0, 0, 0, 0, 0, 1773, 348, 90, 193, 30, 1110, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 1722, 625, 1866, 197, 12, 727, 228, 212, 568, 145, 0, 0, 0, 0, 0, 0, 181, 231, 1833, 840, 2244, 771, 669, 12, 665, 439, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4423, 5776, 9208, 5732, 5077, 4749, 6302, 7885] avg=6144.0000 max=9208 max_rank=2 min=4423 min_rank=0 imbalance_ratio=1.498698 max_load_reduction=39944 max_load_reduction_pct=0.812663 moved_to_echo=24013 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[57, 174, 1141, 139, 24, 233, 583, 1733, 2661, 0, 0, 0, 0, 0, 0, 0, 296, 102, 282, 1007, 152, 341, 653, 482, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 255, 471, 42, 1829, 1075, 141, 340, 0, 0, 0, 0, 0, 0, 0, 0, 237, 455, 1015, 1274, 1282, 73, 16, 7, 691, 0, 0, 0, 0, 0, 0, 0, 14, 427, 105, 1168, 1286, 13, 1372, 801, 1443, 1161, 0, 0, 0, 0, 0, 0, 0, 1320, 517, 169, 915, 280, 1781, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 528, 1318, 1432, 364, 213, 1967, 228, 52, 80, 108, 0, 0, 0, 0, 0, 0, 199, 80, 2424, 425, 294, 751, 967, 301, 94, 1178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6745, 3315, 5247, 5050, 7790, 8002, 6290, 6713] avg=6144.0000 max=8002 max_rank=5 min=3315 min_rank=1 imbalance_ratio=1.302409 max_load_reduction=41150 max_load_reduction_pct=0.837199 moved_to_echo=28795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[326, 146, 1413, 255, 34, 61, 1133, 225, 1663, 0, 0, 0, 0, 0, 0, 0, 55, 1136, 3119, 512, 467, 1772, 91, 486, 0, 0, 0, 0, 0, 0, 0, 0, 2352, 235, 30, 40, 2154, 919, 507, 710, 0, 0, 0, 0, 0, 0, 0, 0, 197, 612, 823, 7, 3964, 557, 11, 47, 327, 0, 0, 0, 0, 0, 0, 0, 80, 2006, 31, 34, 1731, 352, 306, 337, 902, 1366, 0, 0, 0, 0, 0, 0, 0, 625, 44, 8, 531, 610, 1284, 772, 0, 0, 0, 0, 0, 0, 0, 0, 269, 1443, 238, 340, 1309, 671, 87, 180, 911, 51, 0, 0, 0, 0, 0, 0, 21, 267, 304, 850, 482, 624, 1226, 21, 1066, 1387, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5256, 7638, 6947, 6545, 7145, 3874, 5499, 6248] avg=6144.0000 max=7638 max_rank=1 min=3874 min_rank=5 imbalance_ratio=1.243164 max_load_reduction=41514 max_load_reduction_pct=0.844604 moved_to_echo=22766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1377, 765, 305, 136, 239, 730, 215, 319, 2046, 0, 0, 0, 0, 0, 0, 0, 143, 680, 3393, 1428, 52, 495, 186, 638, 0, 0, 0, 0, 0, 0, 0, 0, 696, 183, 320, 83, 957, 1025, 598, 790, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1737, 1073, 587, 3310, 97, 1, 94, 67, 0, 0, 0, 0, 0, 0, 0, 37, 681, 1323, 46, 201, 30, 646, 795, 1108, 606, 0, 0, 0, 0, 0, 0, 0, 130, 395, 638, 2688, 157, 673, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 745, 1536, 1120, 1038, 44, 591, 1468, 791, 545, 10, 0, 0, 0, 0, 0, 0, 35, 564, 907, 490, 41, 1319, 293, 48, 638, 615, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 7015, 4652, 7168, 5473, 5874, 7888, 4950] avg=6144.0000 max=7888 max_rank=6 min=4652 min_rank=2 imbalance_ratio=1.283854 max_load_reduction=41264 max_load_reduction_pct=0.839518 moved_to_echo=24185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[658, 575, 15, 1180, 503, 243, 1886, 637, 1203, 0, 0, 0, 0, 0, 0, 0, 16, 473, 404, 153, 73, 2015, 257, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 100, 56, 39, 1672, 1379, 1279, 654, 704, 0, 0, 0, 0, 0, 0, 0, 0, 531, 1985, 889, 211, 2849, 178, 15, 1268, 536, 0, 0, 0, 0, 0, 0, 0, 13, 844, 151, 547, 248, 36, 1346, 574, 653, 875, 0, 0, 0, 0, 0, 0, 0, 1024, 309, 501, 2906, 37, 643, 587, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 552, 1295, 121, 221, 463, 754, 637, 378, 83, 0, 0, 0, 0, 0, 0, 1656, 165, 556, 13, 36, 149, 576, 1138, 443, 888, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6900, 5379, 5883, 8462, 5287, 6007, 5614, 5620] avg=6144.0000 max=8462 max_rank=3 min=5287 min_rank=4 imbalance_ratio=1.377279 max_load_reduction=40690 max_load_reduction_pct=0.827840 moved_to_echo=22528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[246, 258, 315, 239, 470, 29, 31, 213, 3308, 0, 0, 0, 0, 0, 0, 0, 194, 407, 1656, 1672, 26, 615, 731, 513, 0, 0, 0, 0, 0, 0, 0, 0, 2828, 327, 886, 1718, 1039, 694, 247, 420, 0, 0, 0, 0, 0, 0, 0, 0, 226, 1068, 490, 275, 1461, 609, 144, 314, 515, 0, 0, 0, 0, 0, 0, 0, 112, 847, 197, 322, 237, 23, 829, 462, 1793, 658, 0, 0, 0, 0, 0, 0, 0, 983, 905, 155, 1303, 7, 2101, 852, 0, 0, 0, 0, 0, 0, 0, 0, 747, 197, 3729, 110, 16, 1174, 87, 363, 325, 80, 0, 0, 0, 0, 0, 0, 908, 310, 768, 541, 76, 1550, 241, 911, 381, 668, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5109, 5814, 8159, 5102, 5480, 6306, 6828, 6354] avg=6144.0000 max=8159 max_rank=2 min=5102 min_rank=3 imbalance_ratio=1.327962 max_load_reduction=40993 max_load_reduction_pct=0.834005 moved_to_echo=24968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[509, 1313, 1344, 701, 30, 226, 830, 1671, 1498, 0, 0, 0, 0, 0, 0, 0, 220, 1893, 1236, 3627, 189, 922, 208, 860, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 689, 244, 618, 887, 277, 83, 187, 0, 0, 0, 0, 0, 0, 0, 0, 66, 327, 1116, 59, 647, 192, 35, 92, 448, 0, 0, 0, 0, 0, 0, 0, 54, 604, 230, 335, 1836, 447, 876, 713, 812, 562, 0, 0, 0, 0, 0, 0, 0, 857, 21, 256, 2389, 930, 2571, 820, 0, 0, 0, 0, 0, 0, 0, 0, 199, 880, 241, 277, 380, 605, 338, 653, 1518, 70, 0, 0, 0, 0, 0, 0, 19, 212, 160, 367, 488, 831, 693, 9, 1776, 570, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 9155, 4294, 2982, 6469, 7844, 5161, 5125] avg=6144.0000 max=9155 max_rank=1 min=2982 min_rank=3 imbalance_ratio=1.490072 max_load_reduction=39997 max_load_reduction_pct=0.813741 moved_to_echo=24599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[173, 184, 365, 113, 80, 54, 523, 1332, 3641, 0, 0, 0, 0, 0, 0, 0, 115, 1172, 2198, 119, 411, 956, 25, 64, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 102, 16, 400, 368, 135, 246, 889, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1233, 1001, 112, 4868, 13, 10, 205, 653, 0, 0, 0, 0, 0, 0, 0, 100, 974, 268, 135, 471, 145, 1075, 1046, 1974, 243, 0, 0, 0, 0, 0, 0, 0, 1233, 753, 282, 491, 100, 1216, 2421, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 116, 1149, 115, 890, 642, 107, 135, 950, 105, 0, 0, 0, 0, 0, 0, 486, 181, 496, 141, 954, 1007, 1346, 289, 1109, 248, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 5060, 4762, 8111, 6431, 6496, 5570, 6257] avg=6144.0000 max=8111 max_rank=3 min=4762 min_rank=2 imbalance_ratio=1.320150 max_load_reduction=41041 max_load_reduction_pct=0.834981 moved_to_echo=24754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[246, 258, 315, 239, 470, 29, 31, 213, 3308, 0, 0, 0, 0, 0, 0, 0, 194, 407, 1656, 1672, 26, 615, 731, 513, 0, 0, 0, 0, 0, 0, 0, 0, 2828, 327, 886, 1718, 1039, 694, 247, 420, 0, 0, 0, 0, 0, 0, 0, 0, 226, 1068, 490, 275, 1461, 609, 144, 314, 515, 0, 0, 0, 0, 0, 0, 0, 112, 847, 197, 322, 237, 23, 829, 462, 1793, 658, 0, 0, 0, 0, 0, 0, 0, 983, 905, 155, 1303, 7, 2101, 852, 0, 0, 0, 0, 0, 0, 0, 0, 747, 197, 3729, 110, 16, 1174, 87, 363, 325, 80, 0, 0, 0, 0, 0, 0, 908, 310, 768, 541, 76, 1550, 241, 911, 381, 668, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5109, 5814, 8159, 5102, 5480, 6306, 6828, 6354] avg=6144.0000 max=8159 max_rank=2 min=5102 min_rank=3 imbalance_ratio=1.327962 max_load_reduction=40993 max_load_reduction_pct=0.834005 moved_to_echo=24968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 2048, 7552, 3712, 4736, 7040, 6016, 2816, 6272, 6016, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 8064, 3328, 2176, 11520, 2176, 11392, 10880, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[6784, 6784, 11136, 2688, 3200, 6912, 3328, 3072, 5376, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 7296, 2432, 3072, 6528, 1536, 8064, 4864, 9472, 6016, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 6656, 14720, 8960, 1664, 7936, 2688, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 8064, 3328, 2176, 11520, 2176, 11392, 10880, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12800, 6528, 2688, 4864, 9344, 6400, 2944, 4224, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 7296, 2432, 3072, 6528, 1536, 8064, 4864, 9472, 6016, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 3968, 5504, 2944, 1664, 1792, 5376, 7680, 17408, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1536, 8320, 7424, 2688, 21120, 1920, 384, 2304, 4224, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[87, 70, 102, 577, 1094, 287, 1856, 553, 2014, 95, 230, 255, 922, 774, 423, 1913, 2530, 696, 643, 766, 417, 2093, 837, 967, 168, 891, 552, 227, 943, 1568, 2, 1123, 42, 1721, 341, 225, 697, 618, 1196, 543, 458, 164, 82, 101, 386, 418, 682, 105, 1974, 1258, 1098, 1930, 1009, 690, 238, 16, 1059, 399, 873, 1473, 136, 558, 2480, 507] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[545, 76, 515, 1048, 169, 12, 2468, 253, 1159, 76, 427, 1216, 974, 131, 62, 518, 540, 3167, 374, 990, 160, 2397, 6031, 147, 300, 567, 211, 63, 1940, 2379, 7, 2376, 116, 414, 1011, 244, 176, 464, 2966, 2393, 720, 591, 17, 13, 246, 453, 337, 314, 197, 329, 408, 396, 792, 501, 68, 163, 540, 29, 306, 251, 797, 463, 1700, 439] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[57, 204, 1388, 398, 1998, 48, 453, 800, 1751, 7, 739, 333, 1354, 3014, 221, 972, 760, 1869, 1225, 113, 1097, 4395, 1443, 175, 382, 1141, 204, 1025, 3327, 1174, 304, 399, 129, 1336, 163, 203, 62, 1234, 960, 452, 349, 40, 26, 245, 93, 1335, 466, 500, 111, 392, 116, 1027, 291, 483, 256, 520, 447, 651, 771, 350, 39, 435, 156, 2744] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[438, 1123, 456, 360, 148, 40, 2315, 375, 2013, 184, 2338, 185, 159, 829, 405, 502, 1275, 809, 767, 197, 14, 614, 2123, 1251, 210, 966, 18, 84, 350, 2284, 10, 157, 43, 3521, 1126, 2412, 396, 799, 1251, 1386, 523, 382, 905, 83, 89, 816, 31, 380, 414, 2055, 514, 792, 25, 1535, 793, 31, 858, 918, 1129, 786, 248, 776, 547, 589] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[87, 70, 102, 577, 1094, 287, 1856, 553, 2014, 95, 230, 255, 922, 774, 423, 1913, 2530, 696, 643, 766, 417, 2093, 837, 967, 168, 891, 552, 227, 943, 1568, 2, 1123, 42, 1721, 341, 225, 697, 618, 1196, 543, 458, 164, 82, 101, 386, 418, 682, 105, 1974, 1258, 1098, 1930, 1009, 690, 238, 16, 1059, 399, 873, 1473, 136, 558, 2480, 507] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[227, 50, 67, 220, 1788, 44, 3380, 32, 1211, 850, 128, 262, 305, 1486, 46, 466, 270, 1597, 2001, 1499, 1200, 969, 3869, 1435, 95, 2390, 587, 730, 3143, 472, 102, 301, 103, 1596, 222, 671, 1371, 562, 2610, 145, 111, 392, 4, 169, 29, 275, 274, 105, 288, 688, 796, 417, 94, 2376, 172, 67, 1178, 246, 215, 563, 1199, 79, 486, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[1869, 69, 906, 240, 748, 108, 906, 289, 666, 121, 95, 1602, 1630, 908, 401, 855, 817, 830, 794, 1248, 243, 1282, 978, 2254, 158, 5176, 127, 252, 1445, 1741, 77, 305, 166, 2083, 667, 233, 50, 733, 1180, 656, 643, 838, 23, 223, 40, 95, 308, 1045, 349, 1104, 144, 245, 24, 3156, 119, 44, 1953, 397, 281, 651, 1050, 272, 465, 775] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[3585, 751, 18, 604, 932, 51, 621, 322, 3503, 227, 1559, 266, 1297, 119, 47, 2127, 170, 647, 1648, 33, 129, 522, 2988, 214, 82, 610, 464, 154, 4226, 936, 18, 803, 428, 1729, 969, 2416, 159, 117, 1130, 1866, 418, 106, 212, 11, 3, 832, 513, 349, 229, 252, 160, 449, 310, 872, 1043, 39, 321, 627, 397, 218, 68, 160, 2476, 600] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[853, 599, 378, 505, 1339, 58, 1181, 274, 1933, 50, 417, 1358, 968, 421, 1177, 1188, 375, 2639, 2459, 998, 534, 1629, 919, 1244, 335, 2398, 22, 97, 1590, 1047, 179, 838, 45, 659, 534, 243, 2959, 387, 3416, 446, 68, 241, 119, 549, 49, 458, 510, 600, 874, 113, 592, 959, 31, 708, 14, 36, 492, 190, 537, 1130, 988, 210, 83, 910] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[545, 76, 515, 1048, 169, 12, 2468, 253, 1159, 76, 427, 1216, 974, 131, 62, 518, 540, 3167, 374, 990, 160, 2397, 6031, 147, 300, 567, 211, 63, 1940, 2379, 7, 2376, 116, 414, 1011, 244, 176, 464, 2966, 2393, 720, 591, 17, 13, 246, 453, 337, 314, 197, 329, 408, 396, 792, 501, 68, 163, 540, 29, 306, 251, 797, 463, 1700, 439] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[1869, 69, 906, 240, 748, 108, 906, 289, 666, 121, 95, 1602, 1630, 908, 401, 855, 817, 830, 794, 1248, 243, 1282, 978, 2254, 158, 5176, 127, 252, 1445, 1741, 77, 305, 166, 2083, 667, 233, 50, 733, 1180, 656, 643, 838, 23, 223, 40, 95, 308, 1045, 349, 1104, 144, 245, 24, 3156, 119, 44, 1953, 397, 281, 651, 1050, 272, 465, 775] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[57, 204, 1388, 398, 1998, 48, 453, 800, 1751, 7, 739, 333, 1354, 3014, 221, 972, 760, 1869, 1225, 113, 1097, 4395, 1443, 175, 382, 1141, 204, 1025, 3327, 1174, 304, 399, 129, 1336, 163, 203, 62, 1234, 960, 452, 349, 40, 26, 245, 93, 1335, 466, 500, 111, 392, 116, 1027, 291, 483, 256, 520, 447, 651, 771, 350, 39, 435, 156, 2744] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[87, 70, 102, 577, 1094, 287, 1856, 553, 668, 60, 0, 0, 0, 0, 0, 0, 1346, 95, 230, 255, 922, 774, 423, 1913, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 696, 643, 766, 417, 441, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 168, 891, 552, 227, 550, 1568, 2, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1721, 341, 225, 697, 618, 381, 543, 0, 0, 0, 0, 0, 0, 0, 0, 458, 164, 82, 101, 386, 418, 682, 105, 837, 1355, 0, 0, 0, 0, 0, 0, 1974, 1258, 1098, 1930, 1009, 690, 238, 16, 812, 297, 0, 0, 0, 0, 0, 0, 1059, 399, 873, 1473, 136, 558, 2480, 507, 333, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5354, 5958, 6460, 5081, 4568, 4588, 9322, 7821] avg=6144.0000 max=9322 max_rank=6 min=4568 min_rank=4 imbalance_ratio=1.517253 max_load_reduction=39830 max_load_reduction_pct=0.810343 moved_to_echo=26299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3585, 751, 18, 604, 932, 51, 621, 322, 1163, 272, 0, 0, 0, 0, 0, 0, 2340, 227, 1559, 266, 1297, 119, 47, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 170, 647, 1648, 33, 129, 110, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 82, 610, 464, 154, 2458, 936, 18, 803, 0, 0, 0, 0, 0, 0, 0, 0, 428, 1729, 969, 2416, 159, 117, 360, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 418, 106, 212, 11, 3, 832, 513, 349, 2988, 338, 0, 0, 0, 0, 0, 0, 229, 252, 160, 449, 310, 872, 1043, 39, 767, 74, 0, 0, 0, 0, 0, 0, 321, 627, 397, 218, 68, 160, 2476, 600, 1496, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8319, 7982, 2951, 5525, 8044, 5770, 4195, 6366] avg=6144.0000 max=8319 max_rank=0 min=2951 min_rank=2 imbalance_ratio=1.354004 max_load_reduction=40833 max_load_reduction_pct=0.830750 moved_to_echo=24375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[227, 50, 67, 220, 1788, 44, 3380, 32, 402, 202, 0, 0, 0, 0, 0, 0, 809, 850, 128, 262, 305, 1486, 46, 466, 0, 0, 0, 0, 0, 0, 0, 0, 270, 1597, 2001, 1499, 1200, 205, 0, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 95, 2390, 587, 730, 1829, 472, 102, 301, 0, 0, 0, 0, 0, 0, 0, 0, 103, 1596, 222, 671, 1371, 562, 829, 145, 0, 0, 0, 0, 0, 0, 0, 0, 111, 392, 4, 169, 29, 275, 274, 105, 3869, 627, 0, 0, 0, 0, 0, 0, 288, 688, 796, 417, 94, 2376, 172, 67, 1773, 137, 0, 0, 0, 0, 0, 0, 1178, 246, 215, 563, 1199, 79, 486, 427, 1112, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6412, 4352, 8207, 6506, 5499, 5855, 6808, 5513] avg=6144.0000 max=8207 max_rank=2 min=4352 min_rank=1 imbalance_ratio=1.335775 max_load_reduction=40945 max_load_reduction_pct=0.833028 moved_to_echo=23675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[545, 76, 515, 1048, 169, 12, 2468, 253, 384, 124, 0, 0, 0, 0, 0, 0, 775, 76, 427, 1216, 974, 131, 62, 518, 0, 0, 0, 0, 0, 0, 0, 0, 540, 3167, 374, 990, 160, 505, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 300, 567, 211, 63, 1130, 2379, 7, 2376, 0, 0, 0, 0, 0, 0, 0, 0, 116, 414, 1011, 244, 176, 464, 943, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 720, 591, 17, 13, 246, 453, 337, 314, 6031, 1552, 0, 0, 0, 0, 0, 0, 197, 329, 408, 396, 792, 501, 68, 163, 2014, 340, 0, 0, 0, 0, 0, 0, 540, 29, 306, 251, 797, 463, 1700, 439, 686, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5594, 4179, 5883, 7033, 5761, 10274, 5208, 5220] avg=6144.0000 max=10274 max_rank=5 min=4179 min_rank=1 imbalance_ratio=1.672201 max_load_reduction=38878 max_load_reduction_pct=0.790975 moved_to_echo=26463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[57, 204, 1388, 398, 1998, 48, 453, 800, 581, 214, 0, 0, 0, 0, 0, 0, 1170, 7, 739, 333, 1354, 3014, 221, 972, 0, 0, 0, 0, 0, 0, 0, 0, 760, 1869, 1225, 113, 1097, 925, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 382, 1141, 204, 1025, 1935, 1174, 304, 399, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1336, 163, 203, 62, 1234, 305, 452, 0, 0, 0, 0, 0, 0, 0, 0, 349, 40, 26, 245, 93, 1335, 466, 500, 1443, 2846, 0, 0, 0, 0, 0, 0, 111, 392, 116, 1027, 291, 483, 256, 520, 652, 624, 0, 0, 0, 0, 0, 0, 447, 651, 771, 350, 39, 435, 156, 2744, 1178, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 7810, 6164, 6564, 3884, 7343, 4472, 6774] avg=6144.0000 max=7810 max_rank=1 min=3884 min_rank=4 imbalance_ratio=1.271159 max_load_reduction=41342 max_load_reduction_pct=0.841105 moved_to_echo=22473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[87, 70, 102, 577, 1094, 287, 1856, 553, 668, 60, 0, 0, 0, 0, 0, 0, 1346, 95, 230, 255, 922, 774, 423, 1913, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 696, 643, 766, 417, 441, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 168, 891, 552, 227, 550, 1568, 2, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1721, 341, 225, 697, 618, 381, 543, 0, 0, 0, 0, 0, 0, 0, 0, 458, 164, 82, 101, 386, 418, 682, 105, 837, 1355, 0, 0, 0, 0, 0, 0, 1974, 1258, 1098, 1930, 1009, 690, 238, 16, 812, 297, 0, 0, 0, 0, 0, 0, 1059, 399, 873, 1473, 136, 558, 2480, 507, 333, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5354, 5958, 6460, 5081, 4568, 4588, 9322, 7821] avg=6144.0000 max=9322 max_rank=6 min=4568 min_rank=4 imbalance_ratio=1.517253 max_load_reduction=39830 max_load_reduction_pct=0.810343 moved_to_echo=26299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1869, 69, 906, 240, 748, 108, 906, 289, 221, 93, 0, 0, 0, 0, 0, 0, 445, 121, 95, 1602, 1630, 908, 401, 855, 0, 0, 0, 0, 0, 0, 0, 0, 817, 830, 794, 1248, 243, 270, 0, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 158, 5176, 127, 252, 841, 1741, 77, 305, 0, 0, 0, 0, 0, 0, 0, 0, 166, 2083, 667, 233, 50, 733, 376, 656, 0, 0, 0, 0, 0, 0, 0, 0, 643, 838, 23, 223, 40, 95, 308, 1045, 978, 830, 0, 0, 0, 0, 0, 0, 349, 1104, 144, 245, 24, 3156, 119, 44, 801, 182, 0, 0, 0, 0, 0, 0, 1953, 397, 281, 651, 1050, 272, 465, 775, 511, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5449, 6057, 6456, 8677, 4964, 5023, 6168, 6358] avg=6144.0000 max=8677 max_rank=3 min=4964 min_rank=4 imbalance_ratio=1.412272 max_load_reduction=40475 max_load_reduction_pct=0.823466 moved_to_echo=22513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[853, 599, 378, 505, 1339, 58, 1181, 274, 641, 102, 0, 0, 0, 0, 0, 0, 1292, 50, 417, 1358, 968, 421, 1177, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 375, 2639, 2459, 998, 534, 344, 0, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 335, 2398, 22, 97, 925, 1047, 179, 838, 0, 0, 0, 0, 0, 0, 0, 0, 45, 659, 534, 243, 2959, 387, 1086, 446, 0, 0, 0, 0, 0, 0, 0, 0, 68, 241, 119, 549, 49, 458, 510, 600, 919, 1054, 0, 0, 0, 0, 0, 0, 874, 113, 592, 959, 31, 708, 14, 36, 2320, 231, 0, 0, 0, 0, 0, 0, 492, 190, 537, 1130, 988, 210, 83, 910, 563, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 6871, 8593, 5841, 6359, 4567, 5878, 5113] avg=6144.0000 max=8593 max_rank=2 min=4567 min_rank=5 imbalance_ratio=1.398600 max_load_reduction=40559 max_load_reduction_pct=0.825175 moved_to_echo=21917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[438, 1123, 456, 360, 148, 40, 2315, 375, 672, 26, 0, 0, 0, 0, 0, 0, 1341, 184, 2338, 185, 159, 829, 405, 502, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 809, 767, 197, 14, 125, 0, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 210, 966, 18, 84, 196, 2284, 10, 157, 0, 0, 0, 0, 0, 0, 0, 0, 43, 3521, 1126, 2412, 396, 799, 390, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 523, 382, 905, 83, 89, 816, 31, 380, 2123, 400, 0, 0, 0, 0, 0, 0, 414, 2055, 514, 792, 25, 1535, 793, 31, 853, 89, 0, 0, 0, 0, 0, 0, 858, 918, 1129, 786, 248, 776, 547, 589, 128, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5953, 5943, 4438, 3925, 10073, 5732, 7101, 5987] avg=6144.0000 max=10073 max_rank=4 min=3925 min_rank=3 imbalance_ratio=1.639486 max_load_reduction=39079 max_load_reduction_pct=0.795064 moved_to_echo=28893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[57, 204, 1388, 398, 1998, 48, 453, 800, 581, 214, 0, 0, 0, 0, 0, 0, 1170, 7, 739, 333, 1354, 3014, 221, 972, 0, 0, 0, 0, 0, 0, 0, 0, 760, 1869, 1225, 113, 1097, 925, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 382, 1141, 204, 1025, 1935, 1174, 304, 399, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1336, 163, 203, 62, 1234, 305, 452, 0, 0, 0, 0, 0, 0, 0, 0, 349, 40, 26, 245, 93, 1335, 466, 500, 1443, 2846, 0, 0, 0, 0, 0, 0, 111, 392, 116, 1027, 291, 483, 256, 520, 652, 624, 0, 0, 0, 0, 0, 0, 447, 651, 771, 350, 39, 435, 156, 2744, 1178, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 7810, 6164, 6564, 3884, 7343, 4472, 6774] avg=6144.0000 max=7810 max_rank=1 min=3884 min_rank=4 imbalance_ratio=1.271159 max_load_reduction=41342 max_load_reduction_pct=0.841105 moved_to_echo=22473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[227, 50, 67, 220, 1788, 44, 3380, 32, 402, 202, 0, 0, 0, 0, 0, 0, 809, 850, 128, 262, 305, 1486, 46, 466, 0, 0, 0, 0, 0, 0, 0, 0, 270, 1597, 2001, 1499, 1200, 205, 0, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 95, 2390, 587, 730, 1829, 472, 102, 301, 0, 0, 0, 0, 0, 0, 0, 0, 103, 1596, 222, 671, 1371, 562, 829, 145, 0, 0, 0, 0, 0, 0, 0, 0, 111, 392, 4, 169, 29, 275, 274, 105, 3869, 627, 0, 0, 0, 0, 0, 0, 288, 688, 796, 417, 94, 2376, 172, 67, 1773, 137, 0, 0, 0, 0, 0, 0, 1178, 246, 215, 563, 1199, 79, 486, 427, 1112, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6412, 4352, 8207, 6506, 5499, 5855, 6808, 5513] avg=6144.0000 max=8207 max_rank=2 min=4352 min_rank=1 imbalance_ratio=1.335775 max_load_reduction=40945 max_load_reduction_pct=0.833028 moved_to_echo=23675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[853, 599, 378, 505, 1339, 58, 1181, 274, 641, 102, 0, 0, 0, 0, 0, 0, 1292, 50, 417, 1358, 968, 421, 1177, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 375, 2639, 2459, 998, 534, 344, 0, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 335, 2398, 22, 97, 925, 1047, 179, 838, 0, 0, 0, 0, 0, 0, 0, 0, 45, 659, 534, 243, 2959, 387, 1086, 446, 0, 0, 0, 0, 0, 0, 0, 0, 68, 241, 119, 549, 49, 458, 510, 600, 919, 1054, 0, 0, 0, 0, 0, 0, 874, 113, 592, 959, 31, 708, 14, 36, 2320, 231, 0, 0, 0, 0, 0, 0, 492, 190, 537, 1130, 988, 210, 83, 910, 563, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 6871, 8593, 5841, 6359, 4567, 5878, 5113] avg=6144.0000 max=8593 max_rank=2 min=4567 min_rank=5 imbalance_ratio=1.398600 max_load_reduction=40559 max_load_reduction_pct=0.825175 moved_to_echo=21917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 6272, 3840, 6272, 2688, 10368, 2816, 1024, 10112, 2048, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9600, 1664, 6016, 5504, 7680, 7808, 2816, 8576, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6784, 12288, 9984, 5888, 3840, 2944, 0, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 6272, 3840, 6272, 2688, 10368, 2816, 1024, 10112, 2048, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 2816, 1408, 1408, 1024, 4736, 3200, 3456, 19200, 9088, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 14208, 2304, 2688, 9984, 11648, 768, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6912, 3584, 4608, 5504, 4608, 3072, 8448, 7040, 6016, 128, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1152, 13184, 5120, 6656, 5888, 4992, 4736, 7936, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7680, 2944, 3840, 3968, 8320, 768, 13184, 2944, 4736, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6784, 12288, 9984, 5888, 3840, 2944, 0, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6912, 3584, 4608, 5504, 4608, 3072, 8448, 7040, 6016, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 14208, 2304, 2688, 9984, 11648, 768, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[2066, 118, 340, 229, 755, 550, 52, 1056, 6, 1042, 67, 490, 1496, 3560, 41, 623, 453, 1088, 704, 337, 1643, 242, 100, 513, 1370, 1072, 27, 1027, 71, 610, 451, 265, 192, 779, 554, 325, 122, 154, 878, 672, 37, 310, 4260, 303, 212, 385, 4182, 249, 56, 50, 10, 3163, 214, 953, 250, 3447, 43, 196, 637, 1026, 614, 20, 626, 1769] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[538, 200, 318, 850, 1491, 255, 38, 262, 104, 352, 355, 937, 813, 1169, 39, 289, 109, 3043, 1078, 1156, 1193, 78, 171, 1326, 2160, 2573, 132, 499, 18, 1373, 133, 347, 80, 934, 268, 1693, 1686, 778, 411, 623, 275, 117, 2252, 444, 467, 53, 803, 668, 144, 484, 94, 624, 487, 1647, 1168, 1528, 216, 471, 40, 1716, 1734, 90, 2304, 1454] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[1179, 38, 2050, 760, 3378, 338, 55, 149, 5, 453, 56, 677, 1506, 2401, 42, 102, 62, 463, 327, 454, 631, 245, 261, 24, 1542, 714, 59, 137, 28, 1538, 437, 2757, 124, 731, 342, 1994, 190, 26, 180, 330, 77, 206, 1222, 2507, 2073, 179, 2032, 927, 332, 53, 17, 2719, 0, 2227, 1362, 1513, 377, 78, 34, 2947, 188, 239, 124, 934] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[1972, 813, 1209, 255, 919, 333, 367, 221, 71, 299, 336, 2423, 190, 2142, 6, 587, 333, 4070, 1247, 799, 743, 92, 88, 326, 848, 2360, 44, 439, 14, 2511, 299, 223, 462, 636, 650, 1704, 378, 2271, 680, 91, 81, 356, 4808, 454, 541, 236, 94, 449, 43, 513, 264, 1978, 768, 922, 1225, 1700, 165, 128, 125, 33, 112, 199, 360, 147] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[947, 997, 3045, 876, 829, 116, 82, 1806, 51, 150, 501, 156, 1688, 1376, 118, 407, 949, 1038, 2241, 2645, 108, 527, 389, 607, 1773, 866, 900, 1182, 32, 1655, 415, 2156, 197, 212, 56, 497, 103, 795, 177, 94, 137, 399, 173, 266, 266, 136, 1473, 36, 239, 1656, 39, 722, 114, 2491, 1209, 2720, 204, 854, 1008, 494, 56, 105, 83, 1513] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[330, 21, 136, 996, 4063, 518, 333, 1263, 75, 747, 174, 3251, 380, 858, 193, 213, 113, 1427, 1320, 846, 162, 241, 87, 529, 1027, 475, 10, 711, 12, 1932, 59, 1915, 231, 78, 159, 510, 83, 513, 1427, 67, 770, 566, 1193, 1074, 1461, 141, 2394, 818, 452, 231, 233, 328, 268, 959, 2185, 2106, 41, 13, 72, 961, 3147, 676, 244, 1334] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[1447, 36, 268, 403, 579, 44, 358, 614, 2, 818, 253, 1860, 1345, 1915, 103, 239, 37, 1298, 345, 715, 1153, 813, 16, 498, 1471, 1506, 202, 668, 29, 1053, 405, 2977, 151, 145, 24, 525, 508, 473, 745, 656, 1008, 150, 602, 130, 1093, 758, 2825, 387, 106, 614, 303, 25, 619, 2165, 1627, 3168, 224, 110, 105, 2367, 856, 730, 522, 1961] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[149, 438, 1849, 762, 1863, 1165, 12, 257, 8, 43, 6, 343, 580, 2473, 41, 635, 1021, 1423, 1152, 886, 1740, 14, 441, 215, 1785, 870, 303, 657, 34, 2118, 787, 1234, 299, 95, 245, 2345, 2, 1512, 32, 124, 141, 455, 598, 429, 403, 73, 388, 713, 251, 209, 148, 1545, 12, 4271, 309, 902, 441, 461, 2, 4576, 561, 517, 424, 1365] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[538, 200, 318, 850, 1427, 255, 38, 262, 0, 0, 0, 0, 0, 0, 0, 0, 104, 352, 355, 937, 813, 1169, 39, 289, 587, 0, 0, 0, 0, 0, 0, 0, 109, 3043, 1078, 1156, 1193, 78, 171, 1326, 64, 22, 0, 0, 0, 0, 0, 0, 2160, 2573, 132, 499, 18, 500, 133, 347, 0, 0, 0, 0, 0, 0, 0, 0, 80, 934, 268, 1693, 1686, 778, 411, 623, 1165, 225, 0, 0, 0, 0, 0, 0, 275, 117, 1722, 444, 467, 53, 803, 668, 0, 0, 0, 0, 0, 0, 0, 0, 144, 484, 94, 624, 487, 1647, 1168, 363, 0, 0, 0, 0, 0, 0, 0, 0, 216, 471, 40, 1716, 1734, 90, 2304, 1454, 508, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3888, 4645, 8240, 6362, 7863, 4549, 5011, 8594] avg=6144.0000 max=8594 max_rank=7 min=3888 min_rank=0 imbalance_ratio=1.398763 max_load_reduction=40558 max_load_reduction_pct=0.825155 moved_to_echo=26017 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2066, 118, 340, 229, 723, 550, 52, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1042, 67, 490, 1496, 3560, 41, 623, 260, 0, 0, 0, 0, 0, 0, 0, 453, 1088, 704, 337, 1643, 242, 100, 513, 32, 43, 0, 0, 0, 0, 0, 0, 1370, 1072, 27, 1027, 71, 223, 451, 265, 0, 0, 0, 0, 0, 0, 0, 0, 192, 779, 554, 325, 122, 154, 878, 672, 2628, 100, 0, 0, 0, 0, 0, 0, 37, 310, 3256, 303, 212, 385, 4182, 249, 0, 0, 0, 0, 0, 0, 0, 0, 56, 50, 10, 3163, 214, 953, 250, 819, 0, 0, 0, 0, 0, 0, 0, 0, 43, 196, 637, 1026, 614, 20, 626, 1769, 961, 27, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5134, 7585, 5155, 4506, 6404, 8934, 5515, 5919] avg=6144.0000 max=8934 max_rank=5 min=4506 min_rank=3 imbalance_ratio=1.454102 max_load_reduction=40218 max_load_reduction_pct=0.818237 moved_to_echo=26772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[1179, 38, 2050, 760, 3378, 338, 55, 149, 5, 453, 56, 677, 1506, 2401, 42, 102, 62, 463, 327, 454, 631, 245, 261, 24, 1542, 714, 59, 137, 28, 1538, 437, 2757, 124, 731, 342, 1994, 190, 26, 180, 330, 77, 206, 1222, 2507, 2073, 179, 2032, 927, 332, 53, 17, 2719, 0, 2227, 1362, 1513, 377, 78, 34, 2947, 188, 239, 124, 934] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[2066, 118, 340, 229, 755, 550, 52, 1056, 6, 1042, 67, 490, 1496, 3560, 41, 623, 453, 1088, 704, 337, 1643, 242, 100, 513, 1370, 1072, 27, 1027, 71, 610, 451, 265, 192, 779, 554, 325, 122, 154, 878, 672, 37, 310, 4260, 303, 212, 385, 4182, 249, 56, 50, 10, 3163, 214, 953, 250, 3447, 43, 196, 637, 1026, 614, 20, 626, 1769] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[1972, 813, 1209, 255, 919, 333, 367, 221, 71, 299, 336, 2423, 190, 2142, 6, 587, 333, 4070, 1247, 799, 743, 92, 88, 326, 848, 2360, 44, 439, 14, 2511, 299, 223, 462, 636, 650, 1704, 378, 2271, 680, 91, 81, 356, 4808, 454, 541, 236, 94, 449, 43, 513, 264, 1978, 768, 922, 1225, 1700, 165, 128, 125, 33, 112, 199, 360, 147] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[1447, 36, 268, 403, 579, 44, 358, 614, 2, 818, 253, 1860, 1345, 1915, 103, 239, 37, 1298, 345, 715, 1153, 813, 16, 498, 1471, 1506, 202, 668, 29, 1053, 405, 2977, 151, 145, 24, 525, 508, 473, 745, 656, 1008, 150, 602, 130, 1093, 758, 2825, 387, 106, 614, 303, 25, 619, 2165, 1627, 3168, 224, 110, 105, 2367, 856, 730, 522, 1961] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[947, 997, 3045, 876, 793, 116, 82, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 51, 150, 501, 156, 1688, 1376, 118, 407, 707, 0, 0, 0, 0, 0, 0, 0, 949, 1038, 2241, 2645, 108, 527, 389, 607, 36, 1, 0, 0, 0, 0, 0, 0, 1773, 866, 900, 1182, 32, 603, 415, 2156, 0, 0, 0, 0, 0, 0, 0, 0, 197, 212, 56, 497, 103, 795, 177, 94, 2074, 272, 0, 0, 0, 0, 0, 0, 137, 399, 133, 266, 266, 136, 1473, 36, 0, 0, 0, 0, 0, 0, 0, 0, 239, 1656, 39, 722, 114, 2491, 1209, 646, 0, 0, 0, 0, 0, 0, 0, 0, 204, 854, 1008, 494, 56, 105, 83, 1513, 39, 73, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8662, 5154, 8541, 7927, 4477, 2846, 7116, 4429] avg=6144.0000 max=8662 max_rank=0 min=2846 min_rank=5 imbalance_ratio=1.409831 max_load_reduction=40490 max_load_reduction_pct=0.823771 moved_to_echo=18868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[538, 200, 318, 850, 1427, 255, 38, 262, 0, 0, 0, 0, 0, 0, 0, 0, 104, 352, 355, 937, 813, 1169, 39, 289, 587, 0, 0, 0, 0, 0, 0, 0, 109, 3043, 1078, 1156, 1193, 78, 171, 1326, 64, 22, 0, 0, 0, 0, 0, 0, 2160, 2573, 132, 499, 18, 500, 133, 347, 0, 0, 0, 0, 0, 0, 0, 0, 80, 934, 268, 1693, 1686, 778, 411, 623, 1165, 225, 0, 0, 0, 0, 0, 0, 275, 117, 1722, 444, 467, 53, 803, 668, 0, 0, 0, 0, 0, 0, 0, 0, 144, 484, 94, 624, 487, 1647, 1168, 363, 0, 0, 0, 0, 0, 0, 0, 0, 216, 471, 40, 1716, 1734, 90, 2304, 1454, 508, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3888, 4645, 8240, 6362, 7863, 4549, 5011, 8594] avg=6144.0000 max=8594 max_rank=7 min=3888 min_rank=0 imbalance_ratio=1.398763 max_load_reduction=40558 max_load_reduction_pct=0.825155 moved_to_echo=26017 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1179, 38, 2050, 760, 3231, 338, 55, 149, 0, 0, 0, 0, 0, 0, 0, 0, 5, 453, 56, 677, 1506, 2401, 42, 102, 657, 0, 0, 0, 0, 0, 0, 0, 62, 463, 327, 454, 631, 245, 261, 24, 147, 12, 0, 0, 0, 0, 0, 0, 1542, 714, 59, 137, 28, 560, 437, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 124, 731, 342, 1994, 190, 26, 180, 330, 1153, 253, 0, 0, 0, 0, 0, 0, 77, 206, 935, 2507, 2073, 179, 2032, 927, 0, 0, 0, 0, 0, 0, 0, 0, 332, 53, 17, 2719, 0, 2227, 1362, 360, 0, 0, 0, 0, 0, 0, 0, 0, 377, 78, 34, 2947, 188, 239, 124, 934, 275, 68, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7800, 5899, 2626, 6234, 5323, 8936, 7070, 5264] avg=6144.0000 max=8936 max_rank=5 min=2626 min_rank=2 imbalance_ratio=1.454427 max_load_reduction=40216 max_load_reduction_pct=0.818197 moved_to_echo=26593 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1972, 813, 1209, 255, 880, 333, 367, 221, 0, 0, 0, 0, 0, 0, 0, 0, 71, 299, 336, 2423, 190, 2142, 6, 587, 1074, 0, 0, 0, 0, 0, 0, 0, 333, 4070, 1247, 799, 743, 92, 88, 326, 39, 48, 0, 0, 0, 0, 0, 0, 848, 2360, 44, 439, 14, 913, 299, 223, 0, 0, 0, 0, 0, 0, 0, 0, 462, 636, 650, 1704, 378, 2271, 680, 91, 1296, 413, 0, 0, 0, 0, 0, 0, 81, 356, 3675, 454, 541, 236, 94, 449, 0, 0, 0, 0, 0, 0, 0, 0, 43, 513, 264, 1978, 768, 922, 1225, 404, 0, 0, 0, 0, 0, 0, 0, 0, 165, 128, 125, 33, 112, 199, 360, 147, 1085, 111, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6050, 7128, 7785, 5140, 8581, 5886, 6117, 2465] avg=6144.0000 max=8581 max_rank=4 min=2465 min_rank=7 imbalance_ratio=1.396647 max_load_reduction=40571 max_load_reduction_pct=0.825419 moved_to_echo=23049 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[149, 438, 1849, 762, 1782, 1165, 12, 257, 0, 0, 0, 0, 0, 0, 0, 0, 8, 43, 6, 343, 580, 2473, 41, 635, 905, 0, 0, 0, 0, 0, 0, 0, 1021, 1423, 1152, 886, 1740, 14, 441, 215, 81, 6, 0, 0, 0, 0, 0, 0, 1785, 870, 303, 657, 34, 771, 787, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 299, 95, 245, 2345, 2, 1512, 32, 124, 687, 348, 0, 0, 0, 0, 0, 0, 141, 455, 458, 429, 403, 73, 388, 713, 0, 0, 0, 0, 0, 0, 0, 0, 251, 209, 148, 1545, 12, 4271, 309, 215, 0, 0, 0, 0, 0, 0, 0, 0, 441, 461, 2, 4576, 561, 517, 424, 1365, 134, 94, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6414, 5034, 6979, 6441, 5689, 3060, 6960, 8575] avg=6144.0000 max=8575 max_rank=7 min=3060 min_rank=5 imbalance_ratio=1.395671 max_load_reduction=40577 max_load_reduction_pct=0.825541 moved_to_echo=24284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[330, 21, 136, 996, 3887, 518, 333, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 75, 747, 174, 3251, 380, 858, 193, 213, 826, 0, 0, 0, 0, 0, 0, 0, 113, 1427, 1320, 846, 162, 241, 87, 529, 176, 12, 0, 0, 0, 0, 0, 0, 1027, 475, 10, 711, 12, 704, 59, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 231, 78, 159, 510, 83, 513, 1427, 67, 1606, 317, 0, 0, 0, 0, 0, 0, 770, 566, 912, 1074, 1461, 141, 2394, 818, 0, 0, 0, 0, 0, 0, 0, 0, 452, 231, 233, 328, 268, 959, 2185, 500, 0, 0, 0, 0, 0, 0, 0, 0, 41, 13, 72, 961, 3147, 676, 244, 1334, 269, 85, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7484, 6717, 4913, 4913, 4991, 8136, 5156, 6842] avg=6144.0000 max=8136 max_rank=5 min=4913 min_rank=2 imbalance_ratio=1.324219 max_load_reduction=41016 max_load_reduction_pct=0.834473 moved_to_echo=25125 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2066, 118, 340, 229, 723, 550, 52, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1042, 67, 490, 1496, 3560, 41, 623, 260, 0, 0, 0, 0, 0, 0, 0, 453, 1088, 704, 337, 1643, 242, 100, 513, 32, 43, 0, 0, 0, 0, 0, 0, 1370, 1072, 27, 1027, 71, 223, 451, 265, 0, 0, 0, 0, 0, 0, 0, 0, 192, 779, 554, 325, 122, 154, 878, 672, 2628, 100, 0, 0, 0, 0, 0, 0, 37, 310, 3256, 303, 212, 385, 4182, 249, 0, 0, 0, 0, 0, 0, 0, 0, 56, 50, 10, 3163, 214, 953, 250, 819, 0, 0, 0, 0, 0, 0, 0, 0, 43, 196, 637, 1026, 614, 20, 626, 1769, 961, 27, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5134, 7585, 5155, 4506, 6404, 8934, 5515, 5919] avg=6144.0000 max=8934 max_rank=5 min=4506 min_rank=3 imbalance_ratio=1.454102 max_load_reduction=40218 max_load_reduction_pct=0.818237 moved_to_echo=26772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1447, 36, 268, 403, 550, 44, 358, 614, 0, 0, 0, 0, 0, 0, 0, 0, 2, 818, 253, 1860, 1345, 1915, 103, 239, 455, 0, 0, 0, 0, 0, 0, 0, 37, 1298, 345, 715, 1153, 813, 16, 498, 29, 9, 0, 0, 0, 0, 0, 0, 1471, 1506, 202, 668, 29, 371, 405, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 151, 145, 24, 525, 508, 473, 745, 656, 2420, 177, 0, 0, 0, 0, 0, 0, 1008, 150, 454, 130, 1093, 758, 2825, 387, 0, 0, 0, 0, 0, 0, 0, 0, 106, 614, 303, 25, 619, 2165, 1627, 748, 0, 0, 0, 0, 0, 0, 0, 0, 224, 110, 105, 2367, 856, 730, 522, 1961, 139, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3720, 6990, 4913, 7629, 5824, 6805, 6207, 7064] avg=6144.0000 max=7629 max_rank=3 min=3720 min_rank=0 imbalance_ratio=1.241699 max_load_reduction=41523 max_load_reduction_pct=0.844788 moved_to_echo=25900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[149, 438, 1849, 762, 1782, 1165, 12, 257, 0, 0, 0, 0, 0, 0, 0, 0, 8, 43, 6, 343, 580, 2473, 41, 635, 905, 0, 0, 0, 0, 0, 0, 0, 1021, 1423, 1152, 886, 1740, 14, 441, 215, 81, 6, 0, 0, 0, 0, 0, 0, 1785, 870, 303, 657, 34, 771, 787, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 299, 95, 245, 2345, 2, 1512, 32, 124, 687, 348, 0, 0, 0, 0, 0, 0, 141, 455, 458, 429, 403, 73, 388, 713, 0, 0, 0, 0, 0, 0, 0, 0, 251, 209, 148, 1545, 12, 4271, 309, 215, 0, 0, 0, 0, 0, 0, 0, 0, 441, 461, 2, 4576, 561, 517, 424, 1365, 134, 94, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6414, 5034, 6979, 6441, 5689, 3060, 6960, 8575] avg=6144.0000 max=8575 max_rank=7 min=3060 min_rank=5 imbalance_ratio=1.395671 max_load_reduction=40577 max_load_reduction_pct=0.825541 moved_to_echo=24284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2560, 2560, 11648, 5632, 6528, 2048, 14208, 4352, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8704, 2688, 9216, 5248, 13312, 3328, 1408, 5632, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 3968, 1792, 10240, 8064, 16000, 640, 3200, 5504, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 3712, 2304, 9600, 3072, 6528, 4608, 2688, 13056, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8704, 2688, 9216, 5248, 13312, 3328, 1408, 5632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2560, 2560, 11648, 5632, 6528, 2048, 14208, 4352, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 2432, 2048, 14208, 7296, 2688, 4736, 10496, 3456, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12032, 10496, 1792, 5376, 256, 4736, 3072, 11904, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3200, 13952, 8448, 7936, 7424, 2304, 1664, 4096, 640, 256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1664, 3840, 1152, 11136, 2560, 15744, 9344, 4096, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 3712, 2304, 9600, 3072, 6528, 4608, 2688, 13056, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3200, 13952, 8448, 7936, 7424, 2304, 1664, 4096, 640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12032, 10496, 1792, 5376, 256, 4736, 3072, 11904, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[156, 48, 232, 156, 2087, 267, 102, 306, 379, 199, 1640, 1364, 748, 2551, 133, 581, 14, 3019, 1157, 178, 1203, 458, 2401, 506, 212, 937, 222, 717, 1157, 631, 2332, 722, 72, 1146, 20, 3037, 1978, 80, 98, 1122, 452, 279, 61, 552, 419, 3012, 2929, 183, 319, 109, 449, 144, 2242, 78, 692, 429, 293, 471, 628, 580, 60, 14, 119, 270] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[4104, 118, 281, 7, 1334, 197, 65, 755, 172, 451, 411, 1596, 955, 341, 1325, 760, 527, 802, 184, 551, 1150, 216, 1479, 30, 2015, 1769, 28, 750, 47, 47, 1900, 769, 8, 414, 424, 479, 2414, 94, 62, 2072, 914, 213, 9, 676, 1161, 960, 692, 0, 540, 1102, 1345, 1476, 704, 1468, 4333, 48, 190, 105, 553, 77, 1077, 247, 23, 136] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[1865, 23, 1336, 4, 1061, 149, 249, 130, 2345, 1714, 1093, 1091, 545, 87, 143, 898, 470, 291, 457, 177, 63, 998, 952, 758, 58, 1089, 439, 135, 43, 980, 1523, 2063, 17, 178, 89, 2075, 1386, 23, 1, 1891, 782, 1240, 6, 109, 240, 919, 3846, 86, 899, 560, 1214, 2608, 224, 220, 830, 57, 647, 243, 1494, 523, 2792, 11, 389, 324] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[2962, 790, 242, 68, 1419, 67, 412, 670, 245, 304, 409, 1144, 372, 2226, 2180, 434, 151, 1276, 99, 343, 80, 312, 528, 406, 77, 2327, 525, 327, 1583, 41, 3465, 440, 1, 914, 34, 152, 3992, 53, 59, 1256, 3240, 197, 5, 19, 60, 1063, 195, 197, 448, 1675, 1023, 842, 2660, 1445, 231, 7, 881, 410, 581, 641, 545, 19, 54, 329] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[3638, 83, 41, 42, 806, 406, 73, 676, 1465, 1083, 881, 1430, 1266, 1023, 2593, 296, 268, 1432, 269, 130, 941, 1210, 254, 555, 613, 1226, 132, 285, 121, 254, 1139, 559, 215, 43, 559, 1272, 1431, 29, 107, 1649, 2078, 364, 4, 742, 1145, 2322, 707, 22, 1374, 219, 586, 1843, 248, 857, 889, 160, 76, 21, 639, 784, 2169, 584, 354, 470] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[4104, 118, 281, 7, 1334, 197, 65, 755, 172, 451, 411, 1596, 955, 341, 1325, 760, 527, 802, 184, 551, 1150, 216, 1479, 30, 2015, 1769, 28, 750, 47, 47, 1900, 769, 8, 414, 424, 479, 2414, 94, 62, 2072, 914, 213, 9, 676, 1161, 960, 692, 0, 540, 1102, 1345, 1476, 704, 1468, 4333, 48, 190, 105, 553, 77, 1077, 247, 23, 136] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[5246, 281, 107, 8, 2192, 22, 229, 133, 209, 112, 609, 5299, 165, 1590, 375, 238, 411, 3534, 794, 369, 383, 1230, 383, 276, 901, 373, 66, 886, 741, 230, 1185, 802, 152, 570, 207, 189, 2224, 52, 527, 990, 1660, 374, 7, 793, 2173, 1393, 516, 31, 365, 16, 726, 34, 1013, 3042, 33, 732, 93, 59, 326, 351, 954, 103, 23, 45] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[520, 1741, 101, 9, 1704, 449, 123, 558, 409, 69, 2117, 1264, 869, 564, 1130, 437, 69, 1624, 764, 526, 771, 1405, 1127, 4, 852, 1691, 238, 970, 201, 692, 4453, 400, 156, 709, 8, 1309, 674, 572, 605, 743, 1317, 1128, 67, 298, 946, 3450, 532, 26, 1307, 106, 441, 728, 3418, 86, 677, 458, 178, 118, 364, 3, 137, 3, 528, 209] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[2329, 259, 493, 19, 1156, 761, 139, 85, 385, 92, 1412, 1520, 156, 1342, 977, 454, 849, 686, 993, 89, 437, 220, 1240, 141, 136, 388, 20, 1213, 57, 7, 2885, 1921, 1, 703, 668, 377, 2756, 532, 105, 3833, 294, 291, 4, 252, 158, 1255, 2092, 1, 131, 2372, 655, 365, 3365, 1290, 639, 289, 75, 225, 774, 796, 753, 15, 237, 988] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[156, 48, 232, 156, 2087, 267, 102, 306, 420, 73, 53, 0, 0, 0, 0, 0, 379, 199, 1640, 297, 748, 2551, 133, 581, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3019, 1157, 178, 1203, 458, 2401, 506, 559, 0, 0, 0, 0, 0, 0, 0, 212, 937, 222, 717, 1157, 631, 1611, 722, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1146, 20, 3037, 1925, 80, 98, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 452, 279, 61, 552, 419, 2592, 2929, 183, 0, 0, 0, 0, 0, 0, 0, 0, 319, 109, 449, 144, 670, 78, 692, 429, 0, 0, 0, 0, 0, 0, 0, 0, 293, 471, 628, 580, 60, 14, 119, 270, 1067, 1572, 89, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3900, 6528, 9495, 6209, 7500, 7467, 2890, 5163] avg=6144.0000 max=9495 max_rank=2 min=2890 min_rank=6 imbalance_ratio=1.545410 max_load_reduction=39657 max_load_reduction_pct=0.806824 moved_to_echo=23020 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[520, 1741, 101, 9, 1704, 449, 123, 558, 409, 69, 2117, 1264, 869, 564, 1130, 437, 69, 1624, 764, 526, 771, 1405, 1127, 4, 852, 1691, 238, 970, 201, 692, 4453, 400, 156, 709, 8, 1309, 674, 572, 605, 743, 1317, 1128, 67, 298, 946, 3450, 532, 26, 1307, 106, 441, 728, 3418, 86, 677, 458, 178, 118, 364, 3, 137, 3, 528, 209] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[156, 48, 232, 156, 2087, 267, 102, 306, 379, 199, 1640, 1364, 748, 2551, 133, 581, 14, 3019, 1157, 178, 1203, 458, 2401, 506, 212, 937, 222, 717, 1157, 631, 2332, 722, 72, 1146, 20, 3037, 1978, 80, 98, 1122, 452, 279, 61, 552, 419, 3012, 2929, 183, 319, 109, 449, 144, 2242, 78, 692, 429, 293, 471, 628, 580, 60, 14, 119, 270] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[3638, 83, 41, 42, 806, 406, 73, 676, 1465, 1083, 881, 1430, 1266, 1023, 2593, 296, 268, 1432, 269, 130, 941, 1210, 254, 555, 613, 1226, 132, 285, 121, 254, 1139, 559, 215, 43, 559, 1272, 1431, 29, 107, 1649, 2078, 364, 4, 742, 1145, 2322, 707, 22, 1374, 219, 586, 1843, 248, 857, 889, 160, 76, 21, 639, 784, 2169, 584, 354, 470] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[1865, 23, 1336, 4, 1061, 149, 249, 130, 2345, 1714, 1093, 1091, 545, 87, 143, 898, 470, 291, 457, 177, 63, 998, 952, 758, 58, 1089, 439, 135, 43, 980, 1523, 2063, 17, 178, 89, 2075, 1386, 23, 1, 1891, 782, 1240, 6, 109, 240, 919, 3846, 86, 899, 560, 1214, 2608, 224, 220, 830, 57, 647, 243, 1494, 523, 2792, 11, 389, 324] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[2962, 790, 242, 68, 1419, 67, 412, 670, 148, 109, 108, 0, 0, 0, 0, 0, 245, 304, 409, 249, 372, 2226, 2180, 434, 0, 0, 0, 0, 0, 0, 0, 0, 151, 1276, 99, 343, 80, 312, 528, 406, 831, 0, 0, 0, 0, 0, 0, 0, 77, 2327, 525, 327, 1583, 41, 2393, 440, 0, 0, 0, 0, 0, 0, 0, 0, 1, 914, 34, 152, 3884, 53, 59, 1256, 0, 0, 0, 0, 0, 0, 0, 0, 3240, 197, 5, 19, 60, 915, 195, 197, 0, 0, 0, 0, 0, 0, 0, 0, 448, 1675, 1023, 842, 794, 1445, 231, 7, 0, 0, 0, 0, 0, 0, 0, 0, 881, 410, 581, 641, 545, 19, 54, 329, 895, 1866, 132, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6995, 6419, 4026, 7713, 6353, 4828, 6465, 6353] avg=6144.0000 max=7713 max_rank=3 min=4026 min_rank=2 imbalance_ratio=1.255371 max_load_reduction=41439 max_load_reduction_pct=0.843079 moved_to_echo=23999 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[4104, 118, 281, 7, 1334, 197, 65, 755, 133, 60, 65, 0, 0, 0, 0, 0, 172, 451, 411, 347, 955, 341, 1325, 760, 0, 0, 0, 0, 0, 0, 0, 0, 527, 802, 184, 551, 1150, 216, 1479, 30, 456, 0, 0, 0, 0, 0, 0, 0, 2015, 1769, 28, 750, 47, 47, 1312, 769, 0, 0, 0, 0, 0, 0, 0, 0, 8, 414, 424, 479, 2349, 94, 62, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 914, 213, 9, 676, 1161, 827, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 1102, 1345, 1476, 211, 1468, 4333, 48, 0, 0, 0, 0, 0, 0, 0, 0, 190, 105, 553, 77, 1077, 247, 23, 136, 1249, 493, 72, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7119, 4762, 5395, 6737, 5902, 4492, 10523, 4222] avg=6144.0000 max=10523 max_rank=6 min=4222 min_rank=7 imbalance_ratio=1.712728 max_load_reduction=38629 max_load_reduction_pct=0.785909 moved_to_echo=25139 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[5246, 281, 107, 8, 2192, 22, 229, 133, 194, 37, 60, 0, 0, 0, 0, 0, 209, 112, 609, 1151, 165, 1590, 375, 238, 0, 0, 0, 0, 0, 0, 0, 0, 411, 3534, 794, 369, 383, 1230, 383, 276, 284, 0, 0, 0, 0, 0, 0, 0, 901, 373, 66, 886, 741, 230, 819, 802, 0, 0, 0, 0, 0, 0, 0, 0, 152, 570, 207, 189, 2164, 52, 527, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 374, 7, 793, 2173, 1199, 516, 31, 0, 0, 0, 0, 0, 0, 0, 0, 365, 16, 726, 34, 303, 3042, 33, 732, 0, 0, 0, 0, 0, 0, 0, 0, 93, 59, 326, 351, 954, 103, 23, 45, 4148, 710, 45, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8509, 4449, 7664, 4818, 4851, 6753, 5251, 6857] avg=6144.0000 max=8509 max_rank=0 min=4449 min_rank=1 imbalance_ratio=1.384928 max_load_reduction=40643 max_load_reduction_pct=0.826884 moved_to_echo=23712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[4104, 118, 281, 7, 1334, 197, 65, 755, 133, 60, 65, 0, 0, 0, 0, 0, 172, 451, 411, 347, 955, 341, 1325, 760, 0, 0, 0, 0, 0, 0, 0, 0, 527, 802, 184, 551, 1150, 216, 1479, 30, 456, 0, 0, 0, 0, 0, 0, 0, 2015, 1769, 28, 750, 47, 47, 1312, 769, 0, 0, 0, 0, 0, 0, 0, 0, 8, 414, 424, 479, 2349, 94, 62, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 914, 213, 9, 676, 1161, 827, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 1102, 1345, 1476, 211, 1468, 4333, 48, 0, 0, 0, 0, 0, 0, 0, 0, 190, 105, 553, 77, 1077, 247, 23, 136, 1249, 493, 72, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7119, 4762, 5395, 6737, 5902, 4492, 10523, 4222] avg=6144.0000 max=10523 max_rank=6 min=4222 min_rank=7 imbalance_ratio=1.712728 max_load_reduction=38629 max_load_reduction_pct=0.785909 moved_to_echo=25139 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[2329, 259, 493, 19, 1156, 761, 139, 85, 175, 91, 74, 0, 0, 0, 0, 0, 385, 92, 1412, 330, 156, 1342, 977, 454, 0, 0, 0, 0, 0, 0, 0, 0, 849, 686, 993, 89, 437, 220, 1240, 141, 692, 0, 0, 0, 0, 0, 0, 0, 136, 388, 20, 1213, 57, 7, 1992, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 1, 703, 668, 377, 2682, 532, 105, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 294, 291, 4, 252, 158, 1080, 2092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 131, 2372, 655, 365, 1005, 1290, 639, 289, 0, 0, 0, 0, 0, 0, 0, 0, 75, 225, 774, 796, 753, 15, 237, 988, 1190, 2360, 110, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5581, 5148, 5347, 5734, 8901, 4172, 6746, 7523] avg=6144.0000 max=8901 max_rank=4 min=4172 min_rank=5 imbalance_ratio=1.448730 max_load_reduction=40251 max_load_reduction_pct=0.818909 moved_to_echo=27342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[520, 1741, 101, 9, 1704, 449, 123, 558, 481, 141, 18, 0, 0, 0, 0, 0, 409, 69, 2117, 275, 869, 564, 1130, 437, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1624, 764, 526, 771, 1405, 1127, 4, 1068, 0, 0, 0, 0, 0, 0, 0, 852, 1691, 238, 970, 201, 692, 3074, 400, 0, 0, 0, 0, 0, 0, 0, 0, 156, 709, 8, 1309, 656, 572, 605, 743, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 1128, 67, 298, 946, 2969, 532, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1307, 106, 441, 728, 1021, 86, 677, 458, 0, 0, 0, 0, 0, 0, 0, 0, 178, 118, 364, 3, 137, 3, 528, 209, 989, 2397, 170, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5845, 5870, 7358, 8118, 4758, 7283, 4824, 5096] avg=6144.0000 max=8118 max_rank=3 min=4758 min_rank=4 imbalance_ratio=1.321289 max_load_reduction=41034 max_load_reduction_pct=0.834839 moved_to_echo=21961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[156, 48, 232, 156, 2087, 267, 102, 306, 420, 73, 53, 0, 0, 0, 0, 0, 379, 199, 1640, 297, 748, 2551, 133, 581, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3019, 1157, 178, 1203, 458, 2401, 506, 559, 0, 0, 0, 0, 0, 0, 0, 212, 937, 222, 717, 1157, 631, 1611, 722, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1146, 20, 3037, 1925, 80, 98, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 452, 279, 61, 552, 419, 2592, 2929, 183, 0, 0, 0, 0, 0, 0, 0, 0, 319, 109, 449, 144, 670, 78, 692, 429, 0, 0, 0, 0, 0, 0, 0, 0, 293, 471, 628, 580, 60, 14, 119, 270, 1067, 1572, 89, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3900, 6528, 9495, 6209, 7500, 7467, 2890, 5163] avg=6144.0000 max=9495 max_rank=2 min=2890 min_rank=6 imbalance_ratio=1.545410 max_load_reduction=39657 max_load_reduction_pct=0.806824 moved_to_echo=23020 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[3638, 83, 41, 42, 806, 406, 73, 676, 324, 36, 38, 0, 0, 0, 0, 0, 1465, 1083, 881, 311, 1266, 1023, 2593, 296, 0, 0, 0, 0, 0, 0, 0, 0, 268, 1432, 269, 130, 941, 1210, 254, 555, 273, 0, 0, 0, 0, 0, 0, 0, 613, 1226, 132, 285, 121, 254, 787, 559, 0, 0, 0, 0, 0, 0, 0, 0, 215, 43, 559, 1272, 1393, 29, 107, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 2078, 364, 4, 742, 1145, 1998, 707, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 219, 586, 1843, 75, 857, 889, 160, 0, 0, 0, 0, 0, 0, 0, 0, 76, 21, 639, 784, 2169, 584, 354, 470, 1119, 173, 43, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 8918, 5332, 3977, 5267, 7060, 6003, 6432] avg=6144.0000 max=8918 max_rank=1 min=3977 min_rank=3 imbalance_ratio=1.451497 max_load_reduction=40234 max_load_reduction_pct=0.818563 moved_to_echo=24762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[1865, 23, 1336, 4, 1061, 149, 249, 130, 131, 52, 40, 0, 0, 0, 0, 0, 2345, 1714, 1093, 233, 545, 87, 143, 898, 0, 0, 0, 0, 0, 0, 0, 0, 470, 291, 457, 177, 63, 998, 952, 758, 369, 0, 0, 0, 0, 0, 0, 0, 58, 1089, 439, 135, 43, 980, 1039, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 17, 178, 89, 2075, 1346, 23, 1, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 782, 1240, 6, 109, 240, 788, 3846, 86, 0, 0, 0, 0, 0, 0, 0, 0, 899, 560, 1214, 2608, 62, 220, 830, 57, 0, 0, 0, 0, 0, 0, 0, 0, 647, 243, 1494, 523, 2792, 11, 389, 324, 858, 162, 63, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5040, 7058, 4535, 5846, 5620, 7097, 6450, 7506] avg=6144.0000 max=7506 max_rank=7 min=4535 min_rank=2 imbalance_ratio=1.221680 max_load_reduction=41646 max_load_reduction_pct=0.847290 moved_to_echo=26673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[520, 1741, 101, 9, 1704, 449, 123, 558, 481, 141, 18, 0, 0, 0, 0, 0, 409, 69, 2117, 275, 869, 564, 1130, 437, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1624, 764, 526, 771, 1405, 1127, 4, 1068, 0, 0, 0, 0, 0, 0, 0, 852, 1691, 238, 970, 201, 692, 3074, 400, 0, 0, 0, 0, 0, 0, 0, 0, 156, 709, 8, 1309, 656, 572, 605, 743, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 1128, 67, 298, 946, 2969, 532, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1307, 106, 441, 728, 1021, 86, 677, 458, 0, 0, 0, 0, 0, 0, 0, 0, 178, 118, 364, 3, 137, 3, 528, 209, 989, 2397, 170, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5845, 5870, 7358, 8118, 4758, 7283, 4824, 5096] avg=6144.0000 max=8118 max_rank=3 min=4758 min_rank=4 imbalance_ratio=1.321289 max_load_reduction=41034 max_load_reduction_pct=0.834839 moved_to_echo=21961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[2329, 259, 493, 19, 1156, 761, 139, 85, 175, 91, 74, 0, 0, 0, 0, 0, 385, 92, 1412, 330, 156, 1342, 977, 454, 0, 0, 0, 0, 0, 0, 0, 0, 849, 686, 993, 89, 437, 220, 1240, 141, 692, 0, 0, 0, 0, 0, 0, 0, 136, 388, 20, 1213, 57, 7, 1992, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 1, 703, 668, 377, 2682, 532, 105, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 294, 291, 4, 252, 158, 1080, 2092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 131, 2372, 655, 365, 1005, 1290, 639, 289, 0, 0, 0, 0, 0, 0, 0, 0, 75, 225, 774, 796, 753, 15, 237, 988, 1190, 2360, 110, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5581, 5148, 5347, 5734, 8901, 4172, 6746, 7523] avg=6144.0000 max=8901 max_rank=4 min=4172 min_rank=5 imbalance_ratio=1.448730 max_load_reduction=40251 max_load_reduction_pct=0.818909 moved_to_echo=27342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 1664, 5376, 3840, 8576, 1024, 1792, 2816, 11520, 9856, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5632, 4096, 8576, 3200, 5120, 9728, 8960, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[20864, 3456, 2944, 384, 11776, 2432, 1408, 3328, 2048, 640, 512, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4864, 9856, 1792, 5376, 3968, 2944, 13056, 7680, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 1664, 5376, 3840, 8576, 1024, 1792, 2816, 11520, 9856, 768, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5504, 6272, 6528, 8064, 4224, 8576, 8448, 2304, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10752, 4096, 256, 3456, 6400, 12416, 11520, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[20864, 3456, 2944, 384, 11776, 2432, 1408, 3328, 2048, 640, 512, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 4736, 2048, 8960, 16512, 1536, 1664, 13568, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5632, 4096, 8576, 3200, 5120, 9728, 8960, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10752, 4096, 256, 3456, 6400, 12416, 11520, 640, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 12672, 4736, 2432, 5120, 6144, 8448, 2688, 4608, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True - [2026-08-26 15:39:17] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 846.4 | throughput per GPU (TFLOP/s/GPU): 150.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.092003E+01 | loss scale: 1.0 | grad norm: 30.566 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1402, 812, 527, 574, 544, 933, 1303, 251, 842, 1195, 1287, 527, 146, 1388, 434, 1020, 1115, 865, 750, 830, 585, 413, 979, 601, 842, 820, 304, 469, 876, 806, 457, 918, 863, 503, 788, 883, 513, 383, 876, 717, 456, 572, 329, 521, 1400, 1610, 828, 317, 528, 707, 867, 850, 1116, 1600, 746, 625, 522, 854, 1268, 789, 462, 427, 1164, 253] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1471, 689, 494, 636, 630, 927, 1365, 188, 644, 1199, 1198, 459, 137, 1317, 483, 961, 1160, 888, 750, 864, 589, 484, 951, 594, 789, 685, 318, 394, 836, 1115, 404, 950, 1014, 435, 725, 887, 437, 375, 980, 685, 454, 565, 380, 511, 1373, 1494, 748, 356, 607, 745, 731, 844, 1001, 1704, 800, 552, 551, 1035, 1341, 851, 501, 390, 1189, 322] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 + [2026-08-26 16:35:19] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 839.4 | throughput per GPU (TFLOP/s/GPU): 152.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.092003E+01 | loss scale: 1.0 | grad norm: 30.566 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1452, 729, 512, 511, 592, 911, 1263, 261, 634, 1194, 1207, 410, 170, 1281, 449, 1021, 1334, 852, 737, 842, 628, 490, 990, 543, 894, 742, 311, 447, 826, 1059, 435, 911, 768, 439, 769, 835, 431, 391, 932, 780, 411, 538, 424, 797, 1405, 1548, 805, 350, 631, 644, 783, 797, 1029, 1925, 732, 554, 495, 860, 1311, 733, 559, 416, 1176, 246] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1464, 694, 528, 606, 525, 864, 1325, 194, 825, 1133, 1294, 418, 152, 1278, 465, 985, 1017, 889, 750, 781, 649, 468, 1025, 572, 788, 814, 261, 488, 871, 1011, 376, 934, 984, 441, 873, 872, 533, 398, 947, 787, 456, 501, 353, 477, 1539, 1544, 829, 360, 548, 694, 765, 786, 1157, 1550, 828, 544, 473, 1025, 1366, 764, 520, 393, 1146, 255] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1403, 671, 540, 598, 471, 850, 1298, 236, 786, 1231, 1294, 434, 118, 1434, 423, 986, 1203, 812, 809, 832, 605, 506, 850, 571, 806, 847, 311, 476, 803, 1034, 466, 902, 867, 442, 737, 909, 417, 366, 851, 919, 500, 541, 384, 448, 1525, 1542, 742, 315, 611, 761, 738, 844, 1124, 1603, 820, 544, 617, 924, 1305, 755, 488, 436, 1210, 261] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1452, 729, 512, 511, 592, 911, 1263, 261, 634, 1194, 1207, 410, 170, 1281, 449, 1021, 1334, 852, 737, 842, 628, 490, 990, 543, 894, 742, 311, 447, 826, 1059, 435, 911, 768, 439, 769, 835, 431, 391, 932, 780, 411, 538, 424, 797, 1405, 1548, 805, 350, 631, 644, 783, 797, 1029, 1925, 732, 554, 495, 860, 1311, 733, 559, 416, 1176, 246] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1445, 737, 565, 530, 662, 808, 1413, 259, 792, 1060, 1291, 390, 142, 1423, 497, 914, 1017, 813, 798, 798, 652, 507, 1064, 664, 732, 845, 397, 483, 885, 1033, 458, 913, 783, 487, 738, 748, 552, 379, 879, 745, 512, 552, 383, 675, 1436, 1477, 623, 340, 655, 668, 725, 939, 964, 1593, 699, 452, 573, 971, 1309, 793, 649, 465, 1107, 294] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1402, 812, 527, 574, 544, 933, 1303, 251, 842, 1195, 1287, 527, 146, 1388, 434, 1020, 1115, 865, 750, 830, 585, 413, 979, 601, 842, 820, 304, 469, 876, 806, 457, 918, 863, 503, 788, 883, 513, 383, 876, 717, 456, 572, 329, 521, 1400, 1610, 828, 317, 528, 707, 867, 850, 1116, 1600, 746, 625, 522, 854, 1268, 789, 462, 427, 1164, 253] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1471, 689, 494, 636, 630, 927, 1365, 188, 644, 1199, 1198, 459, 137, 1317, 483, 961, 1160, 888, 750, 864, 589, 484, 951, 594, 789, 685, 318, 394, 836, 1115, 404, 950, 1014, 435, 725, 887, 437, 375, 980, 685, 454, 565, 380, 511, 1373, 1494, 748, 356, 607, 745, 731, 844, 1001, 1704, 800, 552, 551, 1035, 1341, 851, 501, 390, 1189, 322] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1377, 781, 691, 594, 607, 835, 1284, 180, 679, 861, 1238, 560, 157, 1400, 516, 1006, 1141, 908, 698, 861, 614, 476, 1001, 532, 709, 804, 290, 409, 882, 961, 381, 973, 863, 505, 758, 910, 478, 381, 1006, 849, 482, 543, 379, 489, 1595, 1589, 795, 353, 467, 642, 741, 884, 1066, 1574, 717, 534, 456, 1077, 1388, 804, 512, 415, 1182, 312] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1283, 718, 575, 639, 614, 892, 1358, 163, 716, 774, 1282, 454, 125, 1376, 446, 938, 1041, 848, 689, 806, 625, 539, 1059, 548, 815, 783, 294, 368, 889, 1041, 478, 936, 1152, 442, 919, 815, 499, 385, 883, 774, 496, 590, 362, 774, 1388, 1550, 809, 321, 474, 716, 681, 852, 1173, 1668, 839, 574, 573, 981, 1217, 795, 563, 372, 1111, 292] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1445, 737, 565, 530, 662, 808, 1413, 259, 792, 1060, 1291, 390, 142, 1423, 497, 914, 1017, 813, 798, 798, 652, 507, 1064, 664, 732, 845, 397, 483, 885, 1033, 458, 913, 783, 487, 738, 748, 552, 379, 879, 745, 512, 552, 383, 675, 1436, 1477, 623, 340, 655, 668, 725, 939, 964, 1593, 699, 452, 573, 971, 1309, 793, 649, 465, 1107, 294] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1266, 812, 527, 574, 544, 933, 1303, 251, 0, 0, 0, 0, 0, 0, 0, 0, 842, 1195, 1287, 527, 146, 1039, 434, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 865, 750, 830, 585, 413, 979, 601, 0, 0, 0, 0, 0, 0, 0, 0, 842, 820, 304, 469, 876, 806, 457, 918, 556, 0, 0, 0, 0, 0, 0, 0, 863, 503, 788, 883, 513, 383, 876, 717, 349, 191, 17, 0, 0, 0, 0, 0, 456, 572, 329, 521, 1400, 1610, 828, 317, 48, 0, 0, 0, 0, 0, 0, 0, 528, 707, 867, 850, 1116, 853, 746, 625, 0, 0, 0, 0, 0, 0, 0, 0, 522, 854, 1268, 789, 462, 427, 1164, 253, 119, 40, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6210, 6490, 6050, 6048, 6083, 6081, 6292, 5898] avg=6144.0000 max=6490 max_rank=1 min=5898 min_rank=7 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1267, 671, 540, 598, 471, 850, 1298, 236, 0, 0, 0, 0, 0, 0, 0, 0, 786, 1231, 1294, 434, 118, 1073, 423, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 812, 809, 832, 605, 506, 850, 571, 0, 0, 0, 0, 0, 0, 0, 0, 806, 847, 311, 476, 803, 1034, 466, 902, 557, 0, 0, 0, 0, 0, 0, 0, 867, 442, 737, 909, 417, 366, 851, 919, 361, 191, 17, 0, 0, 0, 0, 0, 500, 541, 384, 448, 1525, 1542, 742, 315, 52, 0, 0, 0, 0, 0, 0, 0, 611, 761, 738, 844, 1124, 855, 820, 544, 0, 0, 0, 0, 0, 0, 0, 0, 617, 924, 1305, 755, 488, 436, 1210, 261, 119, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5931, 6345, 6093, 6202, 6077, 6049, 6297, 6158] avg=6144.0000 max=6345 max_rank=1 min=5931 min_rank=0 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1329, 689, 494, 636, 630, 927, 1365, 188, 0, 0, 0, 0, 0, 0, 0, 0, 644, 1199, 1198, 459, 137, 986, 483, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 888, 750, 864, 589, 484, 951, 594, 0, 0, 0, 0, 0, 0, 0, 0, 789, 685, 318, 394, 836, 1115, 404, 950, 592, 0, 0, 0, 0, 0, 0, 0, 1014, 435, 725, 887, 437, 375, 980, 685, 331, 204, 18, 0, 0, 0, 0, 0, 454, 565, 380, 511, 1373, 1494, 748, 356, 50, 0, 0, 0, 0, 0, 0, 0, 607, 745, 731, 844, 1001, 908, 800, 552, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1035, 1341, 851, 501, 390, 1189, 322, 124, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6258, 6067, 6188, 6083, 6091, 5931, 6188, 6346] avg=6144.0000 max=6346 max_rank=7 min=5931 min_rank=5 imbalance_ratio=1.032878 max_load_reduction=42806 max_load_reduction_pct=0.870890 moved_to_echo=24556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1311, 729, 512, 511, 592, 911, 1263, 261, 0, 0, 0, 0, 0, 0, 0, 0, 634, 1194, 1207, 410, 170, 959, 449, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1229, 852, 737, 842, 628, 490, 990, 543, 0, 0, 0, 0, 0, 0, 0, 0, 894, 742, 311, 447, 826, 1059, 435, 911, 669, 0, 0, 0, 0, 0, 0, 0, 768, 439, 769, 835, 431, 391, 932, 780, 322, 230, 18, 0, 0, 0, 0, 0, 411, 538, 424, 797, 1405, 1548, 805, 350, 57, 0, 0, 0, 0, 0, 0, 0, 631, 644, 783, 797, 1029, 1026, 732, 554, 0, 0, 0, 0, 0, 0, 0, 0, 495, 860, 1311, 733, 559, 416, 1176, 246, 123, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6090, 6044, 6311, 6294, 5915, 6335, 6196, 5967] avg=6144.0000 max=6335 max_rank=5 min=5915 min_rank=4 imbalance_ratio=1.031087 max_load_reduction=42817 max_load_reduction_pct=0.871114 moved_to_echo=24413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1322, 694, 528, 606, 525, 864, 1325, 194, 0, 0, 0, 0, 0, 0, 0, 0, 825, 1133, 1294, 418, 152, 957, 465, 985, 0, 0, 0, 0, 0, 0, 0, 0, 936, 889, 750, 781, 649, 468, 1025, 572, 0, 0, 0, 0, 0, 0, 0, 0, 788, 814, 261, 488, 871, 1011, 376, 934, 539, 0, 0, 0, 0, 0, 0, 0, 984, 441, 873, 872, 533, 398, 947, 787, 321, 185, 18, 0, 0, 0, 0, 0, 456, 501, 353, 477, 1539, 1544, 829, 360, 44, 0, 0, 0, 0, 0, 0, 0, 548, 694, 765, 786, 1157, 826, 828, 544, 0, 0, 0, 0, 0, 0, 0, 0, 473, 1025, 1366, 764, 520, 393, 1146, 255, 124, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6058, 6229, 6070, 6082, 6359, 6103, 6148, 6103] avg=6144.0000 max=6359 max_rank=4 min=6058 min_rank=0 imbalance_ratio=1.034993 max_load_reduction=42793 max_load_reduction_pct=0.870626 moved_to_echo=24713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1329, 689, 494, 636, 630, 927, 1365, 188, 0, 0, 0, 0, 0, 0, 0, 0, 644, 1199, 1198, 459, 137, 986, 483, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 888, 750, 864, 589, 484, 951, 594, 0, 0, 0, 0, 0, 0, 0, 0, 789, 685, 318, 394, 836, 1115, 404, 950, 592, 0, 0, 0, 0, 0, 0, 0, 1014, 435, 725, 887, 437, 375, 980, 685, 331, 204, 18, 0, 0, 0, 0, 0, 454, 565, 380, 511, 1373, 1494, 748, 356, 50, 0, 0, 0, 0, 0, 0, 0, 607, 745, 731, 844, 1001, 908, 800, 552, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1035, 1341, 851, 501, 390, 1189, 322, 124, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6258, 6067, 6188, 6083, 6091, 5931, 6188, 6346] avg=6144.0000 max=6346 max_rank=7 min=5931 min_rank=5 imbalance_ratio=1.032878 max_load_reduction=42806 max_load_reduction_pct=0.870890 moved_to_echo=24556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1267, 671, 540, 598, 471, 850, 1298, 236, 0, 0, 0, 0, 0, 0, 0, 0, 786, 1231, 1294, 434, 118, 1073, 423, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 812, 809, 832, 605, 506, 850, 571, 0, 0, 0, 0, 0, 0, 0, 0, 806, 847, 311, 476, 803, 1034, 466, 902, 557, 0, 0, 0, 0, 0, 0, 0, 867, 442, 737, 909, 417, 366, 851, 919, 361, 191, 17, 0, 0, 0, 0, 0, 500, 541, 384, 448, 1525, 1542, 742, 315, 52, 0, 0, 0, 0, 0, 0, 0, 611, 761, 738, 844, 1124, 855, 820, 544, 0, 0, 0, 0, 0, 0, 0, 0, 617, 924, 1305, 755, 488, 436, 1210, 261, 119, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5931, 6345, 6093, 6202, 6077, 6049, 6297, 6158] avg=6144.0000 max=6345 max_rank=1 min=5931 min_rank=0 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1305, 737, 565, 530, 662, 808, 1413, 259, 0, 0, 0, 0, 0, 0, 0, 0, 792, 1060, 1291, 390, 142, 1065, 497, 914, 0, 0, 0, 0, 0, 0, 0, 0, 936, 813, 798, 798, 652, 507, 1064, 664, 0, 0, 0, 0, 0, 0, 0, 0, 732, 845, 397, 483, 885, 1033, 458, 913, 554, 0, 0, 0, 0, 0, 0, 0, 783, 487, 738, 748, 552, 379, 879, 745, 358, 190, 18, 0, 0, 0, 0, 0, 512, 552, 383, 675, 1436, 1477, 623, 340, 44, 0, 0, 0, 0, 0, 0, 0, 655, 668, 725, 939, 964, 849, 699, 452, 0, 0, 0, 0, 0, 0, 0, 0, 573, 971, 1309, 793, 649, 465, 1107, 294, 122, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6279, 6151, 6232, 6300, 5877, 6042, 5951, 6320] avg=6144.0000 max=6320 max_rank=7 min=5877 min_rank=4 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24190 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1266, 812, 527, 574, 544, 933, 1303, 251, 0, 0, 0, 0, 0, 0, 0, 0, 842, 1195, 1287, 527, 146, 1039, 434, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 865, 750, 830, 585, 413, 979, 601, 0, 0, 0, 0, 0, 0, 0, 0, 842, 820, 304, 469, 876, 806, 457, 918, 556, 0, 0, 0, 0, 0, 0, 0, 863, 503, 788, 883, 513, 383, 876, 717, 349, 191, 17, 0, 0, 0, 0, 0, 456, 572, 329, 521, 1400, 1610, 828, 317, 48, 0, 0, 0, 0, 0, 0, 0, 528, 707, 867, 850, 1116, 853, 746, 625, 0, 0, 0, 0, 0, 0, 0, 0, 522, 854, 1268, 789, 462, 427, 1164, 253, 119, 40, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6210, 6490, 6050, 6048, 6083, 6081, 6292, 5898] avg=6144.0000 max=6490 max_rank=1 min=5898 min_rank=7 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1159, 718, 575, 639, 614, 892, 1358, 163, 0, 0, 0, 0, 0, 0, 0, 0, 716, 774, 1282, 454, 125, 1030, 446, 938, 0, 0, 0, 0, 0, 0, 0, 0, 959, 848, 689, 806, 625, 539, 1059, 548, 0, 0, 0, 0, 0, 0, 0, 0, 815, 783, 294, 368, 889, 1041, 478, 936, 580, 0, 0, 0, 0, 0, 0, 0, 1152, 442, 919, 815, 499, 385, 883, 774, 346, 199, 16, 0, 0, 0, 0, 0, 496, 590, 362, 774, 1388, 1550, 809, 321, 45, 0, 0, 0, 0, 0, 0, 0, 474, 716, 681, 852, 1173, 889, 839, 574, 0, 0, 0, 0, 0, 0, 0, 0, 573, 981, 1217, 795, 563, 372, 1111, 292, 108, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 5765, 6073, 6184, 6430, 6335, 6198, 6049] avg=6144.0000 max=6430 max_rank=4 min=5765 min_rank=1 imbalance_ratio=1.046549 max_load_reduction=42722 max_load_reduction_pct=0.869181 moved_to_echo=25012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1236, 781, 691, 594, 607, 835, 1284, 180, 0, 0, 0, 0, 0, 0, 0, 0, 679, 861, 1238, 560, 157, 1044, 516, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 908, 698, 861, 614, 476, 1001, 532, 0, 0, 0, 0, 0, 0, 0, 0, 709, 804, 290, 409, 882, 961, 381, 973, 550, 0, 0, 0, 0, 0, 0, 0, 863, 505, 758, 910, 478, 381, 1006, 849, 356, 193, 21, 0, 0, 0, 0, 0, 482, 543, 379, 489, 1595, 1589, 795, 353, 51, 0, 0, 0, 0, 0, 0, 0, 467, 642, 741, 884, 1066, 831, 717, 534, 0, 0, 0, 0, 0, 0, 0, 0, 456, 1077, 1388, 804, 512, 415, 1182, 312, 120, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6208, 6061, 6135, 5959, 6320, 6276, 5882, 6311] avg=6144.0000 max=6320 max_rank=4 min=5882 min_rank=6 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1305, 737, 565, 530, 662, 808, 1413, 259, 0, 0, 0, 0, 0, 0, 0, 0, 792, 1060, 1291, 390, 142, 1065, 497, 914, 0, 0, 0, 0, 0, 0, 0, 0, 936, 813, 798, 798, 652, 507, 1064, 664, 0, 0, 0, 0, 0, 0, 0, 0, 732, 845, 397, 483, 885, 1033, 458, 913, 554, 0, 0, 0, 0, 0, 0, 0, 783, 487, 738, 748, 552, 379, 879, 745, 358, 190, 18, 0, 0, 0, 0, 0, 512, 552, 383, 675, 1436, 1477, 623, 340, 44, 0, 0, 0, 0, 0, 0, 0, 655, 668, 725, 939, 964, 849, 699, 452, 0, 0, 0, 0, 0, 0, 0, 0, 573, 971, 1309, 793, 649, 465, 1107, 294, 122, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6279, 6151, 6232, 6300, 5877, 6042, 5951, 6320] avg=6144.0000 max=6320 max_rank=7 min=5877 min_rank=4 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24190 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 8704, 10112, 3712, 1152, 8192, 3840, 7936, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6400, 6400, 2560, 3584, 6912, 8064, 3456, 7552, 4608, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7296, 3712, 6400, 6912, 3968, 3072, 7424, 6272, 2816, 1664, 256, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3840, 4480, 3072, 4736, 11776, 12416, 6272, 2816, 512, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10240, 5888, 4480, 4736, 4736, 7040, 10624, 1792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7296, 3712, 6400, 6912, 3968, 3072, 7424, 6272, 2816, 1664, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6400, 6400, 2560, 3584, 6912, 8064, 3456, 7552, 4608, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 5632, 6144, 6912, 8704, 7040, 6272, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4352, 7808, 10624, 6400, 4352, 3328, 9344, 2304, 1024, 384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8320, 6912, 6016, 6656, 4992, 3968, 7936, 4736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4352, 7808, 10624, 6400, 4352, 3328, 9344, 2304, 1024, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3840, 4480, 3072, 4736, 11776, 12416, 6272, 2816, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[203, 424, 332, 530, 1281, 642, 950, 590, 260, 267, 1238, 317, 1153, 576, 852, 2286, 810, 1127, 895, 566, 908, 1030, 1468, 699, 602, 729, 838, 1837, 170, 882, 517, 691, 287, 326, 491, 601, 962, 252, 392, 921, 1569, 793, 37, 733, 343, 458, 41, 799, 3221, 441, 1388, 109, 790, 1143, 842, 720, 619, 902, 674, 502, 1714, 437, 516, 459] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[252, 369, 687, 537, 1216, 572, 933, 533, 290, 327, 859, 254, 1193, 463, 901, 2218, 849, 1285, 970, 744, 882, 900, 1341, 691, 648, 777, 893, 1900, 263, 878, 536, 573, 319, 339, 463, 569, 1061, 224, 329, 983, 1650, 569, 20, 832, 385, 507, 39, 878, 3274, 393, 1395, 63, 591, 1150, 758, 788, 565, 960, 567, 585, 1730, 384, 613, 435] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[262, 440, 354, 616, 1291, 654, 846, 600, 267, 348, 1178, 242, 1126, 573, 947, 2065, 933, 1206, 985, 716, 872, 938, 1501, 624, 622, 771, 963, 1811, 252, 825, 502, 628, 353, 318, 414, 548, 909, 321, 390, 898, 1683, 609, 51, 749, 395, 470, 38, 640, 3113, 389, 1458, 121, 765, 1008, 783, 800, 589, 930, 756, 508, 1648, 401, 682, 457] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[183, 606, 195, 603, 1337, 676, 892, 634, 306, 312, 1153, 221, 1078, 547, 857, 2278, 946, 1245, 888, 424, 948, 1044, 1283, 773, 622, 767, 781, 1852, 163, 786, 440, 720, 353, 310, 436, 558, 1092, 279, 401, 808, 1576, 824, 29, 836, 440, 513, 52, 622, 3293, 460, 1322, 92, 723, 1146, 794, 855, 560, 862, 715, 490, 1844, 371, 464, 472] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[280, 522, 341, 550, 1233, 562, 935, 485, 268, 360, 1322, 244, 1199, 587, 863, 2263, 838, 1356, 930, 519, 997, 962, 1429, 703, 656, 842, 903, 1721, 224, 873, 532, 649, 367, 322, 484, 655, 1020, 335, 329, 927, 1472, 798, 34, 722, 323, 449, 48, 857, 3002, 433, 1221, 152, 615, 1037, 812, 944, 534, 932, 483, 485, 1849, 373, 540, 450] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[212, 442, 462, 498, 1236, 753, 931, 604, 247, 386, 1235, 204, 1256, 630, 913, 2239, 832, 1197, 899, 460, 905, 917, 1216, 664, 594, 810, 925, 1894, 183, 843, 497, 552, 392, 395, 486, 604, 1087, 362, 387, 928, 1529, 784, 39, 769, 333, 486, 52, 758, 3054, 342, 1380, 97, 707, 1085, 792, 889, 502, 954, 497, 637, 1822, 446, 532, 389] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[262, 440, 354, 616, 1291, 654, 846, 600, 267, 348, 1178, 242, 1126, 573, 947, 2065, 933, 1206, 985, 716, 872, 938, 1501, 624, 622, 771, 963, 1811, 252, 825, 502, 628, 353, 318, 414, 548, 909, 321, 390, 898, 1683, 609, 51, 749, 395, 470, 38, 640, 3113, 389, 1458, 121, 765, 1008, 783, 800, 589, 930, 756, 508, 1648, 401, 682, 457] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[228, 423, 488, 586, 1282, 521, 794, 533, 314, 402, 1239, 321, 1236, 614, 895, 2013, 1008, 1263, 1012, 966, 1008, 1079, 1305, 591, 595, 797, 862, 1996, 236, 859, 552, 463, 392, 342, 468, 509, 1009, 260, 374, 982, 1618, 618, 55, 772, 298, 432, 26, 599, 3010, 434, 1415, 133, 675, 981, 708, 870, 607, 878, 660, 502, 1546, 390, 621, 487] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[203, 424, 332, 530, 1281, 642, 950, 590, 260, 267, 1238, 317, 1153, 576, 852, 2286, 810, 1127, 895, 566, 908, 1030, 1468, 699, 602, 729, 838, 1837, 170, 882, 517, 691, 287, 326, 491, 601, 962, 252, 392, 921, 1569, 793, 37, 733, 343, 458, 41, 799, 3221, 441, 1388, 109, 790, 1143, 842, 720, 619, 902, 674, 502, 1714, 437, 516, 459] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[212, 442, 462, 498, 1236, 753, 931, 604, 247, 386, 1235, 204, 1256, 630, 913, 2239, 832, 1197, 899, 460, 905, 917, 1216, 664, 594, 810, 925, 1894, 183, 843, 497, 552, 392, 395, 486, 604, 1087, 362, 387, 928, 1529, 784, 39, 769, 333, 486, 52, 758, 3054, 342, 1380, 97, 707, 1085, 792, 889, 502, 954, 497, 637, 1822, 446, 532, 389] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[257, 531, 731, 567, 1250, 581, 974, 644, 298, 431, 1140, 293, 1085, 560, 715, 1927, 892, 1207, 870, 306, 916, 1224, 1403, 554, 679, 807, 974, 1784, 221, 859, 523, 651, 267, 341, 405, 704, 1210, 282, 300, 978, 1519, 808, 41, 661, 259, 505, 64, 618, 3212, 514, 1451, 128, 830, 1113, 774, 896, 408, 816, 785, 564, 1501, 396, 506, 442] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[262, 440, 354, 616, 1291, 654, 846, 600, 568, 510, 0, 0, 0, 0, 0, 0, 267, 348, 1178, 242, 1126, 573, 947, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 933, 1106, 985, 716, 872, 938, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 622, 771, 963, 1610, 252, 825, 502, 628, 0, 0, 0, 0, 0, 0, 0, 0, 353, 318, 414, 548, 909, 321, 390, 898, 1773, 0, 0, 0, 0, 0, 0, 0, 1683, 609, 51, 749, 395, 470, 38, 640, 991, 543, 0, 0, 0, 0, 0, 0, 797, 389, 1458, 121, 765, 1008, 783, 800, 0, 0, 0, 0, 0, 0, 0, 0, 589, 930, 756, 508, 1648, 401, 682, 457, 201, 100, 88, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6090, 6174, 6173, 5924, 6169, 6121, 6360] avg=6144.0000 max=6360 max_rank=7 min=5924 min_rank=4 imbalance_ratio=1.035156 max_load_reduction=42792 max_load_reduction_pct=0.870605 moved_to_echo=24574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[252, 369, 687, 537, 1216, 572, 933, 533, 610, 455, 0, 0, 0, 0, 0, 0, 290, 327, 859, 254, 1193, 463, 901, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 849, 1179, 970, 744, 882, 900, 0, 691, 0, 0, 0, 0, 0, 0, 0, 0, 648, 777, 893, 1690, 263, 878, 536, 573, 0, 0, 0, 0, 0, 0, 0, 0, 319, 339, 463, 569, 1061, 224, 329, 983, 1864, 0, 0, 0, 0, 0, 0, 0, 1650, 569, 20, 832, 385, 507, 39, 878, 886, 572, 0, 0, 0, 0, 0, 0, 838, 393, 1395, 63, 591, 1150, 758, 788, 0, 0, 0, 0, 0, 0, 0, 0, 565, 960, 567, 585, 1730, 384, 613, 435, 210, 106, 95, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 5800, 6215, 6258, 6151, 6338, 5976, 6250] avg=6144.0000 max=6338 max_rank=5 min=5800 min_rank=1 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[280, 522, 341, 550, 1233, 562, 935, 485, 268, 360, 1322, 244, 1199, 587, 863, 2263, 838, 1356, 930, 519, 997, 962, 1429, 703, 656, 842, 903, 1721, 224, 873, 532, 649, 367, 322, 484, 655, 1020, 335, 329, 927, 1472, 798, 34, 722, 323, 449, 48, 857, 3002, 433, 1221, 152, 615, 1037, 812, 944, 534, 932, 483, 485, 1849, 373, 540, 450] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[252, 369, 687, 537, 1216, 572, 933, 533, 290, 327, 859, 254, 1193, 463, 901, 2218, 849, 1285, 970, 744, 882, 900, 1341, 691, 648, 777, 893, 1900, 263, 878, 536, 573, 319, 339, 463, 569, 1061, 224, 329, 983, 1650, 569, 20, 832, 385, 507, 39, 878, 3274, 393, 1395, 63, 591, 1150, 758, 788, 565, 960, 567, 585, 1730, 384, 613, 435] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[183, 606, 195, 603, 1337, 676, 892, 634, 626, 436, 0, 0, 0, 0, 0, 0, 306, 312, 1153, 221, 1078, 547, 857, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 946, 1142, 888, 424, 948, 1044, 0, 773, 0, 0, 0, 0, 0, 0, 0, 0, 622, 767, 781, 1647, 163, 786, 440, 720, 0, 0, 0, 0, 0, 0, 0, 0, 353, 310, 436, 558, 1092, 279, 401, 808, 1875, 0, 0, 0, 0, 0, 0, 0, 1576, 824, 29, 836, 440, 513, 52, 622, 847, 575, 0, 0, 0, 0, 0, 0, 843, 460, 1322, 92, 723, 1146, 794, 855, 0, 0, 0, 0, 0, 0, 0, 0, 560, 862, 715, 490, 1844, 371, 464, 472, 205, 103, 97, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6188, 6029, 6165, 5926, 6112, 6314, 6235, 6183] avg=6144.0000 max=6314 max_rank=5 min=5926 min_rank=3 imbalance_ratio=1.027669 max_load_reduction=42838 max_load_reduction_pct=0.871541 moved_to_echo=24844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[212, 442, 462, 498, 1236, 753, 931, 604, 615, 414, 0, 0, 0, 0, 0, 0, 247, 386, 1235, 204, 1256, 630, 913, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 832, 1098, 899, 460, 905, 917, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 594, 810, 925, 1684, 183, 843, 497, 552, 0, 0, 0, 0, 0, 0, 0, 0, 392, 395, 486, 604, 1087, 362, 387, 928, 1739, 0, 0, 0, 0, 0, 0, 0, 1529, 784, 39, 769, 333, 486, 52, 758, 802, 533, 0, 0, 0, 0, 0, 0, 782, 342, 1380, 97, 707, 1085, 792, 889, 0, 0, 0, 0, 0, 0, 0, 0, 502, 954, 497, 637, 1822, 446, 532, 389, 210, 99, 96, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6167, 6399, 5775, 6088, 6380, 6085, 6074, 6184] avg=6144.0000 max=6399 max_rank=1 min=5775 min_rank=2 imbalance_ratio=1.041504 max_load_reduction=42753 max_load_reduction_pct=0.869812 moved_to_echo=24723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[262, 440, 354, 616, 1291, 654, 846, 600, 568, 510, 0, 0, 0, 0, 0, 0, 267, 348, 1178, 242, 1126, 573, 947, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 933, 1106, 985, 716, 872, 938, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 622, 771, 963, 1610, 252, 825, 502, 628, 0, 0, 0, 0, 0, 0, 0, 0, 353, 318, 414, 548, 909, 321, 390, 898, 1773, 0, 0, 0, 0, 0, 0, 0, 1683, 609, 51, 749, 395, 470, 38, 640, 991, 543, 0, 0, 0, 0, 0, 0, 797, 389, 1458, 121, 765, 1008, 783, 800, 0, 0, 0, 0, 0, 0, 0, 0, 589, 930, 756, 508, 1648, 401, 682, 457, 201, 100, 88, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6090, 6174, 6173, 5924, 6169, 6121, 6360] avg=6144.0000 max=6360 max_rank=7 min=5924 min_rank=4 imbalance_ratio=1.035156 max_load_reduction=42792 max_load_reduction_pct=0.870605 moved_to_echo=24574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[228, 423, 488, 586, 1282, 521, 794, 533, 553, 444, 0, 0, 0, 0, 0, 0, 314, 402, 1239, 321, 1236, 614, 895, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1159, 1012, 966, 1008, 1079, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 595, 797, 862, 1775, 236, 859, 552, 463, 0, 0, 0, 0, 0, 0, 0, 0, 392, 342, 468, 509, 1009, 260, 374, 982, 1714, 0, 0, 0, 0, 0, 0, 0, 1618, 618, 55, 772, 298, 432, 26, 599, 861, 525, 0, 0, 0, 0, 0, 0, 771, 434, 1415, 133, 675, 981, 708, 870, 0, 0, 0, 0, 0, 0, 0, 0, 607, 878, 660, 502, 1546, 390, 621, 487, 221, 104, 86, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5852, 6395, 6823, 6139, 6050, 5804, 5987, 6102] avg=6144.0000 max=6823 max_rank=2 min=5804 min_rank=5 imbalance_ratio=1.110514 max_load_reduction=42329 max_load_reduction_pct=0.861186 moved_to_echo=23943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[252, 369, 687, 537, 1216, 572, 933, 533, 610, 455, 0, 0, 0, 0, 0, 0, 290, 327, 859, 254, 1193, 463, 901, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 849, 1179, 970, 744, 882, 900, 0, 691, 0, 0, 0, 0, 0, 0, 0, 0, 648, 777, 893, 1690, 263, 878, 536, 573, 0, 0, 0, 0, 0, 0, 0, 0, 319, 339, 463, 569, 1061, 224, 329, 983, 1864, 0, 0, 0, 0, 0, 0, 0, 1650, 569, 20, 832, 385, 507, 39, 878, 886, 572, 0, 0, 0, 0, 0, 0, 838, 393, 1395, 63, 591, 1150, 758, 788, 0, 0, 0, 0, 0, 0, 0, 0, 565, 960, 567, 585, 1730, 384, 613, 435, 210, 106, 95, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 5800, 6215, 6258, 6151, 6338, 5976, 6250] avg=6144.0000 max=6338 max_rank=5 min=5800 min_rank=1 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[203, 424, 332, 530, 1281, 642, 950, 590, 628, 498, 0, 0, 0, 0, 0, 0, 260, 267, 1238, 317, 1153, 576, 852, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 810, 1034, 895, 566, 908, 1030, 0, 699, 0, 0, 0, 0, 0, 0, 0, 0, 602, 729, 838, 1633, 170, 882, 517, 691, 0, 0, 0, 0, 0, 0, 0, 0, 287, 326, 491, 601, 962, 252, 392, 921, 1834, 0, 0, 0, 0, 0, 0, 0, 1569, 793, 37, 733, 343, 458, 41, 799, 970, 562, 0, 0, 0, 0, 0, 0, 825, 441, 1388, 109, 790, 1143, 842, 720, 0, 0, 0, 0, 0, 0, 0, 0, 619, 902, 674, 502, 1714, 437, 516, 459, 204, 93, 98, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6078, 6223, 5942, 6062, 6066, 6305, 6258, 6218] avg=6144.0000 max=6305 max_rank=5 min=5942 min_rank=2 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24847 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[280, 522, 341, 550, 1233, 562, 935, 485, 626, 486, 0, 0, 0, 0, 0, 0, 268, 360, 1322, 244, 1199, 587, 863, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 838, 1240, 930, 519, 997, 962, 0, 703, 0, 0, 0, 0, 0, 0, 0, 0, 656, 842, 903, 1527, 224, 873, 532, 649, 0, 0, 0, 0, 0, 0, 0, 0, 367, 322, 484, 655, 1020, 335, 329, 927, 1714, 0, 0, 0, 0, 0, 0, 0, 1472, 798, 34, 722, 323, 449, 48, 857, 943, 529, 0, 0, 0, 0, 0, 0, 759, 433, 1221, 152, 615, 1037, 812, 944, 0, 0, 0, 0, 0, 0, 0, 0, 534, 932, 483, 485, 1849, 373, 540, 450, 194, 116, 101, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6020, 6379, 6189, 6206, 6153, 6175, 5973, 6057] avg=6144.0000 max=6379 max_rank=1 min=5973 min_rank=6 imbalance_ratio=1.038249 max_load_reduction=42773 max_load_reduction_pct=0.870219 moved_to_echo=24358 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[257, 531, 731, 567, 1250, 581, 974, 644, 530, 477, 0, 0, 0, 0, 0, 0, 298, 431, 1140, 293, 1085, 560, 715, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 892, 1107, 870, 306, 916, 1224, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 679, 807, 974, 1586, 221, 859, 523, 651, 0, 0, 0, 0, 0, 0, 0, 0, 267, 341, 405, 704, 1210, 282, 300, 978, 1829, 0, 0, 0, 0, 0, 0, 0, 1519, 808, 41, 661, 259, 505, 64, 618, 926, 561, 0, 0, 0, 0, 0, 0, 822, 514, 1451, 128, 830, 1113, 774, 896, 0, 0, 0, 0, 0, 0, 0, 0, 408, 816, 785, 564, 1501, 396, 506, 442, 198, 100, 82, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6542, 5837, 5869, 6300, 6316, 5962, 6528, 5798] avg=6144.0000 max=6542 max_rank=0 min=5798 min_rank=7 imbalance_ratio=1.064779 max_load_reduction=42610 max_load_reduction_pct=0.866903 moved_to_echo=24604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[228, 423, 488, 586, 1282, 521, 794, 533, 553, 444, 0, 0, 0, 0, 0, 0, 314, 402, 1239, 321, 1236, 614, 895, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1159, 1012, 966, 1008, 1079, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 595, 797, 862, 1775, 236, 859, 552, 463, 0, 0, 0, 0, 0, 0, 0, 0, 392, 342, 468, 509, 1009, 260, 374, 982, 1714, 0, 0, 0, 0, 0, 0, 0, 1618, 618, 55, 772, 298, 432, 26, 599, 861, 525, 0, 0, 0, 0, 0, 0, 771, 434, 1415, 133, 675, 981, 708, 870, 0, 0, 0, 0, 0, 0, 0, 0, 607, 878, 660, 502, 1546, 390, 621, 487, 221, 104, 86, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5852, 6395, 6823, 6139, 6050, 5804, 5987, 6102] avg=6144.0000 max=6823 max_rank=2 min=5804 min_rank=5 imbalance_ratio=1.110514 max_load_reduction=42329 max_load_reduction_pct=0.861186 moved_to_echo=23943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[280, 522, 341, 550, 1233, 562, 935, 485, 626, 486, 0, 0, 0, 0, 0, 0, 268, 360, 1322, 244, 1199, 587, 863, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 838, 1240, 930, 519, 997, 962, 0, 703, 0, 0, 0, 0, 0, 0, 0, 0, 656, 842, 903, 1527, 224, 873, 532, 649, 0, 0, 0, 0, 0, 0, 0, 0, 367, 322, 484, 655, 1020, 335, 329, 927, 1714, 0, 0, 0, 0, 0, 0, 0, 1472, 798, 34, 722, 323, 449, 48, 857, 943, 529, 0, 0, 0, 0, 0, 0, 759, 433, 1221, 152, 615, 1037, 812, 944, 0, 0, 0, 0, 0, 0, 0, 0, 534, 932, 483, 485, 1849, 373, 540, 450, 194, 116, 101, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6020, 6379, 6189, 6206, 6153, 6175, 5973, 6057] avg=6144.0000 max=6379 max_rank=1 min=5973 min_rank=6 imbalance_ratio=1.038249 max_load_reduction=42773 max_load_reduction_pct=0.870219 moved_to_echo=24358 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6528, 3456, 11136, 896, 5760, 8704, 6272, 6784, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 6400, 7168, 13184, 1792, 6912, 4224, 4992, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4480, 7296, 5248, 4352, 13696, 3200, 4480, 3712, 1664, 896, 768, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7168, 9088, 7552, 4736, 7552, 8192, 0, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12672, 5888, 384, 6144, 2816, 3840, 384, 5888, 7296, 4480, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[1920, 3840, 3712, 4608, 10240, 4992, 7296, 4736, 4864, 3840, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2304, 2944, 9472, 2176, 9344, 4608, 7040, 11904, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12672, 5888, 384, 6144, 2816, 3840, 384, 5888, 7296, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4480, 7296, 5248, 4352, 13696, 3200, 4480, 3712, 1664, 896, 768, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2816, 2816, 3712, 4864, 8448, 2432, 2944, 7552, 14464, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[83, 711, 78, 433, 260, 513, 272, 1981, 28, 960, 244, 2024, 650, 1280, 352, 600, 1902, 547, 655, 929, 367, 305, 1475, 123, 511, 1799, 3072, 1158, 647, 428, 341, 107, 772, 709, 1725, 1191, 1931, 714, 1791, 460, 262, 1055, 155, 136, 1427, 1934, 107, 529, 912, 965, 769, 296, 477, 426, 153, 111, 1673, 558, 266, 437, 306, 772, 547, 751] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[107, 744, 138, 454, 323, 437, 252, 1374, 33, 983, 258, 1794, 541, 1699, 358, 759, 1927, 362, 730, 1041, 342, 289, 1608, 203, 454, 1411, 2786, 1033, 607, 663, 294, 156, 691, 633, 2036, 1174, 1809, 960, 1940, 557, 245, 967, 174, 113, 1529, 1713, 203, 409, 947, 888, 630, 456, 643, 543, 216, 225, 1680, 444, 298, 337, 381, 687, 705, 759] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[142, 802, 98, 368, 260, 878, 252, 1839, 26, 1114, 230, 1895, 668, 1375, 310, 563, 1807, 516, 608, 1125, 420, 296, 1782, 173, 431, 1535, 2775, 1092, 692, 596, 216, 107, 918, 632, 1971, 1209, 1914, 1015, 1863, 429, 373, 768, 117, 123, 1320, 1681, 107, 730, 946, 775, 637, 299, 488, 351, 176, 187, 1522, 622, 297, 413, 274, 764, 545, 695] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[121, 827, 74, 413, 265, 525, 231, 2101, 32, 1242, 282, 1732, 732, 1109, 400, 574, 2327, 440, 561, 1029, 454, 271, 1569, 151, 553, 1628, 3054, 1237, 762, 488, 348, 134, 675, 723, 1773, 1374, 1476, 955, 1983, 428, 309, 915, 116, 137, 1360, 1707, 129, 439, 775, 1037, 714, 300, 584, 442, 196, 184, 1316, 554, 256, 317, 381, 796, 535, 600] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[142, 802, 98, 368, 260, 878, 252, 1839, 26, 1114, 230, 1895, 668, 1375, 310, 563, 1807, 516, 608, 1125, 420, 296, 1782, 173, 431, 1535, 2775, 1092, 692, 596, 216, 107, 918, 632, 1971, 1209, 1914, 1015, 1863, 429, 373, 768, 117, 123, 1320, 1681, 107, 730, 946, 775, 637, 299, 488, 351, 176, 187, 1522, 622, 297, 413, 274, 764, 545, 695] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[107, 744, 138, 454, 323, 437, 252, 1374, 33, 983, 258, 1794, 541, 1699, 358, 759, 1927, 362, 730, 1041, 342, 289, 1608, 203, 454, 1411, 2786, 1033, 607, 663, 294, 156, 691, 633, 2036, 1174, 1809, 960, 1940, 557, 245, 967, 174, 113, 1529, 1713, 203, 409, 947, 888, 630, 456, 643, 543, 216, 225, 1680, 444, 298, 337, 381, 687, 705, 759] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[83, 711, 78, 433, 260, 513, 272, 1981, 28, 960, 244, 2024, 650, 1280, 352, 600, 1902, 547, 655, 929, 367, 305, 1475, 123, 511, 1799, 3072, 1158, 647, 428, 341, 107, 772, 709, 1725, 1191, 1931, 714, 1791, 460, 262, 1055, 155, 136, 1427, 1934, 107, 529, 912, 965, 769, 296, 477, 426, 153, 111, 1673, 558, 266, 437, 306, 772, 547, 751] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[101, 733, 86, 433, 308, 540, 245, 2080, 25, 1204, 224, 2005, 734, 1258, 411, 650, 2104, 504, 519, 1016, 528, 307, 1627, 160, 510, 1644, 2999, 1268, 670, 521, 411, 97, 563, 585, 1887, 1253, 1596, 732, 1885, 434, 318, 784, 136, 168, 1134, 1918, 201, 464, 727, 988, 635, 338, 534, 599, 196, 130, 1502, 593, 262, 406, 292, 746, 522, 702] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[119, 861, 127, 380, 256, 568, 248, 1710, 30, 1402, 304, 1853, 660, 1268, 369, 796, 2059, 518, 676, 969, 489, 306, 1329, 203, 532, 1487, 2972, 1272, 747, 402, 371, 91, 726, 654, 1782, 1093, 1771, 1061, 1693, 501, 318, 974, 122, 132, 1431, 1831, 172, 465, 721, 1034, 758, 383, 550, 293, 178, 199, 1414, 572, 265, 384, 351, 771, 536, 643] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[91, 931, 122, 432, 316, 499, 282, 1496, 58, 1172, 204, 1834, 570, 1400, 374, 656, 1877, 542, 648, 952, 551, 319, 1402, 172, 481, 1668, 2881, 1112, 921, 494, 352, 118, 865, 723, 1926, 1069, 1769, 784, 1901, 531, 338, 911, 128, 171, 1284, 1652, 174, 447, 830, 912, 663, 462, 514, 452, 198, 163, 1462, 538, 306, 485, 459, 725, 565, 818] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[149, 807, 112, 444, 282, 533, 284, 1960, 36, 1187, 259, 1745, 1035, 1252, 285, 676, 1913, 609, 649, 813, 538, 380, 1373, 206, 509, 1689, 2857, 1161, 668, 534, 353, 93, 561, 684, 1709, 1291, 1618, 1070, 1822, 417, 317, 985, 121, 127, 1432, 1715, 139, 572, 962, 963, 704, 327, 564, 552, 180, 135, 1369, 543, 295, 326, 256, 734, 590, 681] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[119, 861, 127, 380, 256, 568, 248, 1710, 30, 1402, 304, 1853, 660, 1268, 369, 796, 2059, 518, 676, 969, 489, 306, 1329, 203, 532, 1487, 2972, 1272, 747, 402, 371, 91, 726, 654, 1782, 1093, 1771, 1061, 1693, 501, 318, 974, 122, 132, 1431, 1831, 172, 465, 721, 1034, 758, 383, 550, 293, 178, 199, 1414, 572, 265, 384, 351, 771, 536, 643] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[142, 802, 98, 368, 260, 878, 252, 1839, 1561, 145, 0, 0, 0, 0, 0, 0, 26, 1114, 230, 1688, 668, 1375, 310, 563, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 516, 608, 1125, 420, 296, 1782, 173, 0, 0, 0, 0, 0, 0, 0, 0, 431, 1535, 1134, 1092, 692, 596, 216, 107, 0, 0, 0, 0, 0, 0, 0, 0, 918, 632, 461, 1209, 1914, 1015, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 373, 768, 117, 123, 1320, 1681, 107, 730, 390, 227, 207, 0, 0, 0, 0, 0, 946, 775, 637, 299, 488, 351, 176, 187, 1863, 80, 0, 0, 0, 0, 0, 0, 1522, 622, 297, 413, 274, 764, 545, 695, 1138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6345, 5974, 6337, 5803, 6578, 6043, 5802, 6270] avg=6144.0000 max=6578 max_rank=4 min=5802 min_rank=6 imbalance_ratio=1.070638 max_load_reduction=42574 max_load_reduction_pct=0.866170 moved_to_echo=24693 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[83, 711, 78, 433, 260, 513, 272, 1981, 1728, 127, 0, 0, 0, 0, 0, 0, 28, 960, 244, 1803, 650, 1280, 352, 600, 0, 0, 0, 0, 0, 0, 0, 0, 1491, 547, 655, 929, 367, 305, 1475, 123, 0, 0, 0, 0, 0, 0, 0, 0, 511, 1799, 1255, 1158, 647, 428, 341, 107, 0, 0, 0, 0, 0, 0, 0, 0, 772, 709, 403, 1191, 1931, 714, 0, 460, 0, 0, 0, 0, 0, 0, 0, 0, 262, 1055, 155, 136, 1427, 1934, 107, 529, 411, 199, 221, 0, 0, 0, 0, 0, 912, 965, 769, 296, 477, 426, 153, 111, 1791, 89, 0, 0, 0, 0, 0, 0, 1673, 558, 266, 437, 306, 772, 547, 751, 996, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6186, 5917, 5892, 6246, 6180, 6436, 5989, 6306] avg=6144.0000 max=6436 max_rank=5 min=5892 min_rank=2 imbalance_ratio=1.047526 max_load_reduction=42716 max_load_reduction_pct=0.869059 moved_to_echo=24911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[121, 827, 74, 413, 265, 525, 231, 2101, 1718, 131, 0, 0, 0, 0, 0, 0, 32, 1242, 282, 1543, 732, 1109, 400, 574, 0, 0, 0, 0, 0, 0, 0, 0, 1825, 440, 561, 1029, 454, 271, 1569, 151, 0, 0, 0, 0, 0, 0, 0, 0, 553, 1628, 1247, 1237, 762, 488, 348, 134, 0, 0, 0, 0, 0, 0, 0, 0, 675, 723, 415, 1374, 1476, 955, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 309, 915, 116, 137, 1360, 1707, 129, 439, 502, 204, 189, 0, 0, 0, 0, 0, 775, 1037, 714, 300, 584, 442, 196, 184, 1983, 89, 0, 0, 0, 0, 0, 0, 1316, 554, 256, 317, 381, 796, 535, 600, 1023, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6406, 5914, 6300, 6397, 6046, 6007, 6304, 5778] avg=6144.0000 max=6406 max_rank=0 min=5778 min_rank=7 imbalance_ratio=1.042643 max_load_reduction=42746 max_load_reduction_pct=0.869670 moved_to_echo=24135 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[101, 733, 86, 433, 308, 540, 245, 2080, 1687, 139, 0, 0, 0, 0, 0, 0, 25, 1204, 224, 1786, 734, 1258, 411, 650, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 504, 519, 1016, 528, 307, 1627, 160, 0, 0, 0, 0, 0, 0, 0, 0, 510, 1644, 1225, 1268, 670, 521, 411, 97, 0, 0, 0, 0, 0, 0, 0, 0, 563, 585, 441, 1253, 1596, 732, 0, 434, 0, 0, 0, 0, 0, 0, 0, 0, 318, 784, 136, 168, 1134, 1918, 201, 464, 454, 218, 219, 0, 0, 0, 0, 0, 727, 988, 635, 338, 534, 599, 196, 130, 1885, 87, 0, 0, 0, 0, 0, 0, 1502, 593, 262, 406, 292, 746, 522, 702, 1089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 6292, 6311, 6346, 5604, 6014, 6119, 6114] avg=6144.0000 max=6352 max_rank=0 min=5604 min_rank=4 imbalance_ratio=1.033854 max_load_reduction=42800 max_load_reduction_pct=0.870768 moved_to_echo=23851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[119, 861, 127, 380, 256, 568, 248, 1710, 1672, 131, 0, 0, 0, 0, 0, 0, 30, 1402, 304, 1650, 660, 1268, 369, 796, 0, 0, 0, 0, 0, 0, 0, 0, 1614, 518, 676, 969, 489, 306, 1329, 203, 0, 0, 0, 0, 0, 0, 0, 0, 532, 1487, 1214, 1272, 747, 402, 371, 91, 0, 0, 0, 0, 0, 0, 0, 0, 726, 654, 418, 1093, 1771, 1061, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 318, 974, 122, 132, 1431, 1831, 172, 465, 445, 205, 203, 0, 0, 0, 0, 0, 721, 1034, 758, 383, 550, 293, 178, 199, 1693, 86, 0, 0, 0, 0, 0, 0, 1414, 572, 265, 384, 351, 771, 536, 643, 1028, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 6479, 6104, 6116, 6224, 6298, 5895, 5964] avg=6144.0000 max=6479 max_rank=1 min=5895 min_rank=6 imbalance_ratio=1.054525 max_load_reduction=42673 max_load_reduction_pct=0.868184 moved_to_echo=24381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[107, 744, 138, 454, 323, 437, 252, 1374, 1567, 150, 0, 0, 0, 0, 0, 0, 33, 983, 258, 1598, 541, 1699, 358, 759, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 362, 730, 1041, 342, 289, 1608, 203, 0, 0, 0, 0, 0, 0, 0, 0, 454, 1411, 1138, 1033, 607, 663, 294, 156, 0, 0, 0, 0, 0, 0, 0, 0, 691, 633, 476, 1174, 1809, 960, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 245, 967, 174, 113, 1529, 1713, 203, 409, 416, 235, 196, 0, 0, 0, 0, 0, 947, 888, 630, 456, 643, 543, 216, 225, 1940, 81, 0, 0, 0, 0, 0, 0, 1680, 444, 298, 337, 381, 687, 705, 759, 1175, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5546, 6229, 6086, 5756, 6300, 6200, 6569, 6466] avg=6144.0000 max=6569 max_rank=6 min=5546 min_rank=0 imbalance_ratio=1.069173 max_load_reduction=42583 max_load_reduction_pct=0.866353 moved_to_echo=25535 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[142, 802, 98, 368, 260, 878, 252, 1839, 1561, 145, 0, 0, 0, 0, 0, 0, 26, 1114, 230, 1688, 668, 1375, 310, 563, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 516, 608, 1125, 420, 296, 1782, 173, 0, 0, 0, 0, 0, 0, 0, 0, 431, 1535, 1134, 1092, 692, 596, 216, 107, 0, 0, 0, 0, 0, 0, 0, 0, 918, 632, 461, 1209, 1914, 1015, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 373, 768, 117, 123, 1320, 1681, 107, 730, 390, 227, 207, 0, 0, 0, 0, 0, 946, 775, 637, 299, 488, 351, 176, 187, 1863, 80, 0, 0, 0, 0, 0, 0, 1522, 622, 297, 413, 274, 764, 545, 695, 1138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6345, 5974, 6337, 5803, 6578, 6043, 5802, 6270] avg=6144.0000 max=6578 max_rank=4 min=5802 min_rank=6 imbalance_ratio=1.070638 max_load_reduction=42574 max_load_reduction_pct=0.866170 moved_to_echo=24693 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[149, 807, 112, 444, 282, 533, 284, 1960, 1607, 126, 0, 0, 0, 0, 0, 0, 36, 1187, 259, 1554, 1035, 1252, 285, 676, 0, 0, 0, 0, 0, 0, 0, 0, 1500, 609, 649, 813, 538, 380, 1373, 206, 0, 0, 0, 0, 0, 0, 0, 0, 509, 1689, 1167, 1161, 668, 534, 353, 93, 0, 0, 0, 0, 0, 0, 0, 0, 561, 684, 400, 1291, 1618, 1070, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 317, 985, 121, 127, 1432, 1715, 139, 572, 413, 197, 191, 0, 0, 0, 0, 0, 962, 963, 704, 327, 564, 552, 180, 135, 1822, 83, 0, 0, 0, 0, 0, 0, 1369, 543, 295, 326, 256, 734, 590, 681, 986, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6304, 6284, 6068, 6174, 6041, 6209, 6292, 5780] avg=6144.0000 max=6304 max_rank=0 min=5780 min_rank=7 imbalance_ratio=1.026042 max_load_reduction=42848 max_load_reduction_pct=0.871745 moved_to_echo=24322 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[101, 733, 86, 433, 308, 540, 245, 2080, 1687, 139, 0, 0, 0, 0, 0, 0, 25, 1204, 224, 1786, 734, 1258, 411, 650, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 504, 519, 1016, 528, 307, 1627, 160, 0, 0, 0, 0, 0, 0, 0, 0, 510, 1644, 1225, 1268, 670, 521, 411, 97, 0, 0, 0, 0, 0, 0, 0, 0, 563, 585, 441, 1253, 1596, 732, 0, 434, 0, 0, 0, 0, 0, 0, 0, 0, 318, 784, 136, 168, 1134, 1918, 201, 464, 454, 218, 219, 0, 0, 0, 0, 0, 727, 988, 635, 338, 534, 599, 196, 130, 1885, 87, 0, 0, 0, 0, 0, 0, 1502, 593, 262, 406, 292, 746, 522, 702, 1089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 6292, 6311, 6346, 5604, 6014, 6119, 6114] avg=6144.0000 max=6352 max_rank=0 min=5604 min_rank=4 imbalance_ratio=1.033854 max_load_reduction=42800 max_load_reduction_pct=0.870768 moved_to_echo=23851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[91, 931, 122, 432, 316, 499, 282, 1496, 1626, 146, 0, 0, 0, 0, 0, 0, 58, 1172, 204, 1629, 570, 1400, 374, 656, 0, 0, 0, 0, 0, 0, 0, 0, 1468, 542, 648, 952, 551, 319, 1402, 172, 0, 0, 0, 0, 0, 0, 0, 0, 481, 1668, 1168, 1112, 921, 494, 352, 118, 0, 0, 0, 0, 0, 0, 0, 0, 865, 723, 438, 1069, 1769, 784, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 338, 911, 128, 171, 1284, 1652, 174, 447, 409, 226, 205, 0, 0, 0, 0, 0, 830, 912, 663, 462, 514, 452, 198, 163, 1901, 87, 0, 0, 0, 0, 0, 0, 1462, 538, 306, 485, 459, 725, 565, 818, 1116, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5941, 6063, 6054, 6314, 6179, 5945, 6182, 6474] avg=6144.0000 max=6474 max_rank=7 min=5941 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=42678 max_load_reduction_pct=0.868286 moved_to_echo=24780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[119, 861, 127, 380, 256, 568, 248, 1710, 1672, 131, 0, 0, 0, 0, 0, 0, 30, 1402, 304, 1650, 660, 1268, 369, 796, 0, 0, 0, 0, 0, 0, 0, 0, 1614, 518, 676, 969, 489, 306, 1329, 203, 0, 0, 0, 0, 0, 0, 0, 0, 532, 1487, 1214, 1272, 747, 402, 371, 91, 0, 0, 0, 0, 0, 0, 0, 0, 726, 654, 418, 1093, 1771, 1061, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 318, 974, 122, 132, 1431, 1831, 172, 465, 445, 205, 203, 0, 0, 0, 0, 0, 721, 1034, 758, 383, 550, 293, 178, 199, 1693, 86, 0, 0, 0, 0, 0, 0, 1414, 572, 265, 384, 351, 771, 536, 643, 1028, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 6479, 6104, 6116, 6224, 6298, 5895, 5964] avg=6144.0000 max=6479 max_rank=1 min=5895 min_rank=6 imbalance_ratio=1.054525 max_load_reduction=42673 max_load_reduction_pct=0.868184 moved_to_echo=24381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12032, 4480, 2304, 3200, 2816, 6016, 4608, 5760, 8576, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1024, 6528, 896, 3456, 2304, 4608, 2176, 14592, 13184, 1152, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12544, 4096, 5120, 7936, 3712, 2560, 12288, 1408, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4096, 12928, 9600, 9344, 5760, 4224, 2688, 1024, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 7424, 1152, 1152, 11008, 14208, 1280, 4096, 3456, 1792, 1664, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1024, 6528, 896, 3456, 2304, 4608, 2176, 14592, 13184, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 9344, 2048, 13312, 5632, 10752, 2944, 5376, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5888, 5376, 3456, 9728, 13952, 7296, 0, 3840, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[6912, 7680, 5632, 2944, 4480, 3712, 1536, 1408, 14976, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 9344, 2048, 13312, 5632, 10752, 2944, 5376, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1710, 676, 780, 58, 96, 226, 1870, 1724, 146, 228, 757, 295, 960, 723, 275, 1681, 2347, 341, 846, 1195, 604, 890, 83, 1130, 174, 270, 1268, 210, 685, 1300, 2678, 801, 1473, 1158, 189, 109, 576, 629, 163, 364, 953, 2744, 577, 755, 246, 444, 475, 385, 1006, 1086, 606, 237, 104, 1289, 320, 2550, 837, 30, 231, 299, 845, 750, 527, 168] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1950, 900, 739, 79, 90, 249, 2113, 1519, 125, 266, 759, 316, 855, 538, 343, 1581, 2599, 234, 951, 1157, 632, 863, 71, 1164, 233, 375, 1085, 214, 792, 1258, 2804, 662, 1209, 718, 227, 161, 558, 692, 151, 421, 1209, 2231, 632, 608, 326, 443, 433, 801, 1245, 919, 677, 237, 123, 1291, 346, 2229, 825, 30, 239, 327, 954, 618, 490, 266] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2000, 679, 814, 77, 121, 275, 1977, 1495, 125, 193, 744, 298, 784, 720, 341, 1814, 2557, 317, 719, 1182, 500, 859, 93, 920, 300, 334, 1178, 234, 794, 1273, 2363, 871, 1385, 756, 249, 145, 564, 623, 301, 361, 1100, 2734, 501, 526, 230, 495, 481, 548, 1199, 881, 1007, 315, 79, 891, 438, 2418, 853, 49, 173, 304, 905, 636, 798, 256] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1728, 951, 793, 49, 170, 264, 1822, 1894, 145, 301, 692, 270, 1068, 556, 291, 1390, 2355, 284, 505, 1188, 506, 908, 47, 1037, 166, 379, 1241, 204, 795, 1293, 2929, 1144, 1297, 707, 198, 184, 285, 574, 220, 450, 1547, 2422, 522, 646, 272, 476, 381, 339, 1029, 954, 805, 207, 109, 889, 345, 2582, 817, 28, 353, 318, 992, 932, 693, 214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2123, 732, 646, 91, 101, 269, 1720, 1607, 118, 221, 597, 393, 1087, 648, 246, 1900, 2506, 273, 716, 1236, 556, 1053, 105, 1204, 169, 386, 1189, 224, 723, 1241, 2506, 771, 1354, 723, 158, 183, 294, 589, 264, 442, 1130, 2361, 686, 633, 301, 731, 504, 488, 1166, 977, 539, 254, 138, 1072, 468, 2166, 1068, 41, 223, 313, 1024, 693, 615, 197] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1889, 795, 1033, 53, 99, 183, 1895, 1891, 78, 202, 663, 322, 899, 645, 381, 1511, 3176, 212, 680, 1248, 458, 1087, 112, 1396, 126, 296, 819, 229, 863, 1167, 2910, 640, 1569, 802, 199, 118, 487, 740, 218, 377, 1009, 2152, 329, 700, 334, 394, 444, 587, 1272, 872, 662, 332, 98, 1188, 223, 2324, 907, 10, 180, 298, 1053, 559, 536, 221] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1710, 676, 780, 58, 96, 226, 1870, 1724, 146, 228, 757, 295, 960, 723, 275, 1681, 2347, 341, 846, 1195, 604, 890, 83, 1130, 174, 270, 1268, 210, 685, 1300, 2678, 801, 1473, 1158, 189, 109, 576, 629, 163, 364, 953, 2744, 577, 755, 246, 444, 475, 385, 1006, 1086, 606, 237, 104, 1289, 320, 2550, 837, 30, 231, 299, 845, 750, 527, 168] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1656, 835, 1034, 45, 63, 296, 1969, 2005, 137, 301, 570, 352, 894, 715, 347, 1689, 2606, 184, 902, 921, 616, 905, 154, 1307, 182, 312, 973, 329, 580, 1182, 2459, 638, 1721, 802, 169, 141, 558, 735, 127, 478, 1116, 2368, 621, 743, 289, 479, 551, 613, 1303, 699, 604, 301, 125, 1097, 354, 2002, 747, 84, 209, 509, 1003, 670, 511, 265] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1950, 900, 739, 79, 90, 249, 2113, 1519, 125, 266, 759, 316, 855, 538, 343, 1581, 2599, 234, 951, 1157, 632, 863, 71, 1164, 233, 375, 1085, 214, 792, 1258, 2804, 662, 1209, 718, 227, 161, 558, 692, 151, 421, 1209, 2231, 632, 608, 326, 443, 433, 801, 1245, 919, 677, 237, 123, 1291, 346, 2229, 825, 30, 239, 327, 954, 618, 490, 266] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1852, 775, 886, 56, 112, 290, 1918, 1858, 115, 263, 479, 343, 1067, 730, 529, 1471, 2770, 266, 764, 1122, 526, 1192, 104, 1089, 203, 430, 1155, 248, 816, 1322, 2488, 1007, 1087, 825, 218, 210, 547, 544, 240, 609, 1092, 2049, 311, 724, 383, 415, 454, 482, 1233, 816, 639, 210, 148, 1039, 349, 2105, 792, 98, 198, 440, 1081, 656, 604, 308] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2123, 732, 646, 91, 101, 269, 1720, 1607, 118, 221, 597, 393, 1087, 648, 246, 1900, 2506, 273, 716, 1236, 556, 1053, 105, 1204, 169, 386, 1189, 224, 723, 1241, 2506, 771, 1354, 723, 158, 183, 294, 589, 264, 442, 1130, 2361, 686, 633, 301, 731, 504, 488, 1166, 977, 539, 254, 138, 1072, 468, 2166, 1068, 41, 223, 313, 1024, 693, 615, 197] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1710, 676, 780, 58, 96, 226, 462, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 146, 228, 757, 295, 960, 723, 275, 1681, 837, 403, 103, 0, 0, 0, 0, 0, 1071, 341, 846, 1195, 604, 890, 83, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 174, 270, 1268, 210, 685, 1300, 1446, 801, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 1158, 189, 109, 576, 629, 163, 364, 1129, 629, 0, 0, 0, 0, 0, 0, 953, 2341, 577, 755, 246, 444, 475, 385, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1086, 606, 237, 104, 1289, 320, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 837, 30, 231, 299, 845, 750, 527, 168, 1408, 647, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5732, 6408, 6160, 6154, 6419, 6176, 6361, 5742] avg=6144.0000 max=6419 max_rank=4 min=5732 min_rank=0 imbalance_ratio=1.044759 max_load_reduction=42733 max_load_reduction_pct=0.869405 moved_to_echo=24698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2000, 679, 814, 77, 121, 275, 488, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 125, 193, 744, 298, 784, 720, 341, 1814, 794, 402, 91, 0, 0, 0, 0, 0, 1167, 317, 719, 1182, 500, 859, 93, 920, 0, 0, 0, 0, 0, 0, 0, 0, 300, 334, 1178, 234, 794, 1273, 1276, 871, 0, 0, 0, 0, 0, 0, 0, 0, 1385, 756, 249, 145, 564, 623, 301, 361, 996, 685, 0, 0, 0, 0, 0, 0, 1100, 2332, 501, 526, 230, 495, 481, 548, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 881, 1007, 315, 79, 891, 438, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 853, 49, 173, 304, 905, 636, 798, 256, 1489, 705, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5949, 6306, 5757, 6260, 6065, 6213, 6434, 6168] avg=6144.0000 max=6434 max_rank=6 min=5757 min_rank=2 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2000, 679, 814, 77, 121, 275, 1977, 1495, 125, 193, 744, 298, 784, 720, 341, 1814, 2557, 317, 719, 1182, 500, 859, 93, 920, 300, 334, 1178, 234, 794, 1273, 2363, 871, 1385, 756, 249, 145, 564, 623, 301, 361, 1100, 2734, 501, 526, 230, 495, 481, 548, 1199, 881, 1007, 315, 79, 891, 438, 2418, 853, 49, 173, 304, 905, 636, 798, 256] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1728, 951, 793, 49, 170, 264, 450, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 145, 301, 692, 270, 1068, 556, 291, 1390, 848, 356, 113, 0, 0, 0, 0, 0, 1075, 284, 505, 1188, 506, 908, 47, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 166, 379, 1241, 204, 795, 1293, 1581, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 1297, 707, 198, 184, 285, 574, 220, 450, 1235, 631, 0, 0, 0, 0, 0, 0, 1547, 2066, 522, 646, 272, 476, 381, 339, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 954, 805, 207, 109, 889, 345, 1734, 0, 0, 0, 0, 0, 0, 0, 0, 817, 28, 353, 318, 992, 932, 693, 214, 1372, 649, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6299, 6030, 5550, 6803, 5781, 6249, 6072, 6368] avg=6144.0000 max=6803 max_rank=3 min=5550 min_rank=2 imbalance_ratio=1.107259 max_load_reduction=42349 max_load_reduction_pct=0.861593 moved_to_echo=24470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1950, 900, 739, 79, 90, 249, 522, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 125, 266, 759, 316, 855, 538, 343, 1581, 732, 328, 108, 0, 0, 0, 0, 0, 1187, 234, 951, 1157, 632, 863, 71, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 233, 375, 1085, 214, 792, 1258, 1513, 662, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 718, 227, 161, 558, 692, 151, 421, 1183, 696, 0, 0, 0, 0, 0, 0, 1209, 1903, 632, 608, 326, 443, 433, 801, 0, 0, 0, 0, 0, 0, 0, 0, 1245, 919, 677, 237, 123, 1291, 346, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 825, 30, 239, 327, 954, 618, 490, 266, 1591, 716, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6048, 5951, 6259, 6132, 6016, 6355, 6335, 6056] avg=6144.0000 max=6355 max_rank=5 min=5951 min_rank=1 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1889, 795, 1033, 53, 99, 183, 468, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 78, 202, 663, 322, 899, 645, 381, 1511, 763, 316, 112, 0, 0, 0, 0, 0, 1450, 212, 680, 1248, 458, 1087, 112, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 126, 296, 819, 229, 863, 1167, 1571, 640, 0, 0, 0, 0, 0, 0, 0, 0, 1569, 802, 199, 118, 487, 740, 218, 377, 1227, 851, 0, 0, 0, 0, 0, 0, 1009, 1836, 329, 700, 334, 394, 444, 587, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 872, 662, 332, 98, 1188, 223, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 907, 10, 180, 298, 1053, 559, 536, 221, 1427, 875, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6411, 5892, 6643, 5711, 6588, 5633, 6208, 6066] avg=6144.0000 max=6643 max_rank=2 min=5633 min_rank=5 imbalance_ratio=1.081217 max_load_reduction=42509 max_load_reduction_pct=0.864848 moved_to_echo=24495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1656, 835, 1034, 45, 63, 296, 486, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 137, 301, 570, 352, 894, 715, 347, 1689, 657, 348, 95, 0, 0, 0, 0, 0, 1190, 184, 902, 921, 616, 905, 154, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 182, 312, 973, 329, 580, 1182, 1327, 638, 0, 0, 0, 0, 0, 0, 0, 0, 1721, 802, 169, 141, 558, 735, 127, 478, 1037, 698, 0, 0, 0, 0, 0, 0, 1116, 2020, 621, 743, 289, 479, 551, 613, 0, 0, 0, 0, 0, 0, 0, 0, 1303, 699, 604, 301, 125, 1097, 354, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 747, 84, 209, 509, 1003, 670, 511, 265, 1483, 718, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6420, 6105, 6179, 5523, 6466, 6432, 5828, 6199] avg=6144.0000 max=6466 max_rank=4 min=5523 min_rank=3 imbalance_ratio=1.052409 max_load_reduction=42686 max_load_reduction_pct=0.868449 moved_to_echo=24925 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1710, 676, 780, 58, 96, 226, 462, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 146, 228, 757, 295, 960, 723, 275, 1681, 837, 403, 103, 0, 0, 0, 0, 0, 1071, 341, 846, 1195, 604, 890, 83, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 174, 270, 1268, 210, 685, 1300, 1446, 801, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 1158, 189, 109, 576, 629, 163, 364, 1129, 629, 0, 0, 0, 0, 0, 0, 953, 2341, 577, 755, 246, 444, 475, 385, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1086, 606, 237, 104, 1289, 320, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 837, 30, 231, 299, 845, 750, 527, 168, 1408, 647, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5732, 6408, 6160, 6154, 6419, 6176, 6361, 5742] avg=6144.0000 max=6419 max_rank=4 min=5732 min_rank=0 imbalance_ratio=1.044759 max_load_reduction=42733 max_load_reduction_pct=0.869405 moved_to_echo=24698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2123, 732, 646, 91, 101, 269, 421, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 118, 221, 597, 393, 1087, 648, 246, 1900, 714, 349, 102, 0, 0, 0, 0, 0, 1134, 273, 716, 1236, 556, 1053, 105, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 169, 386, 1189, 224, 723, 1241, 1342, 771, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 723, 158, 183, 294, 589, 264, 442, 1062, 677, 0, 0, 0, 0, 0, 0, 1130, 2012, 686, 633, 301, 731, 504, 488, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 977, 539, 254, 138, 1072, 468, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 41, 223, 313, 1024, 693, 615, 197, 1299, 695, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5990, 6375, 6277, 6045, 5746, 6485, 6066, 6168] avg=6144.0000 max=6485 max_rank=5 min=5746 min_rank=4 imbalance_ratio=1.055501 max_load_reduction=42667 max_load_reduction_pct=0.868062 moved_to_echo=24465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2000, 679, 814, 77, 121, 275, 488, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 125, 193, 744, 298, 784, 720, 341, 1814, 794, 402, 91, 0, 0, 0, 0, 0, 1167, 317, 719, 1182, 500, 859, 93, 920, 0, 0, 0, 0, 0, 0, 0, 0, 300, 334, 1178, 234, 794, 1273, 1276, 871, 0, 0, 0, 0, 0, 0, 0, 0, 1385, 756, 249, 145, 564, 623, 301, 361, 996, 685, 0, 0, 0, 0, 0, 0, 1100, 2332, 501, 526, 230, 495, 481, 548, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 881, 1007, 315, 79, 891, 438, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 853, 49, 173, 304, 905, 636, 798, 256, 1489, 705, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5949, 6306, 5757, 6260, 6065, 6213, 6434, 6168] avg=6144.0000 max=6434 max_rank=6 min=5757 min_rank=2 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1656, 835, 1034, 45, 63, 296, 486, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 137, 301, 570, 352, 894, 715, 347, 1689, 657, 348, 95, 0, 0, 0, 0, 0, 1190, 184, 902, 921, 616, 905, 154, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 182, 312, 973, 329, 580, 1182, 1327, 638, 0, 0, 0, 0, 0, 0, 0, 0, 1721, 802, 169, 141, 558, 735, 127, 478, 1037, 698, 0, 0, 0, 0, 0, 0, 1116, 2020, 621, 743, 289, 479, 551, 613, 0, 0, 0, 0, 0, 0, 0, 0, 1303, 699, 604, 301, 125, 1097, 354, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 747, 84, 209, 509, 1003, 670, 511, 265, 1483, 718, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6420, 6105, 6179, 5523, 6466, 6432, 5828, 6199] avg=6144.0000 max=6466 max_rank=4 min=5523 min_rank=3 imbalance_ratio=1.052409 max_load_reduction=42686 max_load_reduction_pct=0.868449 moved_to_echo=24925 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1852, 775, 886, 56, 112, 290, 474, 1858, 0, 0, 0, 0, 0, 0, 0, 0, 115, 263, 479, 343, 1067, 730, 529, 1471, 691, 301, 96, 0, 0, 0, 0, 0, 1264, 266, 764, 1122, 526, 1192, 104, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 203, 430, 1155, 248, 816, 1322, 1343, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 825, 218, 210, 547, 544, 240, 609, 1049, 742, 0, 0, 0, 0, 0, 0, 1092, 1748, 311, 724, 383, 415, 454, 482, 0, 0, 0, 0, 0, 0, 0, 0, 1233, 816, 639, 210, 148, 1039, 349, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 792, 98, 198, 440, 1081, 656, 604, 308, 1444, 764, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6303, 6085, 6327, 6524, 6071, 5609, 5848, 6385] avg=6144.0000 max=6524 max_rank=3 min=5609 min_rank=5 imbalance_ratio=1.061849 max_load_reduction=42628 max_load_reduction_pct=0.867269 moved_to_echo=23913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11136, 6528, 1664, 1280, 3968, 5248, 1792, 3584, 8960, 5632, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9216, 16384, 4224, 5376, 2432, 3968, 3840, 4352, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9472, 7296, 5632, 2176, 1024, 8832, 2944, 12416, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9600, 2176, 6144, 9344, 4480, 7808, 896, 9344, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 2816, 8960, 1920, 6144, 10112, 11520, 6656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9216, 16384, 4224, 5376, 2432, 3968, 3840, 4352, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[14976, 6400, 6784, 512, 896, 2176, 3840, 14080, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1024, 2048, 5376, 2688, 7680, 5376, 2816, 13056, 6144, 2816, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9600, 2176, 6144, 9344, 4480, 7808, 896, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 2816, 8960, 1920, 6144, 10112, 11520, 6656, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6912, 384, 1920, 2816, 7936, 5632, 4864, 1920, 11520, 5888, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[486, 704, 1179, 283, 590, 334, 714, 1061, 140, 965, 757, 430, 286, 1273, 2369, 114, 217, 3862, 27, 951, 60, 476, 3551, 344, 494, 90, 2219, 1796, 1325, 2132, 532, 0, 40, 166, 8, 2023, 337, 411, 390, 520, 206, 150, 166, 1445, 1981, 33, 1558, 779, 338, 356, 421, 1624, 1082, 711, 591, 775, 234, 975, 98, 204, 1263, 29, 100, 377] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[942, 996, 1056, 315, 608, 556, 899, 744, 117, 672, 846, 635, 523, 1213, 1874, 172, 281, 3508, 61, 960, 33, 862, 2252, 255, 435, 422, 2109, 1504, 1277, 2021, 771, 3, 65, 246, 19, 1722, 364, 359, 479, 624, 286, 184, 284, 1102, 2391, 20, 889, 1024, 353, 429, 731, 1235, 1070, 894, 493, 808, 170, 905, 95, 292, 2114, 87, 120, 376] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[375, 743, 1192, 389, 392, 526, 800, 957, 157, 929, 720, 445, 265, 1404, 1796, 93, 236, 3578, 34, 1078, 54, 456, 2414, 323, 443, 70, 2331, 1932, 2019, 2203, 565, 1, 65, 112, 9, 2294, 420, 298, 395, 593, 252, 193, 249, 1833, 1894, 43, 1437, 1107, 391, 368, 563, 1279, 957, 1086, 592, 731, 156, 601, 161, 228, 1466, 51, 57, 351] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[409, 850, 967, 470, 662, 388, 706, 1129, 198, 727, 535, 480, 282, 1153, 2401, 112, 288, 3965, 37, 820, 64, 542, 2401, 255, 668, 396, 1864, 1874, 1536, 2481, 870, 1, 29, 186, 4, 2116, 387, 366, 422, 450, 259, 149, 190, 1222, 1876, 28, 1610, 1210, 267, 404, 834, 1154, 818, 794, 458, 934, 129, 1245, 55, 163, 1275, 85, 94, 408] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[260, 796, 1328, 505, 434, 415, 548, 1154, 132, 863, 678, 491, 310, 1150, 2095, 86, 242, 4326, 22, 921, 50, 550, 2430, 286, 401, 540, 2297, 1918, 1478, 2545, 777, 0, 40, 73, 15, 2089, 314, 313, 449, 443, 182, 114, 247, 1177, 2200, 21, 1029, 705, 312, 368, 597, 1674, 1096, 639, 488, 669, 65, 1470, 99, 408, 1420, 32, 52, 324] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[416, 605, 1434, 441, 318, 348, 601, 1232, 124, 1114, 916, 447, 208, 1119, 2263, 128, 230, 4171, 29, 890, 58, 786, 2685, 215, 609, 465, 2342, 1626, 1515, 2497, 668, 0, 53, 107, 10, 1790, 317, 348, 474, 595, 207, 141, 201, 1162, 2094, 16, 1057, 590, 289, 353, 540, 1292, 1082, 581, 557, 590, 92, 1327, 81, 132, 1950, 36, 48, 540] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[307, 736, 1243, 387, 384, 704, 695, 1149, 223, 903, 782, 451, 308, 1113, 1857, 77, 246, 4230, 29, 891, 61, 626, 2352, 309, 519, 124, 2302, 1930, 1507, 2299, 519, 4, 37, 97, 20, 2002, 356, 379, 316, 544, 141, 119, 332, 1365, 2094, 29, 1309, 1147, 376, 383, 772, 1126, 1131, 816, 367, 613, 94, 1454, 135, 233, 1311, 97, 76, 614] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[409, 850, 967, 470, 662, 388, 706, 1129, 198, 727, 535, 480, 282, 1153, 2401, 112, 288, 3965, 37, 820, 64, 542, 2401, 255, 668, 396, 1864, 1874, 1536, 2481, 870, 1, 29, 186, 4, 2116, 387, 366, 422, 450, 259, 149, 190, 1222, 1876, 28, 1610, 1210, 267, 404, 834, 1154, 818, 794, 458, 934, 129, 1245, 55, 163, 1275, 85, 94, 408] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[425, 702, 1032, 352, 793, 520, 729, 1151, 193, 525, 934, 541, 252, 1258, 2158, 168, 248, 3989, 34, 959, 30, 446, 2879, 274, 571, 541, 2467, 1626, 1762, 2110, 619, 0, 40, 153, 8, 2217, 461, 406, 448, 409, 248, 122, 184, 1111, 2174, 24, 1115, 786, 345, 468, 543, 1623, 963, 687, 317, 703, 118, 1159, 68, 199, 1220, 89, 85, 371] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[486, 704, 1179, 283, 590, 334, 714, 1061, 421, 137, 0, 0, 0, 0, 0, 0, 140, 965, 757, 430, 286, 1273, 2369, 114, 106, 40, 0, 0, 0, 0, 0, 0, 217, 1349, 27, 951, 60, 476, 3551, 344, 0, 0, 0, 0, 0, 0, 0, 0, 494, 90, 1290, 1796, 1325, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 166, 8, 2023, 337, 411, 390, 520, 1995, 97, 0, 0, 0, 0, 0, 0, 206, 150, 166, 1445, 1941, 33, 1558, 779, 0, 0, 0, 0, 0, 0, 0, 0, 338, 356, 421, 1624, 1082, 711, 591, 775, 402, 0, 0, 0, 0, 0, 0, 0, 234, 975, 98, 204, 1263, 29, 100, 377, 2416, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5909, 6480, 6975, 5527, 5987, 6278, 6300, 5696] avg=6144.0000 max=6975 max_rank=2 min=5527 min_rank=3 imbalance_ratio=1.135254 max_load_reduction=42177 max_load_reduction_pct=0.858093 moved_to_echo=24261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[416, 605, 1434, 441, 318, 348, 601, 1232, 445, 160, 0, 0, 0, 0, 0, 0, 124, 1114, 916, 447, 208, 1119, 2263, 128, 112, 42, 0, 0, 0, 0, 0, 0, 230, 1457, 29, 890, 58, 786, 2685, 215, 0, 0, 0, 0, 0, 0, 0, 0, 609, 465, 1361, 1626, 1515, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 107, 10, 1790, 317, 348, 474, 595, 2337, 105, 0, 0, 0, 0, 0, 0, 207, 141, 201, 1162, 2052, 16, 1057, 590, 0, 0, 0, 0, 0, 0, 0, 0, 289, 353, 540, 1292, 1082, 581, 557, 590, 424, 0, 0, 0, 0, 0, 0, 0, 92, 1327, 81, 132, 1950, 36, 48, 540, 2609, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6000, 6473, 6350, 6244, 6136, 5426, 5708, 6815] avg=6144.0000 max=6815 max_rank=7 min=5426 min_rank=5 imbalance_ratio=1.109212 max_load_reduction=42337 max_load_reduction_pct=0.861348 moved_to_echo=24085 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[942, 996, 1056, 315, 608, 556, 899, 744, 401, 130, 0, 0, 0, 0, 0, 0, 117, 672, 846, 635, 523, 1213, 1874, 172, 101, 48, 0, 0, 0, 0, 0, 0, 281, 1226, 61, 960, 33, 862, 2252, 255, 0, 0, 0, 0, 0, 0, 0, 0, 435, 422, 1225, 1504, 1277, 0, 771, 3, 0, 0, 0, 0, 0, 0, 0, 0, 65, 246, 19, 1722, 364, 359, 479, 624, 1891, 88, 0, 0, 0, 0, 0, 0, 286, 184, 284, 1102, 2343, 20, 889, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 353, 429, 731, 1235, 1070, 894, 493, 808, 382, 0, 0, 0, 0, 0, 0, 0, 170, 905, 95, 292, 2114, 87, 120, 376, 2194, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6647, 6201, 5930, 5637, 5857, 6132, 6395, 6353] avg=6144.0000 max=6647 max_rank=0 min=5637 min_rank=3 imbalance_ratio=1.081868 max_load_reduction=42505 max_load_reduction_pct=0.864766 moved_to_echo=24737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[260, 796, 1328, 505, 434, 415, 548, 1154, 436, 164, 0, 0, 0, 0, 0, 0, 132, 863, 678, 491, 310, 1150, 2095, 86, 110, 44, 0, 0, 0, 0, 0, 0, 242, 1511, 22, 921, 50, 550, 2430, 286, 0, 0, 0, 0, 0, 0, 0, 0, 401, 540, 1335, 1918, 1478, 0, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 73, 15, 2089, 314, 313, 449, 443, 2381, 109, 0, 0, 0, 0, 0, 0, 182, 114, 247, 1177, 2156, 21, 1029, 705, 0, 0, 0, 0, 0, 0, 0, 0, 312, 368, 597, 1674, 1096, 639, 488, 669, 416, 0, 0, 0, 0, 0, 0, 0, 65, 1470, 99, 408, 1420, 32, 52, 324, 2706, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6040, 5959, 6012, 6449, 6226, 5631, 6259, 6576] avg=6144.0000 max=6576 max_rank=7 min=5631 min_rank=5 imbalance_ratio=1.070312 max_load_reduction=42576 max_load_reduction_pct=0.866211 moved_to_echo=24692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[260, 796, 1328, 505, 434, 415, 548, 1154, 132, 863, 678, 491, 310, 1150, 2095, 86, 242, 4326, 22, 921, 50, 550, 2430, 286, 401, 540, 2297, 1918, 1478, 2545, 777, 0, 40, 73, 15, 2089, 314, 313, 449, 443, 182, 114, 247, 1177, 2200, 21, 1029, 705, 312, 368, 597, 1674, 1096, 639, 488, 669, 65, 1470, 99, 408, 1420, 32, 52, 324] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[942, 996, 1056, 315, 608, 556, 899, 744, 117, 672, 846, 635, 523, 1213, 1874, 172, 281, 3508, 61, 960, 33, 862, 2252, 255, 435, 422, 2109, 1504, 1277, 2021, 771, 3, 65, 246, 19, 1722, 364, 359, 479, 624, 286, 184, 284, 1102, 2391, 20, 889, 1024, 353, 429, 731, 1235, 1070, 894, 493, 808, 170, 905, 95, 292, 2114, 87, 120, 376] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[486, 704, 1179, 283, 590, 334, 714, 1061, 140, 965, 757, 430, 286, 1273, 2369, 114, 217, 3862, 27, 951, 60, 476, 3551, 344, 494, 90, 2219, 1796, 1325, 2132, 532, 0, 40, 166, 8, 2023, 337, 411, 390, 520, 206, 150, 166, 1445, 1981, 33, 1558, 779, 338, 356, 421, 1624, 1082, 711, 591, 775, 234, 975, 98, 204, 1263, 29, 100, 377] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[375, 743, 1192, 389, 392, 526, 800, 957, 157, 929, 720, 445, 265, 1404, 1796, 93, 236, 3578, 34, 1078, 54, 456, 2414, 323, 443, 70, 2331, 1932, 2019, 2203, 565, 1, 65, 112, 9, 2294, 420, 298, 395, 593, 252, 193, 249, 1833, 1894, 43, 1437, 1107, 391, 368, 563, 1279, 957, 1086, 592, 731, 156, 601, 161, 228, 1466, 51, 57, 351] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[307, 736, 1243, 387, 384, 704, 695, 1149, 437, 148, 0, 0, 0, 0, 0, 0, 223, 903, 782, 451, 308, 1113, 1857, 77, 110, 42, 0, 0, 0, 0, 0, 0, 246, 1478, 29, 891, 61, 626, 2352, 309, 0, 0, 0, 0, 0, 0, 0, 0, 519, 124, 1338, 1930, 1507, 0, 519, 4, 0, 0, 0, 0, 0, 0, 0, 0, 37, 97, 20, 2002, 356, 379, 316, 544, 2151, 106, 0, 0, 0, 0, 0, 0, 141, 119, 332, 1365, 2052, 29, 1309, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 376, 383, 772, 1126, 1131, 816, 367, 613, 417, 0, 0, 0, 0, 0, 0, 0, 94, 1454, 135, 233, 1311, 97, 76, 614, 2646, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6190, 5866, 5992, 5941, 6008, 6494, 6001, 6660] avg=6144.0000 max=6660 max_rank=7 min=5866 min_rank=1 imbalance_ratio=1.083984 max_load_reduction=42492 max_load_reduction_pct=0.864502 moved_to_echo=25163 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[409, 850, 967, 470, 662, 388, 706, 1129, 354, 159, 0, 0, 0, 0, 0, 0, 198, 727, 535, 480, 282, 1153, 2401, 112, 89, 38, 0, 0, 0, 0, 0, 0, 288, 1385, 37, 820, 64, 542, 2401, 255, 0, 0, 0, 0, 0, 0, 0, 0, 668, 396, 1084, 1874, 1536, 0, 870, 1, 0, 0, 0, 0, 0, 0, 0, 0, 29, 186, 4, 2116, 387, 366, 422, 450, 2322, 100, 0, 0, 0, 0, 0, 0, 259, 149, 190, 1222, 1838, 28, 1610, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 267, 404, 834, 1154, 818, 794, 458, 934, 337, 0, 0, 0, 0, 0, 0, 0, 129, 1245, 55, 163, 1275, 85, 94, 408, 2480, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 6015, 5792, 6429, 6382, 6506, 6000, 5934] avg=6144.0000 max=6506 max_rank=5 min=5792 min_rank=2 imbalance_ratio=1.058919 max_load_reduction=42646 max_load_reduction_pct=0.867635 moved_to_echo=24822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[416, 605, 1434, 441, 318, 348, 601, 1232, 445, 160, 0, 0, 0, 0, 0, 0, 124, 1114, 916, 447, 208, 1119, 2263, 128, 112, 42, 0, 0, 0, 0, 0, 0, 230, 1457, 29, 890, 58, 786, 2685, 215, 0, 0, 0, 0, 0, 0, 0, 0, 609, 465, 1361, 1626, 1515, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 107, 10, 1790, 317, 348, 474, 595, 2337, 105, 0, 0, 0, 0, 0, 0, 207, 141, 201, 1162, 2052, 16, 1057, 590, 0, 0, 0, 0, 0, 0, 0, 0, 289, 353, 540, 1292, 1082, 581, 557, 590, 424, 0, 0, 0, 0, 0, 0, 0, 92, 1327, 81, 132, 1950, 36, 48, 540, 2609, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6000, 6473, 6350, 6244, 6136, 5426, 5708, 6815] avg=6144.0000 max=6815 max_rank=7 min=5426 min_rank=5 imbalance_ratio=1.109212 max_load_reduction=42337 max_load_reduction_pct=0.861348 moved_to_echo=24085 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[260, 796, 1328, 505, 434, 415, 548, 1154, 436, 164, 0, 0, 0, 0, 0, 0, 132, 863, 678, 491, 310, 1150, 2095, 86, 110, 44, 0, 0, 0, 0, 0, 0, 242, 1511, 22, 921, 50, 550, 2430, 286, 0, 0, 0, 0, 0, 0, 0, 0, 401, 540, 1335, 1918, 1478, 0, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 73, 15, 2089, 314, 313, 449, 443, 2381, 109, 0, 0, 0, 0, 0, 0, 182, 114, 247, 1177, 2156, 21, 1029, 705, 0, 0, 0, 0, 0, 0, 0, 0, 312, 368, 597, 1674, 1096, 639, 488, 669, 416, 0, 0, 0, 0, 0, 0, 0, 65, 1470, 99, 408, 1420, 32, 52, 324, 2706, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6040, 5959, 6012, 6449, 6226, 5631, 6259, 6576] avg=6144.0000 max=6576 max_rank=7 min=5631 min_rank=5 imbalance_ratio=1.070312 max_load_reduction=42576 max_load_reduction_pct=0.866211 moved_to_echo=24692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[942, 996, 1056, 315, 608, 556, 899, 744, 401, 130, 0, 0, 0, 0, 0, 0, 117, 672, 846, 635, 523, 1213, 1874, 172, 101, 48, 0, 0, 0, 0, 0, 0, 281, 1226, 61, 960, 33, 862, 2252, 255, 0, 0, 0, 0, 0, 0, 0, 0, 435, 422, 1225, 1504, 1277, 0, 771, 3, 0, 0, 0, 0, 0, 0, 0, 0, 65, 246, 19, 1722, 364, 359, 479, 624, 1891, 88, 0, 0, 0, 0, 0, 0, 286, 184, 284, 1102, 2343, 20, 889, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 353, 429, 731, 1235, 1070, 894, 493, 808, 382, 0, 0, 0, 0, 0, 0, 0, 170, 905, 95, 292, 2114, 87, 120, 376, 2194, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6647, 6201, 5930, 5637, 5857, 6132, 6395, 6353] avg=6144.0000 max=6647 max_rank=0 min=5637 min_rank=3 imbalance_ratio=1.081868 max_load_reduction=42505 max_load_reduction_pct=0.864766 moved_to_echo=24737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[486, 704, 1179, 283, 590, 334, 714, 1061, 421, 137, 0, 0, 0, 0, 0, 0, 140, 965, 757, 430, 286, 1273, 2369, 114, 106, 40, 0, 0, 0, 0, 0, 0, 217, 1349, 27, 951, 60, 476, 3551, 344, 0, 0, 0, 0, 0, 0, 0, 0, 494, 90, 1290, 1796, 1325, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 166, 8, 2023, 337, 411, 390, 520, 1995, 97, 0, 0, 0, 0, 0, 0, 206, 150, 166, 1445, 1941, 33, 1558, 779, 0, 0, 0, 0, 0, 0, 0, 0, 338, 356, 421, 1624, 1082, 711, 591, 775, 402, 0, 0, 0, 0, 0, 0, 0, 234, 975, 98, 204, 1263, 29, 100, 377, 2416, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5909, 6480, 6975, 5527, 5987, 6278, 6300, 5696] avg=6144.0000 max=6975 max_rank=2 min=5527 min_rank=3 imbalance_ratio=1.135254 max_load_reduction=42177 max_load_reduction_pct=0.858093 moved_to_echo=24261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[375, 743, 1192, 389, 392, 526, 800, 957, 443, 142, 0, 0, 0, 0, 0, 0, 157, 929, 720, 445, 265, 1404, 1796, 93, 111, 38, 0, 0, 0, 0, 0, 0, 236, 1250, 34, 1078, 54, 456, 2414, 323, 0, 0, 0, 0, 0, 0, 0, 0, 443, 70, 1355, 1932, 2019, 0, 565, 1, 0, 0, 0, 0, 0, 0, 0, 0, 65, 112, 9, 2294, 420, 298, 395, 593, 2061, 90, 0, 0, 0, 0, 0, 0, 252, 193, 249, 1833, 1856, 43, 1437, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 391, 368, 563, 1279, 957, 1086, 592, 731, 422, 0, 0, 0, 0, 0, 0, 0, 156, 601, 161, 228, 1466, 51, 57, 351, 2238, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5959, 5958, 5845, 6385, 6337, 6970, 6389, 5309] avg=6144.0000 max=6970 max_rank=5 min=5309 min_rank=7 imbalance_ratio=1.134440 max_load_reduction=42182 max_load_reduction_pct=0.858195 moved_to_echo=25005 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[425, 702, 1032, 352, 793, 520, 729, 1151, 473, 136, 0, 0, 0, 0, 0, 0, 193, 525, 934, 541, 252, 1258, 2158, 168, 122, 49, 0, 0, 0, 0, 0, 0, 248, 1386, 34, 959, 30, 446, 2879, 274, 0, 0, 0, 0, 0, 0, 0, 0, 571, 541, 1421, 1626, 1762, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 153, 8, 2217, 461, 406, 448, 409, 1974, 103, 0, 0, 0, 0, 0, 0, 248, 122, 184, 1111, 2125, 24, 1115, 786, 0, 0, 0, 0, 0, 0, 0, 0, 345, 468, 543, 1623, 963, 687, 317, 703, 451, 0, 0, 0, 0, 0, 0, 0, 118, 1159, 68, 199, 1220, 89, 85, 371, 2500, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6313, 6200, 6256, 6540, 6219, 5715, 6100, 5809] avg=6144.0000 max=6540 max_rank=3 min=5715 min_rank=5 imbalance_ratio=1.064453 max_load_reduction=42612 max_load_reduction_pct=0.866943 moved_to_echo=23843 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 3200, 5120, 11008, 8320, 6272, 3968, 5888, 3328, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1408, 6784, 6272, 3968, 2560, 9728, 16896, 1024, 896, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 3200, 5120, 11008, 8320, 6272, 3968, 5888, 3328, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 2688, 10496, 14208, 12544, 0, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 1280, 1920, 10496, 16384, 256, 10112, 7424, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[384, 1152, 128, 16256, 3072, 2944, 3456, 4224, 17152, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 6144, 9472, 3200, 4224, 3840, 5760, 8704, 3456, 1280, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1152, 9216, 896, 1920, 12032, 512, 640, 3456, 19840, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 1280, 1920, 10496, 16384, 256, 10112, 7424, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 11136, 384, 7552, 512, 4864, 20992, 2304, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 6144, 9472, 3200, 4224, 3840, 5760, 8704, 3456, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[108, 839, 819, 75, 629, 1031, 312, 3521, 553, 123, 161, 2257, 370, 1053, 1613, 508, 3267, 2383, 45, 223, 1921, 55, 1475, 66, 99, 109, 800, 798, 2, 440, 1545, 78, 699, 33, 218, 527, 25, 101, 3823, 2197, 1666, 168, 5, 369, 5, 437, 1357, 403, 288, 388, 1533, 2515, 645, 1027, 109, 418, 143, 484, 687, 60, 72, 767, 11, 694] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[206, 1051, 854, 34, 626, 916, 264, 3784, 327, 87, 230, 2358, 255, 1470, 1362, 359, 3507, 2546, 48, 130, 1166, 62, 1779, 66, 100, 62, 649, 593, 4, 805, 1354, 58, 830, 31, 242, 483, 25, 120, 3762, 2585, 1545, 117, 35, 167, 12, 372, 784, 469, 166, 805, 1642, 2446, 445, 810, 68, 446, 133, 647, 1194, 42, 37, 721, 8, 851] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[256, 865, 668, 56, 584, 750, 176, 4013, 517, 84, 196, 2120, 368, 947, 1141, 557, 3040, 2772, 71, 232, 1335, 21, 1765, 42, 103, 30, 793, 847, 1, 838, 1452, 36, 692, 46, 372, 752, 23, 56, 3814, 2913, 1805, 159, 29, 286, 13, 408, 1269, 201, 151, 695, 1992, 2131, 679, 970, 44, 330, 140, 499, 768, 61, 60, 565, 14, 539] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[245, 990, 597, 51, 550, 711, 199, 3698, 609, 140, 201, 2100, 272, 935, 1263, 219, 3608, 3778, 69, 312, 1470, 34, 1660, 14, 139, 70, 568, 481, 0, 625, 1661, 49, 656, 15, 132, 827, 5, 77, 3505, 2603, 1672, 145, 2, 235, 6, 529, 951, 237, 124, 525, 1814, 2655, 669, 852, 36, 224, 145, 724, 1048, 28, 25, 644, 6, 718] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[210, 944, 773, 33, 661, 1015, 100, 3927, 472, 110, 133, 1800, 302, 737, 1214, 465, 3371, 2948, 75, 203, 1634, 47, 1809, 53, 161, 48, 733, 1008, 0, 496, 1255, 55, 748, 26, 398, 653, 39, 97, 3605, 2371, 1640, 128, 17, 361, 5, 579, 939, 249, 187, 717, 1925, 2659, 440, 788, 93, 619, 93, 739, 920, 43, 33, 599, 19, 631] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[249, 876, 710, 72, 443, 881, 288, 3709, 505, 125, 95, 1795, 363, 836, 1337, 755, 3461, 2403, 60, 387, 2361, 85, 1461, 154, 94, 60, 870, 745, 8, 573, 1229, 124, 824, 79, 337, 488, 26, 142, 3688, 1932, 1676, 130, 24, 383, 34, 653, 1064, 688, 403, 509, 1258, 2898, 288, 855, 126, 566, 71, 781, 669, 48, 29, 823, 33, 513] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[274, 1243, 544, 36, 578, 735, 228, 3353, 673, 123, 182, 2183, 399, 607, 1630, 395, 2868, 2821, 65, 160, 1669, 38, 1372, 46, 86, 52, 609, 941, 1, 559, 1500, 55, 651, 35, 391, 845, 10, 66, 3895, 2278, 1657, 156, 39, 174, 15, 475, 1160, 276, 124, 327, 2055, 2567, 667, 1089, 66, 498, 116, 831, 1136, 64, 58, 703, 8, 695] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[245, 990, 597, 51, 550, 711, 199, 3698, 609, 140, 201, 2100, 272, 935, 1263, 219, 3608, 3778, 69, 312, 1470, 34, 1660, 14, 139, 70, 568, 481, 0, 625, 1661, 49, 656, 15, 132, 827, 5, 77, 3505, 2603, 1672, 145, 2, 235, 6, 529, 951, 237, 124, 525, 1814, 2655, 669, 852, 36, 224, 145, 724, 1048, 28, 25, 644, 6, 718] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[108, 839, 819, 75, 629, 1031, 312, 3521, 553, 123, 161, 2257, 370, 1053, 1613, 508, 3267, 2383, 45, 223, 1921, 55, 1475, 66, 99, 109, 800, 798, 2, 440, 1545, 78, 699, 33, 218, 527, 25, 101, 3823, 2197, 1666, 168, 5, 369, 5, 437, 1357, 403, 288, 388, 1533, 2515, 645, 1027, 109, 418, 143, 484, 687, 60, 72, 767, 11, 694] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[128, 875, 562, 69, 783, 692, 265, 3970, 440, 182, 134, 2123, 352, 562, 1752, 480, 3427, 3050, 62, 156, 1639, 61, 1830, 53, 140, 56, 678, 593, 0, 905, 1780, 41, 712, 37, 264, 626, 19, 92, 3647, 2573, 1611, 230, 17, 244, 22, 367, 999, 188, 188, 271, 1943, 2380, 425, 919, 109, 330, 149, 535, 1115, 108, 30, 618, 12, 532] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[256, 865, 668, 56, 584, 750, 176, 4013, 517, 84, 196, 2120, 368, 947, 1141, 557, 3040, 2772, 71, 232, 1335, 21, 1765, 42, 103, 30, 793, 847, 1, 838, 1452, 36, 692, 46, 372, 752, 23, 56, 3814, 2913, 1805, 159, 29, 286, 13, 408, 1269, 201, 151, 695, 1992, 2131, 679, 970, 44, 330, 140, 499, 768, 61, 60, 565, 14, 539] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[249, 876, 710, 72, 443, 881, 288, 3709, 505, 125, 95, 1795, 363, 836, 1337, 755, 3461, 2403, 60, 387, 2361, 85, 1461, 154, 94, 60, 870, 745, 8, 573, 1229, 124, 824, 79, 337, 488, 26, 142, 3688, 1932, 1676, 130, 24, 383, 34, 653, 1064, 688, 403, 509, 1258, 2898, 288, 855, 126, 566, 71, 781, 669, 48, 29, 823, 33, 513] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[108, 839, 819, 75, 629, 1031, 312, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 553, 123, 161, 2257, 370, 1053, 1613, 508, 119, 0, 0, 0, 0, 0, 0, 0, 0, 2059, 45, 223, 1921, 55, 1475, 66, 0, 0, 0, 0, 0, 0, 0, 0, 99, 109, 800, 798, 2, 440, 1545, 78, 1880, 330, 139, 0, 0, 0, 0, 0, 699, 33, 218, 527, 25, 101, 1943, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 1666, 168, 5, 369, 5, 437, 1357, 403, 982, 841, 205, 0, 0, 0, 0, 0, 288, 388, 1533, 1674, 645, 1027, 109, 418, 0, 0, 0, 0, 0, 0, 0, 0, 143, 484, 687, 60, 72, 767, 11, 694, 2937, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6213, 6757, 5844, 6220, 5743, 6438, 6082, 5855] avg=6144.0000 max=6757 max_rank=1 min=5743 min_rank=4 imbalance_ratio=1.099772 max_load_reduction=42395 max_load_reduction_pct=0.862528 moved_to_echo=24118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[256, 865, 668, 56, 584, 750, 176, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 517, 84, 196, 2120, 368, 947, 1141, 557, 138, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 71, 232, 1335, 21, 1765, 42, 0, 0, 0, 0, 0, 0, 0, 0, 103, 30, 793, 847, 1, 838, 1452, 36, 1876, 307, 159, 0, 0, 0, 0, 0, 692, 46, 372, 752, 23, 56, 1938, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 1805, 159, 29, 286, 13, 408, 1269, 201, 1119, 713, 238, 0, 0, 0, 0, 0, 151, 695, 1992, 1418, 679, 970, 44, 330, 0, 0, 0, 0, 0, 0, 0, 0, 140, 499, 768, 61, 60, 565, 14, 539, 2733, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6090, 6068, 5862, 6442, 6792, 6240, 6279, 5379] avg=6144.0000 max=6792 max_rank=4 min=5379 min_rank=7 imbalance_ratio=1.105469 max_load_reduction=42360 max_load_reduction_pct=0.861816 moved_to_echo=24690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[206, 1051, 854, 34, 626, 916, 264, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 327, 87, 230, 2358, 255, 1470, 1362, 359, 127, 0, 0, 0, 0, 0, 0, 0, 0, 2200, 48, 130, 1166, 62, 1779, 66, 0, 0, 0, 0, 0, 0, 0, 0, 100, 62, 649, 593, 4, 805, 1354, 58, 1850, 354, 150, 0, 0, 0, 0, 0, 830, 31, 242, 483, 25, 120, 1912, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 117, 35, 167, 12, 372, 784, 469, 1055, 818, 219, 0, 0, 0, 0, 0, 166, 805, 1642, 1628, 445, 810, 68, 446, 0, 0, 0, 0, 0, 0, 0, 0, 133, 647, 1194, 42, 37, 721, 8, 851, 3153, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6530, 6575, 5451, 5979, 6228, 5593, 6010, 6786] avg=6144.0000 max=6786 max_rank=7 min=5451 min_rank=2 imbalance_ratio=1.104492 max_load_reduction=42366 max_load_reduction_pct=0.861938 moved_to_echo=24617 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[245, 990, 597, 51, 550, 711, 199, 2521, 0, 0, 0, 0, 0, 0, 0, 0, 609, 140, 201, 2100, 272, 935, 1263, 219, 189, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 69, 312, 1470, 34, 1660, 14, 0, 0, 0, 0, 0, 0, 0, 0, 139, 70, 568, 481, 0, 625, 1661, 49, 1724, 365, 146, 0, 0, 0, 0, 0, 656, 15, 132, 827, 5, 77, 1781, 2603, 0, 0, 0, 0, 0, 0, 0, 0, 1672, 145, 2, 235, 6, 529, 951, 237, 1031, 888, 325, 0, 0, 0, 0, 0, 124, 525, 1814, 1767, 669, 852, 36, 224, 0, 0, 0, 0, 0, 0, 0, 0, 145, 724, 1048, 28, 25, 644, 6, 718, 3243, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5864, 5928, 6823, 5828, 6096, 6021, 6011, 6581] avg=6144.0000 max=6823 max_rank=2 min=5828 min_rank=3 imbalance_ratio=1.110514 max_load_reduction=42329 max_load_reduction_pct=0.861186 moved_to_echo=24709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[210, 944, 773, 33, 661, 1015, 100, 2676, 0, 0, 0, 0, 0, 0, 0, 0, 472, 110, 133, 1800, 302, 737, 1214, 465, 147, 0, 0, 0, 0, 0, 0, 0, 0, 2548, 75, 203, 1634, 47, 1809, 53, 0, 0, 0, 0, 0, 0, 0, 0, 161, 48, 733, 1008, 0, 496, 1255, 55, 1773, 342, 156, 0, 0, 0, 0, 0, 748, 26, 398, 653, 39, 97, 1832, 2371, 0, 0, 0, 0, 0, 0, 0, 0, 1640, 128, 17, 361, 5, 579, 939, 249, 1095, 890, 253, 0, 0, 0, 0, 0, 187, 717, 1925, 1769, 440, 788, 93, 619, 0, 0, 0, 0, 0, 0, 0, 0, 93, 739, 920, 43, 33, 599, 19, 631, 3029, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6412, 5380, 6369, 6027, 6164, 6156, 6538, 6106] avg=6144.0000 max=6538 max_rank=6 min=5380 min_rank=1 imbalance_ratio=1.064128 max_load_reduction=42614 max_load_reduction_pct=0.866984 moved_to_echo=24964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[249, 876, 710, 72, 443, 881, 288, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 505, 125, 95, 1795, 363, 836, 1337, 755, 120, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 60, 387, 2361, 85, 1461, 154, 0, 0, 0, 0, 0, 0, 0, 0, 94, 60, 870, 745, 8, 573, 1229, 124, 1814, 351, 147, 0, 0, 0, 0, 0, 824, 79, 337, 488, 26, 142, 1874, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 1676, 130, 24, 383, 34, 653, 1064, 688, 1034, 970, 206, 0, 0, 0, 0, 0, 403, 509, 1258, 1928, 288, 855, 126, 566, 0, 0, 0, 0, 0, 0, 0, 0, 71, 781, 669, 48, 29, 823, 33, 513, 3110, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6047, 5931, 6585, 6015, 5702, 6862, 5933, 6077] avg=6144.0000 max=6862 max_rank=5 min=5702 min_rank=4 imbalance_ratio=1.116862 max_load_reduction=42290 max_load_reduction_pct=0.860392 moved_to_echo=24574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[108, 839, 819, 75, 629, 1031, 312, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 553, 123, 161, 2257, 370, 1053, 1613, 508, 119, 0, 0, 0, 0, 0, 0, 0, 0, 2059, 45, 223, 1921, 55, 1475, 66, 0, 0, 0, 0, 0, 0, 0, 0, 99, 109, 800, 798, 2, 440, 1545, 78, 1880, 330, 139, 0, 0, 0, 0, 0, 699, 33, 218, 527, 25, 101, 1943, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 1666, 168, 5, 369, 5, 437, 1357, 403, 982, 841, 205, 0, 0, 0, 0, 0, 288, 388, 1533, 1674, 645, 1027, 109, 418, 0, 0, 0, 0, 0, 0, 0, 0, 143, 484, 687, 60, 72, 767, 11, 694, 2937, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6213, 6757, 5844, 6220, 5743, 6438, 6082, 5855] avg=6144.0000 max=6757 max_rank=1 min=5743 min_rank=4 imbalance_ratio=1.099772 max_load_reduction=42395 max_load_reduction_pct=0.862528 moved_to_echo=24118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[274, 1243, 544, 36, 578, 735, 228, 2285, 0, 0, 0, 0, 0, 0, 0, 0, 673, 123, 182, 2183, 399, 607, 1630, 395, 141, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 65, 160, 1669, 38, 1372, 46, 0, 0, 0, 0, 0, 0, 0, 0, 86, 52, 609, 941, 1, 559, 1500, 55, 1916, 290, 133, 0, 0, 0, 0, 0, 651, 35, 391, 845, 10, 66, 1979, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 1657, 156, 39, 174, 15, 475, 1160, 276, 935, 859, 242, 0, 0, 0, 0, 0, 124, 327, 2055, 1708, 667, 1089, 66, 498, 0, 0, 0, 0, 0, 0, 0, 0, 116, 831, 1136, 64, 58, 703, 8, 695, 2578, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5923, 6333, 5788, 6142, 6255, 5988, 6534, 6189] avg=6144.0000 max=6534 max_rank=6 min=5788 min_rank=2 imbalance_ratio=1.063477 max_load_reduction=42618 max_load_reduction_pct=0.867065 moved_to_echo=24966 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[245, 990, 597, 51, 550, 711, 199, 2521, 0, 0, 0, 0, 0, 0, 0, 0, 609, 140, 201, 2100, 272, 935, 1263, 219, 189, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 69, 312, 1470, 34, 1660, 14, 0, 0, 0, 0, 0, 0, 0, 0, 139, 70, 568, 481, 0, 625, 1661, 49, 1724, 365, 146, 0, 0, 0, 0, 0, 656, 15, 132, 827, 5, 77, 1781, 2603, 0, 0, 0, 0, 0, 0, 0, 0, 1672, 145, 2, 235, 6, 529, 951, 237, 1031, 888, 325, 0, 0, 0, 0, 0, 124, 525, 1814, 1767, 669, 852, 36, 224, 0, 0, 0, 0, 0, 0, 0, 0, 145, 724, 1048, 28, 25, 644, 6, 718, 3243, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5864, 5928, 6823, 5828, 6096, 6021, 6011, 6581] avg=6144.0000 max=6823 max_rank=2 min=5828 min_rank=3 imbalance_ratio=1.110514 max_load_reduction=42329 max_load_reduction_pct=0.861186 moved_to_echo=24709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[128, 875, 562, 69, 783, 692, 265, 2699, 0, 0, 0, 0, 0, 0, 0, 0, 440, 182, 134, 2123, 352, 562, 1752, 480, 155, 0, 0, 0, 0, 0, 0, 0, 0, 2629, 62, 156, 1639, 61, 1830, 53, 0, 0, 0, 0, 0, 0, 0, 0, 140, 56, 678, 593, 0, 905, 1780, 41, 1798, 347, 161, 0, 0, 0, 0, 0, 712, 37, 264, 626, 19, 92, 1849, 2573, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 230, 17, 244, 22, 367, 999, 188, 1110, 800, 266, 0, 0, 0, 0, 0, 188, 271, 1943, 1580, 425, 919, 109, 330, 0, 0, 0, 0, 0, 0, 0, 0, 149, 535, 1115, 108, 30, 618, 12, 532, 3080, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6180, 6430, 6499, 6172, 5854, 5765, 6179] avg=6144.0000 max=6499 max_rank=3 min=5765 min_rank=6 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=23970 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[256, 865, 668, 56, 584, 750, 176, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 517, 84, 196, 2120, 368, 947, 1141, 557, 138, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 71, 232, 1335, 21, 1765, 42, 0, 0, 0, 0, 0, 0, 0, 0, 103, 30, 793, 847, 1, 838, 1452, 36, 1876, 307, 159, 0, 0, 0, 0, 0, 692, 46, 372, 752, 23, 56, 1938, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 1805, 159, 29, 286, 13, 408, 1269, 201, 1119, 713, 238, 0, 0, 0, 0, 0, 151, 695, 1992, 1418, 679, 970, 44, 330, 0, 0, 0, 0, 0, 0, 0, 0, 140, 499, 768, 61, 60, 565, 14, 539, 2733, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6090, 6068, 5862, 6442, 6792, 6240, 6279, 5379] avg=6144.0000 max=6792 max_rank=4 min=5379 min_rank=7 imbalance_ratio=1.105469 max_load_reduction=42360 max_load_reduction_pct=0.861816 moved_to_echo=24690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[249, 876, 710, 72, 443, 881, 288, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 505, 125, 95, 1795, 363, 836, 1337, 755, 120, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 60, 387, 2361, 85, 1461, 154, 0, 0, 0, 0, 0, 0, 0, 0, 94, 60, 870, 745, 8, 573, 1229, 124, 1814, 351, 147, 0, 0, 0, 0, 0, 824, 79, 337, 488, 26, 142, 1874, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 1676, 130, 24, 383, 34, 653, 1064, 688, 1034, 970, 206, 0, 0, 0, 0, 0, 403, 509, 1258, 1928, 288, 855, 126, 566, 0, 0, 0, 0, 0, 0, 0, 0, 71, 781, 669, 48, 29, 823, 33, 513, 3110, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6047, 5931, 6585, 6015, 5702, 6862, 5933, 6077] avg=6144.0000 max=6862 max_rank=5 min=5702 min_rank=4 imbalance_ratio=1.116862 max_load_reduction=42290 max_load_reduction_pct=0.860392 moved_to_echo=24574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4096, 1024, 1408, 16768, 2688, 7168, 11392, 3840, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13312, 1280, 256, 2304, 128, 3840, 8576, 2816, 8448, 6784, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4096, 1024, 1408, 16768, 2688, 7168, 11392, 3840, 1152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1024, 512, 5760, 6016, 128, 5248, 11776, 512, 14720, 2688, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 7808, 5632, 512, 4864, 6784, 1920, 20480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5888, 384, 2432, 5248, 256, 768, 15232, 19456, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 4352, 14208, 13568, 4352, 7424, 768, 3456, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1024, 5248, 7552, 512, 384, 5504, 128, 5248, 23936, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 7808, 5632, 512, 4864, 6784, 1920, 20480, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 19712, 512, 1920, 13312, 512, 13184, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5888, 384, 2432, 5248, 256, 768, 15232, 19456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1454, 59, 262, 4398, 1143, 5, 198, 1062, 508, 715, 847, 562, 4101, 1805, 67, 1606, 780, 47, 76, 1122, 26, 874, 831, 820, 39, 897, 341, 944, 279, 25, 49, 24, 51, 319, 876, 269, 780, 2, 160, 307, 742, 5104, 20, 2590, 2504, 86, 1927, 253, 261, 621, 135, 818, 2083, 1027, 838, 49, 257, 86, 8, 38, 76, 7, 881, 11] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1178, 54, 240, 4608, 307, 2, 124, 1264, 560, 899, 987, 352, 4345, 1900, 31, 1366, 663, 16, 24, 964, 35, 418, 938, 556, 16, 776, 126, 713, 260, 135, 23, 98, 19, 273, 937, 391, 723, 1, 140, 415, 575, 5270, 34, 2284, 2398, 42, 2262, 285, 233, 1780, 153, 913, 2415, 1742, 914, 70, 255, 59, 14, 38, 28, 3, 485, 23] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1426, 63, 177, 4118, 701, 9, 131, 1399, 369, 680, 791, 358, 5180, 2367, 15, 1438, 737, 37, 15, 1470, 49, 1083, 623, 846, 15, 858, 364, 916, 273, 42, 30, 35, 26, 353, 806, 171, 584, 1, 107, 377, 796, 5965, 22, 2124, 2417, 62, 1996, 246, 128, 364, 98, 944, 1693, 1539, 257, 34, 207, 137, 13, 74, 59, 1, 927, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1294, 66, 347, 4342, 729, 4, 161, 906, 371, 1171, 802, 444, 4202, 2054, 22, 1831, 709, 95, 26, 1023, 27, 623, 911, 1043, 26, 734, 337, 1176, 396, 26, 49, 44, 26, 276, 1272, 330, 777, 6, 80, 388, 807, 5294, 37, 2046, 2992, 54, 1695, 210, 160, 578, 131, 703, 2427, 646, 858, 20, 237, 32, 32, 61, 58, 2, 908, 18] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1728, 37, 96, 4776, 864, 5, 198, 1049, 533, 831, 765, 414, 5066, 2678, 4, 1478, 713, 21, 40, 970, 31, 609, 904, 645, 22, 747, 168, 1043, 562, 24, 36, 10, 11, 231, 872, 181, 373, 0, 31, 372, 913, 5668, 8, 1996, 3165, 14, 1531, 236, 97, 639, 181, 804, 2500, 715, 315, 3, 228, 41, 29, 31, 34, 2, 847, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1811, 33, 319, 3768, 665, 2, 209, 1238, 452, 716, 830, 253, 4973, 1785, 19, 786, 635, 24, 20, 1069, 30, 565, 882, 1172, 11, 850, 263, 928, 337, 49, 54, 50, 27, 450, 1521, 187, 661, 0, 33, 499, 762, 5528, 11, 2051, 2417, 41, 1863, 311, 179, 1741, 179, 711, 2179, 1194, 718, 35, 228, 74, 7, 47, 18, 4, 664, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1501, 89, 197, 4370, 813, 33, 120, 890, 344, 719, 864, 635, 4681, 2779, 48, 1834, 727, 40, 26, 1479, 47, 523, 625, 708, 42, 846, 448, 1378, 517, 65, 65, 46, 18, 335, 937, 91, 674, 2, 66, 295, 263, 4771, 16, 2310, 2250, 65, 1664, 254, 116, 835, 58, 579, 1730, 2403, 953, 23, 201, 168, 7, 90, 52, 2, 417, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1485, 93, 373, 3839, 1153, 32, 117, 992, 347, 1041, 833, 390, 3987, 1879, 28, 1825, 624, 99, 115, 897, 31, 795, 762, 1019, 14, 821, 340, 1171, 371, 65, 34, 50, 33, 187, 829, 514, 751, 4, 110, 732, 1042, 6016, 16, 2626, 2841, 61, 1217, 259, 150, 542, 226, 786, 2023, 686, 341, 26, 309, 85, 2, 25, 71, 3, 989, 28] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1454, 59, 262, 2346, 1143, 5, 198, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 508, 715, 847, 562, 0, 1714, 67, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 780, 47, 76, 1122, 26, 874, 831, 820, 1498, 285, 2, 0, 0, 0, 0, 0, 39, 897, 341, 944, 279, 25, 49, 24, 2627, 497, 0, 0, 0, 0, 0, 0, 51, 319, 876, 269, 780, 2, 160, 307, 1767, 1474, 0, 0, 0, 0, 0, 0, 742, 0, 20, 2590, 1006, 86, 1927, 253, 0, 0, 0, 0, 0, 0, 0, 0, 261, 621, 135, 818, 2083, 1027, 838, 49, 89, 0, 0, 0, 0, 0, 0, 0, 257, 86, 8, 38, 76, 7, 881, 11, 4607, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6529, 6019, 6361, 5722, 6005, 6624, 5921, 5971] avg=6144.0000 max=6624 max_rank=5 min=5722 min_rank=3 imbalance_ratio=1.078125 max_load_reduction=42528 max_load_reduction_pct=0.865234 moved_to_echo=24521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1294, 66, 347, 2316, 729, 4, 161, 906, 0, 0, 0, 0, 0, 0, 0, 0, 371, 1171, 802, 444, 0, 1951, 22, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 709, 95, 26, 1023, 27, 623, 911, 1043, 1790, 281, 2, 0, 0, 0, 0, 0, 26, 734, 337, 1176, 396, 26, 49, 44, 2693, 516, 0, 0, 0, 0, 0, 0, 26, 276, 1272, 330, 777, 6, 80, 388, 1745, 1509, 0, 0, 0, 0, 0, 0, 807, 0, 37, 2046, 1202, 54, 1695, 210, 0, 0, 0, 0, 0, 0, 0, 0, 160, 578, 131, 703, 2427, 646, 858, 20, 101, 0, 0, 0, 0, 0, 0, 0, 237, 32, 32, 61, 58, 2, 908, 18, 4778, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5823, 6592, 6530, 5997, 6409, 6051, 5624, 6126] avg=6144.0000 max=6592 max_rank=1 min=5624 min_rank=6 imbalance_ratio=1.072917 max_load_reduction=42560 max_load_reduction_pct=0.865885 moved_to_echo=24210 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1178, 54, 240, 2458, 307, 2, 124, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 560, 899, 987, 352, 0, 1805, 31, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 663, 16, 24, 964, 35, 418, 938, 556, 1434, 298, 2, 0, 0, 0, 0, 0, 16, 776, 126, 713, 260, 135, 23, 98, 2784, 514, 0, 0, 0, 0, 0, 0, 19, 273, 937, 391, 723, 1, 140, 415, 1852, 1561, 0, 0, 0, 0, 0, 0, 575, 0, 34, 2284, 964, 42, 2262, 285, 0, 0, 0, 0, 0, 0, 0, 0, 233, 1780, 153, 913, 2415, 1742, 914, 70, 93, 0, 0, 0, 0, 0, 0, 0, 255, 59, 14, 38, 28, 3, 485, 23, 4756, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5627, 6000, 5348, 5445, 6312, 6446, 8313, 5661] avg=6144.0000 max=8313 max_rank=6 min=5348 min_rank=2 imbalance_ratio=1.353027 max_load_reduction=40839 max_load_reduction_pct=0.830872 moved_to_echo=26732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1294, 66, 347, 4342, 729, 4, 161, 906, 371, 1171, 802, 444, 4202, 2054, 22, 1831, 709, 95, 26, 1023, 27, 623, 911, 1043, 26, 734, 337, 1176, 396, 26, 49, 44, 26, 276, 1272, 330, 777, 6, 80, 388, 807, 5294, 37, 2046, 2992, 54, 1695, 210, 160, 578, 131, 703, 2427, 646, 858, 20, 237, 32, 32, 61, 58, 2, 908, 18] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1178, 54, 240, 4608, 307, 2, 124, 1264, 560, 899, 987, 352, 4345, 1900, 31, 1366, 663, 16, 24, 964, 35, 418, 938, 556, 16, 776, 126, 713, 260, 135, 23, 98, 19, 273, 937, 391, 723, 1, 140, 415, 575, 5270, 34, 2284, 2398, 42, 2262, 285, 233, 1780, 153, 913, 2415, 1742, 914, 70, 255, 59, 14, 38, 28, 3, 485, 23] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1454, 59, 262, 4398, 1143, 5, 198, 1062, 508, 715, 847, 562, 4101, 1805, 67, 1606, 780, 47, 76, 1122, 26, 874, 831, 820, 39, 897, 341, 944, 279, 25, 49, 24, 51, 319, 876, 269, 780, 2, 160, 307, 742, 5104, 20, 2590, 2504, 86, 1927, 253, 261, 621, 135, 818, 2083, 1027, 838, 49, 257, 86, 8, 38, 76, 7, 881, 11] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1728, 37, 96, 4776, 864, 5, 198, 1049, 533, 831, 765, 414, 5066, 2678, 4, 1478, 713, 21, 40, 970, 31, 609, 904, 645, 22, 747, 168, 1043, 562, 24, 36, 10, 11, 231, 872, 181, 373, 0, 31, 372, 913, 5668, 8, 1996, 3165, 14, 1531, 236, 97, 639, 181, 804, 2500, 715, 315, 3, 228, 41, 29, 31, 34, 2, 847, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1811, 33, 319, 2010, 665, 2, 209, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 452, 716, 830, 253, 0, 1695, 19, 786, 0, 0, 0, 0, 0, 0, 0, 0, 635, 24, 20, 1069, 30, 565, 882, 1172, 1446, 244, 2, 0, 0, 0, 0, 0, 11, 850, 263, 928, 337, 49, 54, 50, 3186, 538, 0, 0, 0, 0, 0, 0, 27, 450, 1521, 187, 661, 0, 33, 499, 1514, 1787, 0, 0, 0, 0, 0, 0, 762, 0, 11, 2051, 971, 41, 1863, 311, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1741, 179, 711, 2179, 1194, 718, 35, 88, 0, 0, 0, 0, 0, 0, 0, 228, 74, 7, 47, 18, 4, 664, 14, 4990, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6287, 4751, 6089, 6266, 6679, 6010, 7024, 6046] avg=6144.0000 max=7024 max_rank=6 min=4751 min_rank=1 imbalance_ratio=1.143229 max_load_reduction=42128 max_load_reduction_pct=0.857096 moved_to_echo=25759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1426, 63, 177, 2197, 701, 9, 131, 1399, 0, 0, 0, 0, 0, 0, 0, 0, 369, 680, 791, 358, 0, 2247, 15, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 737, 37, 15, 1470, 49, 1083, 623, 846, 1446, 266, 3, 0, 0, 0, 0, 0, 15, 858, 364, 916, 273, 42, 30, 35, 3319, 581, 0, 0, 0, 0, 0, 0, 26, 353, 806, 171, 584, 1, 107, 377, 1655, 1861, 0, 0, 0, 0, 0, 0, 796, 0, 22, 2124, 971, 62, 1996, 246, 0, 0, 0, 0, 0, 0, 0, 0, 128, 364, 98, 944, 1693, 1539, 257, 34, 117, 0, 0, 0, 0, 0, 0, 0, 207, 137, 13, 74, 59, 1, 927, 9, 5384, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6103, 5898, 6575, 6433, 5941, 6217, 5174, 6811] avg=6144.0000 max=6811 max_rank=7 min=5174 min_rank=6 imbalance_ratio=1.108561 max_load_reduction=42341 max_load_reduction_pct=0.861430 moved_to_echo=24143 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1485, 93, 373, 2048, 1153, 32, 117, 992, 0, 0, 0, 0, 0, 0, 0, 0, 347, 1041, 833, 390, 0, 1785, 28, 1825, 0, 0, 0, 0, 0, 0, 0, 0, 624, 99, 115, 897, 31, 795, 762, 1019, 1700, 248, 2, 0, 0, 0, 0, 0, 14, 821, 340, 1171, 371, 65, 34, 50, 2555, 587, 0, 0, 0, 0, 0, 0, 33, 187, 829, 514, 751, 4, 110, 732, 1543, 1432, 0, 0, 0, 0, 0, 0, 1042, 0, 16, 2626, 1141, 61, 1217, 259, 0, 0, 0, 0, 0, 0, 0, 0, 150, 542, 226, 786, 2023, 686, 341, 26, 92, 0, 0, 0, 0, 0, 0, 0, 309, 85, 2, 25, 71, 3, 989, 28, 5429, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6293, 6249, 6292, 6008, 6135, 6362, 4872, 6941] avg=6144.0000 max=6941 max_rank=7 min=4872 min_rank=6 imbalance_ratio=1.129720 max_load_reduction=42211 max_load_reduction_pct=0.858785 moved_to_echo=24310 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1294, 66, 347, 2316, 729, 4, 161, 906, 0, 0, 0, 0, 0, 0, 0, 0, 371, 1171, 802, 444, 0, 1951, 22, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 709, 95, 26, 1023, 27, 623, 911, 1043, 1790, 281, 2, 0, 0, 0, 0, 0, 26, 734, 337, 1176, 396, 26, 49, 44, 2693, 516, 0, 0, 0, 0, 0, 0, 26, 276, 1272, 330, 777, 6, 80, 388, 1745, 1509, 0, 0, 0, 0, 0, 0, 807, 0, 37, 2046, 1202, 54, 1695, 210, 0, 0, 0, 0, 0, 0, 0, 0, 160, 578, 131, 703, 2427, 646, 858, 20, 101, 0, 0, 0, 0, 0, 0, 0, 237, 32, 32, 61, 58, 2, 908, 18, 4778, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5823, 6592, 6530, 5997, 6409, 6051, 5624, 6126] avg=6144.0000 max=6592 max_rank=1 min=5624 min_rank=6 imbalance_ratio=1.072917 max_load_reduction=42560 max_load_reduction_pct=0.865885 moved_to_echo=24210 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1728, 37, 96, 2548, 864, 5, 198, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 533, 831, 765, 414, 0, 2543, 4, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 713, 21, 40, 970, 31, 609, 904, 645, 1893, 309, 3, 0, 0, 0, 0, 0, 22, 747, 168, 1043, 562, 24, 36, 10, 3245, 552, 0, 0, 0, 0, 0, 0, 11, 231, 872, 181, 373, 0, 31, 372, 1919, 1821, 0, 0, 0, 0, 0, 0, 913, 0, 8, 1996, 1272, 14, 1531, 236, 0, 0, 0, 0, 0, 0, 0, 0, 97, 639, 181, 804, 2500, 715, 315, 3, 132, 0, 0, 0, 0, 0, 0, 0, 228, 41, 29, 31, 34, 2, 847, 17, 5116, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6525, 6568, 6138, 6409, 5811, 5970, 5386, 6345] avg=6144.0000 max=6568 max_rank=1 min=5386 min_rank=6 imbalance_ratio=1.069010 max_load_reduction=42584 max_load_reduction_pct=0.866374 moved_to_echo=23512 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1454, 59, 262, 2346, 1143, 5, 198, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 508, 715, 847, 562, 0, 1714, 67, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 780, 47, 76, 1122, 26, 874, 831, 820, 1498, 285, 2, 0, 0, 0, 0, 0, 39, 897, 341, 944, 279, 25, 49, 24, 2627, 497, 0, 0, 0, 0, 0, 0, 51, 319, 876, 269, 780, 2, 160, 307, 1767, 1474, 0, 0, 0, 0, 0, 0, 742, 0, 20, 2590, 1006, 86, 1927, 253, 0, 0, 0, 0, 0, 0, 0, 0, 261, 621, 135, 818, 2083, 1027, 838, 49, 89, 0, 0, 0, 0, 0, 0, 0, 257, 86, 8, 38, 76, 7, 881, 11, 4607, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6529, 6019, 6361, 5722, 6005, 6624, 5921, 5971] avg=6144.0000 max=6624 max_rank=5 min=5722 min_rank=3 imbalance_ratio=1.078125 max_load_reduction=42528 max_load_reduction_pct=0.865234 moved_to_echo=24521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1501, 89, 197, 2322, 813, 33, 120, 890, 0, 0, 0, 0, 0, 0, 0, 0, 344, 719, 864, 635, 0, 2631, 48, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 727, 40, 26, 1479, 47, 523, 625, 708, 1350, 287, 7, 0, 0, 0, 0, 0, 42, 846, 448, 1378, 517, 65, 65, 46, 3000, 465, 0, 0, 0, 0, 0, 0, 18, 335, 937, 91, 674, 2, 66, 295, 1761, 1681, 0, 0, 0, 0, 0, 0, 263, 0, 16, 2310, 900, 65, 1664, 254, 0, 0, 0, 0, 0, 0, 0, 0, 116, 835, 58, 579, 1730, 2403, 953, 23, 141, 0, 0, 0, 0, 0, 0, 0, 201, 168, 7, 90, 52, 2, 417, 8, 4306, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 7075, 5819, 6872, 5860, 5472, 6838, 5251] avg=6144.0000 max=7075 max_rank=1 min=5251 min_rank=7 imbalance_ratio=1.151530 max_load_reduction=42077 max_load_reduction_pct=0.856059 moved_to_echo=23421 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1811, 33, 319, 2010, 665, 2, 209, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 452, 716, 830, 253, 0, 1695, 19, 786, 0, 0, 0, 0, 0, 0, 0, 0, 635, 24, 20, 1069, 30, 565, 882, 1172, 1446, 244, 2, 0, 0, 0, 0, 0, 11, 850, 263, 928, 337, 49, 54, 50, 3186, 538, 0, 0, 0, 0, 0, 0, 27, 450, 1521, 187, 661, 0, 33, 499, 1514, 1787, 0, 0, 0, 0, 0, 0, 762, 0, 11, 2051, 971, 41, 1863, 311, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1741, 179, 711, 2179, 1194, 718, 35, 88, 0, 0, 0, 0, 0, 0, 0, 228, 74, 7, 47, 18, 4, 664, 14, 4990, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6287, 4751, 6089, 6266, 6679, 6010, 7024, 6046] avg=6144.0000 max=7024 max_rank=6 min=4751 min_rank=1 imbalance_ratio=1.143229 max_load_reduction=42128 max_load_reduction_pct=0.857096 moved_to_echo=25759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1485, 93, 373, 2048, 1153, 32, 117, 992, 0, 0, 0, 0, 0, 0, 0, 0, 347, 1041, 833, 390, 0, 1785, 28, 1825, 0, 0, 0, 0, 0, 0, 0, 0, 624, 99, 115, 897, 31, 795, 762, 1019, 1700, 248, 2, 0, 0, 0, 0, 0, 14, 821, 340, 1171, 371, 65, 34, 50, 2555, 587, 0, 0, 0, 0, 0, 0, 33, 187, 829, 514, 751, 4, 110, 732, 1543, 1432, 0, 0, 0, 0, 0, 0, 1042, 0, 16, 2626, 1141, 61, 1217, 259, 0, 0, 0, 0, 0, 0, 0, 0, 150, 542, 226, 786, 2023, 686, 341, 26, 92, 0, 0, 0, 0, 0, 0, 0, 309, 85, 2, 25, 71, 3, 989, 28, 5429, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6293, 6249, 6292, 6008, 6135, 6362, 4872, 6941] avg=6144.0000 max=6941 max_rank=7 min=4872 min_rank=6 imbalance_ratio=1.129720 max_load_reduction=42211 max_load_reduction_pct=0.858785 moved_to_echo=24310 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1178, 54, 240, 2458, 307, 2, 124, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 560, 899, 987, 352, 0, 1805, 31, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 663, 16, 24, 964, 35, 418, 938, 556, 1434, 298, 2, 0, 0, 0, 0, 0, 16, 776, 126, 713, 260, 135, 23, 98, 2784, 514, 0, 0, 0, 0, 0, 0, 19, 273, 937, 391, 723, 1, 140, 415, 1852, 1561, 0, 0, 0, 0, 0, 0, 575, 0, 34, 2284, 964, 42, 2262, 285, 0, 0, 0, 0, 0, 0, 0, 0, 233, 1780, 153, 913, 2415, 1742, 914, 70, 93, 0, 0, 0, 0, 0, 0, 0, 255, 59, 14, 38, 28, 3, 485, 23, 4756, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5627, 6000, 5348, 5445, 6312, 6446, 8313, 5661] avg=6144.0000 max=8313 max_rank=6 min=5348 min_rank=2 imbalance_ratio=1.353027 max_load_reduction=40839 max_load_reduction_pct=0.830872 moved_to_echo=26732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 6656, 2432, 8320, 3072, 512, 384, 384, 23424, 4352, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 7168, 1280, 6272, 17152, 9984, 5248, 384, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3584, 6784, 6784, 3456, 0, 16384, 256, 12288, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5632, 384, 384, 9088, 384, 5504, 6528, 6912, 12672, 2304, 128, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[11904, 512, 2048, 18304, 6400, 128, 1280, 8832, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 2432, 8064, 2176, 5376, 128, 768, 3456, 13824, 13184, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 7168, 1280, 6272, 17152, 9984, 5248, 384, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 6656, 2432, 8320, 3072, 512, 384, 384, 23424, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5632, 384, 384, 9088, 384, 5504, 6528, 6912, 12672, 2304, 128, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 0, 256, 18048, 8448, 512, 14208, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 2432, 8064, 2176, 5376, 128, 768, 3456, 13824, 13184, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 768, 128, 512, 512, 128, 6144, 128, 39424, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[2332, 685, 21, 80, 1714, 25, 625, 231, 15, 159, 5242, 15, 45, 1410, 737, 1202, 2696, 90, 76, 54, 390, 382, 8, 163, 312, 1351, 329, 445, 214, 7308, 2249, 5, 298, 56, 844, 391, 1226, 274, 288, 21, 1244, 851, 5, 53, 115, 516, 601, 15, 3110, 280, 3, 22, 131, 296, 2030, 1163, 102, 865, 11, 1106, 26, 502, 1623, 474] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1811, 578, 20, 50, 2091, 38, 486, 155, 43, 335, 4754, 39, 121, 1464, 563, 1011, 2103, 41, 108, 96, 133, 646, 9, 641, 231, 1696, 672, 790, 298, 6771, 2452, 3, 357, 60, 1378, 320, 1760, 355, 245, 3, 1394, 510, 3, 62, 132, 564, 145, 14, 2509, 281, 19, 152, 345, 331, 3079, 814, 111, 769, 17, 628, 14, 767, 1150, 615] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1151, 715, 70, 31, 2298, 16, 278, 320, 16, 153, 5130, 48, 154, 1586, 440, 1311, 2930, 77, 60, 86, 291, 282, 8, 407, 287, 1017, 406, 765, 375, 7273, 2117, 20, 258, 50, 1671, 456, 1698, 255, 83, 9, 1398, 652, 1, 31, 110, 741, 127, 12, 2506, 343, 6, 253, 453, 338, 2833, 1214, 157, 518, 7, 613, 2, 608, 1249, 382] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1294, 675, 59, 41, 1248, 32, 421, 145, 40, 394, 4649, 28, 25, 1866, 499, 944, 2181, 117, 122, 100, 422, 417, 10, 271, 311, 1490, 278, 683, 457, 7263, 2596, 87, 320, 265, 1090, 350, 1539, 496, 290, 19, 2147, 724, 8, 28, 190, 353, 92, 32, 3059, 818, 26, 203, 254, 405, 2256, 964, 177, 809, 8, 500, 7, 484, 1426, 648] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1500, 461, 19, 74, 2027, 24, 233, 561, 11, 162, 5472, 17, 72, 1396, 469, 1574, 2694, 50, 83, 44, 292, 507, 17, 287, 330, 1072, 610, 856, 210, 6853, 1785, 8, 431, 49, 1431, 553, 1552, 261, 78, 11, 1336, 770, 0, 48, 167, 620, 258, 7, 2500, 338, 13, 59, 285, 351, 2578, 1232, 96, 850, 35, 968, 5, 533, 1317, 650] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1699, 364, 29, 58, 1159, 28, 687, 152, 16, 264, 5338, 20, 122, 1313, 659, 1404, 3381, 65, 71, 103, 201, 655, 11, 264, 161, 1488, 315, 811, 283, 7102, 2033, 15, 293, 48, 1275, 373, 1258, 428, 211, 11, 1166, 1571, 2, 66, 53, 597, 89, 9, 2437, 509, 39, 22, 433, 279, 2547, 1116, 124, 903, 21, 618, 5, 609, 1334, 435] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1282, 369, 17, 67, 2395, 39, 397, 605, 60, 277, 4833, 40, 164, 1572, 675, 1122, 2234, 66, 155, 49, 280, 819, 11, 299, 127, 1302, 436, 478, 508, 7130, 1573, 19, 580, 28, 869, 394, 1225, 275, 162, 26, 1534, 956, 0, 76, 162, 631, 284, 24, 2766, 282, 32, 159, 723, 260, 2141, 1607, 261, 816, 35, 579, 13, 590, 1517, 745] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1711, 700, 61, 66, 1421, 46, 215, 294, 14, 234, 5761, 29, 38, 1490, 482, 1635, 1975, 49, 164, 81, 238, 582, 13, 656, 207, 1441, 654, 604, 293, 7191, 1944, 15, 405, 49, 1142, 411, 1923, 318, 97, 10, 1130, 882, 3, 122, 105, 789, 130, 18, 2779, 261, 8, 31, 145, 439, 2159, 1341, 75, 756, 15, 1076, 9, 458, 1143, 619] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1699, 364, 29, 58, 1159, 28, 687, 152, 16, 264, 5338, 20, 122, 1313, 659, 1404, 3381, 65, 71, 103, 201, 655, 11, 264, 161, 1488, 315, 811, 283, 7102, 2033, 15, 293, 48, 1275, 373, 1258, 428, 211, 11, 1166, 1571, 2, 66, 53, 597, 89, 9, 2437, 509, 39, 22, 433, 279, 2547, 1116, 124, 903, 21, 618, 5, 609, 1334, 435] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1294, 675, 59, 41, 1248, 32, 421, 145, 40, 394, 4649, 28, 25, 1866, 499, 944, 2181, 117, 122, 100, 422, 417, 10, 271, 311, 1490, 278, 683, 457, 7263, 2596, 87, 320, 265, 1090, 350, 1539, 496, 290, 19, 2147, 724, 8, 28, 190, 353, 92, 32, 3059, 818, 26, 203, 254, 405, 2256, 964, 177, 809, 8, 500, 7, 484, 1426, 648] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1811, 578, 20, 50, 2091, 38, 486, 155, 43, 335, 4754, 39, 121, 1464, 563, 1011, 2103, 41, 108, 96, 133, 646, 9, 641, 231, 1696, 672, 790, 298, 6771, 2452, 3, 357, 60, 1378, 320, 1760, 355, 245, 3, 1394, 510, 3, 62, 132, 564, 145, 14, 2509, 281, 19, 152, 345, 331, 3079, 814, 111, 769, 17, 628, 14, 767, 1150, 615] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1500, 461, 19, 74, 2027, 24, 233, 561, 11, 162, 5472, 17, 72, 1396, 469, 1574, 2694, 50, 83, 44, 292, 507, 17, 287, 330, 1072, 610, 856, 210, 6853, 1785, 8, 431, 49, 1431, 553, 1552, 261, 78, 11, 1336, 770, 0, 48, 167, 620, 258, 7, 2500, 338, 13, 59, 285, 351, 2578, 1232, 96, 850, 35, 968, 5, 533, 1317, 650] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1151, 715, 70, 31, 2298, 16, 278, 320, 16, 153, 5130, 48, 154, 1586, 440, 1311, 2930, 77, 60, 86, 291, 282, 8, 407, 287, 1017, 406, 765, 375, 7273, 2117, 20, 258, 50, 1671, 456, 1698, 255, 83, 9, 1398, 652, 1, 31, 110, 741, 127, 12, 2506, 343, 6, 253, 453, 338, 2833, 1214, 157, 518, 7, 613, 2, 608, 1249, 382] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2332, 685, 21, 80, 1714, 25, 625, 231, 1529, 0, 0, 0, 0, 0, 0, 0, 15, 159, 2440, 15, 45, 1410, 737, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 2696, 90, 76, 54, 390, 382, 8, 163, 2227, 0, 0, 0, 0, 0, 0, 0, 312, 1351, 329, 445, 214, 982, 2249, 5, 0, 0, 0, 0, 0, 0, 0, 0, 298, 56, 844, 391, 1226, 274, 288, 21, 1186, 892, 0, 0, 0, 0, 0, 0, 1244, 851, 5, 53, 115, 516, 601, 15, 2913, 0, 0, 0, 0, 0, 0, 0, 1501, 280, 3, 22, 131, 296, 2030, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 102, 865, 11, 1106, 26, 502, 1623, 474, 1910, 80, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7242, 6023, 6086, 5887, 5476, 6313, 5426, 6699] avg=6144.0000 max=7242 max_rank=0 min=5426 min_rank=6 imbalance_ratio=1.178711 max_load_reduction=41910 max_load_reduction_pct=0.852661 moved_to_echo=23914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1711, 700, 61, 66, 1421, 46, 215, 294, 1366, 0, 0, 0, 0, 0, 0, 0, 14, 234, 2681, 29, 38, 1490, 482, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 49, 164, 81, 238, 582, 13, 656, 2191, 0, 0, 0, 0, 0, 0, 0, 207, 1441, 654, 604, 293, 967, 1944, 15, 0, 0, 0, 0, 0, 0, 0, 0, 405, 49, 1142, 411, 1923, 318, 97, 10, 1167, 981, 0, 0, 0, 0, 0, 0, 1130, 882, 3, 122, 105, 789, 130, 18, 2866, 0, 0, 0, 0, 0, 0, 0, 1342, 261, 8, 31, 145, 439, 2159, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 75, 756, 15, 1076, 9, 458, 1143, 619, 2099, 71, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5880, 6603, 5949, 6125, 6503, 6045, 5726, 6321] avg=6144.0000 max=6603 max_rank=1 min=5726 min_rank=6 imbalance_ratio=1.074707 max_load_reduction=42549 max_load_reduction_pct=0.865662 moved_to_echo=24595 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1811, 578, 20, 50, 2091, 38, 486, 155, 1234, 0, 0, 0, 0, 0, 0, 0, 43, 335, 2213, 39, 121, 1464, 563, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 2103, 41, 108, 96, 133, 646, 9, 641, 2063, 0, 0, 0, 0, 0, 0, 0, 231, 1696, 672, 790, 298, 911, 2452, 3, 0, 0, 0, 0, 0, 0, 0, 0, 357, 60, 1378, 320, 1760, 355, 245, 3, 1099, 809, 0, 0, 0, 0, 0, 0, 1394, 510, 3, 62, 132, 564, 145, 14, 2698, 0, 0, 0, 0, 0, 0, 0, 1211, 281, 19, 152, 345, 331, 3079, 814, 0, 0, 0, 0, 0, 0, 0, 0, 111, 769, 17, 628, 14, 767, 1150, 615, 1732, 64, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6463, 5789, 5840, 7053, 6386, 5522, 6232, 5867] avg=6144.0000 max=7053 max_rank=3 min=5522 min_rank=5 imbalance_ratio=1.147949 max_load_reduction=42099 max_load_reduction_pct=0.856506 moved_to_echo=24007 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1500, 461, 19, 74, 2027, 24, 233, 561, 1229, 0, 0, 0, 0, 0, 0, 0, 11, 162, 2547, 17, 72, 1396, 469, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 50, 83, 44, 292, 507, 17, 287, 2088, 0, 0, 0, 0, 0, 0, 0, 330, 1072, 610, 856, 210, 922, 1785, 8, 0, 0, 0, 0, 0, 0, 0, 0, 431, 49, 1431, 553, 1552, 261, 78, 11, 1112, 931, 0, 0, 0, 0, 0, 0, 1336, 770, 0, 48, 167, 620, 258, 7, 2731, 0, 0, 0, 0, 0, 0, 0, 1207, 338, 13, 59, 285, 351, 2578, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 96, 850, 35, 968, 5, 533, 1317, 650, 1994, 64, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6248, 6062, 5793, 6409, 5937, 6063, 6512] avg=6144.0000 max=6512 max_rank=7 min=5793 min_rank=3 imbalance_ratio=1.059896 max_load_reduction=42640 max_load_reduction_pct=0.867513 moved_to_echo=24921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1151, 715, 70, 31, 2298, 16, 278, 320, 1232, 0, 0, 0, 0, 0, 0, 0, 16, 153, 2388, 48, 154, 1586, 440, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 2930, 77, 60, 86, 291, 282, 8, 407, 2216, 0, 0, 0, 0, 0, 0, 0, 287, 1017, 406, 765, 375, 978, 2117, 20, 0, 0, 0, 0, 0, 0, 0, 0, 258, 50, 1671, 456, 1698, 255, 83, 9, 1180, 873, 0, 0, 0, 0, 0, 0, 1398, 652, 1, 31, 110, 741, 127, 12, 2899, 0, 0, 0, 0, 0, 0, 0, 1210, 343, 6, 253, 453, 338, 2833, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 157, 518, 7, 613, 2, 608, 1249, 382, 1869, 64, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 6096, 6357, 5965, 6533, 5971, 6650, 5469] avg=6144.0000 max=6650 max_rank=6 min=5469 min_rank=7 imbalance_ratio=1.082357 max_load_reduction=42502 max_load_reduction_pct=0.864705 moved_to_echo=24623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1711, 700, 61, 66, 1421, 46, 215, 294, 1366, 0, 0, 0, 0, 0, 0, 0, 14, 234, 2681, 29, 38, 1490, 482, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 49, 164, 81, 238, 582, 13, 656, 2191, 0, 0, 0, 0, 0, 0, 0, 207, 1441, 654, 604, 293, 967, 1944, 15, 0, 0, 0, 0, 0, 0, 0, 0, 405, 49, 1142, 411, 1923, 318, 97, 10, 1167, 981, 0, 0, 0, 0, 0, 0, 1130, 882, 3, 122, 105, 789, 130, 18, 2866, 0, 0, 0, 0, 0, 0, 0, 1342, 261, 8, 31, 145, 439, 2159, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 75, 756, 15, 1076, 9, 458, 1143, 619, 2099, 71, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5880, 6603, 5949, 6125, 6503, 6045, 5726, 6321] avg=6144.0000 max=6603 max_rank=1 min=5726 min_rank=6 imbalance_ratio=1.074707 max_load_reduction=42549 max_load_reduction_pct=0.865662 moved_to_echo=24595 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1282, 369, 17, 67, 2395, 39, 397, 605, 1360, 0, 0, 0, 0, 0, 0, 0, 60, 277, 2249, 40, 164, 1572, 675, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 66, 155, 49, 280, 819, 11, 299, 2173, 0, 0, 0, 0, 0, 0, 0, 127, 1302, 436, 478, 508, 958, 1573, 19, 0, 0, 0, 0, 0, 0, 0, 0, 580, 28, 869, 394, 1225, 275, 162, 26, 1157, 823, 0, 0, 0, 0, 0, 0, 1534, 956, 0, 76, 162, 631, 284, 24, 2842, 0, 0, 0, 0, 0, 0, 0, 1335, 282, 32, 159, 723, 260, 2141, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 261, 816, 35, 579, 13, 590, 1517, 745, 1761, 71, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 6159, 6086, 5401, 5539, 6509, 6539, 6388] avg=6144.0000 max=6539 max_rank=6 min=5401 min_rank=3 imbalance_ratio=1.064290 max_load_reduction=42613 max_load_reduction_pct=0.866964 moved_to_echo=24975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1500, 461, 19, 74, 2027, 24, 233, 561, 1229, 0, 0, 0, 0, 0, 0, 0, 11, 162, 2547, 17, 72, 1396, 469, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 50, 83, 44, 292, 507, 17, 287, 2088, 0, 0, 0, 0, 0, 0, 0, 330, 1072, 610, 856, 210, 922, 1785, 8, 0, 0, 0, 0, 0, 0, 0, 0, 431, 49, 1431, 553, 1552, 261, 78, 11, 1112, 931, 0, 0, 0, 0, 0, 0, 1336, 770, 0, 48, 167, 620, 258, 7, 2731, 0, 0, 0, 0, 0, 0, 0, 1207, 338, 13, 59, 285, 351, 2578, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 96, 850, 35, 968, 5, 533, 1317, 650, 1994, 64, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6248, 6062, 5793, 6409, 5937, 6063, 6512] avg=6144.0000 max=6512 max_rank=7 min=5793 min_rank=3 imbalance_ratio=1.059896 max_load_reduction=42640 max_load_reduction_pct=0.867513 moved_to_echo=24921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1699, 364, 29, 58, 1159, 28, 687, 152, 1202, 0, 0, 0, 0, 0, 0, 0, 16, 264, 2476, 20, 122, 1313, 659, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 3381, 65, 71, 103, 201, 655, 11, 264, 2168, 0, 0, 0, 0, 0, 0, 0, 161, 1488, 315, 811, 283, 944, 2033, 15, 0, 0, 0, 0, 0, 0, 0, 0, 293, 48, 1275, 373, 1258, 428, 211, 11, 1157, 913, 0, 0, 0, 0, 0, 0, 1166, 1571, 2, 66, 53, 597, 89, 9, 2833, 0, 0, 0, 0, 0, 0, 0, 1168, 509, 39, 22, 433, 279, 2547, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 124, 903, 21, 618, 5, 609, 1334, 435, 1949, 67, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5378, 6274, 6919, 6050, 5967, 6386, 6113, 6065] avg=6144.0000 max=6919 max_rank=2 min=5378 min_rank=0 imbalance_ratio=1.126139 max_load_reduction=42233 max_load_reduction_pct=0.859233 moved_to_echo=24531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1294, 675, 59, 41, 1248, 32, 421, 145, 1504, 0, 0, 0, 0, 0, 0, 0, 40, 394, 2164, 28, 25, 1866, 499, 944, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 117, 122, 100, 422, 417, 10, 271, 2213, 0, 0, 0, 0, 0, 0, 0, 311, 1490, 278, 683, 457, 976, 2596, 87, 0, 0, 0, 0, 0, 0, 0, 0, 320, 265, 1090, 350, 1539, 496, 290, 19, 1179, 791, 0, 0, 0, 0, 0, 0, 2147, 724, 8, 28, 190, 353, 92, 32, 2895, 0, 0, 0, 0, 0, 0, 0, 1477, 818, 26, 203, 254, 405, 2256, 964, 0, 0, 0, 0, 0, 0, 0, 0, 177, 809, 8, 500, 7, 484, 1426, 648, 1694, 78, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5419, 5960, 5853, 6878, 6339, 6469, 6403, 5831] avg=6144.0000 max=6878 max_rank=3 min=5419 min_rank=0 imbalance_ratio=1.119466 max_load_reduction=42274 max_load_reduction_pct=0.860067 moved_to_echo=25042 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1699, 364, 29, 58, 1159, 28, 687, 152, 1202, 0, 0, 0, 0, 0, 0, 0, 16, 264, 2476, 20, 122, 1313, 659, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 3381, 65, 71, 103, 201, 655, 11, 264, 2168, 0, 0, 0, 0, 0, 0, 0, 161, 1488, 315, 811, 283, 944, 2033, 15, 0, 0, 0, 0, 0, 0, 0, 0, 293, 48, 1275, 373, 1258, 428, 211, 11, 1157, 913, 0, 0, 0, 0, 0, 0, 1166, 1571, 2, 66, 53, 597, 89, 9, 2833, 0, 0, 0, 0, 0, 0, 0, 1168, 509, 39, 22, 433, 279, 2547, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 124, 903, 21, 618, 5, 609, 1334, 435, 1949, 67, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5378, 6274, 6919, 6050, 5967, 6386, 6113, 6065] avg=6144.0000 max=6919 max_rank=2 min=5378 min_rank=0 imbalance_ratio=1.126139 max_load_reduction=42233 max_load_reduction_pct=0.859233 moved_to_echo=24531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10496, 3200, 256, 1024, 2816, 2816, 19712, 9472, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12800, 4608, 384, 512, 14464, 256, 3456, 2560, 10752, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1152, 6400, 256, 6144, 128, 4608, 10880, 4608, 15104, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[20224, 640, 896, 640, 2304, 4352, 128, 3072, 17408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10496, 3200, 256, 1024, 2816, 2816, 19712, 9472, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 2048, 19200, 256, 768, 12160, 4608, 10240, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 640, 9728, 3328, 12288, 2688, 1536, 128, 9344, 7040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[20224, 640, 896, 640, 2304, 4352, 128, 3072, 17408, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2048, 10880, 3712, 5504, 2688, 7680, 16768, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[11392, 7040, 128, 512, 1152, 4864, 1792, 256, 22784, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 2048, 19200, 256, 768, 12160, 4608, 10240, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4480, 430, 109, 800, 53, 339, 1400, 1888, 60, 0, 0, 2148, 1, 1228, 3, 549, 47, 16, 815, 17, 592, 17, 71, 145, 586, 69, 2463, 5, 0, 3427, 966, 35, 207, 2044, 82, 1624, 113, 447, 696, 1, 28, 4055, 35, 18, 3836, 146, 11, 1939, 183, 8, 31, 266, 285, 707, 124, 1, 135, 1554, 255, 321, 4, 523, 5865, 849] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4455, 279, 158, 691, 4, 512, 603, 1707, 61, 0, 0, 1787, 1, 2099, 0, 736, 32, 2, 1712, 7, 233, 36, 54, 96, 1059, 58, 2208, 7, 1, 4045, 757, 16, 88, 1065, 34, 2172, 26, 359, 725, 0, 51, 3425, 39, 8, 4073, 238, 4, 1846, 165, 15, 20, 213, 187, 708, 9, 1, 42, 1995, 309, 509, 1, 988, 5755, 666] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4217, 342, 326, 639, 21, 244, 903, 1021, 51, 0, 0, 1630, 1, 1796, 1, 778, 56, 1, 1436, 25, 466, 40, 12, 223, 712, 141, 2707, 1, 8, 3648, 1246, 63, 132, 1164, 54, 1798, 192, 794, 485, 1, 16, 4397, 30, 7, 4116, 145, 8, 2128, 209, 12, 9, 241, 346, 1143, 30, 3, 116, 1676, 204, 666, 1, 510, 4876, 888] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4288, 412, 545, 570, 34, 318, 762, 1821, 42, 0, 0, 1174, 2, 635, 1, 616, 16, 4, 1170, 36, 449, 53, 44, 132, 975, 71, 2749, 0, 4, 4422, 1056, 20, 98, 1297, 30, 1685, 260, 1111, 585, 0, 44, 3463, 83, 18, 3889, 176, 17, 1990, 215, 10, 14, 351, 670, 971, 31, 6, 199, 1212, 293, 578, 0, 719, 5809, 907] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4006, 337, 402, 514, 27, 342, 1406, 1033, 98, 0, 0, 1742, 1, 917, 0, 395, 60, 8, 1269, 69, 196, 16, 71, 172, 741, 187, 3014, 6, 9, 3910, 859, 26, 213, 1586, 110, 2193, 296, 658, 695, 0, 36, 3064, 74, 3, 3522, 317, 18, 2639, 410, 51, 13, 446, 490, 939, 22, 14, 111, 2050, 441, 775, 15, 483, 5367, 268] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4433, 274, 663, 735, 10, 453, 665, 724, 22, 0, 0, 1584, 2, 1543, 3, 715, 13, 9, 1413, 11, 434, 72, 96, 106, 764, 180, 2876, 4, 6, 3876, 1012, 33, 218, 1107, 79, 1614, 106, 1128, 819, 0, 42, 3235, 47, 18, 3548, 202, 15, 2231, 234, 38, 10, 444, 414, 711, 14, 5, 218, 1558, 392, 506, 4, 627, 5905, 932] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4455, 279, 158, 691, 4, 512, 603, 1707, 61, 0, 0, 1787, 1, 2099, 0, 736, 32, 2, 1712, 7, 233, 36, 54, 96, 1059, 58, 2208, 7, 1, 4045, 757, 16, 88, 1065, 34, 2172, 26, 359, 725, 0, 51, 3425, 39, 8, 4073, 238, 4, 1846, 165, 15, 20, 213, 187, 708, 9, 1, 42, 1995, 309, 509, 1, 988, 5755, 666] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4546, 259, 177, 442, 27, 417, 782, 1472, 54, 0, 0, 1850, 4, 1000, 1, 326, 60, 21, 872, 10, 405, 51, 83, 175, 611, 178, 2702, 4, 6, 4184, 813, 30, 389, 1578, 137, 1625, 267, 320, 666, 0, 29, 3922, 46, 24, 3994, 235, 6, 2374, 276, 25, 80, 259, 691, 659, 205, 9, 212, 1302, 520, 597, 1, 268, 6122, 752] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4480, 430, 109, 800, 53, 339, 1400, 1888, 60, 0, 0, 2148, 1, 1228, 3, 549, 47, 16, 815, 17, 592, 17, 71, 145, 586, 69, 2463, 5, 0, 3427, 966, 35, 207, 2044, 82, 1624, 113, 447, 696, 1, 28, 4055, 35, 18, 3836, 146, 11, 1939, 183, 8, 31, 266, 285, 707, 124, 1, 135, 1554, 255, 321, 4, 523, 5865, 849] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4006, 337, 402, 514, 27, 342, 1406, 1033, 98, 0, 0, 1742, 1, 917, 0, 395, 60, 8, 1269, 69, 196, 16, 71, 172, 741, 187, 3014, 6, 9, 3910, 859, 26, 213, 1586, 110, 2193, 296, 658, 695, 0, 36, 3064, 74, 3, 3522, 317, 18, 2639, 410, 51, 13, 446, 490, 939, 22, 14, 111, 2050, 441, 775, 15, 483, 5367, 268] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4288, 412, 545, 570, 34, 318, 762, 1821, 42, 0, 0, 1174, 2, 635, 1, 616, 16, 4, 1170, 36, 449, 53, 44, 132, 975, 71, 2749, 0, 4, 4422, 1056, 20, 98, 1297, 30, 1685, 260, 1111, 585, 0, 44, 3463, 83, 18, 3889, 176, 17, 1990, 215, 10, 14, 351, 670, 971, 31, 6, 199, 1212, 293, 578, 0, 719, 5809, 907] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4139, 352, 409, 695, 14, 285, 981, 2404, 70, 5, 3, 931, 41, 869, 7, 495, 10, 12, 2070, 26, 472, 51, 55, 151, 1023, 276, 2088, 1, 2, 4004, 1143, 30, 243, 1175, 229, 2055, 187, 839, 643, 7, 61, 2703, 209, 7, 3531, 237, 7, 1862, 179, 31, 90, 399, 460, 801, 111, 6, 320, 2099, 428, 460, 7, 327, 5388, 937] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2042, 274, 663, 735, 10, 453, 665, 724, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1584, 2, 1543, 3, 715, 1356, 1288, 0, 0, 0, 0, 0, 0, 13, 9, 1413, 11, 434, 72, 96, 106, 3224, 976, 0, 0, 0, 0, 0, 0, 764, 180, 2876, 4, 6, 1544, 1012, 33, 0, 0, 0, 0, 0, 0, 0, 0, 218, 1107, 79, 1614, 106, 1128, 819, 0, 1103, 0, 0, 0, 0, 0, 0, 0, 42, 3235, 47, 18, 131, 202, 15, 2231, 0, 0, 0, 0, 0, 0, 0, 0, 234, 38, 10, 444, 414, 711, 14, 5, 3417, 530, 0, 0, 0, 0, 0, 0, 218, 1558, 392, 506, 4, 627, 2151, 932, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5566, 6513, 6354, 6419, 6174, 5921, 5817, 6388] avg=6144.0000 max=6513 max_rank=1 min=5566 min_rank=0 imbalance_ratio=1.060059 max_load_reduction=42639 max_load_reduction_pct=0.867493 moved_to_echo=24300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1976, 412, 545, 570, 34, 318, 762, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 1174, 2, 635, 1, 616, 1547, 1245, 0, 0, 0, 0, 0, 0, 16, 4, 1170, 36, 449, 53, 44, 132, 3171, 1114, 0, 0, 0, 0, 0, 0, 975, 71, 2749, 0, 4, 1761, 1056, 20, 0, 0, 0, 0, 0, 0, 0, 0, 98, 1297, 30, 1685, 260, 1111, 585, 0, 1067, 0, 0, 0, 0, 0, 0, 0, 44, 3463, 83, 18, 144, 176, 17, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 215, 10, 14, 351, 670, 971, 31, 6, 3745, 521, 0, 0, 0, 0, 0, 0, 199, 1212, 293, 578, 0, 719, 2117, 907, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6438, 5262, 6189, 6636, 6133, 5935, 6534, 6025] avg=6144.0000 max=6636 max_rank=3 min=5262 min_rank=1 imbalance_ratio=1.080078 max_load_reduction=42516 max_load_reduction_pct=0.864990 moved_to_echo=24627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2053, 279, 158, 691, 4, 512, 603, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 1787, 1, 2099, 0, 736, 1415, 1294, 0, 0, 0, 0, 0, 0, 32, 2, 1712, 7, 233, 36, 54, 96, 3142, 1019, 0, 0, 0, 0, 0, 0, 1059, 58, 2208, 7, 1, 1611, 757, 16, 0, 0, 0, 0, 0, 0, 0, 0, 88, 1065, 34, 2172, 26, 359, 725, 0, 1108, 0, 0, 0, 0, 0, 0, 0, 51, 3425, 39, 8, 151, 238, 4, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 165, 15, 20, 213, 187, 708, 9, 1, 3922, 516, 0, 0, 0, 0, 0, 0, 42, 1995, 309, 509, 1, 988, 2097, 666, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6007, 7393, 6333, 5717, 5577, 5762, 5756, 6607] avg=6144.0000 max=7393 max_rank=1 min=5577 min_rank=4 imbalance_ratio=1.203288 max_load_reduction=41759 max_load_reduction_pct=0.849589 moved_to_echo=23702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2065, 430, 109, 800, 53, 339, 1400, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 2148, 1, 1228, 3, 549, 1199, 1301, 0, 0, 0, 0, 0, 0, 47, 16, 815, 17, 592, 17, 71, 145, 3202, 863, 0, 0, 0, 0, 0, 0, 586, 69, 2463, 5, 0, 1365, 966, 35, 0, 0, 0, 0, 0, 0, 0, 0, 207, 2044, 82, 1624, 113, 447, 696, 1, 1114, 0, 0, 0, 0, 0, 0, 0, 28, 4055, 35, 18, 142, 146, 11, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 183, 8, 31, 266, 285, 707, 124, 1, 3694, 526, 0, 0, 0, 0, 0, 0, 135, 1554, 255, 321, 4, 523, 2137, 849, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7084, 6489, 5785, 5489, 6328, 6374, 5825, 5778] avg=6144.0000 max=7084 max_rank=0 min=5489 min_rank=3 imbalance_ratio=1.152995 max_load_reduction=42068 max_load_reduction_pct=0.855876 moved_to_echo=24305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1943, 342, 326, 639, 21, 244, 903, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 1630, 1, 1796, 1, 778, 1276, 1225, 0, 0, 0, 0, 0, 0, 56, 1, 1436, 25, 466, 40, 12, 223, 2662, 919, 0, 0, 0, 0, 0, 0, 712, 141, 2707, 1, 8, 1453, 1246, 63, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1164, 54, 1798, 192, 794, 485, 1, 1049, 0, 0, 0, 0, 0, 0, 0, 16, 4397, 30, 7, 152, 145, 8, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 209, 12, 9, 241, 346, 1143, 30, 3, 3964, 437, 0, 0, 0, 0, 0, 0, 116, 1676, 204, 666, 1, 510, 1777, 888, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5439, 6758, 5840, 6331, 5669, 6883, 6394, 5838] avg=6144.0000 max=6883 max_rank=5 min=5439 min_rank=0 imbalance_ratio=1.120280 max_load_reduction=42269 max_load_reduction_pct=0.859965 moved_to_echo=24784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2065, 430, 109, 800, 53, 339, 1400, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 2148, 1, 1228, 3, 549, 1199, 1301, 0, 0, 0, 0, 0, 0, 47, 16, 815, 17, 592, 17, 71, 145, 3202, 863, 0, 0, 0, 0, 0, 0, 586, 69, 2463, 5, 0, 1365, 966, 35, 0, 0, 0, 0, 0, 0, 0, 0, 207, 2044, 82, 1624, 113, 447, 696, 1, 1114, 0, 0, 0, 0, 0, 0, 0, 28, 4055, 35, 18, 142, 146, 11, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 183, 8, 31, 266, 285, 707, 124, 1, 3694, 526, 0, 0, 0, 0, 0, 0, 135, 1554, 255, 321, 4, 523, 2137, 849, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7084, 6489, 5785, 5489, 6328, 6374, 5825, 5778] avg=6144.0000 max=7084 max_rank=0 min=5489 min_rank=3 imbalance_ratio=1.152995 max_load_reduction=42068 max_load_reduction_pct=0.855876 moved_to_echo=24305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2042, 274, 663, 735, 10, 453, 665, 724, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1584, 2, 1543, 3, 715, 1356, 1288, 0, 0, 0, 0, 0, 0, 13, 9, 1413, 11, 434, 72, 96, 106, 3224, 976, 0, 0, 0, 0, 0, 0, 764, 180, 2876, 4, 6, 1544, 1012, 33, 0, 0, 0, 0, 0, 0, 0, 0, 218, 1107, 79, 1614, 106, 1128, 819, 0, 1103, 0, 0, 0, 0, 0, 0, 0, 42, 3235, 47, 18, 131, 202, 15, 2231, 0, 0, 0, 0, 0, 0, 0, 0, 234, 38, 10, 444, 414, 711, 14, 5, 3417, 530, 0, 0, 0, 0, 0, 0, 218, 1558, 392, 506, 4, 627, 2151, 932, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5566, 6513, 6354, 6419, 6174, 5921, 5817, 6388] avg=6144.0000 max=6513 max_rank=1 min=5566 min_rank=0 imbalance_ratio=1.060059 max_load_reduction=42639 max_load_reduction_pct=0.867493 moved_to_echo=24300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1847, 337, 402, 514, 27, 342, 1406, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 1742, 1, 917, 0, 395, 1368, 1163, 0, 0, 0, 0, 0, 0, 60, 8, 1269, 69, 196, 16, 71, 172, 2930, 985, 0, 0, 0, 0, 0, 0, 741, 187, 3014, 6, 9, 1557, 859, 26, 0, 0, 0, 0, 0, 0, 0, 0, 213, 1586, 110, 2193, 296, 658, 695, 0, 996, 0, 0, 0, 0, 0, 0, 0, 36, 3064, 74, 3, 130, 317, 18, 2639, 0, 0, 0, 0, 0, 0, 0, 0, 410, 51, 13, 446, 490, 939, 22, 14, 3392, 481, 0, 0, 0, 0, 0, 0, 111, 2050, 441, 775, 15, 483, 1956, 268, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5908, 5684, 5776, 6399, 6747, 6281, 6258, 6099] avg=6144.0000 max=6747 max_rank=4 min=5684 min_rank=1 imbalance_ratio=1.098145 max_load_reduction=42405 max_load_reduction_pct=0.862732 moved_to_echo=25385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2086, 259, 177, 442, 27, 417, 782, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 1850, 4, 1000, 1, 326, 1469, 1325, 0, 0, 0, 0, 0, 0, 60, 21, 872, 10, 405, 51, 83, 175, 3346, 1056, 0, 0, 0, 0, 0, 0, 611, 178, 2702, 4, 6, 1659, 813, 30, 0, 0, 0, 0, 0, 0, 0, 0, 389, 1578, 137, 1625, 267, 320, 666, 0, 1135, 0, 0, 0, 0, 0, 0, 0, 29, 3922, 46, 24, 143, 235, 6, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 276, 25, 80, 259, 691, 659, 205, 9, 3851, 553, 0, 0, 0, 0, 0, 0, 212, 1302, 520, 597, 1, 268, 2223, 752, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5662, 6029, 6079, 6003, 6117, 6779, 6608, 5875] avg=6144.0000 max=6779 max_rank=5 min=5662 min_rank=0 imbalance_ratio=1.103353 max_load_reduction=42373 max_load_reduction_pct=0.862081 moved_to_echo=25379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1976, 412, 545, 570, 34, 318, 762, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 1174, 2, 635, 1, 616, 1547, 1245, 0, 0, 0, 0, 0, 0, 16, 4, 1170, 36, 449, 53, 44, 132, 3171, 1114, 0, 0, 0, 0, 0, 0, 975, 71, 2749, 0, 4, 1761, 1056, 20, 0, 0, 0, 0, 0, 0, 0, 0, 98, 1297, 30, 1685, 260, 1111, 585, 0, 1067, 0, 0, 0, 0, 0, 0, 0, 44, 3463, 83, 18, 144, 176, 17, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 215, 10, 14, 351, 670, 971, 31, 6, 3745, 521, 0, 0, 0, 0, 0, 0, 199, 1212, 293, 578, 0, 719, 2117, 907, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6438, 5262, 6189, 6636, 6133, 5935, 6534, 6025] avg=6144.0000 max=6636 max_rank=3 min=5262 min_rank=1 imbalance_ratio=1.080078 max_load_reduction=42516 max_load_reduction_pct=0.864990 moved_to_echo=24627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1908, 352, 409, 695, 14, 285, 981, 2404, 0, 0, 0, 0, 0, 0, 0, 0, 70, 5, 3, 931, 41, 869, 7, 495, 1401, 1202, 0, 0, 0, 0, 0, 0, 10, 12, 2070, 26, 472, 51, 55, 151, 2941, 1008, 0, 0, 0, 0, 0, 0, 1023, 276, 2088, 1, 2, 1595, 1143, 30, 0, 0, 0, 0, 0, 0, 0, 0, 243, 1175, 229, 2055, 187, 839, 643, 7, 1029, 0, 0, 0, 0, 0, 0, 0, 61, 2703, 209, 7, 131, 237, 7, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 179, 31, 90, 399, 460, 801, 111, 6, 3400, 483, 0, 0, 0, 0, 0, 0, 320, 2099, 428, 460, 7, 327, 1964, 937, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7048, 5024, 6796, 6158, 6407, 5217, 5960, 6542] avg=6144.0000 max=7048 max_rank=0 min=5024 min_rank=1 imbalance_ratio=1.147135 max_load_reduction=42104 max_load_reduction_pct=0.856608 moved_to_echo=24126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2086, 259, 177, 442, 27, 417, 782, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 1850, 4, 1000, 1, 326, 1469, 1325, 0, 0, 0, 0, 0, 0, 60, 21, 872, 10, 405, 51, 83, 175, 3346, 1056, 0, 0, 0, 0, 0, 0, 611, 178, 2702, 4, 6, 1659, 813, 30, 0, 0, 0, 0, 0, 0, 0, 0, 389, 1578, 137, 1625, 267, 320, 666, 0, 1135, 0, 0, 0, 0, 0, 0, 0, 29, 3922, 46, 24, 143, 235, 6, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 276, 25, 80, 259, 691, 659, 205, 9, 3851, 553, 0, 0, 0, 0, 0, 0, 212, 1302, 520, 597, 1, 268, 2223, 752, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5662, 6029, 6079, 6003, 6117, 6779, 6608, 5875] avg=6144.0000 max=6779 max_rank=5 min=5662 min_rank=0 imbalance_ratio=1.103353 max_load_reduction=42373 max_load_reduction_pct=0.862081 moved_to_echo=25379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[512, 128, 128, 12928, 128, 10112, 128, 4736, 11136, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 13568, 2944, 4480, 128, 4480, 16512, 6272, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1664, 11136, 768, 14848, 1536, 5760, 5376, 128, 8704, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[16000, 2688, 2816, 5120, 256, 2944, 7552, 12160, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6528, 1280, 20864, 128, 128, 12672, 7936, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 28288, 640, 128, 1152, 1792, 128, 17024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 13568, 2944, 4480, 128, 4480, 16512, 6272, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 128, 10880, 256, 3328, 384, 512, 1280, 24704, 8064, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 28288, 640, 128, 1152, 1792, 128, 17024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6528, 1280, 20864, 128, 128, 12672, 7936, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 256, 384, 2688, 3584, 6656, 640, 128, 29440, 4096, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[817, 189, 243, 1077, 174, 933, 1071, 34, 33, 651, 605, 46, 42, 1080, 53, 218, 169, 26, 3, 900, 8, 12, 907, 1461, 1147, 4276, 834, 341, 1958, 3561, 2368, 275, 25, 2123, 824, 245, 1256, 1071, 146, 90, 158, 1795, 57, 23, 1048, 0, 0, 0, 5681, 477, 41, 2570, 54, 75, 36, 222, 5, 2743, 265, 102, 2390, 1, 2, 115] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[238, 116, 168, 1187, 156, 957, 1091, 4, 4, 377, 698, 76, 72, 672, 33, 242, 31, 13, 4, 661, 8, 1, 277, 1097, 940, 4282, 539, 869, 2480, 4327, 3382, 329, 10, 1794, 793, 285, 1833, 151, 30, 84, 49, 1642, 72, 1, 915, 0, 0, 1, 5845, 523, 22, 2910, 57, 19, 9, 185, 2, 2820, 301, 52, 3343, 0, 0, 73] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[700, 112, 598, 1015, 186, 2170, 659, 41, 100, 507, 583, 54, 156, 1003, 126, 92, 35, 19, 10, 1103, 12, 2, 389, 1147, 1231, 3466, 1710, 181, 2748, 2420, 3202, 168, 37, 1796, 843, 258, 1674, 254, 63, 116, 127, 1829, 98, 11, 759, 0, 0, 0, 5489, 648, 21, 2615, 108, 58, 47, 186, 7, 3139, 324, 283, 2292, 0, 1, 124] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[626, 157, 193, 448, 115, 973, 715, 9, 14, 376, 649, 56, 76, 883, 58, 153, 17, 19, 1, 1059, 19, 5, 580, 1822, 1937, 3638, 1107, 319, 2537, 3562, 2942, 384, 59, 1683, 1018, 265, 2153, 269, 46, 80, 106, 2488, 26, 7, 719, 0, 0, 2, 4726, 540, 20, 2571, 32, 2, 45, 205, 11, 2771, 446, 144, 3126, 1, 0, 142] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[554, 183, 313, 541, 160, 2048, 808, 9, 55, 352, 515, 9, 45, 1344, 63, 267, 41, 19, 4, 1009, 5, 1, 497, 1682, 765, 4068, 667, 559, 2908, 2906, 3210, 506, 78, 1520, 899, 254, 1998, 537, 34, 118, 112, 2011, 97, 3, 602, 7, 0, 0, 4506, 370, 43, 3386, 56, 22, 28, 210, 11, 2994, 502, 320, 2212, 0, 0, 109] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[495, 203, 147, 1688, 370, 1295, 895, 37, 27, 292, 747, 49, 30, 285, 49, 272, 40, 16, 4, 713, 4, 2, 452, 1194, 578, 3851, 834, 741, 2459, 3152, 3377, 252, 28, 1041, 1110, 238, 1438, 1076, 61, 147, 92, 1787, 126, 5, 1294, 5, 0, 5, 5238, 466, 15, 2825, 57, 21, 22, 243, 11, 3250, 56, 672, 3149, 0, 1, 123] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[464, 330, 242, 2236, 588, 1560, 924, 26, 34, 93, 553, 57, 60, 352, 61, 140, 73, 13, 3, 1189, 6, 7, 339, 1632, 1168, 3140, 941, 797, 1673, 3710, 2698, 382, 152, 1392, 550, 240, 2790, 230, 83, 126, 200, 1789, 61, 25, 1400, 0, 0, 0, 5027, 539, 67, 3403, 47, 20, 25, 159, 12, 2631, 215, 46, 2333, 6, 0, 93] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[238, 116, 168, 1187, 156, 957, 1091, 4, 4, 377, 698, 76, 72, 672, 33, 242, 31, 13, 4, 661, 8, 1, 277, 1097, 940, 4282, 539, 869, 2480, 4327, 3382, 329, 10, 1794, 793, 285, 1833, 151, 30, 84, 49, 1642, 72, 1, 915, 0, 0, 1, 5845, 523, 22, 2910, 57, 19, 9, 185, 2, 2820, 301, 52, 3343, 0, 0, 73] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[921, 126, 467, 1236, 242, 1724, 925, 20, 18, 699, 1051, 64, 44, 823, 60, 125, 132, 112, 5, 1499, 67, 4, 579, 1822, 1104, 3476, 1173, 698, 2690, 2945, 2719, 504, 81, 1700, 1048, 244, 995, 186, 55, 99, 149, 1807, 48, 44, 976, 2, 21, 3, 5213, 630, 93, 1700, 71, 29, 50, 258, 5, 2953, 253, 389, 1855, 3, 2, 116] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[626, 157, 193, 448, 115, 973, 715, 9, 14, 376, 649, 56, 76, 883, 58, 153, 17, 19, 1, 1059, 19, 5, 580, 1822, 1937, 3638, 1107, 319, 2537, 3562, 2942, 384, 59, 1683, 1018, 265, 2153, 269, 46, 80, 106, 2488, 26, 7, 719, 0, 0, 2, 4726, 540, 20, 2571, 32, 2, 45, 205, 11, 2771, 446, 144, 3126, 1, 0, 142] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[817, 189, 243, 1077, 174, 933, 1071, 34, 1010, 0, 0, 0, 0, 0, 0, 0, 33, 651, 605, 46, 42, 1080, 53, 218, 4276, 71, 0, 0, 0, 0, 0, 0, 169, 26, 3, 900, 8, 12, 907, 1461, 2958, 201, 93, 0, 0, 0, 0, 0, 1147, 0, 834, 341, 1958, 0, 568, 275, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2123, 824, 245, 1256, 1071, 146, 90, 697, 22, 0, 0, 0, 0, 0, 0, 158, 1795, 57, 23, 1048, 0, 0, 0, 3289, 0, 0, 0, 0, 0, 0, 0, 2723, 477, 41, 2570, 54, 75, 36, 222, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2721, 265, 102, 2390, 1, 2, 115, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5548, 7075, 6738, 5123, 6499, 6370, 6198, 5601] avg=6144.0000 max=7075 max_rank=1 min=5123 min_rank=3 imbalance_ratio=1.151530 max_load_reduction=42077 max_load_reduction_pct=0.856059 moved_to_echo=24668 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[817, 189, 243, 1077, 174, 933, 1071, 34, 33, 651, 605, 46, 42, 1080, 53, 218, 169, 26, 3, 900, 8, 12, 907, 1461, 1147, 4276, 834, 341, 1958, 3561, 2368, 275, 25, 2123, 824, 245, 1256, 1071, 146, 90, 158, 1795, 57, 23, 1048, 0, 0, 0, 5681, 477, 41, 2570, 54, 75, 36, 222, 5, 2743, 265, 102, 2390, 1, 2, 115] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[700, 112, 598, 1015, 186, 2170, 659, 41, 1367, 0, 0, 0, 0, 0, 0, 0, 100, 507, 583, 54, 156, 1003, 126, 92, 3466, 48, 0, 0, 0, 0, 0, 0, 35, 19, 10, 1103, 12, 2, 389, 1147, 2858, 137, 126, 0, 0, 0, 0, 0, 1231, 0, 1710, 181, 2748, 0, 767, 168, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1796, 843, 258, 1674, 254, 63, 116, 942, 26, 0, 0, 0, 0, 0, 0, 127, 1829, 98, 11, 759, 0, 0, 0, 2235, 0, 0, 0, 0, 0, 0, 0, 2631, 648, 21, 2615, 108, 58, 47, 186, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3113, 324, 283, 2292, 0, 1, 124, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6848, 6135, 5838, 6805, 6009, 5059, 6314, 6144] avg=6144.0000 max=6848 max_rank=0 min=5059 min_rank=5 imbalance_ratio=1.114583 max_load_reduction=42304 max_load_reduction_pct=0.860677 moved_to_echo=23526 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[554, 183, 313, 541, 160, 2048, 808, 9, 1370, 0, 0, 0, 0, 0, 0, 0, 55, 352, 515, 9, 45, 1344, 63, 267, 4068, 58, 0, 0, 0, 0, 0, 0, 41, 19, 4, 1009, 5, 1, 497, 1682, 2346, 164, 126, 0, 0, 0, 0, 0, 765, 0, 667, 559, 2908, 0, 770, 506, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1520, 899, 254, 1998, 537, 34, 118, 944, 25, 0, 0, 0, 0, 0, 0, 112, 2011, 97, 3, 602, 7, 0, 0, 2684, 0, 0, 0, 0, 0, 0, 0, 2160, 370, 43, 3386, 56, 22, 28, 210, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2969, 502, 320, 2212, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5986, 6776, 5894, 6175, 6407, 5516, 6275, 6123] avg=6144.0000 max=6776 max_rank=1 min=5516 min_rank=5 imbalance_ratio=1.102865 max_load_reduction=42376 max_load_reduction_pct=0.862142 moved_to_echo=24321 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[626, 157, 193, 448, 115, 973, 715, 9, 1256, 0, 0, 0, 0, 0, 0, 0, 14, 376, 649, 56, 76, 883, 58, 153, 3638, 72, 0, 0, 0, 0, 0, 0, 17, 19, 1, 1059, 19, 5, 580, 1822, 2460, 201, 115, 0, 0, 0, 0, 0, 1937, 0, 1107, 319, 2537, 0, 705, 384, 0, 0, 0, 0, 0, 0, 0, 0, 59, 1683, 1018, 265, 2153, 269, 46, 80, 866, 23, 0, 0, 0, 0, 0, 0, 106, 2488, 26, 7, 719, 0, 0, 2, 3289, 0, 0, 0, 0, 0, 0, 0, 2266, 540, 20, 2571, 32, 2, 45, 205, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2748, 446, 144, 3126, 1, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4492, 5975, 6298, 6989, 6462, 6637, 5681, 6618] avg=6144.0000 max=6989 max_rank=3 min=4492 min_rank=0 imbalance_ratio=1.137533 max_load_reduction=42163 max_load_reduction_pct=0.857808 moved_to_echo=25398 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[238, 116, 168, 1187, 156, 957, 1091, 4, 1443, 0, 0, 0, 0, 0, 0, 0, 4, 377, 698, 76, 72, 672, 33, 242, 4282, 86, 0, 0, 0, 0, 0, 0, 31, 13, 4, 661, 8, 1, 277, 1097, 3043, 244, 133, 0, 0, 0, 0, 0, 940, 0, 539, 869, 2480, 0, 811, 329, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1794, 793, 285, 1833, 151, 30, 84, 995, 23, 0, 0, 0, 0, 0, 0, 49, 1642, 72, 1, 915, 0, 0, 1, 3997, 0, 0, 0, 0, 0, 0, 0, 2802, 523, 22, 2910, 57, 19, 9, 185, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2797, 301, 52, 3343, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5360, 6542, 5512, 5968, 5998, 6677, 6527, 6568] avg=6144.0000 max=6677 max_rank=5 min=5360 min_rank=0 imbalance_ratio=1.086751 max_load_reduction=42475 max_load_reduction_pct=0.864156 moved_to_echo=25770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[817, 189, 243, 1077, 174, 933, 1071, 34, 1010, 0, 0, 0, 0, 0, 0, 0, 33, 651, 605, 46, 42, 1080, 53, 218, 4276, 71, 0, 0, 0, 0, 0, 0, 169, 26, 3, 900, 8, 12, 907, 1461, 2958, 201, 93, 0, 0, 0, 0, 0, 1147, 0, 834, 341, 1958, 0, 568, 275, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2123, 824, 245, 1256, 1071, 146, 90, 697, 22, 0, 0, 0, 0, 0, 0, 158, 1795, 57, 23, 1048, 0, 0, 0, 3289, 0, 0, 0, 0, 0, 0, 0, 2723, 477, 41, 2570, 54, 75, 36, 222, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2721, 265, 102, 2390, 1, 2, 115, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5548, 7075, 6738, 5123, 6499, 6370, 6198, 5601] avg=6144.0000 max=7075 max_rank=1 min=5123 min_rank=3 imbalance_ratio=1.151530 max_load_reduction=42077 max_load_reduction_pct=0.856059 moved_to_echo=24668 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[921, 126, 467, 1236, 242, 1724, 925, 20, 1160, 0, 0, 0, 0, 0, 0, 0, 18, 699, 1051, 64, 44, 823, 60, 125, 3476, 58, 0, 0, 0, 0, 0, 0, 132, 112, 5, 1499, 67, 4, 579, 1822, 2714, 168, 107, 0, 0, 0, 0, 0, 1104, 0, 1173, 698, 2690, 0, 652, 504, 0, 0, 0, 0, 0, 0, 0, 0, 81, 1700, 1048, 244, 995, 186, 55, 99, 800, 24, 0, 0, 0, 0, 0, 0, 149, 1807, 48, 44, 976, 2, 21, 3, 2719, 0, 0, 0, 0, 0, 0, 0, 2499, 630, 93, 1700, 71, 29, 50, 258, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2929, 253, 389, 1855, 3, 2, 116, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6821, 6418, 7209, 6821, 5232, 5769, 5330, 5552] avg=6144.0000 max=7209 max_rank=2 min=5232 min_rank=4 imbalance_ratio=1.173340 max_load_reduction=41943 max_load_reduction_pct=0.853333 moved_to_echo=21883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 203, 147, 1688, 370, 1295, 895, 37, 1441, 0, 0, 0, 0, 0, 0, 0, 27, 292, 747, 49, 30, 285, 49, 272, 3851, 63, 0, 0, 0, 0, 0, 0, 40, 16, 4, 713, 4, 2, 452, 1194, 2727, 179, 132, 0, 0, 0, 0, 0, 578, 0, 834, 741, 2459, 0, 810, 252, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1041, 1110, 238, 1438, 1076, 61, 147, 994, 27, 0, 0, 0, 0, 0, 0, 92, 1787, 126, 5, 1294, 5, 0, 5, 2910, 0, 0, 0, 0, 0, 0, 0, 2511, 466, 15, 2825, 57, 21, 22, 243, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3223, 56, 672, 3149, 0, 1, 123, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6571, 5665, 5463, 5674, 6160, 6224, 6160, 7235] avg=6144.0000 max=7235 max_rank=7 min=5463 min_rank=2 imbalance_ratio=1.177572 max_load_reduction=41917 max_load_reduction_pct=0.852804 moved_to_echo=25779 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[554, 183, 313, 541, 160, 2048, 808, 9, 1370, 0, 0, 0, 0, 0, 0, 0, 55, 352, 515, 9, 45, 1344, 63, 267, 4068, 58, 0, 0, 0, 0, 0, 0, 41, 19, 4, 1009, 5, 1, 497, 1682, 2346, 164, 126, 0, 0, 0, 0, 0, 765, 0, 667, 559, 2908, 0, 770, 506, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1520, 899, 254, 1998, 537, 34, 118, 944, 25, 0, 0, 0, 0, 0, 0, 112, 2011, 97, 3, 602, 7, 0, 0, 2684, 0, 0, 0, 0, 0, 0, 0, 2160, 370, 43, 3386, 56, 22, 28, 210, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2969, 502, 320, 2212, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5986, 6776, 5894, 6175, 6407, 5516, 6275, 6123] avg=6144.0000 max=6776 max_rank=1 min=5516 min_rank=5 imbalance_ratio=1.102865 max_load_reduction=42376 max_load_reduction_pct=0.862142 moved_to_echo=24321 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[464, 330, 242, 2236, 588, 1560, 924, 26, 1156, 0, 0, 0, 0, 0, 0, 0, 34, 93, 553, 57, 60, 352, 61, 140, 3140, 76, 0, 0, 0, 0, 0, 0, 73, 13, 3, 1189, 6, 7, 339, 1632, 2620, 209, 109, 0, 0, 0, 0, 0, 1168, 0, 941, 797, 1673, 0, 636, 382, 0, 0, 0, 0, 0, 0, 0, 0, 152, 1392, 550, 240, 2790, 230, 83, 126, 797, 25, 0, 0, 0, 0, 0, 0, 200, 1789, 61, 25, 1400, 0, 0, 0, 3425, 0, 0, 0, 0, 0, 0, 0, 2407, 539, 67, 3403, 47, 20, 25, 159, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2606, 215, 46, 2333, 6, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7526, 4566, 6200, 5597, 6385, 6900, 6667, 5311] avg=6144.0000 max=7526 max_rank=0 min=4566 min_rank=1 imbalance_ratio=1.224935 max_load_reduction=41626 max_load_reduction_pct=0.846883 moved_to_echo=25263 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[921, 126, 467, 1236, 242, 1724, 925, 20, 1160, 0, 0, 0, 0, 0, 0, 0, 18, 699, 1051, 64, 44, 823, 60, 125, 3476, 58, 0, 0, 0, 0, 0, 0, 132, 112, 5, 1499, 67, 4, 579, 1822, 2714, 168, 107, 0, 0, 0, 0, 0, 1104, 0, 1173, 698, 2690, 0, 652, 504, 0, 0, 0, 0, 0, 0, 0, 0, 81, 1700, 1048, 244, 995, 186, 55, 99, 800, 24, 0, 0, 0, 0, 0, 0, 149, 1807, 48, 44, 976, 2, 21, 3, 2719, 0, 0, 0, 0, 0, 0, 0, 2499, 630, 93, 1700, 71, 29, 50, 258, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2929, 253, 389, 1855, 3, 2, 116, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6821, 6418, 7209, 6821, 5232, 5769, 5330, 5552] avg=6144.0000 max=7209 max_rank=2 min=5232 min_rank=4 imbalance_ratio=1.173340 max_load_reduction=41943 max_load_reduction_pct=0.853333 moved_to_echo=21883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[626, 157, 193, 448, 115, 973, 715, 9, 1256, 0, 0, 0, 0, 0, 0, 0, 14, 376, 649, 56, 76, 883, 58, 153, 3638, 72, 0, 0, 0, 0, 0, 0, 17, 19, 1, 1059, 19, 5, 580, 1822, 2460, 201, 115, 0, 0, 0, 0, 0, 1937, 0, 1107, 319, 2537, 0, 705, 384, 0, 0, 0, 0, 0, 0, 0, 0, 59, 1683, 1018, 265, 2153, 269, 46, 80, 866, 23, 0, 0, 0, 0, 0, 0, 106, 2488, 26, 7, 719, 0, 0, 2, 3289, 0, 0, 0, 0, 0, 0, 0, 2266, 540, 20, 2571, 32, 2, 45, 205, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2748, 446, 144, 3126, 1, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4492, 5975, 6298, 6989, 6462, 6637, 5681, 6618] avg=6144.0000 max=6989 max_rank=3 min=4492 min_rank=0 imbalance_ratio=1.137533 max_load_reduction=42163 max_load_reduction_pct=0.857808 moved_to_echo=25398 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[20096, 4224, 384, 22016, 512, 256, 384, 1792, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8960, 0, 7808, 4608, 19456, 0, 5760, 2816, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 1536, 2432, 9472, 2048, 11776, 7168, 256, 10240, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 23168, 2432, 2048, 20736, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 256, 128, 8192, 256, 128, 4096, 11904, 21760, 1536, 1024, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 15232, 640, 128, 7808, 128, 128, 128, 24576, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 13056, 7168, 2048, 14208, 3840, 640, 896, 7040, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 23168, 2432, 2048, 20736, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 3456, 5504, 512, 640, 6528, 512, 1536, 30208, 640, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[241, 761, 482, 294, 1776, 4474, 32, 21, 120, 1448, 544, 27, 618, 388, 1782, 224, 152, 31, 199, 2408, 0, 400, 168, 13, 403, 219, 132, 318, 5259, 518, 4, 2277, 16, 141, 87, 0, 545, 858, 1, 0, 2044, 1486, 1, 5427, 863, 51, 64, 1586, 1, 3729, 26, 737, 31, 3, 79, 2460, 65, 6, 789, 1992, 248, 10, 58, 15] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[105, 731, 183, 280, 2263, 3926, 17, 1, 199, 2013, 1219, 43, 223, 250, 2059, 116, 39, 85, 146, 3047, 0, 266, 196, 2, 449, 238, 216, 35, 4935, 155, 4, 2791, 13, 115, 17, 52, 475, 2630, 0, 0, 1507, 1163, 0, 5533, 722, 49, 25, 890, 0, 3955, 33, 668, 13, 0, 70, 1630, 102, 8, 728, 2257, 188, 5, 39, 33] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[141, 622, 246, 335, 1737, 4076, 72, 2, 171, 1053, 436, 11, 500, 229, 2283, 85, 118, 136, 119, 3779, 0, 324, 361, 0, 165, 142, 238, 279, 5876, 195, 18, 2213, 26, 149, 57, 18, 225, 938, 1, 0, 1169, 1668, 0, 5561, 1015, 12, 301, 1773, 0, 3599, 43, 493, 16, 0, 170, 2718, 19, 28, 647, 2233, 215, 11, 60, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[59, 786, 285, 289, 1949, 4084, 36, 8, 116, 1823, 177, 21, 454, 355, 2444, 141, 49, 67, 27, 3493, 0, 98, 282, 0, 273, 15, 168, 188, 5634, 420, 27, 2886, 45, 96, 63, 41, 324, 1026, 0, 1, 1464, 1484, 0, 4819, 1219, 8, 24, 1207, 2, 4312, 45, 653, 32, 0, 22, 2013, 94, 13, 1207, 1919, 210, 5, 120, 30] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[123, 1055, 672, 491, 2189, 4495, 47, 9, 321, 1973, 602, 78, 380, 430, 1537, 321, 384, 93, 184, 1977, 16, 161, 235, 0, 314, 173, 106, 289, 3173, 397, 7, 3216, 16, 167, 54, 48, 1056, 2109, 3, 33, 1003, 1210, 0, 4803, 1060, 9, 117, 1361, 0, 4191, 27, 1252, 275, 2, 48, 2202, 119, 30, 589, 1594, 237, 4, 73, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[141, 622, 246, 335, 1737, 4076, 72, 2, 171, 1053, 436, 11, 500, 229, 2283, 85, 118, 136, 119, 3779, 0, 324, 361, 0, 165, 142, 238, 279, 5876, 195, 18, 2213, 26, 149, 57, 18, 225, 938, 1, 0, 1169, 1668, 0, 5561, 1015, 12, 301, 1773, 0, 3599, 43, 493, 16, 0, 170, 2718, 19, 28, 647, 2233, 215, 11, 60, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[87, 1030, 202, 300, 1927, 4691, 11, 3, 216, 2097, 585, 21, 325, 469, 2188, 215, 107, 38, 100, 2183, 0, 204, 649, 1, 250, 125, 106, 104, 5150, 278, 4, 3320, 11, 108, 43, 3, 636, 1090, 0, 0, 1550, 1361, 1, 5114, 786, 53, 94, 1013, 0, 4250, 36, 382, 181, 1, 59, 1798, 38, 78, 734, 2563, 143, 11, 22, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[189, 1046, 340, 282, 1059, 4816, 12, 23, 451, 992, 254, 12, 295, 707, 2613, 150, 31, 96, 126, 1862, 0, 311, 338, 8, 323, 197, 313, 163, 4821, 374, 76, 2655, 43, 174, 100, 62, 237, 988, 0, 4, 1991, 2453, 3, 4901, 1155, 19, 52, 2041, 0, 3824, 34, 677, 24, 1, 49, 1407, 108, 103, 1346, 2208, 108, 30, 37, 38] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[263, 550, 314, 302, 1439, 4324, 102, 4, 304, 2054, 519, 30, 647, 731, 1655, 207, 38, 16, 180, 2337, 0, 250, 403, 0, 317, 136, 130, 113, 5017, 378, 11, 3126, 26, 168, 107, 59, 567, 1290, 0, 0, 1472, 1957, 2, 5359, 1006, 29, 89, 1470, 1, 3961, 33, 525, 27, 1, 114, 1819, 61, 34, 842, 2001, 135, 47, 36, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[123, 1055, 672, 491, 2189, 4495, 47, 9, 321, 1973, 602, 78, 380, 430, 1537, 321, 384, 93, 184, 1977, 16, 161, 235, 0, 314, 173, 106, 289, 3173, 397, 7, 3216, 16, 167, 54, 48, 1056, 2109, 3, 33, 1003, 1210, 0, 4803, 1060, 9, 117, 1361, 0, 4191, 27, 1252, 275, 2, 48, 2202, 119, 30, 589, 1594, 237, 4, 73, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[141, 622, 246, 335, 1737, 2492, 72, 2, 0, 0, 0, 0, 0, 0, 0, 0, 171, 1053, 436, 11, 500, 229, 2283, 85, 475, 0, 0, 0, 0, 0, 0, 0, 118, 136, 119, 3779, 0, 324, 361, 0, 1584, 863, 158, 0, 0, 0, 0, 0, 165, 142, 238, 279, 2531, 195, 18, 2213, 0, 0, 0, 0, 0, 0, 0, 0, 26, 149, 57, 18, 225, 938, 1, 0, 4284, 0, 0, 0, 0, 0, 0, 0, 1169, 1668, 0, 537, 1015, 12, 301, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 43, 493, 16, 0, 170, 2718, 0, 0, 0, 0, 0, 0, 0, 0, 19, 28, 647, 2233, 215, 11, 60, 25, 2482, 740, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5647, 5243, 7442, 5781, 5698, 6475, 6406, 6460] avg=6144.0000 max=7442 max_rank=2 min=5243 min_rank=1 imbalance_ratio=1.211263 max_load_reduction=41710 max_load_reduction_pct=0.848592 moved_to_echo=25039 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[189, 1046, 340, 282, 1059, 4816, 12, 23, 451, 992, 254, 12, 295, 707, 2613, 150, 31, 96, 126, 1862, 0, 311, 338, 8, 323, 197, 313, 163, 4821, 374, 76, 2655, 43, 174, 100, 62, 237, 988, 0, 4, 1991, 2453, 3, 4901, 1155, 19, 52, 2041, 0, 3824, 34, 677, 24, 1, 49, 1407, 108, 103, 1346, 2208, 108, 30, 37, 38] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[105, 731, 183, 280, 2263, 3926, 17, 1, 199, 2013, 1219, 43, 223, 250, 2059, 116, 39, 85, 146, 3047, 0, 266, 196, 2, 449, 238, 216, 35, 4935, 155, 4, 2791, 13, 115, 17, 52, 475, 2630, 0, 0, 1507, 1163, 0, 5533, 722, 49, 25, 890, 0, 3955, 33, 668, 13, 0, 70, 1630, 102, 8, 728, 2257, 188, 5, 39, 33] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[241, 761, 482, 294, 1776, 2736, 32, 21, 0, 0, 0, 0, 0, 0, 0, 0, 120, 1448, 544, 27, 618, 388, 1782, 224, 492, 0, 0, 0, 0, 0, 0, 0, 152, 31, 199, 2408, 0, 400, 168, 13, 1738, 772, 164, 0, 0, 0, 0, 0, 403, 219, 132, 318, 2266, 518, 4, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 16, 141, 87, 0, 545, 858, 1, 0, 4181, 0, 0, 0, 0, 0, 0, 0, 2044, 1486, 1, 524, 863, 51, 64, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3073, 26, 737, 31, 3, 79, 2460, 0, 0, 0, 0, 0, 0, 0, 0, 65, 6, 789, 1992, 248, 10, 58, 15, 2221, 722, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6343, 5643, 6045, 6137, 5829, 6619, 6410, 6126] avg=6144.0000 max=6619 max_rank=5 min=5643 min_rank=1 imbalance_ratio=1.077311 max_load_reduction=42533 max_load_reduction_pct=0.865336 moved_to_echo=24984 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[59, 786, 285, 289, 1949, 2497, 36, 8, 0, 0, 0, 0, 0, 0, 0, 0, 116, 1823, 177, 21, 454, 355, 2444, 141, 569, 0, 0, 0, 0, 0, 0, 0, 49, 67, 27, 3493, 0, 98, 282, 0, 1587, 828, 189, 0, 0, 0, 0, 0, 273, 15, 168, 188, 2426, 420, 27, 2886, 0, 0, 0, 0, 0, 0, 0, 0, 45, 96, 63, 41, 324, 1026, 0, 1, 3713, 0, 0, 0, 0, 0, 0, 0, 1464, 1484, 0, 465, 1219, 8, 24, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3554, 45, 653, 32, 0, 22, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 94, 13, 1207, 1919, 210, 5, 120, 30, 2380, 641, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5909, 6100, 6620, 6403, 5309, 5871, 6321, 6619] avg=6144.0000 max=6620 max_rank=2 min=5309 min_rank=4 imbalance_ratio=1.077474 max_load_reduction=42532 max_load_reduction_pct=0.865316 moved_to_echo=24120 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[105, 731, 183, 280, 2263, 2397, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 199, 2013, 1219, 43, 223, 250, 2059, 116, 525, 0, 0, 0, 0, 0, 0, 0, 39, 85, 146, 3047, 0, 266, 196, 2, 1529, 729, 177, 0, 0, 0, 0, 0, 449, 238, 216, 35, 2118, 155, 4, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 13, 115, 17, 52, 475, 2630, 0, 0, 4267, 0, 0, 0, 0, 0, 0, 0, 1507, 1163, 0, 526, 722, 49, 25, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3253, 33, 668, 13, 0, 70, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 102, 8, 728, 2257, 188, 5, 39, 33, 2088, 740, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6647, 6216, 6006, 7569, 4882, 5667, 6188] avg=6144.0000 max=7569 max_rank=4 min=4882 min_rank=5 imbalance_ratio=1.231934 max_load_reduction=41583 max_load_reduction_pct=0.846008 moved_to_echo=24306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[87, 1030, 202, 300, 1927, 2868, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 216, 2097, 585, 21, 325, 469, 2188, 215, 561, 0, 0, 0, 0, 0, 0, 0, 107, 38, 100, 2183, 0, 204, 649, 1, 1823, 756, 187, 0, 0, 0, 0, 0, 250, 125, 106, 104, 2219, 278, 4, 3320, 0, 0, 0, 0, 0, 0, 0, 0, 11, 108, 43, 3, 636, 1090, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 1550, 1361, 1, 494, 786, 53, 94, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 36, 382, 181, 1, 59, 1798, 0, 0, 0, 0, 0, 0, 0, 0, 38, 78, 734, 2563, 143, 11, 22, 7, 2175, 680, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6428, 6677, 6048, 6406, 5831, 5352, 5959, 6451] avg=6144.0000 max=6677 max_rank=1 min=5352 min_rank=5 imbalance_ratio=1.086751 max_load_reduction=42475 max_load_reduction_pct=0.864156 moved_to_echo=23593 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[189, 1046, 340, 282, 1059, 2945, 12, 23, 0, 0, 0, 0, 0, 0, 0, 0, 451, 992, 254, 12, 295, 707, 2613, 150, 504, 0, 0, 0, 0, 0, 0, 0, 31, 96, 126, 1862, 0, 311, 338, 8, 1871, 708, 168, 0, 0, 0, 0, 0, 323, 197, 313, 163, 2077, 374, 76, 2655, 0, 0, 0, 0, 0, 0, 0, 0, 43, 174, 100, 62, 237, 988, 0, 4, 3776, 0, 0, 0, 0, 0, 0, 0, 1991, 2453, 3, 473, 1155, 19, 52, 2041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3152, 34, 677, 24, 1, 49, 1407, 0, 0, 0, 0, 0, 0, 0, 0, 108, 103, 1346, 2208, 108, 30, 37, 38, 2036, 652, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5896, 5978, 5519, 6178, 5384, 8187, 5344, 6666] avg=6144.0000 max=8187 max_rank=5 min=5344 min_rank=6 imbalance_ratio=1.332520 max_load_reduction=40965 max_load_reduction_pct=0.833435 moved_to_echo=25581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[263, 550, 314, 302, 1439, 2644, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 304, 2054, 519, 30, 647, 731, 1655, 207, 522, 0, 0, 0, 0, 0, 0, 0, 38, 16, 180, 2337, 0, 250, 403, 0, 1680, 737, 174, 0, 0, 0, 0, 0, 317, 136, 130, 113, 2161, 378, 11, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 26, 168, 107, 59, 567, 1290, 0, 0, 4129, 0, 0, 0, 0, 0, 0, 0, 1472, 1957, 2, 517, 1006, 29, 89, 1470, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3265, 33, 525, 27, 1, 114, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 61, 34, 842, 2001, 135, 47, 36, 17, 2119, 713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5618, 6669, 5815, 6372, 6346, 6542, 5785, 6005] avg=6144.0000 max=6669 max_rank=1 min=5618 min_rank=0 imbalance_ratio=1.085449 max_load_reduction=42483 max_load_reduction_pct=0.864319 moved_to_echo=24678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[123, 1055, 672, 491, 2189, 2748, 47, 9, 0, 0, 0, 0, 0, 0, 0, 0, 321, 1973, 602, 78, 380, 430, 1537, 321, 553, 0, 0, 0, 0, 0, 0, 0, 384, 93, 184, 1977, 16, 161, 235, 0, 1747, 466, 184, 0, 0, 0, 0, 0, 314, 173, 106, 289, 1367, 397, 7, 3216, 0, 0, 0, 0, 0, 0, 0, 0, 16, 167, 54, 48, 1056, 2109, 3, 33, 3700, 0, 0, 0, 0, 0, 0, 0, 1003, 1210, 0, 464, 1060, 9, 117, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 27, 1252, 275, 2, 48, 2202, 0, 0, 0, 0, 0, 0, 0, 0, 119, 30, 589, 1594, 237, 4, 73, 12, 1340, 639, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7334, 6195, 5447, 5869, 7186, 5224, 7260, 4637] avg=6144.0000 max=7334 max_rank=0 min=4637 min_rank=7 imbalance_ratio=1.193685 max_load_reduction=41818 max_load_reduction_pct=0.850789 moved_to_echo=24307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[141, 622, 246, 335, 1737, 2492, 72, 2, 0, 0, 0, 0, 0, 0, 0, 0, 171, 1053, 436, 11, 500, 229, 2283, 85, 475, 0, 0, 0, 0, 0, 0, 0, 118, 136, 119, 3779, 0, 324, 361, 0, 1584, 863, 158, 0, 0, 0, 0, 0, 165, 142, 238, 279, 2531, 195, 18, 2213, 0, 0, 0, 0, 0, 0, 0, 0, 26, 149, 57, 18, 225, 938, 1, 0, 4284, 0, 0, 0, 0, 0, 0, 0, 1169, 1668, 0, 537, 1015, 12, 301, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 43, 493, 16, 0, 170, 2718, 0, 0, 0, 0, 0, 0, 0, 0, 19, 28, 647, 2233, 215, 11, 60, 25, 2482, 740, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5647, 5243, 7442, 5781, 5698, 6475, 6406, 6460] avg=6144.0000 max=7442 max_rank=2 min=5243 min_rank=1 imbalance_ratio=1.211263 max_load_reduction=41710 max_load_reduction_pct=0.848592 moved_to_echo=25039 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[87, 1030, 202, 300, 1927, 2868, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 216, 2097, 585, 21, 325, 469, 2188, 215, 561, 0, 0, 0, 0, 0, 0, 0, 107, 38, 100, 2183, 0, 204, 649, 1, 1823, 756, 187, 0, 0, 0, 0, 0, 250, 125, 106, 104, 2219, 278, 4, 3320, 0, 0, 0, 0, 0, 0, 0, 0, 11, 108, 43, 3, 636, 1090, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 1550, 1361, 1, 494, 786, 53, 94, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 36, 382, 181, 1, 59, 1798, 0, 0, 0, 0, 0, 0, 0, 0, 38, 78, 734, 2563, 143, 11, 22, 7, 2175, 680, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6428, 6677, 6048, 6406, 5831, 5352, 5959, 6451] avg=6144.0000 max=6677 max_rank=1 min=5352 min_rank=5 imbalance_ratio=1.086751 max_load_reduction=42475 max_load_reduction_pct=0.864156 moved_to_echo=23593 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[189, 1046, 340, 282, 1059, 2945, 12, 23, 0, 0, 0, 0, 0, 0, 0, 0, 451, 992, 254, 12, 295, 707, 2613, 150, 504, 0, 0, 0, 0, 0, 0, 0, 31, 96, 126, 1862, 0, 311, 338, 8, 1871, 708, 168, 0, 0, 0, 0, 0, 323, 197, 313, 163, 2077, 374, 76, 2655, 0, 0, 0, 0, 0, 0, 0, 0, 43, 174, 100, 62, 237, 988, 0, 4, 3776, 0, 0, 0, 0, 0, 0, 0, 1991, 2453, 3, 473, 1155, 19, 52, 2041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3152, 34, 677, 24, 1, 49, 1407, 0, 0, 0, 0, 0, 0, 0, 0, 108, 103, 1346, 2208, 108, 30, 37, 38, 2036, 652, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5896, 5978, 5519, 6178, 5384, 8187, 5344, 6666] avg=6144.0000 max=8187 max_rank=5 min=5344 min_rank=6 imbalance_ratio=1.332520 max_load_reduction=40965 max_load_reduction_pct=0.833435 moved_to_echo=25581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[105, 731, 183, 280, 2263, 2397, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 199, 2013, 1219, 43, 223, 250, 2059, 116, 525, 0, 0, 0, 0, 0, 0, 0, 39, 85, 146, 3047, 0, 266, 196, 2, 1529, 729, 177, 0, 0, 0, 0, 0, 449, 238, 216, 35, 2118, 155, 4, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 13, 115, 17, 52, 475, 2630, 0, 0, 4267, 0, 0, 0, 0, 0, 0, 0, 1507, 1163, 0, 526, 722, 49, 25, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3253, 33, 668, 13, 0, 70, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 102, 8, 728, 2257, 188, 5, 39, 33, 2088, 740, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6647, 6216, 6006, 7569, 4882, 5667, 6188] avg=6144.0000 max=7569 max_rank=4 min=4882 min_rank=5 imbalance_ratio=1.231934 max_load_reduction=41583 max_load_reduction_pct=0.846008 moved_to_echo=24306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 640, 1152, 21120, 128, 2048, 2688, 128, 13568, 5888, 1408, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1920, 13568, 4352, 256, 3456, 3584, 16640, 1536, 4224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12288, 12800, 128, 4096, 7936, 256, 768, 11392, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 640, 1152, 21120, 128, 2048, 2688, 128, 13568, 5888, 1408, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 384, 6912, 16768, 1536, 128, 512, 256, 16896, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 26240, 384, 5504, 640, 128, 640, 16128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 1280, 1536, 1536, 17280, 2816, 256, 22528, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12288, 12800, 128, 4096, 7936, 256, 768, 11392, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 1152, 640, 384, 4096, 11008, 128, 128, 32000, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1280, 6656, 2816, 2688, 14464, 21376, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 26240, 384, 5504, 640, 128, 640, 16128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 1152, 640, 384, 4096, 11008, 128, 128, 32000, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[14, 3224, 599, 4553, 188, 90, 238, 4, 142, 10, 90, 122, 1163, 0, 1873, 14, 1203, 0, 654, 79, 687, 335, 106, 1926, 597, 628, 47, 57, 86, 1644, 447, 352, 4026, 198, 243, 465, 468, 4, 1117, 709, 131, 3, 1325, 211, 6, 10, 6, 259, 267, 589, 3967, 1625, 5889, 1, 1174, 5, 442, 22, 3, 2316, 414, 84, 494, 1507] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[7, 3509, 729, 4422, 108, 55, 74, 12, 150, 11, 129, 53, 623, 0, 2364, 68, 814, 0, 459, 122, 311, 330, 26, 2488, 367, 1043, 25, 50, 208, 1395, 346, 645, 2587, 111, 107, 367, 333, 1, 1001, 1441, 106, 7, 1320, 293, 3, 46, 22, 97, 219, 212, 5070, 1639, 6301, 3, 979, 2, 594, 150, 1, 3350, 479, 179, 163, 1026] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3, 2031, 497, 3759, 85, 127, 61, 1, 135, 6, 380, 59, 605, 17, 2288, 7, 2518, 3, 756, 132, 415, 487, 23, 2436, 265, 1081, 10, 58, 556, 1903, 172, 535, 3708, 96, 214, 173, 220, 18, 1524, 1795, 119, 14, 1450, 183, 15, 26, 31, 147, 141, 106, 4141, 1022, 5847, 0, 1089, 0, 483, 75, 0, 2778, 437, 129, 143, 1617] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[2, 2499, 471, 4308, 128, 39, 254, 10, 115, 8, 147, 20, 394, 1, 1528, 48, 843, 1, 451, 18, 522, 453, 33, 1718, 193, 1822, 13, 133, 67, 1449, 450, 248, 3512, 175, 305, 403, 605, 5, 543, 1122, 349, 19, 760, 363, 10, 62, 4, 158, 270, 219, 4167, 1382, 7030, 36, 1741, 0, 1076, 63, 0, 4191, 624, 124, 362, 1086] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[2, 2231, 840, 4202, 107, 31, 367, 0, 90, 13, 97, 75, 413, 0, 1624, 49, 1604, 2, 563, 32, 660, 285, 46, 930, 197, 2244, 13, 75, 70, 1130, 441, 326, 4222, 99, 271, 378, 407, 0, 874, 1800, 145, 10, 1242, 133, 0, 4, 4, 149, 443, 295, 5596, 1056, 6680, 0, 773, 0, 591, 63, 0, 3654, 432, 34, 295, 743] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[2, 1701, 117, 5125, 171, 41, 158, 10, 198, 27, 291, 152, 898, 0, 1401, 90, 1379, 4, 393, 48, 345, 434, 33, 2718, 224, 1086, 5, 69, 190, 1065, 304, 254, 3066, 145, 324, 257, 282, 5, 2274, 2483, 93, 9, 1255, 355, 8, 8, 0, 150, 894, 191, 4342, 1345, 6796, 0, 604, 1, 1007, 37, 0, 2546, 437, 126, 223, 956] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 1738, 752, 4700, 66, 28, 264, 14, 158, 9, 92, 109, 799, 1, 2402, 18, 1700, 9, 747, 44, 384, 1017, 82, 2321, 222, 1392, 6, 129, 453, 1572, 249, 258, 2752, 157, 94, 328, 268, 8, 914, 2184, 248, 16, 845, 200, 11, 4, 2, 169, 413, 120, 3210, 1729, 6601, 2, 841, 0, 472, 70, 0, 3464, 471, 95, 394, 1335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[2, 1701, 117, 5125, 171, 41, 158, 10, 198, 27, 291, 152, 898, 0, 1401, 90, 1379, 4, 393, 48, 345, 434, 33, 2718, 224, 1086, 5, 69, 190, 1065, 304, 254, 3066, 145, 324, 257, 282, 5, 2274, 2483, 93, 9, 1255, 355, 8, 8, 0, 150, 894, 191, 4342, 1345, 6796, 0, 604, 1, 1007, 37, 0, 2546, 437, 126, 223, 956] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3, 2031, 497, 3759, 85, 127, 61, 1, 135, 6, 380, 59, 605, 17, 2288, 7, 2518, 3, 756, 132, 415, 487, 23, 2436, 265, 1081, 10, 58, 556, 1903, 172, 535, 3708, 96, 214, 173, 220, 18, 1524, 1795, 119, 14, 1450, 183, 15, 26, 31, 147, 141, 106, 4141, 1022, 5847, 0, 1089, 0, 483, 75, 0, 2778, 437, 129, 143, 1617] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[32, 2287, 216, 3327, 167, 108, 164, 32, 156, 10, 1208, 96, 386, 5, 2360, 67, 667, 35, 653, 86, 903, 709, 55, 2430, 356, 793, 43, 48, 210, 1108, 458, 336, 3524, 279, 261, 405, 462, 3, 1720, 2468, 101, 10, 495, 364, 8, 14, 20, 414, 843, 262, 3890, 1633, 6440, 6, 898, 2, 575, 55, 0, 2355, 385, 145, 149, 1455] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[14, 3224, 599, 2972, 188, 90, 238, 4, 0, 0, 0, 0, 0, 0, 0, 0, 142, 10, 90, 122, 1163, 0, 1873, 14, 1880, 902, 0, 0, 0, 0, 0, 0, 1203, 0, 654, 79, 687, 335, 106, 1926, 863, 0, 0, 0, 0, 0, 0, 0, 597, 628, 47, 57, 86, 1644, 447, 352, 1639, 679, 74, 0, 0, 0, 0, 0, 2387, 198, 243, 465, 468, 4, 1117, 709, 0, 0, 0, 0, 0, 0, 0, 0, 131, 3, 1325, 211, 6, 10, 6, 259, 4009, 0, 0, 0, 0, 0, 0, 0, 267, 589, 2779, 1625, 0, 1, 1174, 5, 0, 0, 0, 0, 0, 0, 0, 0, 442, 22, 3, 2316, 414, 84, 494, 1507, 251, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7329, 6196, 5853, 6250, 5591, 5960, 6440, 5533] avg=6144.0000 max=7329 max_rank=0 min=5533 min_rank=7 imbalance_ratio=1.192871 max_load_reduction=41823 max_load_reduction_pct=0.850891 moved_to_echo=23524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 2031, 497, 2454, 85, 127, 61, 1, 0, 0, 0, 0, 0, 0, 0, 0, 135, 6, 380, 59, 605, 17, 2288, 7, 1867, 744, 0, 0, 0, 0, 0, 0, 2518, 3, 756, 132, 415, 487, 23, 2436, 901, 0, 0, 0, 0, 0, 0, 0, 265, 1081, 10, 58, 556, 1903, 172, 535, 1510, 561, 77, 0, 0, 0, 0, 0, 2198, 96, 214, 173, 220, 18, 1524, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 119, 14, 1450, 183, 15, 26, 31, 147, 3980, 0, 0, 0, 0, 0, 0, 0, 141, 106, 2901, 1022, 0, 0, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 75, 0, 2778, 437, 129, 143, 1617, 262, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5259, 6108, 7671, 6728, 6238, 5965, 5259, 5924] avg=6144.0000 max=7671 max_rank=2 min=5259 min_rank=0 imbalance_ratio=1.248535 max_load_reduction=41481 max_load_reduction_pct=0.843933 moved_to_echo=23386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[14, 3224, 599, 4553, 188, 90, 238, 4, 142, 10, 90, 122, 1163, 0, 1873, 14, 1203, 0, 654, 79, 687, 335, 106, 1926, 597, 628, 47, 57, 86, 1644, 447, 352, 4026, 198, 243, 465, 468, 4, 1117, 709, 131, 3, 1325, 211, 6, 10, 6, 259, 267, 589, 3967, 1625, 5889, 1, 1174, 5, 442, 22, 3, 2316, 414, 84, 494, 1507] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[7, 3509, 729, 4422, 108, 55, 74, 12, 150, 11, 129, 53, 623, 0, 2364, 68, 814, 0, 459, 122, 311, 330, 26, 2488, 367, 1043, 25, 50, 208, 1395, 346, 645, 2587, 111, 107, 367, 333, 1, 1001, 1441, 106, 7, 1320, 293, 3, 46, 22, 97, 219, 212, 5070, 1639, 6301, 3, 979, 2, 594, 150, 1, 3350, 479, 179, 163, 1026] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 2231, 840, 2743, 107, 31, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 13, 97, 75, 413, 0, 1624, 49, 2133, 832, 0, 0, 0, 0, 0, 0, 1604, 2, 563, 32, 660, 285, 46, 930, 1218, 0, 0, 0, 0, 0, 0, 0, 197, 2244, 13, 75, 70, 1130, 441, 326, 1719, 627, 104, 0, 0, 0, 0, 0, 2503, 99, 271, 378, 407, 0, 874, 1800, 0, 0, 0, 0, 0, 0, 0, 0, 145, 10, 1242, 133, 0, 4, 4, 149, 4547, 0, 0, 0, 0, 0, 0, 0, 443, 295, 3920, 1056, 0, 0, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 63, 0, 3654, 432, 34, 295, 743, 354, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6321, 5326, 5340, 6946, 6332, 6234, 6487, 6166] avg=6144.0000 max=6946 max_rank=3 min=5326 min_rank=1 imbalance_ratio=1.130534 max_load_reduction=42206 max_load_reduction_pct=0.858683 moved_to_echo=25219 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 2499, 471, 2812, 128, 39, 254, 10, 0, 0, 0, 0, 0, 0, 0, 0, 115, 8, 147, 20, 394, 1, 1528, 48, 2245, 853, 0, 0, 0, 0, 0, 0, 843, 1, 451, 18, 522, 453, 33, 1718, 907, 0, 0, 0, 0, 0, 0, 0, 193, 1822, 13, 133, 67, 1449, 450, 248, 1430, 643, 77, 0, 0, 0, 0, 0, 2082, 175, 305, 403, 605, 5, 543, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 349, 19, 760, 363, 10, 62, 4, 158, 4785, 0, 0, 0, 0, 0, 0, 0, 270, 219, 2920, 1382, 0, 36, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 63, 0, 4191, 624, 124, 362, 1086, 263, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6215, 5359, 4946, 6525, 5240, 6510, 6568, 7789] avg=6144.0000 max=7789 max_rank=7 min=4946 min_rank=2 imbalance_ratio=1.267741 max_load_reduction=41363 max_load_reduction_pct=0.841532 moved_to_echo=26107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 3509, 729, 2880, 108, 55, 74, 12, 0, 0, 0, 0, 0, 0, 0, 0, 150, 11, 129, 53, 623, 0, 2364, 68, 2013, 879, 0, 0, 0, 0, 0, 0, 814, 0, 459, 122, 311, 330, 26, 2488, 1107, 0, 0, 0, 0, 0, 0, 0, 367, 1043, 25, 50, 208, 1395, 346, 645, 1057, 663, 98, 0, 0, 0, 0, 0, 1530, 111, 107, 367, 333, 1, 1001, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 106, 7, 1320, 293, 3, 46, 22, 97, 4288, 0, 0, 0, 0, 0, 0, 0, 219, 212, 3542, 1639, 0, 3, 979, 2, 0, 0, 0, 0, 0, 0, 0, 0, 594, 150, 1, 3350, 479, 179, 163, 1026, 323, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7374, 6290, 5657, 5897, 4891, 6182, 6596, 6265] avg=6144.0000 max=7374 max_rank=0 min=4891 min_rank=4 imbalance_ratio=1.200195 max_load_reduction=41778 max_load_reduction_pct=0.849976 moved_to_echo=23934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 1701, 117, 3346, 171, 41, 158, 10, 0, 0, 0, 0, 0, 0, 0, 0, 198, 27, 291, 152, 898, 0, 1401, 90, 2171, 1015, 0, 0, 0, 0, 0, 0, 1379, 4, 393, 48, 345, 434, 33, 2718, 945, 0, 0, 0, 0, 0, 0, 0, 224, 1086, 5, 69, 190, 1065, 304, 254, 1248, 764, 81, 0, 0, 0, 0, 0, 1818, 145, 324, 257, 282, 5, 2274, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 93, 9, 1255, 355, 8, 8, 0, 150, 4625, 0, 0, 0, 0, 0, 0, 0, 894, 191, 3042, 1345, 0, 0, 604, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 37, 0, 2546, 437, 126, 223, 956, 274, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5546, 6243, 6299, 5290, 7588, 6503, 6077, 5606] avg=6144.0000 max=7588 max_rank=4 min=5290 min_rank=3 imbalance_ratio=1.235026 max_load_reduction=41564 max_load_reduction_pct=0.845622 moved_to_echo=25774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 2031, 497, 2454, 85, 127, 61, 1, 0, 0, 0, 0, 0, 0, 0, 0, 135, 6, 380, 59, 605, 17, 2288, 7, 1867, 744, 0, 0, 0, 0, 0, 0, 2518, 3, 756, 132, 415, 487, 23, 2436, 901, 0, 0, 0, 0, 0, 0, 0, 265, 1081, 10, 58, 556, 1903, 172, 535, 1510, 561, 77, 0, 0, 0, 0, 0, 2198, 96, 214, 173, 220, 18, 1524, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 119, 14, 1450, 183, 15, 26, 31, 147, 3980, 0, 0, 0, 0, 0, 0, 0, 141, 106, 2901, 1022, 0, 0, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 75, 0, 2778, 437, 129, 143, 1617, 262, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5259, 6108, 7671, 6728, 6238, 5965, 5259, 5924] avg=6144.0000 max=7671 max_rank=2 min=5259 min_rank=0 imbalance_ratio=1.248535 max_load_reduction=41481 max_load_reduction_pct=0.843933 moved_to_echo=23386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1738, 752, 3068, 66, 28, 264, 14, 0, 0, 0, 0, 0, 0, 0, 0, 158, 9, 92, 109, 799, 1, 2402, 18, 2107, 931, 0, 0, 0, 0, 0, 0, 1700, 9, 747, 44, 384, 1017, 82, 2321, 698, 0, 0, 0, 0, 0, 0, 0, 222, 1392, 6, 129, 453, 1572, 249, 258, 1120, 701, 59, 0, 0, 0, 0, 0, 1632, 157, 94, 328, 268, 8, 914, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 248, 16, 845, 200, 11, 4, 2, 169, 4494, 0, 0, 0, 0, 0, 0, 0, 413, 120, 2250, 1729, 0, 2, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 70, 0, 3464, 471, 95, 394, 1335, 203, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 6626, 7002, 6161, 5585, 5989, 5355, 6504] avg=6144.0000 max=7002 max_rank=2 min=5355 min_rank=6 imbalance_ratio=1.139648 max_load_reduction=42150 max_load_reduction_pct=0.857544 moved_to_echo=23433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 1701, 117, 3346, 171, 41, 158, 10, 0, 0, 0, 0, 0, 0, 0, 0, 198, 27, 291, 152, 898, 0, 1401, 90, 2171, 1015, 0, 0, 0, 0, 0, 0, 1379, 4, 393, 48, 345, 434, 33, 2718, 945, 0, 0, 0, 0, 0, 0, 0, 224, 1086, 5, 69, 190, 1065, 304, 254, 1248, 764, 81, 0, 0, 0, 0, 0, 1818, 145, 324, 257, 282, 5, 2274, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 93, 9, 1255, 355, 8, 8, 0, 150, 4625, 0, 0, 0, 0, 0, 0, 0, 894, 191, 3042, 1345, 0, 0, 604, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 37, 0, 2546, 437, 126, 223, 956, 274, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5546, 6243, 6299, 5290, 7588, 6503, 6077, 5606] avg=6144.0000 max=7588 max_rank=4 min=5290 min_rank=3 imbalance_ratio=1.235026 max_load_reduction=41564 max_load_reduction_pct=0.845622 moved_to_echo=25774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 2499, 471, 2812, 128, 39, 254, 10, 0, 0, 0, 0, 0, 0, 0, 0, 115, 8, 147, 20, 394, 1, 1528, 48, 2245, 853, 0, 0, 0, 0, 0, 0, 843, 1, 451, 18, 522, 453, 33, 1718, 907, 0, 0, 0, 0, 0, 0, 0, 193, 1822, 13, 133, 67, 1449, 450, 248, 1430, 643, 77, 0, 0, 0, 0, 0, 2082, 175, 305, 403, 605, 5, 543, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 349, 19, 760, 363, 10, 62, 4, 158, 4785, 0, 0, 0, 0, 0, 0, 0, 270, 219, 2920, 1382, 0, 36, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 63, 0, 4191, 624, 124, 362, 1086, 263, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6215, 5359, 4946, 6525, 5240, 6510, 6568, 7789] avg=6144.0000 max=7789 max_rank=7 min=4946 min_rank=2 imbalance_ratio=1.267741 max_load_reduction=41363 max_load_reduction_pct=0.841532 moved_to_echo=26107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[14, 3224, 599, 2972, 188, 90, 238, 4, 0, 0, 0, 0, 0, 0, 0, 0, 142, 10, 90, 122, 1163, 0, 1873, 14, 1880, 902, 0, 0, 0, 0, 0, 0, 1203, 0, 654, 79, 687, 335, 106, 1926, 863, 0, 0, 0, 0, 0, 0, 0, 597, 628, 47, 57, 86, 1644, 447, 352, 1639, 679, 74, 0, 0, 0, 0, 0, 2387, 198, 243, 465, 468, 4, 1117, 709, 0, 0, 0, 0, 0, 0, 0, 0, 131, 3, 1325, 211, 6, 10, 6, 259, 4009, 0, 0, 0, 0, 0, 0, 0, 267, 589, 2779, 1625, 0, 1, 1174, 5, 0, 0, 0, 0, 0, 0, 0, 0, 442, 22, 3, 2316, 414, 84, 494, 1507, 251, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7329, 6196, 5853, 6250, 5591, 5960, 6440, 5533] avg=6144.0000 max=7329 max_rank=0 min=5533 min_rank=7 imbalance_ratio=1.192871 max_load_reduction=41823 max_load_reduction_pct=0.850891 moved_to_echo=23524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[32, 2287, 216, 2172, 167, 108, 164, 32, 0, 0, 0, 0, 0, 0, 0, 0, 156, 10, 1208, 96, 386, 5, 2360, 67, 2057, 659, 0, 0, 0, 0, 0, 0, 667, 35, 653, 86, 903, 709, 55, 2430, 846, 0, 0, 0, 0, 0, 0, 0, 356, 793, 43, 48, 210, 1108, 458, 336, 1435, 496, 72, 0, 0, 0, 0, 0, 2089, 279, 261, 405, 462, 3, 1720, 2468, 0, 0, 0, 0, 0, 0, 0, 0, 101, 10, 495, 364, 8, 14, 20, 414, 4383, 0, 0, 0, 0, 0, 0, 0, 843, 262, 2726, 1633, 0, 6, 898, 2, 0, 0, 0, 0, 0, 0, 0, 0, 575, 55, 0, 2355, 385, 145, 149, 1455, 246, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5178, 7004, 6384, 5355, 7687, 5809, 6370, 5365] avg=6144.0000 max=7687 max_rank=4 min=5178 min_rank=0 imbalance_ratio=1.251139 max_load_reduction=41465 max_load_reduction_pct=0.843608 moved_to_echo=25231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 3509, 729, 2880, 108, 55, 74, 12, 0, 0, 0, 0, 0, 0, 0, 0, 150, 11, 129, 53, 623, 0, 2364, 68, 2013, 879, 0, 0, 0, 0, 0, 0, 814, 0, 459, 122, 311, 330, 26, 2488, 1107, 0, 0, 0, 0, 0, 0, 0, 367, 1043, 25, 50, 208, 1395, 346, 645, 1057, 663, 98, 0, 0, 0, 0, 0, 1530, 111, 107, 367, 333, 1, 1001, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 106, 7, 1320, 293, 3, 46, 22, 97, 4288, 0, 0, 0, 0, 0, 0, 0, 219, 212, 3542, 1639, 0, 3, 979, 2, 0, 0, 0, 0, 0, 0, 0, 0, 594, 150, 1, 3350, 479, 179, 163, 1026, 323, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7374, 6290, 5657, 5897, 4891, 6182, 6596, 6265] avg=6144.0000 max=7374 max_rank=0 min=4891 min_rank=4 imbalance_ratio=1.200195 max_load_reduction=41778 max_load_reduction_pct=0.849976 moved_to_echo=23934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 128, 2560, 768, 5376, 128, 15872, 384, 16512, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2432, 10112, 256, 640, 1920, 11392, 2944, 3072, 11264, 5248, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5248, 640, 128, 24704, 3712, 1024, 2304, 9728, 2176, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[16256, 1280, 1920, 2816, 3072, 128, 9984, 14080, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2432, 10112, 256, 640, 1920, 11392, 2944, 3072, 11264, 5248, 768, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10752, 128, 4736, 640, 4352, 4096, 512, 17024, 7552, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 19328, 4224, 22528, 1024, 640, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5248, 640, 128, 24704, 3712, 1024, 2304, 9728, 2176, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 2048, 24192, 11520, 0, 128, 8192, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 19328, 4224, 22528, 1024, 640, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 128, 8704, 2176, 128, 256, 128, 1664, 35200, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[713, 109, 26, 1, 39, 654, 0, 16, 2212, 790, 13, 0, 596, 2845, 805, 177, 3218, 5767, 0, 355, 1475, 838, 56, 586, 114, 1, 6034, 135, 57, 369, 59, 912, 602, 482, 213, 181, 51, 1, 922, 122, 1606, 642, 334, 61, 43, 5822, 230, 35, 225, 1816, 1, 39, 300, 1109, 1346, 0, 44, 59, 3381, 133, 58, 275, 42, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[974, 31, 9, 7, 59, 425, 0, 22, 2836, 955, 2, 1, 753, 3367, 765, 342, 1631, 4028, 0, 360, 1044, 1212, 108, 666, 141, 0, 4738, 158, 270, 156, 99, 922, 419, 470, 449, 242, 139, 0, 1865, 31, 1457, 1022, 278, 25, 6, 5313, 889, 120, 16, 1826, 2, 130, 243, 1279, 2361, 0, 82, 96, 3783, 176, 86, 232, 4, 30] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1906, 94, 4, 0, 15, 778, 0, 19, 1815, 776, 71, 0, 636, 2442, 597, 526, 1708, 3064, 0, 828, 2009, 1985, 25, 669, 49, 0, 5472, 67, 110, 682, 94, 1567, 316, 141, 559, 195, 207, 0, 941, 37, 1965, 784, 97, 177, 38, 5843, 41, 34, 39, 1875, 3, 125, 270, 840, 1825, 0, 56, 52, 4085, 147, 135, 298, 13, 6] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1150, 336, 3, 1, 40, 643, 0, 23, 2183, 973, 31, 0, 677, 3224, 801, 422, 2364, 3851, 0, 284, 856, 1210, 63, 1184, 54, 2, 5596, 67, 316, 351, 21, 1306, 617, 237, 395, 147, 220, 0, 1308, 5, 1398, 376, 159, 65, 26, 6749, 389, 58, 146, 1450, 1, 75, 53, 1027, 1587, 0, 68, 82, 3922, 138, 150, 255, 5, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[898, 256, 0, 4, 50, 876, 0, 13, 2701, 1255, 43, 0, 1197, 3006, 813, 527, 3144, 3552, 0, 302, 798, 1105, 20, 351, 134, 0, 5956, 126, 113, 510, 54, 1097, 566, 209, 282, 254, 146, 0, 914, 34, 1145, 527, 631, 45, 18, 5502, 80, 26, 155, 1651, 1, 78, 397, 1099, 2008, 5, 19, 85, 3822, 110, 117, 312, 12, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[981, 76, 16, 0, 4, 871, 0, 25, 885, 468, 12, 0, 442, 2598, 780, 228, 3527, 3748, 1, 473, 954, 963, 26, 460, 58, 0, 6182, 111, 167, 520, 54, 1044, 302, 324, 553, 141, 47, 0, 1636, 36, 2425, 390, 281, 107, 24, 6162, 517, 40, 403, 1854, 2, 36, 43, 1040, 1426, 0, 36, 120, 4786, 134, 283, 304, 12, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[713, 109, 26, 1, 39, 654, 0, 16, 2212, 790, 13, 0, 596, 2845, 805, 177, 3218, 5767, 0, 355, 1475, 838, 56, 586, 114, 1, 6034, 135, 57, 369, 59, 912, 602, 482, 213, 181, 51, 1, 922, 122, 1606, 642, 334, 61, 43, 5822, 230, 35, 225, 1816, 1, 39, 300, 1109, 1346, 0, 44, 59, 3381, 133, 58, 275, 42, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1311, 23, 1, 0, 42, 720, 0, 44, 2360, 771, 4, 0, 742, 3510, 397, 485, 2879, 3837, 0, 239, 1868, 1723, 40, 490, 30, 0, 5803, 113, 129, 347, 29, 1086, 465, 258, 431, 216, 21, 0, 1362, 14, 974, 440, 112, 52, 11, 5548, 88, 12, 3, 1929, 0, 39, 501, 1232, 1664, 0, 30, 93, 4252, 114, 86, 150, 4, 28] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[981, 76, 16, 0, 4, 871, 0, 25, 885, 468, 12, 0, 442, 2598, 780, 228, 3527, 3748, 1, 473, 954, 963, 26, 460, 58, 0, 6182, 111, 167, 520, 54, 1044, 302, 324, 553, 141, 47, 0, 1636, 36, 2425, 390, 281, 107, 24, 6162, 517, 40, 403, 1854, 2, 36, 43, 1040, 1426, 0, 36, 120, 4786, 134, 283, 304, 12, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1021, 125, 31, 5, 79, 805, 0, 8, 1670, 523, 152, 2, 609, 2682, 1023, 707, 2699, 4857, 27, 315, 1131, 689, 48, 1405, 47, 11, 5805, 166, 200, 215, 33, 761, 335, 220, 89, 272, 49, 6, 1392, 26, 2069, 443, 101, 43, 41, 6456, 391, 79, 95, 1509, 4, 21, 44, 730, 2273, 3, 119, 112, 3604, 217, 204, 299, 25, 30] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[974, 31, 9, 7, 59, 425, 0, 22, 4028, 7, 0, 0, 0, 0, 0, 0, 2836, 955, 2, 1, 753, 1508, 765, 342, 0, 0, 0, 0, 0, 0, 0, 0, 1535, 0, 0, 360, 1044, 1212, 108, 666, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 3527, 158, 270, 156, 99, 922, 0, 0, 0, 0, 0, 0, 0, 0, 419, 470, 449, 242, 139, 0, 1865, 31, 2426, 767, 0, 0, 0, 0, 0, 0, 1457, 1022, 278, 25, 6, 2880, 889, 120, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1826, 2, 130, 243, 1279, 2361, 0, 793, 96, 0, 0, 0, 0, 0, 0, 82, 96, 3783, 176, 86, 232, 4, 30, 1092, 418, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5562, 7162, 4925, 5273, 6808, 6677, 6746, 5999] avg=6144.0000 max=7162 max_rank=1 min=4925 min_rank=2 imbalance_ratio=1.165690 max_load_reduction=41990 max_load_reduction_pct=0.854289 moved_to_echo=26230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1906, 94, 4, 0, 15, 778, 0, 19, 1815, 776, 71, 0, 636, 2442, 597, 526, 1708, 3064, 0, 828, 2009, 1985, 25, 669, 49, 0, 5472, 67, 110, 682, 94, 1567, 316, 141, 559, 195, 207, 0, 941, 37, 1965, 784, 97, 177, 38, 5843, 41, 34, 39, 1875, 3, 125, 270, 840, 1825, 0, 56, 52, 4085, 147, 135, 298, 13, 6] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[974, 31, 9, 7, 59, 425, 0, 22, 2836, 955, 2, 1, 753, 3367, 765, 342, 1631, 4028, 0, 360, 1044, 1212, 108, 666, 141, 0, 4738, 158, 270, 156, 99, 922, 419, 470, 449, 242, 139, 0, 1865, 31, 1457, 1022, 278, 25, 6, 5313, 889, 120, 16, 1826, 2, 130, 243, 1279, 2361, 0, 82, 96, 3783, 176, 86, 232, 4, 30] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[898, 256, 0, 4, 50, 876, 0, 13, 3552, 7, 0, 0, 0, 0, 0, 0, 2701, 1255, 43, 0, 1197, 1347, 813, 527, 0, 0, 0, 0, 0, 0, 0, 0, 2958, 0, 0, 302, 798, 1105, 20, 351, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 4432, 126, 113, 510, 54, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 566, 209, 282, 254, 146, 0, 914, 34, 2512, 684, 0, 0, 0, 0, 0, 0, 1145, 527, 631, 45, 18, 2983, 80, 26, 0, 0, 0, 0, 0, 0, 0, 0, 155, 1651, 1, 78, 397, 1099, 2008, 5, 998, 186, 0, 0, 0, 0, 0, 0, 19, 85, 3822, 110, 117, 312, 12, 1, 975, 526, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5656, 7883, 5534, 6466, 5601, 5455, 6578, 5979] avg=6144.0000 max=7883 max_rank=1 min=5455 min_rank=5 imbalance_ratio=1.283040 max_load_reduction=41269 max_load_reduction_pct=0.839620 moved_to_echo=23613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[713, 109, 26, 1, 39, 654, 0, 16, 5767, 8, 0, 0, 0, 0, 0, 0, 2212, 790, 13, 0, 596, 1275, 805, 177, 0, 0, 0, 0, 0, 0, 0, 0, 3027, 0, 0, 355, 1475, 838, 56, 586, 0, 0, 0, 0, 0, 0, 0, 0, 114, 1, 4490, 135, 57, 369, 59, 912, 0, 0, 0, 0, 0, 0, 0, 0, 602, 482, 213, 181, 51, 1, 922, 122, 2658, 648, 0, 0, 0, 0, 0, 0, 1606, 642, 334, 61, 43, 3156, 230, 35, 0, 0, 0, 0, 0, 0, 0, 0, 225, 1816, 1, 39, 300, 1109, 1346, 0, 1011, 191, 0, 0, 0, 0, 0, 0, 44, 59, 3381, 133, 58, 275, 42, 5, 922, 533, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7333, 5868, 6337, 6137, 5880, 6107, 6038, 5452] avg=6144.0000 max=7333 max_rank=0 min=5452 min_rank=7 imbalance_ratio=1.193522 max_load_reduction=41819 max_load_reduction_pct=0.850810 moved_to_echo=23477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1906, 94, 4, 0, 15, 778, 0, 19, 3064, 8, 0, 0, 0, 0, 0, 0, 1815, 776, 71, 0, 636, 1094, 597, 526, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 0, 0, 828, 2009, 1985, 25, 669, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4073, 67, 110, 682, 94, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 316, 141, 559, 195, 207, 0, 941, 37, 2668, 556, 0, 0, 0, 0, 0, 0, 1965, 784, 97, 177, 38, 3167, 41, 34, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1875, 3, 125, 270, 840, 1825, 0, 916, 101, 0, 0, 0, 0, 0, 0, 56, 52, 4085, 147, 135, 298, 13, 6, 792, 483, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5888, 5515, 7123, 6642, 5620, 6303, 5994, 6067] avg=6144.0000 max=7123 max_rank=2 min=5515 min_rank=1 imbalance_ratio=1.159342 max_load_reduction=42029 max_load_reduction_pct=0.855082 moved_to_echo=23984 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1150, 336, 3, 1, 40, 643, 0, 23, 3851, 9, 0, 0, 0, 0, 0, 0, 2183, 973, 31, 0, 677, 1445, 801, 422, 0, 0, 0, 0, 0, 0, 0, 0, 2224, 0, 0, 284, 856, 1210, 63, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2, 4165, 67, 316, 351, 21, 1306, 0, 0, 0, 0, 0, 0, 0, 0, 617, 237, 395, 147, 220, 0, 1308, 5, 3082, 734, 0, 0, 0, 0, 0, 0, 1398, 376, 159, 65, 26, 3658, 389, 58, 0, 0, 0, 0, 0, 0, 0, 0, 146, 1450, 1, 75, 53, 1027, 1587, 0, 937, 140, 0, 0, 0, 0, 0, 0, 68, 82, 3922, 138, 150, 255, 5, 12, 1045, 494, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6056, 6532, 5821, 6282, 6745, 6129, 5416, 6171] avg=6144.0000 max=6745 max_rank=4 min=5416 min_rank=6 imbalance_ratio=1.097819 max_load_reduction=42407 max_load_reduction_pct=0.862773 moved_to_echo=24461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1311, 23, 1, 0, 42, 720, 0, 44, 3837, 7, 0, 0, 0, 0, 0, 0, 2360, 771, 4, 0, 742, 1573, 397, 485, 0, 0, 0, 0, 0, 0, 0, 0, 2708, 0, 0, 239, 1868, 1723, 40, 490, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4319, 113, 129, 347, 29, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 465, 258, 431, 216, 21, 0, 1362, 14, 2533, 799, 0, 0, 0, 0, 0, 0, 974, 440, 112, 52, 11, 3008, 88, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1929, 0, 39, 501, 1232, 1664, 0, 972, 171, 0, 0, 0, 0, 0, 0, 30, 93, 4252, 114, 86, 150, 4, 28, 1138, 512, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5985, 6332, 7068, 6053, 6099, 4697, 6511, 6407] avg=6144.0000 max=7068 max_rank=2 min=4697 min_rank=5 imbalance_ratio=1.150391 max_load_reduction=42084 max_load_reduction_pct=0.856201 moved_to_echo=23714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[898, 256, 0, 4, 50, 876, 0, 13, 3552, 7, 0, 0, 0, 0, 0, 0, 2701, 1255, 43, 0, 1197, 1347, 813, 527, 0, 0, 0, 0, 0, 0, 0, 0, 2958, 0, 0, 302, 798, 1105, 20, 351, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 4432, 126, 113, 510, 54, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 566, 209, 282, 254, 146, 0, 914, 34, 2512, 684, 0, 0, 0, 0, 0, 0, 1145, 527, 631, 45, 18, 2983, 80, 26, 0, 0, 0, 0, 0, 0, 0, 0, 155, 1651, 1, 78, 397, 1099, 2008, 5, 998, 186, 0, 0, 0, 0, 0, 0, 19, 85, 3822, 110, 117, 312, 12, 1, 975, 526, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5656, 7883, 5534, 6466, 5601, 5455, 6578, 5979] avg=6144.0000 max=7883 max_rank=1 min=5455 min_rank=5 imbalance_ratio=1.283040 max_load_reduction=41269 max_load_reduction_pct=0.839620 moved_to_echo=23613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[981, 76, 16, 0, 4, 871, 0, 25, 3748, 12, 0, 0, 0, 0, 0, 0, 885, 468, 12, 0, 442, 1157, 780, 228, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 1, 473, 954, 963, 26, 460, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 4592, 111, 167, 520, 54, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 302, 324, 553, 141, 47, 0, 1636, 36, 2817, 595, 0, 0, 0, 0, 0, 0, 2425, 390, 281, 107, 24, 3333, 517, 40, 0, 0, 0, 0, 0, 0, 0, 0, 403, 1854, 2, 36, 43, 1040, 1426, 0, 1040, 214, 0, 0, 0, 0, 0, 0, 36, 120, 4786, 134, 283, 304, 12, 14, 846, 550, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5733, 3972, 6190, 6546, 6451, 7117, 6058, 7085] avg=6144.0000 max=7117 max_rank=5 min=3972 min_rank=1 imbalance_ratio=1.158366 max_load_reduction=42035 max_load_reduction_pct=0.855204 moved_to_echo=26711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1021, 125, 31, 5, 79, 805, 0, 8, 4857, 8, 0, 0, 0, 0, 0, 0, 1670, 523, 152, 2, 609, 1202, 1023, 707, 0, 0, 0, 0, 0, 0, 0, 0, 2539, 0, 27, 315, 1131, 689, 48, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 47, 11, 4320, 166, 200, 215, 33, 761, 0, 0, 0, 0, 0, 0, 0, 0, 335, 220, 89, 272, 49, 6, 1392, 26, 2948, 611, 0, 0, 0, 0, 0, 0, 2069, 443, 101, 43, 41, 3500, 391, 79, 0, 0, 0, 0, 0, 0, 0, 0, 95, 1509, 4, 21, 44, 730, 2273, 3, 972, 160, 0, 0, 0, 0, 0, 0, 119, 112, 3604, 217, 204, 299, 25, 30, 869, 513, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6939, 5888, 6154, 5753, 5948, 6667, 5811, 5992] avg=6144.0000 max=6939 max_rank=0 min=5753 min_rank=3 imbalance_ratio=1.129395 max_load_reduction=42213 max_load_reduction_pct=0.858826 moved_to_echo=24418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[974, 31, 9, 7, 59, 425, 0, 22, 4028, 7, 0, 0, 0, 0, 0, 0, 2836, 955, 2, 1, 753, 1508, 765, 342, 0, 0, 0, 0, 0, 0, 0, 0, 1535, 0, 0, 360, 1044, 1212, 108, 666, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 3527, 158, 270, 156, 99, 922, 0, 0, 0, 0, 0, 0, 0, 0, 419, 470, 449, 242, 139, 0, 1865, 31, 2426, 767, 0, 0, 0, 0, 0, 0, 1457, 1022, 278, 25, 6, 2880, 889, 120, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1826, 2, 130, 243, 1279, 2361, 0, 793, 96, 0, 0, 0, 0, 0, 0, 82, 96, 3783, 176, 86, 232, 4, 30, 1092, 418, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5562, 7162, 4925, 5273, 6808, 6677, 6746, 5999] avg=6144.0000 max=7162 max_rank=1 min=4925 min_rank=2 imbalance_ratio=1.165690 max_load_reduction=41990 max_load_reduction_pct=0.854289 moved_to_echo=26230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1906, 94, 4, 0, 15, 778, 0, 19, 3064, 8, 0, 0, 0, 0, 0, 0, 1815, 776, 71, 0, 636, 1094, 597, 526, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 0, 0, 828, 2009, 1985, 25, 669, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4073, 67, 110, 682, 94, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 316, 141, 559, 195, 207, 0, 941, 37, 2668, 556, 0, 0, 0, 0, 0, 0, 1965, 784, 97, 177, 38, 3167, 41, 34, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1875, 3, 125, 270, 840, 1825, 0, 916, 101, 0, 0, 0, 0, 0, 0, 56, 52, 4085, 147, 135, 298, 13, 6, 792, 483, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5888, 5515, 7123, 6642, 5620, 6303, 5994, 6067] avg=6144.0000 max=7123 max_rank=2 min=5515 min_rank=1 imbalance_ratio=1.159342 max_load_reduction=42029 max_load_reduction_pct=0.855082 moved_to_echo=23984 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[981, 76, 16, 0, 4, 871, 0, 25, 3748, 12, 0, 0, 0, 0, 0, 0, 885, 468, 12, 0, 442, 1157, 780, 228, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 1, 473, 954, 963, 26, 460, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 4592, 111, 167, 520, 54, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 302, 324, 553, 141, 47, 0, 1636, 36, 2817, 595, 0, 0, 0, 0, 0, 0, 2425, 390, 281, 107, 24, 3333, 517, 40, 0, 0, 0, 0, 0, 0, 0, 0, 403, 1854, 2, 36, 43, 1040, 1426, 0, 1040, 214, 0, 0, 0, 0, 0, 0, 36, 120, 4786, 134, 283, 304, 12, 14, 846, 550, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5733, 3972, 6190, 6546, 6451, 7117, 6058, 7085] avg=6144.0000 max=7117 max_rank=5 min=3972 min_rank=1 imbalance_ratio=1.158366 max_load_reduction=42035 max_load_reduction_pct=0.855204 moved_to_echo=26711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3712, 2432, 3072, 1664, 896, 128, 10368, 384, 21760, 5504, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19968, 0, 128, 3200, 10240, 9728, 512, 5888, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16768, 6528, 384, 128, 5760, 10624, 6016, 3456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13056, 4736, 2048, 640, 256, 25728, 2688, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1152, 13952, 128, 640, 1920, 8448, 14592, 128, 7680, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13056, 4736, 2048, 640, 256, 25728, 2688, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 768, 31744, 1280, 1152, 2176, 128, 128, 7680, 4096, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8960, 1152, 128, 128, 384, 5888, 0, 256, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19968, 0, 128, 3200, 10240, 9728, 512, 5888, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 128, 33920, 1024, 1408, 3200, 512, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8960, 1152, 128, 128, 384, 5888, 0, 256, 32768, 128, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[31, 15, 12, 0, 2, 2367, 1065, 1242, 4, 8, 260, 74, 13, 96, 10, 0, 2137, 20, 2169, 1807, 6465, 1145, 216, 22, 1007, 0, 153, 1191, 3, 6001, 56, 418, 23, 488, 885, 2, 0, 449, 415, 0, 27, 36, 2427, 185, 5, 105, 859, 0, 610, 228, 4660, 496, 3831, 182, 190, 274, 3059, 11, 5, 120, 1362, 164, 45, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[27, 245, 13, 9, 4, 4346, 858, 1432, 26, 189, 248, 171, 5, 76, 2, 2, 948, 73, 2177, 1307, 6336, 650, 134, 42, 1388, 0, 53, 870, 29, 3521, 109, 587, 61, 480, 443, 4, 14, 1320, 268, 1, 62, 222, 2174, 399, 102, 67, 593, 0, 778, 418, 4524, 674, 4838, 227, 434, 437, 2867, 11, 2, 318, 1445, 24, 63, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[94, 18, 37, 3, 1, 2462, 1611, 567, 20, 2, 272, 294, 1, 19, 22, 0, 952, 21, 1618, 2111, 5308, 872, 29, 10, 1343, 0, 225, 1093, 6, 5903, 84, 532, 9, 1274, 1661, 5, 0, 743, 95, 0, 21, 22, 2137, 36, 2, 57, 1024, 0, 323, 126, 4683, 547, 4045, 177, 47, 742, 3971, 26, 1, 162, 1533, 3, 149, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[39, 35, 3, 0, 0, 2921, 1208, 1508, 31, 6, 110, 169, 2, 40, 19, 1, 857, 37, 1232, 2980, 6488, 736, 170, 48, 1905, 0, 112, 1091, 1, 4721, 24, 949, 12, 600, 1328, 0, 1, 523, 216, 0, 55, 16, 2459, 178, 1, 72, 1438, 0, 569, 356, 4275, 360, 4336, 182, 28, 352, 2460, 19, 0, 132, 1672, 11, 57, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[195, 4, 29, 5, 1, 2577, 708, 2049, 63, 9, 223, 354, 3, 20, 44, 3, 1744, 36, 867, 3095, 6061, 806, 399, 59, 1102, 0, 206, 1268, 18, 4034, 68, 822, 7, 584, 1433, 15, 0, 384, 252, 1, 172, 12, 2544, 272, 6, 98, 1041, 0, 493, 76, 4484, 280, 4055, 230, 306, 421, 3337, 19, 1, 307, 1312, 1, 136, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[20, 150, 17, 0, 0, 3339, 955, 889, 3, 25, 274, 209, 4, 61, 60, 1, 1154, 15, 1173, 2063, 6415, 1296, 148, 32, 834, 1, 24, 1403, 2, 5228, 44, 808, 58, 1009, 1151, 1, 0, 591, 301, 2, 44, 28, 2598, 108, 1, 305, 966, 0, 667, 268, 5845, 356, 3240, 174, 125, 620, 3124, 42, 0, 119, 714, 5, 42, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[28, 5, 11, 0, 1, 3435, 733, 1045, 150, 0, 224, 137, 0, 21, 1, 0, 1158, 120, 948, 2013, 6731, 375, 89, 174, 2067, 0, 98, 1218, 1, 4506, 41, 478, 5, 396, 826, 0, 0, 1122, 131, 0, 20, 19, 3076, 86, 0, 148, 623, 0, 923, 477, 4391, 413, 5224, 151, 96, 603, 3256, 31, 0, 128, 1112, 1, 86, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[94, 18, 37, 3, 1, 2462, 1611, 567, 20, 2, 272, 294, 1, 19, 22, 0, 952, 21, 1618, 2111, 5308, 872, 29, 10, 1343, 0, 225, 1093, 6, 5903, 84, 532, 9, 1274, 1661, 5, 0, 743, 95, 0, 21, 22, 2137, 36, 2, 57, 1024, 0, 323, 126, 4683, 547, 4045, 177, 47, 742, 3971, 26, 1, 162, 1533, 3, 149, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[99, 14, 9, 2, 6, 2620, 1093, 771, 77, 14, 119, 108, 1, 300, 7, 0, 921, 26, 890, 2536, 6755, 247, 84, 40, 2858, 0, 69, 1117, 1, 5353, 36, 607, 7, 439, 1743, 2, 3, 1809, 215, 0, 53, 13, 1814, 46, 1, 39, 1007, 2, 727, 224, 4717, 533, 3875, 212, 58, 385, 3136, 5, 0, 184, 1072, 0, 51, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[20, 150, 17, 0, 0, 3339, 955, 889, 3, 25, 274, 209, 4, 61, 60, 1, 1154, 15, 1173, 2063, 6415, 1296, 148, 32, 834, 1, 24, 1403, 2, 5228, 44, 808, 58, 1009, 1151, 1, 0, 591, 301, 2, 44, 28, 2598, 108, 1, 305, 966, 0, 667, 268, 5845, 356, 3240, 174, 125, 620, 3124, 42, 0, 119, 714, 5, 42, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[27, 245, 13, 9, 4, 4346, 858, 1432, 26, 189, 248, 171, 5, 76, 2, 2, 948, 73, 2177, 1307, 6336, 650, 134, 42, 1388, 0, 53, 870, 29, 3521, 109, 587, 61, 480, 443, 4, 14, 1320, 268, 1, 62, 222, 2174, 399, 102, 67, 593, 0, 778, 418, 4524, 674, 4838, 227, 434, 437, 2867, 11, 2, 318, 1445, 24, 63, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[31, 15, 12, 0, 2, 2367, 1065, 1242, 4, 8, 260, 74, 13, 96, 10, 0, 2137, 20, 2169, 1807, 6465, 1145, 216, 22, 1007, 0, 153, 1191, 3, 6001, 56, 418, 23, 488, 885, 2, 0, 449, 415, 0, 27, 36, 2427, 185, 5, 105, 859, 0, 610, 228, 4660, 496, 3831, 182, 190, 274, 3059, 11, 5, 120, 1362, 164, 45, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[195, 4, 29, 5, 1, 2577, 708, 2049, 441, 226, 0, 0, 0, 0, 0, 0, 63, 9, 223, 354, 3, 20, 44, 3, 5328, 0, 0, 0, 0, 0, 0, 0, 1744, 36, 867, 3095, 255, 806, 399, 59, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 206, 1268, 18, 2132, 68, 822, 0, 0, 0, 0, 0, 0, 0, 0, 7, 584, 1433, 15, 0, 384, 252, 1, 2548, 478, 0, 0, 0, 0, 0, 0, 172, 12, 2544, 272, 6, 98, 1041, 0, 1936, 308, 0, 0, 0, 0, 0, 0, 493, 76, 0, 280, 3829, 230, 306, 421, 0, 0, 0, 0, 0, 0, 0, 0, 3337, 19, 1, 307, 1312, 1, 136, 1, 1153, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6235, 6047, 7261, 5616, 5702, 6389, 5635, 6267] avg=6144.0000 max=7261 max_rank=2 min=5616 min_rank=3 imbalance_ratio=1.181803 max_load_reduction=41891 max_load_reduction_pct=0.852275 moved_to_echo=23993 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[28, 5, 11, 0, 1, 3435, 733, 1045, 493, 291, 0, 0, 0, 0, 0, 0, 150, 0, 224, 137, 0, 21, 1, 0, 5917, 0, 0, 0, 0, 0, 0, 0, 1158, 120, 948, 2013, 283, 375, 89, 174, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 98, 1218, 1, 2381, 41, 478, 0, 0, 0, 0, 0, 0, 0, 0, 5, 396, 826, 0, 0, 1122, 131, 0, 2495, 531, 0, 0, 0, 0, 0, 0, 20, 19, 3076, 86, 0, 148, 623, 0, 1896, 344, 0, 0, 0, 0, 0, 0, 923, 477, 0, 413, 4933, 151, 96, 603, 0, 0, 0, 0, 0, 0, 0, 0, 3256, 31, 0, 128, 1112, 1, 86, 0, 1288, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6042, 6450, 5160, 6284, 5506, 6212, 7596, 5902] avg=6144.0000 max=7596 max_rank=6 min=5160 min_rank=2 imbalance_ratio=1.236328 max_load_reduction=41556 max_load_reduction_pct=0.845459 moved_to_echo=25216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[39, 35, 3, 0, 0, 2921, 1208, 1508, 516, 242, 0, 0, 0, 0, 0, 0, 31, 6, 110, 169, 2, 40, 19, 1, 5703, 0, 0, 0, 0, 0, 0, 0, 857, 37, 1232, 2980, 273, 736, 170, 48, 0, 0, 0, 0, 0, 0, 0, 0, 1905, 0, 112, 1091, 1, 2494, 24, 949, 0, 0, 0, 0, 0, 0, 0, 0, 12, 600, 1328, 0, 1, 523, 216, 0, 2430, 512, 0, 0, 0, 0, 0, 0, 55, 16, 2459, 178, 1, 72, 1438, 0, 1845, 361, 0, 0, 0, 0, 0, 0, 569, 356, 0, 360, 4094, 182, 28, 352, 0, 0, 0, 0, 0, 0, 0, 0, 2460, 19, 0, 132, 1672, 11, 57, 1, 1350, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6472, 6081, 6333, 6576, 5622, 6425, 5941, 5702] avg=6144.0000 max=6576 max_rank=3 min=5622 min_rank=4 imbalance_ratio=1.070312 max_load_reduction=42576 max_load_reduction_pct=0.866211 moved_to_echo=23690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[20, 150, 17, 0, 0, 3339, 955, 889, 572, 180, 0, 0, 0, 0, 0, 0, 3, 25, 274, 209, 4, 61, 60, 1, 5639, 0, 0, 0, 0, 0, 0, 0, 1154, 15, 1173, 2063, 270, 1296, 148, 32, 0, 0, 0, 0, 0, 0, 0, 0, 834, 1, 24, 1403, 2, 2761, 44, 808, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1009, 1151, 1, 0, 591, 301, 2, 3321, 506, 0, 0, 0, 0, 0, 0, 44, 28, 2598, 108, 1, 305, 966, 0, 2524, 400, 0, 0, 0, 0, 0, 0, 667, 268, 0, 356, 3060, 174, 125, 620, 0, 0, 0, 0, 0, 0, 0, 0, 3124, 42, 0, 119, 714, 5, 42, 1, 1495, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6276, 6151, 5877, 6940, 6974, 5270, 5542] avg=6144.0000 max=6974 max_rank=5 min=5270 min_rank=6 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=24726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[27, 245, 13, 9, 4, 4346, 858, 1432, 385, 270, 0, 0, 0, 0, 0, 0, 26, 189, 248, 171, 5, 76, 2, 2, 5569, 0, 0, 0, 0, 0, 0, 0, 948, 73, 2177, 1307, 267, 650, 134, 42, 0, 0, 0, 0, 0, 0, 0, 0, 1388, 0, 53, 870, 29, 1861, 109, 587, 0, 0, 0, 0, 0, 0, 0, 0, 61, 480, 443, 4, 14, 1320, 268, 1, 2571, 500, 0, 0, 0, 0, 0, 0, 62, 222, 2174, 399, 102, 67, 593, 0, 1953, 269, 0, 0, 0, 0, 0, 0, 778, 418, 0, 674, 4568, 227, 434, 437, 0, 0, 0, 0, 0, 0, 0, 0, 2867, 11, 2, 318, 1445, 24, 63, 5, 1006, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7589, 6288, 5598, 4897, 5662, 5841, 7536, 5741] avg=6144.0000 max=7589 max_rank=0 min=4897 min_rank=3 imbalance_ratio=1.235189 max_load_reduction=41563 max_load_reduction_pct=0.845601 moved_to_echo=24780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[39, 35, 3, 0, 0, 2921, 1208, 1508, 516, 242, 0, 0, 0, 0, 0, 0, 31, 6, 110, 169, 2, 40, 19, 1, 5703, 0, 0, 0, 0, 0, 0, 0, 857, 37, 1232, 2980, 273, 736, 170, 48, 0, 0, 0, 0, 0, 0, 0, 0, 1905, 0, 112, 1091, 1, 2494, 24, 949, 0, 0, 0, 0, 0, 0, 0, 0, 12, 600, 1328, 0, 1, 523, 216, 0, 2430, 512, 0, 0, 0, 0, 0, 0, 55, 16, 2459, 178, 1, 72, 1438, 0, 1845, 361, 0, 0, 0, 0, 0, 0, 569, 356, 0, 360, 4094, 182, 28, 352, 0, 0, 0, 0, 0, 0, 0, 0, 2460, 19, 0, 132, 1672, 11, 57, 1, 1350, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6472, 6081, 6333, 6576, 5622, 6425, 5941, 5702] avg=6144.0000 max=6576 max_rank=3 min=5622 min_rank=4 imbalance_ratio=1.070312 max_load_reduction=42576 max_load_reduction_pct=0.866211 moved_to_echo=23690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[94, 18, 37, 3, 1, 2462, 1611, 567, 646, 225, 0, 0, 0, 0, 0, 0, 20, 2, 272, 294, 1, 19, 22, 0, 4666, 0, 0, 0, 0, 0, 0, 0, 952, 21, 1618, 2111, 224, 872, 29, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 0, 225, 1093, 6, 3118, 84, 532, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1274, 1661, 5, 0, 743, 95, 0, 2661, 418, 0, 0, 0, 0, 0, 0, 21, 22, 2137, 36, 2, 57, 1024, 0, 2022, 451, 0, 0, 0, 0, 0, 0, 323, 126, 0, 547, 3820, 177, 47, 742, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 26, 1, 162, 1533, 3, 149, 1, 1688, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5664, 5296, 5837, 6401, 6866, 5772, 5782, 7534] avg=6144.0000 max=7534 max_rank=7 min=5296 min_rank=1 imbalance_ratio=1.226237 max_load_reduction=41618 max_load_reduction_pct=0.846720 moved_to_echo=25954 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[99, 14, 9, 2, 6, 2620, 1093, 771, 586, 216, 0, 0, 0, 0, 0, 0, 77, 14, 119, 108, 1, 300, 7, 0, 5938, 0, 0, 0, 0, 0, 0, 0, 921, 26, 890, 2536, 284, 247, 84, 40, 0, 0, 0, 0, 0, 0, 0, 0, 2858, 0, 69, 1117, 1, 2828, 36, 607, 0, 0, 0, 0, 0, 0, 0, 0, 7, 439, 1743, 2, 3, 1809, 215, 0, 2680, 533, 0, 0, 0, 0, 0, 0, 53, 13, 1814, 46, 1, 39, 1007, 2, 2037, 409, 0, 0, 0, 0, 0, 0, 727, 224, 0, 533, 3659, 212, 58, 385, 0, 0, 0, 0, 0, 0, 0, 0, 3136, 5, 0, 184, 1072, 0, 51, 0, 1530, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5416, 6564, 5028, 7516, 7431, 5421, 5798, 5978] avg=6144.0000 max=7516 max_rank=3 min=5028 min_rank=2 imbalance_ratio=1.223307 max_load_reduction=41636 max_load_reduction_pct=0.847087 moved_to_echo=24628 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[195, 4, 29, 5, 1, 2577, 708, 2049, 441, 226, 0, 0, 0, 0, 0, 0, 63, 9, 223, 354, 3, 20, 44, 3, 5328, 0, 0, 0, 0, 0, 0, 0, 1744, 36, 867, 3095, 255, 806, 399, 59, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 206, 1268, 18, 2132, 68, 822, 0, 0, 0, 0, 0, 0, 0, 0, 7, 584, 1433, 15, 0, 384, 252, 1, 2548, 478, 0, 0, 0, 0, 0, 0, 172, 12, 2544, 272, 6, 98, 1041, 0, 1936, 308, 0, 0, 0, 0, 0, 0, 493, 76, 0, 280, 3829, 230, 306, 421, 0, 0, 0, 0, 0, 0, 0, 0, 3337, 19, 1, 307, 1312, 1, 136, 1, 1153, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6235, 6047, 7261, 5616, 5702, 6389, 5635, 6267] avg=6144.0000 max=7261 max_rank=2 min=5616 min_rank=3 imbalance_ratio=1.181803 max_load_reduction=41891 max_load_reduction_pct=0.852275 moved_to_echo=23993 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[31, 15, 12, 0, 2, 2367, 1065, 1242, 661, 217, 0, 0, 0, 0, 0, 0, 4, 8, 260, 74, 13, 96, 10, 0, 5685, 0, 0, 0, 0, 0, 0, 0, 2137, 20, 2169, 1807, 268, 1145, 216, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 153, 1191, 3, 3157, 56, 418, 0, 0, 0, 0, 0, 0, 0, 0, 23, 488, 885, 2, 0, 449, 415, 0, 2649, 512, 0, 0, 0, 0, 0, 0, 27, 36, 2427, 185, 5, 105, 859, 0, 2011, 463, 0, 0, 0, 0, 0, 0, 610, 228, 0, 496, 3614, 182, 190, 274, 0, 0, 0, 0, 0, 0, 0, 0, 3059, 11, 5, 120, 1362, 164, 45, 0, 1720, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5612, 6150, 7784, 5985, 5423, 6118, 5594, 6486] avg=6144.0000 max=7784 max_rank=2 min=5423 min_rank=4 imbalance_ratio=1.266927 max_load_reduction=41368 max_load_reduction_pct=0.841634 moved_to_echo=23621 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[27, 245, 13, 9, 4, 4346, 858, 1432, 385, 270, 0, 0, 0, 0, 0, 0, 26, 189, 248, 171, 5, 76, 2, 2, 5569, 0, 0, 0, 0, 0, 0, 0, 948, 73, 2177, 1307, 267, 650, 134, 42, 0, 0, 0, 0, 0, 0, 0, 0, 1388, 0, 53, 870, 29, 1861, 109, 587, 0, 0, 0, 0, 0, 0, 0, 0, 61, 480, 443, 4, 14, 1320, 268, 1, 2571, 500, 0, 0, 0, 0, 0, 0, 62, 222, 2174, 399, 102, 67, 593, 0, 1953, 269, 0, 0, 0, 0, 0, 0, 778, 418, 0, 674, 4568, 227, 434, 437, 0, 0, 0, 0, 0, 0, 0, 0, 2867, 11, 2, 318, 1445, 24, 63, 5, 1006, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7589, 6288, 5598, 4897, 5662, 5841, 7536, 5741] avg=6144.0000 max=7589 max_rank=0 min=4897 min_rank=3 imbalance_ratio=1.235189 max_load_reduction=41563 max_load_reduction_pct=0.845601 moved_to_echo=24780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[99, 14, 9, 2, 6, 2620, 1093, 771, 586, 216, 0, 0, 0, 0, 0, 0, 77, 14, 119, 108, 1, 300, 7, 0, 5938, 0, 0, 0, 0, 0, 0, 0, 921, 26, 890, 2536, 284, 247, 84, 40, 0, 0, 0, 0, 0, 0, 0, 0, 2858, 0, 69, 1117, 1, 2828, 36, 607, 0, 0, 0, 0, 0, 0, 0, 0, 7, 439, 1743, 2, 3, 1809, 215, 0, 2680, 533, 0, 0, 0, 0, 0, 0, 53, 13, 1814, 46, 1, 39, 1007, 2, 2037, 409, 0, 0, 0, 0, 0, 0, 727, 224, 0, 533, 3659, 212, 58, 385, 0, 0, 0, 0, 0, 0, 0, 0, 3136, 5, 0, 184, 1072, 0, 51, 0, 1530, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5416, 6564, 5028, 7516, 7431, 5421, 5798, 5978] avg=6144.0000 max=7516 max_rank=3 min=5028 min_rank=2 imbalance_ratio=1.223307 max_load_reduction=41636 max_load_reduction_pct=0.847087 moved_to_echo=24628 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[28, 5, 11, 0, 1, 3435, 733, 1045, 493, 291, 0, 0, 0, 0, 0, 0, 150, 0, 224, 137, 0, 21, 1, 0, 5917, 0, 0, 0, 0, 0, 0, 0, 1158, 120, 948, 2013, 283, 375, 89, 174, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 98, 1218, 1, 2381, 41, 478, 0, 0, 0, 0, 0, 0, 0, 0, 5, 396, 826, 0, 0, 1122, 131, 0, 2495, 531, 0, 0, 0, 0, 0, 0, 20, 19, 3076, 86, 0, 148, 623, 0, 1896, 344, 0, 0, 0, 0, 0, 0, 923, 477, 0, 413, 4933, 151, 96, 603, 0, 0, 0, 0, 0, 0, 0, 0, 3256, 31, 0, 128, 1112, 1, 86, 0, 1288, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6042, 6450, 5160, 6284, 5506, 6212, 7596, 5902] avg=6144.0000 max=7596 max_rank=6 min=5160 min_rank=2 imbalance_ratio=1.236328 max_load_reduction=41556 max_load_reduction_pct=0.845459 moved_to_echo=25216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 384, 19328, 1408, 128, 896, 7552, 128, 16256, 3072, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[640, 512, 256, 128, 128, 24192, 8320, 9600, 4352, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9984, 384, 11136, 17920, 2176, 6144, 1280, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 5376, 9472, 128, 128, 7040, 1920, 128, 21376, 4096, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[25216, 256, 128, 1536, 10240, 256, 640, 128, 11264, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9984, 384, 11136, 17920, 2176, 6144, 1280, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12544, 128, 1024, 9344, 128, 20736, 512, 5248, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[5120, 2176, 0, 3712, 31616, 1536, 1408, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[25216, 256, 128, 1536, 10240, 256, 640, 128, 11264, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[384, 256, 1792, 1536, 128, 640, 256, 128, 44544, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[12, 454, 0, 1, 447, 89, 82, 2, 24, 42, 4163, 0, 1408, 3226, 1706, 0, 102, 0, 268, 68, 0, 0, 0, 1327, 574, 630, 5175, 1, 7, 37, 0, 22, 282, 1, 201, 575, 0, 2, 1070, 40, 1514, 1724, 96, 7206, 30, 0, 1575, 5806, 746, 18, 144, 189, 152, 5, 227, 2592, 1497, 768, 51, 2088, 14, 0, 468, 204] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[116, 138, 6, 8, 604, 465, 104, 2, 36, 74, 4136, 27, 806, 1732, 553, 2, 190, 0, 247, 76, 4, 1, 1, 2381, 419, 428, 5831, 0, 0, 6, 0, 19, 260, 2, 648, 756, 0, 1, 1222, 25, 3401, 4347, 552, 6564, 126, 0, 1351, 3777, 878, 1, 180, 135, 563, 3, 68, 1467, 1724, 1489, 141, 629, 4, 0, 318, 108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[31, 219, 0, 2, 586, 124, 299, 5, 11, 99, 3458, 0, 1581, 1791, 1853, 0, 168, 1, 92, 17, 3, 1, 0, 1545, 700, 746, 5836, 0, 1, 11, 8, 31, 251, 0, 652, 678, 0, 1, 1562, 4, 1261, 2552, 683, 7277, 29, 0, 1780, 4376, 1132, 37, 47, 172, 344, 0, 453, 2128, 1058, 735, 102, 2207, 9, 1, 286, 116] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[100, 121, 0, 0, 609, 206, 132, 0, 66, 132, 3281, 8, 1199, 2580, 788, 1, 243, 6, 763, 138, 0, 4, 0, 2686, 448, 491, 5725, 0, 0, 26, 0, 3, 214, 3, 338, 669, 0, 0, 1169, 14, 2605, 1948, 139, 7352, 16, 0, 936, 4056, 1828, 11, 137, 139, 146, 4, 542, 2960, 1001, 974, 33, 1831, 3, 2, 257, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[11, 206, 0, 0, 621, 288, 67, 0, 61, 55, 3593, 0, 1503, 1899, 1169, 0, 161, 0, 349, 33, 0, 0, 0, 1822, 785, 742, 4388, 0, 0, 62, 1, 13, 254, 0, 251, 779, 0, 0, 2000, 33, 2471, 2307, 254, 7324, 37, 0, 1860, 3751, 1715, 22, 241, 103, 350, 4, 377, 1483, 1451, 1993, 70, 1748, 10, 0, 380, 55] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[60, 68, 0, 7, 490, 114, 60, 6, 21, 151, 2769, 2, 1600, 2237, 1170, 0, 96, 15, 153, 56, 13, 3, 0, 2134, 392, 578, 6036, 3, 1, 26, 9, 4, 278, 0, 412, 855, 0, 22, 1327, 194, 1876, 3065, 610, 6859, 6, 0, 1123, 5191, 1079, 19, 269, 165, 408, 4, 134, 1839, 1741, 1036, 217, 1773, 5, 0, 177, 194] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[56, 451, 3, 10, 1603, 695, 63, 73, 182, 70, 3677, 2, 1260, 1837, 781, 4, 255, 3, 941, 94, 8, 0, 0, 1502, 663, 925, 5899, 5, 2, 61, 0, 10, 314, 5, 534, 686, 2, 21, 1057, 124, 2316, 1999, 374, 6570, 70, 0, 2254, 4347, 1134, 12, 248, 145, 232, 13, 288, 1250, 1218, 953, 33, 1310, 15, 7, 336, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[100, 121, 0, 0, 609, 206, 132, 0, 66, 132, 3281, 8, 1199, 2580, 788, 1, 243, 6, 763, 138, 0, 4, 0, 2686, 448, 491, 5725, 0, 0, 26, 0, 3, 214, 3, 338, 669, 0, 0, 1169, 14, 2605, 1948, 139, 7352, 16, 0, 936, 4056, 1828, 11, 137, 139, 146, 4, 542, 2960, 1001, 974, 33, 1831, 3, 2, 257, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[116, 138, 6, 8, 604, 465, 104, 2, 36, 74, 4136, 27, 806, 1732, 553, 2, 190, 0, 247, 76, 4, 1, 1, 2381, 419, 428, 5831, 0, 0, 6, 0, 19, 260, 2, 648, 756, 0, 1, 1222, 25, 3401, 4347, 552, 6564, 126, 0, 1351, 3777, 878, 1, 180, 135, 563, 3, 68, 1467, 1724, 1489, 141, 629, 4, 0, 318, 108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4, 177, 0, 0, 772, 646, 170, 5, 24, 59, 4448, 0, 997, 1455, 1014, 0, 265, 5, 281, 83, 14, 0, 0, 1647, 290, 570, 6495, 0, 0, 6, 3, 10, 315, 0, 859, 946, 0, 39, 1450, 152, 1295, 1903, 885, 7304, 24, 0, 1392, 4388, 804, 2, 50, 36, 530, 2, 177, 2266, 885, 1292, 141, 1802, 141, 0, 331, 301] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[31, 219, 0, 2, 586, 124, 299, 5, 11, 99, 3458, 0, 1581, 1791, 1853, 0, 168, 1, 92, 17, 3, 1, 0, 1545, 700, 746, 5836, 0, 1, 11, 8, 31, 251, 0, 652, 678, 0, 1, 1562, 4, 1261, 2552, 683, 7277, 29, 0, 1780, 4376, 1132, 37, 47, 172, 344, 0, 453, 2128, 1058, 735, 102, 2207, 9, 1, 286, 116] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[12, 454, 0, 1, 447, 89, 82, 2, 4782, 0, 0, 0, 0, 0, 0, 0, 24, 42, 1672, 0, 1408, 3226, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 268, 68, 0, 0, 0, 1327, 2424, 1611, 0, 0, 0, 0, 0, 0, 574, 630, 4491, 1, 7, 37, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 282, 1, 201, 575, 0, 2, 1070, 40, 4164, 0, 0, 0, 0, 0, 0, 0, 1514, 1409, 96, 0, 30, 0, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 18, 144, 189, 152, 5, 227, 2592, 2271, 31, 0, 0, 0, 0, 0, 0, 1497, 768, 51, 2088, 14, 0, 468, 204, 684, 315, 220, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5869, 8078, 5800, 5762, 6335, 4624, 6375, 6309] avg=6144.0000 max=8078 max_rank=1 min=4624 min_rank=5 imbalance_ratio=1.314779 max_load_reduction=41074 max_load_reduction_pct=0.835653 moved_to_echo=23643 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[60, 68, 0, 7, 490, 114, 60, 6, 4551, 0, 0, 0, 0, 0, 0, 0, 21, 151, 1113, 2, 1600, 2237, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 15, 153, 56, 13, 3, 0, 2134, 2308, 1438, 0, 0, 0, 0, 0, 0, 392, 578, 5238, 3, 1, 26, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 412, 855, 0, 22, 1327, 194, 3723, 0, 0, 0, 0, 0, 0, 0, 1876, 2504, 610, 0, 6, 0, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 19, 269, 165, 408, 4, 134, 1839, 1510, 30, 0, 0, 0, 0, 0, 0, 1741, 1036, 217, 1773, 5, 0, 177, 194, 798, 561, 146, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5356, 6294, 6216, 6251, 6811, 6119, 5457, 6648] avg=6144.0000 max=6811 max_rank=4 min=5356 min_rank=0 imbalance_ratio=1.108561 max_load_reduction=42341 max_load_reduction_pct=0.861430 moved_to_echo=25035 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[12, 454, 0, 1, 447, 89, 82, 2, 24, 42, 4163, 0, 1408, 3226, 1706, 0, 102, 0, 268, 68, 0, 0, 0, 1327, 574, 630, 5175, 1, 7, 37, 0, 22, 282, 1, 201, 575, 0, 2, 1070, 40, 1514, 1724, 96, 7206, 30, 0, 1575, 5806, 746, 18, 144, 189, 152, 5, 227, 2592, 1497, 768, 51, 2088, 14, 0, 468, 204] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[60, 68, 0, 7, 490, 114, 60, 6, 21, 151, 2769, 2, 1600, 2237, 1170, 0, 96, 15, 153, 56, 13, 3, 0, 2134, 392, 578, 6036, 3, 1, 26, 9, 4, 278, 0, 412, 855, 0, 22, 1327, 194, 1876, 3065, 610, 6859, 6, 0, 1123, 5191, 1079, 19, 269, 165, 408, 4, 134, 1839, 1741, 1036, 217, 1773, 5, 0, 177, 194] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[31, 219, 0, 2, 586, 124, 299, 5, 4828, 0, 0, 0, 0, 0, 0, 0, 11, 99, 1389, 0, 1581, 1791, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 1, 92, 17, 3, 1, 0, 1545, 2449, 1212, 0, 0, 0, 0, 0, 0, 700, 746, 5065, 0, 1, 11, 8, 31, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 652, 678, 0, 1, 1562, 4, 3139, 0, 0, 0, 0, 0, 0, 0, 1261, 2085, 683, 0, 29, 0, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 37, 47, 172, 344, 0, 453, 2128, 1886, 25, 0, 0, 0, 0, 0, 0, 1058, 735, 102, 2207, 9, 1, 286, 116, 771, 467, 183, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 6724, 5488, 6562, 6287, 5838, 6224, 5935] avg=6144.0000 max=6724 max_rank=1 min=5488 min_rank=2 imbalance_ratio=1.094401 max_load_reduction=42428 max_load_reduction_pct=0.863200 moved_to_echo=24284 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[56, 451, 3, 10, 1603, 695, 63, 73, 4360, 0, 0, 0, 0, 0, 0, 0, 182, 70, 1478, 2, 1260, 1837, 781, 4, 0, 0, 0, 0, 0, 0, 0, 0, 255, 3, 941, 94, 8, 0, 0, 1502, 2210, 1205, 0, 0, 0, 0, 0, 0, 663, 925, 5119, 5, 2, 61, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 314, 5, 534, 686, 2, 21, 1057, 124, 3119, 0, 0, 0, 0, 0, 0, 0, 2316, 1633, 374, 0, 70, 0, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 12, 248, 145, 232, 13, 288, 1250, 2005, 23, 0, 0, 0, 0, 0, 0, 1218, 953, 33, 1310, 15, 7, 336, 150, 780, 366, 194, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7314, 5614, 6218, 6785, 5862, 6647, 5350, 5362] avg=6144.0000 max=7314 max_rank=0 min=5350 min_rank=6 imbalance_ratio=1.190430 max_load_reduction=41838 max_load_reduction_pct=0.851196 moved_to_echo=23221 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[11, 206, 0, 0, 621, 288, 67, 0, 4859, 0, 0, 0, 0, 0, 0, 0, 61, 55, 1443, 0, 1503, 1899, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 349, 33, 0, 0, 0, 1822, 2465, 1039, 0, 0, 0, 0, 0, 0, 785, 742, 3808, 0, 0, 62, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 251, 779, 0, 0, 2000, 33, 2691, 0, 0, 0, 0, 0, 0, 0, 2471, 1885, 254, 0, 37, 0, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 22, 241, 103, 350, 4, 377, 1483, 1960, 21, 0, 0, 0, 0, 0, 0, 1451, 1993, 70, 1748, 10, 0, 380, 55, 580, 422, 190, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6052, 6130, 5869, 5411, 6008, 6507, 6276, 6899] avg=6144.0000 max=6899 max_rank=7 min=5411 min_rank=3 imbalance_ratio=1.122884 max_load_reduction=42253 max_load_reduction_pct=0.859639 moved_to_echo=25690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 177, 0, 0, 772, 646, 170, 5, 4847, 0, 0, 0, 0, 0, 0, 0, 24, 59, 1787, 0, 997, 1455, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 5, 281, 83, 14, 0, 0, 1647, 2457, 1216, 0, 0, 0, 0, 0, 0, 290, 570, 5637, 0, 0, 6, 3, 10, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 859, 946, 0, 39, 1450, 152, 3148, 0, 0, 0, 0, 0, 0, 0, 1295, 1555, 885, 0, 24, 0, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 2, 50, 36, 530, 2, 177, 2266, 2426, 24, 0, 0, 0, 0, 0, 0, 885, 1292, 141, 1802, 141, 0, 331, 301, 858, 348, 235, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6621, 5336, 5968, 6516, 6909, 5151, 6317, 6334] avg=6144.0000 max=6909 max_rank=4 min=5151 min_rank=5 imbalance_ratio=1.124512 max_load_reduction=42243 max_load_reduction_pct=0.859436 moved_to_echo=24711 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[100, 121, 0, 0, 609, 206, 132, 0, 4879, 0, 0, 0, 0, 0, 0, 0, 66, 132, 1319, 8, 1199, 2580, 788, 1, 0, 0, 0, 0, 0, 0, 0, 0, 243, 6, 763, 138, 0, 4, 0, 2686, 2473, 1124, 0, 0, 0, 0, 0, 0, 448, 491, 4968, 0, 0, 26, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 214, 3, 338, 669, 0, 0, 1169, 14, 2910, 0, 0, 0, 0, 0, 0, 0, 2605, 1592, 139, 0, 16, 0, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 11, 137, 139, 146, 4, 542, 2960, 1789, 22, 0, 0, 0, 0, 0, 0, 1001, 974, 33, 1831, 3, 2, 257, 69, 757, 356, 173, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6047, 6093, 7437, 5936, 5317, 5288, 7578, 5456] avg=6144.0000 max=7578 max_rank=6 min=5288 min_rank=5 imbalance_ratio=1.233398 max_load_reduction=41574 max_load_reduction_pct=0.845825 moved_to_echo=23639 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[60, 68, 0, 7, 490, 114, 60, 6, 4551, 0, 0, 0, 0, 0, 0, 0, 21, 151, 1113, 2, 1600, 2237, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 15, 153, 56, 13, 3, 0, 2134, 2308, 1438, 0, 0, 0, 0, 0, 0, 392, 578, 5238, 3, 1, 26, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 412, 855, 0, 22, 1327, 194, 3723, 0, 0, 0, 0, 0, 0, 0, 1876, 2504, 610, 0, 6, 0, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 19, 269, 165, 408, 4, 134, 1839, 1510, 30, 0, 0, 0, 0, 0, 0, 1741, 1036, 217, 1773, 5, 0, 177, 194, 798, 561, 146, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5356, 6294, 6216, 6251, 6811, 6119, 5457, 6648] avg=6144.0000 max=6811 max_rank=4 min=5356 min_rank=0 imbalance_ratio=1.108561 max_load_reduction=42341 max_load_reduction_pct=0.861430 moved_to_echo=25035 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[12, 454, 0, 1, 447, 89, 82, 2, 4782, 0, 0, 0, 0, 0, 0, 0, 24, 42, 1672, 0, 1408, 3226, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 268, 68, 0, 0, 0, 1327, 2424, 1611, 0, 0, 0, 0, 0, 0, 574, 630, 4491, 1, 7, 37, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 282, 1, 201, 575, 0, 2, 1070, 40, 4164, 0, 0, 0, 0, 0, 0, 0, 1514, 1409, 96, 0, 30, 0, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 18, 144, 189, 152, 5, 227, 2592, 2271, 31, 0, 0, 0, 0, 0, 0, 1497, 768, 51, 2088, 14, 0, 468, 204, 684, 315, 220, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5869, 8078, 5800, 5762, 6335, 4624, 6375, 6309] avg=6144.0000 max=8078 max_rank=1 min=4624 min_rank=5 imbalance_ratio=1.314779 max_load_reduction=41074 max_load_reduction_pct=0.835653 moved_to_echo=23643 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[116, 138, 6, 8, 604, 465, 104, 2, 4356, 0, 0, 0, 0, 0, 0, 0, 36, 74, 1653, 27, 806, 1732, 553, 2, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 247, 76, 4, 1, 1, 2381, 2208, 1048, 0, 0, 0, 0, 0, 0, 419, 428, 5057, 0, 0, 6, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 260, 2, 648, 756, 0, 1, 1222, 25, 2709, 0, 0, 0, 0, 0, 0, 0, 3401, 3548, 552, 0, 126, 0, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 1, 180, 135, 563, 3, 68, 1467, 2260, 20, 0, 0, 0, 0, 0, 0, 1724, 1489, 141, 629, 4, 0, 318, 108, 774, 799, 223, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5799, 4883, 6156, 5929, 5623, 8978, 5575, 6209] avg=6144.0000 max=8978 max_rank=5 min=4883 min_rank=1 imbalance_ratio=1.461263 max_load_reduction=40174 max_load_reduction_pct=0.817342 moved_to_echo=26385 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[31, 219, 0, 2, 586, 124, 299, 5, 4828, 0, 0, 0, 0, 0, 0, 0, 11, 99, 1389, 0, 1581, 1791, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 1, 92, 17, 3, 1, 0, 1545, 2449, 1212, 0, 0, 0, 0, 0, 0, 700, 746, 5065, 0, 1, 11, 8, 31, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 652, 678, 0, 1, 1562, 4, 3139, 0, 0, 0, 0, 0, 0, 0, 1261, 2085, 683, 0, 29, 0, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 37, 47, 172, 344, 0, 453, 2128, 1886, 25, 0, 0, 0, 0, 0, 0, 1058, 735, 102, 2207, 9, 1, 286, 116, 771, 467, 183, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 6724, 5488, 6562, 6287, 5838, 6224, 5935] avg=6144.0000 max=6724 max_rank=1 min=5488 min_rank=2 imbalance_ratio=1.094401 max_load_reduction=42428 max_load_reduction_pct=0.863200 moved_to_echo=24284 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[56, 451, 3, 10, 1603, 695, 63, 73, 4360, 0, 0, 0, 0, 0, 0, 0, 182, 70, 1478, 2, 1260, 1837, 781, 4, 0, 0, 0, 0, 0, 0, 0, 0, 255, 3, 941, 94, 8, 0, 0, 1502, 2210, 1205, 0, 0, 0, 0, 0, 0, 663, 925, 5119, 5, 2, 61, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 314, 5, 534, 686, 2, 21, 1057, 124, 3119, 0, 0, 0, 0, 0, 0, 0, 2316, 1633, 374, 0, 70, 0, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 12, 248, 145, 232, 13, 288, 1250, 2005, 23, 0, 0, 0, 0, 0, 0, 1218, 953, 33, 1310, 15, 7, 336, 150, 780, 366, 194, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7314, 5614, 6218, 6785, 5862, 6647, 5350, 5362] avg=6144.0000 max=7314 max_rank=0 min=5350 min_rank=6 imbalance_ratio=1.190430 max_load_reduction=41838 max_load_reduction_pct=0.851196 moved_to_echo=23221 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[10624, 9344, 896, 13440, 256, 128, 2560, 1280, 6016, 3712, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9344, 128, 1408, 1152, 2816, 128, 2304, 16000, 16128, 256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[16768, 16256, 3712, 0, 384, 0, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2176, 128, 3968, 6016, 128, 128, 10880, 640, 25728, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[1536, 128, 3200, 640, 128, 128, 128, 15104, 19072, 9984, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 768, 11904, 128, 10368, 16768, 9088, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9344, 128, 1408, 1152, 2816, 128, 2304, 16000, 16128, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[1536, 128, 3200, 640, 128, 128, 128, 15104, 19072, 9984, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2176, 128, 3968, 6016, 128, 128, 10880, 640, 25728, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 1920, 128, 128, 5760, 2688, 1024, 128, 37504, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 5120, 39424, 128, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[2952, 1, 5, 1514, 24, 1, 3409, 1202, 6596, 7561, 123, 19, 0, 113, 13, 266, 398, 15, 1216, 145, 106, 744, 23, 1, 804, 957, 12, 1, 441, 2, 106, 2691, 0, 2, 66, 184, 12, 167, 8, 1, 105, 355, 428, 625, 1967, 501, 83, 308, 2310, 10, 1, 166, 5, 510, 0, 18, 551, 90, 31, 1, 5, 2158, 3605, 3418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[2567, 1, 0, 590, 47, 1, 3863, 466, 5615, 8116, 97, 0, 0, 274, 19, 176, 193, 3, 1937, 232, 517, 876, 51, 1, 1338, 1379, 49, 7, 81, 0, 60, 509, 0, 2, 64, 440, 1, 219, 4, 1, 431, 542, 249, 695, 1858, 918, 404, 229, 2617, 0, 2, 8, 7, 267, 0, 1, 944, 50, 59, 1, 4, 2100, 3792, 4178] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[2804, 3, 1, 727, 20, 49, 3607, 371, 4728, 7734, 84, 6, 0, 126, 8, 298, 185, 20, 1785, 437, 810, 222, 213, 3, 331, 1754, 3, 9, 77, 1, 43, 455, 0, 3, 64, 570, 1, 230, 46, 1, 327, 204, 241, 1249, 3848, 1521, 263, 159, 2346, 4, 1, 90, 22, 283, 1, 4, 1176, 73, 52, 1, 38, 1225, 4516, 3679] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[2717, 7, 9, 645, 80, 14, 4438, 906, 4784, 7271, 106, 6, 0, 57, 109, 80, 224, 2, 1451, 443, 305, 200, 50, 0, 480, 1270, 35, 0, 140, 9, 104, 972, 0, 7, 172, 1115, 8, 105, 21, 1, 465, 396, 181, 1696, 2623, 738, 181, 456, 4182, 4, 1, 46, 107, 129, 0, 3, 731, 76, 54, 2, 20, 1982, 3168, 3568] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[3587, 0, 2, 714, 17, 2, 3047, 344, 5981, 7361, 144, 4, 15, 235, 7, 150, 396, 7, 1499, 137, 422, 221, 226, 13, 640, 729, 25, 31, 44, 2, 88, 268, 0, 23, 198, 186, 4, 309, 7, 2, 279, 371, 102, 750, 2928, 1602, 223, 545, 3026, 10, 6, 11, 11, 283, 0, 5, 1133, 264, 44, 2, 11, 2291, 3999, 4169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[3025, 2, 4, 676, 79, 8, 2335, 475, 5359, 7369, 126, 11, 0, 149, 42, 140, 207, 7, 868, 213, 485, 407, 27, 4, 421, 1241, 37, 32, 538, 0, 73, 644, 0, 3, 60, 195, 0, 443, 1, 0, 661, 526, 295, 1296, 3462, 1335, 200, 1136, 4863, 0, 14, 30, 7, 228, 0, 2, 790, 93, 37, 0, 9, 2071, 2664, 3727] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[3174, 15, 0, 856, 26, 2, 3611, 450, 5510, 7660, 76, 4, 0, 67, 3, 130, 278, 4, 1807, 382, 239, 199, 37, 5, 474, 1618, 32, 17, 239, 0, 53, 394, 0, 15, 270, 497, 1, 625, 15, 0, 257, 661, 297, 842, 2280, 1096, 366, 407, 4363, 0, 5, 10, 3, 196, 0, 5, 700, 41, 92, 10, 3, 1361, 3829, 3543] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[2567, 1, 0, 590, 47, 1, 3863, 466, 5615, 8116, 97, 0, 0, 274, 19, 176, 193, 3, 1937, 232, 517, 876, 51, 1, 1338, 1379, 49, 7, 81, 0, 60, 509, 0, 2, 64, 440, 1, 219, 4, 1, 431, 542, 249, 695, 1858, 918, 404, 229, 2617, 0, 2, 8, 7, 267, 0, 1, 944, 50, 59, 1, 4, 2100, 3792, 4178] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[3587, 0, 2, 714, 17, 2, 3047, 344, 5981, 7361, 144, 4, 15, 235, 7, 150, 396, 7, 1499, 137, 422, 221, 226, 13, 640, 729, 25, 31, 44, 2, 88, 268, 0, 23, 198, 186, 4, 309, 7, 2, 279, 371, 102, 750, 2928, 1602, 223, 545, 3026, 10, 6, 11, 11, 283, 0, 5, 1133, 264, 44, 2, 11, 2291, 3999, 4169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[2952, 1, 5, 1514, 24, 1, 3409, 1202, 6596, 7561, 123, 19, 0, 113, 13, 266, 398, 15, 1216, 145, 106, 744, 23, 1, 804, 957, 12, 1, 441, 2, 106, 2691, 0, 2, 66, 184, 12, 167, 8, 1, 105, 355, 428, 625, 1967, 501, 83, 308, 2310, 10, 1, 166, 5, 510, 0, 18, 551, 90, 31, 1, 5, 2158, 3605, 3418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[2882, 19, 28, 998, 454, 25, 3965, 311, 5184, 7048, 177, 0, 2, 49, 31, 346, 328, 11, 1304, 355, 326, 138, 15, 3, 1173, 1005, 41, 17, 114, 24, 58, 474, 0, 25, 390, 580, 14, 610, 10, 1, 747, 992, 166, 705, 1985, 1888, 458, 653, 2822, 220, 2, 1, 48, 477, 0, 54, 1011, 637, 93, 0, 89, 1319, 3442, 2808] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[3025, 2, 4, 676, 79, 8, 2335, 475, 5359, 7369, 126, 11, 0, 149, 42, 140, 207, 7, 868, 213, 485, 407, 27, 4, 421, 1241, 37, 32, 538, 0, 73, 644, 0, 3, 60, 195, 0, 443, 1, 0, 661, 526, 295, 1296, 3462, 1335, 200, 1136, 4863, 0, 14, 30, 7, 228, 0, 2, 790, 93, 37, 0, 9, 2071, 2664, 3727] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[2804, 3, 1, 727, 20, 49, 3607, 371, 4728, 7734, 84, 6, 0, 126, 8, 298, 185, 20, 1785, 437, 810, 222, 213, 3, 331, 1754, 3, 9, 77, 1, 43, 455, 0, 3, 64, 570, 1, 230, 46, 1, 327, 204, 241, 1249, 3848, 1521, 263, 159, 2346, 4, 1, 90, 22, 283, 1, 4, 1176, 73, 52, 1, 38, 1225, 4516, 3679] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2717, 7, 9, 645, 80, 14, 2081, 906, 0, 0, 0, 0, 0, 0, 0, 0, 4784, 183, 106, 6, 0, 57, 109, 80, 0, 0, 0, 0, 0, 0, 0, 0, 224, 2, 1451, 443, 305, 200, 50, 0, 2111, 1019, 0, 0, 0, 0, 0, 0, 480, 1270, 35, 0, 140, 9, 104, 972, 2549, 582, 0, 0, 0, 0, 0, 0, 0, 7, 172, 1115, 8, 105, 21, 1, 4977, 0, 0, 0, 0, 0, 0, 0, 465, 396, 181, 1696, 2047, 738, 181, 456, 0, 0, 0, 0, 0, 0, 0, 0, 4182, 4, 1, 46, 107, 129, 0, 3, 1775, 576, 369, 0, 0, 0, 0, 0, 731, 76, 54, 2, 20, 1982, 2799, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6459, 5325, 5805, 6141, 6406, 6160, 7192, 5664] avg=6144.0000 max=7192 max_rank=6 min=5325 min_rank=1 imbalance_ratio=1.170573 max_load_reduction=41960 max_load_reduction_pct=0.853678 moved_to_echo=25422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3025, 2, 4, 676, 79, 8, 1095, 475, 0, 0, 0, 0, 0, 0, 0, 0, 5359, 185, 126, 11, 0, 149, 42, 140, 0, 0, 0, 0, 0, 0, 0, 0, 207, 7, 868, 213, 485, 407, 27, 4, 2140, 1065, 0, 0, 0, 0, 0, 0, 421, 1241, 37, 32, 538, 0, 73, 644, 2662, 306, 0, 0, 0, 0, 0, 0, 0, 3, 60, 195, 0, 443, 1, 0, 5044, 0, 0, 0, 0, 0, 0, 0, 661, 526, 295, 1296, 2702, 1335, 200, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 4863, 0, 14, 30, 7, 228, 0, 2, 934, 760, 311, 0, 0, 0, 0, 0, 790, 93, 37, 0, 9, 2071, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5364, 6012, 5423, 5954, 5746, 8151, 7149, 5353] avg=6144.0000 max=8151 max_rank=5 min=5353 min_rank=7 imbalance_ratio=1.326660 max_load_reduction=41001 max_load_reduction_pct=0.834167 moved_to_echo=26399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2567, 1, 0, 590, 47, 1, 1812, 466, 0, 0, 0, 0, 0, 0, 0, 0, 5615, 204, 97, 0, 0, 274, 19, 176, 0, 0, 0, 0, 0, 0, 0, 0, 193, 3, 1937, 232, 517, 876, 51, 1, 2357, 1194, 0, 0, 0, 0, 0, 0, 1338, 1379, 49, 7, 81, 0, 60, 509, 2984, 506, 0, 0, 0, 0, 0, 0, 0, 2, 64, 440, 1, 219, 4, 1, 5555, 0, 0, 0, 0, 0, 0, 0, 431, 542, 249, 695, 1450, 918, 404, 229, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 2, 8, 7, 267, 0, 1, 1545, 408, 442, 0, 0, 0, 0, 0, 944, 50, 59, 1, 4, 2100, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5484, 6385, 7361, 6913, 6286, 4918, 5297, 6508] avg=6144.0000 max=7361 max_rank=2 min=4918 min_rank=5 imbalance_ratio=1.198079 max_load_reduction=41791 max_load_reduction_pct=0.850240 moved_to_echo=23009 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3587, 0, 2, 714, 17, 2, 1429, 344, 0, 0, 0, 0, 0, 0, 0, 0, 5981, 186, 144, 4, 15, 235, 7, 150, 0, 0, 0, 0, 0, 0, 0, 0, 396, 7, 1499, 137, 422, 221, 226, 13, 2137, 1191, 0, 0, 0, 0, 0, 0, 640, 729, 25, 31, 44, 2, 88, 268, 2978, 399, 0, 0, 0, 0, 0, 0, 0, 23, 198, 186, 4, 309, 7, 2, 5038, 0, 0, 0, 0, 0, 0, 0, 279, 371, 102, 750, 2285, 1602, 223, 545, 0, 0, 0, 0, 0, 0, 0, 0, 3026, 10, 6, 11, 11, 283, 0, 5, 1219, 643, 466, 0, 0, 0, 0, 0, 1133, 264, 44, 2, 11, 2291, 3533, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6722, 6249, 5204, 5767, 6157, 5680, 7278] avg=6144.0000 max=7278 max_rank=7 min=5204 min_rank=3 imbalance_ratio=1.184570 max_load_reduction=41874 max_load_reduction_pct=0.851929 moved_to_echo=24882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2952, 1, 5, 1514, 24, 1, 1599, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 6596, 191, 123, 19, 0, 113, 13, 266, 0, 0, 0, 0, 0, 0, 0, 0, 398, 15, 1216, 145, 106, 744, 23, 1, 2195, 977, 0, 0, 0, 0, 0, 0, 804, 957, 12, 1, 441, 2, 106, 2691, 2441, 447, 0, 0, 0, 0, 0, 0, 0, 2, 66, 184, 12, 167, 8, 1, 5175, 0, 0, 0, 0, 0, 0, 0, 105, 355, 428, 625, 1535, 501, 83, 308, 0, 0, 0, 0, 0, 0, 0, 0, 2310, 10, 1, 166, 5, 510, 0, 18, 1363, 432, 420, 0, 0, 0, 0, 0, 551, 90, 31, 1, 5, 2158, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7298, 7321, 5820, 7902, 5615, 3940, 5235, 6021] avg=6144.0000 max=7902 max_rank=3 min=3940 min_rank=5 imbalance_ratio=1.286133 max_load_reduction=41250 max_load_reduction_pct=0.839233 moved_to_echo=20811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2717, 7, 9, 645, 80, 14, 2081, 906, 0, 0, 0, 0, 0, 0, 0, 0, 4784, 183, 106, 6, 0, 57, 109, 80, 0, 0, 0, 0, 0, 0, 0, 0, 224, 2, 1451, 443, 305, 200, 50, 0, 2111, 1019, 0, 0, 0, 0, 0, 0, 480, 1270, 35, 0, 140, 9, 104, 972, 2549, 582, 0, 0, 0, 0, 0, 0, 0, 7, 172, 1115, 8, 105, 21, 1, 4977, 0, 0, 0, 0, 0, 0, 0, 465, 396, 181, 1696, 2047, 738, 181, 456, 0, 0, 0, 0, 0, 0, 0, 0, 4182, 4, 1, 46, 107, 129, 0, 3, 1775, 576, 369, 0, 0, 0, 0, 0, 731, 76, 54, 2, 20, 1982, 2799, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6459, 5325, 5805, 6141, 6406, 6160, 7192, 5664] avg=6144.0000 max=7192 max_rank=6 min=5325 min_rank=1 imbalance_ratio=1.170573 max_load_reduction=41960 max_load_reduction_pct=0.853678 moved_to_echo=25422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3174, 15, 0, 856, 26, 2, 1694, 450, 0, 0, 0, 0, 0, 0, 0, 0, 5510, 193, 76, 4, 0, 67, 3, 130, 0, 0, 0, 0, 0, 0, 0, 0, 278, 4, 1807, 382, 239, 199, 37, 5, 2224, 1013, 0, 0, 0, 0, 0, 0, 474, 1618, 32, 17, 239, 0, 53, 394, 2530, 473, 0, 0, 0, 0, 0, 0, 0, 15, 270, 497, 1, 625, 15, 0, 5243, 0, 0, 0, 0, 0, 0, 0, 257, 661, 297, 842, 1779, 1096, 366, 407, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 0, 5, 10, 3, 196, 0, 5, 1444, 501, 447, 0, 0, 0, 0, 0, 700, 41, 92, 10, 3, 1361, 3382, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6217, 5983, 6188, 5830, 6666, 5705, 6974, 5589] avg=6144.0000 max=6974 max_rank=6 min=5589 min_rank=7 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=24934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2567, 1, 0, 590, 47, 1, 1812, 466, 0, 0, 0, 0, 0, 0, 0, 0, 5615, 204, 97, 0, 0, 274, 19, 176, 0, 0, 0, 0, 0, 0, 0, 0, 193, 3, 1937, 232, 517, 876, 51, 1, 2357, 1194, 0, 0, 0, 0, 0, 0, 1338, 1379, 49, 7, 81, 0, 60, 509, 2984, 506, 0, 0, 0, 0, 0, 0, 0, 2, 64, 440, 1, 219, 4, 1, 5555, 0, 0, 0, 0, 0, 0, 0, 431, 542, 249, 695, 1450, 918, 404, 229, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 2, 8, 7, 267, 0, 1, 1545, 408, 442, 0, 0, 0, 0, 0, 944, 50, 59, 1, 4, 2100, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5484, 6385, 7361, 6913, 6286, 4918, 5297, 6508] avg=6144.0000 max=7361 max_rank=2 min=4918 min_rank=5 imbalance_ratio=1.198079 max_load_reduction=41791 max_load_reduction_pct=0.850240 moved_to_echo=23009 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3025, 2, 4, 676, 79, 8, 1095, 475, 0, 0, 0, 0, 0, 0, 0, 0, 5359, 185, 126, 11, 0, 149, 42, 140, 0, 0, 0, 0, 0, 0, 0, 0, 207, 7, 868, 213, 485, 407, 27, 4, 2140, 1065, 0, 0, 0, 0, 0, 0, 421, 1241, 37, 32, 538, 0, 73, 644, 2662, 306, 0, 0, 0, 0, 0, 0, 0, 3, 60, 195, 0, 443, 1, 0, 5044, 0, 0, 0, 0, 0, 0, 0, 661, 526, 295, 1296, 2702, 1335, 200, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 4863, 0, 14, 30, 7, 228, 0, 2, 934, 760, 311, 0, 0, 0, 0, 0, 790, 93, 37, 0, 9, 2071, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5364, 6012, 5423, 5954, 5746, 8151, 7149, 5353] avg=6144.0000 max=8151 max_rank=5 min=5353 min_rank=7 imbalance_ratio=1.326660 max_load_reduction=41001 max_load_reduction_pct=0.834167 moved_to_echo=26399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2882, 19, 28, 998, 454, 25, 1859, 311, 0, 0, 0, 0, 0, 0, 0, 0, 5184, 178, 177, 0, 2, 49, 31, 346, 0, 0, 0, 0, 0, 0, 0, 0, 328, 11, 1304, 355, 326, 138, 15, 3, 2046, 803, 0, 0, 0, 0, 0, 0, 1173, 1005, 41, 17, 114, 24, 58, 474, 2005, 520, 0, 0, 0, 0, 0, 0, 0, 25, 390, 580, 14, 610, 10, 1, 4824, 0, 0, 0, 0, 0, 0, 0, 747, 992, 166, 705, 1549, 1888, 458, 653, 0, 0, 0, 0, 0, 0, 0, 0, 2822, 220, 2, 1, 48, 477, 0, 54, 1586, 436, 401, 0, 0, 0, 0, 0, 1011, 637, 93, 0, 89, 1319, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6576, 5967, 5329, 5431, 6454, 7158, 6047, 6190] avg=6144.0000 max=7158 max_rank=5 min=5329 min_rank=2 imbalance_ratio=1.165039 max_load_reduction=41994 max_load_reduction_pct=0.854370 moved_to_echo=25849 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2804, 3, 1, 727, 20, 49, 1684, 371, 0, 0, 0, 0, 0, 0, 0, 0, 4728, 187, 84, 6, 0, 126, 8, 298, 0, 0, 0, 0, 0, 0, 0, 0, 185, 20, 1785, 437, 810, 222, 213, 3, 2250, 1052, 0, 0, 0, 0, 0, 0, 331, 1754, 3, 9, 77, 1, 43, 455, 2627, 477, 0, 0, 0, 0, 0, 0, 0, 3, 64, 570, 1, 230, 46, 1, 5297, 0, 0, 0, 0, 0, 0, 0, 327, 204, 241, 1249, 2999, 1521, 263, 159, 0, 0, 0, 0, 0, 0, 0, 0, 2346, 4, 1, 90, 22, 283, 1, 4, 1446, 849, 532, 0, 0, 0, 0, 0, 1176, 73, 52, 1, 38, 1225, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5659, 5437, 6977, 5777, 6212, 6963, 5578, 6549] avg=6144.0000 max=6977 max_rank=2 min=5437 min_rank=1 imbalance_ratio=1.135579 max_load_reduction=42175 max_load_reduction_pct=0.858053 moved_to_echo=25302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2952, 1, 5, 1514, 24, 1, 1599, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 6596, 191, 123, 19, 0, 113, 13, 266, 0, 0, 0, 0, 0, 0, 0, 0, 398, 15, 1216, 145, 106, 744, 23, 1, 2195, 977, 0, 0, 0, 0, 0, 0, 804, 957, 12, 1, 441, 2, 106, 2691, 2441, 447, 0, 0, 0, 0, 0, 0, 0, 2, 66, 184, 12, 167, 8, 1, 5175, 0, 0, 0, 0, 0, 0, 0, 105, 355, 428, 625, 1535, 501, 83, 308, 0, 0, 0, 0, 0, 0, 0, 0, 2310, 10, 1, 166, 5, 510, 0, 18, 1363, 432, 420, 0, 0, 0, 0, 0, 551, 90, 31, 1, 5, 2158, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7298, 7321, 5820, 7902, 5615, 3940, 5235, 6021] avg=6144.0000 max=7902 max_rank=3 min=3940 min_rank=5 imbalance_ratio=1.286133 max_load_reduction=41250 max_load_reduction_pct=0.839233 moved_to_echo=20811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2882, 19, 28, 998, 454, 25, 1859, 311, 0, 0, 0, 0, 0, 0, 0, 0, 5184, 178, 177, 0, 2, 49, 31, 346, 0, 0, 0, 0, 0, 0, 0, 0, 328, 11, 1304, 355, 326, 138, 15, 3, 2046, 803, 0, 0, 0, 0, 0, 0, 1173, 1005, 41, 17, 114, 24, 58, 474, 2005, 520, 0, 0, 0, 0, 0, 0, 0, 25, 390, 580, 14, 610, 10, 1, 4824, 0, 0, 0, 0, 0, 0, 0, 747, 992, 166, 705, 1549, 1888, 458, 653, 0, 0, 0, 0, 0, 0, 0, 0, 2822, 220, 2, 1, 48, 477, 0, 54, 1586, 436, 401, 0, 0, 0, 0, 0, 1011, 637, 93, 0, 89, 1319, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6576, 5967, 5329, 5431, 6454, 7158, 6047, 6190] avg=6144.0000 max=7158 max_rank=5 min=5329 min_rank=2 imbalance_ratio=1.165039 max_load_reduction=41994 max_load_reduction_pct=0.854370 moved_to_echo=25849 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2304, 128, 11904, 2432, 3328, 3072, 768, 128, 17536, 8320, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7040, 1408, 512, 128, 256, 14592, 25728, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[26624, 256, 128, 384, 256, 2432, 128, 128, 11392, 4608, 3456, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 4096, 2048, 7936, 16384, 9600, 2304, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7040, 1408, 512, 128, 256, 14592, 25728, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[23808, 128, 128, 6784, 768, 128, 13312, 4608, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[26624, 256, 128, 384, 256, 2432, 128, 128, 11392, 4608, 3456, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 9984, 256, 128, 1792, 128, 640, 6528, 20864, 3712, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 1408, 3840, 128, 2816, 128, 128, 41216, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[43776, 1536, 1024, 128, 128, 1152, 256, 1664, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 152, 1717, 2816, 587, 1393, 0, 14, 0, 44, 0, 14, 2123, 74, 2038, 2305, 11, 1603, 0, 0, 6, 3, 153, 0, 246, 131, 432, 270, 23, 130, 749, 0, 131, 162, 1619, 515, 99, 5633, 0, 0, 347, 6, 290, 1738, 4, 6, 2398, 253, 205, 2, 311, 1680, 637, 4160, 419, 76, 0, 220, 0, 5, 6249, 4872, 74, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 180, 867, 2529, 766, 1931, 0, 6, 0, 11, 0, 11, 1620, 215, 2439, 1678, 15, 3159, 0, 0, 6, 17, 50, 1, 140, 318, 438, 293, 35, 105, 689, 0, 243, 140, 848, 1094, 101, 4258, 0, 0, 411, 0, 266, 3101, 1, 0, 1673, 754, 108, 3, 161, 2395, 1158, 4020, 446, 0, 1, 311, 0, 19, 4561, 5554, 3, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[1, 461, 1716, 2598, 506, 1906, 0, 4, 0, 30, 0, 84, 1635, 166, 2071, 1330, 7, 2272, 0, 0, 10, 0, 61, 0, 59, 609, 413, 199, 6, 107, 767, 0, 574, 634, 752, 1371, 31, 4091, 0, 2, 698, 1, 228, 2322, 10, 0, 4179, 495, 212, 3, 352, 2446, 373, 3596, 445, 3, 1, 83, 0, 16, 3752, 5447, 17, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[10, 267, 1414, 3148, 1001, 1748, 0, 145, 18, 81, 0, 105, 1088, 25, 2746, 1163, 0, 1945, 0, 8, 5, 4, 48, 35, 100, 284, 510, 730, 15, 67, 653, 5, 472, 49, 1220, 608, 174, 4035, 0, 0, 410, 0, 74, 3302, 38, 8, 2053, 674, 270, 15, 376, 2380, 437, 4944, 450, 9, 1, 249, 0, 23, 4684, 4692, 129, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 289, 1141, 3349, 154, 2812, 0, 53, 0, 16, 0, 7, 1258, 274, 2277, 1844, 1, 3124, 0, 0, 3, 3, 32, 4, 91, 188, 470, 68, 41, 77, 854, 0, 176, 138, 1030, 574, 17, 5175, 0, 3, 298, 0, 207, 2705, 0, 0, 2311, 602, 204, 31, 150, 2821, 1083, 4055, 489, 5, 0, 295, 0, 4, 4952, 3385, 12, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 152, 1717, 2816, 587, 1393, 0, 14, 0, 44, 0, 14, 2123, 74, 2038, 2305, 11, 1603, 0, 0, 6, 3, 153, 0, 246, 131, 432, 270, 23, 130, 749, 0, 131, 162, 1619, 515, 99, 5633, 0, 0, 347, 6, 290, 1738, 4, 6, 2398, 253, 205, 2, 311, 1680, 637, 4160, 419, 76, 0, 220, 0, 5, 6249, 4872, 74, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 103, 1137, 2563, 398, 1113, 0, 69, 0, 12, 0, 43, 2023, 174, 3647, 2550, 5, 2966, 0, 0, 14, 2, 96, 0, 126, 332, 569, 98, 61, 170, 995, 0, 299, 222, 1536, 1954, 124, 3696, 0, 0, 399, 0, 77, 1116, 4, 0, 2044, 341, 262, 5, 99, 2225, 1996, 4574, 484, 1, 0, 231, 0, 11, 3982, 4145, 34, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[1, 202, 1060, 1758, 582, 1626, 0, 71, 0, 3, 0, 49, 1127, 155, 2388, 1767, 6, 2952, 0, 0, 2, 3, 30, 1, 196, 260, 665, 98, 20, 348, 557, 0, 206, 81, 1269, 2934, 121, 4567, 1, 0, 296, 2, 96, 1293, 8, 0, 1571, 750, 158, 0, 856, 2477, 1263, 4382, 540, 0, 0, 290, 0, 11, 5575, 4458, 15, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[1, 461, 1716, 2598, 506, 1906, 0, 4, 0, 30, 0, 84, 1635, 166, 2071, 1330, 7, 2272, 0, 0, 10, 0, 61, 0, 59, 609, 413, 199, 6, 107, 767, 0, 574, 634, 752, 1371, 31, 4091, 0, 2, 698, 1, 228, 2322, 10, 0, 4179, 495, 212, 3, 352, 2446, 373, 3596, 445, 3, 1, 83, 0, 16, 3752, 5447, 17, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[1, 242, 2444, 2706, 255, 2571, 0, 27, 4, 19, 0, 74, 1718, 234, 2830, 1664, 13, 3666, 0, 0, 40, 5, 22, 1, 74, 255, 482, 130, 42, 119, 743, 0, 313, 266, 813, 1404, 208, 4077, 0, 0, 243, 5, 62, 1917, 7, 3, 1930, 487, 7, 3, 245, 2100, 987, 4520, 497, 8, 0, 207, 0, 4, 5358, 3024, 15, 61] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[10, 267, 1414, 3148, 1001, 1748, 0, 145, 18, 81, 0, 105, 1088, 25, 2746, 1163, 0, 1945, 0, 8, 5, 4, 48, 35, 100, 284, 510, 730, 15, 67, 653, 5, 472, 49, 1220, 608, 174, 4035, 0, 0, 410, 0, 74, 3302, 38, 8, 2053, 674, 270, 15, 376, 2380, 437, 4944, 450, 9, 1, 249, 0, 23, 4684, 4692, 129, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 103, 1137, 2563, 398, 1113, 0, 69, 0, 12, 0, 43, 2023, 174, 3647, 2550, 5, 2966, 0, 0, 14, 2, 96, 0, 126, 332, 569, 98, 61, 170, 995, 0, 299, 222, 1536, 1954, 124, 3696, 0, 0, 399, 0, 77, 1116, 4, 0, 2044, 341, 262, 5, 99, 2225, 1996, 4574, 484, 1, 0, 231, 0, 11, 3982, 4145, 34, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 461, 1716, 1939, 506, 1906, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 84, 1635, 166, 2059, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2272, 0, 0, 10, 0, 61, 0, 1563, 1255, 113, 0, 0, 0, 0, 0, 59, 609, 413, 199, 6, 107, 767, 0, 2679, 480, 0, 0, 0, 0, 0, 0, 574, 634, 752, 1371, 31, 2836, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 698, 1, 228, 2322, 10, 0, 4179, 495, 546, 12, 0, 0, 0, 0, 0, 0, 212, 3, 352, 2446, 373, 1553, 445, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 83, 0, 16, 1073, 5447, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6533, 5304, 5281, 5319, 6200, 8491, 5387, 6637] avg=6144.0000 max=8491 max_rank=5 min=5281 min_rank=2 imbalance_ratio=1.381999 max_load_reduction=40661 max_load_reduction_pct=0.827250 moved_to_echo=26715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 180, 867, 1887, 766, 1931, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 11, 1620, 215, 2425, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3159, 0, 0, 6, 17, 50, 1, 1748, 1307, 110, 0, 0, 0, 0, 0, 140, 318, 438, 293, 35, 105, 689, 0, 3256, 536, 0, 0, 0, 0, 0, 0, 243, 140, 848, 1094, 101, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 266, 3101, 1, 0, 1673, 754, 532, 14, 0, 0, 0, 0, 0, 0, 108, 3, 161, 2395, 1158, 1736, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 311, 0, 19, 1305, 5554, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5637, 5960, 6413, 5810, 5377, 6752, 6007, 7196] avg=6144.0000 max=7196 max_rank=7 min=5377 min_rank=4 imbalance_ratio=1.171224 max_load_reduction=41956 max_load_reduction_pct=0.853597 moved_to_echo=25332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 242, 2444, 2019, 255, 2571, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4, 19, 0, 74, 1718, 234, 2814, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3666, 0, 0, 40, 5, 22, 1, 1965, 1251, 118, 0, 0, 0, 0, 0, 74, 255, 482, 130, 42, 119, 743, 0, 3825, 603, 0, 0, 0, 0, 0, 0, 313, 266, 813, 1404, 208, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 5, 62, 1917, 7, 3, 1930, 487, 569, 16, 0, 0, 0, 0, 0, 0, 7, 3, 245, 2100, 987, 1952, 497, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 4, 1533, 3024, 15, 61, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7559, 6527, 7081, 6273, 5830, 5239, 5799, 4844] avg=6144.0000 max=7559 max_rank=0 min=4844 min_rank=7 imbalance_ratio=1.230306 max_load_reduction=41593 max_load_reduction_pct=0.846212 moved_to_echo=21712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[10, 267, 1414, 2349, 1001, 1748, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 18, 81, 0, 105, 1088, 25, 2731, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1945, 0, 8, 5, 4, 48, 35, 2150, 1238, 137, 0, 0, 0, 0, 0, 100, 284, 510, 730, 15, 67, 653, 5, 3344, 659, 0, 0, 0, 0, 0, 0, 472, 49, 1220, 608, 174, 2797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 74, 3302, 38, 8, 2053, 674, 662, 15, 0, 0, 0, 0, 0, 0, 270, 15, 376, 2380, 437, 2135, 450, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 249, 0, 23, 1340, 4692, 129, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6934, 5211, 5570, 6367, 5320, 7236, 6072, 6442] avg=6144.0000 max=7236 max_rank=5 min=5211 min_rank=1 imbalance_ratio=1.177734 max_load_reduction=41916 max_load_reduction_pct=0.852783 moved_to_echo=25070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 289, 1141, 2499, 154, 2812, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 7, 1258, 274, 2264, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3124, 0, 0, 3, 3, 32, 4, 1763, 1588, 146, 0, 0, 0, 0, 0, 91, 188, 470, 68, 41, 77, 854, 0, 3535, 541, 0, 0, 0, 0, 0, 0, 176, 138, 1030, 574, 17, 3587, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 207, 2705, 0, 0, 2311, 602, 704, 13, 0, 0, 0, 0, 0, 0, 204, 31, 150, 2821, 1083, 1751, 489, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 4, 1417, 3385, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6948, 5663, 6664, 5865, 5525, 6840, 6534, 5113] avg=6144.0000 max=6948 max_rank=0 min=5113 min_rank=7 imbalance_ratio=1.130859 max_load_reduction=42204 max_load_reduction_pct=0.858643 moved_to_echo=24012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 461, 1716, 1939, 506, 1906, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 84, 1635, 166, 2059, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2272, 0, 0, 10, 0, 61, 0, 1563, 1255, 113, 0, 0, 0, 0, 0, 59, 609, 413, 199, 6, 107, 767, 0, 2679, 480, 0, 0, 0, 0, 0, 0, 574, 634, 752, 1371, 31, 2836, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 698, 1, 228, 2322, 10, 0, 4179, 495, 546, 12, 0, 0, 0, 0, 0, 0, 212, 3, 352, 2446, 373, 1553, 445, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 83, 0, 16, 1073, 5447, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6533, 5304, 5281, 5319, 6200, 8491, 5387, 6637] avg=6144.0000 max=8491 max_rank=5 min=5281 min_rank=2 imbalance_ratio=1.381999 max_load_reduction=40661 max_load_reduction_pct=0.827250 moved_to_echo=26715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 202, 1060, 1312, 582, 1626, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 49, 1127, 155, 2375, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2952, 0, 0, 2, 3, 30, 1, 1905, 1402, 76, 0, 0, 0, 0, 0, 196, 260, 665, 98, 20, 348, 557, 0, 3980, 584, 0, 0, 0, 0, 0, 0, 206, 81, 1269, 2934, 121, 3165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 2, 96, 1293, 8, 0, 1571, 750, 370, 13, 0, 0, 0, 0, 0, 0, 158, 0, 856, 2477, 1263, 1893, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 11, 1595, 4458, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4854, 5476, 6377, 6708, 7777, 4399, 7187, 6374] avg=6144.0000 max=7777 max_rank=4 min=4399 min_rank=5 imbalance_ratio=1.265788 max_load_reduction=41375 max_load_reduction_pct=0.841777 moved_to_echo=25737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 103, 1137, 1912, 398, 1113, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 43, 2023, 174, 3626, 2550, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2966, 0, 0, 14, 2, 96, 0, 1989, 1134, 112, 0, 0, 0, 0, 0, 126, 332, 569, 98, 61, 170, 995, 0, 2843, 610, 0, 0, 0, 0, 0, 0, 299, 222, 1536, 1954, 124, 2562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 77, 1116, 4, 0, 2044, 341, 539, 21, 0, 0, 0, 0, 0, 0, 262, 5, 99, 2225, 1996, 1975, 484, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 11, 1139, 4145, 34, 25, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4732, 8428, 6318, 5804, 6697, 4541, 7047, 5585] avg=6144.0000 max=8428 max_rank=1 min=4541 min_rank=5 imbalance_ratio=1.371745 max_load_reduction=40724 max_load_reduction_pct=0.828532 moved_to_echo=23870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 152, 1717, 2092, 587, 1393, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 14, 2123, 74, 2023, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1603, 0, 0, 6, 3, 153, 0, 1812, 1733, 127, 0, 0, 0, 0, 0, 246, 131, 432, 270, 23, 130, 749, 0, 4466, 559, 0, 0, 0, 0, 0, 0, 131, 162, 1619, 515, 99, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 6, 290, 1738, 4, 6, 2398, 253, 597, 15, 0, 0, 0, 0, 0, 0, 205, 2, 311, 1680, 637, 1789, 419, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 5, 1783, 4872, 74, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5955, 6583, 5448, 7006, 6426, 5654, 5119, 6961] avg=6144.0000 max=7006 max_rank=3 min=5119 min_rank=6 imbalance_ratio=1.140299 max_load_reduction=42146 max_load_reduction_pct=0.857463 moved_to_echo=24160 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 242, 2444, 2019, 255, 2571, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4, 19, 0, 74, 1718, 234, 2814, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3666, 0, 0, 40, 5, 22, 1, 1965, 1251, 118, 0, 0, 0, 0, 0, 74, 255, 482, 130, 42, 119, 743, 0, 3825, 603, 0, 0, 0, 0, 0, 0, 313, 266, 813, 1404, 208, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 5, 62, 1917, 7, 3, 1930, 487, 569, 16, 0, 0, 0, 0, 0, 0, 7, 3, 245, 2100, 987, 1952, 497, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 4, 1533, 3024, 15, 61, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7559, 6527, 7081, 6273, 5830, 5239, 5799, 4844] avg=6144.0000 max=7559 max_rank=0 min=4844 min_rank=7 imbalance_ratio=1.230306 max_load_reduction=41593 max_load_reduction_pct=0.846212 moved_to_echo=21712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[10, 267, 1414, 2349, 1001, 1748, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 18, 81, 0, 105, 1088, 25, 2731, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1945, 0, 8, 5, 4, 48, 35, 2150, 1238, 137, 0, 0, 0, 0, 0, 100, 284, 510, 730, 15, 67, 653, 5, 3344, 659, 0, 0, 0, 0, 0, 0, 472, 49, 1220, 608, 174, 2797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 74, 3302, 38, 8, 2053, 674, 662, 15, 0, 0, 0, 0, 0, 0, 270, 15, 376, 2380, 437, 2135, 450, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 249, 0, 23, 1340, 4692, 129, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6934, 5211, 5570, 6367, 5320, 7236, 6072, 6442] avg=6144.0000 max=7236 max_rank=5 min=5211 min_rank=1 imbalance_ratio=1.177734 max_load_reduction=41916 max_load_reduction_pct=0.852783 moved_to_echo=25070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 103, 1137, 1912, 398, 1113, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 43, 2023, 174, 3626, 2550, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2966, 0, 0, 14, 2, 96, 0, 1989, 1134, 112, 0, 0, 0, 0, 0, 126, 332, 569, 98, 61, 170, 995, 0, 2843, 610, 0, 0, 0, 0, 0, 0, 299, 222, 1536, 1954, 124, 2562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 77, 1116, 4, 0, 2044, 341, 539, 21, 0, 0, 0, 0, 0, 0, 262, 5, 99, 2225, 1996, 1975, 484, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 11, 1139, 4145, 34, 25, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4732, 8428, 6318, 5804, 6697, 4541, 7047, 5585] avg=6144.0000 max=8428 max_rank=1 min=4541 min_rank=5 imbalance_ratio=1.371745 max_load_reduction=40724 max_load_reduction_pct=0.828532 moved_to_echo=23870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 21760, 0, 128, 128, 128, 512, 128, 14976, 11008, 1024, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 128, 2560, 18560, 7936, 14848, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 1792, 9088, 10496, 896, 24704, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 256, 0, 512, 12672, 1408, 20352, 14336, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1920, 11520, 16128, 4352, 15104, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 2432, 4096, 1920, 256, 1152, 6016, 128, 28032, 4608, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 1920, 0, 128, 11264, 35584, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 1792, 9088, 10496, 896, 24704, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3200, 128, 1408, 17536, 128, 128, 18176, 4480, 4608, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1920, 11520, 16128, 4352, 15104, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 1920, 0, 128, 11264, 35584, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[295, 235, 169, 16, 0, 32, 185, 83, 0, 2, 66, 3781, 0, 2629, 5837, 1, 72, 2, 194, 347, 2737, 4869, 361, 28, 78, 5343, 747, 397, 4, 1, 737, 1, 0, 0, 1, 1356, 16, 4262, 928, 0, 0, 536, 19, 357, 797, 2236, 95, 794, 3665, 16, 6, 84, 0, 2, 1262, 112, 131, 155, 568, 305, 789, 0, 0, 1411] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[351, 225, 160, 13, 0, 183, 318, 647, 0, 3, 6, 4319, 1, 2130, 6328, 8, 130, 25, 303, 679, 3636, 4998, 230, 8, 1, 4094, 505, 172, 9, 0, 386, 21, 0, 0, 1, 787, 0, 2748, 2254, 0, 0, 478, 34, 692, 1503, 3460, 9, 567, 2443, 0, 0, 141, 0, 4, 1441, 136, 206, 540, 153, 371, 12, 0, 0, 1283] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[438, 264, 265, 118, 0, 29, 460, 391, 0, 5, 88, 3491, 0, 1246, 6902, 20, 338, 4, 401, 635, 1593, 5902, 369, 25, 14, 6432, 255, 612, 19, 35, 508, 13, 0, 0, 4, 1265, 10, 3173, 758, 1, 0, 255, 66, 871, 1471, 1756, 128, 1171, 3086, 4, 0, 419, 0, 13, 1477, 325, 102, 54, 139, 453, 106, 0, 8, 1165] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[422, 239, 117, 291, 0, 59, 108, 208, 0, 4, 4, 4823, 0, 2003, 6652, 10, 99, 126, 466, 556, 3218, 5858, 54, 66, 0, 5176, 561, 443, 25, 1, 407, 28, 0, 0, 1, 1315, 1, 2537, 1125, 0, 0, 387, 14, 357, 1451, 1833, 26, 310, 2958, 11, 0, 65, 1, 1, 1147, 353, 155, 466, 248, 1171, 362, 1, 2, 830] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[245, 228, 156, 4, 0, 205, 227, 110, 0, 2, 63, 4221, 0, 1917, 5404, 34, 129, 32, 588, 603, 3097, 3445, 22, 9, 23, 3372, 810, 1218, 14, 0, 1190, 2, 0, 0, 6, 1638, 286, 2475, 2377, 1, 0, 379, 4, 1335, 1172, 2535, 61, 773, 3642, 11, 6, 233, 0, 32, 694, 451, 311, 531, 622, 822, 44, 0, 4, 1337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[349, 501, 173, 110, 0, 175, 421, 269, 3, 18, 145, 3651, 0, 1854, 7269, 2, 49, 6, 318, 236, 3304, 6237, 93, 6, 5, 5972, 236, 260, 40, 2, 932, 1, 0, 0, 0, 1008, 1, 3028, 1746, 0, 0, 377, 34, 377, 1175, 1989, 41, 734, 3172, 10, 1, 198, 0, 11, 796, 135, 131, 269, 200, 331, 57, 0, 5, 689] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[295, 235, 169, 16, 0, 32, 185, 83, 0, 2, 66, 3781, 0, 2629, 5837, 1, 72, 2, 194, 347, 2737, 4869, 361, 28, 78, 5343, 747, 397, 4, 1, 737, 1, 0, 0, 1, 1356, 16, 4262, 928, 0, 0, 536, 19, 357, 797, 2236, 95, 794, 3665, 16, 6, 84, 0, 2, 1262, 112, 131, 155, 568, 305, 789, 0, 0, 1411] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[438, 264, 265, 118, 0, 29, 460, 391, 0, 5, 88, 3491, 0, 1246, 6902, 20, 338, 4, 401, 635, 1593, 5902, 369, 25, 14, 6432, 255, 612, 19, 35, 508, 13, 0, 0, 4, 1265, 10, 3173, 758, 1, 0, 255, 66, 871, 1471, 1756, 128, 1171, 3086, 4, 0, 419, 0, 13, 1477, 325, 102, 54, 139, 453, 106, 0, 8, 1165] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[402, 208, 184, 12, 0, 27, 998, 104, 0, 2, 24, 4656, 8, 1295, 5868, 0, 72, 3, 181, 343, 3238, 5539, 122, 79, 6, 5199, 370, 406, 15, 0, 488, 1, 0, 0, 2, 819, 0, 2144, 628, 0, 0, 361, 14, 1261, 1971, 1869, 67, 1741, 3583, 14, 1, 280, 0, 7, 1908, 61, 409, 238, 267, 772, 161, 0, 0, 724] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[534, 161, 412, 15, 7, 73, 649, 620, 47, 3, 24, 3939, 35, 1110, 6044, 2, 280, 4, 324, 617, 2204, 5135, 83, 222, 22, 5537, 321, 276, 4, 26, 364, 6, 3, 2, 1, 235, 52, 3239, 1126, 0, 4, 416, 0, 863, 1428, 1362, 69, 1970, 4085, 2, 7, 133, 0, 54, 1691, 124, 452, 669, 134, 1018, 253, 0, 1, 659] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[438, 264, 265, 118, 0, 29, 460, 391, 4836, 0, 0, 0, 0, 0, 0, 0, 0, 5, 88, 3491, 0, 1246, 201, 20, 0, 0, 0, 0, 0, 0, 0, 0, 338, 4, 401, 635, 1593, 2277, 369, 25, 0, 0, 0, 0, 0, 0, 0, 0, 14, 5652, 255, 612, 19, 35, 508, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1265, 10, 3173, 758, 1, 547, 296, 0, 0, 0, 0, 0, 0, 0, 255, 66, 871, 1471, 1756, 128, 1171, 484, 0, 0, 0, 0, 0, 0, 0, 3086, 4, 0, 419, 0, 13, 1477, 325, 1214, 0, 0, 0, 0, 0, 0, 0, 102, 54, 139, 453, 106, 0, 8, 1165, 1865, 1864, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6801, 5051, 5642, 7108, 6054, 6202, 6538, 5756] avg=6144.0000 max=7108 max_rank=3 min=5051 min_rank=1 imbalance_ratio=1.156901 max_load_reduction=42044 max_load_reduction_pct=0.855387 moved_to_echo=24550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[245, 228, 156, 4, 0, 205, 227, 110, 0, 2, 63, 4221, 0, 1917, 5404, 34, 129, 32, 588, 603, 3097, 3445, 22, 9, 23, 3372, 810, 1218, 14, 0, 1190, 2, 0, 0, 6, 1638, 286, 2475, 2377, 1, 0, 379, 4, 1335, 1172, 2535, 61, 773, 3642, 11, 6, 233, 0, 32, 694, 451, 311, 531, 622, 822, 44, 0, 4, 1337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[349, 501, 173, 110, 0, 175, 421, 269, 5093, 0, 0, 0, 0, 0, 0, 0, 3, 18, 145, 3651, 0, 1854, 212, 2, 0, 0, 0, 0, 0, 0, 0, 0, 49, 6, 318, 236, 3304, 2405, 93, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5248, 236, 260, 40, 2, 932, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1, 3028, 1746, 0, 579, 275, 0, 0, 0, 0, 0, 0, 0, 377, 34, 377, 1175, 1989, 41, 734, 449, 0, 0, 0, 0, 0, 0, 0, 3172, 10, 1, 198, 0, 11, 796, 135, 1283, 0, 0, 0, 0, 0, 0, 0, 131, 269, 200, 331, 57, 0, 5, 689, 1964, 1970, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7091, 5885, 6417, 6724, 6637, 5176, 5606, 5616] avg=6144.0000 max=7091 max_rank=0 min=5176 min_rank=5 imbalance_ratio=1.154134 max_load_reduction=42061 max_load_reduction_pct=0.855733 moved_to_echo=23035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[295, 235, 169, 16, 0, 32, 185, 83, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 66, 3781, 0, 2629, 170, 1, 0, 0, 0, 0, 0, 0, 0, 0, 72, 2, 194, 347, 2737, 1879, 361, 28, 0, 0, 0, 0, 0, 0, 0, 0, 78, 4695, 747, 397, 4, 1, 737, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1356, 16, 4262, 928, 0, 452, 246, 0, 0, 0, 0, 0, 0, 0, 536, 19, 357, 797, 2236, 95, 794, 402, 0, 0, 0, 0, 0, 0, 0, 3665, 16, 6, 84, 0, 2, 1262, 112, 1001, 0, 0, 0, 0, 0, 0, 0, 131, 155, 568, 305, 789, 0, 0, 1411, 1577, 1537, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5105, 6649, 5620, 6660, 7261, 5236, 6148, 6473] avg=6144.0000 max=7261 max_rank=4 min=5105 min_rank=0 imbalance_ratio=1.181803 max_load_reduction=41891 max_load_reduction_pct=0.852275 moved_to_echo=25118 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[351, 225, 160, 13, 0, 183, 318, 647, 4434, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 4319, 1, 2130, 184, 8, 0, 0, 0, 0, 0, 0, 0, 0, 130, 25, 303, 679, 3636, 1928, 230, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3598, 505, 172, 9, 0, 386, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 787, 0, 2748, 2254, 0, 464, 188, 0, 0, 0, 0, 0, 0, 0, 478, 34, 692, 1503, 3460, 9, 567, 308, 0, 0, 0, 0, 0, 0, 0, 2443, 0, 0, 141, 0, 4, 1441, 136, 1028, 0, 0, 0, 0, 0, 0, 0, 206, 540, 153, 371, 12, 0, 0, 1283, 1710, 1578, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6331, 6651, 6939, 4692, 6442, 7051, 5193, 5853] avg=6144.0000 max=7051 max_rank=5 min=4692 min_rank=3 imbalance_ratio=1.147624 max_load_reduction=42101 max_load_reduction_pct=0.856547 moved_to_echo=24539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[402, 208, 184, 12, 0, 27, 998, 104, 4112, 0, 0, 0, 0, 0, 0, 0, 0, 2, 24, 4656, 8, 1295, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 3, 181, 343, 3238, 2137, 122, 79, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4569, 370, 406, 15, 0, 488, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 819, 0, 2144, 628, 0, 514, 239, 0, 0, 0, 0, 0, 0, 0, 361, 14, 1261, 1971, 1869, 67, 1741, 391, 0, 0, 0, 0, 0, 0, 0, 3583, 14, 1, 280, 0, 7, 1908, 61, 1139, 0, 0, 0, 0, 0, 0, 0, 409, 238, 267, 772, 161, 0, 0, 724, 1585, 1749, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6047, 6156, 6175, 5855, 4346, 7675, 6993, 5905] avg=6144.0000 max=7675 max_rank=5 min=4346 min_rank=4 imbalance_ratio=1.249186 max_load_reduction=41477 max_load_reduction_pct=0.843852 moved_to_echo=24919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[422, 239, 117, 291, 0, 59, 108, 208, 4661, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4823, 0, 2003, 194, 10, 0, 0, 0, 0, 0, 0, 0, 0, 99, 126, 466, 556, 3218, 2260, 54, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4549, 561, 443, 25, 1, 407, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1315, 1, 2537, 1125, 0, 543, 238, 0, 0, 0, 0, 0, 0, 0, 387, 14, 357, 1451, 1833, 26, 310, 389, 0, 0, 0, 0, 0, 0, 0, 2958, 11, 0, 65, 1, 1, 1147, 353, 1205, 0, 0, 0, 0, 0, 0, 0, 155, 466, 248, 1171, 362, 1, 2, 830, 1797, 1850, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 7038, 6845, 6014, 5760, 4767, 5741, 6882] avg=6144.0000 max=7038 max_rank=1 min=4767 min_rank=5 imbalance_ratio=1.145508 max_load_reduction=42114 max_load_reduction_pct=0.856812 moved_to_echo=23150 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[245, 228, 156, 4, 0, 205, 227, 110, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 2, 63, 4221, 0, 1917, 149, 34, 0, 0, 0, 0, 0, 0, 0, 0, 129, 32, 588, 603, 3097, 1318, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2957, 810, 1218, 14, 0, 1190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1638, 286, 2475, 2377, 1, 323, 157, 0, 0, 0, 0, 0, 0, 0, 379, 4, 1335, 1172, 2535, 61, 773, 258, 0, 0, 0, 0, 0, 0, 0, 3642, 11, 6, 233, 0, 32, 694, 451, 712, 0, 0, 0, 0, 0, 0, 0, 311, 531, 622, 822, 44, 0, 4, 1337, 1464, 1092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4966, 6386, 5798, 6214, 7263, 6517, 5781, 6227] avg=6144.0000 max=7263 max_rank=4 min=4966 min_rank=0 imbalance_ratio=1.182129 max_load_reduction=41889 max_load_reduction_pct=0.852234 moved_to_echo=25788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[349, 501, 173, 110, 0, 175, 421, 269, 5093, 0, 0, 0, 0, 0, 0, 0, 3, 18, 145, 3651, 0, 1854, 212, 2, 0, 0, 0, 0, 0, 0, 0, 0, 49, 6, 318, 236, 3304, 2405, 93, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5248, 236, 260, 40, 2, 932, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1, 3028, 1746, 0, 579, 275, 0, 0, 0, 0, 0, 0, 0, 377, 34, 377, 1175, 1989, 41, 734, 449, 0, 0, 0, 0, 0, 0, 0, 3172, 10, 1, 198, 0, 11, 796, 135, 1283, 0, 0, 0, 0, 0, 0, 0, 131, 269, 200, 331, 57, 0, 5, 689, 1964, 1970, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7091, 5885, 6417, 6724, 6637, 5176, 5606, 5616] avg=6144.0000 max=7091 max_rank=0 min=5176 min_rank=5 imbalance_ratio=1.154134 max_load_reduction=42061 max_load_reduction_pct=0.855733 moved_to_echo=23035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[438, 264, 265, 118, 0, 29, 460, 391, 4836, 0, 0, 0, 0, 0, 0, 0, 0, 5, 88, 3491, 0, 1246, 201, 20, 0, 0, 0, 0, 0, 0, 0, 0, 338, 4, 401, 635, 1593, 2277, 369, 25, 0, 0, 0, 0, 0, 0, 0, 0, 14, 5652, 255, 612, 19, 35, 508, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1265, 10, 3173, 758, 1, 547, 296, 0, 0, 0, 0, 0, 0, 0, 255, 66, 871, 1471, 1756, 128, 1171, 484, 0, 0, 0, 0, 0, 0, 0, 3086, 4, 0, 419, 0, 13, 1477, 325, 1214, 0, 0, 0, 0, 0, 0, 0, 102, 54, 139, 453, 106, 0, 8, 1165, 1865, 1864, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6801, 5051, 5642, 7108, 6054, 6202, 6538, 5756] avg=6144.0000 max=7108 max_rank=3 min=5051 min_rank=1 imbalance_ratio=1.156901 max_load_reduction=42044 max_load_reduction_pct=0.855387 moved_to_echo=24550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[534, 161, 412, 15, 7, 73, 649, 620, 4235, 0, 0, 0, 0, 0, 0, 0, 47, 3, 24, 3939, 35, 1110, 176, 2, 0, 0, 0, 0, 0, 0, 0, 0, 280, 4, 324, 617, 2204, 1982, 83, 222, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4866, 321, 276, 4, 26, 364, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 235, 52, 3239, 1126, 0, 476, 255, 0, 0, 0, 0, 0, 0, 4, 416, 0, 863, 1428, 1362, 69, 1970, 416, 0, 0, 0, 0, 0, 0, 0, 4085, 2, 7, 133, 0, 54, 1691, 124, 1056, 0, 0, 0, 0, 0, 0, 0, 452, 669, 134, 1018, 253, 0, 1, 659, 1633, 1621, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6706, 5336, 5716, 5885, 5389, 6528, 7152, 6440] avg=6144.0000 max=7152 max_rank=6 min=5336 min_rank=1 imbalance_ratio=1.164062 max_load_reduction=42000 max_load_reduction_pct=0.854492 moved_to_echo=25509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[402, 208, 184, 12, 0, 27, 998, 104, 4112, 0, 0, 0, 0, 0, 0, 0, 0, 2, 24, 4656, 8, 1295, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 3, 181, 343, 3238, 2137, 122, 79, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4569, 370, 406, 15, 0, 488, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 819, 0, 2144, 628, 0, 514, 239, 0, 0, 0, 0, 0, 0, 0, 361, 14, 1261, 1971, 1869, 67, 1741, 391, 0, 0, 0, 0, 0, 0, 0, 3583, 14, 1, 280, 0, 7, 1908, 61, 1139, 0, 0, 0, 0, 0, 0, 0, 409, 238, 267, 772, 161, 0, 0, 724, 1585, 1749, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6047, 6156, 6175, 5855, 4346, 7675, 6993, 5905] avg=6144.0000 max=7675 max_rank=5 min=4346 min_rank=4 imbalance_ratio=1.249186 max_load_reduction=41477 max_load_reduction_pct=0.843852 moved_to_echo=24919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[245, 228, 156, 4, 0, 205, 227, 110, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 2, 63, 4221, 0, 1917, 149, 34, 0, 0, 0, 0, 0, 0, 0, 0, 129, 32, 588, 603, 3097, 1318, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2957, 810, 1218, 14, 0, 1190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1638, 286, 2475, 2377, 1, 323, 157, 0, 0, 0, 0, 0, 0, 0, 379, 4, 1335, 1172, 2535, 61, 773, 258, 0, 0, 0, 0, 0, 0, 0, 3642, 11, 6, 233, 0, 32, 694, 451, 712, 0, 0, 0, 0, 0, 0, 0, 311, 531, 622, 822, 44, 0, 4, 1337, 1464, 1092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4966, 6386, 5798, 6214, 7263, 6517, 5781, 6227] avg=6144.0000 max=7263 max_rank=4 min=4966 min_rank=0 imbalance_ratio=1.182129 max_load_reduction=41889 max_load_reduction_pct=0.852234 moved_to_echo=25788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[26752, 128, 128, 1664, 128, 128, 10496, 1792, 8704, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 256, 2816, 4096, 23040, 16256, 1408, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 2944, 2432, 5248, 1792, 128, 128, 8192, 13696, 13312, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 8448, 384, 23680, 11008, 128, 3968, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 512, 32896, 128, 14208, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 3200, 256, 6144, 11008, 17152, 512, 8064, 3200, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 2176, 1664, 640, 128, 896, 3456, 2432, 35328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 2944, 2432, 5248, 1792, 128, 128, 8192, 13696, 13312, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 512, 32896, 128, 14208, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 36224, 3840, 3840, 256, 128, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 30, 0, 2145, 217, 47, 5, 16, 75, 732, 1, 5, 0, 49, 3276, 1840, 33, 0, 507, 527, 2746, 3226, 1, 2, 284, 635, 674, 698, 2204, 3693, 29, 2078, 158, 1, 70, 561, 209, 238, 1292, 27, 0, 2, 100, 25, 0, 0, 43, 0, 860, 1, 5492, 2977, 422, 0, 72, 2307, 575, 1897, 4968, 9, 6, 3, 692, 370] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 1, 0, 2637, 410, 4, 1, 14, 260, 997, 4, 0, 0, 55, 1898, 1258, 24, 2, 91, 411, 3304, 3334, 5, 0, 366, 676, 195, 1024, 3135, 2087, 0, 1618, 183, 0, 96, 1323, 154, 129, 1171, 39, 0, 17, 114, 10, 0, 4, 11, 1, 461, 5, 5695, 4365, 782, 0, 328, 1261, 1123, 1823, 4842, 4, 1, 4, 653, 742] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 43, 7, 2953, 353, 77, 2, 367, 897, 1330, 7, 0, 0, 140, 2713, 2444, 115, 10, 196, 205, 2425, 731, 15, 3, 179, 1868, 173, 885, 3794, 1849, 12, 2451, 357, 10, 182, 476, 141, 170, 754, 35, 25, 2, 157, 68, 0, 0, 40, 2, 490, 1, 5621, 1681, 197, 0, 595, 1960, 981, 1629, 4721, 4, 67, 5, 1334, 1203] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 6, 0, 2839, 358, 81, 0, 15, 1198, 1152, 12, 0, 4, 210, 1328, 1783, 47, 7, 120, 633, 3746, 853, 3, 3, 330, 659, 84, 1008, 2499, 3229, 5, 1588, 170, 0, 55, 680, 21, 61, 934, 4, 2, 0, 105, 12, 0, 1, 21, 3, 635, 1, 5327, 3982, 969, 0, 578, 1631, 1352, 804, 5203, 0, 0, 0, 1527, 1274] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 7, 0, 2910, 439, 8, 2, 111, 590, 1120, 15, 0, 2, 110, 1020, 904, 159, 17, 398, 139, 4511, 2001, 19, 0, 306, 1325, 559, 1489, 5231, 1921, 3, 462, 361, 0, 90, 475, 278, 84, 859, 16, 1, 0, 191, 2, 0, 1, 3, 0, 346, 1, 4440, 2169, 447, 1, 834, 1272, 1753, 2111, 5906, 1, 16, 13, 727, 976] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 16, 0, 2369, 407, 3, 8, 24, 335, 606, 1, 0, 4, 98, 2186, 897, 161, 12, 73, 213, 3950, 2412, 0, 0, 13, 1351, 109, 838, 3728, 3827, 0, 1152, 140, 1, 67, 519, 317, 327, 1060, 95, 6, 0, 41, 8, 0, 20, 107, 0, 351, 2, 5766, 3478, 703, 0, 437, 1455, 1184, 1964, 4685, 2, 0, 5, 395, 1224] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[1, 79, 0, 2116, 469, 24, 11, 32, 1014, 1090, 0, 0, 4, 63, 1043, 1375, 8, 17, 76, 154, 4274, 2155, 2, 4, 174, 1501, 111, 563, 2909, 3644, 0, 1838, 358, 0, 88, 2339, 269, 127, 1277, 2, 4, 0, 378, 17, 1, 0, 19, 0, 326, 0, 4213, 3911, 276, 1, 499, 2209, 746, 2381, 2937, 1, 1, 0, 1196, 825] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 16, 0, 2369, 407, 3, 8, 24, 335, 606, 1, 0, 4, 98, 2186, 897, 161, 12, 73, 213, 3950, 2412, 0, 0, 13, 1351, 109, 838, 3728, 3827, 0, 1152, 140, 1, 67, 519, 317, 327, 1060, 95, 6, 0, 41, 8, 0, 20, 107, 0, 351, 2, 5766, 3478, 703, 0, 437, 1455, 1184, 1964, 4685, 2, 0, 5, 395, 1224] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 43, 7, 2953, 353, 77, 2, 367, 897, 1330, 7, 0, 0, 140, 2713, 2444, 115, 10, 196, 205, 2425, 731, 15, 3, 179, 1868, 173, 885, 3794, 1849, 12, 2451, 357, 10, 182, 476, 141, 170, 754, 35, 25, 2, 157, 68, 0, 0, 40, 2, 490, 1, 5621, 1681, 197, 0, 595, 1960, 981, 1629, 4721, 4, 67, 5, 1334, 1203] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 6, 0, 2839, 358, 81, 0, 15, 1198, 1152, 12, 0, 4, 210, 1328, 1783, 47, 7, 120, 633, 3746, 853, 3, 3, 330, 659, 84, 1008, 2499, 3229, 5, 1588, 170, 0, 55, 680, 21, 61, 934, 4, 2, 0, 105, 12, 0, 1, 21, 3, 635, 1, 5327, 3982, 969, 0, 578, 1631, 1352, 804, 5203, 0, 0, 0, 1527, 1274] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 1, 0, 2637, 410, 4, 1, 14, 260, 997, 4, 0, 0, 55, 1898, 1258, 24, 2, 91, 411, 3304, 3334, 5, 0, 366, 676, 195, 1024, 3135, 2087, 0, 1618, 183, 0, 96, 1323, 154, 129, 1171, 39, 0, 17, 114, 10, 0, 4, 11, 1, 461, 5, 5695, 4365, 782, 0, 328, 1261, 1123, 1823, 4842, 4, 1, 4, 653, 742] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 30, 0, 2145, 217, 47, 5, 16, 75, 732, 1, 5, 0, 49, 3276, 1840, 33, 0, 507, 527, 2746, 3226, 1, 2, 284, 635, 674, 698, 2204, 3693, 29, 2078, 158, 1, 70, 561, 209, 238, 1292, 27, 0, 2, 100, 25, 0, 0, 43, 0, 860, 1, 5492, 2977, 422, 0, 72, 2307, 575, 1897, 4968, 9, 6, 3, 692, 370] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2025, 282, 17, 0, 4, 706, 238, 1, 0, 2, 43, 3081, 1914, 40, 0, 88, 204, 5019, 2760, 0, 1, 70, 2078, 116, 345, 4047, 2418, 1, 1830, 188, 0, 69, 1096, 145, 85, 468, 4, 0, 0, 126, 5, 0, 0, 19, 0, 275, 0, 5957, 3586, 565, 0, 145, 2318, 331, 1295, 3454, 0, 0, 0, 739, 952] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 0, 2145, 217, 47, 5, 16, 2551, 1026, 0, 0, 0, 0, 0, 0, 75, 732, 1, 5, 0, 49, 3276, 1840, 278, 343, 157, 0, 0, 0, 0, 0, 33, 0, 507, 527, 2468, 3226, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 284, 635, 674, 698, 0, 2510, 29, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 158, 1, 70, 561, 209, 238, 1292, 27, 1809, 761, 0, 0, 0, 0, 0, 0, 0, 2, 100, 25, 0, 0, 43, 0, 5492, 395, 0, 0, 0, 0, 0, 0, 860, 1, 0, 2634, 422, 0, 72, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 575, 1897, 1656, 9, 6, 3, 692, 370, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6037, 6756, 6764, 6908, 5126, 6057, 6296, 5208] avg=6144.0000 max=6908 max_rank=3 min=5126 min_rank=4 imbalance_ratio=1.124349 max_load_reduction=42244 max_load_reduction_pct=0.859456 moved_to_echo=22687 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 2910, 439, 8, 2, 111, 3033, 534, 0, 0, 0, 0, 0, 0, 590, 1120, 15, 0, 2, 110, 1020, 904, 457, 250, 81, 0, 0, 0, 0, 0, 159, 17, 398, 139, 4054, 2001, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 1325, 559, 1489, 0, 1306, 3, 462, 0, 0, 0, 0, 0, 0, 0, 0, 361, 0, 90, 475, 278, 84, 859, 16, 4295, 905, 0, 0, 0, 0, 0, 0, 1, 0, 191, 2, 0, 1, 3, 0, 4440, 936, 0, 0, 0, 0, 0, 0, 346, 1, 0, 1919, 447, 1, 834, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 1753, 2111, 1968, 1, 16, 13, 727, 976, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7044, 4549, 6787, 5450, 7363, 5574, 4820, 7565] avg=6144.0000 max=7565 max_rank=7 min=4549 min_rank=1 imbalance_ratio=1.231283 max_load_reduction=41587 max_load_reduction_pct=0.846090 moved_to_echo=25322 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 2839, 358, 81, 0, 15, 2672, 897, 0, 0, 0, 0, 0, 0, 1198, 1152, 12, 0, 4, 210, 1328, 1783, 379, 459, 137, 0, 0, 0, 0, 0, 47, 7, 120, 633, 3367, 853, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 330, 659, 84, 1008, 0, 2195, 5, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 55, 680, 21, 61, 934, 4, 2053, 797, 0, 0, 0, 0, 0, 0, 2, 0, 105, 12, 0, 1, 21, 3, 5327, 446, 0, 0, 0, 0, 0, 0, 635, 1, 0, 3523, 969, 0, 578, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 804, 1734, 0, 0, 0, 1527, 1274, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6868, 6662, 5033, 5869, 4775, 5917, 7337, 6691] avg=6144.0000 max=7337 max_rank=6 min=4775 min_rank=4 imbalance_ratio=1.194173 max_load_reduction=41815 max_load_reduction_pct=0.850728 moved_to_echo=24720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2025, 282, 17, 0, 4, 1773, 672, 0, 0, 0, 0, 0, 0, 706, 238, 1, 0, 2, 43, 3081, 1914, 509, 413, 103, 0, 0, 0, 0, 0, 40, 0, 88, 204, 4510, 2760, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 70, 2078, 116, 345, 0, 1643, 1, 1830, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 69, 1096, 145, 85, 468, 4, 3323, 529, 0, 0, 0, 0, 0, 0, 0, 0, 126, 5, 0, 0, 19, 0, 5957, 724, 0, 0, 0, 0, 0, 0, 275, 0, 0, 3173, 565, 0, 145, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 331, 1295, 1152, 0, 0, 0, 739, 952, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4773, 7010, 7603, 6083, 5907, 6831, 6476, 4469] avg=6144.0000 max=7603 max_rank=2 min=4469 min_rank=7 imbalance_ratio=1.237467 max_load_reduction=41549 max_load_reduction_pct=0.845317 moved_to_echo=23683 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 7, 2953, 353, 77, 2, 367, 2424, 514, 0, 0, 0, 0, 0, 0, 897, 1330, 7, 0, 0, 140, 2713, 2444, 245, 193, 78, 0, 0, 0, 0, 0, 115, 10, 196, 205, 2180, 731, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1868, 173, 885, 0, 1257, 12, 2451, 0, 0, 0, 0, 0, 0, 0, 0, 357, 10, 182, 476, 141, 170, 754, 35, 3116, 723, 0, 0, 0, 0, 0, 0, 25, 2, 157, 68, 0, 0, 40, 2, 5621, 678, 0, 0, 0, 0, 0, 0, 490, 1, 0, 1488, 197, 0, 595, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1629, 1574, 4, 67, 5, 1334, 1203, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6740, 8047, 3455, 6825, 5964, 6593, 4731, 6797] avg=6144.0000 max=8047 max_rank=1 min=3455 min_rank=2 imbalance_ratio=1.309733 max_load_reduction=41105 max_load_reduction_pct=0.836283 moved_to_echo=24085 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 2637, 410, 4, 1, 14, 2486, 580, 0, 0, 0, 0, 0, 0, 260, 997, 4, 0, 0, 55, 1898, 1258, 335, 503, 88, 0, 0, 0, 0, 0, 24, 2, 91, 411, 2969, 3334, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 676, 195, 1024, 0, 1419, 0, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 96, 1323, 154, 129, 1171, 39, 2574, 742, 0, 0, 0, 0, 0, 0, 0, 17, 114, 10, 0, 4, 11, 1, 5695, 561, 0, 0, 0, 0, 0, 0, 461, 5, 0, 3862, 782, 0, 328, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 1823, 1614, 4, 1, 4, 653, 742, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 5398, 6836, 5298, 6411, 6413, 6699, 5964] avg=6144.0000 max=6836 max_rank=2 min=5298 min_rank=3 imbalance_ratio=1.112630 max_load_reduction=42316 max_load_reduction_pct=0.860921 moved_to_echo=25487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1, 79, 0, 2116, 469, 24, 11, 32, 1508, 1013, 0, 0, 0, 0, 0, 0, 1014, 1090, 0, 0, 4, 63, 1043, 1375, 433, 451, 155, 0, 0, 0, 0, 0, 8, 17, 76, 154, 3841, 2155, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 174, 1501, 111, 563, 0, 2476, 0, 1838, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 88, 2339, 269, 127, 1277, 2, 2388, 450, 0, 0, 0, 0, 0, 0, 4, 0, 378, 17, 1, 0, 19, 0, 4213, 521, 0, 0, 0, 0, 0, 0, 326, 0, 0, 3460, 276, 1, 499, 2209, 0, 0, 0, 0, 0, 0, 0, 0, 746, 2381, 979, 1, 1, 0, 1196, 825, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5253, 5628, 6257, 6663, 7298, 5153, 6771, 6129] avg=6144.0000 max=7298 max_rank=4 min=5153 min_rank=5 imbalance_ratio=1.187826 max_load_reduction=41854 max_load_reduction_pct=0.851522 moved_to_echo=25351 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 2839, 358, 81, 0, 15, 2672, 897, 0, 0, 0, 0, 0, 0, 1198, 1152, 12, 0, 4, 210, 1328, 1783, 379, 459, 137, 0, 0, 0, 0, 0, 47, 7, 120, 633, 3367, 853, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 330, 659, 84, 1008, 0, 2195, 5, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 55, 680, 21, 61, 934, 4, 2053, 797, 0, 0, 0, 0, 0, 0, 2, 0, 105, 12, 0, 1, 21, 3, 5327, 446, 0, 0, 0, 0, 0, 0, 635, 1, 0, 3523, 969, 0, 578, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 804, 1734, 0, 0, 0, 1527, 1274, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6868, 6662, 5033, 5869, 4775, 5917, 7337, 6691] avg=6144.0000 max=7337 max_rank=6 min=4775 min_rank=4 imbalance_ratio=1.194173 max_load_reduction=41815 max_load_reduction_pct=0.850728 moved_to_echo=24720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2025, 282, 17, 0, 4, 1773, 672, 0, 0, 0, 0, 0, 0, 706, 238, 1, 0, 2, 43, 3081, 1914, 509, 413, 103, 0, 0, 0, 0, 0, 40, 0, 88, 204, 4510, 2760, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 70, 2078, 116, 345, 0, 1643, 1, 1830, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 69, 1096, 145, 85, 468, 4, 3323, 529, 0, 0, 0, 0, 0, 0, 0, 0, 126, 5, 0, 0, 19, 0, 5957, 724, 0, 0, 0, 0, 0, 0, 275, 0, 0, 3173, 565, 0, 145, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 331, 1295, 1152, 0, 0, 0, 739, 952, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4773, 7010, 7603, 6083, 5907, 6831, 6476, 4469] avg=6144.0000 max=7603 max_rank=2 min=4469 min_rank=7 imbalance_ratio=1.237467 max_load_reduction=41549 max_load_reduction_pct=0.845317 moved_to_echo=23683 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 0, 2145, 217, 47, 5, 16, 2551, 1026, 0, 0, 0, 0, 0, 0, 75, 732, 1, 5, 0, 49, 3276, 1840, 278, 343, 157, 0, 0, 0, 0, 0, 33, 0, 507, 527, 2468, 3226, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 284, 635, 674, 698, 0, 2510, 29, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 158, 1, 70, 561, 209, 238, 1292, 27, 1809, 761, 0, 0, 0, 0, 0, 0, 0, 2, 100, 25, 0, 0, 43, 0, 5492, 395, 0, 0, 0, 0, 0, 0, 860, 1, 0, 2634, 422, 0, 72, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 575, 1897, 1656, 9, 6, 3, 692, 370, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6037, 6756, 6764, 6908, 5126, 6057, 6296, 5208] avg=6144.0000 max=6908 max_rank=3 min=5126 min_rank=4 imbalance_ratio=1.124349 max_load_reduction=42244 max_load_reduction_pct=0.859456 moved_to_echo=22687 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 2369, 407, 3, 8, 24, 2410, 1067, 0, 0, 0, 0, 0, 0, 335, 606, 1, 0, 4, 98, 2186, 897, 404, 404, 167, 0, 0, 0, 0, 0, 161, 12, 73, 213, 3546, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1351, 109, 838, 0, 2593, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 140, 1, 67, 519, 317, 327, 1060, 95, 3062, 720, 0, 0, 0, 0, 0, 0, 6, 0, 41, 8, 0, 20, 107, 0, 5766, 666, 0, 0, 0, 0, 0, 0, 351, 2, 0, 3074, 703, 0, 437, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 1964, 1555, 2, 0, 5, 395, 1224, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6304, 5102, 6417, 6056, 6308, 6614, 6022, 6329] avg=6144.0000 max=6614 max_rank=5 min=5102 min_rank=1 imbalance_ratio=1.076497 max_load_reduction=42538 max_load_reduction_pct=0.865438 moved_to_echo=25273 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 7, 2953, 353, 77, 2, 367, 2424, 514, 0, 0, 0, 0, 0, 0, 897, 1330, 7, 0, 0, 140, 2713, 2444, 245, 193, 78, 0, 0, 0, 0, 0, 115, 10, 196, 205, 2180, 731, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1868, 173, 885, 0, 1257, 12, 2451, 0, 0, 0, 0, 0, 0, 0, 0, 357, 10, 182, 476, 141, 170, 754, 35, 3116, 723, 0, 0, 0, 0, 0, 0, 25, 2, 157, 68, 0, 0, 40, 2, 5621, 678, 0, 0, 0, 0, 0, 0, 490, 1, 0, 1488, 197, 0, 595, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1629, 1574, 4, 67, 5, 1334, 1203, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6740, 8047, 3455, 6825, 5964, 6593, 4731, 6797] avg=6144.0000 max=8047 max_rank=1 min=3455 min_rank=2 imbalance_ratio=1.309733 max_load_reduction=41105 max_load_reduction_pct=0.836283 moved_to_echo=24085 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8064, 13952, 12288, 128, 128, 128, 7296, 7680, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 7296, 128, 128, 128, 768, 16640, 12416, 3072, 3072, 1024, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 256, 128, 20096, 2944, 384, 128, 640, 18944, 6400, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 10112, 2048, 6912, 0, 15488, 128, 13056, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8064, 13952, 12288, 128, 128, 128, 7296, 7680, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1920, 128, 768, 7552, 1536, 1280, 7936, 256, 22656, 5632, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 128, 0, 23168, 4480, 128, 3584, 14464, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 128, 1664, 2560, 27008, 17536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 128, 1280, 256, 128, 128, 384, 128, 42624, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 250, 84, 240, 248, 1884, 5, 1, 0, 4564, 0, 504, 22, 726, 10, 690, 0, 1068, 2405, 1, 1441, 627, 0, 33, 0, 6632, 2, 96, 7025, 166, 0, 0, 535, 3, 54, 0, 144, 46, 30, 20, 54, 224, 779, 3643, 1, 0, 1283, 12, 1, 1, 230, 429, 0, 1, 849, 4561, 4394, 2223, 90, 0, 0, 6, 807, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 267, 213, 106, 150, 2116, 5, 0, 11, 5387, 3, 620, 25, 502, 47, 353, 0, 1750, 2055, 8, 1415, 447, 5, 89, 7, 3909, 13, 351, 6473, 262, 0, 0, 607, 1, 302, 0, 652, 277, 117, 124, 318, 1228, 755, 3155, 0, 12, 356, 51, 0, 6, 380, 333, 3, 0, 408, 4639, 4085, 3242, 294, 6, 0, 26, 1186, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 90, 299, 319, 121, 816, 1, 0, 1, 3515, 3, 331, 56, 428, 23, 122, 0, 2393, 2975, 5, 2328, 1046, 2, 12, 0, 4806, 9, 1045, 6033, 282, 0, 0, 438, 41, 119, 0, 235, 32, 236, 86, 231, 1676, 933, 4311, 1, 0, 692, 24, 0, 0, 387, 1095, 0, 0, 421, 2814, 2697, 3119, 30, 0, 1, 0, 2470, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[5, 55, 396, 249, 35, 2691, 15, 0, 0, 5228, 1, 892, 13, 1627, 3, 195, 0, 2459, 2019, 0, 4289, 299, 0, 2, 0, 3242, 11, 83, 4181, 366, 0, 0, 358, 1, 288, 0, 251, 323, 283, 21, 326, 794, 545, 3770, 2, 0, 327, 23, 2, 0, 730, 404, 9, 0, 571, 4860, 2119, 3386, 18, 0, 0, 6, 1379, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 250, 84, 240, 248, 1884, 5, 1, 0, 4564, 0, 504, 22, 726, 10, 690, 0, 1068, 2405, 1, 1441, 627, 0, 33, 0, 6632, 2, 96, 7025, 166, 0, 0, 535, 3, 54, 0, 144, 46, 30, 20, 54, 224, 779, 3643, 1, 0, 1283, 12, 1, 1, 230, 429, 0, 1, 849, 4561, 4394, 2223, 90, 0, 0, 6, 807, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 132, 769, 521, 327, 1386, 15, 1, 0, 5931, 0, 845, 32, 471, 4, 928, 0, 1295, 2754, 3, 2470, 1297, 0, 1, 1, 4472, 1, 14, 6550, 505, 0, 0, 612, 0, 35, 0, 117, 17, 64, 82, 136, 490, 716, 2546, 0, 2, 797, 20, 0, 0, 222, 199, 0, 0, 1415, 2760, 5251, 1907, 59, 1, 1, 0, 978, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[2, 179, 194, 196, 57, 1100, 19, 1, 36, 5339, 0, 486, 77, 993, 2, 556, 5, 1144, 2623, 21, 4167, 1673, 9, 11, 0, 4136, 13, 142, 5313, 456, 0, 2, 285, 15, 12, 37, 496, 73, 53, 69, 311, 533, 1855, 5547, 4, 7, 2052, 13, 1, 3, 449, 364, 3, 0, 396, 1756, 1791, 844, 36, 2, 0, 5, 3122, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 103, 9, 112, 155, 1750, 8, 0, 0, 5498, 0, 633, 3, 473, 17, 432, 0, 1123, 2662, 0, 1802, 630, 0, 13, 5, 6037, 1, 92, 6978, 170, 0, 0, 872, 0, 45, 0, 324, 55, 12, 2, 33, 587, 628, 3544, 0, 0, 1474, 8, 0, 0, 323, 535, 0, 0, 600, 2264, 4368, 3075, 47, 0, 0, 1, 1649, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[3, 92, 291, 73, 346, 1523, 40, 1, 6, 6117, 0, 415, 13, 412, 3, 469, 0, 1078, 2133, 5, 2032, 845, 1, 10, 0, 6107, 21, 161, 7704, 398, 0, 0, 432, 3, 34, 0, 142, 15, 75, 104, 294, 116, 637, 3377, 1, 0, 302, 11, 0, 0, 357, 551, 0, 5, 304, 3427, 4306, 3002, 86, 1, 1, 0, 1270, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 267, 213, 106, 150, 2116, 5, 0, 2015, 645, 730, 0, 0, 0, 0, 0, 11, 4488, 3, 620, 25, 502, 47, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1750, 2055, 8, 719, 447, 5, 89, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3909, 13, 351, 663, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 1, 302, 0, 652, 277, 117, 124, 5080, 0, 0, 0, 0, 0, 0, 0, 318, 1228, 755, 2901, 0, 12, 356, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 380, 333, 3, 0, 408, 4639, 899, 254, 51, 0, 0, 0, 0, 0, 2070, 3242, 294, 6, 0, 26, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6247, 6049, 5073, 5205, 7160, 5621, 6973, 6824] avg=6144.0000 max=7160 max_rank=4 min=5073 min_rank=2 imbalance_ratio=1.165365 max_load_reduction=41992 max_load_reduction_pct=0.854329 moved_to_echo=26578 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 103, 9, 112, 155, 1750, 8, 0, 0, 5498, 0, 633, 3, 473, 17, 432, 0, 1123, 2662, 0, 1802, 630, 0, 13, 5, 6037, 1, 92, 6978, 170, 0, 0, 872, 0, 45, 0, 324, 55, 12, 2, 33, 587, 628, 3544, 0, 0, 1474, 8, 0, 0, 323, 535, 0, 0, 600, 2264, 4368, 3075, 47, 0, 0, 1, 1649, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[2, 179, 194, 196, 57, 1100, 19, 1, 36, 5339, 0, 486, 77, 993, 2, 556, 5, 1144, 2623, 21, 4167, 1673, 9, 11, 0, 4136, 13, 142, 5313, 456, 0, 2, 285, 15, 12, 37, 496, 73, 53, 69, 311, 533, 1855, 5547, 4, 7, 2052, 13, 1, 3, 449, 364, 3, 0, 396, 1756, 1791, 844, 36, 2, 0, 5, 3122, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 90, 299, 319, 121, 816, 1, 0, 1330, 1062, 681, 0, 0, 0, 0, 0, 1, 2929, 3, 331, 56, 428, 23, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2393, 2975, 5, 1182, 1046, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4806, 9, 1045, 617, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 41, 119, 0, 235, 32, 236, 86, 4735, 0, 0, 0, 0, 0, 0, 0, 231, 1676, 933, 3963, 1, 0, 692, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 1095, 0, 0, 421, 2814, 586, 348, 84, 0, 0, 0, 0, 0, 1367, 3119, 30, 0, 1, 0, 2470, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4719, 3893, 7615, 6759, 5922, 7520, 5735, 6989] avg=6144.0000 max=7615 max_rank=2 min=3893 min_rank=1 imbalance_ratio=1.239421 max_load_reduction=41537 max_load_reduction_pct=0.845072 moved_to_echo=26166 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 103, 9, 112, 155, 1750, 8, 0, 2154, 822, 787, 0, 0, 0, 0, 0, 0, 4581, 0, 633, 3, 473, 17, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 2662, 0, 915, 630, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6037, 1, 92, 715, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 0, 45, 0, 324, 55, 12, 2, 5476, 0, 0, 0, 0, 0, 0, 0, 33, 587, 628, 3258, 0, 0, 1474, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 535, 0, 0, 600, 2264, 917, 286, 65, 0, 0, 0, 0, 0, 2214, 3075, 47, 0, 0, 1, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5900, 6139, 5343, 7020, 6786, 5988, 4990, 6986] avg=6144.0000 max=7020 max_rank=3 min=4990 min_rank=6 imbalance_ratio=1.142578 max_load_reduction=42132 max_load_reduction_pct=0.857178 moved_to_echo=24750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 267, 213, 106, 150, 2116, 5, 0, 2015, 645, 730, 0, 0, 0, 0, 0, 11, 4488, 3, 620, 25, 502, 47, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1750, 2055, 8, 719, 447, 5, 89, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3909, 13, 351, 663, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 1, 302, 0, 652, 277, 117, 124, 5080, 0, 0, 0, 0, 0, 0, 0, 318, 1228, 755, 2901, 0, 12, 356, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 380, 333, 3, 0, 408, 4639, 899, 254, 51, 0, 0, 0, 0, 0, 2070, 3242, 294, 6, 0, 26, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6247, 6049, 5073, 5205, 7160, 5621, 6973, 6824] avg=6144.0000 max=7160 max_rank=4 min=5073 min_rank=2 imbalance_ratio=1.165365 max_load_reduction=41992 max_load_reduction_pct=0.854329 moved_to_echo=26578 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 250, 84, 240, 248, 1884, 5, 1, 2167, 657, 793, 0, 0, 0, 0, 0, 0, 3803, 0, 504, 22, 726, 10, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 2405, 1, 732, 627, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6632, 2, 96, 719, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 3, 54, 0, 144, 46, 30, 20, 5513, 0, 0, 0, 0, 0, 0, 0, 54, 224, 779, 3349, 1, 0, 1283, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 230, 429, 0, 1, 849, 4561, 761, 294, 52, 0, 0, 0, 0, 0, 2227, 2223, 90, 0, 0, 6, 807, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6329, 5755, 4866, 7615, 6345, 5702, 7179, 5361] avg=6144.0000 max=7615 max_rank=3 min=4866 min_rank=2 imbalance_ratio=1.239421 max_load_reduction=41537 max_load_reduction_pct=0.845072 moved_to_echo=24587 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[3, 92, 291, 73, 346, 1523, 40, 1, 2124, 927, 869, 0, 0, 0, 0, 0, 6, 5096, 0, 415, 13, 412, 3, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 2133, 5, 1031, 845, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6107, 21, 161, 789, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 3, 34, 0, 142, 15, 75, 104, 6046, 0, 0, 0, 0, 0, 0, 0, 294, 116, 637, 3105, 1, 0, 302, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 551, 0, 5, 304, 3427, 1021, 272, 74, 0, 0, 0, 0, 0, 2182, 3002, 86, 1, 1, 0, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6289, 6414, 5103, 7476, 6851, 4466, 6011, 6542] avg=6144.0000 max=7476 max_rank=3 min=4466 min_rank=5 imbalance_ratio=1.216797 max_load_reduction=41676 max_load_reduction_pct=0.847900 moved_to_echo=23870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 103, 9, 112, 155, 1750, 8, 0, 2154, 822, 787, 0, 0, 0, 0, 0, 0, 4581, 0, 633, 3, 473, 17, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 2662, 0, 915, 630, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6037, 1, 92, 715, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 0, 45, 0, 324, 55, 12, 2, 5476, 0, 0, 0, 0, 0, 0, 0, 33, 587, 628, 3258, 0, 0, 1474, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 535, 0, 0, 600, 2264, 917, 286, 65, 0, 0, 0, 0, 0, 2214, 3075, 47, 0, 0, 1, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5900, 6139, 5343, 7020, 6786, 5988, 4990, 6986] avg=6144.0000 max=7020 max_rank=3 min=4990 min_rank=6 imbalance_ratio=1.142578 max_load_reduction=42132 max_load_reduction_pct=0.857178 moved_to_echo=24750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 132, 769, 521, 327, 1386, 15, 1, 2590, 1127, 739, 0, 0, 0, 0, 0, 0, 4941, 0, 845, 32, 471, 4, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 2754, 3, 1253, 1297, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4472, 1, 14, 670, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 35, 0, 117, 17, 64, 82, 5141, 0, 0, 0, 0, 0, 0, 0, 136, 490, 716, 2341, 0, 2, 797, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 199, 0, 0, 1415, 2760, 990, 205, 90, 0, 0, 0, 0, 0, 2661, 1907, 59, 1, 1, 0, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7607, 7221, 6603, 5663, 6068, 4502, 5881, 5607] avg=6144.0000 max=7607 max_rank=0 min=4502 min_rank=5 imbalance_ratio=1.238118 max_load_reduction=41545 max_load_reduction_pct=0.845235 moved_to_echo=22058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[5, 55, 396, 249, 35, 2691, 15, 0, 1048, 1959, 477, 0, 0, 0, 0, 0, 0, 4352, 1, 892, 13, 1627, 3, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2459, 2019, 0, 2170, 299, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3242, 11, 83, 419, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 1, 288, 0, 251, 323, 283, 21, 3285, 0, 0, 0, 0, 0, 0, 0, 326, 794, 545, 3462, 2, 0, 327, 23, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 730, 404, 9, 0, 571, 4860, 876, 308, 160, 0, 0, 0, 0, 0, 1071, 3386, 18, 0, 0, 6, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6930, 7083, 6949, 4121, 4810, 5479, 7920, 5860] avg=6144.0000 max=7920 max_rank=6 min=4121 min_rank=3 imbalance_ratio=1.289062 max_load_reduction=41232 max_load_reduction_pct=0.838867 moved_to_echo=24069 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[2, 179, 194, 196, 57, 1100, 19, 1, 883, 1901, 599, 0, 0, 0, 0, 0, 36, 4448, 0, 486, 77, 993, 2, 556, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1144, 2623, 21, 2114, 1673, 9, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4136, 13, 142, 544, 456, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 285, 15, 12, 37, 496, 73, 53, 69, 4170, 0, 0, 0, 0, 0, 0, 0, 311, 533, 1855, 5099, 4, 7, 2052, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 449, 364, 3, 0, 396, 1756, 891, 448, 152, 0, 0, 0, 0, 0, 908, 844, 36, 2, 0, 5, 3122, 66, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5131, 6598, 7600, 5293, 5210, 9874, 4463, 4983] avg=6144.0000 max=9874 max_rank=5 min=4463 min_rank=6 imbalance_ratio=1.607096 max_load_reduction=39278 max_load_reduction_pct=0.799113 moved_to_echo=24530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[3, 92, 291, 73, 346, 1523, 40, 1, 2124, 927, 869, 0, 0, 0, 0, 0, 6, 5096, 0, 415, 13, 412, 3, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 2133, 5, 1031, 845, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6107, 21, 161, 789, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 3, 34, 0, 142, 15, 75, 104, 6046, 0, 0, 0, 0, 0, 0, 0, 294, 116, 637, 3105, 1, 0, 302, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 551, 0, 5, 304, 3427, 1021, 272, 74, 0, 0, 0, 0, 0, 2182, 3002, 86, 1, 1, 0, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6289, 6414, 5103, 7476, 6851, 4466, 6011, 6542] avg=6144.0000 max=7476 max_rank=3 min=4466 min_rank=5 imbalance_ratio=1.216797 max_load_reduction=41676 max_load_reduction_pct=0.847900 moved_to_echo=23870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[5, 55, 396, 249, 35, 2691, 15, 0, 1048, 1959, 477, 0, 0, 0, 0, 0, 0, 4352, 1, 892, 13, 1627, 3, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2459, 2019, 0, 2170, 299, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3242, 11, 83, 419, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 1, 288, 0, 251, 323, 283, 21, 3285, 0, 0, 0, 0, 0, 0, 0, 326, 794, 545, 3462, 2, 0, 327, 23, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 730, 404, 9, 0, 571, 4860, 876, 308, 160, 0, 0, 0, 0, 0, 1071, 3386, 18, 0, 0, 6, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6930, 7083, 6949, 4121, 4810, 5479, 7920, 5860] avg=6144.0000 max=7920 max_rank=6 min=4121 min_rank=3 imbalance_ratio=1.289062 max_load_reduction=41232 max_load_reduction_pct=0.838867 moved_to_echo=24069 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 1280, 2304, 1920, 1536, 13312, 128, 128, 14336, 9216, 5760, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14720, 20864, 768, 128, 128, 128, 12928, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 5760, 6912, 27520, 128, 128, 7296, 256, 0, 0, 0, 0, 0, 0, 0, 0] @@ -4200,591 +4200,591 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[1331, 39, 9, 5914, 0, 8, 316, 525, 5, 261, 67, 1, 3, 0, 75, 2248, 0, 5, 56, 0, 0, 94, 38, 5446, 1, 0, 1, 2, 1539, 3, 1, 0, 51, 2079, 1054, 31, 5430, 0, 1468, 4721, 4, 0, 8, 843, 0, 0, 7885, 40, 13, 438, 0, 7, 0, 16, 6, 0, 319, 4, 0, 2979, 12, 1588, 79, 2089] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[740, 67, 8, 4985, 1, 16, 454, 446, 3, 351, 83, 48, 1, 0, 66, 3349, 0, 2, 54, 1, 0, 7, 9, 5196, 7, 0, 15, 0, 2133, 3, 1, 0, 49, 1576, 108, 793, 6266, 5, 1926, 4660, 0, 0, 0, 581, 4, 0, 7556, 6, 27, 334, 34, 2, 0, 53, 27, 0, 390, 7, 0, 3204, 11, 2622, 95, 770] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[2114, 187, 0, 5991, 0, 6, 201, 262, 0, 56, 180, 5, 0, 0, 104, 2211, 0, 1, 6, 0, 0, 26, 185, 4638, 4, 0, 1, 0, 2670, 0, 0, 0, 21, 1627, 407, 51, 5870, 0, 1713, 4785, 0, 0, 3, 948, 17, 0, 8039, 0, 44, 111, 0, 0, 0, 30, 0, 0, 266, 0, 0, 3285, 5, 1351, 43, 1688] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[1103, 64, 2, 6689, 2, 10, 262, 575, 8, 242, 85, 2, 0, 0, 28, 1331, 0, 0, 27, 0, 0, 16, 18, 4946, 3, 0, 0, 7, 2999, 0, 0, 0, 42, 943, 609, 80, 7064, 1, 1915, 5117, 0, 0, 1, 359, 7, 0, 8051, 2, 41, 363, 0, 0, 1, 12, 35, 1, 411, 0, 0, 2920, 2, 1461, 35, 1260] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[998, 240, 8, 5525, 59, 18, 892, 327, 3, 528, 31, 87, 35, 0, 18, 1231, 2, 16, 46, 1, 22, 22, 34, 2698, 3, 3, 281, 0, 3593, 1, 4, 7, 21, 1918, 262, 28, 6567, 3, 3573, 6066, 1, 8, 0, 219, 120, 68, 7942, 0, 14, 968, 0, 1, 0, 223, 1, 21, 442, 9, 6, 1292, 7, 1614, 93, 932] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[880, 145, 39, 5804, 35, 48, 522, 186, 36, 252, 25, 5, 2, 0, 9, 1519, 1, 1, 8, 0, 11, 240, 4, 3807, 1, 8, 6, 3, 2594, 0, 2, 5, 1, 2505, 651, 7, 5625, 0, 2285, 5859, 0, 0, 0, 301, 41, 0, 7865, 2, 22, 413, 1, 0, 0, 107, 0, 0, 315, 0, 1, 2515, 12, 1578, 73, 2775] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[2011, 467, 18, 5528, 0, 33, 213, 224, 8, 162, 58, 11, 0, 0, 13, 1328, 0, 3, 49, 0, 0, 180, 53, 4128, 2, 0, 1, 0, 2522, 0, 0, 0, 71, 1729, 1387, 56, 5220, 0, 2663, 5869, 0, 0, 9, 1115, 8, 0, 8066, 0, 58, 164, 0, 0, 0, 69, 17, 0, 476, 7, 0, 2599, 1, 1065, 44, 1447] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[1999, 55, 1, 6123, 0, 3, 335, 194, 0, 351, 189, 28, 6, 0, 50, 1181, 0, 2, 13, 15, 0, 67, 27, 3689, 0, 0, 3, 0, 3013, 0, 0, 0, 11, 1803, 229, 51, 5578, 0, 4403, 5705, 0, 0, 0, 809, 3, 0, 7598, 0, 95, 192, 0, 19, 0, 9, 4, 3, 443, 1, 17, 2579, 1, 1394, 87, 774] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[1103, 64, 2, 6689, 2, 10, 262, 575, 8, 242, 85, 2, 0, 0, 28, 1331, 0, 0, 27, 0, 0, 16, 18, 4946, 3, 0, 0, 7, 2999, 0, 0, 0, 42, 943, 609, 80, 7064, 1, 1915, 5117, 0, 0, 1, 359, 7, 0, 8051, 2, 41, 363, 0, 0, 1, 12, 35, 1, 411, 0, 0, 2920, 2, 1461, 35, 1260] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[1331, 39, 9, 5914, 0, 8, 316, 525, 5, 261, 67, 1, 3, 0, 75, 2248, 0, 5, 56, 0, 0, 94, 38, 5446, 1, 0, 1, 2, 1539, 3, 1, 0, 51, 2079, 1054, 31, 5430, 0, 1468, 4721, 4, 0, 8, 843, 0, 0, 7885, 40, 13, 438, 0, 7, 0, 16, 6, 0, 319, 4, 0, 2979, 12, 1588, 79, 2089] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[2114, 187, 0, 5991, 0, 6, 201, 262, 0, 56, 180, 5, 0, 0, 104, 2211, 0, 1, 6, 0, 0, 26, 185, 4638, 4, 0, 1, 0, 2670, 0, 0, 0, 21, 1627, 407, 51, 5870, 0, 1713, 4785, 0, 0, 3, 948, 17, 0, 8039, 0, 44, 111, 0, 0, 0, 30, 0, 0, 266, 0, 0, 3285, 5, 1351, 43, 1688] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[998, 240, 8, 5525, 59, 18, 892, 327, 3, 528, 31, 87, 35, 0, 18, 1231, 2, 16, 46, 1, 22, 22, 34, 2698, 3, 3, 281, 0, 3593, 1, 4, 7, 21, 1918, 262, 28, 6567, 3, 3573, 6066, 1, 8, 0, 219, 120, 68, 7942, 0, 14, 968, 0, 1, 0, 223, 1, 21, 442, 9, 6, 1292, 7, 1614, 93, 932] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[740, 67, 8, 3261, 1, 16, 454, 446, 0, 0, 0, 0, 0, 0, 0, 0, 3, 351, 83, 48, 1, 0, 66, 3349, 3127, 324, 0, 0, 0, 0, 0, 0, 0, 2, 54, 1, 0, 7, 9, 5196, 1381, 51, 0, 0, 0, 0, 0, 0, 7, 0, 15, 0, 2133, 3, 1, 0, 2323, 558, 343, 0, 0, 0, 0, 0, 49, 1576, 108, 793, 0, 5, 1926, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 4, 0, 5233, 6, 0, 0, 0, 0, 0, 0, 0, 0, 27, 334, 34, 2, 0, 53, 27, 0, 5942, 0, 0, 0, 0, 0, 0, 0, 390, 7, 0, 3153, 11, 2622, 95, 770, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4993, 7352, 6701, 5383, 5432, 5824, 6419, 7048] avg=6144.0000 max=7352 max_rank=1 min=4993 min_rank=0 imbalance_ratio=1.196615 max_load_reduction=41800 max_load_reduction_pct=0.850423 moved_to_echo=24723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1331, 39, 9, 3869, 0, 8, 316, 525, 0, 0, 0, 0, 0, 0, 0, 0, 5, 261, 67, 1, 3, 0, 75, 2248, 3168, 280, 0, 0, 0, 0, 0, 0, 0, 5, 56, 0, 0, 94, 38, 5446, 1638, 47, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 1539, 3, 1, 0, 2425, 565, 407, 0, 0, 0, 0, 0, 51, 2079, 1054, 31, 0, 0, 1468, 988, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8, 843, 0, 0, 5460, 40, 0, 0, 0, 0, 0, 0, 0, 0, 13, 438, 0, 7, 0, 16, 6, 0, 5150, 0, 0, 0, 0, 0, 0, 0, 319, 4, 0, 2932, 12, 1588, 79, 2089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6108, 7324, 4944, 5671, 6355, 5630, 7023] avg=6144.0000 max=7324 max_rank=2 min=4944 min_rank=3 imbalance_ratio=1.192057 max_load_reduction=41828 max_load_reduction_pct=0.850993 moved_to_echo=24679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2114, 187, 0, 3920, 0, 6, 201, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 180, 5, 0, 0, 104, 2211, 3210, 303, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 26, 185, 4638, 1659, 52, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 2670, 0, 0, 0, 2472, 573, 412, 0, 0, 0, 0, 0, 21, 1627, 407, 51, 0, 0, 1713, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 948, 17, 0, 5567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 111, 0, 0, 0, 30, 0, 0, 5567, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 3233, 5, 1351, 43, 1688, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6690, 6069, 6567, 6132, 4821, 6535, 5752, 6586] avg=6144.0000 max=6690 max_rank=0 min=4821 min_rank=4 imbalance_ratio=1.088867 max_load_reduction=42462 max_load_reduction_pct=0.863892 moved_to_echo=23694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1103, 64, 2, 4376, 2, 10, 262, 575, 0, 0, 0, 0, 0, 0, 0, 0, 8, 242, 85, 2, 0, 0, 28, 1331, 3433, 366, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 16, 18, 4946, 1853, 46, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 2999, 0, 0, 0, 2476, 613, 460, 0, 0, 0, 0, 0, 42, 943, 609, 80, 0, 1, 1915, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 359, 7, 0, 5575, 2, 0, 0, 0, 0, 0, 0, 0, 0, 41, 363, 0, 0, 1, 12, 35, 1, 6698, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 2874, 2, 1461, 35, 1260, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 5495, 6906, 6558, 4661, 5944, 7151, 6043] avg=6144.0000 max=7151 max_rank=6 min=4661 min_rank=4 imbalance_ratio=1.163900 max_load_reduction=42001 max_load_reduction_pct=0.854513 moved_to_echo=23799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[880, 145, 39, 3798, 35, 48, 522, 186, 0, 0, 0, 0, 0, 0, 0, 0, 36, 252, 25, 5, 2, 0, 9, 1519, 3931, 291, 0, 0, 0, 0, 0, 0, 1, 1, 8, 0, 11, 240, 4, 3807, 1607, 40, 0, 0, 0, 0, 0, 0, 1, 8, 6, 3, 2594, 0, 2, 5, 2418, 702, 399, 0, 0, 0, 0, 0, 1, 2505, 651, 7, 0, 0, 2285, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 41, 0, 5447, 2, 0, 0, 0, 0, 0, 0, 0, 0, 22, 413, 1, 0, 0, 107, 0, 0, 5334, 0, 0, 0, 0, 0, 0, 0, 315, 0, 1, 2475, 12, 1578, 73, 2775, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5653, 6070, 5719, 6138, 6675, 5791, 5877, 7229] avg=6144.0000 max=7229 max_rank=7 min=5653 min_rank=0 imbalance_ratio=1.176595 max_load_reduction=41923 max_load_reduction_pct=0.852926 moved_to_echo=25572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[740, 67, 8, 3261, 1, 16, 454, 446, 0, 0, 0, 0, 0, 0, 0, 0, 3, 351, 83, 48, 1, 0, 66, 3349, 3127, 324, 0, 0, 0, 0, 0, 0, 0, 2, 54, 1, 0, 7, 9, 5196, 1381, 51, 0, 0, 0, 0, 0, 0, 7, 0, 15, 0, 2133, 3, 1, 0, 2323, 558, 343, 0, 0, 0, 0, 0, 49, 1576, 108, 793, 0, 5, 1926, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 4, 0, 5233, 6, 0, 0, 0, 0, 0, 0, 0, 0, 27, 334, 34, 2, 0, 53, 27, 0, 5942, 0, 0, 0, 0, 0, 0, 0, 390, 7, 0, 3153, 11, 2622, 95, 770, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4993, 7352, 6701, 5383, 5432, 5824, 6419, 7048] avg=6144.0000 max=7352 max_rank=1 min=4993 min_rank=0 imbalance_ratio=1.196615 max_load_reduction=41800 max_load_reduction_pct=0.850423 moved_to_echo=24723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[998, 240, 8, 3615, 59, 18, 892, 327, 0, 0, 0, 0, 0, 0, 0, 0, 3, 528, 31, 87, 35, 0, 18, 1231, 4070, 340, 0, 0, 0, 0, 0, 0, 2, 16, 46, 1, 22, 22, 34, 2698, 1530, 20, 0, 0, 0, 0, 0, 0, 3, 3, 281, 0, 3593, 1, 4, 7, 2442, 726, 380, 0, 0, 0, 0, 0, 21, 1918, 262, 28, 0, 3, 3573, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 219, 120, 68, 5500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 968, 0, 1, 0, 223, 1, 21, 6227, 0, 0, 0, 0, 0, 0, 0, 442, 9, 6, 1272, 7, 1614, 93, 932, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6343, 4391, 7440, 7075, 5916, 7455, 4375] avg=6144.0000 max=7455 max_rank=6 min=4375 min_rank=7 imbalance_ratio=1.213379 max_load_reduction=41697 max_load_reduction_pct=0.848328 moved_to_echo=24821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2011, 467, 18, 3617, 0, 33, 213, 224, 0, 0, 0, 0, 0, 0, 0, 0, 8, 162, 58, 11, 0, 0, 13, 1328, 3938, 271, 0, 0, 0, 0, 0, 0, 0, 3, 49, 0, 0, 180, 53, 4128, 1531, 41, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 2522, 0, 0, 0, 2480, 703, 380, 0, 0, 0, 0, 0, 71, 1729, 1387, 56, 0, 0, 2663, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1115, 8, 0, 5586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 164, 0, 0, 0, 69, 17, 0, 4949, 0, 0, 0, 0, 0, 0, 0, 476, 7, 0, 2558, 1, 1065, 44, 1447, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6583, 5789, 5985, 6088, 7134, 6718, 5257, 5598] avg=6144.0000 max=7134 max_rank=4 min=5257 min_rank=6 imbalance_ratio=1.161133 max_load_reduction=42018 max_load_reduction_pct=0.854858 moved_to_echo=24707 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1103, 64, 2, 4376, 2, 10, 262, 575, 0, 0, 0, 0, 0, 0, 0, 0, 8, 242, 85, 2, 0, 0, 28, 1331, 3433, 366, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 16, 18, 4946, 1853, 46, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 2999, 0, 0, 0, 2476, 613, 460, 0, 0, 0, 0, 0, 42, 943, 609, 80, 0, 1, 1915, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 359, 7, 0, 5575, 2, 0, 0, 0, 0, 0, 0, 0, 0, 41, 363, 0, 0, 1, 12, 35, 1, 6698, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 2874, 2, 1461, 35, 1260, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 5495, 6906, 6558, 4661, 5944, 7151, 6043] avg=6144.0000 max=7151 max_rank=6 min=4661 min_rank=4 imbalance_ratio=1.163900 max_load_reduction=42001 max_load_reduction_pct=0.854513 moved_to_echo=23799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2114, 187, 0, 3920, 0, 6, 201, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 180, 5, 0, 0, 104, 2211, 3210, 303, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 26, 185, 4638, 1659, 52, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 2670, 0, 0, 0, 2472, 573, 412, 0, 0, 0, 0, 0, 21, 1627, 407, 51, 0, 0, 1713, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 948, 17, 0, 5567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 111, 0, 0, 0, 30, 0, 0, 5567, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 3233, 5, 1351, 43, 1688, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6690, 6069, 6567, 6132, 4821, 6535, 5752, 6586] avg=6144.0000 max=6690 max_rank=0 min=4821 min_rank=4 imbalance_ratio=1.088867 max_load_reduction=42462 max_load_reduction_pct=0.863892 moved_to_echo=23694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1999, 55, 1, 3998, 0, 3, 335, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 189, 28, 6, 0, 50, 1181, 3832, 289, 0, 0, 0, 0, 0, 0, 0, 2, 13, 15, 0, 67, 27, 3689, 1700, 46, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3013, 0, 0, 0, 2341, 687, 425, 0, 0, 0, 0, 0, 11, 1803, 229, 51, 0, 0, 4403, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 809, 3, 0, 5257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 192, 0, 19, 0, 9, 4, 3, 5289, 0, 0, 0, 0, 0, 0, 0, 443, 1, 17, 2533, 1, 1394, 87, 774, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6585, 5926, 5559, 6469, 7683, 6069, 5611, 5250] avg=6144.0000 max=7683 max_rank=4 min=5250 min_rank=7 imbalance_ratio=1.250488 max_load_reduction=41469 max_load_reduction_pct=0.843689 moved_to_echo=24613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1331, 39, 9, 3869, 0, 8, 316, 525, 0, 0, 0, 0, 0, 0, 0, 0, 5, 261, 67, 1, 3, 0, 75, 2248, 3168, 280, 0, 0, 0, 0, 0, 0, 0, 5, 56, 0, 0, 94, 38, 5446, 1638, 47, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 1539, 3, 1, 0, 2425, 565, 407, 0, 0, 0, 0, 0, 51, 2079, 1054, 31, 0, 0, 1468, 988, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8, 843, 0, 0, 5460, 40, 0, 0, 0, 0, 0, 0, 0, 0, 13, 438, 0, 7, 0, 16, 6, 0, 5150, 0, 0, 0, 0, 0, 0, 0, 319, 4, 0, 2932, 12, 1588, 79, 2089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6108, 7324, 4944, 5671, 6355, 5630, 7023] avg=6144.0000 max=7324 max_rank=2 min=4944 min_rank=3 imbalance_ratio=1.192057 max_load_reduction=41828 max_load_reduction_pct=0.850993 moved_to_echo=24679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[998, 240, 8, 3615, 59, 18, 892, 327, 0, 0, 0, 0, 0, 0, 0, 0, 3, 528, 31, 87, 35, 0, 18, 1231, 4070, 340, 0, 0, 0, 0, 0, 0, 2, 16, 46, 1, 22, 22, 34, 2698, 1530, 20, 0, 0, 0, 0, 0, 0, 3, 3, 281, 0, 3593, 1, 4, 7, 2442, 726, 380, 0, 0, 0, 0, 0, 21, 1918, 262, 28, 0, 3, 3573, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 219, 120, 68, 5500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 968, 0, 1, 0, 223, 1, 21, 6227, 0, 0, 0, 0, 0, 0, 0, 442, 9, 6, 1272, 7, 1614, 93, 932, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6343, 4391, 7440, 7075, 5916, 7455, 4375] avg=6144.0000 max=7455 max_rank=6 min=4375 min_rank=7 imbalance_ratio=1.213379 max_load_reduction=41697 max_load_reduction_pct=0.848328 moved_to_echo=24821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 14208, 4736, 1152, 0, 128, 19968, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11264, 1280, 128, 30464, 128, 256, 3200, 2816, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50304 observed_loss=-1152 tokens_per_expert=[128, 128, 384, 128, 21120, 128, 128, 128, 19456, 5248, 3328, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 128, 128, 21120, 128, 12800, 640, 11776, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2304, 768, 256, 128, 0, 384, 14464, 28800, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11264, 1280, 128, 30464, 128, 256, 3200, 2816, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 128, 128, 21120, 128, 12800, 640, 11776, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 384, 128, 128, 768, 384, 34560, 12928, 384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 5248, 256, 128, 43648, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 3072, 128, 128, 128, 640, 128, 128, 45184, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50304 slack_slots=48000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 119, 901, 13, 17, 7510, 5, 70, 102, 0, 2, 1752, 0, 1544, 2, 29, 7283, 824, 0, 0, 0, 0, 112, 29, 8, 0, 0, 0, 39, 112, 127, 1275, 512, 0, 0, 2, 71, 924, 0, 52, 7, 2587, 0, 0, 2347, 3499, 1, 20, 7016, 0, 2379, 12, 0, 6258, 15, 0, 3, 1557, 3, 6, 3, 3, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[2, 0, 120, 1424, 2, 68, 6672, 2, 71, 92, 10, 0, 4435, 0, 762, 14, 1, 6201, 666, 0, 1, 1, 12, 146, 165, 6, 0, 0, 2, 24, 4, 11, 1602, 580, 0, 9, 0, 1, 1922, 0, 44, 100, 3449, 0, 0, 1184, 4894, 0, 18, 5304, 0, 1068, 7, 7, 6769, 5, 0, 0, 1227, 0, 25, 9, 0, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 73, 82, 1671, 14, 128, 7444, 27, 170, 241, 4, 15, 3360, 0, 641, 41, 68, 7621, 704, 4, 0, 1, 13, 174, 279, 1, 0, 0, 2, 68, 7, 22, 943, 2118, 6, 3, 2, 35, 1012, 73, 55, 29, 3141, 1, 0, 1171, 3235, 0, 58, 6745, 0, 1823, 55, 5, 4497, 22, 1, 3, 1206, 1, 7, 1, 0, 29] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 51, 49, 1021, 6, 47, 7434, 4, 338, 738, 2, 0, 3819, 0, 1338, 2, 6, 7593, 793, 0, 1, 1, 2, 171, 31, 0, 0, 0, 0, 8, 11, 0, 1301, 728, 5, 19, 0, 39, 1071, 0, 15, 7, 2778, 0, 1, 1011, 4518, 0, 13, 6475, 1, 1227, 10, 1, 5355, 6, 0, 13, 1089, 0, 0, 1, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[2, 0, 120, 1424, 2, 68, 6672, 2, 71, 92, 10, 0, 4435, 0, 762, 14, 1, 6201, 666, 0, 1, 1, 12, 146, 165, 6, 0, 0, 2, 24, 4, 11, 1602, 580, 0, 9, 0, 1, 1922, 0, 44, 100, 3449, 0, 0, 1184, 4894, 0, 18, 5304, 0, 1068, 7, 7, 6769, 5, 0, 0, 1227, 0, 25, 9, 0, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[66, 101, 50, 2283, 6, 26, 6456, 9, 640, 463, 25, 0, 1192, 1, 910, 173, 0, 6603, 1009, 6, 0, 9, 38, 269, 98, 61, 0, 28, 0, 16, 91, 35, 1637, 841, 2, 4, 43, 30, 1851, 0, 74, 69, 5320, 0, 3, 756, 4555, 9, 82, 6725, 0, 1346, 286, 14, 2994, 164, 5, 6, 1568, 1, 4, 73, 2, 24] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 1, 160, 1260, 5, 10, 6324, 1, 110, 225, 0, 0, 3510, 0, 1498, 1, 5, 7353, 1230, 0, 0, 1, 1, 55, 67, 1, 0, 1, 0, 15, 179, 9, 982, 711, 0, 4, 0, 35, 998, 0, 20, 18, 4218, 0, 0, 2366, 3011, 0, 17, 7081, 0, 1633, 548, 0, 4530, 17, 0, 2, 925, 0, 12, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 31, 18, 1883, 1, 15, 4664, 5, 40, 101, 3, 0, 2053, 0, 1061, 3, 13, 6904, 503, 0, 0, 0, 15, 55, 33, 7, 0, 0, 0, 24, 77, 13, 2137, 735, 14, 13, 2, 9, 1561, 5, 23, 53, 5385, 0, 0, 2255, 4920, 3, 22, 6304, 0, 1042, 11, 0, 6086, 36, 0, 1, 1008, 0, 4, 1, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[1, 1, 115, 856, 2, 6, 6278, 6, 106, 192, 0, 0, 2952, 0, 1557, 2, 1, 7329, 975, 0, 0, 4, 1, 216, 197, 0, 0, 2, 0, 13, 6, 9, 1641, 1457, 3, 0, 0, 26, 2232, 1, 67, 8, 4080, 0, 0, 1107, 3990, 2, 85, 6469, 1, 767, 30, 1, 5274, 16, 0, 4, 1052, 2, 3, 0, 0, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[66, 101, 50, 2283, 6, 26, 6456, 9, 640, 463, 25, 0, 1192, 1, 910, 173, 0, 6603, 1009, 6, 0, 9, 38, 269, 98, 61, 0, 28, 0, 16, 91, 35, 1637, 841, 2, 4, 43, 30, 1851, 0, 74, 69, 5320, 0, 3, 756, 4555, 9, 82, 6725, 0, 1346, 286, 14, 2994, 164, 5, 6, 1568, 1, 4, 73, 2, 24] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 119, 901, 13, 17, 7510, 5, 70, 102, 0, 2, 1752, 0, 1544, 2, 29, 7283, 824, 0, 0, 0, 0, 112, 29, 8, 0, 0, 0, 39, 112, 127, 1275, 512, 0, 0, 2, 71, 924, 0, 52, 7, 2587, 0, 0, 2347, 3499, 1, 20, 7016, 0, 2379, 12, 0, 6258, 15, 0, 3, 1557, 3, 6, 3, 3, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 119, 901, 13, 17, 5178, 5, 0, 0, 0, 0, 0, 0, 0, 0, 70, 102, 0, 2, 1752, 0, 1544, 2, 933, 821, 0, 0, 0, 0, 0, 0, 29, 5252, 824, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 29, 8, 0, 0, 0, 39, 112, 127, 6355, 0, 0, 0, 0, 0, 0, 0, 1275, 512, 0, 0, 2, 71, 924, 0, 1329, 1098, 0, 0, 0, 0, 0, 0, 52, 7, 2587, 0, 0, 2347, 573, 1, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 2379, 12, 0, 5437, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1557, 3, 6, 3, 3, 0, 2926, 1003, 661, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6233, 5226, 6217, 6670, 5211, 5567, 7863, 6165] avg=6144.0000 max=7863 max_rank=6 min=5211 min_rank=4 imbalance_ratio=1.279785 max_load_reduction=41289 max_load_reduction_pct=0.840027 moved_to_echo=24806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 31, 18, 1883, 1, 15, 3216, 5, 0, 0, 0, 0, 0, 0, 0, 0, 40, 101, 3, 0, 2053, 0, 1061, 3, 884, 798, 0, 0, 0, 0, 0, 0, 13, 4979, 503, 0, 0, 0, 15, 55, 0, 0, 0, 0, 0, 0, 0, 0, 33, 7, 0, 0, 0, 24, 77, 13, 5710, 0, 0, 0, 0, 0, 0, 0, 2137, 735, 14, 13, 2, 9, 1561, 5, 825, 1041, 0, 0, 0, 0, 0, 0, 23, 53, 5385, 0, 0, 2255, 805, 3, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1042, 11, 0, 5288, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1008, 0, 4, 1, 0, 0, 4115, 623, 594, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5169, 4943, 5565, 5864, 6342, 8524, 6399, 6346] avg=6144.0000 max=8524 max_rank=5 min=4943 min_rank=1 imbalance_ratio=1.387370 max_load_reduction=40628 max_load_reduction_pct=0.826579 moved_to_echo=27611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 160, 1260, 5, 10, 4361, 1, 0, 0, 0, 0, 0, 0, 0, 0, 110, 225, 0, 0, 3510, 0, 1498, 1, 942, 594, 0, 0, 0, 0, 0, 0, 5, 5302, 1230, 0, 0, 1, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 67, 1, 0, 1, 0, 15, 179, 9, 6414, 0, 0, 0, 0, 0, 0, 0, 982, 711, 0, 4, 0, 35, 998, 0, 1119, 1109, 0, 0, 0, 0, 0, 0, 20, 18, 4218, 0, 0, 2366, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1633, 548, 0, 3936, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 925, 0, 12, 0, 0, 2, 2518, 844, 667, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5798, 6880, 6594, 6686, 4958, 7115, 6151, 4970] avg=6144.0000 max=7115 max_rank=5 min=4958 min_rank=4 imbalance_ratio=1.158040 max_load_reduction=42037 max_load_reduction_pct=0.855245 moved_to_echo=23194 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 73, 82, 1671, 14, 128, 5133, 27, 0, 0, 0, 0, 0, 0, 0, 0, 170, 241, 4, 15, 3360, 0, 641, 41, 976, 590, 0, 0, 0, 0, 0, 0, 68, 5496, 704, 4, 0, 1, 13, 174, 0, 0, 0, 0, 0, 0, 0, 0, 279, 1, 0, 0, 2, 68, 7, 22, 6110, 0, 0, 0, 0, 0, 0, 0, 943, 2118, 6, 3, 2, 35, 1012, 73, 1317, 1149, 0, 0, 0, 0, 0, 0, 55, 29, 3141, 1, 0, 1171, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 1823, 55, 5, 3907, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1206, 1, 7, 1, 0, 29, 2705, 994, 635, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7128, 6038, 6460, 6489, 6658, 4927, 5870, 5582] avg=6144.0000 max=7128 max_rank=0 min=4927 min_rank=5 imbalance_ratio=1.160156 max_load_reduction=42024 max_load_reduction_pct=0.854980 moved_to_echo=23037 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 49, 1021, 6, 47, 5126, 4, 0, 0, 0, 0, 0, 0, 0, 0, 338, 738, 2, 0, 3819, 0, 1338, 2, 972, 702, 0, 0, 0, 0, 0, 0, 6, 5476, 793, 0, 1, 1, 2, 171, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 8, 11, 0, 5866, 0, 0, 0, 0, 0, 0, 0, 1301, 728, 5, 19, 0, 39, 1071, 0, 1315, 1145, 0, 0, 0, 0, 0, 0, 15, 7, 2778, 0, 1, 1011, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 1227, 10, 1, 4653, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1089, 0, 0, 1, 0, 2, 3778, 993, 609, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6304, 7911, 6450, 5916, 5623, 4552, 5911, 6485] avg=6144.0000 max=7911 max_rank=1 min=4552 min_rank=5 imbalance_ratio=1.287598 max_load_reduction=41241 max_load_reduction_pct=0.839050 moved_to_echo=22571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 160, 1260, 5, 10, 4361, 1, 0, 0, 0, 0, 0, 0, 0, 0, 110, 225, 0, 0, 3510, 0, 1498, 1, 942, 594, 0, 0, 0, 0, 0, 0, 5, 5302, 1230, 0, 0, 1, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 67, 1, 0, 1, 0, 15, 179, 9, 6414, 0, 0, 0, 0, 0, 0, 0, 982, 711, 0, 4, 0, 35, 998, 0, 1119, 1109, 0, 0, 0, 0, 0, 0, 20, 18, 4218, 0, 0, 2366, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1633, 548, 0, 3936, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 925, 0, 12, 0, 0, 2, 2518, 844, 667, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5798, 6880, 6594, 6686, 4958, 7115, 6151, 4970] avg=6144.0000 max=7115 max_rank=5 min=4958 min_rank=4 imbalance_ratio=1.158040 max_load_reduction=42037 max_load_reduction_pct=0.855245 moved_to_echo=23194 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 31, 18, 1883, 1, 15, 3216, 5, 0, 0, 0, 0, 0, 0, 0, 0, 40, 101, 3, 0, 2053, 0, 1061, 3, 884, 798, 0, 0, 0, 0, 0, 0, 13, 4979, 503, 0, 0, 0, 15, 55, 0, 0, 0, 0, 0, 0, 0, 0, 33, 7, 0, 0, 0, 24, 77, 13, 5710, 0, 0, 0, 0, 0, 0, 0, 2137, 735, 14, 13, 2, 9, 1561, 5, 825, 1041, 0, 0, 0, 0, 0, 0, 23, 53, 5385, 0, 0, 2255, 805, 3, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1042, 11, 0, 5288, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1008, 0, 4, 1, 0, 0, 4115, 623, 594, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5169, 4943, 5565, 5864, 6342, 8524, 6399, 6346] avg=6144.0000 max=8524 max_rank=5 min=4943 min_rank=1 imbalance_ratio=1.387370 max_load_reduction=40628 max_load_reduction_pct=0.826579 moved_to_echo=27611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 115, 856, 2, 6, 4329, 6, 0, 0, 0, 0, 0, 0, 0, 0, 106, 192, 0, 0, 2952, 0, 1557, 2, 938, 692, 0, 0, 0, 0, 0, 0, 1, 5286, 975, 0, 0, 4, 1, 216, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 2, 0, 13, 6, 9, 5861, 0, 0, 0, 0, 0, 0, 0, 1641, 1457, 3, 0, 0, 26, 2232, 1, 1111, 1105, 0, 0, 0, 0, 0, 0, 67, 8, 4080, 0, 0, 1107, 653, 2, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 1, 767, 30, 1, 4582, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1052, 2, 3, 0, 0, 7, 3337, 838, 608, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5316, 6439, 6483, 6088, 7576, 5917, 5482, 5851] avg=6144.0000 max=7576 max_rank=4 min=5316 min_rank=0 imbalance_ratio=1.233073 max_load_reduction=41576 max_load_reduction_pct=0.845866 moved_to_echo=24826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 120, 1424, 2, 68, 4593, 2, 0, 0, 0, 0, 0, 0, 0, 0, 71, 92, 10, 0, 4435, 0, 762, 14, 798, 892, 0, 0, 0, 0, 0, 0, 1, 4463, 666, 0, 1, 1, 12, 146, 0, 0, 0, 0, 0, 0, 0, 0, 165, 6, 0, 0, 2, 24, 4, 11, 4805, 0, 0, 0, 0, 0, 0, 0, 1602, 580, 0, 9, 0, 1, 1922, 0, 1184, 940, 0, 0, 0, 0, 0, 0, 44, 100, 3449, 0, 0, 1184, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 1068, 7, 7, 5877, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 0, 25, 9, 0, 14, 4098, 895, 499, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 7074, 5290, 5017, 6238, 5573, 6982, 6767] avg=6144.0000 max=7074 max_rank=1 min=5017 min_rank=3 imbalance_ratio=1.151367 max_load_reduction=42078 max_load_reduction_pct=0.856079 moved_to_echo=25560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 119, 901, 13, 17, 5178, 5, 0, 0, 0, 0, 0, 0, 0, 0, 70, 102, 0, 2, 1752, 0, 1544, 2, 933, 821, 0, 0, 0, 0, 0, 0, 29, 5252, 824, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 29, 8, 0, 0, 0, 39, 112, 127, 6355, 0, 0, 0, 0, 0, 0, 0, 1275, 512, 0, 0, 2, 71, 924, 0, 1329, 1098, 0, 0, 0, 0, 0, 0, 52, 7, 2587, 0, 0, 2347, 573, 1, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 2379, 12, 0, 5437, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1557, 3, 6, 3, 3, 0, 2926, 1003, 661, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6233, 5226, 6217, 6670, 5211, 5567, 7863, 6165] avg=6144.0000 max=7863 max_rank=6 min=5211 min_rank=4 imbalance_ratio=1.279785 max_load_reduction=41289 max_load_reduction_pct=0.840027 moved_to_echo=24806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[66, 101, 50, 2283, 6, 26, 4452, 9, 0, 0, 0, 0, 0, 0, 0, 0, 640, 463, 25, 0, 1192, 1, 910, 173, 845, 392, 0, 0, 0, 0, 0, 0, 0, 4762, 1009, 6, 0, 9, 38, 269, 0, 0, 0, 0, 0, 0, 0, 0, 98, 61, 0, 28, 0, 16, 91, 35, 6093, 0, 0, 0, 0, 0, 0, 0, 1637, 841, 2, 4, 43, 30, 1851, 0, 1142, 996, 0, 0, 0, 0, 0, 0, 74, 69, 5320, 0, 3, 756, 746, 9, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 1346, 286, 14, 2602, 164, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 1568, 1, 4, 73, 2, 24, 3809, 862, 632, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6993, 4641, 6093, 6422, 6546, 6977, 4494, 6986] avg=6144.0000 max=6993 max_rank=0 min=4494 min_rank=6 imbalance_ratio=1.138184 max_load_reduction=42159 max_load_reduction_pct=0.857727 moved_to_echo=25003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 120, 1424, 2, 68, 4593, 2, 0, 0, 0, 0, 0, 0, 0, 0, 71, 92, 10, 0, 4435, 0, 762, 14, 798, 892, 0, 0, 0, 0, 0, 0, 1, 4463, 666, 0, 1, 1, 12, 146, 0, 0, 0, 0, 0, 0, 0, 0, 165, 6, 0, 0, 2, 24, 4, 11, 4805, 0, 0, 0, 0, 0, 0, 0, 1602, 580, 0, 9, 0, 1, 1922, 0, 1184, 940, 0, 0, 0, 0, 0, 0, 44, 100, 3449, 0, 0, 1184, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 1068, 7, 7, 5877, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 0, 25, 9, 0, 14, 4098, 895, 499, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 7074, 5290, 5017, 6238, 5573, 6982, 6767] avg=6144.0000 max=7074 max_rank=1 min=5017 min_rank=3 imbalance_ratio=1.151367 max_load_reduction=42078 max_load_reduction_pct=0.856079 moved_to_echo=25560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 115, 856, 2, 6, 4329, 6, 0, 0, 0, 0, 0, 0, 0, 0, 106, 192, 0, 0, 2952, 0, 1557, 2, 938, 692, 0, 0, 0, 0, 0, 0, 1, 5286, 975, 0, 0, 4, 1, 216, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 2, 0, 13, 6, 9, 5861, 0, 0, 0, 0, 0, 0, 0, 1641, 1457, 3, 0, 0, 26, 2232, 1, 1111, 1105, 0, 0, 0, 0, 0, 0, 67, 8, 4080, 0, 0, 1107, 653, 2, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 1, 767, 30, 1, 4582, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1052, 2, 3, 0, 0, 7, 3337, 838, 608, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5316, 6439, 6483, 6088, 7576, 5917, 5482, 5851] avg=6144.0000 max=7576 max_rank=4 min=5316 min_rank=0 imbalance_ratio=1.233073 max_load_reduction=41576 max_load_reduction_pct=0.845866 moved_to_echo=24826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 2176, 128, 128, 23168, 128, 9344, 256, 7296, 5504, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11520, 7808, 128, 128, 128, 256, 11648, 128, 9344, 8704, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 128, 9728, 128, 128, 128, 128, 128, 27392, 7168, 4992, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 384, 768, 11392, 128, 384, 36480, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 0, 128, 11392, 1024, 128, 36352, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 384, 30976, 128, 128, 12288, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 384, 768, 11392, 128, 384, 36480, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 41088, 6784, 128, 128, 128, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1024, 128, 0, 128, 128, 256, 512, 256, 47232, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[2, 0, 5963, 20, 21, 0, 3, 0, 0, 0, 13, 0, 0, 2571, 2459, 157, 4868, 1034, 370, 8, 729, 2383, 1, 435, 0, 0, 0, 53, 986, 85, 1533, 67, 11, 1, 4, 5, 1036, 6, 4, 1065, 380, 12, 1, 2797, 4, 1, 5268, 3, 1040, 0, 743, 4499, 69, 909, 362, 7, 2, 329, 53, 6744, 1, 19, 2, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[6, 1, 6357, 0, 11, 0, 0, 1, 0, 31, 3, 0, 4, 2298, 1011, 140, 4115, 665, 317, 10, 273, 3737, 0, 564, 201, 0, 2, 9, 729, 4, 2441, 34, 0, 1, 25, 1, 770, 15, 5, 2943, 112, 2, 0, 1339, 0, 0, 5964, 1, 1224, 3, 855, 3558, 261, 1022, 1131, 5, 0, 164, 52, 6718, 9, 4, 3, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[45, 0, 6418, 0, 2, 0, 7, 0, 1, 5, 0, 0, 0, 2530, 874, 24, 3112, 723, 76, 2, 216, 2760, 0, 949, 76, 0, 0, 0, 125, 3, 1893, 65, 0, 0, 4, 10, 758, 22, 6, 1836, 404, 0, 0, 1630, 3, 0, 6517, 0, 1845, 0, 1851, 4720, 112, 1649, 843, 0, 0, 309, 61, 6631, 1, 33, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[106, 0, 7135, 0, 3, 0, 0, 2, 0, 2, 1, 0, 2, 2714, 1214, 12, 2721, 1326, 88, 8, 232, 2184, 0, 297, 58, 0, 0, 3, 26, 24, 1105, 137, 3, 0, 0, 1, 2045, 10, 4, 933, 8, 25, 0, 683, 0, 1, 7010, 3, 2120, 12, 1937, 4857, 910, 1517, 258, 0, 4, 322, 43, 7031, 7, 4, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[35, 0, 7307, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 1240, 54, 2750, 1158, 171, 3, 287, 2358, 0, 84, 19, 0, 0, 0, 561, 1, 2521, 40, 0, 0, 0, 0, 886, 9, 2, 2645, 7, 0, 0, 2229, 0, 0, 6821, 0, 2107, 0, 589, 4017, 194, 1364, 280, 0, 0, 428, 40, 6974, 4, 3, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[160, 0, 5990, 0, 3, 0, 0, 6, 0, 2, 1, 0, 71, 1255, 2606, 20, 3212, 566, 437, 0, 181, 1656, 2, 1030, 7, 0, 0, 0, 930, 4, 2080, 40, 2, 0, 1, 22, 1273, 1, 0, 1002, 49, 0, 0, 708, 12, 0, 6627, 0, 4156, 1, 641, 5451, 154, 835, 772, 5, 3, 62, 159, 6940, 5, 8, 4, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[45, 0, 6418, 0, 2, 0, 7, 0, 1, 5, 0, 0, 0, 2530, 874, 24, 3112, 723, 76, 2, 216, 2760, 0, 949, 76, 0, 0, 0, 125, 3, 1893, 65, 0, 0, 4, 10, 758, 22, 6, 1836, 404, 0, 0, 1630, 3, 0, 6517, 0, 1845, 0, 1851, 4720, 112, 1649, 843, 0, 0, 309, 61, 6631, 1, 33, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[84, 0, 7024, 0, 2, 0, 0, 12, 0, 3, 65, 0, 5, 2991, 2520, 266, 1678, 773, 621, 65, 285, 3390, 0, 741, 441, 0, 1, 15, 292, 0, 2161, 120, 0, 0, 1, 22, 1105, 26, 0, 581, 146, 9, 2, 728, 79, 0, 7055, 0, 1759, 0, 851, 4352, 311, 999, 743, 4, 2, 39, 106, 6578, 69, 27, 2, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[2, 0, 5963, 20, 21, 0, 3, 0, 0, 0, 13, 0, 0, 2571, 2459, 157, 4868, 1034, 370, 8, 729, 2383, 1, 435, 0, 0, 0, 53, 986, 85, 1533, 67, 11, 1, 4, 5, 1036, 6, 4, 1065, 380, 12, 1, 2797, 4, 1, 5268, 3, 1040, 0, 743, 4499, 69, 909, 362, 7, 2, 329, 53, 6744, 1, 19, 2, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[160, 0, 5990, 0, 3, 0, 0, 6, 0, 2, 1, 0, 71, 1255, 2606, 20, 3212, 566, 437, 0, 181, 1656, 2, 1030, 7, 0, 0, 0, 930, 4, 2080, 40, 2, 0, 1, 22, 1273, 1, 0, 1002, 49, 0, 0, 708, 12, 0, 6627, 0, 4156, 1, 641, 5451, 154, 835, 772, 5, 3, 62, 159, 6940, 5, 8, 4, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[19, 5, 7244, 0, 70, 0, 0, 28, 0, 1, 0, 0, 11, 2357, 1139, 188, 2666, 1479, 149, 5, 323, 2301, 0, 174, 346, 3, 0, 12, 385, 15, 2922, 14, 6, 0, 22, 39, 850, 31, 11, 455, 12, 1, 13, 965, 4, 0, 6864, 0, 1148, 13, 2089, 2090, 2296, 3347, 603, 52, 14, 326, 292, 5732, 2, 3, 3, 13] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 5412, 20, 21, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2571, 2459, 157, 916, 551, 801, 0, 0, 0, 0, 0, 2234, 1034, 370, 8, 729, 2383, 1, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 986, 85, 1533, 67, 1551, 1833, 235, 0, 0, 0, 0, 0, 11, 1, 4, 5, 1036, 6, 4, 1065, 3840, 0, 0, 0, 0, 0, 0, 0, 380, 12, 1, 2797, 4, 1, 3717, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 743, 424, 69, 909, 362, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 329, 53, 5828, 1, 19, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5458, 7468, 7194, 6343, 5972, 6915, 3554, 6248] avg=6144.0000 max=7468 max_rank=1 min=3554 min_rank=6 imbalance_ratio=1.215495 max_load_reduction=41684 max_load_reduction_pct=0.848063 moved_to_echo=22689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[6, 1, 5769, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3, 0, 4, 2298, 1011, 140, 913, 588, 677, 0, 0, 0, 0, 0, 1889, 665, 317, 10, 273, 3737, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 2, 9, 729, 4, 2441, 34, 1755, 1549, 186, 0, 0, 0, 0, 0, 0, 1, 25, 1, 770, 15, 5, 2943, 3037, 0, 0, 0, 0, 0, 0, 0, 112, 2, 0, 1339, 0, 0, 4209, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 3, 855, 335, 261, 1022, 1131, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 52, 5805, 9, 4, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5788, 5665, 7455, 6910, 6797, 5663, 4836, 6038] avg=6144.0000 max=7455 max_rank=2 min=4836 min_rank=6 imbalance_ratio=1.213379 max_load_reduction=41697 max_load_reduction_pct=0.848328 moved_to_echo=23334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[106, 0, 6475, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 2, 2714, 1214, 12, 955, 660, 447, 0, 0, 0, 0, 0, 1250, 1326, 88, 8, 232, 2184, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 3, 26, 24, 1105, 137, 2063, 1024, 253, 0, 0, 0, 0, 0, 3, 0, 0, 1, 2045, 10, 4, 933, 4146, 0, 0, 0, 0, 0, 0, 0, 8, 25, 0, 683, 0, 1, 4947, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2120, 12, 1937, 458, 910, 1517, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 322, 43, 6076, 7, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6586, 6007, 5385, 4693, 7142, 5667, 7212, 6460] avg=6144.0000 max=7212 max_rank=6 min=4693 min_rank=3 imbalance_ratio=1.173828 max_load_reduction=41940 max_load_reduction_pct=0.853271 moved_to_echo=26481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 6631, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 1240, 54, 948, 676, 452, 0, 0, 0, 0, 0, 1263, 1158, 171, 3, 287, 2358, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 561, 1, 2521, 40, 2008, 1035, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 9, 2, 2645, 3429, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2229, 0, 0, 4813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 0, 589, 378, 194, 1364, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 40, 6026, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6672, 5328, 5324, 6395, 6971, 7049, 4912, 6501] avg=6144.0000 max=7049 max_rank=5 min=4912 min_rank=6 imbalance_ratio=1.147298 max_load_reduction=42103 max_load_reduction_pct=0.856588 moved_to_echo=25433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[6, 1, 6357, 0, 11, 0, 0, 1, 0, 31, 3, 0, 4, 2298, 1011, 140, 4115, 665, 317, 10, 273, 3737, 0, 564, 201, 0, 2, 9, 729, 4, 2441, 34, 0, 1, 25, 1, 770, 15, 5, 2943, 112, 2, 0, 1339, 0, 0, 5964, 1, 1224, 3, 855, 3558, 261, 1022, 1131, 5, 0, 164, 52, 6718, 9, 4, 3, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[45, 0, 5824, 0, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 2530, 874, 24, 901, 594, 512, 0, 0, 0, 0, 0, 1428, 723, 76, 2, 216, 2760, 0, 949, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 125, 3, 1893, 65, 1918, 1172, 246, 0, 0, 0, 0, 0, 0, 0, 4, 10, 758, 22, 6, 1836, 4029, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 1630, 3, 0, 4599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1845, 0, 1851, 445, 112, 1649, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 61, 5730, 1, 33, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5878, 5441, 6154, 5498, 6665, 6636, 6745, 6135] avg=6144.0000 max=6745 max_rank=6 min=5441 min_rank=1 imbalance_ratio=1.097819 max_load_reduction=42407 max_load_reduction_pct=0.862773 moved_to_echo=26181 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[84, 0, 6374, 0, 2, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 65, 0, 5, 2991, 2520, 266, 894, 650, 276, 0, 0, 0, 0, 0, 770, 773, 621, 65, 285, 3390, 0, 741, 0, 0, 0, 0, 0, 0, 0, 0, 441, 0, 1, 15, 292, 0, 2161, 120, 2077, 632, 227, 0, 0, 0, 0, 0, 0, 0, 1, 22, 1105, 26, 0, 581, 3715, 0, 0, 0, 0, 0, 0, 0, 146, 9, 2, 728, 79, 0, 4978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 0, 851, 410, 311, 999, 743, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 39, 106, 5684, 69, 27, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6472, 7670, 6645, 5966, 5450, 5942, 5077, 5930] avg=6144.0000 max=7670 max_rank=1 min=5077 min_rank=6 imbalance_ratio=1.248372 max_load_reduction=41482 max_load_reduction_pct=0.843953 moved_to_echo=22399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[160, 0, 5433, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 71, 1255, 2606, 20, 947, 557, 531, 0, 0, 0, 0, 0, 1468, 566, 437, 0, 181, 1656, 2, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 930, 4, 2080, 40, 1955, 1213, 288, 0, 0, 0, 0, 0, 2, 0, 1, 22, 1273, 1, 0, 1002, 4656, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 708, 12, 0, 4672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4156, 1, 641, 507, 154, 835, 772, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 62, 159, 5993, 5, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5602, 5990, 5340, 6517, 6957, 5441, 7071, 6234] avg=6144.0000 max=7071 max_rank=6 min=5340 min_rank=2 imbalance_ratio=1.150879 max_load_reduction=42081 max_load_reduction_pct=0.856140 moved_to_echo=25703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[106, 0, 6475, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 2, 2714, 1214, 12, 955, 660, 447, 0, 0, 0, 0, 0, 1250, 1326, 88, 8, 232, 2184, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 3, 26, 24, 1105, 137, 2063, 1024, 253, 0, 0, 0, 0, 0, 3, 0, 0, 1, 2045, 10, 4, 933, 4146, 0, 0, 0, 0, 0, 0, 0, 8, 25, 0, 683, 0, 1, 4947, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2120, 12, 1937, 458, 910, 1517, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 322, 43, 6076, 7, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6586, 6007, 5385, 4693, 7142, 5667, 7212, 6460] avg=6144.0000 max=7212 max_rank=6 min=4693 min_rank=3 imbalance_ratio=1.173828 max_load_reduction=41940 max_load_reduction_pct=0.853271 moved_to_echo=26481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 5412, 20, 21, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2571, 2459, 157, 916, 551, 801, 0, 0, 0, 0, 0, 2234, 1034, 370, 8, 729, 2383, 1, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 986, 85, 1533, 67, 1551, 1833, 235, 0, 0, 0, 0, 0, 11, 1, 4, 5, 1036, 6, 4, 1065, 3840, 0, 0, 0, 0, 0, 0, 0, 380, 12, 1, 2797, 4, 1, 3717, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 743, 424, 69, 909, 362, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 329, 53, 5828, 1, 19, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5458, 7468, 7194, 6343, 5972, 6915, 3554, 6248] avg=6144.0000 max=7468 max_rank=1 min=3554 min_rank=6 imbalance_ratio=1.215495 max_load_reduction=41684 max_load_reduction_pct=0.848063 moved_to_echo=22689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[19, 5, 6574, 0, 70, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 2357, 1139, 188, 779, 670, 438, 0, 0, 0, 0, 0, 1224, 1479, 149, 5, 323, 2301, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 346, 3, 0, 12, 385, 15, 2922, 14, 2020, 1004, 109, 0, 0, 0, 0, 0, 6, 0, 22, 39, 850, 31, 11, 455, 1784, 0, 0, 0, 0, 0, 0, 0, 12, 1, 13, 965, 4, 0, 4844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 13, 2089, 197, 2296, 3347, 603, 52, 0, 0, 0, 0, 0, 0, 0, 0, 14, 326, 292, 4953, 2, 3, 3, 13, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6696, 5583, 5655, 6830, 3198, 5839, 9745, 5606] avg=6144.0000 max=9745 max_rank=6 min=3198 min_rank=4 imbalance_ratio=1.586100 max_load_reduction=39407 max_load_reduction_pct=0.801737 moved_to_echo=24388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[6, 1, 5769, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3, 0, 4, 2298, 1011, 140, 913, 588, 677, 0, 0, 0, 0, 0, 1889, 665, 317, 10, 273, 3737, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 2, 9, 729, 4, 2441, 34, 1755, 1549, 186, 0, 0, 0, 0, 0, 0, 1, 25, 1, 770, 15, 5, 2943, 3037, 0, 0, 0, 0, 0, 0, 0, 112, 2, 0, 1339, 0, 0, 4209, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 3, 855, 335, 261, 1022, 1131, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 52, 5805, 9, 4, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5788, 5665, 7455, 6910, 6797, 5663, 4836, 6038] avg=6144.0000 max=7455 max_rank=2 min=4836 min_rank=6 imbalance_ratio=1.213379 max_load_reduction=41697 max_load_reduction_pct=0.848328 moved_to_echo=23334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[160, 0, 5433, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 71, 1255, 2606, 20, 947, 557, 531, 0, 0, 0, 0, 0, 1468, 566, 437, 0, 181, 1656, 2, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 930, 4, 2080, 40, 1955, 1213, 288, 0, 0, 0, 0, 0, 2, 0, 1, 22, 1273, 1, 0, 1002, 4656, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 708, 12, 0, 4672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4156, 1, 641, 507, 154, 835, 772, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 62, 159, 5993, 5, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5602, 5990, 5340, 6517, 6957, 5441, 7071, 6234] avg=6144.0000 max=7071 max_rank=6 min=5340 min_rank=2 imbalance_ratio=1.150879 max_load_reduction=42081 max_load_reduction_pct=0.856140 moved_to_echo=25703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 6631, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 1240, 54, 948, 676, 452, 0, 0, 0, 0, 0, 1263, 1158, 171, 3, 287, 2358, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 561, 1, 2521, 40, 2008, 1035, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 9, 2, 2645, 3429, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2229, 0, 0, 4813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 0, 589, 378, 194, 1364, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 40, 6026, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6672, 5328, 5324, 6395, 6971, 7049, 4912, 6501] avg=6144.0000 max=7049 max_rank=5 min=4912 min_rank=6 imbalance_ratio=1.147298 max_load_reduction=42103 max_load_reduction_pct=0.856588 moved_to_echo=25433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 18688, 13184, 896, 7296, 4992, 4224, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1152, 128, 128, 128, 4096, 256, 16768, 640, 15360, 9472, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 18688, 13184, 896, 7296, 4992, 4224, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11648, 7808, 2304, 128, 2560, 20864, 128, 4352, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[15488, 128, 9600, 3200, 4352, 11648, 4992, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 8832, 128, 128, 11520, 28672, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 128, 128, 11136, 128, 128, 36864, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2048, 896, 46208, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 128, 48512, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True - [2026-08-26 15:39:18] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 805.8 | throughput per GPU (TFLOP/s/GPU): 158.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952471E+00 | loss scale: 1.0 | grad norm: 42.237 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2641, 499, 368, 727, 361, 1383, 1012, 95, 2763, 593, 1185, 567, 22, 1204, 260, 582, 1250, 1154, 521, 768, 296, 308, 439, 578, 556, 901, 100, 377, 939, 793, 402, 1427, 1232, 314, 920, 1830, 650, 408, 994, 525, 215, 421, 399, 171, 1369, 1877, 722, 132, 319, 583, 550, 834, 1020, 1593, 729, 338, 388, 830, 1461, 604, 338, 468, 1554, 293] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2829, 567, 401, 699, 289, 1412, 998, 111, 2585, 638, 1217, 535, 17, 1195, 284, 440, 1283, 1195, 485, 768, 305, 363, 460, 609, 527, 764, 135, 343, 946, 821, 440, 1421, 1332, 369, 949, 1934, 623, 352, 998, 525, 176, 444, 442, 247, 1300, 1913, 760, 144, 313, 575, 615, 780, 981, 1556, 602, 366, 414, 922, 1368, 527, 392, 370, 1464, 317] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 + [2026-08-26 16:35:20] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 807.8 | throughput per GPU (TFLOP/s/GPU): 157.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952471E+00 | loss scale: 1.0 | grad norm: 42.237 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2718, 539, 432, 734, 328, 1375, 976, 112, 2729, 592, 1263, 525, 27, 1201, 269, 510, 1253, 1116, 407, 835, 283, 337, 418, 579, 503, 762, 131, 371, 959, 861, 366, 1014, 1276, 362, 941, 1823, 652, 373, 1058, 530, 185, 448, 412, 235, 1295, 1970, 689, 175, 319, 642, 593, 786, 940, 2016, 780, 392, 398, 755, 1517, 499, 396, 394, 1463, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2626, 522, 371, 762, 326, 1431, 1045, 88, 2496, 616, 1233, 496, 16, 1212, 310, 537, 1360, 1183, 496, 831, 263, 340, 452, 618, 508, 822, 119, 392, 940, 765, 414, 1326, 1276, 331, 1011, 1595, 583, 375, 996, 587, 182, 463, 458, 227, 1374, 1805, 726, 207, 313, 596, 637, 914, 999, 1651, 643, 399, 426, 761, 1393, 559, 413, 471, 1503, 362] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2845, 514, 424, 592, 306, 1569, 1020, 88, 2767, 624, 1197, 489, 34, 1185, 239, 507, 1191, 1149, 463, 893, 316, 339, 427, 628, 545, 794, 136, 385, 918, 843, 372, 1138, 1196, 353, 915, 1909, 587, 406, 1090, 492, 151, 422, 449, 195, 1383, 1976, 750, 132, 292, 620, 590, 775, 823, 1864, 724, 279, 444, 906, 1403, 536, 407, 441, 1467, 238] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2544, 459, 385, 724, 308, 1396, 1035, 113, 2578, 673, 1201, 548, 18, 1130, 279, 447, 1164, 1215, 487, 913, 366, 411, 391, 605, 551, 788, 136, 388, 960, 791, 444, 1248, 1315, 369, 934, 1801, 725, 395, 1069, 535, 187, 423, 421, 237, 1298, 1957, 675, 162, 276, 609, 615, 764, 1021, 1931, 656, 424, 409, 832, 1405, 504, 428, 425, 1415, 239] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2737, 628, 438, 676, 340, 1198, 980, 131, 2529, 603, 1148, 518, 24, 1149, 285, 574, 1235, 1145, 478, 790, 340, 331, 508, 580, 573, 874, 127, 378, 1009, 769, 383, 1196, 1330, 428, 893, 1780, 576, 366, 1152, 544, 178, 499, 443, 245, 1376, 1849, 728, 183, 292, 515, 526, 861, 1000, 1905, 695, 363, 347, 760, 1396, 583, 395, 390, 1545, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2626, 522, 371, 762, 326, 1431, 1045, 88, 2496, 616, 1233, 496, 16, 1212, 310, 537, 1360, 1183, 496, 831, 263, 340, 452, 618, 508, 822, 119, 392, 940, 765, 414, 1326, 1276, 331, 1011, 1595, 583, 375, 996, 587, 182, 463, 458, 227, 1374, 1805, 726, 207, 313, 596, 637, 914, 999, 1651, 643, 399, 426, 761, 1393, 559, 413, 471, 1503, 362] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2544, 459, 385, 724, 308, 1396, 1035, 113, 2578, 673, 1201, 548, 18, 1130, 279, 447, 1164, 1215, 487, 913, 366, 411, 391, 605, 551, 788, 136, 388, 960, 791, 444, 1248, 1315, 369, 934, 1801, 725, 395, 1069, 535, 187, 423, 421, 237, 1298, 1957, 675, 162, 276, 609, 615, 764, 1021, 1931, 656, 424, 409, 832, 1405, 504, 428, 425, 1415, 239] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2829, 567, 401, 699, 289, 1412, 998, 111, 2585, 638, 1217, 535, 17, 1195, 284, 440, 1283, 1195, 485, 768, 305, 363, 460, 609, 527, 764, 135, 343, 946, 821, 440, 1421, 1332, 369, 949, 1934, 623, 352, 998, 525, 176, 444, 442, 247, 1300, 1913, 760, 144, 313, 575, 615, 780, 981, 1556, 602, 366, 414, 922, 1368, 527, 392, 370, 1464, 317] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[2791, 551, 331, 678, 352, 1258, 1149, 105, 2462, 616, 1134, 553, 21, 1206, 279, 647, 1308, 1076, 552, 898, 306, 350, 398, 571, 636, 787, 109, 424, 859, 896, 384, 1092, 1200, 362, 981, 1701, 665, 390, 1109, 550, 202, 429, 404, 226, 1321, 1868, 709, 165, 345, 531, 657, 718, 971, 1651, 619, 276, 374, 994, 1461, 502, 491, 514, 1509, 478] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1635, 499, 368, 727, 361, 1383, 1012, 95, 0, 0, 0, 0, 0, 0, 0, 0, 1889, 593, 1185, 567, 22, 1204, 260, 582, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 1154, 521, 768, 296, 308, 439, 578, 501, 252, 0, 0, 0, 0, 0, 0, 556, 901, 100, 377, 939, 793, 402, 1427, 859, 0, 0, 0, 0, 0, 0, 0, 1232, 314, 920, 979, 650, 408, 994, 525, 0, 0, 0, 0, 0, 0, 0, 0, 215, 421, 399, 171, 1369, 1877, 722, 132, 599, 147, 0, 0, 0, 0, 0, 0, 319, 583, 550, 834, 1020, 1593, 729, 338, 77, 0, 0, 0, 0, 0, 0, 0, 388, 830, 1461, 604, 338, 468, 1554, 293, 296, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6080, 6302, 6067, 6354, 6022, 6052, 6043, 6232] avg=6144.0000 max=6354 max_rank=3 min=6022 min_rank=4 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2641, 499, 368, 727, 361, 1383, 1012, 95, 2763, 593, 1185, 567, 22, 1204, 260, 582, 1250, 1154, 521, 768, 296, 308, 439, 578, 556, 901, 100, 377, 939, 793, 402, 1427, 1232, 314, 920, 1830, 650, 408, 994, 525, 215, 421, 399, 171, 1369, 1877, 722, 132, 319, 583, 550, 834, 1020, 1593, 729, 338, 388, 830, 1461, 604, 338, 468, 1554, 293] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1682, 539, 432, 734, 328, 1375, 976, 112, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 592, 1263, 525, 27, 1201, 269, 510, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 1116, 407, 835, 283, 337, 418, 579, 495, 251, 0, 0, 0, 0, 0, 0, 503, 762, 131, 371, 959, 861, 366, 1014, 884, 0, 0, 0, 0, 0, 0, 0, 1276, 362, 941, 975, 652, 373, 1058, 530, 0, 0, 0, 0, 0, 0, 0, 0, 185, 448, 412, 235, 1295, 1970, 689, 175, 597, 152, 0, 0, 0, 0, 0, 0, 319, 642, 593, 786, 940, 2016, 780, 392, 77, 0, 0, 0, 0, 0, 0, 0, 398, 755, 1517, 499, 396, 394, 1463, 313, 293, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6251, 5974, 5851, 6167, 6158, 6545, 6028] avg=6144.0000 max=6545 max_rank=6 min=5851 min_rank=3 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=24898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1760, 514, 424, 592, 306, 1569, 1020, 88, 0, 0, 0, 0, 0, 0, 0, 0, 1890, 624, 1197, 489, 34, 1185, 239, 507, 0, 0, 0, 0, 0, 0, 0, 0, 1191, 1149, 463, 893, 316, 339, 427, 628, 502, 262, 0, 0, 0, 0, 0, 0, 545, 794, 136, 385, 918, 843, 372, 1138, 926, 0, 0, 0, 0, 0, 0, 0, 1196, 353, 915, 1022, 587, 406, 1090, 492, 0, 0, 0, 0, 0, 0, 0, 0, 151, 422, 449, 195, 1383, 1976, 750, 132, 625, 159, 0, 0, 0, 0, 0, 0, 292, 620, 590, 775, 823, 1864, 724, 279, 78, 0, 0, 0, 0, 0, 0, 0, 444, 906, 1403, 536, 407, 441, 1467, 238, 297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6273, 6165, 6170, 6057, 6061, 6242, 6045, 6139] avg=6144.0000 max=6273 max_rank=0 min=6045 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=42879 max_load_reduction_pct=0.872375 moved_to_echo=24487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1694, 628, 438, 676, 340, 1198, 980, 131, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 603, 1148, 518, 24, 1149, 285, 574, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 1145, 478, 790, 340, 331, 508, 580, 459, 245, 0, 0, 0, 0, 0, 0, 573, 874, 127, 378, 1009, 769, 383, 1196, 890, 0, 0, 0, 0, 0, 0, 0, 1330, 428, 893, 952, 576, 366, 1152, 544, 0, 0, 0, 0, 0, 0, 0, 0, 178, 499, 443, 245, 1376, 1849, 728, 183, 583, 153, 0, 0, 0, 0, 0, 0, 292, 515, 526, 861, 1000, 1905, 695, 363, 71, 0, 0, 0, 0, 0, 0, 0, 347, 760, 1396, 583, 395, 390, 1545, 335, 271, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 6029, 6111, 6199, 6241, 6237, 6228, 6022] avg=6144.0000 max=6241 max_rank=4 min=6022 min_rank=7 imbalance_ratio=1.015788 max_load_reduction=42911 max_load_reduction_pct=0.873027 moved_to_echo=24728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1751, 567, 401, 699, 289, 1412, 998, 111, 0, 0, 0, 0, 0, 0, 0, 0, 1767, 638, 1217, 535, 17, 1195, 284, 440, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 1195, 485, 768, 305, 363, 460, 609, 469, 266, 0, 0, 0, 0, 0, 0, 527, 764, 135, 343, 946, 821, 440, 1421, 920, 0, 0, 0, 0, 0, 0, 0, 1332, 369, 949, 1035, 623, 352, 998, 525, 0, 0, 0, 0, 0, 0, 0, 0, 176, 444, 442, 247, 1300, 1913, 760, 144, 633, 158, 0, 0, 0, 0, 0, 0, 313, 575, 615, 780, 981, 1556, 602, 366, 72, 0, 0, 0, 0, 0, 0, 0, 414, 922, 1368, 527, 392, 370, 1464, 317, 277, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6228, 6093, 6203, 6317, 6183, 6217, 5860, 6051] avg=6144.0000 max=6317 max_rank=3 min=5860 min_rank=6 imbalance_ratio=1.028158 max_load_reduction=42835 max_load_reduction_pct=0.871480 moved_to_echo=24311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1574, 459, 385, 724, 308, 1396, 1035, 113, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 673, 1201, 548, 18, 1130, 279, 447, 0, 0, 0, 0, 0, 0, 0, 0, 1164, 1215, 487, 913, 366, 411, 391, 605, 467, 248, 0, 0, 0, 0, 0, 0, 551, 788, 136, 388, 960, 791, 444, 1248, 828, 0, 0, 0, 0, 0, 0, 0, 1315, 369, 934, 963, 725, 395, 1069, 535, 0, 0, 0, 0, 0, 0, 0, 0, 187, 423, 421, 237, 1298, 1957, 675, 162, 590, 142, 0, 0, 0, 0, 0, 0, 276, 609, 615, 764, 1021, 1931, 656, 424, 72, 0, 0, 0, 0, 0, 0, 0, 409, 832, 1405, 504, 428, 425, 1415, 239, 276, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5994, 6059, 6267, 6134, 6305, 6092, 6368, 5933] avg=6144.0000 max=6368 max_rank=6 min=5933 min_rank=7 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1760, 514, 424, 592, 306, 1569, 1020, 88, 0, 0, 0, 0, 0, 0, 0, 0, 1890, 624, 1197, 489, 34, 1185, 239, 507, 0, 0, 0, 0, 0, 0, 0, 0, 1191, 1149, 463, 893, 316, 339, 427, 628, 502, 262, 0, 0, 0, 0, 0, 0, 545, 794, 136, 385, 918, 843, 372, 1138, 926, 0, 0, 0, 0, 0, 0, 0, 1196, 353, 915, 1022, 587, 406, 1090, 492, 0, 0, 0, 0, 0, 0, 0, 0, 151, 422, 449, 195, 1383, 1976, 750, 132, 625, 159, 0, 0, 0, 0, 0, 0, 292, 620, 590, 775, 823, 1864, 724, 279, 78, 0, 0, 0, 0, 0, 0, 0, 444, 906, 1403, 536, 407, 441, 1467, 238, 297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6273, 6165, 6170, 6057, 6061, 6242, 6045, 6139] avg=6144.0000 max=6273 max_rank=0 min=6045 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=42879 max_load_reduction_pct=0.872375 moved_to_echo=24487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1635, 499, 368, 727, 361, 1383, 1012, 95, 0, 0, 0, 0, 0, 0, 0, 0, 1889, 593, 1185, 567, 22, 1204, 260, 582, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 1154, 521, 768, 296, 308, 439, 578, 501, 252, 0, 0, 0, 0, 0, 0, 556, 901, 100, 377, 939, 793, 402, 1427, 859, 0, 0, 0, 0, 0, 0, 0, 1232, 314, 920, 979, 650, 408, 994, 525, 0, 0, 0, 0, 0, 0, 0, 0, 215, 421, 399, 171, 1369, 1877, 722, 132, 599, 147, 0, 0, 0, 0, 0, 0, 319, 583, 550, 834, 1020, 1593, 729, 338, 77, 0, 0, 0, 0, 0, 0, 0, 388, 830, 1461, 604, 338, 468, 1554, 293, 296, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6080, 6302, 6067, 6354, 6022, 6052, 6043, 6232] avg=6144.0000 max=6354 max_rank=3 min=6022 min_rank=4 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1618, 522, 371, 762, 326, 1431, 1045, 88, 0, 0, 0, 0, 0, 0, 0, 0, 1694, 616, 1233, 496, 16, 1212, 310, 537, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 1183, 496, 831, 263, 340, 452, 618, 456, 222, 0, 0, 0, 0, 0, 0, 508, 822, 119, 392, 940, 765, 414, 1326, 859, 0, 0, 0, 0, 0, 0, 0, 1276, 331, 1011, 847, 583, 375, 996, 587, 0, 0, 0, 0, 0, 0, 0, 0, 182, 463, 458, 227, 1374, 1805, 726, 207, 526, 149, 0, 0, 0, 0, 0, 0, 313, 596, 637, 914, 999, 1651, 643, 399, 74, 0, 0, 0, 0, 0, 0, 0, 426, 761, 1393, 559, 413, 471, 1503, 362, 272, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 6114, 6221, 6145, 6006, 6117, 6226, 6160] avg=6144.0000 max=6226 max_rank=6 min=6006 min_rank=4 imbalance_ratio=1.013346 max_load_reduction=42926 max_load_reduction_pct=0.873332 moved_to_echo=24509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1694, 628, 438, 676, 340, 1198, 980, 131, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 603, 1148, 518, 24, 1149, 285, 574, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 1145, 478, 790, 340, 331, 508, 580, 459, 245, 0, 0, 0, 0, 0, 0, 573, 874, 127, 378, 1009, 769, 383, 1196, 890, 0, 0, 0, 0, 0, 0, 0, 1330, 428, 893, 952, 576, 366, 1152, 544, 0, 0, 0, 0, 0, 0, 0, 0, 178, 499, 443, 245, 1376, 1849, 728, 183, 583, 153, 0, 0, 0, 0, 0, 0, 292, 515, 526, 861, 1000, 1905, 695, 363, 71, 0, 0, 0, 0, 0, 0, 0, 347, 760, 1396, 583, 395, 390, 1545, 335, 271, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 6029, 6111, 6199, 6241, 6237, 6228, 6022] avg=6144.0000 max=6241 max_rank=4 min=6022 min_rank=7 imbalance_ratio=1.015788 max_load_reduction=42911 max_load_reduction_pct=0.873027 moved_to_echo=24728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1727, 551, 331, 678, 352, 1258, 1149, 105, 0, 0, 0, 0, 0, 0, 0, 0, 1683, 616, 1134, 553, 21, 1206, 279, 647, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 1076, 552, 898, 306, 350, 398, 571, 446, 234, 0, 0, 0, 0, 0, 0, 636, 787, 109, 424, 859, 896, 384, 1092, 908, 0, 0, 0, 0, 0, 0, 0, 1200, 362, 981, 910, 665, 390, 1109, 550, 0, 0, 0, 0, 0, 0, 0, 0, 202, 429, 404, 226, 1321, 1868, 709, 165, 557, 156, 0, 0, 0, 0, 0, 0, 345, 531, 657, 718, 971, 1651, 619, 276, 69, 0, 0, 0, 0, 0, 0, 0, 374, 994, 1461, 502, 491, 514, 1509, 478, 264, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 6139, 6139, 6095, 6167, 6037, 5837, 6587] avg=6144.0000 max=6587 max_rank=7 min=5837 min_rank=6 imbalance_ratio=1.072103 max_load_reduction=42565 max_load_reduction_pct=0.865987 moved_to_echo=24628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 6784, 11520, 4352, 3328, 3584, 12032, 2688, 2304, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[10112, 9344, 3968, 6784, 2560, 2816, 3584, 4864, 3840, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 4736, 4864, 6528, 7808, 14208, 5504, 2944, 640, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 3584, 3456, 1792, 10752, 15232, 5760, 1408, 4736, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13568, 4352, 3200, 5632, 2688, 11136, 8320, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 6784, 11520, 4352, 3328, 3584, 12032, 2688, 2304, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10240, 2944, 7552, 7808, 5120, 3072, 8576, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[10112, 9344, 3968, 6784, 2560, 2816, 3584, 4864, 3840, 2048, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14336, 4992, 9600, 4352, 256, 9600, 2304, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13568, 4352, 3200, 5632, 2688, 11136, 8320, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 3584, 3456, 1792, 10752, 15232, 5760, 1408, 4736, 1280, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4480, 6528, 1024, 3072, 7552, 6656, 3328, 9984, 7168, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[178, 368, 180, 143, 1862, 265, 3092, 276, 39, 387, 748, 468, 854, 531, 136, 2727, 599, 997, 1470, 1208, 435, 1536, 866, 412, 215, 311, 1018, 4555, 40, 925, 69, 722, 167, 9, 10, 250, 538, 25, 301, 190, 1733, 654, 0, 432, 145, 118, 3, 861, 5949, 279, 2090, 19, 292, 349, 1065, 757, 936, 950, 813, 471, 1073, 214, 442, 385] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[201, 414, 190, 146, 1662, 263, 2766, 337, 64, 426, 698, 436, 969, 659, 171, 3024, 589, 1198, 1379, 1347, 509, 1647, 850, 362, 223, 333, 1174, 4429, 42, 908, 79, 652, 170, 19, 45, 210, 577, 21, 287, 171, 1476, 629, 0, 466, 164, 139, 10, 858, 5777, 259, 1920, 11, 305, 298, 1113, 802, 1052, 1031, 675, 507, 1002, 193, 411, 407] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[130, 330, 140, 142, 1788, 265, 2875, 314, 30, 351, 643, 353, 891, 689, 105, 2954, 548, 1064, 1447, 1220, 436, 1636, 757, 415, 186, 334, 1178, 4756, 32, 1033, 86, 644, 124, 8, 10, 217, 529, 8, 267, 182, 1604, 644, 0, 470, 141, 101, 9, 1157, 6166, 221, 2150, 8, 352, 263, 931, 671, 1025, 1056, 664, 510, 1030, 160, 359, 343] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[156, 386, 226, 184, 1820, 259, 2778, 283, 62, 375, 649, 425, 879, 709, 141, 2826, 585, 1100, 1384, 1349, 503, 1606, 828, 387, 217, 322, 1035, 4464, 47, 873, 88, 721, 183, 17, 21, 237, 554, 21, 289, 192, 1646, 647, 0, 490, 158, 126, 9, 892, 5962, 280, 1988, 17, 387, 331, 912, 787, 971, 1084, 690, 487, 1115, 226, 389, 377] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[137, 340, 153, 160, 1882, 308, 2892, 248, 38, 306, 597, 399, 921, 644, 147, 2945, 630, 1052, 1643, 1230, 429, 1591, 788, 304, 207, 382, 962, 4619, 18, 893, 50, 698, 121, 19, 16, 245, 448, 17, 269, 154, 1528, 730, 0, 502, 171, 94, 10, 1335, 6046, 219, 2029, 6, 341, 274, 979, 773, 967, 1065, 613, 488, 992, 205, 456, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[190, 417, 168, 147, 1836, 344, 2698, 308, 45, 356, 632, 428, 917, 656, 176, 2835, 508, 1047, 1445, 1311, 504, 1608, 810, 367, 194, 302, 1038, 4456, 32, 929, 87, 628, 136, 16, 18, 259, 569, 20, 209, 187, 1542, 678, 0, 499, 134, 146, 6, 1350, 5910, 242, 2086, 11, 299, 320, 1016, 803, 970, 1029, 715, 532, 976, 205, 451, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[241, 343, 235, 149, 1827, 244, 2785, 320, 65, 414, 681, 448, 868, 712, 170, 2703, 677, 960, 1439, 1342, 538, 1442, 705, 401, 266, 431, 1080, 4182, 39, 950, 103, 591, 181, 16, 13, 230, 623, 12, 285, 216, 1622, 727, 0, 486, 142, 191, 11, 868, 5889, 287, 1969, 11, 292, 357, 1018, 740, 1055, 1118, 848, 562, 958, 271, 414, 389] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[143, 389, 235, 156, 1950, 283, 2662, 291, 47, 366, 691, 394, 1015, 641, 157, 2865, 533, 1061, 1424, 1446, 573, 1471, 817, 423, 243, 366, 845, 4445, 35, 1064, 85, 632, 189, 9, 21, 238, 547, 18, 227, 201, 1620, 643, 0, 488, 119, 113, 9, 1027, 5856, 248, 2012, 22, 402, 275, 1042, 675, 1013, 1024, 625, 497, 1111, 225, 455, 453] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[178, 368, 180, 143, 1862, 265, 3092, 276, 30, 0, 0, 0, 0, 0, 0, 0, 39, 387, 748, 468, 854, 531, 136, 2727, 55, 0, 0, 0, 0, 0, 0, 0, 599, 997, 1470, 1208, 435, 64, 866, 412, 0, 0, 0, 0, 0, 0, 0, 0, 215, 311, 1018, 2855, 40, 925, 69, 722, 0, 0, 0, 0, 0, 0, 0, 0, 167, 9, 10, 250, 538, 25, 301, 190, 4504, 198, 0, 0, 0, 0, 0, 0, 1733, 654, 0, 432, 145, 118, 3, 861, 1502, 579, 0, 0, 0, 0, 0, 0, 1445, 279, 2090, 19, 292, 349, 1065, 757, 0, 0, 0, 0, 0, 0, 0, 0, 936, 950, 813, 471, 1073, 214, 442, 385, 808, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 5945, 6051, 6155, 6192, 6027, 6296, 6092] avg=6144.0000 max=6394 max_rank=0 min=5945 min_rank=1 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24607 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[201, 414, 190, 146, 1662, 263, 2766, 337, 64, 426, 698, 436, 969, 659, 171, 3024, 589, 1198, 1379, 1347, 509, 1647, 850, 362, 223, 333, 1174, 4429, 42, 908, 79, 652, 170, 19, 45, 210, 577, 21, 287, 171, 1476, 629, 0, 466, 164, 139, 10, 858, 5777, 259, 1920, 11, 305, 298, 1113, 802, 1052, 1031, 675, 507, 1002, 193, 411, 407] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[178, 368, 180, 143, 1862, 265, 3092, 276, 39, 387, 748, 468, 854, 531, 136, 2727, 599, 997, 1470, 1208, 435, 1536, 866, 412, 215, 311, 1018, 4555, 40, 925, 69, 722, 167, 9, 10, 250, 538, 25, 301, 190, 1733, 654, 0, 432, 145, 118, 3, 861, 5949, 279, 2090, 19, 292, 349, 1065, 757, 936, 950, 813, 471, 1073, 214, 442, 385] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[137, 340, 153, 160, 1882, 308, 2892, 248, 38, 306, 597, 399, 921, 644, 147, 2945, 630, 1052, 1643, 1230, 429, 1591, 788, 304, 207, 382, 962, 4619, 18, 893, 50, 698, 121, 19, 16, 245, 448, 17, 269, 154, 1528, 730, 0, 502, 171, 94, 10, 1335, 6046, 219, 2029, 6, 341, 274, 979, 773, 967, 1065, 613, 488, 992, 205, 456, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[190, 417, 168, 147, 1836, 344, 2698, 308, 45, 356, 632, 428, 917, 656, 176, 2835, 508, 1047, 1445, 1311, 504, 1608, 810, 367, 194, 302, 1038, 4456, 32, 929, 87, 628, 136, 16, 18, 259, 569, 20, 209, 187, 1542, 678, 0, 499, 134, 146, 6, 1350, 5910, 242, 2086, 11, 299, 320, 1016, 803, 970, 1029, 715, 532, 976, 205, 451, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[156, 386, 226, 184, 1820, 259, 2778, 283, 32, 0, 0, 0, 0, 0, 0, 0, 62, 375, 649, 425, 879, 709, 141, 2826, 58, 0, 0, 0, 0, 0, 0, 0, 585, 1100, 1384, 1349, 503, 66, 828, 387, 0, 0, 0, 0, 0, 0, 0, 0, 217, 322, 1035, 2798, 47, 873, 88, 721, 0, 0, 0, 0, 0, 0, 0, 0, 183, 17, 21, 237, 554, 21, 289, 192, 4514, 194, 0, 0, 0, 0, 0, 0, 1646, 647, 0, 490, 158, 126, 9, 892, 1472, 606, 0, 0, 0, 0, 0, 0, 1448, 280, 1988, 17, 387, 331, 912, 787, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1084, 690, 487, 1115, 226, 389, 377, 844, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6124, 6202, 6101, 6222, 6046, 6150, 6183] avg=6144.0000 max=6222 max_rank=4 min=6046 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=42930 max_load_reduction_pct=0.873413 moved_to_echo=24601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[130, 330, 140, 142, 1788, 265, 2875, 314, 32, 0, 0, 0, 0, 0, 0, 0, 30, 351, 643, 353, 891, 689, 105, 2954, 59, 0, 0, 0, 0, 0, 0, 0, 548, 1064, 1447, 1220, 436, 68, 757, 415, 0, 0, 0, 0, 0, 0, 0, 0, 186, 334, 1178, 2981, 32, 1033, 86, 644, 0, 0, 0, 0, 0, 0, 0, 0, 124, 8, 10, 217, 529, 8, 267, 182, 4668, 207, 0, 0, 0, 0, 0, 0, 1604, 644, 0, 470, 141, 101, 9, 1157, 1568, 617, 0, 0, 0, 0, 0, 0, 1498, 221, 2150, 8, 352, 263, 931, 671, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1056, 664, 510, 1030, 160, 359, 343, 860, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6016, 6075, 5955, 6474, 6220, 6311, 6094, 6007] avg=6144.0000 max=6474 max_rank=3 min=5955 min_rank=2 imbalance_ratio=1.053711 max_load_reduction=42678 max_load_reduction_pct=0.868286 moved_to_echo=24632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[137, 340, 153, 160, 1882, 308, 2892, 248, 31, 0, 0, 0, 0, 0, 0, 0, 38, 306, 597, 399, 921, 644, 147, 2945, 57, 0, 0, 0, 0, 0, 0, 0, 630, 1052, 1643, 1230, 429, 66, 788, 304, 0, 0, 0, 0, 0, 0, 0, 0, 207, 382, 962, 2895, 18, 893, 50, 698, 0, 0, 0, 0, 0, 0, 0, 0, 121, 19, 16, 245, 448, 17, 269, 154, 4577, 201, 0, 0, 0, 0, 0, 0, 1528, 730, 0, 502, 171, 94, 10, 1335, 1523, 600, 0, 0, 0, 0, 0, 0, 1469, 219, 2029, 6, 341, 274, 979, 773, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1065, 613, 488, 992, 205, 456, 427, 837, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 6054, 6142, 6105, 6067, 6493, 6090, 6050] avg=6144.0000 max=6493 max_rank=5 min=6050 min_rank=7 imbalance_ratio=1.056803 max_load_reduction=42659 max_load_reduction_pct=0.867900 moved_to_echo=24700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[156, 386, 226, 184, 1820, 259, 2778, 283, 32, 0, 0, 0, 0, 0, 0, 0, 62, 375, 649, 425, 879, 709, 141, 2826, 58, 0, 0, 0, 0, 0, 0, 0, 585, 1100, 1384, 1349, 503, 66, 828, 387, 0, 0, 0, 0, 0, 0, 0, 0, 217, 322, 1035, 2798, 47, 873, 88, 721, 0, 0, 0, 0, 0, 0, 0, 0, 183, 17, 21, 237, 554, 21, 289, 192, 4514, 194, 0, 0, 0, 0, 0, 0, 1646, 647, 0, 490, 158, 126, 9, 892, 1472, 606, 0, 0, 0, 0, 0, 0, 1448, 280, 1988, 17, 387, 331, 912, 787, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1084, 690, 487, 1115, 226, 389, 377, 844, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6124, 6202, 6101, 6222, 6046, 6150, 6183] avg=6144.0000 max=6222 max_rank=4 min=6046 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=42930 max_load_reduction_pct=0.873413 moved_to_echo=24601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[241, 343, 235, 149, 1827, 244, 2785, 320, 28, 0, 0, 0, 0, 0, 0, 0, 65, 414, 681, 448, 868, 712, 170, 2703, 52, 0, 0, 0, 0, 0, 0, 0, 677, 960, 1439, 1342, 538, 60, 705, 401, 0, 0, 0, 0, 0, 0, 0, 0, 266, 431, 1080, 2621, 39, 950, 103, 591, 0, 0, 0, 0, 0, 0, 0, 0, 181, 16, 13, 230, 623, 12, 285, 216, 4459, 182, 0, 0, 0, 0, 0, 0, 1622, 727, 0, 486, 142, 191, 11, 868, 1379, 544, 0, 0, 0, 0, 0, 0, 1430, 287, 1969, 11, 292, 357, 1018, 740, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1118, 848, 562, 958, 271, 414, 389, 758, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6172, 6113, 6122, 6081, 6217, 5970, 6104, 6373] avg=6144.0000 max=6373 max_rank=7 min=5970 min_rank=5 imbalance_ratio=1.037272 max_load_reduction=42779 max_load_reduction_pct=0.870341 moved_to_echo=24664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[201, 414, 190, 146, 1662, 263, 2766, 337, 33, 0, 0, 0, 0, 0, 0, 0, 64, 426, 698, 436, 969, 659, 171, 3024, 59, 0, 0, 0, 0, 0, 0, 0, 589, 1198, 1379, 1347, 509, 68, 850, 362, 0, 0, 0, 0, 0, 0, 0, 0, 223, 333, 1174, 2775, 42, 908, 79, 652, 0, 0, 0, 0, 0, 0, 0, 0, 170, 19, 45, 210, 577, 21, 287, 171, 4374, 193, 0, 0, 0, 0, 0, 0, 1476, 629, 0, 466, 164, 139, 10, 858, 1461, 621, 0, 0, 0, 0, 0, 0, 1403, 259, 1920, 11, 305, 298, 1113, 802, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1031, 675, 507, 1002, 193, 411, 407, 866, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6012, 6506, 6302, 6186, 6067, 5824, 6111, 6144] avg=6144.0000 max=6506 max_rank=1 min=5824 min_rank=5 imbalance_ratio=1.058919 max_load_reduction=42646 max_load_reduction_pct=0.867635 moved_to_echo=24146 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[178, 368, 180, 143, 1862, 265, 3092, 276, 30, 0, 0, 0, 0, 0, 0, 0, 39, 387, 748, 468, 854, 531, 136, 2727, 55, 0, 0, 0, 0, 0, 0, 0, 599, 997, 1470, 1208, 435, 64, 866, 412, 0, 0, 0, 0, 0, 0, 0, 0, 215, 311, 1018, 2855, 40, 925, 69, 722, 0, 0, 0, 0, 0, 0, 0, 0, 167, 9, 10, 250, 538, 25, 301, 190, 4504, 198, 0, 0, 0, 0, 0, 0, 1733, 654, 0, 432, 145, 118, 3, 861, 1502, 579, 0, 0, 0, 0, 0, 0, 1445, 279, 2090, 19, 292, 349, 1065, 757, 0, 0, 0, 0, 0, 0, 0, 0, 936, 950, 813, 471, 1073, 214, 442, 385, 808, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 5945, 6051, 6155, 6192, 6027, 6296, 6092] avg=6144.0000 max=6394 max_rank=0 min=5945 min_rank=1 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24607 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[190, 417, 168, 147, 1836, 344, 2698, 308, 32, 0, 0, 0, 0, 0, 0, 0, 45, 356, 632, 428, 917, 656, 176, 2835, 58, 0, 0, 0, 0, 0, 0, 0, 508, 1047, 1445, 1311, 504, 65, 810, 367, 0, 0, 0, 0, 0, 0, 0, 0, 194, 302, 1038, 2793, 32, 929, 87, 628, 0, 0, 0, 0, 0, 0, 0, 0, 136, 16, 18, 259, 569, 20, 209, 187, 4474, 194, 0, 0, 0, 0, 0, 0, 1542, 678, 0, 499, 134, 146, 6, 1350, 1469, 607, 0, 0, 0, 0, 0, 0, 1436, 242, 2086, 11, 299, 320, 1016, 803, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1029, 715, 532, 976, 205, 451, 399, 846, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6103, 6057, 6003, 6082, 6431, 6213, 6123] avg=6144.0000 max=6431 max_rank=5 min=6003 min_rank=3 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[143, 389, 235, 156, 1950, 283, 2662, 291, 34, 0, 0, 0, 0, 0, 0, 0, 47, 366, 691, 394, 1015, 641, 157, 2865, 56, 0, 0, 0, 0, 0, 0, 0, 533, 1061, 1424, 1446, 573, 44, 817, 423, 0, 0, 0, 0, 0, 0, 0, 0, 243, 366, 845, 2777, 35, 1064, 85, 632, 0, 0, 0, 0, 0, 0, 0, 0, 189, 9, 21, 238, 547, 18, 227, 201, 4438, 197, 0, 0, 0, 0, 0, 0, 1620, 643, 0, 488, 119, 113, 9, 1027, 1471, 560, 0, 0, 0, 0, 0, 0, 1418, 248, 2012, 22, 402, 275, 1042, 675, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1024, 625, 497, 1111, 225, 455, 453, 777, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6232, 6321, 6047, 6085, 6050, 6094, 6180] avg=6144.0000 max=6321 max_rank=2 min=6047 min_rank=3 imbalance_ratio=1.028809 max_load_reduction=42831 max_load_reduction_pct=0.871399 moved_to_echo=24409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[241, 343, 235, 149, 1827, 244, 2785, 320, 28, 0, 0, 0, 0, 0, 0, 0, 65, 414, 681, 448, 868, 712, 170, 2703, 52, 0, 0, 0, 0, 0, 0, 0, 677, 960, 1439, 1342, 538, 60, 705, 401, 0, 0, 0, 0, 0, 0, 0, 0, 266, 431, 1080, 2621, 39, 950, 103, 591, 0, 0, 0, 0, 0, 0, 0, 0, 181, 16, 13, 230, 623, 12, 285, 216, 4459, 182, 0, 0, 0, 0, 0, 0, 1622, 727, 0, 486, 142, 191, 11, 868, 1379, 544, 0, 0, 0, 0, 0, 0, 1430, 287, 1969, 11, 292, 357, 1018, 740, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1118, 848, 562, 958, 271, 414, 389, 758, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6172, 6113, 6122, 6081, 6217, 5970, 6104, 6373] avg=6144.0000 max=6373 max_rank=7 min=5970 min_rank=5 imbalance_ratio=1.037272 max_load_reduction=42779 max_load_reduction_pct=0.870341 moved_to_echo=24664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12800, 5376, 0, 3840, 1280, 1152, 128, 8448, 11904, 4736, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4736, 8576, 11648, 10496, 3968, 512, 6528, 3072, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8064, 8448, 5760, 4096, 8320, 1792, 3456, 3200, 6656, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11648, 2048, 16256, 128, 2688, 2560, 8192, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4736, 8576, 11648, 10496, 3968, 512, 6528, 3072, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 2816, 8448, 22528, 384, 7680, 768, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8064, 8448, 5760, 4096, 8320, 1792, 3456, 3200, 6656, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 3072, 5376, 3456, 7424, 5248, 1280, 22912, 512, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1408, 3072, 1536, 1280, 14720, 2304, 22656, 2432, 256, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1280, 128, 256, 1920, 4480, 256, 2176, 1536, 36096, 1664, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[6, 345, 16, 151, 45, 63, 2062, 4748, 2, 749, 420, 1515, 1895, 145, 192, 104, 2604, 1286, 980, 791, 69, 5, 4066, 299, 386, 1136, 5027, 1009, 649, 207, 176, 40, 143, 422, 1623, 103, 4261, 434, 497, 141, 242, 356, 1, 131, 1670, 2042, 131, 104, 298, 1402, 218, 52, 72, 83, 308, 4, 1311, 62, 77, 57, 4, 600, 14, 1101] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 341, 10, 176, 53, 51, 1852, 4512, 1, 630, 387, 1413, 2058, 198, 239, 144, 2892, 1465, 914, 996, 100, 4, 4343, 298, 303, 1100, 4736, 1095, 678, 232, 194, 44, 237, 414, 1493, 137, 3594, 600, 687, 175, 317, 368, 4, 102, 1239, 2094, 140, 73, 402, 1285, 215, 60, 53, 99, 383, 7, 1481, 112, 35, 67, 7, 662, 46, 1104] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[2, 193, 3, 218, 41, 50, 1923, 4968, 1, 621, 520, 1324, 2081, 125, 156, 99, 2699, 1481, 1033, 702, 66, 3, 4959, 394, 260, 1088, 5354, 1063, 686, 203, 117, 24, 94, 505, 1590, 93, 3667, 487, 605, 141, 184, 302, 0, 75, 1334, 1942, 123, 61, 231, 1556, 234, 29, 44, 57, 247, 4, 1276, 44, 34, 49, 2, 595, 23, 1067] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3, 302, 16, 218, 42, 25, 2091, 4656, 1, 754, 347, 1417, 1686, 196, 182, 120, 2888, 1460, 893, 851, 101, 3, 4567, 314, 386, 1085, 4793, 1132, 652, 245, 168, 35, 213, 479, 1703, 141, 3660, 433, 529, 174, 213, 426, 0, 83, 1292, 2045, 126, 94, 297, 1647, 261, 74, 55, 75, 273, 11, 1262, 80, 57, 64, 4, 687, 30, 1035] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4, 230, 13, 223, 27, 43, 1970, 4824, 0, 598, 367, 1265, 1979, 133, 174, 73, 2885, 1557, 924, 1012, 58, 1, 4593, 331, 321, 1183, 5192, 1012, 697, 259, 150, 16, 120, 390, 1468, 88, 3751, 436, 544, 145, 250, 351, 2, 94, 1266, 2114, 146, 49, 335, 1580, 241, 27, 45, 66, 325, 2, 1340, 44, 44, 53, 3, 663, 23, 1033] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2, 201, 14, 206, 33, 43, 1904, 4724, 2, 671, 385, 1352, 1920, 131, 149, 81, 2703, 1477, 939, 987, 67, 3, 4815, 323, 296, 1109, 5025, 1106, 635, 237, 188, 26, 132, 448, 1598, 105, 3861, 470, 515, 160, 196, 384, 1, 76, 1169, 2017, 141, 77, 294, 1662, 277, 49, 47, 70, 353, 4, 1420, 64, 63, 61, 7, 652, 27, 998] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4, 281, 5, 188, 35, 76, 2030, 4836, 0, 649, 407, 1335, 1936, 131, 169, 117, 3013, 1440, 853, 1092, 57, 4, 4478, 347, 342, 1150, 5177, 995, 634, 185, 173, 27, 158, 389, 1474, 124, 3844, 452, 564, 157, 185, 381, 0, 68, 1184, 2004, 135, 83, 303, 1572, 255, 28, 66, 76, 354, 4, 1286, 60, 41, 66, 9, 605, 30, 1029] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[6, 345, 16, 151, 45, 63, 2062, 4748, 2, 749, 420, 1515, 1895, 145, 192, 104, 2604, 1286, 980, 791, 69, 5, 4066, 299, 386, 1136, 5027, 1009, 649, 207, 176, 40, 143, 422, 1623, 103, 4261, 434, 497, 141, 242, 356, 1, 131, 1670, 2042, 131, 104, 298, 1402, 218, 52, 72, 83, 308, 4, 1311, 62, 77, 57, 4, 600, 14, 1101] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2, 174, 13, 195, 30, 32, 2124, 5240, 1, 703, 469, 1519, 2085, 134, 133, 74, 2909, 1233, 885, 773, 54, 5, 4679, 338, 250, 1014, 5232, 1055, 702, 219, 131, 20, 110, 432, 1692, 99, 3724, 440, 405, 144, 180, 347, 0, 53, 1312, 1877, 111, 48, 234, 1731, 273, 21, 45, 86, 278, 3, 1276, 54, 22, 39, 2, 738, 12, 937] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4, 230, 13, 223, 27, 43, 1970, 4824, 0, 598, 367, 1265, 1979, 133, 174, 73, 2885, 1557, 924, 1012, 58, 1, 4593, 331, 321, 1183, 5192, 1012, 697, 259, 150, 16, 120, 390, 1468, 88, 3751, 436, 544, 145, 250, 351, 2, 94, 1266, 2114, 146, 49, 335, 1580, 241, 27, 45, 66, 325, 2, 1340, 44, 44, 53, 3, 663, 23, 1033] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3, 302, 16, 218, 42, 25, 2091, 4656, 1, 754, 347, 1417, 1686, 196, 182, 120, 2888, 1460, 893, 851, 101, 3, 4567, 314, 386, 1085, 4793, 1132, 652, 245, 168, 35, 213, 479, 1703, 141, 3660, 433, 529, 174, 213, 426, 0, 83, 1292, 2045, 126, 94, 297, 1647, 261, 74, 55, 75, 273, 11, 1262, 80, 57, 64, 4, 687, 30, 1035] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 341, 10, 176, 53, 51, 1852, 4512, 1, 630, 387, 1413, 2058, 198, 239, 144, 2892, 1465, 914, 996, 100, 4, 4343, 298, 303, 1100, 4736, 1095, 678, 232, 194, 44, 237, 414, 1493, 137, 3594, 600, 687, 175, 317, 368, 4, 102, 1239, 2094, 140, 73, 402, 1285, 215, 60, 53, 99, 383, 7, 1481, 112, 35, 67, 7, 662, 46, 1104] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[2, 193, 3, 218, 41, 50, 1923, 4968, 1, 621, 520, 1324, 2081, 125, 156, 99, 2699, 1481, 1033, 702, 66, 3, 4959, 394, 260, 1088, 5354, 1063, 686, 203, 117, 24, 94, 505, 1590, 93, 3667, 487, 605, 141, 184, 302, 0, 75, 1334, 1942, 123, 61, 231, 1556, 234, 29, 44, 57, 247, 4, 1276, 44, 34, 49, 2, 595, 23, 1067] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 193, 3, 218, 41, 50, 1923, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 1, 621, 520, 1324, 2081, 125, 156, 99, 915, 323, 0, 0, 0, 0, 0, 0, 2376, 1481, 1033, 702, 66, 3, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 260, 1088, 2725, 1063, 686, 203, 117, 24, 0, 0, 0, 0, 0, 0, 0, 0, 94, 505, 1590, 93, 2610, 487, 605, 141, 0, 0, 0, 0, 0, 0, 0, 0, 184, 302, 0, 75, 1334, 1942, 123, 61, 1258, 607, 142, 0, 0, 0, 0, 0, 231, 1556, 234, 29, 44, 57, 247, 4, 3860, 0, 0, 0, 0, 0, 0, 0, 1276, 44, 34, 49, 2, 595, 23, 1067, 2022, 1099, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6165, 6055, 6166, 6125, 6028, 6262, 6211] avg=6144.0000 max=6262 max_rank=6 min=6028 min_rank=5 imbalance_ratio=1.019206 max_load_reduction=42890 max_load_reduction_pct=0.872599 moved_to_echo=24626 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 341, 10, 176, 53, 51, 1852, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 1, 630, 387, 1413, 2058, 198, 239, 144, 897, 346, 0, 0, 0, 0, 0, 0, 2546, 1465, 914, 996, 100, 4, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 303, 1100, 2411, 1095, 678, 232, 194, 44, 0, 0, 0, 0, 0, 0, 0, 0, 237, 414, 1493, 137, 2557, 600, 687, 175, 0, 0, 0, 0, 0, 0, 0, 0, 317, 368, 4, 102, 1239, 2094, 140, 73, 1143, 537, 140, 0, 0, 0, 0, 0, 402, 1285, 215, 60, 53, 99, 383, 7, 3381, 0, 0, 0, 0, 0, 0, 0, 1481, 112, 35, 67, 7, 662, 46, 1104, 1788, 962, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5853, 6313, 6323, 6057, 6300, 6157, 5885, 6264] avg=6144.0000 max=6323 max_rank=2 min=5853 min_rank=0 imbalance_ratio=1.029134 max_load_reduction=42829 max_load_reduction_pct=0.871358 moved_to_echo=24606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 201, 14, 206, 33, 43, 1904, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 2, 671, 385, 1352, 1920, 131, 149, 81, 963, 324, 0, 0, 0, 0, 0, 0, 2379, 1477, 939, 987, 67, 3, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 296, 1109, 2558, 1106, 635, 237, 188, 26, 0, 0, 0, 0, 0, 0, 0, 0, 132, 448, 1598, 105, 2748, 470, 515, 160, 0, 0, 0, 0, 0, 0, 0, 0, 196, 384, 1, 76, 1169, 2017, 141, 77, 1196, 569, 150, 0, 0, 0, 0, 0, 294, 1662, 277, 49, 47, 70, 353, 4, 3748, 0, 0, 0, 0, 0, 0, 0, 1420, 64, 63, 61, 7, 652, 27, 998, 1898, 1067, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5931, 5978, 6175, 6155, 6176, 5976, 6504, 6257] avg=6144.0000 max=6504 max_rank=6 min=5931 min_rank=0 imbalance_ratio=1.058594 max_load_reduction=42648 max_load_reduction_pct=0.867676 moved_to_echo=24913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 174, 13, 195, 30, 32, 2124, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 1, 703, 469, 1519, 2085, 134, 133, 74, 929, 348, 0, 0, 0, 0, 0, 0, 2561, 1233, 885, 773, 54, 5, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 250, 1014, 2663, 1055, 702, 219, 131, 20, 0, 0, 0, 0, 0, 0, 0, 0, 110, 432, 1692, 99, 2650, 440, 405, 144, 0, 0, 0, 0, 0, 0, 0, 0, 180, 347, 0, 53, 1312, 1877, 111, 48, 1327, 593, 145, 0, 0, 0, 0, 0, 234, 1731, 273, 21, 45, 86, 278, 3, 3642, 0, 0, 0, 0, 0, 0, 0, 1276, 54, 22, 39, 2, 738, 12, 937, 1976, 1037, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6483, 6395, 5849, 6054, 5972, 5993, 6313, 6093] avg=6144.0000 max=6483 max_rank=0 min=5849 min_rank=2 imbalance_ratio=1.055176 max_load_reduction=42669 max_load_reduction_pct=0.868103 moved_to_echo=24371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 345, 16, 151, 45, 63, 2062, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 2, 749, 420, 1515, 1895, 145, 192, 104, 1067, 316, 0, 0, 0, 0, 0, 0, 2288, 1286, 980, 791, 69, 5, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 386, 1136, 2552, 1009, 649, 207, 176, 40, 0, 0, 0, 0, 0, 0, 0, 0, 143, 422, 1623, 103, 3025, 434, 497, 141, 0, 0, 0, 0, 0, 0, 0, 0, 242, 356, 1, 131, 1670, 2042, 131, 104, 1207, 573, 169, 0, 0, 0, 0, 0, 298, 1402, 218, 52, 72, 83, 308, 4, 3165, 0, 0, 0, 0, 0, 0, 0, 1311, 62, 77, 57, 4, 600, 14, 1101, 1902, 901, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6229, 6405, 5718, 6155, 6388, 6626, 5602, 6029] avg=6144.0000 max=6626 max_rank=5 min=5602 min_rank=6 imbalance_ratio=1.078451 max_load_reduction=42526 max_load_reduction_pct=0.865194 moved_to_echo=24645 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 302, 16, 218, 42, 25, 2091, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 1, 754, 347, 1417, 1686, 196, 182, 120, 913, 346, 0, 0, 0, 0, 0, 0, 2542, 1460, 893, 851, 101, 3, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 386, 1085, 2440, 1132, 652, 245, 168, 35, 0, 0, 0, 0, 0, 0, 0, 0, 213, 479, 1703, 141, 2605, 433, 529, 174, 0, 0, 0, 0, 0, 0, 0, 0, 213, 426, 0, 83, 1292, 2045, 126, 94, 1179, 543, 142, 0, 0, 0, 0, 0, 297, 1647, 261, 74, 55, 75, 273, 11, 3554, 0, 0, 0, 0, 0, 0, 0, 1262, 80, 57, 64, 4, 687, 30, 1035, 1810, 1013, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6174, 5962, 6164, 6143, 6277, 6143, 6247, 6042] avg=6144.0000 max=6277 max_rank=4 min=5962 min_rank=1 imbalance_ratio=1.021647 max_load_reduction=42875 max_load_reduction_pct=0.872294 moved_to_echo=24709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 230, 13, 223, 27, 43, 1970, 3602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 367, 1265, 1979, 133, 174, 73, 936, 345, 0, 0, 0, 0, 0, 0, 2540, 1557, 924, 1012, 58, 1, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 321, 1183, 2643, 1012, 697, 259, 150, 16, 0, 0, 0, 0, 0, 0, 0, 0, 120, 390, 1468, 88, 2669, 436, 544, 145, 0, 0, 0, 0, 0, 0, 0, 0, 250, 351, 2, 94, 1266, 2114, 146, 49, 1222, 588, 146, 0, 0, 0, 0, 0, 335, 1580, 241, 27, 45, 66, 325, 2, 3574, 0, 0, 0, 0, 0, 0, 0, 1340, 44, 44, 53, 3, 663, 23, 1033, 1961, 1019, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 5870, 6423, 6281, 5860, 6228, 6195, 6183] avg=6144.0000 max=6423 max_rank=2 min=5860 min_rank=4 imbalance_ratio=1.045410 max_load_reduction=42729 max_load_reduction_pct=0.869324 moved_to_echo=24466 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 174, 13, 195, 30, 32, 2124, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 1, 703, 469, 1519, 2085, 134, 133, 74, 929, 348, 0, 0, 0, 0, 0, 0, 2561, 1233, 885, 773, 54, 5, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 250, 1014, 2663, 1055, 702, 219, 131, 20, 0, 0, 0, 0, 0, 0, 0, 0, 110, 432, 1692, 99, 2650, 440, 405, 144, 0, 0, 0, 0, 0, 0, 0, 0, 180, 347, 0, 53, 1312, 1877, 111, 48, 1327, 593, 145, 0, 0, 0, 0, 0, 234, 1731, 273, 21, 45, 86, 278, 3, 3642, 0, 0, 0, 0, 0, 0, 0, 1276, 54, 22, 39, 2, 738, 12, 937, 1976, 1037, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6483, 6395, 5849, 6054, 5972, 5993, 6313, 6093] avg=6144.0000 max=6483 max_rank=0 min=5849 min_rank=2 imbalance_ratio=1.055176 max_load_reduction=42669 max_load_reduction_pct=0.868103 moved_to_echo=24371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 281, 5, 188, 35, 76, 2030, 3611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 407, 1335, 1936, 131, 169, 117, 959, 361, 0, 0, 0, 0, 0, 0, 2652, 1440, 853, 1092, 57, 4, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 342, 1150, 2635, 995, 634, 185, 173, 27, 0, 0, 0, 0, 0, 0, 0, 0, 158, 389, 1474, 124, 2736, 452, 564, 157, 0, 0, 0, 0, 0, 0, 0, 0, 185, 381, 0, 68, 1184, 2004, 135, 83, 1225, 587, 149, 0, 0, 0, 0, 0, 303, 1572, 255, 28, 66, 76, 354, 4, 3486, 0, 0, 0, 0, 0, 0, 0, 1286, 60, 41, 66, 9, 605, 30, 1029, 1955, 992, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6230, 6064, 6445, 6141, 6054, 6001, 6144, 6073] avg=6144.0000 max=6445 max_rank=2 min=6001 min_rank=5 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=24272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 193, 3, 218, 41, 50, 1923, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 1, 621, 520, 1324, 2081, 125, 156, 99, 915, 323, 0, 0, 0, 0, 0, 0, 2376, 1481, 1033, 702, 66, 3, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 260, 1088, 2725, 1063, 686, 203, 117, 24, 0, 0, 0, 0, 0, 0, 0, 0, 94, 505, 1590, 93, 2610, 487, 605, 141, 0, 0, 0, 0, 0, 0, 0, 0, 184, 302, 0, 75, 1334, 1942, 123, 61, 1258, 607, 142, 0, 0, 0, 0, 0, 231, 1556, 234, 29, 44, 57, 247, 4, 3860, 0, 0, 0, 0, 0, 0, 0, 1276, 44, 34, 49, 2, 595, 23, 1067, 2022, 1099, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6165, 6055, 6166, 6125, 6028, 6262, 6211] avg=6144.0000 max=6262 max_rank=6 min=6028 min_rank=5 imbalance_ratio=1.019206 max_load_reduction=42890 max_load_reduction_pct=0.872599 moved_to_echo=24626 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 302, 16, 218, 42, 25, 2091, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 1, 754, 347, 1417, 1686, 196, 182, 120, 913, 346, 0, 0, 0, 0, 0, 0, 2542, 1460, 893, 851, 101, 3, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 386, 1085, 2440, 1132, 652, 245, 168, 35, 0, 0, 0, 0, 0, 0, 0, 0, 213, 479, 1703, 141, 2605, 433, 529, 174, 0, 0, 0, 0, 0, 0, 0, 0, 213, 426, 0, 83, 1292, 2045, 126, 94, 1179, 543, 142, 0, 0, 0, 0, 0, 297, 1647, 261, 74, 55, 75, 273, 11, 3554, 0, 0, 0, 0, 0, 0, 0, 1262, 80, 57, 64, 4, 687, 30, 1035, 1810, 1013, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6174, 5962, 6164, 6143, 6277, 6143, 6247, 6042] avg=6144.0000 max=6277 max_rank=4 min=5962 min_rank=1 imbalance_ratio=1.021647 max_load_reduction=42875 max_load_reduction_pct=0.872294 moved_to_echo=24709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 341, 10, 176, 53, 51, 1852, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 1, 630, 387, 1413, 2058, 198, 239, 144, 897, 346, 0, 0, 0, 0, 0, 0, 2546, 1465, 914, 996, 100, 4, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 303, 1100, 2411, 1095, 678, 232, 194, 44, 0, 0, 0, 0, 0, 0, 0, 0, 237, 414, 1493, 137, 2557, 600, 687, 175, 0, 0, 0, 0, 0, 0, 0, 0, 317, 368, 4, 102, 1239, 2094, 140, 73, 1143, 537, 140, 0, 0, 0, 0, 0, 402, 1285, 215, 60, 53, 99, 383, 7, 3381, 0, 0, 0, 0, 0, 0, 0, 1481, 112, 35, 67, 7, 662, 46, 1104, 1788, 962, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5853, 6313, 6323, 6057, 6300, 6157, 5885, 6264] avg=6144.0000 max=6323 max_rank=2 min=5853 min_rank=0 imbalance_ratio=1.029134 max_load_reduction=42829 max_load_reduction_pct=0.871358 moved_to_echo=24606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 345, 16, 151, 45, 63, 2062, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 2, 749, 420, 1515, 1895, 145, 192, 104, 1067, 316, 0, 0, 0, 0, 0, 0, 2288, 1286, 980, 791, 69, 5, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 386, 1136, 2552, 1009, 649, 207, 176, 40, 0, 0, 0, 0, 0, 0, 0, 0, 143, 422, 1623, 103, 3025, 434, 497, 141, 0, 0, 0, 0, 0, 0, 0, 0, 242, 356, 1, 131, 1670, 2042, 131, 104, 1207, 573, 169, 0, 0, 0, 0, 0, 298, 1402, 218, 52, 72, 83, 308, 4, 3165, 0, 0, 0, 0, 0, 0, 0, 1311, 62, 77, 57, 4, 600, 14, 1101, 1902, 901, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6229, 6405, 5718, 6155, 6388, 6626, 5602, 6029] avg=6144.0000 max=6626 max_rank=5 min=5602 min_rank=6 imbalance_ratio=1.078451 max_load_reduction=42526 max_load_reduction_pct=0.865194 moved_to_echo=24645 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1792, 2944, 128, 768, 10496, 16256, 1152, 640, 9856, 4608, 1280, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 8960, 20736, 8576, 5376, 1792, 1408, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 5376, 3328, 11264, 15744, 1280, 1408, 896, 7680, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10752, 640, 384, 512, 128, 5248, 256, 8320, 15360, 8192, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1280, 3584, 12672, 896, 21632, 3840, 4352, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 12544, 2048, 384, 512, 640, 2560, 128, 28416, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19968, 11520, 7424, 7296, 640, 128, 0, 2688, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2176, 128, 1664, 384, 384, 16000, 28800, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 12544, 2048, 384, 512, 640, 2560, 128, 28416, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1280, 3584, 12672, 896, 21632, 3840, 4352, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10752, 640, 384, 512, 128, 5248, 256, 8320, 15360, 8192, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[918, 49, 1707, 48, 608, 213, 614, 457, 218, 124, 102, 60, 320, 20, 889, 269, 5750, 313, 212, 2, 363, 111, 0, 3564, 283, 0, 2024, 2247, 788, 1188, 5761, 79, 794, 1052, 23, 43, 885, 339, 951, 228, 408, 1659, 61, 2091, 185, 528, 6, 1409, 1, 158, 1978, 928, 78, 650, 26, 1218, 351, 0, 819, 105, 242, 2201, 228, 206] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[920, 41, 1547, 20, 555, 130, 657, 351, 158, 120, 97, 50, 346, 31, 715, 210, 5618, 347, 343, 1, 472, 112, 1, 3751, 242, 0, 2178, 2540, 771, 1054, 5880, 57, 737, 959, 17, 60, 1080, 454, 958, 239, 293, 1441, 72, 2314, 189, 524, 3, 1384, 1, 130, 1895, 806, 57, 593, 32, 1250, 238, 0, 1044, 74, 271, 2226, 184, 312] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[918, 51, 1673, 35, 552, 189, 480, 506, 248, 124, 75, 41, 331, 18, 709, 231, 5853, 353, 225, 2, 408, 127, 0, 3520, 282, 0, 2241, 2163, 805, 1284, 5712, 81, 554, 872, 7, 45, 770, 257, 1076, 230, 452, 1558, 55, 2260, 179, 591, 7, 1254, 0, 156, 2140, 882, 46, 479, 40, 1445, 274, 0, 1053, 114, 241, 2325, 200, 353] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[852, 34, 1922, 21, 543, 141, 630, 476, 226, 121, 84, 29, 306, 6, 901, 159, 5947, 363, 225, 1, 503, 91, 0, 3742, 311, 0, 2200, 2410, 908, 1116, 5765, 82, 621, 992, 27, 38, 965, 409, 941, 234, 347, 1491, 39, 2315, 178, 540, 0, 1335, 1, 138, 1801, 722, 40, 564, 36, 1549, 235, 0, 865, 97, 126, 1905, 153, 333] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[901, 83, 1664, 25, 443, 162, 545, 500, 222, 132, 105, 54, 347, 10, 865, 206, 5682, 258, 250, 2, 408, 132, 0, 3356, 280, 1, 2290, 2407, 817, 1195, 5765, 129, 577, 960, 19, 69, 906, 400, 973, 244, 336, 1714, 67, 2353, 238, 495, 5, 1370, 5, 139, 1952, 766, 62, 600, 35, 1288, 263, 0, 929, 137, 293, 2228, 278, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[858, 40, 1744, 27, 515, 146, 681, 453, 167, 98, 77, 43, 337, 12, 622, 220, 6020, 355, 259, 0, 745, 101, 0, 3493, 287, 0, 2049, 2298, 725, 1021, 5883, 129, 863, 1110, 22, 54, 1124, 355, 887, 233, 342, 1482, 55, 2264, 198, 615, 0, 1447, 3, 122, 1863, 782, 40, 541, 42, 1456, 250, 0, 946, 94, 239, 1814, 154, 350] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[853, 42, 1554, 30, 430, 149, 630, 422, 250, 109, 77, 43, 331, 16, 527, 118, 5653, 402, 312, 1, 526, 141, 1, 3550, 225, 3, 2095, 2441, 962, 1399, 5782, 80, 757, 1016, 27, 43, 900, 325, 968, 268, 386, 1720, 64, 2330, 201, 632, 2, 1192, 1, 161, 1948, 733, 74, 610, 39, 1475, 293, 0, 1037, 85, 287, 1922, 295, 207] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[890, 53, 1621, 46, 621, 218, 501, 465, 261, 140, 130, 114, 318, 25, 876, 193, 5377, 304, 249, 6, 325, 165, 0, 3383, 359, 4, 2432, 2453, 775, 1169, 5691, 112, 762, 991, 18, 46, 1112, 258, 928, 244, 371, 1709, 87, 2254, 158, 443, 5, 1348, 6, 130, 1958, 1068, 52, 509, 42, 1302, 302, 0, 780, 83, 258, 2115, 252, 285] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[918, 49, 1707, 48, 608, 213, 614, 457, 218, 124, 102, 60, 320, 20, 889, 269, 5750, 313, 212, 2, 363, 111, 0, 3564, 283, 0, 2024, 2247, 788, 1188, 5761, 79, 794, 1052, 23, 43, 885, 339, 951, 228, 408, 1659, 61, 2091, 185, 528, 6, 1409, 1, 158, 1978, 928, 78, 650, 26, 1218, 351, 0, 819, 105, 242, 2201, 228, 206] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[918, 51, 1673, 35, 552, 189, 480, 506, 248, 124, 75, 41, 331, 18, 709, 231, 5853, 353, 225, 2, 408, 127, 0, 3520, 282, 0, 2241, 2163, 805, 1284, 5712, 81, 554, 872, 7, 45, 770, 257, 1076, 230, 452, 1558, 55, 2260, 179, 591, 7, 1254, 0, 156, 2140, 882, 46, 479, 40, 1445, 274, 0, 1053, 114, 241, 2325, 200, 353] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[901, 83, 1664, 25, 443, 162, 545, 500, 222, 132, 105, 54, 347, 10, 865, 206, 5682, 258, 250, 2, 408, 132, 0, 3356, 280, 1, 2290, 2407, 817, 1195, 5765, 129, 577, 960, 19, 69, 906, 400, 973, 244, 336, 1714, 67, 2353, 238, 495, 5, 1370, 5, 139, 1952, 766, 62, 600, 35, 1288, 263, 0, 929, 137, 293, 2228, 278, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[920, 41, 1547, 20, 555, 130, 657, 351, 158, 120, 97, 50, 346, 31, 715, 210, 5618, 347, 343, 1, 472, 112, 1, 3751, 242, 0, 2178, 2540, 771, 1054, 5880, 57, 737, 959, 17, 60, 1080, 454, 958, 239, 293, 1441, 72, 2314, 189, 524, 3, 1384, 1, 130, 1895, 806, 57, 593, 32, 1250, 238, 0, 1044, 74, 271, 2226, 184, 312] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[852, 34, 1922, 21, 543, 141, 630, 476, 226, 121, 84, 29, 306, 6, 901, 159, 5947, 363, 225, 1, 503, 91, 0, 3742, 311, 0, 2200, 2410, 908, 1116, 5765, 82, 621, 992, 27, 38, 965, 409, 941, 234, 347, 1491, 39, 2315, 178, 540, 0, 1335, 1, 138, 1801, 722, 40, 564, 36, 1549, 235, 0, 865, 97, 126, 1905, 153, 333] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[858, 40, 1744, 27, 515, 146, 681, 453, 1833, 0, 0, 0, 0, 0, 0, 0, 167, 98, 77, 43, 337, 12, 622, 220, 4424, 0, 0, 0, 0, 0, 0, 0, 1481, 355, 259, 0, 745, 101, 0, 3493, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 2049, 1533, 725, 1021, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 863, 1110, 22, 54, 1124, 355, 887, 233, 1945, 0, 0, 0, 0, 0, 0, 0, 342, 1482, 55, 2028, 198, 615, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 3, 122, 1863, 782, 40, 541, 42, 1456, 765, 236, 169, 0, 0, 0, 0, 0, 250, 0, 946, 94, 239, 1814, 154, 350, 1459, 592, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 6000, 6434, 5744, 6593, 6167, 6019, 5898] avg=6144.0000 max=6593 max_rank=4 min=5744 min_rank=3 imbalance_ratio=1.073079 max_load_reduction=42559 max_load_reduction_pct=0.865865 moved_to_echo=24677 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[918, 49, 1707, 48, 608, 213, 614, 457, 1751, 0, 0, 0, 0, 0, 0, 0, 218, 124, 102, 60, 320, 20, 889, 269, 4331, 0, 0, 0, 0, 0, 0, 0, 1415, 313, 212, 2, 363, 111, 0, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 2024, 1499, 788, 1188, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 794, 1052, 23, 43, 885, 339, 951, 228, 1858, 0, 0, 0, 0, 0, 0, 0, 408, 1659, 61, 1873, 185, 528, 6, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 1, 158, 1978, 928, 78, 650, 26, 1218, 748, 218, 161, 0, 0, 0, 0, 0, 351, 0, 819, 105, 242, 2201, 228, 206, 1430, 565, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6365, 6333, 5980, 5861, 6173, 6129, 6164, 6147] avg=6144.0000 max=6365 max_rank=0 min=5861 min_rank=3 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[853, 42, 1554, 30, 430, 149, 630, 422, 1722, 0, 0, 0, 0, 0, 0, 0, 250, 109, 77, 43, 331, 16, 527, 118, 4347, 0, 0, 0, 0, 0, 0, 0, 1390, 402, 312, 1, 526, 141, 1, 3550, 0, 0, 0, 0, 0, 0, 0, 0, 225, 3, 2095, 1628, 962, 1399, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 757, 1016, 27, 43, 900, 325, 968, 268, 1827, 0, 0, 0, 0, 0, 0, 0, 386, 1720, 64, 2087, 201, 632, 2, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 161, 1948, 733, 74, 610, 39, 1475, 813, 243, 158, 0, 0, 0, 0, 0, 293, 0, 1037, 85, 287, 1922, 295, 207, 1435, 556, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5832, 5818, 6323, 6392, 6131, 6284, 6255, 6117] avg=6144.0000 max=6392 max_rank=3 min=5818 min_rank=1 imbalance_ratio=1.040365 max_load_reduction=42760 max_load_reduction_pct=0.869954 moved_to_echo=24787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[918, 51, 1673, 35, 552, 189, 480, 506, 1782, 0, 0, 0, 0, 0, 0, 0, 248, 124, 75, 41, 331, 18, 709, 231, 4295, 0, 0, 0, 0, 0, 0, 0, 1441, 353, 225, 2, 408, 127, 0, 3520, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 2241, 1443, 805, 1284, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 554, 872, 7, 45, 770, 257, 1076, 230, 1891, 0, 0, 0, 0, 0, 0, 0, 452, 1558, 55, 2024, 179, 591, 7, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 2140, 882, 46, 479, 40, 1445, 720, 236, 164, 0, 0, 0, 0, 0, 274, 0, 1053, 114, 241, 2325, 200, 353, 1417, 575, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6186, 6072, 6076, 6136, 5702, 6120, 6308, 6552] avg=6144.0000 max=6552 max_rank=7 min=5702 min_rank=4 imbalance_ratio=1.066406 max_load_reduction=42600 max_load_reduction_pct=0.866699 moved_to_echo=24682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[858, 40, 1744, 27, 515, 146, 681, 453, 1833, 0, 0, 0, 0, 0, 0, 0, 167, 98, 77, 43, 337, 12, 622, 220, 4424, 0, 0, 0, 0, 0, 0, 0, 1481, 355, 259, 0, 745, 101, 0, 3493, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 2049, 1533, 725, 1021, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 863, 1110, 22, 54, 1124, 355, 887, 233, 1945, 0, 0, 0, 0, 0, 0, 0, 342, 1482, 55, 2028, 198, 615, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 3, 122, 1863, 782, 40, 541, 42, 1456, 765, 236, 169, 0, 0, 0, 0, 0, 250, 0, 946, 94, 239, 1814, 154, 350, 1459, 592, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 6000, 6434, 5744, 6593, 6167, 6019, 5898] avg=6144.0000 max=6593 max_rank=4 min=5744 min_rank=3 imbalance_ratio=1.073079 max_load_reduction=42559 max_load_reduction_pct=0.865865 moved_to_echo=24677 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[920, 41, 1547, 20, 555, 130, 657, 351, 1711, 0, 0, 0, 0, 0, 0, 0, 158, 120, 97, 50, 346, 31, 715, 210, 4422, 0, 0, 0, 0, 0, 0, 0, 1383, 347, 343, 1, 472, 112, 1, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 2178, 1694, 771, 1054, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 737, 959, 17, 60, 1080, 454, 958, 239, 1815, 0, 0, 0, 0, 0, 0, 0, 293, 1441, 72, 2073, 189, 524, 3, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 1, 130, 1895, 806, 57, 593, 32, 1250, 846, 241, 157, 0, 0, 0, 0, 0, 238, 0, 1044, 74, 271, 2226, 184, 312, 1458, 552, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5932, 6149, 6410, 5996, 6319, 5979, 6008, 6359] avg=6144.0000 max=6410 max_rank=2 min=5932 min_rank=0 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[890, 53, 1621, 46, 621, 218, 501, 465, 1637, 0, 0, 0, 0, 0, 0, 0, 261, 140, 130, 114, 318, 25, 876, 193, 4280, 0, 0, 0, 0, 0, 0, 0, 1323, 304, 249, 6, 325, 165, 0, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 359, 4, 2432, 1636, 775, 1169, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 762, 991, 18, 46, 1112, 258, 928, 244, 1737, 0, 0, 0, 0, 0, 0, 0, 371, 1709, 87, 2019, 158, 443, 5, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 6, 130, 1958, 1068, 52, 509, 42, 1302, 817, 235, 151, 0, 0, 0, 0, 0, 302, 0, 780, 83, 258, 2115, 252, 285, 1411, 529, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6052, 6337, 5755, 6487, 6096, 6140, 6270, 6015] avg=6144.0000 max=6487 max_rank=3 min=5755 min_rank=2 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[852, 34, 1922, 21, 543, 141, 630, 476, 1811, 0, 0, 0, 0, 0, 0, 0, 226, 121, 84, 29, 306, 6, 901, 159, 4335, 0, 0, 0, 0, 0, 0, 0, 1462, 363, 225, 1, 503, 91, 0, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 2200, 1607, 908, 1116, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 621, 992, 27, 38, 965, 409, 941, 234, 1922, 0, 0, 0, 0, 0, 0, 0, 347, 1491, 39, 2074, 178, 540, 0, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 1801, 722, 40, 564, 36, 1549, 803, 241, 167, 0, 0, 0, 0, 0, 235, 0, 865, 97, 126, 1905, 153, 333, 1430, 585, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6430, 6167, 6387, 6224, 6149, 6004, 6062, 5729] avg=6144.0000 max=6430 max_rank=0 min=5729 min_rank=7 imbalance_ratio=1.046549 max_load_reduction=42722 max_load_reduction_pct=0.869181 moved_to_echo=23944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[920, 41, 1547, 20, 555, 130, 657, 351, 1711, 0, 0, 0, 0, 0, 0, 0, 158, 120, 97, 50, 346, 31, 715, 210, 4422, 0, 0, 0, 0, 0, 0, 0, 1383, 347, 343, 1, 472, 112, 1, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 2178, 1694, 771, 1054, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 737, 959, 17, 60, 1080, 454, 958, 239, 1815, 0, 0, 0, 0, 0, 0, 0, 293, 1441, 72, 2073, 189, 524, 3, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 1, 130, 1895, 806, 57, 593, 32, 1250, 846, 241, 157, 0, 0, 0, 0, 0, 238, 0, 1044, 74, 271, 2226, 184, 312, 1458, 552, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5932, 6149, 6410, 5996, 6319, 5979, 6008, 6359] avg=6144.0000 max=6410 max_rank=2 min=5932 min_rank=0 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[901, 83, 1664, 25, 443, 162, 545, 500, 1735, 0, 0, 0, 0, 0, 0, 0, 222, 132, 105, 54, 347, 10, 865, 206, 4335, 0, 0, 0, 0, 0, 0, 0, 1381, 258, 250, 2, 408, 132, 0, 3356, 0, 0, 0, 0, 0, 0, 0, 0, 280, 1, 2290, 1600, 817, 1195, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 577, 960, 19, 69, 906, 400, 973, 244, 1841, 0, 0, 0, 0, 0, 0, 0, 336, 1714, 67, 2104, 238, 495, 5, 1370, 0, 0, 0, 0, 0, 0, 0, 0, 5, 139, 1952, 766, 62, 600, 35, 1288, 807, 249, 163, 0, 0, 0, 0, 0, 263, 0, 929, 137, 293, 2228, 278, 215, 1430, 562, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6058, 6276, 5787, 6312, 5989, 6329, 6066, 6335] avg=6144.0000 max=6335 max_rank=7 min=5787 min_rank=2 imbalance_ratio=1.031087 max_load_reduction=42817 max_load_reduction_pct=0.871114 moved_to_echo=24719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[853, 42, 1554, 30, 430, 149, 630, 422, 1722, 0, 0, 0, 0, 0, 0, 0, 250, 109, 77, 43, 331, 16, 527, 118, 4347, 0, 0, 0, 0, 0, 0, 0, 1390, 402, 312, 1, 526, 141, 1, 3550, 0, 0, 0, 0, 0, 0, 0, 0, 225, 3, 2095, 1628, 962, 1399, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 757, 1016, 27, 43, 900, 325, 968, 268, 1827, 0, 0, 0, 0, 0, 0, 0, 386, 1720, 64, 2087, 201, 632, 2, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 161, 1948, 733, 74, 610, 39, 1475, 813, 243, 158, 0, 0, 0, 0, 0, 293, 0, 1037, 85, 287, 1922, 295, 207, 1435, 556, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5832, 5818, 6323, 6392, 6131, 6284, 6255, 6117] avg=6144.0000 max=6392 max_rank=3 min=5818 min_rank=1 imbalance_ratio=1.040365 max_load_reduction=42760 max_load_reduction_pct=0.869954 moved_to_echo=24787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[890, 53, 1621, 46, 621, 218, 501, 465, 1637, 0, 0, 0, 0, 0, 0, 0, 261, 140, 130, 114, 318, 25, 876, 193, 4280, 0, 0, 0, 0, 0, 0, 0, 1323, 304, 249, 6, 325, 165, 0, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 359, 4, 2432, 1636, 775, 1169, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 762, 991, 18, 46, 1112, 258, 928, 244, 1737, 0, 0, 0, 0, 0, 0, 0, 371, 1709, 87, 2019, 158, 443, 5, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 6, 130, 1958, 1068, 52, 509, 42, 1302, 817, 235, 151, 0, 0, 0, 0, 0, 302, 0, 780, 83, 258, 2115, 252, 285, 1411, 529, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6052, 6337, 5755, 6487, 6096, 6140, 6270, 6015] avg=6144.0000 max=6487 max_rank=3 min=5755 min_rank=2 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2944, 12800, 512, 16384, 1536, 4480, 128, 10752, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 8064, 256, 512, 7808, 2816, 7808, 1920, 14848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2304, 128, 17536, 12672, 6656, 9472, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1152, 15616, 6784, 512, 4608, 384, 11008, 6400, 1920, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2944, 12800, 512, 16384, 1536, 4480, 128, 10752, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7168, 512, 13440, 256, 4352, 1408, 4864, 3712, 14080, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2304, 128, 17536, 12672, 6656, 9472, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 8064, 256, 512, 7808, 2816, 7808, 1920, 14848, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2304, 0, 7552, 896, 2048, 16768, 1792, 2304, 11520, 4608, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[11392, 2816, 2176, 128, 3840, 1024, 128, 28416, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 1024, 768, 512, 2688, 256, 6144, 1664, 34816, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[2, 259, 1078, 0, 17, 540, 24, 367, 1, 389, 591, 379, 0, 0, 2, 55, 1, 1120, 0, 7420, 511, 293, 371, 1002, 99, 1, 541, 4605, 102, 5704, 45, 1, 0, 1, 0, 7533, 2644, 5, 60, 609, 13, 8, 39, 1042, 328, 0, 2440, 53, 1, 46, 796, 49, 25, 353, 1391, 1303, 15, 497, 2, 13, 4269, 84, 3, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[12, 248, 1190, 0, 20, 560, 32, 365, 1, 423, 517, 282, 0, 0, 8, 62, 0, 1055, 0, 7116, 576, 180, 303, 919, 86, 2, 445, 4532, 164, 5783, 46, 0, 0, 4, 0, 7621, 2596, 3, 49, 667, 11, 7, 18, 1050, 347, 0, 2325, 77, 0, 57, 1348, 33, 7, 338, 1267, 1465, 8, 628, 1, 10, 4206, 69, 5, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[6, 258, 1254, 0, 22, 607, 18, 368, 1, 470, 474, 274, 1, 0, 14, 40, 0, 1163, 0, 7146, 706, 189, 226, 782, 69, 5, 582, 4711, 142, 5717, 108, 2, 0, 5, 0, 7549, 2384, 4, 51, 660, 7, 4, 25, 1028, 492, 0, 2507, 79, 0, 58, 1101, 59, 12, 338, 1337, 1409, 6, 617, 0, 19, 3933, 97, 6, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[2, 259, 1078, 0, 17, 540, 24, 367, 1, 389, 591, 379, 0, 0, 2, 55, 1, 1120, 0, 7420, 511, 293, 371, 1002, 99, 1, 541, 4605, 102, 5704, 45, 1, 0, 1, 0, 7533, 2644, 5, 60, 609, 13, 8, 39, 1042, 328, 0, 2440, 53, 1, 46, 796, 49, 25, 353, 1391, 1303, 15, 497, 2, 13, 4269, 84, 3, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[19, 297, 867, 2, 25, 600, 23, 411, 6, 373, 649, 322, 2, 0, 22, 46, 0, 1285, 0, 7078, 466, 268, 320, 953, 59, 3, 616, 4771, 148, 5942, 65, 2, 2, 3, 0, 7614, 2487, 7, 60, 488, 20, 4, 64, 1098, 418, 0, 2468, 95, 0, 46, 1030, 61, 21, 342, 1126, 1298, 10, 553, 2, 36, 4026, 111, 8, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[16, 297, 1124, 3, 24, 588, 20, 327, 6, 574, 622, 429, 0, 0, 24, 44, 0, 1130, 0, 6898, 434, 131, 437, 906, 59, 0, 683, 4872, 155, 5811, 59, 1, 0, 5, 0, 7456, 2143, 9, 48, 623, 28, 3, 67, 1098, 417, 0, 2642, 58, 0, 66, 874, 46, 13, 397, 1450, 1482, 7, 574, 1, 23, 3784, 149, 3, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[8, 195, 1119, 1, 22, 492, 20, 419, 6, 551, 610, 451, 0, 0, 18, 45, 0, 1107, 0, 7060, 539, 153, 357, 1107, 73, 0, 442, 4850, 129, 5133, 60, 1, 0, 0, 0, 7427, 2632, 10, 80, 848, 13, 11, 43, 1182, 300, 0, 2678, 79, 0, 97, 924, 53, 11, 383, 1346, 1397, 9, 611, 2, 16, 3899, 109, 4, 20] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[3, 199, 1008, 0, 19, 519, 20, 299, 1, 379, 506, 228, 0, 0, 12, 70, 0, 1060, 0, 7199, 638, 184, 339, 837, 84, 0, 501, 4648, 107, 6170, 56, 1, 0, 2, 0, 7607, 2558, 1, 45, 623, 11, 1, 20, 1013, 281, 0, 2613, 56, 0, 42, 1207, 37, 9, 373, 1264, 1521, 6, 580, 0, 14, 4077, 93, 2, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[13, 163, 1118, 0, 14, 512, 20, 341, 1, 376, 521, 201, 0, 0, 9, 55, 0, 1213, 0, 7179, 540, 207, 275, 780, 100, 0, 565, 4952, 83, 6021, 59, 0, 0, 2, 0, 7660, 2707, 8, 54, 601, 5, 2, 30, 1092, 296, 0, 2632, 52, 0, 40, 1235, 31, 16, 301, 1128, 1242, 8, 483, 2, 8, 4076, 99, 2, 22] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[6, 258, 1254, 0, 22, 607, 18, 368, 1, 470, 474, 274, 1, 0, 14, 40, 0, 1163, 0, 7146, 706, 189, 226, 782, 69, 5, 582, 4711, 142, 5717, 108, 2, 0, 5, 0, 7549, 2384, 4, 51, 660, 7, 4, 25, 1028, 492, 0, 2507, 79, 0, 58, 1101, 59, 12, 338, 1337, 1409, 6, 617, 0, 19, 3933, 97, 6, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 259, 1078, 0, 17, 540, 24, 367, 3379, 445, 0, 0, 0, 0, 0, 0, 1, 389, 591, 379, 0, 0, 2, 55, 4684, 0, 0, 0, 0, 0, 0, 0, 1, 1120, 0, 3133, 511, 293, 371, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 99, 1, 541, 4605, 102, 575, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2912, 2644, 5, 60, 609, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8, 39, 1042, 328, 0, 2440, 53, 1242, 829, 0, 0, 0, 0, 0, 0, 1, 46, 796, 49, 25, 353, 1391, 1303, 2007, 0, 0, 0, 0, 0, 0, 0, 15, 497, 2, 13, 4269, 84, 3, 10, 1451, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 6101, 6431, 5969, 6231, 5994, 5971, 6344] avg=6144.0000 max=6431 max_rank=2 min=5969 min_rank=3 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 248, 1190, 0, 20, 560, 32, 365, 3418, 451, 0, 0, 0, 0, 0, 0, 1, 423, 517, 282, 0, 0, 8, 62, 4749, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 3004, 576, 180, 303, 919, 0, 0, 0, 0, 0, 0, 0, 0, 86, 2, 445, 4532, 164, 583, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2946, 2596, 3, 49, 667, 0, 0, 0, 0, 0, 0, 0, 0, 11, 7, 18, 1050, 347, 0, 2325, 77, 1257, 795, 0, 0, 0, 0, 0, 0, 0, 57, 1348, 33, 7, 338, 1267, 1465, 1925, 0, 0, 0, 0, 0, 0, 0, 8, 628, 1, 10, 4206, 69, 5, 8, 1392, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6296, 6042, 6037, 5858, 6265, 5887, 6440, 6327] avg=6144.0000 max=6440 max_rank=6 min=5858 min_rank=3 imbalance_ratio=1.048177 max_load_reduction=42712 max_load_reduction_pct=0.868978 moved_to_echo=24919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 199, 1008, 0, 19, 519, 20, 299, 3412, 481, 0, 0, 0, 0, 0, 0, 1, 379, 506, 228, 0, 0, 12, 70, 5067, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 3038, 638, 184, 339, 837, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 501, 4648, 107, 622, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2940, 2558, 1, 45, 623, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 20, 1013, 281, 0, 2613, 56, 1255, 805, 0, 0, 0, 0, 0, 0, 0, 42, 1207, 37, 9, 373, 1264, 1521, 1948, 0, 0, 0, 0, 0, 0, 0, 6, 580, 0, 14, 4077, 93, 2, 9, 1408, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5960, 6263, 6096, 6019, 6169, 6055, 6401, 6189] avg=6144.0000 max=6401 max_rank=6 min=5960 min_rank=0 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[3, 199, 1008, 0, 19, 519, 20, 299, 1, 379, 506, 228, 0, 0, 12, 70, 0, 1060, 0, 7199, 638, 184, 339, 837, 84, 0, 501, 4648, 107, 6170, 56, 1, 0, 2, 0, 7607, 2558, 1, 45, 623, 11, 1, 20, 1013, 281, 0, 2613, 56, 0, 42, 1207, 37, 9, 373, 1264, 1521, 6, 580, 0, 14, 4077, 93, 2, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[8, 195, 1119, 1, 22, 492, 20, 419, 6, 551, 610, 451, 0, 0, 18, 45, 0, 1107, 0, 7060, 539, 153, 357, 1107, 73, 0, 442, 4850, 129, 5133, 60, 1, 0, 0, 0, 7427, 2632, 10, 80, 848, 13, 11, 43, 1182, 300, 0, 2678, 79, 0, 97, 924, 53, 11, 383, 1346, 1397, 9, 611, 2, 16, 3899, 109, 4, 20] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[16, 297, 1124, 3, 24, 588, 20, 327, 6, 574, 622, 429, 0, 0, 24, 44, 0, 1130, 0, 6898, 434, 131, 437, 906, 59, 0, 683, 4872, 155, 5811, 59, 1, 0, 5, 0, 7456, 2143, 9, 48, 623, 28, 3, 67, 1098, 417, 0, 2642, 58, 0, 66, 874, 46, 13, 397, 1450, 1482, 7, 574, 1, 23, 3784, 149, 3, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[13, 163, 1118, 0, 14, 512, 20, 341, 3436, 469, 0, 0, 0, 0, 0, 0, 1, 376, 521, 201, 0, 0, 9, 55, 4944, 0, 0, 0, 0, 0, 0, 0, 0, 1213, 0, 3031, 540, 207, 275, 780, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 565, 4952, 83, 608, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2961, 2707, 8, 54, 601, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 30, 1092, 296, 0, 2632, 52, 1263, 802, 0, 0, 0, 0, 0, 0, 0, 40, 1235, 31, 16, 301, 1128, 1242, 1942, 0, 0, 0, 0, 0, 0, 0, 8, 483, 2, 8, 4076, 99, 2, 22, 1404, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6086, 6107, 6046, 6367, 6333, 6174, 5935, 6104] avg=6144.0000 max=6367 max_rank=3 min=5935 min_rank=6 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[19, 297, 867, 2, 25, 600, 23, 411, 3415, 463, 0, 0, 0, 0, 0, 0, 6, 373, 649, 322, 2, 0, 22, 46, 4879, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 0, 2988, 466, 268, 320, 953, 0, 0, 0, 0, 0, 0, 0, 0, 59, 3, 616, 4771, 148, 600, 65, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 2943, 2487, 7, 60, 488, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 64, 1098, 418, 0, 2468, 95, 1256, 791, 0, 0, 0, 0, 0, 0, 0, 46, 1030, 61, 21, 342, 1126, 1298, 1915, 0, 0, 0, 0, 0, 0, 0, 10, 553, 2, 36, 4026, 111, 8, 14, 1384, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6299, 6280, 6264, 5990, 6214, 5839, 6144] avg=6144.0000 max=6299 max_rank=1 min=5839 min_rank=6 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[16, 297, 1124, 3, 24, 588, 20, 327, 3349, 457, 0, 0, 0, 0, 0, 0, 6, 574, 622, 429, 0, 0, 24, 44, 4776, 0, 0, 0, 0, 0, 0, 0, 0, 1130, 0, 2899, 434, 131, 437, 906, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 683, 4872, 155, 578, 59, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2873, 2143, 9, 48, 623, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3, 67, 1098, 417, 0, 2642, 58, 1234, 775, 0, 0, 0, 0, 0, 0, 0, 66, 874, 46, 13, 397, 1450, 1482, 1870, 0, 0, 0, 0, 0, 0, 0, 7, 574, 1, 23, 3784, 149, 3, 12, 1354, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6475, 5937, 6407, 5701, 6322, 6198, 5907] avg=6144.0000 max=6475 max_rank=1 min=5701 min_rank=4 imbalance_ratio=1.053874 max_load_reduction=42677 max_load_reduction_pct=0.868266 moved_to_echo=24128 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 259, 1078, 0, 17, 540, 24, 367, 3379, 445, 0, 0, 0, 0, 0, 0, 1, 389, 591, 379, 0, 0, 2, 55, 4684, 0, 0, 0, 0, 0, 0, 0, 1, 1120, 0, 3133, 511, 293, 371, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 99, 1, 541, 4605, 102, 575, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2912, 2644, 5, 60, 609, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8, 39, 1042, 328, 0, 2440, 53, 1242, 829, 0, 0, 0, 0, 0, 0, 1, 46, 796, 49, 25, 353, 1391, 1303, 2007, 0, 0, 0, 0, 0, 0, 0, 15, 497, 2, 13, 4269, 84, 3, 10, 1451, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 6101, 6431, 5969, 6231, 5994, 5971, 6344] avg=6144.0000 max=6431 max_rank=2 min=5969 min_rank=3 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 248, 1190, 0, 20, 560, 32, 365, 3418, 451, 0, 0, 0, 0, 0, 0, 1, 423, 517, 282, 0, 0, 8, 62, 4749, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 3004, 576, 180, 303, 919, 0, 0, 0, 0, 0, 0, 0, 0, 86, 2, 445, 4532, 164, 583, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2946, 2596, 3, 49, 667, 0, 0, 0, 0, 0, 0, 0, 0, 11, 7, 18, 1050, 347, 0, 2325, 77, 1257, 795, 0, 0, 0, 0, 0, 0, 0, 57, 1348, 33, 7, 338, 1267, 1465, 1925, 0, 0, 0, 0, 0, 0, 0, 8, 628, 1, 10, 4206, 69, 5, 8, 1392, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6296, 6042, 6037, 5858, 6265, 5887, 6440, 6327] avg=6144.0000 max=6440 max_rank=6 min=5858 min_rank=3 imbalance_ratio=1.048177 max_load_reduction=42712 max_load_reduction_pct=0.868978 moved_to_echo=24919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[6, 258, 1254, 0, 22, 607, 18, 368, 3386, 446, 0, 0, 0, 0, 0, 0, 1, 470, 474, 274, 1, 0, 14, 40, 4695, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 0, 3016, 706, 189, 226, 782, 0, 0, 0, 0, 0, 0, 0, 0, 69, 5, 582, 4711, 142, 576, 108, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2918, 2384, 4, 51, 660, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 25, 1028, 492, 0, 2507, 79, 1245, 799, 0, 0, 0, 0, 0, 0, 0, 58, 1101, 59, 12, 338, 1337, 1409, 1933, 0, 0, 0, 0, 0, 0, 0, 6, 617, 0, 19, 3933, 97, 6, 10, 1398, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6365, 5969, 6082, 6195, 6022, 6186, 6247, 6086] avg=6144.0000 max=6365 max_rank=0 min=5969 min_rank=1 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24541 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 199, 1008, 0, 19, 519, 20, 299, 3412, 481, 0, 0, 0, 0, 0, 0, 1, 379, 506, 228, 0, 0, 12, 70, 5067, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 3038, 638, 184, 339, 837, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 501, 4648, 107, 622, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2940, 2558, 1, 45, 623, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 20, 1013, 281, 0, 2613, 56, 1255, 805, 0, 0, 0, 0, 0, 0, 0, 42, 1207, 37, 9, 373, 1264, 1521, 1948, 0, 0, 0, 0, 0, 0, 0, 6, 580, 0, 14, 4077, 93, 2, 9, 1408, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5960, 6263, 6096, 6019, 6169, 6055, 6401, 6189] avg=6144.0000 max=6401 max_rank=6 min=5960 min_rank=0 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 195, 1119, 1, 22, 492, 20, 419, 3331, 400, 0, 0, 0, 0, 0, 0, 6, 551, 610, 451, 0, 0, 18, 45, 4215, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 2980, 539, 153, 357, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 442, 4850, 129, 518, 60, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2871, 2632, 10, 80, 848, 0, 0, 0, 0, 0, 0, 0, 0, 13, 11, 43, 1182, 300, 0, 2678, 79, 1225, 789, 0, 0, 0, 0, 0, 0, 0, 97, 924, 53, 11, 383, 1346, 1397, 1910, 0, 0, 0, 0, 0, 0, 0, 9, 611, 2, 16, 3899, 109, 4, 20, 1381, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6007, 5896, 6243, 6073, 6441, 6320, 6121, 6051] avg=6144.0000 max=6441 max_rank=4 min=5896 min_rank=1 imbalance_ratio=1.048340 max_load_reduction=42711 max_load_reduction_pct=0.868958 moved_to_echo=24933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[16, 297, 1124, 3, 24, 588, 20, 327, 3349, 457, 0, 0, 0, 0, 0, 0, 6, 574, 622, 429, 0, 0, 24, 44, 4776, 0, 0, 0, 0, 0, 0, 0, 0, 1130, 0, 2899, 434, 131, 437, 906, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 683, 4872, 155, 578, 59, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2873, 2143, 9, 48, 623, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3, 67, 1098, 417, 0, 2642, 58, 1234, 775, 0, 0, 0, 0, 0, 0, 0, 66, 874, 46, 13, 397, 1450, 1482, 1870, 0, 0, 0, 0, 0, 0, 0, 7, 574, 1, 23, 3784, 149, 3, 12, 1354, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6475, 5937, 6407, 5701, 6322, 6198, 5907] avg=6144.0000 max=6475 max_rank=1 min=5701 min_rank=4 imbalance_ratio=1.053874 max_load_reduction=42677 max_load_reduction_pct=0.868266 moved_to_echo=24128 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1920, 8832, 128, 256, 4480, 256, 2944, 27136, 3712, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 9216, 0, 24192, 4480, 1664, 2688, 7296, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 512, 8576, 384, 128, 2944, 10368, 11136, 15488, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1920, 8832, 128, 256, 4480, 256, 2944, 27136, 3712, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 23424, 20224, 128, 512, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 9216, 0, 24192, 4480, 1664, 2688, 7296, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 384, 8704, 2944, 0, 20352, 640, 9984, 6400, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 4608, 128, 256, 32384, 896, 128, 128, 11264, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 128, 4480, 38016, 1152, 4736, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 3584, 4608, 2688, 128, 0, 128, 512, 38016, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 500, 1715, 0, 376, 23, 37, 463, 49, 0, 4, 1238, 0, 2, 2160, 1181, 6015, 8012, 26, 2, 5326, 1548, 2285, 0, 0, 6, 342, 0, 0, 1, 20, 0, 7, 0, 2, 0, 0, 2, 3564, 135, 6973, 11, 293, 0, 0, 3, 0, 1743, 11, 6, 244, 3107, 4, 25, 0, 823, 0, 58, 0, 17, 0, 675, 0, 117] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 424, 1707, 0, 332, 25, 34, 352, 125, 0, 13, 1467, 1, 7, 2849, 1293, 5592, 7926, 46, 1, 5523, 1311, 2306, 0, 0, 11, 370, 0, 0, 2, 40, 0, 12, 0, 0, 1, 0, 2, 3520, 194, 6656, 5, 320, 1, 0, 11, 0, 1798, 6, 9, 329, 2997, 3, 20, 0, 772, 0, 58, 0, 14, 1, 548, 1, 117] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 470, 2046, 0, 339, 37, 78, 220, 117, 0, 11, 1318, 0, 2, 2198, 1299, 5723, 8036, 49, 3, 5025, 1646, 2680, 0, 1, 14, 328, 0, 0, 0, 26, 0, 11, 0, 0, 2, 0, 3, 3067, 181, 6831, 5, 238, 1, 0, 16, 1, 1899, 11, 11, 234, 3469, 6, 34, 1, 668, 5, 40, 1, 31, 0, 612, 0, 108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 391, 2067, 0, 272, 20, 39, 233, 65, 0, 1, 1375, 0, 2, 2592, 1160, 5738, 8040, 55, 4, 5426, 1444, 2621, 0, 0, 8, 348, 0, 0, 1, 19, 0, 4, 0, 0, 1, 0, 0, 2933, 155, 7080, 4, 233, 2, 0, 14, 0, 1881, 5, 11, 258, 3198, 3, 11, 0, 696, 0, 38, 0, 17, 0, 597, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3, 432, 1994, 0, 374, 31, 37, 366, 132, 0, 15, 1348, 0, 1, 2211, 1313, 5508, 7954, 56, 4, 5053, 1540, 2499, 0, 0, 21, 361, 0, 0, 0, 35, 4, 6, 0, 5, 3, 0, 3, 3504, 221, 6854, 8, 308, 2, 1, 5, 4, 2007, 13, 10, 271, 3224, 4, 31, 0, 595, 0, 75, 8, 19, 0, 550, 1, 128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 360, 2072, 0, 304, 21, 28, 232, 82, 0, 3, 1241, 0, 3, 2324, 1313, 5669, 8003, 47, 3, 5790, 1436, 2594, 0, 0, 16, 343, 1, 0, 0, 25, 0, 1, 0, 1, 1, 0, 0, 3440, 137, 7009, 5, 170, 1, 0, 4, 0, 1567, 4, 11, 271, 3366, 4, 17, 0, 550, 0, 26, 0, 7, 0, 542, 0, 107] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[5, 493, 1972, 0, 352, 23, 64, 420, 127, 0, 10, 1358, 0, 1, 2414, 1333, 5742, 7952, 42, 2, 5191, 1535, 2196, 0, 0, 16, 375, 0, 0, 1, 26, 0, 15, 0, 0, 1, 0, 2, 3255, 142, 6833, 7, 292, 0, 0, 7, 0, 1528, 12, 17, 255, 3101, 8, 22, 0, 1143, 0, 30, 0, 11, 0, 724, 1, 96] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 424, 1707, 0, 332, 25, 34, 352, 125, 0, 13, 1467, 1, 7, 2849, 1293, 5592, 7926, 46, 1, 5523, 1311, 2306, 0, 0, 11, 370, 0, 0, 2, 40, 0, 12, 0, 0, 1, 0, 2, 3520, 194, 6656, 5, 320, 1, 0, 11, 0, 1798, 6, 9, 329, 2997, 3, 20, 0, 772, 0, 58, 0, 14, 1, 548, 1, 117] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4, 469, 1898, 0, 430, 23, 70, 462, 82, 0, 7, 1451, 1, 0, 2077, 1477, 5606, 7839, 20, 3, 4990, 1540, 2348, 0, 0, 31, 337, 0, 0, 1, 33, 0, 5, 0, 0, 2, 0, 3, 3543, 210, 6980, 4, 272, 2, 0, 2, 0, 1568, 14, 16, 248, 3236, 3, 46, 0, 860, 0, 46, 0, 27, 0, 725, 0, 141] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 500, 1715, 0, 376, 23, 37, 463, 2720, 426, 0, 0, 0, 0, 0, 0, 49, 0, 4, 1238, 0, 2, 2160, 1181, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2, 2173, 1548, 2285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 342, 0, 0, 1, 20, 0, 5778, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 0, 0, 2, 3564, 135, 2624, 0, 0, 0, 0, 0, 0, 0, 4156, 11, 293, 0, 0, 3, 0, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 244, 3107, 4, 25, 0, 823, 1769, 103, 0, 0, 0, 0, 0, 0, 0, 58, 0, 17, 0, 675, 0, 117, 3295, 2234, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 5682, 6034, 6147, 6334, 6206, 6092, 6396] avg=6144.0000 max=6396 max_rank=7 min=5682 min_rank=1 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=25028 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 391, 2067, 0, 272, 20, 39, 233, 2595, 434, 0, 0, 0, 0, 0, 0, 65, 0, 1, 1375, 0, 2, 2592, 1160, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4, 2214, 1444, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 348, 0, 0, 1, 19, 0, 5799, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 2933, 155, 2673, 0, 0, 0, 0, 0, 0, 0, 4220, 4, 233, 2, 0, 14, 0, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 258, 3198, 3, 11, 0, 696, 1796, 105, 0, 0, 0, 0, 0, 0, 0, 38, 0, 17, 0, 597, 0, 90, 3143, 2241, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6051, 6259, 6338, 6175, 5766, 6354, 6083, 6126] avg=6144.0000 max=6354 max_rank=5 min=5766 min_rank=4 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24329 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 470, 2046, 0, 339, 37, 78, 220, 2589, 402, 0, 0, 0, 0, 0, 0, 117, 0, 11, 1318, 0, 2, 2198, 1299, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 3, 2050, 1646, 2680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 328, 0, 0, 0, 26, 0, 5797, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 3, 3067, 181, 2476, 0, 0, 0, 0, 0, 0, 0, 4073, 5, 238, 1, 0, 16, 1, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 234, 3469, 6, 34, 1, 668, 1732, 97, 0, 0, 0, 0, 0, 0, 5, 40, 1, 31, 0, 612, 0, 108, 3134, 2239, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6181, 5971, 6428, 6166, 5740, 6233, 6263, 6170] avg=6144.0000 max=6428 max_rank=2 min=5740 min_rank=4 imbalance_ratio=1.046224 max_load_reduction=42724 max_load_reduction_pct=0.869222 moved_to_echo=24406 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 360, 2072, 0, 304, 21, 28, 232, 2565, 463, 0, 0, 0, 0, 0, 0, 82, 0, 3, 1241, 0, 3, 2324, 1313, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 3, 2362, 1436, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 343, 1, 0, 0, 25, 0, 5773, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 3440, 137, 2853, 0, 0, 0, 0, 0, 0, 0, 4178, 5, 170, 1, 0, 4, 0, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 271, 3366, 4, 17, 0, 550, 1778, 112, 0, 0, 0, 0, 0, 0, 0, 26, 0, 7, 0, 542, 0, 107, 3104, 2230, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 6019, 6442, 6158, 6433, 5925, 6113, 6016] avg=6144.0000 max=6442 max_rank=2 min=5925 min_rank=5 imbalance_ratio=1.048503 max_load_reduction=42710 max_load_reduction_pct=0.868937 moved_to_echo=24487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 391, 2067, 0, 272, 20, 39, 233, 65, 0, 1, 1375, 0, 2, 2592, 1160, 5738, 8040, 55, 4, 5426, 1444, 2621, 0, 0, 8, 348, 0, 0, 1, 19, 0, 4, 0, 0, 1, 0, 0, 2933, 155, 7080, 4, 233, 2, 0, 14, 0, 1881, 5, 11, 258, 3198, 3, 11, 0, 696, 0, 38, 0, 17, 0, 597, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 500, 1715, 0, 376, 23, 37, 463, 49, 0, 4, 1238, 0, 2, 2160, 1181, 6015, 8012, 26, 2, 5326, 1548, 2285, 0, 0, 6, 342, 0, 0, 1, 20, 0, 7, 0, 2, 0, 0, 2, 3564, 135, 6973, 11, 293, 0, 0, 3, 0, 1743, 11, 6, 244, 3107, 4, 25, 0, 823, 0, 58, 0, 17, 0, 675, 0, 117] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 470, 2046, 0, 339, 37, 78, 220, 117, 0, 11, 1318, 0, 2, 2198, 1299, 5723, 8036, 49, 3, 5025, 1646, 2680, 0, 1, 14, 328, 0, 0, 0, 26, 0, 11, 0, 0, 2, 0, 3, 3067, 181, 6831, 5, 238, 1, 0, 16, 1, 1899, 11, 11, 234, 3469, 6, 34, 1, 668, 5, 40, 1, 31, 0, 612, 0, 108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3, 432, 1994, 0, 374, 31, 37, 366, 132, 0, 15, 1348, 0, 1, 2211, 1313, 5508, 7954, 56, 4, 5053, 1540, 2499, 0, 0, 21, 361, 0, 0, 0, 35, 4, 6, 0, 5, 3, 0, 3, 3504, 221, 6854, 8, 308, 2, 1, 5, 4, 2007, 13, 10, 271, 3224, 4, 31, 0, 595, 0, 75, 8, 19, 0, 550, 1, 128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 424, 1707, 0, 332, 25, 34, 352, 2529, 442, 0, 0, 0, 0, 0, 0, 125, 0, 13, 1467, 1, 7, 2849, 1293, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 2253, 1311, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 370, 0, 0, 2, 40, 0, 5716, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 2, 3520, 194, 2721, 0, 0, 0, 0, 0, 0, 0, 3968, 5, 320, 1, 0, 11, 0, 1798, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 329, 2997, 3, 20, 0, 772, 1688, 107, 0, 0, 0, 0, 0, 0, 0, 58, 0, 14, 1, 548, 1, 117, 3063, 2210, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5845, 6755, 5917, 6139, 6450, 6103, 5931, 6012] avg=6144.0000 max=6755 max_rank=1 min=5845 min_rank=0 imbalance_ratio=1.099447 max_load_reduction=42397 max_load_reduction_pct=0.862569 moved_to_echo=24496 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 493, 1972, 0, 352, 23, 64, 420, 2597, 415, 0, 0, 0, 0, 0, 0, 127, 0, 10, 1358, 0, 1, 2414, 1333, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 2, 2119, 1535, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 375, 0, 0, 1, 26, 0, 5735, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 0, 2, 3255, 142, 2557, 0, 0, 0, 0, 0, 0, 0, 4073, 7, 292, 0, 0, 7, 0, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 12, 17, 255, 3101, 8, 22, 0, 1143, 1733, 100, 0, 0, 0, 0, 0, 0, 0, 30, 0, 11, 0, 724, 1, 96, 3145, 2217, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6341, 6270, 5894, 6153, 5972, 5907, 6391, 6224] avg=6144.0000 max=6391 max_rank=6 min=5894 min_rank=2 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24494 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 432, 1994, 0, 374, 31, 37, 366, 2492, 407, 0, 0, 0, 0, 0, 0, 132, 0, 15, 1348, 0, 1, 2211, 1313, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4, 2052, 1540, 2499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 361, 0, 0, 0, 35, 4, 5737, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 3, 0, 3, 3504, 221, 2494, 0, 0, 0, 0, 0, 0, 0, 4080, 8, 308, 2, 1, 5, 4, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 13, 10, 271, 3224, 4, 31, 0, 595, 1742, 100, 0, 0, 0, 0, 0, 0, 0, 75, 8, 19, 0, 550, 1, 128, 3016, 2217, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6052, 6151, 6158, 6236, 6415, 5990, 6014] avg=6144.0000 max=6415 max_rank=5 min=5990 min_rank=6 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=24655 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 360, 2072, 0, 304, 21, 28, 232, 2565, 463, 0, 0, 0, 0, 0, 0, 82, 0, 3, 1241, 0, 3, 2324, 1313, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 3, 2362, 1436, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 343, 1, 0, 0, 25, 0, 5773, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 3440, 137, 2853, 0, 0, 0, 0, 0, 0, 0, 4178, 5, 170, 1, 0, 4, 0, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 271, 3366, 4, 17, 0, 550, 1778, 112, 0, 0, 0, 0, 0, 0, 0, 26, 0, 7, 0, 542, 0, 107, 3104, 2230, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 6019, 6442, 6158, 6433, 5925, 6113, 6016] avg=6144.0000 max=6442 max_rank=2 min=5925 min_rank=5 imbalance_ratio=1.048503 max_load_reduction=42710 max_load_reduction_pct=0.868937 moved_to_echo=24487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 391, 2067, 0, 272, 20, 39, 233, 2595, 434, 0, 0, 0, 0, 0, 0, 65, 0, 1, 1375, 0, 2, 2592, 1160, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4, 2214, 1444, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 348, 0, 0, 1, 19, 0, 5799, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 2933, 155, 2673, 0, 0, 0, 0, 0, 0, 0, 4220, 4, 233, 2, 0, 14, 0, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 258, 3198, 3, 11, 0, 696, 1796, 105, 0, 0, 0, 0, 0, 0, 0, 38, 0, 17, 0, 597, 0, 90, 3143, 2241, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6051, 6259, 6338, 6175, 5766, 6354, 6083, 6126] avg=6144.0000 max=6354 max_rank=5 min=5766 min_rank=4 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24329 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 469, 1898, 0, 430, 23, 70, 462, 2536, 399, 0, 0, 0, 0, 0, 0, 82, 0, 7, 1451, 1, 0, 2077, 1477, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3, 2037, 1540, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 337, 0, 0, 1, 33, 0, 5654, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 3, 3543, 210, 2458, 0, 0, 0, 0, 0, 0, 0, 4161, 4, 272, 2, 0, 2, 0, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 14, 16, 248, 3236, 3, 46, 0, 860, 1770, 96, 0, 0, 0, 0, 0, 0, 0, 46, 0, 27, 0, 725, 0, 141, 3070, 2185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6291, 6144, 5948, 6056, 6221, 6009, 6289, 6194] avg=6144.0000 max=6291 max_rank=0 min=5948 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=42861 max_load_reduction_pct=0.872009 moved_to_echo=24713 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 500, 1715, 0, 376, 23, 37, 463, 2720, 426, 0, 0, 0, 0, 0, 0, 49, 0, 4, 1238, 0, 2, 2160, 1181, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2, 2173, 1548, 2285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 342, 0, 0, 1, 20, 0, 5778, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 0, 0, 2, 3564, 135, 2624, 0, 0, 0, 0, 0, 0, 0, 4156, 11, 293, 0, 0, 3, 0, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 244, 3107, 4, 25, 0, 823, 1769, 103, 0, 0, 0, 0, 0, 0, 0, 58, 0, 17, 0, 675, 0, 117, 3295, 2234, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 5682, 6034, 6147, 6334, 6206, 6092, 6396] avg=6144.0000 max=6396 max_rank=7 min=5682 min_rank=1 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=25028 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 470, 2046, 0, 339, 37, 78, 220, 2589, 402, 0, 0, 0, 0, 0, 0, 117, 0, 11, 1318, 0, 2, 2198, 1299, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 3, 2050, 1646, 2680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 328, 0, 0, 0, 26, 0, 5797, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 3, 3067, 181, 2476, 0, 0, 0, 0, 0, 0, 0, 4073, 5, 238, 1, 0, 16, 1, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 234, 3469, 6, 34, 1, 668, 1732, 97, 0, 0, 0, 0, 0, 0, 5, 40, 1, 31, 0, 612, 0, 108, 3134, 2239, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6181, 5971, 6428, 6166, 5740, 6233, 6263, 6170] avg=6144.0000 max=6428 max_rank=2 min=5740 min_rank=4 imbalance_ratio=1.046224 max_load_reduction=42724 max_load_reduction_pct=0.869222 moved_to_echo=24406 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 432, 1994, 0, 374, 31, 37, 366, 2492, 407, 0, 0, 0, 0, 0, 0, 132, 0, 15, 1348, 0, 1, 2211, 1313, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4, 2052, 1540, 2499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 361, 0, 0, 0, 35, 4, 5737, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 3, 0, 3, 3504, 221, 2494, 0, 0, 0, 0, 0, 0, 0, 4080, 8, 308, 2, 1, 5, 4, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 13, 10, 271, 3224, 4, 31, 0, 595, 1742, 100, 0, 0, 0, 0, 0, 0, 0, 75, 8, 19, 0, 550, 1, 128, 3016, 2217, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6052, 6151, 6158, 6236, 6415, 5990, 6014] avg=6144.0000 max=6415 max_rank=5 min=5990 min_rank=6 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=24655 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 384, 128, 256, 128, 4992, 128, 1024, 25088, 17792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 0, 128, 10880, 128, 128, 18944, 10496, 8320, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 3584, 15488, 0, 2816, 256, 512, 2816, 20736, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 0, 128, 10880, 128, 128, 18944, 10496, 8320, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[33024, 128, 2176, 128, 128, 128, 128, 14080, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 384, 128, 17280, 12032, 19584, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 2176, 25728, 128, 256, 128, 6144, 14080, 896, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 128, 26880, 1408, 20864, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[33024, 128, 2176, 128, 128, 128, 128, 14080, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 2816, 128, 0, 128, 256, 128, 46080, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1971, 0, 7269, 11, 0, 0, 0, 4691, 0, 0, 909, 1701, 6729, 6236, 463, 12, 0, 0, 0, 509, 0, 5, 1917, 669, 0, 82, 115, 2106, 0, 0, 3, 0, 74, 1, 5596, 2, 0, 0, 0, 0, 4, 2019, 0, 19, 6, 0, 701, 92, 16, 0, 4597, 527, 29, 13, 0, 0, 51, 3, 0, 0, 2, 0, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1813, 0, 7414, 12, 0, 0, 0, 5167, 0, 0, 830, 1646, 6795, 6135, 458, 11, 0, 0, 0, 275, 0, 6, 2465, 657, 0, 64, 94, 1409, 0, 1, 7, 0, 69, 0, 5566, 3, 0, 0, 0, 0, 9, 2225, 0, 44, 4, 0, 541, 79, 14, 1, 4773, 505, 0, 1, 0, 0, 52, 3, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[2114, 0, 7028, 12, 0, 0, 1, 4513, 0, 0, 975, 1956, 6747, 5902, 464, 11, 0, 0, 2, 516, 0, 16, 1854, 897, 0, 103, 137, 2146, 0, 0, 0, 0, 40, 7, 5397, 7, 0, 0, 0, 0, 1, 1996, 0, 52, 8, 0, 689, 109, 26, 0, 4670, 479, 148, 22, 0, 0, 93, 7, 0, 0, 7, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[2053, 0, 7423, 8, 0, 0, 0, 4660, 0, 0, 856, 1561, 6549, 5666, 446, 15, 0, 0, 0, 315, 0, 3, 2436, 555, 0, 67, 116, 1930, 0, 0, 1, 0, 48, 0, 5749, 1, 0, 0, 0, 0, 0, 1883, 0, 25, 6, 0, 683, 98, 33, 0, 5282, 604, 19, 2, 0, 0, 53, 2, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1996, 0, 7130, 11, 0, 0, 2, 4400, 0, 2, 993, 1657, 6929, 5975, 432, 10, 0, 0, 0, 576, 0, 8, 2270, 739, 0, 87, 111, 1825, 0, 1, 0, 0, 76, 1, 5574, 1, 0, 1, 0, 0, 1, 1858, 0, 48, 10, 0, 773, 82, 19, 0, 4862, 577, 33, 18, 0, 0, 51, 6, 1, 0, 6, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[2090, 0, 7492, 6, 0, 0, 0, 4302, 0, 0, 1029, 1821, 7031, 6033, 396, 8, 0, 0, 0, 438, 0, 12, 2094, 658, 0, 110, 124, 2082, 0, 1, 2, 0, 19, 0, 4508, 3, 1, 0, 0, 0, 9, 2115, 0, 40, 4, 1, 795, 100, 33, 0, 5235, 485, 14, 12, 0, 0, 31, 9, 0, 0, 8, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[2114, 0, 7028, 12, 0, 0, 1, 4513, 0, 0, 975, 1956, 6747, 5902, 464, 11, 0, 0, 2, 516, 0, 16, 1854, 897, 0, 103, 137, 2146, 0, 0, 0, 0, 40, 7, 5397, 7, 0, 0, 0, 0, 1, 1996, 0, 52, 8, 0, 689, 109, 26, 0, 4670, 479, 148, 22, 0, 0, 93, 7, 0, 0, 7, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2043, 1, 7500, 7, 1, 0, 0, 4780, 0, 0, 1132, 1765, 6434, 6057, 500, 5, 0, 0, 3, 396, 0, 5, 2054, 684, 0, 78, 104, 1722, 0, 0, 5, 0, 65, 2, 5679, 2, 0, 0, 0, 0, 8, 1661, 0, 29, 2, 0, 710, 92, 30, 0, 4819, 679, 18, 12, 1, 0, 56, 1, 0, 0, 0, 0, 8, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1971, 0, 7269, 11, 0, 0, 0, 4691, 0, 0, 909, 1701, 6729, 6236, 463, 12, 0, 0, 0, 509, 0, 5, 1917, 669, 0, 82, 115, 2106, 0, 0, 3, 0, 74, 1, 5596, 2, 0, 0, 0, 0, 4, 2019, 0, 19, 6, 0, 701, 92, 16, 0, 4597, 527, 29, 13, 0, 0, 51, 3, 0, 0, 2, 0, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1996, 0, 7130, 11, 0, 0, 2, 4400, 0, 2, 993, 1657, 6929, 5975, 432, 10, 0, 0, 0, 576, 0, 8, 2270, 739, 0, 87, 111, 1825, 0, 1, 0, 0, 76, 1, 5574, 1, 0, 1, 0, 0, 1, 1858, 0, 48, 10, 0, 773, 82, 19, 0, 4862, 577, 33, 18, 0, 0, 51, 6, 1, 0, 6, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[2053, 0, 7423, 8, 0, 0, 0, 4660, 0, 0, 856, 1561, 6549, 5666, 446, 15, 0, 0, 0, 315, 0, 3, 2436, 555, 0, 67, 116, 1930, 0, 0, 1, 0, 48, 0, 5749, 1, 0, 0, 0, 0, 0, 1883, 0, 25, 6, 0, 683, 98, 33, 0, 5282, 604, 19, 2, 0, 0, 53, 2, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1668, 0, 7349, 12, 0, 0, 0, 4688, 0, 1, 987, 1804, 7035, 5944, 473, 22, 0, 0, 0, 210, 1, 16, 2068, 792, 0, 72, 95, 1788, 0, 0, 3, 0, 84, 8, 5127, 6, 1, 0, 0, 1, 8, 1717, 0, 60, 5, 0, 944, 101, 42, 0, 5322, 547, 73, 6, 0, 0, 60, 1, 1, 0, 8, 0, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1813, 0, 0, 12, 0, 0, 0, 4628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 1646, 0, 3046, 458, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 6, 2465, 657, 2308, 625, 0, 0, 0, 0, 0, 0, 0, 64, 94, 1409, 0, 1, 7, 0, 2828, 1263, 0, 0, 0, 0, 0, 0, 69, 0, 5566, 3, 0, 0, 0, 0, 695, 0, 0, 0, 0, 0, 0, 0, 9, 2225, 0, 44, 4, 0, 541, 79, 3342, 0, 0, 0, 0, 0, 0, 0, 14, 1, 4773, 505, 0, 1, 0, 0, 539, 86, 0, 0, 0, 0, 0, 0, 52, 3, 0, 0, 4, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6453, 5991, 6336, 5666, 6333, 6244, 5919, 6210] avg=6144.0000 max=6453 max_rank=0 min=5666 min_rank=3 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24706 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2090, 0, 0, 6, 0, 0, 0, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1821, 0, 2997, 396, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 12, 2094, 658, 2269, 646, 0, 0, 0, 0, 0, 0, 0, 110, 124, 2082, 0, 1, 2, 0, 2925, 1276, 0, 0, 0, 0, 0, 0, 19, 0, 4508, 3, 1, 0, 0, 0, 683, 0, 0, 0, 0, 0, 0, 0, 9, 2115, 0, 40, 4, 1, 795, 100, 3460, 0, 0, 0, 0, 0, 0, 0, 33, 0, 5235, 485, 14, 12, 0, 0, 448, 84, 0, 0, 0, 0, 0, 0, 31, 9, 0, 0, 8, 0, 1, 0, 6216, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5950, 6251, 6117, 6520, 5214, 6524, 6311, 6265] avg=6144.0000 max=6524 max_rank=5 min=5214 min_rank=4 imbalance_ratio=1.061849 max_load_reduction=42628 max_load_reduction_pct=0.867269 moved_to_echo=24314 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1971, 0, 0, 11, 0, 0, 0, 4202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 1701, 0, 3097, 463, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 0, 5, 1917, 669, 2346, 618, 0, 0, 0, 0, 0, 0, 0, 82, 115, 2106, 0, 0, 3, 0, 2799, 1238, 0, 0, 0, 0, 0, 0, 74, 1, 5596, 2, 0, 0, 0, 0, 706, 0, 0, 0, 0, 0, 0, 0, 4, 2019, 0, 19, 6, 0, 701, 92, 3312, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4597, 527, 29, 13, 0, 0, 489, 87, 0, 0, 0, 0, 0, 0, 51, 3, 0, 0, 2, 0, 2, 0, 6031, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 6182, 6064, 6343, 6379, 6153, 5758, 6089] avg=6144.0000 max=6379 max_rank=4 min=5758 min_rank=6 imbalance_ratio=1.038249 max_load_reduction=42773 max_load_reduction_pct=0.870219 moved_to_echo=24379 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2043, 1, 0, 7, 1, 0, 0, 4282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 1765, 0, 3007, 500, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 396, 0, 5, 2054, 684, 2279, 591, 0, 0, 0, 0, 0, 0, 0, 78, 104, 1722, 0, 0, 5, 0, 2679, 1279, 0, 0, 0, 0, 0, 0, 65, 2, 5679, 2, 0, 0, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 8, 1661, 0, 29, 2, 0, 710, 92, 3164, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4819, 679, 18, 12, 1, 0, 498, 85, 0, 0, 0, 0, 0, 0, 56, 1, 0, 0, 0, 0, 8, 2, 6221, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6334, 6409, 6012, 5867, 6434, 5666, 6142, 6288] avg=6144.0000 max=6434 max_rank=4 min=5666 min_rank=5 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24530 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1996, 0, 0, 11, 0, 0, 2, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 993, 1657, 0, 2967, 432, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 8, 2270, 739, 2248, 637, 0, 0, 0, 0, 0, 0, 0, 87, 111, 1825, 0, 1, 0, 0, 2883, 1215, 0, 0, 0, 0, 0, 0, 76, 1, 5574, 1, 0, 1, 0, 0, 677, 0, 0, 0, 0, 0, 0, 0, 1, 1858, 0, 48, 10, 0, 773, 82, 3409, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4862, 577, 33, 18, 0, 0, 459, 83, 0, 0, 0, 0, 0, 0, 51, 6, 1, 0, 6, 0, 0, 0, 5915, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5950, 6061, 6478, 6122, 6330, 6181, 6051, 5979] avg=6144.0000 max=6478 max_rank=2 min=5950 min_rank=0 imbalance_ratio=1.054362 max_load_reduction=42674 max_load_reduction_pct=0.868205 moved_to_echo=24541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2053, 0, 0, 8, 0, 0, 0, 4174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 856, 1561, 0, 2814, 446, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 3, 2436, 555, 2131, 603, 0, 0, 0, 0, 0, 0, 0, 67, 116, 1930, 0, 0, 1, 0, 2725, 1266, 0, 0, 0, 0, 0, 0, 48, 0, 5749, 1, 0, 0, 0, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 0, 25, 6, 0, 683, 98, 3221, 0, 0, 0, 0, 0, 0, 0, 33, 0, 5282, 604, 19, 2, 0, 0, 486, 79, 0, 0, 0, 0, 0, 0, 53, 2, 0, 0, 4, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6235, 5692, 6043, 6105, 6440, 5916, 6505, 6216] avg=6144.0000 max=6505 max_rank=6 min=5692 min_rank=1 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=25077 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2090, 0, 0, 6, 0, 0, 0, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1821, 0, 2997, 396, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 12, 2094, 658, 2269, 646, 0, 0, 0, 0, 0, 0, 0, 110, 124, 2082, 0, 1, 2, 0, 2925, 1276, 0, 0, 0, 0, 0, 0, 19, 0, 4508, 3, 1, 0, 0, 0, 683, 0, 0, 0, 0, 0, 0, 0, 9, 2115, 0, 40, 4, 1, 795, 100, 3460, 0, 0, 0, 0, 0, 0, 0, 33, 0, 5235, 485, 14, 12, 0, 0, 448, 84, 0, 0, 0, 0, 0, 0, 31, 9, 0, 0, 8, 0, 1, 0, 6216, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5950, 6251, 6117, 6520, 5214, 6524, 6311, 6265] avg=6144.0000 max=6524 max_rank=5 min=5214 min_rank=4 imbalance_ratio=1.061849 max_load_reduction=42628 max_load_reduction_pct=0.867269 moved_to_echo=24314 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2114, 0, 0, 12, 0, 0, 1, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1956, 0, 2932, 464, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 516, 0, 16, 1854, 897, 2220, 622, 0, 0, 0, 0, 0, 0, 0, 103, 137, 2146, 0, 0, 0, 0, 2807, 1198, 0, 0, 0, 0, 0, 0, 40, 7, 5397, 7, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 0, 1, 1996, 0, 52, 8, 0, 689, 109, 3318, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4670, 479, 148, 22, 0, 0, 470, 82, 0, 0, 0, 0, 0, 0, 93, 7, 0, 0, 7, 0, 0, 0, 5830, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 6338, 6127, 6391, 6119, 6173, 5897, 5937] avg=6144.0000 max=6391 max_rank=3 min=5897 min_rank=6 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24126 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2043, 1, 0, 7, 1, 0, 0, 4282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 1765, 0, 3007, 500, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 396, 0, 5, 2054, 684, 2279, 591, 0, 0, 0, 0, 0, 0, 0, 78, 104, 1722, 0, 0, 5, 0, 2679, 1279, 0, 0, 0, 0, 0, 0, 65, 2, 5679, 2, 0, 0, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 8, 1661, 0, 29, 2, 0, 710, 92, 3164, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4819, 679, 18, 12, 1, 0, 498, 85, 0, 0, 0, 0, 0, 0, 56, 1, 0, 0, 0, 0, 8, 2, 6221, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6334, 6409, 6012, 5867, 6434, 5666, 6142, 6288] avg=6144.0000 max=6434 max_rank=4 min=5666 min_rank=5 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24530 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1668, 0, 0, 12, 0, 0, 0, 4196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 987, 1804, 0, 2941, 473, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 1, 16, 2068, 792, 2240, 648, 0, 0, 0, 0, 0, 0, 0, 72, 95, 1788, 0, 0, 3, 0, 2927, 1253, 0, 0, 0, 0, 0, 0, 84, 8, 5127, 6, 1, 0, 0, 1, 676, 0, 0, 0, 0, 0, 0, 0, 8, 1717, 0, 60, 5, 0, 944, 101, 3460, 0, 0, 0, 0, 0, 0, 0, 42, 0, 5322, 547, 73, 6, 0, 0, 492, 87, 0, 0, 0, 0, 0, 0, 60, 1, 1, 0, 8, 0, 2, 0, 6096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5876, 6228, 5975, 6138, 5903, 6295, 6569, 6168] avg=6144.0000 max=6569 max_rank=6 min=5876 min_rank=0 imbalance_ratio=1.069173 max_load_reduction=42583 max_load_reduction_pct=0.866353 moved_to_echo=24935 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1996, 0, 0, 11, 0, 0, 2, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 993, 1657, 0, 2967, 432, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 8, 2270, 739, 2248, 637, 0, 0, 0, 0, 0, 0, 0, 87, 111, 1825, 0, 1, 0, 0, 2883, 1215, 0, 0, 0, 0, 0, 0, 76, 1, 5574, 1, 0, 1, 0, 0, 677, 0, 0, 0, 0, 0, 0, 0, 1, 1858, 0, 48, 10, 0, 773, 82, 3409, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4862, 577, 33, 18, 0, 0, 459, 83, 0, 0, 0, 0, 0, 0, 51, 6, 1, 0, 6, 0, 0, 0, 5915, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5950, 6061, 6478, 6122, 6330, 6181, 6051, 5979] avg=6144.0000 max=6478 max_rank=2 min=5950 min_rank=0 imbalance_ratio=1.054362 max_load_reduction=42674 max_load_reduction_pct=0.868205 moved_to_echo=24541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 7808, 13952, 0, 23808, 3712, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 128, 3328, 128, 128, 17280, 5760, 18048, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 15488, 0, 384, 128, 128, 5888, 768, 26752, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[15872, 128, 0, 128, 128, 0, 128, 33408, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 768, 896, 15104, 0, 128, 128, 0, 22656, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 7808, 13952, 0, 23808, 3712, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[15872, 128, 0, 128, 128, 0, 128, 33408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 15488, 0, 384, 128, 128, 5888, 768, 26752, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 39680, 4480, 384, 128, 128, 0, 3968, 768, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[512, 128, 43264, 128, 128, 128, 0, 128, 5504, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 128, 128, 0, 128, 0, 128, 128, 48640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1468, 499, 0, 112, 0, 0, 0, 0, 2, 73, 7675, 1, 1, 2365, 7264, 94, 0, 0, 0, 0, 1029, 54, 0, 8, 33, 442, 22, 1343, 0, 8189, 1, 49, 2018, 0, 1210, 0, 4682, 0, 56, 0, 410, 2764, 0, 5, 0, 177, 0, 0, 190, 0, 2, 9, 85, 5, 5561, 1, 7, 814, 0, 0, 0, 0, 409, 23] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1202, 456, 0, 62, 1, 0, 0, 0, 5, 52, 7774, 0, 1, 2177, 7421, 96, 0, 0, 0, 0, 1280, 52, 0, 14, 31, 278, 5, 932, 0, 8189, 0, 12, 2453, 1, 940, 0, 5365, 0, 39, 0, 384, 2933, 0, 2, 1, 184, 0, 0, 167, 0, 7, 13, 52, 0, 5520, 0, 12, 668, 0, 0, 0, 0, 337, 34] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1174, 447, 0, 50, 0, 0, 0, 0, 1, 67, 7820, 0, 0, 2338, 7234, 45, 0, 0, 0, 0, 1047, 53, 0, 4, 37, 254, 18, 1214, 0, 8192, 2, 41, 2645, 0, 1173, 0, 5192, 0, 55, 0, 414, 2644, 0, 0, 1, 142, 0, 0, 153, 0, 4, 15, 54, 3, 5404, 0, 7, 885, 0, 0, 0, 0, 292, 31] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1447, 542, 0, 62, 0, 0, 0, 1, 2, 81, 7671, 0, 0, 2158, 7379, 189, 1, 0, 1, 0, 1157, 68, 0, 10, 66, 458, 20, 1373, 0, 8170, 5, 27, 2086, 1, 1042, 0, 5214, 0, 23, 0, 421, 2396, 0, 0, 27, 197, 0, 0, 174, 0, 5, 15, 81, 8, 5347, 0, 9, 807, 0, 0, 0, 0, 353, 58] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1007, 439, 0, 64, 0, 0, 0, 3, 0, 67, 7687, 0, 2, 2339, 7324, 32, 0, 0, 2, 0, 1147, 72, 0, 4, 52, 308, 14, 1647, 3, 8181, 1, 40, 2303, 1, 1397, 0, 5297, 1, 33, 0, 396, 1939, 2, 1, 5, 151, 0, 0, 195, 2, 2, 16, 76, 15, 5667, 0, 12, 897, 0, 1, 0, 1, 288, 19] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1292, 487, 0, 45, 0, 0, 0, 0, 0, 71, 7872, 0, 0, 2003, 7328, 65, 0, 0, 0, 0, 992, 48, 0, 1, 38, 287, 5, 1643, 0, 8192, 1, 9, 2922, 0, 1033, 0, 5454, 0, 29, 0, 371, 2084, 0, 0, 0, 182, 0, 0, 103, 0, 2, 14, 65, 0, 5382, 0, 0, 861, 0, 0, 0, 0, 239, 32] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1223, 392, 0, 54, 0, 0, 0, 2, 1, 72, 7818, 0, 0, 2436, 7208, 92, 0, 1, 0, 0, 1229, 33, 0, 2, 29, 323, 7, 1436, 1, 8191, 5, 11, 2723, 1, 983, 0, 5567, 0, 20, 0, 387, 2149, 0, 1, 0, 122, 0, 0, 122, 0, 2, 12, 80, 6, 5347, 0, 4, 855, 0, 0, 0, 0, 179, 26] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1174, 447, 0, 50, 0, 0, 0, 0, 1, 67, 7820, 0, 0, 2338, 7234, 45, 0, 0, 0, 0, 1047, 53, 0, 4, 37, 254, 18, 1214, 0, 8192, 2, 41, 2645, 0, 1173, 0, 5192, 0, 55, 0, 414, 2644, 0, 0, 1, 142, 0, 0, 153, 0, 4, 15, 54, 3, 5404, 0, 7, 885, 0, 0, 0, 0, 292, 31] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1193, 535, 0, 37, 0, 0, 0, 0, 2, 123, 7694, 2, 0, 2568, 7193, 64, 0, 0, 0, 0, 985, 57, 0, 2, 91, 308, 14, 1253, 0, 8188, 2, 20, 2301, 3, 1062, 0, 5055, 0, 44, 0, 397, 3051, 0, 1, 0, 202, 0, 0, 132, 0, 3, 12, 75, 11, 5196, 0, 12, 824, 0, 0, 1, 0, 405, 34] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1468, 499, 0, 112, 0, 0, 0, 0, 2, 73, 7675, 1, 1, 2365, 7264, 94, 0, 0, 0, 0, 1029, 54, 0, 8, 33, 442, 22, 1343, 0, 8189, 1, 49, 2018, 0, 1210, 0, 4682, 0, 56, 0, 410, 2764, 0, 5, 0, 177, 0, 0, 190, 0, 2, 9, 85, 5, 5561, 1, 7, 814, 0, 0, 0, 0, 409, 23] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1202, 456, 0, 62, 1, 0, 0, 0, 2773, 1631, 0, 0, 0, 0, 0, 0, 5, 52, 0, 0, 1, 2177, 3748, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 52, 0, 14, 4989, 0, 0, 0, 0, 0, 0, 0, 31, 278, 5, 932, 0, 4370, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2453, 1, 940, 0, 2638, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 2933, 0, 2, 1, 184, 0, 0, 2248, 900, 0, 0, 0, 0, 0, 0, 167, 0, 7, 13, 52, 0, 5520, 0, 479, 0, 0, 0, 0, 0, 0, 0, 12, 668, 0, 0, 0, 0, 337, 34, 2785, 2188, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6125, 6079, 6335, 5628, 6071, 6652, 6238, 6024] avg=6144.0000 max=6652 max_rank=5 min=5628 min_rank=3 imbalance_ratio=1.082682 max_load_reduction=42500 max_load_reduction_pct=0.864665 moved_to_echo=24985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1174, 447, 0, 50, 0, 0, 0, 0, 2703, 1631, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 2338, 3654, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 53, 0, 4, 5018, 0, 0, 0, 0, 0, 0, 0, 37, 254, 18, 1214, 0, 4372, 2, 41, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 0, 1173, 0, 2552, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 2644, 0, 0, 1, 142, 0, 0, 2176, 877, 0, 0, 0, 0, 0, 0, 153, 0, 4, 15, 54, 3, 5404, 0, 464, 0, 0, 0, 0, 0, 0, 0, 7, 885, 0, 0, 0, 0, 292, 31, 2802, 2189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6005, 6105, 6122, 5938, 6425, 6254, 6097, 6206] avg=6144.0000 max=6425 max_rank=4 min=5938 min_rank=3 imbalance_ratio=1.045736 max_load_reduction=42727 max_load_reduction_pct=0.869283 moved_to_echo=24982 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1007, 439, 0, 64, 0, 0, 0, 3, 0, 67, 7687, 0, 2, 2339, 7324, 32, 0, 0, 2, 0, 1147, 72, 0, 4, 52, 308, 14, 1647, 3, 8181, 1, 40, 2303, 1, 1397, 0, 5297, 1, 33, 0, 396, 1939, 2, 1, 5, 151, 0, 0, 195, 2, 2, 16, 76, 15, 5667, 0, 12, 897, 0, 1, 0, 1, 288, 19] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1468, 499, 0, 112, 0, 0, 0, 0, 2714, 1631, 0, 0, 0, 0, 0, 0, 2, 73, 0, 1, 1, 2365, 3669, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 54, 0, 8, 4925, 0, 0, 0, 0, 0, 0, 0, 33, 442, 22, 1343, 0, 4370, 1, 49, 0, 0, 0, 0, 0, 0, 0, 0, 2018, 0, 1210, 0, 2302, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 2764, 0, 5, 0, 177, 0, 0, 1962, 881, 0, 0, 0, 0, 0, 0, 190, 0, 2, 9, 85, 5, 5561, 1, 418, 0, 0, 0, 0, 0, 0, 0, 7, 814, 0, 0, 0, 0, 409, 23, 2750, 2188, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6424, 6205, 6016, 6260, 5586, 6199, 6271, 6191] avg=6144.0000 max=6424 max_rank=0 min=5586 min_rank=4 imbalance_ratio=1.045573 max_load_reduction=42728 max_load_reduction_pct=0.869303 moved_to_echo=24247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1223, 392, 0, 54, 0, 0, 0, 2, 2694, 1631, 0, 0, 0, 0, 0, 0, 1, 72, 0, 0, 0, 2436, 3640, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1229, 33, 0, 2, 5017, 0, 0, 0, 0, 0, 0, 0, 29, 323, 7, 1436, 1, 4371, 5, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 1, 983, 0, 2737, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 2149, 0, 1, 0, 122, 0, 0, 2333, 874, 0, 0, 0, 0, 0, 0, 122, 0, 2, 12, 80, 6, 5347, 0, 497, 0, 0, 0, 0, 0, 0, 0, 4, 855, 0, 0, 0, 0, 179, 26, 2801, 2189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6241, 6282, 6183, 6464, 5866, 6066, 6054] avg=6144.0000 max=6464 max_rank=4 min=5866 min_rank=5 imbalance_ratio=1.052083 max_load_reduction=42688 max_load_reduction_pct=0.868490 moved_to_echo=24450 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1007, 439, 0, 64, 0, 0, 0, 3, 2737, 1629, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 2, 2339, 3699, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1147, 72, 0, 4, 4934, 0, 0, 0, 0, 0, 0, 0, 52, 308, 14, 1647, 3, 4366, 1, 40, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 1, 1397, 0, 2604, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 1939, 2, 1, 5, 151, 0, 0, 2220, 888, 0, 0, 0, 0, 0, 0, 195, 2, 2, 16, 76, 15, 5667, 0, 473, 0, 0, 0, 0, 0, 0, 0, 12, 897, 0, 1, 0, 1, 288, 19, 2753, 2186, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5879, 6139, 6159, 6431, 6339, 5602, 6446, 6157] avg=6144.0000 max=6446 max_rank=6 min=5602 min_rank=5 imbalance_ratio=1.049154 max_load_reduction=42706 max_load_reduction_pct=0.868856 moved_to_echo=24544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1174, 447, 0, 50, 0, 0, 0, 0, 2703, 1631, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 2338, 3654, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 53, 0, 4, 5018, 0, 0, 0, 0, 0, 0, 0, 37, 254, 18, 1214, 0, 4372, 2, 41, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 0, 1173, 0, 2552, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 2644, 0, 0, 1, 142, 0, 0, 2176, 877, 0, 0, 0, 0, 0, 0, 153, 0, 4, 15, 54, 3, 5404, 0, 464, 0, 0, 0, 0, 0, 0, 0, 7, 885, 0, 0, 0, 0, 292, 31, 2802, 2189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6005, 6105, 6122, 5938, 6425, 6254, 6097, 6206] avg=6144.0000 max=6425 max_rank=4 min=5938 min_rank=3 imbalance_ratio=1.045736 max_load_reduction=42727 max_load_reduction_pct=0.869283 moved_to_echo=24982 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1202, 456, 0, 62, 1, 0, 0, 0, 2773, 1631, 0, 0, 0, 0, 0, 0, 5, 52, 0, 0, 1, 2177, 3748, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 52, 0, 14, 4989, 0, 0, 0, 0, 0, 0, 0, 31, 278, 5, 932, 0, 4370, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2453, 1, 940, 0, 2638, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 2933, 0, 2, 1, 184, 0, 0, 2248, 900, 0, 0, 0, 0, 0, 0, 167, 0, 7, 13, 52, 0, 5520, 0, 479, 0, 0, 0, 0, 0, 0, 0, 12, 668, 0, 0, 0, 0, 337, 34, 2785, 2188, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6125, 6079, 6335, 5628, 6071, 6652, 6238, 6024] avg=6144.0000 max=6652 max_rank=5 min=5628 min_rank=3 imbalance_ratio=1.082682 max_load_reduction=42500 max_load_reduction_pct=0.864665 moved_to_echo=24985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1292, 487, 0, 45, 0, 0, 0, 0, 2738, 1631, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 2003, 3702, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 992, 48, 0, 1, 5053, 0, 0, 0, 0, 0, 0, 0, 38, 287, 5, 1643, 0, 4372, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2922, 0, 1033, 0, 2681, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 2084, 0, 0, 0, 182, 0, 0, 2286, 888, 0, 0, 0, 0, 0, 0, 103, 0, 2, 14, 65, 0, 5382, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 0, 0, 239, 32, 2819, 2189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6193, 5841, 6094, 6355, 6665, 5811, 6053, 6140] avg=6144.0000 max=6665 max_rank=4 min=5811 min_rank=5 imbalance_ratio=1.084798 max_load_reduction=42487 max_load_reduction_pct=0.864400 moved_to_echo=24669 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1447, 542, 0, 62, 0, 0, 0, 1, 2762, 1632, 0, 0, 0, 0, 0, 0, 2, 81, 0, 0, 0, 2158, 3720, 189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1157, 68, 0, 10, 4924, 0, 0, 0, 0, 0, 0, 0, 66, 458, 20, 1373, 0, 4351, 5, 27, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 1, 1042, 0, 2555, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 2396, 0, 0, 27, 197, 0, 0, 2189, 897, 0, 0, 0, 0, 0, 0, 174, 0, 5, 15, 81, 8, 5347, 0, 470, 0, 0, 0, 0, 0, 0, 0, 9, 807, 0, 0, 0, 0, 353, 58, 2747, 2187, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6446, 6150, 6161, 6300, 5707, 6127, 6100, 6161] avg=6144.0000 max=6446 max_rank=0 min=5707 min_rank=4 imbalance_ratio=1.049154 max_load_reduction=42706 max_load_reduction_pct=0.868856 moved_to_echo=24095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1193, 535, 0, 37, 0, 0, 0, 0, 2688, 1631, 0, 0, 0, 0, 0, 0, 2, 123, 0, 2, 0, 2568, 3633, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 985, 57, 0, 2, 4939, 0, 0, 0, 0, 0, 0, 0, 91, 308, 14, 1253, 0, 4369, 2, 20, 0, 0, 0, 0, 0, 0, 0, 0, 2301, 3, 1062, 0, 2485, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 3051, 0, 1, 0, 202, 0, 0, 2118, 872, 0, 0, 0, 0, 0, 0, 132, 0, 3, 12, 75, 11, 5196, 0, 452, 0, 0, 0, 0, 0, 0, 0, 12, 824, 0, 0, 1, 0, 405, 34, 2755, 2188, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6084, 6392, 5983, 6057, 5895, 6641, 5881, 6219] avg=6144.0000 max=6641 max_rank=5 min=5881 min_rank=6 imbalance_ratio=1.080892 max_load_reduction=42511 max_load_reduction_pct=0.864889 moved_to_echo=24636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1468, 499, 0, 112, 0, 0, 0, 0, 2714, 1631, 0, 0, 0, 0, 0, 0, 2, 73, 0, 1, 1, 2365, 3669, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 54, 0, 8, 4925, 0, 0, 0, 0, 0, 0, 0, 33, 442, 22, 1343, 0, 4370, 1, 49, 0, 0, 0, 0, 0, 0, 0, 0, 2018, 0, 1210, 0, 2302, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 2764, 0, 5, 0, 177, 0, 0, 1962, 881, 0, 0, 0, 0, 0, 0, 190, 0, 2, 9, 85, 5, 5561, 1, 418, 0, 0, 0, 0, 0, 0, 0, 7, 814, 0, 0, 0, 0, 409, 23, 2750, 2188, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6424, 6205, 6016, 6260, 5586, 6199, 6271, 6191] avg=6144.0000 max=6424 max_rank=0 min=5586 min_rank=4 imbalance_ratio=1.045573 max_load_reduction=42728 max_load_reduction_pct=0.869303 moved_to_echo=24247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1447, 542, 0, 62, 0, 0, 0, 1, 2762, 1632, 0, 0, 0, 0, 0, 0, 2, 81, 0, 0, 0, 2158, 3720, 189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1157, 68, 0, 10, 4924, 0, 0, 0, 0, 0, 0, 0, 66, 458, 20, 1373, 0, 4351, 5, 27, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 1, 1042, 0, 2555, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 2396, 0, 0, 27, 197, 0, 0, 2189, 897, 0, 0, 0, 0, 0, 0, 174, 0, 5, 15, 81, 8, 5347, 0, 470, 0, 0, 0, 0, 0, 0, 0, 9, 807, 0, 0, 0, 0, 353, 58, 2747, 2187, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6446, 6150, 6161, 6300, 5707, 6127, 6100, 6161] avg=6144.0000 max=6446 max_rank=0 min=5707 min_rank=4 imbalance_ratio=1.049154 max_load_reduction=42706 max_load_reduction_pct=0.868856 moved_to_echo=24095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1007, 439, 0, 64, 0, 0, 0, 3, 2737, 1629, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 2, 2339, 3699, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1147, 72, 0, 4, 4934, 0, 0, 0, 0, 0, 0, 0, 52, 308, 14, 1647, 3, 4366, 1, 40, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 1, 1397, 0, 2604, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 1939, 2, 1, 5, 151, 0, 0, 2220, 888, 0, 0, 0, 0, 0, 0, 195, 2, 2, 16, 76, 15, 5667, 0, 473, 0, 0, 0, 0, 0, 0, 0, 12, 897, 0, 1, 0, 1, 288, 19, 2753, 2186, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5879, 6139, 6159, 6431, 6339, 5602, 6446, 6157] avg=6144.0000 max=6446 max_rank=6 min=5602 min_rank=5 imbalance_ratio=1.049154 max_load_reduction=42706 max_load_reduction_pct=0.868856 moved_to_echo=24544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10112, 3840, 0, 512, 128, 0, 0, 128, 21888, 13056, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 19968, 128, 128, 128, 1408, 0, 0, 17536, 7168, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19456, 128, 8960, 0, 20608, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 2688, 128, 10880, 128, 34944, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 640, 0, 128, 128, 18432, 29568, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 6656, 0, 128, 128, 128, 2560, 384, 22272, 17536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 2688, 128, 10880, 128, 34944, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 8960, 512, 0, 128, 39808, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 6656, 0, 128, 128, 128, 2560, 384, 22272, 17536, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1280, 128, 128, 128, 640, 128, 43520, 128, 3840, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3650, 803, 0, 183, 8, 0, 368, 0, 0, 0, 0, 2237, 0, 1, 4, 126, 0, 151, 1, 0, 433, 105, 0, 2, 0, 0, 813, 0, 0, 2576, 2654, 2, 1, 1098, 0, 1, 3, 0, 0, 0, 0, 1876, 4, 2, 7760, 0, 0, 4907, 1, 0, 0, 0, 2097, 1, 673, 1, 52, 881, 0, 87, 0, 1, 8051, 7538] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3256, 741, 0, 206, 14, 1, 477, 0, 0, 0, 0, 2054, 0, 0, 4, 143, 0, 101, 0, 1, 466, 124, 0, 0, 0, 0, 835, 0, 0, 2558, 2646, 2, 0, 961, 0, 0, 0, 0, 0, 0, 0, 2186, 2, 1, 7897, 0, 0, 4506, 4, 0, 1, 0, 2685, 19, 593, 1, 139, 906, 0, 132, 0, 0, 7937, 7553] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3325, 725, 0, 208, 2, 0, 389, 0, 0, 0, 0, 1953, 0, 0, 0, 183, 0, 168, 0, 0, 243, 125, 0, 0, 0, 0, 813, 0, 0, 2624, 2747, 0, 0, 1150, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 7866, 0, 0, 5097, 1, 0, 0, 0, 2240, 3, 564, 0, 56, 923, 0, 108, 0, 6, 8014, 7734] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3392, 739, 0, 245, 18, 0, 531, 0, 0, 0, 0, 1969, 0, 0, 4, 169, 0, 136, 1, 0, 451, 144, 0, 0, 0, 0, 953, 0, 0, 2520, 2596, 18, 1, 889, 1, 0, 0, 5, 0, 0, 0, 1428, 3, 0, 7821, 0, 0, 5057, 1, 0, 0, 2, 3023, 1, 520, 0, 97, 914, 0, 106, 0, 6, 7893, 7498] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[3413, 698, 0, 205, 15, 0, 396, 0, 0, 0, 0, 2035, 0, 1, 3, 155, 1, 187, 1, 0, 427, 134, 1, 0, 0, 0, 878, 0, 2, 2453, 2525, 19, 0, 903, 0, 0, 0, 1, 0, 0, 0, 1818, 11, 0, 7848, 0, 0, 4842, 1, 1, 0, 0, 3048, 8, 506, 0, 39, 947, 0, 119, 1, 10, 7933, 7567] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[3257, 775, 0, 223, 8, 1, 378, 0, 0, 0, 0, 1878, 0, 0, 3, 160, 0, 193, 0, 0, 289, 161, 0, 0, 0, 0, 829, 0, 0, 2432, 2465, 2, 0, 1183, 0, 1, 0, 0, 0, 0, 0, 1967, 4, 1, 7919, 0, 0, 5035, 2, 0, 0, 0, 2493, 2, 609, 1, 66, 1072, 0, 100, 0, 2, 7949, 7692] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3703, 679, 0, 195, 2, 0, 375, 0, 0, 0, 0, 1899, 0, 0, 3, 222, 0, 190, 1, 0, 453, 120, 0, 0, 0, 0, 841, 0, 0, 2364, 2617, 2, 0, 940, 0, 0, 0, 0, 0, 0, 0, 1923, 3, 1, 7858, 0, 0, 4953, 2, 0, 0, 0, 2458, 8, 750, 1, 53, 837, 0, 104, 0, 2, 7969, 7624] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3256, 741, 0, 206, 14, 1, 477, 0, 0, 0, 0, 2054, 0, 0, 4, 143, 0, 101, 0, 1, 466, 124, 0, 0, 0, 0, 835, 0, 0, 2558, 2646, 2, 0, 961, 0, 0, 0, 0, 0, 0, 0, 2186, 2, 1, 7897, 0, 0, 4506, 4, 0, 1, 0, 2685, 19, 593, 1, 139, 906, 0, 132, 0, 0, 7937, 7553] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3783, 662, 0, 200, 11, 0, 403, 0, 0, 0, 0, 2180, 0, 0, 1, 175, 0, 132, 2, 0, 495, 120, 0, 1, 0, 0, 861, 0, 0, 2657, 2331, 8, 0, 970, 0, 0, 0, 1, 0, 0, 0, 1921, 2, 0, 7696, 0, 0, 4603, 2, 0, 0, 1, 2594, 12, 639, 0, 112, 1131, 0, 100, 0, 0, 7918, 7428] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3650, 803, 0, 183, 8, 0, 368, 0, 884, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 0, 1, 4, 126, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 151, 1, 0, 433, 105, 0, 2, 5512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 0, 2576, 2654, 2, 189, 0, 0, 0, 0, 0, 0, 0, 1, 1098, 0, 1, 3, 0, 0, 0, 2596, 2539, 0, 0, 0, 0, 0, 0, 0, 1876, 4, 2, 0, 0, 0, 4294, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2097, 1, 673, 1, 1675, 1253, 0, 0, 0, 0, 0, 0, 52, 881, 0, 87, 0, 1, 0, 4979, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6320, 6279, 6204, 6234, 6238, 6176, 5701, 6000] avg=6144.0000 max=6320 max_rank=0 min=5701 min_rank=6 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3325, 725, 0, 208, 2, 0, 389, 0, 907, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1953, 0, 0, 0, 183, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 243, 125, 0, 0, 5488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 0, 2624, 2747, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 0, 0, 0, 0, 0, 0, 2632, 2526, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 0, 0, 0, 4459, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2240, 3, 564, 0, 1719, 1268, 0, 0, 0, 0, 0, 0, 56, 923, 0, 108, 0, 6, 0, 5108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5997, 6102, 6024, 6381, 6308, 6344, 5795, 6201] avg=6144.0000 max=6381 max_rank=3 min=5795 min_rank=6 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24648 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3650, 803, 0, 183, 8, 0, 368, 0, 0, 0, 0, 2237, 0, 1, 4, 126, 0, 151, 1, 0, 433, 105, 0, 2, 0, 0, 813, 0, 0, 2576, 2654, 2, 1, 1098, 0, 1, 3, 0, 0, 0, 0, 1876, 4, 2, 7760, 0, 0, 4907, 1, 0, 0, 0, 2097, 1, 673, 1, 52, 881, 0, 87, 0, 1, 8051, 7538] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3325, 725, 0, 208, 2, 0, 389, 0, 0, 0, 0, 1953, 0, 0, 0, 183, 0, 168, 0, 0, 243, 125, 0, 0, 0, 0, 813, 0, 0, 2624, 2747, 0, 0, 1150, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 7866, 0, 0, 5097, 1, 0, 0, 0, 2240, 3, 564, 0, 56, 923, 0, 108, 0, 6, 8014, 7734] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[3413, 698, 0, 205, 15, 0, 396, 0, 0, 0, 0, 2035, 0, 1, 3, 155, 1, 187, 1, 0, 427, 134, 1, 0, 0, 0, 878, 0, 2, 2453, 2525, 19, 0, 903, 0, 0, 0, 1, 0, 0, 0, 1818, 11, 0, 7848, 0, 0, 4842, 1, 1, 0, 0, 3048, 8, 506, 0, 39, 947, 0, 119, 1, 10, 7933, 7567] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3392, 739, 0, 245, 18, 0, 531, 0, 0, 0, 0, 1969, 0, 0, 4, 169, 0, 136, 1, 0, 451, 144, 0, 0, 0, 0, 953, 0, 0, 2520, 2596, 18, 1, 889, 1, 0, 0, 5, 0, 0, 0, 1428, 3, 0, 7821, 0, 0, 5057, 1, 0, 0, 2, 3023, 1, 520, 0, 97, 914, 0, 106, 0, 6, 7893, 7498] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3257, 775, 0, 223, 8, 1, 378, 0, 902, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 0, 0, 3, 160, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 289, 161, 0, 0, 5443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 0, 0, 2432, 2465, 2, 194, 0, 0, 0, 0, 0, 0, 0, 0, 1183, 0, 1, 0, 0, 0, 0, 2651, 2506, 0, 0, 0, 0, 0, 0, 0, 1967, 4, 1, 0, 0, 0, 4406, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2493, 2, 609, 1, 1709, 1276, 0, 0, 0, 0, 0, 0, 66, 1072, 0, 100, 0, 2, 0, 5081, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5979, 6033, 6086, 5922, 6341, 6378, 6092, 6321] avg=6144.0000 max=6378 max_rank=5 min=5922 min_rank=3 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=25132 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3413, 698, 0, 205, 15, 0, 396, 0, 887, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 0, 1, 3, 155, 3956, 0, 0, 0, 0, 0, 0, 0, 1, 187, 1, 0, 427, 134, 1, 0, 5432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 0, 2, 2453, 2525, 19, 187, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 0, 0, 1, 0, 0, 2627, 2501, 0, 0, 0, 0, 0, 0, 0, 1818, 11, 0, 0, 0, 0, 4236, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3048, 8, 506, 0, 1682, 1265, 0, 0, 0, 0, 0, 0, 39, 947, 0, 119, 1, 10, 0, 4998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6033, 6150, 6183, 6064, 6032, 6065, 6511, 6114] avg=6144.0000 max=6511 max_rank=6 min=6032 min_rank=4 imbalance_ratio=1.059733 max_load_reduction=42641 max_load_reduction_pct=0.867533 moved_to_echo=24722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3703, 679, 0, 195, 2, 0, 375, 0, 894, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 3, 222, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 190, 1, 0, 453, 120, 0, 0, 5456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 0, 0, 2364, 2617, 2, 191, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 0, 0, 0, 2630, 2513, 0, 0, 0, 0, 0, 0, 0, 1923, 3, 1, 0, 0, 0, 4334, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2458, 8, 750, 1, 1694, 1267, 0, 0, 0, 0, 0, 0, 53, 837, 0, 104, 0, 2, 0, 5036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6276, 6085, 6220, 6015, 6083, 6261, 6180, 6032] avg=6144.0000 max=6276 max_rank=0 min=6015 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=42876 max_load_reduction_pct=0.872314 moved_to_echo=24556 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3392, 739, 0, 245, 18, 0, 531, 0, 882, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 0, 4, 169, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 136, 1, 0, 451, 144, 0, 0, 5405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 0, 0, 2520, 2596, 18, 199, 0, 0, 0, 0, 0, 0, 0, 1, 889, 1, 0, 0, 5, 0, 0, 2617, 2488, 0, 0, 0, 0, 0, 0, 0, 1428, 3, 0, 0, 0, 0, 4416, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 3023, 1, 520, 0, 1670, 1261, 0, 0, 0, 0, 0, 0, 97, 914, 0, 106, 0, 6, 0, 4946, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6249, 6085, 6137, 6286, 6001, 5847, 6478, 6069] avg=6144.0000 max=6478 max_rank=6 min=5847 min_rank=5 imbalance_ratio=1.054362 max_load_reduction=42674 max_load_reduction_pct=0.868205 moved_to_echo=24395 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3256, 741, 0, 206, 14, 1, 477, 0, 885, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2054, 0, 0, 4, 143, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 1, 466, 124, 0, 0, 5434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 2558, 2646, 2, 174, 0, 0, 0, 0, 0, 0, 0, 0, 961, 0, 0, 0, 0, 0, 0, 2643, 2503, 0, 0, 0, 0, 0, 0, 0, 2186, 2, 1, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 2685, 19, 593, 1, 1679, 1274, 0, 0, 0, 0, 0, 0, 139, 906, 0, 132, 0, 0, 0, 4989, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5970, 6181, 6126, 6215, 6107, 6131, 6256, 6166] avg=6144.0000 max=6256 max_rank=6 min=5970 min_rank=0 imbalance_ratio=1.018229 max_load_reduction=42896 max_load_reduction_pct=0.872721 moved_to_echo=24660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3783, 662, 0, 200, 11, 0, 403, 0, 871, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2180, 0, 0, 1, 175, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 132, 2, 0, 495, 120, 0, 1, 5422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 2657, 2331, 8, 178, 0, 0, 0, 0, 0, 0, 0, 0, 970, 0, 0, 0, 1, 0, 0, 2575, 2496, 0, 0, 0, 0, 0, 0, 0, 1921, 2, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2594, 12, 639, 0, 1651, 1240, 0, 0, 0, 0, 0, 0, 112, 1131, 0, 100, 0, 0, 0, 4906, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 6237, 6172, 6035, 6042, 5950, 6139, 6249] avg=6144.0000 max=6328 max_rank=0 min=5950 min_rank=5 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3650, 803, 0, 183, 8, 0, 368, 0, 884, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 0, 1, 4, 126, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 151, 1, 0, 433, 105, 0, 2, 5512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 0, 2576, 2654, 2, 189, 0, 0, 0, 0, 0, 0, 0, 1, 1098, 0, 1, 3, 0, 0, 0, 2596, 2539, 0, 0, 0, 0, 0, 0, 0, 1876, 4, 2, 0, 0, 0, 4294, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2097, 1, 673, 1, 1675, 1253, 0, 0, 0, 0, 0, 0, 52, 881, 0, 87, 0, 1, 0, 4979, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6320, 6279, 6204, 6234, 6238, 6176, 5701, 6000] avg=6144.0000 max=6320 max_rank=0 min=5701 min_rank=6 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3325, 725, 0, 208, 2, 0, 389, 0, 907, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1953, 0, 0, 0, 183, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 243, 125, 0, 0, 5488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 0, 2624, 2747, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 0, 0, 0, 0, 0, 0, 2632, 2526, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 0, 0, 0, 4459, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2240, 3, 564, 0, 1719, 1268, 0, 0, 0, 0, 0, 0, 56, 923, 0, 108, 0, 6, 0, 5108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5997, 6102, 6024, 6381, 6308, 6344, 5795, 6201] avg=6144.0000 max=6381 max_rank=3 min=5795 min_rank=6 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24648 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3413, 698, 0, 205, 15, 0, 396, 0, 887, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 0, 1, 3, 155, 3956, 0, 0, 0, 0, 0, 0, 0, 1, 187, 1, 0, 427, 134, 1, 0, 5432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 0, 2, 2453, 2525, 19, 187, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 0, 0, 1, 0, 0, 2627, 2501, 0, 0, 0, 0, 0, 0, 0, 1818, 11, 0, 0, 0, 0, 4236, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3048, 8, 506, 0, 1682, 1265, 0, 0, 0, 0, 0, 0, 39, 947, 0, 119, 1, 10, 0, 4998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6033, 6150, 6183, 6064, 6032, 6065, 6511, 6114] avg=6144.0000 max=6511 max_rank=6 min=6032 min_rank=4 imbalance_ratio=1.059733 max_load_reduction=42641 max_load_reduction_pct=0.867533 moved_to_echo=24722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3392, 739, 0, 245, 18, 0, 531, 0, 882, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 0, 4, 169, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 136, 1, 0, 451, 144, 0, 0, 5405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 0, 0, 2520, 2596, 18, 199, 0, 0, 0, 0, 0, 0, 0, 1, 889, 1, 0, 0, 5, 0, 0, 2617, 2488, 0, 0, 0, 0, 0, 0, 0, 1428, 3, 0, 0, 0, 0, 4416, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 3023, 1, 520, 0, 1670, 1261, 0, 0, 0, 0, 0, 0, 97, 914, 0, 106, 0, 6, 0, 4946, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6249, 6085, 6137, 6286, 6001, 5847, 6478, 6069] avg=6144.0000 max=6478 max_rank=6 min=5847 min_rank=5 imbalance_ratio=1.054362 max_load_reduction=42674 max_load_reduction_pct=0.868205 moved_to_echo=24395 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 8192, 128, 128, 128, 128, 0, 0, 20992, 20096, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[27904, 5888, 0, 1792, 128, 128, 3328, 0, 7168, 3456, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 20736, 128, 4864, 128, 13568, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 15104, 128, 128, 0, 0, 0, 34176, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 6912, 0, 128, 20224, 20608, 128, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 15104, 128, 128, 0, 0, 0, 34176, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 16256, 0, 128, 128, 1408, 31616, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 1280, 128, 128, 3328, 1152, 128, 128, 43648, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 7680, 0, 896, 128, 128, 0, 40064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 1280, 128, 128, 3328, 1152, 128, 128, 43648, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[957, 2499, 5729, 885, 0, 6197, 126, 0, 0, 0, 0, 0, 1, 19, 0, 0, 0, 1, 0, 3746, 0, 0, 0, 342, 3, 198, 2100, 2925, 8191, 0, 193, 1, 0, 0, 1, 0, 3, 0, 105, 0, 1219, 4180, 1, 0, 623, 0, 0, 0, 0, 5316, 0, 947, 0, 0, 0, 0, 0, 29, 1338, 0, 1273, 0, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1085, 2479, 5833, 766, 0, 6006, 355, 0, 1, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 361, 1, 190, 2052, 3139, 8191, 0, 144, 1, 0, 0, 0, 0, 3, 0, 107, 0, 1222, 4690, 0, 0, 557, 0, 0, 0, 0, 5679, 0, 749, 0, 0, 0, 0, 0, 30, 993, 1, 741, 0, 0, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[798, 2549, 5989, 1117, 0, 6822, 259, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 3652, 0, 0, 0, 337, 7, 252, 2084, 3055, 8192, 0, 212, 0, 0, 0, 0, 0, 4, 0, 10, 0, 1426, 3971, 0, 0, 541, 0, 0, 0, 0, 4141, 0, 937, 0, 0, 0, 0, 0, 35, 1555, 0, 1174, 0, 0, 18] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1182, 2436, 5770, 958, 0, 6125, 273, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3614, 0, 0, 0, 326, 1, 269, 2175, 3342, 8188, 0, 149, 0, 1, 0, 3, 0, 5, 0, 45, 1, 1607, 3140, 0, 0, 669, 2, 1, 2, 0, 5730, 0, 876, 0, 0, 1, 0, 1, 31, 1271, 0, 946, 0, 0, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1079, 2641, 5180, 759, 0, 6462, 305, 0, 3, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 3830, 0, 0, 0, 362, 9, 254, 1973, 2911, 8192, 0, 138, 0, 0, 0, 0, 0, 3, 2, 87, 0, 1566, 4178, 0, 0, 538, 0, 0, 0, 0, 5342, 0, 930, 0, 0, 0, 0, 0, 20, 1394, 0, 985, 0, 0, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[957, 2499, 5729, 885, 0, 6197, 126, 0, 0, 0, 0, 0, 1, 19, 0, 0, 0, 1, 0, 3746, 0, 0, 0, 342, 3, 198, 2100, 2925, 8191, 0, 193, 1, 0, 0, 1, 0, 3, 0, 105, 0, 1219, 4180, 1, 0, 623, 0, 0, 0, 0, 5316, 0, 947, 0, 0, 0, 0, 0, 29, 1338, 0, 1273, 0, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1061, 2585, 5817, 821, 0, 6313, 289, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 376, 3, 253, 1929, 2761, 8192, 0, 161, 0, 0, 0, 0, 0, 1, 0, 45, 0, 1291, 4390, 0, 0, 612, 0, 0, 0, 0, 5159, 0, 862, 0, 0, 0, 0, 0, 62, 1537, 0, 932, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[812, 2405, 5624, 779, 0, 6624, 245, 0, 0, 0, 0, 0, 1, 13, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 371, 0, 122, 2009, 2960, 8192, 0, 124, 0, 0, 0, 6, 0, 1, 0, 69, 0, 1198, 4360, 0, 0, 543, 0, 0, 0, 4, 5034, 0, 1061, 0, 0, 0, 0, 0, 19, 1476, 0, 1035, 0, 0, 32] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1182, 2436, 5770, 958, 0, 6125, 273, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3614, 0, 0, 0, 326, 1, 269, 2175, 3342, 8188, 0, 149, 0, 1, 0, 3, 0, 5, 0, 45, 1, 1607, 3140, 0, 0, 669, 2, 1, 2, 0, 5730, 0, 876, 0, 0, 1, 0, 1, 31, 1271, 0, 946, 0, 0, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1112, 2421, 5796, 540, 0, 6023, 229, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 4289, 0, 0, 0, 347, 2, 221, 1767, 3317, 8192, 0, 182, 0, 0, 0, 0, 0, 2, 0, 35, 0, 1725, 4272, 0, 0, 555, 0, 0, 0, 0, 4658, 0, 722, 0, 0, 0, 0, 0, 55, 1621, 0, 1061, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[957, 2499, 1548, 885, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 19, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3746, 0, 0, 0, 342, 1969, 0, 0, 0, 0, 0, 0, 0, 3, 198, 2100, 2925, 696, 0, 193, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 105, 0, 4619, 1364, 0, 0, 0, 0, 0, 0, 1219, 4180, 1, 0, 623, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5316, 0, 947, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1338, 0, 1273, 0, 0, 4, 2078, 1578, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6015, 6151, 6058, 6116, 6092, 6032, 6388, 6300] avg=6144.0000 max=6388 max_rank=6 min=6015 min_rank=0 imbalance_ratio=1.039714 max_load_reduction=42764 max_load_reduction_pct=0.870036 moved_to_echo=24812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1079, 2641, 5180, 759, 0, 6462, 305, 0, 3, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 3830, 0, 0, 0, 362, 9, 254, 1973, 2911, 8192, 0, 138, 0, 0, 0, 0, 0, 3, 2, 87, 0, 1566, 4178, 0, 0, 538, 0, 0, 0, 0, 5342, 0, 930, 0, 0, 0, 0, 0, 20, 1394, 0, 985, 0, 0, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1085, 2479, 1575, 766, 0, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 27, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 361, 2005, 0, 0, 0, 0, 0, 0, 0, 1, 190, 2052, 3139, 696, 0, 144, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 107, 0, 4477, 1364, 0, 0, 0, 0, 0, 0, 1222, 4690, 0, 0, 557, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5679, 0, 749, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 30, 993, 1, 741, 0, 0, 5, 2116, 1529, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6260, 6159, 6110, 6223, 5951, 6478, 6556, 5415] avg=6144.0000 max=6556 max_rank=6 min=5415 min_rank=7 imbalance_ratio=1.067057 max_load_reduction=42596 max_load_reduction_pct=0.866618 moved_to_echo=24400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1182, 2436, 1559, 958, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3614, 0, 0, 0, 326, 1983, 0, 0, 0, 0, 0, 0, 0, 1, 269, 2175, 3342, 696, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 5, 0, 45, 1, 4566, 1363, 0, 0, 0, 0, 0, 0, 1607, 3140, 0, 0, 669, 2, 1, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5730, 0, 876, 0, 0, 1, 0, 126, 0, 0, 0, 0, 0, 0, 0, 1, 31, 1271, 0, 946, 0, 0, 9, 2093, 1559, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6408, 6132, 5923, 6632, 5984, 5430, 6733, 5910] avg=6144.0000 max=6733 max_rank=6 min=5430 min_rank=5 imbalance_ratio=1.095866 max_load_reduction=42419 max_load_reduction_pct=0.863017 moved_to_echo=24057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1112, 2421, 1566, 540, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4289, 0, 0, 0, 347, 1992, 0, 0, 0, 0, 0, 0, 0, 2, 221, 1767, 3317, 696, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 35, 0, 4490, 1364, 0, 0, 0, 0, 0, 0, 1725, 4272, 0, 0, 555, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4658, 0, 722, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 55, 1621, 0, 1061, 0, 0, 0, 2102, 1533, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5868, 6140, 6628, 6185, 5891, 6561, 5507, 6372] avg=6144.0000 max=6628 max_rank=2 min=5507 min_rank=6 imbalance_ratio=1.078776 max_load_reduction=42524 max_load_reduction_pct=0.865153 moved_to_echo=24331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1061, 2585, 1572, 821, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 376, 1999, 0, 0, 0, 0, 0, 0, 0, 3, 253, 1929, 2761, 696, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 45, 0, 4706, 1364, 0, 0, 0, 0, 0, 0, 1291, 4390, 0, 0, 612, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5159, 0, 862, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 62, 1537, 0, 932, 0, 0, 0, 2110, 1607, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 6134, 6073, 5803, 6116, 6302, 6148, 6248] avg=6144.0000 max=6328 max_rank=0 min=5803 min_rank=3 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1079, 2641, 1386, 759, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 4, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 0, 362, 1784, 0, 0, 0, 0, 0, 0, 0, 9, 254, 1973, 2911, 691, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 87, 0, 4818, 1367, 0, 0, 0, 0, 0, 0, 1566, 4178, 0, 0, 538, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 5342, 0, 930, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1394, 0, 985, 0, 0, 3, 1882, 1644, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 6143, 5976, 5976, 6277, 6293, 6389, 5928] avg=6144.0000 max=6389 max_rank=6 min=5928 min_rank=7 imbalance_ratio=1.039876 max_load_reduction=42763 max_load_reduction_pct=0.870015 moved_to_echo=24887 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[812, 2405, 1520, 779, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 371, 1933, 0, 0, 0, 0, 0, 0, 0, 0, 122, 2009, 2960, 696, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 69, 0, 4937, 1364, 0, 0, 0, 0, 0, 0, 1198, 4360, 0, 0, 543, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 5034, 0, 1061, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1476, 0, 1035, 0, 0, 32, 2040, 1687, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5761, 6146, 6337, 5911, 6377, 6109, 6222, 6289] avg=6144.0000 max=6377 max_rank=4 min=5761 min_rank=0 imbalance_ratio=1.037923 max_load_reduction=42775 max_load_reduction_pct=0.870260 moved_to_echo=24997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1061, 2585, 1572, 821, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 376, 1999, 0, 0, 0, 0, 0, 0, 0, 3, 253, 1929, 2761, 696, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 45, 0, 4706, 1364, 0, 0, 0, 0, 0, 0, 1291, 4390, 0, 0, 612, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5159, 0, 862, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 62, 1537, 0, 932, 0, 0, 0, 2110, 1607, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 6134, 6073, 5803, 6116, 6302, 6148, 6248] avg=6144.0000 max=6328 max_rank=0 min=5803 min_rank=3 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[798, 2549, 1619, 1117, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3652, 0, 0, 0, 337, 2058, 0, 0, 0, 0, 0, 0, 0, 7, 252, 2084, 3055, 696, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 10, 0, 5086, 1364, 0, 0, 0, 0, 0, 0, 1426, 3971, 0, 0, 541, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4141, 0, 937, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1555, 0, 1174, 0, 0, 18, 2172, 1736, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6342, 6147, 6047, 6306, 6464, 5947, 5209, 6690] avg=6144.0000 max=6690 max_rank=7 min=5209 min_rank=6 imbalance_ratio=1.088867 max_load_reduction=42462 max_load_reduction_pct=0.863892 moved_to_echo=24310 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[957, 2499, 1548, 885, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 19, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3746, 0, 0, 0, 342, 1969, 0, 0, 0, 0, 0, 0, 0, 3, 198, 2100, 2925, 696, 0, 193, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 105, 0, 4619, 1364, 0, 0, 0, 0, 0, 0, 1219, 4180, 1, 0, 623, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5316, 0, 947, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1338, 0, 1273, 0, 0, 4, 2078, 1578, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6015, 6151, 6058, 6116, 6092, 6032, 6388, 6300] avg=6144.0000 max=6388 max_rank=6 min=6015 min_rank=0 imbalance_ratio=1.039714 max_load_reduction=42764 max_load_reduction_pct=0.870036 moved_to_echo=24812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1112, 2421, 1566, 540, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4289, 0, 0, 0, 347, 1992, 0, 0, 0, 0, 0, 0, 0, 2, 221, 1767, 3317, 696, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 35, 0, 4490, 1364, 0, 0, 0, 0, 0, 0, 1725, 4272, 0, 0, 555, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4658, 0, 722, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 55, 1621, 0, 1061, 0, 0, 0, 2102, 1533, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5868, 6140, 6628, 6185, 5891, 6561, 5507, 6372] avg=6144.0000 max=6628 max_rank=2 min=5507 min_rank=6 imbalance_ratio=1.078776 max_load_reduction=42524 max_load_reduction_pct=0.865153 moved_to_echo=24331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1182, 2436, 1559, 958, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3614, 0, 0, 0, 326, 1983, 0, 0, 0, 0, 0, 0, 0, 1, 269, 2175, 3342, 696, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 5, 0, 45, 1, 4566, 1363, 0, 0, 0, 0, 0, 0, 1607, 3140, 0, 0, 669, 2, 1, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5730, 0, 876, 0, 0, 1, 0, 126, 0, 0, 0, 0, 0, 0, 0, 1, 31, 1271, 0, 946, 0, 0, 9, 2093, 1559, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6408, 6132, 5923, 6632, 5984, 5430, 6733, 5910] avg=6144.0000 max=6733 max_rank=6 min=5430 min_rank=5 imbalance_ratio=1.095866 max_load_reduction=42419 max_load_reduction_pct=0.863017 moved_to_echo=24057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1079, 2641, 1386, 759, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 4, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 0, 362, 1784, 0, 0, 0, 0, 0, 0, 0, 9, 254, 1973, 2911, 691, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 87, 0, 4818, 1367, 0, 0, 0, 0, 0, 0, 1566, 4178, 0, 0, 538, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 5342, 0, 930, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1394, 0, 985, 0, 0, 3, 1882, 1644, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 6143, 5976, 5976, 6277, 6293, 6389, 5928] avg=6144.0000 max=6389 max_rank=6 min=5928 min_rank=7 imbalance_ratio=1.039876 max_load_reduction=42763 max_load_reduction_pct=0.870015 moved_to_echo=24887 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8192, 20096, 12416, 6656, 0, 0, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 384, 11264, 128, 8192, 0, 0, 128, 16640, 12928, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1792, 16128, 24448, 5632, 0, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 30720, 0, 0, 0, 2944, 15744, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[11264, 33280, 128, 0, 4736, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 384, 11264, 128, 8192, 0, 0, 128, 16640, 12928, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 30720, 0, 0, 0, 2944, 15744, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 41088, 0, 7168, 0, 0, 128, 0, 1024, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 128, 512, 128, 37760, 11008, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 128, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7526, 51, 2639, 0, 0, 389, 0, 7038, 0, 0, 0, 0, 121, 0, 1831, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 111, 0, 5830, 0, 0, 0, 0, 1149, 0, 0, 0, 0, 605, 85, 8191, 2160, 0, 0, 6970, 0, 3076, 0, 107, 0, 0, 0, 507, 0, 0, 292, 277, 105, 60, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7560, 42, 2471, 0, 0, 371, 0, 7154, 0, 0, 0, 0, 130, 0, 1617, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 102, 0, 6281, 0, 0, 0, 0, 617, 0, 0, 0, 0, 729, 52, 8192, 1871, 0, 0, 7019, 0, 3521, 0, 118, 0, 0, 0, 486, 0, 0, 235, 366, 122, 44, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4, 7468, 7, 2632, 0, 2, 420, 0, 7122, 0, 0, 1, 0, 113, 0, 1451, 57, 1, 2, 0, 0, 1, 0, 0, 0, 0, 3, 79, 1, 5725, 1, 0, 1, 0, 864, 3, 1, 0, 0, 720, 140, 8187, 2192, 0, 0, 6725, 0, 3748, 0, 78, 0, 0, 0, 529, 0, 0, 321, 334, 123, 96, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7498, 21, 2471, 0, 0, 385, 0, 6927, 0, 0, 0, 0, 145, 0, 1670, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 76, 0, 6331, 0, 0, 0, 0, 659, 0, 0, 0, 0, 829, 90, 8192, 1991, 0, 0, 6997, 0, 3498, 0, 122, 0, 0, 0, 474, 0, 0, 381, 241, 70, 69, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7667, 173, 2659, 0, 0, 369, 0, 7109, 1, 0, 0, 0, 124, 0, 1600, 15, 2, 0, 1, 0, 0, 0, 0, 0, 0, 2, 96, 0, 6217, 0, 0, 1, 0, 646, 0, 0, 0, 0, 847, 78, 8178, 2010, 0, 0, 7062, 0, 3004, 0, 128, 0, 0, 0, 379, 0, 0, 250, 359, 130, 45, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7662, 19, 2617, 0, 0, 416, 0, 6922, 0, 0, 0, 0, 155, 0, 1325, 43, 0, 0, 0, 0, 0, 0, 5, 0, 1, 4, 127, 0, 5660, 1, 0, 9, 0, 714, 2, 0, 1, 3, 862, 299, 8187, 2211, 0, 0, 6678, 0, 3664, 0, 127, 0, 0, 0, 417, 0, 0, 478, 345, 131, 67, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7554, 238, 2500, 0, 0, 451, 0, 7265, 0, 0, 0, 0, 157, 0, 1595, 44, 0, 0, 0, 0, 0, 1, 4, 0, 0, 6, 97, 0, 5899, 0, 0, 1, 1, 1107, 0, 0, 0, 0, 952, 102, 8192, 1618, 0, 0, 6562, 0, 3234, 0, 135, 0, 0, 0, 455, 0, 0, 349, 391, 152, 90, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7498, 21, 2471, 0, 0, 385, 0, 6927, 0, 0, 0, 0, 145, 0, 1670, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 76, 0, 6331, 0, 0, 0, 0, 659, 0, 0, 0, 0, 829, 90, 8192, 1991, 0, 0, 6997, 0, 3498, 0, 122, 0, 0, 0, 474, 0, 0, 381, 241, 70, 69, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4, 7468, 7, 2632, 0, 2, 420, 0, 7122, 0, 0, 1, 0, 113, 0, 1451, 57, 1, 2, 0, 0, 1, 0, 0, 0, 0, 3, 79, 1, 5725, 1, 0, 1, 0, 864, 3, 1, 0, 0, 720, 140, 8187, 2192, 0, 0, 6725, 0, 3748, 0, 78, 0, 0, 0, 529, 0, 0, 321, 334, 123, 96, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7560, 42, 2471, 0, 0, 371, 0, 7154, 0, 0, 0, 0, 130, 0, 1617, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 102, 0, 6281, 0, 0, 0, 0, 617, 0, 0, 0, 0, 729, 52, 8192, 1871, 0, 0, 7019, 0, 3521, 0, 118, 0, 0, 0, 486, 0, 0, 235, 366, 122, 44, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7569, 215, 2305, 0, 0, 379, 0, 7105, 0, 0, 0, 0, 191, 0, 1757, 74, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 101, 0, 5586, 0, 0, 0, 0, 961, 0, 0, 0, 0, 920, 116, 8192, 1959, 0, 0, 6883, 0, 3266, 0, 128, 0, 0, 0, 356, 0, 0, 511, 383, 118, 68, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3495, 51, 2639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 0, 5572, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1831, 23, 2, 0, 0, 0, 0, 2871, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 111, 0, 5830, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1149, 0, 0, 0, 5303, 0, 0, 0, 0, 0, 0, 0, 0, 605, 85, 0, 2160, 0, 0, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3076, 0, 107, 0, 0, 0, 507, 1369, 834, 0, 0, 0, 0, 0, 0, 0, 0, 292, 277, 105, 60, 0, 0, 2888, 2369, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6185, 6082, 6389, 6045, 6452, 6115, 5893, 5991] avg=6144.0000 max=6452 max_rank=4 min=5893 min_rank=6 imbalance_ratio=1.050130 max_load_reduction=42700 max_load_reduction_pct=0.868734 moved_to_echo=24451 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3510, 42, 2471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 5663, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 44, 0, 0, 0, 0, 0, 2891, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 102, 0, 6281, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 5303, 0, 0, 0, 0, 0, 0, 0, 0, 729, 52, 0, 1871, 0, 0, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 0, 118, 0, 0, 0, 486, 1392, 840, 0, 0, 0, 0, 0, 0, 0, 0, 235, 366, 122, 44, 0, 0, 2889, 2380, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 6164, 6222, 6490, 5920, 5940, 6357, 6036] avg=6144.0000 max=6490 max_rank=3 min=5920 min_rank=4 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3508, 238, 2500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 0, 5752, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 44, 0, 0, 0, 0, 0, 2703, 1668, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 6, 97, 0, 5899, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1107, 0, 0, 0, 5303, 0, 0, 0, 0, 0, 0, 0, 0, 952, 102, 0, 1618, 0, 0, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3234, 0, 135, 0, 0, 0, 455, 1413, 785, 0, 0, 0, 0, 0, 0, 0, 0, 349, 391, 152, 90, 0, 0, 2889, 2378, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6246, 6360, 6010, 6107, 6412, 5746, 6022, 6249] avg=6144.0000 max=6412 max_rank=4 min=5746 min_rank=5 imbalance_ratio=1.043620 max_load_reduction=42740 max_load_reduction_pct=0.869548 moved_to_echo=24429 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3560, 173, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 5628, 1, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1600, 15, 2, 0, 1, 0, 0, 2909, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 96, 0, 6217, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 646, 0, 0, 0, 5294, 0, 0, 0, 0, 0, 0, 0, 0, 847, 78, 0, 2010, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3004, 0, 128, 0, 0, 0, 379, 1383, 845, 0, 0, 0, 0, 0, 0, 0, 0, 250, 359, 130, 45, 0, 0, 2884, 2414, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6392, 6122, 6220, 6413, 5941, 6243, 5739, 6082] avg=6144.0000 max=6413 max_rank=3 min=5739 min_rank=6 imbalance_ratio=1.043783 max_load_reduction=42739 max_load_reduction_pct=0.869527 moved_to_echo=24005 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 3468, 7, 2632, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 5639, 0, 0, 1, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 57, 1, 2, 0, 0, 1, 2770, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 79, 1, 5725, 98, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 864, 3, 1, 0, 5301, 0, 0, 0, 0, 0, 0, 0, 0, 720, 140, 0, 2192, 0, 0, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 0, 78, 0, 0, 0, 529, 1385, 805, 0, 0, 0, 0, 0, 0, 0, 0, 321, 334, 123, 96, 0, 0, 2886, 2351, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6173, 5931, 5906, 6171, 6202, 6545, 6111] avg=6144.0000 max=6545 max_rank=6 min=5906 min_rank=3 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=25029 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3510, 42, 2471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 5663, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 44, 0, 0, 0, 0, 0, 2891, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 102, 0, 6281, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 5303, 0, 0, 0, 0, 0, 0, 0, 0, 729, 52, 0, 1871, 0, 0, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 0, 118, 0, 0, 0, 486, 1392, 840, 0, 0, 0, 0, 0, 0, 0, 0, 235, 366, 122, 44, 0, 0, 2889, 2380, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 6164, 6222, 6490, 5920, 5940, 6357, 6036] avg=6144.0000 max=6490 max_rank=3 min=5920 min_rank=4 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3482, 21, 2471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 5483, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1670, 13, 0, 0, 0, 0, 0, 2882, 1656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 76, 0, 6331, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 0, 0, 5304, 0, 0, 0, 0, 0, 0, 0, 0, 829, 90, 0, 1991, 0, 0, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3498, 0, 122, 0, 0, 0, 474, 1348, 837, 0, 0, 0, 0, 0, 0, 0, 0, 381, 241, 70, 69, 0, 0, 2888, 2360, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5974, 6013, 6221, 6505, 5963, 6188, 6279, 6009] avg=6144.0000 max=6505 max_rank=3 min=5963 min_rank=4 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=24439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3508, 238, 2500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 0, 5752, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 44, 0, 0, 0, 0, 0, 2703, 1668, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 6, 97, 0, 5899, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1107, 0, 0, 0, 5303, 0, 0, 0, 0, 0, 0, 0, 0, 952, 102, 0, 1618, 0, 0, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3234, 0, 135, 0, 0, 0, 455, 1413, 785, 0, 0, 0, 0, 0, 0, 0, 0, 349, 391, 152, 90, 0, 0, 2889, 2378, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6246, 6360, 6010, 6107, 6412, 5746, 6022, 6249] avg=6144.0000 max=6412 max_rank=4 min=5746 min_rank=5 imbalance_ratio=1.043620 max_load_reduction=42740 max_load_reduction_pct=0.869548 moved_to_echo=24429 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3549, 19, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 0, 5472, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 43, 0, 0, 0, 0, 0, 2752, 1697, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 4, 127, 0, 5660, 99, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 714, 2, 0, 1, 5301, 0, 0, 0, 0, 0, 0, 0, 3, 862, 299, 0, 2211, 0, 0, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3664, 0, 127, 0, 0, 0, 417, 1351, 802, 0, 0, 0, 0, 0, 0, 0, 0, 478, 345, 131, 67, 0, 0, 2886, 2416, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6185, 6043, 5817, 5896, 6028, 6499, 6361, 6323] avg=6144.0000 max=6499 max_rank=5 min=5817 min_rank=2 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=25211 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 3468, 7, 2632, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 5639, 0, 0, 1, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 57, 1, 2, 0, 0, 1, 2770, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 79, 1, 5725, 98, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 864, 3, 1, 0, 5301, 0, 0, 0, 0, 0, 0, 0, 0, 720, 140, 0, 2192, 0, 0, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 0, 78, 0, 0, 0, 529, 1385, 805, 0, 0, 0, 0, 0, 0, 0, 0, 321, 334, 123, 96, 0, 0, 2886, 2351, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6173, 5931, 5906, 6171, 6202, 6545, 6111] avg=6144.0000 max=6545 max_rank=6 min=5906 min_rank=3 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=25029 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3514, 215, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 5625, 0, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 74, 4, 0, 0, 0, 0, 2835, 1672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 101, 0, 5586, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 0, 0, 0, 5304, 0, 0, 0, 0, 0, 0, 0, 0, 920, 116, 0, 1959, 0, 0, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3266, 0, 128, 0, 0, 0, 356, 1382, 824, 0, 0, 0, 0, 0, 0, 0, 0, 511, 383, 118, 68, 0, 0, 2888, 2383, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6034, 6195, 6342, 5790, 6265, 6219, 5956, 6351] avg=6144.0000 max=6351 max_rank=7 min=5790 min_rank=3 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3549, 19, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 0, 5472, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 43, 0, 0, 0, 0, 0, 2752, 1697, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 4, 127, 0, 5660, 99, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 714, 2, 0, 1, 5301, 0, 0, 0, 0, 0, 0, 0, 3, 862, 299, 0, 2211, 0, 0, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3664, 0, 127, 0, 0, 0, 417, 1351, 802, 0, 0, 0, 0, 0, 0, 0, 0, 478, 345, 131, 67, 0, 0, 2886, 2416, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6185, 6043, 5817, 5896, 6028, 6499, 6361, 6323] avg=6144.0000 max=6499 max_rank=5 min=5817 min_rank=2 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=25211 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 12928, 384, 128, 128, 128, 0, 128, 22656, 13440, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 27136, 0, 1024, 0, 0, 0, 3712, 11136, 6656, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 28160, 768, 20352, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 6528, 1024, 0, 16128, 0, 0, 25728, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 2944, 2816, 1024, 640, 0, 0, 23168, 19072, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 6784, 128, 128, 128, 42496, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 2944, 2816, 1024, 640, 0, 0, 23168, 19072, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3200, 0, 44928, 128, 0, 128, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 896, 128, 47616, 896, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 7940, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 0, 6878, 0, 0, 0, 14, 469, 0, 701, 0, 1680, 0, 1, 0, 1302, 1877, 0, 6, 6685, 0, 0, 1052, 0, 0, 0, 271, 1, 5813, 0, 0, 429, 0, 0, 0, 0, 0, 0, 69, 0, 1628, 0, 753, 0, 0, 0, 0, 8154, 1496, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 8013, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 0, 6954, 0, 0, 0, 1, 608, 0, 933, 0, 1471, 0, 2, 0, 1353, 2368, 0, 4, 5973, 0, 0, 1468, 0, 0, 0, 215, 0, 5822, 0, 0, 481, 0, 0, 0, 0, 0, 0, 17, 0, 1334, 0, 427, 0, 0, 0, 0, 8164, 1465, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 7928, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 0, 6645, 0, 0, 0, 13, 320, 0, 1011, 0, 1703, 0, 2, 0, 1254, 2556, 0, 3, 6711, 0, 2, 1244, 0, 0, 0, 297, 0, 5281, 0, 0, 476, 0, 0, 0, 0, 0, 1, 72, 0, 1665, 0, 685, 0, 0, 0, 1, 8182, 1574, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 7830, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1980, 0, 6307, 0, 1, 0, 8, 413, 0, 1278, 2, 2171, 0, 0, 0, 1466, 2239, 0, 10, 6637, 0, 0, 672, 0, 0, 0, 312, 0, 5434, 0, 0, 442, 0, 0, 0, 0, 0, 0, 52, 0, 1546, 0, 537, 0, 0, 0, 0, 8158, 1647, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 7987, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 7042, 0, 0, 0, 13, 407, 0, 784, 0, 1712, 0, 2, 0, 1158, 1957, 0, 6, 6447, 0, 0, 1420, 0, 0, 0, 243, 0, 5677, 0, 0, 440, 0, 0, 0, 0, 0, 0, 37, 0, 1571, 0, 561, 0, 0, 0, 0, 8180, 1642, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 7991, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 0, 6944, 0, 0, 0, 6, 465, 0, 844, 0, 1041, 0, 3, 0, 1219, 2269, 0, 0, 6221, 0, 0, 1209, 0, 0, 0, 242, 0, 6244, 0, 0, 447, 0, 0, 0, 0, 0, 0, 29, 0, 1639, 0, 418, 0, 0, 0, 0, 8183, 1500, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 7903, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 0, 6644, 0, 0, 0, 9, 377, 0, 1353, 0, 1259, 0, 2, 0, 1363, 2021, 1, 5, 6650, 0, 0, 1113, 1, 0, 0, 271, 0, 5487, 0, 0, 424, 0, 0, 0, 0, 0, 0, 98, 0, 1854, 0, 764, 0, 0, 0, 0, 8172, 1634, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 8013, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 0, 6954, 0, 0, 0, 1, 608, 0, 933, 0, 1471, 0, 2, 0, 1353, 2368, 0, 4, 5973, 0, 0, 1468, 0, 0, 0, 215, 0, 5822, 0, 0, 481, 0, 0, 0, 0, 0, 0, 17, 0, 1334, 0, 427, 0, 0, 0, 0, 8164, 1465, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 7977, 28, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2169, 1, 6953, 2, 0, 0, 11, 419, 0, 1006, 0, 1709, 0, 3, 0, 1352, 1994, 0, 16, 6241, 0, 0, 1354, 0, 0, 0, 237, 0, 5464, 0, 0, 462, 1, 0, 0, 0, 2, 9, 15, 0, 1330, 2, 580, 0, 0, 0, 0, 8182, 1632, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 7987, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 7042, 0, 0, 0, 13, 407, 0, 784, 0, 1712, 0, 2, 0, 1158, 1957, 0, 6, 6447, 0, 0, 1420, 0, 0, 0, 243, 0, 5677, 0, 0, 440, 0, 0, 0, 0, 0, 0, 37, 0, 1571, 0, 561, 0, 0, 0, 0, 8180, 1642, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6123, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 0, 4266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 469, 0, 701, 0, 1680, 1777, 1355, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1302, 1877, 0, 6, 2767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 0, 0, 0, 271, 1, 3918, 910, 0, 0, 0, 0, 0, 0, 5813, 0, 0, 429, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 1628, 0, 753, 0, 2683, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4561, 1496, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6185, 5996, 5953, 6152, 6282, 6390, 6058] avg=6144.0000 max=6390 max_rank=6 min=5953 min_rank=3 imbalance_ratio=1.040039 max_load_reduction=42762 max_load_reduction_pct=0.869995 moved_to_echo=24882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 7940, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 0, 6878, 0, 0, 0, 14, 469, 0, 701, 0, 1680, 0, 1, 0, 1302, 1877, 0, 6, 6685, 0, 0, 1052, 0, 0, 0, 271, 1, 5813, 0, 0, 429, 0, 0, 0, 0, 0, 0, 69, 0, 1628, 0, 753, 0, 0, 0, 0, 8154, 1496, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 7903, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 0, 6644, 0, 0, 0, 9, 377, 0, 1353, 0, 1259, 0, 2, 0, 1363, 2021, 1, 5, 6650, 0, 0, 1113, 1, 0, 0, 271, 0, 5487, 0, 0, 424, 0, 0, 0, 0, 0, 0, 98, 0, 1854, 0, 764, 0, 0, 0, 0, 8172, 1634, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 7830, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1980, 0, 6307, 0, 1, 0, 8, 413, 0, 1278, 2, 2171, 0, 0, 0, 1466, 2239, 0, 10, 6637, 0, 0, 672, 0, 0, 0, 312, 0, 5434, 0, 0, 442, 0, 0, 0, 0, 0, 0, 52, 0, 1546, 0, 537, 0, 0, 0, 0, 8158, 1647, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6159, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 4368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 407, 0, 784, 0, 1712, 1787, 1387, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1158, 1957, 0, 6, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 0, 0, 0, 243, 0, 3779, 913, 0, 0, 0, 0, 0, 0, 5677, 0, 0, 440, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1571, 0, 561, 0, 2691, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4576, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6172, 6221, 6090, 5791, 6355, 6158, 6147, 6218] avg=6144.0000 max=6355 max_rank=4 min=5791 min_rank=3 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6179, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 0, 4313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 608, 0, 933, 0, 1471, 1793, 1370, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1353, 2368, 0, 4, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1468, 0, 0, 0, 215, 0, 3501, 911, 0, 0, 0, 0, 0, 0, 5822, 0, 0, 481, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1334, 0, 427, 0, 2686, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4567, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6193, 6378, 6176, 6199, 6095, 6344, 5735, 6032] avg=6144.0000 max=6378 max_rank=1 min=5735 min_rank=6 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=24206 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6162, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 0, 4307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 465, 0, 844, 0, 1041, 1788, 1368, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1219, 2269, 0, 0, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 0, 0, 0, 242, 0, 3646, 913, 0, 0, 0, 0, 0, 0, 6244, 0, 0, 447, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1639, 0, 418, 0, 2692, 1269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4578, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6541, 5512, 6066, 6010, 6732, 6047, 6078] avg=6144.0000 max=6732 max_rank=5 min=5512 min_rank=2 imbalance_ratio=1.095703 max_load_reduction=42420 max_load_reduction_pct=0.863037 moved_to_echo=24867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6151, 28, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2169, 1, 4312, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 419, 0, 1006, 0, 1709, 1785, 1370, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1352, 1994, 0, 16, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 0, 0, 0, 237, 0, 3658, 913, 0, 0, 0, 0, 0, 0, 5464, 0, 0, 462, 1, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 2, 9, 15, 0, 1330, 2, 580, 0, 2692, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4577, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6180, 6484, 6300, 5948, 6162, 5968, 5901, 6209] avg=6144.0000 max=6484 max_rank=1 min=5901 min_rank=6 imbalance_ratio=1.055339 max_load_reduction=42668 max_load_reduction_pct=0.868083 moved_to_echo=24240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6159, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 4368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 407, 0, 784, 0, 1712, 1787, 1387, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1158, 1957, 0, 6, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 0, 0, 0, 243, 0, 3779, 913, 0, 0, 0, 0, 0, 0, 5677, 0, 0, 440, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1571, 0, 561, 0, 2691, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4576, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6172, 6221, 6090, 5791, 6355, 6158, 6147, 6218] avg=6144.0000 max=6355 max_rank=4 min=5791 min_rank=3 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6038, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 413, 0, 1278, 2, 2171, 1752, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 2239, 0, 10, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 312, 0, 3890, 911, 0, 0, 0, 0, 0, 0, 5434, 0, 0, 442, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1546, 0, 537, 0, 2684, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4563, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6048, 5892, 6867, 6462, 5785, 5916, 5972, 6210] avg=6144.0000 max=6867 max_rank=2 min=5785 min_rank=4 imbalance_ratio=1.117676 max_load_reduction=42285 max_load_reduction_pct=0.860291 moved_to_echo=23883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6151, 28, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2169, 1, 4312, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 419, 0, 1006, 0, 1709, 1785, 1370, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1352, 1994, 0, 16, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 0, 0, 0, 237, 0, 3658, 913, 0, 0, 0, 0, 0, 0, 5464, 0, 0, 462, 1, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 2, 9, 15, 0, 1330, 2, 580, 0, 2692, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4577, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6180, 6484, 6300, 5948, 6162, 5968, 5901, 6209] avg=6144.0000 max=6484 max_rank=1 min=5901 min_rank=6 imbalance_ratio=1.055339 max_load_reduction=42668 max_load_reduction_pct=0.868083 moved_to_echo=24240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6123, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 0, 4266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 469, 0, 701, 0, 1680, 1777, 1355, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1302, 1877, 0, 6, 2767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 0, 0, 0, 271, 1, 3918, 910, 0, 0, 0, 0, 0, 0, 5813, 0, 0, 429, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 1628, 0, 753, 0, 2683, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4561, 1496, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6185, 5996, 5953, 6152, 6282, 6390, 6058] avg=6144.0000 max=6390 max_rank=6 min=5953 min_rank=3 imbalance_ratio=1.040039 max_load_reduction=42762 max_load_reduction_pct=0.869995 moved_to_echo=24882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6107, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 0, 4113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 320, 0, 1011, 0, 1703, 1777, 1312, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1254, 2556, 0, 3, 2774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1244, 0, 0, 0, 297, 0, 3937, 918, 0, 0, 0, 0, 0, 0, 5281, 0, 0, 476, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 1, 72, 0, 1665, 0, 685, 0, 2697, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4567, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 5624, 6136, 6589, 6398, 5801, 6340, 6142] avg=6144.0000 max=6589 max_rank=3 min=5624 min_rank=1 imbalance_ratio=1.072428 max_load_reduction=42563 max_load_reduction_pct=0.865946 moved_to_echo=24681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6095, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 0, 4120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 377, 0, 1353, 0, 1259, 1768, 1309, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1363, 2021, 1, 5, 2752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 1, 0, 0, 271, 0, 3898, 912, 0, 0, 0, 0, 0, 0, 5487, 0, 0, 424, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 1854, 0, 764, 0, 2689, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4571, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6135, 5827, 6075, 6144, 6195, 5951, 6620, 6205] avg=6144.0000 max=6620 max_rank=6 min=5827 min_rank=1 imbalance_ratio=1.077474 max_load_reduction=42532 max_load_reduction_pct=0.865316 moved_to_echo=24971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 3584, 0, 7936, 128, 12800, 14336, 10752, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 512, 0, 12672, 128, 4736, 0, 21632, 9984, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 9600, 128, 0, 0, 2176, 128, 30336, 7424, 0, 0, 0, 0, 0, 0] @@ -4797,314 +4797,314 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 0, 3, 0, 2395, 0, 4011, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1913, 881, 0, 0, 0, 24, 0, 0, 3916, 4, 6515, 8190, 0, 0, 433, 986, 0, 0, 0, 1, 0, 0, 0, 7760, 0, 0, 0, 190, 0, 376, 0, 0, 1006, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 0, 0, 1, 0, 2623, 0, 3907, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 1080, 0, 0, 0, 9, 0, 0, 4178, 0, 5924, 8189, 1, 0, 419, 1157, 0, 0, 0, 3, 0, 0, 0, 7797, 0, 0, 0, 248, 0, 288, 0, 0, 1166, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 0, 0, 0, 2434, 0, 4585, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2112, 784, 0, 0, 0, 26, 0, 0, 3558, 0, 6021, 8192, 0, 0, 536, 871, 0, 0, 0, 0, 0, 0, 0, 7812, 0, 0, 0, 182, 0, 246, 0, 0, 1079, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 0, 0, 0, 2470, 0, 3997, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1834, 916, 0, 0, 0, 26, 0, 0, 3349, 3, 5955, 8192, 0, 0, 446, 867, 0, 0, 0, 1, 0, 0, 0, 7760, 0, 0, 0, 285, 0, 524, 0, 0, 1431, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 0, 0, 3, 0, 2658, 0, 3883, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1832, 776, 0, 0, 0, 39, 0, 0, 4430, 1, 6136, 8190, 0, 0, 647, 902, 0, 0, 0, 0, 0, 0, 0, 7780, 0, 0, 0, 288, 0, 283, 0, 0, 1070, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 0, 0, 1, 0, 2623, 0, 3907, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 1080, 0, 0, 0, 9, 0, 0, 4178, 0, 5924, 8189, 1, 0, 419, 1157, 0, 0, 0, 3, 0, 0, 0, 7797, 0, 0, 0, 248, 0, 288, 0, 0, 1166, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 0, 0, 2, 0, 2513, 0, 3736, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1869, 834, 0, 0, 0, 17, 0, 0, 3967, 1, 6039, 8190, 0, 0, 587, 911, 0, 0, 0, 2, 0, 0, 0, 7787, 0, 0, 0, 324, 0, 380, 0, 0, 1385, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 0, 3, 0, 2395, 0, 4011, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1913, 881, 0, 0, 0, 24, 0, 0, 3916, 4, 6515, 8190, 0, 0, 433, 986, 0, 0, 0, 1, 0, 0, 0, 7760, 0, 0, 0, 190, 0, 376, 0, 0, 1006, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 2346, 0, 0, 2, 0, 2447, 0, 4916, 8190, 1, 2, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 2006, 910, 0, 0, 0, 21, 0, 0, 3354, 0, 5631, 8187, 0, 0, 611, 963, 3, 0, 0, 0, 0, 0, 2, 7813, 0, 3, 0, 317, 2, 402, 2, 0, 1010, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 0, 0, 0, 2470, 0, 3997, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1834, 916, 0, 0, 0, 26, 0, 0, 3349, 3, 5955, 8192, 0, 0, 446, 867, 0, 0, 0, 1, 0, 0, 0, 7760, 0, 0, 0, 285, 0, 524, 0, 0, 1431, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 0, 0, 1, 0, 2566, 0, 4317, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2043, 1015, 0, 0, 0, 11, 0, 0, 3348, 3, 5499, 8192, 0, 0, 456, 824, 0, 0, 0, 1, 0, 0, 0, 7719, 0, 0, 0, 312, 0, 474, 0, 0, 1117, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 0, 3, 0, 2395, 0, 1258, 0, 0, 0, 0, 0, 0, 0, 4011, 1976, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1913, 881, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 3916, 4, 1632, 600, 132, 0, 0, 0, 0, 0, 5125, 0, 0, 0, 433, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 376, 0, 0, 1006, 0, 2299, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6009, 5988, 6113, 6308, 6544, 6129, 5918] avg=6144.0000 max=6544 max_rank=5 min=5918 min_rank=7 imbalance_ratio=1.065104 max_load_reduction=42608 max_load_reduction_pct=0.866862 moved_to_echo=24899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 0, 0, 3, 0, 2658, 0, 1185, 0, 0, 0, 0, 0, 0, 0, 3883, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1832, 776, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 4430, 1, 1636, 600, 124, 0, 0, 0, 0, 0, 4827, 0, 0, 0, 647, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 283, 0, 0, 1070, 0, 2299, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 5885, 5859, 5928, 6830, 6376, 6144, 5987] avg=6144.0000 max=6830 max_rank=4 min=5859 min_rank=2 imbalance_ratio=1.111654 max_load_reduction=42322 max_load_reduction_pct=0.861043 moved_to_echo=25337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 0, 0, 1, 0, 2623, 0, 1144, 0, 0, 0, 0, 0, 0, 0, 3907, 1976, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 1080, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4178, 0, 1640, 600, 120, 0, 0, 0, 0, 0, 4660, 0, 1, 0, 419, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 288, 0, 0, 1166, 0, 2298, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 5963, 5884, 6172, 6547, 6237, 6160, 6047] avg=6144.0000 max=6547 max_rank=4 min=5884 min_rank=2 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=24991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 0, 0, 0, 2470, 0, 1150, 0, 0, 0, 0, 0, 0, 0, 3997, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1834, 916, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3349, 3, 1632, 600, 121, 0, 0, 0, 0, 0, 4684, 0, 0, 0, 446, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 524, 0, 0, 1431, 0, 2299, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6520, 5973, 6071, 5731, 5997, 6129, 6586] avg=6144.0000 max=6586 max_rank=7 min=5731 min_rank=4 imbalance_ratio=1.071940 max_load_reduction=42566 max_load_reduction_pct=0.866007 moved_to_echo=24443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 0, 0, 0, 2434, 0, 1162, 0, 0, 0, 0, 0, 0, 0, 4585, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2112, 784, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3558, 0, 1643, 600, 122, 0, 0, 0, 0, 0, 4737, 0, 0, 0, 536, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 246, 0, 0, 1079, 0, 2299, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6116, 6561, 6215, 5949, 6144, 6169, 5852] avg=6144.0000 max=6561 max_rank=2 min=5852 min_rank=7 imbalance_ratio=1.067871 max_load_reduction=42591 max_load_reduction_pct=0.866516 moved_to_echo=24114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 0, 3, 0, 2395, 0, 1258, 0, 0, 0, 0, 0, 0, 0, 4011, 1976, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1913, 881, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 3916, 4, 1632, 600, 132, 0, 0, 0, 0, 0, 5125, 0, 0, 0, 433, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 376, 0, 0, 1006, 0, 2299, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6009, 5988, 6113, 6308, 6544, 6129, 5918] avg=6144.0000 max=6544 max_rank=5 min=5918 min_rank=7 imbalance_ratio=1.065104 max_load_reduction=42608 max_load_reduction_pct=0.866862 moved_to_echo=24899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 2346, 0, 0, 2, 0, 2447, 0, 1087, 0, 0, 0, 0, 0, 0, 0, 4916, 1976, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 2006, 910, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 3354, 0, 1643, 600, 114, 0, 0, 0, 0, 0, 4430, 0, 0, 0, 611, 963, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6170, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 2, 402, 2, 0, 1010, 1, 2298, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 5882, 6895, 6240, 5732, 6007, 6175, 6078] avg=6144.0000 max=6895 max_rank=2 min=5732 min_rank=4 imbalance_ratio=1.122233 max_load_reduction=42257 max_load_reduction_pct=0.859721 moved_to_echo=23992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 0, 0, 0, 2470, 0, 1150, 0, 0, 0, 0, 0, 0, 0, 3997, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1834, 916, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3349, 3, 1632, 600, 121, 0, 0, 0, 0, 0, 4684, 0, 0, 0, 446, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 524, 0, 0, 1431, 0, 2299, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6520, 5973, 6071, 5731, 5997, 6129, 6586] avg=6144.0000 max=6586 max_rank=7 min=5731 min_rank=4 imbalance_ratio=1.071940 max_load_reduction=42566 max_load_reduction_pct=0.866007 moved_to_echo=24443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 0, 0, 2, 0, 2513, 0, 1168, 0, 0, 0, 0, 0, 0, 0, 3736, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1869, 834, 3322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3967, 1, 1640, 606, 127, 0, 0, 0, 0, 0, 4744, 0, 0, 0, 587, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 380, 0, 0, 1385, 0, 2301, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6095, 5699, 6029, 6358, 6242, 6149, 6436] avg=6144.0000 max=6436 max_rank=7 min=5699 min_rank=2 imbalance_ratio=1.047526 max_load_reduction=42716 max_load_reduction_pct=0.869059 moved_to_echo=25185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 0, 0, 1, 0, 2566, 0, 1062, 0, 0, 0, 0, 0, 0, 0, 4317, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2043, 1015, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3348, 3, 1623, 600, 112, 0, 0, 0, 0, 0, 4325, 0, 0, 0, 456, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 474, 0, 0, 1117, 0, 2299, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6682, 6293, 6384, 5697, 5605, 6097, 6248] avg=6144.0000 max=6682 max_rank=1 min=5605 min_rank=5 imbalance_ratio=1.087565 max_load_reduction=42470 max_load_reduction_pct=0.864054 moved_to_echo=23647 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 0, 0, 2, 0, 2513, 0, 1168, 0, 0, 0, 0, 0, 0, 0, 3736, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1869, 834, 3322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3967, 1, 1640, 606, 127, 0, 0, 0, 0, 0, 4744, 0, 0, 0, 587, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 380, 0, 0, 1385, 0, 2301, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6095, 5699, 6029, 6358, 6242, 6149, 6436] avg=6144.0000 max=6436 max_rank=7 min=5699 min_rank=2 imbalance_ratio=1.047526 max_load_reduction=42716 max_load_reduction_pct=0.869059 moved_to_echo=25185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 128, 15488, 7296, 26624, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 256, 0, 0, 30208, 128, 13184, 4864, 1024, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 19840, 0, 0, 128, 0, 20224, 0, 9216, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[33408, 15872, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 128, 15488, 7296, 26624, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 2176, 128, 3072, 128, 0, 9344, 128, 18432, 16384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[37632, 0, 128, 0, 4224, 7552, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4866, 0, 0, 151, 64, 1, 3227, 3249, 5, 0, 0, 171, 0, 0, 5103, 0, 2, 1, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 7675, 0, 0, 2, 0, 0, 1, 0, 1883, 0, 5414, 69, 0, 0, 7700, 0, 0, 0, 0, 1552, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4367, 0, 0, 174, 80, 0, 3127, 3486, 1, 0, 1, 206, 0, 0, 4864, 0, 2, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 7841, 0, 0, 1, 0, 0, 0, 0, 2205, 0, 5034, 89, 0, 0, 7743, 0, 0, 0, 0, 1846, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5129, 0, 0, 79, 54, 0, 3594, 3160, 0, 0, 0, 120, 0, 0, 4908, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 7536, 0, 0, 1, 0, 0, 0, 0, 2006, 0, 5164, 65, 0, 0, 7633, 0, 0, 0, 0, 1594, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4693, 0, 0, 135, 110, 0, 3328, 3359, 0, 0, 0, 165, 0, 0, 5384, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 7699, 0, 0, 0, 0, 0, 0, 0, 2009, 0, 5177, 76, 0, 0, 7657, 0, 0, 0, 0, 1311, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4444, 0, 0, 143, 162, 0, 3698, 3415, 5, 0, 0, 174, 0, 0, 4905, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 1, 0, 0, 0, 0, 7550, 0, 0, 0, 0, 0, 0, 0, 2108, 0, 5445, 66, 0, 0, 7672, 0, 0, 0, 0, 1360, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4973, 3, 0, 122, 139, 3, 3286, 3554, 0, 0, 1, 165, 0, 1, 4961, 1, 6, 1, 0, 0, 0, 8079, 1, 0, 2, 2, 0, 0, 0, 7548, 0, 0, 0, 0, 0, 0, 0, 2075, 0, 4919, 75, 0, 0, 7656, 1, 0, 0, 1, 1573, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4552, 0, 0, 153, 107, 0, 3509, 3330, 0, 0, 0, 270, 0, 0, 5463, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 7633, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 5268, 92, 0, 0, 7614, 0, 0, 0, 0, 1094, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4693, 0, 0, 135, 110, 0, 3328, 3359, 0, 0, 0, 165, 0, 0, 5384, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 7699, 0, 0, 0, 0, 0, 0, 0, 2009, 0, 5177, 76, 0, 0, 7657, 0, 0, 0, 0, 1311, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4987, 0, 0, 84, 70, 0, 3527, 2969, 0, 0, 0, 155, 0, 0, 5402, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 7480, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 5025, 75, 0, 0, 7664, 0, 0, 0, 0, 1384, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4866, 0, 0, 151, 64, 1, 3227, 3249, 5, 0, 0, 171, 0, 0, 5103, 0, 2, 1, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 7675, 0, 0, 2, 0, 0, 1, 0, 1883, 0, 5414, 69, 0, 0, 7700, 0, 0, 0, 0, 1552, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3017, 1915, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4367, 0, 0, 1130, 235, 0, 0, 0, 0, 0, 0, 174, 80, 0, 2382, 3486, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 4864, 0, 2, 0, 0, 745, 25, 0, 0, 0, 0, 0, 0, 0, 0, 6170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6322, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2205, 0, 3879, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4726, 0, 0, 0, 0, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6216, 5732, 6124, 5842, 6170, 6323, 6173, 6572] avg=6144.0000 max=6572 max_rank=7 min=5732 min_rank=1 imbalance_ratio=1.069661 max_load_reduction=42580 max_load_reduction_pct=0.866292 moved_to_echo=25238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4552, 0, 0, 153, 107, 0, 3509, 3330, 0, 0, 0, 270, 0, 0, 5463, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 7633, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 5268, 92, 0, 0, 7614, 0, 0, 0, 0, 1094, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4444, 0, 0, 143, 162, 0, 3698, 3415, 5, 0, 0, 174, 0, 0, 4905, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 1, 0, 0, 0, 0, 7550, 0, 0, 0, 0, 0, 0, 0, 2108, 0, 5445, 66, 0, 0, 7672, 0, 0, 0, 0, 1360, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2974, 1921, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5129, 0, 0, 1159, 226, 0, 0, 0, 0, 0, 0, 79, 54, 0, 2737, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 4908, 0, 0, 0, 0, 857, 26, 0, 0, 0, 0, 0, 0, 0, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6076, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 3979, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4659, 0, 0, 0, 0, 1594, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6129, 6514, 6030, 5911, 6187, 6077, 6050, 6254] avg=6144.0000 max=6514 max_rank=1 min=5911 min_rank=3 imbalance_ratio=1.060221 max_load_reduction=42638 max_load_reduction_pct=0.867472 moved_to_echo=24568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 0, 0, 0, 0, 2983, 1914, 1236, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4973, 3, 0, 1104, 226, 0, 0, 0, 0, 0, 0, 122, 139, 3, 2503, 3554, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 1, 4961, 1, 6, 1, 0, 783, 25, 0, 0, 0, 0, 0, 0, 0, 0, 6165, 1, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2075, 0, 3790, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4673, 1, 0, 0, 1, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6307, 6322, 5943, 6170, 6086, 5940, 6248] avg=6144.0000 max=6322 max_rank=2 min=5940 min_rank=6 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3000, 1899, 1256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4866, 0, 0, 1215, 230, 0, 0, 0, 0, 0, 0, 151, 64, 1, 2458, 3249, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 5103, 0, 2, 1, 0, 769, 27, 0, 0, 0, 0, 0, 0, 0, 0, 6117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1883, 0, 4172, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4700, 0, 0, 0, 0, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6311, 5928, 6073, 6117, 6191, 6125, 6252] avg=6144.0000 max=6311 max_rank=1 min=5928 min_rank=2 imbalance_ratio=1.027181 max_load_reduction=42841 max_load_reduction_pct=0.871602 moved_to_echo=24685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2983, 1907, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4693, 0, 0, 1162, 231, 0, 0, 0, 0, 0, 0, 135, 110, 0, 2535, 3359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 5384, 0, 0, 0, 0, 793, 26, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2009, 0, 3989, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4674, 0, 0, 0, 0, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6086, 6139, 6368, 6142, 6208, 6074, 5985] avg=6144.0000 max=6368 max_rank=3 min=5985 min_rank=7 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3017, 1915, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4367, 0, 0, 1130, 235, 0, 0, 0, 0, 0, 0, 174, 80, 0, 2382, 3486, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 4864, 0, 2, 0, 0, 745, 25, 0, 0, 0, 0, 0, 0, 0, 0, 6170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6322, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2205, 0, 3879, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4726, 0, 0, 0, 0, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6216, 5732, 6124, 5842, 6170, 6323, 6173, 6572] avg=6144.0000 max=6572 max_rank=7 min=5732 min_rank=1 imbalance_ratio=1.069661 max_load_reduction=42580 max_load_reduction_pct=0.866292 moved_to_echo=25238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2986, 1910, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4987, 0, 0, 1128, 224, 0, 0, 0, 0, 0, 0, 84, 70, 0, 2686, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 5402, 0, 0, 0, 0, 841, 25, 0, 0, 0, 0, 0, 0, 0, 0, 6154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 3872, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4678, 0, 0, 0, 0, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 6339, 5809, 6423, 6154, 6031, 6213, 6062] avg=6144.0000 max=6423 max_rank=3 min=5809 min_rank=2 imbalance_ratio=1.045410 max_load_reduction=42729 max_load_reduction_pct=0.869324 moved_to_echo=24460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2994, 1899, 1239, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4444, 0, 0, 1225, 229, 0, 0, 0, 0, 0, 0, 143, 162, 0, 2812, 3415, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 4905, 0, 0, 0, 0, 886, 32, 0, 0, 0, 0, 0, 0, 0, 0, 6103, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2108, 0, 4188, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4678, 0, 0, 0, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 5900, 6537, 5997, 6104, 6082, 6362, 6038] avg=6144.0000 max=6537 max_rank=2 min=5900 min_rank=1 imbalance_ratio=1.063965 max_load_reduction=42615 max_load_reduction_pct=0.867004 moved_to_echo=24586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3000, 1899, 1256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4866, 0, 0, 1215, 230, 0, 0, 0, 0, 0, 0, 151, 64, 1, 2458, 3249, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 5103, 0, 2, 1, 0, 769, 27, 0, 0, 0, 0, 0, 0, 0, 0, 6117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1883, 0, 4172, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4700, 0, 0, 0, 0, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6311, 5928, 6073, 6117, 6191, 6125, 6252] avg=6144.0000 max=6311 max_rank=1 min=5928 min_rank=2 imbalance_ratio=1.027181 max_load_reduction=42841 max_load_reduction_pct=0.871602 moved_to_echo=24685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 0, 0, 0, 0, 2983, 1914, 1236, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4973, 3, 0, 1104, 226, 0, 0, 0, 0, 0, 0, 122, 139, 3, 2503, 3554, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 1, 4961, 1, 6, 1, 0, 783, 25, 0, 0, 0, 0, 0, 0, 0, 0, 6165, 1, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2075, 0, 3790, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4673, 1, 0, 0, 1, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6307, 6322, 5943, 6170, 6086, 5940, 6248] avg=6144.0000 max=6322 max_rank=2 min=5940 min_rank=6 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2967, 1896, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4552, 0, 0, 1182, 229, 0, 0, 0, 0, 0, 0, 153, 107, 0, 2673, 3330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 5463, 0, 0, 0, 0, 836, 26, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 4060, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4647, 0, 0, 0, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 5963, 6263, 6595, 6108, 6154, 6215, 5741] avg=6144.0000 max=6595 max_rank=3 min=5741 min_rank=7 imbalance_ratio=1.073405 max_load_reduction=42557 max_load_reduction_pct=0.865824 moved_to_echo=24218 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2994, 1899, 1239, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4444, 0, 0, 1225, 229, 0, 0, 0, 0, 0, 0, 143, 162, 0, 2812, 3415, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 4905, 0, 0, 0, 0, 886, 32, 0, 0, 0, 0, 0, 0, 0, 0, 6103, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2108, 0, 4188, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4678, 0, 0, 0, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 5900, 6537, 5997, 6104, 6082, 6362, 6038] avg=6144.0000 max=6537 max_rank=2 min=5900 min_rank=1 imbalance_ratio=1.063965 max_load_reduction=42615 max_load_reduction_pct=0.867004 moved_to_echo=24586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 23936, 15360, 9984, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[37504, 128, 0, 0, 128, 11776, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 16640, 0, 32000, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1152, 896, 128, 20864, 26624, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[37504, 128, 0, 0, 128, 11776, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 38016, 128, 0, 9344, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 16640, 0, 32000, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1536, 0, 128, 41088, 128, 128, 128, 0, 6528, 256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 49152, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 49152, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 49152, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 261, 197, 19, 0, 0, 0, 5019, 0, 0, 0, 8012, 0, 36, 0, 0, 0, 0, 7108, 0, 0, 7033, 0, 0, 0, 0, 0, 0, 0, 610, 136, 0, 0, 3074, 0, 0, 3871, 7089, 197, 0, 0, 214, 0, 0, 0, 0, 0, 5269, 0, 323, 0, 0, 0, 1, 0, 666, 0, 0, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 318, 117, 13, 0, 0, 0, 5063, 0, 0, 0, 8041, 0, 46, 0, 0, 0, 0, 6484, 0, 0, 7354, 0, 0, 0, 0, 0, 0, 0, 809, 134, 0, 0, 2937, 0, 0, 3621, 7234, 369, 0, 0, 176, 0, 0, 0, 0, 0, 5515, 0, 346, 0, 0, 0, 0, 0, 565, 0, 0, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 378, 176, 6, 0, 0, 0, 4713, 0, 0, 0, 7977, 0, 30, 0, 0, 0, 0, 6642, 0, 0, 7100, 0, 0, 0, 0, 0, 0, 0, 823, 91, 0, 0, 3421, 2, 0, 3372, 7252, 344, 0, 0, 227, 0, 0, 0, 0, 0, 5785, 0, 287, 0, 0, 0, 0, 0, 522, 0, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 250, 259, 5, 0, 0, 0, 5515, 0, 0, 0, 8048, 0, 2, 0, 0, 0, 0, 7036, 0, 0, 7017, 0, 0, 0, 0, 1, 0, 0, 524, 110, 0, 0, 3574, 0, 0, 3628, 7096, 201, 0, 0, 144, 0, 0, 0, 0, 0, 4746, 0, 309, 0, 0, 0, 0, 0, 661, 0, 0, 26] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 343, 316, 13, 0, 0, 3, 5201, 0, 0, 2, 8007, 0, 46, 2, 0, 0, 0, 6559, 0, 0, 7216, 0, 0, 0, 0, 0, 0, 0, 836, 113, 1, 0, 3287, 0, 0, 2981, 7235, 308, 1, 0, 138, 0, 0, 0, 0, 0, 5416, 0, 426, 0, 0, 3, 0, 0, 687, 0, 2, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 459, 424, 16, 0, 0, 0, 5541, 0, 0, 0, 8028, 0, 21, 0, 0, 0, 0, 7234, 0, 0, 7281, 0, 0, 0, 0, 0, 0, 0, 522, 112, 0, 0, 2325, 0, 0, 3098, 7259, 443, 0, 0, 168, 0, 0, 0, 0, 0, 5169, 0, 272, 0, 0, 0, 0, 0, 770, 0, 0, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 378, 176, 6, 0, 0, 0, 4713, 0, 0, 0, 7977, 0, 30, 0, 0, 0, 0, 6642, 0, 0, 7100, 0, 0, 0, 0, 0, 0, 0, 823, 91, 0, 0, 3421, 2, 0, 3372, 7252, 344, 0, 0, 227, 0, 0, 0, 0, 0, 5785, 0, 287, 0, 0, 0, 0, 0, 522, 0, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 336, 231, 25, 0, 0, 0, 4660, 0, 0, 0, 8037, 0, 8, 0, 0, 0, 0, 6796, 0, 1, 7076, 0, 0, 0, 0, 0, 0, 0, 785, 141, 0, 0, 3060, 0, 0, 3520, 7174, 264, 0, 0, 146, 0, 0, 0, 0, 0, 5527, 0, 555, 0, 0, 0, 0, 0, 793, 0, 0, 16] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 261, 197, 19, 0, 0, 0, 5019, 0, 0, 0, 8012, 0, 36, 0, 0, 0, 0, 7108, 0, 0, 7033, 0, 0, 0, 0, 0, 0, 0, 610, 136, 0, 0, 3074, 0, 0, 3871, 7089, 197, 0, 0, 214, 0, 0, 0, 0, 0, 5269, 0, 323, 0, 0, 0, 1, 0, 666, 0, 0, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 320, 229, 12, 0, 0, 0, 5278, 0, 0, 0, 8008, 0, 3, 0, 0, 0, 0, 7160, 0, 0, 7213, 0, 0, 0, 0, 0, 0, 0, 544, 113, 0, 0, 3371, 0, 0, 3495, 7213, 216, 0, 0, 187, 0, 0, 0, 0, 0, 4958, 0, 297, 0, 0, 0, 0, 0, 512, 0, 0, 23] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 261, 197, 5561, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 5019, 0, 0, 0, 729, 272, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 6326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 136, 0, 0, 3074, 0, 1908, 270, 0, 0, 0, 0, 0, 0, 0, 3871, 2197, 197, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5269, 0, 323, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 666, 0, 0, 17, 2984, 2451, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6019, 6039, 6362, 6034, 5998, 6479, 6102, 6119] avg=6144.0000 max=6479 max_rank=5 min=5998 min_rank=4 imbalance_ratio=1.054525 max_load_reduction=42673 max_load_reduction_pct=0.868184 moved_to_echo=24698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 250, 259, 5587, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5515, 0, 0, 0, 727, 269, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6021, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 110, 0, 0, 3574, 0, 1910, 269, 0, 0, 0, 0, 0, 0, 0, 3628, 2199, 201, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4746, 0, 309, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 26, 2987, 2461, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6096, 6516, 6264, 6022, 6387, 6172, 5560, 6135] avg=6144.0000 max=6516 max_rank=1 min=5560 min_rank=6 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24254 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 459, 424, 16, 0, 0, 0, 5541, 0, 0, 0, 8028, 0, 21, 0, 0, 0, 0, 7234, 0, 0, 7281, 0, 0, 0, 0, 0, 0, 0, 522, 112, 0, 0, 2325, 0, 0, 3098, 7259, 443, 0, 0, 168, 0, 0, 0, 0, 0, 5169, 0, 272, 0, 0, 0, 0, 0, 770, 0, 0, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 318, 117, 5581, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 5063, 0, 0, 0, 762, 248, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 5771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 809, 134, 0, 0, 2937, 0, 1947, 282, 0, 0, 0, 0, 0, 0, 0, 3621, 2242, 369, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5515, 0, 346, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 10, 3045, 2460, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6016, 6086, 5817, 6310, 6109, 6408, 6326, 6080] avg=6144.0000 max=6408 max_rank=5 min=5817 min_rank=2 imbalance_ratio=1.042969 max_load_reduction=42744 max_load_reduction_pct=0.869629 moved_to_echo=24923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 343, 316, 5559, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3, 5201, 0, 0, 2, 748, 251, 0, 0, 0, 0, 0, 0, 0, 0, 46, 2, 0, 0, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 113, 1, 0, 3287, 0, 1948, 277, 0, 0, 0, 0, 0, 0, 0, 2981, 2241, 308, 1, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5416, 0, 426, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 687, 0, 2, 9, 3046, 2448, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6218, 5885, 6191, 6462, 5669, 6313, 6195] avg=6144.0000 max=6462 max_rank=4 min=5669 min_rank=5 imbalance_ratio=1.051758 max_load_reduction=42690 max_load_reduction_pct=0.868530 moved_to_echo=24639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 459, 424, 5573, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 5541, 0, 0, 0, 757, 281, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 6431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 112, 0, 0, 2325, 0, 1958, 283, 0, 0, 0, 0, 0, 0, 0, 3098, 2240, 443, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5169, 0, 272, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 0, 0, 10, 3061, 2455, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6456, 6595, 6452, 6241, 5200, 5949, 5963, 6296] avg=6144.0000 max=6595 max_rank=1 min=5200 min_rank=4 imbalance_ratio=1.073405 max_load_reduction=42557 max_load_reduction_pct=0.865824 moved_to_echo=23408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 250, 259, 5587, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5515, 0, 0, 0, 727, 269, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6021, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 110, 0, 0, 3574, 0, 1910, 269, 0, 0, 0, 0, 0, 0, 0, 3628, 2199, 201, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4746, 0, 309, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 26, 2987, 2461, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6096, 6516, 6264, 6022, 6387, 6172, 5560, 6135] avg=6144.0000 max=6516 max_rank=1 min=5560 min_rank=6 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24254 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 336, 231, 5578, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4660, 0, 0, 0, 733, 260, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 6048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 785, 141, 0, 0, 3060, 0, 1931, 271, 0, 0, 0, 0, 0, 0, 0, 3520, 2223, 264, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5527, 0, 555, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 0, 0, 16, 3020, 2459, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 5678, 6056, 6073, 6188, 6153, 6570, 6288] avg=6144.0000 max=6570 max_rank=6 min=5678 min_rank=1 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=25199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 378, 176, 5538, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4713, 0, 0, 0, 736, 254, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 91, 0, 0, 3421, 2, 1952, 272, 0, 0, 0, 0, 0, 0, 0, 3372, 2247, 344, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5785, 0, 287, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 0, 0, 4, 3053, 2439, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6092, 5709, 5941, 6092, 6561, 6190, 6549, 6018] avg=6144.0000 max=6561 max_rank=4 min=5709 min_rank=1 imbalance_ratio=1.067871 max_load_reduction=42591 max_load_reduction_pct=0.866516 moved_to_echo=25318 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 343, 316, 5559, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3, 5201, 0, 0, 2, 748, 251, 0, 0, 0, 0, 0, 0, 0, 0, 46, 2, 0, 0, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 113, 1, 0, 3287, 0, 1948, 277, 0, 0, 0, 0, 0, 0, 0, 2981, 2241, 308, 1, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5416, 0, 426, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 687, 0, 2, 9, 3046, 2448, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6218, 5885, 6191, 6462, 5669, 6313, 6195] avg=6144.0000 max=6462 max_rank=4 min=5669 min_rank=5 imbalance_ratio=1.051758 max_load_reduction=42690 max_load_reduction_pct=0.868530 moved_to_echo=24639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 320, 229, 5559, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 5278, 0, 0, 0, 747, 274, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 6372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 113, 0, 0, 3371, 0, 1942, 277, 0, 0, 0, 0, 0, 0, 0, 3495, 2234, 216, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4958, 0, 297, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 23, 3037, 2449, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6108, 6311, 6375, 6189, 6247, 6132, 5769, 6021] avg=6144.0000 max=6375 max_rank=2 min=5769 min_rank=6 imbalance_ratio=1.037598 max_load_reduction=42777 max_load_reduction_pct=0.870300 moved_to_echo=24169 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 261, 197, 5561, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 5019, 0, 0, 0, 729, 272, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 6326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 136, 0, 0, 3074, 0, 1908, 270, 0, 0, 0, 0, 0, 0, 0, 3871, 2197, 197, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5269, 0, 323, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 666, 0, 0, 17, 2984, 2451, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6019, 6039, 6362, 6034, 5998, 6479, 6102, 6119] avg=6144.0000 max=6479 max_rank=5 min=5998 min_rank=4 imbalance_ratio=1.054525 max_load_reduction=42673 max_load_reduction_pct=0.868184 moved_to_echo=24698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 459, 424, 5573, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 5541, 0, 0, 0, 757, 281, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 6431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 112, 0, 0, 2325, 0, 1958, 283, 0, 0, 0, 0, 0, 0, 0, 3098, 2240, 443, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5169, 0, 272, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 0, 0, 10, 3061, 2455, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6456, 6595, 6452, 6241, 5200, 5949, 5963, 6296] avg=6144.0000 max=6595 max_rank=1 min=5200 min_rank=4 imbalance_ratio=1.073405 max_load_reduction=42557 max_load_reduction_pct=0.865824 moved_to_echo=23408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 5504, 1024, 128, 0, 25088, 128, 15616, 2304, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 27648, 17920, 2432, 128, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 41088, 0, 0, 128, 6016, 2176, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 42496, 0, 2816, 0, 3968, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 5248, 0, 128, 128, 24320, 19712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 41088, 0, 0, 128, 6016, 2176, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 2688, 2048, 44544, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 5248, 0, 128, 128, 24320, 19712, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 256, 128, 0, 0, 0, 49024, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 49152, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6822, 0, 0, 0, 8192, 0, 5766, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5273, 0, 0, 3763, 0, 0, 7085, 0, 0, 0, 0, 0, 0, 7451, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 0, 0, 1, 7, 6393, 0, 0, 0, 8188, 0, 6070, 0, 530, 0, 0, 0, 5, 0, 0, 0, 0, 293, 0, 0, 0, 13, 0, 2, 0, 0, 0, 2, 0, 5486, 0, 0, 3755, 0, 0, 7066, 0, 0, 2, 0, 0, 0, 7528, 4, 0, 0, 6, 3798, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 5395, 0, 0, 0, 8191, 0, 5855, 0, 589, 0, 0, 0, 1, 0, 0, 0, 0, 478, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5591, 0, 0, 4022, 0, 0, 7553, 0, 0, 0, 0, 0, 0, 7522, 0, 0, 0, 0, 3945, 0, 0, 0, 5, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6865, 0, 0, 0, 8192, 0, 5659, 0, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5336, 0, 0, 3409, 0, 0, 7153, 0, 0, 0, 0, 0, 0, 7380, 0, 0, 0, 0, 4126, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 0, 6579, 0, 0, 0, 8191, 0, 5378, 0, 479, 0, 0, 0, 11, 1, 0, 0, 0, 598, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 5347, 0, 3, 3786, 0, 0, 6979, 0, 0, 0, 0, 0, 0, 7485, 0, 0, 0, 0, 4301, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6494, 0, 0, 0, 8192, 0, 5070, 0, 489, 0, 0, 1, 5, 0, 0, 0, 0, 553, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 5177, 0, 0, 3788, 0, 0, 7306, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 0, 3, 4543, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6865, 0, 0, 0, 8192, 0, 5659, 0, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5336, 0, 0, 3409, 0, 0, 7153, 0, 0, 0, 0, 0, 0, 7380, 0, 0, 0, 0, 4126, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6420, 0, 0, 0, 8192, 0, 5285, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5676, 0, 0, 3423, 0, 0, 7394, 0, 0, 0, 0, 0, 0, 7524, 0, 0, 0, 1, 4471, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 0, 0, 1, 7, 6393, 0, 0, 0, 8188, 0, 6070, 0, 530, 0, 0, 0, 5, 0, 0, 0, 0, 293, 0, 0, 0, 13, 0, 2, 0, 0, 0, 2, 0, 5486, 0, 0, 3755, 0, 0, 7066, 0, 0, 2, 0, 0, 0, 7528, 4, 0, 0, 6, 3798, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6586, 0, 0, 0, 8192, 0, 5628, 0, 515, 0, 0, 0, 1, 0, 0, 0, 0, 573, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5418, 0, 0, 2447, 0, 0, 7480, 0, 0, 0, 0, 0, 0, 7382, 0, 0, 0, 2, 4926, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 0, 0, 0, 5855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 1, 0, 0, 3530, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 4, 0, 3065, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5591, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 2702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 5, 0, 0, 0, 1321, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6320, 5985, 5596, 6326, 6724, 6182, 5876] avg=6144.0000 max=6724 max_rank=5 min=5596 min_rank=3 imbalance_ratio=1.094401 max_load_reduction=42428 max_load_reduction_pct=0.863200 moved_to_echo=25108 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 587, 0, 0, 0, 0, 0, 5766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 0, 0, 0, 3311, 2359, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 9, 0, 3876, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5273, 0, 728, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 0, 0, 2535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 1239, 599, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6353, 6229, 6088, 6001, 6298, 6124, 5915] avg=6144.0000 max=6353 max_rank=1 min=5915 min_rank=7 imbalance_ratio=1.034017 max_load_reduction=42799 max_load_reduction_pct=0.870748 moved_to_echo=24338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6822, 0, 0, 0, 8192, 0, 5766, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5273, 0, 0, 3763, 0, 0, 7085, 0, 0, 0, 0, 0, 0, 7451, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 5395, 0, 0, 0, 8191, 0, 5855, 0, 589, 0, 0, 0, 1, 0, 0, 0, 0, 478, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5591, 0, 0, 4022, 0, 0, 7553, 0, 0, 0, 0, 0, 0, 7522, 0, 0, 0, 0, 3945, 0, 0, 0, 5, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 0, 6579, 0, 0, 0, 8191, 0, 5378, 0, 479, 0, 0, 0, 11, 1, 0, 0, 0, 598, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 5347, 0, 3, 3786, 0, 0, 6979, 0, 0, 0, 0, 0, 0, 7485, 0, 0, 0, 0, 4301, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 0, 5659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 0, 0, 0, 0, 3343, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 9, 0, 3900, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5336, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 0, 0, 2559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4126, 0, 0, 0, 0, 0, 0, 0, 1251, 593, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6250, 6223, 6474, 6057, 5968, 6066, 5970] avg=6144.0000 max=6474 max_rank=3 min=5968 min_rank=5 imbalance_ratio=1.053711 max_load_reduction=42678 max_load_reduction_pct=0.868286 moved_to_echo=24061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 0, 0, 1, 6141, 0, 0, 0, 0, 0, 0, 0, 7, 551, 0, 0, 0, 0, 0, 6070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 0, 0, 0, 5, 0, 0, 3302, 2210, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 13, 0, 3632, 2047, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 5486, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 2529, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6188, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 1, 0, 0, 0, 0, 0, 1235, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6628, 6047, 5985, 6225, 6286, 6198, 5639] avg=6144.0000 max=6628 max_rank=1 min=5639 min_rank=7 imbalance_ratio=1.078776 max_load_reduction=42524 max_load_reduction_pct=0.865153 moved_to_echo=24348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 5285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 3455, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 3, 0, 3647, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5676, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 3423, 0, 0, 2646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6184, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4471, 0, 0, 0, 0, 0, 0, 0, 1293, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 5838, 6192, 5945, 6411, 6069, 6185, 6369] avg=6144.0000 max=6411 max_rank=4 min=5838 min_rank=1 imbalance_ratio=1.043457 max_load_reduction=42741 max_load_reduction_pct=0.869568 moved_to_echo=25034 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 5378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 0, 0, 11, 1, 0, 3265, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 10, 0, 3741, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5347, 0, 735, 0, 0, 0, 0, 0, 0, 0, 3, 3786, 0, 0, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4301, 0, 0, 0, 0, 0, 0, 0, 1223, 606, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 5938, 6034, 6397, 6082, 6280, 6144, 6130] avg=6144.0000 max=6397 max_rank=3 min=5938 min_rank=1 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 5070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 1, 5, 0, 0, 3414, 2245, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 10, 1, 3689, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5177, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 0, 2615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4543, 0, 0, 0, 0, 0, 0, 0, 1277, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 5630, 6154, 6302, 5911, 6403, 6184, 6425] avg=6144.0000 max=6425 max_rank=7 min=5630 min_rank=1 imbalance_ratio=1.045736 max_load_reduction=42727 max_load_reduction_pct=0.869283 moved_to_echo=24923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 5285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 3455, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 3, 0, 3647, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5676, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 3423, 0, 0, 2646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6184, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4471, 0, 0, 0, 0, 0, 0, 0, 1293, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 5838, 6192, 5945, 6411, 6069, 6185, 6369] avg=6144.0000 max=6411 max_rank=4 min=5838 min_rank=1 imbalance_ratio=1.043457 max_load_reduction=42741 max_load_reduction_pct=0.869568 moved_to_echo=25034 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 587, 0, 0, 0, 0, 0, 5766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 0, 0, 0, 3311, 2359, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 9, 0, 3876, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5273, 0, 728, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 0, 0, 2535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 1239, 599, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6353, 6229, 6088, 6001, 6298, 6124, 5915] avg=6144.0000 max=6353 max_rank=1 min=5915 min_rank=7 imbalance_ratio=1.034017 max_load_reduction=42799 max_load_reduction_pct=0.870748 moved_to_echo=24338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 0, 0, 0, 1, 0, 0, 3495, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 2, 0, 3742, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5418, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 2447, 0, 0, 2677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6067, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4926, 0, 0, 0, 0, 0, 0, 0, 1308, 594, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6195, 6288, 6365, 6139, 5124, 6069, 6828] avg=6144.0000 max=6828 max_rank=7 min=5124 min_rank=5 imbalance_ratio=1.111328 max_load_reduction=42324 max_load_reduction_pct=0.861084 moved_to_echo=24160 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 0, 0, 1, 6141, 0, 0, 0, 0, 0, 0, 0, 7, 551, 0, 0, 0, 0, 0, 6070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 0, 0, 0, 5, 0, 0, 3302, 2210, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 13, 0, 3632, 2047, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 5486, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 2529, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6188, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 1, 0, 0, 0, 0, 0, 1235, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6628, 6047, 5985, 6225, 6286, 6198, 5639] avg=6144.0000 max=6628 max_rank=1 min=5639 min_rank=7 imbalance_ratio=1.078776 max_load_reduction=42524 max_load_reduction_pct=0.865153 moved_to_echo=24348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 0, 0, 0, 5855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 1, 0, 0, 3530, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 4, 0, 3065, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5591, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 2702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 5, 0, 0, 0, 1321, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6320, 5985, 5596, 6326, 6724, 6182, 5876] avg=6144.0000 max=6724 max_rank=5 min=5596 min_rank=3 imbalance_ratio=1.094401 max_load_reduction=42428 max_load_reduction_pct=0.863200 moved_to_echo=25108 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 5378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 0, 0, 11, 1, 0, 3265, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 10, 0, 3741, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5347, 0, 735, 0, 0, 0, 0, 0, 0, 0, 3, 3786, 0, 0, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4301, 0, 0, 0, 0, 0, 0, 0, 1223, 606, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 5938, 6034, 6397, 6082, 6280, 6144, 6130] avg=6144.0000 max=6397 max_rank=3 min=5938 min_rank=1 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 4224, 0, 0, 128, 128, 128, 0, 27136, 17920, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 3456, 0, 0, 0, 128, 128, 29312, 16512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 28416, 0, 0, 20864, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[34304, 0, 128, 0, 128, 0, 0, 0, 10240, 4864, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 128, 0, 43392, 0, 5888, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 4480, 0, 0, 0, 0, 0, 44800, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 128, 0, 43392, 0, 5888, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 49152, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 0, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 2746, 0, 0, 0, 1576, 9, 0, 7708, 0, 0, 1, 0, 0, 73, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 1, 0, 0, 0, 0, 536, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6501, 6344, 7402, 7120] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3031, 0, 0, 0, 1439, 1, 0, 7541, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 4, 0, 0, 0, 534, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6784, 6196, 7543, 7081] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3152, 0, 0, 0, 1270, 0, 0, 7666, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 544, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6691, 6155, 7449, 7191] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 2799, 0, 0, 0, 1579, 1, 0, 7711, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 1, 0, 13, 1, 0, 0, 668, 34, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6190, 6613, 7573, 6913] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3091, 0, 0, 0, 1246, 6, 0, 7675, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 677, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6316, 6288, 7587, 7048] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3595, 0, 0, 0, 1432, 2, 0, 7582, 0, 0, 1, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 783, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 2, 0, 0, 0, 0, 0, 594, 26, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 6768, 5876, 7381, 6857] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3031, 0, 0, 0, 1439, 1, 0, 7541, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 4, 0, 0, 0, 534, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6784, 6196, 7543, 7081] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 2835, 0, 0, 0, 1466, 11, 0, 7627, 1, 0, 0, 1, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 1, 1026, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 1, 1, 5, 0, 0, 3, 509, 6, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 2, 0, 6547, 5999, 7566, 7289] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 2746, 0, 0, 0, 1576, 9, 0, 7708, 0, 0, 1, 0, 0, 73, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 1, 0, 0, 0, 0, 536, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6501, 6344, 7402, 7120] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3595, 0, 0, 0, 1432, 2, 0, 7582, 0, 0, 1, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 783, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 2, 0, 0, 0, 0, 0, 594, 26, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 6768, 5876, 7381, 6857] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3293, 0, 0, 0, 1651, 13, 0, 7735, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 2, 0, 0, 0, 661, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6375, 6171, 7311, 6808] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 2799, 0, 0, 0, 1579, 1, 0, 7711, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 1, 0, 13, 1, 0, 0, 668, 34, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6190, 6613, 7573, 6913] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3031, 0, 0, 0, 1439, 1, 1528, 61, 0, 0, 0, 0, 0, 0, 0, 5984, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 3355, 2024, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 534, 9, 3429, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4839, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6060, 6032, 6197, 6167, 6157, 6218, 6186, 6135] avg=6144.0000 max=6218 max_rank=5 min=6032 min_rank=1 imbalance_ratio=1.012044 max_load_reduction=42934 max_load_reduction_pct=0.873494 moved_to_echo=24696 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3152, 0, 0, 0, 1270, 0, 1553, 61, 0, 0, 0, 0, 0, 0, 0, 6083, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 3309, 2017, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 27, 3382, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4914, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6036, 6100, 6121, 6192, 6137, 6230, 6242, 6094] avg=6144.0000 max=6242 max_rank=6 min=6036 min_rank=0 imbalance_ratio=1.015951 max_load_reduction=42910 max_load_reduction_pct=0.873006 moved_to_echo=24703 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3091, 0, 0, 0, 1246, 6, 1555, 62, 0, 0, 0, 0, 0, 0, 0, 6090, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 3123, 2017, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 33, 3193, 2231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4817, 1353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5960, 6171, 6234, 6121, 6136, 6134, 6170, 6226] avg=6144.0000 max=6234 max_rank=2 min=5960 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=42918 max_load_reduction_pct=0.873169 moved_to_echo=24666 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3031, 0, 0, 0, 1439, 1, 1528, 61, 0, 0, 0, 0, 0, 0, 0, 5984, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 3355, 2024, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 534, 9, 3429, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4839, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6060, 6032, 6197, 6167, 6157, 6218, 6186, 6135] avg=6144.0000 max=6218 max_rank=5 min=6032 min_rank=1 imbalance_ratio=1.012044 max_load_reduction=42934 max_load_reduction_pct=0.873494 moved_to_echo=24696 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2835, 0, 0, 0, 1466, 11, 1546, 59, 0, 0, 0, 0, 0, 0, 0, 6051, 1, 0, 0, 1, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6217, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 1, 1026, 0, 3238, 2020, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 0, 0, 3, 509, 6, 3309, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 4981, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6131, 6217, 6324, 6146, 6142, 6333, 5942] avg=6144.0000 max=6333 max_rank=6 min=5917 min_rank=0 imbalance_ratio=1.030762 max_load_reduction=42819 max_load_reduction_pct=0.871155 moved_to_echo=24563 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2799, 0, 0, 0, 1579, 1, 1563, 65, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 3061, 2022, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 1, 0, 0, 668, 34, 3129, 2188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4725, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6007, 6211, 6222, 5900, 6152, 6034, 6078, 6548] avg=6144.0000 max=6548 max_rank=7 min=5900 min_rank=3 imbalance_ratio=1.065755 max_load_reduction=42604 max_load_reduction_pct=0.866781 moved_to_echo=24812 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2746, 0, 0, 0, 1576, 9, 1567, 66, 0, 0, 0, 0, 0, 0, 0, 6107, 0, 0, 1, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 3215, 2027, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 536, 41, 3286, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4865, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5964, 6181, 6089, 6179, 6158, 6119, 6184, 6278] avg=6144.0000 max=6278 max_rank=7 min=5964 min_rank=0 imbalance_ratio=1.021810 max_load_reduction=42874 max_load_reduction_pct=0.872274 moved_to_echo=24739 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3595, 0, 0, 0, 1432, 2, 1536, 58, 0, 0, 0, 0, 0, 0, 0, 6017, 0, 0, 1, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 783, 0, 3347, 2019, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 594, 26, 3421, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4686, 1317, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6623, 6106, 6064, 6179, 6143, 6214, 6004, 5819] avg=6144.0000 max=6623 max_rank=0 min=5819 min_rank=7 imbalance_ratio=1.077962 max_load_reduction=42529 max_load_reduction_pct=0.865255 moved_to_echo=24180 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2799, 0, 0, 0, 1579, 1, 1563, 65, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 3061, 2022, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 1, 0, 0, 668, 34, 3129, 2188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4725, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6007, 6211, 6222, 5900, 6152, 6034, 6078, 6548] avg=6144.0000 max=6548 max_rank=7 min=5900 min_rank=3 imbalance_ratio=1.065755 max_load_reduction=42604 max_load_reduction_pct=0.866781 moved_to_echo=24812 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2835, 0, 0, 0, 1466, 11, 1546, 59, 0, 0, 0, 0, 0, 0, 0, 6051, 1, 0, 0, 1, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6217, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 1, 1026, 0, 3238, 2020, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 0, 0, 3, 509, 6, 3309, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 4981, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6131, 6217, 6324, 6146, 6142, 6333, 5942] avg=6144.0000 max=6333 max_rank=6 min=5917 min_rank=0 imbalance_ratio=1.030762 max_load_reduction=42819 max_load_reduction_pct=0.871155 moved_to_echo=24563 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3293, 0, 0, 0, 1651, 13, 1567, 61, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 0, 3153, 2012, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 661, 20, 3222, 2156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4652, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6585, 6220, 6008, 6090, 6123, 6061, 5955, 6110] avg=6144.0000 max=6585 max_rank=0 min=5955 min_rank=6 imbalance_ratio=1.071777 max_load_reduction=42567 max_load_reduction_pct=0.866028 moved_to_echo=24249 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2746, 0, 0, 0, 1576, 9, 1567, 66, 0, 0, 0, 0, 0, 0, 0, 6107, 0, 0, 1, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 3215, 2027, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 536, 41, 3286, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4865, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5964, 6181, 6089, 6179, 6158, 6119, 6184, 6278] avg=6144.0000 max=6278 max_rank=7 min=5964 min_rank=0 imbalance_ratio=1.021810 max_load_reduction=42874 max_load_reduction_pct=0.872274 moved_to_echo=24739 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 7040, 0, 25856, 16256, 256, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 24576, 0, 0, 0, 11776, 128, 12416, 512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 128, 128, 0, 38528, 10752, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 4736, 256, 26496, 17920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49152 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 48640, 128, 0, 128, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49152 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49152 slack_slots=49152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8101, 0, 117, 0, 0, 0, 328, 2619, 0, 338, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7345, 8181, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 5147, 0, 0, 47, 0, 0, 0, 0, 0, 0, 167, 0, 18, 0, 6, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8122, 0, 226, 0, 6, 0, 355, 2715, 0, 369, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7356, 8172, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 4825, 0, 0, 40, 0, 0, 0, 0, 0, 0, 97, 0, 27, 0, 7, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8093, 0, 146, 1, 3, 0, 315, 3026, 0, 635, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7047, 8167, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 4748, 0, 0, 57, 0, 0, 0, 0, 0, 0, 83, 0, 17, 0, 5, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49152 slack_slots=49152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8115, 0, 162, 0, 3, 0, 333, 2523, 0, 308, 0, 0, 0, 0, 437, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7362, 8181, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 5155, 0, 0, 42, 0, 0, 0, 0, 0, 0, 126, 0, 12, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 314, 1, 2, 3, 443, 2740, 0, 445, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7312, 8180, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 4432, 0, 0, 51, 0, 0, 0, 0, 0, 0, 140, 0, 10, 0, 6, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 2, 1, 1, 0, 2, 0, 0, 0, 0, 8103, 1, 177, 0, 2, 2, 356, 2645, 2, 401, 0, 2, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7222, 8177, 8188, 0, 0, 7, 0, 0, 0, 3, 0, 8183, 4835, 0, 0, 21, 1, 0, 0, 0, 0, 0, 125, 0, 39, 1, 7, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8093, 0, 146, 1, 3, 0, 315, 3026, 0, 635, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7047, 8167, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 4748, 0, 0, 57, 0, 0, 0, 0, 0, 0, 83, 0, 17, 0, 5, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8101, 0, 117, 0, 0, 0, 328, 2619, 0, 338, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7345, 8181, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 5147, 0, 0, 47, 0, 0, 0, 0, 0, 0, 167, 0, 18, 0, 6, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 98, 1, 0, 1, 271, 3057, 0, 305, 0, 0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7256, 8177, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 4732, 0, 0, 54, 0, 0, 0, 0, 0, 0, 139, 0, 29, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 2, 1, 1, 0, 2, 0, 0, 0, 0, 8103, 1, 177, 0, 2, 2, 356, 2645, 2, 401, 0, 2, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7222, 8177, 8188, 0, 0, 7, 0, 0, 0, 3, 0, 8183, 4835, 0, 0, 21, 1, 0, 0, 0, 0, 0, 125, 0, 39, 1, 7, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 314, 1, 2, 3, 443, 2740, 0, 445, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7312, 8180, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 4432, 0, 0, 51, 0, 0, 0, 0, 0, 0, 140, 0, 10, 0, 6, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8122, 0, 226, 0, 6, 0, 355, 2715, 0, 369, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7356, 8172, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 4825, 0, 0, 40, 0, 0, 0, 0, 0, 0, 97, 0, 27, 0, 7, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8116, 0, 98, 0, 3, 1, 254, 3280, 0, 359, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7242, 8185, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 4338, 0, 0, 36, 0, 0, 0, 0, 0, 0, 229, 0, 67, 0, 5, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5962, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 2619, 0, 338, 0, 0, 0, 1849, 739, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 4256, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 5147, 0, 0, 47, 0, 0, 0, 0, 1135, 200, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 18, 0, 6, 0, 3925, 2049, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6079, 5873, 6007, 6210, 6145, 6529, 6165] avg=6144.0000 max=6529 max_rank=6 min=5873 min_rank=2 imbalance_ratio=1.062663 max_load_reduction=42623 max_load_reduction_pct=0.867167 moved_to_echo=25049 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5978, 0, 226, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 2715, 0, 369, 0, 0, 0, 1849, 741, 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 4250, 1403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 4825, 0, 0, 40, 0, 0, 0, 0, 1137, 200, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 27, 0, 7, 0, 3922, 2050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6210, 6029, 6103, 6219, 6143, 6202, 6103] avg=6144.0000 max=6219 max_rank=4 min=6029 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24667 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5973, 0, 162, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 2523, 0, 308, 0, 0, 0, 1849, 740, 0, 0, 0, 0, 0, 0, 0, 437, 0, 0, 1, 0, 0, 0, 4255, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 5155, 0, 0, 42, 0, 0, 0, 0, 1138, 200, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 12, 0, 1, 0, 3926, 2050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6138, 5753, 6095, 6224, 6149, 6535, 6115] avg=6144.0000 max=6535 max_rank=6 min=5753 min_rank=2 imbalance_ratio=1.063639 max_load_reduction=42617 max_load_reduction_pct=0.867045 moved_to_echo=25023 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 1, 1, 0, 2, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5964, 1, 177, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 356, 2645, 2, 401, 0, 2, 0, 1847, 739, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 4254, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 4835, 0, 0, 21, 1, 0, 0, 0, 1116, 199, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 39, 1, 7, 0, 3923, 2049, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 5994, 6300, 6106, 6147, 6172, 6144] avg=6144.0000 max=6300 max_rank=3 min=5994 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=42852 max_load_reduction_pct=0.871826 moved_to_echo=24569 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5957, 0, 146, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 3026, 0, 635, 0, 0, 0, 1849, 738, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 0, 4248, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 4748, 0, 0, 57, 0, 0, 0, 0, 1089, 200, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 17, 0, 5, 0, 3919, 2050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6107, 6563, 6065, 5958, 6146, 6094, 6074] avg=6144.0000 max=6563 max_rank=2 min=5958 min_rank=4 imbalance_ratio=1.068197 max_load_reduction=42589 max_load_reduction_pct=0.866475 moved_to_echo=24272 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5970, 0, 98, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 271, 3057, 0, 305, 0, 0, 0, 1849, 740, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 4253, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 4732, 0, 0, 54, 0, 0, 0, 0, 1121, 200, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 29, 0, 1, 0, 3924, 2050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6069, 6223, 6189, 6135, 6144, 6107, 6143] avg=6144.0000 max=6223 max_rank=2 min=6069 min_rank=1 imbalance_ratio=1.012858 max_load_reduction=42929 max_load_reduction_pct=0.873393 moved_to_echo=24529 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 1, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6013, 0, 314, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 443, 2740, 0, 445, 0, 0, 0, 1854, 750, 0, 0, 0, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 4255, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 4432, 0, 0, 51, 0, 0, 0, 0, 1135, 202, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 10, 0, 6, 0, 3925, 2049, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6330, 6235, 6179, 6177, 6136, 5820, 6130] avg=6144.0000 max=6330 max_rank=1 min=5820 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24263 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5962, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 2619, 0, 338, 0, 0, 0, 1849, 739, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 4256, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 5147, 0, 0, 47, 0, 0, 0, 0, 1135, 200, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 18, 0, 6, 0, 3925, 2049, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6079, 5873, 6007, 6210, 6145, 6529, 6165] avg=6144.0000 max=6529 max_rank=6 min=5873 min_rank=2 imbalance_ratio=1.062663 max_load_reduction=42623 max_load_reduction_pct=0.867167 moved_to_echo=25049 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5978, 0, 226, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 2715, 0, 369, 0, 0, 0, 1849, 741, 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 4250, 1403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 4825, 0, 0, 40, 0, 0, 0, 0, 1137, 200, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 27, 0, 7, 0, 3922, 2050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6210, 6029, 6103, 6219, 6143, 6202, 6103] avg=6144.0000 max=6219 max_rank=4 min=6029 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24667 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5974, 0, 98, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 254, 3280, 0, 359, 0, 0, 0, 1848, 740, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 4258, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 4338, 0, 0, 36, 0, 0, 0, 0, 1119, 200, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 67, 0, 5, 0, 3927, 2050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6075, 6482, 6214, 6123, 6142, 5693, 6278] avg=6144.0000 max=6482 max_rank=2 min=5693 min_rank=6 imbalance_ratio=1.055013 max_load_reduction=42670 max_load_reduction_pct=0.868123 moved_to_echo=24236 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 1, 1, 0, 2, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5964, 1, 177, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 356, 2645, 2, 401, 0, 2, 0, 1847, 739, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 4254, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 4835, 0, 0, 21, 1, 0, 0, 0, 1116, 199, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 39, 1, 7, 0, 3923, 2049, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 5994, 6300, 6106, 6147, 6172, 6144] avg=6144.0000 max=6300 max_rank=3 min=5994 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=42852 max_load_reduction_pct=0.871826 moved_to_echo=24569 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 1, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6013, 0, 314, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 443, 2740, 0, 445, 0, 0, 0, 1854, 750, 0, 0, 0, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 4255, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 4432, 0, 0, 51, 0, 0, 0, 0, 1135, 202, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 10, 0, 6, 0, 3925, 2049, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6330, 6235, 6179, 6177, 6136, 5820, 6130] avg=6144.0000 max=6330 max_rank=1 min=5820 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24263 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 1152, 0, 256, 128, 128, 0, 31488, 16512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 3968, 0, 0, 128, 0, 0, 0, 34048, 11264, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2688, 22656, 128, 3200, 0, 128, 0, 14848, 6016, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[38272, 0, 0, 384, 128, 0, 0, 0, 9088, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 47872, 128, 1408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49152 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 47872, 128, 1408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49152 slack_slots=49152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7594, 495, 1326, 987, 0, 0, 12, 0, 0, 382, 0, 0, 0, 0, 7630, 0, 1, 0, 0, 0, 560, 0, 0, 696, 0, 0, 0, 0, 0, 0, 0, 0, 7544, 6911, 136, 0, 8157, 0, 1, 2120, 95, 0, 0, 0, 0, 4437] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7539, 400, 1548, 1111, 0, 0, 27, 0, 0, 369, 0, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 438, 0, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 6767, 74, 0, 8148, 15, 0, 2320, 81, 0, 0, 0, 0, 4434] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7663, 532, 1141, 1066, 0, 0, 5, 0, 0, 378, 0, 0, 0, 0, 7652, 0, 0, 0, 0, 0, 457, 0, 0, 742, 0, 0, 0, 0, 0, 0, 0, 0, 7563, 6677, 113, 0, 8150, 4, 0, 2292, 75, 0, 0, 0, 0, 4580] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7433, 444, 1033, 860, 0, 0, 1, 0, 0, 385, 0, 0, 0, 0, 7528, 0, 0, 0, 0, 0, 539, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 7752, 6605, 111, 0, 8091, 8, 0, 2675, 153, 0, 0, 0, 0, 4891] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 71, 0, 0, 0, 1, 0, 1, 0, 3, 1, 7610, 592, 1322, 981, 2, 0, 35, 0, 0, 417, 0, 0, 0, 0, 7599, 0, 1, 0, 0, 0, 478, 1, 1, 794, 0, 3, 0, 1, 0, 2, 1, 0, 7627, 6583, 109, 0, 8114, 10, 1, 2371, 118, 0, 0, 0, 0, 4300] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 553, 1341, 994, 0, 0, 22, 0, 0, 403, 0, 0, 0, 0, 7633, 0, 1, 0, 0, 0, 349, 0, 0, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 7543, 6872, 84, 0, 8088, 1, 0, 1775, 57, 0, 0, 0, 0, 4547] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7515, 488, 890, 845, 0, 0, 2, 0, 0, 417, 0, 0, 0, 0, 7690, 0, 0, 0, 0, 0, 490, 0, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 7847, 6865, 91, 0, 8128, 15, 0, 2690, 78, 0, 0, 0, 0, 4358] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7555, 387, 1486, 999, 0, 0, 34, 0, 0, 382, 0, 0, 0, 0, 7806, 0, 0, 0, 0, 0, 439, 0, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 6654, 145, 0, 8109, 4, 0, 2322, 130, 0, 0, 0, 0, 4309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 6168, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 4071, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 495, 1326, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 382, 0, 0, 0, 0, 3473, 2234, 0, 0, 0, 0, 0, 0, 5667, 0, 1, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 0, 1963, 1941, 749, 730, 0, 0, 0, 0, 0, 0, 0, 6181, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2120, 95, 0, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 6126, 6227, 6101, 6228, 6079, 6317, 5904] avg=6144.0000 max=6317 max_rank=6 min=5904 min_rank=7 imbalance_ratio=1.028158 max_load_reduction=42835 max_load_reduction_pct=0.871480 moved_to_echo=24528 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 71, 0, 0, 0, 1, 0, 1, 0, 3, 1, 7610, 592, 1322, 981, 2, 0, 35, 0, 0, 417, 0, 0, 0, 0, 7599, 0, 1, 0, 0, 0, 478, 1, 1, 794, 0, 3, 0, 1, 0, 2, 1, 0, 7627, 6583, 109, 0, 8114, 10, 1, 2371, 118, 0, 0, 0, 0, 4300] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7539, 400, 1548, 1111, 0, 0, 27, 0, 0, 369, 0, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 438, 0, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 6767, 74, 0, 8148, 15, 0, 2320, 81, 0, 0, 0, 0, 4434] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 553, 1341, 994, 0, 0, 22, 0, 0, 403, 0, 0, 0, 0, 7633, 0, 1, 0, 0, 0, 349, 0, 0, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 7543, 6872, 84, 0, 8088, 1, 0, 1775, 57, 0, 0, 0, 0, 4547] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7663, 532, 1141, 1066, 0, 0, 5, 0, 0, 378, 0, 0, 0, 0, 7652, 0, 0, 0, 0, 0, 457, 0, 0, 742, 0, 0, 0, 0, 0, 0, 0, 0, 7563, 6677, 113, 0, 8150, 4, 0, 2292, 75, 0, 0, 0, 0, 4580] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7594, 495, 1326, 987, 0, 0, 12, 0, 0, 382, 0, 0, 0, 0, 7630, 0, 1, 0, 0, 0, 560, 0, 0, 696, 0, 0, 0, 0, 0, 0, 0, 0, 7544, 6911, 136, 0, 8157, 0, 1, 2120, 95, 0, 0, 0, 0, 4437] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 6119, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 4184, 1972, 0, 0, 0, 0, 0, 0, 0, 2, 3346, 444, 1033, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 385, 0, 0, 0, 0, 3568, 2187, 0, 0, 0, 0, 0, 0, 5591, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 0, 0, 0, 1937, 1900, 826, 698, 0, 0, 0, 0, 0, 0, 0, 5907, 111, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 153, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6120, 6189, 5685, 6141, 6130, 5968, 6026, 6893] avg=6144.0000 max=6893 max_rank=7 min=5685 min_rank=2 imbalance_ratio=1.121908 max_load_reduction=42259 max_load_reduction_pct=0.859762 moved_to_echo=25017 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 1, 0, 4002, 1977, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 387, 1486, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 382, 0, 0, 0, 0, 3414, 2223, 0, 0, 0, 0, 0, 0, 5798, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 0, 0, 0, 0, 0, 2008, 1931, 728, 703, 0, 0, 0, 0, 0, 0, 0, 5951, 145, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2322, 130, 0, 0, 0, 0, 3581, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6027, 6273, 6053, 6237, 6297, 6100, 6033] avg=6144.0000 max=6297 max_rank=5 min=6027 min_rank=1 imbalance_ratio=1.024902 max_load_reduction=42855 max_load_reduction_pct=0.871887 moved_to_echo=24667 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 4236, 1980, 0, 0, 0, 0, 0, 0, 0, 1, 3383, 488, 890, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 417, 0, 0, 0, 0, 3611, 2211, 0, 0, 0, 0, 0, 0, 5712, 0, 0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 1978, 1921, 736, 725, 0, 0, 0, 0, 0, 0, 0, 6140, 91, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2690, 78, 0, 0, 0, 0, 3622, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6285, 5607, 6241, 6202, 6033, 6246, 6390] avg=6144.0000 max=6390 max_rank=7 min=5607 min_rank=2 imbalance_ratio=1.040039 max_load_reduction=42762 max_load_reduction_pct=0.869995 moved_to_echo=24871 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6163, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4081, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 3451, 532, 1141, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 378, 0, 0, 0, 0, 3482, 2254, 0, 0, 0, 0, 0, 0, 5683, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 742, 0, 0, 0, 0, 0, 0, 1969, 1958, 773, 706, 0, 0, 0, 0, 0, 0, 0, 5971, 113, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2292, 75, 0, 0, 0, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 6130, 6190, 6119, 6140, 6148, 6088, 6174] avg=6144.0000 max=6190 max_rank=2 min=6088 min_rank=6 imbalance_ratio=1.007487 max_load_reduction=42962 max_load_reduction_pct=0.874064 moved_to_echo=24550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 6137, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 1, 0, 1, 0, 4117, 1977, 0, 0, 0, 0, 0, 0, 3, 1, 3426, 592, 1322, 981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 417, 0, 0, 0, 0, 3510, 2239, 0, 0, 0, 0, 0, 0, 5644, 0, 1, 0, 0, 0, 478, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 794, 0, 3, 0, 1, 0, 2, 1955, 1945, 726, 696, 0, 0, 0, 0, 1, 0, 0, 5887, 109, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2371, 118, 0, 0, 0, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6167, 6327, 6201, 6124, 6123, 6007, 6064] avg=6144.0000 max=6327 max_rank=2 min=6007 min_rank=6 imbalance_ratio=1.029785 max_load_reduction=42825 max_load_reduction_pct=0.871277 moved_to_echo=24318 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 4236, 1980, 0, 0, 0, 0, 0, 0, 0, 1, 3383, 488, 890, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 417, 0, 0, 0, 0, 3611, 2211, 0, 0, 0, 0, 0, 0, 5712, 0, 0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 1978, 1921, 736, 725, 0, 0, 0, 0, 0, 0, 0, 6140, 91, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2690, 78, 0, 0, 0, 0, 3622, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6285, 5607, 6241, 6202, 6033, 6246, 6390] avg=6144.0000 max=6390 max_rank=7 min=5607 min_rank=2 imbalance_ratio=1.040039 max_load_reduction=42762 max_load_reduction_pct=0.869995 moved_to_echo=24871 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6163, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 4092, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 3387, 400, 1548, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 369, 0, 0, 0, 0, 3489, 2222, 0, 0, 0, 0, 0, 0, 5634, 0, 0, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 0, 0, 0, 0, 0, 0, 1956, 1930, 752, 719, 0, 0, 0, 0, 0, 0, 0, 6048, 74, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 81, 0, 0, 0, 0, 3682, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 6138, 6446, 6107, 6072, 6006, 6137, 6083] avg=6144.0000 max=6446 max_rank=2 min=6006 min_rank=5 imbalance_ratio=1.049154 max_load_reduction=42706 max_load_reduction_pct=0.868856 moved_to_echo=24298 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6117, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 4072, 1971, 0, 0, 0, 0, 0, 0, 0, 0, 3509, 553, 1341, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 403, 0, 0, 0, 0, 3471, 2293, 0, 0, 0, 0, 0, 0, 5669, 0, 1, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 0, 0, 0, 0, 0, 1964, 1992, 768, 726, 0, 0, 0, 0, 0, 0, 0, 6146, 84, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 57, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6117, 6090, 6397, 6189, 6019, 6498, 6231, 5611] avg=6144.0000 max=6498 max_rank=5 min=5611 min_rank=7 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=24359 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 1, 0, 4002, 1977, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 387, 1486, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 382, 0, 0, 0, 0, 3414, 2223, 0, 0, 0, 0, 0, 0, 5798, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 0, 0, 0, 0, 0, 2008, 1931, 728, 703, 0, 0, 0, 0, 0, 0, 0, 5951, 145, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2322, 130, 0, 0, 0, 0, 3581, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6027, 6273, 6053, 6237, 6297, 6100, 6033] avg=6144.0000 max=6297 max_rank=5 min=6027 min_rank=1 imbalance_ratio=1.024902 max_load_reduction=42855 max_load_reduction_pct=0.871887 moved_to_echo=24667 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 6168, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 4071, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 495, 1326, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 382, 0, 0, 0, 0, 3473, 2234, 0, 0, 0, 0, 0, 0, 5667, 0, 1, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 0, 1963, 1941, 749, 730, 0, 0, 0, 0, 0, 0, 0, 6181, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2120, 95, 0, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 6126, 6227, 6101, 6228, 6079, 6317, 5904] avg=6144.0000 max=6317 max_rank=6 min=5904 min_rank=7 imbalance_ratio=1.028158 max_load_reduction=42835 max_load_reduction_pct=0.871480 moved_to_echo=24528 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6163, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 4092, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 3387, 400, 1548, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 369, 0, 0, 0, 0, 3489, 2222, 0, 0, 0, 0, 0, 0, 5634, 0, 0, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 0, 0, 0, 0, 0, 0, 1956, 1930, 752, 719, 0, 0, 0, 0, 0, 0, 0, 6048, 74, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 81, 0, 0, 0, 0, 3682, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 6138, 6446, 6107, 6072, 6006, 6137, 6083] avg=6144.0000 max=6446 max_rank=2 min=6006 min_rank=5 imbalance_ratio=1.049154 max_load_reduction=42706 max_load_reduction_pct=0.868856 moved_to_echo=24298 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 6144, 0, 128, 0, 128, 0, 128, 15744, 15616, 6144, 5760, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 27392, 3968, 10112, 7936, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 18688, 896, 0, 0, 0, 0, 29824, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[256, 0, 0, 3200, 0, 0, 0, 0, 28032, 17920, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[45440, 0, 128, 0, 0, 0, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 48256, 896, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 0, 0, 0, 128, 0, 128, 0, 32896, 15872, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[256, 0, 0, 3200, 0, 0, 0, 0, 28032, 17920, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 0, 0, 0, 128, 0, 128, 0, 32896, 15872, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 48256, 896, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 4294, 0, 10, 0, 0, 0, 0, 0, 1303, 579, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 0, 0, 0, 610, 138, 7904, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 0, 0, 585, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3069, 0, 4286, 0, 0, 0, 5416, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 3807, 1, 0, 0, 0, 0, 0, 0, 1357, 528, 1, 1, 2, 1, 0, 1, 3683, 0, 1, 0, 0, 0, 485, 60, 8027, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 516, 0, 0, 1, 0, 8190, 3, 3, 0, 0, 0, 0, 2, 0, 0, 3911, 0, 4030, 0, 0, 0, 5518, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 3849, 0, 0, 0, 0, 0, 0, 0, 1519, 557, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 514, 134, 8034, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 0, 287, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3135, 0, 4070, 0, 0, 0, 5559, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 4189, 0, 0, 0, 0, 0, 0, 0, 1384, 668, 0, 0, 0, 0, 0, 0, 3017, 0, 0, 0, 0, 0, 789, 106, 7914, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 906, 0, 0, 396, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 4153, 0, 0, 0, 5545, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 3835, 0, 0, 0, 0, 0, 0, 0, 1562, 502, 0, 0, 0, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 620, 83, 8098, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 907, 0, 0, 378, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3337, 0, 4433, 0, 0, 0, 5318, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 3906, 0, 3, 0, 0, 0, 0, 0, 1303, 611, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 0, 923, 75, 8041, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 974, 0, 0, 308, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 4563, 0, 0, 0, 5572, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 4294, 0, 10, 0, 0, 0, 0, 0, 1303, 579, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 0, 0, 0, 610, 138, 7904, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 0, 0, 585, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3069, 0, 4286, 0, 0, 0, 5416, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 4101, 0, 1, 0, 0, 0, 0, 0, 1113, 639, 0, 0, 0, 0, 0, 0, 3219, 0, 0, 0, 0, 0, 477, 94, 8000, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 0, 0, 478, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2869, 0, 4884, 0, 0, 0, 5781, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 4029, 0, 0, 0, 0, 0, 0, 0, 1114, 976, 0, 0, 0, 0, 0, 0, 3578, 0, 0, 0, 0, 0, 561, 102, 7821, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 525, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3045, 0, 4618, 0, 0, 0, 5581, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 4189, 0, 0, 0, 0, 0, 0, 0, 1384, 668, 0, 0, 0, 0, 0, 0, 3017, 0, 0, 0, 0, 0, 789, 106, 7914, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 906, 0, 0, 396, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 4153, 0, 0, 0, 5545, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4294, 0, 10, 0, 0, 0, 0, 1420, 706, 0, 0, 0, 0, 0, 0, 0, 1303, 579, 0, 0, 0, 0, 0, 2352, 1773, 0, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 0, 0, 0, 610, 1547, 496, 0, 0, 0, 0, 0, 0, 138, 5988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 0, 3064, 2048, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 5707, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 3069, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6430, 6007, 6135, 6126, 6209, 6293, 6144, 5808] avg=6144.0000 max=6430 max_rank=0 min=5808 min_rank=7 imbalance_ratio=1.046549 max_load_reduction=42722 max_load_reduction_pct=0.869181 moved_to_echo=24454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 3906, 0, 3, 0, 0, 0, 0, 0, 1303, 611, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 0, 923, 75, 8041, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 974, 0, 0, 308, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 4563, 0, 0, 0, 5572, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 3849, 0, 0, 0, 0, 0, 0, 0, 1519, 557, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 514, 134, 8034, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 0, 287, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3135, 0, 4070, 0, 0, 0, 5559, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 3807, 1, 0, 0, 0, 0, 0, 0, 1357, 528, 1, 1, 2, 1, 0, 1, 3683, 0, 1, 0, 0, 0, 485, 60, 8027, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 516, 0, 0, 1, 0, 8190, 3, 3, 0, 0, 0, 0, 2, 0, 0, 3911, 0, 4030, 0, 0, 0, 5518, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3835, 0, 0, 0, 0, 0, 0, 1454, 707, 0, 0, 0, 0, 0, 0, 0, 1562, 502, 0, 0, 0, 0, 0, 2309, 1774, 0, 0, 0, 0, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 620, 1600, 508, 0, 0, 0, 0, 0, 0, 83, 6136, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 907, 0, 3009, 2048, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 5711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 3337, 0, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6147, 6422, 6220, 5964, 6089, 6144, 6170] avg=6144.0000 max=6422 max_rank=2 min=5964 min_rank=4 imbalance_ratio=1.045247 max_load_reduction=42730 max_load_reduction_pct=0.869344 moved_to_echo=24367 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3849, 0, 0, 0, 0, 0, 0, 1443, 707, 0, 0, 0, 0, 0, 0, 0, 1519, 557, 0, 0, 0, 0, 0, 2415, 1774, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 514, 1469, 504, 0, 0, 0, 0, 0, 0, 134, 6087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 3144, 2049, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 5711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 3135, 0, 2601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 6265, 6534, 6221, 6256, 5998, 6143, 5736] avg=6144.0000 max=6534 max_rank=2 min=5736 min_rank=7 imbalance_ratio=1.063477 max_load_reduction=42618 max_load_reduction_pct=0.867065 moved_to_echo=24133 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3807, 1, 0, 0, 0, 0, 0, 1442, 707, 0, 0, 0, 0, 0, 0, 0, 1357, 528, 1, 1, 2, 1, 0, 2397, 1774, 0, 0, 0, 0, 0, 0, 1, 3683, 0, 1, 0, 0, 0, 485, 1454, 504, 0, 0, 0, 0, 0, 0, 60, 6081, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 3121, 2047, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 1, 0, 5709, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 2576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5957, 6061, 6128, 6142, 6001, 6229, 6147, 6487] avg=6144.0000 max=6487 max_rank=7 min=5957 min_rank=0 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24864 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4189, 0, 0, 0, 0, 0, 0, 1421, 707, 0, 0, 0, 0, 0, 0, 0, 1384, 668, 0, 0, 0, 0, 0, 2408, 1774, 0, 0, 0, 0, 0, 0, 0, 3017, 0, 0, 0, 0, 0, 789, 1499, 497, 0, 0, 0, 0, 0, 0, 106, 5996, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 906, 0, 3137, 2048, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 5711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6317, 6234, 5802, 6103, 6091, 6107, 6144, 6354] avg=6144.0000 max=6354 max_rank=7 min=5802 min_rank=2 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24696 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3906, 0, 3, 0, 0, 0, 0, 1448, 709, 0, 0, 0, 0, 0, 0, 0, 1303, 611, 0, 0, 0, 0, 0, 2420, 1780, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 0, 923, 1649, 510, 0, 0, 0, 0, 0, 0, 75, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 974, 0, 3152, 2049, 0, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 5703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 6114, 5873, 6158, 6176, 6011, 6143, 6611] avg=6144.0000 max=6611 max_rank=7 min=5873 min_rank=2 imbalance_ratio=1.076009 max_load_reduction=42541 max_load_reduction_pct=0.865499 moved_to_echo=24941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4101, 0, 1, 0, 0, 0, 0, 1437, 707, 0, 0, 0, 0, 0, 0, 0, 1113, 639, 0, 0, 0, 0, 0, 2510, 1774, 0, 0, 0, 0, 0, 0, 0, 3219, 0, 0, 0, 0, 0, 477, 1763, 502, 0, 0, 0, 0, 0, 0, 94, 6061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 0, 3271, 2048, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 0, 5711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 2869, 0, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6246, 6036, 5961, 6155, 6431, 6189, 6144, 5990] avg=6144.0000 max=6431 max_rank=4 min=5961 min_rank=2 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4294, 0, 10, 0, 0, 0, 0, 1420, 706, 0, 0, 0, 0, 0, 0, 0, 1303, 579, 0, 0, 0, 0, 0, 2352, 1773, 0, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 0, 0, 0, 610, 1547, 496, 0, 0, 0, 0, 0, 0, 138, 5988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 0, 3064, 2048, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 5707, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 3069, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6430, 6007, 6135, 6126, 6209, 6293, 6144, 5808] avg=6144.0000 max=6430 max_rank=0 min=5808 min_rank=7 imbalance_ratio=1.046549 max_load_reduction=42722 max_load_reduction_pct=0.869181 moved_to_echo=24454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3849, 0, 0, 0, 0, 0, 0, 1443, 707, 0, 0, 0, 0, 0, 0, 0, 1519, 557, 0, 0, 0, 0, 0, 2415, 1774, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 514, 1469, 504, 0, 0, 0, 0, 0, 0, 134, 6087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 3144, 2049, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 5711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 3135, 0, 2601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 6265, 6534, 6221, 6256, 5998, 6143, 5736] avg=6144.0000 max=6534 max_rank=2 min=5736 min_rank=7 imbalance_ratio=1.063477 max_load_reduction=42618 max_load_reduction_pct=0.867065 moved_to_echo=24133 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 0, 0, 0, 0, 0, 0, 1405, 707, 0, 0, 0, 0, 0, 0, 0, 1114, 976, 0, 0, 0, 0, 0, 2424, 1774, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 0, 0, 0, 0, 561, 1667, 491, 0, 0, 0, 0, 0, 0, 102, 5925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 3157, 2049, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 5711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 3045, 0, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6288, 6297, 6027, 6024, 6236, 6143, 5996] avg=6144.0000 max=6297 max_rank=2 min=5996 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=42855 max_load_reduction_pct=0.871887 moved_to_echo=24399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3807, 1, 0, 0, 0, 0, 0, 1442, 707, 0, 0, 0, 0, 0, 0, 0, 1357, 528, 1, 1, 2, 1, 0, 2397, 1774, 0, 0, 0, 0, 0, 0, 1, 3683, 0, 1, 0, 0, 0, 485, 1454, 504, 0, 0, 0, 0, 0, 0, 60, 6081, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 3121, 2047, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 1, 0, 5709, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 2576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5957, 6061, 6128, 6142, 6001, 6229, 6147, 6487] avg=6144.0000 max=6487 max_rank=7 min=5957 min_rank=0 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24864 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3906, 0, 3, 0, 0, 0, 0, 1448, 709, 0, 0, 0, 0, 0, 0, 0, 1303, 611, 0, 0, 0, 0, 0, 2420, 1780, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 0, 923, 1649, 510, 0, 0, 0, 0, 0, 0, 75, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 974, 0, 3152, 2049, 0, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 5703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 6114, 5873, 6158, 6176, 6011, 6143, 6611] avg=6144.0000 max=6611 max_rank=7 min=5873 min_rank=2 imbalance_ratio=1.076009 max_load_reduction=42541 max_load_reduction_pct=0.865499 moved_to_echo=24941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 10752, 5120, 128, 128, 128, 128, 0, 19328, 14208, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 32128, 128, 128, 0, 0, 0, 0, 11520, 5760, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 27520, 0, 128, 0, 0, 0, 4992, 12672, 4096, 0, 0, 0, 0, 0, 0] @@ -5115,1238 +5115,1238 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 48384, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[2291, 0, 0, 8192, 0, 0, 0, 748, 0, 0, 0, 0, 0, 0, 745, 0, 0, 0, 0, 0, 0, 8191, 0, 17, 0, 0, 0, 0, 5157, 0, 0, 0, 0, 1535, 0, 7872, 0, 0, 7483, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 5766, 1021] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[2113, 0, 0, 8190, 0, 2, 0, 842, 0, 0, 0, 1, 1, 0, 948, 0, 0, 0, 0, 0, 0, 8187, 0, 10, 1, 3, 0, 0, 5155, 0, 0, 0, 0, 1664, 0, 8046, 0, 0, 7088, 0, 0, 1, 1, 225, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 5715, 954] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[2004, 0, 0, 8192, 0, 0, 0, 719, 0, 0, 0, 0, 0, 0, 1122, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 5099, 0, 0, 0, 0, 1532, 0, 8135, 0, 0, 7329, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5656, 1004] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[2088, 0, 0, 8192, 0, 0, 0, 691, 0, 0, 0, 0, 0, 0, 609, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 5487, 0, 0, 0, 0, 1474, 0, 8075, 0, 0, 7296, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6048, 884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[1909, 0, 0, 8192, 0, 0, 0, 922, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 0, 0, 5317, 0, 0, 0, 0, 1886, 0, 8042, 0, 0, 7186, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5738, 875] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[2004, 0, 0, 8192, 0, 0, 0, 719, 0, 0, 0, 0, 0, 0, 1122, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 5099, 0, 0, 0, 0, 1532, 0, 8135, 0, 0, 7329, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5656, 1004] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[2222, 0, 0, 8192, 0, 0, 0, 1111, 3, 0, 0, 0, 0, 0, 786, 0, 0, 0, 0, 0, 0, 8192, 0, 20, 0, 0, 0, 0, 4752, 0, 0, 0, 0, 1948, 0, 7972, 0, 0, 7400, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5765, 626] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[2113, 0, 0, 8190, 0, 2, 0, 842, 0, 0, 0, 1, 1, 0, 948, 0, 0, 0, 0, 0, 0, 8187, 0, 10, 1, 3, 0, 0, 5155, 0, 0, 0, 0, 1664, 0, 8046, 0, 0, 7088, 0, 0, 1, 1, 225, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 5715, 954] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[2291, 0, 0, 8192, 0, 0, 0, 748, 0, 0, 0, 0, 0, 0, 745, 0, 0, 0, 0, 0, 0, 8191, 0, 17, 0, 0, 0, 0, 5157, 0, 0, 0, 0, 1535, 0, 7872, 0, 0, 7483, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 5766, 1021] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[2088, 0, 0, 8192, 0, 0, 0, 691, 0, 0, 0, 0, 0, 0, 609, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 5487, 0, 0, 0, 0, 1474, 0, 8075, 0, 0, 7296, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6048, 884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[2020, 0, 0, 8192, 0, 0, 0, 822, 5, 0, 0, 0, 0, 0, 837, 0, 0, 0, 0, 0, 0, 8192, 0, 41, 0, 0, 0, 0, 4997, 0, 0, 0, 0, 1594, 0, 8080, 0, 0, 7577, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 5562, 1045] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[1858, 0, 0, 8192, 0, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 725, 0, 0, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 4803, 0, 0, 0, 0, 1889, 0, 7871, 0, 0, 7374, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5956, 980] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2291, 0, 0, 3216, 0, 0, 0, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 2943, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5157, 0, 0, 0, 558, 488, 0, 0, 0, 0, 0, 0, 0, 1535, 0, 0, 0, 0, 4540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 4100, 1835, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 5208, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6255, 6137, 6147, 6203, 6075, 6065, 6039, 6231] avg=6144.0000 max=6255 max_rank=0 min=6039 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=42897 max_load_reduction_pct=0.872742 moved_to_echo=24410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2088, 0, 0, 3216, 0, 0, 0, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 0, 2869, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5487, 0, 0, 0, 585, 488, 0, 0, 0, 0, 0, 0, 0, 1474, 0, 0, 0, 0, 4427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 4100, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5463, 884, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5995, 5927, 6139, 6560, 5901, 6090, 6193, 6347] avg=6144.0000 max=6560 max_rank=3 min=5901 min_rank=4 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2222, 0, 0, 3216, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 786, 0, 2910, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4752, 0, 0, 0, 558, 488, 0, 0, 0, 0, 0, 0, 0, 1948, 0, 0, 0, 0, 4490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 4100, 1858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6114, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5207, 626, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6549, 6148, 6151, 5798, 6438, 6120, 6114, 5834] avg=6144.0000 max=6549 max_rank=0 min=5798 min_rank=3 imbalance_ratio=1.065918 max_load_reduction=42603 max_load_reduction_pct=0.866760 moved_to_echo=24506 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1909, 0, 0, 3216, 0, 0, 0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 2826, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5317, 0, 0, 0, 555, 488, 0, 0, 0, 0, 0, 0, 0, 1886, 0, 0, 0, 0, 4360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 4100, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5183, 875, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6047, 5946, 6137, 6360, 6246, 6191, 6167, 6058] avg=6144.0000 max=6360 max_rank=3 min=5946 min_rank=1 imbalance_ratio=1.035156 max_load_reduction=42792 max_load_reduction_pct=0.870605 moved_to_echo=24662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2004, 0, 0, 3216, 0, 0, 0, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 2882, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5099, 0, 0, 0, 547, 488, 0, 0, 0, 0, 0, 0, 0, 1532, 0, 0, 0, 0, 4447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 4100, 1897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5109, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5939, 6453, 6139, 6134, 5979, 6157, 6238, 6113] avg=6144.0000 max=6453 max_rank=1 min=5939 min_rank=0 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2088, 0, 0, 3216, 0, 0, 0, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 0, 2869, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5487, 0, 0, 0, 585, 488, 0, 0, 0, 0, 0, 0, 0, 1474, 0, 0, 0, 0, 4427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 4100, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5463, 884, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5995, 5927, 6139, 6560, 5901, 6090, 6193, 6347] avg=6144.0000 max=6560 max_rank=3 min=5901 min_rank=4 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2113, 0, 0, 3216, 0, 2, 0, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 948, 0, 2788, 1572, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 5155, 0, 0, 0, 553, 487, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 4300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 225, 0, 1, 0, 0, 4099, 1877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5162, 954, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6173, 6185, 6138, 6199, 5964, 6204, 6171, 6118] avg=6144.0000 max=6204 max_rank=5 min=5964 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=42948 max_load_reduction_pct=0.873779 moved_to_echo=24457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1909, 0, 0, 3216, 0, 0, 0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 2826, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5317, 0, 0, 0, 555, 488, 0, 0, 0, 0, 0, 0, 0, 1886, 0, 0, 0, 0, 4360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 4100, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5183, 875, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6047, 5946, 6137, 6360, 6246, 6191, 6167, 6058] avg=6144.0000 max=6360 max_rank=3 min=5946 min_rank=1 imbalance_ratio=1.035156 max_load_reduction=42792 max_load_reduction_pct=0.870605 moved_to_echo=24662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2020, 0, 0, 3212, 0, 0, 0, 822, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 837, 0, 2985, 1578, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4997, 0, 0, 0, 544, 490, 0, 0, 0, 0, 0, 0, 0, 1594, 0, 0, 0, 0, 4592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 4103, 1885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 5018, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6054, 6282, 6165, 6031, 6186, 6173, 6195, 6066] avg=6144.0000 max=6282 max_rank=1 min=6031 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=42870 max_load_reduction_pct=0.872192 moved_to_echo=24620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2222, 0, 0, 3216, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 786, 0, 2910, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4752, 0, 0, 0, 558, 488, 0, 0, 0, 0, 0, 0, 0, 1948, 0, 0, 0, 0, 4490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 4100, 1858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6114, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5207, 626, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6549, 6148, 6151, 5798, 6438, 6120, 6114, 5834] avg=6144.0000 max=6549 max_rank=0 min=5798 min_rank=3 imbalance_ratio=1.065918 max_load_reduction=42603 max_load_reduction_pct=0.866760 moved_to_echo=24506 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2291, 0, 0, 3216, 0, 0, 0, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 2943, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5157, 0, 0, 0, 558, 488, 0, 0, 0, 0, 0, 0, 0, 1535, 0, 0, 0, 0, 4540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 4100, 1835, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 5208, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6255, 6137, 6147, 6203, 6075, 6065, 6039, 6231] avg=6144.0000 max=6255 max_rank=0 min=6039 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=42897 max_load_reduction_pct=0.872742 moved_to_echo=24410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1858, 0, 0, 3216, 0, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 2900, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4803, 0, 0, 0, 576, 488, 0, 0, 0, 0, 0, 0, 0, 1889, 0, 0, 0, 0, 4474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 4100, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5380, 980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6074, 6136, 5867, 6363, 6152, 6035, 6385] avg=6144.0000 max=6385 max_rank=7 min=5867 min_rank=3 imbalance_ratio=1.039225 max_load_reduction=42767 max_load_reduction_pct=0.870097 moved_to_echo=24935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2020, 0, 0, 3212, 0, 0, 0, 822, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 837, 0, 2985, 1578, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4997, 0, 0, 0, 544, 490, 0, 0, 0, 0, 0, 0, 0, 1594, 0, 0, 0, 0, 4592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 4103, 1885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 5018, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6054, 6282, 6165, 6031, 6186, 6173, 6195, 6066] avg=6144.0000 max=6282 max_rank=1 min=6031 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=42870 max_load_reduction_pct=0.872192 moved_to_echo=24620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 0, 6528, 0, 23168, 12672, 7040, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[16512, 0, 0, 25728, 0, 128, 0, 7040, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 1408, 0, 128, 0, 0, 32896, 14976, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 13568, 0, 0, 0, 0, 35712, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 40832, 0, 0, 0, 4480, 3968, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 128, 41856, 7424, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[1, 0, 58, 0, 7920, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3292, 0, 0, 0, 2067, 0, 8192, 0, 27, 88, 0, 0, 0, 0, 0, 519, 0, 0, 332, 8192, 0, 0, 8192, 2, 0, 0, 1539, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 65, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3674, 0, 0, 0, 1797, 0, 8192, 0, 18, 73, 0, 0, 0, 0, 1, 461, 0, 0, 248, 8192, 0, 0, 8192, 4, 0, 0, 1647, 0, 0, 0, 0, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 56, 0, 7896, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3567, 0, 0, 0, 1674, 0, 8192, 0, 9, 77, 0, 0, 0, 0, 2, 464, 0, 0, 328, 8192, 0, 0, 8192, 6, 0, 0, 1330, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 67, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3491, 0, 0, 0, 2038, 0, 8192, 0, 18, 69, 0, 0, 0, 0, 3, 405, 0, 0, 244, 8192, 0, 0, 8192, 1, 0, 0, 1526, 0, 0, 0, 0, 390, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 46, 0, 7868, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 1630, 0, 8190, 0, 8, 88, 0, 0, 0, 0, 1, 620, 0, 0, 352, 8192, 0, 0, 8192, 1, 0, 0, 1078, 0, 0, 0, 0, 400, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[9, 0, 62, 0, 7841, 0, 0, 2, 1, 1, 0, 2, 180, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 3410, 0, 0, 2, 1833, 0, 8185, 0, 37, 75, 0, 0, 0, 1, 7, 501, 1, 0, 280, 8189, 0, 4, 8189, 5, 0, 0, 1648, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 56, 0, 7896, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3567, 0, 0, 0, 1674, 0, 8192, 0, 9, 77, 0, 0, 0, 0, 2, 464, 0, 0, 328, 8192, 0, 0, 8192, 6, 0, 0, 1330, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 98, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3332, 0, 0, 0, 2179, 0, 8192, 0, 34, 75, 0, 0, 0, 0, 2, 391, 0, 0, 211, 8192, 0, 0, 8192, 20, 0, 0, 1759, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 65, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3674, 0, 0, 0, 1797, 0, 8192, 0, 18, 73, 0, 0, 0, 0, 1, 461, 0, 0, 248, 8192, 0, 0, 8192, 4, 0, 0, 1647, 0, 0, 0, 0, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 121, 0, 7825, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3401, 0, 0, 0, 1760, 0, 8192, 0, 18, 91, 0, 0, 0, 0, 0, 453, 0, 0, 322, 8192, 0, 0, 8192, 42, 0, 0, 1018, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 58, 0, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 3812, 2049, 0, 0, 0, 0, 0, 0, 0, 2624, 0, 0, 0, 0, 0, 3292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 4252, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 519, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 332, 5845, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 0, 0, 0, 0, 359, 0, 2347, 1820, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 6043, 5916, 6346, 6192, 6179, 6174, 6143] avg=6144.0000 max=6346 max_rank=3 min=5916 min_rank=2 imbalance_ratio=1.032878 max_load_reduction=42806 max_load_reduction_pct=0.870890 moved_to_echo=24688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 46, 0, 6060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 3812, 2048, 0, 0, 0, 0, 0, 0, 0, 2623, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, 0, 4251, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 1, 620, 0, 3830, 1754, 0, 0, 0, 0, 0, 0, 0, 352, 5845, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 0, 0, 0, 0, 400, 0, 2347, 1808, 109, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 6158, 6621, 5889, 6293, 6198, 5742, 6145] avg=6144.0000 max=6621 max_rank=2 min=5742 min_rank=6 imbalance_ratio=1.077637 max_load_reduction=42531 max_load_reduction_pct=0.865295 moved_to_echo=24378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[9, 0, 62, 0, 7841, 0, 0, 2, 1, 1, 0, 2, 180, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 3410, 0, 0, 2, 1833, 0, 8185, 0, 37, 75, 0, 0, 0, 1, 7, 501, 1, 0, 280, 8189, 0, 4, 8189, 5, 0, 0, 1648, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[1, 0, 58, 0, 7920, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3292, 0, 0, 0, 2067, 0, 8192, 0, 27, 88, 0, 0, 0, 0, 0, 519, 0, 0, 332, 8192, 0, 0, 8192, 2, 0, 0, 1539, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 65, 0, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 3812, 2048, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 0, 4252, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1, 461, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 248, 5845, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1647, 0, 0, 0, 0, 373, 0, 2347, 1813, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 5996, 6299, 6067, 6120, 6097, 6289, 6144] avg=6144.0000 max=6299 max_rank=2 min=5996 min_rank=1 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 67, 0, 6117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 3813, 2048, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2038, 0, 4252, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 3, 405, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 244, 5845, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1526, 0, 0, 0, 0, 390, 0, 2347, 1826, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 6049, 6116, 6308, 6062, 6090, 6198, 6145] avg=6144.0000 max=6308 max_rank=3 min=6049 min_rank=1 imbalance_ratio=1.026693 max_load_reduction=42844 max_load_reduction_pct=0.871663 moved_to_echo=24495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 56, 0, 6081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 3812, 2048, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 0, 4252, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 2, 464, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 328, 5845, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 0, 0, 0, 0, 627, 0, 2347, 1815, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6209, 6192, 5935, 6128, 6179, 6228, 6144] avg=6144.0000 max=6228 max_rank=6 min=5935 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=42924 max_load_reduction_pct=0.873291 moved_to_echo=24679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 58, 0, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 3812, 2049, 0, 0, 0, 0, 0, 0, 0, 2624, 0, 0, 0, 0, 0, 3292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 4252, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 519, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 332, 5845, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 0, 0, 0, 0, 359, 0, 2347, 1820, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 6043, 5916, 6346, 6192, 6179, 6174, 6143] avg=6144.0000 max=6346 max_rank=3 min=5916 min_rank=2 imbalance_ratio=1.032878 max_load_reduction=42806 max_load_reduction_pct=0.870890 moved_to_echo=24688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 46, 0, 6060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 3812, 2048, 0, 0, 0, 0, 0, 0, 0, 2623, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, 0, 4251, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 1, 620, 0, 3830, 1754, 0, 0, 0, 0, 0, 0, 0, 352, 5845, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 0, 0, 0, 0, 400, 0, 2347, 1808, 109, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 6158, 6621, 5889, 6293, 6198, 5742, 6145] avg=6144.0000 max=6621 max_rank=2 min=5742 min_rank=6 imbalance_ratio=1.077637 max_load_reduction=42531 max_load_reduction_pct=0.865295 moved_to_echo=24378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 121, 0, 6027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 3813, 2048, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 3401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 0, 4252, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 453, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 322, 5845, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 0, 0, 0, 0, 538, 0, 2347, 1798, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6656, 6026, 6030, 6129, 6209, 5810, 6144] avg=6144.0000 max=6656 max_rank=1 min=5810 min_rank=6 imbalance_ratio=1.083333 max_load_reduction=42496 max_load_reduction_pct=0.864583 moved_to_echo=24292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 62, 0, 6039, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2, 180, 0, 0, 0, 3812, 2047, 0, 0, 0, 0, 0, 0, 0, 2623, 0, 0, 0, 0, 0, 3410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1833, 0, 4248, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 1, 7, 501, 1, 3828, 1754, 0, 0, 0, 0, 0, 0, 0, 280, 5842, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1648, 0, 0, 0, 0, 496, 0, 2347, 1802, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6043, 6033, 6120, 6167, 6131, 6402, 6144] avg=6144.0000 max=6402 max_rank=6 min=6033 min_rank=2 imbalance_ratio=1.041992 max_load_reduction=42750 max_load_reduction_pct=0.869751 moved_to_echo=24844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 98, 0, 6068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 3817, 2049, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 0, 0, 0, 0, 3332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2179, 0, 4244, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 2, 391, 0, 3835, 1758, 0, 0, 0, 0, 0, 0, 0, 211, 5838, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 0, 0, 0, 0, 266, 0, 2354, 1817, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 5998, 5949, 6457, 6061, 6069, 6309, 6143] avg=6144.0000 max=6457 max_rank=3 min=5949 min_rank=2 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 65, 0, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 3812, 2048, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 0, 4252, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1, 461, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 248, 5845, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1647, 0, 0, 0, 0, 373, 0, 2347, 1813, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 5996, 6299, 6067, 6120, 6097, 6289, 6144] avg=6144.0000 max=6299 max_rank=2 min=5996 min_rank=1 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 121, 0, 6027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 3813, 2048, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 3401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 0, 4252, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 453, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 322, 5845, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 0, 0, 0, 0, 538, 0, 2347, 1798, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6656, 6026, 6030, 6129, 6209, 5810, 6144] avg=6144.0000 max=6656 max_rank=1 min=5810 min_rank=6 imbalance_ratio=1.083333 max_load_reduction=42496 max_load_reduction_pct=0.864583 moved_to_echo=24292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 11648, 0, 0, 0, 0, 3456, 0, 18816, 14592, 896, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 2304, 0, 0, 0, 30592, 16512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 20992, 0, 0, 0, 0, 0, 28288, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 15104, 0, 34048, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 0, 0, 0, 128, 128, 3840, 128, 30720, 14080, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 2432, 46848, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 640, 0, 48640, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3495, 0, 0, 0, 0, 0, 0, 258, 8192, 0, 0, 171, 5894, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 105, 0, 0, 0, 0, 0, 1071, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4274, 0, 8192, 0, 0, 8060, 0, 0, 0, 0, 0, 77, 6638, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3249, 0, 0, 0, 0, 1, 0, 524, 8192, 0, 0, 120, 5505, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 1559, 108, 0, 0, 0, 0, 0, 999, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 3987, 0, 8192, 0, 0, 8092, 0, 0, 0, 0, 0, 163, 6723, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3685, 0, 0, 0, 0, 2, 0, 454, 8185, 0, 0, 223, 5484, 1, 0, 3, 0, 96, 1, 0, 0, 0, 0, 0, 0, 0, 1538, 96, 0, 0, 0, 2, 0, 947, 1591, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8, 0, 3629, 0, 8190, 1, 0, 8104, 1, 0, 2, 0, 0, 142, 6761, 0, 3, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3423, 0, 0, 0, 0, 2, 0, 288, 8192, 0, 0, 242, 4925, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 94, 0, 0, 0, 0, 0, 1165, 2218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4050, 0, 8192, 0, 0, 8068, 0, 0, 0, 0, 0, 235, 6684, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3975, 0, 0, 0, 0, 2, 0, 225, 8192, 0, 0, 344, 5232, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 130, 0, 0, 0, 0, 0, 804, 1282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4345, 0, 8192, 0, 0, 8070, 0, 0, 0, 0, 0, 136, 6601, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3411, 0, 0, 0, 0, 2, 0, 366, 8192, 0, 0, 228, 5512, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 57, 0, 0, 0, 0, 0, 770, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4359, 0, 8192, 0, 0, 8144, 0, 0, 0, 0, 0, 139, 6804, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3212, 0, 0, 0, 0, 2, 0, 404, 8192, 0, 0, 245, 5104, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 120, 0, 0, 0, 0, 0, 1474, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 4015, 0, 8191, 0, 0, 7911, 1, 0, 0, 0, 0, 192, 6714, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3411, 0, 0, 0, 0, 2, 0, 366, 8192, 0, 0, 228, 5512, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 57, 0, 0, 0, 0, 0, 770, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4359, 0, 8192, 0, 0, 8144, 0, 0, 0, 0, 0, 139, 6804, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3495, 0, 0, 0, 0, 0, 0, 258, 8192, 0, 0, 171, 5894, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 105, 0, 0, 0, 0, 0, 1071, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4274, 0, 8192, 0, 0, 8060, 0, 0, 0, 0, 0, 77, 6638, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 2994, 0, 0, 0, 0, 3, 0, 226, 8192, 0, 0, 257, 5066, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 171, 0, 0, 0, 0, 0, 1088, 2183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3991, 0, 8192, 0, 0, 7856, 0, 0, 0, 0, 0, 196, 7040, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3495, 0, 0, 0, 0, 0, 1954, 750, 0, 0, 0, 0, 0, 0, 0, 258, 233, 0, 0, 171, 5894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 3981, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 105, 0, 0, 3978, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1286, 0, 0, 0, 3424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 4274, 0, 0, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6199, 6556, 6151, 6015, 5781, 6141, 6344, 5965] avg=6144.0000 max=6556 max_rank=1 min=5781 min_rank=4 imbalance_ratio=1.067057 max_load_reduction=42596 max_load_reduction_pct=0.866618 moved_to_echo=24231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3975, 0, 0, 0, 0, 2, 0, 225, 8192, 0, 0, 344, 5232, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 130, 0, 0, 0, 0, 0, 804, 1282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4345, 0, 8192, 0, 0, 8070, 0, 0, 0, 0, 0, 136, 6601, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3685, 0, 0, 0, 0, 2, 0, 454, 8185, 0, 0, 223, 5484, 1, 0, 3, 0, 96, 1, 0, 0, 0, 0, 0, 0, 0, 1538, 96, 0, 0, 0, 2, 0, 947, 1591, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8, 0, 3629, 0, 8190, 1, 0, 8104, 1, 0, 2, 0, 0, 142, 6761, 0, 3, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3423, 0, 0, 0, 0, 2, 0, 288, 8192, 0, 0, 242, 4925, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 94, 0, 0, 0, 0, 0, 1165, 2218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4050, 0, 8192, 0, 0, 8068, 0, 0, 0, 0, 0, 235, 6684, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3249, 0, 0, 0, 0, 1, 1962, 759, 0, 0, 0, 0, 0, 0, 0, 524, 233, 0, 0, 120, 5505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 3981, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1559, 108, 0, 0, 3978, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1572, 0, 0, 0, 3438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 2077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 5964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5971, 6382, 6183, 6260, 6009, 6156, 6064, 6127] avg=6144.0000 max=6382 max_rank=1 min=5971 min_rank=0 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24356 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3975, 0, 0, 0, 0, 2, 1957, 745, 0, 0, 0, 0, 0, 0, 0, 225, 233, 0, 0, 344, 5232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 3981, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 130, 0, 0, 3978, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 1282, 0, 0, 0, 3428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 4345, 0, 0, 0, 0, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 5856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6679, 6034, 6121, 6251, 5514, 6144, 6417, 5992] avg=6144.0000 max=6679 max_rank=0 min=5514 min_rank=4 imbalance_ratio=1.087077 max_load_reduction=42473 max_load_reduction_pct=0.864115 moved_to_echo=24067 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3212, 0, 0, 0, 0, 2, 1918, 758, 0, 0, 0, 0, 0, 0, 0, 404, 233, 0, 0, 245, 5104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 3981, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 120, 0, 0, 3978, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 1794, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 2031, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 5956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5890, 5986, 6120, 6175, 6629, 6157, 6047, 6148] avg=6144.0000 max=6629 max_rank=4 min=5890 min_rank=0 imbalance_ratio=1.078939 max_load_reduction=42523 max_load_reduction_pct=0.865133 moved_to_echo=24981 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3411, 0, 0, 0, 0, 2, 1975, 768, 0, 0, 0, 0, 0, 0, 0, 366, 233, 0, 0, 228, 5512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 3981, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 57, 0, 0, 3978, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 1582, 0, 0, 0, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 4359, 0, 0, 0, 0, 2090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 6036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 6339, 6142, 5943, 5812, 6136, 6449, 6175] avg=6144.0000 max=6449 max_rank=6 min=5812 min_rank=4 imbalance_ratio=1.049642 max_load_reduction=42703 max_load_reduction_pct=0.868795 moved_to_echo=24572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3495, 0, 0, 0, 0, 0, 1954, 750, 0, 0, 0, 0, 0, 0, 0, 258, 233, 0, 0, 171, 5894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 3981, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 105, 0, 0, 3978, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1286, 0, 0, 0, 3424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 4274, 0, 0, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6199, 6556, 6151, 6015, 5781, 6141, 6344, 5965] avg=6144.0000 max=6556 max_rank=1 min=5781 min_rank=4 imbalance_ratio=1.067057 max_load_reduction=42596 max_load_reduction_pct=0.866618 moved_to_echo=24231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3975, 0, 0, 0, 0, 2, 1957, 745, 0, 0, 0, 0, 0, 0, 0, 225, 233, 0, 0, 344, 5232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 3981, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 130, 0, 0, 3978, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 1282, 0, 0, 0, 3428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 4345, 0, 0, 0, 0, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 5856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6679, 6034, 6121, 6251, 5514, 6144, 6417, 5992] avg=6144.0000 max=6679 max_rank=0 min=5514 min_rank=4 imbalance_ratio=1.087077 max_load_reduction=42473 max_load_reduction_pct=0.864115 moved_to_echo=24067 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2994, 0, 0, 0, 0, 3, 1905, 795, 0, 0, 0, 0, 0, 0, 0, 226, 233, 0, 0, 257, 5066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 3981, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 171, 0, 0, 3978, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 2183, 0, 0, 0, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 6245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5697, 5782, 6166, 6308, 6608, 6142, 6008, 6441] avg=6144.0000 max=6608 max_rank=4 min=5697 min_rank=0 imbalance_ratio=1.075521 max_load_reduction=42544 max_load_reduction_pct=0.865560 moved_to_echo=25199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3685, 0, 0, 0, 0, 2, 1965, 763, 0, 0, 0, 0, 0, 0, 0, 454, 232, 0, 0, 223, 5484, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 96, 1, 0, 0, 0, 3978, 2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1538, 96, 0, 0, 3975, 616, 0, 0, 0, 0, 0, 0, 0, 2, 0, 947, 1591, 0, 0, 3, 3443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 3629, 0, 0, 1, 0, 2080, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 142, 5998, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6415, 6394, 6142, 6225, 5986, 6134, 5711, 6145] avg=6144.0000 max=6415 max_rank=0 min=5711 min_rank=6 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=23976 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3423, 0, 0, 0, 0, 2, 1960, 760, 0, 0, 0, 0, 0, 0, 0, 288, 224, 0, 0, 242, 4925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 3987, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 94, 0, 0, 3981, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 2218, 0, 0, 0, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 5924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 5679, 6127, 5975, 6813, 6142, 6112, 6159] avg=6144.0000 max=6813 max_rank=4 min=5679 min_rank=1 imbalance_ratio=1.108887 max_load_reduction=42339 max_load_reduction_pct=0.861389 moved_to_echo=25226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3212, 0, 0, 0, 0, 2, 1918, 758, 0, 0, 0, 0, 0, 0, 0, 404, 233, 0, 0, 245, 5104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 3981, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 120, 0, 0, 3978, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 1794, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 2031, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 5956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5890, 5986, 6120, 6175, 6629, 6157, 6047, 6148] avg=6144.0000 max=6629 max_rank=4 min=5890 min_rank=0 imbalance_ratio=1.078939 max_load_reduction=42523 max_load_reduction_pct=0.865133 moved_to_echo=24981 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2994, 0, 0, 0, 0, 3, 1905, 795, 0, 0, 0, 0, 0, 0, 0, 226, 233, 0, 0, 257, 5066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 3981, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 171, 0, 0, 3978, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 2183, 0, 0, 0, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 6245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5697, 5782, 6166, 6308, 6608, 6142, 6008, 6441] avg=6144.0000 max=6608 max_rank=4 min=5697 min_rank=0 imbalance_ratio=1.075521 max_load_reduction=42544 max_load_reduction_pct=0.865560 moved_to_echo=25199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 896, 128, 0, 0, 0, 31872, 16640, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 27520, 0, 0, 0, 0, 128, 15616, 6144, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 8320, 13568, 0, 0, 128, 27392, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 0, 16512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 11648, 896, 0, 0, 31872, 4992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 0, 16512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 2816, 1920, 0, 0, 1920, 42752, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 1280, 47872, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 256, 0, 49024, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True - [2026-08-26 15:39:18] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 815.6 | throughput per GPU (TFLOP/s/GPU): 156.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476898E+00 | loss scale: 1.0 | grad norm: 7.154 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2861, 568, 228, 1132, 529, 871, 685, 98, 2628, 259, 421, 1113, 32, 833, 568, 515, 624, 1144, 420, 499, 350, 251, 397, 868, 1066, 836, 274, 346, 961, 442, 1128, 1031, 1214, 586, 800, 1467, 1393, 903, 835, 601, 231, 600, 433, 393, 912, 1514, 799, 160, 179, 457, 493, 1125, 825, 1601, 563, 261, 460, 653, 1308, 603, 200, 647, 1714, 1244] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[2741, 562, 281, 1254, 440, 886, 723, 122, 2643, 329, 424, 1022, 31, 870, 516, 409, 797, 1240, 459, 547, 359, 303, 377, 955, 1011, 949, 222, 411, 906, 584, 1244, 1066, 1220, 596, 803, 1497, 1396, 736, 808, 589, 257, 631, 444, 304, 923, 1536, 787, 156, 171, 484, 471, 1118, 740, 1582, 488, 292, 418, 651, 1313, 543, 124, 652, 1570, 1169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 + [2026-08-26 16:35:21] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 824.4 | throughput per GPU (TFLOP/s/GPU): 154.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476898E+00 | loss scale: 1.0 | grad norm: 7.154 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2782, 582, 228, 1171, 377, 791, 720, 134, 2771, 266, 454, 975, 22, 974, 531, 426, 759, 1139, 425, 605, 318, 291, 361, 988, 988, 899, 287, 430, 820, 539, 1139, 928, 1256, 583, 846, 1453, 1392, 812, 778, 536, 282, 596, 418, 334, 925, 1554, 829, 193, 158, 415, 547, 1002, 695, 1898, 535, 334, 420, 676, 1296, 597, 146, 634, 1753, 1139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2704, 547, 254, 977, 310, 812, 783, 168, 2557, 274, 427, 946, 41, 941, 548, 437, 740, 1101, 443, 545, 354, 267, 387, 967, 983, 942, 314, 372, 1036, 517, 1158, 1064, 1121, 617, 820, 1439, 1342, 797, 844, 671, 329, 671, 453, 341, 997, 1529, 746, 164, 158, 438, 516, 1013, 711, 1872, 550, 373, 392, 677, 1385, 569, 207, 666, 1628, 1200] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2902, 602, 278, 1154, 392, 750, 603, 129, 2936, 230, 402, 1202, 42, 964, 470, 479, 716, 1115, 364, 536, 329, 277, 412, 1019, 1045, 923, 263, 365, 897, 410, 1121, 972, 1216, 580, 726, 1514, 1309, 795, 756, 585, 267, 690, 448, 320, 910, 1504, 825, 179, 160, 427, 493, 1079, 753, 2006, 514, 264, 397, 701, 1339, 607, 140, 624, 1663, 1062] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2931, 585, 219, 1065, 312, 786, 657, 148, 2773, 260, 475, 1026, 28, 978, 521, 386, 676, 1116, 383, 544, 342, 309, 446, 990, 906, 797, 288, 383, 886, 521, 1128, 976, 1339, 592, 853, 1397, 1457, 848, 881, 570, 262, 759, 451, 349, 895, 1431, 822, 154, 149, 463, 542, 1078, 737, 1982, 509, 267, 427, 688, 1354, 544, 166, 675, 1630, 1040] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[2741, 562, 281, 1254, 440, 886, 723, 122, 2643, 329, 424, 1022, 31, 870, 516, 409, 797, 1240, 459, 547, 359, 303, 377, 955, 1011, 949, 222, 411, 906, 584, 1244, 1066, 1220, 596, 803, 1497, 1396, 736, 808, 589, 257, 631, 444, 304, 923, 1536, 787, 156, 171, 484, 471, 1118, 740, 1582, 488, 292, 418, 651, 1313, 543, 124, 652, 1570, 1169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2886, 534, 212, 1293, 376, 798, 653, 163, 2629, 264, 349, 1144, 23, 760, 515, 457, 755, 1235, 281, 583, 363, 234, 404, 895, 1088, 947, 267, 383, 893, 436, 1239, 1076, 1262, 655, 860, 1448, 1331, 764, 803, 647, 317, 673, 456, 383, 944, 1524, 856, 209, 146, 439, 543, 1115, 730, 1601, 504, 304, 432, 665, 1273, 609, 127, 590, 1596, 1211] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2861, 568, 228, 1132, 529, 871, 685, 98, 2628, 259, 421, 1113, 32, 833, 568, 515, 624, 1144, 420, 499, 350, 251, 397, 868, 1066, 836, 274, 346, 961, 442, 1128, 1031, 1214, 586, 800, 1467, 1393, 903, 835, 601, 231, 600, 433, 393, 912, 1514, 799, 160, 179, 457, 493, 1125, 825, 1601, 563, 261, 460, 653, 1308, 603, 200, 647, 1714, 1244] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2650, 579, 279, 1113, 409, 795, 770, 164, 2693, 277, 417, 1087, 55, 800, 560, 473, 871, 1142, 390, 539, 405, 298, 408, 966, 1087, 916, 331, 416, 1084, 431, 1133, 1060, 1121, 663, 911, 1347, 1222, 848, 914, 580, 268, 660, 446, 315, 959, 1490, 809, 209, 163, 414, 547, 1125, 619, 1569, 514, 356, 382, 811, 1272, 561, 184, 606, 1541, 1128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2902, 602, 278, 1154, 392, 750, 603, 129, 2936, 230, 402, 1202, 42, 964, 470, 479, 716, 1115, 364, 536, 329, 277, 412, 1019, 1045, 923, 263, 365, 897, 410, 1121, 972, 1216, 580, 726, 1514, 1309, 795, 756, 585, 267, 690, 448, 320, 910, 1504, 825, 179, 160, 427, 493, 1079, 753, 2006, 514, 264, 397, 701, 1339, 607, 140, 624, 1663, 1062] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2704, 547, 254, 977, 310, 812, 783, 168, 2557, 274, 427, 946, 41, 941, 548, 437, 740, 1101, 443, 545, 354, 267, 387, 967, 983, 942, 314, 372, 1036, 517, 1158, 1064, 1121, 617, 820, 1439, 1342, 797, 844, 671, 329, 671, 453, 341, 997, 1529, 746, 164, 158, 438, 516, 1013, 711, 1872, 550, 373, 392, 677, 1385, 569, 207, 666, 1628, 1200] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2119, 582, 228, 1171, 377, 791, 720, 134, 0, 0, 0, 0, 0, 0, 0, 0, 2550, 266, 454, 975, 22, 974, 531, 426, 0, 0, 0, 0, 0, 0, 0, 0, 759, 1139, 425, 605, 318, 291, 361, 988, 1323, 0, 0, 0, 0, 0, 0, 0, 988, 899, 287, 430, 820, 539, 1089, 928, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 583, 846, 0, 1286, 812, 778, 536, 0, 0, 0, 0, 0, 0, 0, 0, 282, 596, 418, 334, 925, 1554, 829, 193, 663, 205, 130, 0, 0, 0, 0, 0, 158, 415, 547, 1002, 695, 1898, 535, 334, 268, 221, 106, 50, 0, 0, 0, 0, 420, 676, 1296, 597, 146, 634, 1280, 1139, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6198, 6209, 5980, 6097, 6129, 6229, 6188] avg=6144.0000 max=6229 max_rank=6 min=5980 min_rank=3 imbalance_ratio=1.013835 max_load_reduction=42923 max_load_reduction_pct=0.873271 moved_to_echo=24643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2018, 579, 279, 1113, 409, 795, 770, 164, 0, 0, 0, 0, 0, 0, 0, 0, 2479, 277, 417, 1087, 55, 800, 560, 473, 0, 0, 0, 0, 0, 0, 0, 0, 871, 1142, 390, 539, 405, 298, 408, 966, 1227, 0, 0, 0, 0, 0, 0, 0, 1087, 916, 331, 416, 1084, 431, 1084, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 663, 911, 0, 1129, 848, 914, 580, 0, 0, 0, 0, 0, 0, 0, 0, 268, 660, 446, 315, 959, 1490, 809, 209, 632, 180, 120, 0, 0, 0, 0, 0, 163, 414, 547, 1125, 619, 1569, 514, 356, 236, 214, 93, 49, 0, 0, 0, 0, 382, 811, 1272, 561, 184, 606, 1125, 1128, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6148, 6246, 6409, 6166, 6088, 5899, 6069] avg=6144.0000 max=6409 max_rank=3 min=5899 min_rank=6 imbalance_ratio=1.043132 max_load_reduction=42743 max_load_reduction_pct=0.869609 moved_to_echo=24222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2060, 547, 254, 977, 310, 812, 783, 168, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 274, 427, 946, 41, 941, 548, 437, 0, 0, 0, 0, 0, 0, 0, 0, 740, 1101, 443, 545, 354, 267, 387, 967, 1310, 0, 0, 0, 0, 0, 0, 0, 983, 942, 314, 372, 1036, 517, 1108, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 617, 820, 0, 1240, 797, 844, 671, 0, 0, 0, 0, 0, 0, 0, 0, 329, 671, 453, 341, 997, 1529, 746, 164, 644, 191, 129, 0, 0, 0, 0, 0, 158, 438, 516, 1013, 711, 1872, 550, 373, 249, 204, 102, 50, 0, 0, 0, 0, 392, 677, 1385, 569, 207, 666, 1188, 1200, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5911, 5967, 6114, 6336, 6110, 6194, 6236, 6284] avg=6144.0000 max=6336 max_rank=3 min=5911 min_rank=0 imbalance_ratio=1.031250 max_load_reduction=42816 max_load_reduction_pct=0.871094 moved_to_echo=24824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2088, 562, 281, 1254, 440, 886, 723, 122, 0, 0, 0, 0, 0, 0, 0, 0, 2433, 329, 424, 1022, 31, 870, 516, 409, 0, 0, 0, 0, 0, 0, 0, 0, 797, 1240, 459, 547, 359, 303, 377, 955, 1364, 0, 0, 0, 0, 0, 0, 0, 1011, 949, 222, 411, 906, 584, 1190, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1220, 596, 803, 0, 1290, 736, 808, 589, 0, 0, 0, 0, 0, 0, 0, 0, 257, 631, 444, 304, 923, 1536, 787, 156, 653, 184, 133, 0, 0, 0, 0, 0, 171, 484, 471, 1118, 740, 1582, 488, 292, 240, 210, 106, 54, 0, 0, 0, 0, 418, 651, 1313, 543, 124, 652, 1146, 1169, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6356, 6034, 6401, 6339, 6042, 6008, 5956, 6016] avg=6144.0000 max=6401 max_rank=2 min=5956 min_rank=6 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24022 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2179, 568, 228, 1132, 529, 871, 685, 98, 0, 0, 0, 0, 0, 0, 0, 0, 2419, 259, 421, 1113, 32, 833, 568, 515, 0, 0, 0, 0, 0, 0, 0, 0, 624, 1144, 420, 499, 350, 251, 397, 868, 1335, 0, 0, 0, 0, 0, 0, 0, 1066, 836, 274, 346, 961, 442, 1079, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 586, 800, 0, 1287, 903, 835, 601, 0, 0, 0, 0, 0, 0, 0, 0, 231, 600, 433, 393, 912, 1514, 799, 160, 682, 201, 132, 0, 0, 0, 0, 0, 179, 457, 493, 1125, 825, 1601, 563, 261, 262, 209, 106, 49, 0, 0, 0, 0, 460, 653, 1308, 603, 200, 647, 1251, 1244, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6290, 6160, 5888, 6035, 6226, 6057, 6130, 6366] avg=6144.0000 max=6366 max_rank=7 min=5888 min_rank=2 imbalance_ratio=1.036133 max_load_reduction=42786 max_load_reduction_pct=0.870483 moved_to_echo=24779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2198, 534, 212, 1293, 376, 798, 653, 163, 0, 0, 0, 0, 0, 0, 0, 0, 2420, 264, 349, 1144, 23, 760, 515, 457, 0, 0, 0, 0, 0, 0, 0, 0, 755, 1235, 281, 583, 363, 234, 404, 895, 1318, 0, 0, 0, 0, 0, 0, 0, 1088, 947, 267, 383, 893, 436, 1185, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 655, 860, 0, 1229, 764, 803, 647, 0, 0, 0, 0, 0, 0, 0, 0, 317, 673, 456, 383, 944, 1524, 856, 209, 688, 187, 130, 0, 0, 0, 0, 0, 146, 439, 543, 1115, 730, 1601, 504, 304, 244, 209, 102, 54, 0, 0, 0, 0, 432, 665, 1273, 609, 127, 590, 1165, 1211, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6227, 5932, 6068, 6275, 6220, 6367, 5991, 6072] avg=6144.0000 max=6367 max_rank=5 min=5932 min_rank=1 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2210, 602, 278, 1154, 392, 750, 603, 129, 0, 0, 0, 0, 0, 0, 0, 0, 2702, 230, 402, 1202, 42, 964, 470, 479, 0, 0, 0, 0, 0, 0, 0, 0, 716, 1115, 364, 536, 329, 277, 412, 1019, 1379, 0, 0, 0, 0, 0, 0, 0, 1045, 923, 263, 365, 897, 410, 1072, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 580, 726, 0, 1209, 795, 756, 585, 0, 0, 0, 0, 0, 0, 0, 0, 267, 690, 448, 320, 910, 1504, 825, 179, 692, 195, 135, 0, 0, 0, 0, 0, 160, 427, 493, 1079, 753, 2006, 514, 264, 255, 234, 100, 49, 0, 0, 0, 0, 397, 701, 1339, 607, 140, 624, 1213, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 6491, 6147, 5947, 5867, 6165, 6334, 6083] avg=6144.0000 max=6491 max_rank=1 min=5867 min_rank=4 imbalance_ratio=1.056478 max_load_reduction=42661 max_load_reduction_pct=0.867940 moved_to_echo=24449 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2018, 579, 279, 1113, 409, 795, 770, 164, 0, 0, 0, 0, 0, 0, 0, 0, 2479, 277, 417, 1087, 55, 800, 560, 473, 0, 0, 0, 0, 0, 0, 0, 0, 871, 1142, 390, 539, 405, 298, 408, 966, 1227, 0, 0, 0, 0, 0, 0, 0, 1087, 916, 331, 416, 1084, 431, 1084, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 663, 911, 0, 1129, 848, 914, 580, 0, 0, 0, 0, 0, 0, 0, 0, 268, 660, 446, 315, 959, 1490, 809, 209, 632, 180, 120, 0, 0, 0, 0, 0, 163, 414, 547, 1125, 619, 1569, 514, 356, 236, 214, 93, 49, 0, 0, 0, 0, 382, 811, 1272, 561, 184, 606, 1125, 1128, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6148, 6246, 6409, 6166, 6088, 5899, 6069] avg=6144.0000 max=6409 max_rank=3 min=5899 min_rank=6 imbalance_ratio=1.043132 max_load_reduction=42743 max_load_reduction_pct=0.869609 moved_to_echo=24222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2060, 547, 254, 977, 310, 812, 783, 168, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 274, 427, 946, 41, 941, 548, 437, 0, 0, 0, 0, 0, 0, 0, 0, 740, 1101, 443, 545, 354, 267, 387, 967, 1310, 0, 0, 0, 0, 0, 0, 0, 983, 942, 314, 372, 1036, 517, 1108, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 617, 820, 0, 1240, 797, 844, 671, 0, 0, 0, 0, 0, 0, 0, 0, 329, 671, 453, 341, 997, 1529, 746, 164, 644, 191, 129, 0, 0, 0, 0, 0, 158, 438, 516, 1013, 711, 1872, 550, 373, 249, 204, 102, 50, 0, 0, 0, 0, 392, 677, 1385, 569, 207, 666, 1188, 1200, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5911, 5967, 6114, 6336, 6110, 6194, 6236, 6284] avg=6144.0000 max=6336 max_rank=3 min=5911 min_rank=0 imbalance_ratio=1.031250 max_load_reduction=42816 max_load_reduction_pct=0.871094 moved_to_echo=24824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2229, 585, 219, 1065, 312, 786, 657, 148, 0, 0, 0, 0, 0, 0, 0, 0, 2548, 260, 475, 1026, 28, 978, 521, 386, 0, 0, 0, 0, 0, 0, 0, 0, 676, 1116, 383, 544, 342, 309, 446, 990, 1273, 0, 0, 0, 0, 0, 0, 0, 906, 797, 288, 383, 886, 521, 1074, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 592, 853, 0, 1341, 848, 881, 570, 0, 0, 0, 0, 0, 0, 0, 0, 262, 759, 451, 349, 895, 1431, 822, 154, 702, 195, 124, 0, 0, 0, 0, 0, 149, 463, 542, 1078, 737, 1982, 509, 267, 255, 225, 116, 54, 0, 0, 0, 0, 427, 688, 1354, 544, 166, 675, 1180, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6001, 6222, 6079, 5831, 6424, 6144, 6377, 6074] avg=6144.0000 max=6424 max_rank=4 min=5831 min_rank=3 imbalance_ratio=1.045573 max_load_reduction=42728 max_load_reduction_pct=0.869303 moved_to_echo=25019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2088, 562, 281, 1254, 440, 886, 723, 122, 0, 0, 0, 0, 0, 0, 0, 0, 2433, 329, 424, 1022, 31, 870, 516, 409, 0, 0, 0, 0, 0, 0, 0, 0, 797, 1240, 459, 547, 359, 303, 377, 955, 1364, 0, 0, 0, 0, 0, 0, 0, 1011, 949, 222, 411, 906, 584, 1190, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1220, 596, 803, 0, 1290, 736, 808, 589, 0, 0, 0, 0, 0, 0, 0, 0, 257, 631, 444, 304, 923, 1536, 787, 156, 653, 184, 133, 0, 0, 0, 0, 0, 171, 484, 471, 1118, 740, 1582, 488, 292, 240, 210, 106, 54, 0, 0, 0, 0, 418, 651, 1313, 543, 124, 652, 1146, 1169, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6356, 6034, 6401, 6339, 6042, 6008, 5956, 6016] avg=6144.0000 max=6401 max_rank=2 min=5956 min_rank=6 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24022 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2198, 534, 212, 1293, 376, 798, 653, 163, 0, 0, 0, 0, 0, 0, 0, 0, 2420, 264, 349, 1144, 23, 760, 515, 457, 0, 0, 0, 0, 0, 0, 0, 0, 755, 1235, 281, 583, 363, 234, 404, 895, 1318, 0, 0, 0, 0, 0, 0, 0, 1088, 947, 267, 383, 893, 436, 1185, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 655, 860, 0, 1229, 764, 803, 647, 0, 0, 0, 0, 0, 0, 0, 0, 317, 673, 456, 383, 944, 1524, 856, 209, 688, 187, 130, 0, 0, 0, 0, 0, 146, 439, 543, 1115, 730, 1601, 504, 304, 244, 209, 102, 54, 0, 0, 0, 0, 432, 665, 1273, 609, 127, 590, 1165, 1211, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6227, 5932, 6068, 6275, 6220, 6367, 5991, 6072] avg=6144.0000 max=6367 max_rank=5 min=5932 min_rank=1 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19968, 2176, 3456, 8576, 384, 7168, 4352, 3584, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2304, 5376, 3584, 2816, 7552, 12160, 6528, 1536, 5376, 1664, 1152, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[17152, 4608, 2048, 9216, 3200, 6528, 5632, 1152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 9344, 3200, 4480, 2944, 2304, 3200, 7680, 10624, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 5632, 10624, 4736, 1408, 5120, 9600, 9216, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8192, 7296, 2304, 3200, 7552, 3968, 8960, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[17152, 4608, 2048, 9216, 3200, 6528, 5632, 1152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[1408, 3584, 4224, 8704, 5888, 14208, 4224, 2560, 2048, 1792, 896, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2304, 5376, 3584, 2816, 7552, 12160, 6528, 1536, 5376, 1664, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8192, 7296, 2304, 3200, 7552, 3968, 8960, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 5632, 10624, 4736, 1408, 5120, 9600, 9216, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9856, 4992, 6656, 0, 10112, 6528, 6656, 4864, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[320, 864, 353, 551, 2288, 119, 4639, 325, 992, 600, 351, 1173, 555, 1049, 41, 1363, 531, 1003, 1161, 1838, 11, 614, 17, 318, 94, 705, 904, 2454, 31, 640, 210, 2407, 96, 38, 522, 155, 518, 13, 369, 137, 651, 606, 296, 871, 204, 155, 285, 1379, 2722, 224, 1573, 373, 487, 150, 869, 104, 875, 955, 1608, 1210, 576, 533, 1168, 909] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[374, 935, 344, 643, 2446, 83, 4633, 316, 1109, 637, 331, 942, 511, 1053, 44, 1509, 579, 991, 1245, 1930, 33, 607, 13, 337, 114, 803, 903, 2316, 46, 555, 259, 2318, 111, 38, 510, 152, 532, 14, 294, 130, 744, 599, 289, 814, 208, 151, 315, 1354, 2546, 240, 1786, 376, 465, 172, 835, 148, 867, 1025, 1374, 989, 569, 550, 1112, 884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[324, 898, 370, 576, 2344, 103, 4316, 323, 1091, 631, 374, 981, 536, 1156, 68, 1412, 596, 1023, 890, 1844, 40, 628, 26, 380, 98, 767, 985, 2285, 69, 675, 282, 2368, 134, 54, 559, 202, 594, 26, 335, 145, 886, 585, 316, 829, 190, 160, 297, 1245, 2433, 248, 1769, 418, 427, 168, 910, 156, 842, 1025, 1326, 1263, 649, 539, 1049, 914] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[360, 915, 363, 661, 2422, 90, 4766, 287, 1328, 603, 330, 955, 554, 1058, 51, 1349, 580, 1094, 1253, 1898, 19, 533, 13, 377, 110, 734, 897, 1962, 39, 578, 242, 2566, 45, 17, 676, 214, 502, 23, 265, 149, 446, 527, 320, 825, 225, 158, 263, 1380, 2076, 221, 1460, 550, 477, 161, 818, 116, 1008, 1015, 1446, 1452, 712, 527, 1093, 998] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[299, 873, 397, 650, 2240, 92, 4696, 311, 1156, 670, 286, 1011, 519, 1184, 78, 1390, 504, 992, 819, 1741, 29, 561, 18, 363, 134, 650, 839, 2100, 58, 650, 214, 2473, 121, 27, 580, 266, 486, 27, 417, 128, 766, 587, 288, 921, 234, 147, 284, 1332, 2224, 221, 1880, 500, 487, 153, 884, 107, 820, 978, 1537, 1497, 577, 569, 1119, 991] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[320, 864, 353, 551, 2288, 119, 4639, 325, 992, 600, 351, 1173, 555, 1049, 41, 1363, 531, 1003, 1161, 1838, 11, 614, 17, 318, 94, 705, 904, 2454, 31, 640, 210, 2407, 96, 38, 522, 155, 518, 13, 369, 137, 651, 606, 296, 871, 204, 155, 285, 1379, 2722, 224, 1573, 373, 487, 150, 869, 104, 875, 955, 1608, 1210, 576, 533, 1168, 909] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[348, 886, 296, 639, 2380, 84, 4530, 326, 986, 551, 325, 1119, 542, 1038, 63, 1525, 555, 1000, 1294, 2023, 22, 652, 28, 380, 128, 729, 927, 2184, 43, 642, 248, 2341, 121, 21, 501, 183, 513, 18, 380, 121, 848, 529, 289, 814, 218, 151, 274, 1177, 2611, 241, 1904, 390, 482, 156, 853, 133, 955, 1013, 1306, 969, 577, 496, 1078, 996] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[314, 952, 353, 620, 2315, 77, 4498, 321, 1257, 646, 396, 1038, 554, 1114, 57, 1378, 562, 951, 964, 1708, 43, 565, 14, 392, 121, 797, 1021, 2194, 50, 661, 230, 2240, 139, 33, 603, 234, 561, 27, 400, 153, 709, 575, 309, 909, 257, 150, 326, 1341, 2115, 265, 1770, 363, 439, 187, 950, 139, 799, 991, 1409, 1309, 721, 579, 988, 999] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[327, 823, 344, 660, 2230, 119, 4431, 269, 1238, 670, 256, 1062, 570, 1074, 50, 1417, 546, 944, 867, 2023, 38, 550, 22, 419, 99, 718, 960, 2418, 29, 600, 217, 2516, 95, 31, 568, 225, 537, 28, 360, 136, 799, 537, 324, 851, 172, 190, 292, 1324, 2287, 231, 1618, 432, 508, 185, 985, 127, 930, 936, 1460, 1309, 577, 561, 1021, 1010] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[374, 935, 344, 643, 2446, 83, 4633, 316, 1109, 637, 331, 942, 511, 1053, 44, 1509, 579, 991, 1245, 1930, 33, 607, 13, 337, 114, 803, 903, 2316, 46, 555, 259, 2318, 111, 38, 510, 152, 532, 14, 294, 130, 744, 599, 289, 814, 208, 151, 315, 1354, 2546, 240, 1786, 376, 465, 172, 835, 148, 867, 1025, 1374, 989, 569, 550, 1112, 884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[348, 886, 296, 639, 2380, 84, 4530, 326, 986, 551, 325, 1119, 542, 1038, 63, 1525, 555, 1000, 1294, 2023, 22, 652, 28, 380, 128, 729, 927, 2184, 43, 642, 248, 2341, 121, 21, 501, 183, 513, 18, 380, 121, 848, 529, 289, 814, 218, 151, 274, 1177, 2611, 241, 1904, 390, 482, 156, 853, 133, 955, 1013, 1306, 969, 577, 496, 1078, 996] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[320, 864, 353, 551, 2288, 119, 1222, 325, 0, 0, 0, 0, 0, 0, 0, 0, 992, 600, 351, 1173, 555, 1049, 41, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 531, 1003, 1161, 1838, 11, 614, 17, 318, 308, 184, 96, 82, 0, 0, 0, 0, 94, 705, 904, 2454, 31, 640, 210, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 96, 38, 522, 155, 518, 13, 369, 137, 3417, 931, 0, 0, 0, 0, 0, 0, 651, 606, 296, 871, 204, 155, 285, 1379, 1105, 677, 0, 0, 0, 0, 0, 0, 2414, 224, 1573, 373, 487, 150, 869, 104, 0, 0, 0, 0, 0, 0, 0, 0, 875, 955, 0, 1026, 576, 533, 1168, 909, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6042, 6028, 6163, 6258, 6196, 6229, 6194, 6042] avg=6144.0000 max=6258 max_rank=3 min=6028 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=42894 max_load_reduction_pct=0.872681 moved_to_echo=24661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[299, 873, 397, 650, 2240, 92, 4696, 311, 1156, 670, 286, 1011, 519, 1184, 78, 1390, 504, 992, 819, 1741, 29, 561, 18, 363, 134, 650, 839, 2100, 58, 650, 214, 2473, 121, 27, 580, 266, 486, 27, 417, 128, 766, 587, 288, 921, 234, 147, 284, 1332, 2224, 221, 1880, 500, 487, 153, 884, 107, 820, 978, 1537, 1497, 577, 569, 1119, 991] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[324, 898, 370, 576, 2344, 103, 1137, 323, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 631, 374, 981, 536, 1156, 68, 1312, 0, 0, 0, 0, 0, 0, 0, 0, 596, 1023, 890, 1844, 40, 628, 26, 380, 275, 192, 100, 81, 0, 0, 0, 0, 98, 767, 985, 2285, 69, 675, 282, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 134, 54, 559, 202, 594, 26, 335, 145, 3179, 768, 0, 0, 0, 0, 0, 0, 886, 585, 316, 829, 190, 160, 297, 1245, 1087, 558, 0, 0, 0, 0, 0, 0, 2158, 248, 1769, 418, 427, 168, 910, 156, 0, 0, 0, 0, 0, 0, 0, 0, 842, 1025, 0, 1071, 649, 539, 1049, 914, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 6149, 6075, 6361, 5996, 6153, 6254, 6089] avg=6144.0000 max=6361 max_rank=3 min=5996 min_rank=4 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[360, 915, 363, 661, 2422, 90, 1255, 287, 0, 0, 0, 0, 0, 0, 0, 0, 1328, 603, 330, 955, 554, 1058, 51, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 580, 1094, 1253, 1898, 19, 533, 13, 377, 235, 220, 95, 88, 0, 0, 0, 0, 110, 734, 897, 1962, 39, 578, 242, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 45, 17, 676, 214, 502, 23, 265, 149, 3511, 837, 0, 0, 0, 0, 0, 0, 446, 527, 320, 825, 225, 158, 263, 1380, 1178, 609, 0, 0, 0, 0, 0, 0, 1841, 221, 1460, 550, 477, 161, 818, 116, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1015, 0, 1232, 712, 527, 1093, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6353, 6133, 6405, 5862, 6239, 5931, 5644, 6585] avg=6144.0000 max=6585 max_rank=7 min=5644 min_rank=6 imbalance_ratio=1.071777 max_load_reduction=42567 max_load_reduction_pct=0.866028 moved_to_echo=24399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[320, 864, 353, 551, 2288, 119, 1222, 325, 0, 0, 0, 0, 0, 0, 0, 0, 992, 600, 351, 1173, 555, 1049, 41, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 531, 1003, 1161, 1838, 11, 614, 17, 318, 308, 184, 96, 82, 0, 0, 0, 0, 94, 705, 904, 2454, 31, 640, 210, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 96, 38, 522, 155, 518, 13, 369, 137, 3417, 931, 0, 0, 0, 0, 0, 0, 651, 606, 296, 871, 204, 155, 285, 1379, 1105, 677, 0, 0, 0, 0, 0, 0, 2414, 224, 1573, 373, 487, 150, 869, 104, 0, 0, 0, 0, 0, 0, 0, 0, 875, 955, 0, 1026, 576, 533, 1168, 909, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6042, 6028, 6163, 6258, 6196, 6229, 6194, 6042] avg=6144.0000 max=6258 max_rank=3 min=6028 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=42894 max_load_reduction_pct=0.872681 moved_to_echo=24661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[314, 952, 353, 620, 2315, 77, 1184, 321, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 646, 396, 1038, 554, 1114, 57, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 562, 951, 964, 1708, 43, 565, 14, 392, 239, 199, 97, 77, 0, 0, 0, 0, 121, 797, 1021, 2194, 50, 661, 230, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 139, 33, 603, 234, 561, 27, 400, 153, 3314, 817, 0, 0, 0, 0, 0, 0, 709, 575, 309, 909, 257, 150, 326, 1341, 1028, 592, 0, 0, 0, 0, 0, 0, 1876, 265, 1770, 363, 439, 187, 950, 139, 0, 0, 0, 0, 0, 0, 0, 0, 799, 991, 0, 1110, 721, 579, 988, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6343, 5811, 6209, 6281, 6196, 5989, 6187] avg=6144.0000 max=6343 max_rank=1 min=5811 min_rank=2 imbalance_ratio=1.032389 max_load_reduction=42809 max_load_reduction_pct=0.870951 moved_to_echo=24653 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[348, 886, 296, 639, 2380, 84, 1193, 326, 0, 0, 0, 0, 0, 0, 0, 0, 986, 551, 325, 1119, 542, 1038, 63, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 555, 1000, 1294, 2023, 22, 652, 28, 380, 295, 147, 108, 80, 0, 0, 0, 0, 128, 729, 927, 2184, 43, 642, 248, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 121, 21, 501, 183, 513, 18, 380, 121, 3337, 757, 0, 0, 0, 0, 0, 0, 848, 529, 289, 814, 218, 151, 274, 1177, 1075, 549, 0, 0, 0, 0, 0, 0, 2316, 241, 1904, 390, 482, 156, 853, 133, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1013, 0, 822, 577, 496, 1078, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6041, 6584, 6087, 5952, 5924, 6475, 5937] avg=6144.0000 max=6584 max_rank=2 min=5924 min_rank=5 imbalance_ratio=1.071615 max_load_reduction=42568 max_load_reduction_pct=0.866048 moved_to_echo=24288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[360, 915, 363, 661, 2422, 90, 1255, 287, 0, 0, 0, 0, 0, 0, 0, 0, 1328, 603, 330, 955, 554, 1058, 51, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 580, 1094, 1253, 1898, 19, 533, 13, 377, 235, 220, 95, 88, 0, 0, 0, 0, 110, 734, 897, 1962, 39, 578, 242, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 45, 17, 676, 214, 502, 23, 265, 149, 3511, 837, 0, 0, 0, 0, 0, 0, 446, 527, 320, 825, 225, 158, 263, 1380, 1178, 609, 0, 0, 0, 0, 0, 0, 1841, 221, 1460, 550, 477, 161, 818, 116, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1015, 0, 1232, 712, 527, 1093, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6353, 6133, 6405, 5862, 6239, 5931, 5644, 6585] avg=6144.0000 max=6585 max_rank=7 min=5644 min_rank=6 imbalance_ratio=1.071777 max_load_reduction=42567 max_load_reduction_pct=0.866028 moved_to_echo=24399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[299, 873, 397, 650, 2240, 92, 1237, 311, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 670, 286, 1011, 519, 1184, 78, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 504, 992, 819, 1741, 29, 561, 18, 363, 251, 227, 98, 85, 0, 0, 0, 0, 134, 650, 839, 2100, 58, 650, 214, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 121, 27, 580, 266, 486, 27, 417, 128, 3459, 890, 0, 0, 0, 0, 0, 0, 766, 587, 288, 921, 234, 147, 284, 1332, 1135, 647, 0, 0, 0, 0, 0, 0, 1973, 221, 1880, 500, 487, 153, 884, 107, 0, 0, 0, 0, 0, 0, 0, 0, 820, 978, 0, 1270, 577, 569, 1119, 991, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6099, 6196, 5688, 5898, 6401, 6341, 6205, 6324] avg=6144.0000 max=6401 max_rank=4 min=5688 min_rank=2 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=25271 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[327, 823, 344, 660, 2230, 119, 1167, 269, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 670, 256, 1062, 570, 1074, 50, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 546, 944, 867, 2023, 38, 550, 22, 419, 258, 199, 100, 86, 0, 0, 0, 0, 99, 718, 960, 2418, 29, 600, 217, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 95, 31, 568, 225, 537, 28, 360, 136, 3264, 847, 0, 0, 0, 0, 0, 0, 799, 537, 324, 851, 172, 190, 292, 1324, 1155, 613, 0, 0, 0, 0, 0, 0, 2029, 231, 1618, 432, 508, 185, 985, 127, 0, 0, 0, 0, 0, 0, 0, 0, 930, 936, 0, 1110, 577, 561, 1021, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5939, 6237, 6052, 6316, 6091, 6257, 6115, 6145] avg=6144.0000 max=6316 max_rank=3 min=5939 min_rank=0 imbalance_ratio=1.027995 max_load_reduction=42836 max_load_reduction_pct=0.871501 moved_to_echo=24608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[374, 935, 344, 643, 2446, 83, 1215, 316, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 637, 331, 942, 511, 1053, 44, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 579, 991, 1245, 1930, 33, 607, 13, 337, 292, 153, 111, 83, 0, 0, 0, 0, 114, 803, 903, 2316, 46, 555, 259, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 111, 38, 510, 152, 532, 14, 294, 130, 3418, 797, 0, 0, 0, 0, 0, 0, 744, 599, 289, 814, 208, 151, 315, 1354, 1070, 577, 0, 0, 0, 0, 0, 0, 2254, 240, 1786, 376, 465, 172, 835, 148, 0, 0, 0, 0, 0, 0, 0, 0, 867, 1025, 0, 836, 569, 550, 1112, 884, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6356, 6025, 6374, 6161, 5996, 6121, 6276, 5843] avg=6144.0000 max=6374 max_rank=2 min=5843 min_rank=7 imbalance_ratio=1.037435 max_load_reduction=42778 max_load_reduction_pct=0.870321 moved_to_echo=24236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[299, 873, 397, 650, 2240, 92, 1237, 311, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 670, 286, 1011, 519, 1184, 78, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 504, 992, 819, 1741, 29, 561, 18, 363, 251, 227, 98, 85, 0, 0, 0, 0, 134, 650, 839, 2100, 58, 650, 214, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 121, 27, 580, 266, 486, 27, 417, 128, 3459, 890, 0, 0, 0, 0, 0, 0, 766, 587, 288, 921, 234, 147, 284, 1332, 1135, 647, 0, 0, 0, 0, 0, 0, 1973, 221, 1880, 500, 487, 153, 884, 107, 0, 0, 0, 0, 0, 0, 0, 0, 820, 978, 0, 1270, 577, 569, 1119, 991, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6099, 6196, 5688, 5898, 6401, 6341, 6205, 6324] avg=6144.0000 max=6401 max_rank=4 min=5688 min_rank=2 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=25271 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[348, 886, 296, 639, 2380, 84, 1193, 326, 0, 0, 0, 0, 0, 0, 0, 0, 986, 551, 325, 1119, 542, 1038, 63, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 555, 1000, 1294, 2023, 22, 652, 28, 380, 295, 147, 108, 80, 0, 0, 0, 0, 128, 729, 927, 2184, 43, 642, 248, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 121, 21, 501, 183, 513, 18, 380, 121, 3337, 757, 0, 0, 0, 0, 0, 0, 848, 529, 289, 814, 218, 151, 274, 1177, 1075, 549, 0, 0, 0, 0, 0, 0, 2316, 241, 1904, 390, 482, 156, 853, 133, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1013, 0, 822, 577, 496, 1078, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6041, 6584, 6087, 5952, 5924, 6475, 5937] avg=6144.0000 max=6584 max_rank=2 min=5924 min_rank=5 imbalance_ratio=1.071615 max_load_reduction=42568 max_load_reduction_pct=0.866048 moved_to_echo=24288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 7168, 2944, 5120, 18688, 768, 9728, 2560, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 5120, 2688, 8320, 4352, 8832, 512, 10624, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7168, 8064, 0, 8576, 4992, 4480, 8704, 7808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5888, 4608, 2432, 6912, 1792, 1280, 2432, 10624, 8960, 4864, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 8064, 8576, 15104, 256, 4736, 256, 3072, 2176, 1536, 896, 768, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 6016, 7552, 17920, 384, 5120, 1920, 9856, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16896, 1920, 13824, 3456, 3840, 1408, 7168, 1152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 384, 4608, 1664, 4352, 256, 2944, 1152, 27008, 6656, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5888, 4608, 2432, 6912, 1792, 1280, 2432, 10624, 8960, 4864, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[558, 265, 541, 1450, 944, 50, 1172, 758, 1075, 1008, 1986, 759, 665, 1514, 132, 303, 3458, 813, 4259, 326, 461, 24, 1510, 1740, 419, 612, 492, 213, 271, 1175, 743, 1045, 364, 284, 973, 1250, 214, 465, 455, 248, 1045, 483, 15, 256, 236, 639, 353, 44, 2648, 1254, 402, 919, 115, 1017, 1024, 400, 568, 1098, 282, 38, 226, 297, 7, 792] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[560, 233, 378, 1223, 850, 47, 1229, 832, 950, 1068, 1937, 799, 643, 1449, 105, 274, 3481, 866, 4710, 265, 424, 19, 1534, 1844, 417, 559, 630, 229, 289, 1262, 805, 831, 366, 288, 996, 1120, 372, 348, 235, 172, 1155, 542, 14, 284, 206, 700, 328, 38, 2767, 1189, 416, 930, 69, 937, 1173, 313, 595, 1003, 261, 22, 267, 314, 8, 982] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[508, 278, 540, 1294, 768, 83, 1124, 752, 1049, 1114, 1433, 767, 706, 1459, 155, 264, 3156, 923, 4314, 326, 659, 44, 1387, 1881, 342, 689, 632, 233, 231, 1176, 913, 942, 373, 357, 893, 1127, 323, 349, 512, 204, 1133, 520, 38, 270, 284, 772, 361, 110, 2603, 1257, 462, 961, 108, 1060, 1079, 417, 612, 787, 390, 25, 272, 315, 16, 1020] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[499, 287, 514, 1342, 959, 65, 1046, 677, 1190, 779, 1747, 770, 625, 1496, 169, 224, 3230, 1120, 4434, 361, 565, 32, 1359, 1641, 369, 719, 638, 190, 247, 1222, 886, 899, 336, 322, 858, 1301, 263, 438, 405, 251, 1174, 587, 24, 283, 270, 763, 319, 68, 2720, 1137, 450, 906, 131, 1054, 1198, 338, 593, 925, 361, 41, 209, 298, 19, 809] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[699, 259, 438, 1463, 1047, 69, 1029, 611, 1152, 858, 1954, 728, 694, 1595, 122, 254, 3213, 879, 4398, 239, 570, 37, 1392, 1680, 375, 630, 502, 153, 289, 1152, 759, 1149, 412, 280, 882, 1061, 249, 456, 367, 300, 1056, 523, 19, 212, 258, 669, 372, 77, 3042, 1241, 402, 920, 155, 1144, 945, 445, 490, 1134, 297, 27, 237, 296, 14, 781] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[529, 270, 519, 1392, 761, 65, 1197, 808, 1007, 907, 1913, 793, 634, 1308, 124, 289, 3668, 880, 4783, 315, 437, 34, 1617, 1768, 298, 596, 516, 145, 256, 1265, 837, 921, 378, 269, 913, 1012, 386, 397, 362, 144, 1048, 455, 10, 234, 193, 735, 348, 65, 2871, 1113, 508, 947, 89, 934, 994, 439, 618, 1159, 215, 35, 214, 277, 11, 927] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[508, 278, 540, 1294, 768, 83, 1124, 752, 1049, 1114, 1433, 767, 706, 1459, 155, 264, 3156, 923, 4314, 326, 659, 44, 1387, 1881, 342, 689, 632, 233, 231, 1176, 913, 942, 373, 357, 893, 1127, 323, 349, 512, 204, 1133, 520, 38, 270, 284, 772, 361, 110, 2603, 1257, 462, 961, 108, 1060, 1079, 417, 612, 787, 390, 25, 272, 315, 16, 1020] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[699, 259, 438, 1463, 1047, 69, 1029, 611, 1152, 858, 1954, 728, 694, 1595, 122, 254, 3213, 879, 4398, 239, 570, 37, 1392, 1680, 375, 630, 502, 153, 289, 1152, 759, 1149, 412, 280, 882, 1061, 249, 456, 367, 300, 1056, 523, 19, 212, 258, 669, 372, 77, 3042, 1241, 402, 920, 155, 1144, 945, 445, 490, 1134, 297, 27, 237, 296, 14, 781] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[370, 360, 318, 1254, 767, 67, 1216, 739, 1149, 844, 1826, 777, 624, 1489, 100, 342, 3734, 1029, 4392, 309, 560, 40, 1580, 1782, 468, 630, 640, 172, 270, 1359, 978, 728, 386, 219, 758, 1013, 431, 430, 196, 151, 1193, 451, 14, 245, 207, 747, 276, 71, 3132, 1174, 243, 1022, 93, 876, 1101, 351, 533, 973, 343, 30, 212, 302, 13, 1053] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[634, 276, 524, 1457, 900, 61, 1169, 691, 1239, 1025, 1864, 652, 651, 1410, 138, 213, 3521, 874, 4314, 321, 577, 22, 1359, 1741, 404, 616, 487, 137, 216, 1215, 807, 959, 327, 267, 801, 1124, 292, 520, 354, 165, 1230, 507, 7, 284, 242, 661, 310, 52, 3155, 1322, 344, 881, 128, 976, 952, 349, 597, 853, 346, 14, 259, 300, 12, 1047] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[560, 233, 378, 1223, 850, 47, 1229, 832, 709, 0, 0, 0, 0, 0, 0, 0, 950, 1068, 863, 799, 643, 1449, 105, 274, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 866, 0, 265, 424, 19, 1534, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 417, 559, 630, 229, 289, 1262, 805, 831, 739, 365, 0, 0, 0, 0, 0, 0, 366, 288, 996, 1120, 372, 348, 235, 172, 1132, 1020, 0, 0, 0, 0, 0, 0, 1155, 542, 14, 284, 206, 700, 328, 38, 3051, 0, 0, 0, 0, 0, 0, 0, 1008, 1189, 416, 930, 69, 937, 1173, 313, 0, 0, 0, 0, 0, 0, 0, 0, 595, 1003, 261, 22, 267, 314, 8, 982, 1659, 1209, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6061, 6151, 6092, 6126, 6049, 6318, 6035, 6320] avg=6144.0000 max=6320 max_rank=7 min=6035 min_rank=6 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[508, 278, 540, 1294, 768, 83, 1124, 752, 524, 0, 0, 0, 0, 0, 0, 0, 1049, 1114, 639, 767, 706, 1459, 155, 264, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 923, 0, 326, 659, 44, 1387, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 342, 689, 632, 233, 231, 1176, 913, 942, 695, 270, 0, 0, 0, 0, 0, 0, 373, 357, 893, 1127, 323, 349, 512, 204, 1026, 959, 0, 0, 0, 0, 0, 0, 1133, 520, 38, 270, 284, 772, 361, 110, 2795, 0, 0, 0, 0, 0, 0, 0, 949, 1257, 462, 961, 108, 1060, 1079, 417, 0, 0, 0, 0, 0, 0, 0, 0, 612, 787, 390, 25, 272, 315, 16, 1020, 1519, 1096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5871, 6153, 6254, 6123, 6123, 6283, 6293, 6052] avg=6144.0000 max=6293 max_rank=6 min=5871 min_rank=0 imbalance_ratio=1.024251 max_load_reduction=42859 max_load_reduction_pct=0.871969 moved_to_echo=24751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[558, 265, 541, 1450, 944, 50, 1172, 758, 727, 0, 0, 0, 0, 0, 0, 0, 1075, 1008, 884, 759, 665, 1514, 132, 303, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 813, 0, 326, 461, 24, 1510, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 419, 612, 492, 213, 271, 1175, 743, 1045, 708, 375, 0, 0, 0, 0, 0, 0, 364, 284, 973, 1250, 214, 465, 455, 248, 1124, 976, 0, 0, 0, 0, 0, 0, 1045, 483, 15, 256, 236, 639, 353, 44, 2760, 0, 0, 0, 0, 0, 0, 0, 964, 1254, 402, 919, 115, 1017, 1024, 400, 0, 0, 0, 0, 0, 0, 0, 0, 568, 1098, 282, 38, 226, 297, 7, 792, 1499, 1201, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 6340, 6007, 6053, 6353, 5831, 6095, 6008] avg=6144.0000 max=6465 max_rank=0 min=5831 min_rank=5 imbalance_ratio=1.052246 max_load_reduction=42687 max_load_reduction_pct=0.868469 moved_to_echo=24287 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[499, 287, 514, 1342, 959, 65, 1046, 677, 639, 0, 0, 0, 0, 0, 0, 0, 1190, 779, 778, 770, 625, 1496, 169, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1120, 0, 361, 565, 32, 1359, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 369, 719, 638, 190, 247, 1222, 886, 899, 727, 330, 0, 0, 0, 0, 0, 0, 336, 322, 858, 1301, 263, 438, 405, 251, 1050, 1002, 0, 0, 0, 0, 0, 0, 1174, 587, 24, 283, 270, 763, 319, 68, 2873, 0, 0, 0, 0, 0, 0, 0, 991, 1137, 450, 906, 131, 1054, 1198, 338, 0, 0, 0, 0, 0, 0, 0, 0, 593, 925, 361, 41, 209, 298, 19, 809, 1561, 1122, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6028, 6031, 6136, 6227, 6226, 6361, 6205, 5938] avg=6144.0000 max=6361 max_rank=5 min=5938 min_rank=7 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[558, 265, 541, 1450, 944, 50, 1172, 758, 727, 0, 0, 0, 0, 0, 0, 0, 1075, 1008, 884, 759, 665, 1514, 132, 303, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 813, 0, 326, 461, 24, 1510, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 419, 612, 492, 213, 271, 1175, 743, 1045, 708, 375, 0, 0, 0, 0, 0, 0, 364, 284, 973, 1250, 214, 465, 455, 248, 1124, 976, 0, 0, 0, 0, 0, 0, 1045, 483, 15, 256, 236, 639, 353, 44, 2760, 0, 0, 0, 0, 0, 0, 0, 964, 1254, 402, 919, 115, 1017, 1024, 400, 0, 0, 0, 0, 0, 0, 0, 0, 568, 1098, 282, 38, 226, 297, 7, 792, 1499, 1201, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 6340, 6007, 6053, 6353, 5831, 6095, 6008] avg=6144.0000 max=6465 max_rank=0 min=5831 min_rank=5 imbalance_ratio=1.052246 max_load_reduction=42687 max_load_reduction_pct=0.868469 moved_to_echo=24287 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[699, 259, 438, 1463, 1047, 69, 1029, 611, 715, 0, 0, 0, 0, 0, 0, 0, 1152, 858, 870, 728, 694, 1595, 122, 254, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 879, 0, 239, 570, 37, 1392, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 375, 630, 502, 153, 289, 1152, 759, 1149, 813, 369, 0, 0, 0, 0, 0, 0, 412, 280, 882, 1061, 249, 456, 367, 300, 1044, 1121, 0, 0, 0, 0, 0, 0, 1056, 523, 19, 212, 258, 669, 372, 77, 2849, 0, 0, 0, 0, 0, 0, 0, 1108, 1241, 402, 920, 155, 1144, 945, 445, 0, 0, 0, 0, 0, 0, 0, 0, 490, 1134, 297, 27, 237, 296, 14, 781, 1549, 1116, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6330, 6273, 5850, 6191, 6172, 6035, 6360, 5941] avg=6144.0000 max=6360 max_rank=6 min=5850 min_rank=2 imbalance_ratio=1.035156 max_load_reduction=42792 max_load_reduction_pct=0.870605 moved_to_echo=24508 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[529, 270, 519, 1392, 761, 65, 1197, 808, 703, 0, 0, 0, 0, 0, 0, 0, 1007, 907, 846, 793, 634, 1308, 124, 289, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 880, 0, 315, 437, 34, 1617, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 298, 596, 516, 145, 256, 1265, 837, 921, 771, 364, 0, 0, 0, 0, 0, 0, 378, 269, 913, 1012, 386, 397, 362, 144, 1196, 1062, 0, 0, 0, 0, 0, 0, 1048, 455, 10, 234, 193, 735, 348, 65, 3099, 0, 0, 0, 0, 0, 0, 0, 1038, 1113, 508, 947, 89, 934, 994, 439, 0, 0, 0, 0, 0, 0, 0, 0, 618, 1159, 215, 35, 214, 277, 11, 927, 1684, 1280, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6244, 5908, 6243, 5969, 6119, 6187, 6062, 6420] avg=6144.0000 max=6420 max_rank=7 min=5908 min_rank=1 imbalance_ratio=1.044922 max_load_reduction=42732 max_load_reduction_pct=0.869385 moved_to_echo=24788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[634, 276, 524, 1457, 900, 61, 1169, 691, 682, 0, 0, 0, 0, 0, 0, 0, 1239, 1025, 830, 652, 651, 1410, 138, 213, 0, 0, 0, 0, 0, 0, 0, 0, 1153, 874, 0, 321, 577, 22, 1359, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 404, 616, 487, 137, 216, 1215, 807, 959, 843, 352, 0, 0, 0, 0, 0, 0, 327, 267, 801, 1124, 292, 520, 354, 165, 1145, 1163, 0, 0, 0, 0, 0, 0, 1230, 507, 7, 284, 242, 661, 310, 52, 2794, 0, 0, 0, 0, 0, 0, 0, 1149, 1322, 344, 881, 128, 976, 952, 349, 0, 0, 0, 0, 0, 0, 0, 0, 597, 853, 346, 14, 259, 300, 12, 1047, 1520, 1223, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 6158, 6047, 6036, 6158, 6087, 6101, 6171] avg=6144.0000 max=6394 max_rank=0 min=6036 min_rank=3 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24517 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[370, 360, 318, 1254, 767, 67, 1216, 739, 668, 0, 0, 0, 0, 0, 0, 0, 1149, 844, 813, 777, 624, 1489, 100, 342, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 1029, 0, 309, 560, 40, 1580, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 468, 630, 640, 172, 270, 1359, 978, 728, 837, 345, 0, 0, 0, 0, 0, 0, 386, 219, 758, 1013, 431, 430, 196, 151, 1214, 1154, 0, 0, 0, 0, 0, 0, 1193, 451, 14, 245, 207, 747, 276, 71, 2846, 0, 0, 0, 0, 0, 0, 0, 1141, 1174, 243, 1022, 93, 876, 1101, 351, 0, 0, 0, 0, 0, 0, 0, 0, 533, 973, 343, 30, 212, 302, 13, 1053, 1546, 1297, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5759, 6138, 6523, 6427, 5952, 6050, 6001, 6302] avg=6144.0000 max=6523 max_rank=2 min=5759 min_rank=0 imbalance_ratio=1.061686 max_load_reduction=42629 max_load_reduction_pct=0.867289 moved_to_echo=24305 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[529, 270, 519, 1392, 761, 65, 1197, 808, 703, 0, 0, 0, 0, 0, 0, 0, 1007, 907, 846, 793, 634, 1308, 124, 289, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 880, 0, 315, 437, 34, 1617, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 298, 596, 516, 145, 256, 1265, 837, 921, 771, 364, 0, 0, 0, 0, 0, 0, 378, 269, 913, 1012, 386, 397, 362, 144, 1196, 1062, 0, 0, 0, 0, 0, 0, 1048, 455, 10, 234, 193, 735, 348, 65, 3099, 0, 0, 0, 0, 0, 0, 0, 1038, 1113, 508, 947, 89, 934, 994, 439, 0, 0, 0, 0, 0, 0, 0, 0, 618, 1159, 215, 35, 214, 277, 11, 927, 1684, 1280, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6244, 5908, 6243, 5969, 6119, 6187, 6062, 6420] avg=6144.0000 max=6420 max_rank=7 min=5908 min_rank=1 imbalance_ratio=1.044922 max_load_reduction=42732 max_load_reduction_pct=0.869385 moved_to_echo=24788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4480, 2304, 3840, 10880, 7040, 512, 9216, 5888, 5376, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9088, 7424, 0, 2560, 4352, 256, 11776, 14080, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8832, 7680, 6528, 6144, 5248, 11776, 1152, 2176, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8448, 9728, 3328, 7552, 896, 8064, 8576, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8832, 7680, 6528, 6144, 5248, 11776, 1152, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9088, 7424, 0, 2560, 4352, 256, 11776, 14080, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 5120, 4608, 1536, 2176, 9856, 6784, 7552, 6144, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4608, 7936, 2560, 256, 1920, 2432, 128, 7424, 12544, 9600, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 2304, 7168, 9088, 2560, 3456, 2944, 1664, 8960, 8576, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9088, 4096, 256, 2176, 1920, 5760, 2688, 640, 23168, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4608, 7936, 2560, 256, 1920, 2432, 128, 7424, 12544, 9600, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[79, 67, 2, 1623, 1306, 2794, 838, 91, 917, 724, 357, 188, 400, 137, 876, 96, 417, 255, 2, 1, 1248, 139, 254, 136, 3177, 1489, 59, 770, 1366, 393, 1229, 1388, 3, 120, 2261, 427, 5054, 161, 988, 242, 455, 452, 644, 754, 359, 281, 0, 564, 749, 359, 1135, 400, 4458, 89, 612, 103, 524, 25, 53, 1194, 16, 1411, 110, 2331] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[80, 53, 5, 1756, 1178, 3087, 424, 129, 1143, 813, 394, 226, 467, 397, 527, 209, 449, 219, 3, 11, 970, 174, 580, 121, 3397, 1384, 194, 734, 843, 348, 1086, 1497, 12, 159, 2529, 458, 4610, 162, 1222, 266, 589, 426, 972, 634, 342, 320, 3, 470, 754, 465, 998, 393, 3802, 61, 949, 156, 713, 43, 34, 1298, 19, 1304, 141, 1950] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[86, 52, 10, 1543, 1273, 2770, 699, 126, 1072, 855, 297, 341, 442, 148, 746, 265, 537, 264, 5, 3, 1105, 174, 493, 137, 3435, 1332, 123, 758, 1024, 377, 1287, 1455, 13, 286, 2008, 412, 4580, 210, 1253, 264, 618, 395, 847, 707, 362, 242, 1, 458, 790, 458, 907, 377, 3787, 80, 617, 163, 679, 40, 41, 1349, 48, 1639, 157, 2130] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[79, 67, 2, 1623, 1306, 2794, 838, 91, 917, 724, 357, 188, 400, 137, 876, 96, 417, 255, 2, 1, 1248, 139, 254, 136, 3177, 1489, 59, 770, 1366, 393, 1229, 1388, 3, 120, 2261, 427, 5054, 161, 988, 242, 455, 452, 644, 754, 359, 281, 0, 564, 749, 359, 1135, 400, 4458, 89, 612, 103, 524, 25, 53, 1194, 16, 1411, 110, 2331] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[104, 13, 3, 1471, 1399, 2870, 705, 109, 1093, 749, 369, 131, 407, 231, 865, 141, 463, 256, 0, 6, 1305, 177, 240, 126, 3155, 1349, 148, 847, 961, 425, 1183, 1515, 4, 152, 2559, 383, 4902, 204, 1003, 255, 430, 450, 656, 663, 377, 286, 1, 470, 740, 355, 1045, 403, 4271, 88, 560, 188, 553, 30, 71, 1286, 23, 1666, 117, 2145] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[34, 63, 2, 1826, 1199, 3154, 364, 91, 897, 726, 272, 205, 511, 484, 752, 178, 259, 242, 4, 0, 1232, 129, 459, 129, 3567, 1461, 6, 829, 1024, 370, 995, 1357, 19, 209, 2346, 460, 5165, 166, 1050, 268, 522, 393, 851, 597, 353, 215, 0, 765, 803, 426, 915, 470, 4054, 85, 685, 158, 569, 51, 34, 1110, 38, 1470, 117, 1967] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[69, 63, 7, 1727, 1331, 3406, 612, 51, 927, 755, 298, 177, 391, 215, 667, 119, 382, 211, 2, 11, 1053, 134, 379, 122, 3510, 1452, 21, 869, 1019, 407, 1242, 1375, 3, 179, 2264, 386, 5155, 243, 864, 195, 429, 439, 924, 622, 296, 235, 3, 581, 888, 387, 1000, 430, 4126, 83, 725, 150, 676, 52, 27, 1185, 32, 1227, 130, 2212] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[80, 53, 5, 1756, 1178, 3087, 424, 129, 1143, 813, 394, 226, 467, 397, 527, 209, 449, 219, 3, 11, 970, 174, 580, 121, 3397, 1384, 194, 734, 843, 348, 1086, 1497, 12, 159, 2529, 458, 4610, 162, 1222, 266, 589, 426, 972, 634, 342, 320, 3, 470, 754, 465, 998, 393, 3802, 61, 949, 156, 713, 43, 34, 1298, 19, 1304, 141, 1950] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1, 26, 1, 2074, 1034, 3122, 279, 116, 852, 756, 286, 181, 451, 475, 580, 164, 196, 253, 1, 4, 1060, 142, 533, 127, 3470, 1749, 13, 806, 736, 331, 867, 1730, 12, 134, 2671, 621, 4988, 109, 899, 244, 550, 373, 824, 535, 310, 247, 3, 634, 997, 431, 985, 412, 4019, 86, 791, 212, 623, 62, 40, 1286, 17, 1282, 146, 2193] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[34, 63, 2, 1826, 1199, 3154, 364, 91, 897, 726, 272, 205, 511, 484, 752, 178, 259, 242, 4, 0, 1232, 129, 459, 129, 3567, 1461, 6, 829, 1024, 370, 995, 1357, 19, 209, 2346, 460, 5165, 166, 1050, 268, 522, 393, 851, 597, 353, 215, 0, 765, 803, 426, 915, 470, 4054, 85, 685, 158, 569, 51, 34, 1110, 38, 1470, 117, 1967] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[130, 21, 6, 1612, 1508, 3469, 698, 117, 1042, 753, 386, 142, 415, 109, 940, 215, 538, 237, 5, 2, 1036, 143, 276, 160, 3103, 1368, 89, 871, 1109, 406, 1254, 1286, 4, 168, 2168, 362, 4869, 195, 1212, 232, 513, 421, 916, 625, 213, 307, 4, 493, 650, 378, 982, 461, 4149, 143, 458, 105, 544, 43, 36, 980, 49, 1500, 236, 2290] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[79, 67, 2, 1623, 1306, 2137, 838, 91, 0, 0, 0, 0, 0, 0, 0, 0, 917, 724, 357, 188, 400, 137, 876, 96, 1610, 393, 325, 0, 0, 0, 0, 0, 417, 255, 2, 1, 1248, 139, 254, 136, 3177, 335, 0, 0, 0, 0, 0, 0, 0, 1489, 59, 770, 1366, 393, 1229, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 3, 120, 2261, 427, 1730, 161, 988, 242, 0, 0, 0, 0, 0, 0, 0, 0, 455, 452, 644, 754, 359, 281, 0, 564, 2664, 0, 0, 0, 0, 0, 0, 0, 749, 359, 1135, 400, 2848, 89, 612, 103, 0, 0, 0, 0, 0, 0, 0, 0, 524, 25, 53, 1194, 16, 1411, 110, 2331, 264, 173, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6023, 5964, 6521, 5932, 6173, 6295, 6101] avg=6144.0000 max=6521 max_rank=3 min=5932 min_rank=4 imbalance_ratio=1.061361 max_load_reduction=42631 max_load_reduction_pct=0.867330 moved_to_echo=24501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[69, 63, 7, 1727, 1331, 2605, 612, 51, 0, 0, 0, 0, 0, 0, 0, 0, 927, 755, 298, 177, 391, 215, 667, 119, 1490, 479, 331, 0, 0, 0, 0, 0, 382, 211, 2, 11, 1053, 134, 379, 122, 3510, 341, 0, 0, 0, 0, 0, 0, 0, 1452, 21, 869, 1019, 407, 1242, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 3, 179, 2264, 386, 1766, 243, 864, 195, 0, 0, 0, 0, 0, 0, 0, 0, 429, 439, 924, 622, 296, 235, 3, 581, 2717, 0, 0, 0, 0, 0, 0, 0, 888, 387, 1000, 430, 2636, 83, 725, 150, 0, 0, 0, 0, 0, 0, 0, 0, 676, 52, 27, 1185, 32, 1227, 130, 2212, 322, 172, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 5849, 6145, 6213, 5900, 6246, 6299, 6035] avg=6144.0000 max=6465 max_rank=0 min=5849 min_rank=1 imbalance_ratio=1.052246 max_load_reduction=42687 max_load_reduction_pct=0.868469 moved_to_echo=24480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[86, 52, 10, 1543, 1273, 2119, 699, 126, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 855, 297, 341, 442, 148, 746, 265, 1368, 389, 294, 0, 0, 0, 0, 0, 537, 264, 5, 3, 1105, 174, 493, 137, 3435, 303, 0, 0, 0, 0, 0, 0, 0, 1332, 123, 758, 1024, 377, 1287, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 13, 286, 2008, 412, 1569, 210, 1253, 264, 0, 0, 0, 0, 0, 0, 0, 0, 618, 395, 847, 707, 362, 242, 1, 458, 2414, 0, 0, 0, 0, 0, 0, 0, 790, 458, 907, 377, 2419, 80, 617, 163, 0, 0, 0, 0, 0, 0, 0, 0, 679, 40, 41, 1349, 48, 1639, 157, 2130, 262, 182, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5908, 6217, 6456, 6174, 6015, 6044, 5811, 6527] avg=6144.0000 max=6527 max_rank=7 min=5811 min_rank=6 imbalance_ratio=1.062337 max_load_reduction=42625 max_load_reduction_pct=0.867208 moved_to_echo=24397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[80, 53, 5, 1756, 1178, 2361, 424, 129, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 813, 394, 226, 467, 397, 527, 209, 1373, 434, 296, 0, 0, 0, 0, 0, 449, 219, 3, 11, 970, 174, 580, 121, 3397, 305, 0, 0, 0, 0, 0, 0, 0, 1384, 194, 734, 843, 348, 1086, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 12, 159, 2529, 458, 1579, 162, 1222, 266, 0, 0, 0, 0, 0, 0, 0, 0, 589, 426, 972, 634, 342, 320, 3, 470, 2430, 0, 0, 0, 0, 0, 0, 0, 754, 465, 998, 393, 2429, 61, 949, 156, 0, 0, 0, 0, 0, 0, 0, 0, 713, 43, 34, 1298, 19, 1304, 141, 1950, 292, 187, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5986, 6279, 6229, 5899, 6387, 6186, 6205, 5981] avg=6144.0000 max=6387 max_rank=4 min=5899 min_rank=3 imbalance_ratio=1.039551 max_load_reduction=42765 max_load_reduction_pct=0.870056 moved_to_echo=24759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 26, 1, 2074, 1034, 2388, 279, 116, 0, 0, 0, 0, 0, 0, 0, 0, 852, 756, 286, 181, 451, 475, 580, 164, 1452, 439, 321, 0, 0, 0, 0, 0, 196, 253, 1, 4, 1060, 142, 533, 127, 3470, 330, 0, 0, 0, 0, 0, 0, 0, 1749, 13, 806, 736, 331, 867, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 12, 134, 2671, 621, 1708, 109, 899, 244, 0, 0, 0, 0, 0, 0, 0, 0, 550, 373, 824, 535, 310, 247, 3, 634, 2629, 0, 0, 0, 0, 0, 0, 0, 997, 431, 985, 412, 2567, 86, 791, 212, 0, 0, 0, 0, 0, 0, 0, 0, 623, 62, 40, 1286, 17, 1282, 146, 2193, 295, 216, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5919, 5957, 6116, 6016, 6398, 6105, 6481, 6160] avg=6144.0000 max=6481 max_rank=6 min=5919 min_rank=0 imbalance_ratio=1.054850 max_load_reduction=42671 max_load_reduction_pct=0.868144 moved_to_echo=25144 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[34, 63, 2, 1826, 1199, 2413, 364, 91, 0, 0, 0, 0, 0, 0, 0, 0, 897, 726, 272, 205, 511, 484, 752, 178, 1464, 443, 332, 0, 0, 0, 0, 0, 259, 242, 4, 0, 1232, 129, 459, 129, 3567, 342, 0, 0, 0, 0, 0, 0, 0, 1461, 6, 829, 1024, 370, 995, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 19, 209, 2346, 460, 1769, 166, 1050, 268, 0, 0, 0, 0, 0, 0, 0, 0, 522, 393, 851, 597, 353, 215, 0, 765, 2722, 0, 0, 0, 0, 0, 0, 0, 803, 426, 915, 470, 2590, 85, 685, 158, 0, 0, 0, 0, 0, 0, 0, 0, 569, 51, 34, 1110, 38, 1470, 117, 1967, 298, 170, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5992, 6264, 6363, 5872, 6287, 6418, 6132, 5824] avg=6144.0000 max=6418 max_rank=5 min=5824 min_rank=7 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[69, 63, 7, 1727, 1331, 2605, 612, 51, 0, 0, 0, 0, 0, 0, 0, 0, 927, 755, 298, 177, 391, 215, 667, 119, 1490, 479, 331, 0, 0, 0, 0, 0, 382, 211, 2, 11, 1053, 134, 379, 122, 3510, 341, 0, 0, 0, 0, 0, 0, 0, 1452, 21, 869, 1019, 407, 1242, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 3, 179, 2264, 386, 1766, 243, 864, 195, 0, 0, 0, 0, 0, 0, 0, 0, 429, 439, 924, 622, 296, 235, 3, 581, 2717, 0, 0, 0, 0, 0, 0, 0, 888, 387, 1000, 430, 2636, 83, 725, 150, 0, 0, 0, 0, 0, 0, 0, 0, 676, 52, 27, 1185, 32, 1227, 130, 2212, 322, 172, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 5849, 6145, 6213, 5900, 6246, 6299, 6035] avg=6144.0000 max=6465 max_rank=0 min=5849 min_rank=1 imbalance_ratio=1.052246 max_load_reduction=42687 max_load_reduction_pct=0.868469 moved_to_echo=24480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 26, 1, 2074, 1034, 2388, 279, 116, 0, 0, 0, 0, 0, 0, 0, 0, 852, 756, 286, 181, 451, 475, 580, 164, 1452, 439, 321, 0, 0, 0, 0, 0, 196, 253, 1, 4, 1060, 142, 533, 127, 3470, 330, 0, 0, 0, 0, 0, 0, 0, 1749, 13, 806, 736, 331, 867, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 12, 134, 2671, 621, 1708, 109, 899, 244, 0, 0, 0, 0, 0, 0, 0, 0, 550, 373, 824, 535, 310, 247, 3, 634, 2629, 0, 0, 0, 0, 0, 0, 0, 997, 431, 985, 412, 2567, 86, 791, 212, 0, 0, 0, 0, 0, 0, 0, 0, 623, 62, 40, 1286, 17, 1282, 146, 2193, 295, 216, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5919, 5957, 6116, 6016, 6398, 6105, 6481, 6160] avg=6144.0000 max=6481 max_rank=6 min=5919 min_rank=0 imbalance_ratio=1.054850 max_load_reduction=42671 max_load_reduction_pct=0.868144 moved_to_echo=25144 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[79, 67, 2, 1623, 1306, 2137, 838, 91, 0, 0, 0, 0, 0, 0, 0, 0, 917, 724, 357, 188, 400, 137, 876, 96, 1610, 393, 325, 0, 0, 0, 0, 0, 417, 255, 2, 1, 1248, 139, 254, 136, 3177, 335, 0, 0, 0, 0, 0, 0, 0, 1489, 59, 770, 1366, 393, 1229, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 3, 120, 2261, 427, 1730, 161, 988, 242, 0, 0, 0, 0, 0, 0, 0, 0, 455, 452, 644, 754, 359, 281, 0, 564, 2664, 0, 0, 0, 0, 0, 0, 0, 749, 359, 1135, 400, 2848, 89, 612, 103, 0, 0, 0, 0, 0, 0, 0, 0, 524, 25, 53, 1194, 16, 1411, 110, 2331, 264, 173, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6023, 5964, 6521, 5932, 6173, 6295, 6101] avg=6144.0000 max=6521 max_rank=3 min=5932 min_rank=4 imbalance_ratio=1.061361 max_load_reduction=42631 max_load_reduction_pct=0.867330 moved_to_echo=24501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[130, 21, 6, 1612, 1508, 2653, 698, 117, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 753, 386, 142, 415, 109, 940, 215, 1499, 488, 313, 0, 0, 0, 0, 0, 538, 237, 5, 2, 1036, 143, 276, 160, 3103, 322, 0, 0, 0, 0, 0, 0, 0, 1368, 89, 871, 1109, 406, 1254, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 4, 168, 2168, 362, 1668, 195, 1212, 232, 0, 0, 0, 0, 0, 0, 0, 0, 513, 421, 916, 625, 213, 307, 4, 493, 2566, 0, 0, 0, 0, 0, 0, 0, 650, 378, 982, 461, 2650, 143, 458, 105, 0, 0, 0, 0, 0, 0, 0, 0, 544, 43, 36, 980, 49, 1500, 236, 2290, 328, 161, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6745, 6302, 5822, 6222, 6009, 6058, 5827, 6167] avg=6144.0000 max=6745 max_rank=0 min=5822 min_rank=2 imbalance_ratio=1.097819 max_load_reduction=42407 max_load_reduction_pct=0.862773 moved_to_echo=24061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[104, 13, 3, 1471, 1399, 2190, 705, 109, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 749, 369, 131, 407, 231, 865, 141, 1548, 407, 320, 0, 0, 0, 0, 0, 463, 256, 0, 6, 1305, 177, 240, 126, 3155, 329, 0, 0, 0, 0, 0, 0, 0, 1349, 148, 847, 961, 425, 1183, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 4, 152, 2559, 383, 1664, 204, 1003, 255, 0, 0, 0, 0, 0, 0, 0, 0, 430, 450, 656, 663, 377, 286, 1, 470, 2589, 0, 0, 0, 0, 0, 0, 0, 740, 355, 1045, 403, 2723, 88, 560, 188, 0, 0, 0, 0, 0, 0, 0, 0, 553, 30, 71, 1286, 23, 1666, 117, 2145, 273, 193, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5994, 6261, 6057, 6235, 6224, 5922, 6102, 6357] avg=6144.0000 max=6357 max_rank=7 min=5922 min_rank=5 imbalance_ratio=1.034668 max_load_reduction=42795 max_load_reduction_pct=0.870667 moved_to_echo=24605 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[80, 53, 5, 1756, 1178, 2361, 424, 129, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 813, 394, 226, 467, 397, 527, 209, 1373, 434, 296, 0, 0, 0, 0, 0, 449, 219, 3, 11, 970, 174, 580, 121, 3397, 305, 0, 0, 0, 0, 0, 0, 0, 1384, 194, 734, 843, 348, 1086, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 12, 159, 2529, 458, 1579, 162, 1222, 266, 0, 0, 0, 0, 0, 0, 0, 0, 589, 426, 972, 634, 342, 320, 3, 470, 2430, 0, 0, 0, 0, 0, 0, 0, 754, 465, 998, 393, 2429, 61, 949, 156, 0, 0, 0, 0, 0, 0, 0, 0, 713, 43, 34, 1298, 19, 1304, 141, 1950, 292, 187, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5986, 6279, 6229, 5899, 6387, 6186, 6205, 5981] avg=6144.0000 max=6387 max_rank=4 min=5899 min_rank=3 imbalance_ratio=1.039551 max_load_reduction=42765 max_load_reduction_pct=0.870056 moved_to_echo=24759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8064, 6144, 2688, 1664, 3584, 2304, 6016, 1408, 11904, 3584, 2560, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 3456, 6656, 5248, 2688, 2176, 128, 4480, 20736, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 3328, 8064, 3456, 20864, 768, 5504, 1280, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 384, 128, 13696, 10240, 18944, 4736, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 11648, 768, 6528, 8192, 3072, 9216, 10240, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 384, 384, 9728, 256, 11520, 1280, 17280, 2432, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 18816, 3584, 13568, 1536, 8576, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 3328, 8064, 3456, 20864, 768, 5504, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 11648, 768, 6528, 8192, 3072, 9216, 10240, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[3328, 2048, 128, 128, 9088, 1280, 3328, 1152, 26880, 2688, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 18816, 3584, 13568, 1536, 8576, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[2310, 2167, 1654, 47, 81, 3, 0, 227, 281, 1834, 0, 846, 125, 1773, 366, 76, 1023, 472, 21, 533, 696, 242, 572, 1269, 1305, 652, 73, 1051, 0, 1526, 6, 3669, 409, 1749, 728, 328, 2819, 5, 783, 445, 14, 1464, 577, 1389, 1, 589, 234, 1598, 537, 2936, 168, 260, 1007, 305, 662, 485, 1075, 1, 408, 756, 498, 1455, 446, 121] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[2832, 1891, 1532, 97, 23, 0, 0, 232, 342, 1824, 0, 689, 149, 2674, 601, 79, 1009, 344, 165, 251, 687, 236, 425, 862, 1057, 583, 84, 853, 1, 1228, 3, 4214, 735, 2044, 1096, 182, 2053, 0, 719, 479, 11, 758, 562, 898, 1, 782, 221, 1825, 706, 3422, 118, 308, 1155, 373, 359, 463, 1275, 6, 294, 758, 640, 1604, 205, 133] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2145, 2234, 1638, 60, 30, 0, 0, 216, 340, 1971, 0, 698, 177, 2256, 375, 50, 856, 505, 55, 634, 823, 270, 666, 1337, 906, 640, 84, 1033, 3, 1514, 1, 3805, 417, 1570, 762, 366, 2346, 0, 973, 443, 11, 907, 577, 1584, 0, 553, 222, 1275, 454, 3402, 202, 360, 1272, 278, 506, 439, 1026, 10, 263, 884, 797, 1299, 509, 123] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1612, 2372, 1802, 101, 220, 8, 1, 195, 290, 1940, 6, 873, 148, 1401, 379, 106, 865, 559, 78, 601, 635, 344, 513, 1356, 1336, 626, 96, 1212, 4, 1502, 5, 3556, 488, 1646, 548, 407, 2635, 13, 746, 478, 10, 1369, 679, 1188, 3, 614, 320, 1825, 638, 2681, 312, 324, 1241, 449, 641, 542, 975, 27, 358, 798, 591, 1322, 414, 128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[2402, 2354, 1706, 80, 32, 0, 0, 268, 332, 1954, 1, 793, 161, 2248, 425, 76, 1110, 386, 104, 448, 672, 271, 417, 1037, 1393, 575, 81, 1306, 3, 1443, 11, 3779, 463, 1747, 869, 347, 2294, 0, 858, 387, 21, 832, 653, 1107, 0, 540, 194, 1651, 483, 3269, 177, 340, 1161, 327, 510, 499, 1157, 18, 248, 687, 600, 1428, 299, 118] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[2345, 2163, 1641, 149, 238, 2, 0, 290, 391, 1995, 0, 665, 112, 2185, 375, 74, 969, 396, 203, 284, 865, 269, 587, 1256, 1321, 675, 76, 978, 0, 1340, 10, 3925, 522, 1748, 599, 279, 1878, 6, 651, 476, 19, 1080, 621, 1206, 0, 923, 203, 1753, 858, 3015, 128, 325, 1215, 495, 486, 476, 1235, 11, 288, 617, 481, 1341, 306, 132] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2145, 2234, 1638, 60, 30, 0, 0, 216, 340, 1971, 0, 698, 177, 2256, 375, 50, 856, 505, 55, 634, 823, 270, 666, 1337, 906, 640, 84, 1033, 3, 1514, 1, 3805, 417, 1570, 762, 366, 2346, 0, 973, 443, 11, 907, 577, 1584, 0, 553, 222, 1275, 454, 3402, 202, 360, 1272, 278, 506, 439, 1026, 10, 263, 884, 797, 1299, 509, 123] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[2310, 2167, 1654, 47, 81, 3, 0, 227, 281, 1834, 0, 846, 125, 1773, 366, 76, 1023, 472, 21, 533, 696, 242, 572, 1269, 1305, 652, 73, 1051, 0, 1526, 6, 3669, 409, 1749, 728, 328, 2819, 5, 783, 445, 14, 1464, 577, 1389, 1, 589, 234, 1598, 537, 2936, 168, 260, 1007, 305, 662, 485, 1075, 1, 408, 756, 498, 1455, 446, 121] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2135, 2262, 2143, 71, 23, 0, 0, 272, 355, 2132, 0, 783, 161, 1830, 369, 52, 912, 560, 39, 679, 836, 289, 434, 1369, 1122, 652, 91, 1303, 2, 1294, 4, 3577, 429, 1404, 710, 500, 2572, 0, 940, 404, 9, 1001, 678, 1307, 0, 444, 205, 1371, 540, 3105, 210, 312, 1256, 356, 665, 500, 1007, 3, 295, 713, 828, 1123, 377, 137] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1650, 2185, 1673, 80, 132, 0, 2, 262, 307, 2199, 0, 973, 177, 1459, 470, 65, 922, 568, 15, 822, 664, 349, 440, 1367, 1322, 637, 99, 1045, 2, 1597, 10, 3761, 322, 1504, 454, 558, 2775, 0, 793, 470, 5, 1628, 605, 1629, 0, 439, 251, 1501, 387, 2855, 192, 256, 1141, 232, 743, 522, 1092, 7, 276, 784, 482, 1344, 531, 120] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2832, 1555, 1532, 97, 23, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 342, 1824, 0, 689, 149, 2674, 601, 79, 336, 31, 0, 0, 0, 0, 0, 0, 1009, 344, 165, 251, 687, 236, 425, 862, 600, 701, 0, 0, 0, 0, 0, 0, 1057, 583, 84, 853, 1, 1228, 3, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 735, 2044, 1096, 182, 1395, 0, 719, 479, 0, 0, 0, 0, 0, 0, 0, 0, 11, 758, 562, 898, 1, 782, 221, 1825, 646, 58, 0, 0, 0, 0, 0, 0, 706, 2745, 118, 308, 1155, 373, 359, 463, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 6, 294, 758, 640, 1604, 205, 133, 1651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6271, 6725, 5280, 5671, 6650, 5762, 6227, 6566] avg=6144.0000 max=6725 max_rank=1 min=5280 min_rank=2 imbalance_ratio=1.094564 max_load_reduction=42427 max_load_reduction_pct=0.863180 moved_to_echo=25205 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1612, 1950, 1802, 101, 220, 8, 1, 195, 0, 0, 0, 0, 0, 0, 0, 0, 290, 1940, 6, 873, 148, 1401, 379, 106, 422, 24, 0, 0, 0, 0, 0, 0, 865, 559, 78, 601, 635, 344, 513, 1356, 770, 592, 0, 0, 0, 0, 0, 0, 1336, 626, 96, 1212, 4, 1502, 5, 1571, 0, 0, 0, 0, 0, 0, 0, 0, 488, 1646, 548, 407, 1790, 13, 746, 478, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1369, 679, 1188, 3, 614, 320, 1825, 506, 75, 0, 0, 0, 0, 0, 0, 638, 2151, 312, 324, 1241, 449, 641, 542, 0, 0, 0, 0, 0, 0, 0, 0, 975, 27, 358, 798, 591, 1322, 414, 128, 1393, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5889, 5589, 6313, 6352, 6116, 6589, 6298, 6006] avg=6144.0000 max=6589 max_rank=5 min=5589 min_rank=1 imbalance_ratio=1.072428 max_load_reduction=42563 max_load_reduction_pct=0.865946 moved_to_echo=25009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2402, 1935, 1706, 80, 32, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 332, 1954, 1, 793, 161, 2248, 425, 76, 419, 29, 0, 0, 0, 0, 0, 0, 1110, 386, 104, 448, 672, 271, 417, 1037, 671, 629, 0, 0, 0, 0, 0, 0, 1393, 575, 81, 1306, 3, 1443, 11, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 463, 1747, 869, 347, 1558, 0, 858, 387, 0, 0, 0, 0, 0, 0, 0, 0, 21, 832, 653, 1107, 0, 540, 194, 1651, 617, 65, 0, 0, 0, 0, 0, 0, 483, 2623, 177, 340, 1161, 327, 510, 499, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 18, 248, 687, 600, 1428, 299, 118, 1480, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6423, 6438, 5745, 6482, 6229, 5680, 6120, 6035] avg=6144.0000 max=6482 max_rank=3 min=5680 min_rank=5 imbalance_ratio=1.055013 max_load_reduction=42670 max_load_reduction_pct=0.868123 moved_to_echo=24064 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2345, 1778, 1641, 149, 238, 2, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 391, 1995, 0, 665, 112, 2185, 375, 74, 385, 27, 0, 0, 0, 0, 0, 0, 969, 396, 203, 284, 865, 269, 587, 1256, 549, 653, 0, 0, 0, 0, 0, 0, 1321, 675, 76, 978, 0, 1340, 10, 1734, 0, 0, 0, 0, 0, 0, 0, 0, 522, 1748, 599, 279, 1276, 6, 651, 476, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1080, 621, 1206, 0, 923, 203, 1753, 569, 53, 0, 0, 0, 0, 0, 0, 858, 2419, 128, 325, 1215, 495, 486, 476, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 11, 288, 617, 481, 1341, 306, 132, 1538, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6443, 6209, 6031, 6134, 5557, 6427, 6402, 5949] avg=6144.0000 max=6443 max_rank=0 min=5557 min_rank=4 imbalance_ratio=1.048665 max_load_reduction=42709 max_load_reduction_pct=0.868917 moved_to_echo=24335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2135, 2262, 2143, 71, 23, 0, 0, 272, 355, 2132, 0, 783, 161, 1830, 369, 52, 912, 560, 39, 679, 836, 289, 434, 1369, 1122, 652, 91, 1303, 2, 1294, 4, 3577, 429, 1404, 710, 500, 2572, 0, 940, 404, 9, 1001, 678, 1307, 0, 444, 205, 1371, 540, 3105, 210, 312, 1256, 356, 665, 500, 1007, 3, 295, 713, 828, 1123, 377, 137] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[2345, 2163, 1641, 149, 238, 2, 0, 290, 391, 1995, 0, 665, 112, 2185, 375, 74, 969, 396, 203, 284, 865, 269, 587, 1256, 1321, 675, 76, 978, 0, 1340, 10, 3925, 522, 1748, 599, 279, 1878, 6, 651, 476, 19, 1080, 621, 1206, 0, 923, 203, 1753, 858, 3015, 128, 325, 1215, 495, 486, 476, 1235, 11, 288, 617, 481, 1341, 306, 132] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2145, 1837, 1638, 60, 30, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 340, 1971, 0, 698, 177, 2256, 375, 50, 397, 31, 0, 0, 0, 0, 0, 0, 856, 505, 55, 634, 823, 270, 666, 1337, 686, 633, 0, 0, 0, 0, 0, 0, 906, 640, 84, 1033, 3, 1514, 1, 1681, 0, 0, 0, 0, 0, 0, 0, 0, 417, 1570, 762, 366, 1594, 0, 973, 443, 0, 0, 0, 0, 0, 0, 0, 0, 11, 907, 577, 1584, 0, 553, 222, 1275, 642, 66, 0, 0, 0, 0, 0, 0, 454, 2729, 202, 360, 1272, 278, 506, 439, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 10, 263, 884, 797, 1299, 509, 123, 1491, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5926, 6295, 6465, 5862, 6125, 5837, 6240, 6402] avg=6144.0000 max=6465 max_rank=2 min=5837 min_rank=5 imbalance_ratio=1.052246 max_load_reduction=42687 max_load_reduction_pct=0.868469 moved_to_echo=24604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2310, 1782, 1654, 47, 81, 3, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 281, 1834, 0, 846, 125, 1773, 366, 76, 385, 26, 0, 0, 0, 0, 0, 0, 1023, 472, 21, 533, 696, 242, 572, 1269, 824, 610, 0, 0, 0, 0, 0, 0, 1305, 652, 73, 1051, 0, 1526, 6, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 409, 1749, 728, 328, 1915, 5, 783, 445, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1464, 577, 1389, 1, 589, 234, 1598, 554, 80, 0, 0, 0, 0, 0, 0, 537, 2356, 168, 260, 1007, 305, 662, 485, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1, 408, 756, 498, 1455, 446, 121, 1437, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6104, 5712, 6262, 6235, 6362, 6500, 5780, 6197] avg=6144.0000 max=6500 max_rank=5 min=5712 min_rank=1 imbalance_ratio=1.057943 max_load_reduction=42652 max_load_reduction_pct=0.867757 moved_to_echo=24839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2135, 1856, 2143, 71, 23, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 355, 2132, 0, 783, 161, 1830, 369, 52, 406, 32, 0, 0, 0, 0, 0, 0, 912, 560, 39, 679, 836, 289, 434, 1369, 756, 598, 0, 0, 0, 0, 0, 0, 1122, 652, 91, 1303, 2, 1294, 4, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 429, 1404, 710, 500, 1740, 0, 940, 404, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1001, 678, 1307, 0, 444, 205, 1371, 590, 76, 0, 0, 0, 0, 0, 0, 540, 2483, 210, 312, 1256, 356, 665, 500, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 3, 295, 713, 828, 1123, 377, 137, 1405, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6500, 6120, 6472, 6042, 6127, 5681, 6322, 5888] avg=6144.0000 max=6500 max_rank=0 min=5681 min_rank=5 imbalance_ratio=1.057943 max_load_reduction=42652 max_load_reduction_pct=0.867757 moved_to_echo=24018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1612, 1950, 1802, 101, 220, 8, 1, 195, 0, 0, 0, 0, 0, 0, 0, 0, 290, 1940, 6, 873, 148, 1401, 379, 106, 422, 24, 0, 0, 0, 0, 0, 0, 865, 559, 78, 601, 635, 344, 513, 1356, 770, 592, 0, 0, 0, 0, 0, 0, 1336, 626, 96, 1212, 4, 1502, 5, 1571, 0, 0, 0, 0, 0, 0, 0, 0, 488, 1646, 548, 407, 1790, 13, 746, 478, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1369, 679, 1188, 3, 614, 320, 1825, 506, 75, 0, 0, 0, 0, 0, 0, 638, 2151, 312, 324, 1241, 449, 641, 542, 0, 0, 0, 0, 0, 0, 0, 0, 975, 27, 358, 798, 591, 1322, 414, 128, 1393, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5889, 5589, 6313, 6352, 6116, 6589, 6298, 6006] avg=6144.0000 max=6589 max_rank=5 min=5589 min_rank=1 imbalance_ratio=1.072428 max_load_reduction=42563 max_load_reduction_pct=0.865946 moved_to_echo=25009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2832, 1555, 1532, 97, 23, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 342, 1824, 0, 689, 149, 2674, 601, 79, 336, 31, 0, 0, 0, 0, 0, 0, 1009, 344, 165, 251, 687, 236, 425, 862, 600, 701, 0, 0, 0, 0, 0, 0, 1057, 583, 84, 853, 1, 1228, 3, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 735, 2044, 1096, 182, 1395, 0, 719, 479, 0, 0, 0, 0, 0, 0, 0, 0, 11, 758, 562, 898, 1, 782, 221, 1825, 646, 58, 0, 0, 0, 0, 0, 0, 706, 2745, 118, 308, 1155, 373, 359, 463, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 6, 294, 758, 640, 1604, 205, 133, 1651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6271, 6725, 5280, 5671, 6650, 5762, 6227, 6566] avg=6144.0000 max=6725 max_rank=1 min=5280 min_rank=2 imbalance_ratio=1.094564 max_load_reduction=42427 max_load_reduction_pct=0.863180 moved_to_echo=25205 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1650, 1797, 1673, 80, 132, 0, 2, 262, 0, 0, 0, 0, 0, 0, 0, 0, 307, 2199, 0, 973, 177, 1459, 470, 65, 388, 26, 0, 0, 0, 0, 0, 0, 922, 568, 15, 822, 664, 349, 440, 1367, 811, 626, 0, 0, 0, 0, 0, 0, 1322, 637, 99, 1045, 2, 1597, 10, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 322, 1504, 454, 558, 1885, 0, 793, 470, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1628, 605, 1629, 0, 439, 251, 1501, 539, 79, 0, 0, 0, 0, 0, 0, 387, 2290, 192, 256, 1141, 232, 743, 522, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 7, 276, 784, 482, 1344, 531, 120, 1473, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5596, 6064, 6584, 6374, 5986, 6676, 5763, 6109] avg=6144.0000 max=6676 max_rank=5 min=5596 min_rank=0 imbalance_ratio=1.086589 max_load_reduction=42476 max_load_reduction_pct=0.864176 moved_to_echo=24534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2345, 1778, 1641, 149, 238, 2, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 391, 1995, 0, 665, 112, 2185, 375, 74, 385, 27, 0, 0, 0, 0, 0, 0, 969, 396, 203, 284, 865, 269, 587, 1256, 549, 653, 0, 0, 0, 0, 0, 0, 1321, 675, 76, 978, 0, 1340, 10, 1734, 0, 0, 0, 0, 0, 0, 0, 0, 522, 1748, 599, 279, 1276, 6, 651, 476, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1080, 621, 1206, 0, 923, 203, 1753, 569, 53, 0, 0, 0, 0, 0, 0, 858, 2419, 128, 325, 1215, 495, 486, 476, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 11, 288, 617, 481, 1341, 306, 132, 1538, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6443, 6209, 6031, 6134, 5557, 6427, 6402, 5949] avg=6144.0000 max=6443 max_rank=0 min=5557 min_rank=4 imbalance_ratio=1.048665 max_load_reduction=42709 max_load_reduction_pct=0.868917 moved_to_echo=24335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2402, 1935, 1706, 80, 32, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 332, 1954, 1, 793, 161, 2248, 425, 76, 419, 29, 0, 0, 0, 0, 0, 0, 1110, 386, 104, 448, 672, 271, 417, 1037, 671, 629, 0, 0, 0, 0, 0, 0, 1393, 575, 81, 1306, 3, 1443, 11, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 463, 1747, 869, 347, 1558, 0, 858, 387, 0, 0, 0, 0, 0, 0, 0, 0, 21, 832, 653, 1107, 0, 540, 194, 1651, 617, 65, 0, 0, 0, 0, 0, 0, 483, 2623, 177, 340, 1161, 327, 510, 499, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 18, 248, 687, 600, 1428, 299, 118, 1480, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6423, 6438, 5745, 6482, 6229, 5680, 6120, 6035] avg=6144.0000 max=6482 max_rank=3 min=5680 min_rank=5 imbalance_ratio=1.055013 max_load_reduction=42670 max_load_reduction_pct=0.868123 moved_to_echo=24064 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2135, 1856, 2143, 71, 23, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 355, 2132, 0, 783, 161, 1830, 369, 52, 406, 32, 0, 0, 0, 0, 0, 0, 912, 560, 39, 679, 836, 289, 434, 1369, 756, 598, 0, 0, 0, 0, 0, 0, 1122, 652, 91, 1303, 2, 1294, 4, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 429, 1404, 710, 500, 1740, 0, 940, 404, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1001, 678, 1307, 0, 444, 205, 1371, 590, 76, 0, 0, 0, 0, 0, 0, 540, 2483, 210, 312, 1256, 356, 665, 500, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 3, 295, 713, 828, 1123, 377, 137, 1405, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6500, 6120, 6472, 6042, 6127, 5681, 6322, 5888] avg=6144.0000 max=6500 max_rank=0 min=5681 min_rank=5 imbalance_ratio=1.057943 max_load_reduction=42652 max_load_reduction_pct=0.867757 moved_to_echo=24018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3840, 13440, 5888, 3072, 13184, 128, 6528, 3584, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 3840, 768, 4352, 5888, 2304, 4096, 9856, 5760, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4608, 19840, 1536, 2560, 9472, 2816, 4608, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[17536, 14592, 13824, 768, 896, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8960, 128, 2432, 6016, 4992, 11008, 3200, 1024, 11904, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 15872, 128, 6400, 1280, 15872, 3456, 640, 3200, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[17536, 14592, 13824, 768, 896, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 9088, 4992, 10368, 128, 4992, 1920, 12800, 4736, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4608, 19840, 1536, 2560, 9472, 2816, 4608, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3840, 13440, 5888, 3072, 13184, 128, 6528, 3584, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9856, 5120, 768, 8832, 128, 11520, 128, 13440, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 9088, 4992, 10368, 128, 4992, 1920, 12800, 4736, 640, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[109, 1638, 1306, 2029, 175, 1970, 4, 110, 472, 1010, 179, 1071, 2740, 465, 35, 1959, 535, 14, 223, 833, 17, 14, 199, 1383, 89, 836, 523, 143, 162, 25, 286, 2293, 272, 390, 133, 2064, 124, 31, 573, 5, 3, 501, 6, 102, 2406, 1545, 122, 996, 1456, 620, 533, 310, 200, 4, 2588, 436, 486, 3219, 504, 1713, 3189, 603, 1003, 168] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[110, 3142, 1340, 2038, 444, 1953, 37, 71, 506, 672, 89, 1436, 3292, 148, 29, 2523, 484, 0, 179, 677, 11, 26, 461, 851, 36, 1398, 576, 155, 89, 32, 342, 1967, 324, 431, 47, 1567, 112, 2, 760, 14, 1, 294, 7, 144, 2422, 1724, 37, 1206, 1181, 528, 700, 685, 80, 5, 1540, 313, 432, 2762, 503, 1791, 2433, 925, 869, 199] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[126, 2204, 1388, 1805, 457, 1893, 14, 119, 546, 766, 60, 1203, 3346, 198, 23, 2645, 613, 7, 244, 987, 6, 20, 287, 968, 36, 1312, 532, 156, 68, 11, 195, 1484, 364, 434, 28, 1635, 126, 2, 819, 7, 5, 271, 4, 140, 2735, 1929, 38, 1392, 925, 586, 642, 617, 100, 2, 1627, 406, 407, 3193, 564, 1779, 2840, 880, 785, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[110, 2014, 1403, 1624, 437, 2147, 32, 84, 369, 825, 106, 1129, 3005, 287, 23, 2157, 682, 3, 173, 787, 21, 8, 347, 1142, 48, 1215, 522, 158, 82, 4, 233, 2279, 323, 389, 47, 1780, 149, 9, 768, 4, 0, 369, 8, 149, 2298, 1873, 147, 1001, 942, 650, 487, 338, 102, 6, 1749, 436, 500, 3676, 471, 2183, 3092, 772, 856, 122] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[117, 1868, 1408, 1871, 724, 1930, 35, 123, 405, 743, 102, 1238, 2913, 160, 159, 2245, 521, 225, 320, 611, 64, 43, 401, 803, 59, 1327, 535, 173, 46, 197, 255, 1976, 423, 379, 31, 1850, 135, 3, 857, 24, 2, 360, 23, 218, 2480, 1956, 151, 1023, 1015, 562, 579, 430, 135, 13, 1514, 483, 352, 3675, 485, 1637, 2938, 815, 774, 233] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[126, 1307, 1438, 2242, 335, 1977, 16, 174, 363, 944, 127, 1047, 3382, 423, 64, 1887, 579, 73, 208, 1020, 12, 18, 253, 1562, 161, 1085, 539, 169, 134, 173, 205, 2090, 400, 437, 139, 1898, 188, 85, 526, 4, 3, 417, 8, 110, 2091, 1933, 163, 1143, 1488, 603, 562, 278, 302, 19, 1916, 403, 485, 3179, 534, 1407, 2592, 622, 853, 231] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[98, 1855, 1529, 1909, 518, 2288, 55, 50, 416, 639, 107, 1279, 3385, 179, 116, 2250, 631, 61, 171, 753, 27, 12, 650, 746, 29, 1037, 528, 152, 84, 4, 310, 2353, 441, 420, 57, 1736, 107, 1, 927, 18, 7, 360, 20, 99, 2102, 1525, 86, 1161, 1102, 672, 563, 276, 149, 6, 1347, 432, 450, 3551, 491, 1749, 3061, 928, 938, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[117, 1868, 1408, 1871, 724, 1930, 35, 123, 405, 743, 102, 1238, 2913, 160, 159, 2245, 521, 225, 320, 611, 64, 43, 401, 803, 59, 1327, 535, 173, 46, 197, 255, 1976, 423, 379, 31, 1850, 135, 3, 857, 24, 2, 360, 23, 218, 2480, 1956, 151, 1023, 1015, 562, 579, 430, 135, 13, 1514, 483, 352, 3675, 485, 1637, 2938, 815, 774, 233] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[110, 2014, 1403, 1624, 437, 2147, 32, 84, 369, 825, 106, 1129, 3005, 287, 23, 2157, 682, 3, 173, 787, 21, 8, 347, 1142, 48, 1215, 522, 158, 82, 4, 233, 2279, 323, 389, 47, 1780, 149, 9, 768, 4, 0, 369, 8, 149, 2298, 1873, 147, 1001, 942, 650, 487, 338, 102, 6, 1749, 436, 500, 3676, 471, 2183, 3092, 772, 856, 122] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[126, 2204, 1388, 1805, 457, 1893, 14, 119, 546, 766, 60, 1203, 3346, 198, 23, 2645, 613, 7, 244, 987, 6, 20, 287, 968, 36, 1312, 532, 156, 68, 11, 195, 1484, 364, 434, 28, 1635, 126, 2, 819, 7, 5, 271, 4, 140, 2735, 1929, 38, 1392, 925, 586, 642, 617, 100, 2, 1627, 406, 407, 3193, 564, 1779, 2840, 880, 785, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[110, 3142, 1340, 2038, 444, 1953, 37, 71, 506, 672, 89, 1436, 3292, 148, 29, 2523, 484, 0, 179, 677, 11, 26, 461, 851, 36, 1398, 576, 155, 89, 32, 342, 1967, 324, 431, 47, 1567, 112, 2, 760, 14, 1, 294, 7, 144, 2422, 1724, 37, 1206, 1181, 528, 700, 685, 80, 5, 1540, 313, 432, 2762, 503, 1791, 2433, 925, 869, 199] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[108, 2826, 1525, 1731, 977, 2257, 64, 107, 388, 602, 74, 1353, 3203, 96, 78, 2664, 577, 53, 279, 725, 23, 53, 674, 829, 24, 1437, 551, 137, 63, 10, 225, 1903, 524, 374, 38, 1659, 130, 2, 815, 12, 17, 242, 15, 127, 2410, 1900, 120, 1001, 877, 435, 532, 352, 228, 16, 1135, 348, 499, 3134, 502, 1429, 2846, 881, 792, 144] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[109, 10, 1306, 2029, 175, 1970, 4, 110, 0, 0, 0, 0, 0, 0, 0, 0, 472, 1010, 179, 1071, 876, 465, 35, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 535, 14, 223, 833, 17, 14, 199, 1383, 2924, 0, 0, 0, 0, 0, 0, 0, 89, 836, 523, 143, 162, 25, 286, 2293, 1350, 0, 0, 0, 0, 0, 0, 0, 272, 390, 133, 2064, 124, 31, 573, 5, 1864, 295, 113, 0, 0, 0, 0, 0, 3, 501, 6, 102, 2406, 1545, 122, 996, 277, 0, 0, 0, 0, 0, 0, 0, 1456, 620, 533, 310, 200, 4, 2588, 436, 1093, 165, 0, 0, 0, 0, 0, 0, 486, 0, 504, 1713, 1819, 603, 1003, 168, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5713, 6067, 6142, 5707, 5864, 5958, 7405, 6296] avg=6144.0000 max=7405 max_rank=6 min=5707 min_rank=3 imbalance_ratio=1.205241 max_load_reduction=41747 max_load_reduction_pct=0.849345 moved_to_echo=25523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[110, 17, 1340, 2038, 444, 1953, 37, 71, 0, 0, 0, 0, 0, 0, 0, 0, 506, 672, 89, 1436, 1052, 148, 29, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 179, 677, 11, 26, 461, 851, 2509, 0, 0, 0, 0, 0, 0, 0, 36, 1398, 576, 155, 89, 32, 342, 1967, 2590, 0, 0, 0, 0, 0, 0, 0, 324, 431, 47, 1567, 112, 2, 760, 14, 2240, 253, 218, 0, 0, 0, 0, 0, 1, 294, 7, 144, 2422, 1724, 37, 1206, 212, 0, 0, 0, 0, 0, 0, 0, 1181, 528, 700, 685, 80, 5, 1540, 313, 833, 317, 0, 0, 0, 0, 0, 0, 432, 0, 503, 1791, 1388, 925, 869, 199, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6010, 6455, 5198, 7185, 5968, 6047, 6182, 6107] avg=6144.0000 max=7185 max_rank=3 min=5198 min_rank=2 imbalance_ratio=1.169434 max_load_reduction=41967 max_load_reduction_pct=0.853821 moved_to_echo=24304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[117, 12, 1408, 1871, 724, 1930, 35, 123, 0, 0, 0, 0, 0, 0, 0, 0, 405, 743, 102, 1238, 931, 160, 159, 2245, 0, 0, 0, 0, 0, 0, 0, 0, 521, 225, 320, 611, 64, 43, 401, 803, 3338, 0, 0, 0, 0, 0, 0, 0, 59, 1327, 535, 173, 46, 197, 255, 1976, 1539, 0, 0, 0, 0, 0, 0, 0, 423, 379, 31, 1850, 135, 3, 857, 24, 1982, 337, 129, 0, 0, 0, 0, 0, 2, 360, 23, 218, 2480, 1956, 151, 1023, 256, 0, 0, 0, 0, 0, 0, 0, 1015, 562, 579, 430, 135, 13, 1514, 483, 1007, 188, 0, 0, 0, 0, 0, 0, 352, 0, 485, 1637, 1675, 815, 774, 233, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6220, 5983, 6326, 6107, 6150, 6469, 5926, 5971] avg=6144.0000 max=6469 max_rank=5 min=5926 min_rank=6 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[110, 12, 1403, 1624, 437, 2147, 32, 84, 0, 0, 0, 0, 0, 0, 0, 0, 369, 825, 106, 1129, 961, 287, 23, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 682, 3, 173, 787, 21, 8, 347, 1142, 3340, 0, 0, 0, 0, 0, 0, 0, 48, 1215, 522, 158, 82, 4, 233, 2279, 1660, 0, 0, 0, 0, 0, 0, 0, 323, 389, 47, 1780, 149, 9, 768, 4, 2044, 336, 139, 0, 0, 0, 0, 0, 0, 369, 8, 149, 2298, 1873, 147, 1001, 269, 0, 0, 0, 0, 0, 0, 0, 942, 650, 487, 338, 102, 6, 1749, 436, 1059, 203, 0, 0, 0, 0, 0, 0, 500, 0, 471, 2183, 1764, 772, 856, 122, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5849, 5857, 6503, 6201, 5988, 6114, 5972, 6668] avg=6144.0000 max=6668 max_rank=7 min=5849 min_rank=0 imbalance_ratio=1.085286 max_load_reduction=42484 max_load_reduction_pct=0.864339 moved_to_echo=24742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[98, 11, 1529, 1909, 518, 2288, 55, 50, 0, 0, 0, 0, 0, 0, 0, 0, 416, 639, 107, 1279, 1082, 179, 116, 2250, 0, 0, 0, 0, 0, 0, 0, 0, 631, 61, 171, 753, 27, 12, 650, 746, 3225, 0, 0, 0, 0, 0, 0, 0, 29, 1037, 528, 152, 84, 4, 310, 2353, 1529, 0, 0, 0, 0, 0, 0, 0, 441, 420, 57, 1736, 107, 1, 927, 18, 2303, 326, 128, 0, 0, 0, 0, 0, 7, 360, 20, 99, 2102, 1525, 86, 1161, 266, 0, 0, 0, 0, 0, 0, 0, 1102, 672, 563, 276, 149, 6, 1347, 432, 1049, 187, 0, 0, 0, 0, 0, 0, 450, 0, 491, 1749, 1746, 928, 938, 149, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6458, 6068, 6276, 6026, 6464, 5626, 5783, 6451] avg=6144.0000 max=6464 max_rank=4 min=5626 min_rank=5 imbalance_ratio=1.052083 max_load_reduction=42688 max_load_reduction_pct=0.868490 moved_to_echo=24324 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[126, 0, 1388, 1805, 457, 1893, 14, 119, 0, 0, 0, 0, 0, 0, 0, 0, 546, 766, 60, 1203, 1066, 198, 23, 2645, 0, 0, 0, 0, 0, 0, 0, 0, 613, 7, 244, 987, 6, 20, 287, 968, 2901, 0, 0, 0, 0, 0, 0, 0, 36, 1312, 532, 156, 68, 11, 195, 1484, 1820, 0, 0, 0, 0, 0, 0, 0, 364, 434, 28, 1635, 126, 2, 819, 7, 2280, 292, 157, 0, 0, 0, 0, 0, 5, 271, 4, 140, 2735, 1929, 38, 1392, 251, 0, 0, 0, 0, 0, 0, 0, 925, 586, 642, 617, 100, 2, 1627, 406, 977, 227, 0, 0, 0, 0, 0, 0, 407, 0, 564, 1779, 1612, 880, 785, 151, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5802, 6507, 6033, 5614, 6144, 6765, 6109, 6178] avg=6144.0000 max=6765 max_rank=5 min=5614 min_rank=3 imbalance_ratio=1.101074 max_load_reduction=42387 max_load_reduction_pct=0.862366 moved_to_echo=25196 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[108, 16, 1525, 1731, 977, 2257, 64, 107, 0, 0, 0, 0, 0, 0, 0, 0, 388, 602, 74, 1353, 1024, 96, 78, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 577, 53, 279, 725, 23, 53, 674, 829, 2848, 0, 0, 0, 0, 0, 0, 0, 24, 1437, 551, 137, 63, 10, 225, 1903, 2329, 0, 0, 0, 0, 0, 0, 0, 524, 374, 38, 1659, 130, 2, 815, 12, 2179, 286, 196, 0, 0, 0, 0, 0, 17, 242, 15, 127, 2410, 1900, 120, 1001, 248, 0, 0, 0, 0, 0, 0, 0, 877, 435, 532, 352, 228, 16, 1135, 348, 975, 285, 0, 0, 0, 0, 0, 0, 499, 0, 502, 1429, 1623, 881, 792, 144, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6785, 6279, 6061, 6679, 6215, 6080, 5183, 5870] avg=6144.0000 max=6785 max_rank=0 min=5183 min_rank=6 imbalance_ratio=1.104329 max_load_reduction=42367 max_load_reduction_pct=0.861959 moved_to_echo=23348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[109, 10, 1306, 2029, 175, 1970, 4, 110, 0, 0, 0, 0, 0, 0, 0, 0, 472, 1010, 179, 1071, 876, 465, 35, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 535, 14, 223, 833, 17, 14, 199, 1383, 2924, 0, 0, 0, 0, 0, 0, 0, 89, 836, 523, 143, 162, 25, 286, 2293, 1350, 0, 0, 0, 0, 0, 0, 0, 272, 390, 133, 2064, 124, 31, 573, 5, 1864, 295, 113, 0, 0, 0, 0, 0, 3, 501, 6, 102, 2406, 1545, 122, 996, 277, 0, 0, 0, 0, 0, 0, 0, 1456, 620, 533, 310, 200, 4, 2588, 436, 1093, 165, 0, 0, 0, 0, 0, 0, 486, 0, 504, 1713, 1819, 603, 1003, 168, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5713, 6067, 6142, 5707, 5864, 5958, 7405, 6296] avg=6144.0000 max=7405 max_rank=6 min=5707 min_rank=3 imbalance_ratio=1.205241 max_load_reduction=41747 max_load_reduction_pct=0.849345 moved_to_echo=25523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[126, 7, 1438, 2242, 335, 1977, 16, 174, 0, 0, 0, 0, 0, 0, 0, 0, 363, 944, 127, 1047, 1081, 423, 64, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 579, 73, 208, 1020, 12, 18, 253, 1562, 2888, 0, 0, 0, 0, 0, 0, 0, 161, 1085, 539, 169, 134, 173, 205, 2090, 1077, 0, 0, 0, 0, 0, 0, 0, 400, 437, 139, 1898, 188, 85, 526, 4, 2301, 291, 90, 0, 0, 0, 0, 0, 3, 417, 8, 110, 2091, 1933, 163, 1143, 225, 0, 0, 0, 0, 0, 0, 0, 1488, 603, 562, 278, 302, 19, 1916, 403, 888, 133, 0, 0, 0, 0, 0, 0, 485, 0, 534, 1407, 1479, 622, 853, 231, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6315, 5936, 6613, 5633, 6359, 6093, 6592, 5611] avg=6144.0000 max=6613 max_rank=2 min=5611 min_rank=7 imbalance_ratio=1.076335 max_load_reduction=42539 max_load_reduction_pct=0.865458 moved_to_echo=24655 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[110, 17, 1340, 2038, 444, 1953, 37, 71, 0, 0, 0, 0, 0, 0, 0, 0, 506, 672, 89, 1436, 1052, 148, 29, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 179, 677, 11, 26, 461, 851, 2509, 0, 0, 0, 0, 0, 0, 0, 36, 1398, 576, 155, 89, 32, 342, 1967, 2590, 0, 0, 0, 0, 0, 0, 0, 324, 431, 47, 1567, 112, 2, 760, 14, 2240, 253, 218, 0, 0, 0, 0, 0, 1, 294, 7, 144, 2422, 1724, 37, 1206, 212, 0, 0, 0, 0, 0, 0, 0, 1181, 528, 700, 685, 80, 5, 1540, 313, 833, 317, 0, 0, 0, 0, 0, 0, 432, 0, 503, 1791, 1388, 925, 869, 199, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6010, 6455, 5198, 7185, 5968, 6047, 6182, 6107] avg=6144.0000 max=7185 max_rank=3 min=5198 min_rank=2 imbalance_ratio=1.169434 max_load_reduction=41967 max_load_reduction_pct=0.853821 moved_to_echo=24304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[108, 16, 1525, 1731, 977, 2257, 64, 107, 0, 0, 0, 0, 0, 0, 0, 0, 388, 602, 74, 1353, 1024, 96, 78, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 577, 53, 279, 725, 23, 53, 674, 829, 2848, 0, 0, 0, 0, 0, 0, 0, 24, 1437, 551, 137, 63, 10, 225, 1903, 2329, 0, 0, 0, 0, 0, 0, 0, 524, 374, 38, 1659, 130, 2, 815, 12, 2179, 286, 196, 0, 0, 0, 0, 0, 17, 242, 15, 127, 2410, 1900, 120, 1001, 248, 0, 0, 0, 0, 0, 0, 0, 877, 435, 532, 352, 228, 16, 1135, 348, 975, 285, 0, 0, 0, 0, 0, 0, 499, 0, 502, 1429, 1623, 881, 792, 144, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6785, 6279, 6061, 6679, 6215, 6080, 5183, 5870] avg=6144.0000 max=6785 max_rank=0 min=5183 min_rank=6 imbalance_ratio=1.104329 max_load_reduction=42367 max_load_reduction_pct=0.861959 moved_to_echo=23348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[126, 0, 1388, 1805, 457, 1893, 14, 119, 0, 0, 0, 0, 0, 0, 0, 0, 546, 766, 60, 1203, 1066, 198, 23, 2645, 0, 0, 0, 0, 0, 0, 0, 0, 613, 7, 244, 987, 6, 20, 287, 968, 2901, 0, 0, 0, 0, 0, 0, 0, 36, 1312, 532, 156, 68, 11, 195, 1484, 1820, 0, 0, 0, 0, 0, 0, 0, 364, 434, 28, 1635, 126, 2, 819, 7, 2280, 292, 157, 0, 0, 0, 0, 0, 5, 271, 4, 140, 2735, 1929, 38, 1392, 251, 0, 0, 0, 0, 0, 0, 0, 925, 586, 642, 617, 100, 2, 1627, 406, 977, 227, 0, 0, 0, 0, 0, 0, 407, 0, 564, 1779, 1612, 880, 785, 151, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5802, 6507, 6033, 5614, 6144, 6765, 6109, 6178] avg=6144.0000 max=6765 max_rank=5 min=5614 min_rank=3 imbalance_ratio=1.101074 max_load_reduction=42387 max_load_reduction_pct=0.862366 moved_to_echo=25196 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[98, 11, 1529, 1909, 518, 2288, 55, 50, 0, 0, 0, 0, 0, 0, 0, 0, 416, 639, 107, 1279, 1082, 179, 116, 2250, 0, 0, 0, 0, 0, 0, 0, 0, 631, 61, 171, 753, 27, 12, 650, 746, 3225, 0, 0, 0, 0, 0, 0, 0, 29, 1037, 528, 152, 84, 4, 310, 2353, 1529, 0, 0, 0, 0, 0, 0, 0, 441, 420, 57, 1736, 107, 1, 927, 18, 2303, 326, 128, 0, 0, 0, 0, 0, 7, 360, 20, 99, 2102, 1525, 86, 1161, 266, 0, 0, 0, 0, 0, 0, 0, 1102, 672, 563, 276, 149, 6, 1347, 432, 1049, 187, 0, 0, 0, 0, 0, 0, 450, 0, 491, 1749, 1746, 928, 938, 149, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6458, 6068, 6276, 6026, 6464, 5626, 5783, 6451] avg=6144.0000 max=6464 max_rank=4 min=5626 min_rank=5 imbalance_ratio=1.052083 max_load_reduction=42688 max_load_reduction_pct=0.868490 moved_to_echo=24324 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9088, 4736, 4608, 3328, 1408, 128, 13440, 3328, 7936, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 2816, 128, 1152, 18944, 14464, 896, 8960, 2048, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 6272, 896, 9856, 8192, 2048, 640, 18432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 9728, 4352, 1280, 768, 512, 2176, 16384, 13952, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9088, 4736, 4608, 3328, 1408, 128, 13440, 3328, 7936, 1792, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3712, 0, 4096, 13696, 13184, 6528, 6912, 1408, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4736, 512, 1920, 6400, 256, 256, 3328, 8320, 24064, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3072, 3328, 640, 14208, 1152, 256, 6144, 128, 17280, 2432, 1280, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 9728, 4352, 1280, 768, 512, 2176, 16384, 13952, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 128, 11392, 15360, 4096, 16512, 384, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[709, 1168, 91, 7, 138, 164, 687, 4382, 1282, 127, 50, 0, 396, 67, 4648, 95, 1, 9, 24, 739, 161, 9, 69, 13, 36, 1061, 1525, 0, 2171, 141, 38, 3032, 567, 220, 373, 16, 2409, 363, 0, 212, 4, 789, 1924, 291, 31, 213, 43, 188, 1504, 285, 33, 2, 1646, 33, 2, 4713, 79, 150, 32, 3638, 374, 1840, 1838, 2300] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[693, 871, 359, 10, 151, 67, 679, 3988, 961, 64, 826, 3, 411, 594, 3626, 270, 8, 7, 43, 2284, 145, 4, 151, 74, 37, 1010, 1951, 0, 1284, 485, 53, 2435, 276, 125, 1213, 32, 1820, 239, 7, 105, 41, 1650, 1338, 331, 104, 334, 36, 514, 1485, 405, 413, 107, 2046, 87, 15, 1598, 252, 465, 30, 3977, 488, 1410, 2559, 2106] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[609, 1078, 305, 12, 95, 83, 328, 4227, 750, 81, 871, 1, 397, 741, 4622, 176, 6, 4, 108, 1736, 66, 6, 242, 67, 16, 1135, 1876, 0, 1388, 379, 54, 2266, 300, 74, 1171, 56, 1987, 153, 0, 33, 29, 1411, 836, 328, 25, 277, 15, 516, 1708, 277, 225, 39, 2339, 200, 6, 1484, 249, 468, 14, 3801, 447, 1590, 3010, 2359] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[699, 1311, 134, 1, 176, 131, 766, 3431, 1068, 76, 79, 15, 324, 8, 4577, 143, 4, 22, 30, 1316, 205, 5, 74, 18, 16, 1016, 1924, 0, 1522, 243, 60, 2682, 525, 164, 569, 17, 2611, 609, 2, 234, 6, 976, 1992, 157, 29, 266, 12, 290, 1458, 443, 243, 4, 1902, 43, 21, 3954, 139, 701, 38, 3455, 403, 1695, 1884, 2234] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[577, 948, 290, 11, 82, 61, 576, 4425, 910, 56, 1011, 0, 467, 681, 4584, 152, 7, 7, 53, 1880, 81, 24, 181, 69, 35, 873, 2440, 0, 1149, 204, 58, 2370, 359, 60, 1237, 18, 1841, 216, 0, 53, 8, 1094, 974, 433, 19, 196, 95, 452, 1450, 339, 103, 22, 2306, 126, 22, 2420, 312, 894, 29, 3112, 450, 1674, 2541, 2035] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[609, 1078, 305, 12, 95, 83, 328, 4227, 750, 81, 871, 1, 397, 741, 4622, 176, 6, 4, 108, 1736, 66, 6, 242, 67, 16, 1135, 1876, 0, 1388, 379, 54, 2266, 300, 74, 1171, 56, 1987, 153, 0, 33, 29, 1411, 836, 328, 25, 277, 15, 516, 1708, 277, 225, 39, 2339, 200, 6, 1484, 249, 468, 14, 3801, 447, 1590, 3010, 2359] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[649, 1053, 422, 4, 126, 60, 479, 4393, 778, 81, 534, 0, 372, 398, 4894, 133, 4, 6, 42, 1661, 71, 12, 287, 46, 16, 925, 1629, 2, 1385, 302, 44, 2437, 347, 83, 927, 23, 1943, 202, 1, 54, 16, 1537, 952, 341, 29, 232, 45, 413, 1710, 271, 268, 21, 2224, 53, 5, 2177, 202, 468, 19, 4009, 463, 1823, 2686, 2363] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[627, 710, 760, 8, 127, 57, 468, 3837, 603, 21, 1159, 12, 437, 1094, 4752, 192, 3, 2, 69, 1985, 87, 11, 175, 84, 29, 1072, 2515, 0, 1234, 246, 50, 2084, 368, 68, 1305, 35, 1793, 126, 1, 25, 43, 1583, 572, 335, 46, 404, 156, 573, 1451, 289, 285, 28, 2551, 64, 14, 995, 349, 1026, 27, 3194, 396, 1351, 3221, 1968] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[699, 1311, 134, 1, 176, 131, 766, 3431, 1068, 76, 79, 15, 324, 8, 4577, 143, 4, 22, 30, 1316, 205, 5, 74, 18, 16, 1016, 1924, 0, 1522, 243, 60, 2682, 525, 164, 569, 17, 2611, 609, 2, 234, 6, 976, 1992, 157, 29, 266, 12, 290, 1458, 443, 243, 4, 1902, 43, 21, 3954, 139, 701, 38, 3455, 403, 1695, 1884, 2234] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[815, 1399, 270, 15, 132, 73, 502, 4412, 1269, 110, 284, 1, 442, 261, 4304, 107, 7, 9, 58, 1294, 152, 3, 122, 86, 42, 1047, 1918, 0, 2065, 233, 36, 2907, 367, 106, 868, 24, 2030, 265, 0, 133, 8, 1109, 992, 349, 23, 213, 50, 413, 1293, 308, 87, 25, 2047, 81, 7, 3242, 162, 207, 19, 3639, 380, 1737, 2305, 2288] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[709, 1168, 91, 7, 138, 164, 687, 3487, 0, 0, 0, 0, 0, 0, 0, 0, 1282, 127, 50, 0, 396, 67, 3578, 95, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 24, 739, 161, 9, 69, 13, 3638, 375, 0, 0, 0, 0, 0, 0, 36, 1061, 1525, 0, 2171, 141, 38, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 567, 220, 373, 16, 2409, 363, 0, 212, 895, 1339, 667, 0, 0, 0, 0, 0, 4, 789, 1924, 291, 31, 213, 43, 188, 1504, 669, 403, 0, 0, 0, 0, 0, 1504, 285, 33, 2, 1646, 33, 2, 3374, 0, 0, 0, 0, 0, 0, 0, 0, 79, 150, 32, 0, 374, 1840, 794, 2300, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6451, 5595, 5038, 6500, 7061, 6059, 6879, 5569] avg=6144.0000 max=7061 max_rank=4 min=5038 min_rank=2 imbalance_ratio=1.149251 max_load_reduction=42091 max_load_reduction_pct=0.856344 moved_to_echo=25568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[627, 710, 760, 8, 127, 57, 468, 3837, 603, 21, 1159, 12, 437, 1094, 4752, 192, 3, 2, 69, 1985, 87, 11, 175, 84, 29, 1072, 2515, 0, 1234, 246, 50, 2084, 368, 68, 1305, 35, 1793, 126, 1, 25, 43, 1583, 572, 335, 46, 404, 156, 573, 1451, 289, 285, 28, 2551, 64, 14, 995, 349, 1026, 27, 3194, 396, 1351, 3221, 1968] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[693, 871, 359, 10, 151, 67, 679, 3173, 0, 0, 0, 0, 0, 0, 0, 0, 961, 64, 826, 3, 411, 594, 2791, 270, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 43, 2284, 145, 4, 151, 74, 3977, 523, 0, 0, 0, 0, 0, 0, 37, 1010, 1951, 0, 1284, 485, 53, 1227, 0, 0, 0, 0, 0, 0, 0, 0, 276, 125, 1213, 32, 1820, 239, 7, 105, 815, 454, 521, 0, 0, 0, 0, 0, 41, 1650, 1338, 331, 104, 334, 36, 514, 1208, 931, 314, 0, 0, 0, 0, 0, 1485, 405, 413, 107, 2046, 87, 15, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 252, 465, 30, 0, 488, 1410, 1105, 2106, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6003, 5920, 7216, 6047, 5607, 6801, 5702, 5856] avg=6144.0000 max=7216 max_rank=2 min=5607 min_rank=4 imbalance_ratio=1.174479 max_load_reduction=41936 max_load_reduction_pct=0.853190 moved_to_echo=23966 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[577, 948, 290, 11, 82, 61, 576, 3521, 0, 0, 0, 0, 0, 0, 0, 0, 910, 56, 1011, 0, 467, 681, 3529, 152, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 53, 1880, 81, 24, 181, 69, 3112, 519, 0, 0, 0, 0, 0, 0, 35, 873, 2440, 0, 1149, 204, 58, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 359, 60, 1237, 18, 1841, 216, 0, 53, 904, 688, 658, 0, 0, 0, 0, 0, 8, 1094, 974, 433, 19, 196, 95, 452, 1176, 925, 397, 0, 0, 0, 0, 0, 1450, 339, 103, 22, 2306, 126, 22, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 312, 894, 29, 0, 450, 1674, 1097, 2035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 6806, 5933, 5953, 6034, 5769, 6100, 6491] avg=6144.0000 max=6806 max_rank=1 min=5769 min_rank=5 imbalance_ratio=1.107747 max_load_reduction=42346 max_load_reduction_pct=0.861532 moved_to_echo=24394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[709, 1168, 91, 7, 138, 164, 687, 3487, 0, 0, 0, 0, 0, 0, 0, 0, 1282, 127, 50, 0, 396, 67, 3578, 95, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 24, 739, 161, 9, 69, 13, 3638, 375, 0, 0, 0, 0, 0, 0, 36, 1061, 1525, 0, 2171, 141, 38, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 567, 220, 373, 16, 2409, 363, 0, 212, 895, 1339, 667, 0, 0, 0, 0, 0, 4, 789, 1924, 291, 31, 213, 43, 188, 1504, 669, 403, 0, 0, 0, 0, 0, 1504, 285, 33, 2, 1646, 33, 2, 3374, 0, 0, 0, 0, 0, 0, 0, 0, 79, 150, 32, 0, 374, 1840, 794, 2300, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6451, 5595, 5038, 6500, 7061, 6059, 6879, 5569] avg=6144.0000 max=7061 max_rank=4 min=5038 min_rank=2 imbalance_ratio=1.149251 max_load_reduction=42091 max_load_reduction_pct=0.856344 moved_to_echo=25568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[609, 1078, 305, 12, 95, 83, 328, 3363, 0, 0, 0, 0, 0, 0, 0, 0, 750, 81, 871, 1, 397, 741, 3558, 176, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 108, 1736, 66, 6, 242, 67, 3801, 615, 0, 0, 0, 0, 0, 0, 16, 1135, 1876, 0, 1388, 379, 54, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 300, 74, 1171, 56, 1987, 153, 0, 33, 864, 421, 664, 0, 0, 0, 0, 0, 29, 1411, 836, 328, 25, 277, 15, 516, 1124, 1095, 400, 0, 0, 0, 0, 0, 1708, 277, 225, 39, 2339, 200, 6, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 249, 468, 14, 0, 447, 1590, 1300, 2359, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5873, 6575, 6651, 5990, 5723, 6056, 5857, 6427] avg=6144.0000 max=6651 max_rank=2 min=5723 min_rank=4 imbalance_ratio=1.082520 max_load_reduction=42501 max_load_reduction_pct=0.864685 moved_to_echo=24063 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[699, 1311, 134, 1, 176, 131, 766, 2730, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 76, 79, 15, 324, 8, 3524, 143, 0, 0, 0, 0, 0, 0, 0, 0, 4, 22, 30, 1316, 205, 5, 74, 18, 3455, 385, 0, 0, 0, 0, 0, 0, 16, 1016, 1924, 0, 1522, 243, 60, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 525, 164, 569, 17, 2611, 609, 2, 234, 701, 1124, 657, 0, 0, 0, 0, 0, 6, 976, 1992, 157, 29, 266, 12, 290, 1330, 685, 396, 0, 0, 0, 0, 0, 1458, 443, 243, 4, 1902, 43, 21, 2830, 0, 0, 0, 0, 0, 0, 0, 0, 139, 701, 38, 0, 403, 1695, 814, 2234, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5948, 5237, 5514, 6133, 7213, 6139, 6944, 6024] avg=6144.0000 max=7213 max_rank=4 min=5237 min_rank=1 imbalance_ratio=1.173991 max_load_reduction=41939 max_load_reduction_pct=0.853251 moved_to_echo=26320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[815, 1399, 270, 15, 132, 73, 502, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 110, 284, 1, 442, 261, 3313, 107, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 58, 1294, 152, 3, 122, 86, 3639, 471, 0, 0, 0, 0, 0, 0, 42, 1047, 1918, 0, 2065, 233, 36, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 367, 106, 868, 24, 2030, 265, 0, 133, 901, 921, 618, 0, 0, 0, 0, 0, 8, 1109, 992, 349, 23, 213, 50, 413, 1442, 839, 373, 0, 0, 0, 0, 0, 1293, 308, 87, 25, 2047, 81, 7, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 162, 207, 19, 0, 380, 1737, 995, 2288, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6717, 5787, 5841, 6806, 6233, 5811, 6169, 5788] avg=6144.0000 max=6806 max_rank=3 min=5787 min_rank=1 imbalance_ratio=1.107747 max_load_reduction=42346 max_load_reduction_pct=0.861532 moved_to_echo=24001 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[577, 948, 290, 11, 82, 61, 576, 3521, 0, 0, 0, 0, 0, 0, 0, 0, 910, 56, 1011, 0, 467, 681, 3529, 152, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 53, 1880, 81, 24, 181, 69, 3112, 519, 0, 0, 0, 0, 0, 0, 35, 873, 2440, 0, 1149, 204, 58, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 359, 60, 1237, 18, 1841, 216, 0, 53, 904, 688, 658, 0, 0, 0, 0, 0, 8, 1094, 974, 433, 19, 196, 95, 452, 1176, 925, 397, 0, 0, 0, 0, 0, 1450, 339, 103, 22, 2306, 126, 22, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 312, 894, 29, 0, 450, 1674, 1097, 2035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 6806, 5933, 5953, 6034, 5769, 6100, 6491] avg=6144.0000 max=6806 max_rank=1 min=5769 min_rank=5 imbalance_ratio=1.107747 max_load_reduction=42346 max_load_reduction_pct=0.861532 moved_to_echo=24394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[649, 1053, 422, 4, 126, 60, 479, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 778, 81, 534, 0, 372, 398, 3760, 133, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 42, 1661, 71, 12, 287, 46, 4009, 553, 0, 0, 0, 0, 0, 0, 16, 925, 1629, 2, 1385, 302, 44, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 347, 83, 927, 23, 1943, 202, 1, 54, 902, 623, 707, 0, 0, 0, 0, 0, 16, 1537, 952, 341, 29, 232, 45, 413, 1213, 982, 427, 0, 0, 0, 0, 0, 1710, 271, 268, 21, 2224, 53, 5, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 202, 468, 19, 0, 463, 1823, 1151, 2363, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6056, 6691, 5527, 5812, 6187, 6106, 6489] avg=6144.0000 max=6691 max_rank=2 min=5527 min_rank=3 imbalance_ratio=1.089030 max_load_reduction=42461 max_load_reduction_pct=0.863871 moved_to_echo=24594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[627, 710, 760, 8, 127, 57, 468, 3053, 0, 0, 0, 0, 0, 0, 0, 0, 603, 21, 1159, 12, 437, 1094, 3658, 192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 69, 1985, 87, 11, 175, 84, 3194, 658, 0, 0, 0, 0, 0, 0, 29, 1072, 2515, 0, 1234, 246, 50, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 368, 68, 1305, 35, 1793, 126, 1, 25, 784, 282, 682, 0, 0, 0, 0, 0, 43, 1583, 572, 335, 46, 404, 156, 573, 1034, 1172, 412, 0, 0, 0, 0, 0, 1451, 289, 285, 28, 2551, 64, 14, 713, 0, 0, 0, 0, 0, 0, 0, 0, 349, 1026, 27, 0, 396, 1351, 1391, 1968, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5810, 7176, 6268, 6196, 5469, 6330, 5395, 6508] avg=6144.0000 max=7176 max_rank=1 min=5395 min_rank=6 imbalance_ratio=1.167969 max_load_reduction=41976 max_load_reduction_pct=0.854004 moved_to_echo=23702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[649, 1053, 422, 4, 126, 60, 479, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 778, 81, 534, 0, 372, 398, 3760, 133, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 42, 1661, 71, 12, 287, 46, 4009, 553, 0, 0, 0, 0, 0, 0, 16, 925, 1629, 2, 1385, 302, 44, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 347, 83, 927, 23, 1943, 202, 1, 54, 902, 623, 707, 0, 0, 0, 0, 0, 16, 1537, 952, 341, 29, 232, 45, 413, 1213, 982, 427, 0, 0, 0, 0, 0, 1710, 271, 268, 21, 2224, 53, 5, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 202, 468, 19, 0, 463, 1823, 1151, 2363, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6056, 6691, 5527, 5812, 6187, 6106, 6489] avg=6144.0000 max=6691 max_rank=2 min=5527 min_rank=3 imbalance_ratio=1.089030 max_load_reduction=42461 max_load_reduction_pct=0.863871 moved_to_echo=24594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[256, 10240, 9600, 2688, 384, 2176, 512, 3456, 10112, 7424, 3200, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12160, 2688, 1664, 256, 17152, 768, 128, 14848, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 8192, 15872, 128, 12288, 2304, 512, 10240, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 1024, 7680, 256, 16512, 2176, 128, 896, 6784, 5888, 5248, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 8192, 15872, 128, 12288, 2304, 512, 10240, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 512, 12928, 1024, 128, 1408, 512, 28928, 4224, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 4480, 256, 0, 3456, 13184, 8704, 17664, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5504, 8576, 2688, 128, 1152, 768, 4608, 26368, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 512, 12928, 1024, 128, 1408, 512, 28928, 4224, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 640, 4864, 128, 3328, 3968, 27776, 1280, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[255, 341, 752, 94, 454, 3461, 1, 1162, 250, 648, 386, 4100, 160, 254, 375, 899, 1126, 1429, 24, 231, 1028, 240, 2, 1414, 25, 128, 25, 373, 834, 575, 354, 18, 616, 330, 0, 253, 952, 194, 878, 27, 309, 5, 2374, 70, 171, 116, 1114, 27, 287, 432, 73, 165, 1819, 13, 1032, 1978, 4068, 1, 1959, 0, 5734, 1361, 1169, 207] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[253, 379, 3144, 36, 318, 4222, 9, 520, 211, 213, 115, 3637, 361, 1, 368, 190, 706, 709, 293, 198, 395, 272, 2, 843, 13, 25, 148, 203, 768, 372, 175, 0, 571, 882, 0, 179, 309, 104, 355, 7, 335, 5, 3377, 73, 20, 171, 374, 702, 54, 499, 9, 15, 1963, 7, 354, 1046, 4748, 1, 5028, 2, 6558, 1516, 580, 209] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[246, 345, 765, 88, 353, 4212, 2, 969, 255, 557, 301, 4523, 130, 150, 388, 978, 1135, 1138, 30, 212, 574, 308, 17, 1634, 14, 183, 63, 335, 906, 408, 253, 0, 436, 273, 2, 144, 1089, 115, 856, 15, 368, 8, 2307, 37, 93, 75, 933, 85, 447, 421, 374, 197, 1984, 22, 1084, 1681, 4073, 8, 2788, 0, 5286, 1340, 931, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[255, 378, 3195, 21, 341, 3469, 11, 568, 186, 237, 223, 2786, 342, 145, 309, 85, 705, 570, 685, 271, 161, 284, 1, 1033, 21, 22, 178, 170, 896, 314, 266, 0, 734, 684, 0, 121, 487, 41, 386, 7, 309, 0, 3647, 91, 34, 155, 512, 402, 26, 542, 51, 143, 2374, 2, 465, 1404, 4566, 1, 4740, 0, 6672, 1687, 525, 216] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[261, 416, 621, 51, 411, 3328, 1, 796, 184, 676, 309, 4531, 232, 30, 323, 1105, 1204, 866, 24, 245, 532, 310, 6, 1853, 109, 143, 198, 296, 1216, 456, 288, 26, 735, 323, 1, 219, 1196, 104, 1481, 50, 315, 4, 2188, 74, 58, 153, 966, 8, 227, 405, 359, 251, 2039, 12, 1040, 1590, 3805, 0, 2798, 1, 5239, 1248, 946, 270] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[246, 345, 765, 88, 353, 4212, 2, 969, 255, 557, 301, 4523, 130, 150, 388, 978, 1135, 1138, 30, 212, 574, 308, 17, 1634, 14, 183, 63, 335, 906, 408, 253, 0, 436, 273, 2, 144, 1089, 115, 856, 15, 368, 8, 2307, 37, 93, 75, 933, 85, 447, 421, 374, 197, 1984, 22, 1084, 1681, 4073, 8, 2788, 0, 5286, 1340, 931, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[270, 374, 1261, 46, 415, 4124, 5, 1155, 171, 341, 302, 4372, 201, 60, 384, 863, 951, 706, 59, 215, 769, 395, 2, 1313, 13, 96, 28, 358, 1332, 392, 236, 0, 537, 451, 4, 239, 891, 103, 678, 18, 361, 1, 2391, 92, 86, 120, 672, 183, 159, 454, 38, 199, 1984, 14, 913, 1540, 4147, 0, 3386, 0, 5797, 1395, 862, 228] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[239, 398, 995, 68, 404, 3639, 2, 854, 200, 548, 316, 3988, 200, 36, 345, 507, 1009, 1292, 39, 116, 891, 301, 1, 1584, 118, 84, 50, 371, 1317, 409, 377, 0, 368, 237, 0, 215, 1112, 132, 756, 6, 329, 1, 2682, 81, 107, 147, 697, 26, 146, 460, 59, 233, 2028, 6, 986, 1865, 4210, 1, 2700, 0, 6240, 1337, 1054, 233] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[253, 379, 3144, 36, 318, 4222, 9, 520, 211, 213, 115, 3637, 361, 1, 368, 190, 706, 709, 293, 198, 395, 272, 2, 843, 13, 25, 148, 203, 768, 372, 175, 0, 571, 882, 0, 179, 309, 104, 355, 7, 335, 5, 3377, 73, 20, 171, 374, 702, 54, 499, 9, 15, 1963, 7, 354, 1046, 4748, 1, 5028, 2, 6558, 1516, 580, 209] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[255, 378, 3195, 21, 341, 3469, 11, 568, 186, 237, 223, 2786, 342, 145, 309, 85, 705, 570, 685, 271, 161, 284, 1, 1033, 21, 22, 178, 170, 896, 314, 266, 0, 734, 684, 0, 121, 487, 41, 386, 7, 309, 0, 3647, 91, 34, 155, 512, 402, 26, 542, 51, 143, 2374, 2, 465, 1404, 4566, 1, 4740, 0, 6672, 1687, 525, 216] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[251, 331, 584, 35, 506, 3602, 1, 1041, 169, 798, 494, 4541, 169, 215, 375, 837, 1417, 1193, 13, 153, 838, 440, 5, 1381, 11, 149, 14, 431, 1120, 463, 388, 2, 427, 127, 3, 114, 1137, 141, 984, 16, 360, 5, 2092, 70, 160, 159, 1102, 12, 148, 336, 312, 200, 2128, 32, 1138, 1872, 4232, 0, 1776, 0, 5535, 1427, 866, 274] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[253, 379, 3144, 36, 318, 3106, 9, 520, 0, 0, 0, 0, 0, 0, 0, 0, 211, 213, 115, 3365, 361, 1, 368, 190, 0, 0, 0, 0, 0, 0, 0, 0, 706, 709, 293, 198, 395, 272, 2, 843, 840, 754, 0, 0, 0, 0, 0, 0, 13, 25, 148, 203, 768, 372, 175, 0, 4263, 0, 0, 0, 0, 0, 0, 0, 571, 882, 0, 179, 309, 104, 355, 7, 2295, 1239, 0, 0, 0, 0, 0, 0, 335, 5, 3377, 73, 20, 171, 374, 702, 2125, 0, 0, 0, 0, 0, 0, 0, 54, 499, 9, 15, 1963, 7, 354, 1046, 362, 272, 0, 0, 0, 0, 0, 0, 544, 1, 5028, 2, 0, 1516, 580, 209, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7765, 4824, 5012, 5967, 5941, 7182, 4581, 7880] avg=6144.0000 max=7880 max_rank=7 min=4581 min_rank=6 imbalance_ratio=1.282552 max_load_reduction=41272 max_load_reduction_pct=0.839681 moved_to_echo=25584 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[255, 341, 752, 94, 454, 2546, 1, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 250, 648, 386, 3793, 160, 254, 375, 899, 0, 0, 0, 0, 0, 0, 0, 0, 1126, 1429, 24, 231, 1028, 240, 2, 1414, 719, 618, 0, 0, 0, 0, 0, 0, 25, 128, 25, 373, 834, 575, 354, 18, 3727, 0, 0, 0, 0, 0, 0, 0, 616, 330, 0, 253, 952, 194, 878, 27, 2007, 1061, 0, 0, 0, 0, 0, 0, 309, 5, 2374, 70, 171, 116, 1114, 27, 1820, 0, 0, 0, 0, 0, 0, 0, 287, 432, 73, 165, 1819, 13, 1032, 1978, 297, 307, 0, 0, 0, 0, 0, 0, 468, 1, 1959, 0, 0, 1361, 1169, 207, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5605, 6765, 6831, 6059, 6318, 6006, 6403, 5165] avg=6144.0000 max=6831 max_rank=2 min=5165 min_rank=7 imbalance_ratio=1.111816 max_load_reduction=42321 max_load_reduction_pct=0.861023 moved_to_echo=23892 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[255, 378, 3195, 21, 341, 2552, 11, 568, 0, 0, 0, 0, 0, 0, 0, 0, 186, 237, 223, 2578, 342, 145, 309, 85, 0, 0, 0, 0, 0, 0, 0, 0, 705, 570, 685, 271, 161, 284, 1, 1033, 808, 619, 0, 0, 0, 0, 0, 0, 21, 22, 178, 170, 896, 314, 266, 0, 4338, 0, 0, 0, 0, 0, 0, 0, 734, 684, 0, 121, 487, 41, 386, 7, 2334, 1191, 0, 0, 0, 0, 0, 0, 309, 0, 3647, 91, 34, 155, 512, 402, 2043, 0, 0, 0, 0, 0, 0, 0, 26, 542, 51, 143, 2374, 2, 465, 1404, 298, 208, 0, 0, 0, 0, 0, 0, 524, 1, 4740, 0, 0, 1687, 525, 216, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7321, 4105, 5137, 6205, 5985, 7193, 5513, 7693] avg=6144.0000 max=7693 max_rank=7 min=4105 min_rank=1 imbalance_ratio=1.252116 max_load_reduction=41459 max_load_reduction_pct=0.843486 moved_to_echo=26384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[261, 416, 621, 51, 411, 2449, 1, 796, 0, 0, 0, 0, 0, 0, 0, 0, 184, 676, 309, 4192, 232, 30, 323, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 866, 24, 245, 532, 310, 6, 1853, 673, 594, 0, 0, 0, 0, 0, 0, 109, 143, 198, 296, 1216, 456, 288, 26, 3407, 0, 0, 0, 0, 0, 0, 0, 735, 323, 1, 219, 1196, 104, 1481, 50, 1832, 993, 0, 0, 0, 0, 0, 0, 315, 4, 2188, 74, 58, 153, 966, 8, 1703, 0, 0, 0, 0, 0, 0, 0, 227, 405, 359, 251, 2039, 12, 1040, 1590, 285, 339, 0, 0, 0, 0, 0, 0, 436, 0, 2798, 1, 0, 1248, 946, 270, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5006, 7051, 6307, 6139, 6934, 5469, 6547, 5699] avg=6144.0000 max=7051 max_rank=1 min=5006 min_rank=0 imbalance_ratio=1.147624 max_load_reduction=42101 max_load_reduction_pct=0.856547 moved_to_echo=24649 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[270, 374, 1261, 46, 415, 3034, 5, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 171, 341, 302, 4045, 201, 60, 384, 863, 0, 0, 0, 0, 0, 0, 0, 0, 951, 706, 59, 215, 769, 395, 2, 1313, 733, 736, 0, 0, 0, 0, 0, 0, 13, 96, 28, 358, 1332, 392, 236, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 537, 451, 4, 239, 891, 103, 678, 18, 2028, 1082, 0, 0, 0, 0, 0, 0, 361, 1, 2391, 92, 86, 120, 672, 183, 1856, 0, 0, 0, 0, 0, 0, 0, 159, 454, 38, 199, 1984, 14, 913, 1540, 354, 327, 0, 0, 0, 0, 0, 0, 476, 0, 3386, 0, 0, 1395, 862, 228, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6560, 6367, 5879, 6224, 6031, 5762, 5982, 6347] avg=6144.0000 max=6560 max_rank=0 min=5762 min_rank=5 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24122 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[246, 345, 765, 88, 353, 3099, 2, 969, 0, 0, 0, 0, 0, 0, 0, 0, 255, 557, 301, 4185, 130, 150, 388, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 1138, 30, 212, 574, 308, 17, 1634, 720, 752, 0, 0, 0, 0, 0, 0, 14, 183, 63, 335, 906, 408, 253, 0, 3436, 0, 0, 0, 0, 0, 0, 0, 436, 273, 2, 144, 1089, 115, 856, 15, 1850, 1063, 0, 0, 0, 0, 0, 0, 368, 8, 2307, 37, 93, 75, 933, 85, 1823, 0, 0, 0, 0, 0, 0, 0, 447, 421, 374, 197, 1984, 22, 1084, 1681, 361, 338, 0, 0, 0, 0, 0, 0, 467, 8, 2788, 0, 0, 1340, 931, 208, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5867, 6944, 6520, 5598, 5843, 5729, 6909, 5742] avg=6144.0000 max=6944 max_rank=1 min=5598 min_rank=3 imbalance_ratio=1.130208 max_load_reduction=42208 max_load_reduction_pct=0.858724 moved_to_echo=24223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[239, 398, 995, 68, 404, 2669, 2, 854, 0, 0, 0, 0, 0, 0, 0, 0, 200, 548, 316, 3686, 200, 36, 345, 507, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1292, 39, 116, 891, 301, 1, 1584, 749, 653, 0, 0, 0, 0, 0, 0, 118, 84, 50, 371, 1317, 409, 377, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 368, 237, 0, 215, 1112, 132, 756, 6, 2183, 1102, 0, 0, 0, 0, 0, 0, 329, 1, 2682, 81, 107, 147, 697, 26, 1887, 0, 0, 0, 0, 0, 0, 0, 146, 460, 59, 233, 2028, 6, 986, 1865, 317, 302, 0, 0, 0, 0, 0, 0, 472, 1, 2700, 0, 0, 1337, 1054, 233, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5629, 5838, 6635, 6783, 6111, 5957, 6402, 5797] avg=6144.0000 max=6783 max_rank=3 min=5629 min_rank=0 imbalance_ratio=1.104004 max_load_reduction=42369 max_load_reduction_pct=0.862000 moved_to_echo=24267 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[255, 378, 3195, 21, 341, 2552, 11, 568, 0, 0, 0, 0, 0, 0, 0, 0, 186, 237, 223, 2578, 342, 145, 309, 85, 0, 0, 0, 0, 0, 0, 0, 0, 705, 570, 685, 271, 161, 284, 1, 1033, 808, 619, 0, 0, 0, 0, 0, 0, 21, 22, 178, 170, 896, 314, 266, 0, 4338, 0, 0, 0, 0, 0, 0, 0, 734, 684, 0, 121, 487, 41, 386, 7, 2334, 1191, 0, 0, 0, 0, 0, 0, 309, 0, 3647, 91, 34, 155, 512, 402, 2043, 0, 0, 0, 0, 0, 0, 0, 26, 542, 51, 143, 2374, 2, 465, 1404, 298, 208, 0, 0, 0, 0, 0, 0, 524, 1, 4740, 0, 0, 1687, 525, 216, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7321, 4105, 5137, 6205, 5985, 7193, 5513, 7693] avg=6144.0000 max=7693 max_rank=7 min=4105 min_rank=1 imbalance_ratio=1.252116 max_load_reduction=41459 max_load_reduction_pct=0.843486 moved_to_echo=26384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[270, 374, 1261, 46, 415, 3034, 5, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 171, 341, 302, 4045, 201, 60, 384, 863, 0, 0, 0, 0, 0, 0, 0, 0, 951, 706, 59, 215, 769, 395, 2, 1313, 733, 736, 0, 0, 0, 0, 0, 0, 13, 96, 28, 358, 1332, 392, 236, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 537, 451, 4, 239, 891, 103, 678, 18, 2028, 1082, 0, 0, 0, 0, 0, 0, 361, 1, 2391, 92, 86, 120, 672, 183, 1856, 0, 0, 0, 0, 0, 0, 0, 159, 454, 38, 199, 1984, 14, 913, 1540, 354, 327, 0, 0, 0, 0, 0, 0, 476, 0, 3386, 0, 0, 1395, 862, 228, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6560, 6367, 5879, 6224, 6031, 5762, 5982, 6347] avg=6144.0000 max=6560 max_rank=0 min=5762 min_rank=5 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24122 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[251, 331, 584, 35, 506, 2650, 1, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 169, 798, 494, 4201, 169, 215, 375, 837, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 1193, 13, 153, 838, 440, 5, 1381, 748, 643, 0, 0, 0, 0, 0, 0, 11, 149, 14, 431, 1120, 463, 388, 2, 3599, 0, 0, 0, 0, 0, 0, 0, 427, 127, 3, 114, 1137, 141, 984, 16, 1936, 1104, 0, 0, 0, 0, 0, 0, 360, 5, 2092, 70, 160, 159, 1102, 12, 1894, 0, 0, 0, 0, 0, 0, 0, 148, 336, 312, 200, 2128, 32, 1138, 1872, 309, 340, 0, 0, 0, 0, 0, 0, 486, 0, 1776, 0, 0, 1427, 866, 274, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5399, 7258, 6831, 6177, 5989, 5854, 6815, 4829] avg=6144.0000 max=7258 max_rank=1 min=4829 min_rank=7 imbalance_ratio=1.181315 max_load_reduction=41894 max_load_reduction_pct=0.852336 moved_to_echo=23487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[253, 379, 3144, 36, 318, 3106, 9, 520, 0, 0, 0, 0, 0, 0, 0, 0, 211, 213, 115, 3365, 361, 1, 368, 190, 0, 0, 0, 0, 0, 0, 0, 0, 706, 709, 293, 198, 395, 272, 2, 843, 840, 754, 0, 0, 0, 0, 0, 0, 13, 25, 148, 203, 768, 372, 175, 0, 4263, 0, 0, 0, 0, 0, 0, 0, 571, 882, 0, 179, 309, 104, 355, 7, 2295, 1239, 0, 0, 0, 0, 0, 0, 335, 5, 3377, 73, 20, 171, 374, 702, 2125, 0, 0, 0, 0, 0, 0, 0, 54, 499, 9, 15, 1963, 7, 354, 1046, 362, 272, 0, 0, 0, 0, 0, 0, 544, 1, 5028, 2, 0, 1516, 580, 209, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7765, 4824, 5012, 5967, 5941, 7182, 4581, 7880] avg=6144.0000 max=7880 max_rank=7 min=4581 min_rank=6 imbalance_ratio=1.282552 max_load_reduction=41272 max_load_reduction_pct=0.839681 moved_to_echo=25584 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[239, 398, 995, 68, 404, 2669, 2, 854, 0, 0, 0, 0, 0, 0, 0, 0, 200, 548, 316, 3686, 200, 36, 345, 507, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1292, 39, 116, 891, 301, 1, 1584, 749, 653, 0, 0, 0, 0, 0, 0, 118, 84, 50, 371, 1317, 409, 377, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 368, 237, 0, 215, 1112, 132, 756, 6, 2183, 1102, 0, 0, 0, 0, 0, 0, 329, 1, 2682, 81, 107, 147, 697, 26, 1887, 0, 0, 0, 0, 0, 0, 0, 146, 460, 59, 233, 2028, 6, 986, 1865, 317, 302, 0, 0, 0, 0, 0, 0, 472, 1, 2700, 0, 0, 1337, 1054, 233, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5629, 5838, 6635, 6783, 6111, 5957, 6402, 5797] avg=6144.0000 max=6783 max_rank=3 min=5629 min_rank=0 imbalance_ratio=1.104004 max_load_reduction=42369 max_load_reduction_pct=0.862000 moved_to_echo=24267 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1536, 3584, 1280, 1408, 16384, 128, 7040, 13056, 2688, 2560, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8320, 7936, 1280, 1664, 5248, 2560, 128, 11136, 6016, 5376, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 3072, 11392, 512, 3328, 22144, 128, 7168, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 3328, 128, 1536, 7296, 1024, 6400, 256, 16512, 8960, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3968, 128, 25216, 128, 0, 11392, 7040, 1920, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 128, 21120, 640, 768, 1152, 6400, 1536, 15232, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 3328, 128, 1536, 7296, 1024, 6400, 256, 16512, 8960, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 4096, 2560, 30080, 1920, 896, 2944, 5504, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 896, 768, 2560, 8448, 3456, 2432, 128, 30720, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[66, 85, 336, 1693, 51, 651, 170, 0, 240, 346, 9, 144, 250, 1, 281, 286, 1794, 479, 1056, 1521, 0, 513, 1907, 933, 1402, 946, 348, 2527, 0, 0, 90, 941, 1433, 346, 0, 88, 5, 729, 114, 2820, 990, 0, 1037, 87, 32, 12, 715, 1185, 23, 142, 110, 0, 2424, 6421, 152, 26, 4493, 674, 206, 168, 2668, 380, 676, 1930] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[41, 35, 361, 192, 36, 881, 188, 0, 300, 578, 35, 2, 3383, 1, 447, 35, 1738, 36, 375, 1736, 0, 368, 2071, 1105, 1425, 771, 107, 3768, 44, 0, 234, 988, 447, 490, 7, 116, 38, 521, 227, 4442, 2337, 3, 1209, 110, 2, 367, 1092, 48, 0, 573, 105, 0, 2031, 4526, 155, 175, 3204, 821, 228, 105, 3782, 35, 265, 410] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[72, 73, 292, 1235, 117, 662, 205, 0, 243, 405, 20, 62, 362, 8, 147, 401, 1800, 528, 935, 2131, 1, 428, 1721, 1005, 767, 767, 486, 2710, 5, 1, 89, 837, 865, 383, 0, 80, 8, 692, 110, 3562, 1329, 1, 949, 77, 13, 52, 714, 970, 31, 232, 120, 0, 2440, 6572, 116, 96, 4281, 712, 229, 151, 3277, 415, 577, 1583] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[49, 84, 385, 1424, 25, 683, 198, 0, 288, 404, 26, 72, 1205, 3, 160, 218, 1643, 295, 938, 1856, 0, 569, 2288, 981, 1318, 777, 409, 3243, 4, 1, 90, 905, 897, 529, 1, 148, 13, 543, 132, 3557, 1642, 0, 967, 88, 8, 94, 899, 594, 16, 283, 168, 0, 2105, 6038, 71, 161, 4256, 810, 158, 144, 2642, 179, 404, 1064] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[46, 32, 478, 183, 28, 943, 193, 0, 290, 683, 55, 0, 3653, 2, 404, 30, 1762, 30, 135, 1062, 0, 390, 2502, 1176, 1225, 303, 84, 4087, 27, 0, 319, 1025, 261, 606, 0, 210, 83, 680, 265, 4771, 2621, 5, 1295, 278, 0, 619, 1364, 25, 3, 581, 116, 0, 1971, 3554, 150, 288, 2259, 904, 121, 159, 4150, 62, 244, 360] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[67, 77, 368, 1530, 164, 877, 213, 0, 153, 418, 9, 186, 778, 14, 440, 223, 1706, 370, 823, 1540, 8, 478, 2344, 1008, 956, 1119, 309, 2920, 12, 7, 96, 855, 884, 493, 1, 117, 39, 636, 146, 3799, 1417, 5, 1163, 212, 16, 31, 756, 760, 41, 593, 120, 1, 2146, 5848, 145, 118, 4066, 702, 184, 213, 2672, 139, 618, 1003] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[49, 84, 385, 1424, 25, 683, 198, 0, 288, 404, 26, 72, 1205, 3, 160, 218, 1643, 295, 938, 1856, 0, 569, 2288, 981, 1318, 777, 409, 3243, 4, 1, 90, 905, 897, 529, 1, 148, 13, 543, 132, 3557, 1642, 0, 967, 88, 8, 94, 899, 594, 16, 283, 168, 0, 2105, 6038, 71, 161, 4256, 810, 158, 144, 2642, 179, 404, 1064] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[69, 87, 371, 1342, 19, 812, 236, 0, 219, 457, 4, 106, 746, 1, 506, 209, 1673, 275, 744, 1763, 2, 425, 1630, 1485, 1716, 1154, 313, 2896, 1, 1, 85, 1154, 624, 518, 0, 132, 10, 693, 142, 3573, 1368, 3, 1003, 161, 9, 45, 698, 533, 29, 188, 127, 0, 2358, 5536, 238, 125, 4158, 671, 296, 171, 3188, 221, 449, 1384] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[72, 73, 292, 1235, 117, 662, 205, 0, 243, 405, 20, 62, 362, 8, 147, 401, 1800, 528, 935, 2131, 1, 428, 1721, 1005, 767, 767, 486, 2710, 5, 1, 89, 837, 865, 383, 0, 80, 8, 692, 110, 3562, 1329, 1, 949, 77, 13, 52, 714, 970, 31, 232, 120, 0, 2440, 6572, 116, 96, 4281, 712, 229, 151, 3277, 415, 577, 1583] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[66, 85, 336, 1693, 51, 651, 170, 0, 240, 346, 9, 144, 250, 1, 281, 286, 1794, 479, 1056, 1521, 0, 513, 1907, 933, 1402, 946, 348, 2527, 0, 0, 90, 941, 1433, 346, 0, 88, 5, 729, 114, 2820, 990, 0, 1037, 87, 32, 12, 715, 1185, 23, 142, 110, 0, 2424, 6421, 152, 26, 4493, 674, 206, 168, 2668, 380, 676, 1930] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[41, 35, 361, 192, 36, 881, 188, 0, 300, 578, 35, 2, 3383, 1, 447, 35, 1738, 36, 375, 1736, 0, 368, 2071, 1105, 1425, 771, 107, 3768, 44, 0, 234, 988, 447, 490, 7, 116, 38, 521, 227, 4442, 2337, 3, 1209, 110, 2, 367, 1092, 48, 0, 573, 105, 0, 2031, 4526, 155, 175, 3204, 821, 228, 105, 3782, 35, 265, 410] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[60, 92, 303, 1647, 44, 820, 199, 0, 79, 427, 10, 267, 163, 3, 221, 304, 1746, 422, 835, 1870, 0, 602, 1879, 1254, 859, 1107, 275, 2968, 1, 0, 87, 820, 714, 514, 0, 102, 6, 970, 118, 2642, 936, 0, 1248, 115, 30, 20, 632, 1305, 26, 263, 163, 1, 2634, 6580, 121, 20, 4120, 679, 229, 198, 2790, 234, 677, 1701] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[41, 35, 361, 192, 36, 881, 188, 0, 2032, 538, 318, 0, 0, 0, 0, 0, 300, 578, 35, 2, 3383, 1, 447, 35, 2881, 0, 0, 0, 0, 0, 0, 0, 1738, 36, 375, 1736, 0, 368, 103, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 1425, 771, 107, 3224, 44, 0, 234, 988, 0, 0, 0, 0, 0, 0, 0, 0, 447, 490, 7, 116, 38, 521, 227, 4442, 309, 0, 0, 0, 0, 0, 0, 0, 2337, 3, 1209, 110, 2, 367, 1092, 48, 1430, 235, 0, 0, 0, 0, 0, 0, 0, 573, 105, 0, 2031, 2494, 155, 175, 0, 0, 0, 0, 0, 0, 0, 0, 5, 821, 228, 105, 3782, 35, 265, 410, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4622, 7662, 5461, 6793, 6597, 6833, 5533, 5651] avg=6144.0000 max=7662 max_rank=1 min=4622 min_rank=0 imbalance_ratio=1.247070 max_load_reduction=41490 max_load_reduction_pct=0.844116 moved_to_echo=24614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[69, 87, 371, 1342, 19, 812, 236, 0, 219, 457, 4, 106, 746, 1, 506, 209, 1673, 275, 744, 1763, 2, 425, 1630, 1485, 1716, 1154, 313, 2896, 1, 1, 85, 1154, 624, 518, 0, 132, 10, 693, 142, 3573, 1368, 3, 1003, 161, 9, 45, 698, 533, 29, 188, 127, 0, 2358, 5536, 238, 125, 4158, 671, 296, 171, 3188, 221, 449, 1384] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[72, 73, 292, 1235, 117, 662, 205, 0, 2951, 447, 425, 0, 0, 0, 0, 0, 243, 405, 20, 62, 362, 8, 147, 401, 3849, 0, 0, 0, 0, 0, 0, 0, 1800, 528, 935, 2131, 1, 428, 86, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 767, 767, 486, 2319, 5, 1, 89, 837, 0, 0, 0, 0, 0, 0, 0, 0, 865, 383, 0, 80, 8, 692, 110, 3562, 222, 0, 0, 0, 0, 0, 0, 0, 1329, 1, 949, 77, 13, 52, 714, 970, 1188, 169, 0, 0, 0, 0, 0, 0, 31, 232, 120, 0, 2440, 3621, 116, 96, 0, 0, 0, 0, 0, 0, 0, 0, 7, 712, 229, 151, 3277, 415, 577, 1583, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6479, 5497, 6914, 5271, 5922, 5462, 6656, 6951] avg=6144.0000 max=6951 max_rank=7 min=5271 min_rank=3 imbalance_ratio=1.131348 max_load_reduction=42201 max_load_reduction_pct=0.858582 moved_to_echo=24991 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[67, 77, 368, 1530, 164, 877, 213, 0, 2626, 609, 404, 0, 0, 0, 0, 0, 153, 418, 9, 186, 778, 14, 440, 223, 3656, 0, 0, 0, 0, 0, 0, 0, 1706, 370, 823, 1540, 8, 478, 117, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1119, 309, 2499, 12, 7, 96, 855, 0, 0, 0, 0, 0, 0, 0, 0, 884, 493, 1, 117, 39, 636, 146, 3799, 239, 0, 0, 0, 0, 0, 0, 0, 1417, 5, 1163, 212, 16, 31, 756, 760, 1618, 182, 0, 0, 0, 0, 0, 0, 41, 593, 120, 1, 2146, 3222, 145, 118, 0, 0, 0, 0, 0, 0, 0, 0, 6, 702, 184, 213, 2672, 139, 618, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6935, 5877, 6050, 5853, 6354, 6160, 6386, 5537] avg=6144.0000 max=6935 max_rank=0 min=5537 min_rank=7 imbalance_ratio=1.128743 max_load_reduction=42217 max_load_reduction_pct=0.858907 moved_to_echo=24437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[46, 32, 478, 183, 28, 943, 193, 0, 1596, 650, 224, 0, 0, 0, 0, 0, 290, 683, 55, 0, 3653, 2, 404, 30, 2033, 0, 0, 0, 0, 0, 0, 0, 1762, 30, 135, 1062, 0, 390, 124, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 303, 84, 3497, 27, 0, 319, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 261, 606, 0, 210, 83, 680, 265, 4771, 335, 0, 0, 0, 0, 0, 0, 0, 2621, 5, 1295, 278, 0, 619, 1364, 25, 1728, 255, 0, 0, 0, 0, 0, 0, 3, 581, 116, 0, 1971, 1958, 150, 288, 0, 0, 0, 0, 0, 0, 0, 0, 2, 904, 121, 159, 4150, 62, 244, 360, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4373, 7150, 4679, 6480, 7211, 8190, 5067, 6002] avg=6144.0000 max=8190 max_rank=5 min=4373 min_rank=0 imbalance_ratio=1.333008 max_load_reduction=40962 max_load_reduction_pct=0.833374 moved_to_echo=26470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[66, 85, 336, 1693, 51, 651, 170, 0, 2883, 496, 446, 0, 0, 0, 0, 0, 240, 346, 9, 144, 250, 1, 281, 286, 4040, 0, 0, 0, 0, 0, 0, 0, 1794, 479, 1056, 1521, 0, 513, 94, 933, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 946, 348, 2163, 0, 0, 90, 941, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 346, 0, 88, 5, 729, 114, 2820, 207, 0, 0, 0, 0, 0, 0, 0, 990, 0, 1037, 87, 32, 12, 715, 1185, 1317, 157, 0, 0, 0, 0, 0, 0, 23, 142, 110, 0, 2424, 3538, 152, 26, 0, 0, 0, 0, 0, 0, 0, 0, 7, 674, 206, 168, 2668, 380, 676, 1930, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6877, 5597, 6390, 5890, 5742, 5532, 6415, 6709] avg=6144.0000 max=6877 max_rank=0 min=5532 min_rank=5 imbalance_ratio=1.119303 max_load_reduction=42275 max_load_reduction_pct=0.860087 moved_to_echo=24398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[67, 77, 368, 1530, 164, 877, 213, 0, 2626, 609, 404, 0, 0, 0, 0, 0, 153, 418, 9, 186, 778, 14, 440, 223, 3656, 0, 0, 0, 0, 0, 0, 0, 1706, 370, 823, 1540, 8, 478, 117, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1119, 309, 2499, 12, 7, 96, 855, 0, 0, 0, 0, 0, 0, 0, 0, 884, 493, 1, 117, 39, 636, 146, 3799, 239, 0, 0, 0, 0, 0, 0, 0, 1417, 5, 1163, 212, 16, 31, 756, 760, 1618, 182, 0, 0, 0, 0, 0, 0, 41, 593, 120, 1, 2146, 3222, 145, 118, 0, 0, 0, 0, 0, 0, 0, 0, 6, 702, 184, 213, 2672, 139, 618, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6935, 5877, 6050, 5853, 6354, 6160, 6386, 5537] avg=6144.0000 max=6935 max_rank=0 min=5537 min_rank=7 imbalance_ratio=1.128743 max_load_reduction=42217 max_load_reduction_pct=0.858907 moved_to_echo=24437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[49, 84, 385, 1424, 25, 683, 198, 0, 2711, 595, 423, 0, 0, 0, 0, 0, 288, 404, 26, 72, 1205, 3, 160, 218, 3827, 0, 0, 0, 0, 0, 0, 0, 1643, 295, 938, 1856, 0, 569, 113, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 777, 409, 2776, 4, 1, 90, 905, 0, 0, 0, 0, 0, 0, 0, 0, 897, 529, 1, 148, 13, 543, 132, 3557, 265, 0, 0, 0, 0, 0, 0, 0, 1642, 0, 967, 88, 8, 94, 899, 594, 1580, 202, 0, 0, 0, 0, 0, 0, 16, 283, 168, 0, 2105, 3327, 71, 161, 0, 0, 0, 0, 0, 0, 0, 0, 6, 810, 158, 144, 2642, 179, 404, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6577, 6203, 6395, 6280, 6085, 6074, 6131, 5407] avg=6144.0000 max=6577 max_rank=0 min=5407 min_rank=7 imbalance_ratio=1.070475 max_load_reduction=42575 max_load_reduction_pct=0.866191 moved_to_echo=23697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[69, 87, 371, 1342, 19, 812, 236, 0, 2490, 428, 418, 0, 0, 0, 0, 0, 219, 457, 4, 106, 746, 1, 506, 209, 3740, 0, 0, 0, 0, 0, 0, 0, 1673, 275, 744, 1763, 2, 425, 73, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 1716, 1154, 313, 2472, 1, 1, 85, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 624, 518, 0, 132, 10, 693, 142, 3573, 240, 0, 0, 0, 0, 0, 0, 0, 1368, 3, 1003, 161, 9, 45, 698, 533, 1129, 184, 0, 0, 0, 0, 0, 0, 29, 188, 127, 0, 2358, 3046, 238, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 296, 171, 3188, 221, 449, 1384, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6272, 5988, 6440, 6896, 5932, 5133, 6111, 6380] avg=6144.0000 max=6896 max_rank=3 min=5133 min_rank=5 imbalance_ratio=1.122396 max_load_reduction=42256 max_load_reduction_pct=0.859701 moved_to_echo=23556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[72, 73, 292, 1235, 117, 662, 205, 0, 2951, 447, 425, 0, 0, 0, 0, 0, 243, 405, 20, 62, 362, 8, 147, 401, 3849, 0, 0, 0, 0, 0, 0, 0, 1800, 528, 935, 2131, 1, 428, 86, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 767, 767, 486, 2319, 5, 1, 89, 837, 0, 0, 0, 0, 0, 0, 0, 0, 865, 383, 0, 80, 8, 692, 110, 3562, 222, 0, 0, 0, 0, 0, 0, 0, 1329, 1, 949, 77, 13, 52, 714, 970, 1188, 169, 0, 0, 0, 0, 0, 0, 31, 232, 120, 0, 2440, 3621, 116, 96, 0, 0, 0, 0, 0, 0, 0, 0, 7, 712, 229, 151, 3277, 415, 577, 1583, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6479, 5497, 6914, 5271, 5922, 5462, 6656, 6951] avg=6144.0000 max=6951 max_rank=7 min=5271 min_rank=3 imbalance_ratio=1.131348 max_load_reduction=42201 max_load_reduction_pct=0.858582 moved_to_echo=24991 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[41, 35, 361, 192, 36, 881, 188, 0, 2032, 538, 318, 0, 0, 0, 0, 0, 300, 578, 35, 2, 3383, 1, 447, 35, 2881, 0, 0, 0, 0, 0, 0, 0, 1738, 36, 375, 1736, 0, 368, 103, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 1425, 771, 107, 3224, 44, 0, 234, 988, 0, 0, 0, 0, 0, 0, 0, 0, 447, 490, 7, 116, 38, 521, 227, 4442, 309, 0, 0, 0, 0, 0, 0, 0, 2337, 3, 1209, 110, 2, 367, 1092, 48, 1430, 235, 0, 0, 0, 0, 0, 0, 0, 573, 105, 0, 2031, 2494, 155, 175, 0, 0, 0, 0, 0, 0, 0, 0, 5, 821, 228, 105, 3782, 35, 265, 410, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4622, 7662, 5461, 6793, 6597, 6833, 5533, 5651] avg=6144.0000 max=7662 max_rank=1 min=4622 min_rank=0 imbalance_ratio=1.247070 max_load_reduction=41490 max_load_reduction_pct=0.844116 moved_to_echo=24614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[60, 92, 303, 1647, 44, 820, 199, 0, 2955, 488, 409, 0, 0, 0, 0, 0, 79, 427, 10, 267, 163, 3, 221, 304, 3704, 0, 0, 0, 0, 0, 0, 0, 1746, 422, 835, 1870, 0, 602, 94, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 859, 1107, 275, 2540, 1, 0, 87, 820, 0, 0, 0, 0, 0, 0, 0, 0, 714, 514, 0, 102, 6, 970, 118, 2642, 243, 0, 0, 0, 0, 0, 0, 0, 936, 0, 1248, 115, 30, 20, 632, 1305, 1297, 185, 0, 0, 0, 0, 0, 0, 26, 263, 163, 1, 2634, 3625, 121, 20, 0, 0, 0, 0, 0, 0, 0, 0, 7, 679, 229, 198, 2790, 234, 677, 1701, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7017, 5178, 6823, 5689, 5309, 5768, 6853, 6515] avg=6144.0000 max=7017 max_rank=0 min=5178 min_rank=1 imbalance_ratio=1.142090 max_load_reduction=42135 max_load_reduction_pct=0.857239 moved_to_echo=24445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[69, 87, 371, 1342, 19, 812, 236, 0, 2490, 428, 418, 0, 0, 0, 0, 0, 219, 457, 4, 106, 746, 1, 506, 209, 3740, 0, 0, 0, 0, 0, 0, 0, 1673, 275, 744, 1763, 2, 425, 73, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 1716, 1154, 313, 2472, 1, 1, 85, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 624, 518, 0, 132, 10, 693, 142, 3573, 240, 0, 0, 0, 0, 0, 0, 0, 1368, 3, 1003, 161, 9, 45, 698, 533, 1129, 184, 0, 0, 0, 0, 0, 0, 29, 188, 127, 0, 2358, 3046, 238, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 296, 171, 3188, 221, 449, 1384, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6272, 5988, 6440, 6896, 5932, 5133, 6111, 6380] avg=6144.0000 max=6896 max_rank=3 min=5133 min_rank=5 imbalance_ratio=1.122396 max_load_reduction=42256 max_load_reduction_pct=0.859701 moved_to_echo=23556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12672, 128, 8960, 1152, 128, 1280, 6912, 5504, 11392, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 640, 2944, 9344, 512, 6400, 1664, 0, 20352, 4352, 3072, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13952, 2560, 5888, 13568, 128, 3840, 896, 8960, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9728, 7040, 2432, 21504, 128, 128, 1152, 7552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13952, 2560, 5888, 13568, 128, 3840, 896, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12672, 128, 8960, 1152, 128, 1280, 6912, 5504, 11392, 1664, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 6016, 1664, 1408, 25216, 1792, 3968, 9472, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 2944, 1152, 128, 18176, 24832, 1152, 1024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6144, 3968, 128, 1024, 256, 5504, 1280, 29184, 2176, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1920, 3840, 256, 896, 10624, 128, 2688, 1792, 27776, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6144, 3968, 128, 1024, 256, 5504, 1280, 29184, 2176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 2944, 1152, 128, 18176, 24832, 1152, 1024, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 243, 173, 3, 1084, 2156, 5, 9, 12, 1512, 25, 71, 24, 4, 6, 2016, 467, 12, 7487, 2346, 138, 723, 1258, 737, 1060, 818, 28, 227, 0, 5879, 90, 2853, 1420, 447, 87, 516, 0, 2, 2876, 5, 173, 2, 962, 258, 26, 232, 524, 0, 4390, 0, 10, 547, 263, 0, 11, 1, 1310, 0, 151, 3, 263, 13, 3011, 183] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 228, 155, 0, 2002, 1181, 7, 14, 19, 2008, 55, 161, 12, 177, 6, 1610, 498, 5, 7911, 1018, 264, 1146, 1053, 579, 731, 534, 27, 251, 21, 5973, 86, 2281, 1682, 435, 82, 1152, 0, 5, 1655, 3, 128, 1, 1541, 237, 5, 327, 1146, 5, 3588, 1, 21, 262, 458, 0, 96, 8, 1864, 4, 199, 5, 160, 6, 3969, 94] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 354, 270, 7, 1674, 1637, 35, 60, 58, 1566, 42, 131, 37, 1, 4, 1349, 526, 13, 7545, 1789, 240, 975, 1333, 497, 887, 1200, 92, 311, 12, 5442, 222, 1833, 1586, 319, 96, 1066, 3, 31, 2851, 35, 531, 11, 1141, 273, 16, 404, 748, 4, 3668, 1, 165, 501, 468, 0, 13, 9, 1150, 15, 179, 2, 306, 29, 3191, 198] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 243, 173, 3, 1084, 2156, 5, 9, 12, 1512, 25, 71, 24, 4, 6, 2016, 467, 12, 7487, 2346, 138, 723, 1258, 737, 1060, 818, 28, 227, 0, 5879, 90, 2853, 1420, 447, 87, 516, 0, 2, 2876, 5, 173, 2, 962, 258, 26, 232, 524, 0, 4390, 0, 10, 547, 263, 0, 11, 1, 1310, 0, 151, 3, 263, 13, 3011, 183] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 305, 229, 2, 1203, 1805, 29, 27, 51, 1813, 20, 85, 21, 8, 2, 2353, 415, 12, 7486, 2148, 136, 874, 1211, 774, 833, 1040, 64, 273, 42, 5485, 107, 3159, 1091, 363, 86, 665, 0, 36, 2844, 6, 242, 0, 742, 252, 10, 351, 662, 3, 4142, 0, 60, 652, 321, 0, 8, 2, 1032, 3, 226, 6, 259, 10, 2839, 227] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 228, 155, 0, 2002, 1181, 7, 14, 19, 2008, 55, 161, 12, 177, 6, 1610, 498, 5, 7911, 1018, 264, 1146, 1053, 579, 731, 534, 27, 251, 21, 5973, 86, 2281, 1682, 435, 82, 1152, 0, 5, 1655, 3, 128, 1, 1541, 237, 5, 327, 1146, 5, 3588, 1, 21, 262, 458, 0, 96, 8, 1864, 4, 199, 5, 160, 6, 3969, 94] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1, 173, 166, 2, 981, 2244, 20, 22, 23, 1116, 18, 78, 38, 26, 10, 1760, 602, 5, 7630, 3009, 145, 813, 1489, 732, 1091, 996, 128, 247, 13, 5717, 111, 2798, 1323, 469, 160, 352, 0, 15, 3163, 6, 247, 4, 750, 244, 7, 315, 447, 0, 4254, 0, 43, 488, 263, 0, 5, 2, 775, 0, 93, 10, 361, 8, 2913, 231] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1, 12, 18, 0, 2072, 219, 8, 34, 39, 2663, 139, 250, 4, 325, 2, 614, 572, 38, 8035, 135, 324, 2032, 378, 362, 476, 106, 12, 47, 0, 5129, 93, 1420, 2603, 271, 86, 2878, 6, 4, 159, 11, 46, 2, 3536, 287, 5, 463, 2023, 29, 2948, 0, 6, 22, 169, 0, 1, 7, 3122, 0, 137, 6, 121, 4, 4561, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2110, 56, 0, 5, 31, 3444, 339, 295, 10, 308, 0, 213, 1282, 14, 8041, 8, 344, 2131, 191, 283, 419, 1, 0, 72, 0, 4020, 80, 546, 3165, 232, 109, 2761, 1, 1, 97, 3, 4, 0, 3523, 286, 0, 560, 2914, 76, 2785, 0, 39, 0, 189, 0, 0, 46, 2463, 0, 131, 32, 107, 3, 5307, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 133, 58, 4, 2100, 1230, 14, 13, 47, 2394, 49, 134, 18, 58, 9, 1650, 527, 7, 7825, 889, 261, 1121, 1058, 474, 841, 694, 14, 241, 0, 5920, 90, 2056, 2165, 431, 98, 885, 0, 0, 1513, 8, 237, 1, 1847, 279, 4, 378, 953, 1, 4083, 0, 42, 382, 196, 0, 3, 3, 1730, 0, 212, 6, 149, 10, 3507, 100] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1, 173, 166, 2, 981, 2244, 20, 22, 23, 1116, 18, 78, 38, 26, 10, 1760, 602, 5, 7630, 3009, 145, 813, 1489, 732, 1091, 996, 128, 247, 13, 5717, 111, 2798, 1323, 469, 160, 352, 0, 15, 3163, 6, 247, 4, 750, 244, 7, 315, 447, 0, 4254, 0, 43, 488, 263, 0, 5, 2, 775, 0, 93, 10, 361, 8, 2913, 231] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 243, 173, 3, 1084, 2156, 5, 9, 2725, 0, 0, 0, 0, 0, 0, 0, 12, 1512, 25, 71, 24, 4, 6, 2016, 1442, 789, 0, 0, 0, 0, 0, 0, 467, 12, 1058, 2346, 138, 723, 1258, 737, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 818, 28, 227, 0, 2361, 90, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 447, 87, 516, 0, 2, 2876, 5, 638, 0, 0, 0, 0, 0, 0, 0, 173, 2, 962, 258, 26, 232, 524, 0, 2262, 0, 0, 0, 0, 0, 0, 0, 4390, 0, 10, 547, 263, 0, 11, 1, 1823, 0, 0, 0, 0, 0, 0, 0, 1310, 0, 151, 3, 263, 13, 3011, 183, 268, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6398, 5901, 6739, 7437, 5991, 4439, 7045, 5202] avg=6144.0000 max=7437 max_rank=3 min=4439 min_rank=5 imbalance_ratio=1.210449 max_load_reduction=41715 max_load_reduction_pct=0.848694 moved_to_echo=22677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 354, 270, 7, 1674, 1637, 35, 60, 2746, 0, 0, 0, 0, 0, 0, 0, 58, 1566, 42, 131, 37, 1, 4, 1349, 1453, 731, 0, 0, 0, 0, 0, 0, 526, 13, 1066, 1789, 240, 975, 1333, 497, 0, 0, 0, 0, 0, 0, 0, 0, 887, 1200, 92, 311, 12, 2185, 222, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 1586, 319, 96, 1066, 3, 31, 2851, 35, 591, 0, 0, 0, 0, 0, 0, 0, 531, 11, 1141, 273, 16, 404, 748, 4, 2280, 0, 0, 0, 0, 0, 0, 0, 3668, 1, 165, 501, 468, 0, 13, 9, 1687, 0, 0, 0, 0, 0, 0, 0, 1150, 15, 179, 2, 306, 29, 3191, 198, 248, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6783, 5372, 6439, 6742, 6578, 5408, 6512, 5318] avg=6144.0000 max=6783 max_rank=0 min=5318 min_rank=7 imbalance_ratio=1.104004 max_load_reduction=42369 max_load_reduction_pct=0.862000 moved_to_echo=23816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 243, 173, 3, 1084, 2156, 5, 9, 2725, 0, 0, 0, 0, 0, 0, 0, 12, 1512, 25, 71, 24, 4, 6, 2016, 1442, 789, 0, 0, 0, 0, 0, 0, 467, 12, 1058, 2346, 138, 723, 1258, 737, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 818, 28, 227, 0, 2361, 90, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 447, 87, 516, 0, 2, 2876, 5, 638, 0, 0, 0, 0, 0, 0, 0, 173, 2, 962, 258, 26, 232, 524, 0, 2262, 0, 0, 0, 0, 0, 0, 0, 4390, 0, 10, 547, 263, 0, 11, 1, 1823, 0, 0, 0, 0, 0, 0, 0, 1310, 0, 151, 3, 263, 13, 3011, 183, 268, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6398, 5901, 6739, 7437, 5991, 4439, 7045, 5202] avg=6144.0000 max=7437 max_rank=3 min=4439 min_rank=5 imbalance_ratio=1.210449 max_load_reduction=41715 max_load_reduction_pct=0.848694 moved_to_echo=22677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 305, 229, 2, 1203, 1805, 29, 27, 2725, 0, 0, 0, 0, 0, 0, 0, 51, 1813, 20, 85, 21, 8, 2, 2353, 1442, 736, 0, 0, 0, 0, 0, 0, 415, 12, 1057, 2148, 136, 874, 1211, 774, 0, 0, 0, 0, 0, 0, 0, 0, 833, 1040, 64, 273, 42, 2204, 107, 3159, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 363, 86, 665, 0, 36, 2844, 6, 595, 0, 0, 0, 0, 0, 0, 0, 242, 0, 742, 252, 10, 351, 662, 3, 2262, 0, 0, 0, 0, 0, 0, 0, 4142, 0, 60, 652, 321, 0, 8, 2, 1700, 0, 0, 0, 0, 0, 0, 0, 1032, 3, 226, 6, 259, 10, 2839, 227, 250, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6325, 6531, 6627, 7722, 5686, 4524, 6885, 4852] avg=6144.0000 max=7722 max_rank=3 min=4524 min_rank=5 imbalance_ratio=1.256836 max_load_reduction=41430 max_load_reduction_pct=0.842896 moved_to_echo=21947 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2110, 56, 0, 5, 2927, 0, 0, 0, 0, 0, 0, 0, 31, 3444, 339, 295, 10, 308, 0, 213, 1549, 540, 0, 0, 0, 0, 0, 0, 1282, 14, 1135, 8, 344, 2131, 191, 283, 0, 0, 0, 0, 0, 0, 0, 0, 419, 1, 0, 72, 0, 1615, 80, 546, 0, 0, 0, 0, 0, 0, 0, 0, 3165, 232, 109, 2761, 1, 1, 97, 3, 436, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3523, 286, 0, 560, 2914, 76, 2430, 0, 0, 0, 0, 0, 0, 0, 2785, 0, 39, 0, 189, 0, 0, 46, 1246, 0, 0, 0, 0, 0, 0, 0, 2463, 0, 131, 32, 107, 3, 5307, 74, 183, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5099, 6729, 5388, 2733, 6805, 9793, 4305, 8300] avg=6144.0000 max=9793 max_rank=5 min=2733 min_rank=3 imbalance_ratio=1.593913 max_load_reduction=39359 max_load_reduction_pct=0.800761 moved_to_echo=29203 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 228, 155, 0, 2002, 1181, 7, 14, 2880, 0, 0, 0, 0, 0, 0, 0, 19, 2008, 55, 161, 12, 177, 6, 1610, 1523, 802, 0, 0, 0, 0, 0, 0, 498, 5, 1117, 1018, 264, 1146, 1053, 579, 0, 0, 0, 0, 0, 0, 0, 0, 731, 534, 27, 251, 21, 2398, 86, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 1682, 435, 82, 1152, 0, 5, 1655, 3, 648, 0, 0, 0, 0, 0, 0, 0, 128, 1, 1541, 237, 5, 327, 1146, 5, 2391, 0, 0, 0, 0, 0, 0, 0, 3588, 1, 21, 262, 458, 0, 96, 8, 1852, 0, 0, 0, 0, 0, 0, 0, 1864, 4, 199, 5, 160, 6, 3969, 94, 273, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6467, 6373, 5680, 6329, 5662, 5781, 6286, 6574] avg=6144.0000 max=6574 max_rank=7 min=5662 min_rank=4 imbalance_ratio=1.069987 max_load_reduction=42578 max_load_reduction_pct=0.866252 moved_to_echo=24303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 12, 18, 0, 2072, 219, 8, 34, 2925, 0, 0, 0, 0, 0, 0, 0, 39, 2663, 139, 250, 4, 325, 2, 614, 1547, 689, 0, 0, 0, 0, 0, 0, 572, 38, 1135, 135, 324, 2032, 378, 362, 0, 0, 0, 0, 0, 0, 0, 0, 476, 106, 12, 47, 0, 2059, 93, 1420, 0, 0, 0, 0, 0, 0, 0, 0, 2603, 271, 86, 2878, 6, 4, 159, 11, 557, 0, 0, 0, 0, 0, 0, 0, 46, 2, 3536, 287, 5, 463, 2023, 29, 2428, 0, 0, 0, 0, 0, 0, 0, 2948, 0, 6, 22, 169, 0, 1, 7, 1590, 0, 0, 0, 0, 0, 0, 0, 3122, 0, 137, 6, 121, 4, 4561, 80, 234, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5289, 6272, 4976, 4213, 6575, 8819, 4743, 8265] avg=6144.0000 max=8819 max_rank=5 min=4213 min_rank=3 imbalance_ratio=1.435384 max_load_reduction=40333 max_load_reduction_pct=0.820577 moved_to_echo=28402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2110, 56, 0, 5, 2927, 0, 0, 0, 0, 0, 0, 0, 31, 3444, 339, 295, 10, 308, 0, 213, 1549, 540, 0, 0, 0, 0, 0, 0, 1282, 14, 1135, 8, 344, 2131, 191, 283, 0, 0, 0, 0, 0, 0, 0, 0, 419, 1, 0, 72, 0, 1615, 80, 546, 0, 0, 0, 0, 0, 0, 0, 0, 3165, 232, 109, 2761, 1, 1, 97, 3, 436, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3523, 286, 0, 560, 2914, 76, 2430, 0, 0, 0, 0, 0, 0, 0, 2785, 0, 39, 0, 189, 0, 0, 46, 1246, 0, 0, 0, 0, 0, 0, 0, 2463, 0, 131, 32, 107, 3, 5307, 74, 183, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5099, 6729, 5388, 2733, 6805, 9793, 4305, 8300] avg=6144.0000 max=9793 max_rank=5 min=2733 min_rank=3 imbalance_ratio=1.593913 max_load_reduction=39359 max_load_reduction_pct=0.800761 moved_to_echo=29203 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 305, 229, 2, 1203, 1805, 29, 27, 2725, 0, 0, 0, 0, 0, 0, 0, 51, 1813, 20, 85, 21, 8, 2, 2353, 1442, 736, 0, 0, 0, 0, 0, 0, 415, 12, 1057, 2148, 136, 874, 1211, 774, 0, 0, 0, 0, 0, 0, 0, 0, 833, 1040, 64, 273, 42, 2204, 107, 3159, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 363, 86, 665, 0, 36, 2844, 6, 595, 0, 0, 0, 0, 0, 0, 0, 242, 0, 742, 252, 10, 351, 662, 3, 2262, 0, 0, 0, 0, 0, 0, 0, 4142, 0, 60, 652, 321, 0, 8, 2, 1700, 0, 0, 0, 0, 0, 0, 0, 1032, 3, 226, 6, 259, 10, 2839, 227, 250, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6325, 6531, 6627, 7722, 5686, 4524, 6885, 4852] avg=6144.0000 max=7722 max_rank=3 min=4524 min_rank=5 imbalance_ratio=1.256836 max_load_reduction=41430 max_load_reduction_pct=0.842896 moved_to_echo=21947 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 133, 58, 4, 2100, 1230, 14, 13, 2853, 0, 0, 0, 0, 0, 0, 0, 47, 2394, 49, 134, 18, 58, 9, 1650, 1511, 798, 0, 0, 0, 0, 0, 0, 527, 7, 1093, 889, 261, 1121, 1058, 474, 0, 0, 0, 0, 0, 0, 0, 0, 841, 694, 14, 241, 0, 2360, 90, 2056, 0, 0, 0, 0, 0, 0, 0, 0, 2165, 431, 98, 885, 0, 0, 1513, 8, 647, 0, 0, 0, 0, 0, 0, 0, 237, 1, 1847, 279, 4, 378, 953, 1, 2368, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 42, 382, 196, 0, 3, 3, 1840, 0, 0, 0, 0, 0, 0, 0, 1730, 0, 212, 6, 149, 10, 3507, 100, 275, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6405, 6668, 5430, 6296, 5747, 6068, 6549, 5989] avg=6144.0000 max=6668 max_rank=1 min=5430 min_rank=2 imbalance_ratio=1.085286 max_load_reduction=42484 max_load_reduction_pct=0.864339 moved_to_echo=24353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 173, 166, 2, 981, 2244, 20, 22, 2777, 0, 0, 0, 0, 0, 0, 0, 23, 1116, 18, 78, 38, 26, 10, 1760, 1469, 768, 0, 0, 0, 0, 0, 0, 602, 5, 1078, 3009, 145, 813, 1489, 732, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 996, 128, 247, 13, 2296, 111, 2798, 0, 0, 0, 0, 0, 0, 0, 0, 1323, 469, 160, 352, 0, 15, 3163, 6, 620, 0, 0, 0, 0, 0, 0, 0, 247, 4, 750, 244, 7, 315, 447, 0, 2306, 0, 0, 0, 0, 0, 0, 0, 4254, 0, 43, 488, 263, 0, 5, 2, 1772, 0, 0, 0, 0, 0, 0, 0, 775, 0, 93, 10, 361, 8, 2913, 231, 261, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6386, 5306, 7873, 7680, 6108, 4320, 6827, 4652] avg=6144.0000 max=7873 max_rank=2 min=4320 min_rank=5 imbalance_ratio=1.281413 max_load_reduction=41279 max_load_reduction_pct=0.839823 moved_to_echo=21907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 133, 58, 4, 2100, 1230, 14, 13, 2853, 0, 0, 0, 0, 0, 0, 0, 47, 2394, 49, 134, 18, 58, 9, 1650, 1511, 798, 0, 0, 0, 0, 0, 0, 527, 7, 1093, 889, 261, 1121, 1058, 474, 0, 0, 0, 0, 0, 0, 0, 0, 841, 694, 14, 241, 0, 2360, 90, 2056, 0, 0, 0, 0, 0, 0, 0, 0, 2165, 431, 98, 885, 0, 0, 1513, 8, 647, 0, 0, 0, 0, 0, 0, 0, 237, 1, 1847, 279, 4, 378, 953, 1, 2368, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 42, 382, 196, 0, 3, 3, 1840, 0, 0, 0, 0, 0, 0, 0, 1730, 0, 212, 6, 149, 10, 3507, 100, 275, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6405, 6668, 5430, 6296, 5747, 6068, 6549, 5989] avg=6144.0000 max=6668 max_rank=1 min=5430 min_rank=2 imbalance_ratio=1.085286 max_load_reduction=42484 max_load_reduction_pct=0.864339 moved_to_echo=24353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4992, 128, 8832, 11392, 1920, 9856, 8064, 4480, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 128, 14080, 2176, 128, 3072, 9472, 128, 18816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[384, 16640, 768, 1280, 256, 1024, 128, 11648, 12032, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 5504, 384, 1792, 128, 17536, 896, 17024, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1536, 1152, 128, 13312, 10624, 128, 256, 22656, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13568, 128, 1408, 128, 1792, 128, 29312, 1280, 2048, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 5504, 384, 1792, 128, 17536, 896, 17024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[384, 16640, 768, 1280, 256, 1024, 128, 11648, 12032, 5888, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[15104, 3072, 896, 10368, 128, 128, 15232, 128, 4736, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[29952, 128, 512, 2944, 2432, 0, 256, 128, 13568, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[5295, 736, 1864, 14, 275, 24, 1104, 469, 366, 0, 0, 476, 80, 273, 122, 0, 183, 0, 20, 434, 1509, 97, 56, 385, 1223, 4, 1404, 164, 0, 3, 1859, 2, 1851, 2314, 55, 1472, 6, 2184, 674, 3, 53, 0, 2311, 9, 42, 2857, 334, 0, 14, 115, 5554, 1, 168, 194, 332, 52, 489, 7431, 0, 2, 0, 53, 1605, 536] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3897, 1438, 2057, 248, 173, 65, 1934, 21, 956, 0, 38, 157, 271, 356, 2, 9, 819, 1, 244, 19, 852, 118, 8, 30, 2802, 3, 892, 226, 2, 262, 633, 188, 1417, 3866, 633, 3614, 117, 2427, 323, 5, 135, 33, 4237, 303, 12, 1869, 308, 7, 0, 375, 1792, 2, 71, 1223, 181, 0, 1201, 4122, 10, 60, 1, 1341, 329, 417] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[3043, 1043, 2133, 206, 89, 149, 1336, 0, 842, 0, 14, 52, 85, 271, 0, 0, 1023, 0, 252, 16, 754, 105, 10, 36, 2559, 9, 667, 254, 2, 224, 609, 236, 1426, 3568, 1625, 4138, 72, 3332, 49, 1, 134, 9, 5209, 376, 4, 1735, 526, 1, 0, 802, 1021, 0, 22, 1897, 286, 3, 965, 4406, 0, 27, 3, 845, 380, 271] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[2992, 1324, 2167, 205, 106, 149, 1576, 17, 743, 0, 69, 166, 244, 277, 1, 3, 650, 2, 741, 64, 605, 80, 35, 19, 3345, 2, 429, 217, 2, 229, 591, 91, 1215, 4175, 1560, 3852, 233, 2950, 53, 0, 67, 20, 4836, 365, 0, 2192, 158, 2, 0, 517, 1369, 0, 19, 1136, 369, 0, 949, 3558, 0, 119, 0, 1264, 725, 308] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4418, 1141, 1986, 111, 195, 48, 1467, 75, 626, 0, 18, 174, 83, 297, 23, 0, 541, 0, 136, 43, 937, 95, 14, 38, 2433, 0, 875, 248, 0, 156, 772, 41, 1303, 3321, 369, 3027, 79, 3214, 435, 0, 138, 3, 4711, 345, 0, 1990, 196, 0, 0, 294, 2966, 0, 93, 691, 277, 1, 1289, 5303, 0, 51, 1, 716, 828, 520] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4536, 692, 2390, 5, 353, 5, 1538, 1517, 187, 0, 0, 700, 59, 574, 350, 0, 21, 0, 0, 385, 1800, 119, 80, 679, 861, 0, 1644, 263, 0, 1, 2429, 0, 819, 967, 3, 1157, 0, 1389, 645, 10, 18, 0, 1415, 1, 138, 2833, 538, 0, 19, 123, 6299, 0, 445, 70, 232, 143, 542, 7766, 0, 0, 0, 6, 1617, 769] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4672, 959, 2449, 230, 168, 33, 1174, 27, 478, 1, 3, 174, 103, 260, 5, 0, 432, 0, 74, 77, 846, 101, 111, 50, 2716, 2, 986, 248, 0, 149, 983, 4, 1161, 2809, 156, 2926, 23, 4061, 68, 3, 108, 9, 4192, 343, 1, 1994, 785, 0, 0, 264, 2769, 0, 101, 1145, 313, 3, 921, 5844, 0, 32, 0, 434, 905, 267] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3824, 429, 2266, 177, 1, 47, 1120, 1, 749, 0, 22, 65, 342, 243, 0, 0, 1413, 0, 373, 13, 775, 87, 12, 10, 2303, 0, 694, 247, 0, 193, 745, 36, 1283, 3325, 804, 4135, 149, 3985, 54, 0, 117, 23, 5505, 353, 0, 1750, 259, 0, 0, 449, 1269, 0, 43, 1670, 251, 0, 1147, 4505, 0, 57, 0, 801, 820, 211] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1236, 736, 1864, 14, 275, 24, 1104, 469, 0, 0, 0, 0, 0, 0, 0, 0, 366, 0, 0, 476, 80, 273, 122, 0, 4059, 1206, 0, 0, 0, 0, 0, 0, 183, 0, 20, 434, 1509, 97, 56, 385, 3049, 1108, 87, 0, 0, 0, 0, 0, 1223, 4, 1404, 164, 0, 3, 1859, 2, 481, 256, 0, 0, 0, 0, 0, 0, 1851, 0, 55, 421, 6, 2184, 674, 3, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 1830, 9, 42, 2857, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 115, 5554, 1, 168, 194, 332, 52, 708, 0, 0, 0, 0, 0, 0, 0, 489, 4382, 0, 2, 0, 53, 1605, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5722, 6582, 6928, 5396, 5194, 5125, 7138, 7067] avg=6144.0000 max=7138 max_rank=6 min=5125 min_rank=5 imbalance_ratio=1.161784 max_load_reduction=42014 max_load_reduction_pct=0.854777 moved_to_echo=24524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4536, 692, 2390, 5, 353, 5, 1538, 1517, 187, 0, 0, 700, 59, 574, 350, 0, 21, 0, 0, 385, 1800, 119, 80, 679, 861, 0, 1644, 263, 0, 1, 2429, 0, 819, 967, 3, 1157, 0, 1389, 645, 10, 18, 0, 1415, 1, 138, 2833, 538, 0, 19, 123, 6299, 0, 445, 70, 232, 143, 542, 7766, 0, 0, 0, 6, 1617, 769] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[5295, 736, 1864, 14, 275, 24, 1104, 469, 366, 0, 0, 476, 80, 273, 122, 0, 183, 0, 20, 434, 1509, 97, 56, 385, 1223, 4, 1404, 164, 0, 3, 1859, 2, 1851, 2314, 55, 1472, 6, 2184, 674, 3, 53, 0, 2311, 9, 42, 2857, 334, 0, 14, 115, 5554, 1, 168, 194, 332, 52, 489, 7431, 0, 2, 0, 53, 1605, 536] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[2992, 1324, 2167, 205, 106, 149, 1576, 17, 743, 0, 69, 166, 244, 277, 1, 3, 650, 2, 741, 64, 605, 80, 35, 19, 3345, 2, 429, 217, 2, 229, 591, 91, 1215, 4175, 1560, 3852, 233, 2950, 53, 0, 67, 20, 4836, 365, 0, 2192, 158, 2, 0, 517, 1369, 0, 19, 1136, 369, 0, 949, 3558, 0, 119, 0, 1264, 725, 308] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4418, 1141, 1986, 111, 195, 48, 1467, 75, 626, 0, 18, 174, 83, 297, 23, 0, 541, 0, 136, 43, 937, 95, 14, 38, 2433, 0, 875, 248, 0, 156, 772, 41, 1303, 3321, 369, 3027, 79, 3214, 435, 0, 138, 3, 4711, 345, 0, 1990, 196, 0, 0, 294, 2966, 0, 93, 691, 277, 1, 1289, 5303, 0, 51, 1, 716, 828, 520] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[910, 1438, 2057, 248, 173, 65, 1934, 21, 0, 0, 0, 0, 0, 0, 0, 0, 956, 0, 38, 157, 271, 356, 2, 9, 2987, 2015, 0, 0, 0, 0, 0, 0, 819, 1, 244, 19, 852, 118, 8, 30, 1691, 1851, 215, 0, 0, 0, 0, 0, 2802, 3, 892, 226, 2, 262, 633, 188, 883, 629, 0, 0, 0, 0, 0, 0, 1417, 0, 633, 1031, 117, 2427, 323, 5, 0, 0, 0, 0, 0, 0, 0, 0, 135, 33, 3354, 303, 12, 1869, 308, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 1792, 2, 71, 1223, 181, 0, 1739, 0, 0, 0, 0, 0, 0, 0, 1201, 2431, 10, 60, 1, 1341, 329, 417, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6846, 6791, 5848, 6520, 5953, 6021, 5383, 5790] avg=6144.0000 max=6846 max_rank=0 min=5383 min_rank=6 imbalance_ratio=1.114258 max_load_reduction=42306 max_load_reduction_pct=0.860718 moved_to_echo=23147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1032, 1141, 1986, 111, 195, 48, 1467, 75, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 18, 174, 83, 297, 23, 0, 3386, 1731, 0, 0, 0, 0, 0, 0, 541, 0, 136, 43, 937, 95, 14, 38, 2175, 1590, 180, 0, 0, 0, 0, 0, 2433, 0, 875, 248, 0, 156, 772, 41, 981, 526, 0, 0, 0, 0, 0, 0, 1303, 0, 369, 864, 79, 3214, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 3, 3730, 345, 0, 1990, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 2966, 0, 93, 691, 277, 1, 1457, 0, 0, 0, 0, 0, 0, 0, 1289, 3128, 0, 51, 1, 716, 828, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6055, 6338, 5749, 6032, 6264, 6402, 5779, 6533] avg=6144.0000 max=6533 max_rank=7 min=5749 min_rank=2 imbalance_ratio=1.063314 max_load_reduction=42619 max_load_reduction_pct=0.867086 moved_to_echo=24978 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[893, 429, 2266, 177, 1, 47, 1120, 1, 0, 0, 0, 0, 0, 0, 0, 0, 749, 0, 22, 65, 342, 243, 0, 0, 2931, 1733, 0, 0, 0, 0, 0, 0, 1413, 0, 373, 13, 775, 87, 12, 10, 1848, 1592, 247, 0, 0, 0, 0, 0, 2303, 0, 694, 247, 0, 193, 745, 36, 1147, 719, 0, 0, 0, 0, 0, 0, 1283, 0, 804, 1179, 149, 3985, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 23, 4358, 353, 0, 1750, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 1269, 0, 43, 1670, 251, 0, 1990, 0, 0, 0, 0, 0, 0, 0, 1147, 2657, 0, 57, 0, 801, 820, 211, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4934, 6085, 6370, 6084, 7454, 6860, 5672, 5693] avg=6144.0000 max=7454 max_rank=4 min=4934 min_rank=0 imbalance_ratio=1.213216 max_load_reduction=41698 max_load_reduction_pct=0.848348 moved_to_echo=25679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1059, 692, 2390, 5, 353, 5, 1538, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 700, 59, 574, 350, 0, 3477, 505, 0, 0, 0, 0, 0, 0, 21, 0, 0, 385, 1800, 119, 80, 679, 3186, 462, 69, 0, 0, 0, 0, 0, 861, 0, 1644, 263, 0, 1, 2429, 0, 294, 201, 0, 0, 0, 0, 0, 0, 819, 0, 3, 331, 0, 1389, 645, 10, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1121, 1, 138, 2833, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 123, 6299, 0, 445, 70, 232, 143, 556, 0, 0, 0, 0, 0, 0, 0, 542, 4580, 0, 0, 0, 6, 1617, 769, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7559, 5852, 6801, 5693, 3197, 4649, 7887, 7514] avg=6144.0000 max=7887 max_rank=6 min=3197 min_rank=4 imbalance_ratio=1.283691 max_load_reduction=41265 max_load_reduction_pct=0.839539 moved_to_echo=23247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[699, 1324, 2167, 205, 106, 149, 1576, 17, 0, 0, 0, 0, 0, 0, 0, 0, 743, 0, 69, 166, 244, 277, 1, 3, 2293, 2176, 0, 0, 0, 0, 0, 0, 650, 2, 741, 64, 605, 80, 35, 19, 1459, 1999, 230, 0, 0, 0, 0, 0, 3345, 2, 429, 217, 2, 229, 591, 91, 1008, 670, 0, 0, 0, 0, 0, 0, 1215, 0, 1560, 1098, 233, 2950, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 20, 3828, 365, 0, 2192, 158, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 1369, 0, 19, 1136, 369, 0, 1854, 0, 0, 0, 0, 0, 0, 0, 949, 2099, 0, 119, 0, 1264, 725, 308, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6243, 5972, 5884, 6584, 7109, 6632, 5264, 5464] avg=6144.0000 max=7109 max_rank=4 min=5264 min_rank=6 imbalance_ratio=1.157064 max_load_reduction=42043 max_load_reduction_pct=0.855367 moved_to_echo=24469 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1059, 692, 2390, 5, 353, 5, 1538, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 700, 59, 574, 350, 0, 3477, 505, 0, 0, 0, 0, 0, 0, 21, 0, 0, 385, 1800, 119, 80, 679, 3186, 462, 69, 0, 0, 0, 0, 0, 861, 0, 1644, 263, 0, 1, 2429, 0, 294, 201, 0, 0, 0, 0, 0, 0, 819, 0, 3, 331, 0, 1389, 645, 10, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1121, 1, 138, 2833, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 123, 6299, 0, 445, 70, 232, 143, 556, 0, 0, 0, 0, 0, 0, 0, 542, 4580, 0, 0, 0, 6, 1617, 769, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7559, 5852, 6801, 5693, 3197, 4649, 7887, 7514] avg=6144.0000 max=7887 max_rank=6 min=3197 min_rank=4 imbalance_ratio=1.283691 max_load_reduction=41265 max_load_reduction_pct=0.839539 moved_to_echo=23247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[711, 1043, 2133, 206, 89, 149, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 0, 14, 52, 85, 271, 0, 0, 2332, 1861, 0, 0, 0, 0, 0, 0, 1023, 0, 252, 16, 754, 105, 10, 36, 1807, 1707, 247, 0, 0, 0, 0, 0, 2559, 9, 667, 254, 2, 224, 609, 236, 1085, 720, 0, 0, 0, 0, 0, 0, 1426, 0, 1625, 1180, 72, 3332, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 134, 9, 4124, 376, 4, 1735, 526, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 1021, 0, 22, 1897, 286, 3, 1991, 0, 0, 0, 0, 0, 0, 0, 965, 2599, 0, 27, 3, 845, 380, 271, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5667, 5457, 5957, 6365, 7685, 6909, 6022, 5090] avg=6144.0000 max=7685 max_rank=4 min=5090 min_rank=7 imbalance_ratio=1.250814 max_load_reduction=41467 max_load_reduction_pct=0.843648 moved_to_echo=25706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1236, 736, 1864, 14, 275, 24, 1104, 469, 0, 0, 0, 0, 0, 0, 0, 0, 366, 0, 0, 476, 80, 273, 122, 0, 4059, 1206, 0, 0, 0, 0, 0, 0, 183, 0, 20, 434, 1509, 97, 56, 385, 3049, 1108, 87, 0, 0, 0, 0, 0, 1223, 4, 1404, 164, 0, 3, 1859, 2, 481, 256, 0, 0, 0, 0, 0, 0, 1851, 0, 55, 421, 6, 2184, 674, 3, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 1830, 9, 42, 2857, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 115, 5554, 1, 168, 194, 332, 52, 708, 0, 0, 0, 0, 0, 0, 0, 489, 4382, 0, 2, 0, 53, 1605, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5722, 6582, 6928, 5396, 5194, 5125, 7138, 7067] avg=6144.0000 max=7138 max_rank=6 min=5125 min_rank=5 imbalance_ratio=1.161784 max_load_reduction=42014 max_load_reduction_pct=0.854777 moved_to_echo=24524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1032, 1141, 1986, 111, 195, 48, 1467, 75, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 18, 174, 83, 297, 23, 0, 3386, 1731, 0, 0, 0, 0, 0, 0, 541, 0, 136, 43, 937, 95, 14, 38, 2175, 1590, 180, 0, 0, 0, 0, 0, 2433, 0, 875, 248, 0, 156, 772, 41, 981, 526, 0, 0, 0, 0, 0, 0, 1303, 0, 369, 864, 79, 3214, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 3, 3730, 345, 0, 1990, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 2966, 0, 93, 691, 277, 1, 1457, 0, 0, 0, 0, 0, 0, 0, 1289, 3128, 0, 51, 1, 716, 828, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6055, 6338, 5749, 6032, 6264, 6402, 5779, 6533] avg=6144.0000 max=6533 max_rank=7 min=5749 min_rank=2 imbalance_ratio=1.063314 max_load_reduction=42619 max_load_reduction_pct=0.867086 moved_to_echo=24978 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1086, 959, 2449, 230, 168, 33, 1174, 27, 0, 0, 0, 0, 0, 0, 0, 0, 478, 1, 3, 174, 103, 260, 5, 0, 3586, 1465, 0, 0, 0, 0, 0, 0, 432, 0, 74, 77, 846, 101, 111, 50, 2402, 1344, 178, 0, 0, 0, 0, 0, 2716, 2, 986, 248, 0, 149, 983, 4, 878, 512, 0, 0, 0, 0, 0, 0, 1161, 0, 156, 824, 23, 4061, 68, 3, 0, 0, 0, 0, 0, 0, 0, 0, 108, 9, 3314, 343, 1, 1994, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 2769, 0, 101, 1145, 313, 3, 1412, 0, 0, 0, 0, 0, 0, 0, 921, 3442, 0, 32, 0, 434, 905, 267, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6075, 5615, 6478, 6296, 6554, 6007, 6001] avg=6144.0000 max=6554 max_rank=5 min=5615 min_rank=2 imbalance_ratio=1.066732 max_load_reduction=42598 max_load_reduction_pct=0.866659 moved_to_echo=24858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 128, 256, 2048, 1280, 2560, 512, 128, 25088, 12800, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 7808, 17408, 1280, 1408, 640, 11264, 2176, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7552, 25344, 128, 384, 128, 5504, 7296, 3328, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[5120, 128, 1920, 1152, 8192, 896, 384, 1280, 17664, 11776, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7552, 25344, 128, 384, 128, 5504, 7296, 3328, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[18304, 128, 7680, 1920, 128, 1280, 8704, 640, 6784, 4352, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10496, 0, 5248, 7040, 768, 23552, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2944, 23040, 128, 1024, 8064, 2304, 256, 11776, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 128, 25728, 2176, 256, 17280, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[345, 9, 53, 70, 454, 485, 118, 1596, 96, 6417, 169, 138, 271, 3985, 3, 1448, 2886, 92, 0, 2, 72, 139, 664, 379, 488, 250, 5510, 620, 34, 401, 40, 34, 459, 0, 2, 517, 77, 1730, 166, 560, 6231, 59, 338, 0, 329, 3277, 1173, 0, 919, 42, 756, 1816, 273, 172, 120, 628, 1549, 133, 18, 28, 91, 267, 36, 118] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[739, 0, 2, 80, 37, 60, 123, 1255, 51, 7115, 195, 0, 6, 5292, 1, 1004, 700, 64, 2, 0, 64, 1, 809, 0, 326, 194, 5508, 23, 2, 130, 3, 0, 275, 0, 0, 55, 3, 6557, 6, 375, 1302, 73, 1986, 0, 93, 5309, 5207, 1, 254, 5, 47, 271, 35, 96, 34, 3172, 41, 4, 0, 0, 6, 3, 73, 83] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[126, 0, 85, 53, 1061, 354, 41, 1919, 102, 6369, 105, 136, 290, 3160, 6, 1231, 2963, 137, 0, 0, 22, 417, 551, 83, 600, 4, 5580, 475, 20, 277, 6, 17, 413, 0, 0, 639, 139, 1140, 163, 596, 7199, 18, 318, 6, 275, 2942, 663, 0, 982, 54, 1270, 2239, 368, 162, 220, 420, 1940, 51, 3, 22, 235, 379, 18, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[531, 0, 0, 59, 103, 53, 127, 991, 54, 6608, 390, 4, 1, 5766, 0, 922, 68, 75, 0, 0, 93, 1, 934, 1, 428, 254, 4681, 0, 1, 130, 0, 0, 266, 0, 0, 0, 0, 7375, 1, 378, 357, 136, 3060, 0, 67, 4810, 6554, 0, 429, 0, 4, 80, 0, 158, 0, 3074, 15, 0, 0, 0, 0, 0, 32, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[426, 0, 3, 80, 254, 164, 125, 1781, 91, 7412, 463, 56, 68, 3957, 1, 1595, 2857, 263, 0, 0, 64, 52, 923, 0, 572, 107, 5899, 293, 32, 158, 3, 1, 399, 0, 0, 288, 172, 2715, 103, 559, 5512, 83, 637, 6, 169, 4910, 1464, 0, 890, 30, 428, 1214, 242, 84, 45, 1031, 277, 8, 0, 0, 15, 11, 12, 148] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[242, 0, 0, 73, 615, 229, 57, 2307, 95, 6561, 260, 97, 153, 2950, 1, 1219, 3448, 166, 0, 0, 17, 251, 610, 39, 518, 47, 5711, 530, 54, 271, 3, 19, 470, 0, 0, 1061, 107, 1221, 183, 534, 7265, 36, 170, 0, 395, 4099, 661, 0, 712, 59, 931, 2435, 531, 152, 237, 298, 543, 74, 5, 3, 232, 73, 27, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[421, 0, 44, 74, 323, 137, 86, 1204, 14, 6853, 187, 18, 179, 3972, 1, 1322, 2985, 105, 0, 0, 74, 129, 704, 70, 448, 27, 5833, 211, 3, 201, 0, 8, 458, 0, 0, 531, 37, 3060, 124, 409, 5477, 81, 517, 0, 199, 4645, 2048, 0, 879, 25, 749, 1355, 343, 84, 61, 1156, 691, 27, 0, 4, 90, 79, 306, 84] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[426, 0, 3, 80, 254, 164, 125, 1781, 91, 7412, 463, 56, 68, 3957, 1, 1595, 2857, 263, 0, 0, 64, 52, 923, 0, 572, 107, 5899, 293, 32, 158, 3, 1, 399, 0, 0, 288, 172, 2715, 103, 559, 5512, 83, 637, 6, 169, 4910, 1464, 0, 890, 30, 428, 1214, 242, 84, 45, 1031, 277, 8, 0, 0, 15, 11, 12, 148] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[531, 0, 0, 59, 103, 53, 127, 991, 54, 6608, 390, 4, 1, 5766, 0, 922, 68, 75, 0, 0, 93, 1, 934, 1, 428, 254, 4681, 0, 1, 130, 0, 0, 266, 0, 0, 0, 0, 7375, 1, 378, 357, 136, 3060, 0, 67, 4810, 6554, 0, 429, 0, 4, 80, 0, 158, 0, 3074, 15, 0, 0, 0, 0, 0, 32, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[210, 0, 17, 101, 579, 339, 108, 2082, 65, 5988, 138, 338, 253, 3689, 0, 917, 2805, 117, 0, 0, 33, 259, 742, 98, 722, 112, 5726, 502, 32, 370, 35, 6, 807, 0, 0, 772, 148, 653, 207, 522, 7571, 8, 188, 11, 365, 3418, 246, 0, 1333, 38, 930, 2918, 288, 104, 181, 181, 1281, 110, 6, 0, 310, 50, 8, 115] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[126, 0, 85, 53, 1061, 354, 41, 1919, 102, 6369, 105, 136, 290, 3160, 6, 1231, 2963, 137, 0, 0, 22, 417, 551, 83, 600, 4, 5580, 475, 20, 277, 6, 17, 413, 0, 0, 639, 139, 1140, 163, 596, 7199, 18, 318, 6, 275, 2942, 663, 0, 982, 54, 1270, 2239, 368, 162, 220, 420, 1940, 51, 3, 22, 235, 379, 18, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[345, 9, 53, 70, 454, 485, 118, 1596, 2162, 1437, 0, 0, 0, 0, 0, 0, 96, 269, 169, 138, 271, 3985, 3, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 2886, 92, 0, 2, 72, 139, 664, 379, 3218, 0, 0, 0, 0, 0, 0, 0, 488, 250, 4849, 620, 34, 401, 40, 34, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 2, 517, 77, 1730, 166, 560, 601, 851, 0, 0, 0, 0, 0, 0, 0, 59, 338, 0, 329, 2111, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 42, 756, 1816, 273, 172, 120, 628, 565, 661, 0, 0, 0, 0, 0, 0, 1549, 133, 18, 28, 91, 267, 36, 118, 4711, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6729, 6379, 7452, 6716, 4963, 4010, 5952, 6951] avg=6144.0000 max=7452 max_rank=2 min=4010 min_rank=5 imbalance_ratio=1.212891 max_load_reduction=41700 max_load_reduction_pct=0.848389 moved_to_echo=21876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[739, 0, 2, 80, 37, 60, 123, 1255, 451, 1593, 0, 0, 0, 0, 0, 0, 51, 298, 195, 0, 6, 5292, 1, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 700, 64, 2, 0, 64, 1, 809, 0, 672, 0, 0, 0, 0, 0, 0, 0, 326, 194, 4847, 23, 2, 130, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 55, 3, 6557, 6, 375, 973, 179, 0, 0, 0, 0, 0, 0, 0, 73, 1986, 0, 93, 3420, 5207, 1, 0, 0, 0, 0, 0, 0, 0, 0, 254, 5, 47, 271, 35, 96, 34, 3172, 916, 661, 0, 0, 0, 0, 0, 0, 41, 4, 0, 0, 6, 3, 73, 83, 5224, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4340, 6847, 2312, 5525, 8423, 10780, 5491, 5434] avg=6144.0000 max=10780 max_rank=5 min=2312 min_rank=2 imbalance_ratio=1.754557 max_load_reduction=38372 max_load_reduction_pct=0.780680 moved_to_echo=30128 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[242, 0, 0, 73, 615, 229, 57, 2307, 2521, 1469, 0, 0, 0, 0, 0, 0, 95, 275, 260, 97, 153, 2950, 1, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 166, 0, 0, 17, 251, 610, 39, 3750, 0, 0, 0, 0, 0, 0, 0, 518, 47, 5026, 530, 54, 271, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 1061, 107, 1221, 183, 534, 751, 994, 0, 0, 0, 0, 0, 0, 0, 36, 170, 0, 395, 2641, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 59, 931, 2435, 531, 152, 237, 298, 707, 685, 0, 0, 0, 0, 0, 0, 543, 74, 5, 3, 232, 73, 27, 95, 4817, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7513, 5050, 8281, 6468, 5321, 3903, 6747, 5869] avg=6144.0000 max=8281 max_rank=2 min=3903 min_rank=5 imbalance_ratio=1.347819 max_load_reduction=40871 max_load_reduction_pct=0.831523 moved_to_echo=21840 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[531, 0, 0, 59, 103, 53, 127, 991, 125, 1479, 0, 0, 0, 0, 0, 0, 54, 277, 390, 4, 1, 5766, 0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 68, 75, 0, 0, 93, 1, 934, 1, 184, 0, 0, 0, 0, 0, 0, 0, 428, 254, 4119, 0, 1, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 7375, 1, 378, 882, 48, 0, 0, 0, 0, 0, 0, 0, 136, 3060, 0, 67, 3098, 6554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 4, 80, 0, 158, 0, 3074, 830, 562, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 32, 81, 4852, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3468, 7414, 1356, 4932, 8950, 12915, 5137, 4980] avg=6144.0000 max=12915 max_rank=5 min=1356 min_rank=2 imbalance_ratio=2.102051 max_load_reduction=36237 max_load_reduction_pct=0.737244 moved_to_echo=31982 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[242, 0, 0, 73, 615, 229, 57, 2307, 2521, 1469, 0, 0, 0, 0, 0, 0, 95, 275, 260, 97, 153, 2950, 1, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 166, 0, 0, 17, 251, 610, 39, 3750, 0, 0, 0, 0, 0, 0, 0, 518, 47, 5026, 530, 54, 271, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 1061, 107, 1221, 183, 534, 751, 994, 0, 0, 0, 0, 0, 0, 0, 36, 170, 0, 395, 2641, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 59, 931, 2435, 531, 152, 237, 298, 707, 685, 0, 0, 0, 0, 0, 0, 543, 74, 5, 3, 232, 73, 27, 95, 4817, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7513, 5050, 8281, 6468, 5321, 3903, 6747, 5869] avg=6144.0000 max=8281 max_rank=2 min=3903 min_rank=5 imbalance_ratio=1.347819 max_load_reduction=40871 max_load_reduction_pct=0.831523 moved_to_echo=21840 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[421, 0, 44, 74, 323, 137, 86, 1204, 1900, 1534, 0, 0, 0, 0, 0, 0, 14, 287, 187, 18, 179, 3972, 1, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 2985, 105, 0, 0, 74, 129, 704, 70, 2829, 0, 0, 0, 0, 0, 0, 0, 448, 27, 5133, 211, 3, 201, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 531, 37, 3060, 124, 409, 851, 748, 0, 0, 0, 0, 0, 0, 0, 81, 517, 0, 199, 2992, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 879, 25, 749, 1355, 343, 84, 61, 1156, 802, 700, 0, 0, 0, 0, 0, 0, 691, 27, 0, 4, 90, 79, 306, 84, 5032, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5723, 5980, 6896, 6031, 6218, 5837, 6154, 6313] avg=6144.0000 max=6896 max_rank=2 min=5723 min_rank=0 imbalance_ratio=1.122396 max_load_reduction=42256 max_load_reduction_pct=0.859701 moved_to_echo=24522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[210, 0, 17, 101, 579, 339, 108, 2082, 2629, 1341, 0, 0, 0, 0, 0, 0, 65, 251, 138, 338, 253, 3689, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 2805, 117, 0, 0, 33, 259, 742, 98, 3908, 0, 0, 0, 0, 0, 0, 0, 722, 112, 5039, 502, 32, 370, 35, 6, 0, 0, 0, 0, 0, 0, 0, 0, 807, 0, 0, 772, 148, 653, 207, 522, 626, 1034, 0, 0, 0, 0, 0, 0, 0, 8, 188, 11, 365, 2202, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 38, 930, 2918, 288, 104, 181, 181, 590, 687, 0, 0, 0, 0, 0, 0, 1281, 110, 6, 0, 310, 50, 8, 115, 4396, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7406, 5651, 7962, 6818, 4769, 3020, 7250, 6276] avg=6144.0000 max=7962 max_rank=2 min=3020 min_rank=5 imbalance_ratio=1.295898 max_load_reduction=41190 max_load_reduction_pct=0.838013 moved_to_echo=21315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[126, 0, 85, 53, 1061, 354, 41, 1919, 2498, 1426, 0, 0, 0, 0, 0, 0, 102, 267, 105, 136, 290, 3160, 6, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 2963, 137, 0, 0, 22, 417, 551, 83, 3716, 0, 0, 0, 0, 0, 0, 0, 600, 4, 4910, 475, 20, 277, 6, 17, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 0, 639, 139, 1140, 163, 596, 539, 985, 0, 0, 0, 0, 0, 0, 0, 18, 318, 6, 275, 1895, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 982, 54, 1270, 2239, 368, 162, 220, 420, 508, 670, 0, 0, 0, 0, 0, 0, 1940, 51, 3, 22, 235, 379, 18, 88, 4676, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7563, 5297, 7889, 6309, 4614, 3175, 6893, 7412] avg=6144.0000 max=7889 max_rank=2 min=3175 min_rank=5 imbalance_ratio=1.284017 max_load_reduction=41263 max_load_reduction_pct=0.839498 moved_to_echo=22094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[426, 0, 3, 80, 254, 164, 125, 1781, 1914, 1663, 0, 0, 0, 0, 0, 0, 91, 303, 463, 56, 68, 3957, 1, 1595, 0, 0, 0, 0, 0, 0, 0, 0, 2857, 263, 0, 0, 64, 52, 923, 0, 2845, 0, 0, 0, 0, 0, 0, 0, 572, 107, 5187, 293, 32, 158, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 288, 172, 2715, 103, 559, 905, 753, 0, 0, 0, 0, 0, 0, 0, 83, 637, 6, 169, 3153, 1464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 890, 30, 428, 1214, 242, 84, 45, 1031, 852, 712, 0, 0, 0, 0, 0, 0, 277, 8, 0, 0, 15, 11, 12, 148, 5446, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6410, 6534, 7004, 6353, 5894, 5512, 5528, 5917] avg=6144.0000 max=7004 max_rank=2 min=5512 min_rank=5 imbalance_ratio=1.139974 max_load_reduction=42148 max_load_reduction_pct=0.857503 moved_to_echo=22851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[210, 0, 17, 101, 579, 339, 108, 2082, 2629, 1341, 0, 0, 0, 0, 0, 0, 65, 251, 138, 338, 253, 3689, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 2805, 117, 0, 0, 33, 259, 742, 98, 3908, 0, 0, 0, 0, 0, 0, 0, 722, 112, 5039, 502, 32, 370, 35, 6, 0, 0, 0, 0, 0, 0, 0, 0, 807, 0, 0, 772, 148, 653, 207, 522, 626, 1034, 0, 0, 0, 0, 0, 0, 0, 8, 188, 11, 365, 2202, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 38, 930, 2918, 288, 104, 181, 181, 590, 687, 0, 0, 0, 0, 0, 0, 1281, 110, 6, 0, 310, 50, 8, 115, 4396, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7406, 5651, 7962, 6818, 4769, 3020, 7250, 6276] avg=6144.0000 max=7962 max_rank=2 min=3020 min_rank=5 imbalance_ratio=1.295898 max_load_reduction=41190 max_load_reduction_pct=0.838013 moved_to_echo=21315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 128, 256, 640, 3456, 1920, 896, 13184, 14208, 12032, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[18816, 1024, 128, 128, 512, 1280, 6016, 768, 21248, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 512, 7296, 128, 1920, 21632, 18048, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 256, 5120, 12416, 2176, 1024, 1024, 9984, 5888, 5376, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 2304, 1920, 896, 1280, 32896, 128, 9728, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 0, 128, 3968, 768, 24576, 1024, 3968, 6144, 5632, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 1024, 39168, 2688, 256, 2048, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 512, 128, 128, 1024, 896, 512, 896, 39168, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 1024, 39168, 2688, 256, 2048, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[278, 0, 83, 0, 2, 203, 4434, 105, 0, 3, 0, 0, 1, 0, 0, 0, 282, 0, 4910, 163, 83, 407, 155, 4, 0, 7804, 0, 85, 3630, 3013, 4, 0, 8, 0, 4398, 0, 1039, 2, 0, 85, 0, 0, 0, 644, 0, 0, 265, 7295, 3, 0, 0, 3294, 533, 0, 4, 2182, 161, 0, 16, 27, 59, 16, 0, 3472] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[387, 0, 65, 1, 1483, 552, 867, 43, 0, 1706, 119, 40, 507, 64, 912, 6, 6917, 15, 3915, 531, 583, 2292, 108, 13, 0, 4188, 0, 447, 616, 1698, 1605, 16, 2532, 541, 582, 0, 566, 0, 0, 27, 67, 423, 116, 510, 1, 0, 380, 3777, 1777, 3, 0, 1553, 453, 0, 113, 1410, 560, 0, 581, 2529, 0, 1, 1, 953] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[358, 0, 56, 0, 576, 579, 1456, 65, 0, 918, 43, 20, 177, 22, 761, 7, 5762, 15, 5171, 467, 270, 1527, 158, 16, 0, 5591, 2, 665, 917, 2891, 771, 64, 1502, 387, 1782, 0, 548, 0, 4, 18, 24, 243, 78, 651, 3, 2, 308, 5182, 999, 3, 0, 1849, 388, 0, 10, 2052, 739, 0, 252, 1370, 0, 1, 11, 1421] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[146, 0, 85, 1, 52, 137, 3875, 176, 0, 101, 2, 1, 37, 0, 74, 0, 1270, 0, 5494, 244, 46, 515, 110, 0, 0, 7569, 0, 95, 3555, 2270, 77, 0, 42, 181, 4419, 0, 805, 1, 0, 18, 1, 30, 21, 693, 0, 3, 259, 7126, 17, 0, 0, 3112, 245, 0, 59, 2784, 274, 0, 74, 35, 53, 4, 0, 2964] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[449, 0, 60, 1, 1451, 699, 1026, 140, 0, 1251, 157, 47, 320, 82, 815, 13, 6192, 49, 3936, 590, 482, 2574, 136, 14, 1, 4614, 1, 579, 630, 2157, 1201, 133, 2284, 440, 1048, 26, 547, 0, 9, 17, 40, 415, 144, 625, 9, 4, 350, 4035, 1279, 0, 0, 1746, 524, 1, 55, 1365, 849, 4, 262, 2129, 11, 3, 57, 1074] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[358, 0, 56, 0, 576, 579, 1456, 65, 0, 918, 43, 20, 177, 22, 761, 7, 5762, 15, 5171, 467, 270, 1527, 158, 16, 0, 5591, 2, 665, 917, 2891, 771, 64, 1502, 387, 1782, 0, 548, 0, 4, 18, 24, 243, 78, 651, 3, 2, 308, 5182, 999, 3, 0, 1849, 388, 0, 10, 2052, 739, 0, 252, 1370, 0, 1, 11, 1421] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[248, 0, 41, 0, 1358, 585, 752, 56, 0, 1901, 119, 45, 565, 55, 1035, 13, 6639, 21, 3463, 490, 513, 2148, 45, 3, 0, 3863, 8, 485, 543, 1879, 2138, 128, 2631, 445, 820, 7, 499, 0, 0, 151, 36, 1010, 115, 591, 0, 0, 287, 2825, 2293, 5, 0, 1675, 283, 0, 81, 1324, 526, 0, 432, 3176, 0, 0, 14, 787] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[387, 0, 65, 1, 1483, 552, 867, 43, 0, 1706, 119, 40, 507, 64, 912, 6, 6917, 15, 3915, 531, 583, 2292, 108, 13, 0, 4188, 0, 447, 616, 1698, 1605, 16, 2532, 541, 582, 0, 566, 0, 0, 27, 67, 423, 116, 510, 1, 0, 380, 3777, 1777, 3, 0, 1553, 453, 0, 113, 1410, 560, 0, 581, 2529, 0, 1, 1, 953] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[278, 0, 83, 0, 2, 203, 4434, 105, 0, 3, 0, 0, 1, 0, 0, 0, 282, 0, 4910, 163, 83, 407, 155, 4, 0, 7804, 0, 85, 3630, 3013, 4, 0, 8, 0, 4398, 0, 1039, 2, 0, 85, 0, 0, 0, 644, 0, 0, 265, 7295, 3, 0, 0, 3294, 533, 0, 4, 2182, 161, 0, 16, 27, 59, 16, 0, 3472] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[532, 0, 81, 0, 246, 660, 1839, 43, 1, 529, 122, 12, 88, 68, 529, 2, 5635, 17, 5019, 441, 484, 1468, 144, 5, 0, 6132, 2, 457, 900, 2627, 389, 1, 993, 251, 1591, 7, 912, 1, 0, 64, 31, 62, 41, 584, 1, 4, 293, 5812, 311, 4, 0, 1817, 594, 0, 154, 2668, 1105, 1, 262, 1209, 15, 7, 0, 1885] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[248, 0, 41, 0, 1358, 585, 752, 56, 0, 1901, 119, 45, 565, 55, 1035, 13, 6639, 21, 3463, 490, 513, 2148, 45, 3, 0, 3863, 8, 485, 543, 1879, 2138, 128, 2631, 445, 820, 7, 499, 0, 0, 151, 36, 1010, 115, 591, 0, 0, 287, 2825, 2293, 5, 0, 1675, 283, 0, 81, 1324, 526, 0, 432, 3176, 0, 0, 14, 787] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[386, 0, 58, 0, 1430, 519, 602, 36, 3, 2341, 155, 39, 207, 54, 902, 5, 6811, 4, 4008, 355, 1082, 2622, 109, 2, 0, 3857, 2, 397, 547, 1884, 1726, 84, 2074, 173, 685, 20, 413, 0, 7, 144, 75, 669, 131, 332, 2, 0, 313, 2901, 1594, 3, 0, 1585, 494, 0, 234, 1089, 719, 0, 489, 4038, 0, 1, 24, 716] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[146, 0, 85, 1, 52, 137, 3875, 176, 2158, 236, 0, 0, 0, 0, 0, 0, 0, 101, 2, 1, 37, 0, 74, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4228, 244, 46, 515, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 0, 95, 3555, 2270, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 181, 4419, 0, 805, 1, 0, 18, 1659, 587, 0, 0, 0, 0, 0, 0, 1, 30, 21, 693, 0, 3, 259, 7001, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3112, 245, 0, 59, 2784, 679, 125, 0, 0, 0, 0, 0, 0, 274, 0, 74, 35, 53, 4, 0, 2964, 2391, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6866, 1249, 5143, 7358, 7712, 8008, 7021, 5795] avg=6144.0000 max=8008 max_rank=5 min=1249 min_rank=1 imbalance_ratio=1.303385 max_load_reduction=41144 max_load_reduction_pct=0.837077 moved_to_echo=28536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[449, 0, 60, 1, 1451, 699, 1026, 140, 1315, 1152, 0, 0, 0, 0, 0, 0, 0, 1251, 157, 47, 320, 82, 815, 13, 5040, 0, 0, 0, 0, 0, 0, 0, 0, 49, 3029, 590, 482, 2574, 136, 14, 0, 0, 0, 0, 0, 0, 0, 0, 1, 831, 1, 579, 630, 2157, 1201, 133, 0, 0, 0, 0, 0, 0, 0, 0, 2284, 440, 1048, 26, 547, 0, 9, 17, 1011, 420, 0, 0, 0, 0, 0, 0, 40, 415, 144, 625, 9, 4, 350, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 0, 0, 1746, 524, 1, 55, 1365, 487, 71, 0, 0, 0, 0, 0, 0, 849, 4, 262, 2129, 11, 3, 57, 1074, 1457, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6293, 7725, 6874, 5533, 5802, 5551, 5528, 5846] avg=6144.0000 max=7725 max_rank=1 min=5528 min_rank=6 imbalance_ratio=1.257324 max_load_reduction=41427 max_load_reduction_pct=0.842834 moved_to_echo=22727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[358, 0, 56, 0, 576, 579, 1456, 65, 1594, 1072, 0, 0, 0, 0, 0, 0, 0, 918, 43, 20, 177, 22, 761, 7, 4690, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3980, 467, 270, 1527, 158, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 2, 665, 917, 2891, 771, 64, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 387, 1782, 0, 548, 0, 4, 18, 1226, 552, 0, 0, 0, 0, 0, 0, 24, 243, 78, 651, 3, 2, 308, 5091, 0, 0, 0, 0, 0, 0, 0, 0, 999, 3, 0, 1849, 388, 0, 10, 2052, 639, 91, 0, 0, 0, 0, 0, 0, 739, 0, 252, 1370, 0, 1, 11, 1421, 1766, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5756, 6638, 6433, 6315, 6019, 6400, 6031, 5560] avg=6144.0000 max=6638 max_rank=1 min=5560 min_rank=7 imbalance_ratio=1.080404 max_load_reduction=42514 max_load_reduction_pct=0.864950 moved_to_echo=24010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[278, 0, 83, 0, 2, 203, 4434, 105, 2225, 53, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 163, 83, 407, 155, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 85, 3630, 3013, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4398, 0, 1039, 2, 0, 85, 1711, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 0, 0, 265, 7167, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3294, 533, 0, 4, 2182, 607, 128, 0, 0, 0, 0, 0, 0, 161, 0, 16, 27, 59, 16, 0, 3472, 2466, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7383, 233, 4591, 8134, 7767, 8076, 6751, 6217] avg=6144.0000 max=8134 max_rank=3 min=233 min_rank=1 imbalance_ratio=1.323893 max_load_reduction=41018 max_load_reduction_pct=0.834513 moved_to_echo=28811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[387, 0, 65, 1, 1483, 552, 867, 43, 1194, 1287, 0, 0, 0, 0, 0, 0, 0, 1706, 119, 40, 507, 64, 912, 6, 5630, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3013, 531, 583, 2292, 108, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 447, 616, 1698, 1605, 16, 0, 0, 0, 0, 0, 0, 0, 0, 2532, 541, 582, 0, 566, 0, 0, 27, 918, 418, 0, 0, 0, 0, 0, 0, 67, 423, 116, 510, 1, 0, 380, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 3, 0, 1553, 453, 0, 113, 1410, 484, 66, 0, 0, 0, 0, 0, 0, 560, 0, 581, 2529, 0, 1, 1, 953, 1323, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5879, 8984, 6555, 5135, 5584, 5208, 5859, 5948] avg=6144.0000 max=8984 max_rank=1 min=5135 min_rank=3 imbalance_ratio=1.462240 max_load_reduction=40168 max_load_reduction_pct=0.817220 moved_to_echo=22599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[248, 0, 41, 0, 1358, 585, 752, 56, 1101, 1235, 0, 0, 0, 0, 0, 0, 0, 1901, 119, 45, 565, 55, 1035, 13, 5404, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2665, 490, 513, 2148, 45, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 8, 485, 543, 1879, 2138, 128, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 445, 820, 7, 499, 0, 0, 151, 847, 370, 0, 0, 0, 0, 0, 0, 36, 1010, 115, 591, 0, 0, 287, 2776, 0, 0, 0, 0, 0, 0, 0, 0, 2293, 5, 0, 1675, 283, 0, 81, 1324, 428, 49, 0, 0, 0, 0, 0, 0, 526, 0, 432, 3176, 0, 0, 14, 787, 1220, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5376, 9137, 5885, 5876, 5770, 4815, 6138, 6155] avg=6144.0000 max=9137 max_rank=1 min=4815 min_rank=5 imbalance_ratio=1.487142 max_load_reduction=40015 max_load_reduction_pct=0.814107 moved_to_echo=22878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[532, 0, 81, 0, 246, 660, 1839, 43, 1751, 1049, 0, 0, 0, 0, 0, 0, 1, 529, 122, 12, 88, 68, 529, 2, 4586, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3853, 441, 484, 1468, 144, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 2, 457, 900, 2627, 389, 1, 0, 0, 0, 0, 0, 0, 0, 0, 993, 251, 1591, 7, 912, 1, 0, 64, 1349, 541, 0, 0, 0, 0, 0, 0, 31, 62, 41, 584, 1, 4, 293, 5706, 0, 0, 0, 0, 0, 0, 0, 0, 311, 4, 0, 1817, 594, 0, 154, 2668, 625, 106, 0, 0, 0, 0, 0, 0, 1105, 1, 262, 1209, 15, 7, 0, 1885, 1942, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 5937, 6412, 5466, 5709, 6722, 6279, 6426] avg=6144.0000 max=6722 max_rank=5 min=5466 min_rank=3 imbalance_ratio=1.094076 max_load_reduction=42430 max_load_reduction_pct=0.863241 moved_to_echo=25136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[387, 0, 65, 1, 1483, 552, 867, 43, 1194, 1287, 0, 0, 0, 0, 0, 0, 0, 1706, 119, 40, 507, 64, 912, 6, 5630, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3013, 531, 583, 2292, 108, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 447, 616, 1698, 1605, 16, 0, 0, 0, 0, 0, 0, 0, 0, 2532, 541, 582, 0, 566, 0, 0, 27, 918, 418, 0, 0, 0, 0, 0, 0, 67, 423, 116, 510, 1, 0, 380, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 3, 0, 1553, 453, 0, 113, 1410, 484, 66, 0, 0, 0, 0, 0, 0, 560, 0, 581, 2529, 0, 1, 1, 953, 1323, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5879, 8984, 6555, 5135, 5584, 5208, 5859, 5948] avg=6144.0000 max=8984 max_rank=1 min=5135 min_rank=3 imbalance_ratio=1.462240 max_load_reduction=40168 max_load_reduction_pct=0.817220 moved_to_echo=22599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[278, 0, 83, 0, 2, 203, 4434, 105, 2225, 53, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 163, 83, 407, 155, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 85, 3630, 3013, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4398, 0, 1039, 2, 0, 85, 1711, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 0, 0, 265, 7167, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3294, 533, 0, 4, 2182, 607, 128, 0, 0, 0, 0, 0, 0, 161, 0, 16, 27, 59, 16, 0, 3472, 2466, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7383, 233, 4591, 8134, 7767, 8076, 6751, 6217] avg=6144.0000 max=8134 max_rank=3 min=233 min_rank=1 imbalance_ratio=1.323893 max_load_reduction=41018 max_load_reduction_pct=0.834513 moved_to_echo=28811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[146, 0, 85, 1, 52, 137, 3875, 176, 2158, 236, 0, 0, 0, 0, 0, 0, 0, 101, 2, 1, 37, 0, 74, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4228, 244, 46, 515, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 0, 95, 3555, 2270, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 181, 4419, 0, 805, 1, 0, 18, 1659, 587, 0, 0, 0, 0, 0, 0, 1, 30, 21, 693, 0, 3, 259, 7001, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3112, 245, 0, 59, 2784, 679, 125, 0, 0, 0, 0, 0, 0, 274, 0, 74, 35, 53, 4, 0, 2964, 2391, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6866, 1249, 5143, 7358, 7712, 8008, 7021, 5795] avg=6144.0000 max=8008 max_rank=5 min=1249 min_rank=1 imbalance_ratio=1.303385 max_load_reduction=41144 max_load_reduction_pct=0.837077 moved_to_echo=28536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[358, 0, 56, 0, 576, 579, 1456, 65, 1594, 1072, 0, 0, 0, 0, 0, 0, 0, 918, 43, 20, 177, 22, 761, 7, 4690, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3980, 467, 270, 1527, 158, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 2, 665, 917, 2891, 771, 64, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 387, 1782, 0, 548, 0, 4, 18, 1226, 552, 0, 0, 0, 0, 0, 0, 24, 243, 78, 651, 3, 2, 308, 5091, 0, 0, 0, 0, 0, 0, 0, 0, 999, 3, 0, 1849, 388, 0, 10, 2052, 639, 91, 0, 0, 0, 0, 0, 0, 739, 0, 252, 1370, 0, 1, 11, 1421, 1766, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5756, 6638, 6433, 6315, 6019, 6400, 6031, 5560] avg=6144.0000 max=6638 max_rank=1 min=5560 min_rank=7 imbalance_ratio=1.080404 max_load_reduction=42514 max_load_reduction_pct=0.864950 moved_to_echo=24010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[386, 0, 58, 0, 1430, 519, 602, 36, 1099, 1268, 0, 0, 0, 0, 0, 0, 3, 2341, 155, 39, 207, 54, 902, 5, 5543, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3085, 355, 1082, 2622, 109, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 2, 397, 547, 1884, 1726, 84, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 173, 685, 20, 413, 0, 7, 144, 845, 428, 0, 0, 0, 0, 0, 0, 75, 669, 131, 332, 2, 0, 313, 2850, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 3, 0, 1585, 494, 0, 234, 1089, 495, 51, 0, 0, 0, 0, 0, 0, 719, 0, 489, 4038, 0, 1, 24, 716, 1218, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5398, 9249, 7259, 5335, 4789, 4372, 5545, 7205] avg=6144.0000 max=9249 max_rank=1 min=4372 min_rank=5 imbalance_ratio=1.505371 max_load_reduction=39903 max_load_reduction_pct=0.811829 moved_to_echo=21911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[532, 0, 81, 0, 246, 660, 1839, 43, 1751, 1049, 0, 0, 0, 0, 0, 0, 1, 529, 122, 12, 88, 68, 529, 2, 4586, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3853, 441, 484, 1468, 144, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 2, 457, 900, 2627, 389, 1, 0, 0, 0, 0, 0, 0, 0, 0, 993, 251, 1591, 7, 912, 1, 0, 64, 1349, 541, 0, 0, 0, 0, 0, 0, 31, 62, 41, 584, 1, 4, 293, 5706, 0, 0, 0, 0, 0, 0, 0, 0, 311, 4, 0, 1817, 594, 0, 154, 2668, 625, 106, 0, 0, 0, 0, 0, 0, 1105, 1, 262, 1209, 15, 7, 0, 1885, 1942, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 5937, 6412, 5466, 5709, 6722, 6279, 6426] avg=6144.0000 max=6722 max_rank=5 min=5466 min_rank=3 imbalance_ratio=1.094076 max_load_reduction=42430 max_load_reduction_pct=0.863241 moved_to_echo=25136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 0, 640, 128, 6656, 3968, 14976, 768, 12544, 7424, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 7936, 128, 3328, 11392, 18432, 7936, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12160, 2432, 15360, 128, 5376, 128, 128, 640, 9600, 3840, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 128, 2432, 14592, 256, 128, 128, 13312, 13824, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8320, 128, 0, 16640, 3584, 128, 768, 14976, 4480, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 0, 640, 128, 6656, 3968, 14976, 768, 12544, 7424, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 8832, 768, 256, 1920, 384, 5120, 128, 32256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 7936, 128, 3328, 11392, 18432, 7936, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 27648, 3328, 3584, 13568, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 2944, 768, 4736, 128, 128, 2560, 38272, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1853, 1548, 3699, 6030, 912, 22, 24, 4595, 2213, 1176, 364, 225, 0, 0, 615, 50, 41, 844, 0, 409, 21, 28, 0, 1249, 466, 0, 89, 138, 0, 0, 1692, 963, 181, 3, 0, 54, 1015, 481, 499, 0, 4501, 182, 1, 737, 1484, 2, 0, 1002, 589, 4781, 4, 7, 0, 497, 0, 0, 0, 4, 13, 491, 2702, 594, 0, 62] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[3077, 5885, 648, 3519, 3032, 4, 1, 427, 170, 294, 16, 2336, 41, 4, 297, 15, 1, 47, 185, 4, 0, 4, 0, 45, 15, 1, 71, 16, 4, 0, 3078, 1178, 27, 5, 0, 1025, 128, 67, 1081, 1, 2058, 372, 0, 30, 1888, 7, 0, 6581, 594, 7475, 3, 216, 0, 986, 5, 0, 0, 4, 145, 1025, 397, 615, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[2674, 3778, 3288, 5389, 1803, 13, 1, 2249, 758, 436, 251, 767, 1, 0, 304, 8, 9, 206, 1, 61, 11, 4, 1, 398, 162, 4, 153, 14, 0, 0, 2469, 1096, 23, 0, 0, 327, 282, 189, 856, 0, 3824, 431, 5, 87, 1885, 32, 0, 2877, 658, 6723, 15, 124, 2, 741, 2, 0, 0, 15, 256, 952, 1899, 613, 0, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[2631, 1427, 3364, 5179, 1781, 104, 36, 3685, 1720, 1266, 253, 282, 0, 0, 579, 35, 97, 972, 0, 654, 51, 8, 0, 555, 724, 18, 142, 95, 0, 0, 1189, 777, 206, 8, 12, 69, 735, 198, 826, 2, 3212, 371, 55, 743, 1703, 6, 2, 1189, 923, 5605, 1, 33, 17, 514, 0, 0, 120, 21, 51, 1005, 3086, 612, 0, 203] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[2674, 3778, 3288, 5389, 1803, 13, 1, 2249, 758, 436, 251, 767, 1, 0, 304, 8, 9, 206, 1, 61, 11, 4, 1, 398, 162, 4, 153, 14, 0, 0, 2469, 1096, 23, 0, 0, 327, 282, 189, 856, 0, 3824, 431, 5, 87, 1885, 32, 0, 2877, 658, 6723, 15, 124, 2, 741, 2, 0, 0, 15, 256, 952, 1899, 613, 0, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[2620, 1186, 3129, 5590, 1592, 45, 45, 4996, 2557, 1345, 289, 215, 0, 0, 556, 34, 122, 950, 0, 382, 39, 28, 0, 807, 967, 3, 98, 149, 0, 0, 1597, 963, 209, 9, 0, 85, 646, 171, 552, 2, 4507, 214, 29, 1108, 1460, 1, 0, 621, 653, 3821, 1, 2, 60, 476, 0, 0, 28, 16, 1, 1042, 2289, 635, 0, 210] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[2249, 1142, 4259, 5505, 906, 36, 5, 4556, 2052, 1284, 292, 157, 0, 0, 716, 23, 38, 740, 0, 416, 10, 29, 0, 1359, 523, 4, 84, 79, 0, 0, 1691, 1223, 204, 19, 0, 80, 689, 224, 665, 0, 4317, 319, 8, 378, 1610, 3, 0, 729, 601, 4879, 1, 1, 33, 667, 0, 0, 19, 24, 152, 890, 2350, 636, 0, 276] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1853, 1548, 3699, 6030, 912, 22, 24, 4595, 2213, 1176, 364, 225, 0, 0, 615, 50, 41, 844, 0, 409, 21, 28, 0, 1249, 466, 0, 89, 138, 0, 0, 1692, 963, 181, 3, 0, 54, 1015, 481, 499, 0, 4501, 182, 1, 737, 1484, 2, 0, 1002, 589, 4781, 4, 7, 0, 497, 0, 0, 0, 4, 13, 491, 2702, 594, 0, 62] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[3819, 5822, 390, 1381, 2136, 13, 0, 14, 6, 140, 20, 3575, 137, 2, 257, 12, 0, 5, 359, 0, 0, 0, 2, 19, 0, 1, 5, 0, 0, 0, 3228, 786, 5, 0, 0, 2464, 12, 28, 1203, 0, 1742, 963, 0, 4, 1667, 1, 0, 7442, 547, 7685, 0, 240, 1, 1120, 5, 0, 0, 2, 325, 1054, 18, 493, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[3069, 2069, 2522, 6035, 2030, 24, 7, 2442, 1264, 1186, 237, 575, 0, 0, 425, 34, 30, 378, 0, 211, 13, 12, 0, 400, 428, 1, 119, 80, 0, 0, 2489, 1322, 41, 0, 0, 199, 720, 392, 754, 0, 3772, 284, 17, 503, 1831, 11, 0, 2581, 689, 5948, 0, 81, 1, 501, 0, 0, 23, 2, 18, 887, 1803, 628, 0, 64] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[2249, 1142, 4259, 5505, 906, 36, 5, 4556, 2052, 1284, 292, 157, 0, 0, 716, 23, 38, 740, 0, 416, 10, 29, 0, 1359, 523, 4, 84, 79, 0, 0, 1691, 1223, 204, 19, 0, 80, 689, 224, 665, 0, 4317, 319, 8, 378, 1610, 3, 0, 729, 601, 4879, 1, 1, 33, 667, 0, 0, 19, 24, 152, 890, 2350, 636, 0, 276] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1589, 0, 3364, 0, 1781, 104, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 1266, 253, 282, 0, 0, 579, 35, 1042, 1024, 0, 0, 0, 0, 0, 0, 97, 972, 0, 654, 51, 8, 0, 555, 4946, 0, 0, 0, 0, 0, 0, 0, 724, 18, 142, 95, 0, 0, 1189, 777, 2064, 95, 0, 0, 0, 0, 0, 0, 206, 8, 12, 69, 735, 198, 826, 2, 3685, 423, 233, 0, 0, 0, 0, 0, 656, 371, 55, 743, 1703, 6, 2, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 923, 4486, 1, 33, 17, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 21, 51, 1005, 3086, 612, 0, 203, 1004, 492, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6874, 6201, 7283, 5104, 6397, 4725, 5974, 6594] avg=6144.0000 max=7283 max_rank=2 min=4725 min_rank=5 imbalance_ratio=1.185384 max_load_reduction=41869 max_load_reduction_pct=0.851827 moved_to_echo=23690 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1859, 0, 648, 0, 3032, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 294, 16, 2336, 41, 4, 297, 15, 1218, 1366, 0, 0, 0, 0, 0, 0, 1, 47, 185, 4, 0, 4, 0, 45, 3361, 0, 0, 0, 0, 0, 0, 0, 15, 1, 71, 16, 4, 0, 3078, 1178, 1323, 127, 0, 0, 0, 0, 0, 0, 27, 5, 0, 1025, 128, 67, 1081, 1, 427, 1746, 158, 0, 0, 0, 0, 0, 420, 372, 0, 30, 1888, 7, 0, 6581, 0, 0, 0, 0, 0, 0, 0, 0, 594, 5982, 3, 216, 0, 986, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 145, 1025, 397, 615, 0, 2, 4139, 315, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5544, 5757, 3647, 5813, 4665, 9298, 7786, 6642] avg=6144.0000 max=9298 max_rank=5 min=3647 min_rank=2 imbalance_ratio=1.513346 max_load_reduction=39854 max_load_reduction_pct=0.810832 moved_to_echo=28391 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1583, 0, 3129, 0, 1592, 45, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 1345, 289, 215, 0, 0, 556, 34, 1037, 698, 0, 0, 0, 0, 0, 0, 122, 950, 0, 382, 39, 28, 0, 807, 5339, 0, 0, 0, 0, 0, 0, 0, 967, 3, 98, 149, 0, 0, 1597, 963, 2897, 65, 0, 0, 0, 0, 0, 0, 209, 9, 0, 85, 646, 171, 552, 2, 4996, 352, 251, 0, 0, 0, 0, 0, 920, 214, 29, 1108, 1460, 1, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 653, 3058, 1, 2, 60, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 16, 1, 1042, 2289, 635, 0, 210, 834, 690, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 6731, 7667, 6739, 7273, 4353, 4250, 5745] avg=6144.0000 max=7667 max_rank=2 min=4250 min_rank=6 imbalance_ratio=1.247884 max_load_reduction=41485 max_load_reduction_pct=0.844014 moved_to_echo=21621 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1120, 0, 3699, 0, 912, 22, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2213, 1176, 364, 225, 0, 0, 615, 50, 733, 873, 0, 0, 0, 0, 0, 0, 41, 844, 0, 409, 21, 28, 0, 1249, 5759, 0, 0, 0, 0, 0, 0, 0, 466, 0, 89, 138, 0, 0, 1692, 963, 2893, 81, 0, 0, 0, 0, 0, 0, 181, 3, 0, 54, 1015, 481, 499, 0, 4595, 459, 271, 0, 0, 0, 0, 0, 919, 182, 1, 737, 1484, 2, 0, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 589, 3827, 4, 7, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 13, 491, 2702, 594, 0, 62, 1089, 689, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 6249, 8351, 6322, 7558, 4327, 4924, 5644] avg=6144.0000 max=8351 max_rank=2 min=4327 min_rank=5 imbalance_ratio=1.359212 max_load_reduction=40801 max_load_reduction_pct=0.830098 moved_to_echo=22453 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1589, 0, 3364, 0, 1781, 104, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 1266, 253, 282, 0, 0, 579, 35, 1042, 1024, 0, 0, 0, 0, 0, 0, 97, 972, 0, 654, 51, 8, 0, 555, 4946, 0, 0, 0, 0, 0, 0, 0, 724, 18, 142, 95, 0, 0, 1189, 777, 2064, 95, 0, 0, 0, 0, 0, 0, 206, 8, 12, 69, 735, 198, 826, 2, 3685, 423, 233, 0, 0, 0, 0, 0, 656, 371, 55, 743, 1703, 6, 2, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 923, 4486, 1, 33, 17, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 21, 51, 1005, 3086, 612, 0, 203, 1004, 492, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6874, 6201, 7283, 5104, 6397, 4725, 5974, 6594] avg=6144.0000 max=7283 max_rank=2 min=4725 min_rank=5 imbalance_ratio=1.185384 max_load_reduction=41869 max_load_reduction_pct=0.851827 moved_to_echo=23690 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2307, 0, 390, 0, 2136, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 140, 20, 3575, 137, 2, 257, 12, 1512, 1404, 0, 0, 0, 0, 0, 0, 0, 5, 359, 0, 0, 0, 2, 19, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 3228, 786, 1119, 131, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2464, 12, 28, 1203, 0, 14, 1728, 61, 0, 0, 0, 0, 0, 357, 963, 0, 4, 1667, 1, 0, 7442, 0, 0, 0, 0, 0, 0, 0, 0, 547, 6150, 0, 240, 1, 1120, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 325, 1054, 18, 493, 0, 2, 4094, 266, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4846, 7065, 1705, 5270, 5515, 10434, 8063, 6254] avg=6144.0000 max=10434 max_rank=5 min=1705 min_rank=2 imbalance_ratio=1.698242 max_load_reduction=38718 max_load_reduction_pct=0.787720 moved_to_echo=30266 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1359, 0, 4259, 0, 906, 36, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2052, 1284, 292, 157, 0, 0, 716, 23, 890, 891, 0, 0, 0, 0, 0, 0, 38, 740, 0, 416, 10, 29, 0, 1359, 5259, 0, 0, 0, 0, 0, 0, 0, 523, 4, 84, 79, 0, 0, 1691, 1223, 2775, 83, 0, 0, 0, 0, 0, 0, 204, 19, 0, 80, 689, 224, 665, 0, 4556, 339, 246, 0, 0, 0, 0, 0, 881, 319, 8, 378, 1610, 3, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 601, 3905, 1, 1, 33, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 24, 152, 890, 2350, 636, 0, 276, 803, 661, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6565, 6305, 7851, 6462, 7022, 3928, 5208, 5811] avg=6144.0000 max=7851 max_rank=2 min=3928 min_rank=5 imbalance_ratio=1.277832 max_load_reduction=41301 max_load_reduction_pct=0.840271 moved_to_echo=21969 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1610, 0, 3288, 0, 1803, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 436, 251, 767, 1, 0, 304, 8, 1064, 1232, 0, 0, 0, 0, 0, 0, 9, 206, 1, 61, 11, 4, 1, 398, 5148, 0, 0, 0, 0, 0, 0, 0, 162, 4, 153, 14, 0, 0, 2469, 1096, 2462, 118, 0, 0, 0, 0, 0, 0, 23, 0, 0, 327, 282, 189, 856, 0, 2249, 1122, 241, 0, 0, 0, 0, 0, 772, 431, 5, 87, 1885, 32, 0, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 658, 5373, 15, 124, 2, 741, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 256, 952, 1899, 613, 0, 25, 2656, 590, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6715, 4821, 5839, 6478, 5289, 6089, 6915, 7006] avg=6144.0000 max=7006 max_rank=7 min=4821 min_rank=1 imbalance_ratio=1.140299 max_load_reduction=42146 max_load_reduction_pct=0.857463 moved_to_echo=25299 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1854, 0, 2522, 0, 2030, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 1186, 237, 575, 0, 0, 425, 34, 1215, 1087, 0, 0, 0, 0, 0, 0, 30, 378, 0, 211, 13, 12, 0, 400, 5765, 0, 0, 0, 0, 0, 0, 0, 428, 1, 119, 80, 0, 0, 2489, 1322, 2424, 101, 0, 0, 0, 0, 0, 0, 41, 0, 0, 199, 720, 392, 754, 0, 2442, 615, 270, 0, 0, 0, 0, 0, 771, 284, 17, 503, 1831, 11, 0, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 689, 4760, 0, 81, 1, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 18, 887, 1803, 628, 0, 64, 1454, 577, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6437, 6023, 6809, 6964, 5433, 5998, 6032, 5456] avg=6144.0000 max=6964 max_rank=3 min=5433 min_rank=4 imbalance_ratio=1.133464 max_load_reduction=42188 max_load_reduction_pct=0.858317 moved_to_echo=22919 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1583, 0, 3129, 0, 1592, 45, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 1345, 289, 215, 0, 0, 556, 34, 1037, 698, 0, 0, 0, 0, 0, 0, 122, 950, 0, 382, 39, 28, 0, 807, 5339, 0, 0, 0, 0, 0, 0, 0, 967, 3, 98, 149, 0, 0, 1597, 963, 2897, 65, 0, 0, 0, 0, 0, 0, 209, 9, 0, 85, 646, 171, 552, 2, 4996, 352, 251, 0, 0, 0, 0, 0, 920, 214, 29, 1108, 1460, 1, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 653, 3058, 1, 2, 60, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 16, 1, 1042, 2289, 635, 0, 210, 834, 690, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 6731, 7667, 6739, 7273, 4353, 4250, 5745] avg=6144.0000 max=7667 max_rank=2 min=4250 min_rank=6 imbalance_ratio=1.247884 max_load_reduction=41485 max_load_reduction_pct=0.844014 moved_to_echo=21621 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1359, 0, 4259, 0, 906, 36, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2052, 1284, 292, 157, 0, 0, 716, 23, 890, 891, 0, 0, 0, 0, 0, 0, 38, 740, 0, 416, 10, 29, 0, 1359, 5259, 0, 0, 0, 0, 0, 0, 0, 523, 4, 84, 79, 0, 0, 1691, 1223, 2775, 83, 0, 0, 0, 0, 0, 0, 204, 19, 0, 80, 689, 224, 665, 0, 4556, 339, 246, 0, 0, 0, 0, 0, 881, 319, 8, 378, 1610, 3, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 601, 3905, 1, 1, 33, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 24, 152, 890, 2350, 636, 0, 276, 803, 661, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6565, 6305, 7851, 6462, 7022, 3928, 5208, 5811] avg=6144.0000 max=7851 max_rank=2 min=3928 min_rank=5 imbalance_ratio=1.277832 max_load_reduction=41301 max_load_reduction_pct=0.840271 moved_to_echo=21969 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10752, 7168, 1792, 8192, 256, 128, 3840, 256, 8832, 8576, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 3200, 128, 3712, 13568, 128, 128, 23040, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 128, 1024, 7424, 14592, 4864, 0, 896, 16128, 4352, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[13312, 0, 21376, 0, 14208, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 128, 1024, 7424, 14592, 4864, 0, 896, 16128, 4352, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 128, 768, 640, 128, 0, 17536, 8320, 18048, 896, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 128, 128, 4352, 4352, 1792, 6528, 128, 23040, 6784, 1792, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 37632, 128, 768, 128, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 4224, 640, 2176, 256, 128, 128, 4864, 36992, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4136, 28, 309, 196, 0, 528, 12, 3613, 1, 1669, 8, 0, 0, 8148, 0, 388, 20, 4731, 1, 2, 6145, 0, 9, 12, 3, 737, 12, 0, 7, 17, 5145, 328, 0, 18, 8, 726, 1, 0, 79, 0, 29, 5, 249, 3181, 171, 204, 65, 183, 3, 179, 77, 2230, 127, 14, 8, 3726, 38, 3, 196, 16, 0, 1411] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4, 2, 125, 507, 817, 874, 0, 331, 97, 2057, 0, 7058, 0, 2, 10, 7555, 71, 2627, 5, 1086, 101, 20, 2791, 0, 123, 143, 93, 711, 641, 1, 165, 806, 1235, 607, 37, 165, 6, 284, 145, 0, 545, 2, 373, 716, 179, 1549, 2710, 118, 577, 403, 34, 932, 422, 1350, 1690, 321, 56, 2267, 724, 19, 257, 290, 27, 2289] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1, 1, 75, 629, 1366, 698, 0, 294, 23, 1012, 0, 6971, 0, 0, 7, 7744, 109, 2470, 11, 708, 43, 8, 2235, 0, 228, 381, 52, 1285, 450, 1, 196, 625, 814, 532, 7, 222, 4, 166, 384, 0, 529, 5, 441, 480, 157, 1650, 2929, 91, 637, 734, 42, 801, 539, 1586, 2331, 177, 85, 2486, 802, 60, 254, 234, 0, 2350] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4136, 28, 309, 196, 0, 528, 12, 3613, 1, 1669, 8, 0, 0, 8148, 0, 388, 20, 4731, 1, 2, 6145, 0, 9, 12, 3, 737, 12, 0, 7, 17, 5145, 328, 0, 18, 8, 726, 1, 0, 79, 0, 29, 5, 249, 3181, 171, 204, 65, 183, 3, 179, 77, 2230, 127, 14, 8, 3726, 38, 3, 196, 16, 0, 1411] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[6, 1, 22, 963, 831, 575, 0, 254, 33, 1555, 0, 7030, 0, 0, 6, 7519, 70, 2677, 11, 518, 7, 27, 2236, 0, 182, 186, 190, 993, 464, 2, 136, 1061, 728, 895, 14, 336, 7, 202, 238, 1, 634, 1, 548, 886, 110, 974, 3052, 90, 418, 301, 29, 1334, 981, 825, 2355, 284, 71, 2343, 605, 13, 249, 314, 0, 2759] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1, 4, 811, 452, 496, 643, 0, 298, 36, 2147, 0, 6409, 0, 0, 6, 7851, 142, 1949, 5, 1918, 16, 27, 4590, 0, 51, 75, 71, 936, 328, 0, 126, 479, 2112, 579, 5, 163, 5, 441, 82, 0, 205, 2, 235, 359, 212, 1538, 1075, 99, 288, 391, 33, 602, 388, 1495, 923, 143, 52, 3372, 648, 16, 225, 148, 0, 3449] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 7, 1534, 131, 480, 807, 0, 407, 2, 1798, 0, 5261, 0, 0, 2, 8085, 13, 2037, 2, 2337, 7, 2, 3822, 0, 12, 115, 2, 659, 442, 7, 144, 44, 2799, 341, 0, 61, 0, 287, 43, 0, 16, 0, 96, 50, 219, 1637, 1433, 109, 872, 788, 41, 595, 104, 2628, 392, 73, 10, 4222, 323, 2, 250, 114, 0, 3488] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 5664, 3, 287, 50, 0, 543, 0, 3702, 0, 372, 0, 0, 1, 7816, 0, 13, 1, 6427, 0, 1, 5787, 0, 0, 1, 0, 191, 6, 0, 0, 2, 5954, 60, 0, 0, 0, 1162, 0, 0, 1, 0, 0, 5, 164, 3076, 40, 194, 6, 65, 2, 116, 1, 3185, 3, 0, 0, 3303, 0, 1, 28, 0, 0, 918] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1, 4, 811, 452, 496, 643, 0, 298, 36, 2147, 0, 6409, 0, 0, 6, 7851, 142, 1949, 5, 1918, 16, 27, 4590, 0, 51, 75, 71, 936, 328, 0, 126, 479, 2112, 579, 5, 163, 5, 441, 82, 0, 205, 2, 235, 359, 212, 1538, 1075, 99, 288, 391, 33, 602, 388, 1495, 923, 143, 52, 3372, 648, 16, 225, 148, 0, 3449] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1, 1, 75, 629, 1366, 698, 0, 294, 23, 1012, 0, 6971, 0, 0, 7, 7744, 109, 2470, 11, 708, 43, 8, 2235, 0, 228, 381, 52, 1285, 450, 1, 196, 625, 814, 532, 7, 222, 4, 166, 384, 0, 529, 5, 441, 480, 157, 1650, 2929, 91, 637, 734, 42, 801, 539, 1586, 2331, 177, 85, 2486, 802, 60, 254, 234, 0, 2350] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1, 5, 38, 743, 579, 910, 0, 374, 64, 1398, 0, 6797, 1, 0, 22, 7952, 23, 3132, 4, 715, 78, 38, 1745, 0, 77, 216, 27, 804, 731, 0, 234, 497, 1076, 452, 23, 221, 3, 2, 201, 0, 155, 0, 449, 465, 225, 979, 3855, 98, 668, 639, 56, 832, 146, 1528, 2106, 131, 134, 3631, 566, 130, 294, 227, 0, 2655] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 2, 125, 507, 817, 874, 0, 331, 2448, 0, 0, 0, 0, 0, 0, 0, 97, 2057, 0, 5356, 0, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 2627, 5, 1086, 101, 20, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 143, 93, 711, 641, 1, 165, 806, 3975, 0, 0, 0, 0, 0, 0, 0, 1235, 607, 37, 165, 6, 284, 145, 0, 991, 1132, 0, 0, 0, 0, 0, 0, 545, 2, 373, 716, 179, 1549, 2710, 118, 866, 415, 0, 0, 0, 0, 0, 0, 577, 403, 34, 932, 422, 1350, 1690, 321, 836, 268, 0, 0, 0, 0, 0, 0, 56, 1999, 724, 19, 257, 290, 27, 2289, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5108, 7522, 5295, 6658, 4602, 7473, 6833, 5661] avg=6144.0000 max=7522 max_rank=1 min=4602 min_rank=4 imbalance_ratio=1.224284 max_load_reduction=41630 max_load_reduction_pct=0.846965 moved_to_echo=24569 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[6, 1, 22, 963, 831, 575, 0, 254, 2436, 0, 0, 0, 0, 0, 0, 0, 33, 1555, 0, 5335, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 2677, 11, 518, 7, 27, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 186, 190, 993, 464, 2, 136, 1061, 3954, 0, 0, 0, 0, 0, 0, 0, 728, 895, 14, 336, 7, 202, 238, 1, 794, 1129, 0, 0, 0, 0, 0, 0, 634, 1, 548, 886, 110, 974, 3052, 90, 863, 332, 0, 0, 0, 0, 0, 0, 418, 301, 29, 1334, 981, 825, 2355, 284, 832, 277, 0, 0, 0, 0, 0, 0, 71, 2066, 605, 13, 249, 314, 0, 2759, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5088, 6929, 4420, 7168, 4344, 7490, 7636, 6077] avg=6144.0000 max=7636 max_rank=6 min=4344 min_rank=4 imbalance_ratio=1.242839 max_load_reduction=41516 max_load_reduction_pct=0.844645 moved_to_echo=25547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4136, 28, 309, 196, 0, 528, 2640, 0, 0, 0, 0, 0, 0, 0, 12, 3613, 1, 1268, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 20, 4731, 1, 2, 3048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 12, 3, 737, 12, 0, 7, 17, 4285, 0, 0, 0, 0, 0, 0, 0, 5145, 328, 0, 18, 8, 726, 1, 0, 2183, 1223, 0, 0, 0, 0, 0, 0, 79, 0, 29, 5, 249, 3181, 171, 204, 204, 914, 0, 0, 0, 0, 0, 0, 65, 183, 3, 179, 77, 2230, 127, 14, 197, 441, 0, 0, 0, 0, 0, 0, 8, 3285, 38, 3, 196, 16, 0, 1411, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7837, 4902, 8190, 5082, 9632, 5036, 3516, 4957] avg=6144.0000 max=9632 max_rank=4 min=3516 min_rank=6 imbalance_ratio=1.567708 max_load_reduction=39520 max_load_reduction_pct=0.804036 moved_to_echo=23141 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 75, 629, 1366, 698, 0, 294, 2509, 0, 0, 0, 0, 0, 0, 0, 23, 1012, 0, 5290, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 2470, 11, 708, 43, 8, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 381, 52, 1285, 450, 1, 196, 625, 4074, 0, 0, 0, 0, 0, 0, 0, 814, 532, 7, 222, 4, 166, 384, 0, 794, 1161, 0, 0, 0, 0, 0, 0, 529, 5, 441, 480, 157, 1650, 2929, 91, 856, 332, 0, 0, 0, 0, 0, 0, 637, 734, 42, 801, 539, 1586, 2331, 177, 825, 294, 0, 0, 0, 0, 0, 0, 85, 2192, 802, 60, 254, 234, 0, 2350, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5573, 6332, 4458, 7292, 4084, 7470, 7966, 5977] avg=6144.0000 max=7966 max_rank=6 min=4084 min_rank=4 imbalance_ratio=1.296549 max_load_reduction=41186 max_load_reduction_pct=0.837931 moved_to_echo=25497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 5664, 3, 287, 50, 0, 543, 2534, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 0, 283, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 6427, 0, 1, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 191, 6, 0, 0, 2, 4111, 0, 0, 0, 0, 0, 0, 0, 5954, 60, 0, 0, 0, 1162, 0, 0, 2056, 1171, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 164, 3076, 40, 194, 45, 861, 0, 0, 0, 0, 0, 0, 6, 65, 2, 116, 1, 3185, 3, 0, 44, 391, 0, 0, 0, 0, 0, 0, 0, 2912, 0, 1, 28, 0, 0, 918, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9082, 3986, 9312, 4311, 10403, 4386, 3813, 3859] avg=6144.0000 max=10403 max_rank=4 min=3813 min_rank=6 imbalance_ratio=1.693197 max_load_reduction=38749 max_load_reduction_pct=0.788350 moved_to_echo=22461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 2, 125, 507, 817, 874, 0, 331, 2448, 0, 0, 0, 0, 0, 0, 0, 97, 2057, 0, 5356, 0, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 2627, 5, 1086, 101, 20, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 143, 93, 711, 641, 1, 165, 806, 3975, 0, 0, 0, 0, 0, 0, 0, 1235, 607, 37, 165, 6, 284, 145, 0, 991, 1132, 0, 0, 0, 0, 0, 0, 545, 2, 373, 716, 179, 1549, 2710, 118, 866, 415, 0, 0, 0, 0, 0, 0, 577, 403, 34, 932, 422, 1350, 1690, 321, 836, 268, 0, 0, 0, 0, 0, 0, 56, 1999, 724, 19, 257, 290, 27, 2289, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5108, 7522, 5295, 6658, 4602, 7473, 6833, 5661] avg=6144.0000 max=7522 max_rank=1 min=4602 min_rank=4 imbalance_ratio=1.224284 max_load_reduction=41630 max_load_reduction_pct=0.846965 moved_to_echo=24569 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 811, 452, 496, 643, 0, 298, 2544, 0, 0, 0, 0, 0, 0, 0, 36, 2147, 0, 4863, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 1949, 5, 1918, 16, 27, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 75, 71, 936, 328, 0, 126, 479, 4130, 0, 0, 0, 0, 0, 0, 0, 2112, 579, 5, 163, 5, 441, 82, 0, 1630, 1177, 0, 0, 0, 0, 0, 0, 205, 2, 235, 359, 212, 1538, 1075, 99, 787, 683, 0, 0, 0, 0, 0, 0, 288, 391, 33, 602, 388, 1495, 923, 143, 759, 399, 0, 0, 0, 0, 0, 0, 52, 2973, 648, 16, 225, 148, 0, 3449, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5249, 7052, 6334, 6196, 6194, 5195, 5421, 7511] avg=6144.0000 max=7511 max_rank=7 min=5195 min_rank=5 imbalance_ratio=1.222493 max_load_reduction=41641 max_load_reduction_pct=0.847188 moved_to_echo=24321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[6, 1, 22, 963, 831, 575, 0, 254, 2436, 0, 0, 0, 0, 0, 0, 0, 33, 1555, 0, 5335, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 2677, 11, 518, 7, 27, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 186, 190, 993, 464, 2, 136, 1061, 3954, 0, 0, 0, 0, 0, 0, 0, 728, 895, 14, 336, 7, 202, 238, 1, 794, 1129, 0, 0, 0, 0, 0, 0, 634, 1, 548, 886, 110, 974, 3052, 90, 863, 332, 0, 0, 0, 0, 0, 0, 418, 301, 29, 1334, 981, 825, 2355, 284, 832, 277, 0, 0, 0, 0, 0, 0, 71, 2066, 605, 13, 249, 314, 0, 2759, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5088, 6929, 4420, 7168, 4344, 7490, 7636, 6077] avg=6144.0000 max=7636 max_rank=6 min=4344 min_rank=4 imbalance_ratio=1.242839 max_load_reduction=41516 max_load_reduction_pct=0.844645 moved_to_echo=25547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 1534, 131, 480, 807, 0, 407, 2621, 0, 0, 0, 0, 0, 0, 0, 2, 1798, 0, 3984, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2037, 2, 2337, 7, 2, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 115, 2, 659, 442, 7, 144, 44, 4252, 0, 0, 0, 0, 0, 0, 0, 2799, 341, 0, 61, 0, 287, 43, 0, 1361, 1212, 0, 0, 0, 0, 0, 0, 16, 0, 96, 50, 219, 1637, 1433, 109, 650, 573, 0, 0, 0, 0, 0, 0, 872, 788, 41, 595, 104, 2628, 392, 73, 627, 506, 0, 0, 0, 0, 0, 0, 10, 3716, 323, 2, 250, 114, 0, 3488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5987, 5786, 6286, 5677, 6104, 4783, 6626, 7903] avg=6144.0000 max=7903 max_rank=7 min=4783 min_rank=5 imbalance_ratio=1.286296 max_load_reduction=41249 max_load_reduction_pct=0.839213 moved_to_echo=25416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 5, 38, 743, 579, 910, 0, 374, 2578, 0, 0, 0, 0, 0, 0, 0, 64, 1398, 0, 5158, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3132, 4, 715, 78, 38, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 216, 27, 804, 731, 0, 234, 497, 4182, 0, 0, 0, 0, 0, 0, 0, 1076, 452, 23, 221, 3, 2, 201, 0, 619, 1192, 0, 0, 0, 0, 0, 0, 155, 0, 449, 465, 225, 979, 3855, 98, 834, 259, 0, 0, 0, 0, 0, 0, 668, 639, 56, 832, 146, 1528, 2106, 131, 805, 430, 0, 0, 0, 0, 0, 0, 134, 3201, 566, 130, 294, 227, 0, 2655, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5228, 6643, 4857, 6768, 3789, 7319, 7341, 7207] avg=6144.0000 max=7341 max_rank=6 min=3789 min_rank=4 imbalance_ratio=1.194824 max_load_reduction=41811 max_load_reduction_pct=0.850647 moved_to_echo=25656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 1534, 131, 480, 807, 0, 407, 2621, 0, 0, 0, 0, 0, 0, 0, 2, 1798, 0, 3984, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2037, 2, 2337, 7, 2, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 115, 2, 659, 442, 7, 144, 44, 4252, 0, 0, 0, 0, 0, 0, 0, 2799, 341, 0, 61, 0, 287, 43, 0, 1361, 1212, 0, 0, 0, 0, 0, 0, 16, 0, 96, 50, 219, 1637, 1433, 109, 650, 573, 0, 0, 0, 0, 0, 0, 872, 788, 41, 595, 104, 2628, 392, 73, 627, 506, 0, 0, 0, 0, 0, 0, 10, 3716, 323, 2, 250, 114, 0, 3488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5987, 5786, 6286, 5677, 6104, 4783, 6626, 7903] avg=6144.0000 max=7903 max_rank=7 min=4783 min_rank=5 imbalance_ratio=1.286296 max_load_reduction=41249 max_load_reduction_pct=0.839213 moved_to_echo=25416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 22400, 3712, 256, 1792, 1408, 128, 19328, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[19968, 3840, 128, 1280, 128, 3328, 1152, 128, 10496, 9472, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 15360, 128, 18560, 256, 128, 14592, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 3584, 256, 5504, 2688, 14848, 9984, 1152, 4992, 3072, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 128, 2176, 3072, 1536, 14592, 15360, 1024, 5120, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[19968, 3840, 128, 1280, 128, 3328, 1152, 128, 10496, 9472, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 12416, 3456, 5248, 4864, 0, 3072, 20352, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 1152, 512, 6400, 3200, 128, 1024, 3584, 33024, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 3584, 256, 5504, 2688, 14848, 9984, 1152, 4992, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 15360, 128, 18560, 256, 128, 14592, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 17408, 128, 31616, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 1152, 512, 6400, 3200, 128, 1024, 3584, 33024, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[2182, 125, 65, 170, 2042, 211, 0, 0, 0, 119, 336, 137, 4362, 4, 805, 312, 0, 862, 0, 4396, 0, 573, 309, 12, 1661, 0, 323, 2757, 1, 1782, 832, 791, 5067, 341, 1143, 141, 423, 319, 119, 638, 0, 494, 385, 37, 715, 172, 896, 21, 3817, 0, 838, 518, 218, 793, 673, 648, 740, 4498, 185, 0, 15, 0, 7, 122] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[2254, 35, 31, 324, 1766, 247, 2, 0, 0, 194, 337, 129, 3768, 16, 1457, 432, 0, 623, 7, 4221, 0, 584, 220, 2, 1302, 0, 762, 2551, 4, 1908, 652, 787, 4156, 129, 789, 380, 247, 458, 123, 996, 0, 414, 364, 148, 1118, 141, 687, 7, 4148, 0, 353, 699, 500, 1289, 1244, 664, 470, 4745, 135, 0, 5, 0, 17, 111] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[217, 316, 89, 930, 272, 622, 1, 1, 14, 18, 354, 1, 6913, 0, 36, 23, 0, 13, 0, 544, 0, 76, 214, 1013, 5803, 0, 1, 1846, 3, 4430, 13, 84, 8070, 928, 5224, 24, 3652, 25, 238, 20, 0, 625, 119, 0, 42, 0, 71, 154, 1700, 0, 394, 53, 552, 24, 30, 478, 56, 139, 855, 1755, 3, 0, 3, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[3, 221, 86, 1168, 23, 365, 0, 0, 47, 0, 105, 0, 6547, 0, 0, 1, 0, 0, 0, 52, 0, 4, 75, 1728, 6530, 0, 1, 908, 10, 3991, 2, 1, 8185, 1625, 5448, 0, 4381, 0, 126, 1, 0, 696, 31, 1, 1, 3, 2, 185, 945, 0, 473, 6, 372, 5, 5, 481, 2, 14, 684, 3598, 0, 3, 1, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[1344, 215, 56, 515, 1902, 339, 0, 0, 0, 145, 467, 100, 4553, 11, 668, 331, 0, 564, 97, 4348, 0, 700, 344, 43, 2019, 0, 505, 2799, 3, 2757, 386, 612, 5201, 522, 1258, 518, 736, 402, 136, 718, 0, 422, 547, 238, 539, 179, 948, 56, 3001, 0, 396, 405, 663, 578, 1291, 590, 626, 2848, 205, 0, 7, 0, 197, 102] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[2254, 35, 31, 324, 1766, 247, 2, 0, 0, 194, 337, 129, 3768, 16, 1457, 432, 0, 623, 7, 4221, 0, 584, 220, 2, 1302, 0, 762, 2551, 4, 1908, 652, 787, 4156, 129, 789, 380, 247, 458, 123, 996, 0, 414, 364, 148, 1118, 141, 687, 7, 4148, 0, 353, 699, 500, 1289, 1244, 664, 470, 4745, 135, 0, 5, 0, 17, 111] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[2182, 125, 65, 170, 2042, 211, 0, 0, 0, 119, 336, 137, 4362, 4, 805, 312, 0, 862, 0, 4396, 0, 573, 309, 12, 1661, 0, 323, 2757, 1, 1782, 832, 791, 5067, 341, 1143, 141, 423, 319, 119, 638, 0, 494, 385, 37, 715, 172, 896, 21, 3817, 0, 838, 518, 218, 793, 673, 648, 740, 4498, 185, 0, 15, 0, 7, 122] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[1408, 149, 120, 563, 869, 471, 0, 1, 0, 133, 440, 208, 5802, 8, 544, 136, 0, 168, 5, 3688, 0, 499, 137, 75, 3541, 0, 272, 2906, 4, 2744, 359, 742, 6433, 578, 1634, 148, 1284, 229, 121, 659, 0, 483, 355, 55, 420, 90, 487, 65, 3384, 0, 513, 358, 580, 586, 840, 601, 299, 2127, 593, 21, 5, 0, 53, 159] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[916, 488, 107, 185, 1697, 624, 0, 1, 1, 50, 460, 40, 6120, 4, 136, 108, 0, 99, 0, 2479, 0, 639, 137, 101, 2968, 0, 86, 2486, 9, 3361, 177, 691, 7653, 1160, 4035, 58, 2174, 337, 208, 204, 0, 834, 347, 15, 144, 44, 191, 96, 2952, 0, 814, 56, 335, 189, 252, 743, 133, 951, 553, 134, 0, 0, 214, 156] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[3, 221, 86, 1168, 23, 365, 0, 0, 47, 0, 105, 0, 6547, 0, 0, 1, 0, 0, 0, 52, 0, 4, 75, 1728, 6530, 0, 1, 908, 10, 3991, 2, 1, 8185, 1625, 5448, 0, 4381, 0, 126, 1, 0, 696, 31, 1, 1, 3, 2, 185, 945, 0, 473, 6, 372, 5, 5, 481, 2, 14, 684, 3598, 0, 3, 1, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[1408, 149, 120, 563, 869, 471, 0, 1, 0, 133, 440, 208, 5802, 8, 544, 136, 0, 168, 5, 3688, 0, 499, 137, 75, 3541, 0, 272, 2906, 4, 2744, 359, 742, 6433, 578, 1634, 148, 1284, 229, 121, 659, 0, 483, 355, 55, 420, 90, 487, 65, 3384, 0, 513, 358, 580, 586, 840, 601, 299, 2127, 593, 21, 5, 0, 53, 159] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[1976, 189, 34, 173, 2170, 306, 0, 0, 0, 260, 432, 81, 4725, 0, 836, 308, 0, 760, 2, 5245, 0, 674, 270, 0, 486, 0, 188, 2134, 10, 2282, 306, 951, 5026, 554, 1510, 246, 380, 966, 129, 537, 0, 538, 534, 132, 422, 145, 734, 48, 3688, 1, 438, 552, 198, 724, 954, 760, 592, 4152, 149, 0, 9, 2, 107, 127] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2254, 35, 31, 324, 1766, 247, 2, 0, 1442, 69, 0, 0, 0, 0, 0, 0, 0, 194, 337, 129, 3379, 16, 1457, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 7, 4221, 0, 584, 220, 2, 355, 389, 122, 0, 0, 0, 0, 0, 0, 0, 762, 2551, 4, 1740, 652, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 781, 380, 247, 458, 123, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 364, 148, 1118, 141, 687, 7, 2714, 0, 0, 0, 0, 0, 0, 0, 4148, 0, 353, 699, 500, 1289, 1244, 664, 46, 8, 0, 0, 0, 0, 0, 0, 470, 4745, 135, 0, 5, 0, 17, 111, 878, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 5944, 6523, 6496, 3114, 5593, 8951, 6361] avg=6144.0000 max=8951 max_rank=6 min=3114 min_rank=4 imbalance_ratio=1.456868 max_load_reduction=40201 max_load_reduction_pct=0.817891 moved_to_echo=24019 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2182, 125, 65, 170, 2042, 211, 0, 0, 1758, 88, 0, 0, 0, 0, 0, 0, 0, 119, 336, 137, 3912, 4, 805, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 4396, 0, 573, 309, 12, 453, 450, 114, 0, 0, 0, 0, 0, 0, 0, 323, 2757, 1, 1625, 832, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 1131, 141, 423, 319, 119, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 385, 37, 715, 172, 896, 21, 3309, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 838, 518, 218, 793, 673, 648, 43, 12, 0, 0, 0, 0, 0, 0, 740, 4498, 185, 0, 15, 0, 7, 122, 1120, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6641, 5625, 7169, 6329, 3112, 6029, 7560, 6687] avg=6144.0000 max=7560 max_rank=6 min=3112 min_rank=4 imbalance_ratio=1.230469 max_load_reduction=41592 max_load_reduction_pct=0.846191 moved_to_echo=23388 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1344, 215, 56, 515, 1902, 339, 0, 0, 1804, 107, 0, 0, 0, 0, 0, 0, 0, 145, 467, 100, 4083, 11, 668, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 97, 4348, 0, 700, 344, 43, 550, 470, 177, 0, 0, 0, 0, 0, 0, 0, 505, 2799, 3, 2514, 386, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 1245, 518, 736, 402, 136, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 547, 238, 539, 179, 948, 56, 3397, 0, 0, 0, 0, 0, 0, 0, 3001, 0, 396, 405, 663, 578, 1291, 590, 66, 13, 0, 0, 0, 0, 0, 0, 626, 2848, 205, 0, 7, 0, 197, 102, 1362, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6282, 5805, 7293, 6819, 4277, 6326, 7003, 5347] avg=6144.0000 max=7293 max_rank=2 min=4277 min_rank=4 imbalance_ratio=1.187012 max_load_reduction=41859 max_load_reduction_pct=0.851624 moved_to_echo=22953 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3, 221, 86, 1168, 23, 365, 0, 0, 2840, 348, 0, 0, 0, 0, 0, 0, 47, 0, 105, 0, 5871, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4, 75, 1728, 1781, 676, 256, 0, 0, 0, 0, 0, 0, 0, 1, 908, 10, 3639, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1625, 5390, 0, 4381, 0, 126, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 31, 1, 1, 3, 2, 185, 5345, 0, 0, 0, 0, 0, 0, 0, 945, 0, 473, 6, 372, 5, 5, 481, 96, 58, 0, 0, 0, 0, 0, 0, 2, 14, 684, 3598, 0, 3, 1, 10, 4401, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5054, 6024, 4572, 4561, 11523, 6264, 2441, 8713] avg=6144.0000 max=11523 max_rank=4 min=2441 min_rank=6 imbalance_ratio=1.875488 max_load_reduction=37629 max_load_reduction_pct=0.765564 moved_to_echo=28941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[217, 316, 89, 930, 272, 622, 1, 1, 2800, 307, 0, 0, 0, 0, 0, 0, 14, 18, 354, 1, 6199, 0, 36, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 544, 0, 76, 214, 1013, 1583, 714, 284, 0, 0, 0, 0, 0, 0, 0, 1, 1846, 3, 4039, 13, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 5168, 24, 3652, 25, 238, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 119, 0, 42, 0, 71, 154, 5270, 0, 0, 0, 0, 0, 0, 0, 1700, 0, 394, 53, 552, 24, 30, 478, 107, 56, 0, 0, 0, 0, 0, 0, 56, 139, 855, 1755, 3, 0, 3, 71, 3913, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5555, 6645, 4441, 5986, 10055, 6281, 3394, 6795] avg=6144.0000 max=10055 max_rank=4 min=3394 min_rank=6 imbalance_ratio=1.636556 max_load_reduction=39097 max_load_reduction_pct=0.795431 moved_to_echo=26525 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2182, 125, 65, 170, 2042, 211, 0, 0, 1758, 88, 0, 0, 0, 0, 0, 0, 0, 119, 336, 137, 3912, 4, 805, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 4396, 0, 573, 309, 12, 453, 450, 114, 0, 0, 0, 0, 0, 0, 0, 323, 2757, 1, 1625, 832, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 1131, 141, 423, 319, 119, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 385, 37, 715, 172, 896, 21, 3309, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 838, 518, 218, 793, 673, 648, 43, 12, 0, 0, 0, 0, 0, 0, 740, 4498, 185, 0, 15, 0, 7, 122, 1120, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6641, 5625, 7169, 6329, 3112, 6029, 7560, 6687] avg=6144.0000 max=7560 max_rank=6 min=3112 min_rank=4 imbalance_ratio=1.230469 max_load_reduction=41592 max_load_reduction_pct=0.846191 moved_to_echo=23388 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2254, 35, 31, 324, 1766, 247, 2, 0, 1442, 69, 0, 0, 0, 0, 0, 0, 0, 194, 337, 129, 3379, 16, 1457, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 7, 4221, 0, 584, 220, 2, 355, 389, 122, 0, 0, 0, 0, 0, 0, 0, 762, 2551, 4, 1740, 652, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 781, 380, 247, 458, 123, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 364, 148, 1118, 141, 687, 7, 2714, 0, 0, 0, 0, 0, 0, 0, 4148, 0, 353, 699, 500, 1289, 1244, 664, 46, 8, 0, 0, 0, 0, 0, 0, 470, 4745, 135, 0, 5, 0, 17, 111, 878, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 5944, 6523, 6496, 3114, 5593, 8951, 6361] avg=6144.0000 max=8951 max_rank=6 min=3114 min_rank=4 imbalance_ratio=1.456868 max_load_reduction=40201 max_load_reduction_pct=0.817891 moved_to_echo=24019 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1976, 189, 34, 173, 2170, 306, 0, 0, 1744, 25, 0, 0, 0, 0, 0, 0, 0, 260, 432, 81, 4237, 0, 836, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 2, 5245, 0, 674, 270, 0, 134, 488, 146, 0, 0, 0, 0, 0, 0, 0, 188, 2134, 10, 2081, 306, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 1494, 246, 380, 966, 129, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 534, 132, 422, 145, 734, 48, 3282, 0, 0, 0, 0, 0, 0, 0, 3688, 1, 438, 552, 198, 724, 954, 760, 55, 16, 0, 0, 0, 0, 0, 0, 592, 4152, 149, 0, 9, 2, 107, 127, 327, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6617, 6154, 7719, 5670, 4306, 5835, 7386, 5465] avg=6144.0000 max=7719 max_rank=2 min=4306 min_rank=4 imbalance_ratio=1.256348 max_load_reduction=41433 max_load_reduction_pct=0.842957 moved_to_echo=22992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1408, 149, 120, 563, 869, 471, 0, 1, 2232, 187, 0, 0, 0, 0, 0, 0, 0, 133, 440, 208, 5203, 8, 544, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 5, 3688, 0, 499, 137, 75, 968, 599, 176, 0, 0, 0, 0, 0, 0, 0, 272, 2906, 4, 2502, 359, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 1617, 148, 1284, 229, 121, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 355, 55, 420, 90, 487, 65, 4201, 0, 0, 0, 0, 0, 0, 0, 3384, 0, 513, 358, 580, 586, 840, 601, 66, 17, 0, 0, 0, 0, 0, 0, 299, 2127, 593, 21, 5, 0, 53, 159, 2386, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6000, 6672, 6315, 6785, 4636, 6156, 6945, 5643] avg=6144.0000 max=6945 max_rank=6 min=4636 min_rank=4 imbalance_ratio=1.130371 max_load_reduction=42207 max_load_reduction_pct=0.858704 moved_to_echo=23380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3, 221, 86, 1168, 23, 365, 0, 0, 2840, 348, 0, 0, 0, 0, 0, 0, 47, 0, 105, 0, 5871, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4, 75, 1728, 1781, 676, 256, 0, 0, 0, 0, 0, 0, 0, 1, 908, 10, 3639, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1625, 5390, 0, 4381, 0, 126, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 31, 1, 1, 3, 2, 185, 5345, 0, 0, 0, 0, 0, 0, 0, 945, 0, 473, 6, 372, 5, 5, 481, 96, 58, 0, 0, 0, 0, 0, 0, 2, 14, 684, 3598, 0, 3, 1, 10, 4401, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5054, 6024, 4572, 4561, 11523, 6264, 2441, 8713] avg=6144.0000 max=11523 max_rank=4 min=2441 min_rank=6 imbalance_ratio=1.875488 max_load_reduction=37629 max_load_reduction_pct=0.765564 moved_to_echo=28941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[916, 488, 107, 185, 1697, 624, 0, 1, 2656, 159, 0, 0, 0, 0, 0, 0, 1, 50, 460, 40, 5484, 4, 136, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 2479, 0, 639, 137, 101, 809, 636, 220, 0, 0, 0, 0, 0, 0, 0, 86, 2486, 9, 3057, 177, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 3988, 58, 2174, 337, 208, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 347, 15, 144, 44, 191, 96, 4997, 0, 0, 0, 0, 0, 0, 0, 2952, 0, 814, 56, 335, 189, 252, 743, 84, 47, 0, 0, 0, 0, 0, 0, 133, 951, 553, 134, 0, 0, 214, 156, 2000, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6833, 6283, 5120, 6506, 8129, 6668, 5472, 4141] avg=6144.0000 max=8129 max_rank=4 min=4141 min_rank=7 imbalance_ratio=1.323079 max_load_reduction=41023 max_load_reduction_pct=0.834615 moved_to_echo=24410 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1976, 189, 34, 173, 2170, 306, 0, 0, 1744, 25, 0, 0, 0, 0, 0, 0, 0, 260, 432, 81, 4237, 0, 836, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 2, 5245, 0, 674, 270, 0, 134, 488, 146, 0, 0, 0, 0, 0, 0, 0, 188, 2134, 10, 2081, 306, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 1494, 246, 380, 966, 129, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 534, 132, 422, 145, 734, 48, 3282, 0, 0, 0, 0, 0, 0, 0, 3688, 1, 438, 552, 198, 724, 954, 760, 55, 16, 0, 0, 0, 0, 0, 0, 592, 4152, 149, 0, 9, 2, 107, 127, 327, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6617, 6154, 7719, 5670, 4306, 5835, 7386, 5465] avg=6144.0000 max=7719 max_rank=2 min=4306 min_rank=4 imbalance_ratio=1.256348 max_load_reduction=41433 max_load_reduction_pct=0.842957 moved_to_echo=22992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 5888, 20864, 1536, 13312, 2816, 1280, 3840, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10368, 1792, 640, 4096, 10752, 3200, 128, 128, 17280, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 3200, 128, 25088, 0, 3840, 1792, 3072, 6656, 4480, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 5888, 20864, 1536, 13312, 2816, 1280, 3840, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[23680, 128, 4224, 2688, 3456, 4224, 5376, 4992, 640, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2944, 19584, 3456, 5632, 128, 128, 640, 896, 16512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 3200, 128, 25088, 0, 3840, 1792, 3072, 6656, 4480, 1536, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 4608, 2688, 640, 3456, 896, 4096, 640, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2944, 19584, 3456, 5632, 128, 128, 640, 896, 16512, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 2176, 18432, 128, 21248, 2816, 4736, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1024, 2944, 768, 38400, 128, 4608, 1664, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[380, 8, 28, 42, 20, 108, 0, 1, 175, 3, 482, 0, 27, 191, 5262, 0, 661, 922, 3758, 366, 88, 2, 2, 7124, 0, 103, 37, 10, 7529, 2477, 0, 631, 150, 17, 1, 9, 2632, 5, 4534, 0, 5, 5, 277, 141, 0, 7267, 240, 0, 1, 35, 1109, 0, 466, 1, 281, 194, 886, 0, 430, 1, 5, 15, 0, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[385, 3, 0, 0, 45, 239, 0, 0, 96, 0, 311, 0, 4, 410, 5580, 0, 590, 326, 3085, 14, 180, 3, 2, 7397, 0, 0, 2, 0, 7184, 5046, 0, 43, 7, 0, 0, 0, 2492, 0, 3329, 0, 0, 0, 17, 3, 0, 7939, 533, 0, 1, 17, 418, 0, 250, 0, 1230, 271, 1656, 0, 41, 0, 1, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[111, 10, 102, 40, 14, 29, 0, 0, 574, 6, 1651, 0, 1017, 102, 1570, 0, 1305, 1724, 2184, 64, 59, 405, 9, 5080, 0, 259, 14, 852, 6911, 329, 0, 1517, 2109, 0, 0, 0, 3643, 176, 3187, 0, 1, 1, 3907, 774, 0, 3131, 85, 0, 0, 439, 2941, 0, 378, 3, 21, 77, 139, 18, 1447, 0, 585, 123, 0, 29] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[81, 7, 145, 54, 18, 16, 9, 12, 690, 44, 1572, 0, 1420, 80, 1207, 3, 1362, 1565, 929, 120, 57, 799, 6, 4128, 1, 445, 135, 758, 6371, 119, 0, 1385, 2307, 6, 1, 0, 2886, 264, 2538, 2, 2, 17, 6180, 1657, 0, 1448, 44, 8, 2, 447, 3398, 0, 294, 12, 7, 65, 42, 171, 2773, 0, 888, 52, 0, 103] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[10, 20, 530, 234, 13, 24, 0, 0, 1165, 23, 2396, 1, 1272, 30, 931, 1, 1151, 1308, 1586, 60, 33, 434, 6, 3954, 0, 546, 314, 841, 7295, 6, 0, 964, 2901, 0, 0, 0, 2562, 199, 1618, 0, 1, 1, 6721, 2138, 0, 807, 7, 0, 10, 515, 3240, 0, 238, 5, 6, 64, 33, 41, 2277, 0, 470, 92, 2, 56] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 3, 365, 166, 17, 10, 0, 3, 606, 70, 2439, 0, 2501, 23, 607, 3, 1274, 1073, 963, 68, 27, 989, 4, 2699, 0, 467, 141, 1333, 6653, 6, 0, 1193, 3895, 0, 0, 0, 2687, 216, 1511, 0, 0, 27, 7254, 2288, 0, 756, 0, 1, 12, 494, 2643, 0, 123, 9, 0, 37, 15, 38, 1918, 0, 1295, 157, 0, 73] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[111, 10, 102, 40, 14, 29, 0, 0, 574, 6, 1651, 0, 1017, 102, 1570, 0, 1305, 1724, 2184, 64, 59, 405, 9, 5080, 0, 259, 14, 852, 6911, 329, 0, 1517, 2109, 0, 0, 0, 3643, 176, 3187, 0, 1, 1, 3907, 774, 0, 3131, 85, 0, 0, 439, 2941, 0, 378, 3, 21, 77, 139, 18, 1447, 0, 585, 123, 0, 29] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[176, 37, 31, 56, 20, 162, 0, 2, 397, 27, 1227, 0, 121, 107, 2751, 0, 864, 1325, 2763, 87, 210, 21, 20, 6286, 0, 143, 26, 153, 7780, 831, 4, 1302, 641, 1, 0, 0, 2890, 37, 4101, 1, 1, 1, 2094, 343, 0, 5362, 125, 1, 0, 714, 2643, 0, 478, 56, 245, 128, 124, 4, 2064, 0, 126, 0, 2, 41] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[385, 3, 0, 0, 45, 239, 0, 0, 96, 0, 311, 0, 4, 410, 5580, 0, 590, 326, 3085, 14, 180, 3, 2, 7397, 0, 0, 2, 0, 7184, 5046, 0, 43, 7, 0, 0, 0, 2492, 0, 3329, 0, 0, 0, 17, 3, 0, 7939, 533, 0, 1, 17, 418, 0, 250, 0, 1230, 271, 1656, 0, 41, 0, 1, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[10, 20, 530, 234, 13, 24, 0, 0, 1165, 23, 2396, 1, 1272, 30, 931, 1, 1151, 1308, 1586, 60, 33, 434, 6, 3954, 0, 546, 314, 841, 7295, 6, 0, 964, 2901, 0, 0, 0, 2562, 199, 1618, 0, 1, 1, 6721, 2138, 0, 807, 7, 0, 10, 515, 3240, 0, 238, 5, 6, 64, 33, 41, 2277, 0, 470, 92, 2, 56] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[380, 8, 28, 42, 20, 108, 0, 1, 175, 3, 482, 0, 27, 191, 5262, 0, 661, 922, 3758, 366, 88, 2, 2, 7124, 0, 103, 37, 10, 7529, 2477, 0, 631, 150, 17, 1, 9, 2632, 5, 4534, 0, 5, 5, 277, 141, 0, 7267, 240, 0, 1, 35, 1109, 0, 466, 1, 281, 194, 886, 0, 430, 1, 5, 15, 0, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 15, 330, 239, 5, 46, 6, 0, 487, 51, 1956, 0, 1164, 20, 960, 7, 1344, 1669, 1308, 222, 78, 496, 5, 4463, 0, 534, 257, 383, 7356, 15, 0, 1504, 2394, 2, 0, 1, 2224, 209, 2577, 10, 9, 20, 7004, 2113, 0, 1036, 4, 5, 29, 569, 2799, 0, 259, 89, 11, 21, 20, 54, 2380, 0, 270, 26, 9, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[380, 8, 28, 42, 20, 108, 0, 1, 4359, 2021, 0, 0, 0, 0, 0, 0, 175, 3, 482, 0, 27, 191, 5262, 0, 1033, 0, 0, 0, 0, 0, 0, 0, 661, 922, 3758, 366, 88, 2, 2, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 37, 10, 3170, 2477, 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 150, 17, 1, 9, 2632, 5, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 90, 141, 0, 7267, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 1109, 0, 466, 1, 281, 194, 123, 1257, 0, 0, 0, 0, 0, 0, 886, 0, 430, 1, 5, 15, 0, 8, 3512, 64, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6967, 7173, 7390, 6428, 5058, 7748, 3467, 4921] avg=6144.0000 max=7748 max_rank=5 min=3467 min_rank=6 imbalance_ratio=1.261068 max_load_reduction=41404 max_load_reduction_pct=0.842367 moved_to_echo=21194 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[81, 7, 145, 54, 18, 16, 9, 12, 3689, 1171, 0, 0, 0, 0, 0, 0, 690, 44, 1572, 0, 1420, 80, 1207, 3, 578, 0, 0, 0, 0, 0, 0, 0, 1362, 1565, 929, 120, 57, 799, 6, 922, 0, 0, 0, 0, 0, 0, 0, 0, 1, 445, 135, 758, 2682, 119, 0, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 2307, 6, 1, 0, 2886, 264, 1257, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 17, 1997, 1657, 0, 1448, 44, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 447, 3398, 0, 294, 12, 7, 65, 2746, 703, 0, 0, 0, 0, 0, 0, 42, 171, 2773, 0, 888, 52, 0, 103, 2035, 1437, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5202, 5594, 5760, 5525, 6723, 5173, 7674, 7501] avg=6144.0000 max=7674 max_rank=6 min=5173 min_rank=5 imbalance_ratio=1.249023 max_load_reduction=41478 max_load_reduction_pct=0.843872 moved_to_echo=27071 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 365, 166, 17, 10, 0, 3, 3852, 765, 0, 0, 0, 0, 0, 0, 606, 70, 2439, 0, 2501, 23, 607, 3, 344, 0, 0, 0, 0, 0, 0, 0, 1274, 1073, 963, 68, 27, 989, 4, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 141, 1333, 2801, 6, 0, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 0, 0, 0, 2687, 216, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2344, 2288, 0, 756, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 12, 494, 2643, 0, 123, 9, 0, 37, 3224, 419, 0, 0, 0, 0, 0, 0, 15, 38, 1918, 0, 1295, 157, 0, 73, 1330, 1686, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5181, 6593, 5002, 5941, 7546, 5416, 6961, 6512] avg=6144.0000 max=7546 max_rank=4 min=5002 min_rank=2 imbalance_ratio=1.228190 max_load_reduction=41606 max_load_reduction_pct=0.846476 moved_to_echo=26435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[385, 3, 0, 0, 45, 239, 0, 0, 4160, 2099, 0, 0, 0, 0, 0, 0, 96, 0, 311, 0, 4, 410, 5580, 0, 758, 0, 0, 0, 0, 0, 0, 0, 590, 326, 3085, 14, 180, 3, 2, 1651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3024, 5046, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2492, 0, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 7939, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 418, 0, 250, 0, 1230, 271, 7, 923, 0, 0, 0, 0, 0, 0, 1656, 0, 41, 0, 1, 0, 0, 2, 3647, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6931, 7159, 5851, 8115, 4147, 8482, 3117, 5350] avg=6144.0000 max=8482 max_rank=5 min=3117 min_rank=6 imbalance_ratio=1.380534 max_load_reduction=40670 max_load_reduction_pct=0.827433 moved_to_echo=21096 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[10, 20, 530, 234, 13, 24, 0, 0, 4224, 1122, 0, 0, 0, 0, 0, 0, 1165, 23, 2396, 1, 1272, 30, 931, 1, 368, 0, 0, 0, 0, 0, 0, 0, 1151, 1308, 1586, 60, 33, 434, 6, 883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 314, 841, 3071, 6, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 2901, 0, 0, 0, 2562, 199, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2171, 2138, 0, 807, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 515, 3240, 0, 238, 5, 6, 64, 2987, 448, 0, 0, 0, 0, 0, 0, 33, 41, 2277, 0, 470, 92, 2, 56, 1949, 1563, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6187, 5461, 5742, 6464, 5125, 7513, 6483] avg=6144.0000 max=7513 max_rank=6 min=5125 min_rank=5 imbalance_ratio=1.222819 max_load_reduction=41639 max_load_reduction_pct=0.847148 moved_to_echo=25585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[111, 10, 102, 40, 14, 29, 0, 0, 4001, 1441, 0, 0, 0, 0, 0, 0, 574, 6, 1651, 0, 1017, 102, 1570, 0, 726, 0, 0, 0, 0, 0, 0, 0, 1305, 1724, 2184, 64, 59, 405, 9, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 14, 852, 2910, 329, 0, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 2109, 0, 0, 0, 3643, 176, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1263, 774, 0, 3131, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 2941, 0, 378, 3, 21, 77, 1736, 883, 0, 0, 0, 0, 0, 0, 139, 18, 1447, 0, 585, 123, 0, 29, 2504, 908, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5748, 5646, 6885, 5881, 7506, 5255, 6478, 5753] avg=6144.0000 max=7506 max_rank=4 min=5255 min_rank=5 imbalance_ratio=1.221680 max_load_reduction=41646 max_load_reduction_pct=0.847290 moved_to_echo=24992 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 365, 166, 17, 10, 0, 3, 3852, 765, 0, 0, 0, 0, 0, 0, 606, 70, 2439, 0, 2501, 23, 607, 3, 344, 0, 0, 0, 0, 0, 0, 0, 1274, 1073, 963, 68, 27, 989, 4, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 141, 1333, 2801, 6, 0, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 0, 0, 0, 2687, 216, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2344, 2288, 0, 756, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 12, 494, 2643, 0, 123, 9, 0, 37, 3224, 419, 0, 0, 0, 0, 0, 0, 15, 38, 1918, 0, 1295, 157, 0, 73, 1330, 1686, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5181, 6593, 5002, 5941, 7546, 5416, 6961, 6512] avg=6144.0000 max=7546 max_rank=4 min=5002 min_rank=2 imbalance_ratio=1.228190 max_load_reduction=41606 max_load_reduction_pct=0.846476 moved_to_echo=26435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[176, 37, 31, 56, 20, 162, 0, 2, 4509, 1787, 0, 0, 0, 0, 0, 0, 397, 27, 1227, 0, 121, 107, 2751, 0, 938, 0, 0, 0, 0, 0, 0, 0, 864, 1325, 2763, 87, 210, 21, 20, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 26, 153, 3271, 831, 4, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 641, 1, 0, 0, 2890, 37, 2022, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 670, 343, 0, 5362, 125, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 714, 2643, 0, 478, 56, 245, 128, 933, 1141, 0, 0, 0, 0, 0, 0, 124, 4, 2064, 0, 126, 0, 2, 41, 3105, 491, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6780, 5568, 6684, 5730, 5592, 6503, 6338, 5957] avg=6144.0000 max=6780 max_rank=0 min=5568 min_rank=1 imbalance_ratio=1.103516 max_load_reduction=42372 max_load_reduction_pct=0.862061 moved_to_echo=24390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[380, 8, 28, 42, 20, 108, 0, 1, 4359, 2021, 0, 0, 0, 0, 0, 0, 175, 3, 482, 0, 27, 191, 5262, 0, 1033, 0, 0, 0, 0, 0, 0, 0, 661, 922, 3758, 366, 88, 2, 2, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 37, 10, 3170, 2477, 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 150, 17, 1, 9, 2632, 5, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 90, 141, 0, 7267, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 1109, 0, 466, 1, 281, 194, 123, 1257, 0, 0, 0, 0, 0, 0, 886, 0, 430, 1, 5, 15, 0, 8, 3512, 64, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6967, 7173, 7390, 6428, 5058, 7748, 3467, 4921] avg=6144.0000 max=7748 max_rank=5 min=3467 min_rank=6 imbalance_ratio=1.261068 max_load_reduction=41404 max_load_reduction_pct=0.842367 moved_to_echo=21194 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 330, 239, 5, 46, 6, 0, 4259, 1266, 0, 0, 0, 0, 0, 0, 487, 51, 1956, 0, 1164, 20, 960, 7, 587, 0, 0, 0, 0, 0, 0, 0, 1344, 1669, 1308, 222, 78, 496, 5, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 257, 383, 3097, 15, 0, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 2394, 2, 0, 1, 2224, 209, 1276, 10, 0, 0, 0, 0, 0, 0, 0, 0, 9, 20, 2263, 2113, 0, 1036, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 29, 569, 2799, 0, 259, 89, 11, 21, 3113, 714, 0, 0, 0, 0, 0, 0, 20, 54, 2380, 0, 270, 26, 9, 88, 2200, 1628, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 5232, 6119, 5790, 6116, 5450, 7604, 6675] avg=6144.0000 max=7604 max_rank=6 min=5232 min_rank=1 imbalance_ratio=1.237630 max_load_reduction=41548 max_load_reduction_pct=0.845296 moved_to_echo=25845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[176, 37, 31, 56, 20, 162, 0, 2, 4509, 1787, 0, 0, 0, 0, 0, 0, 397, 27, 1227, 0, 121, 107, 2751, 0, 938, 0, 0, 0, 0, 0, 0, 0, 864, 1325, 2763, 87, 210, 21, 20, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 26, 153, 3271, 831, 4, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 641, 1, 0, 0, 2890, 37, 2022, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 670, 343, 0, 5362, 125, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 714, 2643, 0, 478, 56, 245, 128, 933, 1141, 0, 0, 0, 0, 0, 0, 124, 4, 2064, 0, 126, 0, 2, 41, 3105, 491, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6780, 5568, 6684, 5730, 5592, 6503, 6338, 5957] avg=6144.0000 max=6780 max_rank=0 min=5568 min_rank=1 imbalance_ratio=1.103516 max_load_reduction=42372 max_load_reduction_pct=0.862061 moved_to_echo=24390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 128, 1536, 896, 256, 640, 128, 128, 33152, 11776, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8576, 9984, 16640, 1024, 768, 3200, 128, 9216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2560, 1024, 4352, 24064, 8832, 128, 8576, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 3328, 19200, 0, 2560, 256, 1920, 896, 14976, 6528, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2944, 384, 13440, 128, 3712, 512, 128, 512, 20352, 7808, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2560, 1024, 4352, 24064, 8832, 128, 8576, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 256, 12160, 128, 7552, 1024, 18944, 128, 5376, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8576, 9984, 16640, 1024, 768, 3200, 128, 9216, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14464, 128, 128, 128, 22016, 1152, 11648, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 10880, 9472, 0, 27776, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[5, 0, 1408, 1, 8127, 0, 2, 0, 833, 0, 0, 0, 1, 0, 0, 0, 1, 233, 0, 1, 0, 0, 1183, 2, 5, 3, 1, 0, 1, 205, 33, 4326, 2, 3705, 1, 0, 7, 0, 0, 18, 43, 258, 94, 7836, 1, 1, 45, 0, 0, 0, 934, 0, 3, 6042, 0, 26, 5261, 0, 5, 0, 2, 8184, 0, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[748, 66, 288, 205, 873, 69, 0, 1446, 1965, 58, 96, 0, 288, 165, 7, 38, 646, 584, 622, 1173, 1248, 2, 187, 184, 771, 251, 100, 0, 1183, 104, 41, 5866, 1317, 1224, 214, 0, 1881, 27, 410, 4270, 652, 742, 79, 3044, 980, 142, 2031, 848, 0, 554, 10, 135, 1083, 299, 347, 38, 1764, 17, 398, 231, 76, 5682, 191, 1192] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[522, 17, 349, 148, 1853, 22, 9, 635, 2134, 32, 0, 0, 136, 109, 4, 9, 663, 464, 243, 454, 705, 2, 256, 109, 626, 99, 48, 0, 936, 117, 43, 6454, 1563, 2400, 54, 0, 1419, 0, 285, 3863, 567, 660, 22, 4914, 722, 34, 2058, 443, 0, 289, 141, 51, 860, 801, 34, 19, 2485, 0, 411, 324, 78, 6494, 182, 781] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[37, 2, 950, 40, 7144, 1, 22, 35, 1608, 1, 1, 5, 13, 29, 3, 2, 52, 286, 7, 71, 0, 0, 933, 29, 14, 41, 5, 0, 35, 177, 34, 6081, 127, 4126, 13, 0, 85, 1, 0, 349, 109, 453, 42, 7435, 83, 34, 507, 5, 4, 5, 453, 2, 67, 4308, 30, 20, 4601, 0, 38, 111, 25, 8024, 29, 408] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[5, 0, 1408, 1, 8127, 0, 2, 0, 833, 0, 0, 0, 1, 0, 0, 0, 1, 233, 0, 1, 0, 0, 1183, 2, 5, 3, 1, 0, 1, 205, 33, 4326, 2, 3705, 1, 0, 7, 0, 0, 18, 43, 258, 94, 7836, 1, 1, 45, 0, 0, 0, 934, 0, 3, 6042, 0, 26, 5261, 0, 5, 0, 2, 8184, 0, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[174, 10, 1378, 206, 4487, 8, 0, 304, 2260, 11, 0, 1, 88, 79, 1, 0, 133, 772, 125, 77, 111, 0, 541, 267, 57, 37, 13, 0, 205, 238, 37, 6244, 450, 2234, 129, 0, 473, 2, 23, 738, 512, 719, 22, 6536, 572, 59, 1835, 50, 0, 110, 386, 16, 949, 2793, 23, 75, 3306, 0, 210, 276, 85, 7611, 209, 885] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[806, 11, 303, 152, 524, 50, 4, 1340, 1371, 90, 0, 0, 241, 323, 2, 1, 571, 717, 988, 665, 988, 1, 68, 509, 694, 363, 63, 0, 1050, 3, 32, 6545, 1528, 1433, 92, 0, 2588, 0, 147, 4300, 610, 676, 10, 3159, 932, 253, 1967, 810, 0, 785, 7, 66, 2286, 180, 186, 26, 963, 1, 885, 240, 82, 5224, 224, 1017] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[522, 17, 349, 148, 1853, 22, 9, 635, 2134, 32, 0, 0, 136, 109, 4, 9, 663, 464, 243, 454, 705, 2, 256, 109, 626, 99, 48, 0, 936, 117, 43, 6454, 1563, 2400, 54, 0, 1419, 0, 285, 3863, 567, 660, 22, 4914, 722, 34, 2058, 443, 0, 289, 141, 51, 860, 801, 34, 19, 2485, 0, 411, 324, 78, 6494, 182, 781] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[550, 99, 615, 412, 752, 25, 0, 767, 1750, 88, 19, 0, 314, 134, 5, 3, 456, 1042, 412, 357, 207, 4, 145, 662, 1008, 793, 59, 0, 924, 25, 96, 6491, 799, 1371, 128, 0, 1436, 0, 71, 3246, 606, 964, 49, 3920, 1335, 192, 1886, 682, 0, 1071, 33, 182, 2602, 202, 348, 51, 1454, 14, 527, 288, 121, 6190, 282, 888] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[806, 11, 303, 152, 524, 50, 4, 1340, 1371, 90, 0, 0, 241, 323, 2, 1, 571, 717, 988, 665, 988, 1, 68, 509, 694, 363, 63, 0, 1050, 3, 32, 6545, 1528, 1433, 92, 0, 2588, 0, 147, 4300, 610, 676, 10, 3159, 932, 253, 1967, 810, 0, 785, 7, 66, 2286, 180, 186, 26, 963, 1, 885, 240, 82, 5224, 224, 1017] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[1526, 76, 62, 140, 353, 45, 4, 1202, 869, 50, 1, 0, 258, 201, 5, 44, 1409, 551, 715, 777, 1428, 1, 20, 50, 1255, 309, 134, 0, 1941, 56, 15, 6134, 2008, 1058, 84, 0, 2931, 9, 473, 4794, 320, 583, 19, 2717, 712, 337, 2219, 1184, 0, 1096, 0, 101, 1280, 59, 92, 13, 787, 2, 732, 286, 72, 4585, 197, 771] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[748, 66, 288, 205, 873, 69, 0, 1446, 1532, 0, 0, 0, 0, 0, 0, 0, 1965, 58, 96, 0, 288, 165, 7, 38, 3650, 0, 0, 0, 0, 0, 0, 0, 646, 584, 622, 1173, 1248, 2, 187, 184, 1606, 345, 0, 0, 0, 0, 0, 0, 771, 251, 100, 0, 1183, 104, 41, 4250, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 1224, 214, 0, 1881, 27, 410, 2143, 0, 0, 0, 0, 0, 0, 0, 0, 652, 742, 79, 1339, 980, 142, 2031, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 10, 135, 1083, 299, 347, 38, 1616, 595, 99, 0, 0, 0, 0, 0, 1764, 17, 398, 231, 76, 1687, 191, 1192, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5227, 6267, 6597, 6700, 7216, 6813, 4776, 5556] avg=6144.0000 max=7216 max_rank=4 min=4776 min_rank=6 imbalance_ratio=1.174479 max_load_reduction=41936 max_load_reduction_pct=0.853190 moved_to_echo=24361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 1408, 1, 8127, 0, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 1, 0, 0, 0, 5258, 0, 0, 0, 0, 0, 0, 0, 1, 233, 0, 1, 0, 0, 1183, 2, 4135, 497, 0, 0, 0, 0, 0, 0, 5, 3, 1, 0, 1, 205, 33, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3705, 1, 0, 7, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 43, 258, 94, 3444, 1, 1, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 0, 3, 6042, 0, 26, 1192, 2, 257, 0, 0, 0, 0, 0, 5261, 0, 5, 0, 2, 2429, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9549, 6092, 6052, 3382, 3725, 3886, 8456, 8010] avg=6144.0000 max=9549 max_rank=0 min=3382 min_rank=3 imbalance_ratio=1.554199 max_load_reduction=39603 max_load_reduction_pct=0.805725 moved_to_echo=24077 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[37, 2, 950, 40, 7144, 1, 22, 35, 125, 0, 0, 0, 0, 0, 0, 0, 1608, 1, 1, 5, 13, 29, 3, 2, 5155, 0, 0, 0, 0, 0, 0, 0, 52, 286, 7, 71, 0, 0, 933, 29, 3924, 488, 0, 0, 0, 0, 0, 0, 14, 41, 5, 0, 35, 177, 34, 4405, 0, 0, 0, 0, 0, 0, 0, 0, 127, 4126, 13, 0, 85, 1, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 109, 453, 42, 3267, 83, 34, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 453, 2, 67, 4308, 30, 20, 1676, 48, 244, 0, 0, 0, 0, 0, 4601, 0, 38, 111, 25, 2381, 29, 408, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8356, 6817, 5790, 4711, 4528, 4500, 6857, 7593] avg=6144.0000 max=8356 max_rank=0 min=4500 min_rank=5 imbalance_ratio=1.360026 max_load_reduction=40796 max_load_reduction_pct=0.829997 moved_to_echo=23478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[522, 17, 349, 148, 1853, 22, 9, 635, 1386, 0, 0, 0, 0, 0, 0, 0, 2134, 32, 0, 0, 136, 109, 4, 9, 4172, 0, 0, 0, 0, 0, 0, 0, 663, 464, 243, 454, 705, 2, 256, 109, 2593, 395, 0, 0, 0, 0, 0, 0, 626, 99, 48, 0, 936, 117, 43, 4676, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 2400, 54, 0, 1419, 0, 285, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 567, 660, 22, 2160, 722, 34, 2058, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 141, 51, 860, 801, 34, 19, 1778, 539, 161, 0, 0, 0, 0, 0, 2485, 0, 411, 324, 78, 1927, 182, 781, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4941, 6596, 5884, 6545, 7659, 6666, 4673, 6188] avg=6144.0000 max=7659 max_rank=4 min=4673 min_rank=6 imbalance_ratio=1.246582 max_load_reduction=41493 max_load_reduction_pct=0.844177 moved_to_echo=25186 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 1408, 1, 8127, 0, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 1, 0, 0, 0, 5258, 0, 0, 0, 0, 0, 0, 0, 1, 233, 0, 1, 0, 0, 1183, 2, 4135, 497, 0, 0, 0, 0, 0, 0, 5, 3, 1, 0, 1, 205, 33, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3705, 1, 0, 7, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 43, 258, 94, 3444, 1, 1, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 0, 3, 6042, 0, 26, 1192, 2, 257, 0, 0, 0, 0, 0, 5261, 0, 5, 0, 2, 2429, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9549, 6092, 6052, 3382, 3725, 3886, 8456, 8010] avg=6144.0000 max=9549 max_rank=0 min=3382 min_rank=3 imbalance_ratio=1.554199 max_load_reduction=39603 max_load_reduction_pct=0.805725 moved_to_echo=24077 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[806, 11, 303, 152, 524, 50, 4, 1340, 1543, 0, 0, 0, 0, 0, 0, 0, 1371, 90, 0, 0, 241, 323, 2, 1, 3356, 0, 0, 0, 0, 0, 0, 0, 571, 717, 988, 665, 988, 1, 68, 509, 1667, 317, 0, 0, 0, 0, 0, 0, 694, 363, 63, 0, 1050, 3, 32, 4742, 0, 0, 0, 0, 0, 0, 0, 0, 1528, 1433, 92, 0, 2588, 0, 147, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 610, 676, 10, 1389, 932, 253, 1967, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 785, 7, 66, 2286, 180, 186, 26, 1803, 600, 103, 0, 0, 0, 0, 0, 963, 1, 885, 240, 82, 1551, 224, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4733, 5384, 6491, 6947, 7945, 6647, 6042, 4963] avg=6144.0000 max=7945 max_rank=4 min=4733 min_rank=0 imbalance_ratio=1.293132 max_load_reduction=41207 max_load_reduction_pct=0.838359 moved_to_echo=25597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[174, 10, 1378, 206, 4487, 8, 0, 304, 267, 0, 0, 0, 0, 0, 0, 0, 2260, 11, 0, 1, 88, 79, 1, 0, 4893, 0, 0, 0, 0, 0, 0, 0, 133, 772, 125, 77, 111, 0, 541, 267, 3453, 467, 0, 0, 0, 0, 0, 0, 57, 37, 13, 0, 205, 238, 37, 4519, 0, 0, 0, 0, 0, 0, 0, 0, 450, 2234, 129, 0, 473, 2, 23, 364, 0, 0, 0, 0, 0, 0, 0, 0, 512, 719, 22, 2865, 572, 59, 1835, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 386, 16, 949, 2793, 23, 75, 1725, 107, 218, 0, 0, 0, 0, 0, 3306, 0, 210, 276, 85, 2251, 209, 885, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6834, 7333, 5946, 5106, 3675, 6634, 6402, 7222] avg=6144.0000 max=7333 max_rank=1 min=3675 min_rank=4 imbalance_ratio=1.193522 max_load_reduction=41819 max_load_reduction_pct=0.850810 moved_to_echo=23933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[550, 99, 615, 412, 752, 25, 0, 767, 1164, 0, 0, 0, 0, 0, 0, 0, 1750, 88, 19, 0, 314, 134, 5, 3, 3977, 0, 0, 0, 0, 0, 0, 0, 456, 1042, 412, 357, 207, 4, 145, 662, 2068, 376, 0, 0, 0, 0, 0, 0, 1008, 793, 59, 0, 924, 25, 96, 4702, 0, 0, 0, 0, 0, 0, 0, 0, 799, 1371, 128, 0, 1436, 0, 71, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 606, 964, 49, 1724, 1335, 192, 1886, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 33, 182, 2602, 202, 348, 51, 1789, 452, 128, 0, 0, 0, 0, 0, 1454, 14, 527, 288, 121, 1837, 282, 888, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4384, 6290, 5729, 7607, 5435, 7438, 6858, 5411] avg=6144.0000 max=7607 max_rank=3 min=4384 min_rank=0 imbalance_ratio=1.238118 max_load_reduction=41545 max_load_reduction_pct=0.845235 moved_to_echo=25142 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[174, 10, 1378, 206, 4487, 8, 0, 304, 267, 0, 0, 0, 0, 0, 0, 0, 2260, 11, 0, 1, 88, 79, 1, 0, 4893, 0, 0, 0, 0, 0, 0, 0, 133, 772, 125, 77, 111, 0, 541, 267, 3453, 467, 0, 0, 0, 0, 0, 0, 57, 37, 13, 0, 205, 238, 37, 4519, 0, 0, 0, 0, 0, 0, 0, 0, 450, 2234, 129, 0, 473, 2, 23, 364, 0, 0, 0, 0, 0, 0, 0, 0, 512, 719, 22, 2865, 572, 59, 1835, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 386, 16, 949, 2793, 23, 75, 1725, 107, 218, 0, 0, 0, 0, 0, 3306, 0, 210, 276, 85, 2251, 209, 885, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6834, 7333, 5946, 5106, 3675, 6634, 6402, 7222] avg=6144.0000 max=7333 max_rank=1 min=3675 min_rank=4 imbalance_ratio=1.193522 max_load_reduction=41819 max_load_reduction_pct=0.850810 moved_to_echo=23933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1526, 76, 62, 140, 353, 45, 4, 1202, 1720, 0, 0, 0, 0, 0, 0, 0, 869, 50, 1, 0, 258, 201, 5, 44, 2945, 0, 0, 0, 0, 0, 0, 0, 1409, 551, 715, 777, 1428, 1, 20, 50, 1434, 278, 0, 0, 0, 0, 0, 0, 1255, 309, 134, 0, 1941, 56, 15, 4444, 0, 0, 0, 0, 0, 0, 0, 0, 2008, 1058, 84, 0, 2931, 9, 473, 2406, 0, 0, 0, 0, 0, 0, 0, 0, 320, 583, 19, 1194, 712, 337, 2219, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 0, 101, 1280, 59, 92, 13, 1690, 668, 89, 0, 0, 0, 0, 0, 787, 2, 732, 286, 72, 1362, 197, 771, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5128, 4373, 6663, 8154, 8969, 6568, 5088, 4209] avg=6144.0000 max=8969 max_rank=4 min=4209 min_rank=7 imbalance_ratio=1.459798 max_load_reduction=40183 max_load_reduction_pct=0.817525 moved_to_echo=24834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[20736, 128, 3328, 1792, 640, 15488, 1408, 6272, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7808, 17664, 768, 0, 10880, 128, 1536, 10880, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3456, 5120, 384, 17408, 5376, 1152, 12672, 4096, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 3968, 2048, 640, 9216, 14720, 1152, 384, 13312, 3072, 1408, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 4736, 3200, 3584, 4736, 128, 3456, 1920, 20992, 3200, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12800, 384, 128, 128, 1408, 1152, 128, 128, 33408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 3968, 2048, 640, 9216, 14720, 1152, 384, 13312, 3072, 1408, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4480, 384, 5376, 1408, 24192, 256, 128, 5760, 7808, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 4736, 3200, 3584, 4736, 128, 3456, 1920, 20992, 3200, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 1920, 512, 0, 6400, 1024, 384, 34944, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[22, 0, 5860, 33, 63, 7, 0, 0, 0, 733, 3, 3, 3780, 0, 16, 7, 1634, 2, 143, 0, 0, 0, 7073, 341, 0, 1, 0, 5, 0, 42, 370, 0, 5, 408, 58, 4831, 225, 0, 0, 86, 7853, 11, 101, 4072, 132, 10, 133, 841, 0, 4400, 80, 1, 8, 338, 0, 13, 1, 8, 3, 70, 5278, 32, 14, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[73, 0, 67, 1314, 681, 0, 0, 0, 82, 3954, 105, 474, 1521, 3, 64, 18, 3531, 0, 3464, 0, 0, 41, 2194, 892, 1398, 0, 0, 226, 0, 994, 179, 0, 559, 75, 2955, 1439, 82, 0, 1, 1169, 5792, 588, 172, 358, 930, 1819, 4742, 2211, 2, 1236, 373, 1, 2, 334, 0, 4, 0, 9, 64, 644, 329, 1538, 203, 246] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[103, 0, 781, 516, 593, 0, 1, 0, 26, 3837, 56, 431, 2537, 7, 45, 0, 2717, 0, 1905, 0, 4, 2, 4531, 1064, 1189, 0, 0, 213, 0, 577, 354, 0, 358, 167, 1575, 3461, 146, 0, 4, 521, 6296, 223, 184, 1489, 816, 660, 2525, 3177, 3, 2096, 215, 1, 0, 455, 0, 0, 5, 9, 112, 500, 1037, 1393, 96, 139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 7360, 4, 13, 0, 0, 1, 0, 72, 0, 0, 2084, 0, 0, 1, 1042, 0, 4, 0, 0, 0, 7500, 94, 0, 0, 0, 0, 0, 7, 98, 0, 1, 654, 8, 6876, 118, 0, 0, 3, 7762, 0, 13, 3373, 33, 3, 9, 52, 0, 5038, 3, 0, 0, 437, 0, 0, 0, 0, 1, 1, 6478, 5, 1, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[96, 0, 33, 954, 895, 0, 0, 0, 63, 2759, 75, 1371, 1019, 2, 228, 1, 2176, 0, 4450, 0, 22, 29, 1486, 1103, 2634, 0, 0, 597, 0, 543, 103, 0, 574, 20, 3669, 815, 38, 0, 2, 1253, 4487, 483, 141, 125, 1001, 1390, 5115, 2572, 8, 840, 386, 8, 0, 158, 0, 1, 1, 1, 368, 773, 133, 3456, 245, 450] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[73, 0, 67, 1314, 681, 0, 0, 0, 82, 3954, 105, 474, 1521, 3, 64, 18, 3531, 0, 3464, 0, 0, 41, 2194, 892, 1398, 0, 0, 226, 0, 994, 179, 0, 559, 75, 2955, 1439, 82, 0, 1, 1169, 5792, 588, 172, 358, 930, 1819, 4742, 2211, 2, 1236, 373, 1, 2, 334, 0, 4, 0, 9, 64, 644, 329, 1538, 203, 246] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[64, 0, 234, 521, 1034, 0, 8, 0, 49, 3030, 110, 551, 2061, 37, 272, 3, 3094, 0, 2871, 0, 41, 15, 2219, 1316, 1937, 0, 0, 568, 0, 229, 201, 0, 544, 106, 2349, 2154, 165, 0, 2, 568, 5492, 283, 172, 466, 1087, 1484, 4031, 3133, 10, 1234, 475, 14, 0, 307, 1, 4, 0, 9, 260, 599, 555, 2615, 261, 307] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[19, 0, 20, 1954, 1540, 0, 0, 0, 78, 3374, 89, 339, 1954, 32, 36, 61, 4153, 0, 2775, 0, 0, 1, 1233, 1050, 950, 0, 0, 131, 0, 614, 88, 0, 350, 12, 2193, 2517, 113, 0, 23, 1045, 6334, 277, 160, 290, 1116, 2355, 4764, 1237, 8, 1242, 558, 6, 2, 596, 0, 11, 14, 16, 45, 501, 557, 1632, 468, 219] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[22, 0, 5860, 33, 63, 7, 0, 0, 0, 733, 3, 3, 3780, 0, 16, 7, 1634, 2, 143, 0, 0, 0, 7073, 341, 0, 1, 0, 5, 0, 42, 370, 0, 5, 408, 58, 4831, 225, 0, 0, 86, 7853, 11, 101, 4072, 132, 10, 133, 841, 0, 4400, 80, 1, 8, 338, 0, 13, 1, 8, 3, 70, 5278, 32, 14, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 7360, 4, 13, 0, 0, 1, 0, 72, 0, 0, 2084, 0, 0, 1, 1042, 0, 4, 0, 0, 0, 7500, 94, 0, 0, 0, 0, 0, 7, 98, 0, 1, 654, 8, 6876, 118, 0, 0, 3, 7762, 0, 13, 3373, 33, 3, 9, 52, 0, 5038, 3, 0, 0, 437, 0, 0, 0, 0, 1, 1, 6478, 5, 1, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[41, 0, 1566, 156, 295, 0, 0, 0, 39, 3331, 19, 126, 3114, 4, 12, 6, 4190, 0, 777, 0, 0, 5, 4696, 724, 202, 1, 0, 41, 0, 92, 477, 0, 127, 238, 538, 5882, 334, 0, 0, 206, 7427, 115, 73, 2431, 685, 341, 764, 984, 3, 3469, 152, 2, 7, 781, 0, 4, 0, 8, 14, 396, 3850, 305, 77, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 5860, 33, 63, 7, 0, 0, 4797, 7, 0, 0, 0, 0, 0, 0, 0, 733, 3, 3, 3780, 0, 16, 7, 15, 426, 0, 0, 0, 0, 0, 0, 1634, 2, 143, 0, 0, 0, 796, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 42, 370, 0, 5204, 0, 0, 0, 0, 0, 0, 0, 5, 408, 58, 4405, 225, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 101, 4072, 132, 10, 35, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4400, 80, 1, 8, 338, 0, 13, 2649, 1480, 0, 0, 0, 0, 0, 0, 1, 8, 3, 70, 5278, 32, 14, 2, 76, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10789, 4983, 2916, 5622, 5187, 5202, 8969, 5484] avg=6144.0000 max=10789 max_rank=0 min=2916 min_rank=2 imbalance_ratio=1.756022 max_load_reduction=38363 max_load_reduction_pct=0.780497 moved_to_echo=24842 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[19, 0, 20, 1954, 1540, 0, 0, 0, 78, 3374, 89, 339, 1954, 32, 36, 61, 4153, 0, 2775, 0, 0, 1, 1233, 1050, 950, 0, 0, 131, 0, 614, 88, 0, 350, 12, 2193, 2517, 113, 0, 23, 1045, 6334, 277, 160, 290, 1116, 2355, 4764, 1237, 8, 1242, 558, 6, 2, 596, 0, 11, 14, 16, 45, 501, 557, 1632, 468, 219] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[103, 0, 781, 516, 593, 0, 1, 0, 26, 3837, 56, 431, 2537, 7, 45, 0, 2717, 0, 1905, 0, 4, 2, 4531, 1064, 1189, 0, 0, 213, 0, 577, 354, 0, 358, 167, 1575, 3461, 146, 0, 4, 521, 6296, 223, 184, 1489, 816, 660, 2525, 3177, 3, 2096, 215, 1, 0, 455, 0, 0, 5, 9, 112, 500, 1037, 1393, 96, 139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[96, 0, 33, 954, 895, 0, 0, 0, 1007, 289, 0, 0, 0, 0, 0, 0, 63, 2759, 75, 1371, 1019, 2, 228, 1, 597, 71, 0, 0, 0, 0, 0, 0, 2176, 0, 4450, 0, 22, 29, 169, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 0, 0, 597, 0, 543, 103, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 574, 20, 3669, 744, 38, 0, 2, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 141, 125, 1001, 1390, 1271, 2572, 0, 0, 0, 0, 0, 0, 0, 0, 8, 840, 386, 8, 0, 158, 0, 1, 1514, 310, 0, 0, 0, 0, 0, 0, 1, 1, 368, 773, 133, 3456, 245, 450, 2958, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3274, 6186, 7949, 6850, 6300, 6983, 3225, 8385] avg=6144.0000 max=8385 max_rank=7 min=3225 min_rank=6 imbalance_ratio=1.364746 max_load_reduction=40767 max_load_reduction_pct=0.829407 moved_to_echo=24893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[73, 0, 67, 1314, 681, 0, 0, 0, 1488, 268, 0, 0, 0, 0, 0, 0, 82, 3954, 105, 474, 1521, 3, 64, 18, 553, 127, 0, 0, 0, 0, 0, 0, 3531, 0, 3464, 0, 0, 41, 247, 892, 0, 0, 0, 0, 0, 0, 0, 0, 1398, 0, 0, 226, 0, 994, 179, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 559, 75, 2955, 1312, 82, 0, 1, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 172, 358, 930, 1819, 1179, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1236, 373, 1, 2, 334, 0, 4, 1954, 459, 0, 0, 0, 0, 0, 0, 0, 9, 64, 644, 329, 1538, 203, 246, 2742, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3891, 6901, 8175, 6635, 6153, 7257, 4365, 5775] avg=6144.0000 max=8175 max_rank=2 min=3891 min_rank=0 imbalance_ratio=1.330566 max_load_reduction=40977 max_load_reduction_pct=0.833679 moved_to_echo=23550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7360, 4, 13, 0, 0, 1, 5086, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 2084, 0, 0, 1, 1, 607, 0, 0, 0, 0, 0, 0, 1042, 0, 4, 0, 0, 0, 845, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 98, 0, 5144, 0, 0, 0, 0, 0, 0, 0, 1, 654, 8, 6269, 118, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3373, 33, 3, 3, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5038, 3, 0, 0, 437, 0, 0, 2618, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 6478, 5, 1, 3, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12464, 2765, 1985, 5249, 7053, 3477, 9665, 6494] avg=6144.0000 max=12464 max_rank=0 min=1985 min_rank=2 imbalance_ratio=2.028646 max_load_reduction=36688 max_load_reduction_pct=0.746419 moved_to_echo=26689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[103, 0, 781, 516, 593, 0, 1, 0, 3073, 143, 0, 0, 0, 0, 0, 0, 26, 3837, 56, 431, 2537, 7, 45, 0, 294, 305, 0, 0, 0, 0, 0, 0, 2717, 0, 1905, 0, 4, 2, 510, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 0, 213, 0, 577, 354, 0, 4173, 0, 0, 0, 0, 0, 0, 0, 358, 167, 1575, 3156, 146, 0, 4, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 184, 1489, 816, 660, 628, 3177, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2096, 215, 1, 0, 455, 0, 0, 2123, 948, 0, 0, 0, 0, 0, 0, 5, 9, 112, 500, 1037, 1393, 96, 139, 1460, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5210, 7538, 6202, 6506, 5927, 7177, 5841, 4751] avg=6144.0000 max=7538 max_rank=1 min=4751 min_rank=7 imbalance_ratio=1.226888 max_load_reduction=41614 max_load_reduction_pct=0.846639 moved_to_echo=23696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 5860, 33, 63, 7, 0, 0, 4797, 7, 0, 0, 0, 0, 0, 0, 0, 733, 3, 3, 3780, 0, 16, 7, 15, 426, 0, 0, 0, 0, 0, 0, 1634, 2, 143, 0, 0, 0, 796, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 42, 370, 0, 5204, 0, 0, 0, 0, 0, 0, 0, 5, 408, 58, 4405, 225, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 101, 4072, 132, 10, 35, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4400, 80, 1, 8, 338, 0, 13, 2649, 1480, 0, 0, 0, 0, 0, 0, 1, 8, 3, 70, 5278, 32, 14, 2, 76, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10789, 4983, 2916, 5622, 5187, 5202, 8969, 5484] avg=6144.0000 max=10789 max_rank=0 min=2916 min_rank=2 imbalance_ratio=1.756022 max_load_reduction=38363 max_load_reduction_pct=0.780497 moved_to_echo=24842 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[64, 0, 234, 521, 1034, 0, 8, 0, 1504, 228, 0, 0, 0, 0, 0, 0, 49, 3030, 110, 551, 2061, 37, 272, 3, 470, 190, 0, 0, 0, 0, 0, 0, 3094, 0, 2871, 0, 41, 15, 251, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 1937, 0, 0, 568, 0, 229, 201, 0, 3639, 0, 0, 0, 0, 0, 0, 0, 544, 106, 2349, 1964, 165, 0, 2, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 172, 466, 1087, 1484, 1002, 3133, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1234, 475, 14, 0, 307, 1, 4, 1853, 464, 0, 0, 0, 0, 0, 0, 0, 9, 260, 599, 555, 2615, 261, 307, 2331, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3593, 6773, 7588, 6574, 5698, 7627, 4362, 6937] avg=6144.0000 max=7627 max_rank=5 min=3593 min_rank=0 imbalance_ratio=1.241374 max_load_reduction=41525 max_load_reduction_pct=0.844828 moved_to_echo=24624 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[96, 0, 33, 954, 895, 0, 0, 0, 1007, 289, 0, 0, 0, 0, 0, 0, 63, 2759, 75, 1371, 1019, 2, 228, 1, 597, 71, 0, 0, 0, 0, 0, 0, 2176, 0, 4450, 0, 22, 29, 169, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 0, 0, 597, 0, 543, 103, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 574, 20, 3669, 744, 38, 0, 2, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 141, 125, 1001, 1390, 1271, 2572, 0, 0, 0, 0, 0, 0, 0, 0, 8, 840, 386, 8, 0, 158, 0, 1, 1514, 310, 0, 0, 0, 0, 0, 0, 1, 1, 368, 773, 133, 3456, 245, 450, 2958, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3274, 6186, 7949, 6850, 6300, 6983, 3225, 8385] avg=6144.0000 max=8385 max_rank=7 min=3225 min_rank=6 imbalance_ratio=1.364746 max_load_reduction=40767 max_load_reduction_pct=0.829407 moved_to_echo=24893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 20, 1954, 1540, 0, 0, 0, 836, 269, 0, 0, 0, 0, 0, 0, 78, 3374, 89, 339, 1954, 32, 36, 61, 556, 222, 0, 0, 0, 0, 0, 0, 4153, 0, 2775, 0, 0, 1, 139, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 950, 0, 0, 131, 0, 614, 88, 0, 4198, 0, 0, 0, 0, 0, 0, 0, 350, 12, 2193, 2295, 113, 0, 23, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 160, 290, 1116, 2355, 1184, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1242, 558, 6, 2, 596, 0, 11, 2136, 258, 0, 0, 0, 0, 0, 0, 14, 16, 45, 501, 557, 1632, 468, 219, 2755, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4638, 6741, 8118, 5981, 6031, 6619, 4817, 6207] avg=6144.0000 max=8118 max_rank=2 min=4638 min_rank=0 imbalance_ratio=1.321289 max_load_reduction=41034 max_load_reduction_pct=0.834839 moved_to_echo=23674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7360, 4, 13, 0, 0, 1, 5086, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 2084, 0, 0, 1, 1, 607, 0, 0, 0, 0, 0, 0, 1042, 0, 4, 0, 0, 0, 845, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 98, 0, 5144, 0, 0, 0, 0, 0, 0, 0, 1, 654, 8, 6269, 118, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3373, 33, 3, 3, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5038, 3, 0, 0, 437, 0, 0, 2618, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 6478, 5, 1, 3, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12464, 2765, 1985, 5249, 7053, 3477, 9665, 6494] avg=6144.0000 max=12464 max_rank=0 min=1985 min_rank=2 imbalance_ratio=2.028646 max_load_reduction=36688 max_load_reduction_pct=0.746419 moved_to_echo=26689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[103, 0, 781, 516, 593, 0, 1, 0, 3073, 143, 0, 0, 0, 0, 0, 0, 26, 3837, 56, 431, 2537, 7, 45, 0, 294, 305, 0, 0, 0, 0, 0, 0, 2717, 0, 1905, 0, 4, 2, 510, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 0, 213, 0, 577, 354, 0, 4173, 0, 0, 0, 0, 0, 0, 0, 358, 167, 1575, 3156, 146, 0, 4, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 184, 1489, 816, 660, 628, 3177, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2096, 215, 1, 0, 455, 0, 0, 2123, 948, 0, 0, 0, 0, 0, 0, 5, 9, 112, 500, 1037, 1393, 96, 139, 1460, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5210, 7538, 6202, 6506, 5927, 7177, 5841, 4751] avg=6144.0000 max=7538 max_rank=1 min=4751 min_rank=7 imbalance_ratio=1.226888 max_load_reduction=41614 max_load_reduction_pct=0.846639 moved_to_echo=23696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 1566, 156, 295, 0, 0, 0, 3188, 47, 0, 0, 0, 0, 0, 0, 39, 3331, 19, 126, 3114, 4, 12, 6, 92, 522, 0, 0, 0, 0, 0, 0, 4190, 0, 777, 0, 0, 5, 523, 724, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1, 0, 41, 0, 92, 477, 0, 4922, 0, 0, 0, 0, 0, 0, 0, 127, 238, 538, 5360, 334, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 73, 2431, 685, 341, 181, 984, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3469, 152, 2, 7, 781, 0, 4, 2505, 985, 0, 0, 0, 0, 0, 0, 0, 8, 14, 396, 3850, 305, 77, 25, 444, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5293, 7265, 6219, 5735, 6803, 4810, 7908, 5119] avg=6144.0000 max=7908 max_rank=6 min=4810 min_rank=5 imbalance_ratio=1.287109 max_load_reduction=41244 max_load_reduction_pct=0.839111 moved_to_echo=24640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 19584, 2304, 128, 128, 3456, 128, 128, 17408, 6528, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 0, 16000, 5504, 5120, 128, 128, 128, 20992, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 896, 3584, 18304, 11008, 1408, 1408, 12800, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 2048, 1024, 12672, 5888, 8064, 5504, 14208, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 0, 16000, 5504, 5120, 128, 128, 128, 20992, 1280, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 21120, 512, 3328, 18176, 128, 768, 128, 2688, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[22656, 128, 16512, 0, 128, 128, 3584, 6656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 896, 3584, 18304, 11008, 1408, 1408, 12800, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 1792, 13440, 25600, 1280, 0, 128, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[22656, 128, 16512, 0, 128, 128, 3584, 6656, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8320, 128, 0, 1792, 0, 3200, 1920, 0, 34176, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[943, 90, 84, 0, 318, 219, 4241, 0, 47, 909, 1533, 2135, 4, 4815, 911, 2108, 3418, 12, 603, 2869, 133, 0, 1589, 491, 0, 155, 3540, 542, 1399, 115, 1015, 0, 0, 146, 66, 1465, 318, 1020, 38, 2771, 298, 125, 35, 79, 589, 0, 10, 1, 109, 4537, 0, 506, 0, 367, 44, 63, 277, 0, 0, 71, 11, 184, 808, 976] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[1015, 168, 134, 0, 344, 264, 3281, 0, 58, 2184, 1381, 2200, 2, 4167, 230, 2890, 1846, 20, 660, 2451, 33, 4, 997, 617, 0, 232, 3793, 274, 1810, 268, 564, 2, 0, 103, 16, 1878, 153, 1530, 1, 3836, 25, 84, 26, 142, 295, 0, 20, 8, 1, 5889, 0, 799, 2, 106, 35, 148, 350, 0, 0, 10, 0, 170, 1418, 218] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[239, 14, 8, 0, 2, 11, 7097, 0, 9, 97, 62, 41, 3, 2220, 5553, 87, 7868, 31, 11, 249, 189, 0, 2506, 4, 3, 3, 334, 4, 5, 10, 6876, 0, 13, 391, 0, 90, 48, 20, 0, 113, 397, 4, 0, 26, 354, 0, 15, 21, 81, 663, 7, 76, 1, 6898, 4, 10, 4, 0, 2, 403, 0, 372, 21, 5582] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[12, 0, 1, 0, 0, 2, 7335, 1, 0, 4, 0, 9, 108, 1265, 5160, 17, 8178, 0, 0, 8, 250, 0, 1757, 2, 0, 0, 72, 0, 3, 1, 7401, 0, 40, 315, 1, 10, 2, 2, 0, 13, 139, 0, 0, 0, 396, 0, 0, 0, 146, 235, 0, 2, 1, 8095, 0, 2, 1, 0, 0, 802, 0, 465, 2, 6897] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[1156, 48, 29, 0, 183, 180, 5331, 0, 22, 745, 1026, 1032, 3, 4586, 1598, 2021, 4678, 0, 524, 1924, 147, 0, 2452, 317, 0, 199, 3092, 141, 633, 83, 2379, 0, 0, 353, 23, 1047, 106, 871, 11, 1468, 265, 108, 2, 51, 833, 0, 1, 1, 74, 4760, 0, 277, 0, 1286, 20, 124, 175, 0, 0, 204, 5, 252, 587, 1719] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[762, 20, 21, 0, 70, 107, 6890, 0, 0, 287, 307, 229, 5, 4602, 2277, 1114, 7278, 39, 37, 857, 215, 0, 2499, 55, 0, 7, 1012, 110, 191, 28, 3548, 0, 0, 403, 0, 479, 243, 104, 6, 663, 453, 9, 0, 52, 1574, 0, 13, 0, 914, 2997, 0, 276, 0, 4248, 4, 23, 35, 1, 0, 430, 1, 241, 130, 3286] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[954, 92, 92, 0, 112, 246, 4964, 0, 15, 779, 1707, 1777, 9, 4675, 578, 3593, 3982, 261, 730, 1167, 163, 11, 1103, 121, 0, 83, 2400, 634, 1208, 189, 1043, 0, 0, 217, 0, 1391, 811, 423, 10, 1562, 336, 90, 15, 140, 952, 0, 20, 126, 268, 5911, 0, 1112, 0, 522, 97, 374, 229, 0, 0, 110, 0, 127, 1018, 603] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[943, 90, 84, 0, 318, 219, 4241, 0, 47, 909, 1533, 2135, 4, 4815, 911, 2108, 3418, 12, 603, 2869, 133, 0, 1589, 491, 0, 155, 3540, 542, 1399, 115, 1015, 0, 0, 146, 66, 1465, 318, 1020, 38, 2771, 298, 125, 35, 79, 589, 0, 10, 1, 109, 4537, 0, 506, 0, 367, 44, 63, 277, 0, 0, 71, 11, 184, 808, 976] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[762, 20, 21, 0, 70, 107, 6890, 0, 0, 287, 307, 229, 5, 4602, 2277, 1114, 7278, 39, 37, 857, 215, 0, 2499, 55, 0, 7, 1012, 110, 191, 28, 3548, 0, 0, 403, 0, 479, 243, 104, 6, 663, 453, 9, 0, 52, 1574, 0, 13, 0, 914, 2997, 0, 276, 0, 4248, 4, 23, 35, 1, 0, 430, 1, 241, 130, 3286] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[1156, 48, 29, 0, 183, 180, 5331, 0, 22, 745, 1026, 1032, 3, 4586, 1598, 2021, 4678, 0, 524, 1924, 147, 0, 2452, 317, 0, 199, 3092, 141, 633, 83, 2379, 0, 0, 353, 23, 1047, 106, 871, 11, 1468, 265, 108, 2, 51, 833, 0, 1, 1, 74, 4760, 0, 277, 0, 1286, 20, 124, 175, 0, 0, 204, 5, 252, 587, 1719] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[966, 137, 119, 0, 242, 326, 4459, 0, 12, 1074, 1575, 1328, 0, 3962, 492, 2879, 3336, 22, 990, 2514, 121, 10, 1586, 376, 0, 58, 2678, 430, 1008, 166, 814, 0, 0, 125, 7, 2191, 472, 695, 6, 2807, 149, 226, 8, 155, 533, 0, 85, 7, 105, 6401, 0, 758, 0, 339, 56, 130, 168, 0, 0, 28, 0, 249, 1375, 397] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[943, 90, 84, 0, 318, 219, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 909, 1533, 2135, 4, 0, 636, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 12, 603, 2869, 133, 0, 1589, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 3540, 542, 1399, 115, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 66, 1465, 318, 1020, 38, 2771, 1099, 275, 406, 60, 0, 0, 0, 0, 298, 125, 35, 79, 589, 0, 10, 1, 4815, 805, 0, 0, 0, 0, 0, 0, 109, 2984, 0, 506, 0, 367, 44, 63, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 71, 11, 184, 808, 976, 1313, 454, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5489, 7372, 6997, 6706, 7664, 6757, 4073, 4094] avg=6144.0000 max=7664 max_rank=4 min=4073 min_rank=6 imbalance_ratio=1.247396 max_load_reduction=41488 max_load_reduction_pct=0.844076 moved_to_echo=22588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1015, 168, 134, 0, 344, 264, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 2184, 1381, 2200, 2, 0, 161, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 702, 20, 660, 2451, 33, 4, 997, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 3793, 274, 1810, 268, 531, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 16, 1878, 153, 1530, 1, 3836, 1426, 69, 314, 33, 0, 0, 0, 0, 25, 84, 26, 142, 295, 0, 20, 8, 4167, 435, 0, 0, 0, 0, 0, 0, 1, 3873, 0, 799, 2, 106, 35, 148, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 10, 0, 170, 1418, 218, 709, 590, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4892, 8876, 5484, 6910, 9359, 5202, 4964, 3465] avg=6144.0000 max=9359 max_rank=4 min=3465 min_rank=7 imbalance_ratio=1.523275 max_load_reduction=39793 max_load_reduction_pct=0.809591 moved_to_echo=22990 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[239, 14, 8, 0, 2, 11, 6418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 97, 62, 41, 3, 0, 3876, 87, 0, 0, 0, 0, 0, 0, 0, 0, 2992, 31, 11, 249, 189, 0, 2506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 334, 4, 5, 10, 6468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 391, 0, 90, 48, 20, 0, 113, 160, 1677, 679, 408, 0, 0, 0, 0, 397, 4, 0, 26, 354, 0, 15, 21, 2220, 1854, 0, 0, 0, 0, 0, 0, 81, 437, 7, 76, 1, 6898, 4, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 403, 0, 372, 21, 5582, 3022, 66, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6692, 4175, 5982, 6827, 3599, 4891, 7514, 9472] avg=6144.0000 max=9472 max_rank=7 min=3599 min_rank=4 imbalance_ratio=1.541667 max_load_reduction=39680 max_load_reduction_pct=0.807292 moved_to_echo=25476 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 1, 0, 0, 2, 6633, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 9, 108, 0, 3602, 17, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 0, 0, 8, 250, 0, 1757, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 3, 1, 6962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 315, 1, 10, 2, 2, 0, 13, 56, 1558, 702, 439, 0, 0, 0, 0, 139, 0, 0, 0, 396, 0, 0, 0, 1265, 1927, 0, 0, 0, 0, 0, 0, 146, 156, 0, 2, 1, 8095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 802, 0, 465, 2, 6897, 3141, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6649, 3740, 5127, 7038, 3138, 3727, 8402, 11331] avg=6144.0000 max=11331 max_rank=7 min=3138 min_rank=4 imbalance_ratio=1.844238 max_load_reduction=37821 max_load_reduction_pct=0.769470 moved_to_echo=26598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1015, 168, 134, 0, 344, 264, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 2184, 1381, 2200, 2, 0, 161, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 702, 20, 660, 2451, 33, 4, 997, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 3793, 274, 1810, 268, 531, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 16, 1878, 153, 1530, 1, 3836, 1426, 69, 314, 33, 0, 0, 0, 0, 25, 84, 26, 142, 295, 0, 20, 8, 4167, 435, 0, 0, 0, 0, 0, 0, 1, 3873, 0, 799, 2, 106, 35, 148, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 10, 0, 170, 1418, 218, 709, 590, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4892, 8876, 5484, 6910, 9359, 5202, 4964, 3465] avg=6144.0000 max=9359 max_rank=4 min=3465 min_rank=7 imbalance_ratio=1.523275 max_load_reduction=39793 max_load_reduction_pct=0.809591 moved_to_echo=22990 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1156, 48, 29, 0, 183, 180, 4821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 745, 1026, 1032, 3, 0, 1116, 2021, 0, 0, 0, 0, 0, 0, 0, 0, 1779, 0, 524, 1924, 147, 0, 2452, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 3092, 141, 633, 83, 2238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 23, 1047, 106, 871, 11, 1468, 1153, 482, 510, 141, 0, 0, 0, 0, 265, 108, 2, 51, 833, 0, 1, 1, 4586, 1102, 0, 0, 0, 0, 0, 0, 74, 3130, 0, 277, 0, 1286, 20, 124, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 204, 5, 252, 587, 1719, 1797, 477, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 5965, 7143, 6386, 6165, 6949, 4911, 5216] avg=6144.0000 max=7143 max_rank=2 min=4911 min_rank=6 imbalance_ratio=1.162598 max_load_reduction=42009 max_load_reduction_pct=0.854675 moved_to_echo=23241 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[943, 90, 84, 0, 318, 219, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 909, 1533, 2135, 4, 0, 636, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 12, 603, 2869, 133, 0, 1589, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 3540, 542, 1399, 115, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 66, 1465, 318, 1020, 38, 2771, 1099, 275, 406, 60, 0, 0, 0, 0, 298, 125, 35, 79, 589, 0, 10, 1, 4815, 805, 0, 0, 0, 0, 0, 0, 109, 2984, 0, 506, 0, 367, 44, 63, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 71, 11, 184, 808, 976, 1313, 454, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5489, 7372, 6997, 6706, 7664, 6757, 4073, 4094] avg=6144.0000 max=7664 max_rank=4 min=4073 min_rank=6 imbalance_ratio=1.247396 max_load_reduction=41488 max_load_reduction_pct=0.844076 moved_to_echo=22588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[954, 92, 92, 0, 112, 246, 4489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 779, 1707, 1777, 9, 0, 404, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 261, 730, 1167, 163, 11, 1103, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 2400, 634, 1208, 189, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 1391, 811, 423, 10, 1562, 1431, 174, 475, 61, 0, 0, 0, 0, 336, 90, 15, 140, 952, 0, 20, 126, 4675, 938, 0, 0, 0, 0, 0, 0, 268, 3888, 0, 1112, 0, 522, 97, 374, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 110, 0, 127, 1018, 603, 1529, 592, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5985, 8284, 5071, 5496, 6555, 7292, 6261, 4208] avg=6144.0000 max=8284 max_rank=1 min=4208 min_rank=7 imbalance_ratio=1.348307 max_load_reduction=40868 max_load_reduction_pct=0.831462 moved_to_echo=24316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[966, 137, 119, 0, 242, 326, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1074, 1575, 1328, 0, 0, 344, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 22, 990, 2514, 121, 10, 1586, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 2678, 430, 1008, 166, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 7, 2191, 472, 695, 6, 2807, 1550, 148, 427, 48, 0, 0, 0, 0, 149, 226, 8, 155, 533, 0, 85, 7, 3962, 786, 0, 0, 0, 0, 0, 0, 105, 4210, 0, 758, 0, 339, 56, 130, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 28, 0, 249, 1375, 397, 1281, 641, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5822, 7212, 6888, 5106, 8476, 5911, 5598, 4139] avg=6144.0000 max=8476 max_rank=4 min=4139 min_rank=7 imbalance_ratio=1.379557 max_load_reduction=40676 max_load_reduction_pct=0.827555 moved_to_echo=24124 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1156, 48, 29, 0, 183, 180, 4821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 745, 1026, 1032, 3, 0, 1116, 2021, 0, 0, 0, 0, 0, 0, 0, 0, 1779, 0, 524, 1924, 147, 0, 2452, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 3092, 141, 633, 83, 2238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 23, 1047, 106, 871, 11, 1468, 1153, 482, 510, 141, 0, 0, 0, 0, 265, 108, 2, 51, 833, 0, 1, 1, 4586, 1102, 0, 0, 0, 0, 0, 0, 74, 3130, 0, 277, 0, 1286, 20, 124, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 204, 5, 252, 587, 1719, 1797, 477, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 5965, 7143, 6386, 6165, 6949, 4911, 5216] avg=6144.0000 max=7143 max_rank=2 min=4911 min_rank=6 imbalance_ratio=1.162598 max_load_reduction=42009 max_load_reduction_pct=0.854675 moved_to_echo=23241 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[762, 20, 21, 0, 70, 107, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 307, 229, 5, 0, 1586, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 2758, 39, 37, 857, 215, 0, 2499, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1012, 110, 191, 28, 3335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 479, 243, 104, 6, 663, 730, 691, 664, 213, 0, 0, 0, 0, 453, 9, 0, 52, 1574, 0, 13, 0, 4602, 1720, 0, 0, 0, 0, 0, 0, 914, 1964, 0, 276, 0, 4248, 4, 23, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1, 0, 430, 1, 241, 130, 3286, 2800, 303, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7206, 3528, 6460, 4683, 4196, 8423, 7429, 7227] avg=6144.0000 max=8423 max_rank=5 min=3528 min_rank=1 imbalance_ratio=1.370931 max_load_reduction=40729 max_load_reduction_pct=0.828634 moved_to_echo=27275 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[954, 92, 92, 0, 112, 246, 4489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 779, 1707, 1777, 9, 0, 404, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 261, 730, 1167, 163, 11, 1103, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 2400, 634, 1208, 189, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 1391, 811, 423, 10, 1562, 1431, 174, 475, 61, 0, 0, 0, 0, 336, 90, 15, 140, 952, 0, 20, 126, 4675, 938, 0, 0, 0, 0, 0, 0, 268, 3888, 0, 1112, 0, 522, 97, 374, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 110, 0, 127, 1018, 603, 1529, 592, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5985, 8284, 5071, 5496, 6555, 7292, 6261, 4208] avg=6144.0000 max=8284 max_rank=1 min=4208 min_rank=7 imbalance_ratio=1.348307 max_load_reduction=40868 max_load_reduction_pct=0.831462 moved_to_echo=24316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2176, 128, 8576, 2176, 4736, 128, 13312, 7680, 5120, 4224, 1408, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1280, 128, 128, 2176, 128, 2176, 5376, 19712, 15616, 3200, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 6144, 7680, 8832, 256, 0, 11776, 14720, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[15488, 512, 3584, 12160, 1280, 128, 14592, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 6144, 7680, 8832, 256, 0, 11776, 14720, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2176, 768, 128, 768, 5632, 0, 256, 256, 30336, 9600, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 768, 17024, 2176, 6272, 896, 22272, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 20736, 128, 3840, 128, 21888, 384, 896, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6144, 640, 512, 0, 1280, 1408, 39424, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[22, 512, 2219, 2, 0, 174, 321, 0, 0, 920, 3, 1105, 1653, 9, 1, 0, 0, 7098, 75, 0, 0, 0, 2, 0, 86, 27, 167, 16, 19, 250, 4029, 94, 0, 2, 2, 0, 1, 67, 4, 319, 913, 7674, 14, 105, 1669, 6895, 17, 75, 12, 0, 11, 0, 4067, 0, 27, 8069, 4, 95, 116, 0, 7, 95, 85, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[22, 37, 618, 0, 0, 3608, 0, 70, 481, 71, 0, 1833, 5801, 84, 5, 79, 24, 47, 2924, 0, 0, 192, 0, 0, 242, 1, 3175, 144, 19, 28, 629, 2, 387, 0, 0, 0, 0, 3828, 0, 6289, 3175, 426, 17, 4286, 0, 187, 1218, 141, 0, 222, 175, 33, 2, 705, 250, 4024, 1, 15, 513, 0, 48, 1731, 1173, 170] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[16, 328, 1791, 0, 0, 1883, 20, 21, 72, 192, 0, 2815, 6168, 110, 0, 7, 4, 1503, 1896, 0, 0, 146, 0, 0, 261, 25, 835, 52, 10, 160, 1744, 48, 365, 0, 0, 0, 0, 2032, 0, 3657, 3940, 3964, 37, 1849, 0, 2339, 290, 248, 0, 92, 135, 17, 159, 501, 171, 6305, 0, 184, 410, 0, 65, 1128, 1090, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[45, 275, 1025, 3, 0, 2518, 0, 26, 245, 147, 0, 2524, 6383, 161, 38, 30, 31, 355, 3339, 0, 0, 286, 0, 0, 671, 24, 1364, 73, 68, 142, 766, 30, 672, 0, 0, 0, 0, 3146, 0, 5269, 2496, 1913, 62, 2901, 0, 813, 722, 109, 0, 56, 217, 54, 36, 921, 170, 5394, 3, 249, 505, 0, 60, 1483, 1281, 51] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[27, 197, 915, 0, 0, 2310, 0, 28, 146, 107, 0, 3383, 6539, 124, 2, 13, 82, 257, 2297, 0, 0, 119, 0, 0, 590, 1, 2621, 29, 70, 78, 542, 16, 271, 0, 0, 0, 0, 3200, 0, 5585, 3271, 1401, 27, 2591, 0, 549, 435, 108, 0, 175, 353, 14, 3, 286, 355, 5291, 6, 200, 549, 0, 55, 2443, 1425, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[45, 275, 1025, 3, 0, 2518, 0, 26, 245, 147, 0, 2524, 6383, 161, 38, 30, 31, 355, 3339, 0, 0, 286, 0, 0, 671, 24, 1364, 73, 68, 142, 766, 30, 672, 0, 0, 0, 0, 3146, 0, 5269, 2496, 1913, 62, 2901, 0, 813, 722, 109, 0, 56, 217, 54, 36, 921, 170, 5394, 3, 249, 505, 0, 60, 1483, 1281, 51] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[21, 2040, 2159, 0, 0, 856, 150, 10, 21, 404, 0, 2889, 4933, 102, 16, 4, 6, 5394, 1249, 0, 0, 22, 1, 0, 204, 60, 632, 17, 14, 279, 1032, 455, 86, 0, 0, 0, 0, 602, 0, 992, 2333, 6710, 19, 421, 91, 4043, 57, 273, 1, 27, 142, 0, 639, 15, 26, 7694, 37, 768, 464, 0, 24, 401, 300, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[0, 341, 1096, 0, 0, 6, 525, 0, 0, 1117, 0, 377, 373, 0, 0, 1, 0, 7457, 5, 0, 0, 1, 0, 0, 0, 0, 5, 2, 0, 282, 3534, 0, 0, 0, 0, 0, 0, 7, 0, 12, 63, 7912, 0, 9, 4416, 7961, 6, 0, 6, 0, 0, 0, 5140, 0, 6, 8182, 0, 85, 203, 0, 10, 3, 8, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[22, 512, 2219, 2, 0, 174, 321, 0, 0, 920, 3, 1105, 1653, 9, 1, 0, 0, 7098, 75, 0, 0, 0, 2, 0, 86, 27, 167, 16, 19, 250, 4029, 94, 0, 2, 2, 0, 1, 67, 4, 319, 913, 7674, 14, 105, 1669, 6895, 17, 75, 12, 0, 11, 0, 4067, 0, 27, 8069, 4, 95, 116, 0, 7, 95, 85, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[3, 809, 826, 6, 0, 2242, 2, 7, 86, 148, 0, 4027, 6672, 142, 5, 37, 4, 494, 3534, 1, 0, 51, 3, 0, 869, 24, 2310, 146, 108, 142, 386, 21, 105, 0, 0, 0, 0, 2467, 0, 4094, 2172, 2482, 50, 2124, 0, 567, 462, 145, 0, 80, 193, 39, 6, 258, 692, 5964, 8, 294, 492, 0, 71, 2018, 1171, 93] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[22, 512, 2219, 2, 0, 174, 321, 0, 4448, 626, 0, 0, 0, 0, 0, 0, 0, 920, 3, 1105, 1076, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7098, 75, 0, 0, 0, 2, 0, 333, 735, 0, 0, 0, 0, 0, 0, 86, 27, 167, 16, 19, 250, 4029, 94, 2267, 1222, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 1, 67, 4, 319, 244, 0, 0, 0, 0, 0, 0, 0, 913, 0, 14, 105, 1669, 1712, 17, 75, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 11, 0, 4067, 0, 27, 5176, 0, 0, 0, 0, 0, 0, 0, 0, 4, 95, 116, 0, 7, 95, 85, 3, 6452, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 3114, 8243, 8177, 639, 4505, 9293, 6857] avg=6144.0000 max=9293 max_rank=6 min=639 min_rank=4 imbalance_ratio=1.512533 max_load_reduction=39859 max_load_reduction_pct=0.810933 moved_to_echo=21294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[0, 341, 1096, 0, 0, 6, 525, 0, 0, 1117, 0, 377, 373, 0, 0, 1, 0, 7457, 5, 0, 0, 1, 0, 0, 0, 0, 5, 2, 0, 282, 3534, 0, 0, 0, 0, 0, 0, 7, 0, 12, 63, 7912, 0, 9, 4416, 7961, 6, 0, 6, 0, 0, 0, 5140, 0, 6, 8182, 0, 85, 203, 0, 10, 3, 8, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[16, 328, 1791, 0, 0, 1883, 20, 21, 72, 192, 0, 2815, 6168, 110, 0, 7, 4, 1503, 1896, 0, 0, 146, 0, 0, 261, 25, 835, 52, 10, 160, 1744, 48, 365, 0, 0, 0, 0, 2032, 0, 3657, 3940, 3964, 37, 1849, 0, 2339, 290, 248, 0, 92, 135, 17, 159, 501, 171, 6305, 0, 184, 410, 0, 65, 1128, 1090, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[22, 37, 618, 0, 0, 3608, 0, 70, 120, 312, 0, 0, 0, 0, 0, 0, 481, 71, 0, 1833, 3776, 84, 5, 79, 0, 0, 0, 0, 0, 0, 0, 0, 24, 47, 2924, 0, 0, 192, 0, 0, 1168, 19, 0, 0, 0, 0, 0, 0, 242, 1, 3175, 144, 19, 28, 629, 2, 1130, 67, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, 3828, 0, 6289, 857, 0, 0, 0, 0, 0, 0, 0, 3175, 0, 17, 4286, 0, 48, 1218, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 175, 33, 2, 705, 250, 2582, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 513, 0, 48, 1731, 1173, 170, 359, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4787, 6329, 4374, 5437, 11361, 8885, 3969, 4010] avg=6144.0000 max=11361 max_rank=4 min=3969 min_rank=6 imbalance_ratio=1.849121 max_load_reduction=37791 max_load_reduction_pct=0.768860 moved_to_echo=28225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[45, 275, 1025, 3, 0, 2518, 0, 26, 524, 418, 0, 0, 0, 0, 0, 0, 245, 147, 0, 2524, 4155, 161, 38, 30, 0, 0, 0, 0, 0, 0, 0, 0, 31, 355, 3339, 0, 0, 286, 0, 0, 1286, 86, 0, 0, 0, 0, 0, 0, 671, 24, 1364, 73, 68, 142, 766, 30, 1515, 304, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 3146, 0, 5269, 942, 0, 0, 0, 0, 0, 0, 0, 2496, 0, 62, 2901, 0, 203, 722, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 217, 54, 36, 921, 170, 3461, 0, 0, 0, 0, 0, 0, 0, 0, 3, 249, 505, 0, 60, 1483, 1281, 51, 1609, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4834, 7300, 5383, 4957, 10029, 6493, 4915, 5241] avg=6144.0000 max=10029 max_rank=4 min=4834 min_rank=0 imbalance_ratio=1.632324 max_load_reduction=39123 max_load_reduction_pct=0.795959 moved_to_echo=26678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[16, 328, 1791, 0, 0, 1883, 20, 21, 1509, 489, 0, 0, 0, 0, 0, 0, 72, 192, 0, 2815, 4015, 110, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1503, 1896, 0, 0, 146, 0, 0, 1242, 249, 0, 0, 0, 0, 0, 0, 261, 25, 835, 52, 10, 160, 1744, 48, 1771, 632, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 2032, 0, 3657, 911, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 37, 1849, 0, 581, 290, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 135, 17, 159, 501, 171, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 410, 0, 65, 1128, 1090, 67, 3332, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6057, 7211, 5040, 5538, 6965, 6945, 5120, 6276] avg=6144.0000 max=7211 max_rank=1 min=5040 min_rank=2 imbalance_ratio=1.173665 max_load_reduction=41941 max_load_reduction_pct=0.853292 moved_to_echo=25306 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 341, 1096, 0, 0, 6, 525, 0, 5136, 634, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 377, 243, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7457, 5, 0, 0, 1, 0, 0, 75, 849, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 282, 3534, 0, 2298, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 12, 55, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 9, 4416, 1976, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 5140, 0, 6, 5250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 203, 0, 10, 3, 8, 1, 6651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7738, 1738, 8387, 7382, 74, 6470, 10402, 6961] avg=6144.0000 max=10402 max_rank=6 min=74 min_rank=4 imbalance_ratio=1.693034 max_load_reduction=38750 max_load_reduction_pct=0.788371 moved_to_echo=23907 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[27, 197, 915, 0, 0, 2310, 0, 28, 354, 410, 0, 0, 0, 0, 0, 0, 146, 107, 0, 3383, 4256, 124, 2, 13, 0, 0, 0, 0, 0, 0, 0, 0, 82, 257, 2297, 0, 0, 119, 0, 0, 1317, 58, 0, 0, 0, 0, 0, 0, 590, 1, 2621, 29, 70, 78, 542, 16, 1486, 223, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 3200, 0, 5585, 966, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 27, 2591, 0, 137, 435, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 353, 14, 3, 286, 355, 3395, 0, 0, 0, 0, 0, 0, 0, 0, 6, 200, 549, 0, 55, 2443, 1425, 66, 1178, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4241, 8031, 4130, 5656, 10022, 6569, 4581, 5922] avg=6144.0000 max=10022 max_rank=4 min=4130 min_rank=2 imbalance_ratio=1.631185 max_load_reduction=39130 max_load_reduction_pct=0.796102 moved_to_echo=27094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[21, 2040, 2159, 0, 0, 856, 150, 10, 2612, 601, 0, 0, 0, 0, 0, 0, 21, 404, 0, 2889, 3205, 102, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5394, 1249, 0, 0, 22, 1, 0, 997, 435, 0, 0, 0, 0, 0, 0, 204, 60, 632, 17, 14, 279, 1032, 455, 2166, 1069, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 602, 0, 992, 731, 0, 0, 0, 0, 0, 0, 0, 2333, 0, 19, 421, 91, 996, 57, 273, 0, 0, 0, 0, 0, 0, 0, 0, 1, 27, 142, 0, 639, 15, 26, 4927, 0, 0, 0, 0, 0, 0, 0, 0, 37, 768, 464, 0, 24, 401, 300, 17, 5641, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8449, 6641, 8104, 5928, 2411, 4190, 5777, 7652] avg=6144.0000 max=8449 max_rank=0 min=2411 min_rank=4 imbalance_ratio=1.375163 max_load_reduction=40703 max_load_reduction_pct=0.828105 moved_to_echo=20030 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 341, 1096, 0, 0, 6, 525, 0, 5136, 634, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 377, 243, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7457, 5, 0, 0, 1, 0, 0, 75, 849, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 282, 3534, 0, 2298, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 12, 55, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 9, 4416, 1976, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 5140, 0, 6, 5250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 203, 0, 10, 3, 8, 1, 6651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7738, 1738, 8387, 7382, 74, 6470, 10402, 6961] avg=6144.0000 max=10402 max_rank=6 min=74 min_rank=4 imbalance_ratio=1.693034 max_load_reduction=38750 max_load_reduction_pct=0.788371 moved_to_echo=23907 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3, 809, 826, 6, 0, 2242, 2, 7, 365, 462, 0, 0, 0, 0, 0, 0, 86, 148, 0, 4027, 4343, 142, 5, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4, 494, 3534, 1, 0, 51, 3, 0, 1344, 60, 0, 0, 0, 0, 0, 0, 869, 24, 2310, 146, 108, 142, 386, 21, 1675, 396, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 2467, 0, 4094, 985, 0, 0, 0, 0, 0, 0, 0, 2172, 0, 50, 2124, 0, 142, 462, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 193, 39, 6, 258, 692, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 8, 294, 492, 0, 71, 2018, 1171, 93, 2086, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4722, 8788, 5491, 6077, 7651, 5095, 5095, 6233] avg=6144.0000 max=8788 max_rank=1 min=4722 min_rank=0 imbalance_ratio=1.430339 max_load_reduction=40364 max_load_reduction_pct=0.821208 moved_to_echo=24074 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[22, 37, 618, 0, 0, 3608, 0, 70, 120, 312, 0, 0, 0, 0, 0, 0, 481, 71, 0, 1833, 3776, 84, 5, 79, 0, 0, 0, 0, 0, 0, 0, 0, 24, 47, 2924, 0, 0, 192, 0, 0, 1168, 19, 0, 0, 0, 0, 0, 0, 242, 1, 3175, 144, 19, 28, 629, 2, 1130, 67, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, 3828, 0, 6289, 857, 0, 0, 0, 0, 0, 0, 0, 3175, 0, 17, 4286, 0, 48, 1218, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 175, 33, 2, 705, 250, 2582, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 513, 0, 48, 1731, 1173, 170, 359, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4787, 6329, 4374, 5437, 11361, 8885, 3969, 4010] avg=6144.0000 max=11361 max_rank=4 min=3969 min_rank=6 imbalance_ratio=1.849121 max_load_reduction=37791 max_load_reduction_pct=0.768860 moved_to_echo=28225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[22, 512, 2219, 2, 0, 174, 321, 0, 4448, 626, 0, 0, 0, 0, 0, 0, 0, 920, 3, 1105, 1076, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7098, 75, 0, 0, 0, 2, 0, 333, 735, 0, 0, 0, 0, 0, 0, 86, 27, 167, 16, 19, 250, 4029, 94, 2267, 1222, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 1, 67, 4, 319, 244, 0, 0, 0, 0, 0, 0, 0, 913, 0, 14, 105, 1669, 1712, 17, 75, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 11, 0, 4067, 0, 27, 5176, 0, 0, 0, 0, 0, 0, 0, 0, 4, 95, 116, 0, 7, 95, 85, 3, 6452, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 3114, 8243, 8177, 639, 4505, 9293, 6857] avg=6144.0000 max=9293 max_rank=6 min=639 min_rank=4 imbalance_ratio=1.512533 max_load_reduction=39859 max_load_reduction_pct=0.810933 moved_to_echo=21294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[16, 328, 1791, 0, 0, 1883, 20, 21, 1509, 489, 0, 0, 0, 0, 0, 0, 72, 192, 0, 2815, 4015, 110, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1503, 1896, 0, 0, 146, 0, 0, 1242, 249, 0, 0, 0, 0, 0, 0, 261, 25, 835, 52, 10, 160, 1744, 48, 1771, 632, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 2032, 0, 3657, 911, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 37, 1849, 0, 581, 290, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 135, 17, 159, 501, 171, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 410, 0, 65, 1128, 1090, 67, 3332, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6057, 7211, 5040, 5538, 6965, 6945, 5120, 6276] avg=6144.0000 max=7211 max_rank=1 min=5040 min_rank=2 imbalance_ratio=1.173665 max_load_reduction=41941 max_load_reduction_pct=0.853292 moved_to_echo=25306 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[21, 2040, 2159, 0, 0, 856, 150, 10, 2612, 601, 0, 0, 0, 0, 0, 0, 21, 404, 0, 2889, 3205, 102, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5394, 1249, 0, 0, 22, 1, 0, 997, 435, 0, 0, 0, 0, 0, 0, 204, 60, 632, 17, 14, 279, 1032, 455, 2166, 1069, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 602, 0, 992, 731, 0, 0, 0, 0, 0, 0, 0, 2333, 0, 19, 421, 91, 996, 57, 273, 0, 0, 0, 0, 0, 0, 0, 0, 1, 27, 142, 0, 639, 15, 26, 4927, 0, 0, 0, 0, 0, 0, 0, 0, 37, 768, 464, 0, 24, 401, 300, 17, 5641, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8449, 6641, 8104, 5928, 2411, 4190, 5777, 7652] avg=6144.0000 max=8449 max_rank=0 min=2411 min_rank=4 imbalance_ratio=1.375163 max_load_reduction=40703 max_load_reduction_pct=0.828105 moved_to_echo=20030 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 128, 128, 0, 128, 15360, 128, 26240, 5760, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 3200, 128, 19072, 25088, 768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 4608, 10752, 128, 0, 13696, 1024, 256, 15104, 3968, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 256, 11136, 512, 384, 1408, 12672, 768, 14336, 5248, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18432, 0, 256, 14336, 6272, 5888, 3328, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 256, 11136, 512, 384, 1408, 12672, 768, 14336, 5248, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 768, 1280, 256, 10112, 2688, 1792, 32768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1920, 3328, 0, 384, 9344, 6656, 512, 27392, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 22656, 15360, 128, 0, 896, 128, 0, 7808, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1920, 3328, 0, 384, 9344, 6656, 512, 27392, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 1, 1, 343, 54, 2407, 23, 99, 11, 0, 130, 59, 1870, 143, 424, 1258, 6440, 210, 311, 1198, 137, 2779, 194, 0, 162, 1477, 4899, 149, 75, 222, 0, 516, 655, 39, 0, 0, 3285, 76, 138, 1, 68, 346, 0, 31, 87, 1287, 0, 139, 2388, 2359, 411, 214, 230, 2614, 32, 104, 2762, 14, 894, 0, 582, 46, 4724, 34] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[5, 19, 0, 11, 1, 5, 7, 1397, 1953, 0, 223, 14, 9, 1803, 5, 69, 223, 65, 32, 805, 1, 3144, 19, 0, 72, 184, 8072, 1, 0, 7, 0, 9, 0, 432, 0, 2, 7989, 0, 1, 0, 17, 99, 1, 41, 4501, 6, 8, 20, 7814, 58, 403, 665, 27, 65, 19, 80, 218, 4251, 3, 37, 2, 1996, 328, 1914] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 5, 294, 71, 2801, 23, 147, 14, 5, 155, 127, 2073, 449, 746, 1093, 5446, 143, 341, 1019, 225, 2381, 137, 2, 231, 1168, 5136, 511, 39, 659, 0, 339, 973, 162, 0, 2, 2970, 69, 328, 0, 216, 166, 0, 94, 154, 1798, 19, 188, 2716, 956, 564, 408, 257, 2399, 55, 135, 2707, 26, 1275, 0, 868, 415, 3407, 45] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 4, 0, 155, 26, 1790, 23, 538, 50, 0, 178, 308, 1211, 848, 309, 442, 3565, 127, 249, 1351, 211, 3564, 184, 0, 213, 770, 6327, 90, 9, 365, 0, 463, 587, 349, 0, 0, 5108, 58, 231, 2, 215, 49, 0, 74, 409, 1080, 16, 150, 3831, 417, 754, 940, 258, 1387, 35, 151, 2981, 278, 736, 0, 639, 1396, 3272, 379] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 2, 3, 46, 29, 268, 104, 802, 1069, 0, 217, 199, 318, 1045, 17, 333, 1276, 73, 53, 2524, 54, 4306, 110, 0, 127, 320, 7617, 47, 34, 138, 0, 33, 136, 503, 0, 0, 7277, 14, 52, 0, 81, 51, 0, 363, 2786, 171, 3, 35, 7088, 567, 589, 1018, 130, 350, 4, 292, 1029, 1348, 214, 26, 39, 728, 1701, 1393] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 1, 1, 343, 54, 2407, 23, 99, 11, 0, 130, 59, 1870, 143, 424, 1258, 6440, 210, 311, 1198, 137, 2779, 194, 0, 162, 1477, 4899, 149, 75, 222, 0, 516, 655, 39, 0, 0, 3285, 76, 138, 1, 68, 346, 0, 31, 87, 1287, 0, 139, 2388, 2359, 411, 214, 230, 2614, 32, 104, 2762, 14, 894, 0, 582, 46, 4724, 34] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 11, 370, 72, 3477, 18, 87, 0, 0, 72, 90, 2747, 65, 589, 1473, 6749, 308, 469, 708, 73, 1829, 429, 0, 182, 836, 3617, 63, 9, 1071, 0, 728, 1250, 14, 0, 0, 1638, 116, 454, 2, 121, 97, 0, 38, 32, 2515, 16, 176, 608, 1067, 480, 156, 244, 3833, 25, 43, 3083, 1, 1289, 0, 1190, 229, 4292, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 50, 1, 457, 242, 1751, 120, 228, 45, 0, 156, 54, 946, 168, 220, 1148, 6450, 202, 747, 1378, 131, 2466, 150, 0, 154, 1211, 5513, 189, 63, 529, 0, 259, 364, 367, 0, 2, 4084, 356, 57, 2, 7, 307, 0, 310, 192, 729, 0, 169, 3742, 1713, 429, 360, 247, 2155, 18, 127, 2307, 15, 820, 0, 509, 255, 4360, 121] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 5, 294, 71, 2801, 23, 147, 14, 5, 155, 127, 2073, 449, 746, 1093, 5446, 143, 341, 1019, 225, 2381, 137, 2, 231, 1168, 5136, 511, 39, 659, 0, 339, 973, 162, 0, 2, 2970, 69, 328, 0, 216, 166, 0, 94, 154, 1798, 19, 188, 2716, 956, 564, 408, 257, 2399, 55, 135, 2707, 26, 1275, 0, 868, 415, 3407, 45] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 2, 0, 808, 3921, 0, 228, 4, 4, 800, 0, 0, 25, 0, 5, 98, 1, 1542, 0, 0, 1, 2, 8188, 0, 4, 1, 0, 3, 2, 301, 0, 0, 8171, 0, 1, 0, 0, 4, 0, 1, 6868, 3, 0, 0, 8007, 2, 46, 464, 1, 7, 0, 0, 18, 5103, 1, 125, 1, 1347, 19, 3022] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 11, 370, 72, 3477, 18, 87, 0, 0, 72, 90, 2747, 65, 589, 1473, 6749, 308, 469, 708, 73, 1829, 429, 0, 182, 836, 3617, 63, 9, 1071, 0, 728, 1250, 14, 0, 0, 1638, 116, 454, 2, 121, 97, 0, 38, 32, 2515, 16, 176, 608, 1067, 480, 156, 244, 3833, 25, 43, 3083, 1, 1289, 0, 1190, 229, 4292, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[5, 19, 0, 11, 1, 5, 7, 1397, 321, 61, 0, 0, 0, 0, 0, 0, 1953, 0, 223, 14, 9, 1803, 5, 69, 2194, 825, 0, 0, 0, 0, 0, 0, 94, 65, 32, 805, 1, 3144, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 184, 5792, 1, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 2, 7989, 0, 1, 0, 86, 0, 0, 0, 0, 0, 0, 0, 17, 99, 1, 41, 4501, 6, 8, 20, 2932, 68, 0, 0, 0, 0, 0, 0, 4057, 58, 403, 665, 27, 65, 19, 80, 0, 0, 0, 0, 0, 0, 0, 0, 218, 4251, 3, 37, 2, 1996, 7, 1914, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1827, 7095, 4160, 6065, 8510, 7693, 5374, 8428] avg=6144.0000 max=8510 max_rank=4 min=1827 min_rank=0 imbalance_ratio=1.385091 max_load_reduction=40642 max_load_reduction_pct=0.826864 moved_to_echo=30005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 2, 3, 46, 29, 268, 104, 802, 1069, 0, 217, 199, 318, 1045, 17, 333, 1276, 73, 53, 2524, 54, 4306, 110, 0, 127, 320, 7617, 47, 34, 138, 0, 33, 136, 503, 0, 0, 7277, 14, 52, 0, 81, 51, 0, 363, 2786, 171, 3, 35, 7088, 567, 589, 1018, 130, 350, 4, 292, 1029, 1348, 214, 26, 39, 728, 1701, 1393] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 294, 71, 2801, 23, 147, 3339, 1496, 0, 0, 0, 0, 0, 0, 14, 5, 155, 127, 2073, 449, 746, 1093, 1396, 287, 0, 0, 0, 0, 0, 0, 2287, 143, 341, 1019, 225, 2381, 137, 2, 0, 0, 0, 0, 0, 0, 0, 0, 231, 1168, 3685, 511, 39, 659, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 973, 162, 0, 2, 2970, 69, 328, 0, 55, 0, 0, 0, 0, 0, 0, 0, 216, 166, 0, 94, 154, 1798, 19, 188, 1019, 1663, 0, 0, 0, 0, 0, 0, 1410, 956, 564, 408, 257, 2399, 55, 135, 0, 0, 0, 0, 0, 0, 0, 0, 2707, 26, 1275, 0, 868, 415, 68, 45, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 6345, 6535, 6632, 4559, 5317, 6184, 5404] avg=6144.0000 max=8176 max_rank=0 min=4559 min_rank=4 imbalance_ratio=1.330729 max_load_reduction=40976 max_load_reduction_pct=0.833659 moved_to_echo=21464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 343, 54, 2407, 23, 99, 4629, 1769, 0, 0, 0, 0, 0, 0, 11, 0, 130, 59, 1870, 143, 424, 1258, 1331, 252, 0, 0, 0, 0, 0, 0, 2704, 210, 311, 1198, 137, 2779, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 1477, 3516, 149, 75, 222, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 655, 39, 0, 0, 3285, 76, 138, 1, 52, 0, 0, 0, 0, 0, 0, 0, 68, 346, 0, 31, 87, 1287, 0, 139, 896, 1967, 0, 0, 0, 0, 0, 0, 1240, 2359, 411, 214, 230, 2614, 32, 104, 0, 0, 0, 0, 0, 0, 0, 0, 2762, 14, 894, 0, 582, 46, 95, 34, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9326, 5478, 7533, 6117, 4246, 4821, 7204, 4427] avg=6144.0000 max=9326 max_rank=0 min=4246 min_rank=4 imbalance_ratio=1.517904 max_load_reduction=39826 max_load_reduction_pct=0.810262 moved_to_echo=20698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 370, 72, 3477, 18, 87, 4206, 1854, 0, 0, 0, 0, 0, 0, 0, 0, 72, 90, 2747, 65, 589, 1473, 983, 64, 0, 0, 0, 0, 0, 0, 2834, 308, 469, 708, 73, 1829, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 836, 2596, 63, 9, 1071, 0, 728, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 14, 0, 0, 1638, 116, 454, 2, 38, 0, 0, 0, 0, 0, 0, 0, 121, 97, 0, 38, 32, 2515, 16, 176, 228, 2061, 0, 0, 0, 0, 0, 0, 316, 1067, 480, 156, 244, 3833, 25, 43, 0, 0, 0, 0, 0, 0, 0, 0, 3083, 1, 1289, 0, 1190, 229, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10095, 6083, 6650, 5485, 3512, 5284, 6164, 5879] avg=6144.0000 max=10095 max_rank=0 min=3512 min_rank=4 imbalance_ratio=1.643066 max_load_reduction=39057 max_load_reduction_pct=0.794617 moved_to_echo=20839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[5, 19, 0, 11, 1, 5, 7, 1397, 321, 61, 0, 0, 0, 0, 0, 0, 1953, 0, 223, 14, 9, 1803, 5, 69, 2194, 825, 0, 0, 0, 0, 0, 0, 94, 65, 32, 805, 1, 3144, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 184, 5792, 1, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 2, 7989, 0, 1, 0, 86, 0, 0, 0, 0, 0, 0, 0, 17, 99, 1, 41, 4501, 6, 8, 20, 2932, 68, 0, 0, 0, 0, 0, 0, 4057, 58, 403, 665, 27, 65, 19, 80, 0, 0, 0, 0, 0, 0, 0, 0, 218, 4251, 3, 37, 2, 1996, 7, 1914, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1827, 7095, 4160, 6065, 8510, 7693, 5374, 8428] avg=6144.0000 max=8510 max_rank=4 min=1827 min_rank=0 imbalance_ratio=1.385091 max_load_reduction=40642 max_load_reduction_pct=0.826864 moved_to_echo=30005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 2, 0, 808, 18, 6, 0, 0, 0, 0, 0, 0, 3921, 0, 228, 4, 4, 800, 0, 0, 2225, 846, 0, 0, 0, 0, 0, 0, 12, 0, 5, 98, 1, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 5875, 0, 4, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 301, 0, 0, 8171, 0, 1, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 6868, 3, 0, 0, 3004, 7, 0, 0, 0, 0, 0, 0, 4157, 2, 46, 464, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 5103, 1, 125, 1, 1347, 1, 3022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[835, 8028, 1658, 5886, 8563, 9887, 4677, 9618] avg=6144.0000 max=9887 max_rank=5 min=835 min_rank=0 imbalance_ratio=1.609212 max_load_reduction=39265 max_load_reduction_pct=0.798848 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 155, 26, 1790, 23, 538, 3206, 979, 0, 0, 0, 0, 0, 0, 50, 0, 178, 308, 1211, 848, 309, 442, 1719, 404, 0, 0, 0, 0, 0, 0, 1498, 127, 249, 1351, 211, 3564, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 770, 4540, 90, 9, 365, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 587, 349, 0, 0, 5108, 58, 231, 2, 68, 0, 0, 0, 0, 0, 0, 0, 215, 49, 0, 74, 409, 1080, 16, 150, 1437, 1088, 0, 0, 0, 0, 0, 0, 1990, 417, 754, 940, 258, 1387, 35, 151, 0, 0, 0, 0, 0, 0, 0, 0, 2981, 278, 736, 0, 639, 1396, 66, 379, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6721, 5469, 7184, 6450, 6403, 4518, 5932, 6475] avg=6144.0000 max=7184 max_rank=2 min=4518 min_rank=5 imbalance_ratio=1.169271 max_load_reduction=41968 max_load_reduction_pct=0.853841 moved_to_echo=23328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 294, 71, 2801, 23, 147, 3339, 1496, 0, 0, 0, 0, 0, 0, 14, 5, 155, 127, 2073, 449, 746, 1093, 1396, 287, 0, 0, 0, 0, 0, 0, 2287, 143, 341, 1019, 225, 2381, 137, 2, 0, 0, 0, 0, 0, 0, 0, 0, 231, 1168, 3685, 511, 39, 659, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 973, 162, 0, 2, 2970, 69, 328, 0, 55, 0, 0, 0, 0, 0, 0, 0, 216, 166, 0, 94, 154, 1798, 19, 188, 1019, 1663, 0, 0, 0, 0, 0, 0, 1410, 956, 564, 408, 257, 2399, 55, 135, 0, 0, 0, 0, 0, 0, 0, 0, 2707, 26, 1275, 0, 868, 415, 68, 45, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 6345, 6535, 6632, 4559, 5317, 6184, 5404] avg=6144.0000 max=8176 max_rank=0 min=4559 min_rank=4 imbalance_ratio=1.330729 max_load_reduction=40976 max_load_reduction_pct=0.833659 moved_to_echo=21464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 3, 46, 29, 268, 104, 802, 1671, 353, 0, 0, 0, 0, 0, 0, 1069, 0, 217, 199, 318, 1045, 17, 333, 2074, 752, 0, 0, 0, 0, 0, 0, 530, 73, 53, 2524, 54, 4306, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 320, 5457, 47, 34, 138, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 136, 503, 0, 0, 7277, 14, 52, 0, 86, 0, 0, 0, 0, 0, 0, 0, 81, 51, 0, 363, 2786, 171, 3, 35, 2661, 393, 0, 0, 0, 0, 0, 0, 3675, 567, 589, 1018, 130, 350, 4, 292, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1348, 214, 26, 39, 728, 30, 1393, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3278, 6024, 7650, 6156, 8068, 6544, 6625, 4807] avg=6144.0000 max=8068 max_rank=4 min=3278 min_rank=0 imbalance_ratio=1.313151 max_load_reduction=41084 max_load_reduction_pct=0.835856 moved_to_echo=26044 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 370, 72, 3477, 18, 87, 4206, 1854, 0, 0, 0, 0, 0, 0, 0, 0, 72, 90, 2747, 65, 589, 1473, 983, 64, 0, 0, 0, 0, 0, 0, 2834, 308, 469, 708, 73, 1829, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 836, 2596, 63, 9, 1071, 0, 728, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 14, 0, 0, 1638, 116, 454, 2, 38, 0, 0, 0, 0, 0, 0, 0, 121, 97, 0, 38, 32, 2515, 16, 176, 228, 2061, 0, 0, 0, 0, 0, 0, 316, 1067, 480, 156, 244, 3833, 25, 43, 0, 0, 0, 0, 0, 0, 0, 0, 3083, 1, 1289, 0, 1190, 229, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10095, 6083, 6650, 5485, 3512, 5284, 6164, 5879] avg=6144.0000 max=10095 max_rank=0 min=3512 min_rank=4 imbalance_ratio=1.643066 max_load_reduction=39057 max_load_reduction_pct=0.794617 moved_to_echo=20839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 50, 1, 457, 242, 1751, 120, 228, 4273, 1772, 0, 0, 0, 0, 0, 0, 45, 0, 156, 54, 946, 168, 220, 1148, 1498, 395, 0, 0, 0, 0, 0, 0, 2708, 202, 747, 1378, 131, 2466, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 1211, 3956, 189, 63, 529, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 364, 367, 0, 2, 4084, 356, 57, 2, 59, 0, 0, 0, 0, 0, 0, 0, 7, 307, 0, 310, 192, 729, 0, 169, 1404, 1970, 0, 0, 0, 0, 0, 0, 1943, 1713, 429, 360, 247, 2155, 18, 127, 0, 0, 0, 0, 0, 0, 0, 0, 2307, 15, 820, 0, 509, 255, 87, 121, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8894, 4630, 7782, 6361, 5291, 5088, 6992, 4114] avg=6144.0000 max=8894 max_rank=0 min=4114 min_rank=7 imbalance_ratio=1.447591 max_load_reduction=40258 max_load_reduction_pct=0.819051 moved_to_echo=21485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 3, 46, 29, 268, 104, 802, 1671, 353, 0, 0, 0, 0, 0, 0, 1069, 0, 217, 199, 318, 1045, 17, 333, 2074, 752, 0, 0, 0, 0, 0, 0, 530, 73, 53, 2524, 54, 4306, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 320, 5457, 47, 34, 138, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 136, 503, 0, 0, 7277, 14, 52, 0, 86, 0, 0, 0, 0, 0, 0, 0, 81, 51, 0, 363, 2786, 171, 3, 35, 2661, 393, 0, 0, 0, 0, 0, 0, 3675, 567, 589, 1018, 130, 350, 4, 292, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1348, 214, 26, 39, 728, 30, 1393, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3278, 6024, 7650, 6156, 8068, 6544, 6625, 4807] avg=6144.0000 max=8068 max_rank=4 min=3278 min_rank=0 imbalance_ratio=1.313151 max_load_reduction=41084 max_load_reduction_pct=0.835856 moved_to_echo=26044 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 1792, 512, 12544, 384, 4224, 21760, 8320, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7168, 128, 1408, 896, 9216, 5376, 2432, 5888, 13440, 3840, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18816, 7168, 3712, 4352, 1408, 12928, 256, 1024, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12672, 1152, 2304, 9088, 896, 22016, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18816, 7168, 3712, 4352, 1408, 12928, 256, 1024, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[15232, 11136, 5248, 256, 3840, 6528, 512, 6912, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 1152, 128, 1024, 15104, 7680, 128, 896, 13696, 9344, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1152, 6016, 35456, 1152, 256, 3072, 0, 2432, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3968, 2176, 0, 128, 40576, 768, 1280, 128, 640, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[20, 219, 0, 7548, 7688, 7274, 10, 1, 669, 0, 0, 17, 0, 14, 11, 4144, 0, 35, 28, 1, 18, 10, 2622, 0, 3, 83, 150, 318, 0, 0, 18, 0, 3047, 7911, 165, 2, 1, 20, 61, 15, 23, 95, 4497, 390, 28, 14, 0, 147, 13, 692, 0, 694, 0, 12, 6, 83, 179, 73, 71, 4, 1, 7, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[8, 1647, 0, 1648, 677, 1807, 304, 0, 6520, 26, 35, 291, 0, 82, 529, 2471, 0, 35, 176, 69, 0, 1480, 1852, 0, 0, 83, 1414, 371, 83, 276, 151, 0, 242, 3158, 816, 136, 272, 2100, 432, 371, 432, 966, 5170, 400, 18, 199, 0, 2943, 15, 473, 0, 2217, 2, 976, 50, 1244, 0, 1166, 962, 872, 7, 1477, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[34, 2308, 1, 1400, 385, 2032, 1064, 0, 5240, 265, 141, 521, 0, 68, 574, 825, 0, 16, 209, 60, 0, 2481, 1456, 0, 0, 64, 2212, 756, 4, 132, 0, 0, 308, 2142, 215, 317, 188, 2712, 123, 527, 405, 328, 4901, 269, 45, 63, 0, 2409, 65, 250, 0, 1890, 1, 1330, 1, 1827, 0, 1309, 1126, 1742, 0, 2411, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[20, 2331, 1, 1011, 458, 1797, 953, 0, 5291, 156, 620, 533, 0, 26, 434, 1483, 0, 20, 30, 16, 0, 2229, 1335, 5, 0, 57, 2772, 1035, 18, 166, 1, 0, 325, 2454, 242, 163, 216, 2266, 138, 447, 167, 550, 4616, 255, 14, 124, 0, 1538, 246, 615, 0, 2059, 4, 1667, 37, 1210, 0, 1529, 1154, 1533, 0, 2785, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[22, 2009, 0, 2936, 1642, 2994, 701, 0, 4024, 14, 283, 342, 0, 3, 249, 1971, 0, 34, 45, 14, 0, 1432, 2151, 0, 0, 63, 2768, 1110, 12, 168, 0, 0, 1028, 4259, 321, 101, 51, 1549, 39, 213, 128, 410, 5559, 330, 55, 54, 0, 752, 237, 681, 0, 2219, 7, 1146, 3, 747, 1, 1134, 630, 969, 0, 1542, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 2, 0, 7906, 8167, 7663, 1, 0, 27, 0, 0, 6, 0, 0, 3, 2989, 0, 45, 0, 0, 0, 9, 1703, 0, 0, 95, 46, 350, 0, 0, 0, 0, 4896, 8177, 57, 0, 0, 6, 0, 0, 1, 13, 4567, 254, 0, 4, 0, 4, 1, 892, 0, 139, 0, 6, 0, 9, 1106, 2, 2, 1, 0, 3, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[8, 1647, 0, 1648, 677, 1807, 304, 0, 6520, 26, 35, 291, 0, 82, 529, 2471, 0, 35, 176, 69, 0, 1480, 1852, 0, 0, 83, 1414, 371, 83, 276, 151, 0, 242, 3158, 816, 136, 272, 2100, 432, 371, 432, 966, 5170, 400, 18, 199, 0, 2943, 15, 473, 0, 2217, 2, 976, 50, 1244, 0, 1166, 962, 872, 7, 1477, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[71, 2088, 0, 328, 46, 967, 1516, 0, 4115, 163, 486, 851, 0, 28, 814, 746, 0, 0, 92, 28, 0, 3301, 831, 0, 0, 47, 3114, 563, 37, 384, 0, 0, 290, 741, 189, 216, 31, 4249, 86, 460, 550, 643, 3872, 297, 41, 98, 0, 2139, 467, 422, 0, 1499, 3, 2647, 13, 1230, 0, 1355, 1324, 2377, 0, 3297, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[34, 2308, 1, 1400, 385, 2032, 1064, 0, 5240, 265, 141, 521, 0, 68, 574, 825, 0, 16, 209, 60, 0, 2481, 1456, 0, 0, 64, 2212, 756, 4, 132, 0, 0, 308, 2142, 215, 317, 188, 2712, 123, 527, 405, 328, 4901, 269, 45, 63, 0, 2409, 65, 250, 0, 1890, 1, 1330, 1, 1827, 0, 1309, 1126, 1742, 0, 2411, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[20, 219, 0, 7548, 7688, 7274, 10, 1, 669, 0, 0, 17, 0, 14, 11, 4144, 0, 35, 28, 1, 18, 10, 2622, 0, 3, 83, 150, 318, 0, 0, 18, 0, 3047, 7911, 165, 2, 1, 20, 61, 15, 23, 95, 4497, 390, 28, 14, 0, 147, 13, 692, 0, 694, 0, 12, 6, 83, 179, 73, 71, 4, 1, 7, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[3, 749, 0, 5878, 5655, 5022, 177, 0, 4103, 9, 78, 45, 0, 29, 71, 3564, 0, 78, 25, 0, 0, 383, 1281, 0, 0, 85, 610, 1076, 5, 103, 6, 0, 1159, 6962, 601, 67, 8, 495, 55, 40, 45, 36, 5503, 384, 2, 5, 0, 571, 6, 636, 0, 1616, 0, 178, 1, 332, 0, 427, 431, 137, 0, 420, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[20, 219, 0, 2154, 7688, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 17, 0, 14, 11, 4144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 28, 1, 18, 10, 2622, 0, 5394, 402, 298, 0, 0, 0, 0, 0, 3, 83, 150, 318, 0, 0, 18, 0, 6976, 0, 0, 0, 0, 0, 0, 0, 3047, 3775, 165, 2, 1, 20, 61, 15, 0, 0, 0, 0, 0, 0, 0, 0, 23, 95, 3548, 390, 28, 14, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 13, 692, 0, 694, 0, 12, 6, 83, 3609, 0, 0, 0, 0, 0, 0, 0, 179, 73, 71, 4, 1, 7, 0, 0, 949, 152, 125, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10092, 4703, 8808, 7548, 7086, 4245, 5109, 1561] avg=6144.0000 max=10092 max_rank=0 min=1561 min_rank=7 imbalance_ratio=1.642578 max_load_reduction=39060 max_load_reduction_pct=0.794678 moved_to_echo=18001 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 2256, 8167, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 6, 0, 0, 3, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 9, 1703, 0, 5650, 416, 315, 0, 0, 0, 0, 0, 0, 95, 46, 350, 0, 0, 0, 0, 7348, 0, 0, 0, 0, 0, 0, 0, 4896, 3902, 57, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 3603, 254, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 892, 0, 139, 0, 6, 0, 9, 3730, 0, 0, 0, 0, 0, 0, 0, 1106, 2, 2, 1, 0, 3, 0, 0, 964, 6, 129, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10426, 3019, 8138, 7839, 8861, 3879, 4777, 2213] avg=6144.0000 max=10426 max_rank=0 min=2213 min_rank=7 imbalance_ratio=1.696940 max_load_reduction=38726 max_load_reduction_pct=0.787882 moved_to_echo=19730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[22, 2009, 0, 838, 1642, 0, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3106, 14, 283, 342, 0, 3, 249, 1971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 45, 14, 0, 1432, 2151, 0, 2098, 216, 123, 0, 0, 0, 0, 0, 0, 63, 2768, 1110, 12, 168, 0, 0, 2871, 0, 0, 0, 0, 0, 0, 0, 1028, 2033, 321, 101, 51, 1549, 39, 213, 0, 0, 0, 0, 0, 0, 0, 0, 128, 410, 4385, 330, 55, 54, 0, 752, 0, 0, 0, 0, 0, 0, 0, 0, 237, 681, 0, 2219, 7, 1146, 3, 747, 1943, 0, 0, 0, 0, 0, 0, 0, 1, 1134, 630, 969, 0, 1542, 0, 0, 1174, 918, 67, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5212, 5968, 6113, 6992, 5335, 6114, 6983, 6435] avg=6144.0000 max=6992 max_rank=3 min=5212 min_rank=0 imbalance_ratio=1.138021 max_load_reduction=42160 max_load_reduction_pct=0.857747 moved_to_echo=24867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 2, 0, 7906, 8167, 7663, 1, 0, 27, 0, 0, 6, 0, 0, 3, 2989, 0, 45, 0, 0, 0, 9, 1703, 0, 0, 95, 46, 350, 0, 0, 0, 0, 4896, 8177, 57, 0, 0, 6, 0, 0, 1, 13, 4567, 254, 0, 4, 0, 4, 1, 892, 0, 139, 0, 6, 0, 9, 1106, 2, 2, 1, 0, 3, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[22, 2009, 0, 2936, 1642, 2994, 701, 0, 4024, 14, 283, 342, 0, 3, 249, 1971, 0, 34, 45, 14, 0, 1432, 2151, 0, 0, 63, 2768, 1110, 12, 168, 0, 0, 1028, 4259, 321, 101, 51, 1549, 39, 213, 128, 410, 5559, 330, 55, 54, 0, 752, 237, 681, 0, 2219, 7, 1146, 3, 747, 1, 1134, 630, 969, 0, 1542, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[71, 2088, 0, 94, 46, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 163, 486, 851, 0, 28, 814, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 28, 0, 3301, 831, 0, 234, 37, 39, 0, 0, 0, 0, 0, 0, 47, 3114, 563, 37, 384, 0, 0, 928, 0, 0, 0, 0, 0, 0, 0, 290, 355, 189, 216, 31, 4249, 86, 460, 0, 0, 0, 0, 0, 0, 0, 0, 550, 643, 3055, 297, 41, 98, 0, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 467, 422, 0, 1499, 3, 2647, 13, 1230, 338, 0, 0, 0, 0, 0, 0, 0, 0, 1355, 1324, 2377, 0, 3297, 0, 0, 817, 939, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3815, 6264, 4562, 5073, 5876, 6823, 6619, 10120] avg=6144.0000 max=10120 max_rank=7 min=3815 min_rank=0 imbalance_ratio=1.647135 max_load_reduction=39032 max_load_reduction_pct=0.794108 moved_to_echo=29438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[20, 2331, 1, 289, 458, 0, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 156, 620, 533, 0, 26, 434, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 30, 16, 0, 2229, 1335, 5, 722, 124, 73, 0, 0, 0, 0, 0, 0, 57, 2772, 1035, 18, 166, 1, 0, 1724, 0, 0, 0, 0, 0, 0, 0, 325, 1173, 242, 163, 216, 2266, 138, 447, 0, 0, 0, 0, 0, 0, 0, 0, 167, 550, 3642, 255, 14, 124, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 246, 615, 0, 2059, 4, 1667, 37, 1210, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 1529, 1154, 1533, 0, 2785, 0, 0, 974, 1208, 38, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 7335, 4554, 5773, 4970, 6290, 6957, 9221] avg=6144.0000 max=9221 max_rank=7 min=4052 min_rank=0 imbalance_ratio=1.500814 max_load_reduction=39931 max_load_reduction_pct=0.812398 moved_to_echo=27438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[34, 2308, 1, 400, 385, 0, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 265, 141, 521, 0, 68, 574, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 209, 60, 0, 2481, 1456, 0, 1000, 109, 83, 0, 0, 0, 0, 0, 0, 64, 2212, 756, 4, 132, 0, 0, 1949, 0, 0, 0, 0, 0, 0, 0, 308, 1023, 215, 317, 188, 2712, 123, 527, 0, 0, 0, 0, 0, 0, 0, 0, 405, 328, 3866, 269, 45, 63, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 65, 250, 0, 1890, 1, 1330, 1, 1827, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 1126, 1742, 0, 2411, 0, 0, 1035, 1196, 33, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 6438, 5414, 5117, 5413, 7385, 6341, 8852] avg=6144.0000 max=8852 max_rank=7 min=4192 min_rank=0 imbalance_ratio=1.440755 max_load_reduction=40300 max_load_reduction_pct=0.819906 moved_to_echo=27991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[71, 2088, 0, 94, 46, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 163, 486, 851, 0, 28, 814, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 28, 0, 3301, 831, 0, 234, 37, 39, 0, 0, 0, 0, 0, 0, 47, 3114, 563, 37, 384, 0, 0, 928, 0, 0, 0, 0, 0, 0, 0, 290, 355, 189, 216, 31, 4249, 86, 460, 0, 0, 0, 0, 0, 0, 0, 0, 550, 643, 3055, 297, 41, 98, 0, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 467, 422, 0, 1499, 3, 2647, 13, 1230, 338, 0, 0, 0, 0, 0, 0, 0, 0, 1355, 1324, 2377, 0, 3297, 0, 0, 817, 939, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3815, 6264, 4562, 5073, 5876, 6823, 6619, 10120] avg=6144.0000 max=10120 max_rank=7 min=3815 min_rank=0 imbalance_ratio=1.647135 max_load_reduction=39032 max_load_reduction_pct=0.794108 moved_to_echo=29438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[8, 1647, 0, 471, 677, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5032, 26, 35, 291, 0, 82, 529, 2471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 176, 69, 0, 1480, 1852, 0, 1177, 160, 75, 0, 0, 0, 0, 0, 0, 83, 1414, 371, 83, 276, 151, 0, 1732, 0, 0, 0, 0, 0, 0, 0, 242, 1508, 816, 136, 272, 2100, 432, 371, 0, 0, 0, 0, 0, 0, 0, 0, 432, 966, 4078, 400, 18, 199, 0, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 15, 473, 0, 2217, 2, 976, 50, 1244, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 962, 872, 7, 1477, 0, 1, 1092, 1488, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3107, 8466, 5024, 4110, 5877, 9036, 6417, 7115] avg=6144.0000 max=9036 max_rank=5 min=3107 min_rank=0 imbalance_ratio=1.470703 max_load_reduction=40116 max_load_reduction_pct=0.816162 moved_to_echo=28445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[20, 219, 0, 2154, 7688, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 17, 0, 14, 11, 4144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 28, 1, 18, 10, 2622, 0, 5394, 402, 298, 0, 0, 0, 0, 0, 3, 83, 150, 318, 0, 0, 18, 0, 6976, 0, 0, 0, 0, 0, 0, 0, 3047, 3775, 165, 2, 1, 20, 61, 15, 0, 0, 0, 0, 0, 0, 0, 0, 23, 95, 3548, 390, 28, 14, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 13, 692, 0, 694, 0, 12, 6, 83, 3609, 0, 0, 0, 0, 0, 0, 0, 179, 73, 71, 4, 1, 7, 0, 0, 949, 152, 125, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10092, 4703, 8808, 7548, 7086, 4245, 5109, 1561] avg=6144.0000 max=10092 max_rank=0 min=1561 min_rank=7 imbalance_ratio=1.642578 max_load_reduction=39060 max_load_reduction_pct=0.794678 moved_to_echo=18001 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 2256, 8167, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 6, 0, 0, 3, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 9, 1703, 0, 5650, 416, 315, 0, 0, 0, 0, 0, 0, 95, 46, 350, 0, 0, 0, 0, 7348, 0, 0, 0, 0, 0, 0, 0, 4896, 3902, 57, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 3603, 254, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 892, 0, 139, 0, 6, 0, 9, 3730, 0, 0, 0, 0, 0, 0, 0, 1106, 2, 2, 1, 0, 3, 0, 0, 964, 6, 129, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10426, 3019, 8138, 7839, 8861, 3879, 4777, 2213] avg=6144.0000 max=10426 max_rank=0 min=2213 min_rank=7 imbalance_ratio=1.696940 max_load_reduction=38726 max_load_reduction_pct=0.787882 moved_to_echo=19730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[22, 2009, 0, 838, 1642, 0, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3106, 14, 283, 342, 0, 3, 249, 1971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 45, 14, 0, 1432, 2151, 0, 2098, 216, 123, 0, 0, 0, 0, 0, 0, 63, 2768, 1110, 12, 168, 0, 0, 2871, 0, 0, 0, 0, 0, 0, 0, 1028, 2033, 321, 101, 51, 1549, 39, 213, 0, 0, 0, 0, 0, 0, 0, 0, 128, 410, 4385, 330, 55, 54, 0, 752, 0, 0, 0, 0, 0, 0, 0, 0, 237, 681, 0, 2219, 7, 1146, 3, 747, 1943, 0, 0, 0, 0, 0, 0, 0, 1, 1134, 630, 969, 0, 1542, 0, 0, 1174, 918, 67, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5212, 5968, 6113, 6992, 5335, 6114, 6983, 6435] avg=6144.0000 max=6992 max_rank=3 min=5212 min_rank=0 imbalance_ratio=1.138021 max_load_reduction=42160 max_load_reduction_pct=0.857747 moved_to_echo=24867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[3, 749, 0, 1672, 5655, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3163, 9, 78, 45, 0, 29, 71, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 25, 0, 0, 383, 1281, 0, 4206, 359, 207, 0, 0, 0, 0, 0, 0, 85, 610, 1076, 5, 103, 6, 0, 4815, 0, 0, 0, 0, 0, 0, 0, 1159, 3309, 601, 67, 8, 495, 55, 40, 0, 0, 0, 0, 0, 0, 0, 0, 45, 36, 4337, 384, 2, 5, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 6, 636, 0, 1616, 0, 178, 1, 332, 3180, 0, 0, 0, 0, 0, 0, 0, 0, 427, 431, 137, 0, 420, 0, 0, 1166, 940, 114, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 6959, 6539, 6700, 5734, 5380, 5949, 3635] avg=6144.0000 max=8256 max_rank=0 min=3635 min_rank=7 imbalance_ratio=1.343750 max_load_reduction=40896 max_load_reduction_pct=0.832031 moved_to_echo=20698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1408, 7040, 5760, 7680, 128, 12032, 0, 128, 8192, 6912, 640, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 3072, 30592, 2688, 256, 640, 0, 10624, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11392, 17152, 2688, 1024, 768, 13440, 1024, 2176, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 11392, 128, 8192, 24832, 0, 4736, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 384, 640, 256, 128, 11392, 13312, 128, 20608, 1920, 1280, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[23168, 640, 1664, 2688, 0, 256, 2688, 18304, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11392, 17152, 2688, 1024, 768, 13440, 1024, 2176, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 4736, 0, 12416, 128, 8064, 128, 6784, 16384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 3072, 30592, 2688, 256, 640, 0, 10624, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[23168, 640, 1664, 2688, 0, 256, 2688, 18304, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 640, 13184, 5632, 256, 1280, 256, 0, 28416, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True - [2026-08-26 15:39:19] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 811.2 | throughput per GPU (TFLOP/s/GPU): 157.3 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.114793E+01 | loss scale: 1.0 | grad norm: 400.648 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2197, 695, 293, 958, 133, 1232, 807, 123, 3117, 688, 662, 322, 4, 2673, 631, 706, 523, 789, 597, 690, 167, 304, 172, 976, 508, 689, 508, 499, 869, 412, 711, 884, 898, 490, 1067, 1530, 988, 385, 975, 584, 185, 549, 484, 152, 826, 1432, 401, 79, 177, 406, 498, 1026, 736, 2426, 294, 396, 1469, 537, 1720, 577, 146, 604, 1617, 959] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 + [2026-08-26 16:35:22] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 815.5 | throughput per GPU (TFLOP/s/GPU): 156.5 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.114793E+01 | loss scale: 1.0 | grad norm: 400.648 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2158, 577, 209, 1134, 117, 1063, 776, 87, 3122, 615, 866, 351, 0, 2621, 725, 556, 576, 809, 672, 747, 137, 228, 169, 1026, 573, 616, 460, 580, 858, 464, 854, 962, 903, 562, 1224, 1425, 1079, 306, 915, 553, 155, 475, 443, 158, 838, 1405, 425, 64, 135, 352, 534, 1111, 781, 2358, 300, 387, 1306, 469, 1648, 592, 141, 697, 1706, 997] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2055, 671, 271, 1067, 135, 1149, 797, 112, 3030, 562, 775, 380, 4, 2697, 705, 567, 560, 812, 689, 769, 147, 221, 164, 841, 546, 707, 455, 403, 889, 435, 746, 1046, 921, 485, 1114, 1524, 916, 329, 914, 536, 202, 504, 535, 175, 876, 1413, 423, 63, 177, 439, 484, 1207, 790, 2204, 252, 450, 1428, 445, 1802, 678, 94, 727, 1717, 921] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2205, 690, 254, 944, 159, 1044, 876, 118, 3068, 558, 735, 412, 3, 2466, 663, 704, 528, 797, 657, 736, 148, 253, 178, 925, 531, 692, 395, 428, 926, 420, 834, 1075, 858, 546, 1230, 1506, 1038, 345, 987, 653, 173, 535, 415, 161, 805, 1327, 386, 74, 204, 434, 469, 1179, 836, 2062, 289, 433, 1425, 527, 1628, 572, 151, 699, 1737, 1046] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2141, 627, 257, 1006, 153, 1191, 836, 102, 2937, 612, 848, 312, 12, 2574, 686, 570, 583, 834, 663, 727, 174, 252, 174, 925, 664, 745, 449, 519, 897, 468, 724, 1005, 998, 517, 1292, 1401, 1018, 435, 980, 554, 157, 472, 398, 152, 786, 1398, 432, 103, 176, 406, 513, 1045, 729, 2201, 293, 410, 1346, 527, 1637, 652, 159, 698, 1651, 949] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[2060, 633, 245, 1100, 122, 1131, 781, 102, 2964, 639, 726, 417, 2, 2479, 634, 718, 599, 909, 608, 710, 144, 260, 180, 988, 604, 520, 468, 402, 879, 547, 821, 1066, 912, 585, 1333, 1483, 1015, 410, 1089, 535, 196, 539, 459, 173, 823, 1404, 421, 91, 137, 378, 481, 1072, 754, 2101, 254, 486, 1363, 565, 1617, 636, 126, 630, 1624, 1002] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2197, 695, 293, 958, 133, 1232, 807, 123, 3117, 688, 662, 322, 4, 2673, 631, 706, 523, 789, 597, 690, 167, 304, 172, 976, 508, 689, 508, 499, 869, 412, 711, 884, 898, 490, 1067, 1530, 988, 385, 975, 584, 185, 549, 484, 152, 826, 1432, 401, 79, 177, 406, 498, 1026, 736, 2426, 294, 396, 1469, 537, 1720, 577, 146, 604, 1617, 959] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2245, 645, 258, 1045, 133, 1154, 876, 129, 3171, 479, 690, 413, 11, 2677, 634, 610, 537, 889, 599, 662, 159, 250, 164, 889, 591, 649, 458, 462, 915, 499, 799, 936, 890, 545, 1150, 1529, 1054, 387, 944, 609, 173, 532, 471, 155, 791, 1456, 423, 74, 188, 412, 524, 1092, 802, 1893, 276, 404, 1338, 494, 1710, 636, 103, 704, 1683, 1082] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2158, 577, 209, 1134, 117, 1063, 776, 87, 3122, 615, 866, 351, 0, 2621, 725, 556, 576, 809, 672, 747, 137, 228, 169, 1026, 573, 616, 460, 580, 858, 464, 854, 962, 903, 562, 1224, 1425, 1079, 306, 915, 553, 155, 475, 443, 158, 838, 1405, 425, 64, 135, 352, 534, 1111, 781, 2358, 300, 387, 1306, 469, 1648, 592, 141, 697, 1706, 997] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2165, 634, 227, 1148, 114, 1088, 820, 89, 3232, 497, 736, 375, 5, 2646, 716, 651, 480, 845, 582, 689, 117, 355, 131, 906, 572, 486, 408, 461, 859, 384, 705, 976, 872, 525, 1212, 1498, 1062, 304, 1036, 563, 179, 503, 443, 137, 870, 1431, 386, 86, 162, 364, 534, 1161, 851, 2397, 267, 355, 1348, 548, 1643, 698, 149, 737, 1703, 1029] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2205, 690, 254, 944, 159, 1044, 876, 118, 3068, 558, 735, 412, 3, 2466, 663, 704, 528, 797, 657, 736, 148, 253, 178, 925, 531, 692, 395, 428, 926, 420, 834, 1075, 858, 546, 1230, 1506, 1038, 345, 987, 653, 173, 535, 415, 161, 805, 1327, 386, 74, 204, 434, 469, 1179, 836, 2062, 289, 433, 1425, 527, 1628, 572, 151, 699, 1737, 1046] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2043, 695, 293, 958, 133, 1232, 807, 123, 0, 0, 0, 0, 0, 0, 0, 0, 524, 688, 662, 322, 4, 2673, 631, 706, 0, 0, 0, 0, 0, 0, 0, 0, 523, 789, 597, 690, 167, 304, 172, 976, 1390, 456, 0, 0, 0, 0, 0, 0, 508, 689, 508, 499, 869, 412, 711, 884, 819, 104, 0, 0, 0, 0, 0, 0, 898, 490, 1067, 583, 988, 385, 975, 584, 0, 0, 0, 0, 0, 0, 0, 0, 185, 549, 484, 152, 826, 1432, 401, 79, 2137, 0, 0, 0, 0, 0, 0, 0, 177, 406, 498, 1026, 736, 2426, 294, 396, 154, 128, 0, 0, 0, 0, 0, 0, 1469, 537, 1720, 577, 146, 604, 123, 959, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6210, 6064, 6003, 5970, 6245, 6241, 6135] avg=6144.0000 max=6284 max_rank=0 min=5970 min_rank=4 imbalance_ratio=1.022786 max_load_reduction=42868 max_load_reduction_pct=0.872152 moved_to_echo=24591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[2060, 633, 245, 1100, 122, 1131, 781, 102, 2964, 639, 726, 417, 2, 2479, 634, 718, 599, 909, 608, 710, 144, 260, 180, 988, 604, 520, 468, 402, 879, 547, 821, 1066, 912, 585, 1333, 1483, 1015, 410, 1089, 535, 196, 539, 459, 173, 823, 1404, 421, 91, 137, 378, 481, 1072, 754, 2101, 254, 486, 1363, 565, 1617, 636, 126, 630, 1624, 1002] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2007, 577, 209, 1134, 117, 1063, 776, 87, 0, 0, 0, 0, 0, 0, 0, 0, 524, 615, 866, 351, 0, 2621, 725, 556, 0, 0, 0, 0, 0, 0, 0, 0, 576, 809, 672, 747, 137, 228, 169, 1026, 1466, 457, 0, 0, 0, 0, 0, 0, 573, 616, 460, 580, 858, 464, 854, 962, 763, 109, 0, 0, 0, 0, 0, 0, 903, 562, 1224, 543, 1079, 306, 915, 553, 0, 0, 0, 0, 0, 0, 0, 0, 155, 475, 443, 158, 838, 1405, 425, 64, 2141, 0, 0, 0, 0, 0, 0, 0, 135, 352, 534, 1111, 781, 2358, 300, 387, 151, 119, 0, 0, 0, 0, 0, 0, 1306, 469, 1648, 592, 141, 697, 131, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5970, 6258, 6287, 6239, 6085, 6104, 6228, 5981] avg=6144.0000 max=6287 max_rank=2 min=5970 min_rank=0 imbalance_ratio=1.023275 max_load_reduction=42865 max_load_reduction_pct=0.872091 moved_to_echo=24398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1987, 627, 257, 1006, 153, 1191, 836, 102, 0, 0, 0, 0, 0, 0, 0, 0, 486, 612, 848, 312, 12, 2574, 686, 570, 0, 0, 0, 0, 0, 0, 0, 0, 583, 834, 663, 727, 174, 252, 174, 925, 1422, 433, 0, 0, 0, 0, 0, 0, 664, 745, 449, 519, 897, 468, 724, 1005, 755, 110, 0, 0, 0, 0, 0, 0, 998, 517, 1292, 526, 1018, 435, 980, 554, 0, 0, 0, 0, 0, 0, 0, 0, 157, 472, 398, 152, 786, 1398, 432, 103, 2018, 0, 0, 0, 0, 0, 0, 0, 176, 406, 513, 1045, 729, 2201, 293, 410, 154, 120, 0, 0, 0, 0, 0, 0, 1346, 527, 1637, 652, 159, 698, 119, 949, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 6100, 6187, 6336, 6320, 5916, 6047, 6087] avg=6144.0000 max=6336 max_rank=3 min=5916 min_rank=5 imbalance_ratio=1.031250 max_load_reduction=42816 max_load_reduction_pct=0.871094 moved_to_echo=24370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1916, 633, 245, 1100, 122, 1131, 781, 102, 0, 0, 0, 0, 0, 0, 0, 0, 498, 639, 726, 417, 2, 2479, 634, 718, 0, 0, 0, 0, 0, 0, 0, 0, 599, 909, 608, 710, 144, 260, 180, 988, 1396, 434, 0, 0, 0, 0, 0, 0, 604, 520, 468, 402, 879, 547, 821, 1066, 794, 104, 0, 0, 0, 0, 0, 0, 912, 585, 1333, 565, 1015, 410, 1089, 535, 0, 0, 0, 0, 0, 0, 0, 0, 196, 539, 459, 173, 823, 1404, 421, 91, 2032, 0, 0, 0, 0, 0, 0, 0, 137, 378, 481, 1072, 754, 2101, 254, 486, 144, 124, 0, 0, 0, 0, 0, 0, 1363, 565, 1617, 636, 126, 630, 124, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6030, 6113, 6228, 6205, 6444, 6138, 5931, 6063] avg=6144.0000 max=6444 max_rank=4 min=5931 min_rank=6 imbalance_ratio=1.048828 max_load_reduction=42708 max_load_reduction_pct=0.868896 moved_to_echo=24576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2050, 690, 254, 944, 159, 1044, 876, 118, 0, 0, 0, 0, 0, 0, 0, 0, 515, 558, 735, 412, 3, 2466, 663, 704, 0, 0, 0, 0, 0, 0, 0, 0, 528, 797, 657, 736, 148, 253, 178, 925, 1493, 449, 0, 0, 0, 0, 0, 0, 531, 692, 395, 428, 926, 420, 834, 1075, 806, 111, 0, 0, 0, 0, 0, 0, 858, 546, 1230, 574, 1038, 345, 987, 653, 0, 0, 0, 0, 0, 0, 0, 0, 173, 535, 415, 161, 805, 1327, 386, 74, 2104, 0, 0, 0, 0, 0, 0, 0, 204, 434, 469, 1179, 836, 2062, 289, 433, 155, 126, 0, 0, 0, 0, 0, 0, 1425, 527, 1628, 572, 151, 699, 133, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6135, 6056, 6164, 6218, 6231, 5980, 6187, 6181] avg=6144.0000 max=6231 max_rank=4 min=5980 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=42921 max_load_reduction_pct=0.873230 moved_to_echo=24579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1911, 671, 271, 1067, 135, 1149, 797, 112, 0, 0, 0, 0, 0, 0, 0, 0, 509, 562, 775, 380, 4, 2697, 705, 567, 0, 0, 0, 0, 0, 0, 0, 0, 560, 812, 689, 769, 147, 221, 164, 841, 1476, 443, 0, 0, 0, 0, 0, 0, 546, 707, 455, 403, 889, 435, 746, 1046, 816, 110, 0, 0, 0, 0, 0, 0, 921, 485, 1114, 581, 916, 329, 914, 536, 0, 0, 0, 0, 0, 0, 0, 0, 202, 504, 535, 175, 876, 1413, 423, 63, 2078, 0, 0, 0, 0, 0, 0, 0, 177, 439, 484, 1207, 790, 2204, 252, 450, 144, 127, 0, 0, 0, 0, 0, 0, 1428, 445, 1802, 678, 94, 727, 131, 921, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6199, 6122, 6153, 5796, 6269, 6274, 6226] avg=6144.0000 max=6274 max_rank=6 min=5796 min_rank=4 imbalance_ratio=1.021159 max_load_reduction=42878 max_load_reduction_pct=0.872355 moved_to_echo=24565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1987, 627, 257, 1006, 153, 1191, 836, 102, 0, 0, 0, 0, 0, 0, 0, 0, 486, 612, 848, 312, 12, 2574, 686, 570, 0, 0, 0, 0, 0, 0, 0, 0, 583, 834, 663, 727, 174, 252, 174, 925, 1422, 433, 0, 0, 0, 0, 0, 0, 664, 745, 449, 519, 897, 468, 724, 1005, 755, 110, 0, 0, 0, 0, 0, 0, 998, 517, 1292, 526, 1018, 435, 980, 554, 0, 0, 0, 0, 0, 0, 0, 0, 157, 472, 398, 152, 786, 1398, 432, 103, 2018, 0, 0, 0, 0, 0, 0, 0, 176, 406, 513, 1045, 729, 2201, 293, 410, 154, 120, 0, 0, 0, 0, 0, 0, 1346, 527, 1637, 652, 159, 698, 119, 949, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 6100, 6187, 6336, 6320, 5916, 6047, 6087] avg=6144.0000 max=6336 max_rank=3 min=5916 min_rank=5 imbalance_ratio=1.031250 max_load_reduction=42816 max_load_reduction_pct=0.871094 moved_to_echo=24370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2013, 634, 227, 1148, 114, 1088, 820, 89, 0, 0, 0, 0, 0, 0, 0, 0, 543, 497, 736, 375, 5, 2646, 716, 651, 0, 0, 0, 0, 0, 0, 0, 0, 480, 845, 582, 689, 117, 355, 131, 906, 1463, 473, 0, 0, 0, 0, 0, 0, 572, 486, 408, 461, 859, 384, 705, 976, 802, 109, 0, 0, 0, 0, 0, 0, 872, 525, 1212, 571, 1062, 304, 1036, 563, 0, 0, 0, 0, 0, 0, 0, 0, 179, 503, 443, 137, 870, 1431, 386, 86, 2216, 0, 0, 0, 0, 0, 0, 0, 162, 364, 534, 1161, 851, 2397, 267, 355, 152, 125, 0, 0, 0, 0, 0, 0, 1348, 548, 1643, 698, 149, 737, 131, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 6169, 6041, 5762, 6145, 6251, 6368, 6283] avg=6144.0000 max=6368 max_rank=6 min=5762 min_rank=3 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=25047 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2043, 695, 293, 958, 133, 1232, 807, 123, 0, 0, 0, 0, 0, 0, 0, 0, 524, 688, 662, 322, 4, 2673, 631, 706, 0, 0, 0, 0, 0, 0, 0, 0, 523, 789, 597, 690, 167, 304, 172, 976, 1390, 456, 0, 0, 0, 0, 0, 0, 508, 689, 508, 499, 869, 412, 711, 884, 819, 104, 0, 0, 0, 0, 0, 0, 898, 490, 1067, 583, 988, 385, 975, 584, 0, 0, 0, 0, 0, 0, 0, 0, 185, 549, 484, 152, 826, 1432, 401, 79, 2137, 0, 0, 0, 0, 0, 0, 0, 177, 406, 498, 1026, 736, 2426, 294, 396, 154, 128, 0, 0, 0, 0, 0, 0, 1469, 537, 1720, 577, 146, 604, 123, 959, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6210, 6064, 6003, 5970, 6245, 6241, 6135] avg=6144.0000 max=6284 max_rank=0 min=5970 min_rank=4 imbalance_ratio=1.022786 max_load_reduction=42868 max_load_reduction_pct=0.872152 moved_to_echo=24591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2088, 645, 258, 1045, 133, 1154, 876, 129, 0, 0, 0, 0, 0, 0, 0, 0, 533, 479, 690, 413, 11, 2677, 634, 610, 0, 0, 0, 0, 0, 0, 0, 0, 537, 889, 599, 662, 159, 250, 164, 889, 1446, 464, 0, 0, 0, 0, 0, 0, 591, 649, 458, 462, 915, 499, 799, 936, 819, 108, 0, 0, 0, 0, 0, 0, 890, 545, 1150, 582, 1054, 387, 944, 609, 0, 0, 0, 0, 0, 0, 0, 0, 173, 532, 471, 155, 791, 1456, 423, 74, 2174, 0, 0, 0, 0, 0, 0, 0, 188, 412, 524, 1092, 802, 1893, 276, 404, 157, 128, 0, 0, 0, 0, 0, 0, 1338, 494, 1710, 636, 103, 704, 129, 1082, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 6047, 6059, 6236, 6161, 6249, 5876, 6196] avg=6144.0000 max=6328 max_rank=0 min=5876 min_rank=6 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2050, 690, 254, 944, 159, 1044, 876, 118, 0, 0, 0, 0, 0, 0, 0, 0, 515, 558, 735, 412, 3, 2466, 663, 704, 0, 0, 0, 0, 0, 0, 0, 0, 528, 797, 657, 736, 148, 253, 178, 925, 1493, 449, 0, 0, 0, 0, 0, 0, 531, 692, 395, 428, 926, 420, 834, 1075, 806, 111, 0, 0, 0, 0, 0, 0, 858, 546, 1230, 574, 1038, 345, 987, 653, 0, 0, 0, 0, 0, 0, 0, 0, 173, 535, 415, 161, 805, 1327, 386, 74, 2104, 0, 0, 0, 0, 0, 0, 0, 204, 434, 469, 1179, 836, 2062, 289, 433, 155, 126, 0, 0, 0, 0, 0, 0, 1425, 527, 1628, 572, 151, 699, 133, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6135, 6056, 6164, 6218, 6231, 5980, 6187, 6181] avg=6144.0000 max=6231 max_rank=4 min=5980 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=42921 max_load_reduction_pct=0.873230 moved_to_echo=24579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1916, 633, 245, 1100, 122, 1131, 781, 102, 0, 0, 0, 0, 0, 0, 0, 0, 498, 639, 726, 417, 2, 2479, 634, 718, 0, 0, 0, 0, 0, 0, 0, 0, 599, 909, 608, 710, 144, 260, 180, 988, 1396, 434, 0, 0, 0, 0, 0, 0, 604, 520, 468, 402, 879, 547, 821, 1066, 794, 104, 0, 0, 0, 0, 0, 0, 912, 585, 1333, 565, 1015, 410, 1089, 535, 0, 0, 0, 0, 0, 0, 0, 0, 196, 539, 459, 173, 823, 1404, 421, 91, 2032, 0, 0, 0, 0, 0, 0, 0, 137, 378, 481, 1072, 754, 2101, 254, 486, 144, 124, 0, 0, 0, 0, 0, 0, 1363, 565, 1617, 636, 126, 630, 124, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6030, 6113, 6228, 6205, 6444, 6138, 5931, 6063] avg=6144.0000 max=6444 max_rank=4 min=5931 min_rank=6 imbalance_ratio=1.048828 max_load_reduction=42708 max_load_reduction_pct=0.868896 moved_to_echo=24576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16128, 5248, 2048, 8448, 1152, 9088, 6656, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 4224, 3712, 1280, 6656, 11392, 3328, 640, 17024, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 6784, 5120, 5760, 1280, 2176, 1408, 7552, 11648, 3712, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11136, 4224, 13440, 5120, 1152, 5504, 1024, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 5120, 3712, 3840, 7168, 3712, 6272, 8064, 6400, 896, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 4736, 6144, 3072, 128, 20864, 5504, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11136, 4224, 13440, 5120, 1152, 5504, 1024, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 4224, 3712, 1280, 6656, 11392, 3328, 640, 17024, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7296, 4352, 9728, 4608, 8192, 2944, 7936, 4608, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 5120, 3712, 3840, 7168, 3712, 6272, 8064, 6400, 896, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1408, 3200, 4096, 8960, 6400, 17664, 2304, 3328, 1280, 1024, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[247, 497, 30, 140, 4462, 79, 1980, 353, 55, 666, 1478, 431, 686, 751, 56, 3006, 363, 1045, 537, 796, 363, 850, 482, 398, 396, 524, 671, 3869, 1, 145, 139, 684, 89, 1, 264, 19, 511, 0, 259, 103, 750, 455, 253, 475, 12, 659, 2, 260, 6623, 393, 3489, 2, 18, 225, 467, 968, 2698, 337, 986, 459, 421, 182, 1311, 281] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[204, 455, 41, 135, 4298, 85, 1911, 332, 57, 586, 1298, 357, 593, 857, 37, 3024, 321, 1032, 469, 512, 294, 1136, 496, 427, 470, 504, 809, 4205, 3, 138, 122, 593, 51, 3, 246, 8, 432, 1, 271, 106, 797, 494, 244, 423, 14, 588, 0, 285, 6949, 364, 3460, 0, 34, 205, 513, 1047, 2874, 378, 929, 407, 302, 149, 1526, 251] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[238, 459, 38, 160, 4600, 88, 2058, 371, 74, 664, 1282, 352, 638, 776, 49, 2828, 401, 967, 440, 646, 267, 894, 487, 451, 461, 446, 740, 4006, 0, 118, 125, 610, 86, 2, 240, 15, 450, 3, 258, 103, 842, 515, 240, 673, 19, 487, 3, 336, 6883, 345, 3305, 0, 23, 187, 594, 1089, 2631, 331, 926, 651, 351, 154, 1362, 314] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[219, 421, 58, 172, 4703, 51, 2104, 467, 65, 792, 1349, 354, 644, 922, 27, 2973, 490, 1020, 273, 614, 318, 897, 369, 385, 483, 514, 746, 4064, 1, 111, 99, 648, 94, 3, 249, 17, 419, 5, 223, 125, 789, 463, 246, 608, 22, 517, 1, 283, 6723, 351, 3236, 0, 33, 219, 481, 1011, 2666, 401, 937, 629, 339, 139, 1275, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[204, 455, 41, 135, 4298, 85, 1911, 332, 57, 586, 1298, 357, 593, 857, 37, 3024, 321, 1032, 469, 512, 294, 1136, 496, 427, 470, 504, 809, 4205, 3, 138, 122, 593, 51, 3, 246, 8, 432, 1, 271, 106, 797, 494, 244, 423, 14, 588, 0, 285, 6949, 364, 3460, 0, 34, 205, 513, 1047, 2874, 378, 929, 407, 302, 149, 1526, 251] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[247, 497, 30, 140, 4462, 79, 1980, 353, 55, 666, 1478, 431, 686, 751, 56, 3006, 363, 1045, 537, 796, 363, 850, 482, 398, 396, 524, 671, 3869, 1, 145, 139, 684, 89, 1, 264, 19, 511, 0, 259, 103, 750, 455, 253, 475, 12, 659, 2, 260, 6623, 393, 3489, 2, 18, 225, 467, 968, 2698, 337, 986, 459, 421, 182, 1311, 281] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[187, 512, 41, 138, 4588, 80, 1855, 475, 77, 611, 1175, 402, 573, 935, 55, 2862, 397, 1005, 475, 492, 324, 1000, 366, 463, 410, 559, 758, 4072, 6, 136, 96, 538, 114, 0, 260, 9, 381, 1, 262, 111, 795, 527, 257, 624, 28, 584, 2, 307, 6837, 338, 3860, 0, 17, 199, 514, 980, 2737, 110, 840, 620, 316, 160, 1377, 322] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[219, 421, 58, 172, 4703, 51, 2104, 467, 65, 792, 1349, 354, 644, 922, 27, 2973, 490, 1020, 273, 614, 318, 897, 369, 385, 483, 514, 746, 4064, 1, 111, 99, 648, 94, 3, 249, 17, 419, 5, 223, 125, 789, 463, 246, 608, 22, 517, 1, 283, 6723, 351, 3236, 0, 33, 219, 481, 1011, 2666, 401, 937, 629, 339, 139, 1275, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[227, 455, 44, 152, 4652, 53, 2124, 440, 32, 582, 1270, 394, 635, 821, 40, 2998, 398, 1105, 328, 550, 263, 954, 449, 393, 425, 520, 645, 3998, 1, 132, 101, 624, 45, 2, 251, 4, 469, 6, 286, 109, 771, 514, 248, 541, 15, 507, 1, 237, 6897, 305, 3832, 0, 20, 204, 478, 1025, 2940, 215, 971, 529, 270, 132, 1245, 278] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[235, 522, 39, 130, 4377, 54, 1826, 425, 72, 686, 1303, 348, 548, 932, 68, 2748, 456, 971, 473, 794, 354, 1063, 446, 461, 454, 503, 758, 3963, 2, 115, 103, 597, 103, 1, 260, 6, 507, 0, 276, 118, 760, 496, 257, 515, 15, 669, 5, 321, 6816, 375, 3236, 0, 31, 209, 505, 909, 2886, 318, 892, 491, 328, 220, 1506, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[172, 564, 25, 150, 4692, 50, 2213, 350, 71, 683, 1298, 389, 657, 970, 40, 2850, 368, 1072, 533, 430, 364, 860, 405, 411, 424, 601, 657, 3886, 4, 140, 106, 741, 118, 1, 253, 16, 488, 4, 260, 99, 782, 524, 252, 636, 14, 538, 5, 234, 6868, 331, 3663, 0, 17, 196, 483, 988, 2541, 202, 845, 608, 321, 202, 1192, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[227, 455, 44, 152, 4652, 53, 2124, 440, 32, 582, 1270, 394, 635, 821, 40, 2998, 398, 1105, 328, 550, 263, 954, 449, 393, 425, 520, 645, 3998, 1, 132, 101, 624, 45, 2, 251, 4, 469, 6, 286, 109, 771, 514, 248, 541, 15, 507, 1, 237, 6897, 305, 3832, 0, 20, 204, 478, 1025, 2940, 215, 971, 529, 270, 132, 1245, 278] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[238, 459, 38, 160, 2811, 88, 2058, 371, 0, 0, 0, 0, 0, 0, 0, 0, 74, 664, 1282, 352, 638, 776, 49, 2137, 0, 0, 0, 0, 0, 0, 0, 0, 401, 967, 440, 646, 267, 894, 487, 451, 604, 472, 420, 0, 0, 0, 0, 0, 461, 446, 740, 3586, 0, 118, 125, 610, 0, 0, 0, 0, 0, 0, 0, 0, 86, 2, 240, 15, 450, 3, 258, 103, 5002, 0, 0, 0, 0, 0, 0, 0, 842, 515, 240, 673, 19, 487, 3, 336, 1789, 1329, 87, 0, 0, 0, 0, 0, 552, 345, 3305, 0, 23, 187, 594, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 2159, 331, 926, 651, 351, 154, 1362, 314, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6223, 5972, 6049, 6086, 6159, 6320, 6095, 6248] avg=6144.0000 max=6320 max_rank=5 min=5972 min_rank=1 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[187, 512, 41, 138, 2803, 80, 1855, 475, 0, 0, 0, 0, 0, 0, 0, 0, 77, 611, 1175, 402, 573, 935, 55, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 397, 1005, 475, 492, 324, 1000, 366, 463, 611, 491, 427, 0, 0, 0, 0, 0, 410, 559, 758, 3645, 6, 136, 96, 538, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 260, 9, 381, 1, 262, 111, 4968, 0, 0, 0, 0, 0, 0, 0, 795, 527, 257, 624, 28, 584, 2, 307, 1785, 1320, 88, 0, 0, 0, 0, 0, 549, 338, 3860, 0, 17, 199, 514, 980, 0, 0, 0, 0, 0, 0, 0, 0, 2246, 110, 840, 620, 316, 160, 1377, 322, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 5991, 6051, 6148, 6106, 6317, 6457, 5991] avg=6144.0000 max=6457 max_rank=6 min=5991 min_rank=1 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[247, 497, 30, 140, 2727, 79, 1980, 353, 0, 0, 0, 0, 0, 0, 0, 0, 55, 666, 1478, 431, 686, 751, 56, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 363, 1045, 537, 796, 363, 850, 482, 398, 642, 484, 406, 0, 0, 0, 0, 0, 396, 524, 671, 3463, 1, 145, 139, 684, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1, 264, 19, 511, 0, 259, 103, 4813, 0, 0, 0, 0, 0, 0, 0, 750, 455, 253, 475, 12, 659, 2, 260, 1735, 1278, 93, 0, 0, 0, 0, 0, 532, 393, 3489, 2, 18, 225, 467, 968, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 337, 986, 459, 421, 182, 1311, 281, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6053, 6394, 6366, 6023, 6059, 5972, 6094, 6191] avg=6144.0000 max=6394 max_rank=1 min=5972 min_rank=5 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[204, 455, 41, 135, 2626, 85, 1911, 332, 0, 0, 0, 0, 0, 0, 0, 0, 57, 586, 1298, 357, 593, 857, 37, 2285, 0, 0, 0, 0, 0, 0, 0, 0, 321, 1032, 469, 512, 294, 1136, 496, 427, 646, 515, 441, 0, 0, 0, 0, 0, 470, 504, 809, 3764, 3, 138, 122, 593, 0, 0, 0, 0, 0, 0, 0, 0, 51, 3, 246, 8, 432, 1, 271, 106, 5049, 0, 0, 0, 0, 0, 0, 0, 797, 494, 244, 423, 14, 588, 0, 285, 1672, 1341, 93, 0, 0, 0, 0, 0, 559, 364, 3460, 0, 34, 205, 513, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 2359, 378, 929, 407, 302, 149, 1526, 251, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5789, 6070, 6289, 6403, 6167, 5951, 6182, 6301] avg=6144.0000 max=6403 max_rank=3 min=5789 min_rank=0 imbalance_ratio=1.042155 max_load_reduction=42749 max_load_reduction_pct=0.869731 moved_to_echo=24601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[187, 512, 41, 138, 2803, 80, 1855, 475, 0, 0, 0, 0, 0, 0, 0, 0, 77, 611, 1175, 402, 573, 935, 55, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 397, 1005, 475, 492, 324, 1000, 366, 463, 611, 491, 427, 0, 0, 0, 0, 0, 410, 559, 758, 3645, 6, 136, 96, 538, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 260, 9, 381, 1, 262, 111, 4968, 0, 0, 0, 0, 0, 0, 0, 795, 527, 257, 624, 28, 584, 2, 307, 1785, 1320, 88, 0, 0, 0, 0, 0, 549, 338, 3860, 0, 17, 199, 514, 980, 0, 0, 0, 0, 0, 0, 0, 0, 2246, 110, 840, 620, 316, 160, 1377, 322, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 5991, 6051, 6148, 6106, 6317, 6457, 5991] avg=6144.0000 max=6457 max_rank=6 min=5991 min_rank=1 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[219, 421, 58, 172, 2874, 51, 2104, 467, 0, 0, 0, 0, 0, 0, 0, 0, 65, 792, 1349, 354, 644, 922, 27, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 490, 1020, 273, 614, 318, 897, 369, 385, 635, 478, 426, 0, 0, 0, 0, 0, 483, 514, 746, 3638, 1, 111, 99, 648, 0, 0, 0, 0, 0, 0, 0, 0, 94, 3, 249, 17, 419, 5, 223, 125, 4885, 0, 0, 0, 0, 0, 0, 0, 789, 463, 246, 608, 22, 517, 1, 283, 1829, 1298, 92, 0, 0, 0, 0, 0, 540, 351, 3236, 0, 33, 219, 481, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 401, 937, 629, 339, 139, 1275, 295, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6366, 6399, 5905, 6240, 6020, 6148, 5871, 6203] avg=6144.0000 max=6399 max_rank=1 min=5871 min_rank=6 imbalance_ratio=1.041504 max_load_reduction=42753 max_load_reduction_pct=0.869812 moved_to_echo=24242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[238, 459, 38, 160, 2811, 88, 2058, 371, 0, 0, 0, 0, 0, 0, 0, 0, 74, 664, 1282, 352, 638, 776, 49, 2137, 0, 0, 0, 0, 0, 0, 0, 0, 401, 967, 440, 646, 267, 894, 487, 451, 604, 472, 420, 0, 0, 0, 0, 0, 461, 446, 740, 3586, 0, 118, 125, 610, 0, 0, 0, 0, 0, 0, 0, 0, 86, 2, 240, 15, 450, 3, 258, 103, 5002, 0, 0, 0, 0, 0, 0, 0, 842, 515, 240, 673, 19, 487, 3, 336, 1789, 1329, 87, 0, 0, 0, 0, 0, 552, 345, 3305, 0, 23, 187, 594, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 2159, 331, 926, 651, 351, 154, 1362, 314, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6223, 5972, 6049, 6086, 6159, 6320, 6095, 6248] avg=6144.0000 max=6320 max_rank=5 min=5972 min_rank=1 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[227, 455, 44, 152, 2843, 53, 2124, 440, 0, 0, 0, 0, 0, 0, 0, 0, 32, 582, 1270, 394, 635, 821, 40, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 398, 1105, 328, 550, 263, 954, 449, 393, 640, 527, 419, 0, 0, 0, 0, 0, 425, 520, 645, 3579, 1, 132, 101, 624, 0, 0, 0, 0, 0, 0, 0, 0, 45, 2, 251, 4, 469, 6, 286, 109, 5012, 0, 0, 0, 0, 0, 0, 0, 771, 514, 248, 541, 15, 507, 1, 237, 1809, 1331, 93, 0, 0, 0, 0, 0, 554, 305, 3832, 0, 20, 204, 478, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 2413, 215, 971, 529, 270, 132, 1245, 278, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6338, 6039, 6026, 6027, 6184, 6067, 6418, 6053] avg=6144.0000 max=6418 max_rank=6 min=6026 min_rank=2 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[235, 522, 39, 130, 2670, 54, 1826, 425, 0, 0, 0, 0, 0, 0, 0, 0, 72, 686, 1303, 348, 548, 932, 68, 2069, 0, 0, 0, 0, 0, 0, 0, 0, 456, 971, 473, 794, 354, 1063, 446, 461, 590, 521, 421, 0, 0, 0, 0, 0, 454, 503, 758, 3542, 2, 115, 103, 597, 0, 0, 0, 0, 0, 0, 0, 0, 103, 1, 260, 6, 507, 0, 276, 118, 4956, 0, 0, 0, 0, 0, 0, 0, 760, 496, 257, 515, 15, 669, 5, 321, 1707, 1319, 89, 0, 0, 0, 0, 0, 541, 375, 3236, 0, 31, 209, 505, 909, 0, 0, 0, 0, 0, 0, 0, 0, 2365, 318, 892, 491, 328, 220, 1506, 295, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5901, 6026, 6550, 6074, 6227, 6153, 5806, 6415] avg=6144.0000 max=6550 max_rank=2 min=5806 min_rank=6 imbalance_ratio=1.066081 max_load_reduction=42602 max_load_reduction_pct=0.866740 moved_to_echo=24601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[219, 421, 58, 172, 2874, 51, 2104, 467, 0, 0, 0, 0, 0, 0, 0, 0, 65, 792, 1349, 354, 644, 922, 27, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 490, 1020, 273, 614, 318, 897, 369, 385, 635, 478, 426, 0, 0, 0, 0, 0, 483, 514, 746, 3638, 1, 111, 99, 648, 0, 0, 0, 0, 0, 0, 0, 0, 94, 3, 249, 17, 419, 5, 223, 125, 4885, 0, 0, 0, 0, 0, 0, 0, 789, 463, 246, 608, 22, 517, 1, 283, 1829, 1298, 92, 0, 0, 0, 0, 0, 540, 351, 3236, 0, 33, 219, 481, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 401, 937, 629, 339, 139, 1275, 295, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6366, 6399, 5905, 6240, 6020, 6148, 5871, 6203] avg=6144.0000 max=6399 max_rank=1 min=5871 min_rank=6 imbalance_ratio=1.041504 max_load_reduction=42753 max_load_reduction_pct=0.869812 moved_to_echo=24242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[172, 564, 25, 150, 2867, 50, 2213, 350, 0, 0, 0, 0, 0, 0, 0, 0, 71, 683, 1298, 389, 657, 970, 40, 2153, 0, 0, 0, 0, 0, 0, 0, 0, 368, 1072, 533, 430, 364, 860, 405, 411, 609, 456, 408, 0, 0, 0, 0, 0, 424, 601, 657, 3478, 4, 140, 106, 741, 0, 0, 0, 0, 0, 0, 0, 0, 118, 1, 253, 16, 488, 4, 260, 99, 4991, 0, 0, 0, 0, 0, 0, 0, 782, 524, 252, 636, 14, 538, 5, 234, 1825, 1326, 88, 0, 0, 0, 0, 0, 551, 331, 3663, 0, 17, 196, 483, 988, 0, 0, 0, 0, 0, 0, 0, 0, 2085, 202, 845, 608, 321, 202, 1192, 295, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6391, 6261, 5916, 6151, 6230, 6224, 6229, 5750] avg=6144.0000 max=6391 max_rank=0 min=5750 min_rank=7 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[235, 522, 39, 130, 2670, 54, 1826, 425, 0, 0, 0, 0, 0, 0, 0, 0, 72, 686, 1303, 348, 548, 932, 68, 2069, 0, 0, 0, 0, 0, 0, 0, 0, 456, 971, 473, 794, 354, 1063, 446, 461, 590, 521, 421, 0, 0, 0, 0, 0, 454, 503, 758, 3542, 2, 115, 103, 597, 0, 0, 0, 0, 0, 0, 0, 0, 103, 1, 260, 6, 507, 0, 276, 118, 4956, 0, 0, 0, 0, 0, 0, 0, 760, 496, 257, 515, 15, 669, 5, 321, 1707, 1319, 89, 0, 0, 0, 0, 0, 541, 375, 3236, 0, 31, 209, 505, 909, 0, 0, 0, 0, 0, 0, 0, 0, 2365, 318, 892, 491, 328, 220, 1506, 295, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5901, 6026, 6550, 6074, 6227, 6153, 5806, 6415] avg=6144.0000 max=6550 max_rank=2 min=5806 min_rank=6 imbalance_ratio=1.066081 max_load_reduction=42602 max_load_reduction_pct=0.866740 moved_to_echo=24601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18048, 2304, 7424, 4480, 2688, 1408, 10880, 2432, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 8320, 3584, 4864, 2560, 7680, 3584, 3456, 4992, 3968, 3456, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 3968, 384, 1280, 22272, 640, 16128, 3328, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 5376, 10496, 3072, 4992, 7040, 384, 17664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 3968, 384, 1280, 22272, 640, 16128, 3328, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18048, 2304, 7424, 4480, 2688, 1408, 10880, 2432, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[6400, 4096, 2048, 4608, 256, 4608, 128, 2304, 14208, 10624, 768, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 2816, 28160, 128, 256, 1664, 4096, 8064, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 4224, 5888, 28800, 128, 1152, 896, 5120, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 128, 2048, 128, 3712, 128, 2176, 896, 39680, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 105, 47, 163, 0, 126, 311, 1822, 6, 1644, 213, 1812, 439, 325, 0, 2, 4804, 912, 5422, 63, 0, 0, 5527, 659, 160, 544, 3123, 2943, 3, 102, 251, 1, 21, 536, 498, 153, 3819, 229, 108, 90, 1277, 0, 0, 20, 482, 1974, 97, 0, 811, 8, 166, 445, 2, 0, 176, 1, 4365, 184, 3, 4, 0, 1287, 0, 866] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 159, 68, 192, 1, 72, 360, 1943, 24, 1764, 250, 1552, 393, 342, 0, 0, 4583, 961, 5324, 70, 2, 0, 5460, 770, 176, 618, 3057, 2724, 2, 300, 267, 0, 56, 507, 545, 211, 3626, 218, 89, 62, 1436, 0, 3, 32, 541, 2093, 111, 1, 822, 6, 184, 526, 7, 0, 126, 2, 4093, 155, 17, 4, 1, 1285, 1, 957] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 131, 60, 136, 1, 105, 411, 1861, 21, 1720, 229, 1746, 456, 348, 0, 0, 4539, 855, 5526, 54, 0, 0, 5118, 666, 147, 600, 2925, 3219, 1, 173, 271, 0, 42, 613, 556, 171, 3704, 198, 98, 77, 1524, 0, 0, 36, 450, 2020, 109, 0, 861, 4, 143, 467, 10, 0, 167, 0, 4015, 156, 10, 17, 0, 1403, 0, 982] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 146, 72, 160, 0, 145, 424, 1825, 5, 1725, 238, 1572, 446, 343, 0, 0, 4575, 757, 5438, 64, 1, 0, 5526, 779, 152, 634, 2934, 2839, 2, 335, 259, 0, 29, 526, 596, 156, 3539, 199, 90, 80, 1684, 0, 2, 24, 419, 2083, 95, 0, 943, 4, 171, 372, 7, 0, 213, 0, 4045, 150, 10, 3, 0, 1336, 0, 980] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 201, 51, 146, 0, 131, 365, 1927, 24, 1599, 221, 1724, 465, 316, 0, 0, 4654, 948, 5216, 61, 2, 0, 5299, 679, 172, 621, 3146, 3070, 2, 407, 277, 0, 45, 482, 393, 172, 3690, 177, 86, 96, 1567, 0, 1, 17, 416, 1938, 106, 1, 895, 7, 160, 472, 3, 0, 202, 0, 3917, 143, 7, 14, 0, 1490, 0, 931] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 160, 62, 178, 1, 76, 413, 1733, 4, 1712, 197, 1737, 441, 352, 0, 2, 4534, 832, 5253, 52, 1, 1, 5640, 744, 150, 578, 3079, 3345, 3, 238, 290, 0, 38, 503, 339, 116, 4150, 172, 73, 93, 1478, 0, 1, 30, 428, 2049, 91, 0, 838, 3, 182, 351, 4, 0, 131, 0, 3836, 148, 13, 5, 1, 1315, 0, 956] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 135, 81, 154, 2, 67, 384, 2023, 23, 1684, 297, 1577, 439, 346, 0, 0, 4480, 1064, 5407, 66, 1, 0, 5445, 632, 177, 708, 2979, 2549, 3, 192, 266, 0, 48, 531, 597, 187, 3728, 212, 111, 82, 1374, 0, 0, 17, 443, 2118, 97, 1, 937, 7, 157, 454, 4, 0, 126, 0, 4081, 199, 20, 7, 0, 1472, 1, 960] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 172, 78, 156, 0, 123, 396, 1876, 14, 1632, 241, 1770, 442, 379, 1, 1, 4586, 889, 5359, 63, 1, 0, 5451, 752, 149, 620, 2808, 2781, 2, 349, 278, 0, 48, 462, 498, 286, 3645, 228, 78, 66, 1481, 0, 0, 21, 439, 2165, 116, 1, 1009, 10, 224, 397, 5, 0, 187, 0, 3913, 171, 10, 10, 0, 1388, 0, 925] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 131, 60, 136, 1, 105, 411, 1861, 21, 1720, 229, 1746, 456, 348, 0, 0, 4539, 855, 5526, 54, 0, 0, 5118, 666, 147, 600, 2925, 3219, 1, 173, 271, 0, 42, 613, 556, 171, 3704, 198, 98, 77, 1524, 0, 0, 36, 450, 2020, 109, 0, 861, 4, 143, 467, 10, 0, 167, 0, 4015, 156, 10, 17, 0, 1403, 0, 982] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 201, 51, 146, 0, 131, 365, 1927, 24, 1599, 221, 1724, 465, 316, 0, 0, 4654, 948, 5216, 61, 2, 0, 5299, 679, 172, 621, 3146, 3070, 2, 407, 277, 0, 45, 482, 393, 172, 3690, 177, 86, 96, 1567, 0, 1, 17, 416, 1938, 106, 1, 895, 7, 160, 472, 3, 0, 202, 0, 3917, 143, 7, 14, 0, 1490, 0, 931] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 159, 68, 192, 1, 72, 360, 1943, 24, 1764, 250, 1552, 393, 342, 0, 0, 4583, 961, 5324, 70, 2, 0, 5460, 770, 176, 618, 3057, 2724, 2, 300, 267, 0, 56, 507, 545, 211, 3626, 218, 89, 62, 1436, 0, 3, 32, 541, 2093, 111, 1, 822, 6, 184, 526, 7, 0, 126, 2, 4093, 155, 17, 4, 1, 1285, 1, 957] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 105, 47, 163, 0, 126, 311, 1822, 6, 1644, 213, 1812, 439, 325, 0, 2, 4804, 912, 5422, 63, 0, 0, 5527, 659, 160, 544, 3123, 2943, 3, 102, 251, 1, 21, 536, 498, 153, 3819, 229, 108, 90, 1277, 0, 0, 20, 482, 1974, 97, 0, 811, 8, 166, 445, 2, 0, 176, 1, 4365, 184, 3, 4, 0, 1287, 0, 866] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 135, 81, 154, 2, 67, 384, 2023, 23, 1684, 297, 1577, 439, 346, 0, 0, 4480, 1064, 5407, 66, 1, 0, 5445, 632, 177, 708, 2979, 2549, 3, 192, 266, 0, 48, 531, 597, 187, 3728, 212, 111, 82, 1374, 0, 0, 17, 443, 2118, 97, 1, 937, 7, 157, 454, 4, 0, 126, 0, 4081, 199, 20, 7, 0, 1472, 1, 960] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 131, 60, 136, 1, 105, 411, 1861, 2507, 909, 0, 0, 0, 0, 0, 0, 21, 1720, 229, 1746, 456, 348, 0, 0, 928, 813, 0, 0, 0, 0, 0, 0, 4416, 855, 0, 54, 0, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 147, 600, 1849, 3219, 1, 173, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 613, 556, 171, 3704, 198, 98, 77, 387, 263, 123, 0, 0, 0, 0, 0, 1524, 0, 0, 36, 450, 2020, 109, 0, 2091, 0, 0, 0, 0, 0, 0, 0, 861, 4, 143, 467, 10, 0, 167, 0, 4209, 0, 0, 0, 0, 0, 0, 0, 3628, 156, 10, 17, 0, 1403, 0, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 6261, 5991, 6260, 6232, 6230, 5861, 6196] avg=6144.0000 max=6261 max_rank=1 min=5861 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24519 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 146, 72, 160, 0, 145, 424, 1825, 2467, 981, 0, 0, 0, 0, 0, 0, 5, 1725, 238, 1572, 446, 343, 0, 0, 913, 815, 0, 0, 0, 0, 0, 0, 4451, 757, 0, 64, 1, 0, 0, 779, 0, 0, 0, 0, 0, 0, 0, 0, 152, 634, 1856, 2839, 2, 335, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 526, 596, 156, 3539, 199, 90, 80, 389, 263, 124, 0, 0, 0, 0, 0, 1684, 0, 2, 24, 419, 2083, 95, 0, 2058, 0, 0, 0, 0, 0, 0, 0, 943, 4, 171, 372, 7, 0, 213, 0, 4545, 0, 0, 0, 0, 0, 0, 0, 3656, 150, 10, 3, 0, 1336, 0, 980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6220, 6057, 6052, 6077, 5991, 6365, 6255, 6135] avg=6144.0000 max=6365 max_rank=5 min=5991 min_rank=4 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24746 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 105, 47, 163, 0, 126, 311, 1822, 2462, 983, 0, 0, 0, 0, 0, 0, 6, 1644, 213, 1812, 439, 325, 0, 2, 910, 868, 0, 0, 0, 0, 0, 0, 4673, 912, 0, 63, 0, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 160, 544, 1975, 2943, 3, 102, 251, 1, 0, 0, 0, 0, 0, 0, 0, 0, 21, 536, 498, 153, 3819, 229, 108, 90, 420, 280, 131, 0, 0, 0, 0, 0, 1277, 0, 0, 20, 482, 1974, 97, 0, 2050, 0, 0, 0, 0, 0, 0, 0, 811, 8, 166, 445, 2, 0, 176, 1, 4544, 0, 0, 0, 0, 0, 0, 0, 3945, 184, 3, 4, 0, 1287, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6020, 6219, 6307, 5979, 6285, 5900, 6153, 6289] avg=6144.0000 max=6307 max_rank=2 min=5900 min_rank=5 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24627 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 160, 62, 178, 1, 76, 413, 1733, 2384, 1002, 0, 0, 0, 0, 0, 0, 4, 1712, 197, 1737, 441, 352, 0, 2, 882, 855, 0, 0, 0, 0, 0, 0, 4411, 832, 0, 52, 1, 1, 0, 744, 0, 0, 0, 0, 0, 0, 0, 0, 150, 578, 1947, 3345, 3, 238, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 503, 339, 116, 4150, 172, 73, 93, 369, 277, 123, 0, 0, 0, 0, 0, 1478, 0, 1, 30, 428, 2049, 91, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 838, 3, 182, 351, 4, 0, 131, 0, 4638, 0, 0, 0, 0, 0, 0, 0, 3467, 148, 13, 5, 1, 1315, 0, 956, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6009, 6182, 6041, 6551, 6253, 6064, 6147, 5905] avg=6144.0000 max=6551 max_rank=3 min=5905 min_rank=7 imbalance_ratio=1.066243 max_load_reduction=42601 max_load_reduction_pct=0.866720 moved_to_echo=24369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 159, 68, 192, 1, 72, 360, 1943, 2416, 970, 0, 0, 0, 0, 0, 0, 24, 1764, 250, 1552, 393, 342, 0, 0, 894, 849, 0, 0, 0, 0, 0, 0, 4458, 961, 0, 70, 2, 0, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 176, 618, 1933, 2724, 2, 300, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 507, 545, 211, 3626, 218, 89, 62, 394, 275, 125, 0, 0, 0, 0, 0, 1436, 0, 3, 32, 541, 2093, 111, 1, 2014, 0, 0, 0, 0, 0, 0, 0, 822, 6, 184, 526, 7, 0, 126, 2, 4490, 0, 0, 0, 0, 0, 0, 0, 3699, 155, 17, 4, 1, 1285, 1, 957, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6182, 6068, 6261, 6020, 6108, 6231, 6163, 6119] avg=6144.0000 max=6261 max_rank=2 min=6020 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24621 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 131, 60, 136, 1, 105, 411, 1861, 2507, 909, 0, 0, 0, 0, 0, 0, 21, 1720, 229, 1746, 456, 348, 0, 0, 928, 813, 0, 0, 0, 0, 0, 0, 4416, 855, 0, 54, 0, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 147, 600, 1849, 3219, 1, 173, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 613, 556, 171, 3704, 198, 98, 77, 387, 263, 123, 0, 0, 0, 0, 0, 1524, 0, 0, 36, 450, 2020, 109, 0, 2091, 0, 0, 0, 0, 0, 0, 0, 861, 4, 143, 467, 10, 0, 167, 0, 4209, 0, 0, 0, 0, 0, 0, 0, 3628, 156, 10, 17, 0, 1403, 0, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 6261, 5991, 6260, 6232, 6230, 5861, 6196] avg=6144.0000 max=6261 max_rank=1 min=5861 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24519 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 201, 51, 146, 0, 131, 365, 1927, 2369, 942, 0, 0, 0, 0, 0, 0, 24, 1599, 221, 1724, 465, 316, 0, 0, 875, 877, 0, 0, 0, 0, 0, 0, 4523, 948, 0, 61, 2, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 172, 621, 1983, 3070, 2, 407, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 482, 393, 172, 3690, 177, 86, 96, 381, 286, 131, 0, 0, 0, 0, 0, 1567, 0, 1, 17, 416, 1938, 106, 1, 1972, 0, 0, 0, 0, 0, 0, 0, 895, 7, 160, 472, 3, 0, 202, 0, 4357, 0, 0, 0, 0, 0, 0, 0, 3536, 143, 7, 14, 0, 1490, 0, 931, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6101, 6213, 6532, 5939, 6018, 6096, 6121] avg=6144.0000 max=6532 max_rank=3 min=5939 min_rank=4 imbalance_ratio=1.063151 max_load_reduction=42620 max_load_reduction_pct=0.867106 moved_to_echo=24174 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 105, 47, 163, 0, 126, 311, 1822, 2462, 983, 0, 0, 0, 0, 0, 0, 6, 1644, 213, 1812, 439, 325, 0, 2, 910, 868, 0, 0, 0, 0, 0, 0, 4673, 912, 0, 63, 0, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 160, 544, 1975, 2943, 3, 102, 251, 1, 0, 0, 0, 0, 0, 0, 0, 0, 21, 536, 498, 153, 3819, 229, 108, 90, 420, 280, 131, 0, 0, 0, 0, 0, 1277, 0, 0, 20, 482, 1974, 97, 0, 2050, 0, 0, 0, 0, 0, 0, 0, 811, 8, 166, 445, 2, 0, 176, 1, 4544, 0, 0, 0, 0, 0, 0, 0, 3945, 184, 3, 4, 0, 1287, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6020, 6219, 6307, 5979, 6285, 5900, 6153, 6289] avg=6144.0000 max=6307 max_rank=2 min=5900 min_rank=5 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24627 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 172, 78, 156, 0, 123, 396, 1876, 2432, 968, 0, 0, 0, 0, 0, 0, 14, 1632, 241, 1770, 442, 379, 1, 1, 901, 780, 0, 0, 0, 0, 0, 0, 4461, 889, 0, 63, 1, 0, 0, 752, 0, 0, 0, 0, 0, 0, 0, 0, 149, 620, 1776, 2781, 2, 349, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 462, 498, 286, 3645, 228, 78, 66, 377, 252, 125, 0, 0, 0, 0, 0, 1481, 0, 0, 21, 439, 2165, 116, 1, 2026, 0, 0, 0, 0, 0, 0, 0, 1009, 10, 224, 397, 5, 0, 187, 0, 4483, 0, 0, 0, 0, 0, 0, 0, 3536, 171, 10, 10, 0, 1388, 0, 925, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 6161, 6166, 5955, 6065, 6249, 6315, 6040] avg=6144.0000 max=6315 max_rank=6 min=5955 min_rank=3 imbalance_ratio=1.027832 max_load_reduction=42837 max_load_reduction_pct=0.871521 moved_to_echo=24669 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 201, 51, 146, 0, 131, 365, 1927, 2369, 942, 0, 0, 0, 0, 0, 0, 24, 1599, 221, 1724, 465, 316, 0, 0, 875, 877, 0, 0, 0, 0, 0, 0, 4523, 948, 0, 61, 2, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 172, 621, 1983, 3070, 2, 407, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 482, 393, 172, 3690, 177, 86, 96, 381, 286, 131, 0, 0, 0, 0, 0, 1567, 0, 1, 17, 416, 1938, 106, 1, 1972, 0, 0, 0, 0, 0, 0, 0, 895, 7, 160, 472, 3, 0, 202, 0, 4357, 0, 0, 0, 0, 0, 0, 0, 3536, 143, 7, 14, 0, 1490, 0, 931, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6101, 6213, 6532, 5939, 6018, 6096, 6121] avg=6144.0000 max=6532 max_rank=3 min=5939 min_rank=4 imbalance_ratio=1.063151 max_load_reduction=42620 max_load_reduction_pct=0.867106 moved_to_echo=24174 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 135, 81, 154, 2, 67, 384, 2023, 2453, 968, 0, 0, 0, 0, 0, 0, 23, 1684, 297, 1577, 439, 346, 0, 0, 909, 828, 0, 0, 0, 0, 0, 0, 4358, 1064, 0, 66, 1, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 177, 708, 1883, 2549, 3, 192, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 531, 597, 187, 3728, 212, 111, 82, 393, 268, 122, 0, 0, 0, 0, 0, 1374, 0, 0, 17, 443, 2118, 97, 1, 2045, 0, 0, 0, 0, 0, 0, 0, 937, 7, 157, 454, 4, 0, 126, 0, 4477, 0, 0, 0, 0, 0, 0, 0, 3688, 199, 20, 7, 0, 1472, 1, 960, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6267, 6103, 6121, 5778, 6279, 6095, 6162, 6347] avg=6144.0000 max=6347 max_rank=7 min=5778 min_rank=3 imbalance_ratio=1.033040 max_load_reduction=42805 max_load_reduction_pct=0.870870 moved_to_echo=24883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 1280, 640, 1408, 128, 896, 3072, 15104, 19584, 7808, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 13568, 1920, 13568, 3584, 2816, 128, 128, 7296, 6784, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 4992, 15232, 23552, 128, 2176, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[29184, 1408, 128, 128, 128, 11008, 128, 7680, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 4224, 4096, 1536, 29952, 1664, 768, 768, 3200, 2176, 1024, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11904, 0, 128, 256, 3712, 16512, 896, 128, 16256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 13568, 1920, 13568, 3584, 2816, 128, 128, 7296, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 4224, 4096, 1536, 29952, 1664, 768, 768, 3200, 2176, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[29184, 1408, 128, 128, 128, 11008, 128, 7680, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[35840, 7296, 0, 512, 128, 128, 0, 5760, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7168, 128, 1408, 3584, 128, 0, 1408, 128, 35840, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 2107, 1250, 863, 132, 388, 79, 319, 0, 255, 0, 6761, 2, 3541, 1726, 150, 0, 2170, 305, 0, 43, 776, 792, 719, 561, 1165, 745, 465, 0, 0, 0, 82, 26, 1652, 21, 18, 1, 934, 5662, 27, 862, 594, 0, 0, 1725, 0, 0, 14, 498, 30, 1088, 0, 0, 2, 0, 652, 0, 2459, 6585, 3, 891] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 11, 3, 2196, 857, 966, 68, 468, 78, 356, 0, 243, 0, 6845, 1, 3646, 2090, 32, 1, 1913, 290, 0, 35, 647, 819, 679, 624, 1305, 650, 448, 3, 0, 6, 66, 35, 1828, 22, 16, 3, 874, 5596, 14, 759, 525, 3, 2, 1781, 1, 3, 8, 495, 31, 1183, 0, 3, 7, 0, 590, 1, 2554, 6616, 37, 819] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2121, 1157, 928, 116, 426, 86, 370, 0, 246, 0, 6538, 0, 3750, 1878, 103, 0, 1850, 266, 0, 165, 741, 1119, 676, 617, 1054, 601, 480, 0, 0, 8, 79, 20, 1817, 57, 13, 1, 804, 5313, 33, 779, 596, 0, 0, 1691, 0, 0, 7, 504, 40, 1511, 0, 0, 2, 0, 526, 0, 2778, 6502, 2, 768] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 2180, 1161, 967, 96, 456, 75, 418, 0, 252, 0, 6441, 0, 3426, 1799, 103, 0, 2174, 318, 0, 94, 765, 919, 656, 552, 1045, 704, 461, 0, 0, 5, 74, 26, 2038, 35, 10, 0, 865, 5444, 34, 713, 557, 0, 0, 1760, 0, 0, 3, 517, 38, 1423, 0, 0, 2, 0, 595, 0, 2722, 6390, 4, 825] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 2424, 1209, 894, 60, 402, 80, 434, 0, 248, 0, 6720, 1, 3460, 1694, 180, 1, 2035, 283, 0, 88, 697, 993, 662, 597, 853, 530, 473, 4, 0, 3, 79, 38, 2010, 68, 8, 0, 997, 5540, 30, 800, 330, 0, 0, 1955, 0, 0, 1, 482, 21, 1382, 0, 0, 3, 0, 490, 2, 2789, 6309, 2, 784] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 2166, 1259, 798, 122, 374, 87, 405, 0, 256, 0, 6606, 1, 3469, 1611, 166, 1, 1846, 304, 0, 87, 763, 983, 677, 561, 1228, 577, 451, 0, 0, 7, 79, 33, 1887, 41, 14, 1, 991, 5551, 12, 818, 504, 0, 0, 1845, 0, 0, 21, 477, 32, 1395, 0, 0, 1, 0, 699, 0, 2735, 6389, 7, 798] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2121, 1157, 928, 116, 426, 86, 370, 0, 246, 0, 6538, 0, 3750, 1878, 103, 0, 1850, 266, 0, 165, 741, 1119, 676, 617, 1054, 601, 480, 0, 0, 8, 79, 20, 1817, 57, 13, 1, 804, 5313, 33, 779, 596, 0, 0, 1691, 0, 0, 7, 504, 40, 1511, 0, 0, 2, 0, 526, 0, 2778, 6502, 2, 768] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 11, 3, 2196, 857, 966, 68, 468, 78, 356, 0, 243, 0, 6845, 1, 3646, 2090, 32, 1, 1913, 290, 0, 35, 647, 819, 679, 624, 1305, 650, 448, 3, 0, 6, 66, 35, 1828, 22, 16, 3, 874, 5596, 14, 759, 525, 3, 2, 1781, 1, 3, 8, 495, 31, 1183, 0, 3, 7, 0, 590, 1, 2554, 6616, 37, 819] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 2107, 1250, 863, 132, 388, 79, 319, 0, 255, 0, 6761, 2, 3541, 1726, 150, 0, 2170, 305, 0, 43, 776, 792, 719, 561, 1165, 745, 465, 0, 0, 0, 82, 26, 1652, 21, 18, 1, 934, 5662, 27, 862, 594, 0, 0, 1725, 0, 0, 14, 498, 30, 1088, 0, 0, 2, 0, 652, 0, 2459, 6585, 3, 891] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 8, 1, 2154, 1253, 942, 62, 361, 90, 413, 0, 237, 1, 6446, 2, 3640, 1765, 166, 0, 2076, 289, 0, 82, 640, 828, 690, 606, 933, 611, 443, 0, 0, 8, 83, 50, 1874, 35, 19, 0, 1009, 5522, 40, 820, 660, 0, 0, 1797, 1, 1, 9, 477, 31, 1246, 3, 0, 12, 2, 575, 0, 2743, 6600, 2, 792] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 2424, 1209, 894, 60, 402, 80, 434, 0, 248, 0, 6720, 1, 3460, 1694, 180, 1, 2035, 283, 0, 88, 697, 993, 662, 597, 853, 530, 473, 4, 0, 3, 79, 38, 2010, 68, 8, 0, 997, 5540, 30, 800, 330, 0, 0, 1955, 0, 0, 1, 482, 21, 1382, 0, 0, 3, 0, 490, 2, 2789, 6309, 2, 784] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4, 2, 2019, 902, 922, 119, 390, 87, 304, 1, 248, 0, 6739, 3, 3725, 1783, 96, 0, 1994, 277, 0, 114, 707, 981, 729, 571, 1013, 662, 451, 2, 0, 4, 89, 33, 2152, 21, 9, 0, 1144, 5706, 14, 814, 527, 0, 1, 1609, 1, 0, 0, 530, 16, 1578, 0, 1, 2, 2, 519, 2, 2735, 6030, 5, 763] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 2180, 1161, 967, 96, 456, 75, 418, 0, 252, 0, 6441, 0, 3426, 1799, 103, 0, 2174, 318, 0, 94, 765, 919, 656, 552, 1045, 704, 461, 0, 0, 5, 74, 26, 2038, 35, 10, 0, 865, 5444, 34, 713, 557, 0, 0, 1760, 0, 0, 3, 517, 38, 1423, 0, 0, 2, 0, 595, 0, 2722, 6390, 4, 825] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 3, 2196, 857, 966, 68, 1291, 572, 0, 0, 0, 0, 0, 0, 468, 78, 356, 0, 243, 0, 5184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1873, 2090, 32, 1, 1913, 290, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 647, 819, 679, 624, 1305, 650, 448, 3, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 6, 66, 35, 1828, 22, 16, 3, 3512, 482, 118, 0, 0, 0, 0, 0, 874, 2084, 14, 759, 525, 3, 2, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 8, 495, 31, 1183, 0, 3, 4380, 0, 0, 0, 0, 0, 0, 0, 7, 0, 590, 1, 2554, 2118, 37, 819, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5964, 6330, 6234, 6264, 6088, 6042, 6104, 6126] avg=6144.0000 max=6330 max_rank=1 min=5964 min_rank=0 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 2107, 1250, 863, 132, 1254, 565, 0, 0, 0, 0, 0, 0, 388, 79, 319, 0, 255, 0, 5121, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1819, 1726, 150, 0, 2170, 305, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 776, 792, 719, 561, 1165, 745, 465, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 26, 1652, 21, 18, 1, 3554, 468, 118, 0, 0, 0, 0, 0, 934, 2108, 27, 862, 594, 0, 0, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 498, 30, 1088, 0, 0, 4360, 0, 0, 0, 0, 0, 0, 0, 2, 0, 652, 0, 2459, 2107, 3, 891, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6164, 6213, 6298, 5940, 6250, 5990, 6114] avg=6144.0000 max=6298 max_rank=3 min=5940 min_rank=4 imbalance_ratio=1.025065 max_load_reduction=42854 max_load_reduction_pct=0.871867 moved_to_echo=24294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2121, 1157, 928, 116, 1328, 546, 0, 0, 0, 0, 0, 0, 426, 86, 370, 0, 246, 0, 4952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1926, 1878, 103, 0, 1850, 266, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 741, 1119, 676, 617, 1054, 601, 480, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 8, 79, 20, 1817, 57, 13, 1, 3335, 496, 116, 0, 0, 0, 0, 0, 804, 1978, 33, 779, 596, 0, 0, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 504, 40, 1511, 0, 0, 4305, 0, 0, 0, 0, 0, 0, 0, 2, 0, 526, 0, 2778, 2081, 2, 768, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6209, 6080, 6188, 6328, 5942, 5881, 6367, 6157] avg=6144.0000 max=6367 max_rank=6 min=5881 min_rank=5 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 2180, 1161, 967, 96, 1213, 538, 0, 0, 0, 0, 0, 0, 456, 75, 418, 0, 252, 0, 4879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 1799, 103, 0, 2174, 318, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 765, 919, 656, 552, 1045, 704, 461, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 5, 74, 26, 2038, 35, 10, 0, 3417, 453, 114, 0, 0, 0, 0, 0, 865, 2027, 34, 713, 557, 0, 0, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 517, 38, 1423, 0, 0, 4231, 0, 0, 0, 0, 0, 0, 0, 2, 0, 595, 0, 2722, 2045, 4, 825, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6165, 6080, 6248, 6126, 6172, 5956, 6212, 6193] avg=6144.0000 max=6248 max_rank=2 min=5956 min_rank=5 imbalance_ratio=1.016927 max_load_reduction=42904 max_load_reduction_pct=0.872884 moved_to_echo=24533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 2107, 1250, 863, 132, 1254, 565, 0, 0, 0, 0, 0, 0, 388, 79, 319, 0, 255, 0, 5121, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1819, 1726, 150, 0, 2170, 305, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 776, 792, 719, 561, 1165, 745, 465, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 26, 1652, 21, 18, 1, 3554, 468, 118, 0, 0, 0, 0, 0, 934, 2108, 27, 862, 594, 0, 0, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 498, 30, 1088, 0, 0, 4360, 0, 0, 0, 0, 0, 0, 0, 2, 0, 652, 0, 2459, 2107, 3, 891, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6164, 6213, 6298, 5940, 6250, 5990, 6114] avg=6144.0000 max=6298 max_rank=3 min=5940 min_rank=4 imbalance_ratio=1.025065 max_load_reduction=42854 max_load_reduction_pct=0.871867 moved_to_echo=24294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 3, 2196, 857, 966, 68, 1291, 572, 0, 0, 0, 0, 0, 0, 468, 78, 356, 0, 243, 0, 5184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1873, 2090, 32, 1, 1913, 290, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 647, 819, 679, 624, 1305, 650, 448, 3, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 6, 66, 35, 1828, 22, 16, 3, 3512, 482, 118, 0, 0, 0, 0, 0, 874, 2084, 14, 759, 525, 3, 2, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 8, 495, 31, 1183, 0, 3, 4380, 0, 0, 0, 0, 0, 0, 0, 7, 0, 590, 1, 2554, 2118, 37, 819, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5964, 6330, 6234, 6264, 6088, 6042, 6104, 6126] avg=6144.0000 max=6330 max_rank=1 min=5964 min_rank=0 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 8, 1, 2154, 1253, 942, 62, 1289, 538, 0, 0, 0, 0, 0, 0, 361, 90, 413, 0, 237, 1, 4883, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1869, 1765, 166, 0, 2076, 289, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 640, 828, 690, 606, 933, 611, 443, 0, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 8, 83, 50, 1874, 35, 19, 0, 3466, 482, 118, 0, 0, 0, 0, 0, 1009, 2056, 40, 820, 660, 0, 0, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 9, 477, 31, 1246, 3, 0, 4370, 0, 0, 0, 0, 0, 0, 0, 12, 2, 575, 0, 2743, 2112, 2, 792, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6249, 5987, 6247, 5776, 6135, 6382, 6138, 6238] avg=6144.0000 max=6382 max_rank=5 min=5776 min_rank=3 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 2166, 1259, 798, 122, 1234, 556, 0, 0, 0, 0, 0, 0, 374, 87, 405, 0, 256, 0, 4996, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 1611, 166, 1, 1846, 304, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 763, 983, 677, 561, 1228, 577, 451, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 7, 79, 33, 1887, 41, 14, 1, 3489, 463, 117, 0, 0, 0, 0, 0, 991, 2062, 12, 818, 504, 0, 0, 1845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 477, 32, 1395, 0, 0, 4235, 0, 0, 0, 0, 0, 0, 0, 1, 0, 699, 0, 2735, 2037, 7, 798, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6119, 5787, 6294, 6131, 6232, 6160, 6277] avg=6144.0000 max=6294 max_rank=3 min=5787 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2121, 1157, 928, 116, 1328, 546, 0, 0, 0, 0, 0, 0, 426, 86, 370, 0, 246, 0, 4952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1926, 1878, 103, 0, 1850, 266, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 741, 1119, 676, 617, 1054, 601, 480, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 8, 79, 20, 1817, 57, 13, 1, 3335, 496, 116, 0, 0, 0, 0, 0, 804, 1978, 33, 779, 596, 0, 0, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 504, 40, 1511, 0, 0, 4305, 0, 0, 0, 0, 0, 0, 0, 2, 0, 526, 0, 2778, 2081, 2, 768, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6209, 6080, 6188, 6328, 5942, 5881, 6367, 6157] avg=6144.0000 max=6367 max_rank=6 min=5881 min_rank=5 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 2424, 1209, 894, 60, 1225, 561, 0, 0, 0, 0, 0, 0, 402, 80, 434, 0, 248, 0, 5090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 1694, 180, 1, 2035, 283, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 697, 993, 662, 597, 853, 530, 473, 4, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 3, 79, 38, 2010, 68, 8, 0, 3477, 458, 113, 0, 0, 0, 0, 0, 997, 2063, 30, 800, 330, 0, 0, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 482, 21, 1382, 0, 0, 4177, 0, 0, 0, 0, 0, 0, 0, 3, 0, 490, 2, 2789, 2019, 2, 784, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6380, 6255, 6058, 5878, 6254, 6175, 6063, 6089] avg=6144.0000 max=6380 max_rank=0 min=5878 min_rank=3 imbalance_ratio=1.038411 max_load_reduction=42772 max_load_reduction_pct=0.870199 moved_to_echo=24581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 2166, 1259, 798, 122, 1234, 556, 0, 0, 0, 0, 0, 0, 374, 87, 405, 0, 256, 0, 4996, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 1611, 166, 1, 1846, 304, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 763, 983, 677, 561, 1228, 577, 451, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 7, 79, 33, 1887, 41, 14, 1, 3489, 463, 117, 0, 0, 0, 0, 0, 991, 2062, 12, 818, 504, 0, 0, 1845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 477, 32, 1395, 0, 0, 4235, 0, 0, 0, 0, 0, 0, 0, 1, 0, 699, 0, 2735, 2037, 7, 798, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6119, 5787, 6294, 6131, 6232, 6160, 6277] avg=6144.0000 max=6294 max_rank=3 min=5787 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 2, 2019, 902, 922, 119, 1319, 563, 0, 0, 0, 0, 0, 0, 390, 87, 304, 1, 248, 0, 5104, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1913, 1783, 96, 0, 1994, 277, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 707, 981, 729, 571, 1013, 662, 451, 2, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 4, 89, 33, 2152, 21, 9, 0, 3581, 493, 108, 0, 0, 0, 0, 0, 1144, 2125, 14, 814, 527, 0, 1, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 530, 16, 1578, 0, 1, 3992, 0, 0, 0, 0, 0, 0, 0, 2, 2, 519, 2, 2735, 1930, 5, 763, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5850, 6137, 6177, 6188, 6490, 6234, 6118, 5958] avg=6144.0000 max=6490 max_rank=4 min=5850 min_rank=0 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 17408, 9088, 7296, 896, 10240, 4480, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[14720, 14464, 1024, 128, 16128, 2432, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 16512, 256, 6400, 4352, 128, 128, 14208, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 7552, 5504, 4736, 8704, 5120, 3712, 128, 8448, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[14720, 14464, 1024, 128, 16128, 2432, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 4736, 128, 21632, 16512, 128, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 640, 384, 15360, 384, 128, 128, 27904, 3840, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 7552, 5504, 4736, 8704, 5120, 3712, 128, 8448, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 4096, 256, 10880, 128, 128, 34176, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3328, 768, 3072, 128, 2048, 128, 40320, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 640, 384, 15360, 384, 128, 128, 27904, 3840, 1024, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 474, 3551, 1, 0, 26, 3, 25, 0, 1, 0, 86, 63, 0, 0, 0, 0, 3359, 0, 4032, 0, 191, 5969, 4, 2, 0, 386, 12, 0, 2082, 1, 0, 1, 0, 0, 1643, 1594, 3, 3634, 251, 0, 1560, 1, 7598, 55, 2, 130, 0, 1, 765, 7030, 1, 4, 397, 3547, 473, 0, 0, 0, 5, 187, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 527, 3655, 4, 0, 5, 2, 61, 1, 2, 1, 58, 70, 0, 3, 0, 0, 3484, 0, 3564, 0, 260, 5590, 7, 13, 0, 461, 33, 0, 2158, 0, 3, 1, 0, 3, 1596, 1566, 0, 3824, 235, 3, 1395, 0, 7484, 49, 0, 133, 1, 1, 1213, 6987, 9, 0, 401, 3664, 456, 0, 3, 3, 0, 161, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 350, 3753, 0, 0, 3, 0, 54, 0, 1, 0, 61, 72, 0, 0, 0, 1, 3381, 0, 3594, 3, 263, 5672, 0, 14, 0, 394, 25, 0, 2044, 0, 0, 0, 0, 0, 1738, 1679, 1, 3685, 217, 0, 1475, 0, 7493, 58, 0, 150, 0, 0, 1300, 7141, 0, 0, 475, 3440, 432, 0, 0, 0, 1, 182, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 458, 3697, 0, 0, 0, 0, 53, 0, 1, 0, 61, 80, 0, 0, 0, 0, 3096, 0, 3871, 0, 250, 5800, 5, 7, 0, 497, 8, 0, 1904, 0, 0, 0, 0, 0, 1794, 1650, 0, 3546, 328, 0, 1615, 0, 7583, 60, 0, 144, 0, 0, 912, 7107, 0, 0, 468, 3533, 461, 0, 0, 0, 1, 162, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 566, 3864, 0, 0, 3, 0, 35, 0, 0, 0, 94, 60, 0, 0, 0, 0, 3566, 0, 3241, 0, 216, 5993, 1, 7, 0, 373, 9, 0, 2113, 0, 0, 0, 0, 0, 1620, 1713, 0, 3345, 322, 0, 1367, 0, 7543, 83, 0, 78, 0, 0, 1163, 6992, 0, 0, 509, 3625, 475, 0, 0, 0, 1, 174, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 494, 3417, 0, 0, 1, 0, 36, 0, 0, 0, 54, 73, 0, 0, 0, 0, 3589, 0, 3655, 0, 253, 5803, 1, 9, 0, 429, 33, 0, 2148, 0, 0, 0, 0, 0, 1638, 1708, 0, 3644, 269, 0, 1439, 0, 7581, 52, 0, 140, 0, 0, 1112, 7018, 0, 0, 359, 3582, 455, 0, 0, 0, 0, 160, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 517, 3792, 0, 0, 0, 0, 32, 0, 0, 0, 79, 75, 0, 0, 0, 0, 3542, 0, 3780, 1, 257, 5873, 5, 10, 0, 555, 9, 0, 1979, 0, 0, 0, 0, 0, 1673, 1391, 0, 3450, 293, 0, 1416, 0, 7529, 68, 0, 116, 0, 0, 1216, 7100, 0, 0, 423, 3341, 464, 0, 0, 0, 0, 166, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 350, 3753, 0, 0, 3, 0, 54, 0, 1, 0, 61, 72, 0, 0, 0, 1, 3381, 0, 3594, 3, 263, 5672, 0, 14, 0, 394, 25, 0, 2044, 0, 0, 0, 0, 0, 1738, 1679, 1, 3685, 217, 0, 1475, 0, 7493, 58, 0, 150, 0, 0, 1300, 7141, 0, 0, 475, 3440, 432, 0, 0, 0, 1, 182, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 474, 3551, 1, 0, 26, 3, 25, 0, 1, 0, 86, 63, 0, 0, 0, 0, 3359, 0, 4032, 0, 191, 5969, 4, 2, 0, 386, 12, 0, 2082, 1, 0, 1, 0, 0, 1643, 1594, 3, 3634, 251, 0, 1560, 1, 7598, 55, 2, 130, 0, 1, 765, 7030, 1, 4, 397, 3547, 473, 0, 0, 0, 5, 187, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 531, 4123, 1, 0, 0, 0, 36, 0, 1, 0, 92, 125, 0, 0, 0, 0, 3469, 0, 3931, 2, 201, 5709, 3, 7, 0, 359, 8, 0, 2002, 0, 0, 1, 0, 2, 1631, 1473, 0, 3625, 329, 0, 1398, 2, 7477, 79, 0, 136, 0, 0, 930, 7164, 0, 0, 374, 3313, 446, 0, 0, 1, 2, 169, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 566, 3864, 0, 0, 3, 0, 35, 0, 0, 0, 94, 60, 0, 0, 0, 0, 3566, 0, 3241, 0, 216, 5993, 1, 7, 0, 373, 9, 0, 2113, 0, 0, 0, 0, 0, 1620, 1713, 0, 3345, 322, 0, 1367, 0, 7543, 83, 0, 78, 0, 0, 1163, 6992, 0, 0, 509, 3625, 475, 0, 0, 0, 1, 174, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 494, 3417, 0, 0, 1, 0, 36, 0, 0, 0, 54, 73, 0, 0, 0, 0, 3589, 0, 3655, 0, 253, 5803, 1, 9, 0, 429, 33, 0, 2148, 0, 0, 0, 0, 0, 1638, 1708, 0, 3644, 269, 0, 1439, 0, 7581, 52, 0, 140, 0, 0, 1112, 7018, 0, 0, 359, 3582, 455, 0, 0, 0, 0, 160, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 517, 3792, 0, 0, 0, 0, 32, 955, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 75, 0, 0, 0, 6019, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 2517, 1, 257, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 555, 9, 0, 1979, 0, 0, 3039, 367, 231, 0, 0, 0, 0, 0, 0, 0, 0, 1673, 1391, 0, 2495, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1416, 0, 4490, 68, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 681, 0, 0, 423, 3341, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 5642, 400, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6192, 6173, 6322, 6190, 5852, 6090, 6125, 6208] avg=6144.0000 max=6322 max_rank=2 min=5852 min_rank=4 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24275 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 474, 3551, 1, 0, 26, 3, 25, 1006, 956, 0, 0, 0, 0, 0, 0, 0, 1, 0, 86, 63, 0, 0, 0, 5960, 0, 0, 0, 0, 0, 0, 0, 0, 3359, 0, 2684, 0, 191, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 386, 12, 0, 2082, 1, 0, 3067, 392, 235, 0, 0, 0, 0, 0, 1, 0, 0, 1643, 1594, 3, 2628, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1560, 1, 4531, 55, 2, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 765, 674, 1, 4, 397, 3547, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 187, 0, 0, 2, 5734, 396, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6042, 6110, 6238, 6177, 6120, 6279, 5862, 6324] avg=6144.0000 max=6324 max_rank=7 min=5862 min_rank=6 imbalance_ratio=1.029297 max_load_reduction=42828 max_load_reduction_pct=0.871338 moved_to_echo=24585 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 531, 4123, 1, 0, 0, 0, 36, 1004, 932, 0, 0, 0, 0, 0, 0, 0, 1, 0, 92, 125, 0, 0, 0, 6074, 0, 0, 0, 0, 0, 0, 0, 0, 3469, 0, 2617, 2, 201, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 359, 8, 0, 2002, 0, 0, 3018, 382, 225, 0, 0, 0, 0, 0, 1, 0, 2, 1631, 1473, 0, 2621, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1398, 2, 4459, 79, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 686, 0, 0, 374, 3313, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 169, 0, 0, 0, 5484, 404, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6627, 6292, 6292, 6001, 6057, 6074, 5749, 6060] avg=6144.0000 max=6627 max_rank=0 min=5749 min_rank=6 imbalance_ratio=1.078613 max_load_reduction=42525 max_load_reduction_pct=0.865173 moved_to_echo=23940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 350, 3753, 0, 0, 3, 0, 54, 1020, 852, 0, 0, 0, 0, 0, 0, 0, 1, 0, 61, 72, 0, 0, 0, 6054, 0, 0, 0, 0, 0, 0, 0, 1, 3381, 0, 2393, 3, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 394, 25, 0, 2044, 0, 0, 3025, 349, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 1679, 1, 2665, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 0, 4468, 58, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 685, 0, 0, 475, 3440, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 182, 0, 0, 0, 5448, 402, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6032, 6188, 6041, 6075, 6300, 6151, 6332, 6033] avg=6144.0000 max=6332 max_rank=6 min=6032 min_rank=0 imbalance_ratio=1.030599 max_load_reduction=42820 max_load_reduction_pct=0.871175 moved_to_echo=24816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 566, 3864, 0, 0, 3, 0, 35, 926, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 60, 0, 0, 0, 5928, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 2158, 0, 216, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 373, 9, 0, 2113, 0, 0, 3045, 315, 236, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 1713, 0, 2419, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 0, 4498, 83, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 670, 0, 0, 509, 3625, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 174, 0, 1, 0, 5757, 394, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6162, 6082, 5941, 6098, 6074, 6026, 6442, 6327] avg=6144.0000 max=6442 max_rank=6 min=5941 min_rank=2 imbalance_ratio=1.048503 max_load_reduction=42710 max_load_reduction_pct=0.868937 moved_to_echo=24869 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 517, 3792, 0, 0, 0, 0, 32, 955, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 75, 0, 0, 0, 6019, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 2517, 1, 257, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 555, 9, 0, 1979, 0, 0, 3039, 367, 231, 0, 0, 0, 0, 0, 0, 0, 0, 1673, 1391, 0, 2495, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1416, 0, 4490, 68, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 681, 0, 0, 423, 3341, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 5642, 400, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6192, 6173, 6322, 6190, 5852, 6090, 6125, 6208] avg=6144.0000 max=6322 max_rank=2 min=5852 min_rank=4 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24275 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 494, 3417, 0, 0, 1, 0, 36, 1009, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 73, 0, 0, 0, 5950, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 2433, 0, 253, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 429, 33, 0, 2148, 0, 0, 3060, 355, 229, 0, 0, 0, 0, 0, 0, 0, 0, 1638, 1708, 0, 2635, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 4521, 52, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 673, 0, 0, 359, 3582, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 5574, 395, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5824, 6077, 6276, 6263, 6250, 6152, 6181, 6129] avg=6144.0000 max=6276 max_rank=2 min=5824 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=42876 max_load_reduction_pct=0.872314 moved_to_echo=24712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 527, 3655, 4, 0, 5, 2, 61, 1059, 845, 0, 0, 0, 0, 0, 0, 1, 2, 1, 58, 70, 0, 3, 0, 5924, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 0, 2373, 0, 260, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 461, 33, 0, 2158, 0, 3, 3021, 346, 220, 0, 0, 0, 0, 0, 1, 0, 3, 1596, 1566, 0, 2765, 235, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1395, 0, 4463, 49, 0, 133, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1213, 669, 9, 0, 401, 3664, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 161, 0, 0, 2, 5370, 394, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6158, 6059, 6124, 6255, 6166, 6044, 6413, 5933] avg=6144.0000 max=6413 max_rank=6 min=5933 min_rank=7 imbalance_ratio=1.043783 max_load_reduction=42739 max_load_reduction_pct=0.869527 moved_to_echo=24556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 458, 3697, 0, 0, 0, 0, 53, 985, 922, 0, 0, 0, 0, 0, 0, 0, 1, 0, 61, 80, 0, 0, 0, 6029, 0, 0, 0, 0, 0, 0, 0, 0, 3096, 0, 2567, 0, 250, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 497, 8, 0, 1904, 0, 0, 3066, 382, 229, 0, 0, 0, 0, 0, 0, 0, 0, 1794, 1650, 0, 2561, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 0, 4517, 60, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 674, 0, 0, 468, 3533, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 162, 0, 0, 0, 5571, 404, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 6171, 5918, 6093, 6333, 6336, 6048, 6138] avg=6144.0000 max=6336 max_rank=5 min=5918 min_rank=2 imbalance_ratio=1.031250 max_load_reduction=42816 max_load_reduction_pct=0.871094 moved_to_echo=24855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 494, 3417, 0, 0, 1, 0, 36, 1009, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 73, 0, 0, 0, 5950, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 2433, 0, 253, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 429, 33, 0, 2148, 0, 0, 3060, 355, 229, 0, 0, 0, 0, 0, 0, 0, 0, 1638, 1708, 0, 2635, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 4521, 52, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 673, 0, 0, 359, 3582, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 5574, 395, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5824, 6077, 6276, 6263, 6250, 6152, 6181, 6129] avg=6144.0000 max=6276 max_rank=2 min=5824 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=42876 max_load_reduction_pct=0.872314 moved_to_echo=24712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 531, 4123, 1, 0, 0, 0, 36, 1004, 932, 0, 0, 0, 0, 0, 0, 0, 1, 0, 92, 125, 0, 0, 0, 6074, 0, 0, 0, 0, 0, 0, 0, 0, 3469, 0, 2617, 2, 201, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 359, 8, 0, 2002, 0, 0, 3018, 382, 225, 0, 0, 0, 0, 0, 1, 0, 2, 1631, 1473, 0, 2621, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1398, 2, 4459, 79, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 686, 0, 0, 374, 3313, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 169, 0, 0, 0, 5484, 404, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6627, 6292, 6292, 6001, 6057, 6074, 5749, 6060] avg=6144.0000 max=6627 max_rank=0 min=5749 min_rank=6 imbalance_ratio=1.078613 max_load_reduction=42525 max_load_reduction_pct=0.865173 moved_to_echo=23940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 350, 3753, 0, 0, 3, 0, 54, 1020, 852, 0, 0, 0, 0, 0, 0, 0, 1, 0, 61, 72, 0, 0, 0, 6054, 0, 0, 0, 0, 0, 0, 0, 1, 3381, 0, 2393, 3, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 394, 25, 0, 2044, 0, 0, 3025, 349, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 1679, 1, 2665, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 0, 4468, 58, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 685, 0, 0, 475, 3440, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 182, 0, 0, 0, 5448, 402, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6032, 6188, 6041, 6075, 6300, 6151, 6332, 6033] avg=6144.0000 max=6332 max_rank=6 min=6032 min_rank=0 imbalance_ratio=1.030599 max_load_reduction=42820 max_load_reduction_pct=0.871175 moved_to_echo=24816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 8704, 5504, 128, 128, 3456, 28160, 3712, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 3456, 256, 0, 16512, 128, 128, 24448, 2944, 1920, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 3968, 29952, 128, 0, 128, 128, 384, 8064, 7040, 0, 0, 0, 0, 0, 0] @@ -6356,519 +6356,519 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 1408, 0, 128, 128, 44672, 3200, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 640, 640, 0, 128, 0, 48000, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[78, 7232, 76, 4, 5409, 2039, 5303, 0, 0, 0, 0, 863, 194, 0, 0, 2750, 337, 1384, 828, 39, 0, 421, 5773, 0, 0, 1492, 337, 0, 0, 0, 336, 0, 996, 0, 0, 0, 1, 0, 225, 813, 5972, 0, 0, 0, 986, 0, 76, 26, 0, 0, 1860, 106, 0, 0, 0, 778, 0, 0, 76, 0, 3, 2149, 0, 190] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[70, 7356, 77, 3, 5589, 1772, 5145, 0, 0, 0, 1, 825, 117, 0, 0, 2994, 351, 1441, 778, 6, 2, 370, 5789, 0, 2, 1734, 358, 2, 1, 0, 349, 0, 697, 0, 0, 3, 2, 1, 236, 486, 6446, 0, 2, 0, 926, 9, 71, 37, 0, 0, 1771, 57, 0, 0, 1, 700, 0, 0, 75, 0, 2, 2319, 1, 178] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[84, 7299, 81, 0, 5065, 2179, 5349, 0, 0, 0, 0, 824, 182, 0, 0, 2617, 370, 1445, 866, 70, 0, 307, 5673, 0, 0, 1707, 372, 0, 0, 0, 415, 0, 645, 0, 0, 0, 0, 0, 325, 543, 6193, 0, 0, 0, 960, 0, 80, 26, 0, 0, 2001, 52, 0, 0, 0, 725, 0, 0, 81, 0, 4, 2398, 0, 214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[88, 7270, 87, 3, 5261, 2415, 5179, 2, 0, 2, 0, 924, 122, 0, 0, 2855, 322, 1478, 765, 9, 0, 416, 5575, 0, 2, 1689, 322, 0, 1, 0, 383, 0, 552, 1, 2, 0, 1, 3, 261, 577, 6241, 0, 0, 2, 905, 1, 85, 42, 0, 3, 1682, 86, 0, 0, 1, 799, 0, 0, 86, 0, 0, 2459, 2, 191] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[72, 7248, 72, 1, 5404, 2112, 5286, 0, 0, 0, 0, 706, 154, 0, 0, 2642, 349, 1423, 759, 24, 0, 390, 5656, 0, 0, 1784, 349, 0, 0, 0, 420, 0, 820, 0, 0, 0, 0, 0, 280, 640, 6138, 0, 0, 0, 1068, 0, 72, 32, 0, 0, 1864, 75, 0, 0, 0, 744, 0, 0, 72, 0, 2, 2292, 0, 202] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[70, 7356, 77, 3, 5589, 1772, 5145, 0, 0, 0, 1, 825, 117, 0, 0, 2994, 351, 1441, 778, 6, 2, 370, 5789, 0, 2, 1734, 358, 2, 1, 0, 349, 0, 697, 0, 0, 3, 2, 1, 236, 486, 6446, 0, 2, 0, 926, 9, 71, 37, 0, 0, 1771, 57, 0, 0, 1, 700, 0, 0, 75, 0, 2, 2319, 1, 178] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[84, 7337, 87, 1, 5036, 1948, 5247, 6, 0, 0, 3, 826, 149, 0, 0, 2721, 351, 1409, 711, 78, 2, 367, 5603, 0, 0, 1799, 350, 0, 0, 1, 413, 2, 762, 18, 0, 0, 1, 0, 298, 696, 6108, 0, 0, 0, 950, 5, 89, 33, 0, 1, 2168, 56, 0, 0, 1, 699, 0, 0, 91, 0, 8, 2447, 4, 186] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[80, 7239, 78, 0, 5640, 1911, 5610, 31, 0, 0, 0, 723, 121, 0, 0, 2871, 307, 1272, 823, 57, 2, 364, 5501, 0, 0, 1776, 307, 0, 0, 0, 391, 0, 795, 0, 0, 0, 0, 0, 306, 696, 6195, 0, 0, 0, 784, 0, 78, 25, 0, 0, 1795, 63, 0, 0, 0, 731, 0, 0, 78, 0, 4, 2305, 0, 193] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[84, 7299, 81, 0, 5065, 2179, 5349, 0, 0, 0, 0, 824, 182, 0, 0, 2617, 370, 1445, 866, 70, 0, 307, 5673, 0, 0, 1707, 372, 0, 0, 0, 415, 0, 645, 0, 0, 0, 0, 0, 325, 543, 6193, 0, 0, 0, 960, 0, 80, 26, 0, 0, 2001, 52, 0, 0, 0, 725, 0, 0, 81, 0, 4, 2398, 0, 214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[71, 7325, 70, 0, 5315, 2197, 4991, 45, 0, 0, 0, 695, 246, 0, 2, 2722, 335, 1331, 761, 9, 1, 488, 5652, 0, 0, 1420, 337, 0, 0, 0, 380, 0, 913, 1, 1, 0, 0, 0, 265, 641, 6333, 0, 0, 0, 889, 0, 71, 31, 0, 0, 2241, 64, 0, 0, 0, 673, 0, 0, 71, 0, 4, 2386, 0, 175] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[70, 0, 77, 3, 0, 1772, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 825, 117, 0, 0, 2994, 1278, 1147, 0, 0, 0, 0, 0, 0, 351, 1441, 778, 6, 2, 370, 3269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1734, 358, 2, 1, 0, 349, 0, 2858, 950, 0, 0, 0, 0, 0, 0, 697, 0, 0, 3, 2, 1, 236, 486, 4498, 0, 0, 0, 0, 0, 0, 0, 5299, 0, 2, 0, 926, 9, 71, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 57, 0, 0, 1, 700, 2520, 948, 50, 0, 0, 0, 0, 0, 0, 0, 75, 0, 2, 2319, 1, 178, 3691, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5739, 6362, 6217, 6254, 5923, 6344, 6047, 6266] avg=6144.0000 max=6362 max_rank=1 min=5739 min_rank=0 imbalance_ratio=1.035482 max_load_reduction=42790 max_load_reduction_pct=0.870565 moved_to_echo=24580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[78, 0, 76, 4, 0, 2039, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 194, 0, 0, 2750, 1318, 1063, 0, 0, 0, 0, 0, 0, 337, 1384, 828, 39, 0, 421, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 337, 0, 0, 0, 336, 0, 2810, 919, 0, 0, 0, 0, 0, 0, 996, 0, 0, 0, 1, 0, 225, 813, 4422, 0, 0, 0, 0, 0, 0, 0, 4909, 0, 0, 0, 986, 0, 76, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1860, 106, 0, 0, 0, 778, 2513, 919, 52, 0, 0, 0, 0, 0, 0, 0, 76, 0, 3, 2149, 0, 190, 3571, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6188, 6269, 5894, 6457, 5997, 6228, 5989] avg=6144.0000 max=6457 max_rank=4 min=5894 min_rank=3 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24671 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[88, 7270, 87, 3, 5261, 2415, 5179, 2, 0, 2, 0, 924, 122, 0, 0, 2855, 322, 1478, 765, 9, 0, 416, 5575, 0, 2, 1689, 322, 0, 1, 0, 383, 0, 552, 1, 2, 0, 1, 3, 261, 577, 6241, 0, 0, 2, 905, 1, 85, 42, 0, 3, 1682, 86, 0, 0, 1, 799, 0, 0, 86, 0, 0, 2459, 2, 191] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[72, 0, 72, 1, 0, 2112, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 154, 0, 0, 2642, 1313, 1092, 0, 0, 0, 0, 0, 0, 349, 1423, 759, 24, 0, 390, 3194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 349, 0, 0, 0, 420, 0, 2816, 918, 0, 0, 0, 0, 0, 0, 820, 0, 0, 0, 0, 0, 280, 640, 4432, 0, 0, 0, 0, 0, 0, 0, 5046, 0, 0, 0, 1068, 0, 72, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 75, 0, 0, 0, 744, 2462, 919, 52, 0, 0, 0, 0, 0, 0, 0, 72, 0, 2, 2292, 0, 202, 3567, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 5907, 6139, 6287, 6172, 6218, 6116, 6135] avg=6144.0000 max=6287 max_rank=3 min=5907 min_rank=1 imbalance_ratio=1.023275 max_load_reduction=42865 max_load_reduction_pct=0.872091 moved_to_echo=24641 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[78, 0, 76, 4, 0, 2039, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 194, 0, 0, 2750, 1318, 1063, 0, 0, 0, 0, 0, 0, 337, 1384, 828, 39, 0, 421, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 337, 0, 0, 0, 336, 0, 2810, 919, 0, 0, 0, 0, 0, 0, 996, 0, 0, 0, 1, 0, 225, 813, 4422, 0, 0, 0, 0, 0, 0, 0, 4909, 0, 0, 0, 986, 0, 76, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1860, 106, 0, 0, 0, 778, 2513, 919, 52, 0, 0, 0, 0, 0, 0, 0, 76, 0, 3, 2149, 0, 190, 3571, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6188, 6269, 5894, 6457, 5997, 6228, 5989] avg=6144.0000 max=6457 max_rank=4 min=5894 min_rank=3 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24671 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[84, 0, 87, 1, 0, 1948, 3892, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 826, 149, 0, 0, 2721, 1304, 1087, 0, 0, 0, 0, 0, 0, 351, 1409, 711, 78, 2, 367, 3164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 350, 0, 0, 1, 413, 2, 2851, 856, 0, 0, 0, 0, 0, 0, 762, 18, 0, 0, 1, 0, 298, 696, 4486, 0, 0, 0, 0, 0, 0, 0, 5021, 0, 0, 0, 950, 5, 89, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2168, 56, 0, 0, 1, 699, 2439, 854, 51, 0, 0, 0, 0, 0, 0, 0, 91, 0, 8, 2447, 4, 186, 3326, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6018, 6090, 6082, 6272, 6261, 6098, 6269, 6062] avg=6144.0000 max=6272 max_rank=3 min=6018 min_rank=0 imbalance_ratio=1.020833 max_load_reduction=42880 max_load_reduction_pct=0.872396 moved_to_echo=24690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[70, 0, 77, 3, 0, 1772, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 825, 117, 0, 0, 2994, 1278, 1147, 0, 0, 0, 0, 0, 0, 351, 1441, 778, 6, 2, 370, 3269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1734, 358, 2, 1, 0, 349, 0, 2858, 950, 0, 0, 0, 0, 0, 0, 697, 0, 0, 3, 2, 1, 236, 486, 4498, 0, 0, 0, 0, 0, 0, 0, 5299, 0, 2, 0, 926, 9, 71, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 57, 0, 0, 1, 700, 2520, 948, 50, 0, 0, 0, 0, 0, 0, 0, 75, 0, 2, 2319, 1, 178, 3691, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5739, 6362, 6217, 6254, 5923, 6344, 6047, 6266] avg=6144.0000 max=6362 max_rank=1 min=5739 min_rank=0 imbalance_ratio=1.035482 max_load_reduction=42790 max_load_reduction_pct=0.870565 moved_to_echo=24580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[84, 0, 81, 0, 0, 2179, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 182, 0, 0, 2617, 1329, 1102, 0, 0, 0, 0, 0, 0, 370, 1445, 866, 70, 0, 307, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 372, 0, 0, 0, 415, 0, 2836, 861, 0, 0, 0, 0, 0, 0, 645, 0, 0, 0, 0, 0, 325, 543, 4463, 0, 0, 0, 0, 0, 0, 0, 5091, 0, 0, 0, 960, 0, 80, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 52, 0, 0, 0, 725, 2469, 860, 52, 0, 0, 0, 0, 0, 0, 0, 81, 0, 4, 2398, 0, 214, 3344, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6312, 6054, 6262, 6191, 5976, 6157, 6159, 6041] avg=6144.0000 max=6312 max_rank=0 min=5976 min_rank=4 imbalance_ratio=1.027344 max_load_reduction=42840 max_load_reduction_pct=0.871582 moved_to_echo=24333 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[88, 0, 87, 3, 0, 2415, 3841, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 924, 122, 0, 0, 2855, 1287, 1111, 0, 0, 0, 0, 0, 0, 322, 1478, 765, 9, 0, 416, 3148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1689, 322, 0, 1, 0, 383, 0, 2826, 896, 0, 0, 0, 0, 0, 0, 552, 1, 2, 0, 1, 3, 261, 577, 4444, 0, 0, 0, 0, 0, 0, 0, 5130, 0, 0, 2, 905, 1, 85, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1682, 86, 0, 0, 1, 799, 2427, 892, 51, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 2459, 2, 191, 3473, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6436, 6301, 6138, 6119, 5841, 6165, 5941, 6211] avg=6144.0000 max=6436 max_rank=0 min=5841 min_rank=4 imbalance_ratio=1.047526 max_load_reduction=42716 max_load_reduction_pct=0.869059 moved_to_echo=24158 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[84, 0, 87, 1, 0, 1948, 3892, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 826, 149, 0, 0, 2721, 1304, 1087, 0, 0, 0, 0, 0, 0, 351, 1409, 711, 78, 2, 367, 3164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 350, 0, 0, 1, 413, 2, 2851, 856, 0, 0, 0, 0, 0, 0, 762, 18, 0, 0, 1, 0, 298, 696, 4486, 0, 0, 0, 0, 0, 0, 0, 5021, 0, 0, 0, 950, 5, 89, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2168, 56, 0, 0, 1, 699, 2439, 854, 51, 0, 0, 0, 0, 0, 0, 0, 91, 0, 8, 2447, 4, 186, 3326, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6018, 6090, 6082, 6272, 6261, 6098, 6269, 6062] avg=6144.0000 max=6272 max_rank=3 min=6018 min_rank=0 imbalance_ratio=1.020833 max_load_reduction=42880 max_load_reduction_pct=0.872396 moved_to_echo=24690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[80, 0, 78, 0, 0, 1911, 4154, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 723, 121, 0, 0, 2871, 1397, 1106, 0, 0, 0, 0, 0, 0, 307, 1272, 823, 57, 2, 364, 3103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1776, 307, 0, 0, 0, 391, 0, 2813, 960, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 306, 696, 4426, 0, 0, 0, 0, 0, 0, 0, 5089, 0, 0, 0, 784, 0, 78, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1795, 63, 0, 0, 0, 731, 2398, 957, 59, 0, 0, 0, 0, 0, 0, 0, 78, 0, 4, 2305, 0, 193, 3723, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6254, 6218, 5928, 6247, 6223, 5976, 6003, 6303] avg=6144.0000 max=6303 max_rank=7 min=5928 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[71, 0, 70, 0, 0, 2197, 3702, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 246, 0, 2, 2722, 1240, 1127, 0, 0, 0, 0, 0, 0, 335, 1331, 761, 9, 1, 488, 3192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 337, 0, 0, 0, 380, 0, 2847, 904, 0, 0, 0, 0, 0, 0, 913, 1, 1, 0, 0, 0, 265, 641, 4478, 0, 0, 0, 0, 0, 0, 0, 5206, 0, 0, 0, 889, 0, 71, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2241, 64, 0, 0, 0, 673, 2460, 902, 49, 0, 0, 0, 0, 0, 0, 0, 71, 0, 4, 2386, 0, 175, 3509, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 6032, 6117, 5888, 6299, 6197, 6389, 6145] avg=6144.0000 max=6389 max_rank=6 min=5888 min_rank=3 imbalance_ratio=1.039876 max_load_reduction=42763 max_load_reduction_pct=0.870015 moved_to_echo=25030 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[80, 0, 78, 0, 0, 1911, 4154, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 723, 121, 0, 0, 2871, 1397, 1106, 0, 0, 0, 0, 0, 0, 307, 1272, 823, 57, 2, 364, 3103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1776, 307, 0, 0, 0, 391, 0, 2813, 960, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 306, 696, 4426, 0, 0, 0, 0, 0, 0, 0, 5089, 0, 0, 0, 784, 0, 78, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1795, 63, 0, 0, 0, 731, 2398, 957, 59, 0, 0, 0, 0, 0, 0, 0, 78, 0, 4, 2305, 0, 193, 3723, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6254, 6218, 5928, 6247, 6223, 5976, 6003, 6303] avg=6144.0000 max=6303 max_rank=7 min=5928 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 128, 6400, 1408, 0, 128, 22272, 10496, 8960, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 13440, 2816, 128, 128, 128, 3200, 128, 22784, 7296, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 15488, 640, 0, 0, 128, 5888, 19712, 7296, 512, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 11264, 6400, 384, 128, 3200, 25600, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[6272, 128, 128, 128, 128, 128, 2304, 5120, 35712, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 640, 0, 128, 18816, 128, 1536, 28288, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[640, 0, 640, 128, 0, 16640, 31232, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[6272, 128, 128, 128, 128, 128, 2304, 5120, 35712, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[40832, 0, 128, 128, 7552, 128, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[694, 0, 25, 0, 0, 2, 0, 226, 0, 0, 6256, 1, 2323, 7820, 1, 4455, 0, 0, 1, 481, 0, 1, 446, 148, 1, 63, 102, 0, 1, 71, 1, 0, 3, 0, 4697, 0, 3, 1, 0, 1, 2, 7556, 1, 0, 0, 123, 594, 445, 0, 4, 8, 57, 3164, 1259, 1, 0, 3, 2, 0, 1, 0, 0, 8108, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[693, 0, 51, 0, 0, 0, 0, 460, 0, 0, 5621, 0, 2581, 7811, 0, 4676, 0, 0, 0, 620, 0, 0, 486, 129, 0, 67, 129, 0, 0, 81, 0, 0, 0, 0, 5345, 0, 0, 0, 0, 0, 0, 7187, 0, 0, 0, 128, 703, 346, 0, 1, 5, 53, 2730, 1136, 2, 0, 0, 0, 0, 0, 0, 0, 8111, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[714, 0, 22, 0, 0, 0, 0, 338, 0, 0, 6038, 0, 2386, 7836, 0, 4386, 0, 0, 0, 572, 0, 0, 438, 114, 0, 55, 117, 0, 0, 76, 0, 0, 0, 0, 4855, 0, 0, 0, 0, 0, 0, 7336, 0, 0, 0, 118, 663, 461, 0, 0, 0, 92, 3088, 1327, 4, 0, 0, 0, 0, 0, 0, 0, 8116, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[664, 0, 45, 0, 4, 0, 0, 449, 0, 1, 5692, 0, 2490, 7793, 0, 4709, 1, 4, 0, 571, 0, 3, 463, 187, 2, 52, 92, 4, 0, 82, 0, 1, 0, 0, 5412, 0, 1, 0, 3, 0, 5, 7158, 1, 0, 0, 135, 632, 430, 0, 0, 1, 49, 2597, 1302, 6, 3, 0, 0, 6, 0, 1, 3, 8094, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[714, 0, 22, 0, 0, 0, 0, 338, 0, 0, 6038, 0, 2386, 7836, 0, 4386, 0, 0, 0, 572, 0, 0, 438, 114, 0, 55, 117, 0, 0, 76, 0, 0, 0, 0, 4855, 0, 0, 0, 0, 0, 0, 7336, 0, 0, 0, 118, 663, 461, 0, 0, 0, 92, 3088, 1327, 4, 0, 0, 0, 0, 0, 0, 0, 8116, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[719, 0, 16, 0, 0, 0, 0, 443, 0, 0, 5823, 0, 2589, 7831, 0, 4636, 0, 0, 0, 677, 0, 0, 432, 160, 0, 73, 23, 0, 0, 72, 0, 0, 0, 0, 4954, 0, 0, 0, 0, 0, 0, 7448, 0, 0, 0, 120, 558, 339, 0, 0, 0, 74, 2690, 1353, 2, 0, 0, 0, 0, 0, 0, 0, 8120, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[677, 0, 92, 0, 0, 0, 0, 283, 0, 0, 6013, 0, 2570, 7884, 0, 4835, 0, 0, 0, 540, 0, 0, 403, 161, 0, 54, 135, 0, 0, 78, 0, 0, 0, 0, 5053, 0, 0, 0, 0, 0, 1, 7291, 0, 0, 0, 120, 587, 353, 0, 0, 1, 62, 2827, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[679, 0, 40, 0, 0, 0, 2, 436, 0, 0, 5397, 0, 2542, 7863, 0, 4114, 0, 3, 1, 676, 0, 0, 444, 113, 0, 54, 193, 1, 1, 86, 0, 0, 0, 0, 5310, 0, 2, 2, 0, 0, 0, 7476, 0, 0, 0, 136, 665, 523, 0, 0, 0, 96, 2956, 1226, 6, 1, 2, 1, 1, 0, 1, 1, 8102, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[693, 0, 51, 0, 0, 0, 0, 460, 0, 0, 5621, 0, 2581, 7811, 0, 4676, 0, 0, 0, 620, 0, 0, 486, 129, 0, 67, 129, 0, 0, 81, 0, 0, 0, 0, 5345, 0, 0, 0, 0, 0, 0, 7187, 0, 0, 0, 128, 703, 346, 0, 1, 5, 53, 2730, 1136, 2, 0, 0, 0, 0, 0, 0, 0, 8111, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[677, 0, 92, 0, 0, 0, 0, 283, 0, 0, 6013, 0, 2570, 7884, 0, 4835, 0, 0, 0, 540, 0, 0, 403, 161, 0, 54, 135, 0, 0, 78, 0, 0, 0, 0, 5053, 0, 0, 0, 0, 0, 1, 7291, 0, 0, 0, 120, 587, 353, 0, 0, 1, 62, 2827, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[694, 0, 25, 0, 0, 2, 0, 226, 0, 0, 6256, 1, 2323, 7820, 1, 4455, 0, 0, 1, 481, 0, 1, 446, 148, 1, 63, 102, 0, 1, 71, 1, 0, 3, 0, 4697, 0, 3, 1, 0, 1, 2, 7556, 1, 0, 0, 123, 594, 445, 0, 4, 8, 57, 3164, 1259, 1, 0, 3, 2, 0, 1, 0, 0, 8108, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[694, 0, 23, 0, 0, 0, 0, 416, 0, 0, 5646, 0, 2476, 7814, 0, 4120, 0, 0, 0, 643, 0, 0, 427, 159, 0, 71, 90, 0, 0, 70, 0, 0, 0, 0, 5375, 0, 0, 0, 0, 0, 1, 7528, 0, 0, 0, 130, 609, 346, 0, 0, 0, 61, 2801, 1529, 2, 0, 0, 0, 0, 0, 0, 0, 8121, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[694, 0, 25, 0, 0, 2, 0, 226, 3330, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2323, 0, 1, 3620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 481, 0, 1, 446, 148, 2926, 2294, 0, 0, 0, 0, 0, 0, 1, 63, 102, 0, 1, 71, 1, 0, 5886, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4697, 0, 3, 1, 0, 1, 835, 174, 0, 0, 0, 0, 0, 0, 2, 5110, 1, 0, 0, 123, 594, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 57, 3164, 1259, 1, 0, 1794, 152, 0, 0, 0, 0, 0, 0, 3, 2, 0, 1, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 5945, 6297, 6125, 5714, 6275, 6439, 6146] avg=6144.0000 max=6439 max_rank=6 min=5714 min_rank=4 imbalance_ratio=1.048014 max_load_reduction=42713 max_load_reduction_pct=0.868998 moved_to_echo=24574 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[664, 0, 45, 0, 4, 0, 0, 449, 3030, 1927, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2490, 0, 0, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 571, 0, 3, 463, 187, 2662, 2173, 0, 0, 0, 0, 0, 0, 2, 52, 92, 4, 0, 82, 0, 1, 5866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5412, 0, 1, 0, 3, 0, 883, 174, 0, 0, 0, 0, 0, 0, 5, 4841, 1, 0, 0, 135, 632, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 49, 2597, 1302, 6, 3, 1791, 144, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 3, 6129, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6119, 6317, 6064, 6099, 6473, 6044, 5893, 6143] avg=6144.0000 max=6473 max_rank=4 min=5893 min_rank=6 imbalance_ratio=1.053548 max_load_reduction=42679 max_load_reduction_pct=0.868306 moved_to_echo=24553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[719, 0, 16, 0, 0, 0, 0, 443, 3099, 1937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2589, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 0, 0, 432, 160, 2724, 2261, 0, 0, 0, 0, 0, 0, 0, 73, 23, 0, 0, 72, 0, 0, 5894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4954, 0, 0, 0, 0, 0, 869, 174, 0, 0, 0, 0, 0, 0, 0, 5037, 0, 0, 0, 120, 558, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 2690, 1353, 2, 0, 1797, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6149, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 6356, 6254, 6062, 5997, 6054, 6066, 6149] avg=6144.0000 max=6356 max_rank=1 min=5997 min_rank=4 imbalance_ratio=1.034505 max_load_reduction=42796 max_load_reduction_pct=0.870687 moved_to_echo=24266 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[714, 0, 22, 0, 0, 0, 0, 338, 3214, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 0, 0, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 438, 114, 2824, 2227, 0, 0, 0, 0, 0, 0, 0, 55, 117, 0, 0, 76, 0, 0, 5898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4855, 0, 0, 0, 0, 0, 822, 174, 0, 0, 0, 0, 0, 0, 0, 4961, 0, 0, 0, 118, 663, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 3088, 1327, 4, 0, 1796, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6226, 5950, 6175, 6146, 5851, 6203, 6455, 6146] avg=6144.0000 max=6455 max_rank=6 min=5851 min_rank=4 imbalance_ratio=1.050618 max_load_reduction=42697 max_load_reduction_pct=0.868673 moved_to_echo=24655 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[679, 0, 40, 0, 0, 0, 2, 436, 2874, 1946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2542, 0, 0, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 676, 0, 0, 444, 113, 2523, 2270, 0, 0, 0, 0, 0, 0, 0, 54, 193, 1, 1, 86, 0, 0, 5917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5310, 0, 2, 2, 0, 0, 771, 174, 0, 0, 0, 0, 0, 0, 0, 5055, 0, 0, 0, 136, 665, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 2956, 1226, 6, 1, 1793, 151, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 1, 1, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 5885, 6030, 6252, 6259, 6379, 6229, 6141] avg=6144.0000 max=6379 max_rank=5 min=5885 min_rank=1 imbalance_ratio=1.038249 max_load_reduction=42773 max_load_reduction_pct=0.870219 moved_to_echo=25008 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[693, 0, 51, 0, 0, 0, 0, 460, 2992, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2581, 0, 0, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 486, 129, 2629, 2182, 0, 0, 0, 0, 0, 0, 0, 67, 129, 0, 0, 81, 0, 0, 5879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5345, 0, 0, 0, 0, 0, 877, 174, 0, 0, 0, 0, 0, 0, 0, 4860, 0, 0, 0, 128, 703, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 53, 2730, 1136, 2, 0, 1795, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6380, 6046, 6156, 6396, 6037, 5867, 6142] avg=6144.0000 max=6396 max_rank=4 min=5867 min_rank=6 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=24442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[664, 0, 45, 0, 4, 0, 0, 449, 3030, 1927, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2490, 0, 0, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 571, 0, 3, 463, 187, 2662, 2173, 0, 0, 0, 0, 0, 0, 2, 52, 92, 4, 0, 82, 0, 1, 5866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5412, 0, 1, 0, 3, 0, 883, 174, 0, 0, 0, 0, 0, 0, 5, 4841, 1, 0, 0, 135, 632, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 49, 2597, 1302, 6, 3, 1791, 144, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 3, 6129, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6119, 6317, 6064, 6099, 6473, 6044, 5893, 6143] avg=6144.0000 max=6473 max_rank=4 min=5893 min_rank=6 imbalance_ratio=1.053548 max_load_reduction=42679 max_load_reduction_pct=0.868306 moved_to_echo=24553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[694, 0, 25, 0, 0, 2, 0, 226, 3330, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2323, 0, 1, 3620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 481, 0, 1, 446, 148, 2926, 2294, 0, 0, 0, 0, 0, 0, 1, 63, 102, 0, 1, 71, 1, 0, 5886, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4697, 0, 3, 1, 0, 1, 835, 174, 0, 0, 0, 0, 0, 0, 2, 5110, 1, 0, 0, 123, 594, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 57, 3164, 1259, 1, 0, 1794, 152, 0, 0, 0, 0, 0, 0, 3, 2, 0, 1, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 5945, 6297, 6125, 5714, 6275, 6439, 6146] avg=6144.0000 max=6439 max_rank=6 min=5714 min_rank=4 imbalance_ratio=1.048014 max_load_reduction=42713 max_load_reduction_pct=0.868998 moved_to_echo=24574 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[677, 0, 92, 0, 0, 0, 0, 283, 3202, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2570, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 0, 0, 403, 161, 2811, 2217, 0, 0, 0, 0, 0, 0, 0, 54, 135, 0, 0, 78, 0, 0, 5933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5053, 0, 0, 0, 0, 0, 910, 178, 0, 0, 0, 0, 0, 0, 1, 4922, 0, 0, 0, 120, 587, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 62, 2827, 1019, 0, 0, 1800, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6495, 6132, 6200, 6141, 5983, 5861, 6135] avg=6144.0000 max=6495 max_rank=1 min=5861 min_rank=6 imbalance_ratio=1.057129 max_load_reduction=42657 max_load_reduction_pct=0.867859 moved_to_echo=24120 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[679, 0, 40, 0, 0, 0, 2, 436, 2874, 1946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2542, 0, 0, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 676, 0, 0, 444, 113, 2523, 2270, 0, 0, 0, 0, 0, 0, 0, 54, 193, 1, 1, 86, 0, 0, 5917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5310, 0, 2, 2, 0, 0, 771, 174, 0, 0, 0, 0, 0, 0, 0, 5055, 0, 0, 0, 136, 665, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 2956, 1226, 6, 1, 1793, 151, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 1, 1, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 5885, 6030, 6252, 6259, 6379, 6229, 6141] avg=6144.0000 max=6379 max_rank=5 min=5885 min_rank=1 imbalance_ratio=1.038249 max_load_reduction=42773 max_load_reduction_pct=0.870219 moved_to_echo=25008 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[694, 0, 23, 0, 0, 0, 0, 416, 3006, 1933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2476, 0, 0, 3348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 0, 0, 427, 159, 2640, 2285, 0, 0, 0, 0, 0, 0, 0, 71, 90, 0, 0, 70, 0, 0, 5881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5375, 0, 0, 0, 0, 0, 772, 174, 0, 0, 0, 0, 0, 0, 1, 5091, 0, 0, 0, 130, 609, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 2801, 1529, 2, 0, 1797, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 5824, 6154, 6112, 6321, 6177, 6342, 6150] avg=6144.0000 max=6342 max_rank=6 min=5824 min_rank=1 imbalance_ratio=1.032227 max_load_reduction=42810 max_load_reduction_pct=0.870972 moved_to_echo=24990 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5632, 0, 384, 0, 128, 128, 128, 3072, 24832, 15616, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 128, 128, 640, 22912, 10240, 128, 128, 14464, 1280, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 19968, 0, 128, 29312, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 4864, 0, 128, 3584, 1280, 21760, 17920, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 39936, 128, 0, 0, 1024, 5120, 3328, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 41088, 0, 128, 128, 128, 128, 6784, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 4864, 0, 128, 3584, 1280, 21760, 17920, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 512, 896, 128, 128, 640, 128, 128, 47232, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[243, 6, 2, 0, 243, 0, 0, 2, 1, 973, 1144, 2315, 2, 5, 349, 2, 1844, 1546, 3, 0, 7736, 2, 0, 1213, 0, 0, 0, 7445, 1, 371, 77, 2, 1, 1, 0, 0, 1173, 624, 271, 0, 2, 3, 0, 0, 0, 0, 0, 2, 1, 3, 0, 2, 458, 7286, 6370, 0, 0, 0, 0, 0, 0, 1, 7427, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[245, 0, 0, 0, 245, 0, 0, 2, 0, 814, 1283, 2005, 0, 0, 288, 4, 1947, 1723, 0, 0, 7729, 0, 0, 1141, 0, 1, 0, 7328, 0, 383, 118, 1, 0, 0, 2, 0, 1319, 577, 282, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 444, 7393, 6409, 3, 0, 0, 0, 0, 0, 0, 7461, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[255, 1, 0, 0, 255, 0, 0, 0, 0, 790, 1161, 2161, 0, 0, 298, 0, 1838, 1478, 0, 0, 7719, 0, 0, 1273, 0, 2, 0, 7336, 0, 323, 128, 0, 0, 0, 1, 0, 1265, 599, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 451, 7389, 6552, 1, 0, 0, 0, 0, 0, 0, 7433, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[248, 1, 3, 0, 248, 0, 0, 1, 0, 672, 1376, 2157, 1, 1, 109, 7, 2101, 1547, 0, 0, 7612, 0, 0, 1276, 0, 0, 0, 7341, 1, 337, 121, 0, 1, 1, 0, 0, 1296, 568, 284, 0, 0, 1, 1, 0, 0, 0, 0, 2, 1, 2, 0, 1, 519, 7465, 6509, 0, 0, 0, 0, 1, 0, 0, 7338, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[252, 0, 0, 0, 250, 0, 0, 1, 0, 782, 1318, 2224, 0, 0, 314, 5, 2181, 1389, 0, 0, 7668, 0, 0, 1124, 0, 3, 0, 7331, 0, 367, 102, 0, 0, 0, 0, 0, 1146, 589, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 400, 7397, 6410, 1, 0, 0, 0, 0, 0, 0, 7470, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[255, 0, 0, 0, 255, 0, 0, 2, 0, 706, 1259, 1905, 0, 0, 339, 7, 2179, 1486, 0, 0, 7669, 0, 0, 1406, 0, 0, 0, 7395, 0, 355, 129, 0, 0, 0, 0, 0, 1234, 659, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 500, 7226, 6452, 0, 0, 0, 0, 0, 0, 0, 7352, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[237, 0, 0, 4, 236, 1, 1, 2, 0, 765, 1050, 2356, 0, 0, 321, 8, 2016, 1433, 4, 0, 7732, 0, 0, 1356, 0, 0, 1, 7397, 4, 367, 62, 0, 1, 0, 3, 5, 1206, 493, 356, 0, 3, 0, 0, 1, 2, 0, 0, 0, 2, 6, 0, 64, 495, 7278, 6468, 0, 2, 4, 3, 1, 0, 3, 7402, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[248, 1, 3, 0, 248, 0, 0, 1, 0, 672, 1376, 2157, 1, 1, 109, 7, 2101, 1547, 0, 0, 7612, 0, 0, 1276, 0, 0, 0, 7341, 1, 337, 121, 0, 1, 1, 0, 0, 1296, 568, 284, 0, 0, 1, 1, 0, 0, 0, 0, 2, 1, 2, 0, 1, 519, 7465, 6509, 0, 0, 0, 0, 1, 0, 0, 7338, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[245, 0, 0, 0, 245, 0, 0, 2, 0, 814, 1283, 2005, 0, 0, 288, 4, 1947, 1723, 0, 0, 7729, 0, 0, 1141, 0, 1, 0, 7328, 0, 383, 118, 1, 0, 0, 2, 0, 1319, 577, 282, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 444, 7393, 6409, 3, 0, 0, 0, 0, 0, 0, 7461, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[248, 1, 0, 0, 248, 0, 0, 2, 0, 867, 1232, 2028, 0, 0, 143, 3, 2006, 1619, 0, 0, 7792, 0, 0, 1304, 0, 0, 0, 7455, 0, 350, 61, 0, 0, 0, 0, 0, 1168, 565, 260, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 461, 7514, 6389, 1, 0, 0, 0, 0, 0, 0, 7429, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[243, 6, 2, 0, 243, 0, 0, 2, 4434, 1213, 0, 0, 0, 0, 0, 0, 1, 973, 1144, 2315, 2, 5, 349, 2, 890, 802, 0, 0, 0, 0, 0, 0, 1844, 1546, 3, 0, 1345, 2, 0, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5737, 1, 371, 77, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1173, 624, 271, 0, 1957, 1708, 384, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 2, 6073, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 2, 458, 0, 5568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6483, 5953, 6188, 6119, 6080, 6032, 6154] avg=6144.0000 max=6483 max_rank=1 min=5953 min_rank=2 imbalance_ratio=1.055176 max_load_reduction=42669 max_load_reduction_pct=0.868103 moved_to_echo=24385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[245, 0, 0, 0, 245, 0, 0, 2, 4430, 1231, 0, 0, 0, 0, 0, 0, 0, 814, 1283, 2005, 0, 0, 288, 4, 894, 807, 0, 0, 0, 0, 0, 0, 1947, 1723, 0, 0, 1344, 0, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5647, 0, 383, 118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1319, 577, 282, 0, 1955, 1681, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 444, 0, 5602, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6095, 6155, 6150, 6202, 6164, 6052, 6181] avg=6144.0000 max=6202 max_rank=4 min=6052 min_rank=6 imbalance_ratio=1.009440 max_load_reduction=42950 max_load_reduction_pct=0.873820 moved_to_echo=24599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[237, 0, 0, 4, 236, 1, 1, 2, 0, 765, 1050, 2356, 0, 0, 321, 8, 2016, 1433, 4, 0, 7732, 0, 0, 1356, 0, 0, 1, 7397, 4, 367, 62, 0, 1, 0, 3, 5, 1206, 493, 356, 0, 3, 0, 0, 1, 2, 0, 0, 0, 2, 6, 0, 64, 495, 7278, 6468, 0, 2, 4, 3, 1, 0, 3, 7402, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[255, 0, 0, 0, 255, 0, 0, 2, 0, 706, 1259, 1905, 0, 0, 339, 7, 2179, 1486, 0, 0, 7669, 0, 0, 1406, 0, 0, 0, 7395, 0, 355, 129, 0, 0, 0, 0, 0, 1234, 659, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 500, 7226, 6452, 0, 0, 0, 0, 0, 0, 0, 7352, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[255, 1, 0, 0, 255, 0, 0, 0, 0, 790, 1161, 2161, 0, 0, 298, 0, 1838, 1478, 0, 0, 7719, 0, 0, 1273, 0, 2, 0, 7336, 0, 323, 128, 0, 0, 0, 1, 0, 1265, 599, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 451, 7389, 6552, 1, 0, 0, 0, 0, 0, 0, 7433, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[252, 0, 0, 0, 250, 0, 0, 1, 4395, 1231, 0, 0, 0, 0, 0, 0, 0, 782, 1318, 2224, 0, 0, 314, 5, 895, 807, 0, 0, 0, 0, 0, 0, 2181, 1389, 0, 0, 1333, 0, 0, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5650, 0, 367, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1146, 589, 322, 0, 1940, 1681, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 400, 0, 5603, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6129, 6345, 6027, 6122, 6064, 6166, 6110, 6189] avg=6144.0000 max=6345 max_rank=1 min=6027 min_rank=2 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[243, 6, 2, 0, 243, 0, 0, 2, 4434, 1213, 0, 0, 0, 0, 0, 0, 1, 973, 1144, 2315, 2, 5, 349, 2, 890, 802, 0, 0, 0, 0, 0, 0, 1844, 1546, 3, 0, 1345, 2, 0, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5737, 1, 371, 77, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1173, 624, 271, 0, 1957, 1708, 384, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 2, 6073, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 2, 458, 0, 5568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6483, 5953, 6188, 6119, 6080, 6032, 6154] avg=6144.0000 max=6483 max_rank=1 min=5953 min_rank=2 imbalance_ratio=1.055176 max_load_reduction=42669 max_load_reduction_pct=0.868103 moved_to_echo=24385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[255, 0, 0, 0, 255, 0, 0, 2, 4395, 1203, 0, 0, 0, 0, 0, 0, 0, 706, 1259, 1905, 0, 0, 339, 7, 881, 812, 0, 0, 0, 0, 0, 0, 2179, 1486, 0, 0, 1334, 0, 0, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5699, 0, 355, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 659, 378, 0, 1940, 1696, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6023, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 500, 0, 5640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6091, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 5909, 6405, 6183, 6287, 6023, 6144, 6091] avg=6144.0000 max=6405 max_rank=2 min=5909 min_rank=1 imbalance_ratio=1.042480 max_load_reduction=42747 max_load_reduction_pct=0.869690 moved_to_echo=24545 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[248, 1, 3, 0, 248, 0, 0, 1, 4362, 1244, 0, 0, 0, 0, 0, 0, 0, 672, 1376, 2157, 1, 1, 109, 7, 879, 819, 0, 0, 0, 0, 0, 0, 2101, 1547, 0, 0, 1325, 0, 0, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5657, 1, 337, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1296, 568, 284, 0, 1925, 1684, 379, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 2, 6221, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 519, 0, 5690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6080, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6107, 6021, 6249, 6116, 6138, 6225, 6213, 6083] avg=6144.0000 max=6249 max_rank=2 min=6021 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=42903 max_load_reduction_pct=0.872864 moved_to_echo=24659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[255, 1, 0, 0, 255, 0, 0, 0, 4427, 1231, 0, 0, 0, 0, 0, 0, 0, 790, 1161, 2161, 0, 0, 298, 0, 893, 828, 0, 0, 0, 0, 0, 0, 1838, 1478, 0, 0, 1336, 0, 0, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5650, 0, 323, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1265, 599, 290, 0, 1956, 1686, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 451, 0, 5724, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6169, 6131, 5925, 6103, 6184, 6158, 6329, 6153] avg=6144.0000 max=6329 max_rank=6 min=5925 min_rank=2 imbalance_ratio=1.030111 max_load_reduction=42823 max_load_reduction_pct=0.871236 moved_to_echo=24824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[245, 0, 0, 0, 245, 0, 0, 2, 4430, 1231, 0, 0, 0, 0, 0, 0, 0, 814, 1283, 2005, 0, 0, 288, 4, 894, 807, 0, 0, 0, 0, 0, 0, 1947, 1723, 0, 0, 1344, 0, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5647, 0, 383, 118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1319, 577, 282, 0, 1955, 1681, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 444, 0, 5602, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6095, 6155, 6150, 6202, 6164, 6052, 6181] avg=6144.0000 max=6202 max_rank=4 min=6052 min_rank=6 imbalance_ratio=1.009440 max_load_reduction=42950 max_load_reduction_pct=0.873820 moved_to_echo=24599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[237, 0, 0, 4, 236, 1, 1, 2, 4431, 1212, 0, 0, 0, 0, 0, 0, 0, 765, 1050, 2356, 0, 0, 321, 8, 887, 814, 0, 0, 0, 0, 0, 0, 2016, 1433, 4, 0, 1345, 0, 0, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5700, 4, 367, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 5, 1206, 493, 356, 0, 1956, 1697, 383, 0, 0, 0, 0, 0, 3, 0, 0, 1, 2, 0, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 64, 495, 0, 5654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 3, 1, 0, 3, 6132, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6201, 6154, 6134, 6100, 6072, 6221, 6146] avg=6144.0000 max=6221 max_rank=6 min=6072 min_rank=5 imbalance_ratio=1.012533 max_load_reduction=42931 max_load_reduction_pct=0.873433 moved_to_echo=24539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[248, 1, 0, 0, 248, 0, 0, 2, 4466, 1252, 0, 0, 0, 0, 0, 0, 0, 867, 1232, 2028, 0, 0, 143, 3, 890, 804, 0, 0, 0, 0, 0, 0, 2006, 1619, 0, 0, 1355, 0, 0, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5745, 0, 350, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 565, 260, 0, 1971, 1710, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 461, 0, 5585, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6217, 5967, 6284, 6156, 6058, 6264, 6051, 6155] avg=6144.0000 max=6284 max_rank=2 min=5967 min_rank=1 imbalance_ratio=1.022786 max_load_reduction=42868 max_load_reduction_pct=0.872152 moved_to_echo=24528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[255, 1, 0, 0, 255, 0, 0, 0, 4427, 1231, 0, 0, 0, 0, 0, 0, 0, 790, 1161, 2161, 0, 0, 298, 0, 893, 828, 0, 0, 0, 0, 0, 0, 1838, 1478, 0, 0, 1336, 0, 0, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5650, 0, 323, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1265, 599, 290, 0, 1956, 1686, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 451, 0, 5724, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6169, 6131, 5925, 6103, 6184, 6158, 6329, 6153] avg=6144.0000 max=6329 max_rank=6 min=5925 min_rank=2 imbalance_ratio=1.030111 max_load_reduction=42823 max_load_reduction_pct=0.871236 moved_to_echo=24824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 6400, 9856, 17152, 128, 128, 2176, 128, 7168, 6528, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[16128, 12288, 128, 0, 10752, 128, 0, 10112, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[2048, 128, 128, 128, 2048, 128, 128, 128, 35456, 9856, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[16128, 12288, 128, 0, 10752, 128, 0, 10112, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 9856, 4736, 2560, 0, 15616, 13568, 3072, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 384, 3840, 0, 45184, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 128, 45568, 128, 2944, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 128, 45568, 128, 2944, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[7466, 60, 1, 2689, 36, 0, 227, 0, 0, 0, 1, 299, 0, 0, 9, 8042, 0, 7152, 5, 0, 0, 0, 4, 4, 0, 2414, 0, 0, 1, 64, 1, 0, 3915, 0, 0, 1, 0, 1, 11, 0, 0, 3, 0, 0, 7, 3, 5, 6764, 359, 4, 0, 0, 0, 21, 0, 0, 0, 2, 0, 325, 550, 233, 438, 8035] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[7403, 75, 0, 2791, 40, 0, 193, 0, 0, 0, 0, 318, 0, 0, 0, 7931, 0, 6913, 1, 0, 0, 0, 0, 0, 0, 2795, 0, 0, 0, 74, 0, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6861, 303, 0, 0, 0, 0, 42, 0, 0, 4, 0, 0, 344, 550, 234, 487, 8007] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7324, 72, 0, 2738, 43, 0, 208, 0, 0, 0, 0, 298, 0, 0, 0, 8044, 0, 7126, 0, 0, 0, 0, 0, 0, 0, 2660, 0, 0, 0, 71, 0, 0, 3835, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6776, 336, 0, 0, 0, 0, 38, 0, 0, 13, 0, 0, 275, 621, 213, 463, 7997] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[7391, 81, 0, 2699, 71, 1, 201, 0, 1, 0, 4, 298, 8, 0, 3, 8013, 0, 7063, 0, 0, 1, 3, 6, 1, 2, 2618, 0, 4, 0, 89, 1, 0, 3726, 2, 1, 0, 0, 1, 1, 7, 2, 1, 0, 7, 0, 2, 1, 6789, 352, 0, 0, 0, 0, 49, 1, 0, 4, 1, 0, 329, 584, 281, 478, 7974] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[7372, 75, 0, 2473, 61, 0, 219, 0, 0, 0, 0, 305, 0, 0, 0, 8133, 0, 7036, 1, 0, 0, 0, 0, 0, 0, 2756, 0, 0, 0, 72, 0, 0, 3848, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6629, 338, 0, 0, 0, 0, 37, 0, 0, 1, 0, 0, 359, 647, 287, 482, 8017] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7324, 72, 0, 2738, 43, 0, 208, 0, 0, 0, 0, 298, 0, 0, 0, 8044, 0, 7126, 0, 0, 0, 0, 0, 0, 0, 2660, 0, 0, 0, 71, 0, 0, 3835, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6776, 336, 0, 0, 0, 0, 38, 0, 0, 13, 0, 0, 275, 621, 213, 463, 7997] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[7445, 86, 3, 2507, 65, 3, 194, 0, 0, 0, 2, 288, 2, 0, 2, 8129, 0, 6995, 0, 0, 1, 0, 2, 0, 0, 3157, 0, 0, 0, 86, 1, 0, 3310, 0, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 6740, 319, 0, 0, 0, 0, 56, 1, 0, 9, 3, 0, 266, 606, 268, 551, 8049] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[7421, 70, 0, 2758, 49, 0, 175, 0, 0, 0, 0, 307, 0, 0, 0, 8177, 0, 7125, 0, 0, 0, 0, 0, 0, 0, 2763, 0, 0, 0, 68, 0, 0, 3367, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6961, 321, 0, 0, 0, 0, 58, 0, 0, 4, 0, 0, 286, 474, 232, 460, 8073] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[7502, 81, 0, 2856, 69, 0, 205, 0, 0, 0, 0, 298, 0, 0, 1, 8181, 0, 7090, 0, 0, 0, 0, 0, 0, 0, 2370, 0, 0, 0, 79, 0, 0, 3616, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6754, 373, 0, 1, 0, 0, 32, 0, 0, 5, 0, 0, 274, 562, 284, 461, 8056] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3144, 60, 1, 2689, 36, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 299, 0, 0, 9, 5812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 5, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2414, 0, 0, 1, 64, 1, 0, 2032, 1338, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 1, 0, 1, 11, 0, 933, 892, 642, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 3, 5, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 359, 4, 0, 0, 0, 21, 0, 0, 4322, 1472, 0, 0, 0, 0, 0, 0, 0, 2, 0, 325, 550, 233, 438, 4531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6121, 6232, 5850, 6395, 6140, 6178, 6079] avg=6144.0000 max=6395 max_rank=4 min=5850 min_rank=3 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[7466, 60, 1, 2689, 36, 0, 227, 0, 0, 0, 1, 299, 0, 0, 9, 8042, 0, 7152, 5, 0, 0, 0, 4, 4, 0, 2414, 0, 0, 1, 64, 1, 0, 3915, 0, 0, 1, 0, 1, 11, 0, 0, 3, 0, 0, 7, 3, 5, 6764, 359, 4, 0, 0, 0, 21, 0, 0, 0, 2, 0, 325, 550, 233, 438, 8035] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[7421, 70, 0, 2758, 49, 0, 175, 0, 0, 0, 0, 307, 0, 0, 0, 8177, 0, 7125, 0, 0, 0, 0, 0, 0, 0, 2763, 0, 0, 0, 68, 0, 0, 3367, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6961, 321, 0, 0, 0, 0, 58, 0, 0, 4, 0, 0, 286, 474, 232, 460, 8073] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[7403, 75, 0, 2791, 40, 0, 193, 0, 0, 0, 0, 318, 0, 0, 0, 7931, 0, 6913, 1, 0, 0, 0, 0, 0, 0, 2795, 0, 0, 0, 74, 0, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6861, 303, 0, 0, 0, 0, 42, 0, 0, 4, 0, 0, 344, 550, 234, 487, 8007] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[7445, 86, 3, 2507, 65, 3, 194, 0, 0, 0, 2, 288, 2, 0, 2, 8129, 0, 6995, 0, 0, 1, 0, 2, 0, 0, 3157, 0, 0, 0, 86, 1, 0, 3310, 0, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 6740, 319, 0, 0, 0, 0, 56, 1, 0, 9, 3, 0, 266, 606, 268, 551, 8049] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3084, 72, 0, 2738, 43, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 5812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2660, 0, 0, 0, 71, 0, 0, 2022, 1339, 0, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 0, 0, 1, 0, 930, 893, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 0, 0, 0, 38, 0, 0, 4240, 1465, 0, 0, 0, 0, 0, 0, 13, 0, 0, 275, 621, 213, 463, 4510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6110, 6196, 6092, 6302, 6133, 6079, 6095] avg=6144.0000 max=6302 max_rank=4 min=6079 min_rank=6 imbalance_ratio=1.025716 max_load_reduction=42850 max_load_reduction_pct=0.871785 moved_to_echo=24609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3104, 75, 0, 2473, 61, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 5877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2756, 0, 0, 0, 72, 0, 0, 2027, 1354, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 0, 0, 4, 0, 918, 902, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6000, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 37, 0, 0, 4268, 1468, 0, 0, 0, 0, 0, 0, 1, 0, 0, 359, 647, 287, 482, 4522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5932, 6182, 6119, 6209, 6301, 6000, 6111, 6298] avg=6144.0000 max=6301 max_rank=4 min=5932 min_rank=0 imbalance_ratio=1.025553 max_load_reduction=42851 max_load_reduction_pct=0.871806 moved_to_echo=24710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3112, 81, 0, 2699, 71, 1, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 298, 8, 0, 3, 5790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 1, 3, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2618, 0, 4, 0, 89, 1, 0, 2017, 1334, 0, 0, 0, 0, 0, 0, 3726, 2, 1, 0, 0, 1, 1, 7, 922, 889, 644, 0, 0, 0, 0, 0, 2, 1, 0, 7, 0, 2, 1, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 0, 49, 1, 0, 4279, 1460, 0, 0, 0, 0, 0, 0, 4, 1, 0, 329, 584, 281, 478, 4497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6165, 6104, 6152, 6065, 6193, 6158, 6141, 6174] avg=6144.0000 max=6193 max_rank=4 min=6065 min_rank=3 imbalance_ratio=1.007975 max_load_reduction=42959 max_load_reduction_pct=0.874003 moved_to_echo=24666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3114, 75, 0, 2791, 40, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 5723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6007, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 0, 0, 0, 74, 0, 0, 2029, 1323, 0, 0, 0, 0, 0, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 906, 885, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6205, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 0, 42, 0, 0, 4289, 1471, 0, 0, 0, 0, 0, 0, 4, 0, 0, 344, 550, 234, 487, 4507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6213, 6041, 6008, 6221, 6233, 6205, 6105, 6126] avg=6144.0000 max=6233 max_rank=4 min=6008 min_rank=2 imbalance_ratio=1.014486 max_load_reduction=42919 max_load_reduction_pct=0.873189 moved_to_echo=24669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3135, 86, 3, 2507, 65, 3, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 288, 2, 0, 2, 5874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6082, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3157, 0, 0, 0, 86, 1, 0, 2036, 1353, 0, 0, 0, 0, 0, 0, 3310, 0, 0, 0, 0, 2, 0, 1, 913, 902, 640, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 56, 1, 0, 4310, 1474, 0, 0, 0, 0, 0, 0, 9, 3, 0, 266, 606, 268, 551, 4539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5993, 6168, 6085, 6633, 5768, 6103, 6160, 6242] avg=6144.0000 max=6633 max_rank=3 min=5768 min_rank=4 imbalance_ratio=1.079590 max_load_reduction=42519 max_load_reduction_pct=0.865051 moved_to_echo=24273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3104, 75, 0, 2473, 61, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 5877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2756, 0, 0, 0, 72, 0, 0, 2027, 1354, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 0, 0, 4, 0, 918, 902, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6000, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 37, 0, 0, 4268, 1468, 0, 0, 0, 0, 0, 0, 1, 0, 0, 359, 647, 287, 482, 4522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5932, 6182, 6119, 6209, 6301, 6000, 6111, 6298] avg=6144.0000 max=6301 max_rank=4 min=5932 min_rank=0 imbalance_ratio=1.025553 max_load_reduction=42851 max_load_reduction_pct=0.871806 moved_to_echo=24710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3159, 81, 0, 2856, 69, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 1, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2370, 0, 0, 0, 79, 0, 0, 2037, 1362, 0, 0, 0, 0, 0, 0, 3616, 0, 0, 0, 0, 0, 2, 0, 925, 908, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 373, 0, 1, 0, 0, 32, 0, 0, 4343, 1475, 0, 0, 0, 0, 0, 0, 5, 0, 0, 274, 562, 284, 461, 4544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6370, 6210, 6165, 5848, 6092, 6113, 6224, 6130] avg=6144.0000 max=6370 max_rank=0 min=5848 min_rank=3 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3125, 70, 0, 2758, 49, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 5909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2763, 0, 0, 0, 68, 0, 0, 2042, 1361, 0, 0, 0, 0, 0, 0, 3367, 0, 0, 0, 0, 0, 3, 0, 930, 907, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6300, 0, 0, 0, 0, 0, 0, 0, 0, 321, 0, 0, 0, 0, 58, 0, 0, 4296, 1479, 0, 0, 0, 0, 0, 0, 4, 0, 0, 286, 474, 232, 460, 4552, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6216, 6195, 6234, 5868, 6300, 6154, 6008] avg=6144.0000 max=6300 max_rank=5 min=5868 min_rank=4 imbalance_ratio=1.025391 max_load_reduction=42852 max_load_reduction_pct=0.871826 moved_to_echo=24330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3135, 86, 3, 2507, 65, 3, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 288, 2, 0, 2, 5874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6082, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3157, 0, 0, 0, 86, 1, 0, 2036, 1353, 0, 0, 0, 0, 0, 0, 3310, 0, 0, 0, 0, 2, 0, 1, 913, 902, 640, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 56, 1, 0, 4310, 1474, 0, 0, 0, 0, 0, 0, 9, 3, 0, 266, 606, 268, 551, 4539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5993, 6168, 6085, 6633, 5768, 6103, 6160, 6242] avg=6144.0000 max=6633 max_rank=3 min=5768 min_rank=4 imbalance_ratio=1.079590 max_load_reduction=42519 max_load_reduction_pct=0.865051 moved_to_echo=24273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3144, 60, 1, 2689, 36, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 299, 0, 0, 9, 5812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 5, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2414, 0, 0, 1, 64, 1, 0, 2032, 1338, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 1, 0, 1, 11, 0, 933, 892, 642, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 3, 5, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 359, 4, 0, 0, 0, 21, 0, 0, 4322, 1472, 0, 0, 0, 0, 0, 0, 0, 2, 0, 325, 550, 233, 438, 4531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6121, 6232, 5850, 6395, 6140, 6178, 6079] avg=6144.0000 max=6395 max_rank=4 min=5850 min_rank=3 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3114, 75, 0, 2791, 40, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 5723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6007, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 0, 0, 0, 74, 0, 0, 2029, 1323, 0, 0, 0, 0, 0, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 906, 885, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6205, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 0, 42, 0, 0, 4289, 1471, 0, 0, 0, 0, 0, 0, 4, 0, 0, 344, 550, 234, 487, 4507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6213, 6041, 6008, 6221, 6233, 6205, 6105, 6126] avg=6144.0000 max=6233 max_rank=4 min=6008 min_rank=2 imbalance_ratio=1.014486 max_load_reduction=42919 max_load_reduction_pct=0.873189 moved_to_echo=24669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 21632, 0, 128, 128, 640, 128, 0, 16256, 10880, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 128, 128, 0, 0, 384, 128, 0, 34432, 11776, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[29440, 128, 128, 128, 0, 128, 128, 128, 7424, 7296, 5248, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[25088, 640, 128, 21632, 512, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 2560, 4608, 2048, 3840, 36224, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 128, 2432, 128, 0, 128, 46720, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 49152, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 49152, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 7125, 1, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 8192, 0, 0, 364, 8125, 0, 423, 0, 6, 0, 0, 0, 2257, 0, 0, 72, 0, 0, 4557, 7613, 0, 373, 0, 0, 0, 283, 0, 0, 0, 8068, 0, 0, 983, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 7053, 0, 0, 0, 56, 0, 0, 0, 1, 1, 1, 1, 434, 0, 3, 0, 0, 0, 8185, 0, 0, 324, 8121, 0, 399, 0, 0, 0, 0, 0, 2344, 1, 0, 81, 0, 1, 4575, 7559, 0, 377, 0, 4, 3, 287, 1, 2, 2, 8112, 2, 0, 1108, 0, 2, 0, 2, 1, 2, 0, 96, 2, 1, 1, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 7086, 0, 0, 0, 89, 0, 1, 0, 0, 0, 0, 0, 384, 0, 1, 0, 0, 0, 8192, 0, 0, 320, 8152, 0, 424, 0, 0, 0, 0, 0, 2176, 0, 0, 76, 0, 0, 4876, 7518, 0, 379, 0, 0, 0, 291, 0, 0, 0, 8083, 0, 0, 993, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7033, 0, 0, 0, 47, 0, 0, 0, 10, 6, 0, 0, 445, 0, 1, 0, 0, 0, 8182, 1, 3, 307, 8132, 0, 437, 1, 2, 0, 0, 0, 2349, 0, 0, 88, 1, 1, 4626, 7544, 3, 346, 1, 7, 8, 318, 3, 0, 1, 8102, 0, 0, 989, 0, 0, 0, 5, 0, 1, 1, 146, 0, 0, 4, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 7001, 0, 0, 1, 79, 0, 0, 0, 0, 2, 3, 1, 416, 0, 0, 0, 0, 0, 8188, 0, 2, 376, 8132, 0, 430, 1, 0, 0, 0, 0, 2347, 0, 0, 87, 2, 0, 4842, 7516, 0, 384, 0, 1, 0, 273, 0, 0, 2, 8061, 0, 0, 839, 0, 3, 0, 2, 0, 2, 0, 158, 1, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 7086, 0, 0, 0, 89, 0, 1, 0, 0, 0, 0, 0, 384, 0, 1, 0, 0, 0, 8192, 0, 0, 320, 8152, 0, 424, 0, 0, 0, 0, 0, 2176, 0, 0, 76, 0, 0, 4876, 7518, 0, 379, 0, 0, 0, 291, 0, 0, 0, 8083, 0, 0, 993, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 7063, 0, 0, 0, 45, 0, 2, 0, 0, 0, 0, 0, 472, 0, 0, 0, 0, 0, 8192, 0, 0, 375, 8169, 0, 360, 0, 0, 0, 0, 0, 2159, 0, 0, 81, 0, 0, 4677, 7584, 0, 338, 0, 0, 0, 392, 0, 0, 0, 8073, 0, 0, 979, 0, 0, 0, 0, 0, 1, 0, 190, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 7074, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 8192, 0, 0, 338, 8141, 0, 462, 0, 0, 0, 0, 0, 2486, 0, 0, 71, 0, 0, 4318, 7556, 0, 341, 0, 0, 0, 297, 0, 0, 0, 8106, 0, 0, 1128, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 7082, 0, 0, 0, 60, 0, 6, 0, 0, 0, 0, 0, 358, 0, 0, 0, 0, 0, 8192, 0, 0, 413, 8144, 0, 440, 0, 0, 0, 0, 0, 2301, 0, 0, 78, 0, 0, 4470, 7568, 0, 392, 0, 0, 0, 324, 0, 0, 0, 8099, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6070, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 434, 0, 3, 3516, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 5778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 2344, 2343, 983, 85, 0, 0, 0, 0, 0, 1, 0, 81, 0, 1, 4575, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 4, 3, 287, 1, 2, 2, 2878, 2595, 0, 0, 0, 0, 0, 0, 5149, 2, 0, 1108, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 96, 2, 1, 1, 7, 5994, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6148, 6102, 6154, 6106, 6149, 6263, 6104] avg=6144.0000 max=6263 max_rank=6 min=6102 min_rank=2 imbalance_ratio=1.019368 max_load_reduction=42889 max_load_reduction_pct=0.872579 moved_to_echo=24622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 7074, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 8192, 0, 0, 338, 8141, 0, 462, 0, 0, 0, 0, 0, 2486, 0, 0, 71, 0, 0, 4318, 7556, 0, 341, 0, 0, 0, 297, 0, 0, 0, 8106, 0, 0, 1128, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 7053, 0, 0, 0, 56, 0, 0, 0, 1, 1, 1, 1, 434, 0, 3, 0, 0, 0, 8185, 0, 0, 324, 8121, 0, 399, 0, 0, 0, 0, 0, 2344, 1, 0, 81, 0, 1, 4575, 7559, 0, 377, 0, 4, 3, 287, 1, 2, 2, 8112, 2, 0, 1108, 0, 2, 0, 2, 1, 2, 0, 96, 2, 1, 1, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6132, 1, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 3541, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 5781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 6, 0, 0, 0, 2257, 2344, 993, 84, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 4557, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 283, 0, 0, 0, 2862, 2614, 0, 0, 0, 0, 0, 0, 5122, 0, 0, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 5999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6206, 6217, 6145, 6107, 6087, 6132, 6105, 6153] avg=6144.0000 max=6217 max_rank=1 min=6087 min_rank=4 imbalance_ratio=1.011882 max_load_reduction=42935 max_load_reduction_pct=0.873515 moved_to_echo=24477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6053, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 6, 0, 0, 445, 0, 1, 3509, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 307, 5786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 1, 2, 0, 0, 0, 2349, 2346, 980, 85, 0, 0, 0, 0, 0, 0, 0, 88, 1, 1, 4626, 1445, 3, 0, 0, 0, 0, 0, 0, 0, 0, 346, 1, 7, 8, 318, 3, 0, 1, 2874, 2590, 0, 0, 0, 0, 0, 0, 5143, 0, 0, 989, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 146, 0, 0, 4, 1, 5992, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6161, 6097, 6200, 6164, 6148, 6137, 6145] avg=6144.0000 max=6200 max_rank=3 min=6097 min_rank=2 imbalance_ratio=1.009115 max_load_reduction=42952 max_load_reduction_pct=0.873861 moved_to_echo=24594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6098, 0, 0, 0, 89, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 1, 3497, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 5801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 2176, 2351, 988, 84, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 4876, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 291, 0, 0, 0, 2868, 2581, 0, 0, 0, 0, 0, 0, 5131, 0, 0, 993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 5999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6188, 6075, 6121, 6023, 6392, 6119, 6124, 6110] avg=6144.0000 max=6392 max_rank=4 min=6023 min_rank=3 imbalance_ratio=1.040365 max_load_reduction=42760 max_load_reduction_pct=0.869954 moved_to_echo=24745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6025, 0, 0, 1, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 1, 416, 0, 0, 3496, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 376, 5786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 1, 0, 0, 0, 0, 2347, 2346, 976, 84, 0, 0, 0, 0, 0, 0, 0, 87, 2, 0, 4842, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 1, 0, 273, 0, 0, 2, 2860, 2581, 0, 0, 0, 0, 0, 0, 5117, 0, 0, 839, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 158, 1, 0, 0, 0, 5996, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 6110, 6164, 6184, 6370, 6101, 5961, 6157] avg=6144.0000 max=6370 max_rank=4 min=5961 min_rank=6 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6088, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 3515, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 5793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 2486, 2348, 986, 85, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 4318, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 297, 0, 0, 0, 2876, 2594, 0, 0, 0, 0, 0, 0, 5145, 0, 0, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 5999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6170, 6131, 6367, 5836, 6108, 6273, 6123] avg=6144.0000 max=6367 max_rank=3 min=5836 min_rank=4 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6079, 0, 0, 0, 45, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 0, 3528, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 5813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 2159, 2356, 984, 84, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 4677, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 392, 0, 0, 0, 2864, 2604, 0, 0, 0, 0, 0, 0, 5125, 0, 0, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 190, 0, 0, 0, 0, 5999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6193, 6188, 5943, 6210, 6198, 6104, 6190] avg=6144.0000 max=6210 max_rank=4 min=5943 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=42942 max_load_reduction_pct=0.873657 moved_to_echo=24702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6053, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 6, 0, 0, 445, 0, 1, 3509, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 307, 5786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 1, 2, 0, 0, 0, 2349, 2346, 980, 85, 0, 0, 0, 0, 0, 0, 0, 88, 1, 1, 4626, 1445, 3, 0, 0, 0, 0, 0, 0, 0, 0, 346, 1, 7, 8, 318, 3, 0, 1, 2874, 2590, 0, 0, 0, 0, 0, 0, 5143, 0, 0, 989, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 146, 0, 0, 4, 1, 5992, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6161, 6097, 6200, 6164, 6148, 6137, 6145] avg=6144.0000 max=6200 max_rank=3 min=6097 min_rank=2 imbalance_ratio=1.009115 max_load_reduction=42952 max_load_reduction_pct=0.873861 moved_to_echo=24594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6070, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 434, 0, 3, 3516, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 5778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 2344, 2343, 983, 85, 0, 0, 0, 0, 0, 1, 0, 81, 0, 1, 4575, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 4, 3, 287, 1, 2, 2, 2878, 2595, 0, 0, 0, 0, 0, 0, 5149, 2, 0, 1108, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 96, 2, 1, 1, 7, 5994, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6148, 6102, 6154, 6106, 6149, 6263, 6104] avg=6144.0000 max=6263 max_rank=6 min=6102 min_rank=2 imbalance_ratio=1.019368 max_load_reduction=42889 max_load_reduction_pct=0.872579 moved_to_echo=24622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6091, 0, 0, 0, 60, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 0, 3526, 2194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 5791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 2301, 2353, 991, 89, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 4470, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 0, 324, 0, 0, 0, 2878, 2603, 0, 0, 0, 0, 0, 0, 5132, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 5998, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6078, 6204, 6174, 5987, 6197, 6185, 6170] avg=6144.0000 max=6204 max_rank=2 min=5987 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=42948 max_load_reduction_pct=0.873779 moved_to_echo=24539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6088, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 3515, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 5793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 2486, 2348, 986, 85, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 4318, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 297, 0, 0, 0, 2876, 2594, 0, 0, 0, 0, 0, 0, 5145, 0, 0, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 5999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6170, 6131, 6367, 5836, 6108, 6273, 6123] avg=6144.0000 max=6367 max_rank=3 min=5836 min_rank=4 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 3456, 0, 128, 28160, 17664, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 3456, 128, 128, 0, 0, 0, 18432, 18816, 7936, 768, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 640, 128, 128, 36992, 11648, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[41088, 128, 0, 8192, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[2944, 128, 128, 128, 2560, 128, 128, 128, 23040, 20864, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 2944, 46336, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 48640, 128, 0, 128, 512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 2944, 46336, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 1152, 128, 128, 128, 128, 48000, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 922, 0, 520, 3, 3, 6830, 0, 0, 0, 0, 3, 2, 6, 0, 0, 5515, 0, 87, 3, 0, 0, 227, 8, 0, 0, 0, 82, 0, 0, 216, 71, 7973, 8187, 8108, 1, 1, 1, 0, 0, 2193, 7, 1, 0, 5, 58, 4, 0, 5, 0, 0, 8097, 0, 2, 6, 3, 1, 0, 0, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 930, 0, 464, 0, 0, 6975, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5339, 0, 26, 0, 0, 0, 206, 4, 0, 0, 0, 145, 0, 0, 251, 70, 7947, 8192, 8119, 0, 0, 0, 0, 0, 2303, 4, 0, 0, 0, 52, 0, 0, 0, 0, 0, 8120, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 772, 0, 454, 0, 0, 6854, 0, 0, 0, 0, 0, 0, 10, 0, 0, 5612, 0, 13, 0, 0, 0, 193, 46, 0, 0, 0, 136, 0, 0, 194, 77, 7993, 8192, 8113, 0, 0, 0, 0, 0, 2334, 1, 0, 0, 0, 53, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 565, 0, 505, 0, 0, 6799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5257, 0, 12, 0, 0, 0, 204, 36, 0, 0, 0, 129, 0, 0, 198, 81, 7991, 8192, 8108, 0, 0, 0, 0, 0, 2924, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 602, 0, 468, 0, 0, 6768, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5537, 0, 9, 0, 0, 0, 217, 22, 0, 0, 0, 163, 0, 0, 235, 75, 7966, 8192, 8116, 0, 0, 0, 0, 0, 2631, 9, 0, 0, 0, 23, 0, 0, 0, 0, 0, 8114, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 733, 1, 401, 0, 3, 6862, 1, 0, 2, 1, 2, 1, 11, 0, 0, 5698, 0, 4, 0, 0, 0, 192, 1, 0, 0, 1, 141, 2, 0, 301, 76, 7935, 8189, 8097, 1, 1, 0, 0, 0, 2270, 57, 1, 0, 0, 56, 1, 0, 3, 0, 0, 8101, 1, 0, 2, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 491, 7, 491, 0, 7, 6725, 1, 0, 7, 1, 1, 4, 10, 0, 0, 5760, 0, 18, 5, 0, 0, 200, 28, 0, 1, 0, 116, 2, 0, 214, 81, 7976, 8181, 8090, 0, 0, 1, 0, 0, 2541, 24, 1, 3, 1, 49, 2, 1, 5, 0, 1, 8084, 0, 4, 9, 8, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 565, 0, 505, 0, 0, 6799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5257, 0, 12, 0, 0, 0, 204, 36, 0, 0, 0, 129, 0, 0, 198, 81, 7991, 8192, 8108, 0, 0, 0, 0, 0, 2924, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 772, 0, 454, 0, 0, 6854, 0, 0, 0, 0, 0, 0, 10, 0, 0, 5612, 0, 13, 0, 0, 0, 193, 46, 0, 0, 0, 136, 0, 0, 194, 77, 7993, 8192, 8113, 0, 0, 0, 0, 0, 2334, 1, 0, 0, 0, 53, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 922, 0, 520, 3, 3, 6830, 0, 0, 0, 0, 3, 2, 6, 0, 0, 5515, 0, 87, 3, 0, 0, 227, 8, 0, 0, 0, 82, 0, 0, 216, 71, 7973, 8187, 8108, 1, 1, 1, 0, 0, 2193, 7, 1, 0, 5, 58, 4, 0, 5, 0, 0, 8097, 0, 2, 6, 3, 1, 0, 0, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 542, 0, 391, 0, 0, 6865, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5806, 0, 17, 0, 0, 0, 175, 7, 0, 0, 0, 143, 0, 0, 208, 70, 7981, 8192, 8117, 0, 0, 0, 0, 0, 2398, 54, 0, 0, 0, 66, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 602, 0, 468, 0, 0, 6768, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5537, 0, 9, 0, 0, 0, 217, 22, 0, 0, 0, 163, 0, 0, 235, 75, 7966, 8192, 8116, 0, 0, 0, 0, 0, 2631, 9, 0, 0, 0, 23, 0, 0, 0, 0, 0, 8114, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 922, 0, 520, 3, 3, 4368, 616, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 3, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5515, 0, 87, 3, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 227, 8, 0, 0, 0, 82, 0, 0, 3740, 2049, 0, 0, 0, 0, 0, 0, 216, 71, 5844, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2193, 7, 1, 0, 5, 58, 1963, 1513, 145, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 6134, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 1, 0, 0, 0, 1, 0, 6138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6432, 6143, 6158, 6106, 6134, 5885, 6145, 6149] avg=6144.0000 max=6432 max_rank=0 min=5885 min_rank=5 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24313 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 930, 0, 464, 0, 0, 4374, 614, 0, 0, 0, 0, 0, 0, 6262, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5339, 0, 26, 0, 0, 0, 565, 0, 0, 0, 0, 0, 0, 0, 206, 4, 0, 0, 0, 145, 0, 0, 3745, 2051, 0, 0, 0, 0, 0, 0, 251, 70, 5825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 4, 0, 0, 0, 52, 1969, 1508, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6382, 6264, 5930, 6151, 6146, 5984, 6151, 6144] avg=6144.0000 max=6382 max_rank=0 min=5930 min_rank=2 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24425 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 733, 1, 401, 0, 3, 4363, 613, 0, 0, 0, 0, 0, 0, 6161, 1, 0, 2, 1, 2, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5698, 0, 4, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 192, 1, 0, 0, 1, 141, 2, 0, 3734, 2051, 0, 0, 0, 0, 0, 0, 301, 76, 5816, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2270, 57, 1, 0, 0, 56, 1964, 1506, 145, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 6137, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6179, 6258, 6122, 6195, 5999, 6142, 6143] avg=6144.0000 max=6258 max_rank=2 min=5999 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=42894 max_load_reduction_pct=0.872681 moved_to_echo=24479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 922, 0, 520, 3, 3, 4368, 616, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 3, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5515, 0, 87, 3, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 227, 8, 0, 0, 0, 82, 0, 0, 3740, 2049, 0, 0, 0, 0, 0, 0, 216, 71, 5844, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2193, 7, 1, 0, 5, 58, 1963, 1513, 145, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 6134, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 1, 0, 0, 0, 1, 0, 6138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6432, 6143, 6158, 6106, 6134, 5885, 6145, 6149] avg=6144.0000 max=6432 max_rank=0 min=5885 min_rank=5 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24313 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 602, 0, 468, 0, 0, 4372, 615, 0, 0, 0, 0, 0, 0, 6077, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5537, 0, 9, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 217, 22, 0, 0, 0, 163, 0, 0, 3744, 2051, 0, 0, 0, 0, 0, 0, 235, 75, 5839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 9, 0, 0, 0, 23, 1967, 1512, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6057, 6080, 6094, 6197, 6149, 6285, 6147, 6143] avg=6144.0000 max=6285 max_rank=5 min=6057 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=42867 max_load_reduction_pct=0.872131 moved_to_echo=24724 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 565, 0, 505, 0, 0, 4368, 617, 0, 0, 0, 0, 0, 0, 6104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5257, 0, 12, 0, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 204, 36, 0, 0, 0, 129, 0, 0, 3740, 2051, 0, 0, 0, 0, 0, 0, 198, 81, 5858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2924, 16, 0, 0, 0, 24, 1967, 1516, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6055, 6104, 5820, 6160, 6137, 6591, 6144, 6141] avg=6144.0000 max=6591 max_rank=5 min=5820 min_rank=2 imbalance_ratio=1.072754 max_load_reduction=42561 max_load_reduction_pct=0.865906 moved_to_echo=25013 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 733, 1, 401, 0, 3, 4363, 613, 0, 0, 0, 0, 0, 0, 6161, 1, 0, 2, 1, 2, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5698, 0, 4, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 192, 1, 0, 0, 1, 141, 2, 0, 3734, 2051, 0, 0, 0, 0, 0, 0, 301, 76, 5816, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2270, 57, 1, 0, 0, 56, 1964, 1506, 145, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 6137, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6179, 6258, 6122, 6195, 5999, 6142, 6143] avg=6144.0000 max=6258 max_rank=2 min=5999 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=42894 max_load_reduction_pct=0.872681 moved_to_echo=24479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 491, 7, 491, 0, 7, 4358, 616, 0, 0, 0, 0, 0, 0, 6038, 1, 0, 7, 1, 1, 4, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5760, 0, 18, 5, 0, 0, 545, 0, 0, 0, 0, 0, 0, 0, 200, 28, 0, 1, 0, 116, 2, 0, 3732, 2048, 0, 0, 0, 0, 0, 0, 214, 81, 5846, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2541, 24, 1, 3, 1, 49, 1960, 1514, 142, 0, 0, 0, 0, 0, 2, 1, 5, 0, 1, 6124, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 0, 1, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5970, 6062, 6328, 6127, 6142, 6235, 6137, 6151] avg=6144.0000 max=6328 max_rank=2 min=5970 min_rank=0 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 602, 0, 468, 0, 0, 4372, 615, 0, 0, 0, 0, 0, 0, 6077, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5537, 0, 9, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 217, 22, 0, 0, 0, 163, 0, 0, 3744, 2051, 0, 0, 0, 0, 0, 0, 235, 75, 5839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 9, 0, 0, 0, 23, 1967, 1512, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6057, 6080, 6094, 6197, 6149, 6285, 6147, 6143] avg=6144.0000 max=6285 max_rank=5 min=6057 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=42867 max_load_reduction_pct=0.872131 moved_to_echo=24724 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 772, 0, 454, 0, 0, 4371, 622, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5612, 0, 13, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 193, 46, 0, 0, 0, 136, 0, 0, 3742, 2052, 0, 0, 0, 0, 0, 0, 194, 77, 5851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 1, 0, 0, 0, 53, 1969, 1520, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6154, 6185, 6169, 6122, 6027, 6136, 6140] avg=6144.0000 max=6219 max_rank=0 min=6027 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24425 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 542, 0, 391, 0, 0, 4373, 617, 0, 0, 0, 0, 0, 0, 6164, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5806, 0, 17, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 175, 7, 0, 0, 0, 143, 0, 0, 3744, 2051, 0, 0, 0, 0, 0, 0, 208, 70, 5849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2398, 54, 0, 0, 0, 66, 1968, 1515, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5923, 6166, 6379, 6120, 6127, 6146, 6150, 6141] avg=6144.0000 max=6379 max_rank=2 min=5923 min_rank=0 imbalance_ratio=1.038249 max_load_reduction=42773 max_load_reduction_pct=0.870219 moved_to_echo=24564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 0, 19712, 256, 128, 128, 128, 384, 15744, 12160, 1280, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 0, 5632, 128, 3712, 128, 128, 35072, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1664, 256, 0, 128, 128, 1152, 128, 0, 29952, 16512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 44544, 0, 256, 128, 0, 0, 4480, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[49152, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1664, 256, 0, 128, 128, 1152, 128, 0, 29952, 16512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 640, 46848, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[49152, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 6209, 0, 0, 72, 0, 0, 0, 906, 0, 0, 81, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4290, 72, 613, 0, 14, 0, 6300, 0, 0, 0, 1206, 4720, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 7, 1, 9, 0, 1, 3, 1, 0, 1, 4, 0, 3, 3, 0, 0, 0, 0, 0, 1, 8185, 5, 0, 1, 0, 0, 8182, 6336, 0, 1, 33, 0, 0, 0, 1096, 0, 4, 93, 3, 8184, 3, 1, 0, 0, 0, 1, 0, 1, 0, 4113, 71, 603, 0, 25, 0, 6150, 0, 7, 0, 1253, 4763, 3, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 73, 8, 6, 0, 0, 3, 4, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 8178, 5, 0, 0, 0, 1, 8181, 5962, 1, 0, 47, 1, 0, 3, 789, 0, 7, 98, 8, 8163, 2, 13, 0, 0, 2, 5, 0, 1, 0, 4427, 81, 580, 0, 33, 6, 6433, 0, 4, 1, 1172, 4842, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 172, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 5968, 0, 0, 27, 0, 0, 0, 966, 0, 0, 76, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 70, 613, 0, 25, 0, 6466, 0, 0, 0, 1274, 4967, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 84, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 5982, 0, 0, 45, 0, 0, 0, 1024, 0, 0, 81, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4286, 81, 657, 0, 22, 0, 6298, 0, 5, 0, 1151, 4851, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 7, 1, 9, 0, 1, 3, 1, 0, 1, 4, 0, 3, 3, 0, 0, 0, 0, 0, 1, 8185, 5, 0, 1, 0, 0, 8182, 6336, 0, 1, 33, 0, 0, 0, 1096, 0, 4, 93, 3, 8184, 3, 1, 0, 0, 0, 1, 0, 1, 0, 4113, 71, 603, 0, 25, 0, 6150, 0, 7, 0, 1253, 4763, 3, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 6110, 0, 0, 57, 0, 0, 0, 832, 0, 0, 84, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4391, 78, 667, 0, 25, 0, 6465, 0, 0, 0, 1149, 4698, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 60, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 5916, 0, 0, 11, 0, 0, 0, 1160, 0, 0, 83, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4275, 75, 552, 0, 24, 0, 6415, 0, 2, 0, 1204, 4796, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 73, 8, 6, 0, 0, 3, 4, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 8178, 5, 0, 0, 0, 1, 8181, 5962, 1, 0, 47, 1, 0, 3, 789, 0, 7, 98, 8, 8163, 2, 13, 0, 0, 2, 5, 0, 1, 0, 4427, 81, 580, 0, 33, 6, 6433, 0, 4, 1, 1172, 4842, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 15, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 2, 8185, 1, 0, 0, 0, 0, 8189, 5928, 0, 0, 8, 1, 1, 0, 1527, 0, 3, 91, 3, 8187, 3, 2, 0, 0, 1, 2, 0, 1, 0, 3912, 86, 531, 0, 16, 2, 6519, 0, 0, 0, 1281, 4650, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 1, 9, 0, 1, 3, 3931, 1991, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4, 0, 3, 3, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6141, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 56, 6336, 0, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 0, 4, 93, 3, 2079, 2046, 810, 0, 0, 0, 0, 0, 6138, 3, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4113, 71, 603, 0, 25, 0, 1234, 0, 0, 0, 0, 0, 0, 0, 140, 0, 7, 0, 1253, 4763, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5943, 6147, 6147, 6427, 6131, 6143, 6047, 6167] avg=6144.0000 max=6427 max_rank=3 min=5943 min_rank=0 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 1, 0, 0, 0, 4027, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 6209, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 906, 0, 0, 81, 0, 2130, 2048, 811, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4290, 72, 613, 0, 14, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 1206, 4720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6142, 6145, 6337, 5976, 6144, 6225, 6069] avg=6144.0000 max=6337 max_rank=3 min=5976 min_rank=4 imbalance_ratio=1.031413 max_load_reduction=42815 max_load_reduction_pct=0.871073 moved_to_echo=24414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 84, 0, 9, 0, 0, 0, 4026, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 5982, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 81, 0, 2129, 2048, 811, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4286, 81, 657, 0, 22, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 143, 0, 5, 0, 1151, 4851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6142, 6145, 6083, 6093, 6144, 6282, 6150] avg=6144.0000 max=6282 max_rank=6 min=6083 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=42870 max_load_reduction_pct=0.872192 moved_to_echo=24669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 73, 8, 6, 0, 0, 3, 4112, 1991, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6135, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 56, 5962, 1, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 789, 0, 7, 98, 8, 2175, 2041, 810, 0, 0, 0, 0, 0, 6122, 2, 13, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4427, 81, 580, 0, 33, 6, 1233, 0, 0, 0, 0, 0, 0, 0, 146, 0, 4, 1, 1172, 4842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6146, 6143, 6067, 5932, 6144, 6361, 6165] avg=6144.0000 max=6361 max_rank=6 min=5932 min_rank=4 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 172, 0, 3, 0, 0, 0, 4133, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 5968, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 76, 1, 2186, 2048, 811, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 70, 613, 0, 25, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 1274, 4967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6302, 6142, 6145, 6050, 6088, 6144, 5893, 6388] avg=6144.0000 max=6388 max_rank=7 min=5893 min_rank=6 imbalance_ratio=1.039714 max_load_reduction=42764 max_load_reduction_pct=0.870036 moved_to_echo=24513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 60, 0, 3, 0, 0, 0, 4100, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 5916, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 0, 83, 0, 2169, 2048, 811, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4275, 75, 552, 0, 24, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 146, 0, 2, 0, 1204, 4796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6142, 6145, 5983, 6271, 6144, 6162, 6148] avg=6144.0000 max=6271 max_rank=4 min=5983 min_rank=3 imbalance_ratio=1.020671 max_load_reduction=42881 max_load_reduction_pct=0.872416 moved_to_echo=24725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 1, 9, 0, 1, 3, 3931, 1991, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4, 0, 3, 3, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6141, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 56, 6336, 0, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 0, 4, 93, 3, 2079, 2046, 810, 0, 0, 0, 0, 0, 6138, 3, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4113, 71, 603, 0, 25, 0, 1234, 0, 0, 0, 0, 0, 0, 0, 140, 0, 7, 0, 1253, 4763, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5943, 6147, 6147, 6427, 6131, 6143, 6047, 6167] avg=6144.0000 max=6427 max_rank=3 min=5943 min_rank=0 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 1, 1, 0, 0, 0, 4167, 1993, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6141, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 5928, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1527, 0, 3, 91, 3, 2204, 2047, 810, 0, 0, 0, 0, 0, 6140, 3, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3912, 86, 531, 0, 16, 2, 1234, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 1281, 4650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6143, 6144, 5992, 6687, 6148, 5782, 6079] avg=6144.0000 max=6687 max_rank=4 min=5782 min_rank=6 imbalance_ratio=1.088379 max_load_reduction=42465 max_load_reduction_pct=0.863953 moved_to_echo=24696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 84, 0, 9, 0, 0, 0, 4026, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 5982, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 81, 0, 2129, 2048, 811, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4286, 81, 657, 0, 22, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 143, 0, 5, 0, 1151, 4851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6142, 6145, 6083, 6093, 6144, 6282, 6150] avg=6144.0000 max=6282 max_rank=6 min=6083 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=42870 max_load_reduction_pct=0.872192 moved_to_echo=24669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 0, 4136, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 6110, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 0, 0, 84, 2, 2190, 2051, 815, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4391, 78, 667, 0, 25, 0, 1239, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 1149, 4698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6148, 6138, 6213, 5974, 6141, 6400, 5986] avg=6144.0000 max=6400 max_rank=6 min=5974 min_rank=4 imbalance_ratio=1.041667 max_load_reduction=42752 max_load_reduction_pct=0.869792 moved_to_echo=24501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 73, 8, 6, 0, 0, 3, 4112, 1991, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6135, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 56, 5962, 1, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 789, 0, 7, 98, 8, 2175, 2041, 810, 0, 0, 0, 0, 0, 6122, 2, 13, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4427, 81, 580, 0, 33, 6, 1233, 0, 0, 0, 0, 0, 0, 0, 146, 0, 4, 1, 1172, 4842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6146, 6143, 6067, 5932, 6144, 6361, 6165] avg=6144.0000 max=6361 max_rank=6 min=5932 min_rank=4 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 172, 0, 3, 0, 0, 0, 4133, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 5968, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 76, 1, 2186, 2048, 811, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 70, 613, 0, 25, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 1274, 4967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6302, 6142, 6145, 6050, 6088, 6144, 5893, 6388] avg=6144.0000 max=6388 max_rank=7 min=5893 min_rank=6 imbalance_ratio=1.039714 max_load_reduction=42764 max_load_reduction_pct=0.870036 moved_to_echo=24513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 640, 128, 128, 0, 128, 128, 32640, 16000, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 8320, 0, 128, 768, 128, 17280, 16384, 6528, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 33664, 640, 4864, 0, 256, 128, 9984, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 0, 128, 128, 9728, 38400, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 512, 48512, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[49152, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 512, 48512, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 1687, 0, 0, 1, 5, 7893, 0, 0, 8114, 10, 260, 126, 129, 0, 4, 9, 0, 154, 3, 1, 0, 3, 2, 102, 2, 1, 8184, 0, 1, 72, 1, 6, 3, 0, 0, 0, 0, 13, 0, 1, 1, 1, 7562, 3, 7050, 0, 0, 2, 0, 0, 0, 0, 4, 0, 145, 1, 1, 0, 7592, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1399, 0, 0, 0, 0, 7893, 0, 0, 8130, 12, 229, 151, 137, 0, 0, 57, 0, 219, 0, 0, 0, 0, 0, 223, 0, 0, 8192, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7544, 0, 7036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 3, 0, 0, 7676, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1555, 0, 0, 0, 2, 7847, 0, 0, 8125, 34, 241, 140, 121, 0, 2, 69, 0, 154, 0, 0, 0, 3, 1, 242, 1, 0, 8188, 0, 0, 87, 2, 0, 1, 0, 0, 0, 2, 3, 0, 0, 0, 0, 7585, 3, 6855, 5, 0, 2, 1, 0, 0, 0, 2, 0, 195, 7, 1, 0, 7673, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1493, 0, 0, 0, 0, 7871, 0, 0, 8129, 4, 235, 157, 140, 0, 0, 12, 0, 130, 0, 0, 0, 0, 0, 223, 0, 0, 8192, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 7687, 0, 6970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 3, 0, 0, 7620, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1373, 0, 0, 0, 0, 7907, 0, 0, 8116, 2, 244, 136, 138, 0, 0, 12, 0, 161, 0, 0, 0, 0, 0, 168, 0, 0, 8192, 0, 1, 96, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7546, 0, 7083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 3, 0, 0, 7703, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[6, 1, 3, 6, 1709, 0, 0, 0, 6, 7881, 0, 0, 8114, 19, 250, 133, 111, 0, 3, 73, 2, 46, 1, 0, 0, 4, 0, 217, 4, 0, 8181, 0, 0, 83, 5, 0, 7, 3, 0, 0, 0, 15, 0, 0, 0, 1, 7554, 4, 6961, 1, 0, 1, 0, 0, 0, 7, 2, 0, 126, 5, 0, 4, 7601, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1752, 0, 0, 0, 0, 7834, 0, 0, 8114, 4, 236, 138, 129, 0, 0, 15, 0, 39, 0, 0, 0, 0, 0, 205, 0, 0, 8192, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 7578, 0, 7030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 14, 0, 0, 7553, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1555, 0, 0, 0, 2, 7847, 0, 0, 8125, 34, 241, 140, 121, 0, 2, 69, 0, 154, 0, 0, 0, 3, 1, 242, 1, 0, 8188, 0, 0, 87, 2, 0, 1, 0, 0, 0, 2, 3, 0, 0, 0, 0, 7585, 3, 6855, 5, 0, 2, 1, 0, 0, 0, 2, 0, 195, 7, 1, 0, 7673, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1399, 0, 0, 0, 0, 7893, 0, 0, 8130, 12, 229, 151, 137, 0, 0, 57, 0, 219, 0, 0, 0, 0, 0, 223, 0, 0, 8192, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7544, 0, 7036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 3, 0, 0, 7676, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1536, 0, 0, 0, 0, 7879, 0, 0, 8130, 25, 216, 145, 138, 0, 0, 58, 0, 148, 0, 0, 0, 0, 0, 156, 0, 0, 8192, 0, 2, 72, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 7526, 0, 7075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1, 0, 0, 7702, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1399, 0, 0, 0, 1679, 1446, 869, 589, 0, 0, 0, 0, 0, 5763, 0, 0, 0, 12, 229, 151, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 57, 0, 219, 0, 0, 2240, 2067, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 5952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 3, 0, 0, 5997, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5982, 6155, 6261, 6175, 6145, 6098, 6167, 6169] avg=6144.0000 max=6261 max_rank=2 min=5982 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24579 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 1687, 0, 0, 1, 1660, 1450, 871, 589, 0, 0, 0, 0, 5, 5763, 0, 0, 0, 10, 260, 126, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 4, 9, 0, 154, 3, 1, 2238, 2062, 1541, 0, 0, 0, 0, 0, 0, 3, 2, 102, 2, 1, 5946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 72, 1, 6, 3, 0, 0, 0, 6052, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 1, 1, 6112, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6179, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 145, 1, 1, 0, 5932, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 6164, 6141, 6056, 6135, 6131, 6181, 6083] avg=6144.0000 max=6261 max_rank=0 min=6056 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24530 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1555, 0, 0, 0, 1678, 1454, 847, 586, 0, 0, 0, 0, 2, 5729, 0, 0, 0, 34, 241, 140, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 2, 69, 0, 154, 0, 0, 2239, 2066, 1532, 0, 0, 0, 0, 0, 0, 3, 1, 242, 1, 0, 5949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 2, 0, 1, 0, 0, 0, 6059, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 6131, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6008, 5, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 195, 7, 1, 0, 5995, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6146, 6183, 6196, 6149, 6139, 6016, 6201] avg=6144.0000 max=6201 max_rank=7 min=6016 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=42951 max_load_reduction_pct=0.873840 moved_to_echo=24505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 1687, 0, 0, 1, 5, 7893, 0, 0, 8114, 10, 260, 126, 129, 0, 4, 9, 0, 154, 3, 1, 0, 3, 2, 102, 2, 1, 8184, 0, 1, 72, 1, 6, 3, 0, 0, 0, 0, 13, 0, 1, 1, 1, 7562, 3, 7050, 0, 0, 2, 0, 0, 0, 0, 4, 0, 145, 1, 1, 0, 7592, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1752, 0, 0, 0, 0, 7834, 0, 0, 8114, 4, 236, 138, 129, 0, 0, 15, 0, 39, 0, 0, 0, 0, 0, 205, 0, 0, 8192, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 7578, 0, 7030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 14, 0, 0, 7553, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[6, 1, 3, 6, 1709, 0, 0, 0, 6, 7881, 0, 0, 8114, 19, 250, 133, 111, 0, 3, 73, 2, 46, 1, 0, 0, 4, 0, 217, 4, 0, 8181, 0, 0, 83, 5, 0, 7, 3, 0, 0, 0, 15, 0, 0, 0, 1, 7554, 4, 6961, 1, 0, 1, 0, 0, 0, 7, 2, 0, 126, 5, 0, 4, 7601, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1493, 0, 0, 0, 1667, 1474, 861, 588, 0, 0, 0, 0, 0, 5746, 0, 0, 0, 4, 235, 157, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 12, 0, 130, 0, 0, 2240, 2066, 1537, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 5952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 6213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 3, 0, 0, 5953, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6083, 6142, 6125, 6175, 6154, 6254, 6109, 6110] avg=6144.0000 max=6254 max_rank=5 min=6083 min_rank=0 imbalance_ratio=1.017904 max_load_reduction=42898 max_load_reduction_pct=0.872762 moved_to_echo=24627 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1373, 0, 0, 0, 1685, 1447, 875, 590, 0, 0, 0, 0, 0, 5773, 0, 0, 0, 2, 244, 136, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 12, 0, 161, 0, 0, 2240, 2062, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 5952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 96, 0, 0, 0, 0, 0, 0, 6054, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 3, 0, 0, 6018, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5970, 6155, 6157, 6120, 6151, 6100, 6208, 6291] avg=6144.0000 max=6291 max_rank=7 min=5970 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=42861 max_load_reduction_pct=0.872009 moved_to_echo=24750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1555, 0, 0, 0, 1678, 1454, 847, 586, 0, 0, 0, 0, 2, 5729, 0, 0, 0, 34, 241, 140, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 2, 69, 0, 154, 0, 0, 2239, 2066, 1532, 0, 0, 0, 0, 0, 0, 3, 1, 242, 1, 0, 5949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 2, 0, 1, 0, 0, 0, 6059, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 6131, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6008, 5, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 195, 7, 1, 0, 5995, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6146, 6183, 6196, 6149, 6139, 6016, 6201] avg=6144.0000 max=6201 max_rank=7 min=6016 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=42951 max_load_reduction_pct=0.873840 moved_to_echo=24505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1536, 0, 0, 0, 1685, 1443, 874, 588, 0, 0, 0, 0, 0, 5753, 0, 0, 0, 25, 216, 145, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 58, 0, 148, 0, 0, 2240, 2067, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 5952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 72, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1, 0, 0, 6017, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6139, 6189, 6108, 6137, 6102, 6201, 6150] avg=6144.0000 max=6201 max_rank=6 min=6102 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=42951 max_load_reduction_pct=0.873840 moved_to_echo=24590 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 1, 3, 6, 1709, 0, 0, 0, 1662, 1448, 860, 589, 0, 0, 0, 0, 6, 5753, 0, 0, 0, 19, 250, 133, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 3, 73, 2, 46, 1, 0, 2237, 2062, 1539, 0, 0, 0, 0, 0, 0, 4, 0, 217, 4, 0, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 5, 0, 7, 3, 0, 0, 6052, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1, 6106, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 1, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 126, 5, 0, 4, 5939, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6161, 6074, 6169, 6150, 6126, 6110, 6078] avg=6144.0000 max=6284 max_rank=0 min=6074 min_rank=2 imbalance_ratio=1.022786 max_load_reduction=42868 max_load_reduction_pct=0.872152 moved_to_echo=24464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 1687, 0, 0, 1, 1660, 1450, 871, 589, 0, 0, 0, 0, 5, 5763, 0, 0, 0, 10, 260, 126, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 4, 9, 0, 154, 3, 1, 2238, 2062, 1541, 0, 0, 0, 0, 0, 0, 3, 2, 102, 2, 1, 5946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 72, 1, 6, 3, 0, 0, 0, 6052, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 1, 1, 6112, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6179, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 145, 1, 1, 0, 5932, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 6164, 6141, 6056, 6135, 6131, 6181, 6083] avg=6144.0000 max=6261 max_rank=0 min=6056 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24530 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1399, 0, 0, 0, 1679, 1446, 869, 589, 0, 0, 0, 0, 0, 5763, 0, 0, 0, 12, 229, 151, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 57, 0, 219, 0, 0, 2240, 2067, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 5952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 3, 0, 0, 5997, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5982, 6155, 6261, 6175, 6145, 6098, 6167, 6169] avg=6144.0000 max=6261 max_rank=2 min=5982 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24579 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1752, 0, 0, 0, 1656, 1456, 870, 590, 0, 0, 0, 0, 0, 5712, 0, 0, 0, 4, 236, 138, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 15, 0, 39, 0, 0, 2244, 2063, 1532, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 5948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 6051, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 14, 0, 0, 5897, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6324, 6090, 6022, 6153, 6131, 6202, 6160, 6070] avg=6144.0000 max=6324 max_rank=0 min=6022 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=42828 max_load_reduction_pct=0.871338 moved_to_echo=24563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1536, 0, 0, 0, 1685, 1443, 874, 588, 0, 0, 0, 0, 0, 5753, 0, 0, 0, 25, 216, 145, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 58, 0, 148, 0, 0, 2240, 2067, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 5952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 72, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1, 0, 0, 6017, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6139, 6189, 6108, 6137, 6102, 6201, 6150] avg=6144.0000 max=6201 max_rank=6 min=6102 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=42951 max_load_reduction_pct=0.873840 moved_to_echo=24590 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[1152, 0, 128, 384, 128, 1152, 128, 128, 17920, 16640, 12416, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 12544, 0, 0, 128, 13440, 11648, 7040, 4736, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 46080, 0, 0, 0, 128, 1920, 1152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 1408, 128, 128, 128, 47872, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[49152, 128, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 768, 128, 128, 128, 128, 0, 0, 48512, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 1536, 128, 128, 47616, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 1408, 128, 128, 128, 47872, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 256, 0, 128, 128, 128, 49024, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[49152, 128, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 280, 0, 0, 0, 8192, 4816, 0, 0, 0, 0, 0, 0, 0, 3833, 6178, 0, 0, 4544, 0, 0, 1070, 2111, 0, 0, 392, 0, 0, 0, 0, 0, 597, 0, 0, 103, 0, 0, 0, 0, 0, 7509, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 735, 0, 0, 0, 46, 0, 0, 0, 0, 554, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 0, 0, 8192, 5136, 0, 0, 0, 0, 0, 0, 0, 3913, 5903, 0, 0, 4383, 0, 0, 1290, 1969, 0, 0, 435, 0, 0, 0, 0, 0, 480, 0, 0, 71, 0, 0, 0, 0, 0, 7485, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 0, 84, 0, 0, 0, 0, 576, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1, 4, 0, 487, 0, 1, 0, 8184, 4910, 0, 1, 6, 4, 4, 0, 0, 4035, 6158, 4, 0, 4796, 0, 0, 947, 1758, 0, 0, 366, 0, 0, 0, 0, 0, 366, 3, 2, 75, 0, 0, 1, 5, 0, 7459, 8184, 0, 0, 0, 6, 0, 0, 0, 0, 752, 0, 2, 0, 53, 0, 0, 0, 0, 578, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 1, 0, 417, 2, 0, 0, 8179, 4952, 0, 4, 1, 8, 9, 0, 1, 3929, 6132, 9, 0, 4375, 0, 5, 1181, 1998, 0, 0, 438, 3, 0, 1, 0, 0, 392, 1, 0, 70, 0, 1, 0, 2, 4, 7488, 8141, 2, 0, 0, 4, 0, 0, 0, 1, 713, 4, 3, 1, 118, 0, 10, 0, 0, 551, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 209, 0, 0, 0, 8192, 5112, 0, 0, 0, 0, 0, 0, 0, 3841, 6273, 0, 0, 4158, 0, 0, 1257, 1924, 0, 0, 525, 0, 0, 0, 0, 0, 668, 0, 0, 110, 0, 0, 0, 0, 0, 7527, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 86, 0, 0, 0, 0, 555, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 322, 0, 0, 0, 8192, 5282, 0, 0, 0, 0, 0, 1, 0, 3562, 6326, 0, 0, 4392, 0, 0, 1123, 1886, 0, 0, 600, 0, 0, 0, 0, 0, 544, 0, 0, 72, 0, 0, 0, 0, 0, 7564, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 74, 0, 0, 0, 0, 510, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1, 4, 0, 487, 0, 1, 0, 8184, 4910, 0, 1, 6, 4, 4, 0, 0, 4035, 6158, 4, 0, 4796, 0, 0, 947, 1758, 0, 0, 366, 0, 0, 0, 0, 0, 366, 3, 2, 75, 0, 0, 1, 5, 0, 7459, 8184, 0, 0, 0, 6, 0, 0, 0, 0, 752, 0, 2, 0, 53, 0, 0, 0, 0, 578, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 0, 0, 8192, 5136, 0, 0, 0, 0, 0, 0, 0, 3913, 5903, 0, 0, 4383, 0, 0, 1290, 1969, 0, 0, 435, 0, 0, 0, 0, 0, 480, 0, 0, 71, 0, 0, 0, 0, 0, 7485, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 0, 84, 0, 0, 0, 0, 576, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 225, 0, 0, 0, 8192, 5154, 0, 0, 0, 0, 0, 0, 0, 3967, 6106, 0, 0, 4238, 0, 0, 1241, 1950, 0, 0, 384, 0, 0, 0, 0, 0, 598, 0, 0, 83, 0, 0, 0, 0, 0, 7577, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 0, 96, 0, 0, 0, 0, 495, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 1, 0, 417, 2, 0, 0, 8179, 4952, 0, 4, 1, 8, 9, 0, 1, 3929, 6132, 9, 0, 4375, 0, 5, 1181, 1998, 0, 0, 438, 3, 0, 1, 0, 0, 392, 1, 0, 70, 0, 1, 0, 2, 4, 7488, 8141, 2, 0, 0, 4, 0, 0, 0, 1, 713, 4, 3, 1, 118, 0, 10, 0, 0, 551, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 280, 0, 0, 0, 8192, 4816, 0, 0, 0, 0, 0, 0, 0, 3833, 6178, 0, 0, 4544, 0, 0, 1070, 2111, 0, 0, 392, 0, 0, 0, 0, 0, 597, 0, 0, 103, 0, 0, 0, 0, 0, 7509, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 735, 0, 0, 0, 46, 0, 0, 0, 0, 554, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[2, 1, 1, 255, 0, 2, 0, 8187, 4870, 0, 1, 3, 0, 3, 0, 1, 3830, 6223, 4, 0, 4454, 0, 0, 1235, 1853, 0, 0, 519, 0, 0, 0, 0, 0, 533, 1, 0, 128, 0, 0, 0, 5, 1, 7563, 8185, 0, 0, 0, 3, 2, 0, 1, 0, 715, 1, 1, 0, 57, 0, 0, 0, 0, 512, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 322, 0, 0, 0, 8192, 5282, 0, 0, 0, 0, 0, 1, 0, 3562, 6326, 0, 0, 4392, 0, 0, 1123, 1886, 0, 0, 600, 0, 0, 0, 0, 0, 544, 0, 0, 72, 0, 0, 0, 0, 0, 7564, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 74, 0, 0, 0, 0, 510, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 0, 487, 0, 1, 0, 5821, 0, 0, 0, 0, 0, 0, 0, 0, 4910, 0, 1, 6, 4, 4, 0, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 4, 0, 1206, 0, 0, 947, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 0, 0, 366, 0, 0, 0, 0, 2363, 1213, 270, 0, 0, 0, 0, 0, 0, 366, 3, 2, 75, 0, 0, 1, 5536, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6090, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 752, 0, 2, 0, 3281, 2377, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 578, 0, 0, 2877, 2648, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6314, 6024, 6192, 5970, 5983, 6101, 6412, 6156] avg=6144.0000 max=6412 max_rank=6 min=5970 min_rank=3 imbalance_ratio=1.043620 max_load_reduction=42740 max_load_reduction_pct=0.869548 moved_to_echo=24652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 209, 0, 0, 0, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 5112, 0, 0, 0, 0, 0, 0, 0, 1109, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 1045, 0, 0, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 1924, 0, 0, 525, 0, 0, 0, 0, 2365, 1052, 272, 0, 0, 0, 0, 0, 0, 668, 0, 0, 110, 0, 0, 0, 5534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 3342, 2061, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 555, 0, 0, 2931, 2647, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6036, 6221, 6143, 6138, 6312, 6146, 5937, 6219] avg=6144.0000 max=6312 max_rank=4 min=5937 min_rank=6 imbalance_ratio=1.027344 max_load_reduction=42840 max_load_reduction_pct=0.871582 moved_to_echo=24614 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 417, 2, 0, 0, 5818, 0, 0, 0, 0, 0, 0, 0, 0, 4952, 0, 4, 1, 8, 9, 0, 1, 1104, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 9, 0, 1100, 0, 5, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 1998, 0, 0, 438, 3, 0, 1, 0, 2361, 1107, 271, 0, 0, 0, 0, 0, 0, 392, 1, 0, 70, 0, 1, 0, 5507, 0, 0, 0, 0, 0, 0, 0, 2, 4, 6113, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 713, 4, 3, 1, 3267, 2168, 0, 0, 0, 0, 0, 0, 118, 0, 10, 0, 0, 551, 0, 0, 2865, 2634, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6239, 6079, 6224, 6179, 5971, 6125, 6157, 6178] avg=6144.0000 max=6239 max_rank=0 min=5971 min_rank=4 imbalance_ratio=1.015462 max_load_reduction=42913 max_load_reduction_pct=0.873067 moved_to_echo=24431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 280, 0, 0, 0, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 4816, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 3833, 0, 0, 0, 1142, 0, 0, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 0, 0, 392, 0, 0, 0, 0, 2365, 1150, 271, 0, 0, 0, 0, 0, 0, 597, 0, 0, 103, 0, 0, 0, 5541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 735, 0, 0, 0, 3291, 2252, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 554, 0, 0, 2887, 2651, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6107, 5923, 6045, 6289, 6241, 6131, 6278, 6138] avg=6144.0000 max=6289 max_rank=3 min=5923 min_rank=1 imbalance_ratio=1.023600 max_load_reduction=42863 max_load_reduction_pct=0.872050 moved_to_echo=24788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 0, 0, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 5136, 0, 0, 0, 0, 0, 0, 0, 1103, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 1102, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 0, 435, 0, 0, 0, 0, 2365, 1109, 271, 0, 0, 0, 0, 0, 0, 480, 0, 0, 71, 0, 0, 0, 5539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 0, 3145, 2172, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 576, 0, 0, 2758, 2649, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6174, 6239, 6305, 6149, 6090, 6111, 6017, 6067] avg=6144.0000 max=6305 max_rank=2 min=6017 min_rank=6 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24285 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 322, 0, 0, 0, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 5282, 0, 0, 0, 0, 0, 1, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 3562, 0, 0, 0, 1104, 0, 0, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 1886, 0, 0, 600, 0, 0, 0, 0, 2365, 1111, 273, 0, 0, 0, 0, 0, 0, 544, 0, 0, 72, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 3371, 2177, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 510, 0, 0, 2955, 2645, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6398, 5789, 6235, 6147, 6176, 6074, 6184] avg=6144.0000 max=6398 max_rank=1 min=5789 min_rank=2 imbalance_ratio=1.041341 max_load_reduction=42754 max_load_reduction_pct=0.869832 moved_to_echo=24581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 0, 487, 0, 1, 0, 5821, 0, 0, 0, 0, 0, 0, 0, 0, 4910, 0, 1, 6, 4, 4, 0, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 4, 0, 1206, 0, 0, 947, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 0, 0, 366, 0, 0, 0, 0, 2363, 1213, 270, 0, 0, 0, 0, 0, 0, 366, 3, 2, 75, 0, 0, 1, 5536, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6090, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 752, 0, 2, 0, 3281, 2377, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 578, 0, 0, 2877, 2648, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6314, 6024, 6192, 5970, 5983, 6101, 6412, 6156] avg=6144.0000 max=6412 max_rank=6 min=5970 min_rank=3 imbalance_ratio=1.043620 max_load_reduction=42740 max_load_reduction_pct=0.869548 moved_to_echo=24652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 1, 255, 0, 2, 0, 5824, 0, 0, 0, 0, 0, 0, 0, 0, 4870, 0, 1, 3, 0, 3, 0, 1, 1115, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 4, 0, 1120, 0, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 0, 519, 0, 0, 0, 0, 2363, 1127, 273, 0, 0, 0, 0, 0, 0, 533, 1, 0, 128, 0, 0, 0, 5537, 0, 0, 0, 0, 0, 0, 0, 5, 1, 6175, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 715, 1, 1, 0, 3316, 2207, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 512, 0, 0, 2907, 2648, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 5993, 6189, 6135, 6199, 6184, 6243, 6124] avg=6144.0000 max=6243 max_rank=6 min=5993 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=42909 max_load_reduction_pct=0.872986 moved_to_echo=24750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 0, 0, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 5136, 0, 0, 0, 0, 0, 0, 0, 1103, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 1102, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 0, 435, 0, 0, 0, 0, 2365, 1109, 271, 0, 0, 0, 0, 0, 0, 480, 0, 0, 71, 0, 0, 0, 5539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 0, 3145, 2172, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 576, 0, 0, 2758, 2649, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6174, 6239, 6305, 6149, 6090, 6111, 6017, 6067] avg=6144.0000 max=6305 max_rank=2 min=6017 min_rank=6 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24285 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 225, 0, 0, 0, 5823, 0, 0, 0, 0, 0, 0, 0, 0, 5154, 0, 0, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 1057, 0, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 1950, 0, 0, 384, 0, 0, 0, 0, 2369, 1076, 278, 0, 0, 0, 0, 0, 0, 598, 0, 0, 83, 0, 0, 0, 5528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 0, 3254, 2105, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 495, 0, 0, 2852, 2643, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6048, 6275, 6265, 6057, 6209, 6178, 6034, 6086] avg=6144.0000 max=6275 max_rank=1 min=6034 min_rank=6 imbalance_ratio=1.021322 max_load_reduction=42877 max_load_reduction_pct=0.872335 moved_to_echo=24507 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 417, 2, 0, 0, 5818, 0, 0, 0, 0, 0, 0, 0, 0, 4952, 0, 4, 1, 8, 9, 0, 1, 1104, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 9, 0, 1100, 0, 5, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 1998, 0, 0, 438, 3, 0, 1, 0, 2361, 1107, 271, 0, 0, 0, 0, 0, 0, 392, 1, 0, 70, 0, 1, 0, 5507, 0, 0, 0, 0, 0, 0, 0, 2, 4, 6113, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 713, 4, 3, 1, 3267, 2168, 0, 0, 0, 0, 0, 0, 118, 0, 10, 0, 0, 551, 0, 0, 2865, 2634, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6239, 6079, 6224, 6179, 5971, 6125, 6157, 6178] avg=6144.0000 max=6239 max_rank=0 min=5971 min_rank=4 imbalance_ratio=1.015462 max_load_reduction=42913 max_load_reduction_pct=0.873067 moved_to_echo=24431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 322, 0, 0, 0, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 5282, 0, 0, 0, 0, 0, 1, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 3562, 0, 0, 0, 1104, 0, 0, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 1886, 0, 0, 600, 0, 0, 0, 0, 2365, 1111, 273, 0, 0, 0, 0, 0, 0, 544, 0, 0, 72, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 3371, 2177, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 510, 0, 0, 2955, 2645, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6398, 5789, 6235, 6147, 6176, 6074, 6184] avg=6144.0000 max=6398 max_rank=1 min=5789 min_rank=2 imbalance_ratio=1.041341 max_load_reduction=42754 max_load_reduction_pct=0.869832 moved_to_echo=24581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[640, 0, 128, 0, 0, 4352, 0, 0, 23040, 21248, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[15488, 0, 0, 3712, 128, 0, 128, 0, 18944, 8960, 2304, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 5376, 128, 128, 128, 26368, 17536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[40320, 0, 128, 128, 128, 128, 128, 128, 8960, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[30976, 0, 128, 0, 8960, 0, 128, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[40320, 0, 128, 128, 128, 128, 128, 128, 8960, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 4224, 128, 128, 768, 0, 128, 128, 44288, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 2560, 128, 128, 0, 46720, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 49152, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 7907, 164, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 8092, 0, 0, 0, 6, 0, 0, 0, 0, 6517, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5818, 0, 0, 0, 0, 0, 661, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3, 7, 147, 4, 0, 1, 0, 3, 1, 0, 3649, 0, 0, 3, 0, 7959, 130, 18, 3, 3, 0, 3, 0, 1, 0, 7976, 0, 0, 2, 0, 8086, 6, 0, 0, 8, 0, 7, 0, 0, 6551, 0, 0, 8180, 0, 1, 0, 0, 1, 2, 0, 0, 0, 0, 5750, 0, 0, 3, 0, 1, 643, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 3606, 0, 0, 0, 0, 7884, 184, 1, 0, 0, 0, 0, 0, 0, 0, 7880, 0, 0, 0, 0, 8107, 0, 0, 0, 17, 0, 0, 0, 0, 6496, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6050, 0, 0, 0, 0, 0, 574, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 3449, 0, 0, 0, 0, 7868, 193, 3, 0, 0, 0, 0, 0, 0, 0, 7915, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 6765, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 0, 0, 0, 0, 602, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 3443, 0, 0, 0, 0, 7916, 192, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 6607, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6003, 0, 0, 0, 0, 0, 634, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 0, 0, 7976, 120, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 8108, 0, 0, 0, 1, 0, 0, 0, 0, 6459, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 0, 0, 0, 0, 560, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 7907, 164, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 8092, 0, 0, 0, 6, 0, 0, 0, 0, 6517, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5818, 0, 0, 0, 0, 0, 661, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 13, 221, 3, 0, 1, 4, 0, 0, 0, 3586, 1, 11, 0, 1, 7961, 139, 13, 7, 0, 0, 5, 1, 1, 0, 7936, 0, 0, 0, 1, 8080, 8, 0, 0, 0, 0, 2, 0, 0, 6377, 0, 0, 8169, 0, 2, 0, 0, 3, 0, 0, 1, 0, 2, 5965, 0, 0, 11, 0, 0, 621, 0, 2, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 3606, 0, 0, 0, 0, 7884, 184, 1, 0, 0, 0, 0, 0, 0, 0, 7880, 0, 0, 0, 0, 8107, 0, 0, 0, 17, 0, 0, 0, 0, 6496, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6050, 0, 0, 0, 0, 0, 574, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 5, 176, 1, 0, 3, 1, 0, 1, 0, 3824, 1, 0, 0, 0, 7837, 187, 3, 0, 1, 0, 3, 0, 1, 0, 7919, 0, 0, 1, 0, 8073, 4, 0, 0, 3, 0, 0, 0, 0, 6387, 0, 0, 8185, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 5892, 0, 0, 4, 0, 0, 634, 0, 0, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 3449, 0, 0, 0, 0, 7868, 193, 3, 0, 0, 0, 0, 0, 0, 0, 7915, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 6765, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 0, 0, 0, 0, 602, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 0, 0, 7976, 120, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 8108, 0, 0, 0, 1, 0, 0, 0, 0, 6459, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 0, 0, 0, 0, 560, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 7, 147, 4, 0, 1, 0, 3924, 2048, 0, 0, 0, 0, 0, 0, 3, 1, 0, 3649, 0, 0, 3, 0, 1790, 584, 149, 0, 0, 0, 0, 0, 6008, 130, 18, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6186, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8, 0, 7, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 6174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 5750, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 643, 0, 0, 0, 4162, 1367, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6179, 6165, 6189, 6153, 6175, 5981, 6176] avg=6144.0000 max=6189 max_rank=3 min=5981 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=42963 max_load_reduction_pct=0.874084 moved_to_echo=24485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3, 7, 147, 4, 0, 1, 0, 3, 1, 0, 3649, 0, 0, 3, 0, 7959, 130, 18, 3, 3, 0, 3, 0, 1, 0, 7976, 0, 0, 2, 0, 8086, 6, 0, 0, 8, 0, 7, 0, 0, 6551, 0, 0, 8180, 0, 1, 0, 0, 1, 2, 0, 0, 0, 0, 5750, 0, 0, 3, 0, 1, 643, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 149, 0, 0, 0, 0, 3927, 2051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3449, 0, 0, 0, 0, 1777, 578, 154, 0, 0, 0, 0, 0, 5939, 193, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 6376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 4166, 1351, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 5958, 6135, 6138, 6141, 6376, 6158, 6119] avg=6144.0000 max=6376 max_rank=5 min=5958 min_rank=1 imbalance_ratio=1.037760 max_load_reduction=42776 max_load_reduction_pct=0.870280 moved_to_echo=24794 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 129, 0, 0, 0, 0, 3927, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 1789, 581, 149, 0, 0, 0, 0, 0, 5968, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5818, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 4165, 1358, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6108, 6217, 6132, 6179, 6146, 6141, 6045, 6184] avg=6144.0000 max=6217 max_rank=1 min=6045 min_rank=6 imbalance_ratio=1.011882 max_load_reduction=42935 max_load_reduction_pct=0.873515 moved_to_echo=24516 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 157, 0, 0, 0, 0, 3935, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 0, 0, 1764, 586, 147, 0, 0, 0, 0, 0, 6020, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 6087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 4173, 1370, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6295, 6140, 6094, 6141, 6087, 6148, 6103] avg=6144.0000 max=6295 max_rank=1 min=6087 min_rank=5 imbalance_ratio=1.024577 max_load_reduction=42857 max_load_reduction_pct=0.871928 moved_to_echo=24479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 221, 3, 0, 1, 4, 3921, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3586, 1, 11, 0, 1, 1781, 585, 145, 0, 0, 0, 0, 0, 6009, 139, 13, 7, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6155, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 2, 0, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 6010, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 2, 5965, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 621, 0, 2, 4, 4159, 1367, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6209, 6110, 6174, 6157, 6133, 6012, 6193, 6164] avg=6144.0000 max=6209 max_rank=0 min=6012 min_rank=5 imbalance_ratio=1.010579 max_load_reduction=42943 max_load_reduction_pct=0.873678 moved_to_echo=24502 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 176, 1, 0, 3, 1, 3918, 2051, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3824, 1, 0, 0, 0, 1778, 576, 146, 0, 0, 0, 0, 0, 5915, 187, 3, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6141, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 6019, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5892, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 634, 0, 0, 3, 4155, 1346, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6326, 6109, 6143, 6141, 6021, 6115, 6142] avg=6144.0000 max=6326 max_rank=1 min=6021 min_rank=5 imbalance_ratio=1.029622 max_load_reduction=42826 max_load_reduction_pct=0.871297 moved_to_echo=24419 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 145, 0, 0, 0, 0, 3930, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 0, 0, 0, 0, 1779, 581, 151, 0, 0, 0, 0, 0, 5976, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6003, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 0, 0, 0, 4167, 1359, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 5954, 6168, 6144, 6140, 6226, 6233, 6160] avg=6144.0000 max=6233 max_rank=6 min=5954 min_rank=1 imbalance_ratio=1.014486 max_load_reduction=42919 max_load_reduction_pct=0.873189 moved_to_echo=24759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 149, 0, 0, 0, 0, 3927, 2051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3449, 0, 0, 0, 0, 1777, 578, 154, 0, 0, 0, 0, 0, 5939, 193, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 6376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 4166, 1351, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 5958, 6135, 6138, 6141, 6376, 6158, 6119] avg=6144.0000 max=6376 max_rank=5 min=5958 min_rank=1 imbalance_ratio=1.037760 max_load_reduction=42776 max_load_reduction_pct=0.870280 moved_to_echo=24794 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 161, 0, 0, 0, 0, 3935, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3606, 0, 0, 0, 0, 1772, 582, 153, 0, 0, 0, 0, 0, 5944, 184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 6114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6050, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 0, 0, 4172, 1358, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6113, 6129, 6108, 6157, 6114, 6279, 6104] avg=6144.0000 max=6279 max_rank=6 min=6104 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=42873 max_load_reduction_pct=0.872253 moved_to_echo=24654 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 221, 3, 0, 1, 4, 3921, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3586, 1, 11, 0, 1, 1781, 585, 145, 0, 0, 0, 0, 0, 6009, 139, 13, 7, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6155, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 2, 0, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 6010, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 2, 5965, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 621, 0, 2, 4, 4159, 1367, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6209, 6110, 6174, 6157, 6133, 6012, 6193, 6164] avg=6144.0000 max=6209 max_rank=0 min=6012 min_rank=5 imbalance_ratio=1.010579 max_load_reduction=42943 max_load_reduction_pct=0.873678 moved_to_echo=24502 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 157, 0, 0, 0, 0, 3935, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 0, 0, 1764, 586, 147, 0, 0, 0, 0, 0, 6020, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 6087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 4173, 1370, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6295, 6140, 6094, 6141, 6087, 6148, 6103] avg=6144.0000 max=6295 max_rank=1 min=6087 min_rank=5 imbalance_ratio=1.024577 max_load_reduction=42857 max_load_reduction_pct=0.871928 moved_to_echo=24479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 7, 147, 4, 0, 1, 0, 3924, 2048, 0, 0, 0, 0, 0, 0, 3, 1, 0, 3649, 0, 0, 3, 0, 1790, 584, 149, 0, 0, 0, 0, 0, 6008, 130, 18, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6186, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8, 0, 7, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 6174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 5750, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 643, 0, 0, 0, 4162, 1367, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6179, 6165, 6189, 6153, 6175, 5981, 6176] avg=6144.0000 max=6189 max_rank=3 min=5981 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=42963 max_load_reduction_pct=0.874084 moved_to_echo=24485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 128, 128, 1408, 128, 0, 128, 128, 31488, 16512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 29056, 128, 128, 128, 128, 14336, 4736, 1280, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 4992, 0, 128, 128, 33408, 10880, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 128, 47360, 0, 1920, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[47872, 1408, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[49152, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 49152, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[49152, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1785, 2, 0, 0, 0, 2, 2510, 0, 0, 1, 0, 0, 0, 1, 0, 8033, 0, 0, 1, 512, 1, 6, 5, 2, 0, 4, 0, 0, 648, 1, 6, 0, 7958, 8186, 3112, 0, 0, 1, 2, 0, 68, 0, 8183, 0, 0, 0, 8108, 4, 6, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1793, 0, 0, 0, 0, 0, 2458, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 692, 0, 0, 0, 7984, 8192, 3095, 0, 0, 0, 0, 0, 79, 0, 8192, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 654, 0, 0, 0, 7998, 8192, 2868, 0, 0, 0, 0, 0, 58, 0, 8192, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 0, 0, 0, 0, 0, 2796, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 0, 7972, 8192, 2832, 0, 0, 0, 0, 0, 77, 0, 8192, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1793, 0, 0, 0, 0, 0, 2458, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 692, 0, 0, 0, 7984, 8192, 3095, 0, 0, 0, 0, 0, 79, 0, 8192, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 0, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 0, 7987, 8192, 2656, 0, 0, 0, 0, 0, 61, 0, 8192, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1897, 0, 0, 0, 0, 2, 2719, 0, 0, 0, 1, 0, 3, 3, 0, 8011, 0, 0, 0, 563, 0, 4, 6, 3, 0, 2, 0, 0, 558, 0, 0, 0, 7994, 8185, 2858, 0, 0, 3, 0, 0, 46, 0, 8185, 0, 0, 0, 8098, 1, 4, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1785, 2, 0, 0, 0, 2, 2510, 0, 0, 1, 0, 0, 0, 1, 0, 8033, 0, 0, 1, 512, 1, 6, 5, 2, 0, 4, 0, 0, 648, 1, 6, 0, 7958, 8186, 3112, 0, 0, 1, 2, 0, 68, 0, 8183, 0, 0, 0, 8108, 4, 6, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 654, 0, 0, 0, 7998, 8192, 2868, 0, 0, 0, 0, 0, 58, 0, 8192, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[2, 2, 0, 0, 2, 2, 1, 1, 1, 0, 1, 1994, 3, 1, 0, 0, 7, 2850, 0, 0, 0, 0, 0, 8, 5, 1, 8006, 1, 0, 1, 553, 0, 13, 8, 3, 0, 8, 1, 0, 575, 0, 7, 1, 7968, 8178, 2574, 0, 1, 2, 0, 1, 100, 0, 8171, 0, 0, 0, 8082, 3, 11, 0, 1, 1, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 0, 0, 0, 0, 0, 2717, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 680, 0, 0, 0, 8020, 8192, 2731, 0, 0, 0, 0, 0, 45, 0, 8192, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 0, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 0, 7987, 8192, 2656, 0, 0, 0, 0, 0, 61, 0, 8192, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 2, 6139, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1785, 2, 0, 0, 0, 2440, 1260, 552, 0, 0, 0, 0, 0, 2, 2510, 0, 0, 1, 0, 0, 0, 1968, 1558, 0, 0, 0, 0, 0, 0, 1, 0, 5593, 0, 0, 1, 512, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 2, 0, 4, 0, 0, 648, 3407, 2047, 0, 0, 0, 0, 0, 0, 1, 6, 0, 3291, 0, 3112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 68, 0, 6073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6140, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6040, 6039, 6108, 6119, 6410, 6144, 6150] avg=6144.0000 max=6410 max_rank=5 min=6039 min_rank=2 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1793, 0, 0, 0, 0, 2444, 1264, 552, 0, 0, 0, 0, 0, 0, 2458, 0, 0, 0, 0, 0, 0, 1971, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 5604, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 692, 3418, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3302, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 6081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6053, 5988, 6103, 6158, 6397, 6160, 6149] avg=6144.0000 max=6397 max_rank=5 min=5988 min_rank=2 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 0, 0, 0, 0, 2442, 1262, 552, 0, 0, 0, 0, 0, 0, 2796, 0, 0, 0, 0, 0, 0, 1970, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 5599, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 3413, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3297, 0, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 6081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 5883, 6325, 6186, 6181, 6129, 6158, 6146] avg=6144.0000 max=6325 max_rank=2 min=5883 min_rank=1 imbalance_ratio=1.029460 max_load_reduction=42827 max_load_reduction_pct=0.871318 moved_to_echo=24614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 0, 1, 0, 1, 6140, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1897, 0, 0, 0, 0, 2433, 1266, 552, 0, 0, 0, 0, 0, 2, 2719, 0, 0, 0, 1, 0, 3, 1965, 1558, 0, 0, 0, 0, 0, 0, 3, 0, 5578, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 3, 0, 2, 0, 0, 558, 3422, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3306, 0, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 46, 0, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6133, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6150, 6248, 6144, 6040, 6164, 6124, 6138] avg=6144.0000 max=6248 max_rank=2 min=6040 min_rank=4 imbalance_ratio=1.016927 max_load_reduction=42904 max_load_reduction_pct=0.872884 moved_to_echo=24466 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 2446, 1271, 556, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 1973, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 5590, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 3428, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 0, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 6072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6299, 6018, 6123, 6129, 6167, 6130, 6141] avg=6144.0000 max=6299 max_rank=1 min=6018 min_rank=2 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24567 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 0, 2, 2, 1, 1, 6133, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1994, 3, 1, 0, 0, 2432, 1262, 551, 0, 0, 0, 0, 0, 7, 2850, 0, 0, 0, 0, 0, 8, 1961, 1555, 0, 0, 0, 0, 0, 0, 5, 1, 5574, 1, 0, 1, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8, 3, 0, 8, 1, 0, 575, 3411, 2045, 0, 0, 0, 0, 0, 0, 0, 7, 1, 3295, 0, 2574, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 100, 0, 6065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6121, 3, 11, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6245, 6381, 6135, 6064, 5878, 6168, 6138] avg=6144.0000 max=6381 max_rank=2 min=5878 min_rank=5 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24248 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 0, 0, 0, 0, 2439, 1265, 552, 0, 0, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 0, 1969, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 5591, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 3419, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 0, 2656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 6081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6347, 5907, 6209, 6301, 5959, 6142, 6142] avg=6144.0000 max=6347 max_rank=1 min=5907 min_rank=2 imbalance_ratio=1.033040 max_load_reduction=42805 max_load_reduction_pct=0.870870 moved_to_echo=24544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 0, 2, 2, 1, 1, 6133, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1994, 3, 1, 0, 0, 2432, 1262, 551, 0, 0, 0, 0, 0, 7, 2850, 0, 0, 0, 0, 0, 8, 1961, 1555, 0, 0, 0, 0, 0, 0, 5, 1, 5574, 1, 0, 1, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8, 3, 0, 8, 1, 0, 575, 3411, 2045, 0, 0, 0, 0, 0, 0, 0, 7, 1, 3295, 0, 2574, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 100, 0, 6065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6121, 3, 11, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6245, 6381, 6135, 6064, 5878, 6168, 6138] avg=6144.0000 max=6381 max_rank=2 min=5878 min_rank=5 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24248 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 2, 6139, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1785, 2, 0, 0, 0, 2440, 1260, 552, 0, 0, 0, 0, 0, 2, 2510, 0, 0, 1, 0, 0, 0, 1968, 1558, 0, 0, 0, 0, 0, 0, 1, 0, 5593, 0, 0, 1, 512, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 2, 0, 4, 0, 0, 648, 3407, 2047, 0, 0, 0, 0, 0, 0, 1, 6, 0, 3291, 0, 3112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 68, 0, 6073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6140, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6040, 6039, 6108, 6119, 6410, 6144, 6150] avg=6144.0000 max=6410 max_rank=5 min=6039 min_rank=2 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 0, 0, 0, 0, 2445, 1270, 552, 0, 0, 0, 0, 0, 0, 2717, 0, 0, 0, 0, 0, 0, 1970, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 5606, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 3433, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3317, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 6081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6135, 6246, 6144, 6160, 6048, 6126, 6148] avg=6144.0000 max=6246 max_rank=2 min=6048 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=42906 max_load_reduction_pct=0.872925 moved_to_echo=24482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 2446, 1271, 556, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 1973, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 5590, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 3428, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 0, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 6072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6299, 6018, 6123, 6129, 6167, 6130, 6141] avg=6144.0000 max=6299 max_rank=1 min=6018 min_rank=2 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24567 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 15104, 128, 128, 0, 0, 19584, 10240, 4480, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 20992, 0, 0, 128, 128, 0, 128, 15872, 12544, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 26496, 0, 22784, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 5376, 27392, 16384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 44800, 128, 0, 128, 4480, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 49152, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 5376, 27392, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 640, 0, 48640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 640, 0, 48640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4971, 0, 0, 0, 0, 1729, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6416, 8191, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4795, 0, 0, 0, 0, 1857, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6683, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4756, 0, 0, 0, 0, 1895, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6390, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 7598, 0, 6, 0, 17, 0, 0, 0, 6, 0, 0, 4290, 3, 2, 1, 0, 1771, 0, 2, 0, 0, 8180, 8153, 0, 9, 5, 0, 4363, 0, 0, 0, 0, 0, 2, 0, 3, 2, 0, 0, 6545, 8184, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4617, 0, 0, 0, 0, 1849, 0, 0, 0, 0, 8192, 8178, 0, 0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6743, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 1, 3, 7628, 0, 2, 0, 5, 0, 0, 0, 4, 0, 0, 4773, 0, 0, 0, 0, 1768, 0, 1, 0, 0, 8185, 8185, 0, 5, 1, 0, 3583, 5, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 6804, 8188, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 5, 11, 0, 0, 0, 1, 0, 9, 7563, 1, 0, 0, 8, 0, 0, 0, 14, 0, 0, 4749, 2, 0, 0, 0, 2059, 1, 0, 0, 0, 8172, 8171, 3, 14, 6, 0, 3605, 29, 0, 0, 0, 0, 10, 1, 2, 0, 0, 0, 6527, 8179, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 1, 3, 7628, 0, 2, 0, 5, 0, 0, 0, 4, 0, 0, 4773, 0, 0, 0, 0, 1768, 0, 1, 0, 0, 8185, 8185, 0, 5, 1, 0, 3583, 5, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 6804, 8188, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4617, 0, 0, 0, 0, 1849, 0, 0, 0, 0, 8192, 8178, 0, 0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6743, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4772, 0, 0, 0, 0, 1867, 0, 0, 0, 0, 8192, 8186, 0, 0, 0, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6782, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4756, 0, 0, 0, 0, 1895, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6390, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4795, 0, 0, 0, 0, 1857, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6683, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2044, 2, 0, 0, 0, 0, 0, 4331, 0, 0, 0, 0, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 1477, 452, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 6176, 6141, 6226, 6147, 6235, 5938] avg=6144.0000 max=6235 max_rank=6 min=5938 min_rank=7 imbalance_ratio=1.014811 max_load_reduction=42917 max_load_reduction_pct=0.873149 moved_to_echo=24546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2045, 2, 0, 0, 0, 0, 0, 4366, 0, 0, 0, 0, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 1462, 473, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 6114, 6142, 6223, 6147, 6027, 6210] avg=6144.0000 max=6223 max_rank=4 min=6027 min_rank=6 imbalance_ratio=1.012858 max_load_reduction=42929 max_load_reduction_pct=0.873393 moved_to_echo=24607 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4090, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 6, 0, 0, 4090, 2035, 2, 0, 0, 0, 0, 0, 3906, 3, 2, 1, 0, 1771, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 0, 0, 0, 0, 0, 2, 0, 1466, 463, 384, 0, 0, 0, 0, 0, 3, 2, 0, 0, 6082, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6141, 6136, 6150, 5685, 6132, 6678, 6088] avg=6144.0000 max=6678 max_rank=6 min=5685 min_rank=4 imbalance_ratio=1.086914 max_load_reduction=42474 max_load_reduction_pct=0.864136 moved_to_echo=24583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 0, 4087, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 6102, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 14, 0, 0, 4085, 2039, 2, 0, 0, 0, 0, 0, 4324, 2, 0, 0, 0, 2059, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6132, 3, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 29, 0, 0, 0, 0, 10, 1, 1459, 462, 425, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6065, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6147, 6113, 6148, 6386, 6155, 5991, 6069] avg=6144.0000 max=6386 max_rank=4 min=5991 min_rank=6 imbalance_ratio=1.039388 max_load_reduction=42766 max_load_reduction_pct=0.870076 moved_to_echo=24601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4094, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 6154, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 0, 0, 4091, 2043, 2, 0, 0, 0, 0, 0, 4346, 0, 0, 0, 0, 1768, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3583, 5, 0, 0, 0, 0, 5, 1, 1472, 482, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6145, 6160, 6145, 6115, 6148, 5975, 6322] avg=6144.0000 max=6322 max_rank=7 min=5975 min_rank=6 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24560 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2048, 5, 0, 0, 0, 0, 0, 4523, 0, 0, 0, 0, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 1479, 459, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6143, 6153, 6148, 6252, 6143, 6211, 5957] avg=6144.0000 max=6252 max_rank=4 min=5957 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=42900 max_load_reduction_pct=0.872803 moved_to_echo=24563 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4090, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 6, 0, 0, 4090, 2035, 2, 0, 0, 0, 0, 0, 3906, 3, 2, 1, 0, 1771, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 0, 0, 0, 0, 0, 2, 0, 1466, 463, 384, 0, 0, 0, 0, 0, 3, 2, 0, 0, 6082, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6141, 6136, 6150, 5685, 6132, 6678, 6088] avg=6144.0000 max=6678 max_rank=6 min=5685 min_rank=4 imbalance_ratio=1.086914 max_load_reduction=42474 max_load_reduction_pct=0.864136 moved_to_echo=24583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2041, 2, 0, 0, 0, 0, 0, 4204, 0, 0, 0, 0, 1849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 1472, 477, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 6155, 6138, 6053, 6137, 6114, 6266] avg=6144.0000 max=6266 max_rank=7 min=6053 min_rank=4 imbalance_ratio=1.019857 max_load_reduction=42886 max_load_reduction_pct=0.872518 moved_to_echo=24570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 0, 4087, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 6102, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 14, 0, 0, 4085, 2039, 2, 0, 0, 0, 0, 0, 4324, 2, 0, 0, 0, 2059, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6132, 3, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 29, 0, 0, 0, 0, 10, 1, 1459, 462, 425, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6065, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6147, 6113, 6148, 6386, 6155, 5991, 6069] avg=6144.0000 max=6386 max_rank=4 min=5991 min_rank=6 imbalance_ratio=1.039388 max_load_reduction=42766 max_load_reduction_pct=0.870076 moved_to_echo=24601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2043, 2, 0, 0, 0, 0, 0, 4345, 0, 0, 0, 0, 1867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 1470, 480, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 6145, 6140, 6212, 6143, 5921, 6302] avg=6144.0000 max=6302 max_rank=7 min=5921 min_rank=6 imbalance_ratio=1.025716 max_load_reduction=42850 max_load_reduction_pct=0.871785 moved_to_echo=24578 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2048, 5, 0, 0, 0, 0, 0, 4523, 0, 0, 0, 0, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 1479, 459, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6143, 6153, 6148, 6252, 6143, 6211, 5957] avg=6144.0000 max=6252 max_rank=4 min=5957 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=42900 max_load_reduction_pct=0.872803 moved_to_echo=24563 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2045, 2, 0, 0, 0, 0, 0, 4366, 0, 0, 0, 0, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 1462, 473, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 6114, 6142, 6223, 6147, 6027, 6210] avg=6144.0000 max=6223 max_rank=4 min=6027 min_rank=6 imbalance_ratio=1.012858 max_load_reduction=42929 max_load_reduction_pct=0.873393 moved_to_echo=24607 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[30336, 128, 0, 0, 0, 0, 128, 128, 11776, 3840, 3456, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[34432, 128, 128, 128, 0, 14848, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 49152, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 32768, 16384, 128, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 0, 32768, 16384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 49152, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 32768, 16384, 128, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 49152, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7482, 7857, 0, 4929, 0, 8124, 0, 0, 0, 0, 0, 1, 0, 251, 0, 0, 0, 0, 0, 0, 0, 2408, 0, 411, 2, 0, 0, 0, 0, 0, 257, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 354, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[8184, 0, 5, 0, 8, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 7579, 7895, 1, 4836, 0, 8079, 1, 3, 0, 0, 0, 1, 0, 243, 4, 0, 0, 0, 1, 7, 0, 2539, 0, 274, 4, 2, 0, 0, 6, 5, 343, 0, 493, 0, 0, 0, 0, 0, 5, 0, 0, 1, 134, 301, 0, 5, 7, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7687, 7893, 0, 4107, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 3044, 0, 453, 0, 0, 0, 0, 0, 0, 309, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 124, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7593, 7839, 0, 4778, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 365, 1, 0, 0, 0, 0, 0, 341, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 262, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7482, 7857, 0, 4929, 0, 8124, 0, 0, 0, 0, 0, 1, 0, 251, 0, 0, 0, 0, 0, 0, 0, 2408, 0, 411, 2, 0, 0, 0, 0, 0, 257, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 354, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[8175, 0, 11, 0, 15, 0, 0, 8178, 0, 0, 0, 0, 0, 11, 0, 7444, 7810, 4, 4655, 0, 8082, 0, 0, 0, 0, 0, 2, 0, 235, 12, 0, 0, 0, 3, 1, 0, 2688, 0, 371, 1, 2, 0, 0, 8, 23, 398, 0, 469, 0, 0, 0, 1, 0, 13, 0, 0, 9, 119, 393, 0, 2, 13, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 7508, 7904, 0, 4524, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 490, 1, 0, 0, 0, 0, 0, 424, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 317, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[8187, 0, 3, 0, 4, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 1, 7396, 7787, 2, 4863, 0, 8107, 0, 3, 0, 0, 0, 1, 0, 250, 1, 0, 0, 0, 2, 0, 0, 2583, 0, 354, 0, 1, 3, 0, 4, 2, 295, 0, 474, 0, 0, 1, 0, 1, 3, 0, 0, 2, 208, 414, 0, 5, 4, 0, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7512, 7915, 0, 4568, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 2637, 0, 528, 0, 0, 0, 0, 0, 0, 362, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 353, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[8187, 0, 3, 0, 4, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 1, 7396, 7787, 2, 4863, 0, 8107, 0, 3, 0, 0, 0, 1, 0, 250, 1, 0, 0, 0, 2, 0, 0, 2583, 0, 354, 0, 1, 3, 0, 4, 2, 295, 0, 474, 0, 0, 1, 0, 1, 3, 0, 0, 2, 208, 414, 0, 5, 4, 0, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 7508, 7904, 0, 4524, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 490, 1, 0, 0, 0, 0, 0, 424, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 317, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[8184, 0, 5, 0, 8, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 7579, 7895, 1, 4836, 0, 8079, 1, 3, 0, 0, 0, 1, 0, 243, 4, 0, 0, 0, 1, 7, 0, 2539, 0, 274, 4, 2, 0, 0, 6, 5, 343, 0, 493, 0, 0, 0, 0, 0, 5, 0, 0, 1, 134, 301, 0, 5, 7, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7687, 7893, 0, 4107, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 3044, 0, 453, 0, 0, 0, 0, 0, 0, 309, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 124, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 0, 0, 0, 0, 0, 0, 0, 1485, 0, 4929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 251, 0, 0, 0, 3850, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2408, 0, 411, 2, 1723, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 557, 4977, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 135, 354, 0, 0, 0, 0, 0, 4274, 1395, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6108, 6414, 6150, 5918, 6117, 6144, 6158] avg=6144.0000 max=6414 max_rank=2 min=5918 min_rank=4 imbalance_ratio=1.043945 max_load_reduction=42738 max_load_reduction_pct=0.869507 moved_to_echo=24337 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 8, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 1, 4836, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 243, 4, 0, 0, 3828, 2046, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 2539, 0, 274, 4, 1722, 1392, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 5, 343, 0, 493, 5001, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 1, 134, 301, 0, 5, 7, 0, 2, 4251, 1402, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6188, 6333, 6122, 5939, 6176, 6143, 6103] avg=6144.0000 max=6333 max_rank=2 min=5939 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=42819 max_load_reduction_pct=0.871155 moved_to_echo=24361 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6198, 0, 0, 0, 0, 0, 0, 0, 0, 1481, 0, 4778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 3856, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 365, 1, 1723, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 488, 4966, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 189, 262, 0, 0, 0, 0, 0, 4281, 1392, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6198, 6259, 6159, 6004, 6121, 6144, 6124] avg=6144.0000 max=6259 max_rank=2 min=6004 min_rank=4 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24393 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 4524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 3852, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 490, 1, 1723, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 430, 5007, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 181, 317, 0, 0, 0, 0, 0, 4276, 1403, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6129, 6018, 6145, 6210, 6187, 6144, 6177] avg=6144.0000 max=6210 max_rank=4 min=6018 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=42942 max_load_reduction_pct=0.873657 moved_to_echo=24718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 15, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 6077, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 4, 4655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 235, 12, 0, 0, 3829, 2043, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 2688, 0, 371, 1, 1721, 1367, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 23, 398, 0, 469, 4947, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 9, 119, 393, 0, 2, 13, 0, 4, 4253, 1387, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6088, 6135, 6121, 6152, 6173, 6146, 6180] avg=6144.0000 max=6180 max_rank=7 min=6088 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=42972 max_load_reduction_pct=0.874268 moved_to_echo=24651 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6037, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 2, 4863, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 250, 1, 0, 0, 3842, 2047, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2583, 0, 354, 0, 1722, 1359, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4, 2, 295, 0, 474, 4933, 326, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 2, 208, 414, 0, 5, 4, 0, 5, 4265, 1383, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6038, 6339, 6141, 6020, 6038, 6145, 6286] avg=6144.0000 max=6339 max_rank=2 min=6020 min_rank=4 imbalance_ratio=1.031738 max_load_reduction=42813 max_load_reduction_pct=0.871033 moved_to_echo=24489 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 1496, 0, 4568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 3849, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 0, 528, 0, 1723, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 421, 5014, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 100, 353, 0, 0, 0, 0, 0, 4272, 1405, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6132, 6064, 6151, 6268, 6123, 6143, 6130] avg=6144.0000 max=6268 max_rank=4 min=6064 min_rank=2 imbalance_ratio=1.020182 max_load_reduction=42884 max_load_reduction_pct=0.872477 moved_to_echo=24664 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 4524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 3852, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 490, 1, 1723, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 430, 5007, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 181, 317, 0, 0, 0, 0, 0, 4276, 1403, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6129, 6018, 6145, 6210, 6187, 6144, 6177] avg=6144.0000 max=6210 max_rank=4 min=6018 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=42942 max_load_reduction_pct=0.873657 moved_to_echo=24718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 8, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 1, 4836, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 243, 4, 0, 0, 3828, 2046, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 2539, 0, 274, 4, 1722, 1392, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 5, 343, 0, 493, 5001, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 1, 134, 301, 0, 5, 7, 0, 2, 4251, 1402, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6188, 6333, 6122, 5939, 6176, 6143, 6103] avg=6144.0000 max=6333 max_rank=2 min=5939 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=42819 max_load_reduction_pct=0.871155 moved_to_echo=24361 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6271, 0, 0, 0, 0, 0, 0, 0, 0, 1483, 0, 4107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 3857, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3044, 0, 453, 0, 1728, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 572, 5005, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 183, 124, 0, 0, 0, 0, 0, 4282, 1405, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 6271, 5590, 6163, 6641, 6217, 6143, 5994] avg=6144.0000 max=6641 max_rank=4 min=5590 min_rank=2 imbalance_ratio=1.080892 max_load_reduction=42511 max_load_reduction_pct=0.864889 moved_to_echo=24995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 2048, 128, 0, 0, 30848, 16384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 21120, 0, 3328, 128, 13824, 11136, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 2048, 128, 0, 0, 30848, 16384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1280, 2560, 0, 128, 128, 0, 128, 34176, 11264, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11904, 128, 37376, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 2816, 0, 3968, 39936, 2688, 0, 0, 0, 0, 0, 0] @@ -6876,642 +6876,642 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4236, 3, 2, 0, 8180, 0, 0, 0, 3, 0, 7857, 4, 0, 0, 0, 8177, 0, 0, 0, 0, 8184, 7, 0, 0, 0, 2, 0, 0, 0, 6, 0, 2, 4225, 0, 0, 6, 0, 0, 0, 0, 0, 22, 2, 0, 8184, 0, 7, 30, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4572, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 13, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4072, 0, 2, 0, 8184, 0, 0, 0, 2, 0, 7942, 3, 0, 0, 0, 8183, 0, 0, 0, 0, 8187, 5, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 4363, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 8187, 0, 5, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4167, 1, 1, 0, 8170, 0, 0, 0, 9, 0, 7926, 12, 0, 0, 0, 8172, 0, 0, 0, 2, 8177, 15, 0, 0, 2, 0, 2, 0, 0, 15, 1, 0, 4217, 0, 0, 4, 1, 0, 1, 0, 0, 9, 2, 0, 8177, 0, 15, 36, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4572, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 13, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4245, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4238, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8192, 0, 0, 25, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4236, 3, 2, 0, 8180, 0, 0, 0, 3, 0, 7857, 4, 0, 0, 0, 8177, 0, 0, 0, 0, 8184, 7, 0, 0, 0, 2, 0, 0, 0, 6, 0, 2, 4225, 0, 0, 6, 0, 0, 0, 0, 0, 22, 2, 0, 8184, 0, 7, 30, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4274, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 7984, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4167, 1, 1, 0, 8170, 0, 0, 0, 9, 0, 7926, 12, 0, 0, 0, 8172, 0, 0, 0, 2, 8177, 15, 0, 0, 2, 0, 2, 0, 0, 15, 1, 0, 4217, 0, 0, 4, 1, 0, 1, 0, 0, 9, 2, 0, 8177, 0, 15, 36, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4396, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8002, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 11, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 1814, 91, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4315, 0, 1955, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 2046, 2029, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 5959, 6146, 6199, 6144, 6289, 6139, 6132] avg=6144.0000 max=6289 max_rank=5 min=5959 min_rank=1 imbalance_ratio=1.023600 max_load_reduction=42863 max_load_reduction_pct=0.872050 moved_to_echo=24704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 1, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4236, 3, 1779, 91, 0, 0, 0, 0, 0, 0, 2, 0, 6137, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6078, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 7, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 4225, 0, 1952, 19, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 22, 2062, 2043, 2027, 0, 0, 0, 0, 0, 2, 0, 6122, 0, 7, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6115, 6142, 6082, 6147, 6204, 6160, 6161] avg=6144.0000 max=6204 max_rank=5 min=6082 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=42948 max_load_reduction_pct=0.873779 moved_to_echo=24672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4167, 1, 1795, 91, 0, 0, 0, 0, 0, 0, 1, 0, 6130, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6132, 15, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 15, 1, 0, 4217, 0, 1949, 19, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 1, 0, 0, 9, 2060, 2041, 2026, 0, 0, 0, 0, 0, 2, 0, 6117, 0, 15, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6057, 6140, 6143, 6151, 6203, 6142, 6170] avg=6144.0000 max=6203 max_rank=5 min=6057 min_rank=1 imbalance_ratio=1.009603 max_load_reduction=42949 max_load_reduction_pct=0.873800 moved_to_echo=24666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4274, 0, 1808, 91, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4120, 0, 1955, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 2046, 2029, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6173, 6148, 6176, 6144, 6094, 6139, 6132] avg=6144.0000 max=6176 max_rank=3 min=6094 min_rank=5 imbalance_ratio=1.005208 max_load_reduction=42976 max_load_reduction_pct=0.874349 moved_to_echo=24509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4072, 0, 1798, 91, 0, 0, 0, 0, 0, 0, 2, 0, 6140, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6140, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 4363, 0, 1953, 19, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 2063, 2043, 2028, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 5966, 6144, 6147, 6146, 6338, 6139, 6129] avg=6144.0000 max=6338 max_rank=5 min=5966 min_rank=1 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4396, 0, 1812, 91, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 1955, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 2045, 2029, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6299, 6146, 6190, 6144, 5951, 6138, 6139] avg=6144.0000 max=6299 max_rank=1 min=5951 min_rank=5 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4572, 0, 1794, 91, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 0, 1955, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2064, 2045, 2029, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6457, 6146, 6129, 6144, 5853, 6139, 6141] avg=6144.0000 max=6457 max_rank=1 min=5853 min_rank=5 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4274, 0, 1808, 91, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4120, 0, 1955, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 2046, 2029, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6173, 6148, 6176, 6144, 6094, 6139, 6132] avg=6144.0000 max=6176 max_rank=3 min=6094 min_rank=5 imbalance_ratio=1.005208 max_load_reduction=42976 max_load_reduction_pct=0.874349 moved_to_echo=24509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 1, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4236, 3, 1779, 91, 0, 0, 0, 0, 0, 0, 2, 0, 6137, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6078, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 7, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 4225, 0, 1952, 19, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 22, 2062, 2043, 2027, 0, 0, 0, 0, 0, 2, 0, 6122, 0, 7, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6115, 6142, 6082, 6147, 6204, 6160, 6161] avg=6144.0000 max=6204 max_rank=5 min=6082 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=42948 max_load_reduction_pct=0.873779 moved_to_echo=24672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4245, 0, 1785, 96, 0, 0, 0, 0, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4238, 0, 1956, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2069, 2045, 2034, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6126, 6140, 6086, 6132, 6220, 6156, 6148] avg=6144.0000 max=6220 max_rank=5 min=6086 min_rank=3 imbalance_ratio=1.012370 max_load_reduction=42932 max_load_reduction_pct=0.873454 moved_to_echo=24656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4167, 1, 1795, 91, 0, 0, 0, 0, 0, 0, 1, 0, 6130, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6132, 15, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 15, 1, 0, 4217, 0, 1949, 19, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 1, 0, 0, 9, 2060, 2041, 2026, 0, 0, 0, 0, 0, 2, 0, 6117, 0, 15, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6057, 6140, 6143, 6151, 6203, 6142, 6170] avg=6144.0000 max=6203 max_rank=5 min=6057 min_rank=1 imbalance_ratio=1.009603 max_load_reduction=42949 max_load_reduction_pct=0.873800 moved_to_echo=24666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4396, 0, 1812, 91, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 1955, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 2045, 2029, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6299, 6146, 6190, 6144, 5951, 6138, 6139] avg=6144.0000 max=6299 max_rank=1 min=5951 min_rank=5 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 16512, 16384, 16256, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 128, 34048, 128, 14464, 768, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 128, 33408, 0, 15744, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 49152, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 49024, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 49152, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 49152, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 49024, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 9, 0, 0, 7823, 8018, 292, 1, 0, 8, 0, 13, 8, 2501, 0, 10, 1772, 0, 7552, 2, 6893, 0, 61, 0, 0, 14, 0, 0, 0, 0, 0, 4897, 8109, 0, 0, 2, 0, 0, 0, 3, 2, 626, 502, 6, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 1, 0, 1, 1, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 7790, 292, 0, 0, 0, 0, 0, 0, 2644, 0, 0, 1767, 0, 7943, 0, 7008, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4797, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 515, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 7777, 7867, 297, 0, 0, 0, 0, 0, 0, 2577, 0, 0, 1793, 0, 7763, 0, 7126, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 4645, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 558, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 7828, 8021, 252, 2, 0, 3, 3, 8, 3, 2474, 0, 6, 1780, 0, 7560, 0, 7133, 0, 90, 0, 0, 6, 0, 0, 1, 0, 0, 4772, 8112, 0, 0, 1, 0, 0, 0, 2, 2, 546, 526, 2, 0, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 0, 3, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7856, 7991, 284, 0, 0, 0, 0, 0, 0, 2537, 0, 0, 2069, 0, 7509, 0, 7004, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 4730, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 526, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 7924, 351, 3, 0, 3, 0, 4, 2, 3129, 0, 2, 1705, 0, 7499, 1, 6719, 0, 120, 0, 0, 4, 0, 0, 0, 0, 0, 4540, 8130, 0, 0, 1, 0, 0, 0, 2, 0, 702, 431, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 1, 1, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7805, 7933, 298, 0, 0, 0, 0, 0, 0, 2687, 0, 0, 1961, 0, 7825, 0, 6997, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 4361, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 588, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 7790, 292, 0, 0, 0, 0, 0, 0, 2644, 0, 0, 1767, 0, 7943, 0, 7008, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4797, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 515, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7847, 8038, 340, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 1938, 0, 7470, 0, 7008, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 4733, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 585, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 6198, 0, 0, 0, 0, 0, 0, 0, 0, 5841, 0, 252, 2, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 2474, 0, 6, 1780, 0, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6168, 0, 90, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4772, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 546, 526, 2147, 1987, 965, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 3770, 2325, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 5, 4328, 1823, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6200, 6101, 5914, 6264, 6232, 6176, 6105, 6160] avg=6144.0000 max=6264 max_rank=3 min=5914 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=42888 max_load_reduction_pct=0.872559 moved_to_echo=24673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 5870, 0, 351, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 3129, 0, 2, 1705, 0, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5810, 0, 120, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4540, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 702, 431, 2129, 1997, 909, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 3739, 2330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 3, 4338, 1800, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6227, 6473, 5935, 6002, 6171, 6076, 6144] avg=6144.0000 max=6473 max_rank=2 min=5935 min_rank=3 imbalance_ratio=1.053548 max_load_reduction=42679 max_load_reduction_pct=0.868306 moved_to_echo=24393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 6080, 0, 0, 0, 0, 0, 0, 0, 0, 5803, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2577, 0, 0, 1793, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4645, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 505, 2204, 1974, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4336, 1787, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6100, 6058, 6279, 6106, 6205, 6200, 6123] avg=6144.0000 max=6279 max_rank=3 min=6058 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=42873 max_load_reduction_pct=0.872253 moved_to_echo=24634 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 9, 0, 6196, 0, 0, 0, 0, 0, 0, 0, 0, 5837, 0, 292, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8, 2501, 0, 10, 1772, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5961, 0, 61, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4897, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 2, 626, 502, 2144, 1986, 932, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 15, 0, 0, 3766, 2324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 7, 4327, 1822, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6207, 6138, 5946, 6038, 6355, 6197, 6112, 6159] avg=6144.0000 max=6355 max_rank=4 min=5946 min_rank=2 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 9, 0, 0, 7823, 8018, 292, 1, 0, 8, 0, 13, 8, 2501, 0, 10, 1772, 0, 7552, 2, 6893, 0, 61, 0, 0, 14, 0, 0, 0, 0, 0, 4897, 8109, 0, 0, 2, 0, 0, 0, 3, 2, 626, 502, 6, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 1, 0, 1, 1, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 7924, 351, 3, 0, 3, 0, 4, 2, 3129, 0, 2, 1705, 0, 7499, 1, 6719, 0, 120, 0, 0, 4, 0, 0, 0, 0, 0, 4540, 8130, 0, 0, 1, 0, 0, 0, 2, 0, 702, 431, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 1, 1, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7856, 7991, 284, 0, 0, 0, 0, 0, 0, 2537, 0, 0, 2069, 0, 7509, 0, 7004, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 4730, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 526, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6021, 0, 0, 0, 0, 0, 0, 0, 0, 5835, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2644, 0, 0, 1767, 0, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6060, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4797, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 360, 2255, 1985, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4332, 1769, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6021, 6127, 6138, 6156, 6256, 6063, 6288, 6103] avg=6144.0000 max=6288 max_rank=6 min=6021 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=42864 max_load_reduction_pct=0.872070 moved_to_echo=24710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 9, 0, 6196, 0, 0, 0, 0, 0, 0, 0, 0, 5837, 0, 292, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8, 2501, 0, 10, 1772, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5961, 0, 61, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4897, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 2, 626, 502, 2144, 1986, 932, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 15, 0, 0, 3766, 2324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 7, 4327, 1822, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6207, 6138, 5946, 6038, 6355, 6197, 6112, 6159] avg=6144.0000 max=6355 max_rank=4 min=5946 min_rank=2 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6212, 0, 0, 0, 0, 0, 0, 0, 0, 5855, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 1938, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6060, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4733, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 444, 2121, 1992, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4341, 1826, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6212, 6195, 6043, 6192, 6196, 6090, 6057, 6167] avg=6144.0000 max=6212 max_rank=0 min=6043 min_rank=2 imbalance_ratio=1.011068 max_load_reduction=42940 max_load_reduction_pct=0.873617 moved_to_echo=24510 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 5820, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 0, 0, 1961, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6046, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4361, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 438, 2227, 1985, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 2337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4346, 1802, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6118, 6341, 6168, 5815, 6189, 6242, 6148] avg=6144.0000 max=6341 max_rank=2 min=5815 min_rank=4 imbalance_ratio=1.032064 max_load_reduction=42811 max_load_reduction_pct=0.870992 moved_to_echo=24394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 6198, 0, 0, 0, 0, 0, 0, 0, 0, 5841, 0, 252, 2, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 2474, 0, 6, 1780, 0, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6168, 0, 90, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4772, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 546, 526, 2147, 1987, 965, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 3770, 2325, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 5, 4328, 1823, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6200, 6101, 5914, 6264, 6232, 6176, 6105, 6160] avg=6144.0000 max=6264 max_rank=3 min=5914 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=42888 max_load_reduction_pct=0.872559 moved_to_echo=24673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 6080, 0, 0, 0, 0, 0, 0, 0, 0, 5803, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2577, 0, 0, 1793, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4645, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 505, 2204, 1974, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4336, 1787, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6100, 6058, 6279, 6106, 6205, 6200, 6123] avg=6144.0000 max=6279 max_rank=3 min=6058 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=42873 max_load_reduction_pct=0.872253 moved_to_echo=24634 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 5862, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2537, 0, 0, 2069, 0, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4730, 1460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 462, 2132, 1994, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4333, 1815, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 6146, 6239, 6120, 6190, 6061, 6072, 6148] avg=6144.0000 max=6239 max_rank=2 min=6061 min_rank=5 imbalance_ratio=1.015462 max_load_reduction=42913 max_load_reduction_pct=0.873067 moved_to_echo=24471 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 5820, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 0, 0, 1961, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6046, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4361, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 438, 2227, 1985, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 2337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4346, 1802, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6118, 6341, 6168, 5815, 6189, 6242, 6148] avg=6144.0000 max=6341 max_rank=2 min=5815 min_rank=4 imbalance_ratio=1.032064 max_load_reduction=42811 max_load_reduction_pct=0.870992 moved_to_echo=24394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 5870, 0, 351, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 3129, 0, 2, 1705, 0, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5810, 0, 120, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4540, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 702, 431, 2129, 1997, 909, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 3739, 2330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 3, 4338, 1800, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6227, 6473, 5935, 6002, 6171, 6076, 6144] avg=6144.0000 max=6473 max_rank=2 min=5935 min_rank=3 imbalance_ratio=1.053548 max_load_reduction=42679 max_load_reduction_pct=0.868306 moved_to_echo=24393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 4736, 3712, 17408, 16000, 7680, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 21120, 0, 128, 14848, 0, 13312, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 37504, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 128, 34688, 14464, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 46848, 0, 2432, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 46848, 0, 2432, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 30592, 18688, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 48384, 0, 896, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 30592, 18688, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 735, 289, 0, 0, 0, 0, 0, 1, 1104, 0, 0, 4226, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 0, 0, 0, 0, 7829, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 8192, 0, 0, 16, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 3698, 0, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 2, 738, 366, 4, 0, 0, 0, 0, 0, 1093, 0, 8, 4111, 0, 0, 0, 0, 0, 0, 5, 3058, 0, 0, 0, 0, 1, 7791, 4, 1, 0, 0, 8183, 2, 8182, 0, 4, 0, 8182, 9, 7, 32, 0, 2, 3419, 0, 0, 5, 0, 0, 3, 0, 1, 394, 0, 0, 3461, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 801, 344, 0, 0, 0, 0, 0, 0, 1252, 0, 0, 4372, 0, 0, 0, 0, 0, 0, 0, 2781, 0, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 3556, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 3201, 0, 76] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 817, 300, 0, 0, 0, 0, 0, 0, 1073, 0, 0, 4303, 0, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 7, 0, 0, 3790, 0, 1, 0, 2, 0, 0, 0, 0, 327, 0, 0, 3614, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 857, 412, 6, 0, 0, 0, 0, 0, 1042, 0, 1, 4248, 0, 0, 0, 1, 0, 0, 5, 3175, 0, 0, 0, 0, 2, 7757, 4, 0, 0, 0, 8185, 0, 8186, 0, 4, 1, 8186, 4, 4, 9, 0, 0, 3354, 0, 16, 3, 0, 0, 2, 0, 3, 433, 0, 0, 3164, 0, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 679, 338, 0, 0, 0, 0, 0, 0, 1059, 0, 0, 4426, 0, 0, 0, 0, 0, 0, 0, 2607, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 3721, 0, 13, 0, 0, 0, 0, 0, 0, 369, 0, 0, 3458, 0, 78] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 2, 738, 366, 4, 0, 0, 0, 0, 0, 1093, 0, 8, 4111, 0, 0, 0, 0, 0, 0, 5, 3058, 0, 0, 0, 0, 1, 7791, 4, 1, 0, 0, 8183, 2, 8182, 0, 4, 0, 8182, 9, 7, 32, 0, 2, 3419, 0, 0, 5, 0, 0, 3, 0, 1, 394, 0, 0, 3461, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 7, 707, 332, 14, 1, 0, 0, 0, 0, 1173, 0, 5, 4342, 0, 0, 0, 3, 0, 0, 11, 3049, 0, 0, 0, 0, 6, 7755, 12, 2, 0, 0, 8174, 0, 8174, 0, 9, 0, 8172, 15, 12, 12, 0, 1, 3142, 0, 13, 11, 0, 0, 11, 0, 2, 437, 0, 0, 3459, 0, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 857, 412, 6, 0, 0, 0, 0, 0, 1042, 0, 1, 4248, 0, 0, 0, 1, 0, 0, 5, 3175, 0, 0, 0, 0, 2, 7757, 4, 0, 0, 0, 8185, 0, 8186, 0, 4, 1, 8186, 4, 4, 9, 0, 0, 3354, 0, 16, 3, 0, 0, 2, 0, 3, 433, 0, 0, 3164, 0, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 801, 344, 0, 0, 0, 0, 0, 0, 1252, 0, 0, 4372, 0, 0, 0, 0, 0, 0, 0, 2781, 0, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 3556, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 3201, 0, 76] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 719, 341, 0, 0, 0, 0, 0, 0, 1201, 0, 0, 4446, 0, 0, 0, 0, 0, 0, 0, 3013, 0, 0, 0, 0, 0, 7806, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 3061, 0, 1, 0, 1, 0, 0, 0, 0, 386, 0, 0, 3529, 0, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 2, 738, 5381, 0, 0, 0, 0, 0, 0, 0, 366, 4, 0, 0, 0, 0, 0, 1093, 2801, 1872, 0, 0, 0, 0, 0, 0, 0, 8, 4111, 0, 0, 0, 0, 0, 1657, 173, 0, 0, 0, 0, 0, 0, 0, 5, 3058, 0, 0, 0, 0, 1, 3273, 0, 0, 0, 0, 0, 0, 0, 6134, 4, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6122, 9, 7, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3419, 0, 0, 5, 0, 0, 3, 2700, 0, 0, 0, 0, 0, 0, 0, 0, 1, 394, 0, 0, 3461, 0, 80, 1887, 338, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6125, 6136, 5949, 6337, 6141, 6174, 6129, 6161] avg=6144.0000 max=6337 max_rank=3 min=5949 min_rank=2 imbalance_ratio=1.031413 max_load_reduction=42815 max_load_reduction_pct=0.871073 moved_to_echo=24605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 735, 289, 0, 0, 0, 0, 0, 1, 1104, 0, 0, 4226, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 0, 0, 0, 0, 7829, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 8192, 0, 0, 16, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 3698, 0, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 679, 338, 0, 0, 0, 0, 0, 0, 1059, 0, 0, 4426, 0, 0, 0, 0, 0, 0, 0, 2607, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 3721, 0, 13, 0, 0, 0, 0, 0, 0, 369, 0, 0, 3458, 0, 78] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 817, 5388, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 1073, 2804, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 4303, 0, 0, 0, 0, 0, 1672, 173, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 6193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 0, 1, 0, 2, 0, 0, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 3614, 0, 72, 1890, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6051, 6148, 5682, 6193, 6136, 6498, 6239] avg=6144.0000 max=6498 max_rank=6 min=5682 min_rank=3 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=25066 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 7, 707, 5376, 0, 0, 0, 0, 0, 0, 0, 332, 14, 1, 0, 0, 0, 0, 1173, 2798, 1870, 0, 0, 0, 0, 0, 0, 0, 5, 4342, 0, 0, 0, 3, 0, 1649, 173, 0, 0, 0, 0, 0, 0, 0, 11, 3049, 0, 0, 0, 0, 6, 3269, 0, 0, 0, 0, 0, 0, 0, 6106, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 6114, 15, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3142, 0, 13, 11, 0, 0, 11, 2699, 0, 0, 0, 0, 0, 0, 0, 0, 2, 437, 0, 0, 3459, 0, 88, 1885, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 6188, 6172, 6335, 6120, 6162, 5877, 6207] avg=6144.0000 max=6335 max_rank=3 min=5877 min_rank=6 imbalance_ratio=1.031087 max_load_reduction=42817 max_load_reduction_pct=0.871114 moved_to_echo=24366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 801, 5388, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 0, 1252, 2804, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 4372, 0, 0, 0, 0, 0, 1653, 173, 0, 0, 0, 0, 0, 0, 0, 0, 2781, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3556, 0, 0, 0, 0, 0, 0, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 3201, 0, 76, 1890, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6189, 6274, 6198, 6058, 6120, 6130, 6261, 5922] avg=6144.0000 max=6274 max_rank=1 min=5922 min_rank=7 imbalance_ratio=1.021159 max_load_reduction=42878 max_load_reduction_pct=0.872355 moved_to_echo=24433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 735, 5388, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 1, 1104, 2804, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 4226, 0, 0, 0, 0, 0, 1665, 173, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 6164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 2704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 3698, 0, 81, 1890, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6072, 6064, 6108, 6164, 6145, 6108, 6368] avg=6144.0000 max=6368 max_rank=7 min=6064 min_rank=2 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 2, 738, 5381, 0, 0, 0, 0, 0, 0, 0, 366, 4, 0, 0, 0, 0, 0, 1093, 2801, 1872, 0, 0, 0, 0, 0, 0, 0, 8, 4111, 0, 0, 0, 0, 0, 1657, 173, 0, 0, 0, 0, 0, 0, 0, 5, 3058, 0, 0, 0, 0, 1, 3273, 0, 0, 0, 0, 0, 0, 0, 6134, 4, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6122, 9, 7, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3419, 0, 0, 5, 0, 0, 3, 2700, 0, 0, 0, 0, 0, 0, 0, 0, 1, 394, 0, 0, 3461, 0, 80, 1887, 338, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6125, 6136, 5949, 6337, 6141, 6174, 6129, 6161] avg=6144.0000 max=6337 max_rank=3 min=5949 min_rank=2 imbalance_ratio=1.031413 max_load_reduction=42815 max_load_reduction_pct=0.871073 moved_to_echo=24605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 857, 5385, 0, 0, 0, 0, 0, 0, 0, 412, 6, 0, 0, 0, 0, 0, 1042, 2801, 1874, 0, 0, 0, 0, 0, 0, 0, 1, 4248, 0, 0, 0, 1, 0, 1649, 173, 0, 0, 0, 0, 0, 0, 0, 5, 3175, 0, 0, 0, 0, 2, 3274, 0, 0, 0, 0, 0, 0, 0, 6108, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 6125, 4, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3354, 0, 16, 3, 0, 0, 2, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 3, 433, 0, 0, 3164, 0, 86, 1888, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6244, 6135, 6072, 6456, 6112, 6147, 6076, 5910] avg=6144.0000 max=6456 max_rank=3 min=5910 min_rank=7 imbalance_ratio=1.050781 max_load_reduction=42696 max_load_reduction_pct=0.868652 moved_to_echo=24245 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 679, 5389, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 1059, 2803, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 4426, 0, 0, 0, 0, 0, 1667, 177, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 13, 0, 0, 0, 0, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 3458, 0, 78, 1892, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6068, 6076, 6270, 5884, 6156, 6128, 6437, 6133] avg=6144.0000 max=6437 max_rank=6 min=5884 min_rank=3 imbalance_ratio=1.047689 max_load_reduction=42715 max_load_reduction_pct=0.869039 moved_to_echo=24854 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 7, 707, 5376, 0, 0, 0, 0, 0, 0, 0, 332, 14, 1, 0, 0, 0, 0, 1173, 2798, 1870, 0, 0, 0, 0, 0, 0, 0, 5, 4342, 0, 0, 0, 3, 0, 1649, 173, 0, 0, 0, 0, 0, 0, 0, 11, 3049, 0, 0, 0, 0, 6, 3269, 0, 0, 0, 0, 0, 0, 0, 6106, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 6114, 15, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3142, 0, 13, 11, 0, 0, 11, 2699, 0, 0, 0, 0, 0, 0, 0, 0, 2, 437, 0, 0, 3459, 0, 88, 1885, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 6188, 6172, 6335, 6120, 6162, 5877, 6207] avg=6144.0000 max=6335 max_rank=3 min=5877 min_rank=6 imbalance_ratio=1.031087 max_load_reduction=42817 max_load_reduction_pct=0.871114 moved_to_echo=24366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 719, 5388, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 1201, 2804, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 4446, 0, 0, 0, 0, 0, 1660, 173, 0, 0, 0, 0, 0, 0, 0, 0, 3013, 0, 0, 0, 0, 0, 3279, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3061, 0, 1, 0, 1, 0, 0, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 0, 0, 3529, 0, 71, 1890, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6107, 6220, 6279, 6292, 6146, 6130, 5766, 6212] avg=6144.0000 max=6292 max_rank=3 min=5766 min_rank=6 imbalance_ratio=1.024089 max_load_reduction=42860 max_load_reduction_pct=0.871989 moved_to_echo=24254 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 735, 5388, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 1, 1104, 2804, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 4226, 0, 0, 0, 0, 0, 1665, 173, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 6164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 2704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 3698, 0, 81, 1890, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6072, 6064, 6108, 6164, 6145, 6108, 6368] avg=6144.0000 max=6368 max_rank=7 min=6064 min_rank=2 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 679, 5389, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 1059, 2803, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 4426, 0, 0, 0, 0, 0, 1667, 177, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 13, 0, 0, 0, 0, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 3458, 0, 78, 1892, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6068, 6076, 6270, 5884, 6156, 6128, 6437, 6133] avg=6144.0000 max=6437 max_rank=6 min=5884 min_rank=3 imbalance_ratio=1.047689 max_load_reduction=42715 max_load_reduction_pct=0.869039 moved_to_echo=24854 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 27520, 0, 128, 128, 128, 0, 128, 21632, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 3200, 0, 0, 27648, 0, 640, 15232, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 128, 128, 0, 0, 0, 128, 9088, 22528, 15104, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 23040, 0, 0, 0, 0, 128, 26240, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 34560, 0, 0, 0, 128, 0, 13312, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 49024, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 128, 128, 0, 0, 0, 128, 9088, 22528, 15104, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 6144, 43136, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[49152, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 49024, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 6144, 43136, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 8192, 0, 0, 0, 81, 0, 0, 255, 7625, 0, 0, 0, 0, 35, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8185, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 5, 0, 1, 0, 0, 193, 0, 0, 6, 8185, 0, 0, 6, 151, 0, 0, 253, 7538, 0, 0, 0, 3, 53, 5, 8186, 0, 0, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 0, 3, 0, 12, 0, 0, 0, 8171, 0, 0, 2, 8175, 3, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4, 0, 0, 0, 8168, 6, 0, 0, 12, 0, 0, 0, 0, 195, 0, 2, 17, 8174, 0, 0, 19, 125, 0, 1, 242, 7498, 0, 0, 0, 14, 107, 13, 8174, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 8192, 0, 0, 0, 76, 0, 0, 252, 7597, 0, 0, 0, 0, 91, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 8192, 0, 0, 0, 81, 0, 0, 255, 7625, 0, 0, 0, 0, 35, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 0, 1, 0, 9, 0, 1, 0, 8182, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 4, 0, 0, 0, 0, 186, 0, 0, 10, 8182, 0, 0, 9, 98, 0, 0, 254, 7512, 0, 0, 0, 6, 137, 8, 8182, 0, 0, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 8192, 0, 0, 0, 92, 0, 0, 245, 7556, 0, 0, 0, 0, 111, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 8192, 0, 0, 0, 105, 0, 0, 246, 7589, 0, 0, 0, 0, 88, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 0, 3, 0, 12, 0, 0, 0, 8171, 0, 0, 2, 8175, 3, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4, 0, 0, 0, 8168, 6, 0, 0, 12, 0, 0, 0, 0, 195, 0, 2, 17, 8174, 0, 0, 19, 125, 0, 1, 242, 7498, 0, 0, 0, 14, 107, 13, 8174, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 8192, 0, 0, 0, 78, 0, 0, 255, 7531, 0, 0, 0, 0, 146, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 8192, 0, 0, 0, 92, 0, 0, 245, 7556, 0, 0, 0, 0, 111, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 0, 1, 0, 9, 0, 1, 0, 8182, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 4, 0, 0, 0, 0, 186, 0, 0, 10, 8182, 0, 0, 9, 98, 0, 0, 254, 7512, 0, 0, 0, 6, 137, 8, 8182, 0, 0, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2047, 2045, 1776, 284, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 2148, 1763, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 252, 5821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 6044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6145, 6147, 6145, 6145, 6134, 6149, 6135] avg=6144.0000 max=6152 max_rank=0 min=6134 min_rank=5 imbalance_ratio=1.001302 max_load_reduction=43000 max_load_reduction_pct=0.874837 moved_to_echo=24563 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 2044, 2044, 1762, 284, 0, 0, 0, 0, 6, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2232, 2146, 1761, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 6, 5953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 151, 0, 0, 253, 5776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 53, 5, 6040, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6146, 6143, 6140, 6145, 6152, 6186, 6104] avg=6144.0000 max=6186 max_rank=6 min=6104 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=42966 max_load_reduction_pct=0.874146 moved_to_echo=24587 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2047, 2045, 1783, 284, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 2148, 1763, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 255, 5842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 6044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 6145, 6147, 6145, 6145, 6154, 6178, 6079] avg=6144.0000 max=6178 max_rank=6 min=6079 min_rank=7 imbalance_ratio=1.005534 max_load_reduction=42974 max_load_reduction_pct=0.874308 moved_to_echo=24556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 0, 1, 0, 2043, 2043, 1756, 284, 0, 0, 0, 0, 9, 0, 1, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6139, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2231, 2145, 1761, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 10, 5951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 98, 0, 0, 254, 5756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 137, 8, 6037, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6147, 6140, 6139, 6139, 6147, 6117, 6193] avg=6144.0000 max=6193 max_rank=7 min=6117 min_rank=6 imbalance_ratio=1.007975 max_load_reduction=42959 max_load_reduction_pct=0.874003 moved_to_echo=24596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2047, 2045, 1767, 284, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 2148, 1763, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 245, 5789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 6044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6145, 6147, 6145, 6145, 6146, 6126, 6155] avg=6144.0000 max=6155 max_rank=7 min=6126 min_rank=6 imbalance_ratio=1.001790 max_load_reduction=42997 max_load_reduction_pct=0.874776 moved_to_echo=24572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 2044, 2044, 1762, 284, 0, 0, 0, 0, 6, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2232, 2146, 1761, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 6, 5953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 151, 0, 0, 253, 5776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 53, 5, 6040, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6146, 6143, 6140, 6145, 6152, 6186, 6104] avg=6144.0000 max=6186 max_rank=6 min=6104 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=42966 max_load_reduction_pct=0.874146 moved_to_echo=24587 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 0, 3, 0, 2041, 2041, 1753, 284, 0, 0, 0, 0, 12, 0, 0, 0, 6129, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4, 0, 0, 0, 2229, 2143, 1758, 0, 0, 0, 0, 0, 6127, 6, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 2, 17, 5945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 125, 0, 1, 242, 5745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 107, 13, 6031, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6129, 6143, 6137, 6141, 6145, 6159, 6132, 6166] avg=6144.0000 max=6166 max_rank=7 min=6129 min_rank=0 imbalance_ratio=1.003581 max_load_reduction=42986 max_load_reduction_pct=0.874552 moved_to_echo=24602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 0, 1, 0, 2043, 2043, 1756, 284, 0, 0, 0, 0, 9, 0, 1, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6139, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2231, 2145, 1761, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 10, 5951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 98, 0, 0, 254, 5756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 137, 8, 6037, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6147, 6140, 6139, 6139, 6147, 6117, 6193] avg=6144.0000 max=6193 max_rank=7 min=6117 min_rank=6 imbalance_ratio=1.007975 max_load_reduction=42959 max_load_reduction_pct=0.874003 moved_to_echo=24596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2047, 2045, 1774, 284, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 2148, 1763, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 246, 5815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 6044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6145, 6147, 6145, 6145, 6122, 6166, 6132] avg=6144.0000 max=6166 max_rank=6 min=6122 min_rank=5 imbalance_ratio=1.003581 max_load_reduction=42986 max_load_reduction_pct=0.874552 moved_to_echo=24565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2047, 2045, 1767, 284, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 2148, 1763, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 245, 5789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 6044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6145, 6147, 6145, 6145, 6146, 6126, 6155] avg=6144.0000 max=6155 max_rank=7 min=6126 min_rank=6 imbalance_ratio=1.001790 max_load_reduction=42997 max_load_reduction_pct=0.874776 moved_to_echo=24572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2049, 2048, 1766, 290, 0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2236, 2150, 1766, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 5956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 255, 5765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 6042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6136, 6144, 6152, 6143, 6138, 6098, 6188] avg=6144.0000 max=6188 max_rank=7 min=6098 min_rank=6 imbalance_ratio=1.007161 max_load_reduction=42964 max_load_reduction_pct=0.874105 moved_to_echo=24567 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 0, 128, 0, 16384, 16384, 14208, 2304, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 0, 17920, 17280, 14208, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 768, 128, 48384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 1536, 0, 128, 128, 47744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 896, 0, 128, 2048, 46336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 49152, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 768, 128, 48384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[49152, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 49152, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 0, 366, 537, 0, 0, 0, 4839, 0, 0, 0, 0, 12, 0, 8192, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 2388, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7932, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 4, 9, 9, 0, 0, 10, 0, 276, 355, 0, 0, 0, 5598, 0, 0, 4, 0, 26, 8, 8180, 3, 0, 1, 4, 0, 0, 8035, 0, 6, 0, 6, 6, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 543, 0, 5, 0, 1836, 1, 0, 1, 0, 8176, 0, 0, 0, 0, 12, 0, 0, 7922, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[2, 0, 3, 2, 6, 0, 0, 11, 0, 396, 283, 0, 0, 0, 5403, 0, 0, 1, 0, 44, 6, 8184, 3, 0, 1, 3, 0, 0, 8016, 0, 2, 0, 4, 5, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 416, 1, 4, 0, 2161, 1, 0, 0, 0, 8183, 0, 0, 0, 0, 1, 0, 0, 7829, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7, 0, 349, 574, 0, 0, 0, 4645, 0, 0, 0, 0, 13, 0, 8192, 0, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 2428, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 0, 345, 375, 0, 0, 0, 5359, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 2159, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 9, 9, 18, 0, 0, 24, 0, 417, 394, 0, 0, 0, 4857, 0, 0, 11, 0, 36, 17, 8170, 2, 0, 1, 5, 0, 1, 8035, 0, 8, 0, 12, 15, 0, 0, 0, 8127, 5, 0, 0, 0, 0, 0, 574, 0, 5, 0, 2380, 0, 0, 3, 0, 8161, 0, 0, 0, 0, 3, 0, 0, 7848, 5, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 387, 422, 0, 0, 0, 4950, 0, 0, 0, 0, 14, 0, 8192, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 2333, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7931, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 0, 366, 537, 0, 0, 0, 4839, 0, 0, 0, 0, 12, 0, 8192, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 2388, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7932, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 0, 347, 534, 0, 0, 0, 4712, 0, 0, 0, 0, 35, 0, 8192, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 2429, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7914, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 9, 9, 0, 0, 10, 4301, 1817, 0, 0, 0, 0, 0, 0, 0, 276, 355, 0, 0, 0, 5598, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 26, 8, 6110, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 6132, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 6125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 5, 0, 1989, 1903, 1481, 253, 0, 0, 0, 0, 1836, 1, 0, 1, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6138, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6532, 6151, 6143, 6137, 6174, 5713, 6152] avg=6144.0000 max=6532 max_rank=1 min=5713 min_rank=6 imbalance_ratio=1.063151 max_load_reduction=42620 max_load_reduction_pct=0.867106 moved_to_echo=24176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 387, 422, 0, 0, 0, 4950, 0, 0, 0, 0, 14, 0, 8192, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 2333, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7931, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 0, 345, 375, 0, 0, 0, 5359, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 2159, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[2, 0, 3, 2, 6, 0, 0, 11, 0, 396, 283, 0, 0, 0, 5403, 0, 0, 1, 0, 44, 6, 8184, 3, 0, 1, 3, 0, 0, 8016, 0, 2, 0, 4, 5, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 416, 1, 4, 0, 2161, 1, 0, 0, 0, 8183, 0, 0, 0, 0, 1, 0, 0, 7829, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7, 0, 349, 574, 0, 0, 0, 4645, 0, 0, 0, 0, 13, 0, 8192, 0, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 2428, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 4309, 1819, 0, 0, 0, 0, 0, 0, 0, 366, 537, 0, 0, 0, 4839, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 2000, 1907, 1483, 253, 0, 0, 0, 0, 2388, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6046, 6132, 6146, 6156, 6118, 6271, 6145] avg=6144.0000 max=6271 max_rank=6 min=6046 min_rank=1 imbalance_ratio=1.020671 max_load_reduction=42881 max_load_reduction_pct=0.872416 moved_to_echo=24690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3, 2, 6, 0, 0, 11, 4304, 1818, 0, 0, 0, 0, 0, 0, 0, 396, 283, 0, 0, 0, 5403, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 44, 6, 6113, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 6118, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 0, 6174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 1, 4, 0, 2006, 1898, 1464, 253, 0, 0, 0, 0, 2161, 1, 0, 0, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6065, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6382, 6167, 6124, 6183, 6042, 6041, 6067] avg=6144.0000 max=6382 max_rank=1 min=6041 min_rank=6 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 4309, 1819, 0, 0, 0, 0, 0, 0, 0, 345, 375, 0, 0, 0, 5359, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 1961, 1904, 1514, 253, 0, 0, 0, 0, 2159, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6389, 6121, 6137, 6039, 6013, 6042, 6274] avg=6144.0000 max=6389 max_rank=1 min=6013 min_rank=5 imbalance_ratio=1.039876 max_load_reduction=42763 max_load_reduction_pct=0.870015 moved_to_echo=24368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7, 4314, 1824, 0, 0, 0, 0, 0, 0, 0, 349, 574, 0, 0, 0, 4645, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 6110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 2003, 1913, 1489, 258, 0, 0, 0, 0, 2428, 0, 0, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 5876, 6123, 6143, 6155, 6254, 6306, 6150] avg=6144.0000 max=6306 max_rank=6 min=5876 min_rank=1 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 9, 18, 0, 0, 24, 4293, 1815, 0, 0, 0, 0, 0, 0, 0, 417, 394, 0, 0, 0, 4857, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 36, 17, 6103, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 1, 6132, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 15, 0, 0, 0, 6134, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 5, 0, 1993, 1903, 1467, 252, 0, 0, 0, 0, 2380, 0, 0, 3, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6081, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 5968, 6169, 6147, 6166, 6194, 6251, 6089] avg=6144.0000 max=6251 max_rank=6 min=5968 min_rank=1 imbalance_ratio=1.017415 max_load_reduction=42901 max_load_reduction_pct=0.872823 moved_to_echo=24700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 9, 9, 0, 0, 10, 4301, 1817, 0, 0, 0, 0, 0, 0, 0, 276, 355, 0, 0, 0, 5598, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 26, 8, 6110, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 6132, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 6125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 5, 0, 1989, 1903, 1481, 253, 0, 0, 0, 0, 1836, 1, 0, 1, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6138, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6532, 6151, 6143, 6137, 6174, 5713, 6152] avg=6144.0000 max=6532 max_rank=1 min=5713 min_rank=6 imbalance_ratio=1.063151 max_load_reduction=42620 max_load_reduction_pct=0.867106 moved_to_echo=24176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 4309, 1819, 0, 0, 0, 0, 0, 0, 0, 345, 375, 0, 0, 0, 5359, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 1961, 1904, 1514, 253, 0, 0, 0, 0, 2159, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6389, 6121, 6137, 6039, 6013, 6042, 6274] avg=6144.0000 max=6389 max_rank=1 min=6013 min_rank=5 imbalance_ratio=1.039876 max_load_reduction=42763 max_load_reduction_pct=0.870015 moved_to_echo=24368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 4309, 1819, 0, 0, 0, 0, 0, 0, 0, 387, 422, 0, 0, 0, 4950, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 1996, 1912, 1483, 253, 0, 0, 0, 0, 2333, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6063, 6134, 6162, 6144, 6157, 6216, 6144] avg=6144.0000 max=6216 max_rank=6 min=6063 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=42936 max_load_reduction_pct=0.873535 moved_to_echo=24661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 4309, 1819, 0, 0, 0, 0, 0, 0, 0, 347, 534, 0, 0, 0, 4712, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 2005, 1908, 1480, 253, 0, 0, 0, 0, 2429, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 5896, 6155, 6150, 6172, 6200, 6312, 6131] avg=6144.0000 max=6312 max_rank=6 min=5896 min_rank=1 imbalance_ratio=1.027344 max_load_reduction=42840 max_load_reduction_pct=0.871582 moved_to_echo=24815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3, 2, 6, 0, 0, 11, 4304, 1818, 0, 0, 0, 0, 0, 0, 0, 396, 283, 0, 0, 0, 5403, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 44, 6, 6113, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 6118, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 0, 6174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 1, 4, 0, 2006, 1898, 1464, 253, 0, 0, 0, 0, 2161, 1, 0, 0, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6065, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6382, 6167, 6124, 6183, 6042, 6041, 6067] avg=6144.0000 max=6382 max_rank=1 min=6041 min_rank=6 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7, 4314, 1824, 0, 0, 0, 0, 0, 0, 0, 349, 574, 0, 0, 0, 4645, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 6110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 2003, 1913, 1489, 258, 0, 0, 0, 0, 2428, 0, 0, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 5876, 6123, 6143, 6155, 6254, 6306, 6150] avg=6144.0000 max=6306 max_rank=6 min=5876 min_rank=1 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 4096, 128, 128, 0, 16000, 15360, 11904, 2048, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 0, 0, 128, 34560, 14592, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[18176, 128, 0, 128, 0, 31104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 2944, 3584, 0, 0, 0, 40448, 0, 2432, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 256, 128, 49024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 256, 128, 49024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True - [2026-08-26 15:39:20] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 812.6 | throughput per GPU (TFLOP/s/GPU): 157.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.368878E+00 | loss scale: 1.0 | grad norm: 5.139 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1743, 569, 425, 947, 81, 1253, 835, 145, 2600, 592, 919, 261, 5, 2767, 694, 857, 570, 820, 799, 683, 153, 374, 154, 1141, 316, 590, 732, 422, 846, 324, 633, 823, 753, 600, 1524, 1338, 765, 193, 940, 824, 215, 478, 486, 182, 828, 1239, 382, 141, 345, 437, 575, 678, 775, 2241, 338, 632, 2126, 385, 1647, 712, 171, 636, 1615, 878] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 + [2026-08-26 16:35:22] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 824.2 | throughput per GPU (TFLOP/s/GPU): 154.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.368878E+00 | loss scale: 1.0 | grad norm: 5.139 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1726, 518, 457, 921, 88, 1264, 1077, 173, 2523, 591, 1086, 227, 11, 2745, 719, 732, 602, 816, 825, 774, 134, 392, 144, 1052, 402, 575, 723, 449, 840, 411, 730, 1003, 825, 649, 1459, 1415, 671, 214, 923, 763, 251, 490, 488, 179, 851, 1234, 406, 133, 338, 386, 492, 732, 724, 1943, 278, 667, 1950, 348, 1700, 708, 174, 650, 1497, 884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1854, 565, 379, 910, 110, 1378, 1097, 171, 2517, 570, 1035, 255, 10, 2803, 752, 763, 529, 744, 732, 701, 94, 384, 171, 958, 320, 545, 602, 494, 850, 335, 615, 958, 788, 623, 1397, 1418, 560, 197, 968, 755, 196, 589, 525, 130, 921, 1152, 399, 145, 320, 378, 500, 867, 720, 2181, 347, 540, 2036, 441, 1741, 768, 222, 708, 1583, 836] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1818, 556, 358, 963, 128, 1261, 898, 237, 2579, 497, 805, 241, 13, 2643, 750, 848, 556, 777, 730, 769, 136, 419, 201, 1055, 372, 515, 664, 464, 901, 200, 656, 907, 763, 726, 1433, 1445, 822, 228, 888, 738, 229, 579, 520, 186, 960, 1168, 370, 127, 318, 408, 567, 716, 764, 2168, 274, 631, 1976, 411, 1748, 763, 176, 672, 1590, 871] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1724, 594, 414, 933, 96, 1431, 1043, 146, 2662, 574, 773, 269, 5, 2699, 712, 840, 556, 775, 755, 767, 106, 366, 172, 1076, 346, 646, 634, 385, 862, 274, 695, 971, 802, 610, 1415, 1448, 768, 208, 957, 726, 203, 502, 516, 181, 870, 1159, 333, 119, 304, 399, 552, 817, 723, 2062, 337, 632, 1958, 380, 1868, 720, 196, 728, 1559, 799] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1738, 449, 446, 814, 118, 1298, 1166, 204, 2409, 628, 1027, 280, 8, 2751, 753, 715, 546, 797, 778, 645, 136, 416, 168, 990, 309, 569, 716, 465, 848, 269, 676, 1032, 856, 672, 1282, 1453, 666, 256, 965, 709, 262, 535, 552, 165, 928, 1132, 337, 108, 400, 410, 525, 627, 748, 2057, 320, 628, 1909, 379, 1685, 803, 264, 750, 1764, 841] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1854, 565, 379, 910, 110, 1378, 1097, 171, 2517, 570, 1035, 255, 10, 2803, 752, 763, 529, 744, 732, 701, 94, 384, 171, 958, 320, 545, 602, 494, 850, 335, 615, 958, 788, 623, 1397, 1418, 560, 197, 968, 755, 196, 589, 525, 130, 921, 1152, 399, 145, 320, 378, 500, 867, 720, 2181, 347, 540, 2036, 441, 1741, 768, 222, 708, 1583, 836] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1678, 451, 336, 1008, 122, 1384, 1193, 147, 2529, 564, 1020, 338, 10, 2672, 715, 736, 579, 772, 836, 646, 115, 372, 159, 1025, 341, 558, 684, 484, 894, 365, 773, 1046, 775, 647, 1475, 1316, 743, 231, 901, 784, 247, 503, 477, 198, 938, 1137, 319, 159, 325, 421, 480, 717, 724, 1981, 325, 589, 1881, 446, 1646, 830, 145, 668, 1639, 933] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1726, 518, 457, 921, 88, 1264, 1077, 173, 2523, 591, 1086, 227, 11, 2745, 719, 732, 602, 816, 825, 774, 134, 392, 144, 1052, 402, 575, 723, 449, 840, 411, 730, 1003, 825, 649, 1459, 1415, 671, 214, 923, 763, 251, 490, 488, 179, 851, 1234, 406, 133, 338, 386, 492, 732, 724, 1943, 278, 667, 1950, 348, 1700, 708, 174, 650, 1497, 884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1743, 569, 425, 947, 81, 1253, 835, 145, 2600, 592, 919, 261, 5, 2767, 694, 857, 570, 820, 799, 683, 153, 374, 154, 1141, 316, 590, 732, 422, 846, 324, 633, 823, 753, 600, 1524, 1338, 765, 193, 940, 824, 215, 478, 486, 182, 828, 1239, 382, 141, 345, 437, 575, 678, 775, 2241, 338, 632, 2126, 385, 1647, 712, 171, 636, 1615, 878] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1735, 539, 293, 855, 108, 1388, 990, 120, 2658, 512, 976, 258, 10, 2895, 778, 849, 543, 779, 832, 723, 83, 381, 112, 1109, 305, 573, 622, 431, 830, 248, 722, 920, 829, 636, 1492, 1428, 629, 212, 932, 797, 193, 502, 430, 165, 878, 1187, 378, 104, 310, 378, 506, 681, 783, 2254, 353, 583, 2198, 351, 1761, 720, 155, 665, 1580, 905] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1724, 594, 414, 933, 96, 1431, 1043, 146, 2662, 574, 773, 269, 5, 2699, 712, 840, 556, 775, 755, 767, 106, 366, 172, 1076, 346, 646, 634, 385, 862, 274, 695, 971, 802, 610, 1415, 1448, 768, 208, 957, 726, 203, 502, 516, 181, 870, 1159, 333, 119, 304, 399, 552, 817, 723, 2062, 337, 632, 1958, 380, 1868, 720, 196, 728, 1559, 799] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1655, 569, 425, 947, 81, 1253, 835, 145, 0, 0, 0, 0, 0, 0, 0, 0, 2600, 592, 919, 261, 5, 265, 694, 857, 0, 0, 0, 0, 0, 0, 0, 0, 570, 820, 799, 683, 153, 374, 154, 1141, 1328, 330, 0, 0, 0, 0, 0, 0, 316, 590, 732, 422, 846, 324, 633, 823, 798, 588, 0, 0, 0, 0, 0, 0, 753, 600, 718, 1338, 765, 193, 940, 824, 0, 0, 0, 0, 0, 0, 0, 0, 215, 478, 486, 182, 828, 1239, 382, 141, 2172, 0, 0, 0, 0, 0, 0, 0, 345, 437, 575, 678, 775, 2241, 338, 632, 218, 88, 67, 0, 0, 0, 0, 0, 0, 385, 1580, 712, 171, 636, 1615, 878, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5910, 6193, 6352, 6072, 6131, 6123, 6394, 5977] avg=6144.0000 max=6394 max_rank=6 min=5910 min_rank=0 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24625 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1726, 556, 358, 963, 128, 1261, 898, 237, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 497, 805, 241, 13, 253, 750, 848, 0, 0, 0, 0, 0, 0, 0, 0, 556, 777, 730, 769, 136, 419, 201, 1055, 1234, 315, 0, 0, 0, 0, 0, 0, 372, 515, 664, 464, 901, 200, 656, 907, 742, 552, 0, 0, 0, 0, 0, 0, 763, 726, 676, 1445, 822, 228, 888, 738, 0, 0, 0, 0, 0, 0, 0, 0, 229, 579, 520, 186, 960, 1168, 370, 127, 2075, 0, 0, 0, 0, 0, 0, 0, 318, 408, 567, 716, 764, 2168, 274, 631, 205, 92, 71, 0, 0, 0, 0, 0, 0, 411, 1677, 763, 176, 672, 1590, 871, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 5986, 6192, 5973, 6286, 6214, 6214, 6160] avg=6144.0000 max=6286 max_rank=4 min=5973 min_rank=3 imbalance_ratio=1.023112 max_load_reduction=42866 max_load_reduction_pct=0.872111 moved_to_echo=24874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1639, 518, 457, 921, 88, 1264, 1077, 173, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 591, 1086, 227, 11, 262, 719, 732, 0, 0, 0, 0, 0, 0, 0, 0, 602, 816, 825, 774, 134, 392, 144, 1052, 1219, 328, 0, 0, 0, 0, 0, 0, 402, 575, 723, 449, 840, 411, 730, 1003, 731, 563, 0, 0, 0, 0, 0, 0, 825, 649, 688, 1415, 671, 214, 923, 763, 0, 0, 0, 0, 0, 0, 0, 0, 251, 490, 488, 179, 851, 1234, 406, 133, 2155, 0, 0, 0, 0, 0, 0, 0, 338, 386, 492, 732, 724, 1943, 278, 667, 208, 87, 69, 0, 0, 0, 0, 0, 0, 348, 1631, 708, 174, 650, 1497, 884, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6151, 6286, 6427, 6148, 6187, 5924, 5892] avg=6144.0000 max=6427 max_rank=3 min=5892 min_rank=7 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1760, 565, 379, 910, 110, 1378, 1097, 171, 0, 0, 0, 0, 0, 0, 0, 0, 2517, 570, 1035, 255, 10, 267, 752, 763, 0, 0, 0, 0, 0, 0, 0, 0, 529, 744, 732, 701, 94, 384, 171, 958, 1272, 335, 0, 0, 0, 0, 0, 0, 320, 545, 602, 494, 850, 335, 615, 958, 764, 539, 0, 0, 0, 0, 0, 0, 788, 623, 658, 1418, 560, 197, 968, 755, 0, 0, 0, 0, 0, 0, 0, 0, 196, 589, 525, 130, 921, 1152, 399, 145, 2201, 0, 0, 0, 0, 0, 0, 0, 320, 378, 500, 867, 720, 2181, 347, 540, 200, 94, 71, 0, 0, 0, 0, 0, 0, 441, 1670, 768, 222, 708, 1583, 836, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6370, 6169, 5920, 6022, 5967, 6258, 6218, 6228] avg=6144.0000 max=6370 max_rank=0 min=5920 min_rank=2 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24671 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1726, 556, 358, 963, 128, 1261, 898, 237, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 497, 805, 241, 13, 253, 750, 848, 0, 0, 0, 0, 0, 0, 0, 0, 556, 777, 730, 769, 136, 419, 201, 1055, 1234, 315, 0, 0, 0, 0, 0, 0, 372, 515, 664, 464, 901, 200, 656, 907, 742, 552, 0, 0, 0, 0, 0, 0, 763, 726, 676, 1445, 822, 228, 888, 738, 0, 0, 0, 0, 0, 0, 0, 0, 229, 579, 520, 186, 960, 1168, 370, 127, 2075, 0, 0, 0, 0, 0, 0, 0, 318, 408, 567, 716, 764, 2168, 274, 631, 205, 92, 71, 0, 0, 0, 0, 0, 0, 411, 1677, 763, 176, 672, 1590, 871, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 5986, 6192, 5973, 6286, 6214, 6214, 6160] avg=6144.0000 max=6286 max_rank=4 min=5973 min_rank=3 imbalance_ratio=1.023112 max_load_reduction=42866 max_load_reduction_pct=0.872111 moved_to_echo=24874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1637, 594, 414, 933, 96, 1431, 1043, 146, 0, 0, 0, 0, 0, 0, 0, 0, 2662, 574, 773, 269, 5, 258, 712, 840, 0, 0, 0, 0, 0, 0, 0, 0, 556, 775, 755, 767, 106, 366, 172, 1076, 1224, 322, 0, 0, 0, 0, 0, 0, 346, 646, 634, 385, 862, 274, 695, 971, 734, 546, 0, 0, 0, 0, 0, 0, 802, 610, 667, 1448, 768, 208, 957, 726, 0, 0, 0, 0, 0, 0, 0, 0, 203, 502, 516, 181, 870, 1159, 333, 119, 2119, 0, 0, 0, 0, 0, 0, 0, 304, 399, 552, 817, 723, 2062, 337, 632, 202, 87, 76, 0, 0, 0, 0, 0, 0, 380, 1792, 720, 196, 728, 1559, 799, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6294, 6093, 6119, 6093, 6186, 6002, 6191, 6174] avg=6144.0000 max=6294 max_rank=0 min=6002 min_rank=5 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24553 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1593, 451, 336, 1008, 122, 1384, 1193, 147, 0, 0, 0, 0, 0, 0, 0, 0, 2529, 564, 1020, 338, 10, 255, 715, 736, 0, 0, 0, 0, 0, 0, 0, 0, 579, 772, 836, 646, 115, 372, 159, 1025, 1176, 319, 0, 0, 0, 0, 0, 0, 341, 558, 684, 484, 894, 365, 773, 1046, 705, 569, 0, 0, 0, 0, 0, 0, 775, 647, 695, 1316, 743, 231, 901, 784, 0, 0, 0, 0, 0, 0, 0, 0, 247, 503, 477, 198, 938, 1137, 319, 159, 2098, 0, 0, 0, 0, 0, 0, 0, 325, 421, 480, 717, 724, 1981, 325, 589, 211, 85, 67, 0, 0, 0, 0, 0, 0, 446, 1579, 830, 145, 668, 1639, 933, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6234, 6167, 5999, 6419, 6092, 6076, 5925, 6240] avg=6144.0000 max=6419 max_rank=3 min=5925 min_rank=6 imbalance_ratio=1.044759 max_load_reduction=42733 max_load_reduction_pct=0.869405 moved_to_echo=24333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1655, 569, 425, 947, 81, 1253, 835, 145, 0, 0, 0, 0, 0, 0, 0, 0, 2600, 592, 919, 261, 5, 265, 694, 857, 0, 0, 0, 0, 0, 0, 0, 0, 570, 820, 799, 683, 153, 374, 154, 1141, 1328, 330, 0, 0, 0, 0, 0, 0, 316, 590, 732, 422, 846, 324, 633, 823, 798, 588, 0, 0, 0, 0, 0, 0, 753, 600, 718, 1338, 765, 193, 940, 824, 0, 0, 0, 0, 0, 0, 0, 0, 215, 478, 486, 182, 828, 1239, 382, 141, 2172, 0, 0, 0, 0, 0, 0, 0, 345, 437, 575, 678, 775, 2241, 338, 632, 218, 88, 67, 0, 0, 0, 0, 0, 0, 385, 1580, 712, 171, 636, 1615, 878, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5910, 6193, 6352, 6072, 6131, 6123, 6394, 5977] avg=6144.0000 max=6394 max_rank=6 min=5910 min_rank=0 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24625 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1645, 449, 446, 814, 118, 1298, 1166, 204, 0, 0, 0, 0, 0, 0, 0, 0, 2409, 628, 1027, 280, 8, 256, 753, 715, 0, 0, 0, 0, 0, 0, 0, 0, 546, 797, 778, 645, 136, 416, 168, 990, 1194, 332, 0, 0, 0, 0, 0, 0, 309, 569, 716, 465, 848, 269, 676, 1032, 715, 497, 0, 0, 0, 0, 0, 0, 856, 672, 598, 1453, 666, 256, 965, 709, 0, 0, 0, 0, 0, 0, 0, 0, 262, 535, 552, 165, 928, 1132, 337, 108, 2163, 0, 0, 0, 0, 0, 0, 0, 400, 410, 525, 627, 748, 2057, 320, 628, 187, 93, 71, 0, 0, 0, 0, 0, 0, 379, 1614, 803, 264, 750, 1764, 841, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6076, 6002, 6096, 6175, 6182, 6066, 6415] avg=6144.0000 max=6415 max_rank=7 min=6002 min_rank=2 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=24838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1760, 565, 379, 910, 110, 1378, 1097, 171, 0, 0, 0, 0, 0, 0, 0, 0, 2517, 570, 1035, 255, 10, 267, 752, 763, 0, 0, 0, 0, 0, 0, 0, 0, 529, 744, 732, 701, 94, 384, 171, 958, 1272, 335, 0, 0, 0, 0, 0, 0, 320, 545, 602, 494, 850, 335, 615, 958, 764, 539, 0, 0, 0, 0, 0, 0, 788, 623, 658, 1418, 560, 197, 968, 755, 0, 0, 0, 0, 0, 0, 0, 0, 196, 589, 525, 130, 921, 1152, 399, 145, 2201, 0, 0, 0, 0, 0, 0, 0, 320, 378, 500, 867, 720, 2181, 347, 540, 200, 94, 71, 0, 0, 0, 0, 0, 0, 441, 1670, 768, 222, 708, 1583, 836, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6370, 6169, 5920, 6022, 5967, 6258, 6218, 6228] avg=6144.0000 max=6370 max_rank=0 min=5920 min_rank=2 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24671 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1647, 539, 293, 855, 108, 1388, 990, 120, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 512, 976, 258, 10, 276, 778, 849, 0, 0, 0, 0, 0, 0, 0, 0, 543, 779, 832, 723, 83, 381, 112, 1109, 1374, 346, 0, 0, 0, 0, 0, 0, 305, 573, 622, 431, 830, 248, 722, 920, 824, 575, 0, 0, 0, 0, 0, 0, 829, 636, 704, 1428, 629, 212, 932, 797, 0, 0, 0, 0, 0, 0, 0, 0, 193, 502, 430, 165, 878, 1187, 378, 104, 2273, 0, 0, 0, 0, 0, 0, 0, 310, 378, 506, 681, 783, 2254, 353, 583, 213, 88, 71, 0, 0, 0, 0, 0, 0, 351, 1690, 720, 155, 665, 1580, 905, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5940, 6317, 6282, 6050, 6167, 6110, 6220, 6066] avg=6144.0000 max=6317 max_rank=1 min=5940 min_rank=0 imbalance_ratio=1.028158 max_load_reduction=42835 max_load_reduction_pct=0.871480 moved_to_echo=24563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1637, 594, 414, 933, 96, 1431, 1043, 146, 0, 0, 0, 0, 0, 0, 0, 0, 2662, 574, 773, 269, 5, 258, 712, 840, 0, 0, 0, 0, 0, 0, 0, 0, 556, 775, 755, 767, 106, 366, 172, 1076, 1224, 322, 0, 0, 0, 0, 0, 0, 346, 646, 634, 385, 862, 274, 695, 971, 734, 546, 0, 0, 0, 0, 0, 0, 802, 610, 667, 1448, 768, 208, 957, 726, 0, 0, 0, 0, 0, 0, 0, 0, 203, 502, 516, 181, 870, 1159, 333, 119, 2119, 0, 0, 0, 0, 0, 0, 0, 304, 399, 552, 817, 723, 2062, 337, 632, 202, 87, 76, 0, 0, 0, 0, 0, 0, 380, 1792, 720, 196, 728, 1559, 799, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6294, 6093, 6119, 6093, 6186, 6002, 6191, 6174] avg=6144.0000 max=6294 max_rank=0 min=6002 min_rank=5 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24553 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 4224, 4096, 1408, 7296, 9472, 2944, 1152, 17280, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13312, 4352, 3200, 7424, 896, 10752, 8320, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[20480, 4608, 7680, 2176, 128, 2176, 5888, 6400, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6400, 5248, 5504, 11264, 5632, 1792, 7552, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13312, 4352, 3200, 7424, 896, 10752, 8320, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 4224, 4096, 1408, 7296, 9472, 2944, 1152, 17280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 3328, 4224, 5888, 6016, 16896, 2688, 4992, 1664, 768, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 4608, 5504, 3712, 6912, 2432, 5504, 7680, 6016, 4480, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 3200, 13312, 6144, 1536, 5504, 12928, 7040, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[4608, 6400, 6400, 5760, 1024, 3200, 1408, 8448, 10112, 2688, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 4608, 5504, 3712, 6912, 2432, 5504, 7680, 6016, 4480, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 3328, 4224, 5888, 6016, 16896, 2688, 4992, 1664, 768, 640, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[217, 545, 47, 207, 3932, 49, 490, 653, 11, 530, 2701, 226, 552, 650, 171, 3545, 360, 1013, 437, 277, 884, 470, 1534, 890, 893, 502, 697, 3178, 0, 45, 31, 184, 79, 3, 268, 1, 738, 13, 429, 135, 952, 571, 268, 780, 16, 1089, 0, 108, 6505, 561, 3564, 0, 24, 441, 219, 1435, 2157, 333, 848, 277, 547, 88, 586, 196] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[215, 546, 32, 206, 4073, 27, 357, 684, 14, 609, 2838, 262, 496, 676, 187, 3370, 451, 1026, 467, 265, 1007, 391, 1319, 1117, 823, 484, 707, 2877, 0, 43, 41, 144, 129, 0, 252, 3, 709, 15, 510, 178, 989, 513, 255, 708, 15, 1114, 0, 98, 6339, 664, 3767, 0, 14, 578, 195, 1437, 2148, 358, 834, 272, 539, 101, 466, 198] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[232, 485, 45, 230, 3701, 90, 419, 755, 14, 549, 2725, 284, 502, 632, 182, 3294, 387, 975, 528, 312, 893, 404, 1473, 1050, 839, 477, 720, 3036, 0, 60, 52, 154, 86, 7, 249, 6, 720, 6, 380, 148, 1009, 595, 248, 683, 16, 1111, 0, 118, 6337, 610, 4014, 0, 12, 497, 209, 1668, 2074, 376, 843, 226, 577, 94, 491, 243] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[335, 525, 41, 190, 4101, 50, 337, 796, 17, 504, 2811, 319, 515, 444, 132, 3240, 392, 1127, 553, 269, 949, 400, 1532, 933, 799, 524, 642, 3111, 4, 56, 36, 176, 75, 2, 264, 6, 742, 7, 477, 158, 1036, 554, 265, 702, 9, 833, 1, 141, 6368, 623, 3806, 0, 17, 470, 237, 1507, 1971, 385, 848, 372, 572, 79, 532, 233] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[311, 557, 58, 250, 3757, 70, 347, 800, 14, 620, 2647, 292, 577, 656, 204, 3196, 469, 1057, 553, 311, 931, 450, 1526, 945, 809, 485, 755, 2893, 2, 71, 90, 161, 119, 6, 256, 14, 819, 19, 402, 228, 1229, 610, 259, 688, 19, 1145, 3, 114, 5843, 664, 3096, 1, 16, 662, 275, 1679, 2072, 378, 902, 352, 598, 94, 513, 213] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[232, 485, 45, 230, 3701, 90, 419, 755, 14, 549, 2725, 284, 502, 632, 182, 3294, 387, 975, 528, 312, 893, 404, 1473, 1050, 839, 477, 720, 3036, 0, 60, 52, 154, 86, 7, 249, 6, 720, 6, 380, 148, 1009, 595, 248, 683, 16, 1111, 0, 118, 6337, 610, 4014, 0, 12, 497, 209, 1668, 2074, 376, 843, 226, 577, 94, 491, 243] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[252, 478, 47, 222, 3748, 105, 410, 836, 6, 641, 2543, 331, 576, 591, 163, 3220, 352, 1041, 501, 291, 1083, 368, 1476, 1012, 728, 499, 752, 2937, 3, 67, 79, 165, 49, 2, 282, 3, 751, 10, 427, 212, 1123, 671, 279, 812, 17, 840, 1, 107, 6138, 669, 3615, 0, 17, 462, 227, 1770, 2074, 355, 909, 385, 565, 126, 490, 241] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[313, 514, 49, 249, 3892, 53, 428, 829, 14, 654, 2692, 301, 604, 563, 215, 3009, 400, 1057, 530, 276, 1033, 427, 1471, 952, 887, 465, 645, 2765, 0, 60, 53, 163, 118, 9, 267, 9, 668, 12, 517, 131, 1056, 537, 257, 690, 19, 1108, 0, 127, 6191, 708, 3752, 0, 15, 540, 261, 1612, 2059, 356, 922, 316, 521, 85, 503, 223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[215, 546, 32, 206, 4073, 27, 357, 684, 14, 609, 2838, 262, 496, 676, 187, 3370, 451, 1026, 467, 265, 1007, 391, 1319, 1117, 823, 484, 707, 2877, 0, 43, 41, 144, 129, 0, 252, 3, 709, 15, 510, 178, 989, 513, 255, 708, 15, 1114, 0, 98, 6339, 664, 3767, 0, 14, 578, 195, 1437, 2148, 358, 834, 272, 539, 101, 466, 198] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[237, 453, 28, 195, 4158, 62, 337, 717, 6, 530, 2705, 254, 524, 529, 157, 3486, 398, 999, 536, 287, 948, 407, 1387, 950, 870, 487, 667, 3223, 0, 33, 30, 129, 82, 3, 245, 1, 757, 1, 406, 126, 984, 548, 246, 676, 10, 963, 0, 119, 6592, 589, 4034, 0, 6, 488, 201, 1445, 2107, 366, 803, 283, 503, 80, 560, 199] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[217, 545, 47, 207, 3905, 49, 490, 653, 0, 0, 0, 0, 0, 0, 0, 0, 11, 530, 2701, 226, 552, 650, 171, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 360, 1013, 437, 277, 884, 470, 1534, 890, 50, 27, 0, 0, 0, 0, 0, 0, 893, 502, 697, 3178, 0, 45, 31, 184, 456, 409, 0, 0, 0, 0, 0, 0, 79, 3, 268, 1, 738, 13, 429, 135, 4577, 0, 0, 0, 0, 0, 0, 0, 952, 571, 268, 780, 16, 1089, 0, 108, 1928, 563, 0, 0, 0, 0, 0, 0, 0, 561, 3058, 0, 24, 441, 219, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 2157, 333, 848, 277, 547, 88, 586, 196, 1228, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6186, 5942, 6395, 6243, 6275, 5738, 6260] avg=6144.0000 max=6395 max_rank=3 min=5738 min_rank=6 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[313, 514, 49, 249, 3892, 53, 428, 829, 14, 654, 2692, 301, 604, 563, 215, 3009, 400, 1057, 530, 276, 1033, 427, 1471, 952, 887, 465, 645, 2765, 0, 60, 53, 163, 118, 9, 267, 9, 668, 12, 517, 131, 1056, 537, 257, 690, 19, 1108, 0, 127, 6191, 708, 3752, 0, 15, 540, 261, 1612, 2059, 356, 922, 316, 521, 85, 503, 223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[217, 545, 47, 207, 3932, 49, 490, 653, 11, 530, 2701, 226, 552, 650, 171, 3545, 360, 1013, 437, 277, 884, 470, 1534, 890, 893, 502, 697, 3178, 0, 45, 31, 184, 79, 3, 268, 1, 738, 13, 429, 135, 952, 571, 268, 780, 16, 1089, 0, 108, 6505, 561, 3564, 0, 24, 441, 219, 1435, 2157, 333, 848, 277, 547, 88, 586, 196] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[311, 557, 58, 250, 3757, 70, 347, 800, 14, 620, 2647, 292, 577, 656, 204, 3196, 469, 1057, 553, 311, 931, 450, 1526, 945, 809, 485, 755, 2893, 2, 71, 90, 161, 119, 6, 256, 14, 819, 19, 402, 228, 1229, 610, 259, 688, 19, 1145, 3, 114, 5843, 664, 3096, 1, 16, 662, 275, 1679, 2072, 378, 902, 352, 598, 94, 513, 213] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[335, 525, 41, 190, 4072, 50, 337, 796, 0, 0, 0, 0, 0, 0, 0, 0, 17, 504, 2811, 319, 515, 444, 132, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 392, 1127, 553, 269, 949, 400, 1532, 933, 54, 29, 0, 0, 0, 0, 0, 0, 799, 524, 642, 3111, 4, 56, 36, 176, 487, 374, 0, 0, 0, 0, 0, 0, 75, 2, 264, 6, 742, 7, 477, 158, 4481, 0, 0, 0, 0, 0, 0, 0, 1036, 554, 265, 702, 9, 833, 1, 141, 1887, 515, 0, 0, 0, 0, 0, 0, 0, 623, 3265, 0, 17, 470, 237, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 385, 848, 372, 572, 79, 532, 233, 1122, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6346, 5971, 6238, 6209, 6212, 5943, 6119, 6114] avg=6144.0000 max=6346 max_rank=0 min=5943 min_rank=5 imbalance_ratio=1.032878 max_load_reduction=42806 max_load_reduction_pct=0.870890 moved_to_echo=24388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[252, 478, 47, 222, 3722, 105, 410, 836, 0, 0, 0, 0, 0, 0, 0, 0, 6, 641, 2543, 331, 576, 591, 163, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 352, 1041, 501, 291, 1083, 368, 1476, 1012, 51, 26, 0, 0, 0, 0, 0, 0, 728, 499, 752, 2937, 3, 67, 79, 165, 463, 371, 0, 0, 0, 0, 0, 0, 49, 2, 282, 3, 751, 10, 427, 212, 4319, 0, 0, 0, 0, 0, 0, 0, 1123, 671, 279, 812, 17, 840, 1, 107, 1819, 511, 0, 0, 0, 0, 0, 0, 0, 669, 3101, 0, 17, 462, 227, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 355, 909, 385, 565, 126, 490, 241, 1115, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 6074, 6201, 6064, 6055, 6180, 6246, 6260] avg=6144.0000 max=6260 max_rank=7 min=6055 min_rank=4 imbalance_ratio=1.018880 max_load_reduction=42892 max_load_reduction_pct=0.872640 moved_to_echo=24741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[215, 546, 32, 206, 4045, 27, 357, 684, 0, 0, 0, 0, 0, 0, 0, 0, 14, 609, 2838, 262, 496, 676, 187, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 451, 1026, 467, 265, 1007, 391, 1319, 1117, 53, 28, 0, 0, 0, 0, 0, 0, 823, 484, 707, 2877, 0, 43, 41, 144, 482, 389, 0, 0, 0, 0, 0, 0, 129, 0, 252, 3, 709, 15, 510, 178, 4461, 0, 0, 0, 0, 0, 0, 0, 989, 513, 255, 708, 15, 1114, 0, 98, 1878, 535, 0, 0, 0, 0, 0, 0, 0, 664, 3232, 0, 14, 578, 195, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 2148, 358, 834, 272, 539, 101, 466, 198, 1167, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6361, 6124, 5990, 6257, 6105, 6120, 6083] avg=6144.0000 max=6361 max_rank=1 min=5990 min_rank=3 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24565 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[313, 514, 49, 249, 3865, 53, 428, 829, 0, 0, 0, 0, 0, 0, 0, 0, 14, 654, 2692, 301, 604, 563, 215, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 400, 1057, 530, 276, 1033, 427, 1471, 952, 53, 27, 0, 0, 0, 0, 0, 0, 887, 465, 645, 2765, 0, 60, 53, 163, 480, 347, 0, 0, 0, 0, 0, 0, 118, 9, 267, 9, 668, 12, 517, 131, 4357, 0, 0, 0, 0, 0, 0, 0, 1056, 537, 257, 690, 19, 1108, 0, 127, 1834, 478, 0, 0, 0, 0, 0, 0, 0, 708, 3219, 0, 15, 540, 261, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 2059, 356, 922, 316, 521, 85, 503, 223, 1042, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6300, 6185, 6226, 5865, 6088, 6106, 6355, 6027] avg=6144.0000 max=6355 max_rank=6 min=5865 min_rank=3 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[232, 485, 45, 230, 3675, 90, 419, 755, 0, 0, 0, 0, 0, 0, 0, 0, 14, 549, 2725, 284, 502, 632, 182, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 387, 975, 528, 312, 893, 404, 1473, 1050, 56, 26, 0, 0, 0, 0, 0, 0, 839, 477, 720, 3036, 0, 60, 52, 154, 514, 380, 0, 0, 0, 0, 0, 0, 86, 7, 249, 6, 720, 6, 380, 148, 4460, 0, 0, 0, 0, 0, 0, 0, 1009, 595, 248, 683, 16, 1111, 0, 118, 1877, 523, 0, 0, 0, 0, 0, 0, 0, 610, 3444, 0, 12, 497, 209, 1668, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 376, 843, 226, 577, 94, 491, 243, 1141, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5931, 6138, 6104, 6232, 6062, 6180, 6440, 6065] avg=6144.0000 max=6440 max_rank=6 min=5931 min_rank=0 imbalance_ratio=1.048177 max_load_reduction=42712 max_load_reduction_pct=0.868978 moved_to_echo=24747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[311, 557, 58, 250, 3727, 70, 347, 800, 0, 0, 0, 0, 0, 0, 0, 0, 14, 620, 2647, 292, 577, 656, 204, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 469, 1057, 553, 311, 931, 450, 1526, 945, 47, 30, 0, 0, 0, 0, 0, 0, 809, 485, 755, 2893, 2, 71, 90, 161, 401, 372, 0, 0, 0, 0, 0, 0, 119, 6, 256, 14, 819, 19, 402, 228, 4112, 0, 0, 0, 0, 0, 0, 0, 1229, 610, 259, 688, 19, 1145, 3, 114, 1731, 512, 0, 0, 0, 0, 0, 0, 0, 664, 2648, 1, 16, 662, 275, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 2072, 378, 902, 352, 598, 94, 513, 213, 1112, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6120, 6210, 6319, 6039, 5975, 6310, 5945, 6234] avg=6144.0000 max=6319 max_rank=2 min=5945 min_rank=6 imbalance_ratio=1.028483 max_load_reduction=42833 max_load_reduction_pct=0.871440 moved_to_echo=24464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[252, 478, 47, 222, 3722, 105, 410, 836, 0, 0, 0, 0, 0, 0, 0, 0, 6, 641, 2543, 331, 576, 591, 163, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 352, 1041, 501, 291, 1083, 368, 1476, 1012, 51, 26, 0, 0, 0, 0, 0, 0, 728, 499, 752, 2937, 3, 67, 79, 165, 463, 371, 0, 0, 0, 0, 0, 0, 49, 2, 282, 3, 751, 10, 427, 212, 4319, 0, 0, 0, 0, 0, 0, 0, 1123, 671, 279, 812, 17, 840, 1, 107, 1819, 511, 0, 0, 0, 0, 0, 0, 0, 669, 3101, 0, 17, 462, 227, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 355, 909, 385, 565, 126, 490, 241, 1115, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 6074, 6201, 6064, 6055, 6180, 6246, 6260] avg=6144.0000 max=6260 max_rank=7 min=6055 min_rank=4 imbalance_ratio=1.018880 max_load_reduction=42892 max_load_reduction_pct=0.872640 moved_to_echo=24741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[335, 525, 41, 190, 4072, 50, 337, 796, 0, 0, 0, 0, 0, 0, 0, 0, 17, 504, 2811, 319, 515, 444, 132, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 392, 1127, 553, 269, 949, 400, 1532, 933, 54, 29, 0, 0, 0, 0, 0, 0, 799, 524, 642, 3111, 4, 56, 36, 176, 487, 374, 0, 0, 0, 0, 0, 0, 75, 2, 264, 6, 742, 7, 477, 158, 4481, 0, 0, 0, 0, 0, 0, 0, 1036, 554, 265, 702, 9, 833, 1, 141, 1887, 515, 0, 0, 0, 0, 0, 0, 0, 623, 3265, 0, 17, 470, 237, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 385, 848, 372, 572, 79, 532, 233, 1122, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6346, 5971, 6238, 6209, 6212, 5943, 6119, 6114] avg=6144.0000 max=6346 max_rank=0 min=5943 min_rank=5 imbalance_ratio=1.032878 max_load_reduction=42806 max_load_reduction_pct=0.870890 moved_to_echo=24388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[217, 545, 47, 207, 3905, 49, 490, 653, 0, 0, 0, 0, 0, 0, 0, 0, 11, 530, 2701, 226, 552, 650, 171, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 360, 1013, 437, 277, 884, 470, 1534, 890, 50, 27, 0, 0, 0, 0, 0, 0, 893, 502, 697, 3178, 0, 45, 31, 184, 456, 409, 0, 0, 0, 0, 0, 0, 79, 3, 268, 1, 738, 13, 429, 135, 4577, 0, 0, 0, 0, 0, 0, 0, 952, 571, 268, 780, 16, 1089, 0, 108, 1928, 563, 0, 0, 0, 0, 0, 0, 0, 561, 3058, 0, 24, 441, 219, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 2157, 333, 848, 277, 547, 88, 586, 196, 1228, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6186, 5942, 6395, 6243, 6275, 5738, 6260] avg=6144.0000 max=6395 max_rank=3 min=5738 min_rank=6 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[313, 514, 49, 249, 3865, 53, 428, 829, 0, 0, 0, 0, 0, 0, 0, 0, 14, 654, 2692, 301, 604, 563, 215, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 400, 1057, 530, 276, 1033, 427, 1471, 952, 53, 27, 0, 0, 0, 0, 0, 0, 887, 465, 645, 2765, 0, 60, 53, 163, 480, 347, 0, 0, 0, 0, 0, 0, 118, 9, 267, 9, 668, 12, 517, 131, 4357, 0, 0, 0, 0, 0, 0, 0, 1056, 537, 257, 690, 19, 1108, 0, 127, 1834, 478, 0, 0, 0, 0, 0, 0, 0, 708, 3219, 0, 15, 540, 261, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 2059, 356, 922, 316, 521, 85, 503, 223, 1042, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6300, 6185, 6226, 5865, 6088, 6106, 6355, 6027] avg=6144.0000 max=6355 max_rank=6 min=5865 min_rank=3 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[237, 453, 28, 195, 4129, 62, 337, 717, 0, 0, 0, 0, 0, 0, 0, 0, 6, 530, 2705, 254, 524, 529, 157, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 398, 999, 536, 287, 948, 407, 1387, 950, 57, 29, 0, 0, 0, 0, 0, 0, 870, 487, 667, 3223, 0, 33, 30, 129, 517, 402, 0, 0, 0, 0, 0, 0, 82, 3, 245, 1, 757, 1, 406, 126, 4639, 0, 0, 0, 0, 0, 0, 0, 984, 548, 246, 676, 10, 963, 0, 119, 1953, 554, 0, 0, 0, 0, 0, 0, 0, 589, 3460, 0, 6, 488, 201, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 366, 803, 283, 503, 80, 560, 199, 1208, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6158, 6027, 5998, 6358, 6260, 6053, 6189, 6109] avg=6144.0000 max=6358 max_rank=3 min=5998 min_rank=2 imbalance_ratio=1.034831 max_load_reduction=42794 max_load_reduction_pct=0.870646 moved_to_echo=24611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[311, 557, 58, 250, 3727, 70, 347, 800, 0, 0, 0, 0, 0, 0, 0, 0, 14, 620, 2647, 292, 577, 656, 204, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 469, 1057, 553, 311, 931, 450, 1526, 945, 47, 30, 0, 0, 0, 0, 0, 0, 809, 485, 755, 2893, 2, 71, 90, 161, 401, 372, 0, 0, 0, 0, 0, 0, 119, 6, 256, 14, 819, 19, 402, 228, 4112, 0, 0, 0, 0, 0, 0, 0, 1229, 610, 259, 688, 19, 1145, 3, 114, 1731, 512, 0, 0, 0, 0, 0, 0, 0, 664, 2648, 1, 16, 662, 275, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 2072, 378, 902, 352, 598, 94, 513, 213, 1112, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6120, 6210, 6319, 6039, 5975, 6310, 5945, 6234] avg=6144.0000 max=6319 max_rank=2 min=5945 min_rank=6 imbalance_ratio=1.028483 max_load_reduction=42833 max_load_reduction_pct=0.871440 moved_to_echo=24464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8448, 4608, 2176, 5760, 128, 8320, 128, 1024, 14976, 4224, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16768, 2944, 6912, 2560, 4480, 768, 4224, 1792, 9216, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 4736, 21760, 2304, 4352, 4864, 1536, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 5120, 25472, 128, 128, 4224, 1920, 12672, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6656, 3968, 5632, 24064, 128, 512, 512, 1280, 3840, 3072, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 4224, 384, 1792, 31232, 512, 3200, 6144, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 8320, 4224, 2304, 7808, 3328, 11776, 7936, 512, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 5120, 25472, 128, 128, 4224, 1920, 12672, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 128, 2176, 128, 6016, 128, 3584, 1408, 35456, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 4736, 21760, 2304, 4352, 4864, 1536, 10112, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 699, 196, 160, 0, 179, 119, 298, 48, 535, 636, 2088, 647, 340, 1, 0, 2370, 1649, 1964, 70, 1, 1, 3869, 673, 710, 772, 3669, 4221, 4, 6, 269, 0, 139, 965, 553, 79, 5633, 996, 599, 91, 651, 2, 9, 23, 1119, 1231, 86, 83, 242, 2, 176, 550, 0, 0, 95, 8, 3875, 303, 5, 8, 0, 2920, 0, 2514] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 683, 208, 114, 0, 148, 131, 319, 47, 594, 658, 2133, 542, 352, 0, 0, 2304, 1685, 1851, 51, 0, 0, 3987, 670, 569, 723, 3842, 4351, 1, 0, 292, 0, 121, 960, 542, 86, 5968, 820, 508, 74, 628, 0, 2, 17, 846, 1259, 89, 81, 196, 0, 193, 464, 0, 0, 95, 2, 3835, 225, 0, 6, 0, 2815, 0, 3065] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 693, 181, 161, 0, 124, 163, 248, 55, 581, 898, 1986, 559, 317, 0, 0, 2651, 1652, 2045, 51, 0, 0, 3758, 832, 576, 781, 3689, 4512, 0, 8, 302, 0, 116, 993, 371, 72, 6101, 932, 524, 83, 503, 0, 1, 16, 704, 1506, 63, 63, 94, 1, 359, 458, 0, 0, 52, 3, 3549, 240, 0, 15, 0, 2934, 0, 2576] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 567, 207, 138, 0, 164, 121, 380, 42, 665, 731, 2060, 628, 325, 0, 0, 2611, 1534, 1952, 55, 3, 0, 3744, 721, 657, 613, 3667, 4272, 3, 1, 290, 0, 147, 881, 645, 120, 5748, 1149, 598, 114, 529, 0, 4, 13, 1210, 1138, 72, 63, 69, 0, 490, 513, 0, 0, 64, 6, 3528, 319, 0, 3, 0, 2948, 0, 2630] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 616, 204, 159, 0, 214, 168, 322, 45, 623, 664, 2127, 592, 347, 2, 2, 2235, 1434, 1866, 71, 2, 2, 3903, 623, 547, 727, 3680, 4429, 2, 13, 280, 0, 126, 958, 529, 96, 5804, 906, 608, 95, 644, 4, 4, 27, 1256, 1614, 97, 89, 84, 3, 208, 511, 1, 0, 118, 8, 3545, 284, 2, 12, 5, 3132, 0, 2483] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 683, 206, 146, 0, 162, 130, 336, 62, 558, 838, 2095, 641, 330, 0, 0, 2454, 1609, 1962, 75, 0, 0, 4037, 692, 692, 728, 3465, 4198, 1, 4, 253, 0, 116, 866, 402, 87, 5804, 902, 622, 89, 672, 0, 2, 18, 1062, 1288, 102, 97, 227, 0, 156, 496, 0, 0, 97, 5, 3780, 294, 0, 11, 0, 2947, 0, 2653] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 699, 196, 160, 0, 179, 119, 298, 48, 535, 636, 2088, 647, 340, 1, 0, 2370, 1649, 1964, 70, 1, 1, 3869, 673, 710, 772, 3669, 4221, 4, 6, 269, 0, 139, 965, 553, 79, 5633, 996, 599, 91, 651, 2, 9, 23, 1119, 1231, 86, 83, 242, 2, 176, 550, 0, 0, 95, 8, 3875, 303, 5, 8, 0, 2920, 0, 2514] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 616, 202, 144, 0, 163, 130, 254, 58, 545, 859, 2311, 551, 357, 0, 0, 2501, 1599, 2249, 55, 0, 0, 3829, 619, 570, 778, 3708, 4523, 0, 8, 254, 1, 187, 883, 321, 51, 6074, 962, 415, 82, 572, 0, 2, 18, 960, 1105, 85, 87, 162, 0, 156, 502, 0, 0, 91, 1, 3724, 242, 2, 3, 0, 2980, 0, 2601] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 616, 204, 159, 0, 214, 168, 322, 45, 623, 664, 2127, 592, 347, 2, 2, 2235, 1434, 1866, 71, 2, 2, 3903, 623, 547, 727, 3680, 4429, 2, 13, 280, 0, 126, 958, 529, 96, 5804, 906, 608, 95, 644, 4, 4, 27, 1256, 1614, 97, 89, 84, 3, 208, 511, 1, 0, 118, 8, 3545, 284, 2, 12, 5, 3132, 0, 2483] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 683, 208, 114, 0, 148, 131, 319, 47, 594, 658, 2133, 542, 352, 0, 0, 2304, 1685, 1851, 51, 0, 0, 3987, 670, 569, 723, 3842, 4351, 1, 0, 292, 0, 121, 960, 542, 86, 5968, 820, 508, 74, 628, 0, 2, 17, 846, 1259, 89, 81, 196, 0, 193, 464, 0, 0, 95, 2, 3835, 225, 0, 6, 0, 2815, 0, 3065] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 693, 181, 161, 0, 124, 163, 248, 55, 581, 898, 1986, 559, 317, 0, 0, 2651, 1652, 2045, 51, 0, 0, 3758, 832, 576, 781, 3689, 4512, 0, 8, 302, 0, 116, 993, 371, 72, 6101, 932, 524, 83, 503, 0, 1, 16, 704, 1506, 63, 63, 94, 1, 359, 458, 0, 0, 52, 3, 3549, 240, 0, 15, 0, 2934, 0, 2576] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 598, 204, 136, 0, 151, 115, 261, 26, 610, 807, 2229, 499, 392, 0, 0, 2511, 1563, 2131, 55, 2, 0, 3828, 669, 597, 753, 3655, 4546, 0, 1, 264, 0, 135, 932, 344, 54, 6183, 839, 324, 94, 587, 0, 5, 11, 782, 1399, 78, 68, 38, 0, 330, 464, 0, 0, 94, 4, 3619, 251, 1, 6, 0, 2894, 0, 3013] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 616, 204, 159, 0, 214, 168, 322, 2219, 2280, 0, 0, 0, 0, 0, 0, 45, 623, 664, 2127, 592, 347, 2, 2, 1019, 598, 0, 0, 0, 0, 0, 0, 1637, 1434, 1866, 71, 2, 2, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 547, 727, 3680, 844, 2, 13, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 958, 529, 96, 2921, 906, 608, 95, 0, 0, 0, 0, 0, 0, 0, 0, 644, 4, 4, 27, 1256, 1614, 97, 89, 1864, 1114, 0, 0, 0, 0, 0, 0, 84, 3, 208, 511, 1, 0, 118, 8, 3903, 1305, 0, 0, 0, 0, 0, 0, 212, 284, 2, 12, 5, 3132, 0, 2483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6182, 6019, 5635, 6093, 6239, 6713, 6141, 6130] avg=6144.0000 max=6713 max_rank=5 min=5635 min_rank=2 imbalance_ratio=1.092611 max_load_reduction=42439 max_load_reduction_pct=0.863424 moved_to_echo=25223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 699, 196, 160, 0, 179, 119, 298, 2422, 2169, 0, 0, 0, 0, 0, 0, 48, 535, 636, 2088, 647, 340, 1, 0, 987, 630, 0, 0, 0, 0, 0, 0, 1740, 1649, 1964, 70, 1, 1, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 710, 772, 3669, 811, 4, 6, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 965, 553, 79, 2840, 996, 599, 91, 0, 0, 0, 0, 0, 0, 0, 0, 651, 2, 9, 23, 1119, 1231, 86, 83, 1806, 1214, 0, 0, 0, 0, 0, 0, 242, 2, 176, 550, 0, 0, 95, 8, 3869, 1241, 0, 0, 0, 0, 0, 0, 239, 303, 5, 8, 0, 2920, 0, 2514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6243, 5912, 6098, 6241, 6262, 6224, 6183, 5989] avg=6144.0000 max=6262 max_rank=4 min=5912 min_rank=1 imbalance_ratio=1.019206 max_load_reduction=42890 max_load_reduction_pct=0.872599 moved_to_echo=24658 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 567, 207, 138, 0, 164, 121, 380, 2205, 2195, 0, 0, 0, 0, 0, 0, 42, 665, 731, 2060, 628, 325, 0, 0, 1007, 694, 0, 0, 0, 0, 0, 0, 1917, 1534, 1952, 55, 3, 0, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 657, 613, 3667, 821, 3, 1, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 881, 645, 120, 2898, 1149, 598, 114, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 4, 13, 1210, 1138, 72, 63, 1843, 1105, 0, 0, 0, 0, 0, 0, 69, 0, 490, 513, 0, 0, 64, 6, 3744, 1256, 0, 0, 0, 0, 0, 0, 218, 319, 0, 3, 0, 2948, 0, 2630, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6152, 6182, 6052, 6552, 5977, 6142, 6118] avg=6144.0000 max=6552 max_rank=4 min=5977 min_rank=0 imbalance_ratio=1.066406 max_load_reduction=42600 max_load_reduction_pct=0.866699 moved_to_echo=24789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 206, 146, 0, 162, 130, 336, 2363, 2157, 0, 0, 0, 0, 0, 0, 62, 558, 838, 2095, 641, 330, 0, 0, 1017, 652, 0, 0, 0, 0, 0, 0, 1802, 1609, 1962, 75, 0, 0, 0, 692, 0, 0, 0, 0, 0, 0, 0, 0, 692, 728, 3465, 807, 1, 4, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 866, 402, 87, 2926, 902, 622, 89, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 2, 18, 1062, 1288, 102, 97, 1861, 1184, 0, 0, 0, 0, 0, 0, 227, 0, 156, 496, 0, 0, 97, 5, 4037, 1234, 0, 0, 0, 0, 0, 0, 233, 294, 0, 11, 0, 2947, 0, 2653, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6193, 6140, 5950, 6010, 6286, 6252, 6138] avg=6144.0000 max=6286 max_rank=5 min=5950 min_rank=3 imbalance_ratio=1.023112 max_load_reduction=42866 max_load_reduction_pct=0.872111 moved_to_echo=24686 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 693, 181, 161, 0, 124, 163, 248, 2218, 2319, 0, 0, 0, 0, 0, 0, 55, 581, 898, 1986, 559, 317, 0, 0, 1069, 705, 0, 0, 0, 0, 0, 0, 1946, 1652, 2045, 51, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 576, 781, 3689, 866, 0, 8, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 993, 371, 72, 3076, 932, 524, 83, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 1, 16, 704, 1506, 63, 63, 1956, 1112, 0, 0, 0, 0, 0, 0, 94, 1, 359, 458, 0, 0, 52, 3, 3758, 1327, 0, 0, 0, 0, 0, 0, 219, 240, 0, 15, 0, 2934, 0, 2576, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6107, 6170, 6526, 6222, 6167, 5924, 6052, 5984] avg=6144.0000 max=6526 max_rank=2 min=5924 min_rank=5 imbalance_ratio=1.062174 max_load_reduction=42626 max_load_reduction_pct=0.867228 moved_to_echo=24127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 208, 114, 0, 148, 131, 319, 2397, 2236, 0, 0, 0, 0, 0, 0, 47, 594, 658, 2133, 542, 352, 0, 0, 1045, 612, 0, 0, 0, 0, 0, 0, 1692, 1685, 1851, 51, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 569, 723, 3842, 836, 1, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 960, 542, 86, 3009, 820, 508, 74, 0, 0, 0, 0, 0, 0, 0, 0, 628, 0, 2, 17, 846, 1259, 89, 81, 1914, 1201, 0, 0, 0, 0, 0, 0, 196, 0, 193, 464, 0, 0, 95, 2, 3987, 1279, 0, 0, 0, 0, 0, 0, 237, 225, 0, 6, 0, 2815, 0, 3065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6236, 5983, 5949, 6263, 6120, 6037, 6216, 6348] avg=6144.0000 max=6348 max_rank=7 min=5949 min_rank=2 imbalance_ratio=1.033203 max_load_reduction=42804 max_load_reduction_pct=0.870850 moved_to_echo=24721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 567, 207, 138, 0, 164, 121, 380, 2205, 2195, 0, 0, 0, 0, 0, 0, 42, 665, 731, 2060, 628, 325, 0, 0, 1007, 694, 0, 0, 0, 0, 0, 0, 1917, 1534, 1952, 55, 3, 0, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 657, 613, 3667, 821, 3, 1, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 881, 645, 120, 2898, 1149, 598, 114, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 4, 13, 1210, 1138, 72, 63, 1843, 1105, 0, 0, 0, 0, 0, 0, 69, 0, 490, 513, 0, 0, 64, 6, 3744, 1256, 0, 0, 0, 0, 0, 0, 218, 319, 0, 3, 0, 2948, 0, 2630, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6152, 6182, 6052, 6552, 5977, 6142, 6118] avg=6144.0000 max=6552 max_rank=4 min=5977 min_rank=0 imbalance_ratio=1.066406 max_load_reduction=42600 max_load_reduction_pct=0.866699 moved_to_echo=24789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 616, 202, 144, 0, 163, 130, 254, 2328, 2324, 0, 0, 0, 0, 0, 0, 58, 545, 859, 2311, 551, 357, 0, 0, 1064, 665, 0, 0, 0, 0, 0, 0, 1836, 1599, 2249, 55, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, 570, 778, 3708, 869, 0, 8, 254, 1, 0, 0, 0, 0, 0, 0, 0, 0, 187, 883, 321, 51, 3062, 962, 415, 82, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 2, 18, 960, 1105, 85, 87, 1948, 1167, 0, 0, 0, 0, 0, 0, 162, 0, 156, 502, 0, 0, 91, 1, 3829, 1330, 0, 0, 0, 0, 0, 0, 229, 242, 2, 3, 0, 2980, 0, 2601, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6161, 6410, 6358, 6188, 5963, 5944, 6071, 6057] avg=6144.0000 max=6410 max_rank=1 min=5944 min_rank=5 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 699, 196, 160, 0, 179, 119, 298, 2422, 2169, 0, 0, 0, 0, 0, 0, 48, 535, 636, 2088, 647, 340, 1, 0, 987, 630, 0, 0, 0, 0, 0, 0, 1740, 1649, 1964, 70, 1, 1, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 710, 772, 3669, 811, 4, 6, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 965, 553, 79, 2840, 996, 599, 91, 0, 0, 0, 0, 0, 0, 0, 0, 651, 2, 9, 23, 1119, 1231, 86, 83, 1806, 1214, 0, 0, 0, 0, 0, 0, 242, 2, 176, 550, 0, 0, 95, 8, 3869, 1241, 0, 0, 0, 0, 0, 0, 239, 303, 5, 8, 0, 2920, 0, 2514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6243, 5912, 6098, 6241, 6262, 6224, 6183, 5989] avg=6144.0000 max=6262 max_rank=4 min=5912 min_rank=1 imbalance_ratio=1.019206 max_load_reduction=42890 max_load_reduction_pct=0.872599 moved_to_echo=24658 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 208, 114, 0, 148, 131, 319, 2397, 2236, 0, 0, 0, 0, 0, 0, 47, 594, 658, 2133, 542, 352, 0, 0, 1045, 612, 0, 0, 0, 0, 0, 0, 1692, 1685, 1851, 51, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 569, 723, 3842, 836, 1, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 960, 542, 86, 3009, 820, 508, 74, 0, 0, 0, 0, 0, 0, 0, 0, 628, 0, 2, 17, 846, 1259, 89, 81, 1914, 1201, 0, 0, 0, 0, 0, 0, 196, 0, 193, 464, 0, 0, 95, 2, 3987, 1279, 0, 0, 0, 0, 0, 0, 237, 225, 0, 6, 0, 2815, 0, 3065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6236, 5983, 5949, 6263, 6120, 6037, 6216, 6348] avg=6144.0000 max=6348 max_rank=7 min=5949 min_rank=2 imbalance_ratio=1.033203 max_load_reduction=42804 max_load_reduction_pct=0.870850 moved_to_echo=24721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 598, 204, 136, 0, 151, 115, 261, 2262, 2336, 0, 0, 0, 0, 0, 0, 26, 610, 807, 2229, 499, 392, 0, 0, 1083, 667, 0, 0, 0, 0, 0, 0, 1844, 1563, 2131, 55, 2, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 597, 753, 3655, 873, 0, 1, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 932, 344, 54, 3117, 839, 324, 94, 0, 0, 0, 0, 0, 0, 0, 0, 587, 0, 5, 11, 782, 1399, 78, 68, 1983, 1134, 0, 0, 0, 0, 0, 0, 38, 0, 330, 464, 0, 0, 94, 4, 3828, 1337, 0, 0, 0, 0, 0, 0, 223, 251, 1, 6, 0, 2894, 0, 3013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6313, 6264, 6143, 5839, 6047, 6095, 6388] avg=6144.0000 max=6388 max_rank=7 min=5839 min_rank=4 imbalance_ratio=1.039714 max_load_reduction=42764 max_load_reduction_pct=0.870036 moved_to_echo=24369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 616, 204, 159, 0, 214, 168, 322, 2219, 2280, 0, 0, 0, 0, 0, 0, 45, 623, 664, 2127, 592, 347, 2, 2, 1019, 598, 0, 0, 0, 0, 0, 0, 1637, 1434, 1866, 71, 2, 2, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 547, 727, 3680, 844, 2, 13, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 958, 529, 96, 2921, 906, 608, 95, 0, 0, 0, 0, 0, 0, 0, 0, 644, 4, 4, 27, 1256, 1614, 97, 89, 1864, 1114, 0, 0, 0, 0, 0, 0, 84, 3, 208, 511, 1, 0, 118, 8, 3903, 1305, 0, 0, 0, 0, 0, 0, 212, 284, 2, 12, 5, 3132, 0, 2483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6182, 6019, 5635, 6093, 6239, 6713, 6141, 6130] avg=6144.0000 max=6713 max_rank=5 min=5635 min_rank=2 imbalance_ratio=1.092611 max_load_reduction=42439 max_load_reduction_pct=0.863424 moved_to_echo=25223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 693, 181, 161, 0, 124, 163, 248, 2218, 2319, 0, 0, 0, 0, 0, 0, 55, 581, 898, 1986, 559, 317, 0, 0, 1069, 705, 0, 0, 0, 0, 0, 0, 1946, 1652, 2045, 51, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 576, 781, 3689, 866, 0, 8, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 993, 371, 72, 3076, 932, 524, 83, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 1, 16, 704, 1506, 63, 63, 1956, 1112, 0, 0, 0, 0, 0, 0, 94, 1, 359, 458, 0, 0, 52, 3, 3758, 1327, 0, 0, 0, 0, 0, 0, 219, 240, 0, 15, 0, 2934, 0, 2576, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6107, 6170, 6526, 6222, 6167, 5924, 6052, 5984] avg=6144.0000 max=6526 max_rank=2 min=5924 min_rank=5 imbalance_ratio=1.062174 max_load_reduction=42626 max_load_reduction_pct=0.867228 moved_to_echo=24127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 4736, 6144, 17152, 4736, 2816, 128, 128, 8320, 5248, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 5248, 1664, 1280, 0, 1408, 1152, 2432, 18432, 18048, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[4864, 128, 128, 256, 8064, 10624, 768, 640, 15232, 9344, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[14464, 12800, 16128, 512, 128, 128, 0, 5504, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1152, 7552, 3712, 768, 23936, 7552, 4224, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 4736, 6144, 17152, 4736, 2816, 128, 128, 8320, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[4864, 128, 128, 256, 8064, 10624, 768, 640, 15232, 9344, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 128, 2176, 3968, 128, 0, 768, 128, 30976, 10368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1152, 7552, 3712, 768, 23936, 7552, 4224, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 2176, 128, 128, 128, 23680, 0, 21632, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 5888, 29440, 6784, 128, 128, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 493, 0, 1085, 489, 532, 310, 4289, 336, 554, 42, 346, 2, 6913, 25, 2994, 25, 633, 364, 508, 204, 0, 54, 804, 1130, 570, 437, 399, 2627, 421, 60, 0, 4, 70, 4, 340, 43, 8, 0, 12, 4546, 2, 463, 147, 1, 0, 4638, 0, 23, 0, 624, 0, 3312, 0, 0, 1, 0, 460, 1, 2625, 4670, 110, 402] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 453, 0, 1139, 738, 510, 373, 4457, 253, 469, 0, 288, 0, 6950, 29, 3046, 8, 586, 317, 516, 234, 0, 63, 751, 1330, 486, 419, 360, 2412, 447, 92, 0, 2, 91, 5, 273, 33, 9, 0, 16, 4454, 0, 415, 249, 0, 0, 4397, 0, 0, 0, 578, 0, 3249, 0, 0, 0, 0, 470, 0, 2542, 5165, 69, 409] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 476, 0, 935, 655, 556, 350, 4513, 302, 493, 31, 323, 4, 6812, 29, 3275, 13, 694, 326, 506, 268, 0, 91, 736, 1221, 499, 472, 379, 2683, 512, 93, 0, 8, 81, 6, 371, 58, 14, 0, 16, 4072, 2, 405, 231, 1, 1, 4229, 1, 34, 4, 581, 4, 3039, 0, 0, 3, 2, 542, 1, 2613, 5081, 87, 418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 473, 0, 1086, 572, 561, 331, 4534, 273, 458, 0, 291, 0, 7001, 25, 3401, 14, 626, 304, 434, 210, 0, 68, 678, 1192, 522, 478, 384, 2650, 452, 93, 0, 3, 69, 6, 271, 38, 9, 0, 11, 4280, 0, 447, 197, 0, 0, 4404, 0, 10, 0, 555, 0, 3167, 0, 0, 1, 0, 498, 0, 2617, 4991, 74, 393] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 506, 2, 997, 579, 562, 366, 4008, 324, 536, 43, 312, 4, 6819, 32, 3315, 22, 605, 329, 588, 279, 0, 64, 754, 1318, 510, 519, 418, 2606, 424, 112, 2, 4, 90, 8, 347, 67, 8, 6, 16, 4294, 2, 417, 223, 4, 0, 4229, 5, 24, 1, 606, 3, 3330, 0, 5, 8, 7, 545, 4, 2395, 5024, 92, 433] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 469, 0, 1153, 458, 538, 258, 4623, 233, 378, 0, 296, 1, 7122, 13, 3663, 29, 636, 299, 541, 210, 0, 45, 639, 1127, 539, 406, 368, 2565, 477, 47, 1, 1, 61, 4, 330, 44, 8, 0, 8, 4417, 2, 475, 161, 0, 2, 4638, 0, 10, 1, 506, 2, 3346, 0, 0, 7, 0, 415, 0, 2255, 4913, 67, 345] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 476, 0, 935, 655, 556, 350, 4513, 302, 493, 31, 323, 4, 6812, 29, 3275, 13, 694, 326, 506, 268, 0, 91, 736, 1221, 499, 472, 379, 2683, 512, 93, 0, 8, 81, 6, 371, 58, 14, 0, 16, 4072, 2, 405, 231, 1, 1, 4229, 1, 34, 4, 581, 4, 3039, 0, 0, 3, 2, 542, 1, 2613, 5081, 87, 418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 453, 0, 1139, 738, 510, 373, 4457, 253, 469, 0, 288, 0, 6950, 29, 3046, 8, 586, 317, 516, 234, 0, 63, 751, 1330, 486, 419, 360, 2412, 447, 92, 0, 2, 91, 5, 273, 33, 9, 0, 16, 4454, 0, 415, 249, 0, 0, 4397, 0, 0, 0, 578, 0, 3249, 0, 0, 0, 0, 470, 0, 2542, 5165, 69, 409] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 395, 0, 963, 598, 560, 387, 4524, 329, 403, 9, 304, 1, 6895, 28, 3290, 31, 603, 371, 456, 250, 0, 40, 739, 1170, 500, 525, 392, 2751, 468, 86, 0, 1, 77, 1, 397, 51, 2, 0, 9, 4062, 0, 375, 186, 1, 0, 3995, 0, 41, 0, 612, 1, 3264, 0, 0, 3, 1, 526, 0, 2673, 5203, 126, 477] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 473, 0, 1086, 572, 561, 331, 4534, 273, 458, 0, 291, 0, 7001, 25, 3401, 14, 626, 304, 434, 210, 0, 68, 678, 1192, 522, 478, 384, 2650, 452, 93, 0, 3, 69, 6, 271, 38, 9, 0, 11, 4280, 0, 447, 197, 0, 0, 4404, 0, 10, 0, 555, 0, 3167, 0, 0, 1, 0, 498, 0, 2617, 4991, 74, 393] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 493, 0, 1085, 489, 532, 310, 4289, 336, 554, 42, 346, 2, 6913, 25, 2994, 25, 633, 364, 508, 204, 0, 54, 804, 1130, 570, 437, 399, 2627, 421, 60, 0, 4, 70, 4, 340, 43, 8, 0, 12, 4546, 2, 463, 147, 1, 0, 4638, 0, 23, 0, 624, 0, 3312, 0, 0, 1, 0, 460, 1, 2625, 4670, 110, 402] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 351, 0, 1035, 497, 548, 346, 4719, 259, 351, 0, 277, 0, 6947, 10, 3420, 9, 635, 304, 558, 187, 0, 30, 691, 1097, 525, 538, 353, 2891, 424, 89, 0, 2, 64, 1, 266, 30, 5, 0, 18, 4429, 0, 439, 178, 0, 0, 4040, 0, 1, 0, 618, 1, 3267, 0, 0, 2, 0, 497, 0, 2319, 5399, 86, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 506, 2, 997, 579, 562, 366, 4008, 324, 536, 43, 312, 4, 6819, 32, 3315, 22, 605, 329, 588, 279, 0, 64, 754, 1318, 510, 519, 418, 2606, 424, 112, 2, 4, 90, 8, 347, 67, 8, 6, 16, 4294, 2, 417, 223, 4, 0, 4229, 5, 24, 1, 606, 3, 3330, 0, 5, 8, 7, 545, 4, 2395, 5024, 92, 433] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 493, 0, 1085, 489, 532, 310, 2714, 652, 0, 0, 0, 0, 0, 0, 4289, 336, 554, 42, 346, 2, 595, 25, 0, 0, 0, 0, 0, 0, 0, 0, 2994, 25, 633, 364, 508, 204, 0, 54, 676, 335, 0, 0, 0, 0, 0, 0, 804, 1130, 570, 437, 399, 2292, 421, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 70, 4, 340, 43, 8, 0, 5666, 0, 0, 0, 0, 0, 0, 0, 12, 4546, 2, 463, 147, 1, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 624, 0, 3312, 0, 0, 1553, 653, 0, 0, 0, 0, 0, 0, 1, 0, 460, 1, 2625, 2441, 110, 402, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6275, 6189, 5793, 6113, 6135, 6442, 6165, 6040] avg=6144.0000 max=6442 max_rank=5 min=5793 min_rank=2 imbalance_ratio=1.048503 max_load_reduction=42710 max_load_reduction_pct=0.868937 moved_to_echo=24782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 469, 0, 1153, 458, 538, 258, 2714, 671, 0, 0, 0, 0, 0, 0, 4623, 233, 378, 0, 296, 1, 614, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3663, 29, 636, 299, 541, 210, 0, 45, 711, 327, 0, 0, 0, 0, 0, 0, 639, 1127, 539, 406, 368, 2238, 477, 47, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 61, 4, 330, 44, 8, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 8, 4417, 2, 475, 161, 0, 2, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 506, 2, 3346, 0, 0, 1634, 653, 0, 0, 0, 0, 0, 0, 7, 0, 415, 0, 2255, 2568, 67, 345, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 6158, 6461, 5841, 6286, 6336, 6152, 5657] avg=6144.0000 max=6461 max_rank=2 min=5657 min_rank=7 imbalance_ratio=1.051595 max_load_reduction=42691 max_load_reduction_pct=0.868551 moved_to_echo=24431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 453, 0, 1139, 738, 510, 373, 2573, 655, 0, 0, 0, 0, 0, 0, 4457, 253, 469, 0, 288, 0, 599, 29, 0, 0, 0, 0, 0, 0, 0, 0, 3046, 8, 586, 317, 516, 234, 0, 63, 747, 307, 0, 0, 0, 0, 0, 0, 751, 1330, 486, 419, 360, 2105, 447, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 91, 5, 273, 33, 9, 0, 5696, 0, 0, 0, 0, 0, 0, 0, 16, 4454, 0, 415, 249, 0, 0, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 0, 3249, 0, 0, 1718, 619, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 2542, 2700, 69, 409, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6441, 6095, 5824, 5990, 6109, 6339, 6164, 6190] avg=6144.0000 max=6441 max_rank=0 min=5824 min_rank=2 imbalance_ratio=1.048340 max_load_reduction=42711 max_load_reduction_pct=0.868958 moved_to_echo=24802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 476, 0, 935, 655, 556, 350, 2475, 642, 0, 0, 0, 0, 0, 0, 4513, 302, 493, 31, 323, 4, 587, 29, 0, 0, 0, 0, 0, 0, 0, 0, 3275, 13, 694, 326, 506, 268, 0, 91, 735, 342, 0, 0, 0, 0, 0, 0, 736, 1221, 499, 472, 379, 2341, 512, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 81, 6, 371, 58, 14, 0, 5583, 0, 0, 0, 0, 0, 0, 0, 16, 4072, 2, 405, 231, 1, 1, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 4, 581, 4, 3039, 0, 0, 1690, 595, 0, 0, 0, 0, 0, 0, 3, 2, 542, 1, 2613, 2656, 87, 418, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6089, 6282, 6250, 6253, 6121, 5887, 5948, 6322] avg=6144.0000 max=6322 max_rank=7 min=5887 min_rank=5 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 493, 0, 1085, 489, 532, 310, 2714, 652, 0, 0, 0, 0, 0, 0, 4289, 336, 554, 42, 346, 2, 595, 25, 0, 0, 0, 0, 0, 0, 0, 0, 2994, 25, 633, 364, 508, 204, 0, 54, 676, 335, 0, 0, 0, 0, 0, 0, 804, 1130, 570, 437, 399, 2292, 421, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 70, 4, 340, 43, 8, 0, 5666, 0, 0, 0, 0, 0, 0, 0, 12, 4546, 2, 463, 147, 1, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 624, 0, 3312, 0, 0, 1553, 653, 0, 0, 0, 0, 0, 0, 1, 0, 460, 1, 2625, 2441, 110, 402, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6275, 6189, 5793, 6113, 6135, 6442, 6165, 6040] avg=6144.0000 max=6442 max_rank=5 min=5793 min_rank=2 imbalance_ratio=1.048503 max_load_reduction=42710 max_load_reduction_pct=0.868937 moved_to_echo=24782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 473, 0, 1086, 572, 561, 331, 2577, 660, 0, 0, 0, 0, 0, 0, 4534, 273, 458, 0, 291, 0, 603, 25, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 14, 626, 304, 434, 210, 0, 68, 722, 338, 0, 0, 0, 0, 0, 0, 678, 1192, 522, 478, 384, 2312, 452, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 69, 6, 271, 38, 9, 0, 5738, 0, 0, 0, 0, 0, 0, 0, 11, 4280, 0, 447, 197, 0, 0, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 555, 0, 3167, 0, 0, 1660, 620, 0, 0, 0, 0, 0, 0, 1, 0, 498, 0, 2617, 2609, 74, 393, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6260, 6184, 6117, 6111, 6134, 6142, 6012, 6192] avg=6144.0000 max=6260 max_rank=0 min=6012 min_rank=6 imbalance_ratio=1.018880 max_load_reduction=42892 max_load_reduction_pct=0.872640 moved_to_echo=24480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 395, 0, 963, 598, 560, 387, 2338, 650, 0, 0, 0, 0, 0, 0, 4524, 329, 403, 9, 304, 1, 594, 28, 0, 0, 0, 0, 0, 0, 0, 0, 3290, 31, 603, 371, 456, 250, 0, 40, 753, 351, 0, 0, 0, 0, 0, 0, 739, 1170, 500, 525, 392, 2400, 468, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 77, 1, 397, 51, 2, 0, 5651, 0, 0, 0, 0, 0, 0, 0, 9, 4062, 0, 375, 186, 1, 0, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 612, 1, 3264, 0, 0, 1731, 562, 0, 0, 0, 0, 0, 0, 3, 1, 526, 0, 2673, 2719, 126, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5891, 6192, 6145, 6280, 6180, 5728, 6211, 6525] avg=6144.0000 max=6525 max_rank=7 min=5728 min_rank=5 imbalance_ratio=1.062012 max_load_reduction=42627 max_load_reduction_pct=0.867249 moved_to_echo=24644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 506, 2, 997, 579, 562, 366, 2480, 647, 0, 0, 0, 0, 0, 0, 4008, 324, 536, 43, 312, 4, 579, 32, 0, 0, 0, 0, 0, 0, 0, 0, 3315, 22, 605, 329, 588, 279, 0, 64, 731, 336, 0, 0, 0, 0, 0, 0, 754, 1318, 510, 519, 418, 2270, 424, 112, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 90, 8, 347, 67, 8, 6, 5593, 0, 0, 0, 0, 0, 0, 0, 16, 4294, 2, 417, 223, 4, 0, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 5, 24, 1, 606, 3, 3330, 0, 5, 1676, 598, 0, 0, 0, 0, 0, 0, 8, 7, 545, 4, 2395, 2617, 92, 433, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 5838, 6269, 6325, 6125, 6107, 6248, 6101] avg=6144.0000 max=6325 max_rank=3 min=5838 min_rank=1 imbalance_ratio=1.029460 max_load_reduction=42827 max_load_reduction_pct=0.871318 moved_to_echo=24581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 469, 0, 1153, 458, 538, 258, 2714, 671, 0, 0, 0, 0, 0, 0, 4623, 233, 378, 0, 296, 1, 614, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3663, 29, 636, 299, 541, 210, 0, 45, 711, 327, 0, 0, 0, 0, 0, 0, 639, 1127, 539, 406, 368, 2238, 477, 47, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 61, 4, 330, 44, 8, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 8, 4417, 2, 475, 161, 0, 2, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 506, 2, 3346, 0, 0, 1634, 653, 0, 0, 0, 0, 0, 0, 7, 0, 415, 0, 2255, 2568, 67, 345, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 6158, 6461, 5841, 6286, 6336, 6152, 5657] avg=6144.0000 max=6461 max_rank=2 min=5657 min_rank=7 imbalance_ratio=1.051595 max_load_reduction=42691 max_load_reduction_pct=0.868551 moved_to_echo=24431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 351, 0, 1035, 497, 548, 346, 2364, 655, 0, 0, 0, 0, 0, 0, 4719, 259, 351, 0, 277, 0, 598, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 9, 635, 304, 558, 187, 0, 30, 781, 369, 0, 0, 0, 0, 0, 0, 691, 1097, 525, 538, 353, 2522, 424, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 64, 1, 266, 30, 5, 0, 5694, 0, 0, 0, 0, 0, 0, 0, 18, 4429, 0, 439, 178, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 618, 1, 3267, 0, 0, 1796, 569, 0, 0, 0, 0, 0, 0, 2, 0, 497, 0, 2319, 2822, 86, 399, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5796, 6214, 6293, 6239, 6062, 6171, 6252, 6125] avg=6144.0000 max=6293 max_rank=2 min=5796 min_rank=0 imbalance_ratio=1.024251 max_load_reduction=42859 max_load_reduction_pct=0.871969 moved_to_echo=24610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 395, 0, 963, 598, 560, 387, 2338, 650, 0, 0, 0, 0, 0, 0, 4524, 329, 403, 9, 304, 1, 594, 28, 0, 0, 0, 0, 0, 0, 0, 0, 3290, 31, 603, 371, 456, 250, 0, 40, 753, 351, 0, 0, 0, 0, 0, 0, 739, 1170, 500, 525, 392, 2400, 468, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 77, 1, 397, 51, 2, 0, 5651, 0, 0, 0, 0, 0, 0, 0, 9, 4062, 0, 375, 186, 1, 0, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 612, 1, 3264, 0, 0, 1731, 562, 0, 0, 0, 0, 0, 0, 3, 1, 526, 0, 2673, 2719, 126, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5891, 6192, 6145, 6280, 6180, 5728, 6211, 6525] avg=6144.0000 max=6525 max_rank=7 min=5728 min_rank=5 imbalance_ratio=1.062012 max_load_reduction=42627 max_load_reduction_pct=0.867249 moved_to_echo=24644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[26496, 256, 5120, 2688, 4224, 1920, 0, 512, 5888, 2816, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 3712, 128, 8448, 4608, 4480, 2816, 20352, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 3968, 128, 20096, 21248, 768, 3328, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5888, 9600, 4224, 3840, 3072, 18560, 3712, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 34560, 128, 3456, 1664, 128, 128, 9472, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 3968, 128, 20096, 21248, 768, 3328, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 256, 128, 4736, 128, 25984, 0, 128, 13568, 4992, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[35712, 2432, 3712, 128, 2560, 128, 4864, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 640, 128, 2688, 384, 128, 128, 45568, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[26496, 256, 5120, 2688, 4224, 1920, 0, 512, 5888, 2816, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 1457, 1492, 96, 122, 1784, 0, 2685, 0, 794, 537, 70, 110, 0, 0, 0, 0, 458, 0, 129, 0, 746, 5380, 1, 335, 0, 6174, 2, 475, 74, 60, 0, 1, 0, 0, 720, 931, 0, 21, 1220, 0, 1697, 0, 4139, 70, 0, 354, 0, 0, 1356, 2710, 0, 0, 6137, 5276, 492, 1, 86, 0, 400, 555, 0, 5, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 1367, 1436, 78, 124, 2356, 0, 2955, 0, 605, 446, 92, 126, 0, 0, 0, 0, 485, 0, 110, 0, 620, 5070, 2, 257, 0, 6210, 0, 605, 100, 44, 0, 0, 0, 0, 873, 779, 0, 15, 1211, 0, 1671, 0, 4354, 90, 0, 355, 0, 0, 1214, 2468, 0, 2, 6155, 5376, 431, 0, 96, 0, 397, 577, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[2, 1558, 1386, 96, 156, 2154, 3, 2931, 0, 672, 486, 73, 88, 0, 1, 2, 0, 491, 0, 108, 0, 855, 5210, 12, 322, 0, 6120, 2, 594, 101, 68, 0, 3, 0, 0, 856, 996, 0, 19, 1145, 0, 1617, 0, 4311, 84, 1, 281, 1, 0, 1193, 2366, 0, 0, 6138, 5175, 437, 5, 86, 0, 377, 563, 0, 7, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 1557, 1332, 77, 134, 2226, 2, 2880, 0, 656, 438, 42, 65, 0, 0, 0, 0, 455, 0, 162, 2, 478, 5410, 0, 255, 0, 6429, 1, 602, 76, 65, 0, 0, 0, 0, 715, 947, 0, 33, 1188, 0, 1515, 0, 4206, 35, 0, 377, 0, 0, 1087, 2412, 0, 1, 6222, 5511, 470, 1, 37, 0, 435, 615, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1477, 1373, 60, 138, 2158, 0, 2764, 0, 649, 380, 60, 74, 0, 0, 0, 0, 520, 0, 141, 0, 708, 5289, 7, 259, 0, 6121, 0, 489, 87, 69, 0, 0, 0, 0, 766, 752, 0, 15, 1413, 0, 1739, 0, 4274, 76, 0, 358, 0, 0, 1107, 2660, 0, 0, 6159, 5503, 440, 0, 61, 0, 424, 582, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 1540, 1431, 44, 82, 2278, 0, 2734, 0, 615, 375, 65, 86, 0, 0, 0, 0, 513, 0, 134, 0, 572, 5382, 1, 226, 0, 6154, 1, 598, 83, 62, 0, 0, 0, 0, 782, 779, 0, 21, 1350, 0, 1659, 0, 4354, 67, 0, 301, 0, 0, 1300, 2253, 0, 0, 6264, 5682, 436, 0, 68, 0, 350, 510, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 1557, 1332, 77, 134, 2226, 2, 2880, 0, 656, 438, 42, 65, 0, 0, 0, 0, 455, 0, 162, 2, 478, 5410, 0, 255, 0, 6429, 1, 602, 76, 65, 0, 0, 0, 0, 715, 947, 0, 33, 1188, 0, 1515, 0, 4206, 35, 0, 377, 0, 0, 1087, 2412, 0, 1, 6222, 5511, 470, 1, 37, 0, 435, 615, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 1367, 1436, 78, 124, 2356, 0, 2955, 0, 605, 446, 92, 126, 0, 0, 0, 0, 485, 0, 110, 0, 620, 5070, 2, 257, 0, 6210, 0, 605, 100, 44, 0, 0, 0, 0, 873, 779, 0, 15, 1211, 0, 1671, 0, 4354, 90, 0, 355, 0, 0, 1214, 2468, 0, 2, 6155, 5376, 431, 0, 96, 0, 397, 577, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 1457, 1492, 96, 122, 1784, 0, 2685, 0, 794, 537, 70, 110, 0, 0, 0, 0, 458, 0, 129, 0, 746, 5380, 1, 335, 0, 6174, 2, 475, 74, 60, 0, 1, 0, 0, 720, 931, 0, 21, 1220, 0, 1697, 0, 4139, 70, 0, 354, 0, 0, 1356, 2710, 0, 0, 6137, 5276, 492, 1, 86, 0, 400, 555, 0, 5, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 1509, 1408, 64, 138, 2114, 0, 2885, 0, 761, 366, 59, 69, 0, 0, 0, 0, 451, 0, 113, 0, 668, 5254, 4, 352, 0, 6140, 0, 544, 98, 81, 0, 3, 0, 0, 793, 1033, 0, 31, 1234, 0, 1504, 0, 4277, 71, 0, 405, 0, 0, 1129, 2489, 0, 0, 6166, 5490, 433, 1, 60, 0, 381, 574, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[2, 1558, 1386, 96, 156, 2154, 3, 2931, 0, 672, 486, 73, 88, 0, 1, 2, 0, 491, 0, 108, 0, 855, 5210, 12, 322, 0, 6120, 2, 594, 101, 68, 0, 3, 0, 0, 856, 996, 0, 19, 1145, 0, 1617, 0, 4311, 84, 1, 281, 1, 0, 1193, 2366, 0, 0, 6138, 5175, 437, 5, 86, 0, 377, 563, 0, 7, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[5, 1553, 1448, 93, 115, 2163, 1, 2758, 0, 515, 639, 70, 92, 0, 5, 2, 0, 478, 7, 144, 1, 687, 5190, 3, 254, 6, 6260, 1, 609, 104, 59, 8, 32, 5, 0, 766, 974, 0, 25, 1224, 2, 1584, 0, 4239, 107, 5, 340, 3, 1, 1185, 2230, 2, 5, 5977, 5471, 439, 0, 83, 0, 372, 810, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1557, 1332, 77, 134, 2226, 2, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 438, 42, 65, 0, 0, 0, 3432, 1085, 420, 0, 0, 0, 0, 0, 0, 455, 0, 162, 2, 478, 4987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 5330, 1, 602, 76, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 947, 0, 33, 1188, 1571, 1099, 423, 168, 0, 0, 0, 0, 0, 1515, 0, 4038, 35, 0, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 2412, 0, 1, 0, 2079, 470, 0, 0, 0, 0, 0, 0, 0, 0, 1, 37, 0, 435, 615, 0, 1, 0, 5137, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6217, 6138, 6084, 6329, 6144, 5965, 6049, 6226] avg=6144.0000 max=6329 max_rank=3 min=5965 min_rank=5 imbalance_ratio=1.030111 max_load_reduction=42823 max_load_reduction_pct=0.871236 moved_to_echo=24384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 1558, 1386, 96, 156, 2154, 3, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 486, 73, 88, 0, 1, 2, 3222, 1071, 428, 0, 0, 0, 0, 0, 0, 491, 0, 108, 0, 855, 4803, 12, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 5074, 2, 594, 101, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 856, 996, 0, 19, 1145, 1599, 1046, 407, 173, 0, 0, 0, 0, 0, 1617, 0, 4138, 84, 1, 281, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 2366, 0, 0, 0, 1953, 437, 0, 0, 0, 0, 0, 0, 0, 0, 5, 86, 0, 377, 563, 0, 7, 0, 5067, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6259, 6043, 6269, 6161, 6244, 6122, 5949, 6105] avg=6144.0000 max=6269 max_rank=2 min=5949 min_rank=6 imbalance_ratio=1.020345 max_load_reduction=42883 max_load_reduction_pct=0.872457 moved_to_echo=24420 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1509, 1408, 64, 138, 2114, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 366, 59, 69, 0, 0, 0, 3418, 1076, 421, 0, 0, 0, 0, 0, 0, 451, 0, 113, 0, 668, 4844, 4, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 5090, 0, 544, 98, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 793, 1033, 0, 31, 1234, 1574, 1050, 410, 171, 0, 0, 0, 0, 0, 1504, 0, 4106, 71, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 2489, 0, 0, 0, 2072, 433, 0, 0, 0, 0, 0, 0, 0, 0, 1, 60, 0, 381, 574, 0, 0, 0, 5090, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6170, 6080, 6165, 6299, 6086, 6123, 6106] avg=6144.0000 max=6299 max_rank=4 min=6080 min_rank=2 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1367, 1436, 78, 124, 2356, 0, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 446, 92, 126, 0, 0, 0, 3347, 1073, 431, 0, 0, 0, 0, 0, 0, 485, 0, 110, 0, 620, 4674, 2, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 5148, 0, 605, 100, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 779, 0, 15, 1211, 1612, 1062, 396, 174, 0, 0, 0, 0, 0, 1671, 0, 4180, 90, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 2468, 0, 2, 0, 2029, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 397, 577, 0, 0, 0, 5082, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6273, 6120, 5891, 6154, 6122, 6296, 6144, 6152] avg=6144.0000 max=6296 max_rank=5 min=5891 min_rank=2 imbalance_ratio=1.024740 max_load_reduction=42856 max_load_reduction_pct=0.871908 moved_to_echo=24714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1457, 1492, 96, 122, 1784, 0, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 794, 537, 70, 110, 0, 0, 0, 3285, 1071, 392, 0, 0, 0, 0, 0, 0, 458, 0, 129, 0, 746, 4960, 1, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 5119, 2, 475, 74, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 720, 931, 0, 21, 1220, 1465, 1055, 420, 166, 0, 0, 0, 0, 0, 1697, 0, 3973, 70, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 2710, 0, 0, 0, 1991, 492, 0, 0, 0, 0, 0, 0, 0, 0, 1, 86, 0, 400, 555, 0, 5, 0, 5066, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5779, 6259, 6294, 6065, 5999, 6094, 6549, 6113] avg=6144.0000 max=6549 max_rank=6 min=5779 min_rank=0 imbalance_ratio=1.065918 max_load_reduction=42603 max_load_reduction_pct=0.866760 moved_to_echo=24755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1477, 1373, 60, 138, 2158, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 380, 60, 74, 0, 0, 0, 3427, 1075, 403, 0, 0, 0, 0, 0, 0, 520, 0, 141, 0, 708, 4876, 7, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 5075, 0, 489, 87, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 752, 0, 15, 1413, 1508, 1046, 413, 171, 0, 0, 0, 0, 0, 1739, 0, 4103, 76, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 2660, 0, 0, 0, 2076, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 424, 582, 0, 0, 0, 5084, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6059, 6068, 6252, 5979, 6084, 6276, 6283, 6151] avg=6144.0000 max=6283 max_rank=6 min=5979 min_rank=3 imbalance_ratio=1.022624 max_load_reduction=42869 max_load_reduction_pct=0.872172 moved_to_echo=24794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1367, 1436, 78, 124, 2356, 0, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 446, 92, 126, 0, 0, 0, 3347, 1073, 431, 0, 0, 0, 0, 0, 0, 485, 0, 110, 0, 620, 4674, 2, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 5148, 0, 605, 100, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 779, 0, 15, 1211, 1612, 1062, 396, 174, 0, 0, 0, 0, 0, 1671, 0, 4180, 90, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 2468, 0, 2, 0, 2029, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 397, 577, 0, 0, 0, 5082, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6273, 6120, 5891, 6154, 6122, 6296, 6144, 6152] avg=6144.0000 max=6296 max_rank=5 min=5891 min_rank=2 imbalance_ratio=1.024740 max_load_reduction=42856 max_load_reduction_pct=0.871908 moved_to_echo=24714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1509, 1408, 64, 138, 2114, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 366, 59, 69, 0, 0, 0, 3418, 1076, 421, 0, 0, 0, 0, 0, 0, 451, 0, 113, 0, 668, 4844, 4, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 5090, 0, 544, 98, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 793, 1033, 0, 31, 1234, 1574, 1050, 410, 171, 0, 0, 0, 0, 0, 1504, 0, 4106, 71, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 2489, 0, 0, 0, 2072, 433, 0, 0, 0, 0, 0, 0, 0, 0, 1, 60, 0, 381, 574, 0, 0, 0, 5090, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6170, 6080, 6165, 6299, 6086, 6123, 6106] avg=6144.0000 max=6299 max_rank=4 min=6080 min_rank=2 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1557, 1332, 77, 134, 2226, 2, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 438, 42, 65, 0, 0, 0, 3432, 1085, 420, 0, 0, 0, 0, 0, 0, 455, 0, 162, 2, 478, 4987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 5330, 1, 602, 76, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 947, 0, 33, 1188, 1571, 1099, 423, 168, 0, 0, 0, 0, 0, 1515, 0, 4038, 35, 0, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 2412, 0, 1, 0, 2079, 470, 0, 0, 0, 0, 0, 0, 0, 0, 1, 37, 0, 435, 615, 0, 1, 0, 5137, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6217, 6138, 6084, 6329, 6144, 5965, 6049, 6226] avg=6144.0000 max=6329 max_rank=3 min=5965 min_rank=5 imbalance_ratio=1.030111 max_load_reduction=42823 max_load_reduction_pct=0.871236 moved_to_echo=24384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1540, 1431, 44, 82, 2278, 0, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 375, 65, 86, 0, 0, 0, 3538, 1093, 399, 0, 0, 0, 0, 0, 0, 513, 0, 134, 0, 572, 4962, 1, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 5102, 1, 598, 83, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 779, 0, 21, 1350, 1491, 1052, 420, 174, 0, 0, 0, 0, 0, 1659, 0, 4180, 67, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 2253, 0, 0, 0, 2144, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 350, 510, 0, 0, 0, 5171, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6171, 6182, 6072, 6069, 6207, 6133, 6099] avg=6144.0000 max=6219 max_rank=0 min=6069 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24508 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 1558, 1386, 96, 156, 2154, 3, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 486, 73, 88, 0, 1, 2, 3222, 1071, 428, 0, 0, 0, 0, 0, 0, 491, 0, 108, 0, 855, 4803, 12, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 5074, 2, 594, 101, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 856, 996, 0, 19, 1145, 1599, 1046, 407, 173, 0, 0, 0, 0, 0, 1617, 0, 4138, 84, 1, 281, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 2366, 0, 0, 0, 1953, 437, 0, 0, 0, 0, 0, 0, 0, 0, 5, 86, 0, 377, 563, 0, 7, 0, 5067, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6259, 6043, 6269, 6161, 6244, 6122, 5949, 6105] avg=6144.0000 max=6269 max_rank=2 min=5949 min_rank=6 imbalance_ratio=1.020345 max_load_reduction=42883 max_load_reduction_pct=0.872457 moved_to_echo=24420 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[5, 1553, 1448, 93, 115, 2163, 1, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 639, 70, 92, 0, 5, 2, 3411, 1043, 406, 0, 0, 0, 0, 0, 0, 478, 7, 144, 1, 687, 4780, 3, 0, 0, 0, 0, 0, 0, 0, 0, 254, 6, 5186, 1, 609, 104, 59, 8, 0, 0, 0, 0, 0, 0, 0, 0, 32, 5, 0, 766, 974, 0, 25, 1224, 1507, 1074, 410, 174, 0, 0, 0, 0, 2, 1584, 0, 4065, 107, 5, 340, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1185, 2230, 2, 5, 0, 2060, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 372, 810, 0, 1, 0, 4934, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6223, 6183, 6100, 6227, 6191, 6106, 5922, 6200] avg=6144.0000 max=6227 max_rank=3 min=5922 min_rank=6 imbalance_ratio=1.013509 max_load_reduction=42925 max_load_reduction_pct=0.873311 moved_to_echo=24419 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 5376, 3712, 640, 768, 0, 128, 128, 27136, 8704, 3328, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 12032, 11392, 640, 1024, 17280, 128, 7040, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 5376, 3712, 640, 768, 0, 128, 128, 27136, 8704, 3328, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 0, 6272, 7296, 0, 256, 10112, 12416, 8576, 3328, 1408, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 9600, 19712, 128, 128, 0, 16512, 3584, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2304, 128, 41216, 128, 4608, 768, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 13056, 0, 32896, 640, 128, 2816, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2304, 128, 41216, 128, 4608, 768, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 3968, 128, 1152, 128, 5376, 38912, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 640, 0, 3200, 4864, 0, 128, 0, 40704, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[172, 2687, 83, 159, 69, 2359, 1672, 165, 0, 0, 1, 527, 1231, 0, 1, 706, 334, 4635, 7293, 2, 2, 908, 5368, 3, 1, 1394, 2489, 1, 0, 0, 985, 0, 4368, 189, 0, 0, 0, 1, 366, 265, 325, 0, 0, 93, 3, 0, 195, 1009, 82, 559, 745, 619, 0, 0, 1, 1167, 23, 0, 86, 0, 48, 506, 0, 5255] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[109, 3122, 64, 134, 68, 1901, 1667, 85, 0, 0, 0, 587, 1310, 0, 0, 787, 312, 4643, 7554, 9, 0, 870, 5379, 0, 0, 1694, 2286, 0, 0, 0, 1191, 0, 4191, 41, 0, 0, 0, 0, 289, 181, 349, 0, 0, 66, 1, 0, 139, 1148, 53, 623, 878, 753, 0, 0, 0, 998, 1, 0, 63, 0, 2, 415, 1, 5188] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[131, 2909, 70, 166, 61, 2431, 1428, 121, 0, 0, 0, 574, 1119, 0, 0, 803, 323, 4716, 7513, 11, 0, 931, 5237, 0, 0, 1546, 2294, 0, 0, 0, 1079, 0, 4376, 69, 0, 0, 0, 0, 274, 178, 301, 0, 0, 70, 20, 0, 181, 1032, 62, 674, 919, 710, 0, 0, 0, 1088, 9, 2, 70, 0, 1, 405, 0, 5248] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[186, 2688, 94, 172, 81, 2321, 1747, 183, 0, 0, 0, 503, 1155, 0, 0, 772, 326, 4705, 7384, 3, 0, 854, 5399, 0, 0, 1466, 2415, 0, 0, 0, 975, 0, 4264, 66, 0, 0, 0, 0, 364, 296, 315, 0, 0, 95, 0, 0, 199, 1021, 87, 560, 778, 636, 0, 0, 0, 1064, 4, 0, 94, 0, 4, 465, 0, 5411] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[194, 2580, 92, 159, 55, 2407, 1746, 171, 0, 5, 2, 542, 998, 0, 2, 800, 341, 4603, 7151, 17, 0, 1033, 5251, 8, 5, 1704, 2530, 8, 0, 1, 991, 6, 4204, 291, 3, 0, 15, 8, 341, 228, 348, 2, 0, 114, 10, 2, 191, 864, 75, 550, 788, 718, 0, 1, 8, 1129, 34, 1, 89, 0, 7, 457, 0, 5272] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[171, 2791, 85, 133, 41, 2550, 1694, 171, 0, 0, 0, 534, 1207, 0, 0, 764, 354, 4529, 7269, 9, 0, 1044, 5346, 0, 0, 1489, 2518, 0, 0, 0, 992, 0, 4236, 60, 0, 0, 1, 0, 294, 252, 463, 0, 0, 84, 2, 1, 157, 1055, 75, 409, 801, 701, 0, 0, 0, 1125, 12, 0, 85, 0, 5, 421, 0, 5222] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[166, 2790, 76, 141, 62, 2185, 1510, 109, 0, 0, 0, 547, 1089, 0, 0, 792, 392, 4889, 7476, 13, 0, 990, 5372, 0, 0, 1429, 2458, 0, 0, 0, 1062, 0, 4077, 62, 0, 0, 0, 0, 306, 182, 339, 0, 0, 75, 2, 0, 143, 1146, 62, 575, 899, 767, 0, 0, 0, 1044, 2, 0, 76, 0, 1, 367, 0, 5479] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[109, 3122, 64, 134, 68, 1901, 1667, 85, 0, 0, 0, 587, 1310, 0, 0, 787, 312, 4643, 7554, 9, 0, 870, 5379, 0, 0, 1694, 2286, 0, 0, 0, 1191, 0, 4191, 41, 0, 0, 0, 0, 289, 181, 349, 0, 0, 66, 1, 0, 139, 1148, 53, 623, 878, 753, 0, 0, 0, 998, 1, 0, 63, 0, 2, 415, 1, 5188] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[172, 2789, 80, 143, 69, 2146, 1935, 132, 0, 0, 0, 536, 1162, 0, 0, 797, 346, 4338, 7422, 35, 0, 1018, 5403, 0, 0, 1707, 2514, 0, 0, 0, 914, 0, 4196, 33, 0, 0, 1, 0, 344, 265, 439, 0, 0, 81, 0, 0, 98, 963, 45, 538, 801, 760, 0, 0, 0, 1051, 4, 0, 80, 0, 17, 373, 0, 5405] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[109, 1768, 64, 134, 68, 1901, 1667, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 587, 1310, 0, 0, 787, 2948, 786, 0, 0, 0, 0, 0, 0, 312, 4643, 0, 9, 0, 870, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1694, 2286, 0, 0, 0, 1191, 0, 1013, 220, 0, 0, 0, 0, 0, 0, 4191, 41, 0, 0, 0, 0, 289, 181, 1254, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 66, 1, 0, 139, 1148, 4606, 0, 0, 0, 0, 0, 0, 0, 53, 623, 878, 753, 0, 0, 0, 998, 2915, 0, 0, 0, 0, 0, 0, 0, 1, 0, 63, 0, 2, 415, 1, 5188, 341, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5796, 6418, 6038, 6404, 5956, 6309, 6220, 6011] avg=6144.0000 max=6418 max_rank=1 min=5796 min_rank=0 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[194, 2580, 92, 159, 55, 2407, 1746, 171, 0, 5, 2, 542, 998, 0, 2, 800, 341, 4603, 7151, 17, 0, 1033, 5251, 8, 5, 1704, 2530, 8, 0, 1, 991, 6, 4204, 291, 3, 0, 15, 8, 341, 228, 348, 2, 0, 114, 10, 2, 191, 864, 75, 550, 788, 718, 0, 1, 8, 1129, 34, 1, 89, 0, 7, 457, 0, 5272] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[172, 2687, 83, 159, 69, 2359, 1672, 165, 0, 0, 1, 527, 1231, 0, 1, 706, 334, 4635, 7293, 2, 2, 908, 5368, 3, 1, 1394, 2489, 1, 0, 0, 985, 0, 4368, 189, 0, 0, 0, 1, 366, 265, 325, 0, 0, 93, 3, 0, 195, 1009, 82, 559, 745, 619, 0, 0, 1, 1167, 23, 0, 86, 0, 48, 506, 0, 5255] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[186, 1523, 94, 172, 81, 2321, 1747, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 1155, 0, 0, 772, 2882, 788, 0, 0, 0, 0, 0, 0, 326, 4705, 0, 3, 0, 854, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 2415, 0, 0, 0, 975, 0, 872, 221, 0, 0, 0, 0, 0, 0, 4264, 66, 0, 0, 0, 0, 364, 296, 1259, 0, 0, 0, 0, 0, 0, 0, 315, 0, 0, 95, 0, 0, 199, 1021, 4502, 0, 0, 0, 0, 0, 0, 0, 87, 560, 778, 636, 0, 0, 0, 1064, 2926, 0, 0, 0, 0, 0, 0, 0, 4, 0, 94, 0, 4, 465, 0, 5411, 293, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6307, 6100, 6093, 5949, 6249, 6132, 6051, 6271] avg=6144.0000 max=6307 max_rank=0 min=5949 min_rank=3 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[131, 1648, 70, 166, 61, 2431, 1428, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 1119, 0, 0, 803, 2932, 765, 0, 0, 0, 0, 0, 0, 323, 4716, 0, 11, 0, 931, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1546, 2294, 0, 0, 0, 1079, 0, 944, 214, 0, 0, 0, 0, 0, 0, 4376, 69, 0, 0, 0, 0, 274, 178, 1221, 0, 0, 0, 0, 0, 0, 0, 301, 0, 0, 70, 20, 0, 181, 1032, 4581, 0, 0, 0, 0, 0, 0, 0, 62, 674, 919, 710, 0, 0, 0, 1088, 2838, 0, 0, 0, 0, 0, 0, 0, 9, 2, 70, 0, 1, 405, 0, 5248, 317, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6056, 6193, 6180, 6077, 6118, 6185, 6291, 6052] avg=6144.0000 max=6291 max_rank=6 min=6052 min_rank=7 imbalance_ratio=1.023926 max_load_reduction=42861 max_load_reduction_pct=0.872009 moved_to_echo=24646 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[172, 1522, 83, 159, 69, 2359, 1672, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 527, 1231, 0, 1, 706, 2847, 784, 0, 0, 0, 0, 0, 0, 334, 4635, 0, 2, 2, 908, 204, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1394, 2489, 1, 0, 0, 985, 0, 872, 219, 0, 0, 0, 0, 0, 0, 4368, 189, 0, 0, 0, 1, 366, 265, 1252, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 93, 3, 0, 195, 1009, 4446, 0, 0, 0, 0, 0, 0, 0, 82, 559, 745, 619, 0, 0, 1, 1167, 2909, 0, 0, 0, 0, 0, 0, 0, 23, 0, 86, 0, 48, 506, 0, 5255, 293, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 6097, 6088, 5961, 6441, 6071, 6082, 6211] avg=6144.0000 max=6441 max_rank=4 min=5961 min_rank=3 imbalance_ratio=1.048340 max_load_reduction=42711 max_load_reduction_pct=0.868958 moved_to_echo=24805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[172, 1580, 80, 143, 69, 2146, 1935, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 1162, 0, 0, 797, 2897, 789, 0, 0, 0, 0, 0, 0, 346, 4338, 0, 35, 0, 1018, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 2514, 0, 0, 0, 914, 0, 905, 221, 0, 0, 0, 0, 0, 0, 4196, 33, 0, 0, 1, 0, 344, 265, 1260, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 81, 0, 0, 98, 963, 4525, 0, 0, 0, 0, 0, 0, 0, 45, 538, 801, 760, 0, 0, 0, 1051, 2928, 0, 0, 0, 0, 0, 0, 0, 4, 0, 80, 0, 17, 373, 0, 5405, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6257, 6181, 5942, 6261, 6099, 6106, 6123, 6183] avg=6144.0000 max=6261 max_rank=3 min=5942 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24511 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[171, 1582, 85, 133, 41, 2550, 1694, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 1207, 0, 0, 764, 2836, 781, 0, 0, 0, 0, 0, 0, 354, 4529, 0, 9, 0, 1044, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1489, 2518, 0, 0, 0, 992, 0, 905, 219, 0, 0, 0, 0, 0, 0, 4236, 60, 0, 0, 1, 0, 294, 252, 1247, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 84, 2, 1, 157, 1055, 4433, 0, 0, 0, 0, 0, 0, 0, 75, 409, 801, 701, 0, 0, 0, 1125, 2897, 0, 0, 0, 0, 0, 0, 0, 12, 0, 85, 0, 5, 421, 0, 5222, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6427, 6122, 6138, 6123, 6090, 6195, 6008, 6049] avg=6144.0000 max=6427 max_rank=0 min=6008 min_rank=6 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[109, 1768, 64, 134, 68, 1901, 1667, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 587, 1310, 0, 0, 787, 2948, 786, 0, 0, 0, 0, 0, 0, 312, 4643, 0, 9, 0, 870, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1694, 2286, 0, 0, 0, 1191, 0, 1013, 220, 0, 0, 0, 0, 0, 0, 4191, 41, 0, 0, 0, 0, 289, 181, 1254, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 66, 1, 0, 139, 1148, 4606, 0, 0, 0, 0, 0, 0, 0, 53, 623, 878, 753, 0, 0, 0, 998, 2915, 0, 0, 0, 0, 0, 0, 0, 1, 0, 63, 0, 2, 415, 1, 5188, 341, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5796, 6418, 6038, 6404, 5956, 6309, 6220, 6011] avg=6144.0000 max=6418 max_rank=1 min=5796 min_rank=0 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[166, 1581, 76, 141, 62, 2185, 1510, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 1089, 0, 0, 792, 2918, 785, 0, 0, 0, 0, 0, 0, 392, 4889, 0, 13, 0, 990, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 2458, 0, 0, 0, 1062, 0, 905, 220, 0, 0, 0, 0, 0, 0, 4077, 62, 0, 0, 0, 0, 306, 182, 1253, 0, 0, 0, 0, 0, 0, 0, 339, 0, 0, 75, 2, 0, 143, 1146, 4558, 0, 0, 0, 0, 0, 0, 0, 62, 575, 899, 767, 0, 0, 0, 1044, 2911, 0, 0, 0, 0, 0, 0, 0, 2, 0, 76, 0, 1, 367, 0, 5479, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5830, 6131, 6487, 6074, 5880, 6263, 6258, 6229] avg=6144.0000 max=6487 max_rank=2 min=5830 min_rank=0 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[172, 1522, 83, 159, 69, 2359, 1672, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 527, 1231, 0, 1, 706, 2847, 784, 0, 0, 0, 0, 0, 0, 334, 4635, 0, 2, 2, 908, 204, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1394, 2489, 1, 0, 0, 985, 0, 872, 219, 0, 0, 0, 0, 0, 0, 4368, 189, 0, 0, 0, 1, 366, 265, 1252, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 93, 3, 0, 195, 1009, 4446, 0, 0, 0, 0, 0, 0, 0, 82, 559, 745, 619, 0, 0, 1, 1167, 2909, 0, 0, 0, 0, 0, 0, 0, 23, 0, 86, 0, 48, 506, 0, 5255, 293, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 6097, 6088, 5961, 6441, 6071, 6082, 6211] avg=6144.0000 max=6441 max_rank=4 min=5961 min_rank=3 imbalance_ratio=1.048340 max_load_reduction=42711 max_load_reduction_pct=0.868958 moved_to_echo=24805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[172, 1580, 80, 143, 69, 2146, 1935, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 1162, 0, 0, 797, 2897, 789, 0, 0, 0, 0, 0, 0, 346, 4338, 0, 35, 0, 1018, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 2514, 0, 0, 0, 914, 0, 905, 221, 0, 0, 0, 0, 0, 0, 4196, 33, 0, 0, 1, 0, 344, 265, 1260, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 81, 0, 0, 98, 963, 4525, 0, 0, 0, 0, 0, 0, 0, 45, 538, 801, 760, 0, 0, 0, 1051, 2928, 0, 0, 0, 0, 0, 0, 0, 4, 0, 80, 0, 17, 373, 0, 5405, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6257, 6181, 5942, 6261, 6099, 6106, 6123, 6183] avg=6144.0000 max=6261 max_rank=3 min=5942 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24511 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[194, 1454, 92, 159, 55, 2407, 1746, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 542, 998, 0, 2, 800, 2791, 770, 0, 0, 0, 0, 0, 0, 341, 4603, 0, 17, 0, 1033, 184, 8, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1704, 2530, 8, 0, 1, 991, 6, 840, 218, 0, 0, 0, 0, 0, 0, 4204, 291, 3, 0, 15, 8, 341, 228, 1229, 0, 0, 0, 0, 0, 0, 0, 348, 2, 0, 114, 10, 2, 191, 864, 4360, 0, 0, 0, 0, 0, 0, 0, 75, 550, 788, 718, 0, 1, 8, 1129, 2850, 0, 0, 0, 0, 0, 0, 0, 34, 1, 89, 0, 7, 457, 0, 5272, 286, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 5910, 6186, 6303, 6319, 5891, 6119, 6146] avg=6144.0000 max=6319 max_rank=4 min=5891 min_rank=5 imbalance_ratio=1.028483 max_load_reduction=42833 max_load_reduction_pct=0.871440 moved_to_echo=24475 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 4352, 9344, 0, 128, 6272, 23168, 6272, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 4608, 6656, 5760, 0, 128, 128, 8704, 23296, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1408, 12672, 768, 1280, 512, 18304, 13440, 1152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 12544, 19584, 128, 0, 128, 8192, 128, 7296, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[33920, 896, 128, 0, 128, 128, 2688, 1920, 9984, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 4608, 6656, 5760, 0, 128, 128, 8704, 23296, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 37120, 0, 128, 128, 7680, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[33920, 896, 128, 0, 128, 128, 2688, 1920, 9984, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2944, 128, 0, 768, 128, 128, 1408, 8320, 36096, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 768, 0, 128, 3456, 128, 42496, 2560, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[3885, 0, 0, 0, 0, 0, 0, 0, 0, 144, 151, 0, 2023, 6388, 0, 221, 0, 0, 0, 0, 0, 245, 103, 194, 0, 81, 15, 0, 0, 94, 0, 0, 0, 4860, 3220, 0, 0, 0, 0, 0, 688, 918, 0, 0, 0, 94, 2508, 1308, 0, 94, 0, 166, 92, 6678, 6199, 0, 691, 0, 0, 0, 0, 0, 8092, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3780, 0, 0, 0, 0, 0, 0, 0, 0, 165, 177, 0, 2172, 6511, 0, 156, 0, 0, 1, 0, 0, 181, 105, 179, 0, 26, 4, 0, 0, 85, 0, 0, 0, 4482, 3411, 0, 0, 0, 0, 0, 683, 940, 0, 0, 1, 85, 2451, 1682, 0, 85, 0, 289, 86, 6554, 6126, 0, 641, 0, 0, 0, 0, 0, 8094, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[3557, 0, 0, 0, 0, 0, 0, 0, 0, 156, 165, 0, 2168, 6451, 0, 203, 0, 0, 2, 1, 1, 164, 89, 178, 0, 62, 2, 1, 0, 83, 0, 1, 1, 4570, 3468, 0, 0, 1, 0, 0, 677, 872, 1, 0, 0, 83, 2779, 1642, 1, 84, 0, 258, 89, 6662, 6145, 1, 470, 0, 0, 0, 0, 1, 8063, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3859, 0, 0, 0, 0, 0, 0, 0, 0, 108, 84, 0, 2207, 6440, 0, 128, 0, 0, 0, 0, 0, 227, 75, 133, 0, 41, 1, 0, 0, 63, 0, 0, 0, 4775, 3335, 0, 0, 0, 0, 0, 738, 997, 0, 0, 0, 63, 2719, 1655, 0, 63, 0, 163, 46, 6571, 5954, 0, 588, 0, 0, 0, 0, 0, 8119, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3735, 0, 0, 0, 0, 0, 0, 0, 0, 143, 152, 0, 2346, 6416, 0, 179, 0, 0, 0, 0, 0, 130, 81, 170, 0, 18, 1, 0, 0, 80, 0, 0, 0, 4733, 3313, 0, 0, 0, 0, 0, 704, 908, 0, 0, 0, 80, 2755, 1703, 0, 80, 0, 167, 92, 6336, 5969, 0, 754, 0, 0, 0, 0, 0, 8107, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3856, 0, 0, 0, 0, 0, 0, 0, 0, 171, 130, 0, 1865, 6615, 0, 107, 0, 0, 0, 0, 0, 116, 72, 153, 0, 59, 0, 0, 0, 70, 0, 0, 0, 4647, 3701, 0, 0, 0, 0, 0, 746, 868, 0, 0, 0, 70, 2320, 1555, 0, 70, 0, 288, 60, 6697, 6024, 0, 827, 0, 0, 0, 0, 0, 8065, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3948, 0, 0, 0, 0, 0, 0, 0, 0, 111, 155, 0, 2129, 6449, 0, 153, 0, 0, 0, 0, 0, 337, 131, 153, 0, 29, 1, 0, 0, 76, 0, 0, 0, 4748, 3408, 0, 0, 0, 0, 0, 707, 977, 0, 0, 0, 76, 2578, 1306, 0, 76, 0, 225, 89, 6625, 6031, 0, 518, 0, 0, 0, 0, 0, 8116, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3423, 0, 3, 1, 6, 0, 3, 15, 4, 144, 171, 2, 2142, 6312, 2, 188, 0, 0, 10, 0, 5, 150, 137, 190, 1, 43, 27, 2, 0, 90, 1, 0, 1, 4890, 3322, 0, 0, 7, 4, 0, 700, 1050, 10, 0, 0, 88, 2749, 1739, 8, 90, 12, 252, 114, 6252, 6006, 8, 772, 0, 0, 0, 8, 5, 7993, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[3885, 0, 0, 0, 0, 0, 0, 0, 0, 144, 151, 0, 2023, 6388, 0, 221, 0, 0, 0, 0, 0, 245, 103, 194, 0, 81, 15, 0, 0, 94, 0, 0, 0, 4860, 3220, 0, 0, 0, 0, 0, 688, 918, 0, 0, 0, 94, 2508, 1308, 0, 94, 0, 166, 92, 6678, 6199, 0, 691, 0, 0, 0, 0, 0, 8092, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3735, 0, 0, 0, 0, 0, 0, 0, 0, 143, 152, 0, 2346, 6416, 0, 179, 0, 0, 0, 0, 0, 130, 81, 170, 0, 18, 1, 0, 0, 80, 0, 0, 0, 4733, 3313, 0, 0, 0, 0, 0, 704, 908, 0, 0, 0, 80, 2755, 1703, 0, 80, 0, 167, 92, 6336, 5969, 0, 754, 0, 0, 0, 0, 0, 8107, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3856, 0, 0, 0, 0, 0, 0, 0, 0, 171, 130, 0, 1865, 6615, 0, 107, 0, 0, 0, 0, 0, 116, 72, 153, 0, 59, 0, 0, 0, 70, 0, 0, 0, 4647, 3701, 0, 0, 0, 0, 0, 746, 868, 0, 0, 0, 70, 2320, 1555, 0, 70, 0, 288, 60, 6697, 6024, 0, 827, 0, 0, 0, 0, 0, 8065, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[3557, 0, 0, 0, 0, 0, 0, 0, 0, 156, 165, 0, 2168, 6451, 0, 203, 0, 0, 2, 1, 1, 164, 89, 178, 0, 62, 2, 1, 0, 83, 0, 1, 1, 4570, 3468, 0, 0, 1, 0, 0, 677, 872, 1, 0, 0, 83, 2779, 1642, 1, 84, 0, 258, 89, 6662, 6145, 1, 470, 0, 0, 0, 0, 1, 8063, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3780, 0, 0, 0, 0, 0, 0, 0, 1871, 346, 72, 0, 0, 0, 0, 0, 0, 165, 177, 0, 2172, 3589, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 181, 105, 179, 2922, 2254, 535, 0, 0, 0, 0, 0, 0, 26, 4, 0, 0, 85, 0, 0, 6019, 0, 0, 0, 0, 0, 0, 0, 0, 2611, 3411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 940, 0, 0, 1, 85, 2451, 1682, 237, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 289, 86, 0, 5817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 0, 5494, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6069, 6259, 6177, 6134, 6022, 6079, 6277, 6135] avg=6144.0000 max=6277 max_rank=6 min=6022 min_rank=4 imbalance_ratio=1.021647 max_load_reduction=42875 max_load_reduction_pct=0.872294 moved_to_echo=24513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3885, 0, 0, 0, 0, 0, 0, 0, 2029, 346, 73, 0, 0, 0, 0, 0, 0, 144, 151, 0, 2023, 3521, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 103, 194, 2867, 2253, 546, 0, 0, 0, 0, 0, 0, 81, 15, 0, 0, 94, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 918, 0, 0, 0, 94, 2508, 1308, 240, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 166, 92, 0, 5886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 0, 0, 0, 0, 5493, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6333, 6060, 6208, 6322, 6051, 5756, 6238, 6184] avg=6144.0000 max=6333 max_rank=0 min=5756 min_rank=5 imbalance_ratio=1.030762 max_load_reduction=42819 max_load_reduction_pct=0.871155 moved_to_echo=24229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3948, 0, 0, 0, 0, 0, 0, 0, 1982, 347, 71, 0, 0, 0, 0, 0, 0, 111, 155, 0, 2129, 3555, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 131, 153, 2894, 2260, 542, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 76, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 2766, 3408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 977, 0, 0, 0, 76, 2578, 1306, 233, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 225, 89, 0, 5727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 5509, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6348, 6103, 6317, 6189, 6174, 5877, 6117, 6027] avg=6144.0000 max=6348 max_rank=0 min=5877 min_rank=5 imbalance_ratio=1.033203 max_load_reduction=42804 max_load_reduction_pct=0.870850 moved_to_echo=24195 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3859, 0, 0, 0, 0, 0, 0, 0, 1993, 347, 70, 0, 0, 0, 0, 0, 0, 108, 84, 0, 2207, 3550, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 75, 133, 2890, 2261, 536, 0, 0, 0, 0, 0, 0, 41, 1, 0, 0, 63, 0, 0, 6035, 0, 0, 0, 0, 0, 0, 0, 0, 2782, 3335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 997, 0, 0, 0, 63, 2719, 1655, 230, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 163, 46, 0, 5654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 0, 0, 0, 0, 0, 5511, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6269, 6077, 6122, 6140, 6117, 6402, 5926, 6099] avg=6144.0000 max=6402 max_rank=5 min=5926 min_rank=6 imbalance_ratio=1.041992 max_load_reduction=42750 max_load_reduction_pct=0.869751 moved_to_echo=24544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3557, 0, 0, 0, 0, 0, 0, 0, 1908, 344, 72, 0, 0, 0, 0, 0, 0, 156, 165, 0, 2168, 3556, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 164, 89, 178, 2895, 2245, 545, 0, 0, 0, 0, 0, 0, 62, 2, 1, 0, 83, 0, 1, 6117, 0, 0, 0, 0, 0, 0, 0, 1, 2662, 3468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 872, 1, 0, 0, 83, 2779, 1642, 238, 0, 0, 0, 0, 0, 0, 0, 1, 84, 0, 258, 89, 0, 5835, 1, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 0, 0, 1, 5474, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5881, 6248, 6120, 6266, 6132, 6292, 6268, 5945] avg=6144.0000 max=6292 max_rank=5 min=5881 min_rank=0 imbalance_ratio=1.024089 max_load_reduction=42860 max_load_reduction_pct=0.871989 moved_to_echo=24637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3856, 0, 0, 0, 0, 0, 0, 0, 1940, 344, 70, 0, 0, 0, 0, 0, 0, 171, 130, 0, 1865, 3646, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 72, 153, 2969, 2246, 547, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 70, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 2707, 3701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 868, 0, 0, 0, 70, 2320, 1555, 233, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 288, 60, 0, 5721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 5475, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6210, 5919, 6103, 6279, 6408, 5792, 6139, 6302] avg=6144.0000 max=6408 max_rank=4 min=5792 min_rank=5 imbalance_ratio=1.042969 max_load_reduction=42744 max_load_reduction_pct=0.869629 moved_to_echo=24641 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3780, 0, 0, 0, 0, 0, 0, 0, 1871, 346, 72, 0, 0, 0, 0, 0, 0, 165, 177, 0, 2172, 3589, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 181, 105, 179, 2922, 2254, 535, 0, 0, 0, 0, 0, 0, 26, 4, 0, 0, 85, 0, 0, 6019, 0, 0, 0, 0, 0, 0, 0, 0, 2611, 3411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 940, 0, 0, 1, 85, 2451, 1682, 237, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 289, 86, 0, 5817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 0, 5494, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6069, 6259, 6177, 6134, 6022, 6079, 6277, 6135] avg=6144.0000 max=6277 max_rank=6 min=6022 min_rank=4 imbalance_ratio=1.021647 max_load_reduction=42875 max_load_reduction_pct=0.872294 moved_to_echo=24513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3735, 0, 0, 0, 0, 0, 0, 0, 1976, 346, 70, 0, 0, 0, 0, 0, 0, 143, 152, 0, 2346, 3537, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 81, 170, 2879, 2258, 518, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 80, 0, 0, 5818, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 3313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 908, 0, 0, 0, 80, 2755, 1703, 231, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 167, 92, 0, 5668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 754, 0, 0, 0, 0, 0, 5503, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6357, 6036, 5917, 6070, 6381, 6007, 6257] avg=6144.0000 max=6381 max_rank=5 min=5917 min_rank=3 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3948, 0, 0, 0, 0, 0, 0, 0, 1982, 347, 71, 0, 0, 0, 0, 0, 0, 111, 155, 0, 2129, 3555, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 131, 153, 2894, 2260, 542, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 76, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 2766, 3408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 977, 0, 0, 0, 76, 2578, 1306, 233, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 225, 89, 0, 5727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 5509, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6348, 6103, 6317, 6189, 6174, 5877, 6117, 6027] avg=6144.0000 max=6348 max_rank=0 min=5877 min_rank=5 imbalance_ratio=1.033203 max_load_reduction=42804 max_load_reduction_pct=0.870850 moved_to_echo=24195 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3423, 0, 3, 1, 6, 0, 3, 15, 2046, 345, 73, 0, 0, 0, 0, 0, 4, 144, 171, 2, 2142, 3476, 2, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 5, 150, 137, 190, 2836, 2230, 511, 0, 0, 0, 0, 0, 1, 43, 27, 2, 0, 90, 1, 0, 5741, 0, 0, 0, 0, 0, 0, 0, 1, 2844, 3322, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 1050, 10, 0, 0, 88, 2749, 1739, 237, 0, 0, 0, 0, 0, 0, 0, 8, 90, 12, 252, 114, 0, 5696, 8, 0, 0, 0, 0, 0, 0, 0, 0, 772, 0, 0, 0, 8, 5, 5418, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5915, 6129, 6069, 5905, 6178, 6573, 6180, 6203] avg=6144.0000 max=6573 max_rank=5 min=5905 min_rank=3 imbalance_ratio=1.069824 max_load_reduction=42579 max_load_reduction_pct=0.866272 moved_to_echo=25134 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3557, 0, 0, 0, 0, 0, 0, 0, 1908, 344, 72, 0, 0, 0, 0, 0, 0, 156, 165, 0, 2168, 3556, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 164, 89, 178, 2895, 2245, 545, 0, 0, 0, 0, 0, 0, 62, 2, 1, 0, 83, 0, 1, 6117, 0, 0, 0, 0, 0, 0, 0, 1, 2662, 3468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 872, 1, 0, 0, 83, 2779, 1642, 238, 0, 0, 0, 0, 0, 0, 0, 1, 84, 0, 258, 89, 0, 5835, 1, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 0, 0, 1, 5474, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5881, 6248, 6120, 6266, 6132, 6292, 6268, 5945] avg=6144.0000 max=6292 max_rank=5 min=5881 min_rank=0 imbalance_ratio=1.024089 max_load_reduction=42860 max_load_reduction_pct=0.871989 moved_to_echo=24637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 1664, 896, 1408, 23168, 18048, 4352, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 7552, 128, 0, 128, 640, 20864, 12672, 1920, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 22016, 27264, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[30080, 0, 128, 128, 128, 0, 128, 128, 15872, 2816, 640, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1152, 1280, 128, 17152, 28544, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[30080, 0, 128, 128, 128, 0, 128, 128, 15872, 2816, 640, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5376, 0, 0, 0, 128, 128, 43904, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 384, 128, 128, 0, 768, 128, 128, 48128, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 768, 128, 1920, 768, 0, 46080, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[250, 683, 14, 59, 248, 0, 0, 0, 1, 3816, 115, 5084, 4, 1, 1688, 5232, 91, 900, 13, 3, 599, 0, 0, 362, 476, 1163, 0, 6229, 7, 4, 0, 0, 2, 2, 0, 9, 3916, 1125, 303, 0, 239, 0, 1, 3, 0, 104, 0, 10, 3773, 13, 0, 6190, 454, 2103, 1241, 0, 0, 3, 8, 2, 2, 512, 1991, 104] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[264, 601, 0, 19, 264, 0, 0, 0, 0, 4006, 67, 5189, 0, 0, 1664, 5112, 65, 919, 0, 0, 575, 0, 0, 400, 470, 738, 0, 6482, 0, 0, 0, 0, 0, 0, 0, 0, 4147, 1081, 304, 1, 151, 0, 0, 0, 0, 47, 0, 0, 3616, 0, 0, 6309, 468, 2201, 1137, 0, 0, 0, 0, 0, 0, 433, 2376, 46] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[251, 605, 0, 15, 251, 0, 0, 0, 0, 4269, 101, 5104, 0, 0, 1778, 5120, 69, 939, 0, 0, 572, 0, 0, 332, 560, 824, 0, 6318, 0, 0, 0, 0, 0, 0, 0, 0, 4156, 1103, 267, 0, 54, 0, 0, 0, 0, 92, 0, 0, 3377, 0, 0, 6267, 428, 2277, 1185, 0, 0, 0, 0, 0, 0, 525, 2222, 91] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[252, 747, 1, 11, 252, 0, 0, 0, 0, 4000, 125, 5017, 0, 0, 1856, 5270, 89, 856, 1, 0, 585, 0, 0, 337, 584, 895, 0, 6090, 0, 0, 0, 0, 1, 0, 0, 1, 4252, 1150, 307, 0, 63, 1, 1, 0, 0, 89, 0, 1, 3516, 1, 0, 6201, 420, 2390, 1245, 0, 0, 0, 0, 0, 0, 388, 2067, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[244, 601, 0, 50, 245, 0, 0, 0, 0, 3915, 117, 5110, 0, 0, 1765, 5039, 86, 878, 0, 0, 553, 0, 0, 372, 590, 841, 0, 6593, 0, 0, 0, 0, 0, 0, 0, 0, 4382, 1103, 297, 0, 87, 0, 0, 0, 0, 90, 0, 0, 3283, 0, 0, 6273, 454, 2212, 1191, 0, 0, 0, 0, 0, 0, 408, 2284, 89] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[274, 772, 0, 39, 274, 0, 0, 0, 0, 3777, 116, 5038, 0, 0, 1642, 5225, 110, 910, 0, 0, 574, 0, 0, 399, 559, 1056, 0, 6314, 0, 11, 0, 0, 0, 0, 0, 0, 4378, 936, 287, 0, 19, 0, 0, 0, 0, 60, 0, 0, 3589, 0, 0, 6083, 469, 2410, 1188, 0, 0, 0, 0, 0, 0, 432, 2151, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[250, 683, 14, 59, 248, 0, 0, 0, 1, 3816, 115, 5084, 4, 1, 1688, 5232, 91, 900, 13, 3, 599, 0, 0, 362, 476, 1163, 0, 6229, 7, 4, 0, 0, 2, 2, 0, 9, 3916, 1125, 303, 0, 239, 0, 1, 3, 0, 104, 0, 10, 3773, 13, 0, 6190, 454, 2103, 1241, 0, 0, 3, 8, 2, 2, 512, 1991, 104] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[245, 686, 0, 30, 245, 0, 0, 0, 0, 3898, 122, 4922, 0, 0, 1669, 5109, 74, 847, 0, 0, 555, 0, 0, 377, 483, 1011, 0, 6435, 3, 0, 0, 0, 0, 0, 0, 0, 4384, 1100, 274, 0, 167, 0, 0, 0, 0, 92, 0, 0, 3418, 0, 0, 6260, 458, 2304, 1242, 0, 0, 0, 0, 0, 0, 452, 2198, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[260, 759, 0, 116, 260, 0, 0, 0, 0, 4073, 144, 5006, 0, 0, 1821, 5051, 126, 757, 0, 0, 591, 0, 0, 334, 465, 841, 0, 6456, 0, 0, 0, 0, 0, 0, 0, 0, 4267, 1040, 258, 0, 189, 0, 0, 0, 0, 89, 0, 0, 3115, 0, 0, 6258, 419, 2333, 1177, 0, 0, 0, 0, 0, 0, 537, 2324, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[264, 601, 0, 19, 264, 0, 0, 0, 4586, 295, 0, 0, 0, 0, 0, 0, 0, 4006, 67, 334, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 919, 0, 0, 575, 0, 0, 400, 3804, 526, 0, 0, 0, 0, 0, 0, 470, 738, 0, 4783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4147, 1081, 304, 1, 564, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 47, 0, 0, 6014, 0, 0, 0, 0, 0, 0, 0, 2316, 0, 0, 0, 468, 2201, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 2376, 46, 1699, 1051, 736, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6029, 6071, 6289, 5991, 6097, 6212, 6122, 6341] avg=6144.0000 max=6341 max_rank=7 min=5991 min_rank=3 imbalance_ratio=1.032064 max_load_reduction=42811 max_load_reduction_pct=0.870992 moved_to_echo=24772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[244, 601, 0, 50, 245, 0, 0, 0, 0, 3915, 117, 5110, 0, 0, 1765, 5039, 86, 878, 0, 0, 553, 0, 0, 372, 590, 841, 0, 6593, 0, 0, 0, 0, 0, 0, 0, 0, 4382, 1103, 297, 0, 87, 0, 0, 0, 0, 90, 0, 0, 3283, 0, 0, 6273, 454, 2212, 1191, 0, 0, 0, 0, 0, 0, 408, 2284, 89] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[251, 605, 0, 15, 251, 0, 0, 0, 4594, 294, 0, 0, 0, 0, 0, 0, 0, 4269, 101, 329, 0, 0, 1778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 939, 0, 0, 572, 0, 0, 332, 3742, 526, 0, 0, 0, 0, 0, 0, 560, 824, 0, 4662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4156, 1103, 267, 0, 526, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 92, 0, 0, 5973, 0, 0, 0, 0, 0, 0, 0, 2164, 0, 0, 0, 428, 2277, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 2222, 91, 1656, 1033, 687, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6010, 6477, 6180, 6046, 6052, 6119, 6054, 6214] avg=6144.0000 max=6477 max_rank=1 min=6010 min_rank=0 imbalance_ratio=1.054199 max_load_reduction=42675 max_load_reduction_pct=0.868225 moved_to_echo=24439 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[252, 747, 1, 11, 252, 0, 0, 0, 4729, 291, 0, 0, 0, 0, 0, 0, 0, 4000, 125, 323, 0, 0, 1856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 856, 1, 0, 585, 0, 0, 337, 3678, 541, 0, 0, 0, 0, 0, 0, 584, 895, 0, 4494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4252, 1150, 307, 0, 548, 0, 0, 0, 0, 0, 0, 0, 63, 1, 1, 0, 0, 89, 0, 1, 5910, 0, 0, 0, 0, 0, 0, 0, 2253, 1, 0, 0, 420, 2390, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 2067, 90, 1596, 1016, 715, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 6304, 6087, 5973, 6259, 6065, 6309, 5872] avg=6144.0000 max=6309 max_rank=6 min=5872 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=42843 max_load_reduction_pct=0.871643 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[264, 601, 0, 19, 264, 0, 0, 0, 4586, 295, 0, 0, 0, 0, 0, 0, 0, 4006, 67, 334, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 919, 0, 0, 575, 0, 0, 400, 3804, 526, 0, 0, 0, 0, 0, 0, 470, 738, 0, 4783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4147, 1081, 304, 1, 564, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 47, 0, 0, 6014, 0, 0, 0, 0, 0, 0, 0, 2316, 0, 0, 0, 468, 2201, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 2376, 46, 1699, 1051, 736, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6029, 6071, 6289, 5991, 6097, 6212, 6122, 6341] avg=6144.0000 max=6341 max_rank=7 min=5991 min_rank=3 imbalance_ratio=1.032064 max_load_reduction=42811 max_load_reduction_pct=0.870992 moved_to_echo=24772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[274, 772, 0, 39, 274, 0, 0, 0, 4687, 285, 0, 0, 0, 0, 0, 0, 0, 3777, 116, 325, 0, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 910, 0, 0, 574, 0, 0, 399, 3693, 538, 0, 0, 0, 0, 0, 0, 559, 1056, 0, 4660, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4378, 936, 287, 0, 559, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 60, 0, 0, 5798, 0, 0, 0, 0, 0, 0, 0, 2300, 0, 0, 0, 469, 2410, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 2151, 60, 1654, 1020, 730, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6331, 5860, 6224, 6286, 6160, 5877, 6367, 6047] avg=6144.0000 max=6367 max_rank=6 min=5860 min_rank=1 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24451 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[260, 759, 0, 116, 260, 0, 0, 0, 4531, 293, 0, 0, 0, 0, 0, 0, 0, 4073, 144, 323, 0, 0, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 757, 0, 0, 591, 0, 0, 334, 3670, 520, 0, 0, 0, 0, 0, 0, 465, 841, 0, 4764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4267, 1040, 258, 0, 485, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 89, 0, 0, 5965, 0, 0, 0, 0, 0, 0, 0, 1996, 0, 0, 0, 419, 2333, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 2324, 86, 1692, 1013, 634, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6361, 5998, 6070, 6050, 6243, 5925, 6286] avg=6144.0000 max=6361 max_rank=1 min=5925 min_rank=6 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24504 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[252, 747, 1, 11, 252, 0, 0, 0, 4729, 291, 0, 0, 0, 0, 0, 0, 0, 4000, 125, 323, 0, 0, 1856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 856, 1, 0, 585, 0, 0, 337, 3678, 541, 0, 0, 0, 0, 0, 0, 584, 895, 0, 4494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4252, 1150, 307, 0, 548, 0, 0, 0, 0, 0, 0, 0, 63, 1, 1, 0, 0, 89, 0, 1, 5910, 0, 0, 0, 0, 0, 0, 0, 2253, 1, 0, 0, 420, 2390, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 2067, 90, 1596, 1016, 715, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 6304, 6087, 5973, 6259, 6065, 6309, 5872] avg=6144.0000 max=6309 max_rank=6 min=5872 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=42843 max_load_reduction_pct=0.871643 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[245, 686, 0, 30, 245, 0, 0, 0, 4584, 294, 0, 0, 0, 0, 0, 0, 0, 3898, 122, 318, 0, 0, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 847, 0, 0, 555, 0, 0, 377, 3608, 525, 0, 0, 0, 0, 0, 0, 483, 1011, 0, 4749, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4384, 1100, 274, 0, 533, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 92, 0, 0, 5966, 0, 0, 0, 0, 0, 0, 0, 2190, 0, 0, 0, 458, 2304, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 2198, 92, 1686, 996, 695, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6084, 6007, 5986, 6246, 6291, 6225, 6194, 6119] avg=6144.0000 max=6291 max_rank=4 min=5986 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=42861 max_load_reduction_pct=0.872009 moved_to_echo=24829 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[250, 683, 14, 59, 248, 0, 0, 0, 4695, 291, 0, 0, 0, 0, 0, 0, 1, 3816, 115, 321, 4, 1, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 900, 13, 3, 599, 0, 0, 362, 3730, 537, 0, 0, 0, 0, 0, 0, 476, 1163, 0, 4594, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 9, 3916, 1125, 303, 0, 592, 0, 0, 0, 0, 0, 0, 0, 239, 0, 1, 3, 0, 104, 0, 10, 5899, 0, 0, 0, 0, 0, 0, 0, 2410, 13, 0, 0, 454, 2103, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 2, 2, 512, 1991, 104, 1635, 1033, 771, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6240, 5946, 6235, 6244, 5949, 6256, 6221, 6061] avg=6144.0000 max=6256 max_rank=5 min=5946 min_rank=1 imbalance_ratio=1.018229 max_load_reduction=42896 max_load_reduction_pct=0.872721 moved_to_echo=24487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[244, 601, 0, 50, 245, 0, 0, 0, 4521, 295, 0, 0, 0, 0, 0, 0, 0, 3915, 117, 329, 0, 0, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 878, 0, 0, 553, 0, 0, 372, 3746, 518, 0, 0, 0, 0, 0, 0, 590, 841, 0, 4865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4382, 1103, 297, 0, 512, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 90, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 2103, 0, 0, 0, 454, 2212, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 2284, 89, 1728, 1035, 668, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5956, 6126, 6153, 6296, 6294, 6155, 5960, 6212] avg=6144.0000 max=6296 max_rank=3 min=5956 min_rank=0 imbalance_ratio=1.024740 max_load_reduction=42856 max_load_reduction_pct=0.871908 moved_to_echo=24621 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[260, 759, 0, 116, 260, 0, 0, 0, 4531, 293, 0, 0, 0, 0, 0, 0, 0, 4073, 144, 323, 0, 0, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 757, 0, 0, 591, 0, 0, 334, 3670, 520, 0, 0, 0, 0, 0, 0, 465, 841, 0, 4764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4267, 1040, 258, 0, 485, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 89, 0, 0, 5965, 0, 0, 0, 0, 0, 0, 0, 1996, 0, 0, 0, 419, 2333, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 2324, 86, 1692, 1013, 634, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6361, 5998, 6070, 6050, 6243, 5925, 6286] avg=6144.0000 max=6361 max_rank=1 min=5925 min_rank=6 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24504 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[17792, 128, 0, 0, 3584, 18304, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 3712, 17664, 768, 13440, 8320, 5760, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 33920, 8704, 2304, 128, 4352, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2048, 5504, 128, 384, 2048, 0, 0, 0, 36992, 2432, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 31872, 1024, 2688, 128, 128, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 33920, 8704, 2304, 128, 4352, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 7040, 128, 128, 4608, 0, 0, 2944, 29696, 4352, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4224, 7424, 0, 37632, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 31872, 1024, 2688, 128, 128, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1024, 128, 128, 128, 0, 768, 0, 128, 47616, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[7164, 63, 0, 219, 5076, 0, 405, 114, 0, 2, 0, 722, 0, 0, 0, 7817, 0, 1597, 4061, 0, 60, 0, 417, 0, 0, 965, 0, 0, 0, 98, 0, 0, 1930, 0, 0, 34, 0, 1322, 5192, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1329, 0, 0, 0, 0, 1588, 55, 0, 2328, 0, 0, 201, 459, 3663, 65, 1201] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[7013, 80, 0, 241, 4843, 0, 370, 186, 0, 0, 0, 860, 0, 0, 0, 7766, 0, 1742, 4623, 0, 3, 0, 528, 0, 0, 934, 0, 0, 0, 140, 0, 0, 1789, 0, 0, 22, 0, 1263, 5243, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1248, 0, 0, 0, 0, 1734, 35, 0, 2042, 0, 0, 227, 459, 3780, 74, 882] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7077, 94, 0, 185, 5181, 0, 350, 195, 0, 0, 0, 944, 0, 0, 0, 7772, 0, 1594, 4243, 0, 27, 0, 519, 0, 0, 914, 0, 0, 0, 153, 0, 0, 1793, 0, 0, 8, 0, 1302, 5199, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1208, 0, 0, 0, 0, 1924, 70, 0, 2078, 0, 0, 263, 408, 3591, 69, 953] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[6917, 88, 0, 227, 4726, 3, 366, 216, 3, 8, 11, 825, 16, 0, 2, 7735, 0, 1782, 4106, 0, 131, 0, 601, 4, 0, 932, 0, 0, 0, 161, 9, 4, 1795, 0, 2, 85, 0, 1300, 5289, 10, 8, 5, 1, 2, 0, 12, 15, 862, 1279, 3, 0, 0, 1, 1834, 53, 11, 2157, 3, 0, 295, 493, 3746, 102, 916] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[6938, 85, 0, 259, 5034, 0, 369, 194, 0, 0, 0, 820, 0, 0, 0, 7754, 0, 1730, 4426, 0, 2, 0, 407, 0, 0, 877, 0, 0, 0, 175, 0, 0, 1858, 0, 0, 50, 0, 1297, 5071, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1214, 0, 0, 0, 0, 1783, 33, 0, 2188, 0, 0, 270, 513, 3539, 126, 1156] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[6958, 70, 0, 216, 5343, 0, 370, 149, 0, 0, 0, 887, 0, 0, 0, 7798, 0, 1665, 4304, 0, 12, 0, 340, 0, 0, 948, 0, 0, 0, 140, 0, 0, 1888, 0, 0, 11, 0, 1296, 4652, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1243, 0, 0, 0, 0, 1923, 52, 0, 2753, 0, 0, 217, 487, 3248, 110, 1063] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7077, 94, 0, 185, 5181, 0, 350, 195, 0, 0, 0, 944, 0, 0, 0, 7772, 0, 1594, 4243, 0, 27, 0, 519, 0, 0, 914, 0, 0, 0, 153, 0, 0, 1793, 0, 0, 8, 0, 1302, 5199, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1208, 0, 0, 0, 0, 1924, 70, 0, 2078, 0, 0, 263, 408, 3591, 69, 953] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[7103, 76, 0, 235, 5087, 0, 396, 176, 0, 0, 0, 891, 0, 1, 0, 7724, 0, 1822, 3926, 0, 0, 0, 581, 0, 0, 939, 0, 0, 0, 140, 0, 0, 1615, 0, 0, 5, 1, 1384, 5204, 0, 0, 0, 0, 0, 0, 0, 0, 929, 1225, 0, 0, 0, 0, 1710, 60, 0, 2200, 0, 0, 238, 439, 3792, 85, 1168] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[7013, 80, 0, 241, 4843, 0, 370, 186, 0, 0, 0, 860, 0, 0, 0, 7766, 0, 1742, 4623, 0, 3, 0, 528, 0, 0, 934, 0, 0, 0, 140, 0, 0, 1789, 0, 0, 22, 0, 1263, 5243, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1248, 0, 0, 0, 0, 1734, 35, 0, 2042, 0, 0, 227, 459, 3780, 74, 882] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[6870, 83, 1, 213, 4956, 0, 363, 186, 0, 4, 1, 811, 1, 0, 0, 7767, 0, 1465, 4371, 0, 61, 0, 609, 0, 1, 815, 2, 0, 0, 152, 0, 1, 1959, 0, 0, 75, 2, 1266, 5219, 1, 0, 0, 0, 0, 0, 2, 0, 1029, 1269, 0, 0, 0, 0, 2071, 62, 1, 2362, 2, 0, 224, 456, 3511, 89, 819] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[264, 63, 0, 219, 5076, 0, 405, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 722, 0, 0, 0, 5328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1597, 3751, 0, 60, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 0, 0, 0, 98, 0, 0, 2489, 1630, 1025, 0, 0, 0, 0, 0, 1930, 0, 0, 34, 0, 1322, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 5270, 0, 0, 0, 0, 0, 0, 0, 1329, 0, 0, 0, 0, 1588, 55, 0, 1567, 1159, 310, 0, 0, 0, 0, 0, 2328, 0, 0, 201, 459, 2096, 65, 1201, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6052, 5825, 6207, 6294, 6275, 6008, 6350] avg=6144.0000 max=6350 max_rank=7 min=5825 min_rank=2 imbalance_ratio=1.033529 max_load_reduction=42802 max_load_reduction_pct=0.870809 moved_to_echo=24927 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[259, 80, 0, 241, 4843, 0, 370, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 0, 0, 0, 5293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 4270, 0, 3, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 0, 0, 0, 140, 0, 0, 2473, 1595, 1035, 0, 0, 0, 0, 0, 1789, 0, 0, 22, 0, 1263, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 5159, 0, 0, 0, 0, 0, 0, 0, 1248, 0, 0, 0, 0, 1734, 35, 0, 1618, 1170, 353, 0, 0, 0, 0, 0, 2042, 0, 0, 227, 459, 2162, 74, 882, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5979, 6153, 6543, 6177, 6112, 6184, 6158, 5846] avg=6144.0000 max=6543 max_rank=2 min=5846 min_rank=7 imbalance_ratio=1.064941 max_load_reduction=42609 max_load_reduction_pct=0.866882 moved_to_echo=24300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[7103, 76, 0, 235, 5087, 0, 396, 176, 0, 0, 0, 891, 0, 1, 0, 7724, 0, 1822, 3926, 0, 0, 0, 581, 0, 0, 939, 0, 0, 0, 140, 0, 0, 1615, 0, 0, 5, 1, 1384, 5204, 0, 0, 0, 0, 0, 0, 0, 0, 929, 1225, 0, 0, 0, 0, 1710, 60, 0, 2200, 0, 0, 238, 439, 3792, 85, 1168] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[6938, 85, 0, 259, 5034, 0, 369, 194, 0, 0, 0, 820, 0, 0, 0, 7754, 0, 1730, 4426, 0, 2, 0, 407, 0, 0, 877, 0, 0, 0, 175, 0, 0, 1858, 0, 0, 50, 0, 1297, 5071, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1214, 0, 0, 0, 0, 1783, 33, 0, 2188, 0, 0, 270, 513, 3539, 126, 1156] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[7164, 63, 0, 219, 5076, 0, 405, 114, 0, 2, 0, 722, 0, 0, 0, 7817, 0, 1597, 4061, 0, 60, 0, 417, 0, 0, 965, 0, 0, 0, 98, 0, 0, 1930, 0, 0, 34, 0, 1322, 5192, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1329, 0, 0, 0, 0, 1588, 55, 0, 2328, 0, 0, 201, 459, 3663, 65, 1201] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[261, 94, 0, 185, 5181, 0, 350, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 0, 0, 0, 5297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 3919, 0, 27, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 914, 0, 0, 0, 153, 0, 0, 2475, 1610, 1026, 0, 0, 0, 0, 0, 1793, 0, 0, 8, 0, 1302, 3013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 5206, 0, 0, 0, 0, 0, 0, 0, 1208, 0, 0, 0, 0, 1924, 70, 0, 1537, 1160, 324, 0, 0, 0, 0, 0, 2078, 0, 0, 263, 408, 2054, 69, 953, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6266, 6241, 6059, 6178, 6116, 6244, 6223, 5825] avg=6144.0000 max=6266 max_rank=0 min=5825 min_rank=7 imbalance_ratio=1.019857 max_load_reduction=42886 max_load_reduction_pct=0.872518 moved_to_echo=24408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[257, 70, 0, 216, 5343, 0, 370, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 0, 0, 0, 5315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1665, 3975, 0, 12, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 0, 0, 0, 140, 0, 0, 2483, 1583, 918, 0, 0, 0, 0, 0, 1888, 0, 0, 11, 0, 1296, 2696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 5118, 0, 0, 0, 0, 0, 0, 0, 1243, 0, 0, 0, 0, 1923, 52, 0, 1390, 1038, 329, 0, 0, 0, 0, 0, 2753, 0, 0, 217, 487, 1858, 110, 1063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6405, 6202, 5992, 6072, 5891, 6127, 5975, 6488] avg=6144.0000 max=6488 max_rank=7 min=5891 min_rank=4 imbalance_ratio=1.055990 max_load_reduction=42664 max_load_reduction_pct=0.868001 moved_to_echo=24481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[257, 85, 0, 259, 5034, 0, 369, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 0, 0, 0, 5285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1730, 4088, 0, 2, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 0, 0, 0, 175, 0, 0, 2469, 1578, 1001, 0, 0, 0, 0, 0, 1858, 0, 0, 50, 0, 1297, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 5103, 0, 0, 0, 0, 0, 0, 0, 1214, 0, 0, 0, 0, 1783, 33, 0, 1514, 1132, 338, 0, 0, 0, 0, 0, 2188, 0, 0, 270, 513, 2025, 126, 1156, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6198, 6105, 6227, 6100, 6143, 6087, 6014, 6278] avg=6144.0000 max=6278 max_rank=7 min=6014 min_rank=6 imbalance_ratio=1.021810 max_load_reduction=42874 max_load_reduction_pct=0.872274 moved_to_echo=24522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[254, 83, 1, 213, 4956, 0, 363, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 811, 1, 0, 0, 5294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 4037, 0, 61, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 815, 2, 0, 0, 152, 0, 1, 2473, 1563, 1030, 0, 0, 0, 0, 0, 1959, 0, 0, 75, 2, 1266, 3024, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1029, 5053, 0, 0, 0, 0, 0, 0, 0, 1269, 0, 0, 0, 0, 2071, 62, 1, 1502, 1165, 334, 0, 0, 0, 0, 0, 2362, 2, 0, 224, 456, 2009, 89, 819, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6056, 6111, 6172, 6037, 6327, 6084, 6404, 5961] avg=6144.0000 max=6404 max_rank=6 min=5961 min_rank=7 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[259, 80, 0, 241, 4843, 0, 370, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 0, 0, 0, 5293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 4270, 0, 3, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 0, 0, 0, 140, 0, 0, 2473, 1595, 1035, 0, 0, 0, 0, 0, 1789, 0, 0, 22, 0, 1263, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 5159, 0, 0, 0, 0, 0, 0, 0, 1248, 0, 0, 0, 0, 1734, 35, 0, 1618, 1170, 353, 0, 0, 0, 0, 0, 2042, 0, 0, 227, 459, 2162, 74, 882, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5979, 6153, 6543, 6177, 6112, 6184, 6158, 5846] avg=6144.0000 max=6543 max_rank=2 min=5846 min_rank=7 imbalance_ratio=1.064941 max_load_reduction=42609 max_load_reduction_pct=0.866882 moved_to_echo=24300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[264, 63, 0, 219, 5076, 0, 405, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 722, 0, 0, 0, 5328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1597, 3751, 0, 60, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 0, 0, 0, 98, 0, 0, 2489, 1630, 1025, 0, 0, 0, 0, 0, 1930, 0, 0, 34, 0, 1322, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 5270, 0, 0, 0, 0, 0, 0, 0, 1329, 0, 0, 0, 0, 1588, 55, 0, 1567, 1159, 310, 0, 0, 0, 0, 0, 2328, 0, 0, 201, 459, 2096, 65, 1201, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6052, 5825, 6207, 6294, 6275, 6008, 6350] avg=6144.0000 max=6350 max_rank=7 min=5825 min_rank=2 imbalance_ratio=1.033529 max_load_reduction=42802 max_load_reduction_pct=0.870809 moved_to_echo=24927 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[249, 88, 0, 227, 4726, 3, 366, 216, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 11, 825, 16, 0, 2, 5267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 3787, 0, 131, 0, 601, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 0, 0, 0, 161, 9, 4, 2468, 1576, 1049, 0, 0, 0, 0, 0, 1795, 0, 2, 85, 0, 1300, 3056, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 1, 2, 0, 12, 15, 862, 5092, 0, 0, 0, 0, 0, 0, 0, 1279, 3, 0, 0, 1, 1834, 53, 11, 1607, 1184, 319, 0, 0, 0, 0, 0, 2157, 3, 0, 295, 493, 2139, 102, 916, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5875, 6132, 6305, 6199, 6248, 5997, 6291, 6105] avg=6144.0000 max=6305 max_rank=2 min=5875 min_rank=0 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[257, 85, 0, 259, 5034, 0, 369, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 0, 0, 0, 5285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1730, 4088, 0, 2, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 0, 0, 0, 175, 0, 0, 2469, 1578, 1001, 0, 0, 0, 0, 0, 1858, 0, 0, 50, 0, 1297, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 5103, 0, 0, 0, 0, 0, 0, 0, 1214, 0, 0, 0, 0, 1783, 33, 0, 1514, 1132, 338, 0, 0, 0, 0, 0, 2188, 0, 0, 270, 513, 2025, 126, 1156, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6198, 6105, 6227, 6100, 6143, 6087, 6014, 6278] avg=6144.0000 max=6278 max_rank=7 min=6014 min_rank=6 imbalance_ratio=1.021810 max_load_reduction=42874 max_load_reduction_pct=0.872274 moved_to_echo=24522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[262, 76, 0, 235, 5087, 0, 396, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 891, 0, 1, 0, 5264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1822, 3626, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 939, 0, 0, 0, 140, 0, 0, 2460, 1616, 1027, 0, 0, 0, 0, 0, 1615, 0, 0, 5, 1, 1384, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 5225, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 0, 0, 0, 1710, 60, 0, 1623, 1161, 300, 0, 0, 0, 0, 0, 2200, 0, 0, 238, 439, 2169, 85, 1168, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6232, 6156, 6029, 6182, 6021, 6154, 6079, 6299] avg=6144.0000 max=6299 max_rank=7 min=6021 min_rank=4 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 7424, 128, 0, 0, 1280, 128, 128, 19840, 12800, 8192, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[10112, 128, 0, 0, 128, 14592, 512, 128, 12416, 9216, 2688, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18176, 128, 0, 2048, 3840, 16512, 768, 8192, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14720, 0, 128, 384, 128, 10496, 23808, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18176, 128, 0, 2048, 3840, 16512, 768, 8192, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 13440, 31488, 0, 384, 0, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 6784, 128, 128, 128, 42368, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 7936, 41344, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2176, 640, 128, 1920, 40320, 128, 3072, 1536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 7936, 41344, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 6784, 128, 128, 128, 42368, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 8121, 166, 0, 0, 70, 0, 41, 0, 0, 0, 0, 0, 1051, 0, 140, 0, 2, 0, 8122, 0, 0, 0, 7677, 0, 274, 0, 61, 0, 0, 0, 0, 0, 289, 70, 0, 0, 0, 7518, 0, 2712, 0, 0, 0, 96, 0, 0, 0, 7468, 0, 69, 0, 0, 0, 0, 0, 0, 0, 1423, 3712, 0, 0, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 8082, 223, 0, 0, 66, 1, 30, 0, 5, 11, 1, 0, 1011, 2, 154, 1, 7, 1, 8064, 4, 10, 0, 7657, 0, 248, 2, 53, 0, 0, 1, 0, 8, 270, 88, 9, 1, 1, 7550, 0, 2816, 2, 19, 0, 98, 7, 7, 21, 7472, 5, 88, 2, 0, 2, 0, 17, 0, 2, 1321, 3616, 2, 0, 2, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 8098, 130, 0, 0, 59, 0, 11, 0, 0, 0, 0, 0, 989, 0, 153, 0, 0, 0, 8098, 0, 0, 0, 7696, 0, 251, 0, 81, 0, 0, 0, 0, 0, 261, 94, 0, 0, 0, 7607, 0, 2668, 0, 0, 0, 110, 0, 0, 0, 7517, 0, 94, 0, 0, 0, 0, 0, 0, 0, 1332, 3809, 0, 0, 0, 94] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 8107, 80, 0, 0, 89, 0, 13, 0, 0, 0, 0, 0, 982, 0, 175, 0, 2, 0, 8107, 0, 0, 0, 7681, 0, 260, 0, 40, 0, 0, 0, 0, 0, 266, 85, 0, 0, 0, 7596, 0, 2751, 0, 0, 0, 105, 0, 0, 0, 7420, 0, 85, 0, 0, 0, 0, 0, 0, 0, 1269, 3954, 0, 0, 0, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 8107, 187, 0, 0, 69, 0, 28, 0, 0, 1, 0, 0, 988, 0, 152, 0, 2, 0, 8099, 0, 2, 0, 7699, 0, 251, 1, 68, 0, 0, 0, 0, 0, 277, 83, 1, 1, 2, 7600, 0, 2697, 0, 2, 0, 107, 0, 0, 1, 7506, 1, 83, 0, 0, 0, 0, 1, 0, 0, 1411, 3639, 3, 0, 0, 83] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 8128, 159, 0, 0, 35, 0, 10, 0, 0, 0, 0, 0, 1048, 0, 98, 0, 0, 0, 8124, 0, 0, 0, 7737, 0, 264, 0, 64, 0, 0, 0, 0, 0, 280, 63, 0, 0, 1, 7519, 0, 2646, 0, 0, 0, 71, 0, 0, 0, 7556, 0, 63, 2, 0, 0, 0, 0, 0, 0, 1358, 3861, 0, 0, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 8112, 88, 0, 0, 59, 0, 4, 0, 0, 0, 0, 0, 1035, 0, 140, 0, 0, 0, 8112, 0, 0, 0, 7723, 0, 244, 0, 69, 0, 0, 0, 0, 0, 252, 80, 0, 0, 0, 7569, 0, 2588, 0, 0, 0, 89, 0, 0, 0, 7544, 0, 80, 0, 0, 0, 0, 0, 0, 0, 1325, 3959, 0, 0, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 8116, 108, 0, 0, 64, 0, 10, 0, 0, 0, 0, 0, 1066, 0, 140, 0, 0, 0, 8116, 0, 0, 0, 7734, 0, 241, 0, 60, 0, 0, 0, 0, 0, 256, 76, 0, 0, 0, 7570, 0, 2698, 0, 0, 0, 106, 0, 0, 0, 7565, 0, 76, 0, 0, 0, 0, 0, 0, 0, 1392, 3683, 0, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 8107, 80, 0, 0, 89, 0, 13, 0, 0, 0, 0, 0, 982, 0, 175, 0, 2, 0, 8107, 0, 0, 0, 7681, 0, 260, 0, 40, 0, 0, 0, 0, 0, 266, 85, 0, 0, 0, 7596, 0, 2751, 0, 0, 0, 105, 0, 0, 0, 7420, 0, 85, 0, 0, 0, 0, 0, 0, 0, 1269, 3954, 0, 0, 0, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5933, 159, 0, 0, 35, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 98, 2283, 2195, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 64, 0, 0, 0, 0, 5841, 0, 0, 0, 0, 0, 0, 0, 0, 280, 63, 0, 0, 1, 5757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 0, 0, 0, 71, 0, 0, 0, 1567, 1257, 519, 0, 0, 0, 0, 0, 6102, 0, 63, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 3861, 0, 0, 0, 65, 935, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6129, 6170, 6169, 6101, 6060, 6167, 6219] avg=6144.0000 max=6219 max_rank=7 min=6060 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 8082, 223, 0, 0, 66, 1, 30, 0, 5, 11, 1, 0, 1011, 2, 154, 1, 7, 1, 8064, 4, 10, 0, 7657, 0, 248, 2, 53, 0, 0, 1, 0, 8, 270, 88, 9, 1, 1, 7550, 0, 2816, 2, 19, 0, 98, 7, 7, 21, 7472, 5, 88, 2, 0, 2, 0, 17, 0, 2, 1321, 3616, 2, 0, 2, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 8112, 88, 0, 0, 59, 0, 4, 0, 0, 0, 0, 0, 1035, 0, 140, 0, 0, 0, 8112, 0, 0, 0, 7723, 0, 244, 0, 69, 0, 0, 0, 0, 0, 252, 80, 0, 0, 0, 7569, 0, 2588, 0, 0, 0, 89, 0, 0, 0, 7544, 0, 80, 0, 0, 0, 0, 0, 0, 0, 1325, 3959, 0, 0, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 8128, 159, 0, 0, 35, 0, 10, 0, 0, 0, 0, 0, 1048, 0, 98, 0, 0, 0, 8124, 0, 0, 0, 7737, 0, 264, 0, 64, 0, 0, 0, 0, 0, 280, 63, 0, 0, 1, 7519, 0, 2646, 0, 0, 0, 71, 0, 0, 0, 7556, 0, 63, 2, 0, 0, 0, 0, 0, 0, 1358, 3861, 0, 0, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5912, 130, 0, 0, 59, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 989, 0, 153, 2275, 2186, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 81, 0, 0, 0, 0, 5823, 0, 0, 0, 0, 0, 0, 0, 0, 261, 94, 0, 0, 0, 5824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 0, 0, 0, 110, 0, 0, 0, 1559, 1272, 516, 0, 0, 0, 0, 0, 6071, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 3809, 0, 0, 0, 94, 930, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6114, 6137, 6155, 6179, 6125, 6165, 6165] avg=6144.0000 max=6179 max_rank=4 min=6112 min_rank=0 imbalance_ratio=1.005697 max_load_reduction=42973 max_load_reduction_pct=0.874288 moved_to_echo=24634 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5918, 187, 0, 0, 69, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 988, 0, 152, 2277, 2189, 510, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1, 68, 0, 0, 0, 0, 5822, 0, 0, 0, 0, 0, 0, 0, 0, 277, 83, 1, 1, 2, 5819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2697, 0, 2, 0, 107, 0, 0, 1, 1560, 1271, 516, 0, 0, 0, 0, 0, 6061, 1, 83, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1411, 3639, 3, 0, 0, 83, 929, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6202, 6117, 6143, 6142, 6183, 6154, 6146, 6065] avg=6144.0000 max=6202 max_rank=0 min=6065 min_rank=7 imbalance_ratio=1.009440 max_load_reduction=42950 max_load_reduction_pct=0.873820 moved_to_echo=24548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5928, 166, 0, 0, 70, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 0, 140, 2282, 2193, 505, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 0, 61, 0, 0, 0, 0, 5840, 0, 0, 0, 0, 0, 0, 0, 0, 289, 70, 0, 0, 0, 5756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2712, 0, 0, 0, 96, 0, 0, 0, 1555, 1257, 513, 0, 0, 0, 0, 0, 6031, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 3712, 0, 0, 0, 70, 924, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6171, 6124, 6175, 6115, 6133, 6100, 6129] avg=6144.0000 max=6205 max_rank=0 min=6100 min_rank=6 imbalance_ratio=1.009928 max_load_reduction=42947 max_load_reduction_pct=0.873759 moved_to_echo=24477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5922, 88, 0, 0, 59, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 0, 140, 2280, 2190, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 69, 0, 0, 0, 0, 5832, 0, 0, 0, 0, 0, 0, 0, 0, 252, 80, 0, 0, 0, 5796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2588, 0, 0, 0, 89, 0, 0, 0, 1564, 1265, 518, 0, 0, 0, 0, 0, 6092, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 3959, 0, 0, 0, 80, 934, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6153, 6159, 6145, 6128, 6024, 6172, 6298] avg=6144.0000 max=6298 max_rank=7 min=6024 min_rank=5 imbalance_ratio=1.025065 max_load_reduction=42854 max_load_reduction_pct=0.871867 moved_to_echo=24622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5896, 223, 0, 0, 66, 1, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 1, 0, 1011, 2, 154, 2267, 2186, 511, 0, 0, 0, 0, 0, 1, 7, 1, 0, 4, 10, 0, 6102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 2, 53, 0, 0, 1, 0, 5797, 0, 0, 0, 0, 0, 0, 0, 8, 270, 88, 9, 1, 1, 5774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 2, 19, 0, 98, 7, 7, 21, 1555, 1265, 517, 0, 0, 0, 0, 0, 6026, 5, 88, 2, 0, 2, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1321, 3616, 2, 0, 2, 92, 929, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6216, 6148, 6125, 6101, 6151, 6307, 6140, 5964] avg=6144.0000 max=6307 max_rank=5 min=5964 min_rank=7 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5918, 80, 0, 0, 89, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 982, 0, 175, 2278, 2189, 510, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 40, 0, 0, 0, 0, 5829, 0, 0, 0, 0, 0, 0, 0, 0, 266, 85, 0, 0, 0, 5816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2751, 0, 0, 0, 105, 0, 0, 0, 1556, 1270, 510, 0, 0, 0, 0, 0, 5992, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 3954, 0, 0, 0, 85, 918, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6134, 6127, 6129, 6167, 6192, 6077, 6226] avg=6144.0000 max=6226 max_rank=7 min=6077 min_rank=6 imbalance_ratio=1.013346 max_load_reduction=42926 max_load_reduction_pct=0.873332 moved_to_echo=24662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5918, 187, 0, 0, 69, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 988, 0, 152, 2277, 2189, 510, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1, 68, 0, 0, 0, 0, 5822, 0, 0, 0, 0, 0, 0, 0, 0, 277, 83, 1, 1, 2, 5819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2697, 0, 2, 0, 107, 0, 0, 1, 1560, 1271, 516, 0, 0, 0, 0, 0, 6061, 1, 83, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1411, 3639, 3, 0, 0, 83, 929, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6202, 6117, 6143, 6142, 6183, 6154, 6146, 6065] avg=6144.0000 max=6202 max_rank=0 min=6065 min_rank=7 imbalance_ratio=1.009440 max_load_reduction=42950 max_load_reduction_pct=0.873820 moved_to_echo=24548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5925, 108, 0, 0, 64, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 0, 140, 2281, 2191, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 60, 0, 0, 0, 0, 5835, 0, 0, 0, 0, 0, 0, 0, 0, 256, 76, 0, 0, 0, 5796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2698, 0, 0, 0, 106, 0, 0, 0, 1567, 1266, 520, 0, 0, 0, 0, 0, 6109, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1392, 3683, 0, 0, 0, 75, 936, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6107, 6186, 6167, 6136, 6128, 6157, 6185, 6086] avg=6144.0000 max=6186 max_rank=1 min=6086 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=42966 max_load_reduction_pct=0.874146 moved_to_echo=24556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5933, 159, 0, 0, 35, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 98, 2283, 2195, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 64, 0, 0, 0, 0, 5841, 0, 0, 0, 0, 0, 0, 0, 0, 280, 63, 0, 0, 1, 5757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 0, 0, 0, 71, 0, 0, 0, 1567, 1257, 519, 0, 0, 0, 0, 0, 6102, 0, 63, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 3861, 0, 0, 0, 65, 935, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6129, 6170, 6169, 6101, 6060, 6167, 6219] avg=6144.0000 max=6219 max_rank=7 min=6060 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5922, 88, 0, 0, 59, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 0, 140, 2280, 2190, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 69, 0, 0, 0, 0, 5832, 0, 0, 0, 0, 0, 0, 0, 0, 252, 80, 0, 0, 0, 5796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2588, 0, 0, 0, 89, 0, 0, 0, 1564, 1265, 518, 0, 0, 0, 0, 0, 6092, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 3959, 0, 0, 0, 80, 934, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6153, 6159, 6145, 6128, 6024, 6172, 6298] avg=6144.0000 max=6298 max_rank=7 min=6024 min_rank=5 imbalance_ratio=1.025065 max_load_reduction=42854 max_load_reduction_pct=0.871867 moved_to_echo=24622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5896, 223, 0, 0, 66, 1, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 1, 0, 1011, 2, 154, 2267, 2186, 511, 0, 0, 0, 0, 0, 1, 7, 1, 0, 4, 10, 0, 6102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 2, 53, 0, 0, 1, 0, 5797, 0, 0, 0, 0, 0, 0, 0, 8, 270, 88, 9, 1, 1, 5774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 2, 19, 0, 98, 7, 7, 21, 1555, 1265, 517, 0, 0, 0, 0, 0, 6026, 5, 88, 2, 0, 2, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1321, 3616, 2, 0, 2, 92, 929, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6216, 6148, 6125, 6101, 6151, 6307, 6140, 5964] avg=6144.0000 max=6307 max_rank=5 min=5964 min_rank=7 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 0, 8192, 128, 1152, 18304, 17536, 4096, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[21632, 128, 128, 0, 896, 128, 128, 128, 12544, 10240, 4224, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 10880, 30336, 128, 0, 128, 768, 7552, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 47360, 1152, 0, 0, 512, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 2048, 128, 512, 0, 0, 128, 0, 46720, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[48512, 128, 640, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2176, 640, 128, 128, 128, 46464, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 47360, 1152, 0, 0, 512, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[48512, 128, 640, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 547, 0, 0, 0, 0, 100, 36, 0, 0, 1093, 1, 0, 0, 0, 0, 1738, 0, 288, 0, 0, 0, 2486, 8031, 0, 3282, 0, 7967, 0, 0, 177, 4309, 0, 7426, 0, 7926, 0, 0, 0, 0, 176, 0, 0, 459, 889, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 33, 35, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 455, 0, 0, 0, 0, 128, 59, 0, 0, 1128, 0, 0, 0, 0, 0, 1664, 0, 324, 0, 0, 0, 2506, 7945, 0, 2894, 0, 7863, 0, 0, 240, 4310, 0, 7363, 0, 7941, 0, 0, 0, 0, 277, 0, 0, 470, 1041, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 91, 59, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 521, 0, 0, 0, 0, 183, 70, 0, 0, 1216, 1, 0, 0, 0, 0, 2238, 0, 198, 0, 0, 0, 2586, 7982, 0, 2152, 0, 7921, 0, 0, 225, 4199, 0, 7375, 0, 7918, 0, 0, 0, 0, 230, 0, 0, 439, 816, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 2736, 37, 70, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 513, 0, 0, 0, 0, 150, 64, 0, 0, 1247, 0, 0, 0, 0, 0, 1814, 0, 364, 0, 0, 0, 2375, 7976, 0, 3215, 0, 7914, 0, 0, 216, 4068, 0, 7341, 0, 7948, 0, 0, 0, 0, 207, 0, 0, 414, 1040, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 6, 64, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 575, 0, 0, 0, 0, 216, 90, 0, 0, 1112, 2, 0, 0, 0, 0, 1652, 0, 349, 0, 0, 0, 2388, 7932, 0, 2908, 0, 7890, 0, 0, 246, 4376, 1, 7390, 0, 7932, 0, 0, 0, 0, 266, 0, 0, 447, 1045, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 2131, 48, 90, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 0, 0, 1, 2, 159, 69, 0, 0, 1214, 9, 1, 1, 0, 0, 1798, 0, 318, 0, 0, 1, 2261, 7955, 1, 2854, 0, 7884, 1, 0, 239, 4469, 0, 7341, 0, 7925, 0, 0, 0, 0, 297, 0, 0, 466, 1096, 5, 0, 89, 1, 2, 0, 0, 0, 0, 0, 2, 2034, 108, 70, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 521, 0, 0, 0, 0, 183, 70, 0, 0, 1216, 1, 0, 0, 0, 0, 2238, 0, 198, 0, 0, 0, 2586, 7982, 0, 2152, 0, 7921, 0, 0, 225, 4199, 0, 7375, 0, 7918, 0, 0, 0, 0, 230, 0, 0, 439, 816, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 2736, 37, 70, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 455, 0, 0, 0, 0, 128, 59, 0, 0, 1128, 0, 0, 0, 0, 0, 1664, 0, 324, 0, 0, 0, 2506, 7945, 0, 2894, 0, 7863, 0, 0, 240, 4310, 0, 7363, 0, 7941, 0, 0, 0, 0, 277, 0, 0, 470, 1041, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 91, 59, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[5, 1, 0, 528, 3, 0, 2, 19, 164, 67, 0, 15, 1391, 15, 7, 0, 0, 1, 1694, 0, 356, 12, 0, 4, 2453, 7914, 0, 2393, 11, 7837, 5, 5, 241, 4185, 0, 7203, 1, 7898, 10, 4, 0, 0, 326, 6, 10, 523, 1096, 8, 0, 104, 19, 1, 2, 1, 0, 5, 2, 17, 2414, 106, 66, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 513, 0, 0, 0, 0, 150, 64, 0, 0, 1247, 0, 0, 0, 0, 0, 1814, 0, 364, 0, 0, 0, 2375, 7976, 0, 3215, 0, 7914, 0, 0, 216, 4068, 0, 7341, 0, 7948, 0, 0, 0, 0, 207, 0, 0, 414, 1040, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 6, 64, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 0, 0, 1, 2, 159, 69, 0, 0, 1214, 9, 1, 1, 0, 0, 1798, 0, 318, 0, 0, 1, 2261, 7955, 1, 2854, 0, 7884, 1, 0, 239, 4469, 0, 7341, 0, 7925, 0, 0, 0, 0, 297, 0, 0, 466, 1096, 5, 0, 89, 1, 2, 0, 0, 0, 0, 0, 2, 2034, 108, 70, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 547, 0, 0, 0, 0, 100, 36, 0, 0, 1093, 1, 0, 0, 0, 0, 1738, 0, 288, 0, 0, 0, 2486, 8031, 0, 3282, 0, 7967, 0, 0, 177, 4309, 0, 7426, 0, 7926, 0, 0, 0, 0, 176, 0, 0, 459, 889, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 33, 35, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 515, 0, 0, 0, 0, 134, 60, 0, 0, 1196, 0, 0, 0, 0, 0, 1827, 0, 317, 0, 0, 0, 2375, 7972, 0, 2741, 0, 7903, 0, 0, 231, 4400, 0, 7391, 0, 7947, 0, 0, 0, 0, 193, 0, 0, 433, 1086, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 2251, 37, 60, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 547, 0, 0, 0, 0, 3744, 1893, 0, 0, 0, 0, 0, 0, 100, 36, 0, 0, 1093, 1, 0, 0, 4182, 540, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 288, 0, 0, 0, 2048, 2013, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 3282, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 4309, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 459, 889, 1, 4462, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 33, 35, 0, 0, 0, 3771, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 5952, 6087, 6685, 6128, 5987, 6146, 5983] avg=6144.0000 max=6685 max_rank=3 min=5952 min_rank=1 imbalance_ratio=1.088053 max_load_reduction=42467 max_load_reduction_pct=0.863993 moved_to_echo=24244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 455, 0, 0, 0, 0, 3751, 1873, 0, 0, 0, 0, 0, 0, 128, 59, 0, 0, 1128, 0, 0, 0, 4190, 533, 0, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 324, 0, 0, 0, 2021, 1996, 0, 0, 0, 0, 0, 0, 2506, 0, 0, 2894, 0, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 4310, 0, 1628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 470, 1041, 0, 4404, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 6072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 91, 59, 0, 0, 0, 3739, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6079, 6038, 6005, 6305, 6178, 6192, 6115, 6240] avg=6144.0000 max=6305 max_rank=3 min=6005 min_rank=2 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 575, 0, 0, 0, 0, 3746, 1870, 0, 0, 0, 0, 0, 0, 216, 90, 0, 0, 1112, 2, 0, 0, 4186, 534, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 0, 349, 0, 0, 0, 2028, 2003, 0, 0, 0, 0, 0, 0, 2388, 0, 0, 2908, 0, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 4376, 1, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 447, 1045, 0, 4419, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 6062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2131, 48, 90, 0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6191, 6140, 6032, 6205, 6258, 6177, 6128, 6021] avg=6144.0000 max=6258 max_rank=4 min=6021 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=42894 max_load_reduction_pct=0.872681 moved_to_echo=24584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 521, 0, 0, 0, 0, 3740, 1882, 0, 0, 0, 0, 0, 0, 183, 70, 0, 0, 1216, 1, 0, 0, 4178, 537, 0, 0, 0, 0, 0, 0, 0, 0, 2238, 0, 198, 0, 0, 0, 2036, 1999, 0, 0, 0, 0, 0, 0, 2586, 0, 0, 2152, 0, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 4199, 0, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 439, 816, 0, 4436, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2736, 37, 70, 0, 0, 0, 3745, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6185, 6471, 5650, 6055, 5921, 6139, 6588] avg=6144.0000 max=6588 max_rank=7 min=5650 min_rank=3 imbalance_ratio=1.072266 max_load_reduction=42564 max_load_reduction_pct=0.865967 moved_to_echo=24703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 513, 0, 0, 0, 0, 3755, 1881, 0, 0, 0, 0, 0, 0, 150, 64, 0, 0, 1247, 0, 0, 0, 4193, 536, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 364, 0, 0, 0, 2034, 1990, 0, 0, 0, 0, 0, 0, 2375, 0, 0, 3215, 0, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 4068, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 414, 1040, 0, 4433, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 6095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 6, 64, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6190, 6202, 6501, 5908, 6094, 6099, 6009] avg=6144.0000 max=6501 max_rank=3 min=5908 min_rank=4 imbalance_ratio=1.058105 max_load_reduction=42651 max_load_reduction_pct=0.867737 moved_to_echo=24110 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 0, 0, 1, 2, 3744, 1876, 0, 0, 0, 0, 0, 0, 159, 69, 0, 0, 1214, 9, 1, 1, 4181, 534, 0, 0, 0, 0, 0, 0, 0, 0, 1798, 0, 318, 0, 0, 1, 2027, 1990, 0, 0, 0, 0, 0, 0, 2261, 0, 1, 2854, 0, 907, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 4469, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 466, 1096, 5, 4416, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1, 2, 0, 0, 0, 0, 6079, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2034, 108, 70, 0, 0, 1, 3727, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6101, 6168, 6134, 6024, 6332, 6280, 6171, 5942] avg=6144.0000 max=6332 max_rank=4 min=5942 min_rank=7 imbalance_ratio=1.030599 max_load_reduction=42820 max_load_reduction_pct=0.871175 moved_to_echo=24725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 575, 0, 0, 0, 0, 3746, 1870, 0, 0, 0, 0, 0, 0, 216, 90, 0, 0, 1112, 2, 0, 0, 4186, 534, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 0, 349, 0, 0, 0, 2028, 2003, 0, 0, 0, 0, 0, 0, 2388, 0, 0, 2908, 0, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 4376, 1, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 447, 1045, 0, 4419, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 6062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2131, 48, 90, 0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6191, 6140, 6032, 6205, 6258, 6177, 6128, 6021] avg=6144.0000 max=6258 max_rank=4 min=6021 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=42894 max_load_reduction_pct=0.872681 moved_to_echo=24584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 547, 0, 0, 0, 0, 3744, 1893, 0, 0, 0, 0, 0, 0, 100, 36, 0, 0, 1093, 1, 0, 0, 4182, 540, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 288, 0, 0, 0, 2048, 2013, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 3282, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 4309, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 459, 889, 1, 4462, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 33, 35, 0, 0, 0, 3771, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 5952, 6087, 6685, 6128, 5987, 6146, 5983] avg=6144.0000 max=6685 max_rank=3 min=5952 min_rank=1 imbalance_ratio=1.088053 max_load_reduction=42467 max_load_reduction_pct=0.863993 moved_to_echo=24244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 515, 0, 0, 0, 0, 3754, 1880, 0, 0, 0, 0, 0, 0, 134, 60, 0, 0, 1196, 0, 0, 0, 4193, 535, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 0, 317, 0, 0, 0, 2032, 2003, 0, 0, 0, 0, 0, 0, 2375, 0, 0, 2741, 0, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 4400, 0, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 433, 1086, 0, 4426, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 6092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2251, 37, 60, 0, 0, 0, 3753, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6118, 6179, 6026, 6266, 6138, 6175, 6101] avg=6144.0000 max=6266 max_rank=4 min=6026 min_rank=3 imbalance_ratio=1.019857 max_load_reduction=42886 max_load_reduction_pct=0.872518 moved_to_echo=24680 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[5, 1, 0, 528, 3, 0, 2, 19, 3731, 1867, 0, 0, 0, 0, 0, 0, 164, 67, 0, 15, 1391, 15, 7, 0, 4167, 535, 0, 0, 0, 0, 0, 0, 0, 1, 1694, 0, 356, 12, 0, 4, 2019, 1956, 0, 0, 0, 0, 0, 0, 2453, 0, 0, 2393, 11, 889, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 241, 4185, 0, 1586, 1, 0, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 6, 10, 523, 1096, 8, 4394, 0, 0, 0, 0, 0, 0, 0, 0, 104, 19, 1, 2, 1, 0, 5, 6047, 0, 0, 0, 0, 0, 0, 0, 2, 17, 2414, 106, 66, 0, 0, 2, 3661, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 6361, 6042, 5756, 6027, 6363, 6179, 6268] avg=6144.0000 max=6363 max_rank=5 min=5756 min_rank=3 imbalance_ratio=1.035645 max_load_reduction=42789 max_load_reduction_pct=0.870544 moved_to_echo=24837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 513, 0, 0, 0, 0, 3755, 1881, 0, 0, 0, 0, 0, 0, 150, 64, 0, 0, 1247, 0, 0, 0, 4193, 536, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 364, 0, 0, 0, 2034, 1990, 0, 0, 0, 0, 0, 0, 2375, 0, 0, 3215, 0, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 4068, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 414, 1040, 0, 4433, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 6095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 6, 64, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6190, 6202, 6501, 5908, 6094, 6099, 6009] avg=6144.0000 max=6501 max_rank=3 min=5908 min_rank=4 imbalance_ratio=1.058105 max_load_reduction=42651 max_load_reduction_pct=0.867737 moved_to_echo=24110 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 455, 0, 0, 0, 0, 3751, 1873, 0, 0, 0, 0, 0, 0, 128, 59, 0, 0, 1128, 0, 0, 0, 4190, 533, 0, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 324, 0, 0, 0, 2021, 1996, 0, 0, 0, 0, 0, 0, 2506, 0, 0, 2894, 0, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 4310, 0, 1628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 470, 1041, 0, 4404, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 6072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 91, 59, 0, 0, 0, 3739, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6079, 6038, 6005, 6305, 6178, 6192, 6115, 6240] avg=6144.0000 max=6305 max_rank=3 min=6005 min_rank=2 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 14464, 0, 2560, 128, 0, 128, 16256, 16000, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 0, 4224, 128, 0, 128, 128, 30080, 15104, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1920, 34432, 128, 13056, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 14464, 0, 2560, 128, 0, 128, 16256, 16000, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[19456, 0, 128, 22528, 128, 7296, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1920, 34432, 128, 13056, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 18304, 512, 640, 0, 0, 128, 29952, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 2048, 128, 128, 3712, 8192, 128, 35456, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1280, 640, 0, 128, 9600, 128, 128, 128, 33536, 4352, 0, 0, 0, 0, 0, 0] @@ -7524,350 +7524,350 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3972, 74, 2391, 2323, 0, 160, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 949, 0, 0, 44, 8192, 0, 0, 0, 0, 382, 0, 0, 636, 0, 0, 135, 447, 297, 0, 0, 5878, 0, 0, 0, 0, 7641, 0, 0, 0, 5901, 0, 1, 0, 6, 0, 195, 299, 863, 91, 0, 79] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3775, 65, 2737, 2526, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 697, 0, 0, 30, 8192, 0, 0, 0, 1, 342, 0, 0, 564, 0, 0, 118, 476, 230, 0, 0, 5865, 0, 0, 0, 0, 7701, 0, 0, 0, 5998, 0, 4, 0, 2, 0, 237, 176, 857, 43, 0, 63] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3914, 126, 2486, 2258, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 1007, 0, 0, 39, 8192, 0, 0, 0, 0, 363, 0, 0, 676, 0, 0, 147, 375, 398, 0, 0, 5730, 0, 0, 0, 0, 7614, 0, 0, 0, 5780, 0, 3, 0, 14, 0, 225, 304, 951, 83, 0, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4318, 110, 2150, 2454, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 8192, 0, 688, 0, 0, 14, 8192, 0, 0, 0, 0, 277, 0, 0, 591, 0, 0, 174, 460, 336, 0, 0, 5827, 0, 0, 0, 0, 7614, 0, 0, 0, 6098, 0, 3, 0, 24, 0, 153, 160, 856, 57, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4097, 69, 2579, 2439, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 633, 0, 0, 46, 8192, 0, 0, 0, 1, 292, 0, 0, 640, 0, 0, 140, 413, 323, 0, 0, 5861, 0, 0, 0, 0, 7615, 0, 0, 0, 5665, 0, 0, 0, 11, 0, 306, 298, 861, 92, 0, 94] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4329, 88, 2249, 2295, 0, 284, 0, 0, 0, 0, 2, 1, 0, 0, 2, 0, 0, 0, 1, 8185, 0, 808, 0, 0, 49, 8188, 0, 0, 0, 1, 309, 0, 0, 636, 0, 1, 164, 406, 331, 1, 2, 5652, 0, 0, 2, 0, 7598, 0, 5, 1, 5973, 0, 8, 1, 19, 0, 241, 281, 872, 83, 1, 83] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3790, 86, 2717, 2604, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 1027, 0, 0, 59, 8192, 0, 0, 0, 1, 245, 0, 0, 590, 0, 0, 94, 439, 306, 0, 0, 5718, 0, 0, 0, 0, 7689, 0, 0, 0, 5764, 0, 7, 0, 8, 0, 247, 226, 943, 82, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1, 3, 4408, 99, 1776, 2462, 6, 340, 0, 0, 0, 4, 6, 10, 0, 5, 7, 0, 0, 0, 20, 8153, 10, 848, 0, 0, 81, 8162, 0, 2, 0, 16, 378, 3, 3, 646, 0, 12, 177, 410, 348, 16, 27, 5617, 0, 0, 20, 1, 7546, 0, 0, 1, 5891, 0, 1, 4, 34, 0, 312, 241, 845, 104, 8, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 937, 65, 2737, 2526, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6199, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 5308, 0, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 6100, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 564, 0, 0, 118, 476, 2838, 1243, 328, 0, 0, 0, 0, 0, 230, 0, 0, 5865, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 5998, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 237, 176, 857, 43, 0, 63, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6523, 6199, 6008, 6131, 5909, 6140, 6261, 5981] avg=6144.0000 max=6523 max_rank=0 min=5909 min_rank=4 imbalance_ratio=1.061686 max_load_reduction=42629 max_load_reduction_pct=0.867289 moved_to_echo=24291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 986, 74, 2391, 2323, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5308, 0, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 636, 0, 0, 135, 447, 2986, 1233, 328, 0, 0, 0, 0, 0, 297, 0, 0, 5878, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 5901, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 195, 299, 863, 91, 0, 79, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5934, 6153, 6259, 6144, 6147, 6220, 6159, 6136] avg=6144.0000 max=6259 max_rank=2 min=5934 min_rank=0 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3914, 126, 2486, 2258, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 1007, 0, 0, 39, 8192, 0, 0, 0, 0, 363, 0, 0, 676, 0, 0, 147, 375, 398, 0, 0, 5730, 0, 0, 0, 0, 7614, 0, 0, 0, 5780, 0, 3, 0, 14, 0, 225, 304, 951, 83, 0, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3790, 86, 2717, 2604, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 1027, 0, 0, 59, 8192, 0, 0, 0, 1, 245, 0, 0, 590, 0, 0, 94, 439, 306, 0, 0, 5718, 0, 0, 0, 0, 7689, 0, 0, 0, 5764, 0, 7, 0, 8, 0, 247, 226, 943, 82, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3972, 74, 2391, 2323, 0, 160, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 949, 0, 0, 44, 8192, 0, 0, 0, 0, 382, 0, 0, 636, 0, 0, 135, 447, 297, 0, 0, 5878, 0, 0, 0, 0, 7641, 0, 0, 0, 5901, 0, 1, 0, 6, 0, 195, 299, 863, 91, 0, 79] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4329, 88, 2249, 2295, 0, 284, 0, 0, 0, 0, 2, 1, 0, 0, 2, 0, 0, 0, 1, 8185, 0, 808, 0, 0, 49, 8188, 0, 0, 0, 1, 309, 0, 0, 636, 0, 1, 164, 406, 331, 1, 2, 5652, 0, 0, 2, 0, 7598, 0, 5, 1, 5973, 0, 8, 1, 19, 0, 241, 281, 872, 83, 1, 83] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3775, 65, 2737, 2526, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 697, 0, 0, 30, 8192, 0, 0, 0, 1, 342, 0, 0, 564, 0, 0, 118, 476, 230, 0, 0, 5865, 0, 0, 0, 0, 7701, 0, 0, 0, 5998, 0, 4, 0, 2, 0, 237, 176, 857, 43, 0, 63] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1017, 69, 2579, 2439, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5308, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 6100, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 640, 0, 0, 140, 413, 3080, 1229, 328, 0, 0, 0, 0, 0, 323, 0, 0, 5861, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 5665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 306, 298, 861, 92, 0, 94, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6396, 6130, 5942, 6147, 6122, 6229, 5921, 6265] avg=6144.0000 max=6396 max_rank=0 min=5921 min_rank=6 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=24537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1075, 88, 2249, 2295, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 6116, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 5303, 0, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 6097, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 636, 0, 1, 164, 406, 3254, 1226, 328, 0, 0, 0, 0, 0, 331, 1, 2, 5652, 0, 0, 2, 0, 45, 0, 0, 0, 0, 0, 0, 0, 256, 0, 5, 1, 5973, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 241, 281, 872, 83, 1, 83, 2554, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5991, 6119, 6114, 6147, 6324, 6033, 6244, 6180] avg=6144.0000 max=6324 max_rank=4 min=5991 min_rank=0 imbalance_ratio=1.029297 max_load_reduction=42828 max_load_reduction_pct=0.871338 moved_to_echo=24781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1072, 110, 2150, 2454, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 5308, 0, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 591, 0, 0, 174, 460, 3246, 1229, 328, 0, 0, 0, 0, 0, 336, 0, 0, 5827, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 6098, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 153, 160, 856, 57, 0, 70, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6108, 6129, 6008, 6114, 6305, 6208, 6357, 5923] avg=6144.0000 max=6357 max_rank=6 min=5923 min_rank=7 imbalance_ratio=1.034668 max_load_reduction=42795 max_load_reduction_pct=0.870667 moved_to_echo=24793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 972, 126, 2486, 2258, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 5308, 0, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 676, 0, 0, 147, 375, 2942, 1229, 328, 0, 0, 0, 0, 0, 398, 0, 0, 5730, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 5780, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 225, 304, 951, 83, 0, 85, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6026, 6129, 6321, 6139, 6060, 6173, 6039, 6265] avg=6144.0000 max=6321 max_rank=2 min=6026 min_rank=0 imbalance_ratio=1.028809 max_load_reduction=42831 max_load_reduction_pct=0.871399 moved_to_echo=24537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 1090, 99, 1776, 2462, 6, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 10, 0, 5, 6079, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 20, 5278, 10, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 6071, 0, 2, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 378, 3, 3, 646, 0, 12, 177, 410, 3318, 1223, 329, 0, 0, 0, 0, 0, 348, 16, 27, 5617, 0, 0, 20, 1, 51, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 1, 5891, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 312, 241, 845, 104, 8, 88, 2546, 2040, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 6104, 6163, 6170, 6499, 6080, 6141, 6218] avg=6144.0000 max=6499 max_rank=4 min=5777 min_rank=0 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=24938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 941, 86, 2717, 2604, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5308, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 6100, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 590, 0, 0, 94, 439, 2849, 1241, 328, 0, 0, 0, 0, 0, 306, 0, 0, 5718, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 5764, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 247, 226, 943, 82, 0, 75, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6397, 6189, 6337, 6160, 5786, 6069, 6030, 6184] avg=6144.0000 max=6397 max_rank=0 min=5786 min_rank=4 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24069 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 986, 74, 2391, 2323, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5308, 0, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 636, 0, 0, 135, 447, 2986, 1233, 328, 0, 0, 0, 0, 0, 297, 0, 0, 5878, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 5901, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 195, 299, 863, 91, 0, 79, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5934, 6153, 6259, 6144, 6147, 6220, 6159, 6136] avg=6144.0000 max=6259 max_rank=2 min=5934 min_rank=0 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1075, 88, 2249, 2295, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 6116, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 5303, 0, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 6097, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 636, 0, 1, 164, 406, 3254, 1226, 328, 0, 0, 0, 0, 0, 331, 1, 2, 5652, 0, 0, 2, 0, 45, 0, 0, 0, 0, 0, 0, 0, 256, 0, 5, 1, 5973, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 241, 281, 872, 83, 1, 83, 2554, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5991, 6119, 6114, 6147, 6324, 6033, 6244, 6180] avg=6144.0000 max=6324 max_rank=4 min=5991 min_rank=0 imbalance_ratio=1.029297 max_load_reduction=42828 max_load_reduction_pct=0.871338 moved_to_echo=24781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 937, 65, 2737, 2526, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6199, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 5308, 0, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 6100, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 564, 0, 0, 118, 476, 2838, 1243, 328, 0, 0, 0, 0, 0, 230, 0, 0, 5865, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 5998, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 237, 176, 857, 43, 0, 63, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6523, 6199, 6008, 6131, 5909, 6140, 6261, 5981] avg=6144.0000 max=6523 max_rank=0 min=5909 min_rank=4 imbalance_ratio=1.061686 max_load_reduction=42629 max_load_reduction_pct=0.867289 moved_to_echo=24291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 1090, 99, 1776, 2462, 6, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 10, 0, 5, 6079, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 20, 5278, 10, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 6071, 0, 2, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 378, 3, 3, 646, 0, 12, 177, 410, 3318, 1223, 329, 0, 0, 0, 0, 0, 348, 16, 27, 5617, 0, 0, 20, 1, 51, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 1, 5891, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 312, 241, 845, 104, 8, 88, 2546, 2040, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 6104, 6163, 6170, 6499, 6080, 6141, 6218] avg=6144.0000 max=6499 max_rank=4 min=5777 min_rank=0 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=24938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 1920, 2048, 7168, 640, 128, 640, 20480, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 128, 128, 4992, 0, 128, 1152, 3456, 24576, 9856, 2688, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 8192, 768, 19200, 19456, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 128, 128, 46208, 0, 0, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 0, 128, 128, 47104, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 128, 128, 4992, 0, 128, 1152, 3456, 24576, 9856, 2688, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 42496, 128, 6784, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 0, 128, 128, 47104, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 128, 128, 46208, 0, 0, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 384, 48768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[804, 0, 0, 0, 1811, 65, 0, 0, 0, 474, 0, 0, 226, 0, 7507, 8129, 6250, 5925, 0, 63, 0, 0, 0, 0, 195, 0, 0, 255, 0, 63, 7606, 1196, 264, 63, 0, 78, 0, 0, 0, 0, 1, 0, 0, 0, 0, 348, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7565, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[839, 0, 0, 0, 1812, 69, 0, 0, 0, 548, 0, 0, 293, 0, 7426, 8098, 6387, 5872, 0, 94, 0, 0, 0, 0, 170, 0, 0, 251, 0, 94, 7488, 1151, 250, 94, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7479, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[745, 0, 0, 0, 1881, 87, 0, 0, 0, 570, 0, 0, 284, 0, 7410, 8106, 6251, 5994, 1, 83, 0, 0, 1, 1, 171, 1, 0, 251, 1, 85, 7450, 1184, 253, 83, 1, 105, 2, 0, 0, 0, 2, 0, 0, 1, 0, 409, 251, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 7483, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[967, 0, 0, 0, 1533, 85, 0, 0, 0, 535, 0, 0, 268, 0, 7388, 8116, 6511, 5914, 0, 76, 0, 0, 0, 0, 193, 0, 0, 179, 0, 76, 7469, 1190, 245, 76, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7510, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[833, 0, 0, 0, 1532, 126, 0, 0, 0, 584, 0, 0, 279, 0, 7386, 8107, 6419, 5924, 0, 85, 0, 0, 0, 0, 162, 0, 0, 260, 0, 88, 7370, 1192, 260, 85, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7565, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[804, 0, 0, 0, 1811, 65, 0, 0, 0, 474, 0, 0, 226, 0, 7507, 8129, 6250, 5925, 0, 63, 0, 0, 0, 0, 195, 0, 0, 255, 0, 63, 7606, 1196, 264, 63, 0, 78, 0, 0, 0, 0, 1, 0, 0, 0, 0, 348, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7565, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[718, 0, 0, 0, 1835, 110, 0, 0, 0, 581, 0, 0, 277, 0, 7462, 8122, 6375, 5973, 0, 70, 0, 0, 0, 0, 200, 0, 0, 274, 0, 70, 7474, 1100, 275, 70, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7432, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[989, 0, 3, 1, 1751, 91, 1, 0, 8, 635, 0, 0, 306, 0, 7333, 8063, 6275, 5687, 10, 88, 0, 0, 8, 7, 167, 19, 0, 248, 18, 108, 7342, 1280, 246, 89, 4, 124, 12, 2, 0, 0, 3, 0, 4, 10, 4, 420, 255, 24, 0, 0, 0, 3, 2, 0, 9, 0, 12, 0, 7470, 0, 9, 8, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[745, 0, 0, 0, 1881, 87, 0, 0, 0, 570, 0, 0, 284, 0, 7410, 8106, 6251, 5994, 1, 83, 0, 0, 1, 1, 171, 1, 0, 251, 1, 85, 7450, 1184, 253, 83, 1, 105, 2, 0, 0, 0, 2, 0, 0, 1, 0, 409, 251, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 7483, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[833, 0, 0, 0, 1861, 74, 0, 0, 0, 554, 0, 0, 277, 0, 7421, 8112, 6489, 5737, 0, 80, 0, 0, 0, 0, 183, 0, 0, 233, 0, 85, 7497, 1095, 244, 80, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7541, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[833, 0, 0, 0, 1532, 126, 0, 0, 0, 584, 0, 0, 279, 0, 7386, 8107, 6419, 5924, 0, 85, 0, 0, 0, 0, 162, 0, 0, 260, 0, 88, 7370, 1192, 260, 85, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7565, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[804, 0, 0, 0, 1811, 65, 0, 0, 2121, 1376, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 226, 0, 5372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 5925, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 255, 0, 63, 4545, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 264, 63, 0, 78, 0, 0, 0, 0, 3661, 1970, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 348, 264, 0, 3061, 2409, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6072, 6168, 6254, 6036, 6097, 6159, 6189] avg=6144.0000 max=6254 max_rank=3 min=6036 min_rank=4 imbalance_ratio=1.017904 max_load_reduction=42898 max_load_reduction_pct=0.872762 moved_to_echo=24481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[745, 0, 0, 0, 1881, 87, 0, 0, 2094, 1361, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 284, 0, 5303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 5994, 1, 83, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 171, 1, 0, 251, 1, 85, 4451, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 253, 83, 1, 105, 2, 0, 0, 0, 3662, 1965, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 409, 251, 1, 2999, 2410, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 6141, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 6157, 6259, 6144, 6071, 6086, 6144, 6123] avg=6144.0000 max=6259 max_rank=2 min=6071 min_rank=4 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[989, 0, 3, 1, 1751, 91, 1, 0, 8, 635, 0, 0, 306, 0, 7333, 8063, 6275, 5687, 10, 88, 0, 0, 8, 7, 167, 19, 0, 248, 18, 108, 7342, 1280, 246, 89, 4, 124, 12, 2, 0, 0, 3, 0, 4, 10, 4, 420, 255, 24, 0, 0, 0, 3, 2, 0, 9, 0, 12, 0, 7470, 0, 9, 8, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[967, 0, 0, 0, 1533, 85, 0, 0, 0, 535, 0, 0, 268, 0, 7388, 8116, 6511, 5914, 0, 76, 0, 0, 0, 0, 193, 0, 0, 179, 0, 76, 7469, 1190, 245, 76, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7510, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[839, 0, 0, 0, 1812, 69, 0, 0, 2098, 1361, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 293, 0, 5315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 5872, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 251, 0, 94, 4474, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 250, 94, 0, 101, 0, 0, 0, 0, 3741, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 251, 0, 3014, 2462, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6179, 6156, 6150, 6140, 6148, 6125, 6136, 6118] avg=6144.0000 max=6179 max_rank=0 min=6118 min_rank=7 imbalance_ratio=1.005697 max_load_reduction=42973 max_load_reduction_pct=0.874288 moved_to_echo=24527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[833, 0, 0, 0, 1532, 126, 0, 0, 2087, 1376, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 279, 0, 5286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 5924, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 260, 0, 88, 4404, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 260, 85, 0, 190, 0, 0, 0, 0, 3760, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 260, 0, 2966, 2474, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5954, 6149, 6194, 6106, 6259, 6158, 6143, 6189] avg=6144.0000 max=6259 max_rank=4 min=5954 min_rank=0 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[718, 0, 0, 0, 1835, 110, 0, 0, 2108, 1352, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 277, 0, 5341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 5973, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 274, 0, 70, 4466, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 275, 70, 0, 60, 0, 0, 0, 0, 3734, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 274, 0, 3008, 2457, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6199, 6227, 6110, 6107, 6152, 6154, 6080] avg=6144.0000 max=6227 max_rank=2 min=6080 min_rank=7 imbalance_ratio=1.013509 max_load_reduction=42925 max_load_reduction_pct=0.873311 moved_to_echo=24493 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[745, 0, 0, 0, 1881, 87, 0, 0, 2094, 1361, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 284, 0, 5303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 5994, 1, 83, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 171, 1, 0, 251, 1, 85, 4451, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 253, 83, 1, 105, 2, 0, 0, 0, 3662, 1965, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 409, 251, 1, 2999, 2410, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 6141, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 6157, 6259, 6144, 6071, 6086, 6144, 6123] avg=6144.0000 max=6259 max_rank=2 min=6071 min_rank=4 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[804, 0, 0, 0, 1811, 65, 0, 0, 2121, 1376, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 226, 0, 5372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 5925, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 255, 0, 63, 4545, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 264, 63, 0, 78, 0, 0, 0, 0, 3661, 1970, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 348, 264, 0, 3061, 2409, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6072, 6168, 6254, 6036, 6097, 6159, 6189] avg=6144.0000 max=6254 max_rank=3 min=6036 min_rank=4 imbalance_ratio=1.017904 max_load_reduction=42898 max_load_reduction_pct=0.872762 moved_to_echo=24481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[833, 0, 0, 0, 1861, 74, 0, 0, 2097, 1372, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 277, 0, 5311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 5737, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 233, 0, 85, 4479, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 244, 80, 0, 105, 0, 0, 0, 0, 3801, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 245, 0, 3018, 2501, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6237, 6142, 6004, 6075, 6196, 6183, 6146, 6169] avg=6144.0000 max=6237 max_rank=0 min=6004 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=42915 max_load_reduction_pct=0.873108 moved_to_echo=24694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[967, 0, 0, 0, 1533, 85, 0, 0, 2088, 1366, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 268, 0, 5287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 5914, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 179, 0, 76, 4463, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 245, 76, 0, 121, 0, 0, 0, 0, 3814, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 244, 0, 3006, 2510, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6039, 6090, 6177, 6101, 6224, 6229, 6148, 6144] avg=6144.0000 max=6229 max_rank=5 min=6039 min_rank=0 imbalance_ratio=1.013835 max_load_reduction=42923 max_load_reduction_pct=0.873271 moved_to_echo=24745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[989, 0, 3, 1, 1751, 91, 1, 0, 2076, 1363, 0, 0, 0, 0, 0, 0, 8, 635, 0, 0, 306, 0, 5238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 5687, 10, 88, 0, 0, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 167, 19, 0, 248, 18, 108, 4382, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 246, 89, 4, 124, 12, 2, 0, 0, 3679, 1955, 0, 0, 0, 0, 0, 0, 3, 0, 4, 10, 4, 420, 255, 24, 2960, 2423, 19, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 9, 0, 6108, 0, 0, 0, 0, 0, 0, 0, 12, 0, 6107, 0, 9, 8, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6275, 6187, 5973, 6222, 6111, 6122, 6122, 6140] avg=6144.0000 max=6275 max_rank=0 min=5973 min_rank=2 imbalance_ratio=1.021322 max_load_reduction=42877 max_load_reduction_pct=0.872335 moved_to_echo=24495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[833, 0, 0, 0, 1532, 126, 0, 0, 2087, 1376, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 279, 0, 5286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 5924, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 260, 0, 88, 4404, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 260, 85, 0, 190, 0, 0, 0, 0, 3760, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 260, 0, 2966, 2474, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5954, 6149, 6194, 6106, 6259, 6158, 6143, 6189] avg=6144.0000 max=6259 max_rank=4 min=5954 min_rank=0 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[967, 0, 0, 0, 1533, 85, 0, 0, 2088, 1366, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 268, 0, 5287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 5914, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 179, 0, 76, 4463, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 245, 76, 0, 121, 0, 0, 0, 0, 3814, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 244, 0, 3006, 2510, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6039, 6090, 6177, 6101, 6224, 6229, 6148, 6144] avg=6144.0000 max=6229 max_rank=5 min=6039 min_rank=0 imbalance_ratio=1.013835 max_load_reduction=42923 max_load_reduction_pct=0.873271 moved_to_echo=24745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6784, 0, 128, 128, 14080, 768, 128, 0, 16896, 11008, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 640, 128, 896, 128, 128, 0, 0, 29952, 15744, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 128, 0, 2048, 128, 768, 35712, 9472, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 4608, 0, 0, 2304, 0, 42496, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1536, 47104, 128, 640, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 49152, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 3328, 2048, 128, 24064, 19712, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 49152, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1536, 47104, 128, 640, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 788, 0, 0, 0, 63, 0, 0, 789, 0, 0, 0, 759, 0, 6465, 693, 0, 0, 7506, 0, 0, 0, 7156, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 7767, 0, 0, 0, 7882, 0, 178, 168, 0, 0, 0, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 153, 0, 0, 0, 0, 737, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 895, 0, 0, 0, 48, 0, 0, 774, 0, 0, 0, 829, 0, 6270, 764, 0, 0, 7546, 0, 0, 0, 7178, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 0, 0, 0, 7784, 0, 130, 186, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 140, 0, 0, 0, 0, 767, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 768, 0, 0, 0, 20, 0, 0, 808, 0, 0, 0, 800, 0, 6354, 759, 0, 0, 7623, 0, 0, 0, 7173, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 0, 0, 0, 7881, 0, 110, 128, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 98, 0, 0, 0, 0, 722, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[6, 3, 0, 842, 0, 0, 0, 87, 0, 0, 777, 11, 7, 26, 808, 6, 6218, 716, 18, 0, 7488, 0, 3, 1, 7065, 0, 0, 178, 2, 2, 0, 4, 0, 0, 5, 0, 7765, 1, 0, 2, 7758, 20, 163, 308, 1, 0, 0, 24, 1, 6, 0, 2, 7884, 16, 8, 0, 153, 0, 4, 0, 0, 762, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 840, 0, 0, 0, 72, 0, 0, 868, 0, 0, 0, 811, 0, 6361, 738, 0, 0, 7506, 0, 0, 0, 7074, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 7808, 0, 0, 0, 7849, 0, 132, 144, 0, 0, 0, 0, 0, 0, 0, 0, 7922, 0, 0, 0, 175, 0, 0, 0, 0, 761, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 812, 0, 0, 0, 52, 0, 0, 832, 1, 0, 2, 759, 0, 6373, 706, 2, 0, 7527, 0, 0, 2, 7179, 0, 0, 110, 0, 0, 0, 1, 0, 0, 0, 0, 7787, 0, 0, 0, 7838, 2, 168, 133, 0, 0, 0, 1, 2, 1, 0, 1, 7944, 1, 0, 0, 152, 0, 0, 0, 3, 761, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 863, 0, 0, 0, 37, 0, 0, 765, 0, 0, 0, 781, 0, 6352, 708, 0, 0, 7550, 0, 0, 0, 7166, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 7801, 0, 0, 0, 7789, 0, 161, 185, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 140, 0, 0, 0, 0, 778, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 788, 0, 0, 0, 63, 0, 0, 789, 0, 0, 0, 759, 0, 6465, 693, 0, 0, 7506, 0, 0, 0, 7156, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 7767, 0, 0, 0, 7882, 0, 178, 168, 0, 0, 0, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 153, 0, 0, 0, 0, 737, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 702, 0, 0, 0, 52, 0, 0, 848, 0, 0, 0, 786, 0, 6299, 727, 0, 0, 7561, 0, 0, 0, 7101, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 7813, 0, 0, 0, 7980, 0, 151, 112, 0, 0, 0, 0, 0, 0, 0, 0, 7977, 0, 0, 0, 139, 0, 0, 0, 0, 828, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 895, 0, 0, 0, 48, 0, 0, 774, 0, 0, 0, 829, 0, 6270, 764, 0, 0, 7546, 0, 0, 0, 7178, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 0, 0, 0, 7784, 0, 130, 186, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 140, 0, 0, 0, 0, 767, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 768, 0, 0, 0, 20, 5334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 808, 0, 0, 0, 800, 0, 1651, 1101, 924, 878, 0, 0, 0, 0, 5430, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6072, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5845, 0, 110, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 722, 0, 0, 2289, 2036, 954, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6162, 6189, 6143, 6153, 6083, 6201, 6099] avg=6144.0000 max=6201 max_rank=6 min=6083 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=42951 max_load_reduction_pct=0.873840 moved_to_echo=24536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 812, 0, 0, 0, 52, 5268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 1, 0, 2, 759, 0, 1648, 1102, 927, 869, 0, 0, 0, 0, 5446, 706, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6077, 0, 0, 110, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5814, 2, 168, 133, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 6132, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 3, 761, 0, 0, 2259, 2024, 943, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6140, 6156, 6188, 6139, 6118, 6137, 6142] avg=6144.0000 max=6188 max_rank=3 min=6118 min_rank=5 imbalance_ratio=1.007161 max_load_reduction=42964 max_load_reduction_pct=0.874105 moved_to_echo=24536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 840, 0, 0, 0, 72, 0, 0, 868, 0, 0, 0, 811, 0, 6361, 738, 0, 0, 7506, 0, 0, 0, 7074, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 7808, 0, 0, 0, 7849, 0, 132, 144, 0, 0, 0, 0, 0, 0, 0, 0, 7922, 0, 0, 0, 175, 0, 0, 0, 0, 761, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 812, 0, 0, 0, 52, 0, 0, 832, 1, 0, 2, 759, 0, 6373, 706, 2, 0, 7527, 0, 0, 2, 7179, 0, 0, 110, 0, 0, 0, 1, 0, 0, 0, 0, 7787, 0, 0, 0, 7838, 2, 168, 133, 0, 0, 0, 1, 2, 1, 0, 1, 7944, 1, 0, 0, 152, 0, 0, 0, 3, 761, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[6, 3, 0, 842, 0, 0, 0, 87, 0, 0, 777, 11, 7, 26, 808, 6, 6218, 716, 18, 0, 7488, 0, 3, 1, 7065, 0, 0, 178, 2, 2, 0, 4, 0, 0, 5, 0, 7765, 1, 0, 2, 7758, 20, 163, 308, 1, 0, 0, 24, 1, 6, 0, 2, 7884, 16, 8, 0, 153, 0, 4, 0, 0, 762, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 788, 0, 0, 0, 63, 5252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 789, 0, 0, 0, 759, 0, 1644, 1099, 940, 869, 0, 0, 0, 0, 5525, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5846, 0, 178, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 737, 0, 0, 2254, 2036, 943, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6103, 6100, 6218, 6162, 6123, 6192, 6131, 6123] avg=6144.0000 max=6218 max_rank=2 min=6100 min_rank=1 imbalance_ratio=1.012044 max_load_reduction=42934 max_load_reduction_pct=0.873494 moved_to_echo=24569 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 863, 0, 0, 0, 37, 5284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 0, 0, 781, 0, 1651, 1100, 924, 874, 0, 0, 0, 0, 5428, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6066, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5777, 0, 161, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 778, 0, 0, 2266, 2012, 949, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 6095, 6136, 6150, 6150, 6123, 6169, 6145] avg=6144.0000 max=6184 max_rank=0 min=6095 min_rank=1 imbalance_ratio=1.006510 max_load_reduction=42968 max_load_reduction_pct=0.874186 moved_to_echo=24587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 895, 0, 0, 0, 48, 5280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 0, 0, 829, 0, 1649, 1102, 912, 868, 0, 0, 0, 0, 5358, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6076, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5774, 0, 130, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 767, 0, 0, 2266, 2010, 943, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6223, 6134, 6122, 6182, 6145, 6090, 6130, 6126] avg=6144.0000 max=6223 max_rank=0 min=6090 min_rank=5 imbalance_ratio=1.012858 max_load_reduction=42929 max_load_reduction_pct=0.873393 moved_to_echo=24491 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 3, 0, 842, 0, 0, 0, 87, 5240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 11, 7, 26, 808, 6, 1647, 1090, 909, 867, 0, 0, 0, 0, 5309, 716, 18, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5975, 0, 0, 178, 2, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6118, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5750, 20, 163, 308, 1, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 2, 6078, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 4, 0, 0, 762, 1, 0, 2248, 2008, 939, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6148, 6047, 6161, 6126, 6266, 6111, 6115] avg=6144.0000 max=6266 max_rank=5 min=6047 min_rank=2 imbalance_ratio=1.019857 max_load_reduction=42886 max_load_reduction_pct=0.872518 moved_to_echo=24618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 768, 0, 0, 0, 20, 5334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 808, 0, 0, 0, 800, 0, 1651, 1101, 924, 878, 0, 0, 0, 0, 5430, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6072, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5845, 0, 110, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 722, 0, 0, 2289, 2036, 954, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6162, 6189, 6143, 6153, 6083, 6201, 6099] avg=6144.0000 max=6201 max_rank=6 min=6083 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=42951 max_load_reduction_pct=0.873840 moved_to_echo=24536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 840, 0, 0, 0, 72, 5252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 0, 0, 0, 811, 0, 1652, 1086, 925, 866, 0, 0, 0, 0, 5436, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5988, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5822, 0, 132, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 761, 0, 0, 2254, 2027, 941, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 6208, 6174, 6079, 6156, 6098, 6115, 6158] avg=6144.0000 max=6208 max_rank=1 min=6079 min_rank=3 imbalance_ratio=1.010417 max_load_reduction=42944 max_load_reduction_pct=0.873698 moved_to_echo=24527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 812, 0, 0, 0, 52, 5268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 1, 0, 2, 759, 0, 1648, 1102, 927, 869, 0, 0, 0, 0, 5446, 706, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6077, 0, 0, 110, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5814, 2, 168, 133, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 6132, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 3, 761, 0, 0, 2259, 2024, 943, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6140, 6156, 6188, 6139, 6118, 6137, 6142] avg=6144.0000 max=6188 max_rank=3 min=6118 min_rank=5 imbalance_ratio=1.007161 max_load_reduction=42964 max_load_reduction_pct=0.874105 moved_to_echo=24536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 863, 0, 0, 0, 37, 5284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 0, 0, 781, 0, 1651, 1100, 924, 874, 0, 0, 0, 0, 5428, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6066, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5777, 0, 161, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 778, 0, 0, 2266, 2012, 949, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 6095, 6136, 6150, 6150, 6123, 6169, 6145] avg=6144.0000 max=6184 max_rank=0 min=6095 min_rank=1 imbalance_ratio=1.006510 max_load_reduction=42968 max_load_reduction_pct=0.874186 moved_to_echo=24587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 702, 0, 0, 0, 52, 5292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 0, 786, 0, 1653, 1090, 916, 872, 0, 0, 0, 0, 5383, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6011, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5919, 0, 151, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 828, 0, 0, 2269, 2061, 947, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 6165, 6110, 6087, 6160, 6182, 6158, 6244] avg=6144.0000 max=6244 max_rank=7 min=6046 min_rank=0 imbalance_ratio=1.016276 max_load_reduction=42908 max_load_reduction_pct=0.872965 moved_to_echo=24744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 3, 0, 842, 0, 0, 0, 87, 5240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 11, 7, 26, 808, 6, 1647, 1090, 909, 867, 0, 0, 0, 0, 5309, 716, 18, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5975, 0, 0, 178, 2, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6118, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5750, 20, 163, 308, 1, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 2, 6078, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 4, 0, 0, 762, 1, 0, 2248, 2008, 939, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6148, 6047, 6161, 6126, 6266, 6111, 6115] avg=6144.0000 max=6266 max_rank=5 min=6047 min_rank=2 imbalance_ratio=1.019857 max_load_reduction=42886 max_load_reduction_pct=0.872518 moved_to_echo=24618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 0, 6528, 128, 128, 128, 6400, 128, 13312, 8832, 7424, 7040, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 0, 128, 0, 128, 6144, 128, 0, 18176, 16256, 7680, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 6528, 0, 0, 0, 512, 42240, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 49152, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[46592, 128, 1280, 1408, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 49152, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[48384, 0, 0, 896, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[43392, 5888, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[48384, 0, 0, 896, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[59, 0, 0, 63, 0, 0, 0, 0, 7803, 0, 0, 0, 0, 0, 0, 0, 1855, 0, 8191, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 1, 0, 0, 1571, 0, 1137, 249, 4037, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6910, 17, 0, 0, 0, 0, 571, 8064, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[94, 0, 0, 94, 0, 0, 0, 0, 7760, 0, 0, 0, 0, 0, 0, 0, 1772, 0, 8192, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 94, 0, 1, 0, 0, 0, 0, 0, 0, 1437, 0, 1157, 330, 4041, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6833, 10, 0, 0, 0, 0, 562, 8029, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[85, 0, 0, 85, 0, 0, 0, 0, 7764, 0, 0, 0, 0, 0, 0, 0, 1811, 0, 8192, 0, 0, 0, 0, 0, 541, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 0, 1261, 355, 3703, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6945, 11, 0, 0, 0, 0, 640, 7981, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[87, 0, 28, 87, 6, 18, 22, 3, 7724, 0, 0, 0, 3, 5, 0, 4, 1524, 1, 8164, 11, 2, 1, 12, 0, 518, 0, 0, 10, 2, 88, 0, 16, 17, 0, 0, 0, 0, 9, 1744, 0, 1120, 339, 4142, 8157, 0, 15, 0, 0, 6, 3, 0, 0, 0, 6, 6643, 11, 6, 22, 0, 10, 608, 7947, 1, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[80, 0, 0, 80, 0, 0, 0, 0, 7788, 0, 0, 0, 0, 0, 0, 0, 1797, 0, 8192, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 1542, 0, 1092, 323, 4026, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6836, 6, 0, 0, 0, 0, 593, 8038, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[74, 0, 0, 75, 0, 0, 0, 0, 7808, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 8192, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 1242, 332, 3873, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6861, 9, 0, 0, 0, 0, 668, 8031, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[83, 0, 2, 83, 2, 0, 1, 0, 7771, 0, 1, 0, 0, 1, 0, 1, 1841, 0, 8189, 0, 1, 0, 1, 0, 447, 0, 0, 0, 0, 83, 0, 0, 2, 0, 0, 0, 0, 1, 1564, 0, 1294, 339, 3899, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6750, 7, 0, 2, 0, 0, 577, 8019, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[94, 0, 0, 94, 0, 0, 0, 0, 7760, 0, 0, 0, 0, 0, 0, 0, 1772, 0, 8192, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 94, 0, 1, 0, 0, 0, 0, 0, 0, 1437, 0, 1157, 330, 4041, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6833, 10, 0, 0, 0, 0, 562, 8029, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[59, 0, 0, 63, 0, 0, 0, 0, 7803, 0, 0, 0, 0, 0, 0, 0, 1855, 0, 8191, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 1, 0, 0, 1571, 0, 1137, 249, 4037, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6910, 17, 0, 0, 0, 0, 571, 8064, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[70, 0, 0, 69, 1, 0, 0, 0, 7778, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 8191, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 70, 0, 23, 0, 0, 0, 0, 0, 0, 1473, 0, 1245, 289, 4204, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6786, 8, 0, 0, 0, 0, 624, 8012, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[59, 0, 0, 63, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 63, 0, 0, 3813, 1574, 195, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1571, 0, 2302, 1638, 697, 0, 0, 0, 0, 0, 1137, 249, 4037, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6213, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 5567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6165, 6233, 6014, 6209, 6063, 6230, 6138] avg=6144.0000 max=6233 max_rank=2 min=6014 min_rank=3 imbalance_ratio=1.014486 max_load_reduction=42919 max_load_reduction_pct=0.873189 moved_to_echo=24640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[74, 0, 0, 75, 0, 0, 0, 0, 7808, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 8192, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 1242, 332, 3873, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6861, 9, 0, 0, 0, 0, 668, 8031, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[87, 0, 28, 87, 6, 18, 22, 3, 7724, 0, 0, 0, 3, 5, 0, 4, 1524, 1, 8164, 11, 2, 1, 12, 0, 518, 0, 0, 10, 2, 88, 0, 16, 17, 0, 0, 0, 0, 9, 1744, 0, 1120, 339, 4142, 8157, 0, 15, 0, 0, 6, 3, 0, 0, 0, 6, 6643, 11, 6, 22, 0, 10, 608, 7947, 1, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[94, 0, 0, 94, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 94, 0, 1, 3814, 1574, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1437, 0, 2292, 1629, 689, 0, 0, 0, 0, 0, 1157, 330, 4041, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 5543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6131, 6150, 6231, 6047, 6168, 6154, 6105] avg=6144.0000 max=6231 max_rank=3 min=6047 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=42921 max_load_reduction_pct=0.873230 moved_to_echo=24474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[80, 0, 0, 80, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 80, 0, 0, 3814, 1574, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1542, 0, 2295, 1635, 689, 0, 0, 0, 0, 0, 1092, 323, 4026, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 5549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6153, 6175, 6149, 6161, 6081, 6153, 6142] avg=6144.0000 max=6175 max_rank=2 min=6081 min_rank=5 imbalance_ratio=1.005046 max_load_reduction=42977 max_load_reduction_pct=0.874369 moved_to_echo=24537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[70, 0, 0, 69, 1, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 70, 0, 23, 3813, 1574, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 2287, 1633, 684, 0, 0, 0, 0, 0, 1245, 289, 4204, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6102, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 624, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 6145, 6063, 6106, 6077, 6378, 6110, 6155] avg=6144.0000 max=6378 max_rank=5 min=6063 min_rank=2 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=24720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[87, 0, 28, 87, 6, 18, 22, 3, 5956, 0, 0, 0, 0, 0, 0, 0, 6100, 0, 0, 0, 3, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1524, 1, 4359, 11, 2, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 10, 2, 88, 0, 16, 3805, 1573, 197, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 9, 1744, 0, 2272, 1624, 674, 0, 0, 0, 0, 0, 1120, 339, 4142, 628, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 6, 5969, 11, 0, 0, 0, 0, 0, 0, 0, 0, 6, 22, 0, 10, 608, 5478, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6207, 6112, 5910, 6209, 6340, 6244, 5995, 6135] avg=6144.0000 max=6340 max_rank=4 min=5910 min_rank=2 imbalance_ratio=1.031901 max_load_reduction=42812 max_load_reduction_pct=0.871012 moved_to_echo=24714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[83, 0, 2, 83, 2, 0, 1, 0, 5977, 0, 0, 0, 0, 0, 0, 0, 6140, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1841, 0, 4377, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 83, 0, 0, 3812, 1574, 194, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1564, 0, 2289, 1631, 681, 0, 0, 0, 0, 0, 1294, 339, 3899, 639, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6069, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 577, 5536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6143, 6220, 6110, 6168, 6172, 6076, 6115] avg=6144.0000 max=6220 max_rank=2 min=6076 min_rank=6 imbalance_ratio=1.012370 max_load_reduction=42932 max_load_reduction_pct=0.873454 moved_to_echo=24531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[59, 0, 0, 63, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 63, 0, 0, 3813, 1574, 195, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1571, 0, 2302, 1638, 697, 0, 0, 0, 0, 0, 1137, 249, 4037, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6213, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 5567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6165, 6233, 6014, 6209, 6063, 6230, 6138] avg=6144.0000 max=6233 max_rank=2 min=6014 min_rank=3 imbalance_ratio=1.014486 max_load_reduction=42919 max_load_reduction_pct=0.873189 moved_to_echo=24640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[85, 0, 0, 85, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1811, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 0, 0, 0, 0, 85, 0, 0, 3814, 1574, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 0, 2278, 1630, 700, 0, 0, 0, 0, 0, 1261, 355, 3703, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6245, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 5510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6134, 6189, 6207, 6109, 5959, 6256, 6150] avg=6144.0000 max=6256 max_rank=6 min=5959 min_rank=5 imbalance_ratio=1.018229 max_load_reduction=42896 max_load_reduction_pct=0.872721 moved_to_echo=24474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[70, 0, 0, 69, 1, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 70, 0, 23, 3813, 1574, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 2287, 1633, 684, 0, 0, 0, 0, 0, 1245, 289, 4204, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6102, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 624, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 6145, 6063, 6106, 6077, 6378, 6110, 6155] avg=6144.0000 max=6378 max_rank=5 min=6063 min_rank=2 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=24720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[80, 0, 0, 80, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 80, 0, 0, 3814, 1574, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1542, 0, 2295, 1635, 689, 0, 0, 0, 0, 0, 1092, 323, 4026, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 5549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6153, 6175, 6149, 6161, 6081, 6153, 6142] avg=6144.0000 max=6175 max_rank=2 min=6081 min_rank=5 imbalance_ratio=1.005046 max_load_reduction=42977 max_load_reduction_pct=0.874369 moved_to_echo=24537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[74, 0, 0, 75, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 76, 0, 0, 3814, 1574, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 2293, 1639, 692, 0, 0, 0, 0, 0, 1242, 332, 3873, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6169, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 5544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6169, 6212, 6126, 6041, 6087, 6178, 6212] avg=6144.0000 max=6212 max_rank=2 min=6041 min_rank=4 imbalance_ratio=1.011068 max_load_reduction=42940 max_load_reduction_pct=0.873617 moved_to_echo=24518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[83, 0, 2, 83, 2, 0, 1, 0, 5977, 0, 0, 0, 0, 0, 0, 0, 6140, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1841, 0, 4377, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 83, 0, 0, 3812, 1574, 194, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1564, 0, 2289, 1631, 681, 0, 0, 0, 0, 0, 1294, 339, 3899, 639, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6069, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 577, 5536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6143, 6220, 6110, 6168, 6172, 6076, 6115] avg=6144.0000 max=6220 max_rank=2 min=6076 min_rank=6 imbalance_ratio=1.012370 max_load_reduction=42932 max_load_reduction_pct=0.873454 moved_to_echo=24531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[87, 0, 28, 87, 6, 18, 22, 3, 5956, 0, 0, 0, 0, 0, 0, 0, 6100, 0, 0, 0, 3, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1524, 1, 4359, 11, 2, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 10, 2, 88, 0, 16, 3805, 1573, 197, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 9, 1744, 0, 2272, 1624, 674, 0, 0, 0, 0, 0, 1120, 339, 4142, 628, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 6, 5969, 11, 0, 0, 0, 0, 0, 0, 0, 0, 6, 22, 0, 10, 608, 5478, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6207, 6112, 5910, 6209, 6340, 6244, 5995, 6135] avg=6144.0000 max=6340 max_rank=4 min=5910 min_rank=2 imbalance_ratio=1.031901 max_load_reduction=42812 max_load_reduction_pct=0.871012 moved_to_echo=24714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 128, 12288, 0, 18432, 13184, 5632, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[14208, 128, 35072, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[9600, 2560, 32000, 5120, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 0, 0, 128, 128, 640, 0, 128, 30592, 12672, 1664, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 4864, 44288, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 0, 128, 640, 128, 128, 128, 128, 47872, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[49152, 0, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 0, 128, 640, 128, 128, 128, 128, 47872, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 302, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 7521, 0, 0, 0, 3505, 0, 0, 0, 0, 8066, 0, 0, 0, 35, 4414, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6095, 8192, 2493, 4669, 0, 0, 0, 0, 559, 0, 2979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 313, 0, 0, 247, 0, 1, 0, 0, 0, 0, 0, 7469, 0, 0, 0, 3096, 0, 2, 1, 0, 8024, 0, 0, 0, 69, 4564, 2, 0, 0, 0, 2, 0, 0, 20, 0, 2, 0, 6133, 8190, 2489, 4583, 0, 0, 0, 0, 652, 0, 3219, 0, 0, 0, 0, 0, 2, 0, 0, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 338, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 7496, 0, 0, 0, 3232, 0, 0, 0, 0, 8004, 0, 0, 0, 59, 4282, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6437, 8192, 2461, 4533, 0, 0, 0, 0, 672, 0, 3099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 332, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 7478, 0, 0, 0, 3098, 0, 0, 0, 0, 8052, 0, 0, 0, 70, 4790, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 6026, 8192, 2412, 4646, 0, 0, 0, 0, 595, 0, 3117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 2, 0, 337, 0, 21, 254, 7, 0, 2, 0, 0, 0, 11, 7399, 0, 0, 3, 3436, 1, 5, 3, 2, 7969, 2, 1, 8, 66, 4399, 30, 32, 2, 0, 19, 0, 0, 15, 12, 20, 4, 6222, 8161, 2400, 5087, 7, 4, 0, 4, 684, 0, 2434, 0, 0, 0, 0, 0, 29, 0, 2, 0, 53] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 343, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 7529, 0, 0, 0, 3246, 0, 0, 0, 0, 8022, 0, 0, 0, 90, 4292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 8192, 2854, 4484, 0, 0, 0, 0, 582, 0, 3042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 338, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 7496, 0, 0, 0, 3232, 0, 0, 0, 0, 8004, 0, 0, 0, 59, 4282, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6437, 8192, 2461, 4533, 0, 0, 0, 0, 672, 0, 3099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 301, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 7486, 0, 0, 0, 3367, 0, 0, 0, 0, 8032, 0, 0, 0, 60, 4371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6086, 8192, 2568, 4697, 0, 0, 0, 0, 541, 0, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 302, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 7521, 0, 0, 0, 3505, 0, 0, 0, 0, 8066, 0, 0, 0, 35, 4414, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6095, 8192, 2493, 4669, 0, 0, 0, 0, 559, 0, 2979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 2, 0, 337, 0, 21, 254, 7, 0, 2, 0, 0, 0, 11, 7399, 0, 0, 3, 3436, 1, 5, 3, 2, 7969, 2, 1, 8, 66, 4399, 30, 32, 2, 0, 19, 0, 0, 15, 12, 20, 4, 6222, 8161, 2400, 5087, 7, 4, 0, 4, 684, 0, 2434, 0, 0, 0, 0, 0, 29, 0, 2, 0, 53] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 315, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 7568, 0, 0, 0, 3832, 0, 0, 0, 0, 8040, 0, 0, 0, 64, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6235, 8192, 2607, 4381, 0, 1, 0, 0, 637, 0, 3109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 313, 0, 0, 247, 0, 1, 0, 0, 0, 0, 0, 7469, 0, 0, 0, 3096, 0, 2, 1, 0, 8024, 0, 0, 0, 69, 4564, 2, 0, 0, 0, 2, 0, 0, 20, 0, 2, 0, 6133, 8190, 2489, 4583, 0, 0, 0, 0, 652, 0, 3219, 0, 0, 0, 0, 0, 2, 0, 0, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 301, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 7486, 0, 0, 0, 3367, 0, 0, 0, 0, 8032, 0, 0, 0, 60, 4371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6086, 8192, 2568, 4697, 0, 0, 0, 0, 541, 0, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 338, 0, 3224, 2706, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 3731, 2295, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 0, 0, 0, 59, 4282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 3099, 0, 0, 1481, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4004, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6268, 6288, 6023, 6046, 6133, 5985, 6261, 6148] avg=6144.0000 max=6288 max_rank=1 min=5985 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=42864 max_load_reduction_pct=0.872070 moved_to_echo=24527 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 343, 0, 3238, 2602, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 3587, 2300, 0, 0, 0, 0, 0, 0, 0, 2804, 0, 0, 0, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 0, 0, 0, 90, 4292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2854, 3486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 0, 3042, 0, 0, 1487, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4013, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6134, 6050, 6091, 6129, 6340, 6109, 6116] avg=6144.0000 max=6340 max_rank=5 min=6050 min_rank=2 imbalance_ratio=1.031901 max_load_reduction=42812 max_load_reduction_pct=0.871012 moved_to_echo=24694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 332, 0, 3216, 2533, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 3493, 2309, 0, 0, 0, 0, 0, 0, 0, 2785, 0, 0, 0, 3098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 0, 0, 0, 70, 4790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 3117, 0, 0, 1477, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4028, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6070, 5883, 6575, 6144, 6024, 6223, 6152] avg=6144.0000 max=6575 max_rank=3 min=5883 min_rank=2 imbalance_ratio=1.070150 max_load_reduction=42577 max_load_reduction_pct=0.866231 moved_to_echo=24543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 302, 0, 3234, 2562, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 3533, 2313, 0, 0, 0, 0, 0, 0, 0, 2802, 0, 0, 0, 3505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 0, 0, 0, 35, 4414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2493, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 2979, 0, 0, 1485, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4035, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6098, 6103, 6307, 6167, 6130, 6123, 6062, 6162] avg=6144.0000 max=6307 max_rank=2 min=6062 min_rank=6 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24477 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 313, 0, 3212, 2579, 0, 0, 0, 0, 0, 0, 0, 247, 0, 1, 0, 0, 0, 0, 3554, 2301, 0, 0, 0, 0, 0, 0, 0, 2782, 0, 0, 0, 3096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1709, 0, 0, 0, 69, 4564, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 20, 6128, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2489, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 3219, 0, 0, 1475, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 70, 4014, 2062, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 6103, 5880, 6343, 6152, 6054, 6366, 6148] avg=6144.0000 max=6366 max_rank=6 min=5880 min_rank=2 imbalance_ratio=1.036133 max_load_reduction=42786 max_load_reduction_pct=0.870483 moved_to_echo=24720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 332, 0, 3216, 2533, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 3493, 2309, 0, 0, 0, 0, 0, 0, 0, 2785, 0, 0, 0, 3098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 0, 0, 0, 70, 4790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 3117, 0, 0, 1477, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4028, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6070, 5883, 6575, 6144, 6024, 6223, 6152] avg=6144.0000 max=6575 max_rank=3 min=5883 min_rank=2 imbalance_ratio=1.070150 max_load_reduction=42577 max_load_reduction_pct=0.866231 moved_to_echo=24543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 338, 0, 3224, 2706, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 3731, 2295, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 0, 0, 0, 59, 4282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 3099, 0, 0, 1481, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4004, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6268, 6288, 6023, 6046, 6133, 5985, 6261, 6148] avg=6144.0000 max=6288 max_rank=1 min=5985 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=42864 max_load_reduction_pct=0.872070 moved_to_echo=24527 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 315, 0, 3255, 2622, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 3613, 2305, 0, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, 0, 0, 64, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 637, 0, 3109, 0, 0, 1495, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4022, 2062, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6192, 6184, 6650, 5621, 6130, 6013, 6217, 6145] avg=6144.0000 max=6650 max_rank=2 min=5621 min_rank=3 imbalance_ratio=1.082357 max_load_reduction=42502 max_load_reduction_pct=0.864705 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 301, 0, 3219, 2559, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 3527, 2303, 0, 0, 0, 0, 0, 0, 0, 2788, 0, 0, 0, 3367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 0, 0, 60, 4371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2568, 3652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 0, 3121, 0, 0, 1479, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 4018, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6079, 6091, 6155, 6142, 6129, 6220, 6186, 6150] avg=6144.0000 max=6220 max_rank=5 min=6079 min_rank=0 imbalance_ratio=1.012370 max_load_reduction=42932 max_load_reduction_pct=0.873454 moved_to_echo=24685 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 2, 0, 337, 0, 3186, 2617, 0, 0, 0, 0, 0, 0, 21, 254, 7, 0, 2, 0, 0, 0, 3605, 2290, 0, 0, 0, 0, 0, 0, 11, 2748, 0, 0, 3, 3436, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1686, 2, 1, 8, 66, 4399, 0, 0, 0, 0, 0, 0, 0, 0, 30, 32, 2, 0, 19, 0, 0, 15, 6107, 0, 0, 0, 0, 0, 0, 0, 12, 20, 4, 0, 0, 2400, 3950, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 684, 0, 2434, 0, 0, 1465, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 2, 0, 53, 3993, 2054, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6179, 6204, 6167, 6205, 6393, 5728, 6131] avg=6144.0000 max=6393 max_rank=5 min=5728 min_rank=6 imbalance_ratio=1.040527 max_load_reduction=42759 max_load_reduction_pct=0.869934 moved_to_echo=24457 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 315, 0, 3255, 2622, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 3613, 2305, 0, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, 0, 0, 64, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 637, 0, 3109, 0, 0, 1495, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4022, 2062, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6192, 6184, 6650, 5621, 6130, 6013, 6217, 6145] avg=6144.0000 max=6650 max_rank=2 min=5621 min_rank=3 imbalance_ratio=1.082357 max_load_reduction=42502 max_load_reduction_pct=0.864705 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 343, 0, 3238, 2602, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 3587, 2300, 0, 0, 0, 0, 0, 0, 0, 2804, 0, 0, 0, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 0, 0, 0, 90, 4292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2854, 3486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 0, 3042, 0, 0, 1487, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4013, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6134, 6050, 6091, 6129, 6340, 6109, 6116] avg=6144.0000 max=6340 max_rank=5 min=6050 min_rank=2 imbalance_ratio=1.031901 max_load_reduction=42812 max_load_reduction_pct=0.871012 moved_to_echo=24694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 128, 2688, 0, 25856, 20864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 22400, 0, 0, 128, 26880, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 128, 4992, 0, 24192, 0, 0, 11904, 8320, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 13696, 128, 128, 128, 640, 35072, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 22400, 0, 0, 128, 26880, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 20352, 28928, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 128, 0, 512, 32128, 16512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2176, 128, 128, 128, 0, 0, 0, 28672, 18432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 128, 0, 512, 32128, 16512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 128, 0, 0, 128, 49024, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 7271, 0, 0, 1262, 0, 0, 0, 0, 0, 6584, 0, 0, 0, 0, 0, 138, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 6720, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 7296, 0, 0, 1136, 0, 0, 0, 0, 0, 6545, 0, 0, 0, 0, 0, 162, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 6488, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[2, 1, 1, 0, 0, 0, 1, 0, 0, 0, 93, 0, 7216, 10, 33, 1364, 2, 0, 1, 0, 12, 6453, 3, 12, 1, 0, 0, 320, 0, 32, 8156, 0, 0, 0, 1, 0, 0, 8154, 2, 3, 0, 0, 2498, 0, 4, 32, 33, 0, 8, 49, 0, 0, 0, 0, 23, 4, 8, 138, 0, 3, 6322, 8154, 2, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 7325, 0, 0, 1394, 0, 0, 0, 0, 0, 6553, 0, 0, 0, 0, 0, 189, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 6333, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 83, 0, 7329, 1, 2, 1261, 0, 0, 1, 0, 1, 6534, 0, 0, 0, 0, 0, 213, 0, 2, 8189, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2374, 0, 0, 2, 2, 0, 0, 2, 0, 0, 0, 0, 2, 0, 1, 197, 0, 0, 6577, 8186, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 7280, 0, 0, 1226, 0, 0, 0, 0, 0, 6489, 0, 0, 0, 0, 0, 231, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 6696, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 7309, 0, 0, 1251, 0, 0, 0, 0, 0, 6537, 0, 0, 0, 0, 0, 232, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 6566, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 7296, 0, 0, 1136, 0, 0, 0, 0, 0, 6545, 0, 0, 0, 0, 0, 162, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 6488, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 83, 0, 7329, 1, 2, 1261, 0, 0, 1, 0, 1, 6534, 0, 0, 0, 0, 0, 213, 0, 2, 8189, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2374, 0, 0, 2, 2, 0, 0, 2, 0, 0, 0, 0, 2, 0, 1, 197, 0, 0, 6577, 8186, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 7254, 0, 0, 1258, 0, 0, 0, 0, 0, 6582, 0, 0, 0, 0, 0, 169, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2536, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 6584, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 4780, 0, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 0, 0, 0, 0, 0, 2044, 661, 565, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2491, 2045, 1594, 0, 0, 0, 0, 0, 0, 152, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6105, 6187, 6075, 6148, 6053, 6130, 6307] avg=6144.0000 max=6307 max_rank=7 min=6053 min_rank=5 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[2, 1, 1, 0, 0, 0, 1, 0, 0, 0, 93, 0, 7216, 10, 33, 1364, 2, 0, 1, 0, 12, 6453, 3, 12, 1, 0, 0, 320, 0, 32, 8156, 0, 0, 0, 1, 0, 0, 8154, 2, 3, 0, 0, 2498, 0, 4, 32, 33, 0, 8, 49, 0, 0, 0, 0, 23, 4, 8, 138, 0, 3, 6322, 8154, 2, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 7271, 0, 0, 1262, 0, 0, 0, 0, 0, 6584, 0, 0, 0, 0, 0, 138, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 6720, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 4816, 0, 0, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2524, 0, 0, 0, 0, 0, 2044, 661, 532, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2509, 2045, 1594, 0, 0, 0, 0, 0, 0, 164, 0, 0, 5801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6304, 6158, 6126, 6148, 6156, 6148, 5965] avg=6144.0000 max=6304 max_rank=1 min=5965 min_rank=7 imbalance_ratio=1.026042 max_load_reduction=42848 max_load_reduction_pct=0.871745 moved_to_echo=24417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 4796, 0, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 0, 0, 0, 0, 0, 2044, 661, 545, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 2044, 1594, 0, 0, 0, 0, 0, 0, 192, 0, 0, 5943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6012, 6150, 6099, 6148, 6322, 6138, 6135] avg=6144.0000 max=6322 max_rank=5 min=6012 min_rank=1 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 1, 83, 0, 4818, 1, 2, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 2, 5935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2374, 0, 0, 2, 2, 0, 2044, 661, 553, 394, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2511, 2043, 1593, 0, 0, 0, 0, 0, 1, 197, 0, 0, 6024, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6166, 6142, 6150, 6146, 6030, 6151, 6223] avg=6144.0000 max=6223 max_rank=7 min=6030 min_rank=5 imbalance_ratio=1.012858 max_load_reduction=42929 max_load_reduction_pct=0.873393 moved_to_echo=24550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 4805, 0, 0, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 0, 0, 0, 2044, 661, 552, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2504, 2045, 1594, 0, 0, 0, 0, 0, 0, 125, 0, 0, 6014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6126, 6143, 6169, 6148, 6137, 6143, 6139] avg=6144.0000 max=6169 max_rank=3 min=6126 min_rank=1 imbalance_ratio=1.004069 max_load_reduction=42983 max_load_reduction_pct=0.874491 moved_to_echo=24567 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 1, 0, 0, 0, 1, 0, 6119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 4740, 10, 33, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 12, 6059, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 320, 0, 32, 5906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6112, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2498, 0, 4, 32, 33, 0, 2042, 661, 536, 394, 0, 0, 0, 0, 8, 49, 0, 0, 0, 0, 23, 4, 2476, 2035, 1589, 0, 0, 0, 0, 0, 8, 138, 0, 3, 5786, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6240, 6089, 6259, 6118, 6200, 6184, 5938] avg=6144.0000 max=6259 max_rank=3 min=5938 min_rank=7 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 4786, 0, 0, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2403, 0, 0, 0, 0, 0, 2044, 661, 563, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2494, 2045, 1594, 0, 0, 0, 0, 0, 0, 166, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6097, 6098, 6168, 6148, 6062, 6133, 6299] avg=6144.0000 max=6299 max_rank=7 min=6062 min_rank=5 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24642 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 4805, 0, 0, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 0, 0, 0, 2044, 661, 552, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2504, 2045, 1594, 0, 0, 0, 0, 0, 0, 125, 0, 0, 6014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6126, 6143, 6169, 6148, 6137, 6143, 6139] avg=6144.0000 max=6169 max_rank=3 min=6126 min_rank=1 imbalance_ratio=1.004069 max_load_reduction=42983 max_load_reduction_pct=0.874491 moved_to_echo=24567 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 1, 83, 0, 4818, 1, 2, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 2, 5935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2374, 0, 0, 2, 2, 0, 2044, 661, 553, 394, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2511, 2043, 1593, 0, 0, 0, 0, 0, 1, 197, 0, 0, 6024, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6166, 6142, 6150, 6146, 6030, 6151, 6223] avg=6144.0000 max=6223 max_rank=7 min=6030 min_rank=5 imbalance_ratio=1.012858 max_load_reduction=42929 max_load_reduction_pct=0.873393 moved_to_echo=24550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 4796, 0, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 0, 0, 0, 0, 0, 2044, 661, 545, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 2044, 1594, 0, 0, 0, 0, 0, 0, 192, 0, 0, 5943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6012, 6150, 6099, 6148, 6322, 6138, 6135] avg=6144.0000 max=6322 max_rank=5 min=6012 min_rank=1 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 4780, 0, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 0, 0, 0, 0, 0, 2044, 661, 565, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2491, 2045, 1594, 0, 0, 0, 0, 0, 0, 152, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6105, 6187, 6075, 6148, 6053, 6130, 6307] avg=6144.0000 max=6307 max_rank=7 min=6053 min_rank=5 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 4769, 0, 0, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2536, 0, 0, 0, 0, 0, 2044, 661, 554, 397, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2485, 2044, 1594, 0, 0, 0, 0, 0, 0, 116, 0, 0, 6030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6102, 6185, 6106, 6148, 6192, 6125, 6146] avg=6144.0000 max=6192 max_rank=5 min=6102 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=42960 max_load_reduction_pct=0.874023 moved_to_echo=24611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 1, 0, 0, 0, 1, 0, 6119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 4740, 10, 33, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 12, 6059, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 320, 0, 32, 5906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6112, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2498, 0, 4, 32, 33, 0, 2042, 661, 536, 394, 0, 0, 0, 0, 8, 49, 0, 0, 0, 0, 23, 4, 2476, 2035, 1589, 0, 0, 0, 0, 0, 8, 138, 0, 3, 5786, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6240, 6089, 6259, 6118, 6200, 6184, 5938] avg=6144.0000 max=6259 max_rank=3 min=5938 min_rank=7 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 19968, 0, 128, 128, 128, 0, 16384, 5376, 4480, 3200, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 0, 128, 128, 20096, 16384, 12800, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 768, 0, 38400, 128, 128, 10240, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1280, 0, 128, 48000, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 1664, 0, 128, 47488, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[7648, 0, 0, 0, 0, 0, 0, 532, 0, 0, 1, 0, 0, 0, 0, 0, 118, 0, 1270, 164, 8192, 0, 0, 0, 0, 0, 97, 0, 973, 0, 0, 0, 480, 0, 0, 0, 0, 0, 1, 63, 0, 0, 0, 0, 8041, 7011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7018, 7168, 0, 63, 0, 0, 312] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[7541, 0, 25, 0, 32, 0, 0, 511, 0, 0, 1, 0, 0, 0, 0, 0, 182, 3, 1150, 132, 8134, 0, 15, 0, 0, 0, 129, 0, 1001, 12, 1, 0, 528, 12, 27, 0, 2, 0, 0, 88, 0, 12, 0, 25, 7888, 6891, 0, 0, 0, 0, 3, 0, 0, 27, 0, 0, 10, 7214, 7059, 0, 90, 31, 6, 370] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[7647, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 1158, 135, 8192, 0, 0, 0, 0, 0, 94, 0, 1029, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 7966, 6984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7140, 7108, 0, 80, 0, 0, 347] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[7603, 0, 2, 0, 2, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 156, 1, 1299, 127, 8189, 1, 2, 0, 0, 0, 108, 0, 1215, 0, 0, 0, 541, 1, 0, 0, 0, 0, 0, 83, 0, 2, 0, 2, 7943, 7006, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 6990, 6962, 0, 83, 2, 2, 334] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[7658, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 1178, 114, 8192, 0, 0, 0, 0, 0, 107, 0, 1134, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 7972, 7044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7081, 7001, 0, 94, 0, 0, 326] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[7615, 0, 0, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 1314, 137, 8192, 0, 0, 0, 0, 0, 132, 0, 1206, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 7952, 6906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7079, 6921, 0, 71, 0, 0, 323] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[7658, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 1178, 114, 8192, 0, 0, 0, 0, 0, 107, 0, 1134, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 7972, 7044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7081, 7001, 0, 94, 0, 0, 326] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[7648, 0, 0, 0, 0, 0, 0, 532, 0, 0, 1, 0, 0, 0, 0, 0, 118, 0, 1270, 164, 8192, 0, 0, 0, 0, 0, 97, 0, 973, 0, 0, 0, 480, 0, 0, 0, 0, 0, 1, 63, 0, 0, 0, 0, 8041, 7011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7018, 7168, 0, 63, 0, 0, 312] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[7569, 0, 0, 0, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 1250, 148, 8192, 0, 0, 0, 0, 0, 167, 0, 1120, 0, 0, 0, 570, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 7890, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7123, 6989, 0, 85, 0, 0, 354] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[7541, 0, 25, 0, 32, 0, 0, 511, 0, 0, 1, 0, 0, 0, 0, 0, 182, 3, 1150, 132, 8134, 0, 15, 0, 0, 0, 129, 0, 1001, 12, 1, 0, 528, 12, 27, 0, 2, 0, 0, 88, 0, 12, 0, 25, 7888, 6891, 0, 0, 0, 0, 3, 0, 0, 27, 0, 0, 10, 7214, 7059, 0, 90, 31, 6, 370] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[7624, 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 1303, 109, 8192, 0, 0, 0, 0, 0, 115, 0, 894, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 7952, 6918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7079, 7224, 0, 76, 0, 0, 392] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5656, 0, 0, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6213, 0, 0, 0, 0, 0, 0, 0, 118, 0, 1270, 164, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 973, 0, 0, 0, 1992, 1363, 826, 807, 0, 0, 0, 0, 480, 0, 0, 0, 0, 0, 1, 63, 2766, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4290, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 5805, 0, 63, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6188, 6214, 6171, 6058, 6038, 6185, 6118, 6180] avg=6144.0000 max=6214 max_rank=1 min=6038 min_rank=4 imbalance_ratio=1.011393 max_load_reduction=42938 max_load_reduction_pct=0.873576 moved_to_echo=24521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5623, 0, 2, 0, 2, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 156, 1, 1299, 127, 4617, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 1215, 0, 0, 0, 1980, 1324, 825, 807, 0, 0, 0, 0, 541, 1, 0, 0, 0, 0, 0, 83, 2765, 2719, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4271, 1805, 0, 0, 0, 0, 0, 0, 1, 0, 5638, 0, 83, 2, 2, 334, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 6138, 6203, 6259, 6109, 6185, 6077, 6060] avg=6144.0000 max=6259 max_rank=3 min=6060 min_rank=7 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5655, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 167, 0, 1158, 135, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 1029, 0, 0, 0, 1992, 1351, 823, 807, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 80, 2766, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 1810, 0, 0, 0, 0, 0, 0, 0, 0, 5757, 0, 80, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6156, 6079, 6096, 6161, 6161, 6173, 6184] avg=6144.0000 max=6184 max_rank=7 min=6079 min_rank=2 imbalance_ratio=1.006510 max_load_reduction=42968 max_load_reduction_pct=0.874186 moved_to_echo=24679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5663, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 0, 0, 0, 0, 0, 0, 159, 0, 1178, 114, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 1134, 0, 0, 0, 1995, 1331, 830, 807, 0, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 94, 2766, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4328, 1811, 0, 0, 0, 0, 0, 0, 0, 0, 5670, 0, 94, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6120, 6161, 6070, 6204, 6154, 6214, 6139, 6090] avg=6144.0000 max=6214 max_rank=5 min=6070 min_rank=2 imbalance_ratio=1.011393 max_load_reduction=42938 max_load_reduction_pct=0.873576 moved_to_echo=24597 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5632, 0, 0, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 149, 0, 1314, 137, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 1206, 0, 0, 0, 1983, 1316, 813, 807, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 70, 2766, 2752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4327, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 5605, 0, 71, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 6145, 6219, 6257, 6154, 6093, 6134, 5999] avg=6144.0000 max=6257 max_rank=3 min=5999 min_rank=7 imbalance_ratio=1.018392 max_load_reduction=42895 max_load_reduction_pct=0.872701 moved_to_echo=24380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5573, 0, 25, 0, 32, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6096, 0, 0, 0, 0, 0, 0, 0, 182, 3, 1150, 132, 4578, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 1001, 12, 1, 0, 1968, 1345, 815, 808, 0, 0, 0, 0, 528, 12, 27, 0, 2, 0, 0, 88, 2748, 2806, 0, 0, 0, 0, 0, 0, 0, 12, 0, 25, 0, 6076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 27, 0, 0, 4408, 1792, 0, 0, 0, 0, 0, 0, 10, 0, 5714, 0, 90, 31, 6, 370, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6097, 6060, 6079, 6211, 6113, 6230, 6221] avg=6144.0000 max=6230 max_rank=6 min=6060 min_rank=2 imbalance_ratio=1.013997 max_load_reduction=42922 max_load_reduction_pct=0.873250 moved_to_echo=24775 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5597, 0, 0, 0, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6096, 0, 0, 0, 0, 0, 0, 0, 143, 0, 1250, 148, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 1120, 0, 0, 0, 1972, 1329, 814, 807, 0, 0, 0, 0, 570, 0, 0, 0, 0, 0, 0, 85, 2766, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4354, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 5660, 0, 85, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6096, 6160, 6209, 6190, 6098, 6148, 6099] avg=6144.0000 max=6209 max_rank=3 min=6096 min_rank=1 imbalance_ratio=1.010579 max_load_reduction=42943 max_load_reduction_pct=0.873678 moved_to_echo=24535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5623, 0, 2, 0, 2, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 156, 1, 1299, 127, 4617, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 1215, 0, 0, 0, 1980, 1324, 825, 807, 0, 0, 0, 0, 541, 1, 0, 0, 0, 0, 0, 83, 2765, 2719, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4271, 1805, 0, 0, 0, 0, 0, 0, 1, 0, 5638, 0, 83, 2, 2, 334, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 6138, 6203, 6259, 6109, 6185, 6077, 6060] avg=6144.0000 max=6259 max_rank=3 min=6060 min_rank=7 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5656, 0, 0, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6213, 0, 0, 0, 0, 0, 0, 0, 118, 0, 1270, 164, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 973, 0, 0, 0, 1992, 1363, 826, 807, 0, 0, 0, 0, 480, 0, 0, 0, 0, 0, 1, 63, 2766, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4290, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 5805, 0, 63, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6188, 6214, 6171, 6058, 6038, 6185, 6118, 6180] avg=6144.0000 max=6214 max_rank=1 min=6038 min_rank=4 imbalance_ratio=1.011393 max_load_reduction=42938 max_load_reduction_pct=0.873576 moved_to_echo=24521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5655, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 167, 0, 1158, 135, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 1029, 0, 0, 0, 1992, 1351, 823, 807, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 80, 2766, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 1810, 0, 0, 0, 0, 0, 0, 0, 0, 5757, 0, 80, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6156, 6079, 6096, 6161, 6161, 6173, 6184] avg=6144.0000 max=6184 max_rank=7 min=6079 min_rank=2 imbalance_ratio=1.006510 max_load_reduction=42968 max_load_reduction_pct=0.874186 moved_to_echo=24679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5638, 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 159, 0, 1303, 109, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 894, 0, 0, 0, 1986, 1373, 815, 807, 0, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 75, 2766, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4326, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 5851, 0, 76, 0, 0, 392, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6145, 6190, 5990, 6135, 6103, 6133, 6319] avg=6144.0000 max=6319 max_rank=7 min=5990 min_rank=3 imbalance_ratio=1.028483 max_load_reduction=42833 max_load_reduction_pct=0.871440 moved_to_echo=24690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5573, 0, 25, 0, 32, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6096, 0, 0, 0, 0, 0, 0, 0, 182, 3, 1150, 132, 4578, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 1001, 12, 1, 0, 1968, 1345, 815, 808, 0, 0, 0, 0, 528, 12, 27, 0, 2, 0, 0, 88, 2748, 2806, 0, 0, 0, 0, 0, 0, 0, 12, 0, 25, 0, 6076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 27, 0, 0, 4408, 1792, 0, 0, 0, 0, 0, 0, 10, 0, 5714, 0, 90, 31, 6, 370, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6097, 6060, 6079, 6211, 6113, 6230, 6221] avg=6144.0000 max=6230 max_rank=6 min=6060 min_rank=2 imbalance_ratio=1.013997 max_load_reduction=42922 max_load_reduction_pct=0.873250 moved_to_echo=24775 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 1024, 0, 8576, 128, 128, 0, 15872, 10752, 6656, 6528, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 128, 128, 0, 128, 0, 128, 640, 22144, 22144, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 34688, 14464, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1280, 128, 9984, 1152, 36992, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 45824, 0, 768, 128, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[45056, 0, 128, 0, 128, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1561, 1, 0, 0, 96, 0, 8126, 5010, 1, 0, 7318, 2, 0, 0, 0, 789, 0, 0, 0, 2, 8106, 3, 1, 0, 0, 2966, 0, 7563, 0, 1, 0, 0, 46, 0, 0, 0, 0, 83, 0, 0, 0, 242, 1257, 1, 5887, 0, 2, 83, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 0, 0, 98, 0, 8066, 5014, 0, 0, 7363, 0, 0, 0, 0, 854, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 3081, 0, 7593, 0, 0, 0, 0, 16, 0, 0, 0, 0, 94, 0, 0, 0, 245, 1266, 0, 5834, 0, 0, 94, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1788, 0, 0, 0, 64, 0, 8079, 4951, 0, 0, 7323, 0, 0, 0, 0, 863, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 3182, 0, 7573, 0, 0, 0, 0, 7, 0, 0, 0, 0, 70, 0, 0, 0, 200, 1331, 0, 5529, 0, 0, 70, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 0, 0, 0, 91, 0, 8097, 5129, 0, 0, 7271, 0, 0, 0, 0, 678, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 2906, 0, 7610, 0, 0, 0, 0, 10, 0, 0, 0, 0, 76, 0, 0, 0, 230, 1344, 0, 6161, 0, 0, 76, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 92, 0, 8101, 4724, 0, 0, 7305, 0, 0, 0, 0, 838, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 3270, 0, 7511, 0, 0, 0, 0, 24, 0, 0, 0, 0, 85, 0, 0, 0, 261, 1243, 0, 5929, 0, 0, 85, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 0, 0, 93, 0, 8071, 4962, 0, 0, 7296, 0, 0, 0, 0, 898, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 3027, 0, 7621, 0, 0, 0, 0, 31, 0, 0, 0, 0, 80, 0, 0, 0, 232, 1311, 0, 6082, 0, 0, 80, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 25, 0, 5, 2, 0, 0, 1, 0, 0, 0, 3, 0, 1640, 17, 1, 0, 105, 0, 7996, 4717, 15, 1, 7187, 20, 0, 0, 0, 865, 3, 0, 0, 12, 8055, 30, 0, 0, 0, 3308, 7, 7499, 3, 34, 1, 0, 80, 0, 10, 11, 2, 88, 6, 2, 1, 309, 1304, 5, 5657, 0, 37, 88, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 0, 0, 98, 0, 8066, 5014, 0, 0, 7363, 0, 0, 0, 0, 854, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 3081, 0, 7593, 0, 0, 0, 0, 16, 0, 0, 0, 0, 94, 0, 0, 0, 245, 1266, 0, 5834, 0, 0, 94, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 0, 0, 0, 51, 0, 8072, 5306, 0, 0, 7352, 0, 0, 0, 0, 826, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 2646, 0, 7689, 0, 0, 0, 0, 19, 0, 0, 0, 0, 63, 0, 0, 0, 144, 1320, 0, 6220, 0, 0, 63, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 2004, 1475, 1135, 46, 0, 0, 0, 0, 0, 0, 98, 0, 1078, 5014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5359, 0, 0, 0, 0, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3087, 0, 0, 0, 0, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 245, 4965, 856, 0, 0, 0, 0, 0, 0, 1266, 0, 4699, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6096, 6190, 6213, 6168, 6134, 6160, 6059] avg=6144.0000 max=6213 max_rank=3 min=6059 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=42939 max_load_reduction_pct=0.873596 moved_to_echo=24521 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 92, 0, 8101, 4724, 0, 0, 7305, 0, 0, 0, 0, 838, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 3270, 0, 7511, 0, 0, 0, 0, 24, 0, 0, 0, 0, 85, 0, 0, 0, 261, 1243, 0, 5929, 0, 0, 85, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 0, 0, 0, 91, 0, 8097, 5129, 0, 0, 7271, 0, 0, 0, 0, 678, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 2906, 0, 7610, 0, 0, 0, 0, 10, 0, 0, 0, 0, 76, 0, 0, 0, 230, 1344, 0, 6161, 0, 0, 76, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 25, 0, 5, 2, 0, 0, 1, 0, 0, 0, 3, 0, 1640, 17, 1, 0, 105, 0, 7996, 4717, 15, 1, 7187, 20, 0, 0, 0, 865, 3, 0, 0, 12, 8055, 30, 0, 0, 0, 3308, 7, 7499, 3, 34, 1, 0, 80, 0, 10, 11, 2, 88, 6, 2, 1, 309, 1304, 5, 5657, 0, 37, 88, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1561, 1, 0, 0, 96, 0, 8126, 5010, 1, 0, 7318, 2, 0, 0, 0, 789, 0, 0, 0, 2, 8106, 3, 1, 0, 0, 2966, 0, 7563, 0, 1, 0, 0, 46, 0, 0, 0, 0, 83, 0, 0, 0, 242, 1257, 1, 5887, 0, 2, 83, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1788, 0, 1993, 1471, 1076, 46, 0, 0, 0, 0, 0, 0, 64, 0, 1080, 4951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5330, 0, 0, 0, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3097, 0, 0, 0, 0, 3182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6102, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 200, 4979, 857, 0, 0, 0, 0, 0, 0, 1331, 0, 4453, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6374, 6095, 6193, 6279, 6109, 6106, 5854] avg=6144.0000 max=6374 max_rank=1 min=5854 min_rank=7 imbalance_ratio=1.037435 max_load_reduction=42778 max_load_reduction_pct=0.870321 moved_to_echo=24348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 6177, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1561, 1, 1991, 1469, 1145, 46, 0, 0, 0, 0, 0, 0, 96, 0, 1087, 5010, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5327, 2, 0, 0, 0, 789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3090, 3, 1, 0, 0, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6094, 0, 1, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 242, 4970, 862, 0, 0, 0, 0, 0, 0, 1257, 1, 4742, 0, 2, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6179, 6216, 6194, 6118, 6062, 6141, 6157, 6085] avg=6144.0000 max=6216 max_rank=1 min=6062 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=42936 max_load_reduction_pct=0.873535 moved_to_echo=24445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 2004, 1475, 1135, 46, 0, 0, 0, 0, 0, 0, 98, 0, 1078, 5014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5359, 0, 0, 0, 0, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3087, 0, 0, 0, 0, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 245, 4965, 856, 0, 0, 0, 0, 0, 0, 1266, 0, 4699, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6096, 6190, 6213, 6168, 6134, 6160, 6059] avg=6144.0000 max=6213 max_rank=3 min=6059 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=42939 max_load_reduction_pct=0.873596 moved_to_echo=24521 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 1988, 1459, 1154, 46, 0, 0, 0, 0, 0, 0, 92, 0, 1083, 4724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5317, 0, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3091, 0, 0, 0, 0, 3270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6052, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 261, 4970, 860, 0, 0, 0, 0, 0, 0, 1243, 0, 4775, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6158, 6224, 5899, 6155, 6361, 6076, 6176, 6103] avg=6144.0000 max=6361 max_rank=4 min=5899 min_rank=2 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 0, 2001, 1493, 1210, 46, 0, 0, 0, 0, 0, 0, 51, 0, 1080, 5306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5351, 0, 0, 0, 0, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3099, 0, 0, 0, 0, 2646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6196, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 144, 4984, 856, 0, 0, 0, 0, 0, 0, 1320, 0, 5010, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6002, 6437, 6177, 5745, 6215, 6047, 6393] avg=6144.0000 max=6437 max_rank=2 min=5745 min_rank=4 imbalance_ratio=1.047689 max_load_reduction=42715 max_load_reduction_pct=0.869039 moved_to_echo=24400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 1985, 1480, 1183, 46, 0, 0, 0, 0, 0, 0, 93, 0, 1080, 4962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5311, 0, 0, 0, 0, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 0, 0, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 232, 4973, 856, 0, 0, 0, 0, 0, 0, 1311, 0, 4899, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6135, 5950, 6135, 6209, 6120, 6172, 6141, 6290] avg=6144.0000 max=6290 max_rank=7 min=5950 min_rank=1 imbalance_ratio=1.023763 max_load_reduction=42862 max_load_reduction_pct=0.872030 moved_to_echo=24723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 1988, 1459, 1154, 46, 0, 0, 0, 0, 0, 0, 92, 0, 1083, 4724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5317, 0, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3091, 0, 0, 0, 0, 3270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6052, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 261, 4970, 860, 0, 0, 0, 0, 0, 0, 1243, 0, 4775, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6158, 6224, 5899, 6155, 6361, 6076, 6176, 6103] avg=6144.0000 max=6361 max_rank=4 min=5899 min_rank=2 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 25, 0, 5, 2, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 1640, 17, 1959, 1461, 1105, 47, 0, 0, 0, 0, 1, 0, 105, 0, 1060, 4717, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5228, 20, 0, 0, 0, 865, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3066, 30, 0, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6038, 3, 34, 1, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 11, 2, 88, 6, 2, 1, 309, 4942, 853, 0, 0, 0, 0, 0, 0, 1304, 5, 4552, 0, 37, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 6233, 5899, 6116, 6416, 6163, 6224, 5986] avg=6144.0000 max=6416 max_rank=4 min=5899 min_rank=2 imbalance_ratio=1.044271 max_load_reduction=42736 max_load_reduction_pct=0.869466 moved_to_echo=24789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 0, 1978, 1478, 1199, 46, 0, 0, 0, 0, 0, 0, 91, 0, 1083, 5129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5293, 0, 0, 0, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3095, 0, 0, 0, 0, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 230, 4976, 859, 0, 0, 0, 0, 0, 0, 1344, 0, 4962, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6057, 6303, 5971, 6001, 6142, 6141, 6382] avg=6144.0000 max=6382 max_rank=7 min=5971 min_rank=3 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 25, 0, 5, 2, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 1640, 17, 1959, 1461, 1105, 47, 0, 0, 0, 0, 1, 0, 105, 0, 1060, 4717, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5228, 20, 0, 0, 0, 865, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3066, 30, 0, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6038, 3, 34, 1, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 11, 2, 88, 6, 2, 1, 309, 4942, 853, 0, 0, 0, 0, 0, 0, 1304, 5, 4552, 0, 37, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 6233, 5899, 6116, 6416, 6163, 6224, 5986] avg=6144.0000 max=6416 max_rank=4 min=5899 min_rank=2 imbalance_ratio=1.044271 max_load_reduction=42736 max_load_reduction_pct=0.869466 moved_to_echo=24789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 6177, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1561, 1, 1991, 1469, 1145, 46, 0, 0, 0, 0, 0, 0, 96, 0, 1087, 5010, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5327, 2, 0, 0, 0, 789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3090, 3, 1, 0, 0, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6094, 0, 1, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 242, 4970, 862, 0, 0, 0, 0, 0, 0, 1257, 1, 4742, 0, 2, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6179, 6216, 6194, 6118, 6062, 6141, 6157, 6085] avg=6144.0000 max=6216 max_rank=1 min=6062 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=42936 max_load_reduction_pct=0.873535 moved_to_echo=24445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 0, 11904, 128, 16000, 11904, 9216, 384, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 24832, 128, 128, 0, 0, 24448, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[42624, 128, 0, 0, 0, 6656, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 768, 0, 8704, 39936, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10496, 128, 38144, 0, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 768, 0, 8704, 39936, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[42624, 128, 0, 0, 0, 6656, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 640, 128, 128, 128, 1920, 39808, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 48896, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 48896, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 152, 1, 8062, 660, 80, 0, 0, 1, 0, 7854, 0, 1, 0, 2, 7865, 2, 0, 347, 0, 0, 0, 0, 0, 0, 4131, 0, 8189, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 24, 0, 0, 0, 176, 0, 2, 0, 3009, 1, 0, 2, 0, 0, 1, 3, 0, 5, 0, 394, 0, 0, 0, 0, 1, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 98, 0, 8094, 655, 55, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 8012, 0, 0, 219, 0, 0, 0, 0, 0, 0, 4118, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 11, 0, 0, 0, 104, 0, 0, 0, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 153, 0, 8049, 654, 94, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7879, 0, 0, 260, 0, 0, 0, 0, 0, 0, 4017, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 3, 0, 0, 0, 161, 0, 0, 0, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 140, 0, 8072, 654, 79, 0, 0, 0, 0, 7868, 0, 0, 0, 0, 7888, 0, 0, 242, 0, 0, 0, 0, 0, 0, 4110, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 37, 0, 0, 0, 166, 0, 0, 0, 3152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 140, 0, 8051, 736, 70, 0, 0, 0, 0, 7848, 0, 0, 0, 0, 7883, 0, 0, 402, 0, 0, 0, 0, 0, 0, 4013, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 30, 0, 0, 0, 170, 0, 0, 0, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 153, 0, 8049, 654, 94, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7879, 0, 0, 260, 0, 0, 0, 0, 0, 0, 4017, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 3, 0, 0, 0, 161, 0, 0, 0, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 175, 0, 8066, 720, 83, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7811, 0, 0, 312, 0, 0, 0, 0, 0, 0, 3708, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 0, 212, 0, 0, 0, 3451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 154, 1, 8000, 753, 86, 1, 0, 0, 1, 7820, 0, 19, 0, 23, 7822, 32, 8, 267, 0, 21, 0, 0, 0, 1, 3320, 0, 8152, 1, 0, 11, 0, 0, 0, 8129, 0, 0, 0, 86, 0, 6, 0, 195, 0, 8, 1, 3617, 23, 4, 2, 1, 0, 5, 36, 0, 1, 0, 506, 0, 11, 0, 3, 0, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 140, 0, 8072, 654, 79, 0, 0, 0, 0, 7868, 0, 0, 0, 0, 7888, 0, 0, 242, 0, 0, 0, 0, 0, 0, 4110, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 37, 0, 0, 0, 166, 0, 0, 0, 3152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 152, 1, 8062, 660, 80, 0, 0, 1, 0, 7854, 0, 1, 0, 2, 7865, 2, 0, 347, 0, 0, 0, 0, 0, 0, 4131, 0, 8189, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 24, 0, 0, 0, 176, 0, 2, 0, 3009, 1, 0, 2, 0, 0, 1, 3, 0, 5, 0, 394, 0, 0, 0, 0, 1, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 140, 0, 8070, 670, 71, 0, 0, 0, 0, 7872, 0, 0, 0, 0, 7885, 0, 0, 236, 0, 0, 0, 0, 0, 0, 3570, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 3, 0, 0, 0, 171, 0, 0, 0, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 98, 0, 5259, 655, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 4109, 1773, 0, 0, 0, 0, 0, 0, 0, 4118, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6126, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 3225, 0, 1717, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6239, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 2835, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6067, 6148, 6101, 6389, 6137, 5984, 6239, 6087] avg=6144.0000 max=6389 max_rank=3 min=5984 min_rank=5 imbalance_ratio=1.039876 max_load_reduction=42763 max_load_reduction_pct=0.870015 moved_to_echo=24447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 152, 1, 5238, 660, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6140, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 347, 0, 0, 0, 0, 0, 4107, 1741, 0, 0, 0, 0, 0, 0, 0, 4131, 0, 2271, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6121, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 2, 0, 3009, 1, 1714, 938, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3, 0, 5, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 1, 2, 2824, 1811, 1125, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6144, 6197, 6403, 6145, 5840, 6135, 6157] avg=6144.0000 max=6403 max_rank=3 min=5840 min_rank=5 imbalance_ratio=1.042155 max_load_reduction=42749 max_load_reduction_pct=0.869731 moved_to_echo=24277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 153, 0, 5229, 654, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 4109, 1743, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6125, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 3303, 0, 1713, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 2820, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6134, 6112, 6288, 6128, 6115, 6136, 6109] avg=6144.0000 max=6288 max_rank=3 min=6109 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=42864 max_load_reduction_pct=0.872070 moved_to_echo=24488 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 154, 1, 8000, 753, 86, 1, 0, 0, 1, 7820, 0, 19, 0, 23, 7822, 32, 8, 267, 0, 21, 0, 0, 0, 1, 3320, 0, 8152, 1, 0, 11, 0, 0, 0, 8129, 0, 0, 0, 86, 0, 6, 0, 195, 0, 8, 1, 3617, 23, 4, 2, 1, 0, 5, 36, 0, 1, 0, 506, 0, 11, 0, 3, 0, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 98, 0, 8094, 655, 55, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 8012, 0, 0, 219, 0, 0, 0, 0, 0, 0, 4118, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 11, 0, 0, 0, 104, 0, 0, 0, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 140, 0, 5231, 736, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 4109, 1744, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6127, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 3076, 0, 1713, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 2820, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6135, 6255, 6284, 6157, 5898, 6139, 6107] avg=6144.0000 max=6284 max_rank=3 min=5898 min_rank=5 imbalance_ratio=1.022786 max_load_reduction=42868 max_load_reduction_pct=0.872152 moved_to_echo=24301 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 140, 0, 5244, 654, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 4109, 1746, 0, 0, 0, 0, 0, 0, 0, 4110, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6127, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 3152, 0, 1717, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 2828, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6117, 6151, 6097, 6381, 6164, 5974, 6142, 6126] avg=6144.0000 max=6381 max_rank=3 min=5974 min_rank=5 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 154, 1, 5193, 753, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6109, 0, 19, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 8, 267, 0, 21, 0, 0, 0, 4090, 1732, 0, 0, 0, 0, 0, 0, 1, 3320, 0, 2254, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6070, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 195, 0, 8, 1, 3617, 23, 1711, 935, 0, 0, 0, 0, 0, 0, 4, 2, 1, 0, 5, 36, 0, 1, 6090, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 11, 0, 3, 0, 25, 2807, 1808, 1124, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6188, 6152, 6150, 5587, 6156, 6496, 6139, 6284] avg=6144.0000 max=6496 max_rank=5 min=5587 min_rank=3 imbalance_ratio=1.057292 max_load_reduction=42656 max_load_reduction_pct=0.867839 moved_to_echo=25075 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 175, 0, 5240, 720, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 4109, 1728, 0, 0, 0, 0, 0, 0, 0, 3708, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6127, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 3451, 0, 1713, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 0, 0, 0, 0, 2826, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6218, 6134, 6149, 5979, 6135, 6315, 6083, 6139] avg=6144.0000 max=6315 max_rank=5 min=5979 min_rank=3 imbalance_ratio=1.027832 max_load_reduction=42837 max_load_reduction_pct=0.871521 moved_to_echo=24672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 153, 0, 5229, 654, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 4109, 1743, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6125, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 3303, 0, 1713, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 2820, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6134, 6112, 6288, 6128, 6115, 6136, 6109] avg=6144.0000 max=6288 max_rank=3 min=6109 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=42864 max_load_reduction_pct=0.872070 moved_to_echo=24488 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 140, 0, 5244, 654, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 4109, 1746, 0, 0, 0, 0, 0, 0, 0, 4110, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6127, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 3152, 0, 1717, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 2828, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6117, 6151, 6097, 6381, 6164, 5974, 6142, 6126] avg=6144.0000 max=6381 max_rank=3 min=5974 min_rank=5 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 152, 1, 5238, 660, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6140, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 347, 0, 0, 0, 0, 0, 4107, 1741, 0, 0, 0, 0, 0, 0, 0, 4131, 0, 2271, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6121, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 2, 0, 3009, 1, 1714, 938, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3, 0, 5, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 1, 2, 2824, 1811, 1125, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6144, 6197, 6403, 6145, 5840, 6135, 6157] avg=6144.0000 max=6403 max_rank=3 min=5840 min_rank=5 imbalance_ratio=1.042155 max_load_reduction=42749 max_load_reduction_pct=0.869731 moved_to_echo=24277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 98, 0, 5259, 655, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 4109, 1773, 0, 0, 0, 0, 0, 0, 0, 4118, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6126, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 3225, 0, 1717, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6239, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 2835, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6067, 6148, 6101, 6389, 6137, 5984, 6239, 6087] avg=6144.0000 max=6389 max_rank=3 min=5984 min_rank=5 imbalance_ratio=1.039876 max_load_reduction=42763 max_load_reduction_pct=0.870015 moved_to_echo=24447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 140, 0, 5243, 670, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 0, 4109, 1746, 0, 0, 0, 0, 0, 0, 0, 3570, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6127, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 3703, 0, 1718, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0, 2827, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6154, 6091, 5841, 6130, 6530, 6139, 6143] avg=6144.0000 max=6530 max_rank=5 min=5841 min_rank=3 imbalance_ratio=1.062826 max_load_reduction=42622 max_load_reduction_pct=0.867147 moved_to_echo=24942 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 154, 1, 5193, 753, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6109, 0, 19, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 8, 267, 0, 21, 0, 0, 0, 4090, 1732, 0, 0, 0, 0, 0, 0, 1, 3320, 0, 2254, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6070, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 195, 0, 8, 1, 3617, 23, 1711, 935, 0, 0, 0, 0, 0, 0, 4, 2, 1, 0, 5, 36, 0, 1, 6090, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 11, 0, 3, 0, 25, 2807, 1808, 1124, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6188, 6152, 6150, 5587, 6156, 6496, 6139, 6284] avg=6144.0000 max=6496 max_rank=5 min=5587 min_rank=3 imbalance_ratio=1.057292 max_load_reduction=42656 max_load_reduction_pct=0.867839 moved_to_echo=25075 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 2304, 0, 128, 0, 0, 0, 32896, 14080, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 1408, 0, 128, 128, 26624, 128, 13824, 7552, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 3072, 0, 128, 0, 128, 128, 128, 22656, 14592, 9088, 0, 0, 0, 0, 0] @@ -7886,829 +7886,829 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8192, 478, 0, 1046, 0, 0, 0, 0, 506, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 0, 0, 0, 0, 0, 5885, 0, 0, 0, 0, 1334, 0, 8192, 0, 0, 0, 8192, 0, 0, 101, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 310, 0, 6130] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8192, 401, 0, 1096, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 638, 0, 0, 0, 0, 0, 0, 0, 0, 6033, 0, 0, 0, 0, 1213, 0, 8192, 0, 0, 0, 8192, 0, 0, 100, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 326, 0, 6152] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[6, 2, 0, 0, 0, 0, 1, 8185, 426, 2, 1194, 0, 0, 0, 0, 464, 0, 0, 1, 0, 0, 0, 0, 601, 0, 0, 0, 0, 0, 0, 0, 1, 5943, 1, 0, 0, 0, 1091, 0, 8189, 0, 2, 0, 8187, 1, 2, 77, 0, 0, 0, 1, 3, 2, 8170, 0, 2, 0, 1, 0, 0, 0, 335, 0, 6262] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8192, 460, 0, 1098, 0, 0, 0, 0, 459, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 6043, 0, 0, 0, 0, 1070, 0, 8192, 0, 0, 0, 8192, 0, 0, 53, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 352, 0, 6258] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[39, 23, 0, 0, 0, 0, 3, 8108, 439, 37, 1589, 0, 0, 0, 0, 533, 0, 10, 7, 0, 0, 0, 1, 534, 0, 0, 0, 0, 0, 1, 3, 18, 6026, 21, 1, 0, 0, 1144, 0, 8148, 0, 31, 0, 8132, 26, 34, 58, 0, 0, 0, 0, 23, 20, 7997, 0, 20, 0, 9, 4, 1, 1, 347, 0, 5764] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8192, 454, 0, 1108, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 846, 0, 8192, 0, 0, 0, 8192, 0, 0, 100, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 345, 0, 6300] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 8192, 504, 0, 914, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 6059, 0, 0, 0, 0, 1170, 0, 8192, 0, 0, 0, 8192, 0, 0, 76, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 323, 0, 6206] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8192, 466, 0, 1012, 0, 0, 0, 0, 484, 0, 0, 0, 0, 0, 0, 0, 628, 0, 0, 0, 0, 0, 0, 0, 0, 5696, 0, 0, 0, 0, 1356, 0, 8192, 0, 0, 0, 8192, 0, 0, 85, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 392, 0, 6267] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 1046, 0, 0, 0, 0, 506, 2034, 975, 681, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 2135, 2046, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 4910, 0, 0, 0, 0, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 5766, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6084, 6142, 6146, 6244, 6158, 6158, 6076] avg=6144.0000 max=6244 max_rank=4 min=6076 min_rank=7 imbalance_ratio=1.016276 max_load_reduction=42908 max_load_reduction_pct=0.872965 moved_to_echo=24636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[6, 2, 0, 0, 0, 0, 1, 8185, 426, 2, 1194, 0, 0, 0, 0, 464, 0, 0, 1, 0, 0, 0, 0, 601, 0, 0, 0, 0, 0, 0, 0, 1, 5943, 1, 0, 0, 0, 1091, 0, 8189, 0, 2, 0, 8187, 1, 2, 77, 0, 0, 0, 1, 3, 2, 8170, 0, 2, 0, 1, 0, 0, 0, 335, 0, 6262] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 8192, 504, 0, 914, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 6059, 0, 0, 0, 0, 1170, 0, 8192, 0, 0, 0, 8192, 0, 0, 76, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 323, 0, 6206] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 1096, 0, 0, 0, 0, 434, 2031, 1000, 681, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 2135, 2046, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 5033, 0, 0, 0, 0, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 5787, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6008, 6186, 6146, 6246, 6157, 6152, 6113] avg=6144.0000 max=6246 max_rank=4 min=6008 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=42906 max_load_reduction_pct=0.872925 moved_to_echo=24668 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 1046, 0, 0, 0, 0, 506, 2034, 975, 681, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 2135, 2046, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 4910, 0, 0, 0, 0, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 5766, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6084, 6142, 6146, 6244, 6158, 6158, 6076] avg=6144.0000 max=6244 max_rank=4 min=6076 min_rank=7 imbalance_ratio=1.016276 max_load_reduction=42908 max_load_reduction_pct=0.872965 moved_to_echo=24636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[6, 2, 0, 0, 0, 0, 1, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 426, 2, 1194, 0, 0, 0, 0, 464, 2028, 985, 681, 372, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 601, 2133, 2046, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6143, 0, 0, 0, 0, 0, 0, 0, 4958, 1, 0, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6054, 1, 2, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 2, 6142, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 335, 0, 5890, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6152, 6146, 6144, 6050, 6136, 6150, 6226] avg=6144.0000 max=6226 max_rank=7 min=6050 min_rank=4 imbalance_ratio=1.013346 max_load_reduction=42926 max_load_reduction_pct=0.873332 moved_to_echo=24562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 504, 0, 914, 0, 0, 0, 0, 549, 2031, 1004, 681, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 2135, 2046, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 5055, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 5838, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6051, 6139, 6146, 6225, 6133, 6152, 6161] avg=6144.0000 max=6225 max_rank=4 min=6051 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=42927 max_load_reduction_pct=0.873352 moved_to_echo=24671 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 1108, 0, 0, 0, 0, 474, 2033, 1020, 681, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 2135, 2047, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 5133, 0, 0, 0, 0, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 0, 5926, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6144, 6155, 6145, 5979, 6157, 6157, 6271] avg=6144.0000 max=6271 max_rank=7 min=5979 min_rank=4 imbalance_ratio=1.020671 max_load_reduction=42881 max_load_reduction_pct=0.872416 moved_to_echo=24564 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[39, 23, 0, 0, 0, 0, 3, 6074, 0, 0, 0, 0, 0, 0, 0, 0, 439, 37, 1589, 0, 0, 0, 0, 533, 1989, 1003, 680, 346, 0, 0, 0, 0, 0, 10, 7, 0, 0, 0, 1, 534, 2122, 2036, 1354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 18, 6112, 0, 0, 0, 0, 0, 0, 0, 5023, 21, 1, 0, 0, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 6010, 26, 34, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 20, 6008, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 1, 1, 347, 0, 5418, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6616, 6064, 6134, 6189, 6159, 6071, 5780] avg=6144.0000 max=6616 max_rank=1 min=5780 min_rank=7 imbalance_ratio=1.076823 max_load_reduction=42536 max_load_reduction_pct=0.865397 moved_to_echo=24199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 1098, 0, 0, 0, 0, 459, 2033, 1002, 681, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 2135, 2046, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 5041, 0, 0, 0, 0, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 5886, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6105, 6143, 6146, 6111, 6110, 6155, 6238] avg=6144.0000 max=6238 max_rank=7 min=6105 min_rank=1 imbalance_ratio=1.015299 max_load_reduction=42914 max_load_reduction_pct=0.873088 moved_to_echo=24614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 504, 0, 914, 0, 0, 0, 0, 549, 2031, 1004, 681, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 2135, 2046, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 5055, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 5838, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6051, 6139, 6146, 6225, 6133, 6152, 6161] avg=6144.0000 max=6225 max_rank=4 min=6051 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=42927 max_load_reduction_pct=0.873352 moved_to_echo=24671 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[39, 23, 0, 0, 0, 0, 3, 6074, 0, 0, 0, 0, 0, 0, 0, 0, 439, 37, 1589, 0, 0, 0, 0, 533, 1989, 1003, 680, 346, 0, 0, 0, 0, 0, 10, 7, 0, 0, 0, 1, 534, 2122, 2036, 1354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 18, 6112, 0, 0, 0, 0, 0, 0, 0, 5023, 21, 1, 0, 0, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 6010, 26, 34, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 20, 6008, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 1, 1, 347, 0, 5418, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6616, 6064, 6134, 6189, 6159, 6071, 5780] avg=6144.0000 max=6616 max_rank=1 min=5780 min_rank=7 imbalance_ratio=1.076823 max_load_reduction=42536 max_load_reduction_pct=0.865397 moved_to_echo=24199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 1012, 0, 0, 0, 0, 484, 2033, 944, 681, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 628, 2135, 2047, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 4752, 0, 0, 0, 0, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 5895, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 5992, 6177, 6145, 6108, 6142, 6157, 6287] avg=6144.0000 max=6287 max_rank=7 min=5992 min_rank=1 imbalance_ratio=1.023275 max_load_reduction=42865 max_load_reduction_pct=0.872091 moved_to_echo=24694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 4864, 17152, 16384, 11008, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3712, 128, 9088, 0, 0, 0, 0, 3968, 16256, 7936, 5504, 2944, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[39936, 128, 128, 0, 0, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 2816, 0, 46464, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 48512, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 2816, 0, 46464, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2142, 5550, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 5212, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 1928, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2348, 5698, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 355, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 4596, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 0, 2065, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[11, 0, 2479, 5460, 0, 0, 0, 1, 35, 0, 0, 0, 8148, 0, 0, 0, 15, 469, 0, 0, 0, 0, 0, 0, 108, 0, 11, 0, 0, 0, 0, 0, 8126, 1, 0, 0, 34, 0, 54, 0, 0, 7, 0, 0, 38, 8137, 8137, 0, 38, 5023, 0, 2, 3, 0, 1, 0, 0, 38, 953, 31, 1788, 0, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2351, 5593, 0, 0, 0, 0, 3, 0, 0, 0, 8189, 0, 0, 0, 2, 405, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 2, 0, 68, 0, 0, 1, 0, 1, 3, 8184, 8187, 0, 3, 4958, 0, 2, 0, 0, 0, 0, 0, 3, 1006, 1, 1912, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2349, 5536, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 4785, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 0, 1989, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2142, 5550, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 5212, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 1928, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2272, 5787, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 4972, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 1881, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2200, 5537, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 0, 1936, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 1993, 5589, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 417, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 0, 1836, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2272, 5787, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 4972, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 1881, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2142, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 2124, 2108, 1526, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5212, 0, 0, 0, 0, 0, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 1928, 0, 0, 0, 2048, 530, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6144, 6095, 6111, 6148, 6144, 6385, 6136] avg=6144.0000 max=6385 max_rank=6 min=5989 min_rank=0 imbalance_ratio=1.039225 max_load_reduction=42767 max_load_reduction_pct=0.870097 moved_to_echo=24813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[11, 0, 2479, 5460, 0, 0, 0, 1, 35, 0, 0, 0, 8148, 0, 0, 0, 15, 469, 0, 0, 0, 0, 0, 0, 108, 0, 11, 0, 0, 0, 0, 0, 8126, 1, 0, 0, 34, 0, 54, 0, 0, 7, 0, 0, 38, 8137, 8137, 0, 38, 5023, 0, 2, 3, 0, 1, 0, 0, 38, 953, 31, 1788, 0, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2349, 5536, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 4785, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 0, 1989, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2351, 5593, 0, 0, 0, 0, 3, 0, 0, 0, 8189, 0, 0, 0, 2, 405, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 2, 0, 68, 0, 0, 1, 0, 1, 3, 8184, 8187, 0, 3, 4958, 0, 2, 0, 0, 0, 0, 0, 3, 1006, 1, 1912, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2348, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 0, 0, 0, 0, 2125, 2108, 1526, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 6067, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4596, 0, 0, 0, 0, 0, 0, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 0, 2065, 0, 0, 0, 2048, 544, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 6144, 6114, 6149, 6165, 6144, 5801, 6338] avg=6144.0000 max=6338 max_rank=7 min=5801 min_rank=6 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24448 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2272, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 2125, 2108, 1526, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 6067, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4972, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 1881, 0, 0, 0, 2048, 553, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 6144, 6116, 6127, 6143, 6144, 6195, 6000] avg=6144.0000 max=6283 max_rank=0 min=6000 min_rank=7 imbalance_ratio=1.022624 max_load_reduction=42869 max_load_reduction_pct=0.872172 moved_to_echo=24482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2142, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 2124, 2108, 1526, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5212, 0, 0, 0, 0, 0, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 1928, 0, 0, 0, 2048, 530, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6144, 6095, 6111, 6148, 6144, 6385, 6136] avg=6144.0000 max=6385 max_rank=6 min=5989 min_rank=0 imbalance_ratio=1.039225 max_load_reduction=42767 max_load_reduction_pct=0.870097 moved_to_echo=24813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2200, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 2124, 2108, 1526, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 6068, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4973, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 0, 1936, 0, 0, 0, 2048, 529, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6038, 6144, 6143, 6152, 6125, 6144, 6143, 6263] avg=6144.0000 max=6263 max_rank=7 min=6038 min_rank=0 imbalance_ratio=1.019368 max_load_reduction=42889 max_load_reduction_pct=0.872579 moved_to_echo=24675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2351, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 405, 0, 0, 0, 0, 0, 0, 2124, 2107, 1525, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 0, 0, 0, 0, 0, 6081, 0, 0, 0, 2, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4958, 0, 2, 0, 0, 0, 0, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1006, 1, 1912, 0, 0, 0, 2046, 534, 521, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6228, 6146, 6163, 6153, 6151, 6143, 6145, 6023] avg=6144.0000 max=6228 max_rank=0 min=6023 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=42924 max_load_reduction_pct=0.873291 moved_to_echo=24462 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2349, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 2125, 2108, 1526, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 6067, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4785, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 0, 1989, 0, 0, 0, 2048, 529, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6186, 6144, 6138, 6159, 6153, 6144, 5955, 6273] avg=6144.0000 max=6273 max_rank=7 min=5955 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=42879 max_load_reduction_pct=0.872375 moved_to_echo=24525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2272, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 2125, 2108, 1526, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 6067, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4972, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 1881, 0, 0, 0, 2048, 553, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 6144, 6116, 6127, 6143, 6144, 6195, 6000] avg=6144.0000 max=6283 max_rank=0 min=6000 min_rank=7 imbalance_ratio=1.022624 max_load_reduction=42869 max_load_reduction_pct=0.872172 moved_to_echo=24482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 2479, 3773, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 6107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 469, 0, 0, 0, 0, 0, 0, 2111, 2092, 1520, 0, 0, 0, 0, 0, 108, 0, 11, 0, 0, 0, 0, 0, 6026, 0, 0, 0, 0, 0, 0, 0, 6034, 1, 0, 0, 34, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 38, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 5023, 0, 2, 3, 0, 1, 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 38, 953, 31, 1788, 0, 0, 4, 2037, 527, 521, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 6142, 6207, 6145, 6123, 6145, 6227, 5899] avg=6144.0000 max=6264 max_rank=0 min=5899 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=42888 max_load_reduction_pct=0.872559 moved_to_echo=24394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1993, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, 0, 0, 0, 2125, 2108, 1526, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 6067, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5120, 0, 0, 0, 0, 0, 0, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 0, 1836, 0, 0, 0, 2048, 534, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5867, 6144, 6176, 6156, 6144, 6144, 6301, 6220] avg=6144.0000 max=6301 max_rank=6 min=5867 min_rank=0 imbalance_ratio=1.025553 max_load_reduction=42851 max_load_reduction_pct=0.871806 moved_to_echo=24809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2200, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 2124, 2108, 1526, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 6068, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4973, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 0, 1936, 0, 0, 0, 2048, 529, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6038, 6144, 6143, 6152, 6125, 6144, 6143, 6263] avg=6144.0000 max=6263 max_rank=7 min=6038 min_rank=0 imbalance_ratio=1.019368 max_load_reduction=42889 max_load_reduction_pct=0.872579 moved_to_echo=24675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 3200, 0, 0, 0, 0, 0, 0, 17024, 16896, 12288, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 8960, 128, 15360, 0, 0, 128, 16384, 4352, 4224, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 18176, 31104, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 18176, 31104, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 39680, 0, 128, 128, 0, 128, 0, 9472, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[48640, 128, 0, 0, 128, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[768, 0, 128, 0, 0, 0, 0, 0, 48512, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3, 2, 0, 0, 2, 8189, 7855, 0, 0, 0, 320, 0, 0, 1, 0, 0, 3, 8189, 0, 0, 0, 1, 0, 0, 8185, 561, 1, 0, 2, 3, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 93, 0, 1, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 333, 0, 0, 7132, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7848, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 344, 0, 0, 7357, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 24, 30, 40, 16, 0, 2, 1, 8143, 7773, 0, 0, 1, 313, 0, 0, 7, 0, 1, 38, 8136, 4, 0, 7, 20, 0, 0, 8145, 620, 16, 0, 18, 34, 0, 0, 0, 0, 0, 8127, 1, 0, 6, 0, 201, 6, 62, 0, 6, 2, 0, 1, 0, 104, 0, 0, 0, 0, 340, 0, 0, 6907, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7868, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 325, 0, 0, 7474, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7868, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 325, 0, 0, 7367, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7847, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 345, 0, 0, 7280, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7850, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 344, 0, 0, 7254, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7848, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 344, 0, 0, 7357, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3, 2, 0, 0, 2, 8189, 7855, 0, 0, 0, 320, 0, 0, 1, 0, 0, 3, 8189, 0, 0, 0, 1, 0, 0, 8185, 561, 1, 0, 2, 3, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 93, 0, 1, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 333, 0, 0, 7132, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7873, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 319, 0, 0, 7385, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7847, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 345, 0, 0, 7280, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7868, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 325, 0, 0, 7367, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 24, 30, 40, 16, 0, 2, 1, 8143, 7773, 0, 0, 1, 313, 0, 0, 7, 0, 1, 38, 8136, 4, 0, 7, 20, 0, 0, 8145, 620, 16, 0, 18, 34, 0, 0, 0, 0, 0, 8127, 1, 0, 6, 0, 201, 6, 62, 0, 6, 2, 0, 1, 0, 104, 0, 0, 0, 0, 340, 0, 0, 6907, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5826, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 2047, 2022, 1477, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6136, 6144, 6064, 6142, 6151, 6157, 6224] avg=6144.0000 max=6224 max_rank=7 min=6064 min_rank=3 imbalance_ratio=1.013021 max_load_reduction=42928 max_load_reduction_pct=0.873372 moved_to_echo=24674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5841, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 2047, 2027, 1500, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 2470, 2059, 1537, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 5974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 6139, 6144, 6033, 6144, 6148, 6112, 6299] avg=6144.0000 max=6299 max_rank=7 min=6033 min_rank=3 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5825, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 2047, 2022, 1461, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 345, 0, 0, 5819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6175, 6144, 6113, 6145, 6126, 6151, 6164] avg=6144.0000 max=6175 max_rank=1 min=6113 min_rank=3 imbalance_ratio=1.005046 max_load_reduction=42977 max_load_reduction_pct=0.874369 moved_to_echo=24586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3, 2, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5831, 0, 0, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5717, 561, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1, 0, 2046, 2024, 1432, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 2468, 2057, 1536, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 5700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6153, 6146, 6280, 6144, 6107, 6150, 6033] avg=6144.0000 max=6280 max_rank=3 min=6033 min_rank=7 imbalance_ratio=1.022135 max_load_reduction=42872 max_load_reduction_pct=0.872233 moved_to_echo=24434 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5826, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 2047, 2022, 1477, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6136, 6144, 6064, 6142, 6151, 6157, 6224] avg=6144.0000 max=6224 max_rank=7 min=6064 min_rank=3 imbalance_ratio=1.013021 max_load_reduction=42928 max_load_reduction_pct=0.873372 moved_to_echo=24674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5828, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 2047, 2022, 1456, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 5798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6170, 6144, 6185, 6145, 6107, 6125, 6142] avg=6144.0000 max=6185 max_rank=3 min=6107 min_rank=5 imbalance_ratio=1.006673 max_load_reduction=42967 max_load_reduction_pct=0.874166 moved_to_echo=24519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5841, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 2047, 2027, 1479, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6145, 6144, 6071, 6144, 6144, 6157, 6213] avg=6144.0000 max=6213 max_rank=7 min=6071 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=42939 max_load_reduction_pct=0.873596 moved_to_echo=24658 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5825, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 2047, 2022, 1461, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 345, 0, 0, 5819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6175, 6144, 6113, 6145, 6126, 6151, 6164] avg=6144.0000 max=6175 max_rank=1 min=6113 min_rank=3 imbalance_ratio=1.005046 max_load_reduction=42977 max_load_reduction_pct=0.874369 moved_to_echo=24586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5845, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 2047, 2028, 1483, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 5902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6153, 6144, 6055, 6145, 6145, 6155, 6221] avg=6144.0000 max=6221 max_rank=7 min=6055 min_rank=3 imbalance_ratio=1.012533 max_load_reduction=42931 max_load_reduction_pct=0.873433 moved_to_echo=24666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 30, 40, 16, 0, 2, 6098, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5766, 0, 0, 1, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 38, 6092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 20, 0, 0, 5688, 620, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 34, 0, 0, 0, 0, 0, 6091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 201, 6, 62, 0, 2036, 2007, 1391, 514, 0, 0, 0, 0, 6, 2, 0, 1, 0, 104, 0, 0, 2457, 2045, 1530, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 5516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6210, 6081, 6142, 6351, 6143, 6224, 6145, 5856] avg=6144.0000 max=6351 max_rank=3 min=5856 min_rank=7 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5845, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 2047, 2028, 1483, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 5902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6153, 6144, 6055, 6145, 6145, 6155, 6221] avg=6144.0000 max=6221 max_rank=7 min=6055 min_rank=3 imbalance_ratio=1.012533 max_load_reduction=42931 max_load_reduction_pct=0.873433 moved_to_echo=24666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 768, 128, 128, 0, 16384, 16256, 11776, 4096, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 768, 0, 0, 19840, 16512, 12288, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 45824, 3456, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 46720, 0, 0, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 2688, 0, 0, 46592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True - [2026-08-26 15:39:21] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 808.7 | throughput per GPU (TFLOP/s/GPU): 157.8 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.975649E+00 | loss scale: 1.0 | grad norm: 3.681 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True + [2026-08-26 16:35:23] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 819.5 | throughput per GPU (TFLOP/s/GPU): 155.7 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.975649E+00 | loss scale: 1.0 | grad norm: 3.681 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1470, 390, 680, 1019, 193, 1012, 797, 481, 1603, 391, 746, 287, 97, 1897, 790, 1034, 678, 992, 779, 605, 232, 450, 221, 1346, 399, 659, 727, 310, 793, 134, 771, 1086, 849, 869, 1417, 1250, 750, 198, 650, 863, 490, 570, 665, 548, 944, 996, 503, 369, 590, 553, 684, 716, 949, 1622, 651, 849, 1780, 320, 1536, 774, 195, 657, 1314, 962] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1536, 393, 544, 888, 168, 922, 879, 591, 1623, 322, 820, 311, 108, 1937, 839, 911, 677, 897, 791, 642, 182, 475, 220, 1291, 393, 740, 836, 398, 765, 125, 737, 1105, 848, 741, 1435, 1321, 709, 229, 700, 925, 470, 558, 660, 456, 1000, 973, 463, 343, 690, 556, 699, 644, 911, 1721, 519, 730, 1862, 342, 1519, 816, 165, 701, 1414, 966] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1502, 353, 666, 875, 155, 1200, 999, 549, 1536, 267, 846, 270, 120, 1811, 934, 1038, 657, 901, 917, 729, 198, 446, 205, 1377, 426, 714, 709, 346, 905, 142, 745, 1150, 780, 774, 1107, 1229, 706, 199, 625, 907, 426, 577, 593, 494, 982, 1048, 572, 368, 618, 466, 678, 636, 1011, 1607, 541, 742, 1881, 370, 1550, 772, 170, 761, 1394, 880] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1439, 374, 614, 848, 196, 1074, 829, 473, 1555, 295, 754, 297, 116, 1979, 802, 950, 717, 896, 882, 645, 201, 472, 241, 1288, 398, 617, 743, 338, 751, 166, 767, 1141, 860, 830, 1542, 1305, 704, 217, 720, 874, 372, 507, 614, 523, 968, 1049, 494, 302, 606, 535, 699, 640, 948, 1682, 588, 829, 1941, 375, 1647, 696, 173, 751, 1356, 947] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1470, 390, 680, 1019, 193, 1012, 797, 481, 1603, 391, 746, 287, 97, 1897, 790, 1034, 678, 992, 779, 605, 232, 450, 221, 1346, 399, 659, 727, 310, 793, 134, 771, 1086, 849, 869, 1417, 1250, 750, 198, 650, 863, 490, 570, 665, 548, 944, 996, 503, 369, 590, 553, 684, 716, 949, 1622, 651, 849, 1780, 320, 1536, 774, 195, 657, 1314, 962] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1426, 353, 675, 937, 168, 1022, 872, 563, 1571, 263, 776, 280, 87, 1992, 900, 917, 770, 958, 827, 680, 188, 429, 247, 1221, 355, 683, 809, 388, 742, 114, 807, 1493, 838, 834, 1519, 1310, 486, 227, 736, 868, 405, 556, 626, 503, 1004, 973, 390, 320, 674, 520, 621, 616, 966, 1780, 485, 665, 1877, 318, 1621, 746, 191, 704, 1408, 852] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1436, 371, 553, 773, 172, 1122, 1011, 473, 1539, 333, 879, 392, 102, 1901, 884, 962, 674, 862, 849, 744, 184, 452, 209, 1110, 341, 647, 803, 415, 827, 199, 645, 1282, 913, 786, 1487, 1352, 476, 232, 751, 804, 475, 551, 635, 462, 977, 950, 463, 330, 639, 559, 634, 598, 997, 1582, 523, 799, 1655, 419, 1699, 829, 169, 861, 1480, 919] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1439, 374, 614, 848, 196, 1074, 829, 473, 1555, 295, 754, 297, 116, 1979, 802, 950, 717, 896, 882, 645, 201, 472, 241, 1288, 398, 617, 743, 338, 751, 166, 767, 1141, 860, 830, 1542, 1305, 704, 217, 720, 874, 372, 507, 614, 523, 968, 1049, 494, 302, 606, 535, 699, 640, 948, 1682, 588, 829, 1941, 375, 1647, 696, 173, 751, 1356, 947] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1432, 334, 628, 992, 233, 1055, 955, 530, 1610, 291, 872, 321, 98, 1924, 924, 1009, 672, 903, 877, 716, 188, 451, 230, 1172, 339, 707, 720, 437, 799, 114, 726, 1044, 820, 727, 1451, 1250, 803, 244, 583, 915, 475, 623, 603, 588, 911, 928, 458, 340, 626, 481, 757, 704, 952, 1708, 578, 777, 1725, 312, 1570, 819, 179, 687, 1311, 944] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1484, 280, 523, 822, 163, 1019, 922, 398, 1719, 340, 798, 237, 82, 2054, 881, 1000, 736, 913, 958, 721, 186, 478, 222, 1242, 392, 683, 737, 519, 798, 184, 655, 1093, 793, 717, 1526, 1265, 688, 241, 769, 935, 405, 502, 619, 465, 1047, 1114, 467, 364, 709, 544, 681, 545, 819, 1704, 442, 715, 1736, 350, 1652, 701, 252, 781, 1423, 942] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1536, 393, 544, 888, 168, 922, 879, 591, 172, 0, 0, 0, 0, 0, 0, 0, 1623, 322, 820, 311, 108, 1179, 839, 911, 0, 0, 0, 0, 0, 0, 0, 0, 677, 897, 791, 642, 182, 475, 220, 1291, 665, 104, 86, 0, 0, 0, 0, 0, 393, 740, 836, 398, 765, 125, 737, 1105, 672, 417, 0, 0, 0, 0, 0, 0, 848, 741, 770, 1321, 709, 229, 700, 925, 0, 0, 0, 0, 0, 0, 0, 0, 470, 558, 660, 456, 1000, 973, 463, 343, 1252, 0, 0, 0, 0, 0, 0, 0, 690, 556, 699, 644, 911, 1445, 519, 730, 0, 0, 0, 0, 0, 0, 0, 0, 193, 342, 1519, 816, 165, 701, 1414, 966, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6093, 6113, 6030, 6188, 6243, 6175, 6194, 6116] avg=6144.0000 max=6243 max_rank=4 min=6030 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=42909 max_load_reduction_pct=0.872986 moved_to_echo=24728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1439, 374, 614, 848, 196, 1074, 829, 473, 168, 0, 0, 0, 0, 0, 0, 0, 1555, 295, 754, 297, 116, 1205, 802, 950, 0, 0, 0, 0, 0, 0, 0, 0, 717, 896, 882, 645, 201, 472, 241, 1288, 715, 101, 88, 0, 0, 0, 0, 0, 398, 617, 743, 338, 751, 166, 767, 1141, 686, 435, 0, 0, 0, 0, 0, 0, 860, 830, 827, 1305, 704, 217, 720, 874, 0, 0, 0, 0, 0, 0, 0, 0, 372, 507, 614, 523, 968, 1049, 494, 302, 1305, 0, 0, 0, 0, 0, 0, 0, 606, 535, 699, 640, 948, 1413, 588, 829, 0, 0, 0, 0, 0, 0, 0, 0, 201, 375, 1647, 696, 173, 751, 1356, 947, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6015, 5974, 6246, 6042, 6337, 6134, 6258, 6146] avg=6144.0000 max=6337 max_rank=4 min=5974 min_rank=1 imbalance_ratio=1.031413 max_load_reduction=42815 max_load_reduction_pct=0.871073 moved_to_echo=24875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1436, 371, 553, 773, 172, 1122, 1011, 473, 158, 0, 0, 0, 0, 0, 0, 0, 1539, 333, 879, 392, 102, 1157, 884, 962, 0, 0, 0, 0, 0, 0, 0, 0, 674, 862, 849, 744, 184, 452, 209, 1110, 689, 95, 85, 0, 0, 0, 0, 0, 341, 647, 803, 415, 827, 199, 645, 1282, 659, 371, 0, 0, 0, 0, 0, 0, 913, 786, 798, 1352, 476, 232, 751, 804, 0, 0, 0, 0, 0, 0, 0, 0, 475, 551, 635, 462, 977, 950, 463, 330, 1113, 0, 0, 0, 0, 0, 0, 0, 639, 559, 634, 598, 997, 1329, 523, 799, 0, 0, 0, 0, 0, 0, 0, 0, 171, 419, 1699, 829, 169, 861, 1480, 919, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6069, 6248, 5953, 6189, 6112, 5956, 6078, 6547] avg=6144.0000 max=6547 max_rank=7 min=5953 min_rank=2 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=24693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1484, 280, 523, 822, 163, 1019, 922, 398, 171, 0, 0, 0, 0, 0, 0, 0, 1719, 340, 798, 237, 82, 1251, 881, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 736, 913, 958, 721, 186, 478, 222, 1242, 707, 103, 91, 0, 0, 0, 0, 0, 392, 683, 737, 519, 798, 184, 655, 1093, 712, 389, 0, 0, 0, 0, 0, 0, 793, 717, 819, 1265, 688, 241, 769, 935, 0, 0, 0, 0, 0, 0, 0, 0, 405, 502, 619, 465, 1047, 1114, 467, 364, 1167, 0, 0, 0, 0, 0, 0, 0, 709, 544, 681, 545, 819, 1430, 442, 715, 0, 0, 0, 0, 0, 0, 0, 0, 180, 350, 1652, 701, 252, 781, 1423, 942, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5782, 6308, 6357, 6162, 6227, 6150, 5885, 6281] avg=6144.0000 max=6357 max_rank=2 min=5782 min_rank=0 imbalance_ratio=1.034668 max_load_reduction=42795 max_load_reduction_pct=0.870667 moved_to_echo=24543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1502, 353, 666, 875, 155, 1200, 999, 549, 166, 0, 0, 0, 0, 0, 0, 0, 1536, 267, 846, 270, 120, 1093, 934, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 657, 901, 917, 729, 198, 446, 205, 1377, 517, 101, 85, 0, 0, 0, 0, 0, 426, 714, 709, 346, 905, 142, 745, 1150, 633, 424, 0, 0, 0, 0, 0, 0, 780, 774, 590, 1229, 706, 199, 625, 907, 0, 0, 0, 0, 0, 0, 0, 0, 426, 577, 593, 494, 982, 1048, 572, 368, 1270, 0, 0, 0, 0, 0, 0, 0, 618, 466, 678, 636, 1011, 1340, 541, 742, 0, 0, 0, 0, 0, 0, 0, 0, 187, 370, 1550, 772, 170, 761, 1394, 880, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 6104, 6133, 6194, 5810, 6330, 6032, 6084] avg=6144.0000 max=6465 max_rank=0 min=5810 min_rank=4 imbalance_ratio=1.052246 max_load_reduction=42687 max_load_reduction_pct=0.868469 moved_to_echo=24256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1432, 334, 628, 992, 233, 1055, 955, 530, 171, 0, 0, 0, 0, 0, 0, 0, 1610, 291, 872, 321, 98, 1171, 924, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 672, 903, 877, 716, 188, 451, 230, 1172, 673, 103, 86, 0, 0, 0, 0, 0, 339, 707, 720, 437, 799, 114, 726, 1044, 667, 386, 0, 0, 0, 0, 0, 0, 820, 727, 778, 1250, 803, 244, 583, 915, 0, 0, 0, 0, 0, 0, 0, 0, 475, 623, 603, 588, 911, 928, 458, 340, 1160, 0, 0, 0, 0, 0, 0, 0, 626, 481, 757, 704, 952, 1434, 578, 777, 0, 0, 0, 0, 0, 0, 0, 0, 179, 312, 1570, 819, 179, 687, 1311, 944, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6330, 6296, 6071, 5939, 6120, 6086, 6309, 6001] avg=6144.0000 max=6330 max_rank=0 min=5939 min_rank=3 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1470, 390, 680, 1019, 193, 1012, 797, 481, 162, 0, 0, 0, 0, 0, 0, 0, 1603, 391, 746, 287, 97, 1155, 790, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 678, 992, 779, 605, 232, 450, 221, 1346, 657, 98, 84, 0, 0, 0, 0, 0, 399, 659, 727, 310, 793, 134, 771, 1086, 658, 399, 0, 0, 0, 0, 0, 0, 849, 869, 760, 1250, 750, 198, 650, 863, 0, 0, 0, 0, 0, 0, 0, 0, 490, 570, 665, 548, 944, 996, 503, 369, 1197, 0, 0, 0, 0, 0, 0, 0, 590, 553, 684, 716, 949, 1362, 651, 849, 0, 0, 0, 0, 0, 0, 0, 0, 184, 320, 1536, 774, 195, 657, 1314, 962, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6204, 6103, 6142, 5936, 6189, 6282, 6354, 5942] avg=6144.0000 max=6354 max_rank=6 min=5936 min_rank=3 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1436, 371, 553, 773, 172, 1122, 1011, 473, 158, 0, 0, 0, 0, 0, 0, 0, 1539, 333, 879, 392, 102, 1157, 884, 962, 0, 0, 0, 0, 0, 0, 0, 0, 674, 862, 849, 744, 184, 452, 209, 1110, 689, 95, 85, 0, 0, 0, 0, 0, 341, 647, 803, 415, 827, 199, 645, 1282, 659, 371, 0, 0, 0, 0, 0, 0, 913, 786, 798, 1352, 476, 232, 751, 804, 0, 0, 0, 0, 0, 0, 0, 0, 475, 551, 635, 462, 977, 950, 463, 330, 1113, 0, 0, 0, 0, 0, 0, 0, 639, 559, 634, 598, 997, 1329, 523, 799, 0, 0, 0, 0, 0, 0, 0, 0, 171, 419, 1699, 829, 169, 861, 1480, 919, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6069, 6248, 5953, 6189, 6112, 5956, 6078, 6547] avg=6144.0000 max=6547 max_rank=7 min=5953 min_rank=2 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=24693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1439, 374, 614, 848, 196, 1074, 829, 473, 168, 0, 0, 0, 0, 0, 0, 0, 1555, 295, 754, 297, 116, 1205, 802, 950, 0, 0, 0, 0, 0, 0, 0, 0, 717, 896, 882, 645, 201, 472, 241, 1288, 715, 101, 88, 0, 0, 0, 0, 0, 398, 617, 743, 338, 751, 166, 767, 1141, 686, 435, 0, 0, 0, 0, 0, 0, 860, 830, 827, 1305, 704, 217, 720, 874, 0, 0, 0, 0, 0, 0, 0, 0, 372, 507, 614, 523, 968, 1049, 494, 302, 1305, 0, 0, 0, 0, 0, 0, 0, 606, 535, 699, 640, 948, 1413, 588, 829, 0, 0, 0, 0, 0, 0, 0, 0, 201, 375, 1647, 696, 173, 751, 1356, 947, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6015, 5974, 6246, 6042, 6337, 6134, 6258, 6146] avg=6144.0000 max=6337 max_rank=4 min=5974 min_rank=1 imbalance_ratio=1.031413 max_load_reduction=42815 max_load_reduction_pct=0.871073 moved_to_echo=24875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1426, 353, 675, 937, 168, 1022, 872, 563, 178, 0, 0, 0, 0, 0, 0, 0, 1571, 263, 776, 280, 87, 1212, 900, 917, 0, 0, 0, 0, 0, 0, 0, 0, 770, 958, 827, 680, 188, 429, 247, 1221, 704, 107, 89, 0, 0, 0, 0, 0, 355, 683, 809, 388, 742, 114, 807, 1493, 691, 420, 0, 0, 0, 0, 0, 0, 838, 834, 815, 1310, 486, 227, 736, 868, 0, 0, 0, 0, 0, 0, 0, 0, 405, 556, 626, 503, 1004, 973, 390, 320, 1262, 0, 0, 0, 0, 0, 0, 0, 674, 520, 621, 616, 966, 1495, 485, 665, 0, 0, 0, 0, 0, 0, 0, 0, 195, 318, 1621, 746, 191, 704, 1408, 852, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6006, 6220, 6502, 6114, 6039, 6042, 6035] avg=6144.0000 max=6502 max_rank=3 min=6006 min_rank=1 imbalance_ratio=1.058268 max_load_reduction=42650 max_load_reduction_pct=0.867716 moved_to_echo=24230 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1432, 334, 628, 992, 233, 1055, 955, 530, 171, 0, 0, 0, 0, 0, 0, 0, 1610, 291, 872, 321, 98, 1171, 924, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 672, 903, 877, 716, 188, 451, 230, 1172, 673, 103, 86, 0, 0, 0, 0, 0, 339, 707, 720, 437, 799, 114, 726, 1044, 667, 386, 0, 0, 0, 0, 0, 0, 820, 727, 778, 1250, 803, 244, 583, 915, 0, 0, 0, 0, 0, 0, 0, 0, 475, 623, 603, 588, 911, 928, 458, 340, 1160, 0, 0, 0, 0, 0, 0, 0, 626, 481, 757, 704, 952, 1434, 578, 777, 0, 0, 0, 0, 0, 0, 0, 0, 179, 312, 1570, 819, 179, 687, 1311, 944, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6330, 6296, 6071, 5939, 6120, 6086, 6309, 6001] avg=6144.0000 max=6330 max_rank=0 min=5939 min_rank=3 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1502, 353, 666, 875, 155, 1200, 999, 549, 166, 0, 0, 0, 0, 0, 0, 0, 1536, 267, 846, 270, 120, 1093, 934, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 657, 901, 917, 729, 198, 446, 205, 1377, 517, 101, 85, 0, 0, 0, 0, 0, 426, 714, 709, 346, 905, 142, 745, 1150, 633, 424, 0, 0, 0, 0, 0, 0, 780, 774, 590, 1229, 706, 199, 625, 907, 0, 0, 0, 0, 0, 0, 0, 0, 426, 577, 593, 494, 982, 1048, 572, 368, 1270, 0, 0, 0, 0, 0, 0, 0, 618, 466, 678, 636, 1011, 1340, 541, 742, 0, 0, 0, 0, 0, 0, 0, 0, 187, 370, 1550, 772, 170, 761, 1394, 880, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 6104, 6133, 6194, 5810, 6330, 6032, 6084] avg=6144.0000 max=6465 max_rank=0 min=5810 min_rank=4 imbalance_ratio=1.052246 max_load_reduction=42687 max_load_reduction_pct=0.868469 moved_to_echo=24256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1484, 280, 523, 822, 163, 1019, 922, 398, 171, 0, 0, 0, 0, 0, 0, 0, 1719, 340, 798, 237, 82, 1251, 881, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 736, 913, 958, 721, 186, 478, 222, 1242, 707, 103, 91, 0, 0, 0, 0, 0, 392, 683, 737, 519, 798, 184, 655, 1093, 712, 389, 0, 0, 0, 0, 0, 0, 793, 717, 819, 1265, 688, 241, 769, 935, 0, 0, 0, 0, 0, 0, 0, 0, 405, 502, 619, 465, 1047, 1114, 467, 364, 1167, 0, 0, 0, 0, 0, 0, 0, 709, 544, 681, 545, 819, 1430, 442, 715, 0, 0, 0, 0, 0, 0, 0, 0, 180, 350, 1652, 701, 252, 781, 1423, 942, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5782, 6308, 6357, 6162, 6227, 6150, 5885, 6281] avg=6144.0000 max=6357 max_rank=2 min=5782 min_rank=0 imbalance_ratio=1.034668 max_load_reduction=42795 max_load_reduction_pct=0.870667 moved_to_echo=24543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 4480, 5120, 4096, 7936, 8064, 3840, 2816, 9728, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6784, 6400, 6272, 10368, 5376, 1792, 5632, 7168, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11776, 2944, 4992, 7168, 1536, 8448, 7296, 4096, 1408, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12800, 2560, 6528, 2432, 896, 9472, 7040, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5248, 4224, 5504, 5120, 7680, 11264, 4352, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11776, 2944, 4992, 7168, 1536, 8448, 7296, 4096, 1408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6784, 6400, 6272, 10368, 5376, 1792, 5632, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 4480, 5120, 4096, 7936, 8064, 3840, 2816, 9728, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 2816, 12800, 6272, 1536, 6016, 11136, 7424, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5632, 7424, 6912, 5504, 1664, 3712, 1920, 10112, 5376, 896, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5248, 4224, 5504, 5120, 7680, 11264, 4352, 6144, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 5504, 6144, 3200, 6400, 1280, 5888, 9472, 5504, 3328, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[564, 697, 299, 435, 2185, 88, 308, 1382, 40, 854, 2177, 365, 614, 874, 485, 2260, 843, 1224, 699, 369, 1151, 228, 1856, 1596, 798, 749, 782, 1453, 27, 149, 103, 68, 257, 40, 276, 44, 1122, 526, 800, 778, 1918, 1203, 424, 979, 128, 1318, 7, 142, 2661, 967, 2212, 0, 38, 997, 182, 1224, 1326, 255, 1089, 453, 755, 561, 346, 402] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[544, 688, 296, 403, 2206, 123, 382, 1124, 39, 969, 2261, 379, 628, 976, 486, 2199, 838, 1226, 830, 425, 1113, 222, 1787, 1442, 776, 669, 800, 1420, 21, 151, 161, 158, 337, 46, 291, 64, 1071, 579, 835, 785, 1824, 1028, 471, 854, 155, 1207, 8, 184, 2327, 1059, 2036, 0, 45, 1101, 234, 1534, 1403, 308, 943, 464, 872, 585, 394, 366] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[482, 809, 355, 409, 1673, 200, 363, 1138, 47, 871, 2054, 306, 781, 1021, 663, 2132, 791, 1344, 855, 369, 1460, 168, 1999, 1208, 784, 632, 709, 1339, 15, 120, 156, 139, 312, 48, 340, 72, 1102, 829, 938, 839, 2040, 1182, 642, 1127, 170, 1062, 18, 184, 2017, 986, 1621, 2, 45, 1274, 243, 1586, 1267, 220, 891, 492, 961, 558, 330, 362] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[534, 693, 424, 462, 2020, 141, 367, 1141, 69, 849, 2095, 307, 713, 1073, 596, 2086, 906, 1327, 754, 422, 1411, 176, 1843, 1331, 758, 716, 799, 1306, 13, 150, 116, 115, 272, 31, 304, 62, 1129, 788, 852, 825, 1888, 1094, 561, 1038, 154, 1093, 8, 190, 1880, 912, 1905, 2, 46, 1178, 266, 1514, 1348, 295, 957, 491, 1010, 525, 383, 438] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[549, 703, 372, 408, 2367, 127, 345, 1132, 28, 922, 2216, 348, 671, 1036, 521, 2313, 745, 1229, 837, 319, 1246, 225, 1947, 1299, 801, 699, 780, 1299, 17, 133, 137, 138, 300, 57, 262, 46, 1030, 614, 813, 869, 1749, 1178, 475, 980, 165, 1206, 15, 217, 2330, 970, 2091, 0, 31, 1182, 216, 1415, 1278, 290, 904, 488, 828, 500, 426, 318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[612, 840, 349, 426, 1965, 114, 375, 1216, 34, 908, 2203, 334, 658, 1019, 516, 2193, 750, 1345, 882, 278, 1275, 159, 1912, 1397, 765, 653, 763, 1300, 25, 80, 103, 141, 320, 30, 318, 46, 1165, 701, 890, 816, 1931, 1087, 515, 950, 139, 1203, 7, 136, 2310, 1015, 1893, 0, 20, 1191, 257, 1607, 1369, 210, 944, 483, 865, 499, 302, 343] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[518, 677, 345, 410, 2088, 162, 375, 989, 49, 826, 2193, 305, 685, 952, 515, 2213, 759, 1351, 807, 287, 1342, 166, 1937, 1472, 757, 742, 838, 1351, 16, 103, 138, 122, 290, 38, 290, 45, 1220, 609, 896, 836, 1924, 1149, 526, 1116, 181, 1227, 13, 142, 2074, 1068, 1806, 0, 41, 1185, 202, 1485, 1306, 267, 1000, 489, 932, 576, 363, 366] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[544, 688, 296, 403, 2206, 123, 382, 1124, 39, 969, 2261, 379, 628, 976, 486, 2199, 838, 1226, 830, 425, 1113, 222, 1787, 1442, 776, 669, 800, 1420, 21, 151, 161, 158, 337, 46, 291, 64, 1071, 579, 835, 785, 1824, 1028, 471, 854, 155, 1207, 8, 184, 2327, 1059, 2036, 0, 45, 1101, 234, 1534, 1403, 308, 943, 464, 872, 585, 394, 366] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[549, 703, 372, 408, 2367, 127, 345, 1132, 28, 922, 2216, 348, 671, 1036, 521, 2313, 745, 1229, 837, 319, 1246, 225, 1947, 1299, 801, 699, 780, 1299, 17, 133, 137, 138, 300, 57, 262, 46, 1030, 614, 813, 869, 1749, 1178, 475, 980, 165, 1206, 15, 217, 2330, 970, 2091, 0, 31, 1182, 216, 1415, 1278, 290, 904, 488, 828, 500, 426, 318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[483, 843, 470, 381, 2036, 171, 359, 1171, 45, 877, 2245, 332, 652, 952, 640, 2312, 751, 1374, 805, 279, 1515, 193, 1966, 1298, 794, 655, 804, 1177, 13, 130, 104, 138, 310, 44, 320, 75, 1154, 827, 812, 859, 1836, 1202, 553, 988, 168, 1068, 11, 173, 2044, 1058, 1882, 1, 34, 1181, 216, 1445, 1160, 271, 896, 465, 958, 526, 324, 326] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[544, 688, 296, 403, 2206, 123, 382, 1124, 352, 0, 0, 0, 0, 0, 0, 0, 39, 969, 2261, 379, 628, 976, 486, 483, 0, 0, 0, 0, 0, 0, 0, 0, 838, 1226, 830, 425, 1113, 222, 5, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 776, 669, 800, 1420, 21, 151, 161, 158, 1782, 273, 0, 0, 0, 0, 0, 0, 337, 46, 291, 64, 1071, 579, 835, 785, 1729, 315, 0, 0, 0, 0, 0, 0, 1824, 1028, 471, 854, 155, 1207, 8, 184, 77, 0, 0, 0, 0, 0, 0, 0, 325, 1059, 2036, 0, 45, 1101, 234, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 308, 943, 464, 872, 585, 394, 366, 972, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 6221, 6101, 6211, 6052, 5808, 6334, 6307] avg=6144.0000 max=6334 max_rank=6 min=5808 min_rank=5 imbalance_ratio=1.030924 max_load_reduction=42818 max_load_reduction_pct=0.871134 moved_to_echo=24501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[612, 840, 349, 426, 1965, 114, 375, 1216, 34, 908, 2203, 334, 658, 1019, 516, 2193, 750, 1345, 882, 278, 1275, 159, 1912, 1397, 765, 653, 763, 1300, 25, 80, 103, 141, 320, 30, 318, 46, 1165, 701, 890, 816, 1931, 1087, 515, 950, 139, 1203, 7, 136, 2310, 1015, 1893, 0, 20, 1191, 257, 1607, 1369, 210, 944, 483, 865, 499, 302, 343] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[534, 693, 424, 462, 2020, 141, 367, 1141, 69, 849, 2095, 307, 713, 1073, 596, 2086, 906, 1327, 754, 422, 1411, 176, 1843, 1331, 758, 716, 799, 1306, 13, 150, 116, 115, 272, 31, 304, 62, 1129, 788, 852, 825, 1888, 1094, 561, 1038, 154, 1093, 8, 190, 1880, 912, 1905, 2, 46, 1178, 266, 1514, 1348, 295, 957, 491, 1010, 525, 383, 438] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[564, 697, 299, 435, 2185, 88, 308, 1382, 362, 0, 0, 0, 0, 0, 0, 0, 40, 854, 2177, 365, 614, 874, 485, 496, 0, 0, 0, 0, 0, 0, 0, 0, 843, 1224, 699, 369, 1151, 228, 5, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 798, 749, 782, 1453, 27, 149, 103, 68, 1851, 313, 0, 0, 0, 0, 0, 0, 257, 40, 276, 44, 1122, 526, 800, 778, 1978, 324, 0, 0, 0, 0, 0, 0, 1918, 1203, 424, 979, 128, 1318, 7, 142, 79, 0, 0, 0, 0, 0, 0, 0, 370, 967, 2212, 0, 38, 997, 182, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 1326, 255, 1089, 453, 755, 561, 346, 402, 999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6320, 5905, 6115, 6293, 6145, 6198, 5990, 6186] avg=6144.0000 max=6320 max_rank=0 min=5905 min_rank=1 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[518, 677, 345, 410, 2088, 162, 375, 989, 354, 0, 0, 0, 0, 0, 0, 0, 49, 826, 2193, 305, 685, 952, 515, 487, 0, 0, 0, 0, 0, 0, 0, 0, 759, 1351, 807, 287, 1342, 166, 5, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 757, 742, 838, 1351, 16, 103, 138, 122, 1932, 244, 0, 0, 0, 0, 0, 0, 290, 38, 290, 45, 1220, 609, 896, 836, 1541, 317, 0, 0, 0, 0, 0, 0, 1924, 1149, 526, 1116, 181, 1227, 13, 142, 77, 0, 0, 0, 0, 0, 0, 0, 289, 1068, 1806, 0, 41, 1185, 202, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 267, 1000, 489, 932, 576, 363, 366, 978, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5918, 6012, 6189, 6243, 6082, 6355, 6076, 6277] avg=6144.0000 max=6355 max_rank=5 min=5918 min_rank=0 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[549, 703, 372, 408, 2367, 127, 345, 1132, 370, 0, 0, 0, 0, 0, 0, 0, 28, 922, 2216, 348, 671, 1036, 521, 509, 0, 0, 0, 0, 0, 0, 0, 0, 745, 1229, 837, 319, 1246, 225, 5, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 801, 699, 780, 1299, 17, 133, 137, 138, 1942, 274, 0, 0, 0, 0, 0, 0, 300, 57, 262, 46, 1030, 614, 813, 869, 1732, 331, 0, 0, 0, 0, 0, 0, 1749, 1178, 475, 980, 165, 1206, 15, 217, 81, 0, 0, 0, 0, 0, 0, 0, 324, 970, 2091, 0, 31, 1182, 216, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 1278, 290, 904, 488, 828, 500, 426, 318, 1022, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6373, 6251, 5905, 6220, 6054, 6066, 6229, 6054] avg=6144.0000 max=6373 max_rank=0 min=5905 min_rank=2 imbalance_ratio=1.037272 max_load_reduction=42779 max_load_reduction_pct=0.870341 moved_to_echo=24403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[544, 688, 296, 403, 2206, 123, 382, 1124, 352, 0, 0, 0, 0, 0, 0, 0, 39, 969, 2261, 379, 628, 976, 486, 483, 0, 0, 0, 0, 0, 0, 0, 0, 838, 1226, 830, 425, 1113, 222, 5, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 776, 669, 800, 1420, 21, 151, 161, 158, 1782, 273, 0, 0, 0, 0, 0, 0, 337, 46, 291, 64, 1071, 579, 835, 785, 1729, 315, 0, 0, 0, 0, 0, 0, 1824, 1028, 471, 854, 155, 1207, 8, 184, 77, 0, 0, 0, 0, 0, 0, 0, 325, 1059, 2036, 0, 45, 1101, 234, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 308, 943, 464, 872, 585, 394, 366, 972, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 6221, 6101, 6211, 6052, 5808, 6334, 6307] avg=6144.0000 max=6334 max_rank=6 min=5808 min_rank=5 imbalance_ratio=1.030924 max_load_reduction=42818 max_load_reduction_pct=0.871134 moved_to_echo=24501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[482, 809, 355, 409, 1673, 200, 363, 1138, 341, 0, 0, 0, 0, 0, 0, 0, 47, 871, 2054, 306, 781, 1021, 663, 470, 0, 0, 0, 0, 0, 0, 0, 0, 791, 1344, 855, 369, 1460, 168, 6, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 784, 632, 709, 1339, 15, 120, 156, 139, 1993, 237, 0, 0, 0, 0, 0, 0, 312, 48, 340, 72, 1102, 829, 938, 839, 1499, 305, 0, 0, 0, 0, 0, 0, 2040, 1182, 642, 1127, 170, 1062, 18, 184, 74, 0, 0, 0, 0, 0, 0, 0, 281, 986, 1621, 2, 45, 1274, 243, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 1267, 220, 891, 492, 961, 558, 330, 362, 942, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5770, 6213, 6201, 6124, 6284, 6499, 6038, 6023] avg=6144.0000 max=6499 max_rank=5 min=5770 min_rank=0 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=24844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[518, 677, 345, 410, 2088, 162, 375, 989, 354, 0, 0, 0, 0, 0, 0, 0, 49, 826, 2193, 305, 685, 952, 515, 487, 0, 0, 0, 0, 0, 0, 0, 0, 759, 1351, 807, 287, 1342, 166, 5, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 757, 742, 838, 1351, 16, 103, 138, 122, 1932, 244, 0, 0, 0, 0, 0, 0, 290, 38, 290, 45, 1220, 609, 896, 836, 1541, 317, 0, 0, 0, 0, 0, 0, 1924, 1149, 526, 1116, 181, 1227, 13, 142, 77, 0, 0, 0, 0, 0, 0, 0, 289, 1068, 1806, 0, 41, 1185, 202, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 267, 1000, 489, 932, 576, 363, 366, 978, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5918, 6012, 6189, 6243, 6082, 6355, 6076, 6277] avg=6144.0000 max=6355 max_rank=5 min=5918 min_rank=0 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[483, 843, 470, 381, 2036, 171, 359, 1171, 370, 0, 0, 0, 0, 0, 0, 0, 45, 877, 2245, 332, 652, 952, 640, 508, 0, 0, 0, 0, 0, 0, 0, 0, 751, 1374, 805, 279, 1515, 193, 6, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 794, 655, 804, 1177, 13, 130, 104, 138, 1960, 240, 0, 0, 0, 0, 0, 0, 310, 44, 320, 75, 1154, 827, 812, 859, 1519, 331, 0, 0, 0, 0, 0, 0, 1836, 1202, 553, 988, 168, 1068, 11, 173, 81, 0, 0, 0, 0, 0, 0, 0, 285, 1058, 1882, 1, 34, 1181, 216, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 271, 896, 465, 958, 526, 324, 326, 1022, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6251, 6221, 6015, 6251, 6080, 6102, 5948] avg=6144.0000 max=6284 max_rank=0 min=5948 min_rank=7 imbalance_ratio=1.022786 max_load_reduction=42868 max_load_reduction_pct=0.872152 moved_to_echo=24381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[534, 693, 424, 462, 2020, 141, 367, 1141, 339, 0, 0, 0, 0, 0, 0, 0, 69, 849, 2095, 307, 713, 1073, 596, 442, 0, 0, 0, 0, 0, 0, 0, 0, 906, 1327, 754, 422, 1411, 176, 2, 1331, 0, 0, 0, 0, 0, 0, 0, 0, 758, 716, 799, 1306, 13, 150, 116, 115, 1841, 224, 0, 0, 0, 0, 0, 0, 272, 31, 304, 62, 1129, 788, 852, 825, 1401, 303, 0, 0, 0, 0, 0, 0, 1888, 1094, 561, 1038, 154, 1093, 8, 190, 76, 0, 0, 0, 0, 0, 0, 0, 255, 912, 1905, 2, 46, 1178, 266, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 295, 957, 491, 1010, 525, 383, 438, 926, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 6144, 6329, 6038, 5967, 6102, 6078, 6373] avg=6144.0000 max=6373 max_rank=7 min=5967 min_rank=4 imbalance_ratio=1.037272 max_load_reduction=42779 max_load_reduction_pct=0.870341 moved_to_echo=24520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[612, 840, 349, 426, 1965, 114, 375, 1216, 351, 0, 0, 0, 0, 0, 0, 0, 34, 908, 2203, 334, 658, 1019, 516, 483, 0, 0, 0, 0, 0, 0, 0, 0, 750, 1345, 882, 278, 1275, 159, 5, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 765, 653, 763, 1300, 25, 80, 103, 141, 1907, 271, 0, 0, 0, 0, 0, 0, 320, 30, 318, 46, 1165, 701, 890, 816, 1717, 314, 0, 0, 0, 0, 0, 0, 1931, 1087, 515, 950, 139, 1203, 7, 136, 76, 0, 0, 0, 0, 0, 0, 0, 322, 1015, 1893, 0, 20, 1191, 257, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 210, 944, 483, 865, 499, 302, 343, 969, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6248, 6155, 6091, 6008, 6317, 6044, 6305, 5984] avg=6144.0000 max=6317 max_rank=4 min=5984 min_rank=7 imbalance_ratio=1.028158 max_load_reduction=42835 max_load_reduction_pct=0.871480 moved_to_echo=24650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[534, 693, 424, 462, 2020, 141, 367, 1141, 339, 0, 0, 0, 0, 0, 0, 0, 69, 849, 2095, 307, 713, 1073, 596, 442, 0, 0, 0, 0, 0, 0, 0, 0, 906, 1327, 754, 422, 1411, 176, 2, 1331, 0, 0, 0, 0, 0, 0, 0, 0, 758, 716, 799, 1306, 13, 150, 116, 115, 1841, 224, 0, 0, 0, 0, 0, 0, 272, 31, 304, 62, 1129, 788, 852, 825, 1401, 303, 0, 0, 0, 0, 0, 0, 1888, 1094, 561, 1038, 154, 1093, 8, 190, 76, 0, 0, 0, 0, 0, 0, 0, 255, 912, 1905, 2, 46, 1178, 266, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 295, 957, 491, 1010, 525, 383, 438, 926, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 6144, 6329, 6038, 5967, 6102, 6078, 6373] avg=6144.0000 max=6373 max_rank=7 min=5967 min_rank=4 imbalance_ratio=1.037272 max_load_reduction=42779 max_load_reduction_pct=0.870341 moved_to_echo=24520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[15232, 9216, 4224, 8064, 1280, 9472, 128, 1408, 640, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 384, 2432, 512, 9088, 5504, 6912, 6656, 13184, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[384, 7168, 17536, 2688, 5504, 7936, 4480, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 8064, 15488, 128, 384, 9344, 1920, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 384, 2432, 512, 9088, 5504, 6912, 6656, 13184, 2560, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6272, 5632, 6400, 10752, 256, 1024, 1024, 1024, 15232, 2176, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 6016, 2944, 3456, 16640, 1152, 2944, 9344, 2944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[15232, 9216, 4224, 8064, 1280, 9472, 128, 1408, 640, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10496, 2176, 7680, 3840, 7296, 4352, 2944, 2944, 7936, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 8064, 15488, 128, 384, 9344, 1920, 11904, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 10496, 6528, 2816, 10624, 1664, 128, 11136, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 1341, 564, 231, 3, 979, 476, 213, 365, 457, 1420, 1734, 1461, 549, 0, 32, 1402, 1838, 607, 239, 32, 0, 1541, 741, 819, 669, 1655, 1819, 22, 13, 790, 27, 295, 1284, 1032, 487, 4587, 2189, 1344, 220, 689, 80, 183, 200, 1610, 1561, 105, 502, 392, 0, 518, 790, 5, 2, 129, 183, 2421, 527, 9, 176, 0, 2114, 8, 3471] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1, 1333, 551, 227, 4, 1149, 567, 199, 315, 444, 1453, 1614, 1281, 571, 0, 90, 1274, 1853, 412, 257, 20, 10, 1612, 671, 822, 626, 1581, 1914, 40, 74, 781, 29, 301, 1379, 946, 563, 4275, 2278, 1570, 232, 538, 77, 187, 105, 1645, 1644, 120, 475, 450, 1, 538, 823, 10, 1, 107, 267, 2394, 454, 18, 239, 0, 2626, 12, 3102] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 1206, 527, 238, 2, 1229, 574, 199, 258, 387, 1334, 1941, 1594, 459, 1, 21, 1347, 1621, 713, 190, 16, 0, 1519, 680, 871, 660, 1776, 2388, 19, 6, 799, 13, 300, 1534, 965, 396, 4662, 1946, 1408, 169, 728, 90, 141, 55, 1507, 1396, 118, 370, 426, 1, 337, 765, 0, 1, 182, 210, 2542, 485, 10, 127, 0, 2223, 2, 3468] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1197, 503, 260, 16, 1112, 636, 210, 355, 493, 1296, 1734, 1252, 606, 1, 225, 1217, 1555, 432, 247, 35, 56, 1809, 737, 635, 664, 1519, 1804, 26, 264, 754, 51, 312, 1617, 978, 559, 3871, 1649, 1593, 278, 612, 76, 214, 144, 2026, 1647, 129, 608, 542, 27, 400, 864, 3, 10, 166, 308, 2337, 496, 39, 331, 3, 2347, 42, 3223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 1286, 563, 236, 2, 1074, 703, 194, 378, 436, 1434, 1514, 1251, 541, 4, 189, 1232, 1636, 498, 278, 24, 42, 1880, 866, 763, 622, 1496, 1754, 30, 35, 994, 40, 289, 1403, 1001, 494, 4718, 1915, 1357, 248, 577, 68, 163, 183, 1731, 1564, 117, 492, 500, 7, 402, 774, 5, 3, 121, 274, 2232, 386, 34, 301, 0, 2528, 13, 3257] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 1119, 550, 259, 1, 1163, 506, 229, 315, 416, 1171, 1992, 1185, 504, 0, 38, 1377, 1714, 711, 256, 30, 0, 1574, 677, 869, 775, 1942, 2203, 29, 29, 750, 32, 303, 1475, 965, 486, 4192, 1998, 1417, 194, 770, 95, 175, 165, 1593, 1753, 162, 334, 436, 2, 319, 693, 7, 1, 225, 186, 2492, 562, 25, 196, 0, 2272, 13, 3229] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1407, 566, 275, 6, 1217, 565, 206, 332, 413, 1325, 1771, 1598, 636, 3, 156, 1271, 1575, 433, 261, 22, 19, 1700, 726, 823, 573, 1589, 1786, 45, 55, 816, 33, 300, 1480, 1035, 662, 3984, 1921, 1629, 243, 702, 95, 156, 108, 1727, 1369, 128, 505, 461, 7, 572, 840, 5, 3, 182, 272, 2274, 438, 15, 311, 0, 2429, 24, 3072] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 1362, 502, 248, 11, 1047, 579, 181, 260, 402, 1275, 1908, 1290, 542, 0, 121, 1314, 1638, 451, 204, 30, 5, 1747, 608, 665, 728, 1850, 2113, 22, 12, 849, 16, 355, 1401, 927, 504, 4507, 1981, 1276, 189, 556, 84, 195, 95, 1894, 1557, 126, 497, 471, 1, 376, 837, 5, 8, 104, 228, 2332, 495, 17, 193, 0, 2662, 18, 3281] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1341, 564, 231, 3, 979, 476, 213, 1983, 321, 0, 0, 0, 0, 0, 0, 365, 457, 1420, 1734, 1461, 549, 0, 32, 276, 0, 0, 0, 0, 0, 0, 0, 1402, 1793, 607, 239, 32, 0, 1541, 741, 0, 0, 0, 0, 0, 0, 0, 0, 819, 669, 1655, 1819, 22, 13, 790, 27, 76, 45, 0, 0, 0, 0, 0, 0, 295, 1284, 1032, 487, 0, 1347, 1344, 220, 0, 0, 0, 0, 0, 0, 0, 0, 689, 80, 183, 200, 1610, 1561, 105, 502, 834, 490, 0, 0, 0, 0, 0, 0, 392, 0, 518, 790, 5, 2, 129, 183, 4266, 0, 0, 0, 0, 0, 0, 0, 2421, 527, 9, 176, 0, 2114, 8, 654, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 6294, 6355, 5935, 6009, 6254, 6285, 5909] avg=6144.0000 max=6355 max_rank=2 min=5909 min_rank=7 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 1286, 563, 236, 2, 1074, 703, 194, 378, 436, 1434, 1514, 1251, 541, 4, 189, 1232, 1636, 498, 278, 24, 42, 1880, 866, 763, 622, 1496, 1754, 30, 35, 994, 40, 289, 1403, 1001, 494, 4718, 1915, 1357, 248, 577, 68, 163, 183, 1731, 1564, 117, 492, 500, 7, 402, 774, 5, 3, 121, 274, 2232, 386, 34, 301, 0, 2528, 13, 3257] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 1119, 550, 259, 1, 1163, 506, 229, 315, 416, 1171, 1992, 1185, 504, 0, 38, 1377, 1714, 711, 256, 30, 0, 1574, 677, 869, 775, 1942, 2203, 29, 29, 750, 32, 303, 1475, 965, 486, 4192, 1998, 1417, 194, 770, 95, 175, 165, 1593, 1753, 162, 334, 436, 2, 319, 693, 7, 1, 225, 186, 2492, 562, 25, 196, 0, 2272, 13, 3229] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 1341, 564, 231, 3, 979, 476, 213, 365, 457, 1420, 1734, 1461, 549, 0, 32, 1402, 1838, 607, 239, 32, 0, 1541, 741, 819, 669, 1655, 1819, 22, 13, 790, 27, 295, 1284, 1032, 487, 4587, 2189, 1344, 220, 689, 80, 183, 200, 1610, 1561, 105, 502, 392, 0, 518, 790, 5, 2, 129, 183, 2421, 527, 9, 176, 0, 2114, 8, 3471] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 1206, 527, 238, 2, 1229, 574, 199, 258, 387, 1334, 1941, 1594, 459, 1, 21, 1347, 1621, 713, 190, 16, 0, 1519, 680, 871, 660, 1776, 2388, 19, 6, 799, 13, 300, 1534, 965, 396, 4662, 1946, 1408, 169, 728, 90, 141, 55, 1507, 1396, 118, 370, 426, 1, 337, 765, 0, 1, 182, 210, 2542, 485, 10, 127, 0, 2223, 2, 3468] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1197, 503, 260, 16, 1112, 636, 210, 355, 493, 1296, 1734, 1252, 606, 1, 225, 1217, 1555, 432, 247, 35, 56, 1809, 737, 635, 664, 1519, 1804, 26, 264, 754, 51, 312, 1617, 978, 559, 3871, 1649, 1593, 278, 612, 76, 214, 144, 2026, 1647, 129, 608, 542, 27, 400, 864, 3, 10, 166, 308, 2337, 496, 39, 331, 3, 2347, 42, 3223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1333, 551, 227, 4, 1149, 567, 199, 1772, 298, 0, 0, 0, 0, 0, 0, 315, 444, 1453, 1614, 1281, 571, 0, 90, 288, 0, 0, 0, 0, 0, 0, 0, 1274, 1808, 412, 257, 20, 10, 1612, 671, 0, 0, 0, 0, 0, 0, 0, 0, 822, 626, 1581, 1914, 40, 74, 781, 29, 79, 45, 0, 0, 0, 0, 0, 0, 301, 1379, 946, 563, 0, 1401, 1570, 232, 0, 0, 0, 0, 0, 0, 0, 0, 538, 77, 187, 105, 1645, 1644, 120, 475, 745, 510, 0, 0, 0, 0, 0, 0, 450, 1, 538, 823, 10, 1, 107, 267, 3977, 0, 0, 0, 0, 0, 0, 0, 2394, 454, 18, 239, 0, 2626, 12, 585, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6101, 6056, 6064, 5991, 6392, 6046, 6174, 6328] avg=6144.0000 max=6392 max_rank=4 min=5991 min_rank=3 imbalance_ratio=1.040365 max_load_reduction=42760 max_load_reduction_pct=0.869954 moved_to_echo=24940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1197, 503, 260, 16, 1112, 636, 210, 1841, 270, 0, 0, 0, 0, 0, 0, 355, 493, 1296, 1734, 1252, 606, 1, 225, 208, 0, 0, 0, 0, 0, 0, 0, 1217, 1517, 432, 247, 35, 56, 1809, 737, 0, 0, 0, 0, 0, 0, 0, 0, 635, 664, 1519, 1804, 26, 264, 754, 51, 57, 38, 0, 0, 0, 0, 0, 0, 312, 1617, 978, 559, 0, 1015, 1593, 278, 0, 0, 0, 0, 0, 0, 0, 0, 612, 76, 214, 144, 2026, 1647, 129, 608, 775, 369, 0, 0, 0, 0, 0, 0, 542, 27, 400, 864, 3, 10, 166, 308, 3601, 0, 0, 0, 0, 0, 0, 0, 2337, 496, 39, 331, 3, 2347, 42, 607, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6045, 6170, 6050, 5812, 6352, 6600, 5921, 6202] avg=6144.0000 max=6600 max_rank=5 min=5812 min_rank=3 imbalance_ratio=1.074219 max_load_reduction=42552 max_load_reduction_pct=0.865723 moved_to_echo=25075 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1362, 502, 248, 11, 1047, 579, 181, 1875, 315, 0, 0, 0, 0, 0, 0, 260, 402, 1275, 1908, 1290, 542, 0, 121, 250, 0, 0, 0, 0, 0, 0, 0, 1314, 1598, 451, 204, 30, 5, 1747, 608, 0, 0, 0, 0, 0, 0, 0, 0, 665, 728, 1850, 2113, 22, 12, 849, 16, 69, 40, 0, 0, 0, 0, 0, 0, 355, 1401, 927, 504, 0, 1219, 1276, 189, 0, 0, 0, 0, 0, 0, 0, 0, 556, 84, 195, 95, 1894, 1557, 126, 497, 788, 443, 0, 0, 0, 0, 0, 0, 471, 1, 376, 837, 5, 8, 104, 228, 4192, 0, 0, 0, 0, 0, 0, 0, 2332, 495, 17, 193, 0, 2662, 18, 618, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6120, 6048, 5957, 6364, 5871, 6235, 6222, 6335] avg=6144.0000 max=6364 max_rank=3 min=5871 min_rank=4 imbalance_ratio=1.035807 max_load_reduction=42788 max_load_reduction_pct=0.870524 moved_to_echo=24663 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1206, 527, 238, 2, 1229, 574, 199, 1981, 325, 0, 0, 0, 0, 0, 0, 258, 387, 1334, 1941, 1594, 459, 1, 21, 246, 0, 0, 0, 0, 0, 0, 0, 1347, 1582, 713, 190, 16, 0, 1519, 680, 0, 0, 0, 0, 0, 0, 0, 0, 871, 660, 1776, 2388, 19, 6, 799, 13, 67, 39, 0, 0, 0, 0, 0, 0, 300, 1534, 965, 396, 0, 1198, 1408, 169, 0, 0, 0, 0, 0, 0, 0, 0, 728, 90, 141, 55, 1507, 1396, 118, 370, 833, 435, 0, 0, 0, 0, 0, 0, 426, 1, 337, 765, 0, 1, 182, 210, 4337, 0, 0, 0, 0, 0, 0, 0, 2542, 485, 10, 127, 0, 2223, 2, 654, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6281, 6241, 6047, 6638, 5970, 5673, 6259, 6043] avg=6144.0000 max=6638 max_rank=3 min=5673 min_rank=5 imbalance_ratio=1.080404 max_load_reduction=42514 max_load_reduction_pct=0.864950 moved_to_echo=23945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1119, 550, 259, 1, 1163, 506, 229, 1845, 292, 0, 0, 0, 0, 0, 0, 315, 416, 1171, 1992, 1185, 504, 0, 38, 252, 0, 0, 0, 0, 0, 0, 0, 1377, 1672, 711, 256, 30, 0, 1574, 677, 0, 0, 0, 0, 0, 0, 0, 0, 869, 775, 1942, 2203, 29, 29, 750, 32, 69, 42, 0, 0, 0, 0, 0, 0, 303, 1475, 965, 486, 0, 1230, 1417, 194, 0, 0, 0, 0, 0, 0, 0, 0, 770, 95, 175, 165, 1593, 1753, 162, 334, 776, 447, 0, 0, 0, 0, 0, 0, 436, 2, 319, 693, 7, 1, 225, 186, 3900, 0, 0, 0, 0, 0, 0, 0, 2492, 562, 25, 196, 0, 2272, 13, 608, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 5873, 6297, 6740, 6070, 6270, 5769, 6168] avg=6144.0000 max=6740 max_rank=3 min=5769 min_rank=6 imbalance_ratio=1.097005 max_load_reduction=42412 max_load_reduction_pct=0.862874 moved_to_echo=24277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1286, 563, 236, 2, 1074, 703, 194, 1861, 329, 0, 0, 0, 0, 0, 0, 378, 436, 1434, 1514, 1251, 541, 4, 189, 242, 0, 0, 0, 0, 0, 0, 0, 1232, 1596, 498, 278, 24, 42, 1880, 866, 0, 0, 0, 0, 0, 0, 0, 0, 763, 622, 1496, 1754, 30, 35, 994, 40, 66, 40, 0, 0, 0, 0, 0, 0, 289, 1403, 1001, 494, 0, 1178, 1357, 248, 0, 0, 0, 0, 0, 0, 0, 0, 577, 68, 163, 183, 1731, 1564, 117, 492, 783, 429, 0, 0, 0, 0, 0, 0, 500, 7, 402, 774, 5, 3, 121, 274, 4389, 0, 0, 0, 0, 0, 0, 0, 2232, 386, 34, 301, 0, 2528, 13, 613, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6248, 5989, 6416, 5840, 5970, 6107, 6475, 6107] avg=6144.0000 max=6475 max_rank=6 min=5840 min_rank=3 imbalance_ratio=1.053874 max_load_reduction=42677 max_load_reduction_pct=0.868266 moved_to_echo=24659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1197, 503, 260, 16, 1112, 636, 210, 1841, 270, 0, 0, 0, 0, 0, 0, 355, 493, 1296, 1734, 1252, 606, 1, 225, 208, 0, 0, 0, 0, 0, 0, 0, 1217, 1517, 432, 247, 35, 56, 1809, 737, 0, 0, 0, 0, 0, 0, 0, 0, 635, 664, 1519, 1804, 26, 264, 754, 51, 57, 38, 0, 0, 0, 0, 0, 0, 312, 1617, 978, 559, 0, 1015, 1593, 278, 0, 0, 0, 0, 0, 0, 0, 0, 612, 76, 214, 144, 2026, 1647, 129, 608, 775, 369, 0, 0, 0, 0, 0, 0, 542, 27, 400, 864, 3, 10, 166, 308, 3601, 0, 0, 0, 0, 0, 0, 0, 2337, 496, 39, 331, 3, 2347, 42, 607, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6045, 6170, 6050, 5812, 6352, 6600, 5921, 6202] avg=6144.0000 max=6600 max_rank=5 min=5812 min_rank=3 imbalance_ratio=1.074219 max_load_reduction=42552 max_load_reduction_pct=0.865723 moved_to_echo=25075 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1341, 564, 231, 3, 979, 476, 213, 1983, 321, 0, 0, 0, 0, 0, 0, 365, 457, 1420, 1734, 1461, 549, 0, 32, 276, 0, 0, 0, 0, 0, 0, 0, 1402, 1793, 607, 239, 32, 0, 1541, 741, 0, 0, 0, 0, 0, 0, 0, 0, 819, 669, 1655, 1819, 22, 13, 790, 27, 76, 45, 0, 0, 0, 0, 0, 0, 295, 1284, 1032, 487, 0, 1347, 1344, 220, 0, 0, 0, 0, 0, 0, 0, 0, 689, 80, 183, 200, 1610, 1561, 105, 502, 834, 490, 0, 0, 0, 0, 0, 0, 392, 0, 518, 790, 5, 2, 129, 183, 4266, 0, 0, 0, 0, 0, 0, 0, 2421, 527, 9, 176, 0, 2114, 8, 654, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 6294, 6355, 5935, 6009, 6254, 6285, 5909] avg=6144.0000 max=6355 max_rank=2 min=5909 min_rank=7 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1407, 566, 275, 6, 1217, 565, 206, 1760, 279, 0, 0, 0, 0, 0, 0, 332, 413, 1325, 1771, 1598, 636, 3, 156, 247, 0, 0, 0, 0, 0, 0, 0, 1271, 1534, 433, 261, 22, 19, 1700, 726, 0, 0, 0, 0, 0, 0, 0, 0, 823, 573, 1589, 1786, 45, 55, 816, 33, 71, 41, 0, 0, 0, 0, 0, 0, 300, 1480, 1035, 662, 0, 1169, 1629, 243, 0, 0, 0, 0, 0, 0, 0, 0, 702, 95, 156, 108, 1727, 1369, 128, 505, 743, 434, 0, 0, 0, 0, 0, 0, 461, 7, 572, 840, 5, 3, 182, 272, 3705, 0, 0, 0, 0, 0, 0, 0, 2274, 438, 15, 311, 0, 2429, 24, 569, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6281, 6481, 5966, 5832, 6518, 5967, 6047, 6060] avg=6144.0000 max=6518 max_rank=4 min=5832 min_rank=3 imbalance_ratio=1.060872 max_load_reduction=42634 max_load_reduction_pct=0.867391 moved_to_echo=24592 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1362, 502, 248, 11, 1047, 579, 181, 1875, 315, 0, 0, 0, 0, 0, 0, 260, 402, 1275, 1908, 1290, 542, 0, 121, 250, 0, 0, 0, 0, 0, 0, 0, 1314, 1598, 451, 204, 30, 5, 1747, 608, 0, 0, 0, 0, 0, 0, 0, 0, 665, 728, 1850, 2113, 22, 12, 849, 16, 69, 40, 0, 0, 0, 0, 0, 0, 355, 1401, 927, 504, 0, 1219, 1276, 189, 0, 0, 0, 0, 0, 0, 0, 0, 556, 84, 195, 95, 1894, 1557, 126, 497, 788, 443, 0, 0, 0, 0, 0, 0, 471, 1, 376, 837, 5, 8, 104, 228, 4192, 0, 0, 0, 0, 0, 0, 0, 2332, 495, 17, 193, 0, 2662, 18, 618, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6120, 6048, 5957, 6364, 5871, 6235, 6222, 6335] avg=6144.0000 max=6364 max_rank=3 min=5871 min_rank=4 imbalance_ratio=1.035807 max_load_reduction=42788 max_load_reduction_pct=0.870524 moved_to_echo=24663 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5248, 768, 1536, 1152, 13824, 12544, 1024, 3840, 6400, 3584, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 11648, 7936, 4224, 0, 9856, 11648, 1792, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 3456, 10752, 14208, 11008, 4480, 128, 896, 2048, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[19072, 3968, 256, 1920, 128, 19328, 256, 4992, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10496, 13184, 4352, 2048, 256, 256, 13440, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 11648, 7936, 4224, 0, 9856, 11648, 1792, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 10368, 4352, 2048, 128, 9088, 4608, 1664, 14976, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10496, 13184, 4352, 2048, 256, 256, 13440, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[19072, 3968, 256, 1920, 128, 19328, 256, 4992, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6272, 5376, 13440, 15872, 256, 512, 6656, 256, 640, 384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 128, 3584, 6400, 128, 128, 1280, 2048, 32384, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[113, 82, 1391, 5, 505, 935, 438, 683, 3445, 2654, 313, 262, 828, 316, 5169, 144, 2878, 24, 1818, 352, 435, 240, 0, 313, 986, 1379, 507, 328, 458, 1717, 829, 501, 134, 147, 91, 160, 287, 184, 219, 58, 25, 3645, 47, 452, 158, 82, 0, 4617, 0, 114, 89, 973, 65, 1977, 0, 161, 81, 3, 1479, 40, 1970, 1418, 87, 341] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[128, 99, 1424, 1, 505, 705, 540, 679, 2975, 2457, 217, 284, 960, 369, 4791, 101, 2549, 18, 2432, 408, 470, 246, 0, 402, 827, 1299, 494, 380, 447, 2107, 805, 421, 311, 187, 84, 172, 423, 213, 264, 52, 19, 3285, 41, 385, 234, 58, 0, 4798, 0, 143, 81, 1169, 112, 1985, 0, 72, 27, 2, 1537, 48, 2276, 1157, 113, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[143, 83, 1543, 7, 437, 449, 583, 613, 2429, 2604, 206, 290, 991, 361, 4561, 128, 2596, 32, 2614, 379, 471, 296, 0, 447, 731, 1388, 556, 393, 531, 1968, 923, 439, 296, 261, 71, 158, 517, 232, 390, 111, 53, 3211, 68, 476, 248, 120, 0, 4844, 0, 145, 74, 1049, 157, 2016, 78, 78, 48, 9, 1512, 305, 1905, 1066, 149, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[156, 46, 1491, 3, 591, 876, 537, 736, 3056, 2483, 318, 283, 838, 341, 4894, 134, 3011, 17, 2210, 329, 367, 215, 0, 317, 630, 1505, 456, 351, 387, 1713, 899, 525, 308, 166, 100, 177, 343, 206, 197, 93, 19, 3405, 68, 400, 222, 87, 0, 4373, 0, 105, 94, 1129, 79, 1962, 0, 174, 91, 0, 1574, 308, 2062, 1325, 57, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[148, 81, 1536, 2, 415, 614, 549, 638, 2627, 2487, 232, 275, 940, 354, 4862, 170, 2705, 43, 2323, 345, 484, 263, 0, 326, 975, 1390, 523, 373, 462, 1784, 879, 398, 191, 180, 81, 177, 421, 218, 235, 90, 22, 3628, 58, 457, 173, 110, 0, 5070, 0, 131, 78, 1070, 87, 2214, 0, 65, 60, 2, 1515, 61, 2021, 1192, 71, 271] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[149, 74, 1595, 1, 438, 628, 568, 597, 2734, 2659, 221, 264, 849, 342, 4868, 184, 2817, 23, 2432, 359, 509, 304, 0, 324, 906, 1327, 530, 349, 522, 1819, 1036, 374, 101, 200, 67, 178, 428, 225, 335, 155, 23, 3777, 23, 454, 195, 81, 0, 4796, 0, 95, 63, 945, 116, 2164, 0, 48, 61, 2, 1437, 12, 1930, 1076, 96, 267] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[143, 83, 1543, 7, 437, 449, 583, 613, 2429, 2604, 206, 290, 991, 361, 4561, 128, 2596, 32, 2614, 379, 471, 296, 0, 447, 731, 1388, 556, 393, 531, 1968, 923, 439, 296, 261, 71, 158, 517, 232, 390, 111, 53, 3211, 68, 476, 248, 120, 0, 4844, 0, 145, 74, 1049, 157, 2016, 78, 78, 48, 9, 1512, 305, 1905, 1066, 149, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[135, 71, 1487, 3, 517, 779, 488, 697, 3258, 2621, 232, 252, 892, 354, 4879, 138, 2659, 28, 2037, 341, 434, 288, 0, 283, 1033, 1408, 590, 355, 412, 1956, 691, 394, 54, 123, 72, 149, 417, 143, 254, 41, 33, 3865, 91, 513, 191, 76, 0, 4828, 0, 95, 71, 1101, 57, 2008, 0, 81, 147, 1, 1308, 36, 2030, 1285, 77, 293] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[118, 86, 1634, 6, 527, 813, 537, 690, 2893, 2259, 247, 283, 943, 358, 4630, 149, 2506, 42, 2415, 345, 401, 272, 0, 385, 851, 1452, 498, 366, 519, 1733, 820, 487, 154, 225, 102, 163, 499, 211, 356, 111, 38, 3599, 69, 431, 193, 79, 1, 4838, 0, 131, 93, 1106, 109, 2031, 16, 140, 77, 1, 1399, 241, 1894, 1179, 71, 330] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[128, 99, 1424, 1, 505, 705, 540, 679, 2975, 2457, 217, 284, 960, 369, 4791, 101, 2549, 18, 2432, 408, 470, 246, 0, 402, 827, 1299, 494, 380, 447, 2107, 805, 421, 311, 187, 84, 172, 423, 213, 264, 52, 19, 3285, 41, 385, 234, 58, 0, 4798, 0, 143, 81, 1169, 112, 1985, 0, 72, 27, 2, 1537, 48, 2276, 1157, 113, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[113, 82, 1391, 5, 505, 935, 438, 683, 1449, 667, 74, 0, 0, 0, 0, 0, 1996, 2654, 313, 262, 828, 316, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 24, 1818, 352, 435, 240, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1379, 507, 328, 458, 1144, 829, 501, 0, 0, 0, 0, 0, 0, 0, 0, 134, 147, 91, 160, 287, 184, 219, 58, 4838, 0, 0, 0, 0, 0, 0, 0, 25, 3645, 47, 452, 158, 82, 0, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 89, 973, 65, 1977, 0, 161, 2236, 331, 0, 0, 0, 0, 0, 0, 81, 3, 1479, 40, 1970, 1418, 87, 341, 499, 321, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6342, 6513, 5739, 6132, 6118, 6123, 5946, 6239] avg=6144.0000 max=6513 max_rank=1 min=5739 min_rank=2 imbalance_ratio=1.060059 max_load_reduction=42639 max_load_reduction_pct=0.867493 moved_to_echo=24426 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[148, 81, 1536, 2, 415, 614, 549, 638, 1105, 732, 76, 0, 0, 0, 0, 0, 1522, 2487, 232, 275, 940, 354, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 2403, 43, 2323, 345, 484, 263, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1390, 523, 373, 462, 1190, 879, 398, 0, 0, 0, 0, 0, 0, 0, 0, 191, 180, 81, 177, 421, 218, 235, 90, 4551, 0, 0, 0, 0, 0, 0, 0, 22, 3628, 58, 457, 173, 110, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 78, 1070, 87, 2214, 0, 65, 2455, 311, 0, 0, 0, 0, 0, 0, 60, 2, 1515, 61, 2021, 1192, 71, 271, 518, 302, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5896, 5980, 6187, 6190, 6144, 6331, 6411, 6013] avg=6144.0000 max=6411 max_rank=6 min=5896 min_rank=0 imbalance_ratio=1.043457 max_load_reduction=42741 max_load_reduction_pct=0.869568 moved_to_echo=24899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[128, 99, 1424, 1, 505, 705, 540, 679, 1251, 693, 90, 0, 0, 0, 0, 0, 1724, 2457, 217, 284, 960, 369, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 2265, 18, 2432, 408, 470, 246, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 827, 1299, 494, 380, 447, 1405, 805, 421, 0, 0, 0, 0, 0, 0, 0, 0, 311, 187, 84, 172, 423, 213, 264, 52, 4485, 0, 0, 0, 0, 0, 0, 0, 19, 3285, 41, 385, 234, 58, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 81, 1169, 112, 1985, 0, 72, 2324, 306, 0, 0, 0, 0, 0, 0, 27, 2, 1537, 48, 2276, 1157, 113, 364, 612, 284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 6112, 6241, 6078, 6191, 5803, 6192, 6420] avg=6144.0000 max=6420 max_rank=7 min=5803 min_rank=5 imbalance_ratio=1.044922 max_load_reduction=42732 max_load_reduction_pct=0.869385 moved_to_echo=24606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[149, 74, 1595, 1, 438, 628, 568, 597, 1150, 692, 78, 0, 0, 0, 0, 0, 1584, 2659, 221, 264, 849, 342, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 2503, 23, 2432, 359, 509, 304, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 906, 1327, 530, 349, 522, 1213, 1036, 374, 0, 0, 0, 0, 0, 0, 0, 0, 101, 200, 67, 178, 428, 225, 335, 155, 4556, 0, 0, 0, 0, 0, 0, 0, 23, 3777, 23, 454, 195, 81, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 63, 945, 116, 2164, 0, 48, 2323, 312, 0, 0, 0, 0, 0, 0, 61, 2, 1437, 12, 1930, 1076, 96, 267, 528, 314, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5970, 6103, 6454, 6257, 6245, 6334, 6066, 5723] avg=6144.0000 max=6454 max_rank=2 min=5723 min_rank=7 imbalance_ratio=1.050456 max_load_reduction=42698 max_load_reduction_pct=0.868693 moved_to_echo=24368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[156, 46, 1491, 3, 591, 876, 537, 736, 1285, 631, 73, 0, 0, 0, 0, 0, 1771, 2483, 318, 283, 838, 341, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 17, 2210, 329, 367, 215, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 630, 1505, 456, 351, 387, 1142, 899, 525, 0, 0, 0, 0, 0, 0, 0, 0, 308, 166, 100, 177, 343, 206, 197, 93, 4581, 0, 0, 0, 0, 0, 0, 0, 19, 3405, 68, 400, 222, 87, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 94, 1129, 79, 1962, 0, 174, 2118, 313, 0, 0, 0, 0, 0, 0, 91, 0, 1574, 308, 2062, 1325, 57, 313, 498, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6425, 6168, 6130, 5895, 6171, 5825, 5974, 6564] avg=6144.0000 max=6564 max_rank=7 min=5825 min_rank=5 imbalance_ratio=1.068359 max_load_reduction=42588 max_load_reduction_pct=0.866455 moved_to_echo=24534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[143, 83, 1543, 7, 437, 449, 583, 613, 1021, 699, 84, 0, 0, 0, 0, 0, 1408, 2604, 206, 290, 991, 361, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 32, 2614, 379, 471, 296, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 731, 1388, 556, 393, 531, 1312, 923, 439, 0, 0, 0, 0, 0, 0, 0, 0, 296, 261, 71, 158, 517, 232, 390, 111, 4270, 0, 0, 0, 0, 0, 0, 0, 53, 3211, 68, 476, 248, 120, 0, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 74, 1049, 157, 2016, 78, 78, 2346, 291, 0, 0, 0, 0, 0, 0, 48, 9, 1512, 305, 1905, 1066, 149, 313, 572, 290, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5662, 5988, 6545, 6273, 6306, 5975, 6234, 6169] avg=6144.0000 max=6545 max_rank=2 min=5662 min_rank=0 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=24684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[148, 81, 1536, 2, 415, 614, 549, 638, 1105, 732, 76, 0, 0, 0, 0, 0, 1522, 2487, 232, 275, 940, 354, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 2403, 43, 2323, 345, 484, 263, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1390, 523, 373, 462, 1190, 879, 398, 0, 0, 0, 0, 0, 0, 0, 0, 191, 180, 81, 177, 421, 218, 235, 90, 4551, 0, 0, 0, 0, 0, 0, 0, 22, 3628, 58, 457, 173, 110, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 78, 1070, 87, 2214, 0, 65, 2455, 311, 0, 0, 0, 0, 0, 0, 60, 2, 1515, 61, 2021, 1192, 71, 271, 518, 302, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5896, 5980, 6187, 6190, 6144, 6331, 6411, 6013] avg=6144.0000 max=6411 max_rank=6 min=5896 min_rank=0 imbalance_ratio=1.043457 max_load_reduction=42741 max_load_reduction_pct=0.869568 moved_to_echo=24899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[135, 71, 1487, 3, 517, 779, 488, 697, 1370, 697, 84, 0, 0, 0, 0, 0, 1888, 2621, 232, 252, 892, 354, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 2362, 28, 2037, 341, 434, 288, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1408, 590, 355, 412, 1304, 691, 394, 0, 0, 0, 0, 0, 0, 0, 0, 54, 123, 72, 149, 417, 143, 254, 41, 4568, 0, 0, 0, 0, 0, 0, 0, 33, 3865, 91, 513, 191, 76, 0, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 71, 1101, 57, 2008, 0, 81, 2338, 311, 0, 0, 0, 0, 0, 0, 147, 1, 1308, 36, 2030, 1285, 77, 293, 568, 297, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 6377, 5773, 6187, 5821, 6562, 6062, 6042] avg=6144.0000 max=6562 max_rank=5 min=5773 min_rank=2 imbalance_ratio=1.068034 max_load_reduction=42590 max_load_reduction_pct=0.866496 moved_to_echo=24487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[149, 74, 1595, 1, 438, 628, 568, 597, 1150, 692, 78, 0, 0, 0, 0, 0, 1584, 2659, 221, 264, 849, 342, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 2503, 23, 2432, 359, 509, 304, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 906, 1327, 530, 349, 522, 1213, 1036, 374, 0, 0, 0, 0, 0, 0, 0, 0, 101, 200, 67, 178, 428, 225, 335, 155, 4556, 0, 0, 0, 0, 0, 0, 0, 23, 3777, 23, 454, 195, 81, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 63, 945, 116, 2164, 0, 48, 2323, 312, 0, 0, 0, 0, 0, 0, 61, 2, 1437, 12, 1930, 1076, 96, 267, 528, 314, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5970, 6103, 6454, 6257, 6245, 6334, 6066, 5723] avg=6144.0000 max=6454 max_rank=2 min=5723 min_rank=7 imbalance_ratio=1.050456 max_load_reduction=42698 max_load_reduction_pct=0.868693 moved_to_echo=24368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[143, 83, 1543, 7, 437, 449, 583, 613, 1021, 699, 84, 0, 0, 0, 0, 0, 1408, 2604, 206, 290, 991, 361, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 32, 2614, 379, 471, 296, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 731, 1388, 556, 393, 531, 1312, 923, 439, 0, 0, 0, 0, 0, 0, 0, 0, 296, 261, 71, 158, 517, 232, 390, 111, 4270, 0, 0, 0, 0, 0, 0, 0, 53, 3211, 68, 476, 248, 120, 0, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 74, 1049, 157, 2016, 78, 78, 2346, 291, 0, 0, 0, 0, 0, 0, 48, 9, 1512, 305, 1905, 1066, 149, 313, 572, 290, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5662, 5988, 6545, 6273, 6306, 5975, 6234, 6169] avg=6144.0000 max=6545 max_rank=2 min=5662 min_rank=0 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=24684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[118, 86, 1634, 6, 527, 813, 537, 690, 1221, 703, 79, 0, 0, 0, 0, 0, 1672, 2259, 247, 283, 943, 358, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 42, 2415, 345, 401, 272, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 851, 1452, 498, 366, 519, 1147, 820, 487, 0, 0, 0, 0, 0, 0, 0, 0, 154, 225, 102, 163, 499, 211, 356, 111, 4335, 0, 0, 0, 0, 0, 0, 0, 38, 3599, 69, 431, 193, 79, 1, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 93, 1106, 109, 2031, 16, 140, 2346, 295, 0, 0, 0, 0, 0, 0, 77, 1, 1399, 241, 1894, 1179, 71, 330, 507, 283, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6414, 5911, 6083, 6140, 6156, 6199, 6267, 5982] avg=6144.0000 max=6414 max_rank=0 min=5911 min_rank=1 imbalance_ratio=1.043945 max_load_reduction=42738 max_load_reduction_pct=0.869507 moved_to_echo=24604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[128, 99, 1424, 1, 505, 705, 540, 679, 1251, 693, 90, 0, 0, 0, 0, 0, 1724, 2457, 217, 284, 960, 369, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 2265, 18, 2432, 408, 470, 246, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 827, 1299, 494, 380, 447, 1405, 805, 421, 0, 0, 0, 0, 0, 0, 0, 0, 311, 187, 84, 172, 423, 213, 264, 52, 4485, 0, 0, 0, 0, 0, 0, 0, 19, 3285, 41, 385, 234, 58, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 81, 1169, 112, 1985, 0, 72, 2324, 306, 0, 0, 0, 0, 0, 0, 27, 2, 1537, 48, 2276, 1157, 113, 364, 612, 284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 6112, 6241, 6078, 6191, 5803, 6192, 6420] avg=6144.0000 max=6420 max_rank=7 min=5803 min_rank=5 imbalance_ratio=1.044922 max_load_reduction=42732 max_load_reduction_pct=0.869385 moved_to_echo=24606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 28416, 512, 3584, 1664, 768, 128, 14208, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 640, 12160, 128, 3968, 5888, 4352, 5376, 9856, 5632, 640, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[19328, 256, 18304, 2944, 3584, 2176, 0, 2816, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 128, 11776, 1152, 16128, 9728, 768, 2560, 4352, 2432, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7040, 11264, 4224, 2944, 3840, 9984, 6912, 3584, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 640, 12160, 128, 3968, 5888, 4352, 5376, 9856, 5632, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 28416, 512, 3584, 1664, 768, 128, 14208, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[13568, 20224, 2048, 2304, 7296, 2816, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 128, 11776, 1152, 16128, 9728, 768, 2560, 4352, 2432, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 1024, 768, 8576, 896, 16384, 128, 896, 18560, 2560, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 1536, 768, 1408, 3456, 1664, 2304, 768, 36224, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 488, 1179, 1331, 815, 2221, 254, 1802, 7, 1261, 1710, 341, 99, 149, 28, 13, 1073, 0, 0, 3, 32, 1240, 717, 2, 1106, 0, 4049, 103, 4381, 121, 117, 0, 166, 0, 0, 872, 289, 0, 6, 812, 87, 755, 0, 1242, 147, 0, 4232, 0, 0, 942, 104, 375, 113, 5739, 4833, 664, 385, 88, 7, 1496, 671, 256, 3, 226] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 440, 1206, 1042, 697, 2495, 247, 1768, 1, 1565, 1757, 81, 153, 233, 20, 10, 1031, 6, 0, 13, 53, 1158, 828, 1, 867, 0, 3724, 96, 4483, 108, 162, 0, 150, 0, 0, 861, 267, 0, 4, 815, 61, 757, 0, 1197, 161, 0, 4472, 0, 0, 1009, 96, 374, 349, 5947, 4883, 606, 368, 115, 6, 1314, 650, 238, 3, 204] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 731, 1220, 1358, 869, 2315, 254, 1824, 17, 1263, 1583, 79, 152, 371, 159, 23, 1455, 0, 0, 14, 62, 911, 647, 6, 979, 0, 4565, 79, 4124, 103, 246, 0, 138, 0, 0, 692, 147, 0, 6, 695, 125, 883, 1, 1058, 140, 0, 4312, 0, 0, 1115, 70, 265, 397, 4932, 4706, 729, 392, 178, 10, 1608, 753, 207, 15, 169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 530, 1231, 1030, 925, 2620, 248, 1791, 10, 1556, 1715, 212, 132, 302, 37, 14, 1374, 0, 0, 7, 49, 1237, 797, 3, 1037, 0, 4015, 89, 4186, 129, 205, 0, 144, 0, 0, 643, 187, 0, 3, 695, 75, 784, 0, 1138, 146, 0, 4117, 0, 0, 1005, 91, 233, 265, 5432, 4797, 715, 480, 142, 7, 1390, 716, 193, 6, 267] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 661, 1208, 1246, 882, 2183, 252, 1679, 32, 1550, 1715, 63, 123, 248, 248, 17, 1468, 0, 1, 43, 60, 1202, 694, 5, 1244, 0, 3748, 113, 3803, 117, 185, 1, 164, 0, 2, 820, 307, 0, 13, 791, 99, 866, 4, 1133, 149, 0, 4234, 1, 0, 1050, 117, 417, 407, 5158, 4513, 691, 492, 168, 17, 1329, 732, 383, 10, 294] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 440, 1206, 1042, 697, 2495, 247, 1768, 1, 1565, 1757, 81, 153, 233, 20, 10, 1031, 6, 0, 13, 53, 1158, 828, 1, 867, 0, 3724, 96, 4483, 108, 162, 0, 150, 0, 0, 861, 267, 0, 4, 815, 61, 757, 0, 1197, 161, 0, 4472, 0, 0, 1009, 96, 374, 349, 5947, 4883, 606, 368, 115, 6, 1314, 650, 238, 3, 204] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 477, 1278, 975, 827, 2340, 239, 1772, 1, 1511, 1754, 170, 136, 286, 5, 18, 1293, 0, 0, 2, 23, 1173, 806, 4, 1045, 0, 4296, 83, 4328, 93, 142, 0, 163, 0, 0, 796, 175, 0, 1, 746, 85, 790, 0, 1159, 147, 0, 4341, 0, 0, 1133, 81, 231, 331, 5253, 5008, 657, 320, 135, 6, 1458, 632, 188, 6, 233] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 731, 1220, 1358, 869, 2315, 254, 1824, 17, 1263, 1583, 79, 152, 371, 159, 23, 1455, 0, 0, 14, 62, 911, 647, 6, 979, 0, 4565, 79, 4124, 103, 246, 0, 138, 0, 0, 692, 147, 0, 6, 695, 125, 883, 1, 1058, 140, 0, 4312, 0, 0, 1115, 70, 265, 397, 4932, 4706, 729, 392, 178, 10, 1608, 753, 207, 15, 169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 595, 1294, 1012, 836, 2384, 264, 1713, 5, 1467, 1770, 204, 203, 153, 55, 20, 1255, 2, 0, 3, 24, 1109, 737, 0, 998, 0, 4284, 95, 4545, 108, 196, 0, 126, 0, 0, 747, 190, 0, 7, 631, 53, 842, 0, 1106, 184, 0, 4184, 0, 0, 1007, 82, 342, 290, 5344, 4848, 669, 433, 147, 3, 1555, 677, 193, 6, 155] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1090, 1279, 1063, 913, 2518, 285, 1766, 17, 1254, 1997, 65, 132, 236, 254, 37, 1200, 0, 0, 35, 55, 1070, 563, 9, 1120, 0, 4251, 91, 3672, 118, 220, 0, 136, 0, 0, 615, 206, 0, 13, 770, 103, 755, 0, 1090, 148, 0, 4195, 0, 0, 977, 113, 343, 403, 4943, 4407, 744, 521, 156, 10, 1670, 887, 306, 34, 297] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 661, 1208, 1246, 882, 2183, 252, 1679, 32, 1550, 1715, 63, 123, 248, 248, 17, 1468, 0, 1, 43, 60, 1202, 694, 5, 1244, 0, 3748, 113, 3803, 117, 185, 1, 164, 0, 2, 820, 307, 0, 13, 791, 99, 866, 4, 1133, 149, 0, 4234, 1, 0, 1050, 117, 417, 407, 5158, 4513, 691, 492, 168, 17, 1329, 732, 383, 10, 294] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 595, 1294, 1012, 836, 2384, 264, 1713, 5, 1467, 1770, 204, 203, 153, 55, 20, 1255, 2, 0, 3, 24, 1109, 737, 0, 998, 0, 4284, 95, 4545, 108, 196, 0, 126, 0, 0, 747, 190, 0, 7, 631, 53, 842, 0, 1106, 184, 0, 4184, 0, 0, 1007, 82, 342, 290, 5344, 4848, 669, 433, 147, 3, 1555, 677, 193, 6, 155] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 731, 1220, 1358, 869, 276, 254, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1263, 1583, 79, 152, 371, 159, 23, 1045, 910, 304, 0, 0, 0, 0, 0, 1455, 0, 0, 14, 62, 911, 647, 6, 1844, 993, 0, 0, 0, 0, 0, 0, 979, 0, 4565, 79, 579, 103, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 692, 147, 0, 6, 695, 3939, 0, 0, 0, 0, 0, 0, 0, 125, 883, 1, 1058, 140, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1115, 70, 265, 397, 0, 3661, 729, 0, 0, 0, 0, 0, 0, 0, 0, 392, 178, 10, 1608, 753, 207, 15, 169, 1701, 1129, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6532, 5906, 5932, 6551, 5617, 6215, 6237, 6162] avg=6144.0000 max=6551 max_rank=3 min=5617 min_rank=4 imbalance_ratio=1.066243 max_load_reduction=42601 max_load_reduction_pct=0.866720 moved_to_echo=24231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 530, 1231, 1030, 925, 312, 248, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1556, 1715, 212, 132, 302, 37, 14, 1066, 1030, 290, 0, 0, 0, 0, 0, 1374, 0, 0, 7, 49, 1237, 797, 3, 1871, 1093, 0, 0, 0, 0, 0, 0, 1037, 0, 4015, 89, 589, 129, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 643, 187, 0, 3, 695, 4339, 0, 0, 0, 0, 0, 0, 0, 75, 784, 0, 1138, 146, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 91, 233, 265, 0, 3731, 715, 0, 0, 0, 0, 0, 0, 0, 0, 480, 142, 7, 1390, 716, 193, 6, 267, 1726, 1278, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6067, 6364, 6431, 6064, 6011, 5970, 6040, 6205] avg=6144.0000 max=6431 max_rank=2 min=5970 min_rank=5 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 488, 1179, 1331, 815, 265, 254, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1261, 1710, 341, 99, 149, 28, 13, 1074, 873, 298, 0, 0, 0, 0, 0, 1073, 0, 0, 3, 32, 1240, 717, 2, 1959, 1155, 0, 0, 0, 0, 0, 0, 1106, 0, 4049, 103, 615, 121, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 872, 289, 0, 6, 812, 4584, 0, 0, 0, 0, 0, 0, 0, 87, 755, 0, 1242, 147, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 942, 104, 375, 113, 0, 3759, 664, 0, 0, 0, 0, 0, 0, 0, 0, 385, 88, 7, 1496, 671, 256, 3, 226, 1807, 1083, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 5853, 6181, 6111, 6729, 6165, 5957, 6022] avg=6144.0000 max=6729 max_rank=4 min=5853 min_rank=1 imbalance_ratio=1.095215 max_load_reduction=42423 max_load_reduction_pct=0.863098 moved_to_echo=24873 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 440, 1206, 1042, 697, 297, 247, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1565, 1757, 81, 153, 233, 20, 10, 1085, 981, 315, 0, 0, 0, 0, 0, 1031, 6, 0, 13, 53, 1158, 828, 1, 2004, 1197, 0, 0, 0, 0, 0, 0, 867, 0, 3724, 96, 630, 108, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 861, 267, 0, 4, 815, 4750, 0, 0, 0, 0, 0, 0, 0, 61, 757, 0, 1197, 161, 0, 4157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 96, 374, 349, 0, 3798, 606, 0, 0, 0, 0, 0, 0, 0, 0, 368, 115, 6, 1314, 650, 238, 3, 204, 1849, 1217, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5697, 6201, 6291, 5587, 6847, 6333, 6232, 5964] avg=6144.0000 max=6847 max_rank=4 min=5587 min_rank=3 imbalance_ratio=1.114421 max_load_reduction=42305 max_load_reduction_pct=0.860697 moved_to_echo=25376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 477, 1278, 975, 827, 279, 239, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1511, 1754, 170, 136, 286, 5, 18, 1113, 920, 306, 0, 0, 0, 0, 0, 1293, 0, 0, 2, 23, 1173, 806, 4, 1935, 1058, 0, 0, 0, 0, 0, 0, 1045, 0, 4296, 83, 608, 93, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 796, 175, 0, 1, 746, 4195, 0, 0, 0, 0, 0, 0, 0, 85, 790, 0, 1159, 147, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 81, 231, 331, 0, 3895, 657, 0, 0, 0, 0, 0, 0, 0, 0, 320, 135, 6, 1458, 632, 188, 6, 233, 1785, 1141, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5847, 6220, 6294, 6267, 6076, 6216, 6328, 5904] avg=6144.0000 max=6328 max_rank=6 min=5847 min_rank=0 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 530, 1231, 1030, 925, 312, 248, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1556, 1715, 212, 132, 302, 37, 14, 1066, 1030, 290, 0, 0, 0, 0, 0, 1374, 0, 0, 7, 49, 1237, 797, 3, 1871, 1093, 0, 0, 0, 0, 0, 0, 1037, 0, 4015, 89, 589, 129, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 643, 187, 0, 3, 695, 4339, 0, 0, 0, 0, 0, 0, 0, 75, 784, 0, 1138, 146, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 91, 233, 265, 0, 3731, 715, 0, 0, 0, 0, 0, 0, 0, 0, 480, 142, 7, 1390, 716, 193, 6, 267, 1726, 1278, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6067, 6364, 6431, 6064, 6011, 5970, 6040, 6205] avg=6144.0000 max=6431 max_rank=2 min=5970 min_rank=5 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 440, 1206, 1042, 697, 297, 247, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1565, 1757, 81, 153, 233, 20, 10, 1085, 981, 315, 0, 0, 0, 0, 0, 1031, 6, 0, 13, 53, 1158, 828, 1, 2004, 1197, 0, 0, 0, 0, 0, 0, 867, 0, 3724, 96, 630, 108, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 861, 267, 0, 4, 815, 4750, 0, 0, 0, 0, 0, 0, 0, 61, 757, 0, 1197, 161, 0, 4157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 96, 374, 349, 0, 3798, 606, 0, 0, 0, 0, 0, 0, 0, 0, 368, 115, 6, 1314, 650, 238, 3, 204, 1849, 1217, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5697, 6201, 6291, 5587, 6847, 6333, 6232, 5964] avg=6144.0000 max=6847 max_rank=4 min=5587 min_rank=3 imbalance_ratio=1.114421 max_load_reduction=42305 max_load_reduction_pct=0.860697 moved_to_echo=25376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1090, 1279, 1063, 913, 300, 285, 1766, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1254, 1997, 65, 132, 236, 254, 37, 979, 990, 296, 0, 0, 0, 0, 0, 1200, 0, 0, 35, 55, 1070, 563, 9, 1642, 996, 0, 0, 0, 0, 0, 0, 1120, 0, 4251, 91, 516, 118, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 615, 206, 0, 13, 770, 3947, 0, 0, 0, 0, 0, 0, 0, 103, 755, 0, 1090, 148, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 977, 113, 343, 403, 0, 3428, 744, 0, 0, 0, 0, 0, 0, 0, 0, 521, 156, 10, 1670, 887, 306, 34, 297, 1514, 1228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6696, 6257, 5570, 6316, 5687, 5995, 6008, 6623] avg=6144.0000 max=6696 max_rank=0 min=5570 min_rank=2 imbalance_ratio=1.089844 max_load_reduction=42456 max_load_reduction_pct=0.863770 moved_to_echo=24313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 731, 1220, 1358, 869, 276, 254, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1263, 1583, 79, 152, 371, 159, 23, 1045, 910, 304, 0, 0, 0, 0, 0, 1455, 0, 0, 14, 62, 911, 647, 6, 1844, 993, 0, 0, 0, 0, 0, 0, 979, 0, 4565, 79, 579, 103, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 692, 147, 0, 6, 695, 3939, 0, 0, 0, 0, 0, 0, 0, 125, 883, 1, 1058, 140, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1115, 70, 265, 397, 0, 3661, 729, 0, 0, 0, 0, 0, 0, 0, 0, 392, 178, 10, 1608, 753, 207, 15, 169, 1701, 1129, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6532, 5906, 5932, 6551, 5617, 6215, 6237, 6162] avg=6144.0000 max=6551 max_rank=3 min=5617 min_rank=4 imbalance_ratio=1.066243 max_load_reduction=42601 max_load_reduction_pct=0.866720 moved_to_echo=24231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 595, 1294, 1012, 836, 284, 264, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1467, 1770, 204, 203, 153, 55, 20, 1077, 937, 295, 0, 0, 0, 0, 0, 1255, 2, 0, 3, 24, 1109, 737, 0, 2032, 1076, 0, 0, 0, 0, 0, 0, 998, 0, 4284, 95, 639, 108, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 747, 190, 0, 7, 631, 4268, 0, 0, 0, 0, 0, 0, 0, 53, 842, 0, 1106, 184, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 82, 342, 290, 0, 3771, 669, 0, 0, 0, 0, 0, 0, 0, 0, 433, 147, 3, 1555, 677, 193, 6, 155, 1874, 1163, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5998, 6186, 6238, 6320, 5969, 6074, 6161, 6206] avg=6144.0000 max=6320 max_rank=3 min=5969 min_rank=4 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 661, 1208, 1246, 882, 253, 252, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1550, 1715, 63, 123, 248, 248, 17, 1006, 861, 302, 0, 0, 0, 0, 0, 1468, 0, 1, 43, 60, 1202, 694, 5, 1703, 1039, 0, 0, 0, 0, 0, 0, 1244, 0, 3748, 113, 528, 117, 185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 2, 820, 307, 0, 13, 791, 4119, 0, 0, 0, 0, 0, 0, 0, 99, 866, 4, 1133, 149, 0, 3932, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 117, 417, 407, 0, 3507, 691, 0, 0, 0, 0, 0, 0, 0, 0, 492, 168, 17, 1329, 732, 383, 10, 294, 1572, 1069, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6181, 6165, 6215, 5936, 6216, 6184, 6189, 6066] avg=6144.0000 max=6216 max_rank=4 min=5936 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=42936 max_load_reduction_pct=0.873535 moved_to_echo=24655 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 5120, 9984, 9088, 6784, 2304, 2048, 14208, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[10240, 128, 128, 128, 384, 9216, 5888, 128, 15104, 8704, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 11520, 14080, 1280, 1152, 2048, 896, 256, 8448, 7552, 2432, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 1152, 128, 11904, 5760, 2048, 128, 1920, 13952, 9344, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 8320, 768, 2688, 2560, 0, 29568, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 11520, 14080, 1280, 1152, 2048, 896, 256, 8448, 7552, 2432, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8448, 0, 33024, 768, 4736, 1024, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 0, 128, 6144, 1792, 0, 128, 6016, 34176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 8320, 768, 2688, 2560, 0, 29568, 5504, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 6528, 128, 9216, 1280, 0, 31744, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 0, 128, 6144, 1792, 0, 128, 6016, 34176, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[264, 1355, 287, 1807, 60, 115, 1344, 654, 0, 0, 12, 576, 3299, 0, 0, 811, 304, 735, 6406, 0, 1, 698, 1409, 0, 0, 4169, 1374, 0, 1, 0, 2800, 0, 4163, 402, 2, 0, 90, 0, 310, 145, 0, 0, 0, 435, 52, 58, 3001, 1355, 539, 1238, 1094, 1376, 0, 0, 0, 902, 626, 234, 169, 8, 72, 184, 155, 4061] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[285, 1474, 161, 1659, 72, 107, 1178, 907, 0, 0, 7, 482, 3263, 0, 0, 818, 337, 678, 6348, 0, 0, 791, 1538, 0, 0, 4262, 1310, 0, 1, 0, 2272, 0, 4451, 329, 13, 0, 76, 0, 347, 121, 0, 0, 0, 417, 42, 117, 3084, 1219, 619, 1254, 1217, 1345, 0, 0, 0, 912, 719, 194, 122, 11, 92, 200, 195, 4106] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[166, 1611, 368, 1409, 92, 102, 1169, 514, 0, 0, 6, 588, 3417, 0, 0, 765, 349, 758, 6262, 0, 0, 742, 1525, 0, 0, 4378, 1235, 0, 0, 0, 2598, 0, 4434, 431, 29, 0, 101, 0, 383, 113, 0, 0, 0, 351, 49, 117, 2969, 1259, 546, 1391, 1142, 1270, 0, 0, 0, 861, 776, 228, 126, 8, 109, 156, 197, 4052] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[160, 1657, 415, 1349, 69, 125, 1397, 467, 0, 0, 8, 574, 3073, 0, 0, 705, 348, 709, 6380, 0, 0, 637, 1223, 0, 0, 4451, 1122, 0, 0, 0, 2664, 0, 4585, 403, 1, 1, 159, 0, 458, 93, 0, 0, 0, 414, 42, 98, 3128, 1212, 615, 1582, 1034, 1240, 0, 2, 0, 911, 747, 258, 82, 24, 96, 164, 191, 4079] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[133, 1525, 400, 1450, 75, 161, 1406, 407, 0, 0, 5, 538, 3157, 0, 0, 743, 337, 715, 6474, 0, 0, 674, 1236, 0, 0, 4814, 1237, 0, 0, 0, 2483, 0, 4528, 386, 16, 1, 117, 0, 400, 74, 1, 0, 0, 365, 46, 170, 3211, 1280, 470, 1321, 1331, 1229, 0, 0, 0, 911, 650, 237, 73, 20, 76, 139, 163, 3967] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[192, 1761, 412, 1418, 66, 160, 1644, 417, 0, 0, 29, 605, 2900, 0, 0, 581, 345, 755, 6333, 1, 0, 569, 1114, 0, 0, 4554, 1212, 0, 1, 0, 2640, 0, 4512, 414, 31, 0, 119, 0, 419, 95, 2, 0, 0, 361, 29, 163, 3028, 1111, 610, 1526, 1286, 1350, 0, 7, 0, 957, 679, 229, 305, 41, 87, 166, 184, 3732] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[264, 1355, 287, 1807, 60, 115, 1344, 654, 0, 0, 12, 576, 3299, 0, 0, 811, 304, 735, 6406, 0, 1, 698, 1409, 0, 0, 4169, 1374, 0, 1, 0, 2800, 0, 4163, 402, 2, 0, 90, 0, 310, 145, 0, 0, 0, 435, 52, 58, 3001, 1355, 539, 1238, 1094, 1376, 0, 0, 0, 902, 626, 234, 169, 8, 72, 184, 155, 4061] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[268, 1759, 439, 1694, 69, 201, 1439, 510, 1, 1, 4, 545, 2913, 0, 1, 680, 347, 701, 6189, 2, 0, 639, 1327, 0, 1, 4628, 1164, 0, 1, 0, 2558, 0, 4167, 374, 32, 4, 100, 1, 457, 120, 0, 2, 0, 407, 44, 138, 3269, 1069, 610, 1401, 965, 1173, 0, 0, 0, 949, 774, 233, 270, 33, 81, 213, 184, 4001] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[285, 1474, 161, 1659, 72, 107, 1178, 907, 0, 0, 7, 482, 3263, 0, 0, 818, 337, 678, 6348, 0, 0, 791, 1538, 0, 0, 4262, 1310, 0, 1, 0, 2272, 0, 4451, 329, 13, 0, 76, 0, 347, 121, 0, 0, 0, 417, 42, 117, 3084, 1219, 619, 1254, 1217, 1345, 0, 0, 0, 912, 719, 194, 122, 11, 92, 200, 195, 4106] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[160, 1657, 415, 1349, 69, 125, 1397, 467, 0, 0, 8, 574, 3073, 0, 0, 705, 348, 709, 6380, 0, 0, 637, 1223, 0, 0, 4451, 1122, 0, 0, 0, 2664, 0, 4585, 403, 1, 1, 159, 0, 458, 93, 0, 0, 0, 414, 42, 98, 3128, 1212, 615, 1582, 1034, 1240, 0, 2, 0, 911, 747, 258, 82, 24, 96, 164, 191, 4079] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[181, 1569, 428, 1610, 44, 248, 1347, 389, 0, 0, 11, 582, 3053, 0, 0, 719, 362, 732, 6424, 1, 0, 776, 1435, 0, 0, 4158, 1383, 0, 0, 0, 2440, 0, 4449, 346, 32, 0, 135, 0, 328, 103, 1, 0, 0, 360, 40, 125, 3217, 1196, 595, 1384, 1111, 1185, 0, 0, 0, 965, 799, 224, 107, 16, 62, 171, 173, 4136] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[285, 1474, 161, 1659, 72, 107, 1178, 907, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 482, 3263, 0, 0, 818, 1708, 0, 0, 0, 0, 0, 0, 0, 337, 678, 3038, 0, 0, 791, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2247, 1310, 0, 1, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4451, 329, 13, 0, 76, 0, 347, 121, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 42, 117, 3084, 1219, 1261, 0, 0, 0, 0, 0, 0, 0, 619, 1254, 1217, 1345, 0, 0, 0, 912, 480, 341, 0, 0, 0, 0, 0, 0, 719, 194, 122, 11, 92, 200, 195, 4106, 539, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6278, 6382, 5830, 6029, 6140, 6168, 6178] avg=6144.0000 max=6382 max_rank=2 min=5830 min_rank=3 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[192, 1761, 412, 1418, 66, 160, 1644, 417, 0, 0, 29, 605, 2900, 0, 0, 581, 345, 755, 6333, 1, 0, 569, 1114, 0, 0, 4554, 1212, 0, 1, 0, 2640, 0, 4512, 414, 31, 0, 119, 0, 419, 95, 2, 0, 0, 361, 29, 163, 3028, 1111, 610, 1526, 1286, 1350, 0, 7, 0, 957, 679, 229, 305, 41, 87, 166, 184, 3732] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[133, 1525, 400, 1450, 75, 161, 1406, 407, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 538, 3157, 0, 0, 743, 1742, 0, 0, 0, 0, 0, 0, 0, 337, 715, 3098, 0, 0, 674, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2538, 1237, 0, 0, 0, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4528, 386, 16, 1, 117, 0, 400, 74, 781, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 365, 46, 170, 3211, 1280, 1286, 0, 0, 0, 0, 0, 0, 0, 470, 1321, 1331, 1229, 0, 0, 0, 911, 542, 348, 0, 0, 0, 0, 0, 0, 650, 237, 73, 20, 76, 139, 163, 3967, 609, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5901, 6185, 6060, 6258, 6303, 6359, 6152, 5934] avg=6144.0000 max=6359 max_rank=5 min=5901 min_rank=0 imbalance_ratio=1.034993 max_load_reduction=42793 max_load_reduction_pct=0.870626 moved_to_echo=24748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[160, 1657, 415, 1349, 69, 125, 1397, 467, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 574, 3073, 0, 0, 705, 1717, 0, 0, 0, 0, 0, 0, 0, 348, 709, 3053, 0, 0, 637, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2347, 1122, 0, 0, 0, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4585, 403, 1, 1, 159, 0, 458, 93, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 42, 98, 3128, 1212, 1267, 0, 0, 0, 0, 0, 0, 0, 615, 1582, 1034, 1240, 0, 2, 0, 911, 501, 343, 0, 0, 0, 0, 0, 0, 747, 258, 82, 24, 96, 164, 191, 4079, 563, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5957, 6077, 5970, 6133, 6422, 6161, 6228, 6204] avg=6144.0000 max=6422 max_rank=4 min=5957 min_rank=0 imbalance_ratio=1.045247 max_load_reduction=42730 max_load_reduction_pct=0.869344 moved_to_echo=25015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[264, 1355, 287, 1807, 60, 115, 1344, 654, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 576, 3299, 0, 0, 811, 1724, 0, 0, 0, 0, 0, 0, 0, 304, 735, 3066, 0, 1, 698, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2198, 1374, 0, 1, 0, 2800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4163, 402, 2, 0, 90, 0, 310, 145, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 52, 58, 3001, 1355, 1272, 0, 0, 0, 0, 0, 0, 0, 539, 1238, 1094, 1376, 0, 0, 0, 902, 469, 344, 0, 0, 0, 0, 0, 0, 626, 234, 169, 8, 72, 184, 155, 4061, 528, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6422, 6213, 6373, 5789, 6173, 5962, 6037] avg=6144.0000 max=6422 max_rank=1 min=5789 min_rank=4 imbalance_ratio=1.045247 max_load_reduction=42730 max_load_reduction_pct=0.869344 moved_to_echo=23961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[166, 1611, 368, 1409, 92, 102, 1169, 514, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 588, 3417, 0, 0, 765, 1685, 0, 0, 0, 0, 0, 0, 0, 349, 758, 2996, 0, 0, 742, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2308, 1235, 0, 0, 0, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4434, 431, 29, 0, 101, 0, 383, 113, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 49, 117, 2969, 1259, 1244, 0, 0, 0, 0, 0, 0, 0, 546, 1391, 1142, 1270, 0, 0, 0, 861, 493, 337, 0, 0, 0, 0, 0, 0, 776, 228, 126, 8, 109, 156, 197, 4052, 554, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5743, 6461, 6370, 6141, 6202, 5989, 6040, 6206] avg=6144.0000 max=6461 max_rank=1 min=5743 min_rank=0 imbalance_ratio=1.051595 max_load_reduction=42691 max_load_reduction_pct=0.868551 moved_to_echo=24437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[192, 1761, 412, 1418, 66, 160, 1644, 417, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 605, 2900, 0, 0, 581, 1704, 0, 0, 0, 0, 0, 0, 0, 345, 755, 3031, 1, 0, 569, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2401, 1212, 0, 1, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4512, 414, 31, 0, 119, 0, 419, 95, 739, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 361, 29, 163, 3028, 1111, 1258, 0, 0, 0, 0, 0, 0, 0, 610, 1526, 1286, 1350, 0, 7, 0, 957, 513, 340, 0, 0, 0, 0, 0, 0, 679, 229, 305, 41, 87, 166, 184, 3732, 576, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6395, 5819, 5815, 6254, 6329, 5952, 6589, 5999] avg=6144.0000 max=6589 max_rank=6 min=5815 min_rank=2 imbalance_ratio=1.072428 max_load_reduction=42563 max_load_reduction_pct=0.865946 moved_to_echo=24869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[133, 1525, 400, 1450, 75, 161, 1406, 407, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 538, 3157, 0, 0, 743, 1742, 0, 0, 0, 0, 0, 0, 0, 337, 715, 3098, 0, 0, 674, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2538, 1237, 0, 0, 0, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4528, 386, 16, 1, 117, 0, 400, 74, 781, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 365, 46, 170, 3211, 1280, 1286, 0, 0, 0, 0, 0, 0, 0, 470, 1321, 1331, 1229, 0, 0, 0, 911, 542, 348, 0, 0, 0, 0, 0, 0, 650, 237, 73, 20, 76, 139, 163, 3967, 609, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5901, 6185, 6060, 6258, 6303, 6359, 6152, 5934] avg=6144.0000 max=6359 max_rank=5 min=5901 min_rank=0 imbalance_ratio=1.034993 max_load_reduction=42793 max_load_reduction_pct=0.870626 moved_to_echo=24748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[268, 1759, 439, 1694, 69, 201, 1439, 510, 334, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 545, 2913, 0, 1, 680, 1671, 0, 0, 0, 0, 0, 0, 0, 347, 701, 2946, 2, 0, 639, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2424, 1164, 0, 1, 0, 2558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4167, 374, 32, 4, 100, 1, 457, 120, 755, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 407, 44, 138, 3269, 1069, 1234, 0, 0, 0, 0, 0, 0, 0, 610, 1401, 965, 1173, 0, 0, 0, 949, 524, 338, 0, 0, 0, 0, 0, 0, 774, 233, 270, 33, 81, 213, 184, 4001, 591, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6713, 5816, 5962, 6148, 6010, 6163, 5960, 6380] avg=6144.0000 max=6713 max_rank=0 min=5816 min_rank=1 imbalance_ratio=1.092611 max_load_reduction=42439 max_load_reduction_pct=0.863424 moved_to_echo=24513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[285, 1474, 161, 1659, 72, 107, 1178, 907, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 482, 3263, 0, 0, 818, 1708, 0, 0, 0, 0, 0, 0, 0, 337, 678, 3038, 0, 0, 791, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2247, 1310, 0, 1, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4451, 329, 13, 0, 76, 0, 347, 121, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 42, 117, 3084, 1219, 1261, 0, 0, 0, 0, 0, 0, 0, 619, 1254, 1217, 1345, 0, 0, 0, 912, 480, 341, 0, 0, 0, 0, 0, 0, 719, 194, 122, 11, 92, 200, 195, 4106, 539, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6278, 6382, 5830, 6029, 6140, 6168, 6178] avg=6144.0000 max=6382 max_rank=2 min=5830 min_rank=3 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[181, 1569, 428, 1610, 44, 248, 1347, 389, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 582, 3053, 0, 0, 719, 1729, 0, 0, 0, 0, 0, 0, 0, 362, 732, 3074, 1, 0, 776, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 1383, 0, 0, 0, 2440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4449, 346, 32, 0, 135, 0, 328, 103, 675, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 360, 40, 125, 3217, 1196, 1276, 0, 0, 0, 0, 0, 0, 0, 595, 1384, 1111, 1185, 0, 0, 0, 965, 468, 345, 0, 0, 0, 0, 0, 0, 799, 224, 107, 16, 62, 171, 173, 4136, 526, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6094, 6380, 6015, 6068, 6215, 6053, 6214] avg=6144.0000 max=6380 max_rank=2 min=6015 min_rank=3 imbalance_ratio=1.038411 max_load_reduction=42772 max_load_reduction_pct=0.870199 moved_to_echo=24550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[268, 1759, 439, 1694, 69, 201, 1439, 510, 334, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 545, 2913, 0, 1, 680, 1671, 0, 0, 0, 0, 0, 0, 0, 347, 701, 2946, 2, 0, 639, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2424, 1164, 0, 1, 0, 2558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4167, 374, 32, 4, 100, 1, 457, 120, 755, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 407, 44, 138, 3269, 1069, 1234, 0, 0, 0, 0, 0, 0, 0, 610, 1401, 965, 1173, 0, 0, 0, 949, 524, 338, 0, 0, 0, 0, 0, 0, 774, 233, 270, 33, 81, 213, 184, 4001, 591, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6713, 5816, 5962, 6148, 6010, 6163, 5960, 6380] avg=6144.0000 max=6713 max_rank=0 min=5816 min_rank=1 imbalance_ratio=1.092611 max_load_reduction=42439 max_load_reduction_pct=0.863424 moved_to_echo=24513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[192, 1761, 412, 1418, 66, 160, 1644, 417, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 605, 2900, 0, 0, 581, 1704, 0, 0, 0, 0, 0, 0, 0, 345, 755, 3031, 1, 0, 569, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2401, 1212, 0, 1, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4512, 414, 31, 0, 119, 0, 419, 95, 739, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 361, 29, 163, 3028, 1111, 1258, 0, 0, 0, 0, 0, 0, 0, 610, 1526, 1286, 1350, 0, 7, 0, 957, 513, 340, 0, 0, 0, 0, 0, 0, 679, 229, 305, 41, 87, 166, 184, 3732, 576, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6395, 5819, 5815, 6254, 6329, 5952, 6589, 5999] avg=6144.0000 max=6589 max_rank=6 min=5815 min_rank=2 imbalance_ratio=1.072428 max_load_reduction=42563 max_load_reduction_pct=0.865946 moved_to_echo=24869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 11136, 9216, 10240, 0, 128, 0, 7424, 4096, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 12800, 2944, 12416, 640, 1280, 11008, 4352, 2560, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5888, 1920, 1280, 256, 768, 1408, 1536, 32256, 4608, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 18688, 10112, 0, 128, 0, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 4608, 25088, 0, 128, 5888, 13696, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 18688, 10112, 0, 128, 0, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 3200, 384, 1024, 24960, 9728, 10112, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 12800, 2944, 12416, 640, 1280, 11008, 4352, 2560, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 5888, 24320, 128, 128, 5632, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[35328, 3200, 256, 128, 1024, 128, 3200, 896, 5760, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 5888, 24320, 128, 128, 5632, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1195, 65, 0, 5165, 0, 0, 42, 0, 0, 177, 127, 29, 850, 375, 387, 447, 0, 0, 202, 0, 0, 485, 351, 435, 0, 69, 407, 0, 64, 75, 0, 0, 69, 6024, 222, 0, 0, 0, 0, 1067, 775, 512, 0, 653, 5020, 75, 347, 4947, 387, 75, 0, 1136, 0, 5248, 6815, 0, 3592, 0, 0, 0, 0, 0, 1241, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1063, 45, 0, 5041, 0, 0, 90, 1, 0, 174, 216, 0, 816, 403, 308, 360, 0, 0, 223, 0, 0, 579, 331, 344, 0, 72, 401, 0, 170, 95, 0, 0, 57, 5643, 176, 0, 0, 0, 0, 1022, 725, 478, 0, 519, 4415, 96, 354, 5409, 326, 95, 0, 935, 4, 5826, 6809, 0, 3466, 0, 0, 0, 0, 0, 2064, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[999, 74, 0, 4944, 0, 0, 123, 1, 0, 195, 131, 26, 852, 380, 402, 517, 0, 0, 137, 0, 0, 543, 413, 491, 0, 66, 298, 0, 78, 69, 0, 0, 79, 5810, 260, 0, 0, 0, 0, 1180, 814, 529, 0, 610, 4956, 69, 381, 4993, 402, 69, 0, 1194, 0, 5386, 6761, 0, 3685, 0, 0, 0, 0, 0, 1235, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1021, 60, 0, 5078, 0, 0, 6, 0, 0, 202, 141, 1, 881, 391, 406, 350, 0, 0, 179, 0, 0, 555, 221, 453, 0, 92, 309, 0, 77, 71, 0, 0, 76, 5932, 192, 0, 0, 0, 0, 1358, 766, 585, 0, 501, 4872, 74, 372, 4844, 421, 71, 0, 1027, 4, 5586, 6791, 0, 3659, 0, 0, 0, 0, 0, 1526, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[999, 74, 0, 4944, 0, 0, 123, 1, 0, 195, 131, 26, 852, 380, 402, 517, 0, 0, 137, 0, 0, 543, 413, 491, 0, 66, 298, 0, 78, 69, 0, 0, 79, 5810, 260, 0, 0, 0, 0, 1180, 814, 529, 0, 610, 4956, 69, 381, 4993, 402, 69, 0, 1194, 0, 5386, 6761, 0, 3685, 0, 0, 0, 0, 0, 1235, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1195, 65, 0, 5165, 0, 0, 42, 0, 0, 177, 127, 29, 850, 375, 387, 447, 0, 0, 202, 0, 0, 485, 351, 435, 0, 69, 407, 0, 64, 75, 0, 0, 69, 6024, 222, 0, 0, 0, 0, 1067, 775, 512, 0, 653, 5020, 75, 347, 4947, 387, 75, 0, 1136, 0, 5248, 6815, 0, 3592, 0, 0, 0, 0, 0, 1241, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1211, 59, 0, 5166, 0, 1, 169, 7, 1, 195, 194, 13, 823, 408, 358, 433, 0, 0, 138, 1, 0, 531, 419, 549, 0, 69, 418, 0, 142, 92, 0, 0, 69, 5884, 292, 0, 1, 1, 0, 1024, 773, 558, 12, 927, 4639, 93, 342, 4902, 361, 91, 1, 1107, 0, 4965, 6726, 1, 3861, 0, 0, 0, 2, 0, 1123, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1019, 60, 0, 5035, 0, 0, 20, 0, 0, 220, 108, 4, 805, 369, 371, 439, 0, 0, 239, 0, 0, 536, 385, 403, 0, 75, 196, 0, 48, 65, 0, 0, 92, 5913, 264, 0, 0, 0, 0, 1346, 768, 530, 0, 546, 5061, 65, 356, 4889, 376, 63, 0, 948, 0, 5937, 6761, 0, 3535, 0, 0, 0, 0, 0, 1305, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1063, 45, 0, 5041, 0, 0, 90, 1, 0, 174, 216, 0, 816, 403, 308, 360, 0, 0, 223, 0, 0, 579, 331, 344, 0, 72, 401, 0, 170, 95, 0, 0, 57, 5643, 176, 0, 0, 0, 0, 1022, 725, 478, 0, 519, 4415, 96, 354, 5409, 326, 95, 0, 935, 4, 5826, 6809, 0, 3466, 0, 0, 0, 0, 0, 2064, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1137, 66, 0, 4824, 0, 0, 6, 0, 0, 194, 227, 0, 831, 319, 357, 314, 0, 0, 155, 0, 0, 600, 175, 254, 0, 60, 388, 0, 138, 90, 0, 0, 87, 5828, 163, 0, 0, 0, 0, 1225, 753, 388, 0, 585, 4799, 90, 324, 5147, 392, 90, 0, 1059, 23, 5752, 6714, 0, 3539, 0, 0, 0, 0, 0, 2059, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1211, 59, 0, 5166, 0, 1, 169, 7, 1, 195, 194, 13, 823, 408, 358, 433, 0, 0, 138, 1, 0, 531, 419, 549, 0, 69, 418, 0, 142, 92, 0, 0, 69, 5884, 292, 0, 1, 1, 0, 1024, 773, 558, 12, 927, 4639, 93, 342, 4902, 361, 91, 1, 1107, 0, 4965, 6726, 1, 3861, 0, 0, 0, 2, 0, 1123, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1082, 91, 0, 4967, 0, 0, 0, 0, 0, 192, 162, 0, 818, 411, 298, 589, 0, 0, 400, 0, 0, 587, 419, 442, 0, 44, 172, 0, 72, 82, 0, 0, 113, 5665, 214, 0, 0, 0, 0, 959, 753, 468, 0, 517, 5021, 82, 454, 5163, 340, 82, 0, 1030, 0, 5670, 6793, 0, 3604, 0, 0, 0, 0, 0, 1396, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1063, 45, 0, 4944, 0, 0, 90, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 216, 0, 816, 403, 308, 360, 1731, 1106, 922, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 579, 331, 344, 3678, 1204, 0, 0, 0, 0, 0, 0, 0, 72, 401, 0, 170, 95, 0, 0, 5605, 0, 0, 0, 0, 0, 0, 0, 57, 4537, 176, 0, 0, 0, 0, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 725, 478, 0, 519, 3477, 96, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 95, 0, 935, 4, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 0, 0, 0, 0, 0, 2064, 1, 962, 97, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6036, 6359, 6343, 5792, 5649, 6224, 6606] avg=6144.0000 max=6606 max_rank=7 min=5649 min_rank=5 imbalance_ratio=1.075195 max_load_reduction=42546 max_load_reduction_pct=0.865601 moved_to_echo=24271 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1195, 65, 0, 5065, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 127, 29, 850, 375, 387, 447, 1583, 1180, 1048, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 485, 351, 435, 3364, 1206, 0, 0, 0, 0, 0, 0, 0, 69, 407, 0, 64, 75, 0, 0, 5609, 0, 0, 0, 0, 0, 0, 0, 69, 4844, 222, 0, 0, 0, 0, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 775, 512, 0, 653, 3953, 75, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 75, 0, 1136, 0, 4382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 0, 0, 0, 1241, 0, 866, 100, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6367, 6203, 6043, 6224, 6202, 6315, 5980, 5818] avg=6144.0000 max=6367 max_rank=0 min=5818 min_rank=7 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24315 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1021, 60, 0, 4980, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 141, 1, 881, 391, 406, 350, 1551, 1162, 1017, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 555, 221, 453, 3293, 1202, 0, 0, 0, 0, 0, 0, 0, 92, 309, 0, 77, 71, 0, 0, 5589, 0, 0, 0, 0, 0, 0, 0, 76, 4770, 192, 0, 0, 0, 0, 1358, 0, 0, 0, 0, 0, 0, 0, 0, 766, 585, 0, 501, 3837, 74, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 71, 0, 1027, 4, 4664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 1526, 1, 922, 98, 18, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6067, 6102, 5903, 6138, 6396, 6135, 6187, 6224] avg=6144.0000 max=6396 max_rank=4 min=5903 min_rank=2 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=24942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1019, 60, 0, 4938, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 108, 4, 805, 369, 371, 439, 1565, 1159, 1057, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 536, 385, 403, 3324, 1195, 0, 0, 0, 0, 0, 0, 0, 75, 196, 0, 48, 65, 0, 0, 5566, 0, 0, 0, 0, 0, 0, 0, 92, 4754, 264, 0, 0, 0, 0, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 768, 530, 0, 546, 3985, 65, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 63, 0, 948, 0, 4957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 0, 0, 0, 0, 0, 1305, 0, 980, 97, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6037, 6097, 6082, 5950, 6456, 6250, 6344, 5936] avg=6144.0000 max=6456 max_rank=4 min=5936 min_rank=7 imbalance_ratio=1.050781 max_load_reduction=42696 max_load_reduction_pct=0.868652 moved_to_echo=24986 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1137, 66, 0, 4731, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 227, 0, 831, 319, 357, 314, 1647, 1142, 1002, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 600, 175, 254, 3500, 1187, 0, 0, 0, 0, 0, 0, 0, 60, 388, 0, 138, 90, 0, 0, 5527, 0, 0, 0, 0, 0, 0, 0, 87, 4686, 163, 0, 0, 0, 0, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 753, 388, 0, 585, 3779, 90, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 90, 0, 1059, 23, 4803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 0, 0, 0, 0, 0, 2059, 0, 949, 93, 18, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5940, 6033, 5871, 6203, 6161, 5919, 6367, 6658] avg=6144.0000 max=6658 max_rank=7 min=5871 min_rank=2 imbalance_ratio=1.083659 max_load_reduction=42494 max_load_reduction_pct=0.864543 moved_to_echo=25105 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[999, 74, 0, 4848, 0, 0, 123, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 131, 26, 852, 380, 402, 517, 1599, 1139, 1035, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 543, 413, 491, 3394, 1196, 0, 0, 0, 0, 0, 0, 0, 66, 298, 0, 78, 69, 0, 0, 5565, 0, 0, 0, 0, 0, 0, 0, 79, 4671, 260, 0, 0, 0, 0, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 814, 529, 0, 610, 3902, 69, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 69, 0, 1194, 0, 4497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 1235, 0, 889, 96, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6045, 6276, 6174, 6076, 6190, 6305, 6162, 5924] avg=6144.0000 max=6305 max_rank=5 min=5924 min_rank=7 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1019, 60, 0, 4938, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 108, 4, 805, 369, 371, 439, 1565, 1159, 1057, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 536, 385, 403, 3324, 1195, 0, 0, 0, 0, 0, 0, 0, 75, 196, 0, 48, 65, 0, 0, 5566, 0, 0, 0, 0, 0, 0, 0, 92, 4754, 264, 0, 0, 0, 0, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 768, 530, 0, 546, 3985, 65, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 63, 0, 948, 0, 4957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 0, 0, 0, 0, 0, 1305, 0, 980, 97, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6037, 6097, 6082, 5950, 6456, 6250, 6344, 5936] avg=6144.0000 max=6456 max_rank=4 min=5936 min_rank=7 imbalance_ratio=1.050781 max_load_reduction=42696 max_load_reduction_pct=0.868652 moved_to_echo=24986 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1211, 59, 0, 5062, 0, 1, 169, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 195, 194, 13, 823, 408, 358, 433, 1570, 1157, 972, 0, 0, 0, 0, 0, 0, 0, 138, 1, 0, 531, 419, 549, 3332, 1190, 0, 0, 0, 0, 0, 0, 0, 69, 418, 0, 142, 92, 0, 0, 5536, 0, 0, 0, 0, 0, 0, 0, 69, 4727, 292, 0, 1, 1, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 773, 558, 12, 927, 3646, 93, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 91, 1, 1107, 0, 4141, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 2, 0, 1123, 0, 824, 104, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6509, 6124, 6160, 6257, 6114, 6351, 5702, 5935] avg=6144.0000 max=6509 max_rank=0 min=5702 min_rank=6 imbalance_ratio=1.059408 max_load_reduction=42643 max_load_reduction_pct=0.867574 moved_to_echo=24102 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1063, 45, 0, 4944, 0, 0, 90, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 216, 0, 816, 403, 308, 360, 1731, 1106, 922, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 579, 331, 344, 3678, 1204, 0, 0, 0, 0, 0, 0, 0, 72, 401, 0, 170, 95, 0, 0, 5605, 0, 0, 0, 0, 0, 0, 0, 57, 4537, 176, 0, 0, 0, 0, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 725, 478, 0, 519, 3477, 96, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 95, 0, 935, 4, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 0, 0, 0, 0, 0, 2064, 1, 962, 97, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6036, 6359, 6343, 5792, 5649, 6224, 6606] avg=6144.0000 max=6606 max_rank=7 min=5649 min_rank=5 imbalance_ratio=1.075195 max_load_reduction=42546 max_load_reduction_pct=0.865601 moved_to_echo=24271 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1082, 91, 0, 4871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 162, 0, 818, 411, 298, 589, 1653, 1110, 1048, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 587, 419, 442, 3510, 1202, 0, 0, 0, 0, 0, 0, 0, 44, 172, 0, 72, 82, 0, 0, 5591, 0, 0, 0, 0, 0, 0, 0, 113, 4555, 214, 0, 0, 0, 0, 959, 0, 0, 0, 0, 0, 0, 0, 0, 753, 468, 0, 517, 3954, 82, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 82, 0, 1030, 0, 4734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 0, 0, 0, 0, 1396, 0, 936, 96, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6044, 6281, 6560, 5961, 5841, 6228, 6186, 6051] avg=6144.0000 max=6560 max_rank=2 min=5841 min_rank=4 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24306 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1211, 59, 0, 5062, 0, 1, 169, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 195, 194, 13, 823, 408, 358, 433, 1570, 1157, 972, 0, 0, 0, 0, 0, 0, 0, 138, 1, 0, 531, 419, 549, 3332, 1190, 0, 0, 0, 0, 0, 0, 0, 69, 418, 0, 142, 92, 0, 0, 5536, 0, 0, 0, 0, 0, 0, 0, 69, 4727, 292, 0, 1, 1, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 773, 558, 12, 927, 3646, 93, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 91, 1, 1107, 0, 4141, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 2, 0, 1123, 0, 824, 104, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6509, 6124, 6160, 6257, 6114, 6351, 5702, 5935] avg=6144.0000 max=6509 max_rank=0 min=5702 min_rank=6 imbalance_ratio=1.059408 max_load_reduction=42643 max_load_reduction_pct=0.867574 moved_to_echo=24102 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1664, 1408, 128, 6784, 3072, 2944, 3456, 12928, 9216, 8192, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[29056, 0, 0, 0, 128, 0, 12032, 128, 7424, 896, 256, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 37632, 1792, 0, 128, 128, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8832, 640, 0, 39552, 0, 128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[29056, 0, 0, 0, 128, 0, 12032, 128, 7424, 896, 256, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 640, 128, 8448, 128, 37120, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 1792, 128, 0, 4480, 2816, 3456, 27520, 9600, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8832, 640, 0, 39552, 0, 128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6144, 4096, 128, 4864, 30592, 768, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 1792, 128, 0, 4480, 2816, 3456, 27520, 9600, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 640, 2688, 0, 896, 640, 0, 0, 44672, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[820, 7576, 0, 0, 371, 0, 42, 0, 0, 300, 42, 3122, 0, 57, 517, 1016, 57, 2721, 0, 0, 176, 42, 0, 392, 6889, 4556, 470, 381, 0, 0, 0, 36, 0, 348, 201, 0, 98, 1250, 484, 28, 512, 0, 0, 72, 0, 3635, 1, 0, 5044, 0, 0, 4266, 420, 56, 1, 0, 0, 0, 0, 0, 0, 2436, 376, 341] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[726, 7467, 0, 0, 347, 0, 66, 0, 0, 299, 66, 2942, 0, 87, 519, 1157, 87, 3174, 0, 0, 240, 66, 0, 480, 6858, 4524, 606, 418, 0, 0, 0, 8, 0, 299, 241, 0, 96, 1083, 480, 20, 486, 0, 0, 60, 0, 3998, 0, 0, 5598, 0, 7, 3477, 478, 16, 0, 0, 0, 0, 0, 0, 0, 1916, 449, 311] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[661, 7521, 0, 0, 328, 0, 63, 0, 0, 348, 64, 3099, 0, 69, 516, 1521, 69, 2648, 0, 0, 267, 63, 0, 486, 6809, 4704, 596, 336, 0, 0, 1, 42, 0, 328, 275, 0, 88, 1021, 482, 20, 464, 0, 0, 69, 0, 3853, 0, 0, 5501, 0, 4, 3919, 462, 73, 2, 0, 0, 0, 0, 0, 0, 1887, 285, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[716, 7517, 0, 0, 350, 0, 53, 0, 0, 300, 53, 2852, 0, 79, 513, 1609, 79, 2484, 0, 0, 285, 53, 2, 514, 6788, 4932, 625, 412, 0, 0, 0, 29, 0, 275, 188, 0, 69, 908, 476, 64, 571, 0, 0, 66, 0, 4120, 0, 0, 5817, 0, 12, 3612, 471, 69, 0, 0, 0, 0, 0, 0, 0, 1604, 372, 213] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[753, 7490, 0, 0, 324, 0, 60, 0, 0, 353, 60, 3196, 0, 76, 529, 1514, 76, 2476, 0, 0, 213, 60, 0, 511, 6691, 4692, 506, 355, 0, 0, 0, 50, 0, 294, 249, 0, 82, 1437, 433, 7, 431, 0, 0, 92, 0, 3555, 0, 0, 5547, 0, 0, 3705, 491, 17, 0, 0, 0, 0, 0, 0, 0, 2181, 399, 247] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[661, 7521, 0, 0, 328, 0, 63, 0, 0, 348, 64, 3099, 0, 69, 516, 1521, 69, 2648, 0, 0, 267, 63, 0, 486, 6809, 4704, 596, 336, 0, 0, 1, 42, 0, 328, 275, 0, 88, 1021, 482, 20, 464, 0, 0, 69, 0, 3853, 0, 0, 5501, 0, 4, 3919, 462, 73, 2, 0, 0, 0, 0, 0, 0, 1887, 285, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[775, 7523, 0, 1, 366, 0, 57, 0, 0, 324, 54, 3095, 1, 70, 530, 1555, 69, 2425, 1, 2, 283, 54, 0, 518, 6794, 4527, 557, 375, 0, 1, 0, 43, 1, 385, 247, 0, 106, 1012, 459, 20, 565, 0, 0, 69, 2, 3791, 0, 1, 5691, 1, 7, 3769, 451, 91, 0, 0, 0, 1, 2, 0, 0, 1865, 316, 300] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[713, 7485, 0, 0, 358, 0, 90, 0, 0, 331, 90, 2929, 0, 113, 588, 1529, 112, 2785, 0, 0, 291, 90, 0, 405, 6803, 4767, 513, 396, 1, 0, 0, 40, 0, 307, 234, 0, 88, 1359, 462, 30, 433, 0, 0, 44, 0, 3825, 0, 0, 5329, 0, 0, 3495, 422, 74, 0, 0, 0, 0, 0, 0, 0, 2049, 382, 190] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[720, 7536, 0, 0, 331, 0, 60, 0, 0, 343, 60, 2705, 0, 92, 539, 1478, 92, 2547, 0, 0, 232, 60, 0, 431, 6832, 4793, 544, 402, 0, 0, 0, 47, 0, 301, 216, 0, 69, 1289, 419, 36, 422, 0, 0, 75, 0, 3820, 0, 0, 5621, 0, 3, 3724, 437, 56, 0, 0, 0, 0, 0, 0, 0, 2223, 409, 188] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[820, 5042, 0, 0, 371, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 42, 3122, 0, 57, 517, 1016, 737, 0, 0, 0, 0, 0, 0, 0, 57, 2721, 0, 0, 176, 42, 0, 392, 2386, 18, 0, 0, 0, 0, 0, 0, 490, 4556, 470, 381, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 201, 0, 98, 1250, 484, 28, 3894, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 72, 0, 3635, 1, 0, 1779, 0, 0, 0, 0, 0, 0, 0, 1722, 0, 0, 4266, 420, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2436, 376, 341, 2505, 936, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6275, 5791, 5792, 5933, 6303, 5999, 6465, 6594] avg=6144.0000 max=6594 max_rank=7 min=5791 min_rank=1 imbalance_ratio=1.073242 max_load_reduction=42558 max_load_reduction_pct=0.865845 moved_to_echo=25361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[726, 4971, 0, 0, 347, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 66, 2942, 0, 87, 519, 1157, 726, 0, 0, 0, 0, 0, 0, 0, 87, 3174, 0, 0, 240, 66, 0, 480, 2648, 17, 0, 0, 0, 0, 0, 0, 488, 4524, 606, 418, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 241, 0, 96, 1083, 480, 20, 3876, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 60, 0, 3998, 0, 0, 1753, 0, 0, 0, 0, 0, 0, 0, 1912, 0, 7, 3477, 478, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1916, 449, 311, 2494, 1038, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 5796, 6712, 6044, 6095, 6297, 5890, 6208] avg=6144.0000 max=6712 max_rank=2 min=5796 min_rank=1 imbalance_ratio=1.092448 max_load_reduction=42440 max_load_reduction_pct=0.863444 moved_to_echo=24490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[713, 7485, 0, 0, 358, 0, 90, 0, 0, 331, 90, 2929, 0, 113, 588, 1529, 112, 2785, 0, 0, 291, 90, 0, 405, 6803, 4767, 513, 396, 1, 0, 0, 40, 0, 307, 234, 0, 88, 1359, 462, 30, 433, 0, 0, 44, 0, 3825, 0, 0, 5329, 0, 0, 3495, 422, 74, 0, 0, 0, 0, 0, 0, 0, 2049, 382, 190] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[726, 7467, 0, 0, 347, 0, 66, 0, 0, 299, 66, 2942, 0, 87, 519, 1157, 87, 3174, 0, 0, 240, 66, 0, 480, 6858, 4524, 606, 418, 0, 0, 0, 8, 0, 299, 241, 0, 96, 1083, 480, 20, 486, 0, 0, 60, 0, 3998, 0, 0, 5598, 0, 7, 3477, 478, 16, 0, 0, 0, 0, 0, 0, 0, 1916, 449, 311] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[775, 7523, 0, 1, 366, 0, 57, 0, 0, 324, 54, 3095, 1, 70, 530, 1555, 69, 2425, 1, 2, 283, 54, 0, 518, 6794, 4527, 557, 375, 0, 1, 0, 43, 1, 385, 247, 0, 106, 1012, 459, 20, 565, 0, 0, 69, 2, 3791, 0, 1, 5691, 1, 7, 3769, 451, 91, 0, 0, 0, 1, 2, 0, 0, 1865, 316, 300] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[820, 7576, 0, 0, 371, 0, 42, 0, 0, 300, 42, 3122, 0, 57, 517, 1016, 57, 2721, 0, 0, 176, 42, 0, 392, 6889, 4556, 470, 381, 0, 0, 0, 36, 0, 348, 201, 0, 98, 1250, 484, 28, 512, 0, 0, 72, 0, 3635, 1, 0, 5044, 0, 0, 4266, 420, 56, 1, 0, 0, 0, 0, 0, 0, 2436, 376, 341] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[661, 5006, 0, 0, 328, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 64, 3099, 0, 69, 516, 1521, 732, 0, 0, 0, 0, 0, 0, 0, 69, 2648, 0, 0, 267, 63, 0, 486, 2603, 17, 0, 0, 0, 0, 0, 0, 484, 4704, 596, 336, 0, 0, 1, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 275, 0, 88, 1021, 482, 20, 3849, 0, 0, 0, 0, 0, 0, 0, 464, 0, 0, 69, 0, 3853, 0, 0, 1766, 0, 0, 0, 0, 0, 0, 0, 1878, 0, 4, 3919, 462, 73, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1887, 285, 208, 2476, 1020, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6058, 6349, 6153, 6163, 6063, 6152, 6338, 5876] avg=6144.0000 max=6349 max_rank=1 min=5876 min_rank=7 imbalance_ratio=1.033366 max_load_reduction=42803 max_load_reduction_pct=0.870829 moved_to_echo=24429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[720, 5016, 0, 0, 331, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 60, 2705, 0, 92, 539, 1478, 733, 0, 0, 0, 0, 0, 0, 0, 92, 2547, 0, 0, 232, 60, 0, 431, 2659, 17, 0, 0, 0, 0, 0, 0, 486, 4793, 544, 402, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 216, 0, 69, 1289, 419, 36, 3862, 0, 0, 0, 0, 0, 0, 0, 422, 0, 0, 75, 0, 3820, 0, 0, 1770, 0, 0, 0, 0, 0, 0, 0, 1919, 0, 3, 3724, 437, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 409, 188, 2484, 1043, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 5950, 6038, 6272, 6192, 6087, 6139, 6347] avg=6144.0000 max=6347 max_rank=7 min=5950 min_rank=1 imbalance_ratio=1.033040 max_load_reduction=42805 max_load_reduction_pct=0.870870 moved_to_echo=24765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[753, 4985, 0, 0, 324, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 60, 3196, 0, 76, 529, 1514, 729, 0, 0, 0, 0, 0, 0, 0, 76, 2476, 0, 0, 213, 60, 0, 511, 2624, 17, 0, 0, 0, 0, 0, 0, 476, 4692, 506, 355, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 249, 0, 82, 1437, 433, 7, 3782, 0, 0, 0, 0, 0, 0, 0, 431, 0, 0, 92, 0, 3555, 0, 0, 1759, 0, 0, 0, 0, 0, 0, 0, 1894, 0, 0, 3705, 491, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 399, 247, 2433, 1029, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6457, 5977, 6079, 6284, 5837, 6107, 6289] avg=6144.0000 max=6457 max_rank=1 min=5837 min_rank=5 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[716, 5004, 0, 0, 350, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 53, 2852, 0, 79, 513, 1609, 731, 0, 0, 0, 0, 0, 0, 0, 79, 2484, 0, 0, 285, 53, 2, 514, 2752, 17, 0, 0, 0, 0, 0, 0, 483, 4932, 625, 412, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 188, 0, 69, 908, 476, 64, 3837, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 66, 0, 4120, 0, 0, 1765, 0, 0, 0, 0, 0, 0, 0, 1986, 0, 12, 3612, 471, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1604, 372, 213, 2468, 1079, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6137, 6186, 6481, 5817, 6522, 6150, 5736] avg=6144.0000 max=6522 max_rank=5 min=5736 min_rank=7 imbalance_ratio=1.061523 max_load_reduction=42630 max_load_reduction_pct=0.867310 moved_to_echo=24225 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[775, 4995, 0, 1, 366, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 54, 3095, 1, 70, 530, 1555, 735, 0, 0, 0, 0, 0, 0, 0, 69, 2425, 1, 2, 283, 54, 0, 518, 2698, 23, 0, 0, 0, 0, 0, 0, 476, 4527, 557, 375, 0, 1, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 1, 385, 247, 0, 106, 1012, 459, 20, 3843, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 69, 2, 3791, 0, 1, 1770, 0, 0, 0, 0, 0, 0, 0, 1933, 1, 7, 3769, 451, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 1865, 316, 300, 2475, 1060, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6364, 6073, 5979, 6073, 6198, 6252, 6019] avg=6144.0000 max=6364 max_rank=1 min=5979 min_rank=3 imbalance_ratio=1.035807 max_load_reduction=42788 max_load_reduction_pct=0.870524 moved_to_echo=24542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[720, 5016, 0, 0, 331, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 60, 2705, 0, 92, 539, 1478, 733, 0, 0, 0, 0, 0, 0, 0, 92, 2547, 0, 0, 232, 60, 0, 431, 2659, 17, 0, 0, 0, 0, 0, 0, 486, 4793, 544, 402, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 216, 0, 69, 1289, 419, 36, 3862, 0, 0, 0, 0, 0, 0, 0, 422, 0, 0, 75, 0, 3820, 0, 0, 1770, 0, 0, 0, 0, 0, 0, 0, 1919, 0, 3, 3724, 437, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 409, 188, 2484, 1043, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 5950, 6038, 6272, 6192, 6087, 6139, 6347] avg=6144.0000 max=6347 max_rank=7 min=5950 min_rank=1 imbalance_ratio=1.033040 max_load_reduction=42805 max_load_reduction_pct=0.870870 moved_to_echo=24765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[726, 4971, 0, 0, 347, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 66, 2942, 0, 87, 519, 1157, 726, 0, 0, 0, 0, 0, 0, 0, 87, 3174, 0, 0, 240, 66, 0, 480, 2648, 17, 0, 0, 0, 0, 0, 0, 488, 4524, 606, 418, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 241, 0, 96, 1083, 480, 20, 3876, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 60, 0, 3998, 0, 0, 1753, 0, 0, 0, 0, 0, 0, 0, 1912, 0, 7, 3477, 478, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1916, 449, 311, 2494, 1038, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 5796, 6712, 6044, 6095, 6297, 5890, 6208] avg=6144.0000 max=6712 max_rank=2 min=5796 min_rank=1 imbalance_ratio=1.092448 max_load_reduction=42440 max_load_reduction_pct=0.863444 moved_to_echo=24490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[713, 4982, 0, 0, 358, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 90, 2929, 0, 113, 588, 1529, 728, 0, 0, 0, 0, 0, 0, 0, 112, 2785, 0, 0, 291, 90, 0, 405, 2521, 17, 0, 0, 0, 0, 0, 0, 484, 4767, 513, 396, 1, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 234, 0, 88, 1359, 462, 30, 3845, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 44, 0, 3825, 0, 0, 1758, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 3495, 422, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2049, 382, 190, 2474, 988, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6308, 6221, 6201, 6325, 6060, 5811, 6083] avg=6144.0000 max=6325 max_rank=4 min=5811 min_rank=6 imbalance_ratio=1.029460 max_load_reduction=42827 max_load_reduction_pct=0.871318 moved_to_echo=24279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[820, 5042, 0, 0, 371, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 42, 3122, 0, 57, 517, 1016, 737, 0, 0, 0, 0, 0, 0, 0, 57, 2721, 0, 0, 176, 42, 0, 392, 2386, 18, 0, 0, 0, 0, 0, 0, 490, 4556, 470, 381, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 201, 0, 98, 1250, 484, 28, 3894, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 72, 0, 3635, 1, 0, 1779, 0, 0, 0, 0, 0, 0, 0, 1722, 0, 0, 4266, 420, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2436, 376, 341, 2505, 936, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6275, 5791, 5792, 5933, 6303, 5999, 6465, 6594] avg=6144.0000 max=6594 max_rank=7 min=5791 min_rank=1 imbalance_ratio=1.073242 max_load_reduction=42558 max_load_reduction_pct=0.865845 moved_to_echo=25361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[775, 4995, 0, 1, 366, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 54, 3095, 1, 70, 530, 1555, 735, 0, 0, 0, 0, 0, 0, 0, 69, 2425, 1, 2, 283, 54, 0, 518, 2698, 23, 0, 0, 0, 0, 0, 0, 476, 4527, 557, 375, 0, 1, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 1, 385, 247, 0, 106, 1012, 459, 20, 3843, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 69, 2, 3791, 0, 1, 1770, 0, 0, 0, 0, 0, 0, 0, 1933, 1, 7, 3769, 451, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 1865, 316, 300, 2475, 1060, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6364, 6073, 5979, 6073, 6198, 6252, 6019] avg=6144.0000 max=6364 max_rank=1 min=5979 min_rank=3 imbalance_ratio=1.035807 max_load_reduction=42788 max_load_reduction_pct=0.870524 moved_to_echo=24542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 16256, 3072, 2048, 19840, 8320, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[768, 21376, 128, 128, 2048, 512, 128, 3840, 20992, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 16256, 3072, 2048, 19840, 8320, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 2688, 512, 24064, 128, 768, 4352, 11392, 5888, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 0, 0, 640, 128, 30720, 128, 128, 14208, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[15104, 128, 128, 30080, 3712, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[5888, 40064, 0, 128, 2816, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2560, 1920, 0, 768, 9472, 3712, 256, 30848, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 37504, 4480, 3200, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[5888, 40064, 0, 128, 2816, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[624, 75, 345, 295, 7093, 0, 7393, 4623, 0, 150, 0, 644, 0, 187, 0, 4108, 245, 5, 3274, 0, 393, 51, 311, 69, 1, 467, 76, 0, 0, 138, 0, 1, 5640, 0, 0, 75, 120, 1228, 2423, 0, 144, 0, 0, 0, 0, 0, 0, 1, 1084, 0, 0, 0, 0, 949, 433, 0, 4094, 136, 0, 549, 1644, 64, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[589, 69, 698, 300, 6927, 0, 7308, 4853, 0, 138, 0, 663, 0, 204, 0, 3666, 260, 36, 3161, 0, 822, 42, 6, 79, 0, 412, 69, 0, 0, 123, 0, 0, 5671, 0, 0, 69, 145, 990, 2785, 0, 148, 0, 0, 0, 0, 0, 0, 4, 1123, 0, 0, 0, 1, 840, 402, 0, 4423, 199, 0, 406, 1443, 78, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[621, 82, 372, 293, 6937, 0, 7272, 4404, 0, 164, 0, 713, 0, 260, 0, 4525, 264, 0, 2656, 0, 462, 43, 236, 113, 1, 422, 82, 0, 0, 173, 0, 0, 5757, 0, 0, 82, 142, 1123, 2898, 0, 195, 0, 0, 0, 0, 0, 0, 2, 1005, 0, 0, 0, 1, 760, 445, 0, 3962, 282, 0, 545, 1785, 72, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[634, 90, 239, 157, 6769, 0, 7165, 4258, 0, 178, 0, 791, 0, 336, 0, 4226, 248, 0, 2959, 0, 387, 46, 317, 87, 23, 487, 90, 0, 0, 157, 0, 0, 5691, 0, 0, 90, 114, 1163, 2863, 0, 177, 0, 0, 0, 0, 0, 0, 0, 949, 0, 0, 0, 0, 783, 413, 0, 4233, 158, 0, 714, 1999, 161, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[599, 91, 1016, 234, 6952, 1, 6991, 4665, 1, 182, 2, 715, 3, 287, 0, 4218, 251, 5, 2869, 0, 534, 49, 74, 69, 5, 486, 94, 0, 0, 146, 1, 9, 5796, 2, 0, 91, 127, 1020, 2512, 2, 160, 1, 2, 2, 0, 3, 1, 0, 970, 1, 1, 0, 1, 865, 440, 1, 4101, 394, 0, 477, 1491, 141, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[728, 95, 240, 191, 6790, 0, 7113, 4453, 0, 191, 0, 869, 0, 321, 0, 4083, 253, 0, 2884, 0, 524, 28, 237, 57, 15, 449, 96, 0, 0, 138, 0, 0, 5689, 0, 0, 96, 108, 1065, 2769, 0, 154, 0, 0, 0, 0, 0, 0, 1, 1056, 0, 0, 0, 0, 862, 428, 0, 4276, 135, 0, 590, 1994, 174, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[618, 63, 776, 229, 6931, 0, 7355, 4706, 0, 127, 0, 593, 0, 178, 0, 3981, 255, 69, 3234, 0, 403, 42, 238, 92, 8, 436, 65, 0, 0, 125, 0, 0, 5807, 0, 0, 65, 95, 1015, 2909, 0, 155, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 0, 0, 2, 818, 361, 0, 3984, 151, 0, 491, 1611, 65, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[611, 71, 402, 181, 6859, 0, 7311, 4458, 0, 145, 0, 729, 0, 212, 0, 4514, 238, 0, 2671, 0, 419, 43, 117, 76, 2, 494, 74, 0, 0, 134, 0, 3, 5815, 0, 0, 74, 93, 1130, 2593, 0, 147, 0, 0, 0, 0, 0, 0, 0, 952, 0, 0, 0, 0, 682, 487, 0, 4355, 313, 0, 567, 2099, 81, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[621, 82, 372, 293, 6937, 0, 7272, 4404, 0, 164, 0, 713, 0, 260, 0, 4525, 264, 0, 2656, 0, 462, 43, 236, 113, 1, 422, 82, 0, 0, 173, 0, 0, 5757, 0, 0, 82, 142, 1123, 2898, 0, 195, 0, 0, 0, 0, 0, 0, 2, 1005, 0, 0, 0, 1, 760, 445, 0, 3962, 282, 0, 545, 1785, 72, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[618, 63, 776, 229, 6931, 0, 7355, 4706, 0, 127, 0, 593, 0, 178, 0, 3981, 255, 69, 3234, 0, 403, 42, 238, 92, 8, 436, 65, 0, 0, 125, 0, 0, 5807, 0, 0, 65, 95, 1015, 2909, 0, 155, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 0, 0, 2, 818, 361, 0, 3984, 151, 0, 491, 1611, 65, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[589, 69, 698, 300, 6927, 0, 7308, 4853, 0, 138, 0, 663, 0, 204, 0, 3666, 260, 36, 3161, 0, 822, 42, 6, 79, 0, 412, 69, 0, 0, 123, 0, 0, 5671, 0, 0, 69, 145, 990, 2785, 0, 148, 0, 0, 0, 0, 0, 0, 4, 1123, 0, 0, 0, 1, 840, 402, 0, 4423, 199, 0, 406, 1443, 78, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[599, 91, 1016, 234, 6952, 1, 6991, 4665, 1, 182, 2, 715, 3, 287, 0, 4218, 251, 5, 2869, 0, 534, 49, 74, 69, 5, 486, 94, 0, 0, 146, 1, 9, 5796, 2, 0, 91, 127, 1020, 2512, 2, 160, 1, 2, 2, 0, 3, 1, 0, 970, 1, 1, 0, 1, 865, 440, 1, 4101, 394, 0, 477, 1491, 141, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[624, 75, 345, 295, 143, 0, 0, 4623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 644, 0, 187, 0, 4108, 647, 191, 0, 0, 0, 0, 0, 0, 245, 5, 3274, 0, 393, 51, 311, 69, 2071, 0, 0, 0, 0, 0, 0, 0, 1, 467, 76, 0, 0, 138, 0, 1, 4312, 1284, 0, 0, 0, 0, 0, 0, 2102, 0, 0, 75, 120, 1228, 2423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 1, 6109, 0, 0, 0, 0, 0, 0, 0, 1084, 0, 0, 0, 0, 949, 433, 0, 2638, 1276, 0, 0, 0, 0, 0, 0, 3447, 136, 0, 549, 1644, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 5927, 6419, 6279, 5948, 6254, 6380, 5840] avg=6144.0000 max=6419 max_rank=2 min=5840 min_rank=7 imbalance_ratio=1.044759 max_load_reduction=42733 max_load_reduction_pct=0.869405 moved_to_echo=24422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[589, 69, 698, 300, 139, 0, 0, 4853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 663, 0, 204, 0, 3666, 699, 193, 0, 0, 0, 0, 0, 0, 260, 36, 3161, 0, 822, 42, 6, 79, 2083, 0, 0, 0, 0, 0, 0, 0, 0, 412, 69, 0, 0, 123, 0, 0, 4211, 1269, 0, 0, 0, 0, 0, 0, 2112, 0, 0, 69, 145, 990, 2785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 4, 6039, 0, 0, 0, 0, 0, 0, 0, 1123, 0, 0, 0, 1, 840, 402, 0, 2577, 1283, 0, 0, 0, 0, 0, 0, 3724, 199, 0, 406, 1443, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6648, 5563, 6489, 6084, 6101, 6191, 6226, 5850] avg=6144.0000 max=6648 max_rank=0 min=5563 min_rank=1 imbalance_ratio=1.082031 max_load_reduction=42504 max_load_reduction_pct=0.864746 moved_to_echo=24368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[728, 95, 240, 191, 136, 0, 0, 4453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 869, 0, 321, 0, 4083, 676, 193, 0, 0, 0, 0, 0, 0, 253, 0, 2884, 0, 524, 28, 237, 57, 2089, 0, 0, 0, 0, 0, 0, 0, 15, 449, 96, 0, 0, 138, 0, 0, 4128, 1234, 0, 0, 0, 0, 0, 0, 2120, 0, 0, 96, 108, 1065, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 1, 5879, 0, 0, 0, 0, 0, 0, 0, 1056, 0, 0, 0, 0, 862, 428, 0, 2526, 1287, 0, 0, 0, 0, 0, 0, 3600, 135, 0, 590, 1994, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5843, 6333, 6072, 6060, 6158, 6034, 6159, 6493] avg=6144.0000 max=6493 max_rank=7 min=5843 min_rank=0 imbalance_ratio=1.056803 max_load_reduction=42659 max_load_reduction_pct=0.867900 moved_to_echo=24844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[634, 90, 239, 157, 136, 0, 0, 4258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 791, 0, 336, 0, 4226, 669, 193, 0, 0, 0, 0, 0, 0, 248, 0, 2959, 0, 387, 46, 317, 87, 2090, 0, 0, 0, 0, 0, 0, 0, 23, 487, 90, 0, 0, 157, 0, 0, 4115, 1243, 0, 0, 0, 0, 0, 0, 2121, 0, 0, 90, 114, 1163, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 5922, 0, 0, 0, 0, 0, 0, 0, 949, 0, 0, 0, 0, 783, 413, 0, 2518, 1287, 0, 0, 0, 0, 0, 0, 3564, 158, 0, 714, 1999, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5514, 6393, 6134, 6115, 6351, 6099, 5950, 6596] avg=6144.0000 max=6596 max_rank=7 min=5514 min_rank=0 imbalance_ratio=1.073568 max_load_reduction=42556 max_load_reduction_pct=0.865804 moved_to_echo=24996 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[728, 95, 240, 191, 136, 0, 0, 4453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 869, 0, 321, 0, 4083, 676, 193, 0, 0, 0, 0, 0, 0, 253, 0, 2884, 0, 524, 28, 237, 57, 2089, 0, 0, 0, 0, 0, 0, 0, 15, 449, 96, 0, 0, 138, 0, 0, 4128, 1234, 0, 0, 0, 0, 0, 0, 2120, 0, 0, 96, 108, 1065, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 1, 5879, 0, 0, 0, 0, 0, 0, 0, 1056, 0, 0, 0, 0, 862, 428, 0, 2526, 1287, 0, 0, 0, 0, 0, 0, 3600, 135, 0, 590, 1994, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5843, 6333, 6072, 6060, 6158, 6034, 6159, 6493] avg=6144.0000 max=6493 max_rank=7 min=5843 min_rank=0 imbalance_ratio=1.056803 max_load_reduction=42659 max_load_reduction_pct=0.867900 moved_to_echo=24844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[611, 71, 402, 181, 138, 0, 0, 4458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 729, 0, 212, 0, 4514, 688, 197, 0, 0, 0, 0, 0, 0, 238, 0, 2671, 0, 419, 43, 117, 76, 2136, 0, 0, 0, 0, 0, 0, 0, 2, 494, 74, 0, 0, 134, 0, 3, 4170, 1270, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 74, 93, 1130, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 6041, 0, 0, 0, 0, 0, 0, 0, 952, 0, 0, 0, 0, 682, 487, 0, 2551, 1316, 0, 0, 0, 0, 0, 0, 3667, 313, 0, 567, 2099, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5861, 6485, 5700, 6147, 6056, 6188, 5988, 6727] avg=6144.0000 max=6727 max_rank=7 min=5700 min_rank=2 imbalance_ratio=1.094889 max_load_reduction=42425 max_load_reduction_pct=0.863139 moved_to_echo=24959 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[599, 91, 1016, 234, 132, 1, 0, 4665, 0, 0, 0, 0, 0, 0, 0, 0, 1, 182, 2, 715, 3, 287, 0, 4218, 652, 202, 0, 0, 0, 0, 0, 0, 251, 5, 2869, 0, 534, 49, 74, 69, 2133, 0, 0, 0, 0, 0, 0, 0, 5, 486, 94, 0, 0, 146, 1, 9, 4230, 1214, 0, 0, 0, 0, 0, 0, 2146, 2, 0, 91, 127, 1020, 2512, 2, 0, 0, 0, 0, 0, 0, 0, 0, 160, 1, 2, 2, 0, 3, 1, 0, 5777, 0, 0, 0, 0, 0, 0, 0, 970, 1, 1, 0, 1, 865, 440, 1, 2590, 1315, 0, 0, 0, 0, 0, 0, 3449, 394, 0, 477, 1491, 141, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6738, 6262, 5984, 6185, 5900, 5946, 6184, 5953] avg=6144.0000 max=6738 max_rank=0 min=5900 min_rank=4 imbalance_ratio=1.096680 max_load_reduction=42414 max_load_reduction_pct=0.862915 moved_to_echo=23983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[618, 63, 776, 229, 139, 0, 0, 4706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 593, 0, 178, 0, 3981, 630, 197, 0, 0, 0, 0, 0, 0, 255, 69, 3234, 0, 403, 42, 238, 92, 2133, 0, 0, 0, 0, 0, 0, 0, 8, 436, 65, 0, 0, 125, 0, 0, 4214, 1276, 0, 0, 0, 0, 0, 0, 2163, 0, 0, 65, 95, 1015, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 6079, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 0, 0, 2, 818, 361, 0, 2578, 1314, 0, 0, 0, 0, 0, 0, 3354, 151, 0, 491, 1611, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 5706, 6466, 6124, 6247, 6234, 6172, 5672] avg=6144.0000 max=6531 max_rank=0 min=5672 min_rank=7 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[621, 82, 372, 293, 140, 0, 0, 4404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 713, 0, 260, 0, 4525, 626, 195, 0, 0, 0, 0, 0, 0, 264, 0, 2656, 0, 462, 43, 236, 113, 2114, 0, 0, 0, 0, 0, 0, 0, 1, 422, 82, 0, 0, 173, 0, 0, 4217, 1263, 0, 0, 0, 0, 0, 0, 2146, 0, 0, 82, 142, 1123, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 2, 6009, 0, 0, 0, 0, 0, 0, 0, 1005, 0, 0, 0, 1, 760, 445, 0, 2580, 1302, 0, 0, 0, 0, 0, 0, 3336, 282, 0, 545, 1785, 72, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5912, 6483, 5888, 6158, 6391, 6206, 6093, 6021] avg=6144.0000 max=6483 max_rank=1 min=5888 min_rank=2 imbalance_ratio=1.055176 max_load_reduction=42669 max_load_reduction_pct=0.868103 moved_to_echo=24711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[618, 63, 776, 229, 139, 0, 0, 4706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 593, 0, 178, 0, 3981, 630, 197, 0, 0, 0, 0, 0, 0, 255, 69, 3234, 0, 403, 42, 238, 92, 2133, 0, 0, 0, 0, 0, 0, 0, 8, 436, 65, 0, 0, 125, 0, 0, 4214, 1276, 0, 0, 0, 0, 0, 0, 2163, 0, 0, 65, 95, 1015, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 6079, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 0, 0, 2, 818, 361, 0, 2578, 1314, 0, 0, 0, 0, 0, 0, 3354, 151, 0, 491, 1611, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 5706, 6466, 6124, 6247, 6234, 6172, 5672] avg=6144.0000 max=6531 max_rank=0 min=5672 min_rank=7 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[589, 69, 698, 300, 139, 0, 0, 4853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 663, 0, 204, 0, 3666, 699, 193, 0, 0, 0, 0, 0, 0, 260, 36, 3161, 0, 822, 42, 6, 79, 2083, 0, 0, 0, 0, 0, 0, 0, 0, 412, 69, 0, 0, 123, 0, 0, 4211, 1269, 0, 0, 0, 0, 0, 0, 2112, 0, 0, 69, 145, 990, 2785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 4, 6039, 0, 0, 0, 0, 0, 0, 0, 1123, 0, 0, 0, 1, 840, 402, 0, 2577, 1283, 0, 0, 0, 0, 0, 0, 3724, 199, 0, 406, 1443, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6648, 5563, 6489, 6084, 6101, 6191, 6226, 5850] avg=6144.0000 max=6648 max_rank=0 min=5563 min_rank=1 imbalance_ratio=1.082031 max_load_reduction=42504 max_load_reduction_pct=0.864746 moved_to_echo=24368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[599, 91, 1016, 234, 132, 1, 0, 4665, 0, 0, 0, 0, 0, 0, 0, 0, 1, 182, 2, 715, 3, 287, 0, 4218, 652, 202, 0, 0, 0, 0, 0, 0, 251, 5, 2869, 0, 534, 49, 74, 69, 2133, 0, 0, 0, 0, 0, 0, 0, 5, 486, 94, 0, 0, 146, 1, 9, 4230, 1214, 0, 0, 0, 0, 0, 0, 2146, 2, 0, 91, 127, 1020, 2512, 2, 0, 0, 0, 0, 0, 0, 0, 0, 160, 1, 2, 2, 0, 3, 1, 0, 5777, 0, 0, 0, 0, 0, 0, 0, 970, 1, 1, 0, 1, 865, 440, 1, 2590, 1315, 0, 0, 0, 0, 0, 0, 3449, 394, 0, 477, 1491, 141, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6738, 6262, 5984, 6185, 5900, 5946, 6184, 5953] avg=6144.0000 max=6738 max_rank=0 min=5900 min_rank=4 imbalance_ratio=1.096680 max_load_reduction=42414 max_load_reduction_pct=0.862915 moved_to_echo=23983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[17152, 128, 0, 768, 1024, 8832, 21760, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 3712, 768, 0, 0, 1152, 128, 128, 33664, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[28160, 1792, 0, 4352, 14080, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[8320, 128, 128, 0, 128, 6656, 3456, 128, 20608, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[28160, 1792, 0, 4352, 14080, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2048, 128, 23808, 0, 3968, 384, 1536, 768, 16896, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1280, 128, 5760, 128, 2048, 0, 33408, 5376, 1664, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5120, 640, 4096, 1920, 1152, 128, 0, 36480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1280, 128, 5760, 128, 2048, 0, 33408, 5376, 1664, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1280, 128, 128, 128, 0, 128, 128, 128, 47872, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 8117, 5393, 0, 0, 63, 0, 457, 0, 0, 0, 0, 0, 730, 3559, 138, 0, 75, 0, 316, 0, 0, 0, 1808, 64, 246, 0, 69, 0, 63, 139, 0, 172, 7912, 75, 0, 741, 0, 6783, 0, 1174, 0, 0, 0, 500, 0, 0, 0, 1357, 0, 75, 0, 0, 0, 0, 0, 0, 0, 7594, 702, 75, 100, 0, 655] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 8097, 5434, 0, 0, 42, 0, 423, 0, 0, 0, 0, 0, 734, 3165, 138, 0, 96, 0, 311, 0, 0, 0, 1597, 174, 254, 0, 72, 0, 42, 269, 0, 210, 7916, 96, 0, 1101, 0, 6619, 0, 961, 0, 0, 0, 455, 0, 0, 0, 1455, 0, 96, 1, 0, 0, 0, 0, 0, 0, 7437, 1150, 95, 102, 0, 610] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 8094, 5574, 0, 1, 54, 2, 589, 0, 1, 4, 0, 0, 721, 3360, 145, 0, 92, 0, 318, 1, 2, 0, 1635, 140, 249, 0, 74, 0, 54, 231, 1, 235, 7889, 91, 1, 791, 2, 6757, 1, 1201, 1, 4, 0, 571, 2, 0, 2, 1186, 1, 93, 0, 0, 0, 0, 4, 0, 1, 7454, 710, 91, 113, 2, 607] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 8110, 5416, 0, 0, 90, 0, 467, 0, 0, 0, 0, 0, 757, 3128, 172, 0, 82, 0, 377, 0, 0, 0, 1681, 72, 264, 0, 44, 0, 90, 154, 0, 256, 7843, 82, 0, 954, 0, 6794, 0, 1353, 0, 0, 0, 382, 0, 0, 0, 1411, 0, 83, 0, 0, 0, 0, 0, 0, 0, 7518, 798, 82, 91, 0, 601] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 8102, 5563, 0, 0, 66, 0, 466, 0, 0, 0, 0, 0, 687, 3519, 156, 0, 90, 0, 334, 0, 0, 0, 1568, 161, 247, 0, 60, 0, 66, 251, 0, 281, 7870, 90, 0, 811, 0, 6733, 0, 946, 0, 0, 0, 404, 0, 0, 0, 1524, 0, 90, 0, 0, 0, 0, 0, 0, 0, 7449, 891, 89, 97, 0, 541] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 8129, 5517, 0, 0, 60, 0, 545, 0, 0, 0, 0, 0, 740, 3246, 125, 0, 65, 0, 346, 0, 0, 0, 1945, 48, 254, 0, 76, 0, 60, 111, 0, 178, 7891, 65, 0, 1092, 0, 6686, 0, 880, 0, 0, 0, 508, 0, 0, 0, 1202, 0, 66, 0, 0, 0, 0, 0, 0, 0, 7634, 928, 63, 93, 0, 599] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 8121, 5488, 0, 0, 60, 1, 461, 0, 0, 0, 0, 0, 733, 3328, 135, 0, 77, 0, 315, 0, 0, 0, 1689, 81, 239, 0, 92, 0, 60, 152, 0, 257, 7868, 74, 0, 1004, 0, 6708, 0, 1077, 0, 0, 0, 437, 0, 0, 0, 1372, 0, 76, 3, 0, 0, 0, 0, 0, 0, 7591, 875, 71, 86, 0, 621] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 8123, 5380, 0, 5, 53, 0, 823, 0, 0, 0, 0, 0, 732, 3420, 122, 0, 70, 0, 340, 0, 0, 0, 1922, 78, 261, 0, 70, 0, 53, 147, 0, 211, 7900, 69, 0, 758, 0, 6742, 0, 861, 0, 0, 0, 480, 0, 0, 0, 1187, 0, 69, 0, 0, 0, 0, 0, 0, 0, 7583, 865, 69, 84, 0, 675] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5434, 0, 0, 42, 0, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 3165, 138, 1766, 134, 0, 0, 0, 0, 0, 0, 0, 96, 0, 311, 0, 0, 0, 1597, 2869, 1097, 0, 0, 0, 0, 0, 0, 174, 254, 0, 72, 0, 42, 269, 0, 5467, 0, 0, 0, 0, 0, 0, 0, 210, 0, 96, 0, 1101, 0, 4853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 0, 0, 0, 455, 0, 0, 0, 4636, 0, 0, 0, 0, 0, 0, 0, 1455, 0, 96, 1, 0, 0, 0, 0, 2547, 2183, 0, 0, 0, 0, 0, 0, 0, 0, 4434, 1150, 95, 102, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5982, 5937, 5970, 6278, 6260, 6052, 6282, 6391] avg=6144.0000 max=6391 max_rank=7 min=5937 min_rank=1 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 8102, 5563, 0, 0, 66, 0, 466, 0, 0, 0, 0, 0, 687, 3519, 156, 0, 90, 0, 334, 0, 0, 0, 1568, 161, 247, 0, 60, 0, 66, 251, 0, 281, 7870, 90, 0, 811, 0, 6733, 0, 946, 0, 0, 0, 404, 0, 0, 0, 1524, 0, 90, 0, 0, 0, 0, 0, 0, 0, 7449, 891, 89, 97, 0, 541] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 8097, 5434, 0, 0, 42, 0, 423, 0, 0, 0, 0, 0, 734, 3165, 138, 0, 96, 0, 311, 0, 0, 0, 1597, 174, 254, 0, 72, 0, 42, 269, 0, 210, 7916, 96, 0, 1101, 0, 6619, 0, 961, 0, 0, 0, 455, 0, 0, 0, 1455, 0, 96, 1, 0, 0, 0, 0, 0, 0, 7437, 1150, 95, 102, 0, 610] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 8117, 5393, 0, 0, 63, 0, 457, 0, 0, 0, 0, 0, 730, 3559, 138, 0, 75, 0, 316, 0, 0, 0, 1808, 64, 246, 0, 69, 0, 63, 139, 0, 172, 7912, 75, 0, 741, 0, 6783, 0, 1174, 0, 0, 0, 500, 0, 0, 0, 1357, 0, 75, 0, 0, 0, 0, 0, 0, 0, 7594, 702, 75, 100, 0, 655] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 8121, 5488, 0, 0, 60, 1, 461, 0, 0, 0, 0, 0, 733, 3328, 135, 0, 77, 0, 315, 0, 0, 0, 1689, 81, 239, 0, 92, 0, 60, 152, 0, 257, 7868, 74, 0, 1004, 0, 6708, 0, 1077, 0, 0, 0, 437, 0, 0, 0, 1372, 0, 76, 3, 0, 0, 0, 0, 0, 0, 7591, 875, 71, 86, 0, 621] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 8110, 5416, 0, 0, 90, 0, 467, 0, 0, 0, 0, 0, 757, 3128, 172, 0, 82, 0, 377, 0, 0, 0, 1681, 72, 264, 0, 44, 0, 90, 154, 0, 256, 7843, 82, 0, 954, 0, 6794, 0, 1353, 0, 0, 0, 382, 0, 0, 0, 1411, 0, 83, 0, 0, 0, 0, 0, 0, 0, 7518, 798, 82, 91, 0, 601] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 8094, 5574, 0, 1, 54, 2, 589, 0, 1, 4, 0, 0, 721, 3360, 145, 0, 92, 0, 318, 1, 2, 0, 1635, 140, 249, 0, 74, 0, 54, 231, 1, 235, 7889, 91, 1, 791, 2, 6757, 1, 1201, 1, 4, 0, 571, 2, 0, 2, 1186, 1, 93, 0, 0, 0, 0, 4, 0, 1, 7454, 710, 91, 113, 2, 607] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5517, 0, 0, 60, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 3246, 125, 1784, 137, 0, 0, 0, 0, 0, 0, 0, 65, 0, 346, 0, 0, 0, 1945, 2945, 1094, 0, 0, 0, 0, 0, 0, 48, 254, 0, 76, 0, 60, 111, 0, 5489, 0, 0, 0, 0, 0, 0, 0, 178, 0, 65, 0, 1092, 0, 4902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 880, 0, 0, 0, 508, 0, 0, 0, 4621, 0, 0, 0, 0, 0, 0, 0, 1202, 0, 66, 0, 0, 0, 0, 0, 2557, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 4552, 928, 63, 93, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6032, 6395, 6038, 6237, 6009, 6001, 6235] avg=6144.0000 max=6395 max_rank=2 min=6001 min_rank=6 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5393, 0, 0, 63, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 3559, 138, 1810, 137, 0, 0, 0, 0, 0, 0, 0, 75, 0, 316, 0, 0, 0, 1808, 2930, 1097, 0, 0, 0, 0, 0, 0, 64, 246, 0, 69, 0, 63, 139, 0, 5481, 0, 0, 0, 0, 0, 0, 0, 172, 0, 75, 0, 741, 0, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 0, 0, 0, 500, 0, 0, 0, 4635, 0, 0, 0, 0, 0, 0, 0, 1357, 0, 75, 0, 0, 0, 0, 0, 2553, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 4527, 702, 75, 100, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6374, 6226, 6062, 5961, 6309, 6165, 6059] avg=6144.0000 max=6374 max_rank=1 min=5961 min_rank=4 imbalance_ratio=1.037435 max_load_reduction=42778 max_load_reduction_pct=0.870321 moved_to_echo=24494 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5563, 0, 0, 66, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 3519, 156, 1797, 134, 0, 0, 0, 0, 0, 0, 0, 90, 0, 334, 0, 0, 0, 1568, 2874, 1091, 0, 0, 0, 0, 0, 0, 161, 247, 0, 60, 0, 66, 251, 0, 5471, 0, 0, 0, 0, 0, 0, 0, 281, 0, 90, 0, 811, 0, 4936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 0, 0, 404, 0, 0, 0, 4611, 0, 0, 0, 0, 0, 0, 0, 1524, 0, 90, 0, 0, 0, 0, 0, 2548, 2168, 0, 0, 0, 0, 0, 0, 0, 0, 4441, 891, 89, 97, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6293, 5957, 6256, 6118, 5961, 6330, 6059] avg=6144.0000 max=6330 max_rank=6 min=5957 min_rank=2 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5434, 0, 0, 42, 0, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 3165, 138, 1766, 134, 0, 0, 0, 0, 0, 0, 0, 96, 0, 311, 0, 0, 0, 1597, 2869, 1097, 0, 0, 0, 0, 0, 0, 174, 254, 0, 72, 0, 42, 269, 0, 5467, 0, 0, 0, 0, 0, 0, 0, 210, 0, 96, 0, 1101, 0, 4853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 0, 0, 0, 455, 0, 0, 0, 4636, 0, 0, 0, 0, 0, 0, 0, 1455, 0, 96, 1, 0, 0, 0, 0, 2547, 2183, 0, 0, 0, 0, 0, 0, 0, 0, 4434, 1150, 95, 102, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5982, 5937, 5970, 6278, 6260, 6052, 6282, 6391] avg=6144.0000 max=6391 max_rank=7 min=5937 min_rank=1 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5416, 0, 0, 90, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 3128, 172, 1813, 135, 0, 0, 0, 0, 0, 0, 0, 82, 0, 377, 0, 0, 0, 1681, 2900, 1088, 0, 0, 0, 0, 0, 0, 72, 264, 0, 44, 0, 90, 154, 0, 5476, 0, 0, 0, 0, 0, 0, 0, 256, 0, 82, 0, 954, 0, 4981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 0, 0, 0, 382, 0, 0, 0, 4594, 0, 0, 0, 0, 0, 0, 0, 1411, 0, 83, 0, 0, 0, 0, 0, 2551, 2161, 0, 0, 0, 0, 0, 0, 0, 0, 4483, 798, 82, 91, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6056, 6005, 6128, 6100, 6273, 6329, 6206, 6055] avg=6144.0000 max=6329 max_rank=5 min=6005 min_rank=1 imbalance_ratio=1.030111 max_load_reduction=42823 max_load_reduction_pct=0.871236 moved_to_echo=24863 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 84, 5488, 0, 0, 60, 1, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 3328, 135, 1790, 137, 0, 0, 0, 0, 0, 0, 0, 77, 0, 315, 0, 0, 0, 1689, 2928, 1092, 0, 0, 0, 0, 0, 0, 81, 239, 0, 92, 0, 60, 152, 0, 5483, 0, 0, 0, 0, 0, 0, 0, 257, 0, 74, 0, 1004, 0, 4918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 0, 437, 0, 0, 0, 4608, 0, 0, 0, 0, 0, 0, 0, 1372, 0, 76, 3, 0, 0, 0, 0, 2554, 2168, 0, 0, 0, 0, 0, 0, 0, 0, 4526, 875, 71, 86, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 6123, 6101, 6107, 6253, 6122, 6173, 6179] avg=6144.0000 max=6253 max_rank=4 min=6094 min_rank=0 imbalance_ratio=1.017741 max_load_reduction=42899 max_load_reduction_pct=0.872782 moved_to_echo=24727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 5574, 0, 1, 54, 2, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 721, 3360, 145, 1809, 138, 0, 0, 0, 0, 0, 0, 0, 92, 0, 318, 1, 2, 0, 1635, 2880, 1095, 0, 0, 0, 0, 0, 0, 140, 249, 0, 74, 0, 54, 231, 1, 5468, 0, 0, 0, 0, 0, 0, 0, 235, 0, 91, 1, 791, 2, 4948, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 1, 4, 0, 571, 2, 0, 2, 4620, 0, 0, 0, 0, 0, 0, 0, 1186, 1, 93, 0, 0, 0, 0, 4, 2549, 2174, 0, 0, 0, 0, 0, 0, 0, 1, 4436, 710, 91, 113, 2, 607, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 6178, 6023, 6217, 6069, 6401, 6007, 5960] avg=6144.0000 max=6401 max_rank=5 min=5960 min_rank=7 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5380, 0, 5, 53, 0, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 732, 3420, 122, 1799, 137, 0, 0, 0, 0, 0, 0, 0, 70, 0, 340, 0, 0, 0, 1922, 2925, 1095, 0, 0, 0, 0, 0, 0, 78, 261, 0, 70, 0, 53, 147, 0, 5485, 0, 0, 0, 0, 0, 0, 0, 211, 0, 69, 0, 758, 0, 4943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 0, 480, 0, 0, 0, 4628, 0, 0, 0, 0, 0, 0, 0, 1187, 0, 69, 0, 0, 0, 0, 0, 2555, 2177, 0, 0, 0, 0, 0, 0, 0, 0, 4521, 865, 69, 84, 0, 675, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6344, 6210, 6352, 6094, 5981, 5969, 5988, 6214] avg=6144.0000 max=6352 max_rank=2 min=5969 min_rank=5 imbalance_ratio=1.033854 max_load_reduction=42800 max_load_reduction_pct=0.870768 moved_to_echo=24152 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5517, 0, 0, 60, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 3246, 125, 1784, 137, 0, 0, 0, 0, 0, 0, 0, 65, 0, 346, 0, 0, 0, 1945, 2945, 1094, 0, 0, 0, 0, 0, 0, 48, 254, 0, 76, 0, 60, 111, 0, 5489, 0, 0, 0, 0, 0, 0, 0, 178, 0, 65, 0, 1092, 0, 4902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 880, 0, 0, 0, 508, 0, 0, 0, 4621, 0, 0, 0, 0, 0, 0, 0, 1202, 0, 66, 0, 0, 0, 0, 0, 2557, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 4552, 928, 63, 93, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6032, 6395, 6038, 6237, 6009, 6001, 6235] avg=6144.0000 max=6395 max_rank=2 min=6001 min_rank=6 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5416, 0, 0, 90, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 3128, 172, 1813, 135, 0, 0, 0, 0, 0, 0, 0, 82, 0, 377, 0, 0, 0, 1681, 2900, 1088, 0, 0, 0, 0, 0, 0, 72, 264, 0, 44, 0, 90, 154, 0, 5476, 0, 0, 0, 0, 0, 0, 0, 256, 0, 82, 0, 954, 0, 4981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 0, 0, 0, 382, 0, 0, 0, 4594, 0, 0, 0, 0, 0, 0, 0, 1411, 0, 83, 0, 0, 0, 0, 0, 2551, 2161, 0, 0, 0, 0, 0, 0, 0, 0, 4483, 798, 82, 91, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6056, 6005, 6128, 6100, 6273, 6329, 6206, 6055] avg=6144.0000 max=6329 max_rank=5 min=6005 min_rank=1 imbalance_ratio=1.030111 max_load_reduction=42823 max_load_reduction_pct=0.871236 moved_to_echo=24863 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 5574, 0, 1, 54, 2, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 721, 3360, 145, 1809, 138, 0, 0, 0, 0, 0, 0, 0, 92, 0, 318, 1, 2, 0, 1635, 2880, 1095, 0, 0, 0, 0, 0, 0, 140, 249, 0, 74, 0, 54, 231, 1, 5468, 0, 0, 0, 0, 0, 0, 0, 235, 0, 91, 1, 791, 2, 4948, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 1, 4, 0, 571, 2, 0, 2, 4620, 0, 0, 0, 0, 0, 0, 0, 1186, 1, 93, 0, 0, 0, 0, 4, 2549, 2174, 0, 0, 0, 0, 0, 0, 0, 1, 4436, 710, 91, 113, 2, 607, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 6178, 6023, 6217, 6069, 6401, 6007, 5960] avg=6144.0000 max=6401 max_rank=5 min=5960 min_rank=7 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10752, 128, 768, 128, 0, 0, 0, 128, 20480, 17408, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 768, 0, 2688, 128, 128, 0, 13952, 23296, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 5888, 26752, 1152, 14464, 1152, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 35968, 7040, 640, 768, 128, 4992, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1920, 0, 768, 128, 7296, 128, 39552, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8576, 128, 128, 0, 3840, 128, 0, 128, 36992, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 5888, 26752, 1152, 14464, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1920, 0, 768, 128, 7296, 128, 39552, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[896, 2048, 0, 640, 0, 512, 1536, 128, 43904, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 768, 43776, 0, 128, 512, 128, 4352, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 352, 0, 0, 0, 0, 203, 214, 25, 0, 1998, 82, 0, 3, 0, 0, 0, 0, 0, 113, 0, 0, 464, 7838, 0, 1695, 0, 2860, 0, 0, 164, 1154, 0, 0, 0, 7458, 0, 343, 2351, 194, 289, 0, 339, 2470, 4150, 0, 0, 72, 0, 184, 0, 0, 0, 0, 43, 0, 6737, 6478, 797, 0, 0, 82] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 0, 0, 0, 136, 145, 33, 0, 2082, 71, 0, 3, 0, 0, 0, 0, 0, 76, 0, 0, 536, 7920, 0, 1584, 0, 2946, 0, 0, 145, 1107, 0, 0, 0, 7573, 0, 253, 2631, 187, 192, 0, 420, 2727, 3739, 0, 0, 81, 0, 163, 0, 0, 0, 0, 13, 0, 6846, 6421, 704, 0, 0, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 376, 0, 0, 0, 0, 132, 95, 36, 0, 2007, 75, 0, 10, 1, 0, 0, 0, 0, 69, 0, 0, 616, 7876, 0, 1556, 0, 2954, 0, 0, 150, 1232, 0, 0, 0, 7563, 0, 341, 2680, 194, 276, 0, 377, 2241, 3886, 0, 0, 64, 0, 143, 0, 0, 0, 0, 10, 0, 6835, 6561, 716, 0, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 318, 0, 0, 0, 0, 153, 99, 20, 0, 1807, 90, 0, 2, 0, 0, 0, 0, 0, 87, 0, 0, 527, 7861, 0, 2016, 0, 2591, 0, 0, 180, 1457, 0, 0, 0, 7430, 0, 250, 2616, 277, 254, 0, 392, 2508, 3799, 0, 0, 161, 0, 168, 0, 0, 0, 0, 92, 0, 6819, 6383, 705, 0, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 0, 0, 0, 136, 145, 33, 0, 2082, 71, 0, 3, 0, 0, 0, 0, 0, 76, 0, 0, 536, 7920, 0, 1584, 0, 2946, 0, 0, 145, 1107, 0, 0, 0, 7573, 0, 253, 2631, 187, 192, 0, 420, 2727, 3739, 0, 0, 81, 0, 163, 0, 0, 0, 0, 13, 0, 6846, 6421, 704, 0, 0, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 309, 0, 0, 0, 0, 99, 110, 22, 0, 2029, 95, 0, 2, 0, 0, 0, 0, 0, 57, 0, 0, 425, 7871, 0, 1916, 0, 2535, 0, 0, 191, 1240, 0, 0, 0, 7433, 0, 255, 2405, 260, 182, 0, 326, 2930, 4079, 0, 0, 174, 0, 152, 0, 0, 0, 0, 17, 0, 6746, 6457, 740, 0, 0, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 333, 0, 0, 0, 0, 132, 152, 72, 0, 2454, 69, 0, 0, 2, 0, 0, 0, 0, 79, 0, 0, 450, 7871, 0, 1418, 0, 3031, 0, 0, 138, 1172, 0, 0, 0, 7545, 0, 440, 2527, 127, 286, 0, 402, 2159, 3822, 0, 0, 78, 0, 215, 0, 0, 0, 0, 1, 0, 6686, 6589, 824, 0, 0, 78] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 332, 0, 0, 0, 0, 152, 172, 36, 0, 1997, 63, 0, 0, 1, 0, 0, 0, 0, 92, 0, 0, 554, 7905, 0, 1711, 0, 2930, 0, 0, 128, 1108, 0, 0, 0, 7579, 0, 340, 2869, 82, 233, 0, 370, 2379, 3604, 0, 0, 48, 0, 207, 0, 0, 0, 0, 13, 0, 6799, 6613, 772, 0, 0, 63] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 342, 3, 0, 0, 5, 140, 150, 35, 2, 2241, 91, 3, 11, 2, 0, 0, 0, 7, 70, 0, 0, 467, 7834, 0, 1526, 3, 3071, 2, 2, 182, 1320, 0, 0, 0, 7469, 1, 392, 2535, 263, 271, 0, 362, 2149, 3745, 2, 0, 140, 2, 201, 0, 3, 0, 0, 2, 4, 6720, 6447, 842, 0, 0, 93] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 352, 0, 0, 0, 0, 203, 214, 25, 0, 1998, 82, 0, 3, 0, 0, 0, 0, 0, 113, 0, 0, 464, 7838, 0, 1695, 0, 2860, 0, 0, 164, 1154, 0, 0, 0, 7458, 0, 343, 2351, 194, 289, 0, 339, 2470, 4150, 0, 0, 72, 0, 184, 0, 0, 0, 0, 43, 0, 6737, 6478, 797, 0, 0, 82] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 376, 0, 0, 0, 0, 3581, 1641, 632, 0, 0, 0, 0, 0, 132, 95, 36, 0, 2007, 75, 0, 10, 2463, 1230, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 69, 0, 0, 6065, 0, 0, 0, 0, 0, 0, 0, 616, 1097, 0, 1556, 0, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 1232, 0, 0, 0, 4468, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 2680, 194, 276, 0, 377, 2241, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 143, 0, 0, 0, 0, 5194, 714, 0, 0, 0, 0, 0, 0, 10, 0, 0, 5331, 716, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6230, 6048, 6135, 6223, 6191, 6073, 6115, 6137] avg=6144.0000 max=6230 max_rank=0 min=6048 min_rank=1 imbalance_ratio=1.013997 max_load_reduction=42922 max_load_reduction_pct=0.873250 moved_to_echo=24516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 309, 0, 0, 0, 0, 3759, 1619, 621, 0, 0, 0, 0, 0, 99, 110, 22, 0, 2029, 95, 0, 2, 2420, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 6061, 0, 0, 0, 0, 0, 0, 0, 425, 1096, 0, 1916, 0, 2535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 1240, 0, 0, 0, 4392, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 260, 182, 0, 326, 2930, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 152, 0, 0, 0, 0, 5127, 714, 0, 0, 0, 0, 0, 0, 17, 0, 0, 5246, 740, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6308, 5988, 6118, 5972, 6078, 6423, 6167, 6098] avg=6144.0000 max=6423 max_rank=5 min=5972 min_rank=3 imbalance_ratio=1.045410 max_load_reduction=42729 max_load_reduction_pct=0.869324 moved_to_echo=24766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 318, 0, 0, 0, 0, 3500, 1636, 621, 0, 0, 0, 0, 0, 153, 99, 20, 0, 1807, 90, 0, 2, 2419, 1197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 6053, 0, 0, 0, 0, 0, 0, 0, 527, 1095, 0, 2016, 0, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 1457, 0, 0, 0, 4390, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 2616, 277, 254, 0, 392, 2508, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 168, 0, 0, 0, 0, 5183, 713, 0, 0, 0, 0, 0, 0, 92, 0, 0, 5186, 705, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 5787, 6140, 6229, 6277, 6346, 6225, 6073] avg=6144.0000 max=6346 max_rank=5 min=5787 min_rank=1 imbalance_ratio=1.032878 max_load_reduction=42806 max_load_reduction_pct=0.870890 moved_to_echo=24921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 332, 0, 0, 0, 0, 3321, 1631, 633, 0, 0, 0, 0, 0, 152, 172, 36, 0, 1997, 63, 0, 0, 2468, 1240, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 92, 0, 0, 6087, 0, 0, 0, 0, 0, 0, 0, 554, 1101, 0, 1711, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 1108, 0, 0, 0, 4478, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 2869, 82, 233, 0, 370, 2379, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 207, 0, 0, 0, 0, 5168, 717, 0, 0, 0, 0, 0, 0, 13, 0, 0, 5373, 772, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6128, 6180, 6296, 6054, 6216, 6140, 6221] avg=6144.0000 max=6296 max_rank=3 min=5917 min_rank=0 imbalance_ratio=1.024740 max_load_reduction=42856 max_load_reduction_pct=0.871908 moved_to_echo=24631 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 0, 0, 0, 3445, 1644, 633, 0, 0, 0, 0, 0, 136, 145, 33, 0, 2082, 71, 0, 3, 2466, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 6098, 0, 0, 0, 0, 0, 0, 0, 536, 1104, 0, 1584, 0, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 1107, 0, 0, 0, 4474, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 187, 192, 0, 420, 2727, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 163, 0, 0, 0, 0, 5202, 718, 0, 0, 0, 0, 0, 0, 13, 0, 0, 5217, 704, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6069, 6140, 6174, 6170, 5979, 6451, 6164, 6005] avg=6144.0000 max=6451 max_rank=5 min=5979 min_rank=4 imbalance_ratio=1.049967 max_load_reduction=42701 max_load_reduction_pct=0.868754 moved_to_echo=24599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 352, 0, 0, 0, 0, 3824, 1617, 623, 0, 0, 0, 0, 0, 203, 214, 25, 0, 1998, 82, 0, 3, 2429, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 6035, 0, 0, 0, 0, 0, 0, 0, 464, 1092, 0, 1695, 0, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 1154, 0, 0, 0, 4406, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 194, 289, 0, 339, 2470, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 184, 0, 0, 0, 0, 5120, 711, 0, 0, 0, 0, 0, 0, 43, 0, 0, 5263, 797, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6416, 6169, 6148, 6111, 6067, 5969, 6087, 6185] avg=6144.0000 max=6416 max_rank=0 min=5969 min_rank=5 imbalance_ratio=1.044271 max_load_reduction=42736 max_load_reduction_pct=0.869466 moved_to_echo=24308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 309, 0, 0, 0, 0, 3759, 1619, 621, 0, 0, 0, 0, 0, 99, 110, 22, 0, 2029, 95, 0, 2, 2420, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 6061, 0, 0, 0, 0, 0, 0, 0, 425, 1096, 0, 1916, 0, 2535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 1240, 0, 0, 0, 4392, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 260, 182, 0, 326, 2930, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 152, 0, 0, 0, 0, 5127, 714, 0, 0, 0, 0, 0, 0, 17, 0, 0, 5246, 740, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6308, 5988, 6118, 5972, 6078, 6423, 6167, 6098] avg=6144.0000 max=6423 max_rank=5 min=5972 min_rank=3 imbalance_ratio=1.045410 max_load_reduction=42729 max_load_reduction_pct=0.869324 moved_to_echo=24766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 333, 0, 0, 0, 0, 3522, 1605, 631, 0, 0, 0, 0, 0, 132, 152, 72, 0, 2454, 69, 0, 0, 2457, 1235, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 79, 0, 0, 6061, 0, 0, 0, 0, 0, 0, 0, 450, 1096, 0, 1418, 0, 3031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 1172, 0, 0, 0, 4457, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 127, 286, 0, 402, 2159, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 215, 0, 0, 0, 0, 5081, 714, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5354, 824, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 6571, 6142, 5995, 6207, 5801, 6088, 6257] avg=6144.0000 max=6571 max_rank=1 min=5801 min_rank=5 imbalance_ratio=1.069499 max_load_reduction=42581 max_load_reduction_pct=0.866313 moved_to_echo=24353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 332, 0, 0, 0, 0, 3321, 1631, 633, 0, 0, 0, 0, 0, 152, 172, 36, 0, 1997, 63, 0, 0, 2468, 1240, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 92, 0, 0, 6087, 0, 0, 0, 0, 0, 0, 0, 554, 1101, 0, 1711, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 1108, 0, 0, 0, 4478, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 2869, 82, 233, 0, 370, 2379, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 207, 0, 0, 0, 0, 5168, 717, 0, 0, 0, 0, 0, 0, 13, 0, 0, 5373, 772, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6128, 6180, 6296, 6054, 6216, 6140, 6221] avg=6144.0000 max=6296 max_rank=3 min=5917 min_rank=0 imbalance_ratio=1.024740 max_load_reduction=42856 max_load_reduction_pct=0.871908 moved_to_echo=24631 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 342, 3, 0, 0, 5, 3454, 1613, 629, 0, 0, 0, 0, 0, 140, 150, 35, 2, 2241, 91, 3, 11, 2436, 1212, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7, 70, 0, 0, 6036, 0, 0, 0, 0, 0, 0, 0, 467, 1085, 0, 1526, 3, 3071, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 182, 1320, 0, 0, 0, 4404, 1, 392, 0, 0, 0, 0, 0, 0, 0, 0, 2535, 263, 271, 0, 362, 2149, 291, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 2, 201, 0, 3, 0, 0, 5107, 713, 0, 0, 0, 0, 0, 0, 2, 4, 0, 5235, 842, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 6321, 6115, 6156, 6299, 5873, 6166, 6176] avg=6144.0000 max=6321 max_rank=1 min=5873 min_rank=5 imbalance_ratio=1.028809 max_load_reduction=42831 max_load_reduction_pct=0.871399 moved_to_echo=24514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 352, 0, 0, 0, 0, 3824, 1617, 623, 0, 0, 0, 0, 0, 203, 214, 25, 0, 1998, 82, 0, 3, 2429, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 6035, 0, 0, 0, 0, 0, 0, 0, 464, 1092, 0, 1695, 0, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 1154, 0, 0, 0, 4406, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 194, 289, 0, 339, 2470, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 184, 0, 0, 0, 0, 5120, 711, 0, 0, 0, 0, 0, 0, 43, 0, 0, 5263, 797, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6416, 6169, 6148, 6111, 6067, 5969, 6087, 6185] avg=6144.0000 max=6416 max_rank=0 min=5969 min_rank=5 imbalance_ratio=1.044271 max_load_reduction=42736 max_load_reduction_pct=0.869466 moved_to_echo=24308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 1152, 384, 128, 16640, 640, 128, 128, 19584, 9856, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 2816, 128, 0, 0, 128, 28416, 13056, 5120, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[20736, 1664, 2048, 0, 3072, 19584, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 1152, 384, 128, 16640, 640, 128, 128, 19584, 9856, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4096, 8832, 0, 13440, 128, 23040, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1280, 9856, 0, 0, 0, 35584, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[20736, 1664, 2048, 0, 3072, 19584, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 896, 128, 1536, 0, 128, 0, 0, 41216, 5760, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 128, 0, 42240, 6144, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 128, 768, 0, 0, 48512, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 262, 79, 0, 7597, 0, 414, 0, 0, 1, 0, 69, 0, 0, 0, 646, 0, 0, 0, 0, 5339, 0, 7219, 0, 0, 79, 7562, 0, 0, 0, 53, 1292, 0, 0, 534, 0, 0, 148, 136, 201, 0, 0, 2923, 0, 0, 0, 0, 6424, 0, 751, 0, 6067, 0, 0, 0, 0, 78, 0, 399, 719, 91, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 246, 112, 0, 7554, 0, 418, 0, 0, 0, 0, 75, 0, 0, 0, 509, 0, 0, 0, 0, 5127, 0, 7219, 0, 0, 81, 7654, 0, 0, 0, 63, 1271, 0, 0, 497, 0, 0, 144, 144, 207, 0, 0, 3450, 0, 0, 0, 0, 6537, 0, 753, 0, 5915, 0, 0, 0, 0, 64, 0, 398, 560, 79, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 249, 171, 0, 7536, 3, 434, 0, 0, 0, 0, 91, 4, 0, 0, 548, 0, 0, 0, 0, 4976, 3, 7119, 0, 0, 46, 7567, 0, 1, 0, 55, 1154, 9, 1, 597, 0, 4, 160, 161, 214, 5, 8, 3372, 0, 0, 6, 1, 6483, 0, 763, 0, 6048, 0, 0, 1, 1, 140, 0, 432, 541, 153, 1, 93] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 239, 98, 0, 7684, 0, 419, 0, 0, 0, 0, 74, 0, 0, 0, 477, 0, 0, 0, 0, 5303, 0, 7182, 0, 0, 74, 7675, 0, 0, 0, 60, 1091, 0, 0, 523, 0, 0, 147, 166, 207, 0, 0, 3572, 0, 0, 0, 0, 6463, 0, 733, 0, 5761, 0, 0, 0, 0, 81, 0, 388, 565, 96, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 254, 96, 0, 7585, 0, 399, 0, 0, 0, 0, 65, 0, 0, 0, 475, 0, 0, 0, 0, 5374, 0, 7267, 0, 0, 66, 7695, 0, 0, 0, 60, 1228, 0, 0, 515, 0, 0, 155, 140, 215, 0, 0, 3180, 0, 0, 0, 0, 6561, 0, 679, 0, 5988, 0, 0, 0, 0, 48, 0, 382, 595, 65, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 246, 112, 0, 7554, 0, 418, 0, 0, 0, 0, 75, 0, 0, 0, 509, 0, 0, 0, 0, 5127, 0, 7219, 0, 0, 81, 7654, 0, 0, 0, 63, 1271, 0, 0, 497, 0, 0, 144, 144, 207, 0, 0, 3450, 0, 0, 0, 0, 6537, 0, 753, 0, 5915, 0, 0, 0, 0, 64, 0, 398, 560, 79, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 254, 90, 0, 7641, 0, 432, 0, 0, 0, 0, 96, 0, 0, 0, 466, 0, 0, 0, 0, 5427, 0, 7256, 0, 0, 60, 7614, 0, 0, 0, 41, 973, 0, 1, 629, 0, 0, 152, 168, 194, 0, 0, 3367, 0, 0, 0, 0, 6528, 0, 651, 0, 5617, 0, 0, 0, 0, 174, 0, 486, 562, 177, 0, 96] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 239, 98, 0, 7684, 0, 419, 0, 0, 0, 0, 74, 0, 0, 0, 477, 0, 0, 0, 0, 5303, 0, 7182, 0, 0, 74, 7675, 0, 0, 0, 60, 1091, 0, 0, 523, 0, 0, 147, 166, 207, 0, 0, 3572, 0, 0, 0, 0, 6463, 0, 733, 0, 5761, 0, 0, 0, 0, 81, 0, 388, 565, 96, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 247, 151, 0, 7644, 0, 393, 0, 0, 0, 0, 90, 0, 0, 0, 443, 0, 0, 0, 0, 5303, 0, 7151, 0, 0, 89, 7593, 0, 0, 0, 65, 1060, 0, 1, 646, 0, 0, 177, 149, 243, 0, 0, 3602, 0, 0, 0, 0, 6498, 0, 658, 0, 5555, 0, 0, 0, 0, 161, 0, 442, 526, 175, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 262, 79, 0, 7597, 0, 414, 0, 0, 1, 0, 69, 0, 0, 0, 646, 0, 0, 0, 0, 5339, 0, 7219, 0, 0, 79, 7562, 0, 0, 0, 53, 1292, 0, 0, 534, 0, 0, 148, 136, 201, 0, 0, 2923, 0, 0, 0, 0, 6424, 0, 751, 0, 6067, 0, 0, 0, 0, 78, 0, 399, 719, 91, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 264, 116, 0, 7576, 0, 442, 0, 0, 0, 0, 82, 0, 0, 0, 543, 0, 0, 0, 0, 5397, 0, 7111, 0, 0, 63, 7673, 0, 0, 0, 90, 1081, 0, 0, 574, 0, 0, 195, 126, 285, 0, 0, 3283, 0, 0, 0, 0, 6444, 0, 733, 0, 5770, 0, 0, 0, 0, 72, 0, 440, 630, 80, 0, 82] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 254, 90, 0, 5387, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 6118, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 0, 5427, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 6002, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 1, 629, 0, 0, 152, 168, 2445, 1702, 0, 0, 0, 0, 0, 0, 194, 0, 0, 3367, 0, 0, 0, 0, 1612, 552, 395, 0, 0, 0, 0, 0, 0, 0, 651, 0, 5222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 486, 562, 177, 0, 96, 4083, 784, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 6214, 6247, 6103, 6070, 6120, 5873, 6362] avg=6144.0000 max=6362 max_rank=7 min=5873 min_rank=6 imbalance_ratio=1.035482 max_load_reduction=42790 max_load_reduction_pct=0.870565 moved_to_echo=24425 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 262, 79, 0, 5356, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 69, 0, 0, 0, 6087, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, 0, 5339, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 5961, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 0, 0, 534, 0, 0, 148, 136, 2407, 1692, 0, 0, 0, 0, 0, 0, 201, 0, 0, 2923, 0, 0, 0, 0, 1601, 549, 427, 0, 0, 0, 0, 0, 0, 0, 751, 0, 5640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 399, 719, 91, 0, 69, 4017, 780, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 6157, 6337, 6093, 6209, 5701, 6391, 6153] avg=6144.0000 max=6391 max_rank=6 min=5701 min_rank=5 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 246, 112, 0, 5326, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 6087, 0, 0, 0, 0, 0, 0, 0, 509, 0, 0, 0, 0, 5127, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 6033, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 0, 0, 497, 0, 0, 144, 144, 2448, 1682, 0, 0, 0, 0, 0, 0, 207, 0, 0, 3450, 0, 0, 0, 0, 1621, 546, 416, 0, 0, 0, 0, 0, 0, 0, 753, 0, 5499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 398, 560, 79, 0, 75, 4089, 780, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 6162, 5988, 6177, 6186, 6240, 6252, 6045] avg=6144.0000 max=6252 max_rank=6 min=5988 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=42900 max_load_reduction_pct=0.872803 moved_to_echo=24723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 247, 151, 0, 5390, 0, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 6029, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 0, 5303, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 5985, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 1, 646, 0, 0, 177, 149, 2433, 1702, 0, 0, 0, 0, 0, 0, 243, 0, 0, 3602, 0, 0, 0, 0, 1608, 552, 391, 0, 0, 0, 0, 0, 0, 0, 658, 0, 5164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 442, 526, 175, 0, 90, 4065, 773, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6181, 6119, 6095, 6139, 6168, 6396, 5822, 6232] avg=6144.0000 max=6396 max_rank=5 min=5822 min_rank=6 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=24618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 249, 171, 0, 5302, 3, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 4, 0, 0, 6006, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 4976, 3, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 5961, 0, 1, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 9, 1, 597, 0, 4, 160, 161, 2429, 1684, 0, 0, 0, 0, 0, 0, 214, 5, 8, 3372, 0, 0, 6, 1, 1606, 550, 430, 0, 0, 0, 0, 0, 0, 0, 763, 0, 5618, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 140, 0, 432, 541, 153, 1, 93, 4054, 775, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6101, 5865, 6063, 6199, 6192, 6382, 6190] avg=6144.0000 max=6382 max_rank=6 min=5865 min_rank=2 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 239, 98, 0, 5418, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 6055, 0, 0, 0, 0, 0, 0, 0, 477, 0, 0, 0, 0, 5303, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 6050, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 523, 0, 0, 147, 166, 2421, 1711, 0, 0, 0, 0, 0, 0, 207, 0, 0, 3572, 0, 0, 0, 0, 1625, 555, 405, 0, 0, 0, 0, 0, 0, 0, 733, 0, 5356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 388, 565, 96, 0, 74, 4042, 776, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6174, 6129, 6131, 6184, 6059, 6364, 6089, 6022] avg=6144.0000 max=6364 max_rank=5 min=6022 min_rank=7 imbalance_ratio=1.035807 max_load_reduction=42788 max_load_reduction_pct=0.870524 moved_to_echo=24534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 254, 96, 0, 5348, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 0, 5374, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 6066, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 0, 0, 515, 0, 0, 155, 140, 2457, 1689, 0, 0, 0, 0, 0, 0, 215, 0, 0, 3180, 0, 0, 0, 0, 1629, 548, 421, 0, 0, 0, 0, 0, 0, 0, 679, 0, 5567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 382, 595, 65, 0, 65, 4104, 786, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6192, 6203, 6192, 6184, 5993, 6246, 6045] avg=6144.0000 max=6246 max_rank=6 min=5993 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=42906 max_load_reduction_pct=0.872925 moved_to_echo=24468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 239, 98, 0, 5418, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 6055, 0, 0, 0, 0, 0, 0, 0, 477, 0, 0, 0, 0, 5303, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 6050, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 523, 0, 0, 147, 166, 2421, 1711, 0, 0, 0, 0, 0, 0, 207, 0, 0, 3572, 0, 0, 0, 0, 1625, 555, 405, 0, 0, 0, 0, 0, 0, 0, 733, 0, 5356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 388, 565, 96, 0, 74, 4042, 776, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6174, 6129, 6131, 6184, 6059, 6364, 6089, 6022] avg=6144.0000 max=6364 max_rank=5 min=6022 min_rank=7 imbalance_ratio=1.035807 max_load_reduction=42788 max_load_reduction_pct=0.870524 moved_to_echo=24534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 247, 151, 0, 5390, 0, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 6029, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 0, 5303, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 5985, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 1, 646, 0, 0, 177, 149, 2433, 1702, 0, 0, 0, 0, 0, 0, 243, 0, 0, 3602, 0, 0, 0, 0, 1608, 552, 391, 0, 0, 0, 0, 0, 0, 0, 658, 0, 5164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 442, 526, 175, 0, 90, 4065, 773, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6181, 6119, 6095, 6139, 6168, 6396, 5822, 6232] avg=6144.0000 max=6396 max_rank=5 min=5822 min_rank=6 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=24618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 254, 90, 0, 5387, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 6118, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 0, 5427, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 6002, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 1, 629, 0, 0, 152, 168, 2445, 1702, 0, 0, 0, 0, 0, 0, 194, 0, 0, 3367, 0, 0, 0, 0, 1612, 552, 395, 0, 0, 0, 0, 0, 0, 0, 651, 0, 5222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 486, 562, 177, 0, 96, 4083, 784, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 6214, 6247, 6103, 6070, 6120, 5873, 6362] avg=6144.0000 max=6362 max_rank=7 min=5873 min_rank=6 imbalance_ratio=1.035482 max_load_reduction=42790 max_load_reduction_pct=0.870565 moved_to_echo=24425 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 262, 79, 0, 5356, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 69, 0, 0, 0, 6087, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, 0, 5339, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 5961, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 0, 0, 534, 0, 0, 148, 136, 2407, 1692, 0, 0, 0, 0, 0, 0, 201, 0, 0, 2923, 0, 0, 0, 0, 1601, 549, 427, 0, 0, 0, 0, 0, 0, 0, 751, 0, 5640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 399, 719, 91, 0, 69, 4017, 780, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 6157, 6337, 6093, 6209, 5701, 6391, 6153] avg=6144.0000 max=6391 max_rank=6 min=5701 min_rank=5 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 264, 116, 0, 5342, 0, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 5996, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 5397, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 6048, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 0, 0, 574, 0, 0, 195, 126, 2414, 1687, 0, 0, 0, 0, 0, 0, 285, 0, 0, 3283, 0, 0, 0, 0, 1625, 547, 406, 0, 0, 0, 0, 0, 0, 0, 733, 0, 5364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 440, 630, 80, 0, 82, 4030, 769, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 6078, 6286, 6201, 6077, 6146, 6097, 6103] avg=6144.0000 max=6286 max_rank=2 min=6077 min_rank=4 imbalance_ratio=1.023112 max_load_reduction=42866 max_load_reduction_pct=0.872111 moved_to_echo=24423 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 249, 171, 0, 5302, 3, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 4, 0, 0, 6006, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 4976, 3, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 5961, 0, 1, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 9, 1, 597, 0, 4, 160, 161, 2429, 1684, 0, 0, 0, 0, 0, 0, 214, 5, 8, 3372, 0, 0, 6, 1, 1606, 550, 430, 0, 0, 0, 0, 0, 0, 0, 763, 0, 5618, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 140, 0, 432, 541, 153, 1, 93, 4054, 775, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6101, 5865, 6063, 6199, 6192, 6382, 6190] avg=6144.0000 max=6382 max_rank=6 min=5865 min_rank=2 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9216, 128, 128, 4608, 0, 128, 1280, 1280, 19456, 13568, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 128, 128, 26752, 0, 0, 128, 128, 12928, 4480, 3328, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 896, 0, 3456, 4736, 1024, 128, 768, 32512, 6272, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4224, 0, 0, 0, 0, 42368, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 5760, 0, 43520, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 2048, 1024, 0, 42880, 128, 3456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 640, 48128, 0, 128, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 5760, 0, 43520, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 768, 128, 0, 0, 48512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 640, 48128, 0, 128, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[6227, 0, 0, 0, 42, 0, 0, 0, 0, 374, 0, 0, 95, 0, 585, 5447, 4, 7301, 0, 96, 0, 0, 72, 0, 7473, 0, 1, 254, 0, 3638, 95, 7517, 2873, 95, 0, 0, 0, 0, 0, 0, 29, 0, 291, 0, 0, 4791, 254, 0, 0, 0, 0, 1451, 0, 0, 0, 0, 0, 133, 14, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[5281, 0, 0, 0, 53, 0, 0, 0, 0, 455, 0, 0, 69, 0, 691, 4843, 66, 7329, 0, 69, 0, 0, 66, 0, 7580, 0, 0, 261, 0, 3818, 69, 7546, 4363, 69, 0, 0, 0, 0, 0, 0, 69, 0, 252, 0, 0, 4210, 261, 0, 0, 0, 0, 1574, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[5553, 0, 0, 0, 54, 0, 1, 0, 2, 397, 0, 0, 91, 0, 643, 5267, 62, 7290, 2, 91, 0, 0, 76, 0, 7483, 5, 2, 249, 3, 3584, 92, 7607, 3776, 91, 0, 0, 1, 1, 0, 0, 91, 0, 225, 4, 0, 4450, 249, 5, 0, 0, 0, 1537, 2, 0, 0, 0, 5, 155, 1, 0, 1, 4, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[5777, 0, 0, 0, 60, 0, 0, 0, 0, 312, 0, 0, 65, 0, 652, 5237, 43, 7391, 0, 65, 0, 0, 75, 0, 7610, 0, 0, 254, 0, 4097, 63, 7638, 3199, 63, 0, 0, 0, 0, 0, 0, 56, 0, 195, 0, 0, 4389, 254, 0, 0, 0, 0, 1536, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[6125, 0, 0, 0, 60, 0, 0, 0, 0, 375, 0, 0, 74, 0, 732, 5612, 6, 7320, 0, 74, 0, 0, 92, 0, 7540, 0, 0, 239, 0, 3709, 71, 7596, 2687, 71, 0, 0, 0, 0, 0, 0, 29, 0, 221, 0, 0, 4830, 239, 0, 0, 0, 0, 1325, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[6100, 0, 0, 0, 66, 0, 0, 0, 0, 322, 0, 0, 90, 0, 665, 5732, 25, 7236, 0, 90, 0, 0, 60, 0, 7512, 0, 0, 246, 0, 3554, 90, 7651, 2536, 90, 0, 0, 0, 0, 0, 0, 33, 0, 223, 0, 0, 4893, 247, 0, 0, 0, 0, 1589, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[5677, 0, 0, 0, 63, 0, 0, 0, 0, 378, 0, 0, 75, 0, 688, 5414, 42, 7357, 0, 75, 0, 0, 69, 0, 7577, 0, 0, 246, 0, 3947, 75, 7612, 3075, 75, 0, 0, 0, 0, 0, 0, 74, 0, 207, 0, 0, 4565, 246, 0, 0, 0, 0, 1476, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[6100, 0, 0, 0, 66, 0, 0, 0, 0, 322, 0, 0, 90, 0, 665, 5732, 25, 7236, 0, 90, 0, 0, 60, 0, 7512, 0, 0, 246, 0, 3554, 90, 7651, 2536, 90, 0, 0, 0, 0, 0, 0, 33, 0, 223, 0, 0, 4893, 247, 0, 0, 0, 0, 1589, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[6227, 0, 0, 0, 42, 0, 0, 0, 0, 374, 0, 0, 95, 0, 585, 5447, 4, 7301, 0, 96, 0, 0, 72, 0, 7473, 0, 1, 254, 0, 3638, 95, 7517, 2873, 95, 0, 0, 0, 0, 0, 0, 29, 0, 291, 0, 0, 4791, 254, 0, 0, 0, 0, 1451, 0, 0, 0, 0, 0, 133, 14, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[6125, 0, 0, 0, 60, 0, 0, 0, 0, 375, 0, 0, 74, 0, 732, 5612, 6, 7320, 0, 74, 0, 0, 92, 0, 7540, 0, 0, 239, 0, 3709, 71, 7596, 2687, 71, 0, 0, 0, 0, 0, 0, 29, 0, 221, 0, 0, 4830, 239, 0, 0, 0, 0, 1325, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[5281, 0, 0, 0, 53, 0, 0, 0, 0, 455, 0, 0, 69, 0, 691, 4843, 66, 7329, 0, 69, 0, 0, 66, 0, 7580, 0, 0, 261, 0, 3818, 69, 7546, 4363, 69, 0, 0, 0, 0, 0, 0, 69, 0, 252, 0, 0, 4210, 261, 0, 0, 0, 0, 1574, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[6036, 0, 0, 0, 90, 0, 0, 0, 0, 352, 0, 0, 80, 0, 592, 5494, 3, 7344, 0, 82, 0, 0, 44, 0, 7542, 0, 0, 264, 0, 4059, 82, 7703, 2989, 82, 0, 0, 0, 0, 0, 0, 52, 0, 196, 0, 0, 4267, 264, 0, 0, 0, 0, 1402, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[5553, 0, 0, 0, 54, 0, 1, 0, 2, 397, 0, 0, 91, 0, 643, 5267, 62, 7290, 2, 91, 0, 0, 76, 0, 7483, 5, 2, 249, 3, 3584, 92, 7607, 3776, 91, 0, 0, 1, 1, 0, 0, 91, 0, 225, 4, 0, 4450, 249, 5, 0, 0, 0, 1537, 2, 0, 0, 0, 5, 155, 1, 0, 1, 4, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6227, 0, 0, 0, 42, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 95, 0, 585, 5101, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5947, 0, 96, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 1, 254, 0, 3638, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 95, 0, 0, 0, 0, 0, 0, 2450, 403, 0, 0, 0, 0, 0, 0, 29, 0, 291, 0, 0, 4791, 254, 0, 951, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 0, 0, 0, 0, 3029, 1581, 0, 0, 0, 0, 0, 0, 0, 133, 14, 0, 0, 0, 0, 0, 5936, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6507, 6155, 6119, 5982, 5821, 6424, 6061, 6083] avg=6144.0000 max=6507 max_rank=0 min=5821 min_rank=4 imbalance_ratio=1.059082 max_load_reduction=42645 max_load_reduction_pct=0.867615 moved_to_echo=24389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5281, 0, 0, 0, 53, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 69, 0, 691, 4535, 0, 0, 0, 0, 0, 0, 0, 0, 66, 5971, 0, 69, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 0, 0, 261, 0, 3818, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 69, 0, 0, 0, 0, 0, 0, 2485, 404, 0, 0, 0, 0, 0, 0, 69, 0, 252, 0, 0, 4210, 261, 0, 954, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 0, 0, 0, 0, 3072, 1588, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5546, 5750, 6172, 6171, 7321, 5842, 6234, 6116] avg=6144.0000 max=7321 max_rank=4 min=5546 min_rank=0 imbalance_ratio=1.191569 max_load_reduction=41831 max_load_reduction_pct=0.851054 moved_to_echo=25513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6125, 0, 0, 0, 60, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 74, 0, 732, 5254, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5963, 0, 74, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 0, 239, 0, 3709, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 71, 0, 0, 0, 0, 0, 0, 2472, 404, 0, 0, 0, 0, 0, 0, 29, 0, 221, 0, 0, 4830, 239, 0, 953, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 0, 0, 0, 0, 3056, 1599, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 5997, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6431, 6435, 6135, 6031, 5634, 6384, 5980, 6122] avg=6144.0000 max=6435 max_rank=1 min=5634 min_rank=4 imbalance_ratio=1.047363 max_load_reduction=42717 max_load_reduction_pct=0.869080 moved_to_echo=24120 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6100, 0, 0, 0, 66, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 90, 0, 665, 5367, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5895, 0, 90, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2004, 0, 0, 246, 0, 3554, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2536, 90, 0, 0, 0, 0, 0, 0, 2463, 399, 0, 0, 0, 0, 0, 0, 33, 0, 223, 0, 0, 4893, 247, 0, 942, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 0, 0, 0, 0, 3045, 1610, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 6041, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 6444, 6070, 5894, 5488, 6452, 6244, 6143] avg=6144.0000 max=6452 max_rank=5 min=5488 min_rank=4 imbalance_ratio=1.050130 max_load_reduction=42700 max_load_reduction_pct=0.868734 moved_to_echo=24327 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5677, 0, 0, 0, 63, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 75, 0, 688, 5069, 0, 0, 0, 0, 0, 0, 0, 0, 42, 5993, 0, 75, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2022, 0, 0, 246, 0, 3947, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3075, 75, 0, 0, 0, 0, 0, 0, 2484, 406, 0, 0, 0, 0, 0, 0, 74, 0, 207, 0, 0, 4565, 246, 0, 958, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 0, 0, 0, 0, 3071, 1602, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 6010, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6210, 6179, 6290, 6040, 6158, 6149, 6149] avg=6144.0000 max=6290 max_rank=3 min=5977 min_rank=0 imbalance_ratio=1.023763 max_load_reduction=42862 max_load_reduction_pct=0.872030 moved_to_echo=24496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5777, 0, 0, 0, 60, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 65, 0, 652, 4904, 0, 0, 0, 0, 0, 0, 0, 0, 43, 6021, 0, 65, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2031, 0, 0, 254, 0, 4097, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3199, 63, 0, 0, 0, 0, 0, 0, 2495, 408, 0, 0, 0, 0, 0, 0, 56, 0, 195, 0, 0, 4389, 254, 0, 962, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1536, 0, 0, 0, 0, 3084, 1607, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 6031, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 5933, 6204, 6445, 6165, 5960, 6227, 6152] avg=6144.0000 max=6445 max_rank=3 min=5933 min_rank=1 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=24504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6100, 0, 0, 0, 66, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 90, 0, 665, 5367, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5895, 0, 90, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2004, 0, 0, 246, 0, 3554, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2536, 90, 0, 0, 0, 0, 0, 0, 2463, 399, 0, 0, 0, 0, 0, 0, 33, 0, 223, 0, 0, 4893, 247, 0, 942, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 0, 0, 0, 0, 3045, 1610, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 6041, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 6444, 6070, 5894, 5488, 6452, 6244, 6143] avg=6144.0000 max=6452 max_rank=5 min=5488 min_rank=4 imbalance_ratio=1.050130 max_load_reduction=42700 max_load_reduction_pct=0.868734 moved_to_echo=24327 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6227, 0, 0, 0, 42, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 95, 0, 585, 5101, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5947, 0, 96, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 1, 254, 0, 3638, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 95, 0, 0, 0, 0, 0, 0, 2450, 403, 0, 0, 0, 0, 0, 0, 29, 0, 291, 0, 0, 4791, 254, 0, 951, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 0, 0, 0, 0, 3029, 1581, 0, 0, 0, 0, 0, 0, 0, 133, 14, 0, 0, 0, 0, 0, 5936, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6507, 6155, 6119, 5982, 5821, 6424, 6061, 6083] avg=6144.0000 max=6507 max_rank=0 min=5821 min_rank=4 imbalance_ratio=1.059082 max_load_reduction=42645 max_load_reduction_pct=0.867615 moved_to_echo=24389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6125, 0, 0, 0, 60, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 74, 0, 732, 5254, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5963, 0, 74, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 0, 239, 0, 3709, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 71, 0, 0, 0, 0, 0, 0, 2472, 404, 0, 0, 0, 0, 0, 0, 29, 0, 221, 0, 0, 4830, 239, 0, 953, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 0, 0, 0, 0, 3056, 1599, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 5997, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6431, 6435, 6135, 6031, 5634, 6384, 5980, 6122] avg=6144.0000 max=6435 max_rank=1 min=5634 min_rank=4 imbalance_ratio=1.047363 max_load_reduction=42717 max_load_reduction_pct=0.869080 moved_to_echo=24120 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5553, 0, 0, 0, 54, 0, 1, 0, 234, 0, 0, 0, 0, 0, 0, 0, 2, 397, 0, 0, 91, 0, 643, 4923, 0, 0, 0, 0, 0, 0, 0, 0, 62, 5930, 2, 91, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 5, 2, 249, 3, 3584, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 91, 0, 0, 1, 1, 0, 0, 2459, 406, 0, 0, 0, 0, 0, 0, 91, 0, 225, 4, 0, 4450, 249, 5, 954, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 2, 0, 0, 0, 3037, 1601, 0, 0, 0, 0, 0, 0, 5, 155, 1, 0, 1, 4, 0, 0, 6006, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5842, 6056, 6161, 5922, 6734, 6088, 6177, 6172] avg=6144.0000 max=6734 max_rank=4 min=5842 min_rank=0 imbalance_ratio=1.096029 max_load_reduction=42418 max_load_reduction_pct=0.862996 moved_to_echo=25171 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5281, 0, 0, 0, 53, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 69, 0, 691, 4535, 0, 0, 0, 0, 0, 0, 0, 0, 66, 5971, 0, 69, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 0, 0, 261, 0, 3818, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 69, 0, 0, 0, 0, 0, 0, 2485, 404, 0, 0, 0, 0, 0, 0, 69, 0, 252, 0, 0, 4210, 261, 0, 954, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 0, 0, 0, 0, 3072, 1588, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5546, 5750, 6172, 6171, 7321, 5842, 6234, 6116] avg=6144.0000 max=7321 max_rank=4 min=5546 min_rank=0 imbalance_ratio=1.191569 max_load_reduction=41831 max_load_reduction_pct=0.851054 moved_to_echo=25513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6036, 0, 0, 0, 90, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 80, 0, 592, 5145, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5983, 0, 82, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 0, 264, 0, 4059, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2989, 82, 0, 0, 0, 0, 0, 0, 2473, 405, 0, 0, 0, 0, 0, 0, 52, 0, 196, 0, 0, 4267, 264, 0, 956, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 0, 3057, 1621, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 6082, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6366, 6169, 6112, 6417, 5949, 5844, 6080, 6215] avg=6144.0000 max=6417 max_rank=3 min=5844 min_rank=5 imbalance_ratio=1.044434 max_load_reduction=42735 max_load_reduction_pct=0.869446 moved_to_echo=24088 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[25600, 640, 0, 0, 128, 128, 0, 0, 19840, 3328, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16128, 128, 128, 2048, 128, 30464, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 0, 1920, 128, 0, 36480, 2048, 128, 7680, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 11904, 128, 0, 0, 0, 24576, 12928, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 3072, 0, 0, 640, 0, 5248, 40320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 11904, 128, 0, 0, 0, 24576, 12928, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[46848, 0, 0, 0, 512, 0, 128, 0, 1920, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1152, 128, 0, 128, 128, 0, 0, 48128, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 47744, 128, 768, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1152, 128, 0, 128, 128, 0, 0, 48128, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 530, 0, 0, 911, 0, 0, 3795, 7836, 0, 0, 0, 729, 0, 87, 335, 0, 137, 1973, 0, 0, 0, 1109, 0, 0, 330, 0, 0, 0, 0, 0, 1, 0, 0, 7379, 155, 0, 90, 357, 0, 1608, 0, 0, 0, 0, 0, 2125, 0, 4, 0, 7846, 1, 85, 0, 4749, 0, 0, 0, 6146, 649, 185, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 516, 0, 0, 889, 0, 0, 4082, 7799, 0, 0, 0, 695, 0, 92, 346, 0, 153, 1888, 0, 0, 0, 1154, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 7450, 126, 0, 63, 485, 0, 1837, 0, 0, 0, 0, 0, 1869, 0, 0, 0, 7866, 0, 92, 0, 4411, 0, 0, 0, 6335, 685, 33, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 538, 0, 0, 915, 0, 0, 4300, 7809, 0, 0, 0, 709, 0, 69, 317, 0, 132, 1524, 0, 0, 0, 1245, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 7433, 139, 0, 75, 463, 0, 1776, 0, 0, 0, 0, 0, 2139, 0, 0, 0, 7902, 0, 69, 0, 4212, 0, 0, 0, 6253, 747, 61, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 0, 0, 924, 0, 0, 3887, 7848, 0, 0, 0, 676, 0, 57, 313, 0, 98, 1993, 0, 0, 0, 1043, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 7403, 140, 0, 95, 349, 0, 1894, 0, 0, 0, 0, 0, 1938, 0, 3, 0, 7827, 1, 57, 0, 4870, 0, 0, 0, 6086, 683, 156, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 530, 0, 0, 911, 0, 0, 3795, 7836, 0, 0, 0, 729, 0, 87, 335, 0, 137, 1973, 0, 0, 0, 1109, 0, 0, 330, 0, 0, 0, 0, 0, 1, 0, 0, 7379, 155, 0, 90, 357, 0, 1608, 0, 0, 0, 0, 0, 2125, 0, 4, 0, 7846, 1, 85, 0, 4749, 0, 0, 0, 6146, 649, 185, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 544, 0, 0, 900, 0, 0, 4359, 7811, 0, 0, 0, 733, 0, 79, 345, 0, 132, 1318, 0, 0, 0, 1115, 0, 0, 344, 0, 0, 0, 0, 0, 0, 0, 0, 7444, 123, 0, 69, 207, 0, 1942, 0, 0, 0, 0, 0, 2104, 0, 0, 0, 7866, 0, 76, 0, 4365, 0, 0, 0, 6257, 706, 313, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 472, 0, 0, 995, 0, 0, 4142, 7770, 0, 0, 0, 686, 0, 113, 380, 0, 183, 1859, 0, 0, 0, 1057, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 7430, 172, 0, 82, 336, 0, 1765, 0, 0, 0, 0, 0, 2016, 0, 0, 0, 7801, 0, 109, 0, 4412, 0, 0, 0, 6165, 720, 144, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 523, 1, 1, 850, 0, 0, 4509, 7786, 0, 4, 7, 700, 0, 69, 328, 4, 124, 1419, 0, 0, 0, 1363, 0, 0, 340, 1, 1, 0, 0, 0, 0, 4, 3, 7355, 145, 1, 91, 443, 3, 1889, 0, 1, 0, 0, 7, 2109, 2, 3, 0, 7845, 2, 69, 0, 4230, 0, 2, 0, 6148, 700, 69, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 516, 0, 0, 889, 0, 0, 4082, 7799, 0, 0, 0, 695, 0, 92, 346, 0, 153, 1888, 0, 0, 0, 1154, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 7450, 126, 0, 63, 485, 0, 1837, 0, 0, 0, 0, 0, 1869, 0, 0, 0, 7866, 0, 92, 0, 4411, 0, 0, 0, 6335, 685, 33, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 600, 0, 0, 987, 0, 0, 3857, 7822, 0, 0, 0, 734, 0, 76, 318, 0, 135, 1754, 0, 0, 0, 1130, 0, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 7408, 135, 0, 71, 449, 0, 1900, 0, 0, 0, 0, 0, 1896, 0, 1, 0, 7859, 0, 68, 0, 4622, 0, 0, 0, 6148, 742, 137, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 544, 0, 0, 900, 0, 0, 4359, 7811, 0, 0, 0, 733, 0, 79, 345, 0, 132, 1318, 0, 0, 0, 1115, 0, 0, 344, 0, 0, 0, 0, 0, 0, 0, 0, 7444, 123, 0, 69, 207, 0, 1942, 0, 0, 0, 0, 0, 2104, 0, 0, 0, 7866, 0, 76, 0, 4365, 0, 0, 0, 6257, 706, 313, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 0, 0, 924, 0, 4719, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 1326, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 313, 0, 98, 1993, 0, 0, 0, 2376, 1489, 0, 0, 0, 0, 0, 0, 1043, 0, 0, 333, 0, 0, 0, 0, 2822, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5914, 140, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 1894, 0, 0, 0, 0, 0, 2458, 1423, 0, 0, 0, 0, 0, 0, 1938, 0, 3, 0, 4028, 1, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4870, 0, 0, 0, 806, 683, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 5889, 6326, 6001, 6149, 6124, 6027, 6515] avg=6144.0000 max=6515 max_rank=7 min=5889 min_rank=1 imbalance_ratio=1.060384 max_load_reduction=42637 max_load_reduction_pct=0.867452 moved_to_echo=24815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 523, 1, 1, 850, 0, 0, 4509, 7786, 0, 4, 7, 700, 0, 69, 328, 4, 124, 1419, 0, 0, 0, 1363, 0, 0, 340, 1, 1, 0, 0, 0, 0, 4, 3, 7355, 145, 1, 91, 443, 3, 1889, 0, 1, 0, 0, 7, 2109, 2, 3, 0, 7845, 2, 69, 0, 4230, 0, 2, 0, 6148, 700, 69, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 538, 0, 0, 915, 0, 4696, 0, 0, 0, 0, 0, 0, 0, 0, 4300, 1319, 0, 0, 0, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 317, 0, 132, 1524, 0, 0, 0, 2399, 1495, 0, 0, 0, 0, 0, 0, 1245, 0, 0, 325, 0, 0, 0, 0, 2899, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5938, 139, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 1776, 0, 0, 0, 0, 0, 2526, 1437, 0, 0, 0, 0, 0, 0, 2139, 0, 0, 0, 4066, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4212, 0, 0, 0, 828, 747, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6328, 5936, 6263, 6152, 6202, 6274, 5848] avg=6144.0000 max=6328 max_rank=1 min=5848 min_rank=7 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 516, 0, 0, 889, 0, 4690, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1318, 0, 0, 0, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 346, 0, 153, 1888, 0, 0, 0, 2388, 1498, 0, 0, 0, 0, 0, 0, 1154, 0, 0, 286, 0, 0, 0, 0, 2937, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5952, 126, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 485, 0, 1837, 0, 0, 0, 0, 0, 2559, 1430, 0, 0, 0, 0, 0, 0, 1869, 0, 0, 0, 4048, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4411, 0, 0, 0, 839, 685, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6095, 6365, 6168, 6141, 6311, 6009, 5968] avg=6144.0000 max=6365 max_rank=2 min=5968 min_rank=7 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 530, 0, 0, 911, 0, 4712, 0, 0, 0, 0, 0, 0, 0, 0, 3795, 1324, 0, 0, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 335, 0, 137, 1973, 0, 0, 0, 2382, 1484, 0, 0, 0, 0, 0, 0, 1109, 0, 0, 330, 0, 0, 0, 0, 2850, 1800, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5895, 155, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 1608, 0, 0, 0, 0, 0, 2483, 1427, 0, 0, 0, 0, 0, 0, 2125, 0, 4, 0, 4037, 1, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4749, 0, 0, 0, 813, 649, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 5848, 6398, 6089, 6141, 5875, 6252, 6396] avg=6144.0000 max=6398 max_rank=2 min=5848 min_rank=1 imbalance_ratio=1.041341 max_load_reduction=42754 max_load_reduction_pct=0.869832 moved_to_echo=24664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 600, 0, 0, 987, 0, 4704, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 1321, 0, 0, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 318, 0, 135, 1754, 0, 0, 0, 2386, 1490, 0, 0, 0, 0, 0, 0, 1130, 0, 0, 303, 0, 0, 0, 0, 2851, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5918, 135, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 1900, 0, 0, 0, 0, 0, 2483, 1429, 0, 0, 0, 0, 0, 0, 1896, 0, 1, 0, 4044, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4622, 0, 0, 0, 814, 742, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6291, 5912, 6159, 6081, 6124, 6261, 6009, 6315] avg=6144.0000 max=6315 max_rank=7 min=5912 min_rank=1 imbalance_ratio=1.027832 max_load_reduction=42837 max_load_reduction_pct=0.871521 moved_to_echo=24709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 0, 0, 924, 0, 4719, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 1326, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 313, 0, 98, 1993, 0, 0, 0, 2376, 1489, 0, 0, 0, 0, 0, 0, 1043, 0, 0, 333, 0, 0, 0, 0, 2822, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5914, 140, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 1894, 0, 0, 0, 0, 0, 2458, 1423, 0, 0, 0, 0, 0, 0, 1938, 0, 3, 0, 4028, 1, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4870, 0, 0, 0, 806, 683, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 5889, 6326, 6001, 6149, 6124, 6027, 6515] avg=6144.0000 max=6515 max_rank=7 min=5889 min_rank=1 imbalance_ratio=1.060384 max_load_reduction=42637 max_load_reduction_pct=0.867452 moved_to_echo=24815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 544, 0, 0, 900, 0, 4697, 0, 0, 0, 0, 0, 0, 0, 0, 4359, 1320, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 345, 0, 132, 1318, 0, 0, 0, 2388, 1497, 0, 0, 0, 0, 0, 0, 1115, 0, 0, 344, 0, 0, 0, 0, 2901, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5947, 123, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 1942, 0, 0, 0, 0, 0, 2528, 1430, 0, 0, 0, 0, 0, 0, 2104, 0, 0, 0, 4048, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4365, 0, 0, 0, 828, 706, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6412, 5759, 6154, 6139, 6107, 6228, 6212] avg=6144.0000 max=6412 max_rank=1 min=5759 min_rank=2 imbalance_ratio=1.043620 max_load_reduction=42740 max_load_reduction_pct=0.869548 moved_to_echo=24686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 523, 1, 1, 850, 0, 4687, 0, 0, 0, 0, 0, 0, 0, 0, 4509, 1307, 0, 4, 7, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 328, 4, 124, 1419, 0, 0, 0, 2385, 1483, 0, 0, 0, 0, 0, 0, 1363, 0, 0, 340, 1, 1, 0, 0, 2855, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 5872, 145, 1, 91, 0, 0, 0, 0, 0, 0, 0, 0, 443, 3, 1889, 0, 1, 0, 0, 7, 2488, 1431, 0, 0, 0, 0, 0, 0, 2109, 2, 3, 0, 4029, 2, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4230, 0, 2, 0, 805, 700, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6527, 5812, 6352, 6116, 6262, 6214, 5806] avg=6144.0000 max=6527 max_rank=1 min=5806 min_rank=7 imbalance_ratio=1.062337 max_load_reduction=42625 max_load_reduction_pct=0.867208 moved_to_echo=24398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 516, 0, 0, 889, 0, 4690, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1318, 0, 0, 0, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 346, 0, 153, 1888, 0, 0, 0, 2388, 1498, 0, 0, 0, 0, 0, 0, 1154, 0, 0, 286, 0, 0, 0, 0, 2937, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5952, 126, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 485, 0, 1837, 0, 0, 0, 0, 0, 2559, 1430, 0, 0, 0, 0, 0, 0, 1869, 0, 0, 0, 4048, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4411, 0, 0, 0, 839, 685, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6095, 6365, 6168, 6141, 6311, 6009, 5968] avg=6144.0000 max=6365 max_rank=2 min=5968 min_rank=7 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 472, 0, 0, 995, 0, 4672, 0, 0, 0, 0, 0, 0, 0, 0, 4142, 1313, 0, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 380, 0, 183, 1859, 0, 0, 0, 2368, 1494, 0, 0, 0, 0, 0, 0, 1057, 0, 0, 343, 0, 0, 0, 0, 2859, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5936, 172, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 1765, 0, 0, 0, 0, 0, 2490, 1419, 0, 0, 0, 0, 0, 0, 2016, 0, 0, 0, 4014, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4412, 0, 0, 0, 816, 720, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6141, 6397, 6044, 6190, 6010, 6139, 6092] avg=6144.0000 max=6397 max_rank=2 min=6010 min_rank=5 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 523, 1, 1, 850, 0, 4687, 0, 0, 0, 0, 0, 0, 0, 0, 4509, 1307, 0, 4, 7, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 328, 4, 124, 1419, 0, 0, 0, 2385, 1483, 0, 0, 0, 0, 0, 0, 1363, 0, 0, 340, 1, 1, 0, 0, 2855, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 5872, 145, 1, 91, 0, 0, 0, 0, 0, 0, 0, 0, 443, 3, 1889, 0, 1, 0, 0, 7, 2488, 1431, 0, 0, 0, 0, 0, 0, 2109, 2, 3, 0, 4029, 2, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4230, 0, 2, 0, 805, 700, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6527, 5812, 6352, 6116, 6262, 6214, 5806] avg=6144.0000 max=6527 max_rank=1 min=5806 min_rank=7 imbalance_ratio=1.062337 max_load_reduction=42625 max_load_reduction_pct=0.867208 moved_to_echo=24398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 2688, 128, 1152, 13824, 0, 0, 0, 19072, 12032, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3200, 128, 14720, 0, 128, 0, 0, 128, 20096, 11520, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16256, 128, 128, 0, 32384, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[35968, 0, 128, 0, 6656, 5632, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3200, 128, 14720, 0, 128, 0, 0, 128, 20096, 11520, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 33024, 10624, 0, 128, 128, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 4224, 128, 128, 7424, 0, 37632, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 0, 0, 2688, 128, 128, 0, 0, 23040, 14464, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[35968, 0, 128, 0, 6656, 5632, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 4224, 128, 128, 7424, 0, 37632, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 47488, 1152, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[572, 0, 0, 90, 337, 6202, 0, 0, 3420, 85, 119, 0, 0, 0, 0, 0, 0, 72, 1438, 0, 0, 27, 0, 0, 149, 0, 0, 0, 0, 90, 0, 0, 0, 0, 161, 0, 0, 0, 2377, 292, 0, 4226, 7757, 8126, 0, 0, 8, 0, 0, 0, 0, 90, 0, 0, 392, 7087, 5019, 0, 0, 0, 581, 244, 0, 191] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[536, 0, 0, 75, 321, 6222, 0, 0, 3732, 68, 93, 0, 0, 0, 0, 0, 0, 75, 1118, 0, 0, 74, 0, 0, 150, 0, 0, 0, 18, 75, 0, 0, 0, 0, 65, 0, 0, 0, 2537, 291, 0, 4442, 7676, 8129, 0, 0, 5, 0, 0, 0, 0, 75, 0, 0, 387, 7103, 4760, 0, 0, 0, 628, 282, 0, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[573, 0, 0, 96, 350, 6047, 0, 0, 4185, 57, 107, 0, 0, 0, 0, 0, 0, 94, 1249, 0, 0, 88, 0, 0, 167, 0, 0, 0, 2, 96, 0, 0, 0, 0, 174, 0, 0, 0, 2268, 283, 0, 4189, 7703, 8151, 0, 0, 7, 0, 0, 0, 0, 96, 0, 0, 326, 7036, 4718, 0, 0, 0, 545, 390, 4, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[546, 0, 0, 69, 330, 6281, 0, 0, 3896, 79, 85, 0, 0, 0, 0, 0, 0, 69, 924, 0, 0, 69, 0, 0, 136, 0, 0, 0, 59, 69, 0, 0, 0, 0, 92, 0, 0, 0, 3497, 313, 7, 4154, 7695, 8139, 0, 0, 17, 0, 0, 0, 0, 69, 0, 0, 402, 7098, 3963, 0, 0, 0, 606, 315, 4, 169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[502, 0, 0, 65, 319, 6184, 0, 0, 4009, 91, 81, 0, 0, 0, 0, 0, 0, 65, 1043, 0, 0, 63, 0, 0, 143, 0, 0, 0, 30, 65, 0, 0, 0, 0, 58, 0, 0, 0, 2905, 311, 0, 3972, 7742, 8132, 0, 0, 6, 0, 0, 0, 0, 65, 0, 0, 376, 7144, 4744, 0, 0, 0, 614, 247, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[508, 0, 0, 74, 313, 6172, 0, 0, 3780, 76, 98, 0, 0, 0, 0, 0, 0, 73, 1296, 0, 0, 59, 0, 0, 162, 0, 0, 0, 4, 74, 0, 0, 0, 0, 81, 0, 0, 0, 2233, 272, 0, 4397, 7729, 8132, 0, 0, 9, 0, 0, 0, 0, 74, 0, 0, 421, 7118, 4936, 0, 0, 0, 621, 243, 0, 197] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[582, 0, 8, 91, 340, 6043, 1, 0, 3773, 69, 113, 0, 0, 0, 0, 1, 0, 91, 1039, 5, 4, 91, 5, 0, 161, 0, 0, 3, 6, 91, 0, 0, 3, 0, 148, 0, 0, 0, 2850, 285, 15, 4656, 7644, 8128, 0, 6, 6, 0, 1, 2, 0, 91, 0, 0, 361, 7017, 4299, 8, 0, 3, 588, 358, 3, 163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[572, 0, 0, 90, 337, 6202, 0, 0, 3420, 85, 119, 0, 0, 0, 0, 0, 0, 72, 1438, 0, 0, 27, 0, 0, 149, 0, 0, 0, 0, 90, 0, 0, 0, 0, 161, 0, 0, 0, 2377, 292, 0, 4226, 7757, 8126, 0, 0, 8, 0, 0, 0, 0, 90, 0, 0, 392, 7087, 5019, 0, 0, 0, 581, 244, 0, 191] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[508, 0, 0, 74, 313, 6172, 0, 0, 3780, 76, 98, 0, 0, 0, 0, 0, 0, 73, 1296, 0, 0, 59, 0, 0, 162, 0, 0, 0, 4, 74, 0, 0, 0, 0, 81, 0, 0, 0, 2233, 272, 0, 4397, 7729, 8132, 0, 0, 9, 0, 0, 0, 0, 74, 0, 0, 421, 7118, 4936, 0, 0, 0, 621, 243, 0, 197] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[573, 0, 0, 96, 350, 6047, 0, 0, 4185, 57, 107, 0, 0, 0, 0, 0, 0, 94, 1249, 0, 0, 88, 0, 0, 167, 0, 0, 0, 2, 96, 0, 0, 0, 0, 174, 0, 0, 0, 2268, 283, 0, 4189, 7703, 8151, 0, 0, 7, 0, 0, 0, 0, 96, 0, 0, 326, 7036, 4718, 0, 0, 0, 545, 390, 4, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[576, 0, 0, 82, 346, 6187, 0, 0, 3903, 111, 90, 0, 0, 0, 0, 0, 0, 81, 1262, 0, 0, 77, 0, 0, 132, 0, 0, 0, 4, 82, 0, 0, 0, 0, 72, 0, 0, 0, 2620, 343, 0, 4236, 7710, 8105, 0, 0, 7, 0, 0, 0, 0, 82, 0, 0, 339, 7037, 4565, 0, 0, 0, 659, 260, 0, 184] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[536, 0, 0, 75, 321, 5228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3732, 68, 93, 0, 0, 0, 0, 0, 1396, 553, 187, 0, 0, 0, 0, 0, 0, 75, 1118, 0, 0, 74, 0, 0, 2582, 2232, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 18, 75, 0, 0, 5897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 2537, 291, 3065, 0, 0, 0, 0, 0, 0, 0, 0, 4442, 1842, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 387, 5707, 0, 0, 0, 0, 0, 0, 0, 0, 4760, 0, 0, 0, 628, 282, 0, 215, 441, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6029, 6081, 6140, 5958, 6289, 6169, 6326] avg=6144.0000 max=6326 max_rank=7 min=5958 min_rank=4 imbalance_ratio=1.029622 max_load_reduction=42826 max_load_reduction_pct=0.871297 moved_to_echo=24742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[573, 0, 0, 96, 350, 5082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4185, 57, 107, 0, 0, 0, 0, 0, 1383, 537, 188, 0, 0, 0, 0, 0, 0, 94, 1249, 0, 0, 88, 0, 0, 2591, 2237, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 2, 96, 0, 0, 5914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 2268, 283, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 4189, 1848, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 326, 5653, 0, 0, 0, 0, 0, 0, 0, 0, 4718, 0, 0, 0, 545, 390, 4, 151, 428, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6101, 6457, 6259, 6179, 5801, 6044, 6075, 6236] avg=6144.0000 max=6457 max_rank=1 min=5801 min_rank=4 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[546, 0, 0, 69, 330, 5278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 79, 85, 0, 0, 0, 0, 0, 1395, 558, 187, 0, 0, 0, 0, 0, 0, 69, 924, 0, 0, 69, 0, 0, 2588, 2235, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 59, 69, 0, 0, 5904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 3497, 313, 3072, 0, 0, 0, 0, 0, 0, 0, 7, 4154, 1848, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 402, 5703, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 606, 315, 4, 169, 445, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6223, 6200, 5885, 6168, 6974, 6026, 6174, 5502] avg=6144.0000 max=6974 max_rank=4 min=5502 min_rank=7 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=24676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[508, 0, 0, 74, 313, 5186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 76, 98, 0, 0, 0, 0, 0, 1399, 549, 188, 0, 0, 0, 0, 0, 0, 73, 1296, 0, 0, 59, 0, 0, 2600, 2233, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 4, 74, 0, 0, 5899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 2233, 272, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 4397, 1855, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 421, 5719, 0, 0, 0, 0, 0, 0, 0, 0, 4936, 0, 0, 0, 621, 243, 0, 197, 437, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6090, 6261, 6139, 5672, 6261, 6214, 6434] avg=6144.0000 max=6434 max_rank=7 min=5672 min_rank=4 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[572, 0, 0, 90, 337, 5212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 85, 119, 0, 0, 0, 0, 0, 1393, 551, 189, 0, 0, 0, 0, 0, 0, 72, 1438, 0, 0, 27, 0, 0, 2609, 2231, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 90, 0, 0, 5895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 2377, 292, 3097, 0, 0, 0, 0, 0, 0, 0, 0, 4226, 1862, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 392, 5694, 0, 0, 0, 0, 0, 0, 0, 0, 5019, 0, 0, 0, 581, 244, 0, 191, 439, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 5757, 6377, 6134, 5927, 6096, 6176, 6474] avg=6144.0000 max=6474 max_rank=7 min=5757 min_rank=1 imbalance_ratio=1.053711 max_load_reduction=42678 max_load_reduction_pct=0.868286 moved_to_echo=24673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[502, 0, 0, 65, 319, 5196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 91, 81, 0, 0, 0, 0, 0, 1404, 550, 188, 0, 0, 0, 0, 0, 0, 65, 1043, 0, 0, 63, 0, 0, 2604, 2232, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 30, 65, 0, 0, 5900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 2905, 311, 3091, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 1859, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 376, 5740, 0, 0, 0, 0, 0, 0, 0, 0, 4744, 0, 0, 0, 614, 247, 0, 176, 438, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6082, 6323, 6007, 6138, 6365, 5837, 6181, 6219] avg=6144.0000 max=6365 max_rank=4 min=5837 min_rank=5 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[582, 0, 8, 91, 340, 5069, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 69, 113, 0, 0, 0, 0, 1, 1384, 541, 190, 0, 0, 0, 0, 0, 0, 91, 1039, 5, 4, 91, 5, 0, 2576, 2232, 0, 0, 0, 0, 0, 0, 161, 0, 0, 3, 6, 91, 0, 0, 5896, 0, 0, 0, 0, 0, 0, 0, 3, 0, 148, 0, 0, 0, 2850, 285, 3056, 0, 0, 0, 0, 0, 0, 0, 15, 4656, 1822, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 91, 0, 0, 361, 5633, 0, 0, 0, 0, 0, 0, 0, 0, 4299, 8, 0, 3, 588, 358, 3, 163, 433, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 6071, 6043, 6157, 6342, 6505, 6088, 5855] avg=6144.0000 max=6505 max_rank=5 min=5855 min_rank=7 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=24790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[508, 0, 0, 74, 313, 5186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 76, 98, 0, 0, 0, 0, 0, 1399, 549, 188, 0, 0, 0, 0, 0, 0, 73, 1296, 0, 0, 59, 0, 0, 2600, 2233, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 4, 74, 0, 0, 5899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 2233, 272, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 4397, 1855, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 421, 5719, 0, 0, 0, 0, 0, 0, 0, 0, 4936, 0, 0, 0, 621, 243, 0, 197, 437, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6090, 6261, 6139, 5672, 6261, 6214, 6434] avg=6144.0000 max=6434 max_rank=7 min=5672 min_rank=4 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[546, 0, 0, 69, 330, 5278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 79, 85, 0, 0, 0, 0, 0, 1395, 558, 187, 0, 0, 0, 0, 0, 0, 69, 924, 0, 0, 69, 0, 0, 2588, 2235, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 59, 69, 0, 0, 5904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 3497, 313, 3072, 0, 0, 0, 0, 0, 0, 0, 7, 4154, 1848, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 402, 5703, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 606, 315, 4, 169, 445, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6223, 6200, 5885, 6168, 6974, 6026, 6174, 5502] avg=6144.0000 max=6974 max_rank=4 min=5502 min_rank=7 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=24676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[573, 0, 0, 96, 350, 5082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4185, 57, 107, 0, 0, 0, 0, 0, 1383, 537, 188, 0, 0, 0, 0, 0, 0, 94, 1249, 0, 0, 88, 0, 0, 2591, 2237, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 2, 96, 0, 0, 5914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 2268, 283, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 4189, 1848, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 326, 5653, 0, 0, 0, 0, 0, 0, 0, 0, 4718, 0, 0, 0, 545, 390, 4, 151, 428, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6101, 6457, 6259, 6179, 5801, 6044, 6075, 6236] avg=6144.0000 max=6457 max_rank=1 min=5801 min_rank=4 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[576, 0, 0, 82, 346, 5199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 111, 90, 0, 0, 0, 0, 0, 1383, 550, 188, 0, 0, 0, 0, 0, 0, 81, 1262, 0, 0, 77, 0, 0, 2593, 2226, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 4, 82, 0, 0, 5879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 2620, 343, 3078, 0, 0, 0, 0, 0, 0, 0, 0, 4236, 1851, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 339, 5654, 0, 0, 0, 0, 0, 0, 0, 0, 4565, 0, 0, 0, 659, 260, 0, 184, 438, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6203, 6225, 6239, 6097, 6113, 6094, 6075, 6106] avg=6144.0000 max=6239 max_rank=2 min=6075 min_rank=6 imbalance_ratio=1.015462 max_load_reduction=42913 max_load_reduction_pct=0.873067 moved_to_echo=24388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[582, 0, 8, 91, 340, 5069, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 69, 113, 0, 0, 0, 0, 1, 1384, 541, 190, 0, 0, 0, 0, 0, 0, 91, 1039, 5, 4, 91, 5, 0, 2576, 2232, 0, 0, 0, 0, 0, 0, 161, 0, 0, 3, 6, 91, 0, 0, 5896, 0, 0, 0, 0, 0, 0, 0, 3, 0, 148, 0, 0, 0, 2850, 285, 3056, 0, 0, 0, 0, 0, 0, 0, 15, 4656, 1822, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 91, 0, 0, 361, 5633, 0, 0, 0, 0, 0, 0, 0, 0, 4299, 8, 0, 3, 588, 358, 3, 163, 433, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 6071, 6043, 6157, 6342, 6505, 6088, 5855] avg=6144.0000 max=6505 max_rank=5 min=5855 min_rank=7 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=24790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 640, 9472, 128, 128, 640, 128, 0, 20864, 17920, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[30720, 640, 896, 0, 0, 0, 0, 128, 11264, 4480, 1536, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 896, 0, 0, 0, 21376, 2432, 24704, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[37120, 128, 0, 128, 4864, 2432, 128, 1536, 3584, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 34304, 14848, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 768, 0, 0, 3072, 45568, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1280, 0, 0, 128, 128, 768, 0, 0, 47232, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 0, 128, 768, 2688, 41472, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1280, 0, 0, 128, 128, 768, 0, 0, 47232, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[759, 441, 0, 0, 0, 0, 5286, 0, 0, 367, 0, 0, 0, 0, 16, 0, 0, 191, 0, 0, 0, 947, 0, 0, 0, 0, 7747, 237, 206, 0, 42, 7416, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 5654, 99, 7856, 98, 7432, 0, 3265, 0, 0, 423, 0, 95, 0, 0, 0, 0, 0, 0, 44, 0, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[794, 430, 0, 0, 0, 0, 4754, 0, 2, 462, 2, 0, 5, 0, 17, 0, 0, 183, 0, 0, 0, 955, 0, 2, 0, 0, 7747, 247, 242, 5, 54, 7438, 7, 4, 6, 0, 7, 0, 0, 497, 0, 3, 5342, 123, 7855, 91, 7410, 3, 3844, 0, 1, 396, 0, 91, 0, 0, 0, 0, 0, 8, 55, 0, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[840, 331, 0, 0, 0, 0, 4965, 0, 0, 400, 0, 0, 0, 0, 2, 0, 0, 145, 0, 0, 0, 1042, 0, 0, 0, 0, 7808, 237, 213, 0, 60, 7525, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 5422, 136, 7876, 74, 7441, 0, 3533, 0, 0, 404, 0, 71, 0, 0, 0, 0, 0, 0, 60, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[805, 317, 0, 0, 0, 0, 4836, 0, 0, 476, 0, 0, 0, 0, 16, 0, 0, 150, 0, 0, 0, 1049, 0, 0, 0, 0, 7796, 230, 214, 0, 63, 7558, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 5653, 132, 7807, 75, 7389, 0, 3411, 0, 0, 392, 0, 75, 0, 0, 0, 0, 0, 0, 63, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[772, 319, 0, 0, 0, 0, 4931, 0, 0, 476, 0, 0, 0, 0, 14, 0, 0, 128, 0, 0, 0, 1040, 0, 0, 0, 0, 7810, 241, 339, 0, 60, 7590, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 5785, 152, 7798, 77, 7432, 0, 3037, 0, 0, 395, 0, 63, 0, 0, 0, 0, 0, 0, 60, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[780, 347, 0, 0, 0, 0, 5143, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 1027, 0, 0, 0, 0, 7764, 263, 170, 0, 90, 7520, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 5542, 203, 7833, 85, 7290, 0, 3395, 0, 0, 391, 0, 82, 0, 0, 0, 0, 0, 0, 90, 0, 0, 44] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[785, 436, 0, 0, 0, 0, 5191, 0, 0, 428, 0, 0, 0, 0, 4, 0, 0, 180, 0, 0, 0, 936, 0, 0, 0, 0, 7764, 243, 165, 0, 66, 7429, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 5719, 153, 7845, 90, 7338, 0, 3298, 0, 0, 397, 0, 90, 0, 0, 0, 0, 0, 0, 66, 0, 0, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[840, 331, 0, 0, 0, 0, 4965, 0, 0, 400, 0, 0, 0, 0, 2, 0, 0, 145, 0, 0, 0, 1042, 0, 0, 0, 0, 7808, 237, 213, 0, 60, 7525, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 5422, 136, 7876, 74, 7441, 0, 3533, 0, 0, 404, 0, 71, 0, 0, 0, 0, 0, 0, 60, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[772, 319, 0, 0, 0, 0, 4931, 0, 0, 476, 0, 0, 0, 0, 14, 0, 0, 128, 0, 0, 0, 1040, 0, 0, 0, 0, 7810, 241, 339, 0, 60, 7590, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 5785, 152, 7798, 77, 7432, 0, 3037, 0, 0, 395, 0, 63, 0, 0, 0, 0, 0, 0, 60, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[759, 441, 0, 0, 0, 0, 5286, 0, 0, 367, 0, 0, 0, 0, 16, 0, 0, 191, 0, 0, 0, 947, 0, 0, 0, 0, 7747, 237, 206, 0, 42, 7416, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 5654, 99, 7856, 98, 7432, 0, 3265, 0, 0, 423, 0, 95, 0, 0, 0, 0, 0, 0, 44, 0, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[816, 365, 0, 0, 0, 0, 4760, 0, 0, 466, 0, 0, 0, 0, 20, 0, 0, 138, 0, 0, 0, 1038, 0, 0, 0, 0, 7793, 250, 264, 0, 53, 7566, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 5253, 132, 7876, 69, 7448, 0, 3762, 0, 0, 389, 0, 69, 0, 0, 0, 0, 0, 0, 53, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[780, 347, 0, 0, 0, 0, 5143, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 1027, 0, 0, 0, 0, 7764, 263, 170, 0, 90, 7520, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 5542, 203, 7833, 85, 7290, 0, 3395, 0, 0, 391, 0, 82, 0, 0, 0, 0, 0, 0, 90, 0, 0, 44] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[759, 441, 0, 0, 0, 0, 5280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 16, 0, 3846, 1833, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 947, 0, 0, 5002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 206, 0, 42, 5546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 3901, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 5654, 99, 0, 98, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3265, 0, 0, 423, 0, 95, 0, 0, 1870, 325, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 72, 6023, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6480, 6062, 6140, 6031, 6089, 6227, 5984, 6139] avg=6144.0000 max=6480 max_rank=0 min=5984 min_rank=6 imbalance_ratio=1.054688 max_load_reduction=42672 max_load_reduction_pct=0.868164 moved_to_echo=24439 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[794, 430, 0, 0, 0, 0, 4754, 0, 2, 462, 2, 0, 5, 0, 17, 0, 0, 183, 0, 0, 0, 955, 0, 2, 0, 0, 7747, 247, 242, 5, 54, 7438, 7, 4, 6, 0, 7, 0, 0, 497, 0, 3, 5342, 123, 7855, 91, 7410, 3, 3844, 0, 1, 396, 0, 91, 0, 0, 0, 0, 0, 8, 55, 0, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[805, 317, 0, 0, 0, 0, 4830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 16, 0, 3871, 1822, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 1049, 0, 0, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 214, 0, 63, 5652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 3925, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 5653, 132, 0, 75, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3411, 0, 0, 392, 0, 75, 0, 0, 1906, 323, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 69, 5985, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5952, 6185, 6172, 6159, 6220, 6234, 6113, 6117] avg=6144.0000 max=6234 max_rank=5 min=5952 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=42918 max_load_reduction_pct=0.873169 moved_to_echo=24684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[840, 331, 0, 0, 0, 0, 4959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 2, 0, 3877, 1838, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 1042, 0, 0, 5008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 213, 0, 60, 5628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 3931, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 5422, 136, 0, 74, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3533, 0, 0, 404, 0, 71, 0, 0, 1897, 325, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 92, 6038, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6117, 6195, 6138, 6137, 6009, 6236, 6190] avg=6144.0000 max=6236 max_rank=6 min=6009 min_rank=5 imbalance_ratio=1.014974 max_load_reduction=42916 max_load_reduction_pct=0.873128 moved_to_echo=24572 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[794, 430, 0, 0, 0, 0, 4745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 462, 2, 0, 5, 0, 17, 0, 3847, 1833, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 955, 0, 2, 4993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 242, 5, 54, 5558, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 6, 0, 7, 0, 0, 497, 3900, 1726, 0, 0, 0, 0, 0, 0, 0, 3, 5342, 123, 0, 91, 364, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 0, 1, 396, 0, 91, 0, 0, 1880, 327, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8, 55, 0, 0, 70, 6022, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5969, 6168, 6133, 6106, 6147, 5926, 6548, 6155] avg=6144.0000 max=6548 max_rank=6 min=5926 min_rank=5 imbalance_ratio=1.065755 max_load_reduction=42604 max_load_reduction_pct=0.866781 moved_to_echo=24776 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[785, 436, 0, 0, 0, 0, 5185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 4, 0, 3855, 1830, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 936, 0, 0, 4939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 165, 0, 66, 5556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 3909, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 5719, 153, 0, 90, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3298, 0, 0, 397, 0, 90, 0, 0, 1873, 320, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 60, 6015, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6406, 6117, 6055, 6030, 6085, 6334, 5984, 6141] avg=6144.0000 max=6406 max_rank=0 min=5984 min_rank=6 imbalance_ratio=1.042643 max_load_reduction=42746 max_load_reduction_pct=0.869670 moved_to_echo=24544 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[816, 365, 0, 0, 0, 0, 4754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 0, 20, 0, 3870, 1838, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 1038, 0, 0, 5013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 264, 0, 53, 5658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 3923, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 5253, 132, 0, 69, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 389, 0, 69, 0, 0, 1908, 325, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 66, 6038, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5935, 6194, 6189, 6225, 6161, 5832, 6459, 6157] avg=6144.0000 max=6459 max_rank=6 min=5832 min_rank=5 imbalance_ratio=1.051270 max_load_reduction=42693 max_load_reduction_pct=0.868591 moved_to_echo=24609 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[772, 319, 0, 0, 0, 0, 4925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 14, 0, 3877, 1819, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 1040, 0, 0, 5002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 339, 0, 60, 5676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 3933, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 5785, 152, 0, 77, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3037, 0, 0, 395, 0, 63, 0, 0, 1914, 325, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 75, 5979, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6016, 6186, 6170, 6316, 6220, 6390, 5740, 6114] avg=6144.0000 max=6390 max_rank=5 min=5740 min_rank=6 imbalance_ratio=1.040039 max_load_reduction=42762 max_load_reduction_pct=0.869995 moved_to_echo=24464 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[780, 347, 0, 0, 0, 0, 5137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 3855, 1828, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 1027, 0, 0, 4907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 170, 0, 90, 5624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 3909, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 5542, 203, 0, 85, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3395, 0, 0, 391, 0, 82, 0, 0, 1896, 318, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 44, 6005, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 6123, 6098, 6147, 6093, 6200, 6088, 6139] avg=6144.0000 max=6264 max_rank=0 min=6088 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=42888 max_load_reduction_pct=0.872559 moved_to_echo=24520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[759, 441, 0, 0, 0, 0, 5280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 16, 0, 3846, 1833, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 947, 0, 0, 5002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 206, 0, 42, 5546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 3901, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 5654, 99, 0, 98, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3265, 0, 0, 423, 0, 95, 0, 0, 1870, 325, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 72, 6023, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6480, 6062, 6140, 6031, 6089, 6227, 5984, 6139] avg=6144.0000 max=6480 max_rank=0 min=5984 min_rank=6 imbalance_ratio=1.054688 max_load_reduction=42672 max_load_reduction_pct=0.868164 moved_to_echo=24439 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[840, 331, 0, 0, 0, 0, 4959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 2, 0, 3877, 1838, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 1042, 0, 0, 5008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 213, 0, 60, 5628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 3931, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 5422, 136, 0, 74, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3533, 0, 0, 404, 0, 71, 0, 0, 1897, 325, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 92, 6038, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6117, 6195, 6138, 6137, 6009, 6236, 6190] avg=6144.0000 max=6236 max_rank=6 min=6009 min_rank=5 imbalance_ratio=1.014974 max_load_reduction=42916 max_load_reduction_pct=0.873128 moved_to_echo=24572 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[816, 365, 0, 0, 0, 0, 4754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 0, 20, 0, 3870, 1838, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 1038, 0, 0, 5013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 264, 0, 53, 5658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 3923, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 5253, 132, 0, 69, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 389, 0, 69, 0, 0, 1908, 325, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 66, 6038, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5935, 6194, 6189, 6225, 6161, 5832, 6459, 6157] avg=6144.0000 max=6459 max_rank=6 min=5832 min_rank=5 imbalance_ratio=1.051270 max_load_reduction=42693 max_load_reduction_pct=0.868591 moved_to_echo=24609 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[794, 430, 0, 0, 0, 0, 4745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 462, 2, 0, 5, 0, 17, 0, 3847, 1833, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 955, 0, 2, 4993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 242, 5, 54, 5558, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 6, 0, 7, 0, 0, 497, 3900, 1726, 0, 0, 0, 0, 0, 0, 0, 3, 5342, 123, 0, 91, 364, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 0, 1, 396, 0, 91, 0, 0, 1880, 327, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8, 55, 0, 0, 70, 6022, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5969, 6168, 6133, 6106, 6147, 5926, 6548, 6155] avg=6144.0000 max=6548 max_rank=6 min=5926 min_rank=5 imbalance_ratio=1.065755 max_load_reduction=42604 max_load_reduction_pct=0.866781 moved_to_echo=24776 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 3584, 128, 0, 128, 0, 128, 0, 30976, 14720, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 0, 0, 4096, 31360, 13824, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 3584, 128, 0, 128, 0, 128, 0, 30976, 14720, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[27648, 0, 128, 3200, 0, 640, 0, 0, 15232, 2688, 128, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[6400, 3072, 0, 0, 0, 0, 39936, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 1280, 0, 0, 0, 8064, 0, 128, 39936, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 44416, 1152, 0, 768, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 2048, 1920, 128, 512, 44928, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 512, 0, 0, 640, 48128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 247, 0, 0, 0, 1084, 0, 0, 0, 63, 0, 7402, 0, 0, 7901, 75, 0, 0, 0, 0, 1025, 0, 0, 0, 0, 0, 8117, 0, 0, 8151, 0, 0, 0, 75, 0, 0, 8129, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 310, 0, 0, 657, 5707, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 308, 0, 0, 0, 1058, 0, 0, 0, 69, 0, 7483, 0, 0, 7780, 69, 0, 0, 0, 0, 2297, 0, 0, 0, 0, 0, 8123, 0, 0, 8112, 0, 0, 0, 69, 0, 0, 8139, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 339, 0, 0, 545, 4532, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 408, 0, 0, 0, 1121, 0, 0, 0, 48, 0, 7424, 0, 0, 7624, 90, 0, 0, 0, 0, 803, 0, 0, 0, 0, 0, 8102, 0, 0, 8180, 0, 0, 0, 90, 0, 0, 8126, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 494, 5991, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 235, 0, 0, 0, 1127, 0, 0, 0, 77, 0, 7449, 0, 0, 7771, 82, 0, 0, 0, 0, 1036, 0, 0, 0, 0, 0, 8110, 0, 0, 8140, 0, 0, 0, 82, 0, 0, 8102, 0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 336, 0, 0, 544, 5772, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 316, 0, 0, 0, 1196, 0, 0, 0, 30, 0, 7397, 0, 0, 7799, 95, 0, 0, 0, 0, 930, 0, 0, 0, 0, 0, 8097, 0, 0, 8186, 0, 0, 0, 94, 0, 0, 8150, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 428, 0, 0, 543, 5691, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 247, 0, 0, 0, 1084, 0, 0, 0, 63, 0, 7402, 0, 0, 7901, 75, 0, 0, 0, 0, 1025, 0, 0, 0, 0, 0, 8117, 0, 0, 8151, 0, 0, 0, 75, 0, 0, 8129, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 310, 0, 0, 657, 5707, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 383, 0, 0, 0, 1024, 0, 0, 0, 63, 0, 7481, 0, 0, 7693, 65, 0, 0, 0, 0, 974, 0, 0, 0, 0, 0, 8129, 0, 0, 8151, 0, 0, 0, 63, 0, 0, 8132, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 302, 0, 0, 506, 5893, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 315, 0, 0, 0, 1165, 0, 0, 0, 40, 0, 7421, 0, 0, 7850, 74, 0, 0, 0, 0, 735, 0, 0, 0, 0, 0, 8121, 0, 0, 8191, 0, 0, 0, 71, 0, 0, 8132, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 320, 0, 0, 543, 5966, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 308, 0, 0, 0, 1058, 0, 0, 0, 69, 0, 7483, 0, 0, 7780, 69, 0, 0, 0, 0, 2297, 0, 0, 0, 0, 0, 8123, 0, 0, 8112, 0, 0, 0, 69, 0, 0, 8139, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 339, 0, 0, 545, 4532, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[1, 0, 322, 0, 0, 0, 1012, 0, 0, 0, 91, 0, 7408, 0, 8, 7701, 94, 0, 0, 0, 2, 1409, 0, 6, 0, 0, 0, 8086, 0, 8, 8112, 0, 0, 0, 91, 0, 0, 8129, 6, 3, 1, 0, 214, 0, 2, 9, 7, 0, 0, 1, 0, 0, 0, 4, 4, 0, 1, 389, 0, 1, 739, 5290, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 383, 0, 0, 0, 1024, 0, 0, 0, 63, 0, 7481, 0, 0, 7693, 65, 0, 0, 0, 0, 974, 0, 0, 0, 0, 0, 8129, 0, 0, 8151, 0, 0, 0, 63, 0, 0, 8132, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 302, 0, 0, 506, 5893, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 235, 0, 0, 0, 1127, 0, 0, 0, 77, 0, 7449, 0, 0, 7771, 82, 0, 0, 0, 0, 1036, 0, 0, 0, 0, 0, 8110, 0, 0, 8140, 0, 0, 0, 82, 0, 0, 8102, 0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 336, 0, 0, 544, 5772, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 316, 0, 0, 0, 1196, 0, 1964, 1343, 1026, 392, 0, 0, 0, 0, 0, 0, 30, 0, 6054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 930, 0, 0, 3885, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 6079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 3914, 2033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 543, 5299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6237, 6084, 5955, 6133, 6173, 6141, 6159, 6270] avg=6144.0000 max=6270 max_rank=7 min=5955 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=42882 max_load_reduction_pct=0.872437 moved_to_echo=24743 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 247, 0, 0, 0, 1084, 0, 1969, 1344, 1023, 394, 0, 0, 0, 0, 0, 0, 63, 0, 6058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 1025, 0, 0, 3936, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 6064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 3965, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 657, 5313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6061, 6121, 6078, 6148, 6139, 6196, 6129, 6280] avg=6144.0000 max=6280 max_rank=7 min=6061 min_rank=0 imbalance_ratio=1.022135 max_load_reduction=42872 max_load_reduction_pct=0.872233 moved_to_echo=24744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 308, 0, 0, 0, 1058, 0, 1970, 1359, 1025, 312, 0, 0, 0, 0, 0, 0, 69, 0, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 2297, 0, 0, 3877, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 6071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 3903, 2015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 6097, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 0, 545, 4220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6032, 6193, 7286, 6153, 6140, 6119, 6125, 5104] avg=6144.0000 max=7286 max_rank=2 min=5104 min_rank=7 imbalance_ratio=1.185872 max_load_reduction=41866 max_load_reduction_pct=0.851766 moved_to_echo=23488 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 408, 0, 0, 0, 1121, 0, 1965, 1348, 1023, 413, 0, 0, 0, 0, 0, 0, 48, 0, 6076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 803, 0, 0, 3798, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 6061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 3826, 2031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6149, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 494, 5578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 6124, 5733, 6137, 6151, 6100, 6149, 6480] avg=6144.0000 max=6480 max_rank=7 min=5733 min_rank=2 imbalance_ratio=1.054688 max_load_reduction=42672 max_load_reduction_pct=0.868164 moved_to_echo=24880 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 315, 0, 0, 0, 1165, 0, 1970, 1348, 1024, 411, 0, 0, 0, 0, 0, 0, 40, 0, 6073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 735, 0, 0, 3912, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 3938, 2035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 543, 5555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6233, 6113, 5763, 6151, 6137, 6180, 6157, 6418] avg=6144.0000 max=6418 max_rank=7 min=5763 min_rank=2 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 322, 0, 0, 0, 1012, 0, 1964, 1349, 1026, 370, 0, 0, 0, 0, 0, 0, 91, 0, 6059, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 2, 1409, 0, 6, 3838, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6122, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 6057, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 214, 0, 2, 9, 7, 0, 3863, 2015, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 4, 0, 6097, 0, 0, 0, 0, 0, 0, 0, 1, 389, 0, 1, 739, 4920, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6044, 6158, 6395, 6130, 6157, 6111, 6106, 6051] avg=6144.0000 max=6395 max_rank=2 min=6044 min_rank=0 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24425 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 383, 0, 0, 0, 1024, 0, 1972, 1358, 1024, 406, 0, 0, 0, 0, 0, 0, 63, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 974, 0, 0, 3833, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 3860, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 506, 5487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6167, 6186, 5914, 6157, 6129, 6099, 6205, 6295] avg=6144.0000 max=6295 max_rank=7 min=5914 min_rank=2 imbalance_ratio=1.024577 max_load_reduction=42857 max_load_reduction_pct=0.871928 moved_to_echo=24728 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 316, 0, 0, 0, 1196, 0, 1964, 1343, 1026, 392, 0, 0, 0, 0, 0, 0, 30, 0, 6054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 930, 0, 0, 3885, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 6079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 3914, 2033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 543, 5299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6237, 6084, 5955, 6133, 6173, 6141, 6159, 6270] avg=6144.0000 max=6270 max_rank=7 min=5955 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=42882 max_load_reduction_pct=0.872437 moved_to_echo=24743 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 235, 0, 0, 0, 1127, 0, 1967, 1353, 1020, 398, 0, 0, 0, 0, 0, 0, 77, 0, 6096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 1036, 0, 0, 3872, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 6044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 3899, 2022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6118, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 0, 544, 5374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6173, 6028, 6143, 6126, 6206, 6122, 6254] avg=6144.0000 max=6254 max_rank=7 min=6028 min_rank=2 imbalance_ratio=1.017904 max_load_reduction=42898 max_load_reduction_pct=0.872762 moved_to_echo=24708 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 315, 0, 0, 0, 1165, 0, 1970, 1348, 1024, 411, 0, 0, 0, 0, 0, 0, 40, 0, 6073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 735, 0, 0, 3912, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 3938, 2035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 543, 5555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6233, 6113, 5763, 6151, 6137, 6180, 6157, 6418] avg=6144.0000 max=6418 max_rank=7 min=5763 min_rank=2 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 308, 0, 0, 0, 1058, 0, 1970, 1359, 1025, 312, 0, 0, 0, 0, 0, 0, 69, 0, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 2297, 0, 0, 3877, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 6071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 3903, 2015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 6097, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 0, 545, 4220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6032, 6193, 7286, 6153, 6140, 6119, 6125, 5104] avg=6144.0000 max=7286 max_rank=2 min=5104 min_rank=7 imbalance_ratio=1.185872 max_load_reduction=41866 max_load_reduction_pct=0.851766 moved_to_echo=23488 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 322, 0, 0, 0, 1012, 0, 1964, 1349, 1026, 370, 0, 0, 0, 0, 0, 0, 91, 0, 6059, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 2, 1409, 0, 6, 3838, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6122, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 6057, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 214, 0, 2, 9, 7, 0, 3863, 2015, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 4, 0, 6097, 0, 0, 0, 0, 0, 0, 0, 1, 389, 0, 1, 739, 4920, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6044, 6158, 6395, 6130, 6157, 6111, 6106, 6051] avg=6144.0000 max=6395 max_rank=2 min=6044 min_rank=0 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24425 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 2560, 0, 0, 0, 8832, 0, 15744, 10880, 8192, 3200, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 0, 0, 0, 128, 9216, 0, 128, 30976, 8448, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 640, 0, 0, 48512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 1792, 0, 128, 128, 128, 0, 31232, 16256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 512, 0, 48768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 1792, 0, 128, 128, 128, 0, 31232, 16256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 49024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2944, 0, 128, 4608, 41856, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 207, 0, 1732, 0, 150, 7986, 8192, 0, 0, 0, 0, 0, 782, 0, 7710, 0, 63, 0, 7887, 0, 0, 0, 0, 730, 7160, 75, 0, 0, 0, 0, 109, 468, 0, 0, 0, 3817, 0, 65, 0, 0, 0, 0, 0, 0, 63, 1, 75, 0, 1115, 344] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 450, 0, 1477, 0, 191, 7837, 8192, 0, 0, 0, 0, 0, 940, 0, 7717, 0, 42, 0, 7921, 0, 0, 0, 0, 497, 7154, 96, 0, 0, 0, 0, 102, 306, 0, 0, 0, 4187, 0, 53, 0, 0, 0, 0, 0, 3, 42, 0, 95, 0, 1106, 337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 398, 0, 1737, 0, 180, 7855, 8192, 0, 0, 0, 0, 0, 913, 0, 7733, 0, 66, 0, 7865, 0, 0, 0, 0, 556, 7088, 90, 0, 0, 0, 0, 111, 424, 0, 0, 0, 3623, 0, 90, 0, 0, 0, 0, 0, 49, 66, 2, 98, 0, 1245, 303] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 0, 229, 0, 1864, 0, 128, 8016, 8192, 0, 0, 0, 0, 0, 847, 0, 7731, 0, 60, 0, 7894, 0, 0, 0, 0, 697, 7150, 65, 0, 0, 0, 0, 57, 337, 0, 0, 0, 3900, 0, 71, 0, 0, 0, 0, 0, 0, 60, 17, 63, 0, 999, 337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 276, 0, 1819, 0, 164, 7961, 8192, 0, 0, 0, 0, 0, 909, 0, 7659, 0, 90, 0, 7862, 0, 0, 0, 0, 638, 7052, 82, 0, 0, 0, 0, 141, 454, 0, 0, 0, 3659, 0, 110, 0, 0, 0, 0, 0, 2, 91, 3, 83, 0, 1144, 361] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 233, 0, 1824, 0, 138, 7976, 8192, 0, 0, 0, 0, 0, 787, 0, 7725, 0, 53, 0, 7930, 0, 0, 0, 0, 899, 7156, 69, 0, 0, 0, 0, 105, 300, 0, 0, 0, 3790, 0, 64, 0, 0, 0, 0, 0, 0, 54, 0, 69, 0, 1010, 345] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 8, 0, 1, 4, 0, 0, 443, 3, 0, 0, 317, 0, 2052, 1, 193, 7856, 8176, 0, 5, 0, 0, 0, 735, 0, 7686, 1, 55, 0, 7872, 2, 4, 0, 1, 761, 7152, 91, 0, 3, 0, 6, 127, 438, 0, 0, 0, 3393, 0, 70, 0, 7, 0, 0, 5, 0, 70, 2, 91, 8, 1159, 346] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 207, 0, 1732, 0, 150, 7986, 8192, 0, 0, 0, 0, 0, 782, 0, 7710, 0, 63, 0, 7887, 0, 0, 0, 0, 730, 7160, 75, 0, 0, 0, 0, 109, 468, 0, 0, 0, 3817, 0, 65, 0, 0, 0, 0, 0, 0, 63, 1, 75, 0, 1115, 344] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 0, 484, 0, 1745, 0, 145, 7973, 8192, 0, 0, 0, 0, 0, 1032, 0, 7710, 0, 60, 0, 7877, 0, 0, 0, 0, 407, 7059, 74, 0, 0, 0, 0, 93, 526, 0, 0, 0, 3600, 0, 75, 0, 0, 0, 0, 0, 10, 60, 1, 72, 0, 1122, 349] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 276, 0, 1819, 0, 164, 7961, 8192, 0, 0, 0, 0, 0, 909, 0, 7659, 0, 90, 0, 7862, 0, 0, 0, 0, 638, 7052, 82, 0, 0, 0, 0, 141, 454, 0, 0, 0, 3659, 0, 110, 0, 0, 0, 0, 0, 2, 91, 3, 83, 0, 1144, 361] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 450, 0, 4324, 1062, 0, 0, 0, 0, 0, 0, 1477, 0, 191, 4152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 5221, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 5440, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 306, 0, 0, 3597, 2050, 0, 0, 0, 0, 0, 0, 0, 4187, 0, 53, 0, 0, 0, 0, 1714, 616, 0, 0, 0, 0, 0, 0, 0, 3, 42, 0, 95, 0, 1106, 337, 2623, 1880, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6243, 5820, 6203, 6033, 6055, 6570, 6086] avg=6144.0000 max=6570 max_rank=6 min=5820 min_rank=2 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=24744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 398, 0, 1737, 0, 180, 7855, 8192, 0, 0, 0, 0, 0, 913, 0, 7733, 0, 66, 0, 7865, 0, 0, 0, 0, 556, 7088, 90, 0, 0, 0, 0, 111, 424, 0, 0, 0, 3623, 0, 90, 0, 0, 0, 0, 0, 49, 66, 2, 98, 0, 1245, 303] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 450, 0, 1477, 0, 191, 7837, 8192, 0, 0, 0, 0, 0, 940, 0, 7717, 0, 42, 0, 7921, 0, 0, 0, 0, 497, 7154, 96, 0, 0, 0, 0, 102, 306, 0, 0, 0, 4187, 0, 53, 0, 0, 0, 0, 0, 3, 42, 0, 95, 0, 1106, 337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 8, 0, 1, 4, 0, 0, 443, 3, 0, 0, 317, 0, 2052, 1, 193, 7856, 8176, 0, 5, 0, 0, 0, 735, 0, 7686, 1, 55, 0, 7872, 2, 4, 0, 1, 761, 7152, 91, 0, 3, 0, 6, 127, 438, 0, 0, 0, 3393, 0, 70, 0, 7, 0, 0, 5, 0, 70, 2, 91, 8, 1159, 346] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 0, 229, 0, 4309, 1087, 0, 0, 0, 0, 0, 0, 1864, 0, 128, 4246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 0, 5230, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 5437, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 337, 0, 0, 3585, 2049, 0, 0, 0, 0, 0, 0, 0, 3900, 0, 71, 0, 0, 0, 0, 1713, 617, 0, 0, 0, 0, 0, 0, 0, 0, 60, 17, 63, 0, 999, 337, 2683, 1884, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6063, 6238, 6137, 6199, 6028, 6301, 6043] avg=6144.0000 max=6301 max_rank=6 min=6028 min_rank=5 imbalance_ratio=1.025553 max_load_reduction=42851 max_load_reduction_pct=0.871806 moved_to_echo=24571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 207, 0, 4305, 1083, 0, 0, 0, 0, 0, 0, 1732, 0, 150, 4230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 5217, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 5445, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 468, 0, 0, 3582, 2049, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 65, 0, 0, 0, 0, 1715, 615, 0, 0, 0, 0, 0, 0, 0, 0, 63, 1, 75, 0, 1115, 344, 2673, 1878, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6016, 6112, 6062, 6250, 6208, 6212, 6149] avg=6144.0000 max=6250 max_rank=4 min=6016 min_rank=1 imbalance_ratio=1.017253 max_load_reduction=42902 max_load_reduction_pct=0.872843 moved_to_echo=24819 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 233, 0, 4329, 1081, 0, 0, 0, 0, 0, 0, 1824, 0, 138, 4225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 5227, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 5442, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 300, 0, 0, 3601, 2049, 0, 0, 0, 0, 0, 0, 0, 3790, 0, 64, 0, 0, 0, 0, 1714, 616, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 69, 0, 1010, 345, 2670, 1882, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6076, 6187, 6067, 6410, 6055, 6184, 6030] avg=6144.0000 max=6410 max_rank=4 min=6030 min_rank=7 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 276, 0, 4293, 1079, 0, 0, 0, 0, 0, 0, 1819, 0, 164, 4217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 0, 5182, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 5363, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 454, 0, 0, 3569, 2049, 0, 0, 0, 0, 0, 0, 0, 3659, 0, 110, 0, 0, 0, 0, 1689, 611, 0, 0, 0, 0, 0, 0, 0, 2, 91, 3, 83, 0, 1144, 361, 2665, 1866, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6048, 6200, 6181, 6083, 6213, 6069, 6215] avg=6144.0000 max=6215 max_rank=7 min=6048 min_rank=1 imbalance_ratio=1.011556 max_load_reduction=42937 max_load_reduction_pct=0.873556 moved_to_echo=24580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 398, 0, 4293, 1065, 0, 0, 0, 0, 0, 0, 1737, 0, 180, 4161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 0, 5232, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 5390, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 424, 0, 0, 3572, 2049, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 90, 0, 0, 0, 0, 1698, 617, 0, 0, 0, 0, 0, 0, 0, 49, 66, 2, 98, 0, 1245, 303, 2629, 1884, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6224, 6078, 6211, 6036, 6156, 6028, 6276] avg=6144.0000 max=6276 max_rank=7 min=6028 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=42876 max_load_reduction_pct=0.872314 moved_to_echo=24496 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 0, 229, 0, 4309, 1087, 0, 0, 0, 0, 0, 0, 1864, 0, 128, 4246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 0, 5230, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 5437, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 337, 0, 0, 3585, 2049, 0, 0, 0, 0, 0, 0, 0, 3900, 0, 71, 0, 0, 0, 0, 1713, 617, 0, 0, 0, 0, 0, 0, 0, 0, 60, 17, 63, 0, 999, 337, 2683, 1884, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6063, 6238, 6137, 6199, 6028, 6301, 6043] avg=6144.0000 max=6301 max_rank=6 min=6028 min_rank=5 imbalance_ratio=1.025553 max_load_reduction=42851 max_load_reduction_pct=0.871806 moved_to_echo=24571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 0, 484, 0, 4301, 1081, 0, 0, 0, 0, 0, 0, 1745, 0, 145, 4223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 0, 5217, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 5368, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 526, 0, 0, 3576, 2049, 0, 0, 0, 0, 0, 0, 0, 3600, 0, 75, 0, 0, 0, 0, 1691, 615, 0, 0, 0, 0, 0, 0, 0, 10, 60, 1, 72, 0, 1122, 349, 2669, 1878, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6352, 6113, 6309, 5849, 6244, 5981, 6161] avg=6144.0000 max=6352 max_rank=1 min=5849 min_rank=4 imbalance_ratio=1.033854 max_load_reduction=42800 max_load_reduction_pct=0.870768 moved_to_echo=24235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 233, 0, 4329, 1081, 0, 0, 0, 0, 0, 0, 1824, 0, 138, 4225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 5227, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 5442, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 300, 0, 0, 3601, 2049, 0, 0, 0, 0, 0, 0, 0, 3790, 0, 64, 0, 0, 0, 0, 1714, 616, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 69, 0, 1010, 345, 2670, 1882, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6076, 6187, 6067, 6410, 6055, 6184, 6030] avg=6144.0000 max=6410 max_rank=4 min=6030 min_rank=7 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 450, 0, 4324, 1062, 0, 0, 0, 0, 0, 0, 1477, 0, 191, 4152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 5221, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 5440, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 306, 0, 0, 3597, 2050, 0, 0, 0, 0, 0, 0, 0, 4187, 0, 53, 0, 0, 0, 0, 1714, 616, 0, 0, 0, 0, 0, 0, 0, 3, 42, 0, 95, 0, 1106, 337, 2623, 1880, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6243, 5820, 6203, 6033, 6055, 6570, 6086] avg=6144.0000 max=6570 max_rank=6 min=5820 min_rank=2 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=24744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 8, 0, 1, 4, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 3, 0, 0, 317, 0, 4298, 1069, 0, 0, 0, 0, 0, 0, 2052, 1, 193, 4153, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 735, 0, 5191, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 1, 761, 5433, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 127, 438, 0, 0, 3574, 2045, 0, 0, 0, 0, 0, 0, 0, 3393, 0, 70, 0, 7, 0, 0, 1719, 618, 0, 0, 0, 0, 0, 0, 5, 0, 70, 2, 91, 8, 1159, 346, 2634, 1877, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6130, 6404, 5982, 6292, 6193, 5807, 6192] avg=6144.0000 max=6404 max_rank=2 min=5807 min_rank=6 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24484 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 276, 0, 4293, 1079, 0, 0, 0, 0, 0, 0, 1819, 0, 164, 4217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 0, 5182, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 5363, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 454, 0, 0, 3569, 2049, 0, 0, 0, 0, 0, 0, 0, 3659, 0, 110, 0, 0, 0, 0, 1689, 611, 0, 0, 0, 0, 0, 0, 0, 2, 91, 3, 83, 0, 1144, 361, 2665, 1866, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6048, 6200, 6181, 6083, 6213, 6069, 6215] avg=6144.0000 max=6215 max_rank=7 min=6048 min_rank=1 imbalance_ratio=1.011556 max_load_reduction=42937 max_load_reduction_pct=0.873556 moved_to_echo=24580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 896, 3328, 0, 0, 28672, 16512, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 512, 128, 768, 128, 8960, 2816, 21248, 15104, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 3584, 128, 0, 0, 2688, 0, 34560, 8704, 0, 0, 0, 0, 0, 0] @@ -8725,340 +8725,340 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 69, 460, 8185, 1020, 0, 0, 7979, 0, 0, 7349, 64, 63, 0, 0, 0, 0, 139, 0, 0, 0, 0, 137, 0, 7816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 6816, 0, 152, 23, 7341, 75, 0, 0, 0, 75, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 52, 456, 8192, 1007, 0, 0, 8004, 0, 0, 7344, 48, 60, 0, 0, 0, 0, 111, 0, 0, 0, 0, 158, 0, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 6795, 0, 158, 18, 7256, 63, 0, 0, 0, 63, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 179, 505, 8179, 958, 0, 0, 7959, 0, 0, 7227, 174, 42, 0, 0, 0, 0, 271, 0, 0, 0, 0, 106, 0, 7725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 6766, 0, 73, 107, 7428, 95, 0, 0, 0, 96, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 78, 434, 8192, 1053, 0, 0, 8001, 0, 0, 7295, 78, 53, 0, 0, 0, 0, 148, 0, 0, 0, 0, 142, 0, 7794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 6749, 0, 165, 55, 7063, 69, 0, 0, 0, 69, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 69, 460, 8185, 1020, 0, 0, 7979, 0, 0, 7349, 64, 63, 0, 0, 0, 0, 139, 0, 0, 0, 0, 137, 0, 7816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 6816, 0, 152, 23, 7341, 75, 0, 0, 0, 75, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 166, 452, 8145, 970, 0, 0, 7946, 0, 0, 7363, 161, 66, 0, 0, 0, 0, 284, 0, 0, 0, 0, 157, 0, 7736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 6621, 0, 80, 27, 7436, 90, 0, 0, 0, 90, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 7, 7, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 397, 9, 0, 0, 142, 496, 8134, 995, 1, 0, 7930, 6, 0, 7293, 140, 54, 0, 0, 0, 7, 248, 9, 0, 0, 0, 126, 5, 7737, 1, 8, 0, 0, 0, 0, 2, 1, 1, 1061, 6792, 1, 176, 77, 7087, 94, 0, 0, 9, 91, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 84, 467, 8186, 1040, 0, 0, 7986, 0, 0, 7325, 81, 60, 0, 0, 0, 0, 162, 0, 0, 0, 0, 149, 0, 7813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 6742, 0, 121, 33, 7438, 71, 0, 0, 0, 71, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 78, 434, 8192, 1053, 0, 0, 8001, 0, 0, 7295, 78, 53, 0, 0, 0, 0, 148, 0, 0, 0, 0, 142, 0, 7794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 6749, 0, 165, 55, 7063, 69, 0, 0, 0, 69, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 7, 7, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 397, 9, 0, 0, 142, 496, 8134, 995, 1, 0, 7930, 6, 0, 7293, 140, 54, 0, 0, 0, 7, 248, 9, 0, 0, 0, 126, 5, 7737, 1, 8, 0, 0, 0, 0, 2, 1, 1, 1061, 6792, 1, 176, 77, 7087, 94, 0, 0, 9, 91, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 75, 462, 8187, 1022, 0, 0, 7938, 0, 0, 7278, 72, 90, 0, 0, 0, 0, 165, 0, 0, 0, 0, 213, 0, 7773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 6755, 0, 136, 55, 7263, 82, 0, 0, 0, 82, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 52, 456, 8192, 1007, 0, 0, 8004, 0, 0, 7344, 48, 60, 0, 0, 0, 0, 111, 0, 0, 0, 0, 158, 0, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 6795, 0, 158, 18, 7256, 63, 0, 0, 0, 63, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 1624, 1458, 1314, 1330, 0, 0, 0, 0, 0, 0, 179, 505, 4566, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5913, 174, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 106, 3613, 1826, 363, 0, 0, 0, 0, 0, 0, 6101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 4945, 0, 73, 107, 0, 0, 0, 0, 0, 0, 0, 0, 6098, 95, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6105, 6208, 6129, 6179, 6101, 5999, 6289] avg=6144.0000 max=6289 max_rank=7 min=5999 min_rank=6 imbalance_ratio=1.023600 max_load_reduction=42863 max_load_reduction_pct=0.872050 moved_to_echo=24568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 1646, 1465, 1335, 1299, 0, 0, 0, 0, 0, 0, 52, 456, 4574, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6009, 48, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 158, 3618, 1837, 365, 0, 0, 0, 0, 0, 0, 6182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 4965, 0, 158, 18, 0, 0, 0, 0, 0, 0, 0, 0, 5957, 63, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6167, 6170, 6089, 6117, 6089, 6182, 6255, 6083] avg=6144.0000 max=6255 max_rank=6 min=6083 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=42897 max_load_reduction_pct=0.872742 moved_to_echo=24609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 1643, 1469, 1336, 1314, 0, 0, 0, 0, 0, 0, 69, 460, 4570, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6013, 64, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 137, 3615, 1831, 366, 0, 0, 0, 0, 0, 0, 6173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 4981, 0, 152, 23, 0, 0, 0, 0, 0, 0, 0, 0, 6027, 75, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6160, 6119, 6140, 6088, 6173, 6147, 6177] avg=6144.0000 max=6177 max_rank=7 min=6088 min_rank=4 imbalance_ratio=1.005371 max_load_reduction=42975 max_load_reduction_pct=0.874329 moved_to_echo=24585 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 1626, 1427, 1339, 1331, 0, 0, 0, 0, 0, 0, 166, 452, 4547, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6024, 161, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 0, 0, 0, 157, 3598, 1823, 356, 0, 0, 0, 0, 0, 0, 6110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 4838, 0, 80, 27, 0, 0, 0, 0, 0, 0, 0, 0, 6105, 90, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6151, 6135, 6251, 6218, 6110, 5879, 6285] avg=6144.0000 max=6285 max_rank=7 min=5879 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=42867 max_load_reduction_pct=0.872131 moved_to_echo=24492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 1624, 1458, 1314, 1330, 0, 0, 0, 0, 0, 0, 179, 505, 4566, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5913, 174, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 106, 3613, 1826, 363, 0, 0, 0, 0, 0, 0, 6101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 4945, 0, 73, 107, 0, 0, 0, 0, 0, 0, 0, 0, 6098, 95, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6105, 6208, 6129, 6179, 6101, 5999, 6289] avg=6144.0000 max=6289 max_rank=7 min=5999 min_rank=6 imbalance_ratio=1.023600 max_load_reduction=42863 max_load_reduction_pct=0.872050 moved_to_echo=24568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 1642, 1453, 1332, 1332, 0, 0, 0, 0, 0, 0, 84, 467, 4570, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5993, 81, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 149, 3616, 1831, 362, 0, 0, 0, 0, 0, 0, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 4927, 0, 121, 33, 0, 0, 0, 0, 0, 0, 0, 0, 6106, 71, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6159, 6161, 6134, 6120, 6171, 6004, 6248] avg=6144.0000 max=6248 max_rank=7 min=6004 min_rank=6 imbalance_ratio=1.016927 max_load_reduction=42904 max_load_reduction_pct=0.872884 moved_to_echo=24543 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 1638, 1455, 1326, 1265, 0, 0, 0, 0, 0, 0, 78, 434, 4574, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 78, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 142, 3618, 1835, 363, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 4931, 0, 165, 55, 0, 0, 0, 0, 0, 0, 0, 0, 5798, 69, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6096, 6139, 6100, 6106, 6156, 6453, 5936] avg=6144.0000 max=6453 max_rank=6 min=5936 min_rank=7 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24651 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 7, 3, 5, 0, 0, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 9, 1631, 1468, 1329, 1274, 0, 0, 0, 0, 0, 0, 142, 496, 4537, 995, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5964, 140, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 248, 9, 0, 0, 0, 126, 3597, 1819, 369, 0, 0, 0, 0, 0, 5, 6106, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 1061, 4955, 1, 176, 77, 0, 0, 0, 0, 0, 0, 0, 0, 5813, 94, 0, 0, 9, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 6108, 6171, 6164, 6175, 6120, 6274, 6007] avg=6144.0000 max=6274 max_rank=6 min=6007 min_rank=7 imbalance_ratio=1.021159 max_load_reduction=42878 max_load_reduction_pct=0.872355 moved_to_echo=24576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 1646, 1465, 1335, 1299, 0, 0, 0, 0, 0, 0, 52, 456, 4574, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6009, 48, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 158, 3618, 1837, 365, 0, 0, 0, 0, 0, 0, 6182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 4965, 0, 158, 18, 0, 0, 0, 0, 0, 0, 0, 0, 5957, 63, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6167, 6170, 6089, 6117, 6089, 6182, 6255, 6083] avg=6144.0000 max=6255 max_rank=6 min=6083 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=42897 max_load_reduction_pct=0.872742 moved_to_echo=24609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 1634, 1456, 1323, 1300, 0, 0, 0, 0, 0, 0, 75, 462, 4571, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5955, 72, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 213, 3616, 1820, 363, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 4936, 0, 136, 55, 0, 0, 0, 0, 0, 0, 0, 0, 5963, 82, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 6203, 6130, 6117, 6177, 6139, 6141, 6127] avg=6144.0000 max=6203 max_rank=1 min=6117 min_rank=3 imbalance_ratio=1.009603 max_load_reduction=42949 max_load_reduction_pct=0.873800 moved_to_echo=24584 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 7, 3, 5, 0, 0, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 9, 1631, 1468, 1329, 1274, 0, 0, 0, 0, 0, 0, 142, 496, 4537, 995, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5964, 140, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 248, 9, 0, 0, 0, 126, 3597, 1819, 369, 0, 0, 0, 0, 0, 5, 6106, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 1061, 4955, 1, 176, 77, 0, 0, 0, 0, 0, 0, 0, 0, 5813, 94, 0, 0, 9, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 6108, 6171, 6164, 6175, 6120, 6274, 6007] avg=6144.0000 max=6274 max_rank=6 min=6007 min_rank=7 imbalance_ratio=1.021159 max_load_reduction=42878 max_load_reduction_pct=0.872355 moved_to_echo=24576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 1638, 1455, 1326, 1265, 0, 0, 0, 0, 0, 0, 78, 434, 4574, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 78, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 142, 3618, 1835, 363, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 4931, 0, 165, 55, 0, 0, 0, 0, 0, 0, 0, 0, 5798, 69, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6096, 6139, 6100, 6106, 6156, 6453, 5936] avg=6144.0000 max=6453 max_rank=6 min=5936 min_rank=7 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24651 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 3456, 128, 13184, 11776, 10752, 10496, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 896, 3840, 36608, 8192, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 1536, 128, 0, 0, 0, 1280, 28928, 14720, 2944, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 8320, 39552, 128, 1152, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[47872, 640, 0, 0, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 47872, 896, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 47872, 896, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 8320, 39552, 128, 1152, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 366, 299, 1540, 7983, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5707, 0, 0, 5362, 0, 0, 336, 0, 0, 0, 0, 0, 7783, 148, 0, 0, 0, 0, 0, 7826, 0, 0, 0, 2742, 0, 0, 0, 231, 0, 0, 0, 174, 0, 0, 246, 0, 0, 0, 0, 0, 7866, 95, 263, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 485, 339, 1456, 8004, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5525, 0, 0, 5914, 0, 0, 319, 0, 0, 0, 0, 0, 7796, 144, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 2428, 0, 0, 0, 157, 0, 0, 0, 78, 0, 0, 234, 0, 0, 0, 0, 0, 7966, 69, 127, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 433, 315, 1428, 7985, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5555, 0, 0, 5763, 0, 0, 352, 0, 0, 0, 0, 1, 7802, 111, 0, 0, 0, 0, 0, 7977, 0, 0, 0, 2586, 0, 0, 0, 133, 0, 0, 0, 64, 0, 0, 272, 0, 0, 0, 0, 0, 7984, 75, 168, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 414, 332, 1416, 7976, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5571, 0, 0, 5520, 0, 0, 313, 0, 0, 0, 0, 1, 7801, 200, 0, 0, 0, 0, 0, 7848, 0, 0, 0, 2745, 0, 0, 0, 248, 0, 0, 0, 161, 0, 0, 278, 0, 0, 0, 0, 0, 7854, 90, 204, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 433, 315, 1428, 7985, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5555, 0, 0, 5763, 0, 0, 352, 0, 0, 0, 0, 1, 7802, 111, 0, 0, 0, 0, 0, 7977, 0, 0, 0, 2586, 0, 0, 0, 133, 0, 0, 0, 64, 0, 0, 272, 0, 0, 0, 0, 0, 7984, 75, 168, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 450, 345, 1366, 7994, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5645, 0, 0, 5324, 0, 0, 314, 0, 0, 0, 0, 0, 7799, 83, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 3055, 0, 0, 0, 140, 0, 0, 0, 48, 0, 0, 247, 0, 0, 0, 0, 0, 7989, 63, 157, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 422, 307, 1407, 7969, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5490, 0, 0, 5541, 0, 0, 336, 0, 0, 0, 0, 0, 7797, 83, 0, 0, 0, 0, 0, 7966, 0, 0, 0, 2946, 0, 0, 0, 157, 0, 0, 0, 81, 0, 0, 265, 0, 0, 0, 0, 0, 7964, 71, 206, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 502, 318, 1358, 7958, 176, 1, 0, 2, 0, 5, 0, 4, 0, 7, 5598, 10, 0, 5915, 0, 6, 344, 0, 0, 0, 0, 5, 7762, 201, 0, 8, 0, 0, 0, 7852, 0, 0, 0, 2336, 0, 0, 0, 209, 0, 5, 0, 141, 7, 1, 245, 3, 0, 0, 9, 0, 7877, 91, 188, 1, 7, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 366, 299, 1540, 7983, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5707, 0, 0, 5362, 0, 0, 336, 0, 0, 0, 0, 0, 7783, 148, 0, 0, 0, 0, 0, 7826, 0, 0, 0, 2742, 0, 0, 0, 231, 0, 0, 0, 174, 0, 0, 246, 0, 0, 0, 0, 0, 7866, 95, 263, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 464, 367, 1439, 7976, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5425, 0, 0, 5629, 0, 0, 310, 0, 0, 0, 0, 0, 7813, 131, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 2790, 0, 0, 0, 185, 0, 0, 0, 72, 0, 0, 231, 0, 0, 0, 0, 0, 7925, 82, 199, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 366, 299, 1540, 3793, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5707, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5362, 0, 0, 336, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5998, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 0, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 174, 0, 2801, 2022, 1017, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 5844, 0, 0, 0, 0, 0, 0, 0, 0, 95, 263, 0, 0, 0, 0, 0, 0, 4429, 1389, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6283, 5890, 6146, 6139, 6245, 6090, 6176] avg=6144.0000 max=6283 max_rank=1 min=5890 min_rank=2 imbalance_ratio=1.022624 max_load_reduction=42869 max_load_reduction_pct=0.872172 moved_to_echo=24650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 485, 339, 1456, 8004, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5525, 0, 0, 5914, 0, 0, 319, 0, 0, 0, 0, 0, 7796, 144, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 2428, 0, 0, 0, 157, 0, 0, 0, 78, 0, 0, 234, 0, 0, 0, 0, 0, 7966, 69, 127, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 433, 315, 1428, 3795, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5555, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5763, 0, 0, 352, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6012, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 0, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 64, 0, 2801, 2052, 1020, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 5932, 0, 0, 0, 0, 0, 0, 0, 0, 75, 168, 0, 0, 0, 0, 0, 0, 4514, 1389, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6119, 6132, 6308, 6124, 6049, 6070, 6204, 6146] avg=6144.0000 max=6308 max_rank=2 min=6049 min_rank=4 imbalance_ratio=1.026693 max_load_reduction=42844 max_load_reduction_pct=0.871663 moved_to_echo=24469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 450, 345, 1366, 3800, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5645, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5324, 0, 0, 314, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6009, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 0, 0, 0, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 48, 0, 2804, 2053, 1020, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 5936, 0, 0, 0, 0, 0, 0, 0, 0, 63, 157, 0, 0, 0, 0, 0, 0, 4536, 1390, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6078, 6222, 5831, 6092, 6535, 6065, 6183, 6146] avg=6144.0000 max=6535 max_rank=4 min=5831 min_rank=2 imbalance_ratio=1.063639 max_load_reduction=42617 max_load_reduction_pct=0.867045 moved_to_echo=24929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 414, 332, 1416, 3791, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5571, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5520, 0, 0, 313, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6011, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 0, 0, 2745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 161, 0, 2798, 2018, 1020, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 5836, 0, 0, 0, 0, 0, 0, 0, 0, 90, 204, 0, 0, 0, 0, 0, 0, 4441, 1387, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 6148, 6026, 6212, 6152, 6245, 6114, 6122] avg=6144.0000 max=6245 max_rank=5 min=6026 min_rank=2 imbalance_ratio=1.016439 max_load_reduction=42907 max_load_reduction_pct=0.872945 moved_to_echo=24633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 485, 339, 1456, 3804, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5525, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5914, 0, 0, 319, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6007, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 0, 2428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 78, 0, 2808, 2047, 1019, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 5919, 0, 0, 0, 0, 0, 0, 0, 0, 69, 127, 0, 0, 0, 0, 0, 0, 4513, 1392, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6102, 6426, 6151, 5891, 6109, 6153, 6101] avg=6144.0000 max=6426 max_rank=2 min=5891 min_rank=4 imbalance_ratio=1.045898 max_load_reduction=42726 max_load_reduction_pct=0.869263 moved_to_echo=24254 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 422, 307, 1407, 3787, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5490, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5541, 0, 0, 336, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6008, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 0, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 81, 0, 2796, 2047, 1019, 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 0, 5917, 0, 0, 0, 0, 0, 0, 0, 0, 71, 206, 0, 0, 0, 0, 0, 0, 4508, 1386, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6067, 6067, 6070, 6091, 6404, 6100, 6182, 6171] avg=6144.0000 max=6404 max_rank=4 min=6067 min_rank=0 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 450, 345, 1366, 3800, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5645, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5324, 0, 0, 314, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6009, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 0, 0, 0, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 48, 0, 2804, 2053, 1020, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 5936, 0, 0, 0, 0, 0, 0, 0, 0, 63, 157, 0, 0, 0, 0, 0, 0, 4536, 1390, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6078, 6222, 5831, 6092, 6535, 6065, 6183, 6146] avg=6144.0000 max=6535 max_rank=4 min=5831 min_rank=2 imbalance_ratio=1.063639 max_load_reduction=42617 max_load_reduction_pct=0.867045 moved_to_echo=24929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 502, 318, 1358, 3773, 176, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 4, 0, 7, 5598, 579, 0, 0, 0, 0, 0, 0, 0, 10, 0, 5915, 0, 6, 344, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5970, 201, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 0, 0, 0, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 5, 0, 141, 7, 2796, 2028, 1019, 0, 0, 0, 0, 0, 1, 245, 3, 0, 0, 9, 0, 5849, 0, 0, 0, 0, 0, 0, 0, 0, 91, 188, 1, 7, 0, 0, 0, 0, 4447, 1389, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6195, 6469, 6184, 5741, 6205, 6107, 6123] avg=6144.0000 max=6469 max_rank=2 min=5741 min_rank=4 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 366, 299, 1540, 3793, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5707, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5362, 0, 0, 336, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5998, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 0, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 174, 0, 2801, 2022, 1017, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 5844, 0, 0, 0, 0, 0, 0, 0, 0, 95, 263, 0, 0, 0, 0, 0, 0, 4429, 1389, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6283, 5890, 6146, 6139, 6245, 6090, 6176] avg=6144.0000 max=6283 max_rank=1 min=5890 min_rank=2 imbalance_ratio=1.022624 max_load_reduction=42869 max_load_reduction_pct=0.872172 moved_to_echo=24650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 464, 367, 1439, 3791, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5425, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5629, 0, 0, 310, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6021, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3451, 0, 0, 0, 2790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 72, 0, 2798, 2037, 1021, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 82, 199, 0, 0, 0, 0, 0, 0, 4499, 1387, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6225, 6003, 6132, 6152, 6241, 6113, 6119, 6167] avg=6144.0000 max=6241 max_rank=4 min=6003 min_rank=1 imbalance_ratio=1.015788 max_load_reduction=42911 max_load_reduction_pct=0.873027 moved_to_echo=24640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 485, 339, 1456, 3804, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5525, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5914, 0, 0, 319, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6007, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 0, 2428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 78, 0, 2808, 2047, 1019, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 5919, 0, 0, 0, 0, 0, 0, 0, 0, 69, 127, 0, 0, 0, 0, 0, 0, 4513, 1392, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6102, 6426, 6151, 5891, 6109, 6153, 6101] avg=6144.0000 max=6426 max_rank=2 min=5891 min_rank=4 imbalance_ratio=1.045898 max_load_reduction=42726 max_load_reduction_pct=0.869263 moved_to_echo=24254 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 502, 318, 1358, 3773, 176, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 4, 0, 7, 5598, 579, 0, 0, 0, 0, 0, 0, 0, 10, 0, 5915, 0, 6, 344, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5970, 201, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 0, 0, 0, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 5, 0, 141, 7, 2796, 2028, 1019, 0, 0, 0, 0, 0, 1, 245, 3, 0, 0, 9, 0, 5849, 0, 0, 0, 0, 0, 0, 0, 0, 91, 188, 1, 7, 0, 0, 0, 0, 4447, 1389, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6195, 6469, 6184, 5741, 6205, 6107, 6123] avg=6144.0000 max=6469 max_rank=2 min=5741 min_rank=4 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 1536, 0, 128, 0, 896, 128, 22528, 16384, 8192, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 3584, 2688, 11520, 30336, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 1536, 128, 128, 0, 0, 0, 0, 35968, 11136, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 3584, 2688, 11520, 30336, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 27648, 0, 0, 0, 21632, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 44544, 4736, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 48128, 1152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 45056, 0, 128, 2688, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2048, 128, 0, 0, 128, 0, 47232, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 48128, 1152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[7880, 0, 0, 0, 0, 0, 0, 1912, 327, 0, 6392, 0, 0, 0, 0, 0, 0, 0, 0, 148, 94, 174, 0, 7596, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 7458, 0, 0, 200, 0, 0, 0, 0, 254, 0, 7431, 0, 0, 0, 0, 0, 330, 96, 0, 0, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[8018, 0, 0, 0, 0, 0, 0, 1828, 377, 0, 6466, 0, 0, 0, 0, 0, 0, 0, 0, 133, 54, 48, 0, 7750, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 7477, 0, 0, 184, 0, 0, 0, 0, 254, 0, 7656, 0, 0, 0, 0, 0, 197, 66, 0, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[7991, 0, 0, 0, 0, 0, 0, 2026, 402, 0, 6174, 0, 0, 0, 0, 0, 0, 0, 0, 155, 19, 78, 0, 7719, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 7458, 0, 0, 177, 0, 0, 0, 0, 261, 0, 7708, 0, 0, 0, 0, 0, 209, 70, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[7989, 0, 0, 0, 0, 0, 0, 1855, 387, 0, 6433, 0, 0, 0, 0, 0, 0, 0, 0, 158, 69, 64, 0, 7738, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 7453, 0, 0, 193, 0, 0, 0, 0, 246, 0, 7607, 0, 0, 0, 0, 0, 210, 76, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[7891, 5, 0, 0, 0, 0, 7, 1994, 360, 10, 6190, 0, 1, 0, 0, 0, 0, 1, 1, 153, 87, 142, 0, 7633, 0, 0, 0, 0, 0, 2, 0, 2, 578, 4, 0, 0, 0, 0, 0, 8169, 0, 10, 0, 7434, 6, 8, 197, 0, 0, 0, 0, 250, 4, 7556, 0, 4, 0, 0, 0, 260, 99, 3, 0, 91] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[7875, 0, 0, 0, 0, 0, 0, 1929, 392, 0, 6338, 0, 0, 0, 0, 0, 0, 0, 0, 176, 78, 161, 0, 7634, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 7463, 0, 0, 164, 0, 0, 0, 0, 247, 0, 7502, 0, 0, 0, 0, 0, 292, 90, 0, 0, 89] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[7880, 0, 0, 0, 0, 0, 0, 1912, 327, 0, 6392, 0, 0, 0, 0, 0, 0, 0, 0, 148, 94, 174, 0, 7596, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 7458, 0, 0, 200, 0, 0, 0, 0, 254, 0, 7431, 0, 0, 0, 0, 0, 330, 96, 0, 0, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[7980, 0, 0, 0, 0, 0, 0, 1990, 421, 0, 6255, 0, 0, 0, 0, 0, 0, 0, 0, 150, 63, 81, 0, 7706, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 7418, 0, 0, 218, 0, 0, 0, 0, 239, 0, 7616, 0, 0, 0, 0, 0, 218, 71, 0, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[7991, 0, 0, 0, 0, 0, 0, 2026, 402, 0, 6174, 0, 0, 0, 0, 0, 0, 0, 0, 155, 19, 78, 0, 7719, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 7458, 0, 0, 177, 0, 0, 0, 0, 261, 0, 7708, 0, 0, 0, 0, 0, 209, 70, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[7948, 0, 0, 0, 0, 0, 0, 1961, 340, 0, 6373, 0, 0, 0, 0, 0, 0, 0, 0, 184, 79, 72, 0, 7730, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 7499, 0, 0, 175, 0, 0, 0, 0, 264, 0, 7582, 0, 0, 0, 0, 0, 199, 82, 0, 0, 82] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4171, 0, 0, 0, 0, 0, 0, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 5826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 94, 174, 0, 5747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 2407, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 5649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5953, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 5774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 96, 0, 0, 95, 1849, 1657, 1505, 566, 114, 0, 0, 0] rerouted_tokens_per_ep_rank=[6083, 6153, 6163, 6116, 6244, 6153, 6028, 6212] avg=6144.0000 max=6244 max_rank=4 min=6028 min_rank=6 imbalance_ratio=1.016276 max_load_reduction=42908 max_load_reduction_pct=0.872965 moved_to_echo=24637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[7891, 5, 0, 0, 0, 0, 7, 1994, 360, 10, 6190, 0, 1, 0, 0, 0, 0, 1, 1, 153, 87, 142, 0, 7633, 0, 0, 0, 0, 0, 2, 0, 2, 578, 4, 0, 0, 0, 0, 0, 8169, 0, 10, 0, 7434, 6, 8, 197, 0, 0, 0, 0, 250, 4, 7556, 0, 4, 0, 0, 0, 260, 99, 3, 0, 91] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4244, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 5893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 54, 48, 0, 5863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 2398, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 5949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 66, 0, 0, 65, 1887, 1707, 1508, 573, 113, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 6270, 6098, 6172, 6068, 6153, 6203, 6116] avg=6144.0000 max=6270 max_rank=1 min=6068 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=42882 max_load_reduction_pct=0.872437 moved_to_echo=24540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4229, 0, 0, 0, 0, 0, 0, 1855, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 5863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 69, 64, 0, 5854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 2398, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 0, 0, 0, 5629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5949, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 76, 0, 0, 75, 1884, 1696, 1504, 570, 113, 0, 0, 0] rerouted_tokens_per_ep_rank=[6084, 6250, 6145, 6158, 6088, 6142, 6157, 6128] avg=6144.0000 max=6250 max_rank=1 min=6084 min_rank=0 imbalance_ratio=1.017253 max_load_reduction=42902 max_load_reduction_pct=0.872843 moved_to_echo=24515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4230, 0, 0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 5627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 19, 78, 0, 5840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 2405, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 5643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5953, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 5989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 70, 0, 0, 69, 1879, 1719, 1505, 547, 114, 0, 0, 0] rerouted_tokens_per_ep_rank=[6256, 6029, 6092, 6166, 6117, 6130, 6250, 6112] avg=6144.0000 max=6256 max_rank=0 min=6029 min_rank=1 imbalance_ratio=1.018229 max_load_reduction=42896 max_load_reduction_pct=0.872721 moved_to_echo=24609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4224, 0, 0, 0, 0, 0, 0, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 5701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 63, 81, 0, 5830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 2407, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 5648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5921, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 5918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 71, 0, 0, 74, 1876, 1698, 1497, 554, 114, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 6122, 6124, 6163, 6131, 6139, 6157, 6102] avg=6144.0000 max=6214 max_rank=0 min=6102 min_rank=7 imbalance_ratio=1.011393 max_load_reduction=42938 max_load_reduction_pct=0.873576 moved_to_echo=24529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4168, 0, 0, 0, 0, 0, 0, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 5777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 78, 161, 0, 5776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 2405, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 5643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5957, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 5829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 90, 0, 0, 89, 1858, 1673, 1506, 561, 114, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6169, 6191, 6112, 6203, 6121, 6076, 6183] avg=6144.0000 max=6203 max_rank=4 min=6076 min_rank=6 imbalance_ratio=1.009603 max_load_reduction=42949 max_load_reduction_pct=0.873800 moved_to_echo=24583 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4244, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 5893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 54, 48, 0, 5863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 2398, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 5949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 66, 0, 0, 65, 1887, 1707, 1508, 573, 113, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 6270, 6098, 6172, 6068, 6153, 6203, 6116] avg=6144.0000 max=6270 max_rank=1 min=6068 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=42882 max_load_reduction_pct=0.872437 moved_to_echo=24540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4172, 5, 0, 0, 0, 0, 7, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 360, 10, 5639, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 153, 87, 142, 0, 5772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 3719, 2409, 0, 0, 0, 0, 0, 0, 578, 4, 0, 0, 0, 0, 0, 5642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 5932, 6, 8, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 4, 5868, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 99, 3, 0, 91, 1861, 1688, 1502, 551, 118, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6010, 6156, 6132, 6224, 6153, 6126, 6173] avg=6144.0000 max=6224 max_rank=4 min=6010 min_rank=1 imbalance_ratio=1.013021 max_load_reduction=42928 max_load_reduction_pct=0.873372 moved_to_echo=24676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4171, 0, 0, 0, 0, 0, 0, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 5826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 94, 174, 0, 5747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 2407, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 5649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5953, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 5774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 96, 0, 0, 95, 1849, 1657, 1505, 566, 114, 0, 0, 0] rerouted_tokens_per_ep_rank=[6083, 6153, 6163, 6116, 6244, 6153, 6028, 6212] avg=6144.0000 max=6244 max_rank=4 min=6028 min_rank=6 imbalance_ratio=1.016276 max_load_reduction=42908 max_load_reduction_pct=0.872965 moved_to_echo=24637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4224, 0, 0, 0, 0, 0, 0, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 5701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 63, 81, 0, 5830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 2407, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 5648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5921, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 5918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 71, 0, 0, 74, 1876, 1698, 1497, 554, 114, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 6122, 6124, 6163, 6131, 6139, 6157, 6102] avg=6144.0000 max=6214 max_rank=0 min=6102 min_rank=7 imbalance_ratio=1.011393 max_load_reduction=42938 max_load_reduction_pct=0.873576 moved_to_echo=24529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4230, 0, 0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 5627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 19, 78, 0, 5840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 2405, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 5643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5953, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 5989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 70, 0, 0, 69, 1879, 1719, 1505, 547, 114, 0, 0, 0] rerouted_tokens_per_ep_rank=[6256, 6029, 6092, 6166, 6117, 6130, 6250, 6112] avg=6144.0000 max=6256 max_rank=0 min=6029 min_rank=1 imbalance_ratio=1.018229 max_load_reduction=42896 max_load_reduction_pct=0.872721 moved_to_echo=24609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4207, 0, 0, 0, 0, 0, 0, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 5809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 79, 72, 0, 5848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 2392, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 5615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5986, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 5891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 82, 0, 0, 82, 1882, 1691, 1513, 564, 113, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 6149, 6183, 6133, 6077, 6161, 6155, 6126] avg=6144.0000 max=6183 max_rank=2 min=6077 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=42969 max_load_reduction_pct=0.874207 moved_to_echo=24519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4172, 5, 0, 0, 0, 0, 7, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 360, 10, 5639, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 153, 87, 142, 0, 5772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 3719, 2409, 0, 0, 0, 0, 0, 0, 578, 4, 0, 0, 0, 0, 0, 5642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 5932, 6, 8, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 4, 5868, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 99, 3, 0, 91, 1861, 1688, 1502, 551, 118, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6010, 6156, 6132, 6224, 6153, 6126, 6173] avg=6144.0000 max=6224 max_rank=4 min=6010 min_rank=1 imbalance_ratio=1.013021 max_load_reduction=42928 max_load_reduction_pct=0.873372 moved_to_echo=24676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 0, 1920, 768, 128, 0, 640, 14976, 13568, 12160, 4608, 1024, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 29952, 19328, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[33664, 128, 0, 0, 0, 0, 128, 15616, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4096, 128, 0, 0, 0, 0, 0, 45184, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 2048, 128, 47232, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 47744, 128, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 1280, 640, 896, 0, 46592, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3072, 128, 46208, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 47744, 128, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3072, 128, 46208, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 63, 0, 0, 303, 1, 0, 0, 0, 65, 0, 268, 0, 0, 0, 0, 0, 0, 4742, 7820, 0, 1, 0, 0, 0, 0, 0, 136, 155, 0, 0, 0, 4896, 8192, 0, 0, 90, 0, 0, 33, 0, 90, 0, 0, 0, 6156, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 174, 0, 0, 344, 18, 0, 0, 0, 63, 0, 134, 0, 0, 0, 0, 0, 0, 5289, 7782, 0, 5, 0, 0, 0, 1, 0, 116, 133, 0, 0, 0, 4745, 8192, 0, 0, 75, 0, 0, 56, 0, 75, 0, 0, 0, 5773, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 88, 0, 0, 338, 29, 0, 0, 0, 60, 0, 168, 0, 0, 0, 0, 0, 0, 5868, 7796, 0, 0, 0, 0, 0, 11, 0, 68, 123, 0, 0, 0, 4635, 8192, 0, 0, 65, 0, 0, 154, 0, 65, 0, 0, 0, 5325, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 76, 0, 0, 348, 13, 0, 0, 0, 60, 0, 177, 0, 0, 0, 0, 0, 0, 4943, 7779, 0, 0, 0, 0, 0, 0, 0, 101, 131, 0, 0, 0, 5021, 8192, 0, 0, 74, 0, 0, 20, 0, 74, 0, 0, 0, 5969, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 59, 0, 0, 345, 48, 0, 0, 0, 53, 0, 153, 0, 0, 0, 0, 0, 0, 6033, 7793, 0, 3, 0, 0, 0, 7, 0, 72, 119, 0, 0, 0, 4717, 8192, 0, 0, 69, 0, 0, 201, 0, 69, 0, 0, 0, 5036, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 50, 0, 0, 361, 12, 0, 0, 0, 90, 0, 193, 0, 0, 0, 0, 0, 0, 5109, 7740, 0, 1, 0, 0, 0, 0, 0, 152, 171, 0, 0, 0, 4643, 8192, 0, 0, 82, 0, 0, 198, 0, 82, 0, 0, 0, 5977, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 17, 0, 0, 337, 5, 0, 0, 0, 42, 0, 254, 0, 0, 0, 0, 0, 0, 4953, 7810, 0, 0, 0, 0, 0, 0, 0, 110, 137, 0, 0, 0, 4757, 8192, 0, 0, 95, 0, 0, 65, 0, 96, 0, 0, 0, 6092, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 63, 0, 0, 303, 1, 0, 0, 0, 65, 0, 268, 0, 0, 0, 0, 0, 0, 4742, 7820, 0, 1, 0, 0, 0, 0, 0, 136, 155, 0, 0, 0, 4896, 8192, 0, 0, 90, 0, 0, 33, 0, 90, 0, 0, 0, 6156, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8051, 0, 2, 0, 0, 1, 9, 1, 0, 0, 8086, 0, 155, 0, 0, 345, 48, 0, 0, 0, 54, 0, 221, 2, 1, 0, 0, 0, 0, 5565, 7767, 0, 0, 0, 9, 0, 8, 0, 101, 145, 0, 2, 10, 4717, 8182, 0, 11, 93, 0, 0, 94, 0, 91, 0, 0, 11, 5361, 9, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 50, 0, 0, 361, 12, 0, 0, 0, 90, 0, 193, 0, 0, 0, 0, 0, 0, 5109, 7740, 0, 1, 0, 0, 0, 0, 0, 152, 171, 0, 0, 0, 4643, 8192, 0, 0, 82, 0, 0, 198, 0, 82, 0, 0, 0, 5977, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 76, 0, 0, 348, 13, 0, 0, 0, 60, 0, 177, 0, 0, 0, 0, 0, 0, 4943, 7779, 0, 0, 0, 0, 0, 0, 0, 101, 131, 0, 0, 0, 5021, 8192, 0, 0, 74, 0, 0, 20, 0, 74, 0, 0, 0, 5969, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6048, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 5, 0, 0, 0, 42, 0, 2049, 1915, 1181, 584, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 4953, 636, 0, 0, 0, 0, 0, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 137, 0, 0, 0, 4757, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 65, 0, 96, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6092, 0, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6065, 6113, 5843, 6159, 6135, 6136, 6523] avg=6144.0000 max=6523 max_rank=7 min=5843 min_rank=3 imbalance_ratio=1.061686 max_load_reduction=42629 max_load_reduction_pct=0.867289 moved_to_echo=24953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6063, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 18, 0, 0, 0, 63, 0, 2054, 1907, 1181, 582, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 5289, 634, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 133, 0, 0, 0, 4745, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 56, 0, 75, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5773, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6237, 6149, 6057, 6142, 6125, 6086, 6203] avg=6144.0000 max=6237 max_rank=1 min=6057 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=42915 max_load_reduction_pct=0.873108 moved_to_echo=24556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6068, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 48, 0, 0, 0, 53, 0, 2055, 1907, 1181, 582, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 6033, 635, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 3, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 119, 0, 0, 0, 4717, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 201, 0, 69, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5036, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6127, 6171, 6821, 6156, 6039, 6219, 5466] avg=6144.0000 max=6821 max_rank=3 min=5466 min_rank=7 imbalance_ratio=1.110189 max_load_reduction=42331 max_load_reduction_pct=0.861226 moved_to_echo=23880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6071, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 29, 0, 0, 0, 60, 0, 2056, 1902, 1181, 583, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 5868, 635, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 123, 0, 0, 0, 4635, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 154, 0, 65, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5325, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6159, 6149, 6671, 6159, 5957, 6164, 5755] avg=6144.0000 max=6671 max_rank=3 min=5755 min_rank=7 imbalance_ratio=1.085775 max_load_reduction=42481 max_load_reduction_pct=0.864278 moved_to_echo=24035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6052, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 1, 0, 0, 0, 65, 0, 2050, 1902, 1181, 584, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 4742, 637, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 155, 0, 0, 0, 4896, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 33, 0, 90, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6115, 6086, 5647, 6168, 6318, 6093, 6588] avg=6144.0000 max=6588 max_rank=7 min=5647 min_rank=3 imbalance_ratio=1.072266 max_load_reduction=42564 max_load_reduction_pct=0.865967 moved_to_echo=25167 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6064, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 13, 0, 0, 0, 60, 0, 2054, 1906, 1181, 581, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 4943, 634, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 131, 0, 0, 0, 5021, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 20, 0, 74, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6140, 6143, 5754, 6135, 6384, 6048, 6398] avg=6144.0000 max=6398 max_rank=7 min=5754 min_rank=3 imbalance_ratio=1.041341 max_load_reduction=42754 max_load_reduction_pct=0.869832 moved_to_echo=24965 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6071, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 29, 0, 0, 0, 60, 0, 2056, 1902, 1181, 583, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 5868, 635, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 123, 0, 0, 0, 4635, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 154, 0, 65, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5325, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6159, 6149, 6671, 6159, 5957, 6164, 5755] avg=6144.0000 max=6671 max_rank=3 min=5755 min_rank=7 imbalance_ratio=1.085775 max_load_reduction=42481 max_load_reduction_pct=0.864278 moved_to_echo=24035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6141, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 6036, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 48, 0, 0, 0, 54, 0, 2050, 1910, 1185, 585, 0, 0, 0, 0, 221, 2, 1, 0, 0, 0, 0, 5565, 637, 0, 0, 0, 0, 0, 0, 0, 6111, 0, 0, 0, 9, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 145, 0, 2, 10, 4717, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 93, 0, 0, 94, 0, 91, 0, 5875, 0, 0, 0, 0, 0, 0, 0, 0, 11, 5361, 9, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6201, 6177, 6426, 6128, 6097, 6164, 5815] avg=6144.0000 max=6426 max_rank=3 min=5815 min_rank=7 imbalance_ratio=1.045898 max_load_reduction=42726 max_load_reduction_pct=0.869263 moved_to_echo=24204 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6048, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 5, 0, 0, 0, 42, 0, 2049, 1915, 1181, 584, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 4953, 636, 0, 0, 0, 0, 0, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 137, 0, 0, 0, 4757, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 65, 0, 96, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6092, 0, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6065, 6113, 5843, 6159, 6135, 6136, 6523] avg=6144.0000 max=6523 max_rank=7 min=5843 min_rank=3 imbalance_ratio=1.061686 max_load_reduction=42629 max_load_reduction_pct=0.867289 moved_to_echo=24953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6068, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 48, 0, 0, 0, 53, 0, 2055, 1907, 1181, 582, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 6033, 635, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 3, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 119, 0, 0, 0, 4717, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 201, 0, 69, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5036, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6127, 6171, 6821, 6156, 6039, 6219, 5466] avg=6144.0000 max=6821 max_rank=3 min=5466 min_rank=7 imbalance_ratio=1.110189 max_load_reduction=42331 max_load_reduction_pct=0.861226 moved_to_echo=23880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6058, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 12, 0, 0, 0, 90, 0, 2052, 1890, 1181, 578, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 5109, 631, 0, 0, 0, 0, 0, 0, 0, 6104, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 171, 0, 0, 0, 4643, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 198, 0, 82, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5977, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6099, 6108, 6164, 5933, 6105, 6097, 6242, 6404] avg=6144.0000 max=6404 max_rank=7 min=5933 min_rank=3 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6141, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 6036, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 48, 0, 0, 0, 54, 0, 2050, 1910, 1185, 585, 0, 0, 0, 0, 221, 2, 1, 0, 0, 0, 0, 5565, 637, 0, 0, 0, 0, 0, 0, 0, 6111, 0, 0, 0, 9, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 145, 0, 2, 10, 4717, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 93, 0, 0, 94, 0, 91, 0, 5875, 0, 0, 0, 0, 0, 0, 0, 0, 11, 5361, 9, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6201, 6177, 6426, 6128, 6097, 6164, 5815] avg=6144.0000 max=6426 max_rank=3 min=5815 min_rank=7 imbalance_ratio=1.045898 max_load_reduction=42726 max_load_reduction_pct=0.869263 moved_to_echo=24204 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 2816, 256, 0, 0, 0, 512, 0, 16512, 15360, 9472, 4736, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 1152, 0, 128, 128, 38144, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 1152, 0, 128, 128, 38144, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 128, 128, 0, 0, 0, 0, 42624, 5120, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 45696, 128, 0, 0, 0, 0, 3456, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 48512, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 768, 0, 0, 896, 0, 768, 0, 47104, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 49152, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[49152, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 49152, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 279, 0, 0, 7888, 7084, 0, 0, 91, 77, 0, 0, 0, 0, 0, 0, 6801, 0, 0, 0, 0, 0, 0, 5083, 687, 0, 0, 0, 0, 0, 0, 407, 7177, 0, 8097, 0, 0, 0, 21, 95, 0, 84, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 349, 4686, 0, 0, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 295, 0, 0, 7860, 7110, 0, 0, 90, 76, 0, 0, 0, 0, 0, 0, 6469, 0, 0, 0, 0, 0, 0, 5030, 753, 0, 0, 0, 0, 0, 0, 490, 7164, 0, 8102, 0, 0, 0, 2, 90, 0, 66, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 337, 4997, 0, 0, 0, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 299, 0, 0, 7856, 7069, 0, 0, 69, 28, 0, 0, 0, 0, 0, 0, 6807, 0, 0, 0, 0, 0, 0, 4092, 701, 0, 0, 0, 0, 0, 0, 466, 7371, 0, 8123, 0, 0, 0, 3, 69, 0, 24, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 330, 5701, 0, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 295, 0, 0, 7846, 7138, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 6601, 0, 0, 0, 0, 0, 0, 4596, 682, 0, 0, 0, 0, 0, 0, 361, 7540, 0, 8129, 0, 0, 0, 16, 63, 0, 6, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 319, 5374, 0, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 347, 0, 0, 7878, 7052, 0, 0, 73, 3, 0, 0, 0, 0, 0, 0, 6841, 0, 0, 0, 0, 0, 0, 4182, 650, 0, 0, 0, 0, 0, 0, 433, 7361, 0, 8117, 0, 0, 0, 18, 75, 0, 45, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 321, 5623, 0, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 295, 0, 0, 7846, 7138, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 6601, 0, 0, 0, 0, 0, 0, 4596, 682, 0, 0, 0, 0, 0, 0, 361, 7540, 0, 8129, 0, 0, 0, 16, 63, 0, 6, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 319, 5374, 0, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 319, 0, 0, 7867, 7070, 0, 0, 80, 12, 0, 0, 0, 0, 0, 0, 6644, 0, 0, 0, 0, 0, 0, 4879, 706, 0, 0, 0, 0, 0, 0, 460, 7261, 0, 8110, 0, 0, 0, 2, 82, 0, 78, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 346, 5121, 0, 0, 0, 43] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 7, 7, 11, 5, 306, 1, 0, 7860, 7010, 0, 0, 91, 17, 0, 0, 2, 0, 0, 10, 6570, 7, 0, 0, 2, 0, 0, 4530, 761, 7, 0, 1, 10, 0, 0, 510, 7250, 0, 8090, 0, 0, 0, 12, 96, 0, 100, 0, 1, 0, 0, 0, 0, 137, 0, 0, 0, 0, 341, 5329, 0, 1, 1, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 295, 0, 0, 7860, 7110, 0, 0, 90, 76, 0, 0, 0, 0, 0, 0, 6469, 0, 0, 0, 0, 0, 0, 5030, 753, 0, 0, 0, 0, 0, 0, 490, 7164, 0, 8102, 0, 0, 0, 2, 90, 0, 66, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 337, 4997, 0, 0, 0, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 279, 0, 0, 7888, 7084, 0, 0, 91, 77, 0, 0, 0, 0, 0, 0, 6801, 0, 0, 0, 0, 0, 0, 5083, 687, 0, 0, 0, 0, 0, 0, 407, 7177, 0, 8097, 0, 0, 0, 21, 95, 0, 84, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 349, 4686, 0, 0, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 299, 0, 0, 7856, 7069, 0, 0, 69, 28, 0, 0, 0, 0, 0, 0, 6807, 0, 0, 0, 0, 0, 0, 4092, 701, 0, 0, 0, 0, 0, 0, 466, 7371, 0, 8123, 0, 0, 0, 3, 69, 0, 24, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 330, 5701, 0, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 321, 0, 0, 7893, 7076, 0, 0, 70, 98, 0, 0, 0, 0, 0, 0, 6929, 0, 0, 0, 0, 0, 0, 4944, 726, 0, 0, 0, 0, 0, 0, 578, 7042, 0, 8121, 0, 0, 0, 1, 71, 0, 56, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 313, 4740, 0, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 279, 0, 3950, 1570, 301, 0, 0, 0, 0, 0, 0, 0, 6034, 0, 0, 91, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5083, 687, 0, 0, 749, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 5607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 95, 0, 84, 0, 3938, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 6032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 4686, 0, 0, 0, 72, 554, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6202, 6228, 6538, 6014, 6203, 6206, 5661] avg=6144.0000 max=6538 max_rank=3 min=5661 min_rank=7 imbalance_ratio=1.064128 max_load_reduction=42614 max_load_reduction_pct=0.866984 moved_to_echo=24084 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 347, 0, 3945, 1611, 299, 0, 0, 0, 0, 0, 0, 0, 6007, 0, 0, 73, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4182, 650, 0, 0, 746, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 5750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 75, 0, 45, 0, 3933, 2071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 6046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 5623, 0, 0, 0, 69, 557, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6202, 6083, 6265, 5597, 6183, 6142, 6110, 6570] avg=6144.0000 max=6570 max_rank=7 min=5597 min_rank=3 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=25005 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 295, 0, 3929, 1650, 303, 0, 0, 0, 0, 0, 0, 0, 6080, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4596, 682, 0, 0, 755, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 5890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 63, 0, 6, 0, 3917, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 6056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 5374, 0, 0, 0, 75, 538, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6143, 6045, 6051, 6251, 6075, 6104, 6306] avg=6144.0000 max=6306 max_rank=7 min=6045 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24736 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 295, 0, 3936, 1567, 302, 0, 0, 0, 0, 0, 0, 0, 6056, 0, 0, 90, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5030, 753, 0, 0, 752, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 5597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 90, 0, 66, 0, 3924, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 6036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 4997, 0, 0, 0, 60, 527, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6222, 5924, 6553, 6087, 6148, 6197, 5921] avg=6144.0000 max=6553 max_rank=3 min=5921 min_rank=7 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=24353 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 319, 0, 3940, 1589, 300, 0, 0, 0, 0, 0, 0, 0, 6023, 0, 0, 80, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4879, 706, 0, 0, 747, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 5672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 82, 0, 78, 0, 3927, 2069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 6041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 5121, 0, 0, 0, 43, 541, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6115, 6085, 6350, 6132, 6158, 6113, 6051] avg=6144.0000 max=6350 max_rank=3 min=6051 min_rank=7 imbalance_ratio=1.033529 max_load_reduction=42802 max_load_reduction_pct=0.870809 moved_to_echo=24454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 299, 0, 3934, 1613, 300, 0, 0, 0, 0, 0, 0, 0, 6022, 0, 0, 69, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 701, 0, 0, 747, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 5758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 69, 0, 24, 0, 3922, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 6050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 5701, 0, 0, 0, 66, 555, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6119, 6233, 5559, 6224, 6091, 6128, 6652] avg=6144.0000 max=6652 max_rank=7 min=5559 min_rank=3 imbalance_ratio=1.082682 max_load_reduction=42500 max_load_reduction_pct=0.864665 moved_to_echo=25095 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 347, 0, 3945, 1611, 299, 0, 0, 0, 0, 0, 0, 0, 6007, 0, 0, 73, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4182, 650, 0, 0, 746, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 5750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 75, 0, 45, 0, 3933, 2071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 6046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 5623, 0, 0, 0, 69, 557, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6202, 6083, 6265, 5597, 6183, 6142, 6110, 6570] avg=6144.0000 max=6570 max_rank=7 min=5597 min_rank=3 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=25005 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 279, 0, 3950, 1570, 301, 0, 0, 0, 0, 0, 0, 0, 6034, 0, 0, 91, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5083, 687, 0, 0, 749, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 5607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 95, 0, 84, 0, 3938, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 6032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 4686, 0, 0, 0, 72, 554, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6202, 6228, 6538, 6014, 6203, 6206, 5661] avg=6144.0000 max=6538 max_rank=3 min=5661 min_rank=7 imbalance_ratio=1.064128 max_load_reduction=42614 max_load_reduction_pct=0.866984 moved_to_echo=24084 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 321, 0, 3953, 1541, 300, 0, 0, 0, 0, 0, 0, 0, 6028, 0, 0, 70, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4944, 726, 0, 0, 748, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 5501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 71, 0, 56, 0, 3940, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 6049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 4740, 0, 0, 0, 92, 565, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 6196, 6345, 6437, 6079, 6140, 6130, 5710] avg=6144.0000 max=6437 max_rank=3 min=5710 min_rank=7 imbalance_ratio=1.047689 max_load_reduction=42715 max_load_reduction_pct=0.869039 moved_to_echo=24059 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 295, 0, 3929, 1650, 303, 0, 0, 0, 0, 0, 0, 0, 6080, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4596, 682, 0, 0, 755, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 5890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 63, 0, 6, 0, 3917, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 6056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 5374, 0, 0, 0, 75, 538, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6143, 6045, 6051, 6251, 6075, 6104, 6306] avg=6144.0000 max=6306 max_rank=7 min=6045 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24736 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 7, 11, 5, 306, 1, 3937, 1589, 301, 0, 0, 0, 0, 0, 0, 0, 5964, 0, 0, 91, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 10, 6008, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4530, 761, 7, 0, 745, 22, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 510, 5661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 96, 0, 100, 0, 3923, 2064, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 137, 0, 0, 6026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 5329, 0, 1, 1, 69, 540, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 6072, 6027, 6067, 6182, 6195, 6164, 6281] avg=6144.0000 max=6281 max_rank=7 min=6027 min_rank=2 imbalance_ratio=1.022298 max_load_reduction=42871 max_load_reduction_pct=0.872213 moved_to_echo=24822 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 319, 0, 3940, 1589, 300, 0, 0, 0, 0, 0, 0, 0, 6023, 0, 0, 80, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4879, 706, 0, 0, 747, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 5672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 82, 0, 78, 0, 3927, 2069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 6041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 5121, 0, 0, 0, 43, 541, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6115, 6085, 6350, 6132, 6158, 6113, 6051] avg=6144.0000 max=6350 max_rank=3 min=6051 min_rank=7 imbalance_ratio=1.033529 max_load_reduction=42802 max_load_reduction_pct=0.870809 moved_to_echo=24454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 128, 2560, 128, 31616, 12800, 2432, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 37376, 5760, 128, 0, 6016, 256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 2688, 41600, 0, 128, 128, 640, 4480, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 768, 0, 512, 0, 31488, 16640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 48256, 0, 0, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 896, 0, 0, 48384, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 3712, 45440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 48256, 0, 0, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 768, 0, 512, 0, 31488, 16640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True - [2026-08-26 15:39:22] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 800.5 | throughput per GPU (TFLOP/s/GPU): 159.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.785368E+00 | loss scale: 1.0 | grad norm: 5.697 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1407, 310, 859, 913, 255, 895, 802, 896, 1408, 226, 704, 312, 301, 1535, 879, 1050, 670, 955, 880, 587, 281, 530, 251, 1357, 453, 720, 806, 369, 717, 96, 822, 1408, 848, 913, 1133, 1094, 484, 227, 571, 936, 574, 586, 529, 739, 877, 974, 490, 601, 688, 564, 842, 586, 1071, 1359, 779, 805, 1524, 290, 1485, 766, 187, 722, 1311, 943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 + [2026-08-26 16:35:24] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 810.4 | throughput per GPU (TFLOP/s/GPU): 157.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.785368E+00 | loss scale: 1.0 | grad norm: 5.697 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1328, 290, 766, 932, 265, 1052, 864, 796, 1376, 209, 641, 330, 241, 1491, 966, 1088, 722, 950, 853, 593, 235, 520, 234, 1361, 514, 647, 785, 430, 764, 146, 843, 1235, 832, 778, 1206, 1129, 463, 203, 648, 948, 509, 517, 529, 712, 921, 1002, 521, 575, 754, 569, 769, 508, 1024, 1447, 766, 825, 1596, 407, 1586, 771, 132, 796, 1295, 947] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1320, 293, 790, 888, 241, 957, 865, 870, 1417, 190, 659, 305, 262, 1565, 893, 1080, 691, 1002, 821, 677, 227, 510, 227, 1413, 472, 696, 737, 349, 711, 124, 804, 1145, 845, 784, 1243, 1149, 641, 193, 622, 1007, 538, 586, 651, 756, 879, 915, 508, 502, 732, 571, 762, 489, 1019, 1499, 812, 757, 1612, 317, 1562, 725, 184, 761, 1354, 976] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1434, 296, 789, 881, 246, 858, 880, 829, 1403, 217, 641, 280, 262, 1703, 1008, 1213, 663, 995, 819, 590, 248, 569, 251, 1372, 418, 719, 828, 375, 737, 109, 774, 1181, 752, 840, 1211, 1081, 616, 221, 604, 915, 568, 574, 543, 776, 929, 941, 453, 478, 729, 611, 766, 511, 1016, 1441, 850, 744, 1680, 247, 1484, 776, 199, 760, 1351, 897] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1407, 310, 859, 913, 255, 895, 802, 896, 1408, 226, 704, 312, 301, 1535, 879, 1050, 670, 955, 880, 587, 281, 530, 251, 1357, 453, 720, 806, 369, 717, 96, 822, 1408, 848, 913, 1133, 1094, 484, 227, 571, 936, 574, 586, 529, 739, 877, 974, 490, 601, 688, 564, 842, 586, 1071, 1359, 779, 805, 1524, 290, 1485, 766, 187, 722, 1311, 943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1374, 304, 702, 1002, 264, 1013, 836, 849, 1469, 248, 705, 352, 305, 1588, 963, 1199, 659, 920, 867, 664, 295, 543, 265, 1240, 470, 582, 742, 390, 722, 104, 819, 1296, 752, 885, 917, 1108, 664, 195, 634, 924, 537, 611, 531, 695, 952, 940, 486, 545, 713, 512, 749, 570, 1016, 1458, 716, 827, 1570, 369, 1495, 741, 158, 803, 1292, 1036] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1379, 347, 778, 928, 257, 938, 839, 921, 1312, 283, 733, 289, 284, 1647, 847, 1085, 683, 1042, 808, 598, 256, 456, 244, 1340, 431, 735, 825, 359, 727, 80, 886, 1179, 816, 844, 1335, 1121, 638, 200, 571, 839, 624, 650, 593, 731, 987, 950, 473, 481, 656, 587, 743, 512, 989, 1471, 754, 816, 1681, 263, 1472, 686, 155, 726, 1318, 954] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1328, 290, 766, 932, 265, 1052, 864, 796, 1376, 209, 641, 330, 241, 1491, 966, 1088, 722, 950, 853, 593, 235, 520, 234, 1361, 514, 647, 785, 430, 764, 146, 843, 1235, 832, 778, 1206, 1129, 463, 203, 648, 948, 509, 517, 529, 712, 921, 1002, 521, 575, 754, 569, 769, 508, 1024, 1447, 766, 825, 1596, 407, 1586, 771, 132, 796, 1295, 947] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1390, 278, 725, 981, 243, 967, 738, 720, 1410, 235, 622, 276, 233, 1713, 838, 1120, 719, 1030, 857, 603, 247, 471, 294, 1392, 455, 618, 764, 381, 785, 114, 855, 1141, 899, 920, 1140, 1118, 733, 215, 561, 1004, 679, 576, 592, 746, 904, 993, 543, 577, 630, 564, 777, 513, 1013, 1445, 696, 918, 1520, 358, 1457, 665, 164, 710, 1282, 1025] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1434, 296, 789, 881, 246, 858, 880, 829, 1403, 217, 641, 280, 262, 1703, 1008, 1213, 663, 995, 819, 590, 248, 569, 251, 1372, 418, 719, 828, 375, 737, 109, 774, 1181, 752, 840, 1211, 1081, 616, 221, 604, 915, 568, 574, 543, 776, 929, 941, 453, 478, 729, 611, 766, 511, 1016, 1441, 850, 744, 1680, 247, 1484, 776, 199, 760, 1351, 897] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1370, 317, 742, 959, 303, 836, 834, 858, 1419, 238, 555, 323, 323, 1587, 894, 1133, 656, 990, 781, 548, 288, 506, 272, 1217, 437, 709, 803, 344, 739, 104, 868, 1122, 814, 782, 1257, 1126, 788, 191, 657, 941, 533, 587, 645, 818, 896, 991, 503, 600, 684, 586, 758, 517, 1017, 1375, 758, 799, 1644, 327, 1584, 679, 163, 734, 1254, 1069] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1379, 347, 778, 928, 257, 938, 839, 921, 1312, 283, 733, 289, 284, 1647, 847, 1085, 683, 1042, 808, 598, 256, 456, 244, 1340, 431, 735, 825, 359, 727, 80, 886, 1179, 816, 844, 1335, 1121, 638, 200, 571, 839, 624, 650, 593, 731, 987, 950, 473, 481, 656, 587, 743, 512, 989, 1471, 754, 816, 1681, 263, 1472, 686, 155, 726, 1318, 954] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1212, 293, 790, 888, 241, 957, 865, 870, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 190, 659, 305, 262, 1209, 893, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 691, 1002, 821, 677, 227, 510, 227, 1413, 349, 207, 108, 0, 0, 0, 0, 0, 472, 696, 737, 349, 711, 124, 804, 1145, 959, 0, 0, 0, 0, 0, 0, 0, 845, 784, 1036, 1149, 641, 193, 622, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 538, 586, 651, 756, 879, 915, 508, 502, 476, 278, 7, 0, 0, 0, 0, 0, 732, 571, 762, 489, 1019, 1023, 812, 757, 0, 0, 0, 0, 0, 0, 0, 0, 375, 317, 1562, 725, 184, 761, 1354, 976, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6116, 6015, 6232, 5997, 6277, 6096, 6165, 6254] avg=6144.0000 max=6277 max_rank=4 min=5997 min_rank=3 imbalance_ratio=1.021647 max_load_reduction=42875 max_load_reduction_pct=0.872294 moved_to_echo=24792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1292, 310, 859, 913, 255, 895, 802, 896, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 226, 704, 312, 301, 1186, 879, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 670, 955, 880, 587, 281, 530, 251, 1357, 342, 188, 115, 0, 0, 0, 0, 0, 453, 720, 806, 369, 717, 96, 822, 1408, 906, 0, 0, 0, 0, 0, 0, 0, 848, 913, 945, 1094, 484, 227, 571, 936, 0, 0, 0, 0, 0, 0, 0, 0, 574, 586, 529, 739, 877, 974, 490, 601, 431, 262, 7, 0, 0, 0, 0, 0, 688, 564, 842, 586, 1071, 928, 779, 805, 0, 0, 0, 0, 0, 0, 0, 0, 356, 290, 1485, 766, 187, 722, 1311, 943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6222, 6066, 6156, 6297, 6018, 6070, 6263, 6060] avg=6144.0000 max=6297 max_rank=3 min=6018 min_rank=4 imbalance_ratio=1.024902 max_load_reduction=42855 max_load_reduction_pct=0.871887 moved_to_echo=24411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1316, 296, 789, 881, 246, 858, 880, 829, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 217, 641, 280, 262, 1315, 1008, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 663, 995, 819, 590, 248, 569, 251, 1372, 380, 201, 118, 0, 0, 0, 0, 0, 418, 719, 828, 375, 737, 109, 774, 1181, 999, 0, 0, 0, 0, 0, 0, 0, 752, 840, 1010, 1081, 616, 221, 604, 915, 0, 0, 0, 0, 0, 0, 0, 0, 568, 574, 543, 776, 929, 941, 453, 478, 457, 289, 8, 0, 0, 0, 0, 0, 729, 611, 766, 511, 1016, 984, 850, 744, 0, 0, 0, 0, 0, 0, 0, 0, 392, 247, 1484, 776, 199, 760, 1351, 897, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6339, 6206, 6140, 6039, 6016, 6211, 6106] avg=6144.0000 max=6339 max_rank=1 min=6016 min_rank=5 imbalance_ratio=1.031738 max_load_reduction=42813 max_load_reduction_pct=0.871033 moved_to_echo=24372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1261, 304, 702, 1002, 264, 1013, 836, 849, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 248, 705, 352, 305, 1226, 963, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 659, 920, 867, 664, 295, 543, 265, 1240, 354, 152, 113, 0, 0, 0, 0, 0, 470, 582, 742, 390, 722, 104, 819, 1296, 934, 0, 0, 0, 0, 0, 0, 0, 752, 885, 765, 1108, 664, 195, 634, 924, 0, 0, 0, 0, 0, 0, 0, 0, 537, 611, 531, 695, 952, 940, 486, 545, 463, 270, 8, 0, 0, 0, 0, 0, 713, 512, 749, 570, 1016, 995, 716, 827, 0, 0, 0, 0, 0, 0, 0, 0, 366, 369, 1495, 741, 158, 803, 1292, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6231, 6467, 6072, 6059, 5927, 6038, 6098, 6260] avg=6144.0000 max=6467 max_rank=1 min=5927 min_rank=4 imbalance_ratio=1.052572 max_load_reduction=42685 max_load_reduction_pct=0.868429 moved_to_echo=24323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1266, 347, 778, 928, 257, 938, 839, 921, 0, 0, 0, 0, 0, 0, 0, 0, 1312, 283, 733, 289, 284, 1272, 847, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 683, 1042, 808, 598, 256, 456, 244, 1340, 367, 222, 113, 0, 0, 0, 0, 0, 431, 735, 825, 359, 727, 80, 886, 1179, 1000, 0, 0, 0, 0, 0, 0, 0, 816, 844, 1113, 1121, 638, 200, 571, 839, 0, 0, 0, 0, 0, 0, 0, 0, 624, 650, 593, 731, 987, 950, 473, 481, 467, 290, 8, 0, 0, 0, 0, 0, 656, 587, 743, 512, 989, 1004, 754, 816, 0, 0, 0, 0, 0, 0, 0, 0, 391, 263, 1472, 686, 155, 726, 1318, 954, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 6105, 6129, 6222, 6142, 6254, 6061, 5965] avg=6144.0000 max=6274 max_rank=0 min=5965 min_rank=7 imbalance_ratio=1.021159 max_load_reduction=42878 max_load_reduction_pct=0.872355 moved_to_echo=24422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1276, 278, 725, 981, 243, 967, 738, 720, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 235, 622, 276, 233, 1323, 838, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 719, 1030, 857, 603, 247, 471, 294, 1392, 382, 190, 114, 0, 0, 0, 0, 0, 455, 618, 764, 381, 785, 114, 855, 1141, 904, 0, 0, 0, 0, 0, 0, 0, 899, 920, 950, 1118, 733, 215, 561, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 679, 576, 592, 746, 904, 993, 543, 577, 459, 262, 8, 0, 0, 0, 0, 0, 630, 564, 777, 513, 1013, 986, 696, 918, 0, 0, 0, 0, 0, 0, 0, 0, 354, 358, 1457, 665, 164, 710, 1282, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5928, 6057, 6299, 6017, 6400, 6339, 6097, 6015] avg=6144.0000 max=6400 max_rank=4 min=5928 min_rank=0 imbalance_ratio=1.041667 max_load_reduction=42752 max_load_reduction_pct=0.869792 moved_to_echo=24851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1219, 290, 766, 932, 265, 1052, 864, 796, 0, 0, 0, 0, 0, 0, 0, 0, 1376, 209, 641, 330, 241, 1151, 966, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 722, 950, 853, 593, 235, 520, 234, 1361, 333, 201, 109, 0, 0, 0, 0, 0, 514, 647, 785, 430, 764, 146, 843, 1235, 949, 0, 0, 0, 0, 0, 0, 0, 832, 778, 1005, 1129, 463, 203, 648, 948, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 529, 712, 921, 1002, 521, 575, 459, 275, 7, 0, 0, 0, 0, 0, 754, 569, 769, 508, 1024, 988, 766, 825, 0, 0, 0, 0, 0, 0, 0, 0, 372, 407, 1586, 771, 132, 796, 1295, 947, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 6002, 6111, 6313, 6006, 6027, 6203, 6306] avg=6144.0000 max=6313 max_rank=3 min=6002 min_rank=1 imbalance_ratio=1.027507 max_load_reduction=42839 max_load_reduction_pct=0.871562 moved_to_echo=24542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1266, 347, 778, 928, 257, 938, 839, 921, 0, 0, 0, 0, 0, 0, 0, 0, 1312, 283, 733, 289, 284, 1272, 847, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 683, 1042, 808, 598, 256, 456, 244, 1340, 367, 222, 113, 0, 0, 0, 0, 0, 431, 735, 825, 359, 727, 80, 886, 1179, 1000, 0, 0, 0, 0, 0, 0, 0, 816, 844, 1113, 1121, 638, 200, 571, 839, 0, 0, 0, 0, 0, 0, 0, 0, 624, 650, 593, 731, 987, 950, 473, 481, 467, 290, 8, 0, 0, 0, 0, 0, 656, 587, 743, 512, 989, 1004, 754, 816, 0, 0, 0, 0, 0, 0, 0, 0, 391, 263, 1472, 686, 155, 726, 1318, 954, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 6105, 6129, 6222, 6142, 6254, 6061, 5965] avg=6144.0000 max=6274 max_rank=0 min=5965 min_rank=7 imbalance_ratio=1.021159 max_load_reduction=42878 max_load_reduction_pct=0.872355 moved_to_echo=24422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1253, 317, 742, 959, 303, 836, 834, 858, 0, 0, 0, 0, 0, 0, 0, 0, 1419, 238, 555, 323, 323, 1216, 894, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 656, 990, 781, 548, 288, 506, 272, 1217, 359, 213, 117, 0, 0, 0, 0, 0, 437, 709, 803, 344, 739, 104, 868, 1122, 981, 0, 0, 0, 0, 0, 0, 0, 814, 782, 1044, 1126, 788, 191, 657, 941, 0, 0, 0, 0, 0, 0, 0, 0, 533, 587, 645, 818, 896, 991, 503, 600, 440, 287, 12, 0, 0, 0, 0, 0, 684, 586, 758, 517, 1017, 935, 758, 799, 0, 0, 0, 0, 0, 0, 0, 0, 376, 327, 1584, 679, 163, 734, 1254, 1069, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 6101, 5947, 6107, 6343, 6312, 6054, 6186] avg=6144.0000 max=6343 max_rank=4 min=5947 min_rank=2 imbalance_ratio=1.032389 max_load_reduction=42809 max_load_reduction_pct=0.870951 moved_to_echo=24895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1276, 278, 725, 981, 243, 967, 738, 720, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 235, 622, 276, 233, 1323, 838, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 719, 1030, 857, 603, 247, 471, 294, 1392, 382, 190, 114, 0, 0, 0, 0, 0, 455, 618, 764, 381, 785, 114, 855, 1141, 904, 0, 0, 0, 0, 0, 0, 0, 899, 920, 950, 1118, 733, 215, 561, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 679, 576, 592, 746, 904, 993, 543, 577, 459, 262, 8, 0, 0, 0, 0, 0, 630, 564, 777, 513, 1013, 986, 696, 918, 0, 0, 0, 0, 0, 0, 0, 0, 354, 358, 1457, 665, 164, 710, 1282, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5928, 6057, 6299, 6017, 6400, 6339, 6097, 6015] avg=6144.0000 max=6400 max_rank=4 min=5928 min_rank=0 imbalance_ratio=1.041667 max_load_reduction=42752 max_load_reduction_pct=0.869792 moved_to_echo=24851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10112, 2560, 6272, 7552, 2176, 7552, 6784, 6784, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6656, 6784, 7936, 8960, 5120, 1664, 4992, 7552, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11264, 1920, 5376, 2560, 2304, 9984, 7296, 9088, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[4608, 4736, 4736, 6016, 7424, 7808, 4096, 4480, 3712, 2304, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10112, 2560, 6272, 7552, 2176, 7552, 6784, 6784, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5632, 4608, 6272, 4224, 8192, 7936, 6144, 6528, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 2688, 12160, 5888, 1408, 6016, 10496, 7936, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5504, 7936, 6784, 4864, 2176, 4224, 2048, 10752, 2944, 1664, 1024, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 5504, 6400, 3072, 6016, 896, 6784, 9728, 7680, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[4608, 4736, 4736, 6016, 7424, 7808, 4096, 4480, 3712, 2304, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 2688, 12160, 5888, 1408, 6016, 10496, 7936, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[623, 893, 673, 450, 1068, 291, 320, 1191, 61, 904, 1598, 320, 601, 1088, 722, 1708, 1083, 1319, 820, 484, 1512, 129, 1837, 1042, 714, 719, 740, 892, 81, 306, 176, 123, 425, 162, 462, 205, 1103, 1536, 1018, 1051, 2107, 1386, 1057, 1438, 313, 913, 16, 186, 882, 1139, 1001, 20, 121, 1583, 199, 1440, 881, 321, 863, 535, 1042, 565, 317, 377] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[639, 856, 703, 449, 1089, 207, 322, 1283, 107, 932, 1539, 338, 619, 1018, 769, 1630, 1204, 1277, 963, 454, 1322, 138, 1684, 1085, 621, 711, 660, 915, 65, 292, 168, 105, 413, 208, 463, 161, 1256, 1546, 1009, 938, 2115, 1476, 997, 1334, 305, 1080, 15, 154, 917, 1238, 1126, 7, 109, 1551, 241, 1332, 966, 308, 908, 561, 946, 573, 318, 417] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[609, 818, 901, 389, 1059, 233, 323, 1148, 67, 960, 1488, 335, 613, 980, 707, 1618, 1070, 1213, 889, 533, 1391, 205, 1829, 1120, 763, 746, 751, 910, 82, 323, 165, 133, 340, 219, 434, 148, 1196, 1560, 1007, 935, 2202, 1483, 997, 1232, 357, 1096, 16, 191, 867, 1198, 1104, 9, 134, 1498, 239, 1414, 997, 338, 781, 514, 949, 572, 375, 379] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[623, 850, 584, 453, 1404, 276, 383, 1274, 112, 962, 1644, 361, 709, 1035, 700, 1767, 974, 1173, 825, 349, 1378, 148, 1693, 1079, 750, 633, 788, 966, 71, 201, 190, 132, 424, 169, 431, 101, 1163, 1291, 926, 930, 1982, 1406, 930, 1355, 309, 1092, 35, 218, 1124, 1317, 1304, 9, 84, 1375, 236, 1319, 996, 329, 926, 606, 940, 576, 352, 410] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[582, 783, 884, 428, 1297, 233, 307, 1152, 70, 933, 1644, 348, 640, 943, 688, 1781, 1001, 1228, 936, 451, 1385, 183, 1833, 1185, 735, 793, 846, 899, 85, 337, 200, 125, 343, 232, 390, 114, 1137, 1258, 848, 917, 2185, 1332, 939, 1245, 276, 947, 20, 191, 1184, 1238, 1259, 8, 115, 1268, 219, 1428, 977, 318, 921, 587, 930, 586, 355, 450] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[650, 837, 797, 401, 1326, 236, 351, 1134, 92, 891, 1544, 347, 622, 1113, 804, 1761, 1001, 1154, 886, 432, 1526, 168, 1783, 1152, 681, 712, 717, 932, 80, 251, 205, 144, 369, 154, 421, 114, 1099, 1515, 844, 1013, 2177, 1358, 972, 1311, 280, 998, 37, 218, 1035, 1170, 998, 7, 120, 1398, 244, 1413, 1002, 358, 920, 628, 908, 602, 322, 417] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[582, 783, 884, 428, 1297, 233, 307, 1152, 70, 933, 1644, 348, 640, 943, 688, 1781, 1001, 1228, 936, 451, 1385, 183, 1833, 1185, 735, 793, 846, 899, 85, 337, 200, 125, 343, 232, 390, 114, 1137, 1258, 848, 917, 2185, 1332, 939, 1245, 276, 947, 20, 191, 1184, 1238, 1259, 8, 115, 1268, 219, 1428, 977, 318, 921, 587, 930, 586, 355, 450] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[623, 850, 584, 453, 1404, 276, 383, 1274, 112, 962, 1644, 361, 709, 1035, 700, 1767, 974, 1173, 825, 349, 1378, 148, 1693, 1079, 750, 633, 788, 966, 71, 201, 190, 132, 424, 169, 431, 101, 1163, 1291, 926, 930, 1982, 1406, 930, 1355, 309, 1092, 35, 218, 1124, 1317, 1304, 9, 84, 1375, 236, 1319, 996, 329, 926, 606, 940, 576, 352, 410] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[623, 893, 673, 450, 1068, 291, 320, 1191, 61, 904, 1598, 320, 601, 1088, 722, 1708, 1083, 1319, 820, 484, 1512, 129, 1837, 1042, 714, 719, 740, 892, 81, 306, 176, 123, 425, 162, 462, 205, 1103, 1536, 1018, 1051, 2107, 1386, 1057, 1438, 313, 913, 16, 186, 882, 1139, 1001, 20, 121, 1583, 199, 1440, 881, 321, 863, 535, 1042, 565, 317, 377] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[644, 733, 810, 585, 1269, 163, 260, 1186, 83, 987, 1590, 334, 618, 1062, 801, 1656, 1159, 1155, 860, 455, 1290, 160, 1610, 1298, 691, 701, 765, 916, 64, 371, 224, 128, 362, 181, 389, 109, 1115, 1386, 1014, 920, 2187, 1382, 975, 1223, 272, 1094, 10, 201, 1070, 1240, 1276, 5, 107, 1463, 191, 1320, 934, 347, 950, 561, 945, 541, 320, 434] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[523, 998, 570, 420, 1290, 260, 338, 1190, 79, 1011, 1814, 367, 603, 1027, 779, 1778, 1068, 1188, 817, 416, 1289, 195, 1639, 1234, 752, 805, 750, 884, 63, 205, 180, 121, 353, 170, 458, 120, 1203, 1213, 975, 998, 1920, 1384, 971, 1275, 267, 1024, 17, 214, 1093, 1315, 1447, 4, 67, 1388, 217, 1459, 895, 307, 893, 530, 943, 575, 373, 431] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[623, 893, 673, 450, 1068, 291, 320, 1191, 389, 177, 0, 0, 0, 0, 0, 0, 61, 904, 1598, 320, 601, 1088, 722, 753, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1319, 820, 484, 1335, 129, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 714, 719, 740, 892, 81, 306, 176, 123, 1837, 623, 0, 0, 0, 0, 0, 0, 425, 162, 462, 205, 1103, 1536, 1018, 1051, 407, 126, 0, 0, 0, 0, 0, 0, 916, 1386, 1057, 1438, 313, 913, 16, 186, 0, 0, 0, 0, 0, 0, 0, 0, 882, 1139, 1001, 20, 121, 1068, 199, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 881, 321, 863, 535, 1042, 565, 317, 377, 568, 548, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 6047, 6212, 6211, 6495, 6225, 5870, 6017] avg=6144.0000 max=6495 max_rank=4 min=5870 min_rank=6 imbalance_ratio=1.057129 max_load_reduction=42657 max_load_reduction_pct=0.867859 moved_to_echo=24607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[639, 856, 703, 449, 1089, 207, 322, 1283, 381, 154, 0, 0, 0, 0, 0, 0, 107, 932, 1539, 338, 619, 1018, 769, 719, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 1277, 963, 454, 1168, 138, 0, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 621, 711, 660, 915, 65, 292, 168, 105, 1684, 625, 0, 0, 0, 0, 0, 0, 413, 208, 463, 161, 1256, 1546, 1009, 938, 388, 123, 0, 0, 0, 0, 0, 0, 920, 1476, 997, 1334, 305, 1080, 15, 154, 0, 0, 0, 0, 0, 0, 0, 0, 917, 1238, 1126, 7, 109, 1047, 241, 1332, 0, 0, 0, 0, 0, 0, 0, 0, 966, 308, 908, 561, 946, 573, 318, 417, 570, 523, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6083, 6041, 6289, 5846, 6505, 6281, 6017, 6090] avg=6144.0000 max=6505 max_rank=4 min=5846 min_rank=3 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=24893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[623, 850, 584, 453, 1404, 276, 383, 1274, 337, 161, 0, 0, 0, 0, 0, 0, 112, 962, 1644, 361, 709, 1035, 700, 779, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1173, 825, 349, 1217, 148, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 750, 633, 788, 966, 71, 201, 190, 132, 1693, 586, 0, 0, 0, 0, 0, 0, 424, 169, 431, 101, 1163, 1291, 926, 930, 421, 109, 0, 0, 0, 0, 0, 0, 862, 1406, 930, 1355, 309, 1092, 35, 218, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 1317, 1304, 9, 84, 929, 236, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 996, 329, 926, 606, 940, 576, 352, 410, 534, 567, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6345, 6302, 5765, 6010, 5965, 6207, 6322, 6236] avg=6144.0000 max=6345 max_rank=0 min=5765 min_rank=2 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[609, 818, 901, 389, 1059, 233, 323, 1148, 368, 162, 0, 0, 0, 0, 0, 0, 67, 960, 1488, 335, 613, 980, 707, 713, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1213, 889, 533, 1229, 205, 0, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 763, 746, 751, 910, 82, 323, 165, 133, 1829, 651, 0, 0, 0, 0, 0, 0, 340, 219, 434, 148, 1196, 1560, 1007, 935, 386, 119, 0, 0, 0, 0, 0, 0, 957, 1483, 997, 1232, 357, 1096, 16, 191, 0, 0, 0, 0, 0, 0, 0, 0, 867, 1198, 1104, 9, 134, 1011, 239, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 997, 338, 781, 514, 949, 572, 375, 379, 594, 519, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6010, 5863, 6259, 6353, 6344, 6329, 5976, 6018] avg=6144.0000 max=6353 max_rank=3 min=5863 min_rank=1 imbalance_ratio=1.034017 max_load_reduction=42799 max_load_reduction_pct=0.870748 moved_to_echo=24667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[582, 783, 884, 428, 1297, 233, 307, 1152, 311, 162, 0, 0, 0, 0, 0, 0, 70, 933, 1644, 348, 640, 943, 688, 784, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1228, 936, 451, 1223, 183, 0, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 735, 793, 846, 899, 85, 337, 200, 125, 1833, 646, 0, 0, 0, 0, 0, 0, 343, 232, 390, 114, 1137, 1258, 848, 917, 425, 101, 0, 0, 0, 0, 0, 0, 950, 1332, 939, 1245, 276, 947, 20, 191, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 1238, 1259, 8, 115, 856, 219, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 977, 318, 921, 587, 930, 586, 355, 450, 589, 572, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6050, 6207, 6499, 5765, 5900, 6307, 6285] avg=6144.0000 max=6499 max_rank=3 min=5765 min_rank=4 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=24257 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[623, 893, 673, 450, 1068, 291, 320, 1191, 389, 177, 0, 0, 0, 0, 0, 0, 61, 904, 1598, 320, 601, 1088, 722, 753, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1319, 820, 484, 1335, 129, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 714, 719, 740, 892, 81, 306, 176, 123, 1837, 623, 0, 0, 0, 0, 0, 0, 425, 162, 462, 205, 1103, 1536, 1018, 1051, 407, 126, 0, 0, 0, 0, 0, 0, 916, 1386, 1057, 1438, 313, 913, 16, 186, 0, 0, 0, 0, 0, 0, 0, 0, 882, 1139, 1001, 20, 121, 1068, 199, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 881, 321, 863, 535, 1042, 565, 317, 377, 568, 548, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 6047, 6212, 6211, 6495, 6225, 5870, 6017] avg=6144.0000 max=6495 max_rank=4 min=5870 min_rank=6 imbalance_ratio=1.057129 max_load_reduction=42657 max_load_reduction_pct=0.867859 moved_to_echo=24607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[623, 850, 584, 453, 1404, 276, 383, 1274, 337, 161, 0, 0, 0, 0, 0, 0, 112, 962, 1644, 361, 709, 1035, 700, 779, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1173, 825, 349, 1217, 148, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 750, 633, 788, 966, 71, 201, 190, 132, 1693, 586, 0, 0, 0, 0, 0, 0, 424, 169, 431, 101, 1163, 1291, 926, 930, 421, 109, 0, 0, 0, 0, 0, 0, 862, 1406, 930, 1355, 309, 1092, 35, 218, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 1317, 1304, 9, 84, 929, 236, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 996, 329, 926, 606, 940, 576, 352, 410, 534, 567, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6345, 6302, 5765, 6010, 5965, 6207, 6322, 6236] avg=6144.0000 max=6345 max_rank=0 min=5765 min_rank=2 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[644, 733, 810, 585, 1269, 163, 260, 1186, 359, 151, 0, 0, 0, 0, 0, 0, 83, 987, 1590, 334, 618, 1062, 801, 729, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 1155, 860, 455, 1139, 160, 0, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 691, 701, 765, 916, 64, 371, 224, 128, 1610, 646, 0, 0, 0, 0, 0, 0, 362, 181, 389, 109, 1115, 1386, 1014, 920, 395, 116, 0, 0, 0, 0, 0, 0, 951, 1382, 975, 1223, 272, 1094, 10, 201, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1240, 1276, 5, 107, 988, 191, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 934, 347, 950, 561, 945, 541, 320, 434, 590, 532, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6204, 6226, 6116, 5987, 6108, 6197, 6154] avg=6144.0000 max=6226 max_rank=2 min=5987 min_rank=4 imbalance_ratio=1.013346 max_load_reduction=42926 max_load_reduction_pct=0.873332 moved_to_echo=24446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[650, 837, 797, 401, 1326, 236, 351, 1134, 346, 182, 0, 0, 0, 0, 0, 0, 92, 891, 1544, 347, 622, 1113, 804, 769, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1154, 886, 432, 1344, 168, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 681, 712, 717, 932, 80, 251, 205, 144, 1783, 646, 0, 0, 0, 0, 0, 0, 369, 154, 421, 114, 1099, 1515, 844, 1013, 423, 115, 0, 0, 0, 0, 0, 0, 941, 1358, 972, 1311, 280, 998, 37, 218, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1170, 998, 7, 120, 937, 244, 1413, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 358, 920, 628, 908, 602, 322, 417, 590, 569, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6260, 6182, 6137, 6151, 6067, 6115, 5924, 6316] avg=6144.0000 max=6316 max_rank=7 min=5924 min_rank=6 imbalance_ratio=1.027995 max_load_reduction=42836 max_load_reduction_pct=0.871501 moved_to_echo=24422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 998, 570, 420, 1290, 260, 338, 1190, 341, 150, 0, 0, 0, 0, 0, 0, 79, 1011, 1814, 367, 603, 1027, 779, 783, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1188, 817, 416, 1139, 195, 0, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 752, 805, 750, 884, 63, 205, 180, 121, 1639, 567, 0, 0, 0, 0, 0, 0, 353, 170, 458, 120, 1203, 1213, 975, 998, 424, 110, 0, 0, 0, 0, 0, 0, 835, 1384, 971, 1275, 267, 1024, 17, 214, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 1315, 1447, 4, 67, 937, 217, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 895, 307, 893, 530, 943, 575, 373, 431, 518, 571, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6080, 6463, 6057, 5966, 6024, 5987, 6539, 6036] avg=6144.0000 max=6539 max_rank=6 min=5966 min_rank=3 imbalance_ratio=1.064290 max_load_reduction=42613 max_load_reduction_pct=0.866964 moved_to_echo=24586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[650, 837, 797, 401, 1326, 236, 351, 1134, 346, 182, 0, 0, 0, 0, 0, 0, 92, 891, 1544, 347, 622, 1113, 804, 769, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1154, 886, 432, 1344, 168, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 681, 712, 717, 932, 80, 251, 205, 144, 1783, 646, 0, 0, 0, 0, 0, 0, 369, 154, 421, 114, 1099, 1515, 844, 1013, 423, 115, 0, 0, 0, 0, 0, 0, 941, 1358, 972, 1311, 280, 998, 37, 218, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1170, 998, 7, 120, 937, 244, 1413, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 358, 920, 628, 908, 602, 322, 417, 590, 569, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6260, 6182, 6137, 6151, 6067, 6115, 5924, 6316] avg=6144.0000 max=6316 max_rank=7 min=5924 min_rank=6 imbalance_ratio=1.027995 max_load_reduction=42836 max_load_reduction_pct=0.871501 moved_to_echo=24422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 6784, 6016, 3584, 9856, 1920, 2688, 9600, 2944, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 11264, 7936, 10496, 2432, 8320, 256, 1664, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 2688, 7168, 4608, 7680, 4608, 2816, 3328, 4608, 4480, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 7680, 12928, 2816, 5120, 8320, 6016, 6144, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 11264, 7936, 10496, 2432, 8320, 256, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 1536, 3456, 1152, 9344, 11392, 7680, 7808, 3328, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8192, 9856, 9600, 128, 896, 7808, 1792, 11136, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 5888, 6144, 7424, 640, 2304, 1536, 1024, 13952, 4992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 6784, 6016, 3584, 9856, 1920, 2688, 9600, 2944, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 1536, 3456, 1152, 9344, 11392, 7680, 7808, 3328, 1024, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8576, 9728, 7040, 3584, 9856, 1408, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8192, 9856, 9600, 128, 896, 7808, 1792, 11136, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[16, 983, 510, 330, 25, 1385, 738, 234, 598, 426, 1216, 1561, 1566, 882, 38, 861, 1137, 1527, 422, 278, 81, 167, 1341, 1041, 532, 691, 1215, 1115, 230, 413, 982, 168, 401, 1361, 1234, 599, 3057, 1974, 1555, 516, 526, 157, 227, 266, 1555, 1209, 139, 1270, 767, 99, 847, 770, 33, 97, 267, 595, 1580, 455, 73, 442, 118, 1772, 128, 2354] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[35, 1272, 607, 344, 30, 1328, 834, 251, 656, 372, 1231, 1359, 1851, 1095, 65, 851, 1018, 1561, 331, 306, 66, 225, 1267, 931, 571, 709, 1053, 910, 269, 484, 1046, 208, 353, 1175, 1068, 653, 2635, 1907, 1802, 482, 555, 234, 322, 201, 1672, 1219, 193, 1208, 833, 148, 732, 752, 56, 102, 317, 558, 1651, 393, 66, 431, 159, 1564, 187, 2388] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[23, 879, 555, 311, 47, 1422, 859, 256, 572, 418, 1251, 1386, 1619, 1002, 43, 1219, 1073, 1611, 421, 313, 92, 165, 1253, 1150, 517, 640, 1115, 1100, 291, 488, 1068, 206, 390, 1165, 1156, 596, 2403, 2020, 1652, 483, 517, 157, 318, 331, 1539, 1116, 145, 1153, 832, 128, 938, 849, 43, 104, 248, 635, 1562, 372, 83, 463, 90, 1653, 198, 2448] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[46, 1015, 549, 370, 69, 1379, 819, 269, 565, 254, 1107, 1416, 1641, 1103, 202, 1391, 1064, 1320, 270, 362, 83, 261, 1332, 872, 499, 680, 1118, 973, 254, 494, 882, 247, 398, 1161, 1153, 736, 2164, 1811, 1937, 520, 371, 207, 269, 143, 1855, 1271, 157, 1130, 908, 118, 795, 771, 105, 148, 284, 769, 1595, 413, 67, 432, 151, 1600, 392, 2415] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[31, 1052, 521, 318, 34, 1375, 795, 269, 646, 462, 1096, 1421, 1547, 996, 48, 956, 1050, 1614, 366, 339, 80, 162, 1270, 955, 581, 730, 1065, 981, 226, 498, 982, 176, 383, 1301, 1217, 557, 2678, 1859, 1766, 466, 438, 198, 316, 221, 1668, 1284, 159, 1165, 716, 87, 937, 858, 47, 126, 232, 655, 1652, 526, 56, 503, 97, 1679, 181, 2482] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[12, 1129, 664, 292, 15, 1207, 765, 236, 618, 443, 1138, 1373, 1600, 832, 20, 741, 1095, 1750, 366, 329, 86, 122, 1176, 995, 639, 731, 1161, 1076, 230, 432, 831, 156, 394, 1140, 1196, 697, 2892, 2346, 1660, 477, 503, 186, 316, 237, 1630, 1244, 137, 1105, 640, 80, 945, 976, 49, 58, 165, 552, 1817, 505, 43, 406, 47, 1927, 133, 2389] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[16, 983, 510, 330, 25, 1385, 738, 234, 598, 426, 1216, 1561, 1566, 882, 38, 861, 1137, 1527, 422, 278, 81, 167, 1341, 1041, 532, 691, 1215, 1115, 230, 413, 982, 168, 401, 1361, 1234, 599, 3057, 1974, 1555, 516, 526, 157, 227, 266, 1555, 1209, 139, 1270, 767, 99, 847, 770, 33, 97, 267, 595, 1580, 455, 73, 442, 118, 1772, 128, 2354] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[92, 991, 564, 263, 82, 1274, 856, 300, 621, 341, 1085, 1279, 1624, 1169, 176, 1325, 968, 1551, 348, 368, 73, 219, 1239, 911, 449, 684, 1071, 827, 294, 495, 935, 338, 405, 1103, 1201, 718, 2255, 1817, 1817, 533, 350, 244, 337, 192, 1717, 1201, 138, 1193, 758, 134, 1112, 790, 106, 165, 268, 745, 1553, 408, 102, 505, 120, 1540, 420, 2393] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[12, 1129, 664, 292, 15, 1207, 765, 236, 618, 443, 1138, 1373, 1600, 832, 20, 741, 1095, 1750, 366, 329, 86, 122, 1176, 995, 639, 731, 1161, 1076, 230, 432, 831, 156, 394, 1140, 1196, 697, 2892, 2346, 1660, 477, 503, 186, 316, 237, 1630, 1244, 137, 1105, 640, 80, 945, 976, 49, 58, 165, 552, 1817, 505, 43, 406, 47, 1927, 133, 2389] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[31, 1052, 521, 318, 34, 1375, 795, 269, 646, 462, 1096, 1421, 1547, 996, 48, 956, 1050, 1614, 366, 339, 80, 162, 1270, 955, 581, 730, 1065, 981, 226, 498, 982, 176, 383, 1301, 1217, 557, 2678, 1859, 1766, 466, 438, 198, 316, 221, 1668, 1284, 159, 1165, 716, 87, 937, 858, 47, 126, 232, 655, 1652, 526, 56, 503, 97, 1679, 181, 2482] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[14, 1042, 566, 337, 19, 1195, 745, 279, 697, 413, 1277, 1285, 1780, 952, 17, 805, 1080, 1768, 371, 326, 218, 175, 1237, 890, 684, 686, 1222, 879, 125, 439, 840, 196, 396, 1067, 1169, 620, 2855, 2376, 1712, 408, 467, 209, 387, 278, 1650, 1295, 132, 865, 748, 98, 673, 809, 44, 112, 227, 515, 1788, 512, 73, 377, 79, 1842, 180, 2630] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[31, 1052, 521, 318, 34, 1375, 795, 269, 1379, 149, 111, 0, 0, 0, 0, 0, 646, 462, 1096, 1421, 433, 996, 48, 956, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1614, 366, 339, 80, 162, 1270, 955, 298, 0, 0, 0, 0, 0, 0, 0, 581, 730, 1065, 981, 226, 498, 982, 176, 861, 0, 0, 0, 0, 0, 0, 0, 383, 1301, 1217, 557, 0, 480, 1766, 466, 0, 0, 0, 0, 0, 0, 0, 0, 438, 198, 316, 221, 1668, 1284, 159, 1165, 663, 104, 0, 0, 0, 0, 0, 0, 716, 87, 937, 858, 47, 126, 232, 655, 2567, 0, 0, 0, 0, 0, 0, 0, 1652, 526, 56, 503, 97, 1679, 181, 1521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6034, 6058, 6134, 6100, 6170, 6216, 6225, 6215] avg=6144.0000 max=6225 max_rank=6 min=6034 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=42927 max_load_reduction_pct=0.873352 moved_to_echo=24826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[16, 983, 510, 330, 25, 1385, 738, 234, 1458, 149, 125, 0, 0, 0, 0, 0, 598, 426, 1216, 1561, 448, 882, 38, 861, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 1527, 422, 278, 81, 167, 1341, 1041, 279, 0, 0, 0, 0, 0, 0, 0, 532, 691, 1215, 1115, 230, 413, 982, 168, 867, 0, 0, 0, 0, 0, 0, 0, 401, 1361, 1234, 599, 0, 516, 1555, 516, 0, 0, 0, 0, 0, 0, 0, 0, 526, 157, 227, 266, 1555, 1209, 139, 1270, 627, 102, 0, 0, 0, 0, 0, 0, 767, 99, 847, 770, 33, 97, 267, 595, 2932, 0, 0, 0, 0, 0, 0, 0, 1580, 455, 73, 442, 118, 1772, 128, 1448, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5953, 6030, 6273, 6213, 6182, 6078, 6407, 6016] avg=6144.0000 max=6407 max_rank=6 min=5953 min_rank=0 imbalance_ratio=1.042806 max_load_reduction=42745 max_load_reduction_pct=0.869649 moved_to_echo=24683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[23, 879, 555, 311, 47, 1422, 859, 256, 572, 418, 1251, 1386, 1619, 1002, 43, 1219, 1073, 1611, 421, 313, 92, 165, 1253, 1150, 517, 640, 1115, 1100, 291, 488, 1068, 206, 390, 1165, 1156, 596, 2403, 2020, 1652, 483, 517, 157, 318, 331, 1539, 1116, 145, 1153, 832, 128, 938, 849, 43, 104, 248, 635, 1562, 372, 83, 463, 90, 1653, 198, 2448] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[35, 1272, 607, 344, 30, 1328, 834, 251, 656, 372, 1231, 1359, 1851, 1095, 65, 851, 1018, 1561, 331, 306, 66, 225, 1267, 931, 571, 709, 1053, 910, 269, 484, 1046, 208, 353, 1175, 1068, 653, 2635, 1907, 1802, 482, 555, 234, 322, 201, 1672, 1219, 193, 1208, 833, 148, 732, 752, 56, 102, 317, 558, 1651, 393, 66, 431, 159, 1564, 187, 2388] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[46, 1015, 549, 370, 69, 1379, 819, 269, 1338, 156, 89, 0, 0, 0, 0, 0, 565, 254, 1107, 1416, 470, 1103, 202, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1320, 270, 362, 83, 261, 1332, 872, 286, 0, 0, 0, 0, 0, 0, 0, 499, 680, 1118, 973, 254, 494, 882, 247, 908, 0, 0, 0, 0, 0, 0, 0, 398, 1161, 1153, 736, 0, 473, 1937, 520, 0, 0, 0, 0, 0, 0, 0, 0, 371, 207, 269, 143, 1855, 1271, 157, 1130, 643, 107, 0, 0, 0, 0, 0, 0, 908, 118, 795, 771, 105, 148, 284, 769, 2075, 0, 0, 0, 0, 0, 0, 0, 1595, 413, 67, 432, 151, 1600, 392, 1486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6099, 6508, 5850, 6055, 6378, 6153, 5973, 6136] avg=6144.0000 max=6508 max_rank=1 min=5850 min_rank=2 imbalance_ratio=1.059245 max_load_reduction=42644 max_load_reduction_pct=0.867594 moved_to_echo=24640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[23, 879, 555, 311, 47, 1422, 859, 256, 1492, 154, 99, 0, 0, 0, 0, 0, 572, 418, 1251, 1386, 464, 1002, 43, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1611, 421, 313, 92, 165, 1253, 1150, 290, 0, 0, 0, 0, 0, 0, 0, 517, 640, 1115, 1100, 291, 488, 1068, 206, 896, 0, 0, 0, 0, 0, 0, 0, 390, 1165, 1156, 596, 0, 528, 1652, 483, 0, 0, 0, 0, 0, 0, 0, 0, 517, 157, 318, 331, 1539, 1116, 145, 1153, 652, 105, 0, 0, 0, 0, 0, 0, 832, 128, 938, 849, 43, 104, 248, 635, 2304, 0, 0, 0, 0, 0, 0, 0, 1562, 372, 83, 463, 90, 1653, 198, 1506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6355, 6368, 6321, 5970, 6033, 6081, 5927] avg=6144.0000 max=6368 max_rank=2 min=5927 min_rank=7 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[16, 983, 510, 330, 25, 1385, 738, 234, 1458, 149, 125, 0, 0, 0, 0, 0, 598, 426, 1216, 1561, 448, 882, 38, 861, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 1527, 422, 278, 81, 167, 1341, 1041, 279, 0, 0, 0, 0, 0, 0, 0, 532, 691, 1215, 1115, 230, 413, 982, 168, 867, 0, 0, 0, 0, 0, 0, 0, 401, 1361, 1234, 599, 0, 516, 1555, 516, 0, 0, 0, 0, 0, 0, 0, 0, 526, 157, 227, 266, 1555, 1209, 139, 1270, 627, 102, 0, 0, 0, 0, 0, 0, 767, 99, 847, 770, 33, 97, 267, 595, 2932, 0, 0, 0, 0, 0, 0, 0, 1580, 455, 73, 442, 118, 1772, 128, 1448, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5953, 6030, 6273, 6213, 6182, 6078, 6407, 6016] avg=6144.0000 max=6407 max_rank=6 min=5953 min_rank=0 imbalance_ratio=1.042806 max_load_reduction=42745 max_load_reduction_pct=0.869649 moved_to_echo=24683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[92, 991, 564, 263, 82, 1274, 856, 300, 1342, 154, 92, 0, 0, 0, 0, 0, 621, 341, 1085, 1279, 465, 1169, 176, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1551, 348, 368, 73, 219, 1239, 911, 283, 0, 0, 0, 0, 0, 0, 0, 449, 684, 1071, 827, 294, 495, 935, 338, 899, 0, 0, 0, 0, 0, 0, 0, 405, 1103, 1201, 718, 0, 475, 1817, 533, 0, 0, 0, 0, 0, 0, 0, 0, 350, 244, 337, 192, 1717, 1201, 138, 1193, 637, 106, 0, 0, 0, 0, 0, 0, 758, 134, 1112, 790, 106, 165, 268, 745, 2163, 0, 0, 0, 0, 0, 0, 0, 1553, 408, 102, 505, 120, 1540, 420, 1473, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6010, 6461, 5960, 5992, 6252, 6115, 6241, 6121] avg=6144.0000 max=6461 max_rank=1 min=5960 min_rank=2 imbalance_ratio=1.051595 max_load_reduction=42691 max_load_reduction_pct=0.868551 moved_to_echo=24729 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[12, 1129, 664, 292, 15, 1207, 765, 236, 1733, 152, 120, 0, 0, 0, 0, 0, 618, 443, 1138, 1373, 459, 832, 20, 741, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 1750, 366, 329, 86, 122, 1176, 995, 283, 0, 0, 0, 0, 0, 0, 0, 639, 731, 1161, 1076, 230, 432, 831, 156, 885, 0, 0, 0, 0, 0, 0, 0, 394, 1140, 1196, 697, 0, 613, 1660, 477, 0, 0, 0, 0, 0, 0, 0, 0, 503, 186, 316, 237, 1630, 1244, 137, 1105, 636, 104, 0, 0, 0, 0, 0, 0, 640, 80, 945, 976, 49, 58, 165, 552, 2772, 0, 0, 0, 0, 0, 0, 0, 1817, 505, 43, 406, 47, 1927, 133, 1470, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6325, 5624, 6202, 6141, 6177, 6098, 6237, 6348] avg=6144.0000 max=6348 max_rank=7 min=5624 min_rank=1 imbalance_ratio=1.033203 max_load_reduction=42804 max_load_reduction_pct=0.870850 moved_to_echo=24860 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[35, 1272, 607, 344, 30, 1328, 834, 251, 1409, 176, 108, 0, 0, 0, 0, 0, 656, 372, 1231, 1359, 531, 1095, 65, 851, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1561, 331, 306, 66, 225, 1267, 931, 283, 0, 0, 0, 0, 0, 0, 0, 571, 709, 1053, 910, 269, 484, 1046, 208, 1024, 0, 0, 0, 0, 0, 0, 0, 353, 1175, 1068, 653, 0, 498, 1802, 482, 0, 0, 0, 0, 0, 0, 0, 0, 555, 234, 322, 201, 1672, 1219, 193, 1208, 636, 120, 0, 0, 0, 0, 0, 0, 833, 148, 732, 752, 56, 102, 317, 558, 2527, 0, 0, 0, 0, 0, 0, 0, 1651, 393, 66, 431, 159, 1564, 187, 1469, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 6160, 5988, 6274, 6031, 6360, 6025, 5920] avg=6144.0000 max=6394 max_rank=0 min=5920 min_rank=7 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[14, 1042, 566, 337, 19, 1195, 745, 279, 1756, 169, 118, 0, 0, 0, 0, 0, 697, 413, 1277, 1285, 510, 952, 17, 805, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1768, 371, 326, 218, 175, 1237, 890, 312, 0, 0, 0, 0, 0, 0, 0, 684, 686, 1222, 879, 125, 439, 840, 196, 985, 0, 0, 0, 0, 0, 0, 0, 396, 1067, 1169, 620, 0, 620, 1712, 408, 0, 0, 0, 0, 0, 0, 0, 0, 467, 209, 387, 278, 1650, 1295, 132, 865, 700, 116, 0, 0, 0, 0, 0, 0, 748, 98, 673, 809, 44, 112, 227, 515, 2737, 0, 0, 0, 0, 0, 0, 0, 1788, 512, 73, 377, 79, 1842, 180, 1618, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6240, 5956, 6377, 6056, 5992, 6099, 5963, 6469] avg=6144.0000 max=6469 max_rank=7 min=5956 min_rank=1 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[23, 879, 555, 311, 47, 1422, 859, 256, 1492, 154, 99, 0, 0, 0, 0, 0, 572, 418, 1251, 1386, 464, 1002, 43, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1611, 421, 313, 92, 165, 1253, 1150, 290, 0, 0, 0, 0, 0, 0, 0, 517, 640, 1115, 1100, 291, 488, 1068, 206, 896, 0, 0, 0, 0, 0, 0, 0, 390, 1165, 1156, 596, 0, 528, 1652, 483, 0, 0, 0, 0, 0, 0, 0, 0, 517, 157, 318, 331, 1539, 1116, 145, 1153, 652, 105, 0, 0, 0, 0, 0, 0, 832, 128, 938, 849, 43, 104, 248, 635, 2304, 0, 0, 0, 0, 0, 0, 0, 1562, 372, 83, 463, 90, 1653, 198, 1506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6355, 6368, 6321, 5970, 6033, 6081, 5927] avg=6144.0000 max=6368 max_rank=2 min=5927 min_rank=7 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[31, 1052, 521, 318, 34, 1375, 795, 269, 1379, 149, 111, 0, 0, 0, 0, 0, 646, 462, 1096, 1421, 433, 996, 48, 956, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1614, 366, 339, 80, 162, 1270, 955, 298, 0, 0, 0, 0, 0, 0, 0, 581, 730, 1065, 981, 226, 498, 982, 176, 861, 0, 0, 0, 0, 0, 0, 0, 383, 1301, 1217, 557, 0, 480, 1766, 466, 0, 0, 0, 0, 0, 0, 0, 0, 438, 198, 316, 221, 1668, 1284, 159, 1165, 663, 104, 0, 0, 0, 0, 0, 0, 716, 87, 937, 858, 47, 126, 232, 655, 2567, 0, 0, 0, 0, 0, 0, 0, 1652, 526, 56, 503, 97, 1679, 181, 1521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6034, 6058, 6134, 6100, 6170, 6216, 6225, 6215] avg=6144.0000 max=6225 max_rank=6 min=6034 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=42927 max_load_reduction_pct=0.873352 moved_to_echo=24826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4992, 3200, 9472, 11136, 3840, 8064, 640, 8192, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4480, 5632, 9088, 7936, 1920, 3840, 7680, 1792, 7424, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 1664, 2560, 1920, 13312, 9856, 1280, 9216, 5248, 896, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6272, 896, 7040, 6656, 512, 1024, 2048, 5120, 20096, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[384, 8448, 4608, 2688, 384, 10624, 6528, 2176, 12032, 1280, 896, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 9600, 9472, 5248, 0, 4224, 13952, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4480, 5632, 9088, 7936, 1920, 3840, 7680, 1792, 7424, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13312, 3584, 640, 3584, 896, 13696, 1920, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 9600, 9472, 5248, 0, 4224, 13952, 3968, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8576, 12800, 2944, 2688, 896, 1536, 10240, 7808, 2432, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[230, 491, 1395, 220, 515, 626, 672, 820, 2358, 2909, 274, 326, 1267, 419, 1943, 193, 2006, 55, 2265, 418, 455, 310, 6, 552, 1219, 1149, 616, 460, 535, 1003, 1304, 855, 308, 578, 214, 325, 927, 494, 1051, 450, 204, 2722, 98, 480, 268, 379, 370, 3831, 3, 224, 164, 979, 328, 1303, 202, 182, 247, 76, 1502, 616, 1017, 1047, 340, 357] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[394, 759, 1332, 222, 425, 577, 720, 800, 1693, 2938, 260, 466, 1325, 378, 1574, 407, 1787, 76, 2586, 491, 459, 303, 8, 703, 1222, 886, 597, 476, 545, 918, 1348, 725, 162, 764, 177, 301, 793, 522, 1184, 470, 310, 2465, 111, 467, 313, 542, 348, 3657, 36, 237, 188, 1025, 415, 1307, 219, 280, 277, 143, 1574, 627, 1144, 948, 411, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[292, 452, 1426, 188, 539, 606, 718, 807, 2075, 2834, 340, 396, 1280, 430, 1850, 306, 2203, 79, 2186, 458, 632, 378, 4, 683, 934, 1211, 632, 456, 551, 1075, 1239, 616, 237, 521, 159, 290, 838, 506, 1141, 412, 206, 2960, 100, 507, 299, 315, 369, 3585, 6, 228, 126, 1074, 289, 1036, 140, 131, 232, 115, 1570, 643, 1268, 927, 449, 597] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[200, 286, 1587, 110, 588, 608, 699, 850, 2512, 3042, 364, 366, 1343, 423, 2254, 200, 2196, 51, 2248, 430, 654, 404, 1, 645, 934, 1337, 549, 445, 620, 1012, 1218, 597, 382, 522, 119, 249, 796, 466, 996, 456, 155, 2981, 91, 434, 289, 296, 360, 3472, 2, 169, 123, 984, 410, 1084, 138, 134, 197, 71, 1441, 563, 1295, 959, 318, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[322, 660, 1479, 162, 445, 534, 700, 807, 1711, 2694, 269, 409, 1468, 376, 1456, 398, 1720, 59, 2406, 445, 471, 243, 10, 746, 1261, 1060, 651, 429, 531, 974, 1240, 630, 250, 601, 183, 287, 763, 650, 1316, 477, 311, 2658, 78, 498, 312, 513, 372, 4012, 16, 208, 172, 1011, 410, 1369, 187, 213, 161, 148, 1609, 647, 1276, 891, 408, 379] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[226, 259, 1695, 106, 607, 744, 590, 914, 2373, 3022, 341, 369, 1694, 384, 1936, 141, 1969, 51, 2469, 421, 708, 329, 0, 633, 971, 1389, 641, 415, 441, 1244, 1056, 469, 319, 524, 117, 320, 852, 384, 862, 388, 108, 3110, 91, 469, 244, 294, 272, 4050, 4, 187, 90, 1079, 303, 1193, 66, 65, 156, 62, 1376, 580, 1316, 913, 360, 391] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[209, 390, 1509, 206, 497, 581, 734, 801, 2062, 2862, 258, 412, 1438, 350, 1838, 322, 2096, 63, 2616, 452, 578, 343, 3, 644, 1228, 1223, 646, 404, 652, 1068, 1240, 632, 252, 625, 142, 274, 987, 597, 1074, 449, 154, 2573, 94, 474, 266, 325, 303, 3893, 15, 229, 99, 1088, 370, 1190, 117, 146, 242, 114, 1387, 618, 1110, 851, 412, 325] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[292, 452, 1426, 188, 539, 606, 718, 807, 2075, 2834, 340, 396, 1280, 430, 1850, 306, 2203, 79, 2186, 458, 632, 378, 4, 683, 934, 1211, 632, 456, 551, 1075, 1239, 616, 237, 521, 159, 290, 838, 506, 1141, 412, 206, 2960, 100, 507, 299, 315, 369, 3585, 6, 228, 126, 1074, 289, 1036, 140, 131, 232, 115, 1570, 643, 1268, 927, 449, 597] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[230, 491, 1395, 220, 515, 626, 672, 820, 2358, 2909, 274, 326, 1267, 419, 1943, 193, 2006, 55, 2265, 418, 455, 310, 6, 552, 1219, 1149, 616, 460, 535, 1003, 1304, 855, 308, 578, 214, 325, 927, 494, 1051, 450, 204, 2722, 98, 480, 268, 379, 370, 3831, 3, 224, 164, 979, 328, 1303, 202, 182, 247, 76, 1502, 616, 1017, 1047, 340, 357] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[226, 259, 1695, 106, 607, 744, 590, 914, 2373, 3022, 341, 369, 1694, 384, 1936, 141, 1969, 51, 2469, 421, 708, 329, 0, 633, 971, 1389, 641, 415, 441, 1244, 1056, 469, 319, 524, 117, 320, 852, 384, 862, 388, 108, 3110, 91, 469, 244, 294, 272, 4050, 4, 187, 90, 1079, 303, 1193, 66, 65, 156, 62, 1376, 580, 1316, 913, 360, 391] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[322, 660, 1479, 162, 445, 534, 700, 807, 1711, 2694, 269, 409, 1468, 376, 1456, 398, 1720, 59, 2406, 445, 471, 243, 10, 746, 1261, 1060, 651, 429, 531, 974, 1240, 630, 250, 601, 183, 287, 763, 650, 1316, 477, 311, 2658, 78, 498, 312, 513, 372, 4012, 16, 208, 172, 1011, 410, 1369, 187, 213, 161, 148, 1609, 647, 1276, 891, 408, 379] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[230, 329, 1786, 105, 587, 707, 693, 906, 2020, 2865, 248, 436, 1560, 341, 1914, 420, 2066, 30, 2545, 420, 681, 323, 1, 683, 1051, 1277, 661, 377, 409, 1078, 1341, 511, 504, 727, 166, 331, 874, 442, 806, 521, 203, 2898, 101, 506, 248, 454, 276, 3739, 4, 192, 116, 1025, 231, 1264, 119, 94, 238, 57, 1547, 547, 1130, 725, 159, 307] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[394, 759, 1332, 222, 425, 577, 720, 800, 719, 302, 64, 0, 0, 0, 0, 0, 1209, 0, 260, 466, 1325, 378, 1574, 407, 0, 0, 0, 0, 0, 0, 0, 0, 1787, 76, 2211, 491, 459, 303, 8, 703, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 886, 597, 476, 545, 918, 629, 725, 0, 0, 0, 0, 0, 0, 0, 0, 162, 764, 177, 301, 793, 522, 1184, 470, 1841, 6, 0, 0, 0, 0, 0, 0, 310, 2465, 111, 467, 313, 542, 348, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 36, 237, 188, 1025, 415, 1307, 219, 280, 2932, 0, 0, 0, 0, 0, 0, 0, 277, 143, 1574, 627, 1144, 948, 411, 335, 420, 375, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6314, 5619, 6038, 5998, 6220, 6070, 6639, 6254] avg=6144.0000 max=6639 max_rank=6 min=5619 min_rank=1 imbalance_ratio=1.080566 max_load_reduction=42513 max_load_reduction_pct=0.864929 moved_to_echo=25183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[322, 660, 1479, 162, 445, 534, 700, 807, 662, 332, 65, 0, 0, 0, 0, 0, 1222, 0, 269, 409, 1468, 376, 1456, 398, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 59, 2057, 445, 471, 243, 10, 746, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 1060, 651, 429, 531, 974, 578, 630, 0, 0, 0, 0, 0, 0, 0, 0, 250, 601, 183, 287, 763, 650, 1316, 477, 2020, 6, 0, 0, 0, 0, 0, 0, 311, 2658, 78, 498, 312, 513, 372, 1660, 0, 0, 0, 0, 0, 0, 0, 0, 16, 208, 172, 1011, 410, 1369, 187, 213, 2688, 0, 0, 0, 0, 0, 0, 0, 161, 148, 1609, 647, 1276, 891, 408, 379, 424, 349, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 5598, 5751, 6114, 6553, 6402, 6274, 6292] avg=6144.0000 max=6553 max_rank=4 min=5598 min_rank=1 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=25521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[209, 390, 1509, 206, 497, 581, 734, 801, 662, 322, 78, 0, 0, 0, 0, 0, 1472, 0, 258, 412, 1438, 350, 1838, 322, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 63, 2237, 452, 578, 343, 3, 644, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 1223, 646, 404, 652, 1068, 578, 632, 0, 0, 0, 0, 0, 0, 0, 0, 252, 625, 142, 274, 987, 597, 1074, 449, 1960, 7, 0, 0, 0, 0, 0, 0, 154, 2573, 94, 474, 266, 325, 303, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 15, 229, 99, 1088, 370, 1190, 117, 146, 2855, 0, 0, 0, 0, 0, 0, 0, 242, 114, 1387, 618, 1110, 851, 412, 325, 512, 379, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6090, 6416, 6431, 6367, 5800, 6109, 5950] avg=6144.0000 max=6431 max_rank=3 min=5800 min_rank=5 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[200, 286, 1587, 110, 588, 608, 699, 850, 650, 287, 95, 0, 0, 0, 0, 0, 1794, 0, 364, 366, 1343, 423, 2254, 200, 0, 0, 0, 0, 0, 0, 0, 0, 2196, 51, 1922, 430, 654, 404, 1, 645, 0, 0, 0, 0, 0, 0, 0, 0, 934, 1337, 549, 445, 620, 1012, 568, 597, 0, 0, 0, 0, 0, 0, 0, 0, 382, 522, 119, 249, 796, 466, 996, 456, 1748, 6, 0, 0, 0, 0, 0, 0, 155, 2981, 91, 434, 289, 296, 360, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 2, 169, 123, 984, 410, 1084, 138, 134, 3036, 0, 0, 0, 0, 0, 0, 0, 197, 71, 1441, 563, 1295, 959, 318, 427, 623, 326, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5960, 6744, 6303, 6062, 5740, 6043, 6080, 6220] avg=6144.0000 max=6744 max_rank=1 min=5740 min_rank=4 imbalance_ratio=1.097656 max_load_reduction=42408 max_load_reduction_pct=0.862793 moved_to_echo=24083 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[226, 259, 1695, 106, 607, 744, 590, 914, 563, 335, 90, 0, 0, 0, 0, 0, 1694, 0, 341, 369, 1694, 384, 1936, 141, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 51, 2111, 421, 708, 329, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1389, 641, 415, 441, 1244, 493, 469, 0, 0, 0, 0, 0, 0, 0, 0, 319, 524, 117, 320, 852, 384, 862, 388, 2039, 7, 0, 0, 0, 0, 0, 0, 108, 3110, 91, 469, 244, 294, 272, 1676, 0, 0, 0, 0, 0, 0, 0, 0, 4, 187, 90, 1079, 303, 1193, 66, 65, 3015, 0, 0, 0, 0, 0, 0, 0, 156, 62, 1376, 580, 1316, 913, 360, 391, 589, 358, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6129, 6559, 6222, 6063, 5812, 6264, 6002, 6101] avg=6144.0000 max=6559 max_rank=1 min=5812 min_rank=4 imbalance_ratio=1.067546 max_load_reduction=42593 max_load_reduction_pct=0.866557 moved_to_echo=24179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[209, 390, 1509, 206, 497, 581, 734, 801, 662, 322, 78, 0, 0, 0, 0, 0, 1472, 0, 258, 412, 1438, 350, 1838, 322, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 63, 2237, 452, 578, 343, 3, 644, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 1223, 646, 404, 652, 1068, 578, 632, 0, 0, 0, 0, 0, 0, 0, 0, 252, 625, 142, 274, 987, 597, 1074, 449, 1960, 7, 0, 0, 0, 0, 0, 0, 154, 2573, 94, 474, 266, 325, 303, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 15, 229, 99, 1088, 370, 1190, 117, 146, 2855, 0, 0, 0, 0, 0, 0, 0, 242, 114, 1387, 618, 1110, 851, 412, 325, 512, 379, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6090, 6416, 6431, 6367, 5800, 6109, 5950] avg=6144.0000 max=6431 max_rank=3 min=5800 min_rank=5 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[322, 660, 1479, 162, 445, 534, 700, 807, 662, 332, 65, 0, 0, 0, 0, 0, 1222, 0, 269, 409, 1468, 376, 1456, 398, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 59, 2057, 445, 471, 243, 10, 746, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 1060, 651, 429, 531, 974, 578, 630, 0, 0, 0, 0, 0, 0, 0, 0, 250, 601, 183, 287, 763, 650, 1316, 477, 2020, 6, 0, 0, 0, 0, 0, 0, 311, 2658, 78, 498, 312, 513, 372, 1660, 0, 0, 0, 0, 0, 0, 0, 0, 16, 208, 172, 1011, 410, 1369, 187, 213, 2688, 0, 0, 0, 0, 0, 0, 0, 161, 148, 1609, 647, 1276, 891, 408, 379, 424, 349, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 5598, 5751, 6114, 6553, 6402, 6274, 6292] avg=6144.0000 max=6553 max_rank=4 min=5598 min_rank=1 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=25521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[292, 452, 1426, 188, 539, 606, 718, 807, 664, 301, 83, 0, 0, 0, 0, 0, 1473, 0, 340, 396, 1280, 430, 1850, 306, 0, 0, 0, 0, 0, 0, 0, 0, 2203, 79, 1866, 458, 632, 378, 4, 683, 0, 0, 0, 0, 0, 0, 0, 0, 934, 1211, 632, 456, 551, 1075, 575, 616, 0, 0, 0, 0, 0, 0, 0, 0, 237, 521, 159, 290, 838, 506, 1141, 412, 1808, 7, 0, 0, 0, 0, 0, 0, 206, 2960, 100, 507, 299, 315, 369, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 6, 228, 126, 1074, 289, 1036, 140, 131, 2827, 0, 0, 0, 0, 0, 0, 0, 232, 115, 1570, 643, 1268, 927, 449, 597, 519, 320, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 6075, 6303, 6050, 5919, 6232, 5857, 6640] avg=6144.0000 max=6640 max_rank=7 min=5857 min_rank=6 imbalance_ratio=1.080729 max_load_reduction=42512 max_load_reduction_pct=0.864909 moved_to_echo=24648 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[230, 491, 1395, 220, 515, 626, 672, 820, 696, 317, 89, 0, 0, 0, 0, 0, 1684, 0, 274, 326, 1267, 419, 1943, 193, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 55, 1937, 418, 455, 310, 6, 552, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 1149, 616, 460, 535, 1003, 608, 855, 0, 0, 0, 0, 0, 0, 0, 0, 308, 578, 214, 325, 927, 494, 1051, 450, 1928, 6, 0, 0, 0, 0, 0, 0, 204, 2722, 98, 480, 268, 379, 370, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 3, 224, 164, 979, 328, 1303, 202, 182, 2903, 0, 0, 0, 0, 0, 0, 0, 247, 76, 1502, 616, 1017, 1047, 340, 357, 585, 328, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6071, 6106, 5739, 6445, 6281, 6107, 6288, 6115] avg=6144.0000 max=6445 max_rank=3 min=5739 min_rank=2 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=24791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[230, 329, 1786, 105, 587, 707, 693, 906, 716, 309, 77, 0, 0, 0, 0, 0, 1442, 0, 248, 436, 1560, 341, 1914, 420, 0, 0, 0, 0, 0, 0, 0, 0, 2066, 30, 2176, 420, 681, 323, 1, 683, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1277, 661, 377, 409, 1078, 625, 511, 0, 0, 0, 0, 0, 0, 0, 0, 504, 727, 166, 331, 874, 442, 806, 521, 1882, 7, 0, 0, 0, 0, 0, 0, 203, 2898, 101, 506, 248, 454, 276, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 4, 192, 116, 1025, 231, 1264, 119, 94, 2858, 0, 0, 0, 0, 0, 0, 0, 238, 57, 1547, 547, 1130, 725, 159, 307, 501, 369, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6445, 6361, 6380, 5989, 6260, 6234, 5903, 5580] avg=6144.0000 max=6445 max_rank=0 min=5580 min_rank=7 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=23977 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 @@ -9066,370 +9066,370 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_me WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[16128, 512, 16640, 3584, 4736, 2688, 128, 5376, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[1664, 22400, 768, 3840, 2304, 3200, 2688, 12544, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2176, 3712, 12288, 1408, 4224, 4992, 5632, 6784, 5376, 2560, 768, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2432, 4864, 1280, 2432, 6912, 4096, 8448, 3712, 15232, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 896, 12032, 4864, 9600, 7296, 2944, 3200, 4224, 2816, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8832, 9600, 5120, 3584, 4352, 8448, 4736, 5120, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12032, 0, 2432, 3200, 11392, 3200, 14848, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2176, 3712, 12288, 1408, 4224, 4992, 5632, 6784, 5376, 2560, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[16128, 512, 16640, 3584, 4736, 2688, 128, 5376, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1792, 1152, 8320, 2816, 9856, 1280, 1280, 23168, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12032, 0, 2432, 3200, 11392, 3200, 14848, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 896, 12032, 4864, 9600, 7296, 2944, 3200, 4224, 2816, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 1815, 1484, 1161, 1133, 1142, 681, 1411, 277, 1461, 2618, 116, 171, 479, 324, 564, 1664, 131, 132, 111, 120, 729, 326, 119, 944, 199, 1904, 310, 2251, 772, 513, 54, 242, 0, 8, 1013, 828, 50, 71, 1091, 278, 531, 96, 895, 475, 0, 2600, 0, 106, 934, 263, 1236, 389, 1743, 1234, 1144, 1253, 418, 178, 1800, 1171, 2067, 833, 1089] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 1593, 1407, 1055, 1048, 1310, 672, 1700, 256, 1325, 2436, 75, 92, 508, 266, 385, 1963, 4, 156, 70, 98, 979, 293, 130, 988, 192, 2425, 294, 2622, 660, 329, 67, 221, 2, 2, 1356, 690, 17, 50, 1180, 270, 747, 59, 969, 432, 1, 2994, 1, 32, 964, 287, 761, 435, 2018, 1765, 968, 1031, 332, 163, 1601, 1101, 1634, 677, 994] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 1245, 1407, 1442, 1067, 1259, 641, 2050, 169, 1777, 2310, 59, 138, 404, 245, 278, 2003, 10, 8, 22, 100, 1425, 272, 94, 1104, 0, 2274, 190, 2864, 437, 404, 51, 203, 0, 0, 1501, 559, 11, 60, 934, 195, 676, 3, 922, 380, 0, 3162, 0, 5, 1068, 177, 660, 386, 2365, 2099, 1060, 714, 396, 46, 1858, 944, 1581, 513, 925] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1671, 1343, 1074, 1156, 1251, 683, 1575, 185, 1395, 2805, 76, 156, 497, 284, 510, 1788, 6, 69, 99, 90, 866, 337, 103, 875, 5, 2813, 265, 2672, 716, 418, 44, 161, 0, 0, 1337, 621, 27, 66, 1082, 292, 672, 18, 942, 377, 0, 2861, 0, 17, 915, 236, 688, 362, 1723, 1634, 1047, 893, 484, 128, 2193, 1208, 1813, 714, 814] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 1798, 1547, 1000, 1083, 1184, 685, 1380, 225, 1268, 2490, 73, 151, 467, 336, 678, 1737, 78, 100, 64, 93, 834, 289, 176, 945, 323, 2302, 402, 2297, 727, 624, 52, 222, 0, 11, 930, 834, 77, 95, 1028, 315, 446, 46, 915, 636, 0, 2649, 2, 102, 976, 250, 1001, 391, 1633, 1380, 1135, 1231, 446, 170, 1932, 1153, 1755, 1001, 982] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1327, 1332, 1342, 1103, 1142, 621, 1629, 137, 1567, 2836, 61, 130, 452, 244, 200, 2028, 2, 0, 17, 70, 1430, 347, 134, 1026, 0, 2933, 134, 3494, 346, 336, 0, 159, 0, 0, 1587, 468, 2, 36, 863, 209, 677, 7, 910, 297, 0, 3225, 0, 4, 1204, 151, 532, 407, 2512, 2135, 876, 744, 362, 32, 1734, 1012, 1155, 506, 926] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 1593, 1407, 1055, 1048, 1310, 672, 1700, 256, 1325, 2436, 75, 92, 508, 266, 385, 1963, 4, 156, 70, 98, 979, 293, 130, 988, 192, 2425, 294, 2622, 660, 329, 67, 221, 2, 2, 1356, 690, 17, 50, 1180, 270, 747, 59, 969, 432, 1, 2994, 1, 32, 964, 287, 761, 435, 2018, 1765, 968, 1031, 332, 163, 1601, 1101, 1634, 677, 994] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 1350, 1326, 1215, 1034, 1342, 618, 1705, 147, 1362, 2430, 71, 117, 468, 256, 304, 1746, 8, 16, 15, 81, 1113, 311, 95, 797, 0, 3207, 180, 3334, 457, 308, 22, 178, 0, 0, 1357, 510, 4, 39, 978, 288, 674, 4, 991, 340, 0, 3322, 0, 3, 1062, 192, 680, 432, 2379, 2085, 1038, 727, 416, 50, 1929, 1048, 1600, 490, 901] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 1245, 1407, 1442, 1067, 1259, 641, 2050, 169, 1777, 2310, 59, 138, 404, 245, 278, 2003, 10, 8, 22, 100, 1425, 272, 94, 1104, 0, 2274, 190, 2864, 437, 404, 51, 203, 0, 0, 1501, 559, 11, 60, 934, 195, 676, 3, 922, 380, 0, 3162, 0, 5, 1068, 177, 660, 386, 2365, 2099, 1060, 714, 396, 46, 1858, 944, 1581, 513, 925] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 1588, 1491, 1048, 1127, 1574, 695, 1709, 236, 1077, 2425, 65, 147, 597, 262, 436, 1725, 6, 86, 57, 102, 968, 367, 104, 828, 17, 2739, 257, 2861, 535, 418, 60, 187, 0, 2, 1254, 603, 31, 60, 1040, 265, 704, 20, 935, 420, 0, 3008, 0, 20, 1077, 251, 715, 433, 1879, 1698, 1018, 855, 508, 128, 1674, 1316, 1617, 754, 1073] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 614, 1547, 1000, 1083, 1184, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 1268, 2490, 73, 151, 467, 336, 678, 696, 0, 0, 0, 0, 0, 0, 0, 1737, 78, 100, 64, 93, 834, 289, 176, 1278, 561, 339, 0, 0, 0, 0, 0, 945, 323, 2302, 402, 1019, 727, 624, 52, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 11, 930, 834, 77, 95, 1028, 1380, 1319, 0, 0, 0, 0, 0, 0, 315, 446, 46, 915, 636, 0, 2649, 2, 365, 488, 0, 0, 0, 0, 0, 0, 102, 976, 250, 1001, 391, 707, 1380, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 446, 170, 274, 1153, 1755, 1001, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6384, 5549, 6394, 5896, 5862, 5942, 7012] avg=6144.0000 max=7012 max_rank=7 min=5549 min_rank=2 imbalance_ratio=1.141276 max_load_reduction=42140 max_load_reduction_pct=0.857340 moved_to_echo=24712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1327, 1332, 1342, 1103, 1142, 621, 1629, 137, 1567, 2836, 61, 130, 452, 244, 200, 2028, 2, 0, 17, 70, 1430, 347, 134, 1026, 0, 2933, 134, 3494, 346, 336, 0, 159, 0, 0, 1587, 468, 2, 36, 863, 209, 677, 7, 910, 297, 0, 3225, 0, 4, 1204, 151, 532, 407, 2512, 2135, 876, 744, 362, 32, 1734, 1012, 1155, 506, 926] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 621, 1484, 1161, 1133, 1142, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 1461, 2618, 116, 171, 479, 324, 564, 702, 0, 0, 0, 0, 0, 0, 0, 1664, 131, 132, 111, 120, 729, 326, 119, 1252, 598, 316, 0, 0, 0, 0, 0, 944, 199, 1904, 310, 999, 772, 513, 54, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 8, 1013, 828, 50, 71, 1091, 1411, 1229, 0, 0, 0, 0, 0, 0, 278, 531, 96, 895, 475, 0, 2600, 0, 389, 492, 0, 0, 0, 0, 0, 0, 106, 934, 263, 1236, 389, 756, 1234, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 418, 178, 255, 1171, 2067, 833, 1089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6222, 6712, 5498, 5695, 5943, 5756, 6062, 7264] avg=6144.0000 max=7264 max_rank=7 min=5498 min_rank=2 imbalance_ratio=1.182292 max_load_reduction=41888 max_load_reduction_pct=0.852214 moved_to_echo=25025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 425, 1407, 1442, 1067, 1259, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 1777, 2310, 59, 138, 404, 245, 278, 482, 0, 0, 0, 0, 0, 0, 0, 2003, 10, 8, 22, 100, 1425, 272, 94, 1594, 812, 326, 0, 0, 0, 0, 0, 1104, 0, 2274, 190, 1270, 437, 404, 51, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 1501, 559, 11, 60, 934, 2050, 1269, 0, 0, 0, 0, 0, 0, 195, 676, 3, 922, 380, 0, 3162, 0, 528, 338, 0, 0, 0, 0, 0, 0, 5, 1068, 177, 660, 386, 1025, 2099, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 714, 396, 46, 263, 944, 1581, 513, 925, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6241, 5862, 6666, 5730, 6587, 6204, 6480, 5382] avg=6144.0000 max=6666 max_rank=2 min=5382 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=42486 max_load_reduction_pct=0.864380 moved_to_echo=24653 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 545, 1407, 1055, 1048, 1310, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 1325, 2436, 75, 92, 508, 266, 385, 616, 0, 0, 0, 0, 0, 0, 0, 1963, 4, 156, 70, 98, 979, 293, 130, 1459, 693, 281, 0, 0, 0, 0, 0, 988, 192, 2425, 294, 1163, 660, 329, 67, 0, 0, 0, 0, 0, 0, 0, 0, 221, 2, 2, 1356, 690, 17, 50, 1180, 1700, 1093, 0, 0, 0, 0, 0, 0, 270, 747, 59, 969, 432, 1, 2994, 1, 451, 432, 0, 0, 0, 0, 0, 0, 32, 964, 287, 761, 435, 874, 1765, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 332, 163, 227, 1101, 1634, 677, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6037, 5959, 6126, 6118, 6311, 6356, 6086, 6159] avg=6144.0000 max=6356 max_rank=5 min=5959 min_rank=1 imbalance_ratio=1.034505 max_load_reduction=42796 max_load_reduction_pct=0.870687 moved_to_echo=24912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 572, 1343, 1074, 1156, 1251, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 1395, 2805, 76, 156, 497, 284, 510, 646, 0, 0, 0, 0, 0, 0, 0, 1788, 6, 69, 99, 90, 866, 337, 103, 1487, 592, 385, 0, 0, 0, 0, 0, 875, 5, 2813, 265, 1185, 716, 418, 44, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 1337, 621, 27, 66, 1082, 1575, 1497, 0, 0, 0, 0, 0, 0, 292, 672, 18, 942, 377, 0, 2861, 0, 385, 453, 0, 0, 0, 0, 0, 0, 17, 915, 236, 688, 362, 746, 1634, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 893, 484, 128, 311, 1208, 1813, 714, 814, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6079, 6554, 5822, 6321, 6366, 6000, 5645, 6365] avg=6144.0000 max=6554 max_rank=1 min=5645 min_rank=6 imbalance_ratio=1.066732 max_load_reduction=42598 max_load_reduction_pct=0.866659 moved_to_echo=24376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 454, 1332, 1342, 1103, 1142, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 1567, 2836, 61, 130, 452, 244, 200, 513, 0, 0, 0, 0, 0, 0, 0, 2028, 2, 0, 17, 70, 1430, 347, 134, 1944, 863, 304, 0, 0, 0, 0, 0, 1026, 0, 2933, 134, 1550, 346, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 1587, 468, 2, 36, 863, 1629, 1184, 0, 0, 0, 0, 0, 0, 209, 677, 7, 910, 297, 0, 3225, 0, 561, 360, 0, 0, 0, 0, 0, 0, 4, 1204, 151, 532, 407, 1088, 2135, 876, 0, 0, 0, 0, 0, 0, 0, 0, 744, 362, 32, 246, 1012, 1155, 506, 926, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5994, 6140, 7139, 6325, 5928, 6246, 6397, 4983] avg=6144.0000 max=7139 max_rank=2 min=4983 min_rank=7 imbalance_ratio=1.161947 max_load_reduction=42013 max_load_reduction_pct=0.854757 moved_to_echo=23554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 462, 1326, 1215, 1034, 1342, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 1362, 2430, 71, 117, 468, 256, 304, 522, 0, 0, 0, 0, 0, 0, 0, 1746, 8, 16, 15, 81, 1113, 311, 95, 1855, 817, 338, 0, 0, 0, 0, 0, 797, 0, 3207, 180, 1479, 457, 308, 22, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 1357, 510, 4, 39, 978, 1705, 1317, 0, 0, 0, 0, 0, 0, 288, 674, 4, 991, 340, 0, 3322, 0, 532, 366, 0, 0, 0, 0, 0, 0, 3, 1062, 192, 680, 432, 1030, 2085, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 727, 416, 50, 274, 1048, 1600, 490, 901, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5997, 5677, 6395, 6450, 6088, 6517, 6522, 5506] avg=6144.0000 max=6522 max_rank=6 min=5506 min_rank=7 imbalance_ratio=1.061523 max_load_reduction=42630 max_load_reduction_pct=0.867310 moved_to_echo=24633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 545, 1407, 1055, 1048, 1310, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 1325, 2436, 75, 92, 508, 266, 385, 616, 0, 0, 0, 0, 0, 0, 0, 1963, 4, 156, 70, 98, 979, 293, 130, 1459, 693, 281, 0, 0, 0, 0, 0, 988, 192, 2425, 294, 1163, 660, 329, 67, 0, 0, 0, 0, 0, 0, 0, 0, 221, 2, 2, 1356, 690, 17, 50, 1180, 1700, 1093, 0, 0, 0, 0, 0, 0, 270, 747, 59, 969, 432, 1, 2994, 1, 451, 432, 0, 0, 0, 0, 0, 0, 32, 964, 287, 761, 435, 874, 1765, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 332, 163, 227, 1101, 1634, 677, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6037, 5959, 6126, 6118, 6311, 6356, 6086, 6159] avg=6144.0000 max=6356 max_rank=5 min=5959 min_rank=1 imbalance_ratio=1.034505 max_load_reduction=42796 max_load_reduction_pct=0.870687 moved_to_echo=24912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 614, 1547, 1000, 1083, 1184, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 1268, 2490, 73, 151, 467, 336, 678, 696, 0, 0, 0, 0, 0, 0, 0, 1737, 78, 100, 64, 93, 834, 289, 176, 1278, 561, 339, 0, 0, 0, 0, 0, 945, 323, 2302, 402, 1019, 727, 624, 52, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 11, 930, 834, 77, 95, 1028, 1380, 1319, 0, 0, 0, 0, 0, 0, 315, 446, 46, 915, 636, 0, 2649, 2, 365, 488, 0, 0, 0, 0, 0, 0, 102, 976, 250, 1001, 391, 707, 1380, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 446, 170, 274, 1153, 1755, 1001, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6384, 5549, 6394, 5896, 5862, 5942, 7012] avg=6144.0000 max=7012 max_rank=7 min=5549 min_rank=2 imbalance_ratio=1.141276 max_load_reduction=42140 max_load_reduction_pct=0.857340 moved_to_echo=24712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 454, 1332, 1342, 1103, 1142, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 1567, 2836, 61, 130, 452, 244, 200, 513, 0, 0, 0, 0, 0, 0, 0, 2028, 2, 0, 17, 70, 1430, 347, 134, 1944, 863, 304, 0, 0, 0, 0, 0, 1026, 0, 2933, 134, 1550, 346, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 1587, 468, 2, 36, 863, 1629, 1184, 0, 0, 0, 0, 0, 0, 209, 677, 7, 910, 297, 0, 3225, 0, 561, 360, 0, 0, 0, 0, 0, 0, 4, 1204, 151, 532, 407, 1088, 2135, 876, 0, 0, 0, 0, 0, 0, 0, 0, 744, 362, 32, 246, 1012, 1155, 506, 926, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5994, 6140, 7139, 6325, 5928, 6246, 6397, 4983] avg=6144.0000 max=7139 max_rank=2 min=4983 min_rank=7 imbalance_ratio=1.161947 max_load_reduction=42013 max_load_reduction_pct=0.854757 moved_to_echo=23554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 534, 1491, 1048, 1127, 1574, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 1077, 2425, 65, 147, 597, 262, 436, 619, 0, 0, 0, 0, 0, 0, 0, 1725, 6, 86, 57, 102, 968, 367, 104, 1596, 649, 297, 0, 0, 0, 0, 0, 828, 17, 2739, 257, 1265, 535, 418, 60, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 2, 1254, 603, 31, 60, 1040, 1709, 1147, 0, 0, 0, 0, 0, 0, 265, 704, 20, 935, 420, 0, 3008, 0, 424, 435, 0, 0, 0, 0, 0, 0, 20, 1077, 251, 715, 433, 806, 1698, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 855, 508, 128, 230, 1316, 1617, 754, 1073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6469, 5864, 5957, 6119, 6033, 6211, 6018, 6481] avg=6144.0000 max=6481 max_rank=7 min=5864 min_rank=1 imbalance_ratio=1.054850 max_load_reduction=42671 max_load_reduction_pct=0.868144 moved_to_echo=24743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 425, 1407, 1442, 1067, 1259, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 1777, 2310, 59, 138, 404, 245, 278, 482, 0, 0, 0, 0, 0, 0, 0, 2003, 10, 8, 22, 100, 1425, 272, 94, 1594, 812, 326, 0, 0, 0, 0, 0, 1104, 0, 2274, 190, 1270, 437, 404, 51, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 1501, 559, 11, 60, 934, 2050, 1269, 0, 0, 0, 0, 0, 0, 195, 676, 3, 922, 380, 0, 3162, 0, 528, 338, 0, 0, 0, 0, 0, 0, 5, 1068, 177, 660, 386, 1025, 2099, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 714, 396, 46, 263, 944, 1581, 513, 925, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6241, 5862, 6666, 5730, 6587, 6204, 6480, 5382] avg=6144.0000 max=6666 max_rank=2 min=5382 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=42486 max_load_reduction_pct=0.864380 moved_to_echo=24653 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 4352, 11392, 9344, 8832, 10240, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 8320, 1920, 6400, 3328, 7040, 14080, 8320, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7552, 3456, 896, 2176, 8960, 13312, 5504, 7808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7552, 768, 20608, 2048, 9984, 4736, 3456, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14720, 256, 640, 512, 768, 8448, 2560, 1024, 12544, 5632, 2688, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 11264, 20352, 640, 1152, 3968, 2304, 3456, 4864, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7552, 768, 20608, 2048, 9984, 4736, 3456, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 128, 128, 10368, 5120, 256, 512, 8320, 13184, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 8320, 1920, 6400, 3328, 7040, 14080, 8320, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2176, 5248, 256, 7552, 3456, 128, 23936, 128, 3712, 3456, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[447, 1810, 753, 474, 78, 361, 2630, 397, 1, 0, 260, 670, 2272, 0, 0, 578, 447, 1015, 4470, 23, 57, 554, 402, 0, 182, 2970, 773, 93, 149, 204, 3787, 161, 2878, 487, 73, 271, 488, 180, 1297, 723, 36, 0, 14, 1724, 61, 415, 2292, 595, 869, 1624, 1094, 895, 32, 541, 0, 1022, 1171, 416, 494, 667, 168, 370, 647, 1590] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[374, 2124, 583, 480, 62, 400, 2428, 496, 0, 0, 168, 664, 2499, 0, 0, 472, 378, 859, 4834, 24, 32, 854, 453, 0, 62, 3691, 756, 118, 66, 204, 3889, 116, 2916, 422, 77, 215, 375, 0, 1122, 442, 20, 0, 4, 1441, 80, 471, 2334, 805, 745, 1675, 1702, 935, 15, 458, 0, 1051, 1174, 275, 474, 457, 108, 340, 614, 1319] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[484, 1747, 853, 561, 83, 198, 2622, 579, 1, 0, 247, 674, 2075, 0, 0, 672, 480, 909, 4781, 18, 69, 592, 400, 0, 222, 2744, 765, 118, 130, 154, 3944, 125, 2980, 542, 79, 333, 500, 109, 1366, 558, 29, 0, 8, 1490, 85, 315, 2197, 622, 839, 1522, 1087, 923, 78, 500, 0, 1138, 1069, 440, 460, 665, 186, 431, 716, 1638] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[359, 2055, 609, 578, 59, 414, 2234, 718, 0, 0, 148, 566, 2594, 0, 0, 512, 417, 787, 4948, 11, 23, 980, 580, 0, 1, 3443, 829, 41, 62, 134, 3760, 92, 3056, 385, 70, 198, 386, 0, 1084, 522, 19, 0, 1, 1295, 57, 452, 2270, 955, 804, 1334, 1679, 1333, 8, 479, 0, 1018, 1251, 271, 199, 418, 99, 221, 535, 1799] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[447, 1810, 753, 474, 78, 361, 2630, 397, 1, 0, 260, 670, 2272, 0, 0, 578, 447, 1015, 4470, 23, 57, 554, 402, 0, 182, 2970, 773, 93, 149, 204, 3787, 161, 2878, 487, 73, 271, 488, 180, 1297, 723, 36, 0, 14, 1724, 61, 415, 2292, 595, 869, 1624, 1094, 895, 32, 541, 0, 1022, 1171, 416, 494, 667, 168, 370, 647, 1590] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[499, 1949, 841, 602, 77, 232, 2490, 571, 0, 0, 173, 587, 2348, 0, 1, 668, 525, 847, 4658, 19, 30, 777, 533, 0, 88, 3252, 763, 94, 156, 191, 3438, 128, 3254, 464, 77, 293, 400, 12, 1407, 521, 36, 1, 8, 1411, 69, 416, 2026, 799, 862, 1274, 1354, 1045, 20, 516, 1, 1151, 1153, 369, 500, 506, 134, 288, 608, 1640] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[484, 1747, 853, 561, 83, 198, 2622, 579, 1, 0, 247, 674, 2075, 0, 0, 672, 480, 909, 4781, 18, 69, 592, 400, 0, 222, 2744, 765, 118, 130, 154, 3944, 125, 2980, 542, 79, 333, 500, 109, 1366, 558, 29, 0, 8, 1490, 85, 315, 2197, 622, 839, 1522, 1087, 923, 78, 500, 0, 1138, 1069, 440, 460, 665, 186, 431, 716, 1638] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[240, 1969, 522, 669, 71, 398, 2134, 628, 0, 0, 157, 607, 3042, 0, 0, 349, 391, 794, 5049, 15, 30, 1056, 555, 0, 0, 3711, 759, 6, 46, 84, 3846, 42, 3264, 399, 80, 115, 403, 0, 854, 473, 7, 0, 0, 1103, 55, 457, 2735, 827, 828, 1492, 1492, 1565, 7, 390, 0, 1004, 1245, 264, 218, 326, 108, 211, 536, 1524] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[367, 1697, 619, 488, 75, 425, 2556, 681, 0, 0, 279, 658, 2337, 0, 0, 552, 468, 915, 4550, 19, 32, 860, 496, 0, 45, 3519, 897, 121, 71, 196, 3696, 129, 2806, 495, 90, 218, 485, 0, 1069, 509, 41, 0, 7, 1569, 55, 426, 2315, 808, 817, 1348, 1702, 1068, 9, 494, 0, 1071, 1195, 271, 482, 430, 132, 283, 597, 1612] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[317, 2261, 536, 326, 88, 412, 2055, 677, 0, 0, 165, 640, 2900, 0, 0, 314, 399, 940, 5311, 9, 34, 1184, 463, 0, 56, 2858, 692, 31, 75, 47, 3899, 85, 3610, 421, 113, 218, 315, 0, 760, 502, 23, 0, 0, 1201, 52, 435, 2488, 900, 777, 1772, 1442, 1207, 21, 444, 0, 1044, 1249, 297, 443, 385, 132, 247, 511, 1369] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[374, 2124, 583, 480, 62, 400, 2428, 496, 0, 0, 168, 664, 2499, 0, 0, 472, 378, 859, 4834, 24, 32, 854, 453, 0, 62, 3691, 756, 118, 66, 204, 3889, 116, 2916, 422, 77, 215, 375, 0, 1122, 442, 20, 0, 4, 1441, 80, 471, 2334, 805, 745, 1675, 1702, 935, 15, 458, 0, 1051, 1174, 275, 474, 457, 108, 340, 614, 1319] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[447, 1810, 753, 474, 78, 361, 1756, 397, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 260, 670, 2272, 0, 0, 578, 2209, 75, 0, 0, 0, 0, 0, 0, 447, 1015, 3174, 23, 57, 554, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 2970, 773, 93, 149, 204, 1578, 161, 0, 0, 0, 0, 0, 0, 0, 0, 2878, 487, 73, 271, 488, 180, 1297, 723, 187, 0, 0, 0, 0, 0, 0, 0, 36, 0, 14, 1724, 61, 415, 2292, 595, 874, 156, 144, 0, 0, 0, 0, 0, 869, 1293, 1094, 895, 32, 541, 0, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 416, 494, 667, 168, 370, 647, 1590, 1065, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 6065, 5672, 6110, 6584, 6311, 5746, 6588] avg=6144.0000 max=6588 max_rank=7 min=5672 min_rank=2 imbalance_ratio=1.072266 max_load_reduction=42564 max_load_reduction_pct=0.865967 moved_to_echo=25229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[240, 1969, 522, 669, 71, 398, 2134, 628, 0, 0, 157, 607, 3042, 0, 0, 349, 391, 794, 5049, 15, 30, 1056, 555, 0, 0, 3711, 759, 6, 46, 84, 3846, 42, 3264, 399, 80, 115, 403, 0, 854, 473, 7, 0, 0, 1103, 55, 457, 2735, 827, 828, 1492, 1492, 1565, 7, 390, 0, 1004, 1245, 264, 218, 326, 108, 211, 536, 1524] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[484, 1747, 853, 561, 83, 198, 1750, 579, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 247, 674, 2075, 0, 0, 672, 2301, 80, 0, 0, 0, 0, 0, 0, 480, 909, 3394, 18, 69, 592, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 2744, 765, 118, 130, 154, 1643, 125, 0, 0, 0, 0, 0, 0, 0, 0, 2980, 542, 79, 333, 500, 109, 1366, 558, 175, 0, 0, 0, 0, 0, 0, 0, 29, 0, 8, 1490, 85, 315, 2197, 622, 872, 167, 135, 0, 0, 0, 0, 0, 839, 1212, 1087, 923, 78, 500, 0, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 440, 460, 665, 186, 431, 716, 1638, 1140, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6255, 6050, 5862, 5901, 6642, 5920, 5777, 6745] avg=6144.0000 max=6745 max_rank=7 min=5777 min_rank=6 imbalance_ratio=1.097819 max_load_reduction=42407 max_load_reduction_pct=0.862773 moved_to_echo=25084 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[374, 2124, 583, 480, 62, 400, 1621, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 664, 2499, 0, 0, 472, 2268, 81, 0, 0, 0, 0, 0, 0, 378, 859, 3432, 24, 32, 854, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 3691, 756, 118, 66, 204, 1621, 116, 0, 0, 0, 0, 0, 0, 0, 0, 2916, 422, 77, 215, 375, 0, 1122, 442, 193, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4, 1441, 80, 471, 2334, 805, 807, 169, 149, 0, 0, 0, 0, 0, 745, 1333, 1702, 935, 15, 458, 0, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 275, 474, 457, 108, 340, 614, 1319, 1152, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6152, 6032, 6634, 5762, 6280, 6239, 5913] avg=6144.0000 max=6634 max_rank=3 min=5762 min_rank=4 imbalance_ratio=1.079753 max_load_reduction=42518 max_load_reduction_pct=0.865031 moved_to_echo=24194 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[359, 2055, 609, 578, 59, 414, 1491, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 566, 2594, 0, 0, 512, 2193, 83, 0, 0, 0, 0, 0, 0, 417, 787, 3513, 11, 23, 980, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3443, 829, 41, 62, 134, 1567, 92, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 385, 70, 198, 386, 0, 1084, 522, 154, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1, 1295, 57, 452, 2270, 955, 743, 173, 118, 0, 0, 0, 0, 0, 804, 1062, 1679, 1333, 8, 479, 0, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 271, 199, 418, 99, 221, 535, 1799, 1179, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 6096, 6311, 6169, 5855, 6083, 6383, 5972] avg=6144.0000 max=6383 max_rank=6 min=5855 min_rank=4 imbalance_ratio=1.038900 max_load_reduction=42769 max_load_reduction_pct=0.870138 moved_to_echo=24293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 1949, 841, 602, 77, 232, 1662, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 587, 2348, 0, 1, 668, 2005, 78, 0, 0, 0, 0, 0, 0, 525, 847, 3307, 19, 30, 777, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 3252, 763, 94, 156, 191, 1433, 128, 0, 0, 0, 0, 0, 0, 0, 0, 3254, 464, 77, 293, 400, 12, 1407, 521, 147, 0, 0, 0, 0, 0, 0, 0, 36, 1, 8, 1411, 69, 416, 2026, 799, 828, 163, 113, 0, 0, 0, 0, 0, 862, 1014, 1354, 1045, 20, 516, 1, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 1153, 369, 500, 506, 134, 288, 608, 1640, 1110, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6433, 5860, 6038, 6105, 6575, 5870, 5963, 6308] avg=6144.0000 max=6575 max_rank=4 min=5860 min_rank=1 imbalance_ratio=1.070150 max_load_reduction=42577 max_load_reduction_pct=0.866231 moved_to_echo=24716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[447, 1810, 753, 474, 78, 361, 1756, 397, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 260, 670, 2272, 0, 0, 578, 2209, 75, 0, 0, 0, 0, 0, 0, 447, 1015, 3174, 23, 57, 554, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 2970, 773, 93, 149, 204, 1578, 161, 0, 0, 0, 0, 0, 0, 0, 0, 2878, 487, 73, 271, 488, 180, 1297, 723, 187, 0, 0, 0, 0, 0, 0, 0, 36, 0, 14, 1724, 61, 415, 2292, 595, 874, 156, 144, 0, 0, 0, 0, 0, 869, 1293, 1094, 895, 32, 541, 0, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 416, 494, 667, 168, 370, 647, 1590, 1065, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 6065, 5672, 6110, 6584, 6311, 5746, 6588] avg=6144.0000 max=6588 max_rank=7 min=5672 min_rank=2 imbalance_ratio=1.072266 max_load_reduction=42564 max_load_reduction_pct=0.865967 moved_to_echo=25229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[317, 2261, 536, 326, 88, 412, 1372, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 640, 2900, 0, 0, 314, 2274, 89, 0, 0, 0, 0, 0, 0, 399, 940, 3771, 9, 34, 1184, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 2858, 692, 31, 75, 47, 1625, 85, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 421, 113, 218, 315, 0, 760, 502, 204, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 1201, 52, 435, 2488, 900, 683, 185, 158, 0, 0, 0, 0, 0, 777, 1410, 1442, 1207, 21, 444, 0, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 297, 443, 385, 132, 247, 511, 1369, 1266, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6382, 6800, 5469, 6143, 6125, 6345, 5899] avg=6144.0000 max=6800 max_rank=2 min=5469 min_rank=3 imbalance_ratio=1.106771 max_load_reduction=42352 max_load_reduction_pct=0.861654 moved_to_echo=24512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[240, 1969, 522, 669, 71, 398, 1425, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 607, 3042, 0, 0, 349, 2243, 84, 0, 0, 0, 0, 0, 0, 391, 794, 3585, 15, 30, 1056, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 759, 6, 46, 84, 1603, 42, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 399, 80, 115, 403, 0, 854, 473, 172, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1103, 55, 457, 2735, 827, 709, 176, 133, 0, 0, 0, 0, 0, 828, 1187, 1492, 1565, 7, 390, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1245, 264, 218, 326, 108, 211, 536, 1524, 1204, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5922, 6482, 6426, 6251, 5760, 6202, 6473, 5636] avg=6144.0000 max=6482 max_rank=1 min=5636 min_rank=7 imbalance_ratio=1.055013 max_load_reduction=42670 max_load_reduction_pct=0.868123 moved_to_echo=24071 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 1949, 841, 602, 77, 232, 1662, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 587, 2348, 0, 1, 668, 2005, 78, 0, 0, 0, 0, 0, 0, 525, 847, 3307, 19, 30, 777, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 3252, 763, 94, 156, 191, 1433, 128, 0, 0, 0, 0, 0, 0, 0, 0, 3254, 464, 77, 293, 400, 12, 1407, 521, 147, 0, 0, 0, 0, 0, 0, 0, 36, 1, 8, 1411, 69, 416, 2026, 799, 828, 163, 113, 0, 0, 0, 0, 0, 862, 1014, 1354, 1045, 20, 516, 1, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 1153, 369, 500, 506, 134, 288, 608, 1640, 1110, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6433, 5860, 6038, 6105, 6575, 5870, 5963, 6308] avg=6144.0000 max=6575 max_rank=4 min=5860 min_rank=1 imbalance_ratio=1.070150 max_load_reduction=42577 max_load_reduction_pct=0.866231 moved_to_echo=24716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[374, 2124, 583, 480, 62, 400, 1621, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 664, 2499, 0, 0, 472, 2268, 81, 0, 0, 0, 0, 0, 0, 378, 859, 3432, 24, 32, 854, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 3691, 756, 118, 66, 204, 1621, 116, 0, 0, 0, 0, 0, 0, 0, 0, 2916, 422, 77, 215, 375, 0, 1122, 442, 193, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4, 1441, 80, 471, 2334, 805, 807, 169, 149, 0, 0, 0, 0, 0, 745, 1333, 1702, 935, 15, 458, 0, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 275, 474, 457, 108, 340, 614, 1319, 1152, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6152, 6032, 6634, 5762, 6280, 6239, 5913] avg=6144.0000 max=6634 max_rank=3 min=5762 min_rank=4 imbalance_ratio=1.079753 max_load_reduction=42518 max_load_reduction_pct=0.865031 moved_to_echo=24194 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[367, 1697, 619, 488, 75, 425, 1702, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 658, 2337, 0, 0, 552, 2161, 78, 0, 0, 0, 0, 0, 0, 468, 915, 3221, 19, 32, 860, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 3519, 897, 121, 71, 196, 1535, 129, 0, 0, 0, 0, 0, 0, 0, 0, 2806, 495, 90, 218, 485, 0, 1069, 509, 159, 0, 0, 0, 0, 0, 0, 0, 41, 0, 7, 1569, 55, 426, 2315, 808, 854, 162, 124, 0, 0, 0, 0, 0, 817, 1065, 1702, 1068, 9, 494, 0, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1195, 271, 482, 430, 132, 283, 597, 1612, 1089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6054, 6065, 6011, 6513, 5831, 6361, 6226, 6091] avg=6144.0000 max=6513 max_rank=3 min=5831 min_rank=4 imbalance_ratio=1.060059 max_load_reduction=42639 max_load_reduction_pct=0.867493 moved_to_echo=24509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[317, 2261, 536, 326, 88, 412, 1372, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 640, 2900, 0, 0, 314, 2274, 89, 0, 0, 0, 0, 0, 0, 399, 940, 3771, 9, 34, 1184, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 2858, 692, 31, 75, 47, 1625, 85, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 421, 113, 218, 315, 0, 760, 502, 204, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 1201, 52, 435, 2488, 900, 683, 185, 158, 0, 0, 0, 0, 0, 777, 1410, 1442, 1207, 21, 444, 0, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 297, 443, 385, 132, 247, 511, 1369, 1266, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6382, 6800, 5469, 6143, 6125, 6345, 5899] avg=6144.0000 max=6800 max_rank=2 min=5469 min_rank=3 imbalance_ratio=1.106771 max_load_reduction=42352 max_load_reduction_pct=0.861654 moved_to_echo=24512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 15616, 5376, 4224, 640, 2944, 12800, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6656, 9600, 11648, 9088, 256, 3840, 128, 8576, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 128, 11264, 640, 3456, 18688, 6400, 6400, 1408, 1152, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[24832, 3712, 768, 1920, 3456, 384, 8960, 4352, 1408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6656, 9600, 11648, 9088, 256, 3840, 128, 8576, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9600, 2688, 3328, 3968, 1152, 2432, 4864, 12544, 9216, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 1664, 5120, 20096, 0, 128, 4224, 17664, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 7168, 27520, 256, 384, 6912, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[768, 26240, 6272, 640, 768, 1280, 12672, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 7168, 27520, 256, 384, 6912, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[718, 77, 0, 5326, 3, 0, 188, 79, 0, 329, 128, 226, 823, 722, 396, 814, 0, 0, 227, 0, 0, 464, 275, 1255, 120, 71, 420, 0, 54, 64, 0, 0, 79, 2448, 248, 0, 0, 0, 0, 2736, 872, 275, 184, 1399, 5731, 65, 572, 4831, 391, 65, 0, 2028, 52, 2908, 6705, 0, 4336, 0, 0, 0, 0, 0, 386, 62] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[690, 91, 0, 4952, 63, 0, 202, 82, 0, 388, 239, 203, 915, 681, 443, 994, 0, 0, 443, 0, 0, 420, 399, 1353, 120, 66, 497, 0, 195, 399, 0, 0, 82, 2578, 204, 0, 0, 0, 0, 2852, 936, 273, 199, 1417, 5589, 77, 822, 4339, 431, 80, 0, 1531, 107, 2571, 6486, 0, 4205, 0, 0, 0, 0, 0, 461, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[704, 80, 0, 5097, 25, 0, 201, 63, 0, 304, 233, 330, 847, 644, 367, 693, 0, 0, 344, 0, 0, 477, 359, 993, 90, 59, 436, 0, 128, 77, 0, 0, 66, 2602, 173, 0, 0, 0, 0, 2940, 926, 276, 192, 1375, 5673, 78, 653, 4818, 365, 77, 0, 1772, 112, 2954, 6581, 0, 4391, 0, 0, 0, 0, 0, 500, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[706, 102, 0, 4881, 84, 0, 204, 88, 0, 338, 249, 160, 898, 756, 443, 1064, 0, 0, 368, 0, 1, 382, 412, 2003, 104, 50, 564, 0, 314, 471, 0, 0, 88, 2644, 347, 0, 0, 0, 0, 1832, 896, 256, 201, 1695, 5475, 89, 889, 4080, 422, 88, 0, 1718, 259, 1828, 6398, 0, 4763, 0, 0, 1, 0, 0, 420, 121] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[688, 103, 0, 4848, 88, 0, 158, 94, 0, 330, 349, 191, 878, 686, 409, 923, 0, 0, 331, 0, 1, 393, 396, 1772, 122, 54, 562, 0, 334, 358, 0, 0, 94, 2567, 341, 0, 0, 0, 0, 1976, 895, 215, 148, 1839, 5420, 92, 1068, 4350, 397, 105, 0, 1930, 349, 1573, 6421, 0, 4733, 0, 0, 0, 0, 0, 458, 113] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[717, 97, 0, 5098, 62, 0, 207, 95, 0, 383, 251, 194, 847, 772, 427, 900, 0, 0, 534, 0, 0, 462, 375, 1406, 121, 53, 443, 0, 213, 153, 0, 0, 95, 2534, 235, 0, 0, 0, 0, 2308, 940, 268, 208, 1587, 5624, 75, 757, 4781, 410, 72, 0, 2206, 86, 2068, 6458, 0, 4133, 0, 0, 1, 0, 0, 420, 76] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[718, 77, 0, 5326, 3, 0, 188, 79, 0, 329, 128, 226, 823, 722, 396, 814, 0, 0, 227, 0, 0, 464, 275, 1255, 120, 71, 420, 0, 54, 64, 0, 0, 79, 2448, 248, 0, 0, 0, 0, 2736, 872, 275, 184, 1399, 5731, 65, 572, 4831, 391, 65, 0, 2028, 52, 2908, 6705, 0, 4336, 0, 0, 0, 0, 0, 386, 62] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[704, 80, 0, 5097, 25, 0, 201, 63, 0, 304, 233, 330, 847, 644, 367, 693, 0, 0, 344, 0, 0, 477, 359, 993, 90, 59, 436, 0, 128, 77, 0, 0, 66, 2602, 173, 0, 0, 0, 0, 2940, 926, 276, 192, 1375, 5673, 78, 653, 4818, 365, 77, 0, 1772, 112, 2954, 6581, 0, 4391, 0, 0, 0, 0, 0, 500, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[733, 66, 0, 4684, 107, 0, 191, 50, 0, 397, 310, 383, 946, 571, 414, 905, 0, 0, 369, 0, 2, 456, 469, 1309, 93, 66, 532, 0, 159, 166, 0, 1, 49, 2546, 234, 0, 0, 1, 0, 2257, 959, 262, 192, 1739, 5447, 124, 871, 4486, 406, 133, 0, 1741, 359, 2223, 6462, 1, 4635, 0, 0, 1, 1, 0, 537, 107] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[706, 102, 0, 4881, 84, 0, 204, 88, 0, 338, 249, 160, 898, 756, 443, 1064, 0, 0, 368, 0, 1, 382, 412, 2003, 104, 50, 564, 0, 314, 471, 0, 0, 88, 2644, 347, 0, 0, 0, 0, 1832, 896, 256, 201, 1695, 5475, 89, 889, 4080, 422, 88, 0, 1718, 259, 1828, 6398, 0, 4763, 0, 0, 1, 0, 0, 420, 121] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[707, 85, 0, 5241, 23, 0, 180, 82, 0, 406, 174, 273, 832, 731, 412, 722, 0, 0, 337, 1, 0, 423, 282, 1411, 112, 88, 411, 0, 84, 73, 0, 0, 86, 2559, 268, 0, 0, 0, 0, 2304, 916, 273, 175, 1455, 5665, 74, 827, 4567, 416, 73, 0, 2025, 82, 2874, 6531, 0, 4327, 0, 0, 0, 0, 0, 495, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[718, 77, 0, 5326, 3, 0, 188, 79, 5, 0, 0, 0, 0, 0, 0, 0, 0, 329, 128, 226, 823, 722, 396, 814, 2306, 214, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 464, 275, 1255, 3128, 510, 0, 0, 0, 0, 0, 0, 120, 71, 420, 0, 54, 64, 0, 0, 5221, 0, 0, 0, 0, 0, 0, 0, 79, 2448, 248, 0, 0, 0, 0, 2736, 903, 0, 0, 0, 0, 0, 0, 0, 872, 275, 184, 1399, 0, 65, 572, 2477, 0, 0, 0, 0, 0, 0, 0, 0, 391, 65, 0, 2028, 52, 2908, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4336, 0, 0, 0, 0, 0, 386, 62, 1232, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6396, 5958, 5859, 5950, 6414, 5844, 6715, 6016] avg=6144.0000 max=6715 max_rank=6 min=5844 min_rank=5 imbalance_ratio=1.092936 max_load_reduction=42437 max_load_reduction_pct=0.863383 moved_to_echo=24989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[706, 102, 0, 4881, 84, 0, 204, 88, 4, 0, 0, 0, 0, 0, 0, 0, 0, 338, 249, 160, 898, 756, 443, 1064, 2200, 181, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 1, 382, 412, 2003, 2984, 486, 0, 0, 0, 0, 0, 0, 104, 50, 564, 0, 314, 471, 0, 0, 4989, 0, 0, 0, 0, 0, 0, 0, 88, 2644, 347, 0, 0, 0, 0, 1832, 762, 0, 0, 0, 0, 0, 0, 0, 896, 256, 201, 1695, 0, 89, 889, 2093, 0, 0, 0, 0, 0, 0, 0, 0, 422, 88, 0, 1718, 259, 1828, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4763, 0, 0, 1, 0, 0, 420, 121, 1040, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6069, 6289, 6636, 6492, 5673, 6119, 5529, 6345] avg=6144.0000 max=6636 max_rank=2 min=5529 min_rank=6 imbalance_ratio=1.080078 max_load_reduction=42516 max_load_reduction_pct=0.864990 moved_to_echo=23666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[690, 91, 0, 4952, 63, 0, 202, 82, 8, 0, 0, 0, 0, 0, 0, 0, 0, 388, 239, 203, 915, 681, 443, 994, 2233, 196, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 420, 399, 1353, 3030, 498, 0, 0, 0, 0, 0, 0, 120, 66, 497, 0, 195, 399, 0, 0, 5091, 0, 0, 0, 0, 0, 0, 0, 82, 2578, 204, 0, 0, 0, 0, 2852, 815, 0, 0, 0, 0, 0, 0, 0, 936, 273, 199, 1417, 0, 77, 822, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 431, 80, 0, 1531, 107, 2571, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4205, 0, 0, 0, 0, 0, 461, 77, 1109, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6088, 6292, 6143, 6368, 6531, 5935, 5943, 5852] avg=6144.0000 max=6531 max_rank=4 min=5852 min_rank=7 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[733, 66, 0, 4684, 107, 0, 191, 50, 5, 0, 0, 0, 0, 0, 0, 0, 0, 397, 310, 383, 946, 571, 414, 905, 2222, 199, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 2, 456, 469, 1309, 3014, 484, 0, 0, 0, 0, 0, 0, 93, 66, 532, 0, 159, 166, 0, 1, 4963, 0, 0, 0, 0, 0, 0, 0, 49, 2546, 234, 0, 0, 1, 0, 2257, 838, 0, 0, 0, 0, 0, 0, 0, 959, 262, 192, 1739, 0, 124, 871, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 406, 133, 0, 1741, 359, 2223, 1226, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4635, 0, 0, 1, 1, 0, 537, 107, 1144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5836, 6347, 6103, 5980, 5925, 6447, 6089, 6425] avg=6144.0000 max=6447 max_rank=5 min=5836 min_rank=0 imbalance_ratio=1.049316 max_load_reduction=42705 max_load_reduction_pct=0.868835 moved_to_echo=24886 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[707, 85, 0, 5241, 23, 0, 180, 82, 5, 0, 0, 0, 0, 0, 0, 0, 0, 406, 174, 273, 832, 731, 412, 722, 2246, 202, 0, 0, 0, 0, 0, 0, 0, 0, 337, 1, 0, 423, 282, 1411, 3046, 504, 0, 0, 0, 0, 0, 0, 112, 88, 411, 0, 84, 73, 0, 0, 5161, 0, 0, 0, 0, 0, 0, 0, 86, 2559, 268, 0, 0, 0, 0, 2304, 854, 0, 0, 0, 0, 0, 0, 0, 916, 273, 175, 1455, 0, 74, 827, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 416, 73, 0, 2025, 82, 2874, 1239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4327, 0, 0, 0, 0, 0, 495, 70, 1164, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6323, 5998, 6004, 5929, 6071, 6062, 6709, 6056] avg=6144.0000 max=6709 max_rank=6 min=5929 min_rank=3 imbalance_ratio=1.091960 max_load_reduction=42443 max_load_reduction_pct=0.863505 moved_to_echo=24898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[690, 91, 0, 4952, 63, 0, 202, 82, 0, 388, 239, 203, 915, 681, 443, 994, 0, 0, 443, 0, 0, 420, 399, 1353, 120, 66, 497, 0, 195, 399, 0, 0, 82, 2578, 204, 0, 0, 0, 0, 2852, 936, 273, 199, 1417, 5589, 77, 822, 4339, 431, 80, 0, 1531, 107, 2571, 6486, 0, 4205, 0, 0, 0, 0, 0, 461, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[688, 103, 0, 4848, 88, 0, 158, 94, 5, 0, 0, 0, 0, 0, 0, 0, 0, 330, 349, 191, 878, 686, 409, 923, 2208, 193, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 1, 393, 396, 1772, 2995, 482, 0, 0, 0, 0, 0, 0, 122, 54, 562, 0, 334, 358, 0, 0, 4938, 0, 0, 0, 0, 0, 0, 0, 94, 2567, 341, 0, 0, 0, 0, 1976, 813, 0, 0, 0, 0, 0, 0, 0, 895, 215, 148, 1839, 0, 92, 1068, 2230, 0, 0, 0, 0, 0, 0, 0, 0, 397, 105, 0, 1930, 349, 1573, 1218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4733, 0, 0, 0, 0, 0, 458, 113, 1109, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5984, 6167, 6370, 6368, 5791, 6487, 5572, 6413] avg=6144.0000 max=6487 max_rank=5 min=5572 min_rank=6 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24263 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[733, 66, 0, 4684, 107, 0, 191, 50, 5, 0, 0, 0, 0, 0, 0, 0, 0, 397, 310, 383, 946, 571, 414, 905, 2222, 199, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 2, 456, 469, 1309, 3014, 484, 0, 0, 0, 0, 0, 0, 93, 66, 532, 0, 159, 166, 0, 1, 4963, 0, 0, 0, 0, 0, 0, 0, 49, 2546, 234, 0, 0, 1, 0, 2257, 838, 0, 0, 0, 0, 0, 0, 0, 959, 262, 192, 1739, 0, 124, 871, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 406, 133, 0, 1741, 359, 2223, 1226, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4635, 0, 0, 1, 1, 0, 537, 107, 1144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5836, 6347, 6103, 5980, 5925, 6447, 6089, 6425] avg=6144.0000 max=6447 max_rank=5 min=5836 min_rank=0 imbalance_ratio=1.049316 max_load_reduction=42705 max_load_reduction_pct=0.868835 moved_to_echo=24886 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[704, 80, 0, 5097, 25, 0, 201, 63, 5, 0, 0, 0, 0, 0, 0, 0, 0, 304, 233, 330, 847, 644, 367, 693, 2263, 213, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 477, 359, 993, 3070, 504, 0, 0, 0, 0, 0, 0, 90, 59, 436, 0, 128, 77, 0, 0, 5169, 0, 0, 0, 0, 0, 0, 0, 66, 2602, 173, 0, 0, 0, 0, 2940, 900, 0, 0, 0, 0, 0, 0, 0, 926, 276, 192, 1375, 0, 78, 653, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 365, 77, 0, 1772, 112, 2954, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4391, 0, 0, 0, 0, 0, 500, 77, 1228, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6175, 5894, 5747, 5959, 6681, 5972, 6528, 6196] avg=6144.0000 max=6681 max_rank=4 min=5747 min_rank=2 imbalance_ratio=1.087402 max_load_reduction=42471 max_load_reduction_pct=0.864075 moved_to_echo=25377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[717, 97, 0, 5098, 62, 0, 207, 95, 5, 0, 0, 0, 0, 0, 0, 0, 0, 383, 251, 194, 847, 772, 427, 900, 2221, 212, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 462, 375, 1406, 3012, 501, 0, 0, 0, 0, 0, 0, 121, 53, 443, 0, 213, 153, 0, 0, 5123, 0, 0, 0, 0, 0, 0, 0, 95, 2534, 235, 0, 0, 0, 0, 2308, 894, 0, 0, 0, 0, 0, 0, 0, 940, 268, 208, 1587, 0, 75, 757, 2451, 0, 0, 0, 0, 0, 0, 0, 0, 410, 72, 0, 2206, 86, 2068, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4133, 0, 0, 1, 0, 0, 420, 76, 1219, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6281, 6207, 6290, 6106, 6066, 6286, 6067, 5849] avg=6144.0000 max=6290 max_rank=2 min=5849 min_rank=7 imbalance_ratio=1.023763 max_load_reduction=42862 max_load_reduction_pct=0.872030 moved_to_echo=24268 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[718, 77, 0, 5326, 3, 0, 188, 79, 5, 0, 0, 0, 0, 0, 0, 0, 0, 329, 128, 226, 823, 722, 396, 814, 2306, 214, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 464, 275, 1255, 3128, 510, 0, 0, 0, 0, 0, 0, 120, 71, 420, 0, 54, 64, 0, 0, 5221, 0, 0, 0, 0, 0, 0, 0, 79, 2448, 248, 0, 0, 0, 0, 2736, 903, 0, 0, 0, 0, 0, 0, 0, 872, 275, 184, 1399, 0, 65, 572, 2477, 0, 0, 0, 0, 0, 0, 0, 0, 391, 65, 0, 2028, 52, 2908, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4336, 0, 0, 0, 0, 0, 386, 62, 1232, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6396, 5958, 5859, 5950, 6414, 5844, 6715, 6016] avg=6144.0000 max=6715 max_rank=6 min=5844 min_rank=5 imbalance_ratio=1.092936 max_load_reduction=42437 max_load_reduction_pct=0.863383 moved_to_echo=24989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[706, 102, 0, 4881, 84, 0, 204, 88, 4, 0, 0, 0, 0, 0, 0, 0, 0, 338, 249, 160, 898, 756, 443, 1064, 2200, 181, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 1, 382, 412, 2003, 2984, 486, 0, 0, 0, 0, 0, 0, 104, 50, 564, 0, 314, 471, 0, 0, 4989, 0, 0, 0, 0, 0, 0, 0, 88, 2644, 347, 0, 0, 0, 0, 1832, 762, 0, 0, 0, 0, 0, 0, 0, 896, 256, 201, 1695, 0, 89, 889, 2093, 0, 0, 0, 0, 0, 0, 0, 0, 422, 88, 0, 1718, 259, 1828, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4763, 0, 0, 1, 0, 0, 420, 121, 1040, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6069, 6289, 6636, 6492, 5673, 6119, 5529, 6345] avg=6144.0000 max=6636 max_rank=2 min=5529 min_rank=6 imbalance_ratio=1.080078 max_load_reduction=42516 max_load_reduction_pct=0.864990 moved_to_echo=23666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[707, 85, 0, 5241, 23, 0, 180, 82, 5, 0, 0, 0, 0, 0, 0, 0, 0, 406, 174, 273, 832, 731, 412, 722, 2246, 202, 0, 0, 0, 0, 0, 0, 0, 0, 337, 1, 0, 423, 282, 1411, 3046, 504, 0, 0, 0, 0, 0, 0, 112, 88, 411, 0, 84, 73, 0, 0, 5161, 0, 0, 0, 0, 0, 0, 0, 86, 2559, 268, 0, 0, 0, 0, 2304, 854, 0, 0, 0, 0, 0, 0, 0, 916, 273, 175, 1455, 0, 74, 827, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 416, 73, 0, 2025, 82, 2874, 1239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4327, 0, 0, 0, 0, 0, 495, 70, 1164, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6323, 5998, 6004, 5929, 6071, 6062, 6709, 6056] avg=6144.0000 max=6709 max_rank=6 min=5929 min_rank=3 imbalance_ratio=1.091960 max_load_reduction=42443 max_load_reduction_pct=0.863505 moved_to_echo=24898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[690, 91, 0, 4952, 63, 0, 202, 82, 8, 0, 0, 0, 0, 0, 0, 0, 0, 388, 239, 203, 915, 681, 443, 994, 2233, 196, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 420, 399, 1353, 3030, 498, 0, 0, 0, 0, 0, 0, 120, 66, 497, 0, 195, 399, 0, 0, 5091, 0, 0, 0, 0, 0, 0, 0, 82, 2578, 204, 0, 0, 0, 0, 2852, 815, 0, 0, 0, 0, 0, 0, 0, 936, 273, 199, 1417, 0, 77, 822, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 431, 80, 0, 1531, 107, 2571, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4205, 0, 0, 0, 0, 0, 461, 77, 1109, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6088, 6292, 6143, 6368, 6531, 5935, 5943, 5852] avg=6144.0000 max=6531 max_rank=4 min=5852 min_rank=7 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 2944, 2048, 2048, 7040, 5632, 3328, 7040, 17920, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7424, 2176, 1536, 12544, 0, 768, 6528, 18688, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 20480, 2176, 0, 0, 128, 0, 19328, 6784, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 2944, 2048, 2048, 7040, 5632, 3328, 7040, 17920, 1664, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 768, 0, 14976, 1408, 19072, 9984, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[35584, 0, 0, 128, 128, 0, 3712, 768, 9344, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 0, 3072, 128, 128, 3584, 3072, 11520, 24320, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 20480, 2176, 0, 0, 128, 0, 19328, 6784, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[35584, 0, 0, 128, 128, 0, 3712, 768, 9344, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5760, 768, 0, 40192, 512, 0, 1536, 640, 128, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 512, 3968, 0, 1536, 1792, 0, 128, 40704, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2351, 7284, 0, 19, 418, 0, 59, 0, 0, 295, 61, 648, 65, 349, 377, 794, 144, 1709, 0, 0, 209, 93, 372, 873, 5160, 4716, 2047, 381, 0, 0, 87, 236, 0, 278, 398, 0, 72, 2071, 481, 75, 570, 0, 0, 53, 303, 6255, 23, 0, 5898, 0, 186, 1005, 481, 72, 0, 0, 0, 10, 0, 0, 0, 1740, 210, 224] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2581, 7518, 0, 2, 340, 0, 50, 0, 0, 318, 50, 644, 16, 216, 429, 898, 90, 1930, 0, 0, 141, 52, 189, 652, 5684, 4251, 2222, 204, 0, 0, 97, 174, 0, 373, 256, 0, 65, 2154, 576, 65, 574, 1, 0, 71, 69, 6325, 2, 0, 5519, 0, 77, 957, 453, 63, 0, 0, 0, 6, 0, 0, 0, 2243, 346, 209] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2161, 7293, 1, 16, 596, 0, 37, 0, 0, 291, 78, 626, 152, 291, 509, 861, 160, 1836, 0, 0, 236, 58, 549, 841, 5143, 4206, 2144, 315, 0, 1, 136, 205, 0, 344, 395, 0, 107, 1916, 586, 108, 758, 0, 0, 66, 262, 6004, 26, 1, 5654, 1, 359, 989, 512, 107, 2, 0, 0, 69, 1, 0, 1, 1576, 234, 332] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1977, 7343, 0, 12, 354, 0, 59, 0, 0, 316, 59, 404, 86, 320, 493, 933, 159, 1920, 0, 0, 231, 80, 365, 859, 5253, 4296, 2204, 295, 0, 0, 55, 202, 0, 358, 349, 0, 77, 2070, 623, 79, 683, 0, 0, 66, 183, 6304, 13, 0, 5724, 0, 283, 1018, 507, 74, 0, 0, 0, 46, 0, 0, 0, 1984, 197, 239] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1842, 7203, 0, 40, 607, 1, 57, 0, 0, 316, 118, 495, 100, 307, 425, 999, 192, 1331, 0, 0, 264, 96, 973, 1287, 4373, 4730, 2125, 421, 0, 0, 89, 223, 0, 246, 580, 0, 86, 1415, 565, 99, 631, 1, 0, 50, 393, 6386, 21, 0, 5920, 0, 380, 951, 504, 85, 163, 0, 0, 302, 0, 0, 0, 1249, 241, 270] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1607, 7212, 1, 45, 585, 3, 66, 0, 0, 169, 103, 656, 132, 221, 410, 922, 303, 1210, 0, 0, 179, 84, 1063, 1220, 4461, 4343, 2201, 412, 0, 1, 225, 198, 0, 259, 551, 0, 91, 1546, 604, 121, 647, 0, 0, 54, 374, 6539, 34, 0, 5896, 0, 433, 1017, 483, 91, 266, 0, 1, 367, 0, 0, 0, 1262, 183, 301] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1977, 7343, 0, 12, 354, 0, 59, 0, 0, 316, 59, 404, 86, 320, 493, 933, 159, 1920, 0, 0, 231, 80, 365, 859, 5253, 4296, 2204, 295, 0, 0, 55, 202, 0, 358, 349, 0, 77, 2070, 623, 79, 683, 0, 0, 66, 183, 6304, 13, 0, 5724, 0, 283, 1018, 507, 74, 0, 0, 0, 46, 0, 0, 0, 1984, 197, 239] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2581, 7518, 0, 2, 340, 0, 50, 0, 0, 318, 50, 644, 16, 216, 429, 898, 90, 1930, 0, 0, 141, 52, 189, 652, 5684, 4251, 2222, 204, 0, 0, 97, 174, 0, 373, 256, 0, 65, 2154, 576, 65, 574, 1, 0, 71, 69, 6325, 2, 0, 5519, 0, 77, 957, 453, 63, 0, 0, 0, 6, 0, 0, 0, 2243, 346, 209] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[2337, 7499, 0, 6, 366, 0, 50, 0, 0, 357, 50, 500, 98, 304, 490, 821, 69, 1972, 0, 0, 164, 54, 139, 610, 5906, 4252, 2015, 294, 0, 0, 77, 209, 0, 376, 266, 0, 79, 2570, 579, 77, 711, 0, 0, 59, 126, 6232, 13, 0, 5403, 0, 84, 931, 441, 74, 0, 0, 0, 11, 0, 0, 0, 1975, 235, 271] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2304, 7472, 0, 14, 349, 0, 51, 0, 0, 320, 55, 609, 31, 322, 372, 679, 114, 1695, 0, 0, 146, 78, 381, 716, 5396, 4672, 2280, 310, 1, 0, 118, 179, 0, 356, 301, 0, 75, 2142, 459, 76, 542, 0, 0, 88, 75, 6321, 4, 0, 5683, 0, 60, 1092, 482, 73, 0, 0, 0, 10, 0, 0, 0, 2176, 229, 244] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2351, 3442, 0, 19, 418, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 61, 648, 65, 349, 377, 794, 2101, 1399, 0, 0, 0, 0, 0, 0, 144, 1709, 0, 0, 209, 93, 372, 873, 1088, 1156, 465, 0, 0, 0, 0, 0, 0, 3560, 2047, 381, 0, 0, 87, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 398, 0, 72, 2071, 481, 75, 1741, 993, 0, 0, 0, 0, 0, 0, 570, 0, 0, 53, 303, 5167, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4440, 0, 186, 1005, 481, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1740, 210, 224, 3761, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6289, 6089, 6109, 6311, 6109, 6116, 6184, 5945] avg=6144.0000 max=6311 max_rank=3 min=5945 min_rank=7 imbalance_ratio=1.027181 max_load_reduction=42841 max_load_reduction_pct=0.871602 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1842, 7203, 0, 40, 607, 1, 57, 0, 0, 316, 118, 495, 100, 307, 425, 999, 192, 1331, 0, 0, 264, 96, 973, 1287, 4373, 4730, 2125, 421, 0, 0, 89, 223, 0, 246, 580, 0, 86, 1415, 565, 99, 631, 1, 0, 50, 393, 6386, 21, 0, 5920, 0, 380, 951, 504, 85, 163, 0, 0, 302, 0, 0, 0, 1249, 241, 270] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2161, 7293, 1, 16, 596, 0, 37, 0, 0, 291, 78, 626, 152, 291, 509, 861, 160, 1836, 0, 0, 236, 58, 549, 841, 5143, 4206, 2144, 315, 0, 1, 136, 205, 0, 344, 395, 0, 107, 1916, 586, 108, 758, 0, 0, 66, 262, 6004, 26, 1, 5654, 1, 359, 989, 512, 107, 2, 0, 0, 69, 1, 0, 1, 1576, 234, 332] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2351, 7284, 0, 19, 418, 0, 59, 0, 0, 295, 61, 648, 65, 349, 377, 794, 144, 1709, 0, 0, 209, 93, 372, 873, 5160, 4716, 2047, 381, 0, 0, 87, 236, 0, 278, 398, 0, 72, 2071, 481, 75, 570, 0, 0, 53, 303, 6255, 23, 0, 5898, 0, 186, 1005, 481, 72, 0, 0, 0, 10, 0, 0, 0, 1740, 210, 224] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1607, 3407, 1, 45, 585, 3, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 103, 656, 132, 221, 410, 922, 2081, 1209, 0, 0, 0, 0, 0, 0, 303, 1210, 0, 0, 179, 84, 1063, 1220, 1137, 1064, 465, 0, 0, 0, 0, 0, 0, 3279, 2201, 412, 0, 1, 225, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 551, 0, 91, 1546, 604, 121, 1724, 992, 0, 0, 0, 0, 0, 0, 647, 0, 0, 54, 374, 5402, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4439, 0, 433, 1017, 483, 91, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 367, 0, 0, 0, 1262, 183, 301, 3252, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5714, 5903, 6725, 6316, 5888, 6511, 6729, 5366] avg=6144.0000 max=6729 max_rank=6 min=5366 min_rank=7 imbalance_ratio=1.095215 max_load_reduction=42423 max_load_reduction_pct=0.863098 moved_to_echo=24494 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2581, 3552, 0, 2, 340, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 50, 644, 16, 216, 429, 898, 2169, 1542, 0, 0, 0, 0, 0, 0, 90, 1930, 0, 0, 141, 52, 189, 652, 1100, 1042, 435, 0, 0, 0, 0, 0, 0, 3209, 2222, 204, 0, 0, 97, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 256, 0, 65, 2154, 576, 65, 1797, 929, 0, 0, 0, 0, 0, 0, 574, 1, 0, 71, 69, 5225, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4155, 0, 77, 957, 453, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2243, 346, 209, 4142, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6525, 6282, 5631, 5906, 6215, 5942, 5705, 6946] avg=6144.0000 max=6946 max_rank=7 min=5631 min_rank=2 imbalance_ratio=1.130534 max_load_reduction=42206 max_load_reduction_pct=0.858683 moved_to_echo=24808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1842, 3404, 0, 40, 607, 1, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 118, 495, 100, 307, 425, 999, 2078, 1185, 0, 0, 0, 0, 0, 0, 192, 1331, 0, 0, 264, 96, 973, 1287, 1110, 1159, 467, 0, 0, 0, 0, 0, 0, 3571, 2125, 421, 0, 0, 89, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 580, 0, 86, 1415, 565, 99, 1721, 996, 0, 0, 0, 0, 0, 0, 631, 1, 0, 50, 393, 5276, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4457, 0, 380, 951, 504, 85, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 0, 1249, 241, 270, 3188, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5951, 6023, 6879, 6429, 5708, 6372, 6540, 5250] avg=6144.0000 max=6879 max_rank=2 min=5250 min_rank=7 imbalance_ratio=1.119629 max_load_reduction=42273 max_load_reduction_pct=0.860046 moved_to_echo=23870 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1607, 3407, 1, 45, 585, 3, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 103, 656, 132, 221, 410, 922, 2081, 1209, 0, 0, 0, 0, 0, 0, 303, 1210, 0, 0, 179, 84, 1063, 1220, 1137, 1064, 465, 0, 0, 0, 0, 0, 0, 3279, 2201, 412, 0, 1, 225, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 551, 0, 91, 1546, 604, 121, 1724, 992, 0, 0, 0, 0, 0, 0, 647, 0, 0, 54, 374, 5402, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4439, 0, 433, 1017, 483, 91, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 367, 0, 0, 0, 1262, 183, 301, 3252, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5714, 5903, 6725, 6316, 5888, 6511, 6729, 5366] avg=6144.0000 max=6729 max_rank=6 min=5366 min_rank=7 imbalance_ratio=1.095215 max_load_reduction=42423 max_load_reduction_pct=0.863098 moved_to_echo=24494 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2337, 3544, 0, 6, 366, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 50, 500, 98, 304, 490, 821, 2163, 1601, 0, 0, 0, 0, 0, 0, 69, 1972, 0, 0, 164, 54, 139, 610, 1084, 1042, 426, 0, 0, 0, 0, 0, 0, 3210, 2015, 294, 0, 0, 77, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 266, 0, 79, 2570, 579, 77, 1792, 909, 0, 0, 0, 0, 0, 0, 711, 0, 0, 59, 126, 5148, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 84, 931, 441, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 1975, 235, 271, 4305, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6303, 6384, 5560, 5805, 6648, 6057, 5598, 6797] avg=6144.0000 max=6797 max_rank=7 min=5560 min_rank=2 imbalance_ratio=1.106283 max_load_reduction=42355 max_load_reduction_pct=0.861715 moved_to_echo=25100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2161, 3446, 1, 16, 596, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 78, 626, 152, 291, 509, 861, 2104, 1394, 0, 0, 0, 0, 0, 0, 160, 1836, 0, 0, 236, 58, 549, 841, 1044, 1031, 446, 0, 0, 0, 0, 0, 0, 3175, 2144, 315, 0, 1, 136, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 395, 0, 107, 1916, 586, 108, 1743, 952, 0, 0, 0, 0, 0, 0, 758, 0, 0, 66, 262, 4960, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4256, 1, 359, 989, 512, 107, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1, 0, 1, 1576, 234, 332, 3749, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6257, 6306, 6201, 5976, 6151, 6073, 6226, 5962] avg=6144.0000 max=6306 max_rank=1 min=5962 min_rank=7 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1977, 3463, 0, 12, 354, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 59, 404, 86, 320, 493, 933, 2122, 1425, 0, 0, 0, 0, 0, 0, 159, 1920, 0, 0, 231, 80, 365, 859, 1099, 1058, 454, 0, 0, 0, 0, 0, 0, 3238, 2204, 295, 0, 0, 55, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 349, 0, 77, 2070, 623, 79, 1758, 968, 0, 0, 0, 0, 0, 0, 683, 0, 0, 66, 183, 5205, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4302, 0, 283, 1018, 507, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 1984, 197, 239, 3828, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5865, 6158, 6225, 5994, 6282, 6150, 6184, 6294] avg=6144.0000 max=6294 max_rank=7 min=5865 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2304, 3530, 0, 14, 349, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 55, 609, 31, 322, 372, 679, 2156, 1463, 0, 0, 0, 0, 0, 0, 114, 1695, 0, 0, 146, 78, 381, 716, 1099, 1145, 448, 0, 0, 0, 0, 0, 0, 3527, 2280, 310, 1, 0, 118, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 301, 0, 75, 2142, 459, 76, 1786, 956, 0, 0, 0, 0, 0, 0, 542, 0, 0, 88, 75, 5222, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4279, 0, 60, 1092, 482, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2176, 229, 244, 3933, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6248, 6007, 5822, 6415, 6151, 5931, 5986, 6592] avg=6144.0000 max=6592 max_rank=7 min=5822 min_rank=2 imbalance_ratio=1.072917 max_load_reduction=42560 max_load_reduction_pct=0.865885 moved_to_echo=24660 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1977, 3463, 0, 12, 354, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 59, 404, 86, 320, 493, 933, 2122, 1425, 0, 0, 0, 0, 0, 0, 159, 1920, 0, 0, 231, 80, 365, 859, 1099, 1058, 454, 0, 0, 0, 0, 0, 0, 3238, 2204, 295, 0, 0, 55, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 349, 0, 77, 2070, 623, 79, 1758, 968, 0, 0, 0, 0, 0, 0, 683, 0, 0, 66, 183, 5205, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4302, 0, 283, 1018, 507, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 1984, 197, 239, 3828, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5865, 6158, 6225, 5994, 6282, 6150, 6184, 6294] avg=6144.0000 max=6294 max_rank=7 min=5865 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2351, 3442, 0, 19, 418, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 61, 648, 65, 349, 377, 794, 2101, 1399, 0, 0, 0, 0, 0, 0, 144, 1709, 0, 0, 209, 93, 372, 873, 1088, 1156, 465, 0, 0, 0, 0, 0, 0, 3560, 2047, 381, 0, 0, 87, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 398, 0, 72, 2071, 481, 75, 1741, 993, 0, 0, 0, 0, 0, 0, 570, 0, 0, 53, 303, 5167, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4440, 0, 186, 1005, 481, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1740, 210, 224, 3761, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6289, 6089, 6109, 6311, 6109, 6116, 6184, 5945] avg=6144.0000 max=6311 max_rank=3 min=5945 min_rank=7 imbalance_ratio=1.027181 max_load_reduction=42841 max_load_reduction_pct=0.871602 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2161, 3446, 1, 16, 596, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 78, 626, 152, 291, 509, 861, 2104, 1394, 0, 0, 0, 0, 0, 0, 160, 1836, 0, 0, 236, 58, 549, 841, 1044, 1031, 446, 0, 0, 0, 0, 0, 0, 3175, 2144, 315, 0, 1, 136, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 395, 0, 107, 1916, 586, 108, 1743, 952, 0, 0, 0, 0, 0, 0, 758, 0, 0, 66, 262, 4960, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4256, 1, 359, 989, 512, 107, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1, 0, 1, 1576, 234, 332, 3749, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6257, 6306, 6201, 5976, 6151, 6073, 6226, 5962] avg=6144.0000 max=6306 max_rank=1 min=5962 min_rank=7 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 2432, 640, 4608, 768, 2432, 3584, 6912, 17024, 11264, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1280, 13696, 0, 0, 1664, 640, 4096, 7168, 8832, 8704, 3712, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 2688, 3200, 0, 768, 16000, 4480, 768, 14080, 7808, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[17280, 27904, 128, 256, 3712, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 2688, 3200, 0, 768, 16000, 4480, 768, 14080, 7808, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 26880, 17280, 2688, 128, 128, 896, 1664, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[34432, 128, 1920, 8064, 3968, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 896, 128, 0, 128, 14208, 1920, 2176, 30208, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5120, 128, 0, 512, 1792, 41728, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[625, 64, 4795, 1359, 3766, 0, 6587, 5092, 0, 127, 0, 812, 0, 507, 0, 783, 243, 0, 2338, 0, 912, 53, 152, 82, 50, 64, 476, 0, 0, 115, 0, 176, 6045, 611, 0, 65, 195, 1264, 763, 0, 143, 0, 0, 0, 0, 0, 0, 3, 1733, 0, 0, 71, 1, 1618, 659, 0, 2849, 455, 1, 1583, 1915, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[582, 77, 5041, 1360, 3337, 0, 6651, 5285, 0, 155, 0, 798, 0, 435, 0, 738, 251, 0, 2618, 0, 934, 57, 277, 85, 50, 77, 461, 0, 0, 128, 0, 183, 5995, 614, 0, 78, 199, 1110, 760, 0, 190, 0, 0, 0, 0, 0, 0, 0, 1938, 0, 0, 59, 6, 1802, 581, 0, 2349, 441, 1, 1559, 1881, 9, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[596, 110, 5429, 1473, 2995, 1, 6449, 5535, 0, 214, 0, 809, 1, 424, 1, 442, 246, 7, 2513, 0, 992, 47, 390, 116, 36, 107, 559, 0, 0, 144, 1, 181, 5869, 652, 0, 107, 207, 862, 756, 0, 258, 1, 0, 0, 0, 1, 3, 0, 1969, 0, 61, 66, 1, 1580, 475, 1, 2803, 512, 14, 1410, 1725, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[586, 104, 5712, 1401, 3796, 0, 6389, 5667, 0, 182, 0, 753, 0, 516, 0, 444, 252, 0, 3258, 0, 949, 50, 320, 148, 64, 91, 521, 0, 0, 157, 0, 129, 5660, 517, 0, 91, 214, 585, 713, 0, 212, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 18, 57, 0, 1768, 400, 0, 2740, 483, 5, 872, 1235, 49, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[562, 86, 5520, 1591, 3748, 0, 6546, 5549, 0, 167, 0, 707, 0, 432, 0, 460, 241, 0, 2890, 0, 972, 61, 352, 139, 57, 84, 549, 0, 0, 142, 0, 80, 5790, 599, 0, 85, 192, 621, 730, 0, 188, 0, 0, 0, 0, 0, 0, 0, 2111, 0, 1, 50, 8, 1848, 415, 0, 2829, 504, 1, 1058, 1176, 11, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[535, 72, 5271, 1695, 3218, 0, 6465, 5605, 0, 144, 0, 744, 0, 370, 0, 618, 259, 0, 2697, 0, 979, 58, 394, 151, 59, 72, 503, 0, 0, 131, 0, 168, 5990, 572, 0, 72, 208, 802, 703, 0, 193, 0, 0, 0, 0, 0, 0, 0, 2053, 0, 0, 53, 2, 1703, 407, 0, 2619, 482, 2, 1354, 1722, 7, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[566, 77, 5064, 1393, 3150, 0, 6446, 5569, 0, 153, 0, 708, 0, 546, 0, 582, 246, 0, 2806, 0, 990, 50, 271, 138, 59, 77, 549, 0, 0, 136, 0, 180, 5984, 665, 0, 77, 226, 993, 812, 0, 221, 0, 0, 0, 0, 0, 0, 0, 2040, 0, 0, 66, 3, 1651, 535, 0, 2655, 505, 4, 1365, 1592, 2, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[586, 104, 5712, 1401, 3796, 0, 6389, 5667, 0, 182, 0, 753, 0, 516, 0, 444, 252, 0, 3258, 0, 949, 50, 320, 148, 64, 91, 521, 0, 0, 157, 0, 129, 5660, 517, 0, 91, 214, 585, 713, 0, 212, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 18, 57, 0, 1768, 400, 0, 2740, 483, 5, 872, 1235, 49, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[600, 73, 4935, 1141, 3772, 0, 6539, 5057, 0, 147, 0, 766, 0, 516, 0, 537, 241, 0, 3025, 0, 916, 50, 212, 121, 51, 73, 491, 0, 0, 125, 0, 140, 5781, 482, 0, 74, 194, 1131, 741, 0, 239, 0, 0, 0, 0, 0, 0, 22, 2010, 0, 0, 88, 4, 2178, 489, 0, 2485, 489, 0, 1293, 1923, 1, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[582, 77, 5041, 1360, 3337, 0, 6651, 5285, 0, 155, 0, 798, 0, 435, 0, 738, 251, 0, 2618, 0, 934, 57, 277, 85, 50, 77, 461, 0, 0, 128, 0, 183, 5995, 614, 0, 78, 199, 1110, 760, 0, 190, 0, 0, 0, 0, 0, 0, 0, 1938, 0, 0, 59, 6, 1802, 581, 0, 2349, 441, 1, 1559, 1881, 9, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[596, 110, 5429, 1473, 2995, 1, 6449, 5535, 0, 214, 0, 809, 1, 424, 1, 442, 246, 7, 2513, 0, 992, 47, 390, 116, 36, 107, 559, 0, 0, 144, 1, 181, 5869, 652, 0, 107, 207, 862, 756, 0, 258, 1, 0, 0, 0, 1, 3, 0, 1969, 0, 61, 66, 1, 1580, 475, 1, 2803, 512, 14, 1410, 1725, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[625, 64, 533, 1359, 3766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 812, 0, 507, 0, 783, 3113, 740, 0, 0, 0, 0, 0, 0, 243, 0, 2338, 0, 912, 53, 152, 82, 1745, 0, 0, 0, 0, 0, 0, 0, 50, 64, 476, 0, 0, 115, 0, 176, 4352, 582, 0, 0, 0, 0, 0, 0, 3698, 611, 0, 65, 195, 1264, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 3, 6005, 0, 0, 0, 0, 0, 0, 0, 1733, 0, 0, 71, 1, 1618, 659, 0, 1149, 579, 0, 0, 0, 0, 0, 0, 2849, 455, 1, 1583, 1915, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6347, 6082, 5525, 5815, 6596, 6151, 5810, 6826] avg=6144.0000 max=6826 max_rank=7 min=5525 min_rank=2 imbalance_ratio=1.111003 max_load_reduction=42326 max_load_reduction_pct=0.861125 moved_to_echo=25383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[562, 86, 615, 1591, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 707, 0, 432, 0, 460, 3583, 805, 0, 0, 0, 0, 0, 0, 241, 0, 2890, 0, 972, 61, 352, 139, 1671, 0, 0, 0, 0, 0, 0, 0, 57, 84, 549, 0, 0, 142, 0, 80, 4744, 577, 0, 0, 0, 0, 0, 0, 3542, 599, 0, 85, 192, 621, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 0, 0, 0, 0, 0, 2111, 0, 1, 50, 8, 1848, 415, 0, 1322, 555, 0, 0, 0, 0, 0, 0, 2829, 504, 1, 1058, 1176, 11, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6602, 6154, 6326, 6233, 5769, 6157, 6310, 5601] avg=6144.0000 max=6602 max_rank=0 min=5601 min_rank=7 imbalance_ratio=1.074544 max_load_reduction=42550 max_load_reduction_pct=0.865682 moved_to_echo=23837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[535, 72, 586, 1695, 3218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 744, 0, 370, 0, 618, 3422, 815, 0, 0, 0, 0, 0, 0, 259, 0, 2697, 0, 979, 58, 394, 151, 1729, 0, 0, 0, 0, 0, 0, 0, 59, 72, 503, 0, 0, 131, 0, 168, 4790, 571, 0, 0, 0, 0, 0, 0, 3664, 572, 0, 72, 208, 802, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 5894, 0, 0, 0, 0, 0, 0, 0, 2053, 0, 0, 53, 2, 1703, 407, 0, 1263, 574, 0, 0, 0, 0, 0, 0, 2619, 482, 2, 1354, 1722, 7, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 6113, 6267, 6294, 6021, 6087, 6055, 6209] avg=6144.0000 max=6294 max_rank=3 min=6021 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[586, 104, 636, 1401, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 753, 0, 516, 0, 444, 3708, 823, 0, 0, 0, 0, 0, 0, 252, 0, 3258, 0, 949, 50, 320, 148, 1634, 0, 0, 0, 0, 0, 0, 0, 64, 91, 521, 0, 0, 157, 0, 129, 4844, 564, 0, 0, 0, 0, 0, 0, 3462, 517, 0, 91, 214, 585, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 5825, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 18, 57, 0, 1768, 400, 0, 1368, 543, 0, 0, 0, 0, 0, 0, 2740, 483, 5, 872, 1235, 49, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6523, 6426, 6611, 6370, 5582, 6037, 6198, 5405] avg=6144.0000 max=6611 max_rank=2 min=5405 min_rank=7 imbalance_ratio=1.076009 max_load_reduction=42541 max_load_reduction_pct=0.865499 moved_to_echo=23222 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[535, 72, 5271, 1695, 3218, 0, 6465, 5605, 0, 144, 0, 744, 0, 370, 0, 618, 259, 0, 2697, 0, 979, 58, 394, 151, 59, 72, 503, 0, 0, 131, 0, 168, 5990, 572, 0, 72, 208, 802, 703, 0, 193, 0, 0, 0, 0, 0, 0, 0, 2053, 0, 0, 53, 2, 1703, 407, 0, 2619, 482, 2, 1354, 1722, 7, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[625, 64, 4795, 1359, 3766, 0, 6587, 5092, 0, 127, 0, 812, 0, 507, 0, 783, 243, 0, 2338, 0, 912, 53, 152, 82, 50, 64, 476, 0, 0, 115, 0, 176, 6045, 611, 0, 65, 195, 1264, 763, 0, 143, 0, 0, 0, 0, 0, 0, 3, 1733, 0, 0, 71, 1, 1618, 659, 0, 2849, 455, 1, 1583, 1915, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[582, 77, 561, 1360, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 798, 0, 435, 0, 738, 3272, 767, 0, 0, 0, 0, 0, 0, 251, 0, 2618, 0, 934, 57, 277, 85, 1730, 0, 0, 0, 0, 0, 0, 0, 50, 77, 461, 0, 0, 128, 0, 183, 4518, 586, 0, 0, 0, 0, 0, 0, 3667, 614, 0, 78, 199, 1110, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 6065, 0, 0, 0, 0, 0, 0, 0, 1938, 0, 0, 59, 6, 1802, 581, 0, 1208, 575, 0, 0, 0, 0, 0, 0, 2349, 441, 1, 1559, 1881, 9, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6165, 5952, 6003, 6428, 6255, 6169, 6263] avg=6144.0000 max=6428 max_rank=4 min=5917 min_rank=0 imbalance_ratio=1.046224 max_load_reduction=42724 max_load_reduction_pct=0.869222 moved_to_echo=25115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[566, 77, 556, 1393, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 708, 0, 546, 0, 582, 3291, 809, 0, 0, 0, 0, 0, 0, 246, 0, 2806, 0, 990, 50, 271, 138, 1731, 0, 0, 0, 0, 0, 0, 0, 59, 77, 549, 0, 0, 136, 0, 180, 4760, 569, 0, 0, 0, 0, 0, 0, 3651, 665, 0, 77, 226, 993, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 5877, 0, 0, 0, 0, 0, 0, 0, 2040, 0, 0, 66, 3, 1651, 535, 0, 1217, 577, 0, 0, 0, 0, 0, 0, 2655, 505, 4, 1365, 1592, 2, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5742, 6089, 6232, 6330, 6424, 6098, 6089, 6148] avg=6144.0000 max=6424 max_rank=4 min=5742 min_rank=0 imbalance_ratio=1.045573 max_load_reduction=42728 max_load_reduction_pct=0.869303 moved_to_echo=24759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[586, 104, 636, 1401, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 753, 0, 516, 0, 444, 3708, 823, 0, 0, 0, 0, 0, 0, 252, 0, 3258, 0, 949, 50, 320, 148, 1634, 0, 0, 0, 0, 0, 0, 0, 64, 91, 521, 0, 0, 157, 0, 129, 4844, 564, 0, 0, 0, 0, 0, 0, 3462, 517, 0, 91, 214, 585, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 5825, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 18, 57, 0, 1768, 400, 0, 1368, 543, 0, 0, 0, 0, 0, 0, 2740, 483, 5, 872, 1235, 49, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6523, 6426, 6611, 6370, 5582, 6037, 6198, 5405] avg=6144.0000 max=6611 max_rank=2 min=5405 min_rank=7 imbalance_ratio=1.076009 max_load_reduction=42541 max_load_reduction_pct=0.865499 moved_to_echo=23222 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[596, 110, 605, 1473, 2995, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 809, 1, 424, 1, 442, 3524, 803, 0, 0, 0, 0, 0, 0, 246, 7, 2513, 0, 992, 47, 390, 116, 1694, 0, 0, 0, 0, 0, 0, 0, 36, 107, 559, 0, 0, 144, 1, 181, 4732, 568, 0, 0, 0, 0, 0, 0, 3590, 652, 0, 107, 207, 862, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 1, 0, 0, 0, 1, 3, 0, 5881, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 61, 66, 1, 1580, 475, 1, 1300, 563, 0, 0, 0, 0, 0, 0, 2803, 512, 14, 1410, 1725, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5780, 6218, 6005, 6328, 6174, 6144, 6016, 6487] avg=6144.0000 max=6487 max_rank=7 min=5780 min_rank=0 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24821 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[562, 86, 615, 1591, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 707, 0, 432, 0, 460, 3583, 805, 0, 0, 0, 0, 0, 0, 241, 0, 2890, 0, 972, 61, 352, 139, 1671, 0, 0, 0, 0, 0, 0, 0, 57, 84, 549, 0, 0, 142, 0, 80, 4744, 577, 0, 0, 0, 0, 0, 0, 3542, 599, 0, 85, 192, 621, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 0, 0, 0, 0, 0, 2111, 0, 1, 50, 8, 1848, 415, 0, 1322, 555, 0, 0, 0, 0, 0, 0, 2829, 504, 1, 1058, 1176, 11, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6602, 6154, 6326, 6233, 5769, 6157, 6310, 5601] avg=6144.0000 max=6602 max_rank=0 min=5601 min_rank=7 imbalance_ratio=1.074544 max_load_reduction=42550 max_load_reduction_pct=0.865682 moved_to_echo=23837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[600, 73, 549, 1141, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 766, 0, 516, 0, 537, 3204, 735, 0, 0, 0, 0, 0, 0, 241, 0, 3025, 0, 916, 50, 212, 121, 1669, 0, 0, 0, 0, 0, 0, 0, 51, 73, 491, 0, 0, 125, 0, 140, 4322, 577, 0, 0, 0, 0, 0, 0, 3536, 482, 0, 74, 194, 1131, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 22, 5962, 0, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 88, 4, 2178, 489, 0, 1182, 554, 0, 0, 0, 0, 0, 0, 2485, 489, 0, 1293, 1923, 1, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6135, 5905, 6234, 5779, 6158, 6223, 6505, 6213] avg=6144.0000 max=6505 max_rank=6 min=5779 min_rank=3 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=25099 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[535, 72, 586, 1695, 3218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 744, 0, 370, 0, 618, 3422, 815, 0, 0, 0, 0, 0, 0, 259, 0, 2697, 0, 979, 58, 394, 151, 1729, 0, 0, 0, 0, 0, 0, 0, 59, 72, 503, 0, 0, 131, 0, 168, 4790, 571, 0, 0, 0, 0, 0, 0, 3664, 572, 0, 72, 208, 802, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 5894, 0, 0, 0, 0, 0, 0, 0, 2053, 0, 0, 53, 2, 1703, 407, 0, 1263, 574, 0, 0, 0, 0, 0, 0, 2619, 482, 2, 1354, 1722, 7, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 6113, 6267, 6294, 6021, 6087, 6055, 6209] avg=6144.0000 max=6294 max_rank=3 min=6021 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[625, 64, 533, 1359, 3766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 812, 0, 507, 0, 783, 3113, 740, 0, 0, 0, 0, 0, 0, 243, 0, 2338, 0, 912, 53, 152, 82, 1745, 0, 0, 0, 0, 0, 0, 0, 50, 64, 476, 0, 0, 115, 0, 176, 4352, 582, 0, 0, 0, 0, 0, 0, 3698, 611, 0, 65, 195, 1264, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 3, 6005, 0, 0, 0, 0, 0, 0, 0, 1733, 0, 0, 71, 1, 1618, 659, 0, 1149, 579, 0, 0, 0, 0, 0, 0, 2849, 455, 1, 1583, 1915, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6347, 6082, 5525, 5815, 6596, 6151, 5810, 6826] avg=6144.0000 max=6826 max_rank=7 min=5525 min_rank=2 imbalance_ratio=1.111003 max_load_reduction=42326 max_load_reduction_pct=0.861125 moved_to_echo=25383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[566, 77, 556, 1393, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 708, 0, 546, 0, 582, 3291, 809, 0, 0, 0, 0, 0, 0, 246, 0, 2806, 0, 990, 50, 271, 138, 1731, 0, 0, 0, 0, 0, 0, 0, 59, 77, 549, 0, 0, 136, 0, 180, 4760, 569, 0, 0, 0, 0, 0, 0, 3651, 665, 0, 77, 226, 993, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 5877, 0, 0, 0, 0, 0, 0, 0, 2040, 0, 0, 66, 3, 1651, 535, 0, 1217, 577, 0, 0, 0, 0, 0, 0, 2655, 505, 4, 1365, 1592, 2, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5742, 6089, 6232, 6330, 6424, 6098, 6089, 6148] avg=6144.0000 max=6424 max_rank=4 min=5742 min_rank=0 imbalance_ratio=1.045573 max_load_reduction=42728 max_load_reduction_pct=0.869303 moved_to_echo=24759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 1408, 0, 6144, 128, 3840, 128, 4608, 27136, 6400, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[16000, 0, 128, 512, 128, 14208, 3968, 128, 10112, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[21376, 3968, 128, 10496, 13184, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[28928, 4736, 0, 768, 1664, 7424, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 1408, 0, 6144, 128, 3840, 128, 4608, 27136, 6400, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 128, 22272, 0, 7680, 512, 2432, 1024, 13696, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[21376, 3968, 128, 10496, 13184, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4736, 768, 4736, 11520, 27904, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 768, 4224, 0, 0, 1152, 128, 1280, 37120, 4608, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 128, 0, 0, 0, 128, 128, 128, 47488, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 7537, 3040, 0, 11, 57, 350, 4404, 0, 0, 0, 0, 0, 622, 2192, 145, 0, 216, 94, 249, 75, 0, 0, 1085, 220, 249, 0, 115, 62, 57, 184, 0, 106, 7853, 85, 0, 1468, 0, 3401, 0, 1697, 0, 0, 0, 2186, 176, 0, 0, 85, 0, 85, 395, 0, 59, 83, 0, 0, 414, 6945, 146, 84, 1292, 0, 1628] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7474, 3807, 0, 0, 37, 368, 3561, 0, 1, 0, 0, 0, 674, 2863, 145, 0, 255, 152, 291, 3, 1, 1, 1133, 315, 247, 1, 127, 4, 37, 259, 0, 251, 7847, 107, 1, 2445, 0, 2439, 0, 1284, 0, 1, 0, 2029, 241, 0, 1, 107, 1, 107, 466, 0, 43, 107, 1, 0, 405, 6912, 191, 108, 651, 1, 1650] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 7540, 3326, 0, 4, 66, 324, 4311, 0, 0, 0, 0, 0, 623, 2376, 157, 0, 229, 126, 301, 32, 0, 0, 1047, 268, 253, 0, 117, 90, 66, 225, 0, 67, 7800, 91, 0, 1436, 0, 3759, 0, 1579, 0, 0, 0, 1881, 153, 0, 0, 91, 0, 92, 410, 0, 107, 90, 0, 0, 373, 6911, 148, 91, 1095, 0, 1497] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 7568, 3489, 0, 1, 59, 305, 3788, 0, 0, 0, 0, 0, 614, 2605, 131, 0, 245, 72, 299, 54, 0, 0, 1075, 213, 258, 0, 88, 0, 59, 152, 0, 261, 7879, 72, 0, 2575, 0, 2890, 0, 1597, 0, 0, 0, 2205, 249, 0, 0, 73, 0, 72, 344, 0, 61, 71, 0, 0, 408, 7011, 277, 72, 643, 0, 1317] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 7680, 3439, 0, 0, 50, 387, 3072, 0, 0, 0, 0, 0, 657, 3298, 115, 0, 249, 0, 321, 0, 0, 0, 1077, 157, 246, 0, 79, 0, 50, 116, 0, 264, 7911, 65, 0, 3538, 0, 2425, 0, 1499, 0, 0, 0, 2141, 113, 0, 0, 65, 0, 66, 353, 0, 50, 49, 0, 0, 329, 7145, 285, 64, 535, 0, 1262] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 7537, 3040, 0, 11, 57, 350, 4404, 0, 0, 0, 0, 0, 622, 2192, 145, 0, 216, 94, 249, 75, 0, 0, 1085, 220, 249, 0, 115, 62, 57, 184, 0, 106, 7853, 85, 0, 1468, 0, 3401, 0, 1697, 0, 0, 0, 2186, 176, 0, 0, 85, 0, 85, 395, 0, 59, 83, 0, 0, 414, 6945, 146, 84, 1292, 0, 1628] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 7507, 3909, 0, 0, 50, 376, 3309, 0, 0, 0, 0, 0, 685, 2819, 128, 0, 225, 37, 287, 0, 0, 0, 1105, 259, 252, 0, 79, 0, 50, 189, 0, 272, 7858, 78, 0, 3407, 0, 2443, 0, 1349, 0, 0, 0, 2019, 241, 0, 0, 78, 0, 78, 403, 0, 50, 60, 0, 0, 335, 7069, 298, 77, 500, 0, 1271] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 7507, 3812, 0, 1, 59, 382, 3865, 0, 0, 0, 0, 0, 679, 2639, 137, 0, 241, 81, 261, 45, 0, 0, 1157, 213, 255, 0, 95, 0, 59, 163, 0, 274, 7815, 77, 0, 2807, 0, 2267, 0, 1473, 0, 0, 0, 2075, 244, 0, 0, 77, 0, 80, 409, 0, 59, 75, 0, 0, 425, 6997, 243, 77, 658, 0, 1369] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 7680, 3439, 0, 0, 50, 387, 3072, 0, 0, 0, 0, 0, 657, 3298, 115, 0, 249, 0, 321, 0, 0, 0, 1077, 157, 246, 0, 79, 0, 50, 116, 0, 264, 7911, 65, 0, 3538, 0, 2425, 0, 1499, 0, 0, 0, 2141, 113, 0, 0, 65, 0, 66, 353, 0, 50, 49, 0, 0, 329, 7145, 285, 64, 535, 0, 1262] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 7610, 3611, 0, 0, 51, 301, 3051, 0, 0, 0, 0, 0, 630, 3244, 125, 0, 204, 30, 329, 0, 0, 0, 1029, 209, 244, 0, 123, 0, 51, 155, 0, 261, 7878, 74, 0, 3171, 0, 2844, 0, 1454, 0, 0, 0, 2057, 212, 0, 0, 76, 0, 74, 328, 0, 51, 59, 0, 0, 389, 7060, 276, 73, 510, 0, 1308] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3040, 0, 11, 57, 350, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 2192, 145, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 216, 94, 249, 75, 0, 0, 1085, 4214, 211, 0, 0, 0, 0, 0, 0, 220, 249, 0, 115, 62, 57, 184, 0, 3209, 2090, 0, 0, 0, 0, 0, 0, 106, 430, 85, 0, 1468, 0, 3401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1697, 0, 0, 0, 2186, 176, 0, 0, 1782, 881, 0, 0, 0, 0, 0, 0, 85, 0, 85, 395, 0, 59, 83, 0, 5447, 0, 0, 0, 0, 0, 0, 0, 0, 414, 3270, 146, 84, 1292, 0, 1628, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6080, 5542, 6144, 6186, 5490, 6722, 6154, 6834] avg=6144.0000 max=6834 max_rank=7 min=5490 min_rank=4 imbalance_ratio=1.112305 max_load_reduction=42318 max_load_reduction_pct=0.860962 moved_to_echo=25200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7474, 3807, 0, 0, 37, 368, 3561, 0, 1, 0, 0, 0, 674, 2863, 145, 0, 255, 152, 291, 3, 1, 1, 1133, 315, 247, 1, 127, 4, 37, 259, 0, 251, 7847, 107, 1, 2445, 0, 2439, 0, 1284, 0, 1, 0, 2029, 241, 0, 1, 107, 1, 107, 466, 0, 43, 107, 1, 0, 405, 6912, 191, 108, 651, 1, 1650] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 7507, 3812, 0, 1, 59, 382, 3865, 0, 0, 0, 0, 0, 679, 2639, 137, 0, 241, 81, 261, 45, 0, 0, 1157, 213, 255, 0, 95, 0, 59, 163, 0, 274, 7815, 77, 0, 2807, 0, 2267, 0, 1473, 0, 0, 0, 2075, 244, 0, 0, 77, 0, 80, 409, 0, 59, 75, 0, 0, 425, 6997, 243, 77, 658, 0, 1369] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3326, 0, 4, 66, 324, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 2376, 157, 2570, 0, 0, 0, 0, 0, 0, 0, 0, 229, 126, 301, 32, 0, 0, 1047, 4186, 210, 0, 0, 0, 0, 0, 0, 268, 253, 0, 117, 90, 66, 225, 0, 3187, 2091, 0, 0, 0, 0, 0, 0, 67, 427, 91, 0, 1436, 0, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1579, 0, 0, 0, 1881, 153, 0, 0, 1744, 877, 0, 0, 0, 0, 0, 0, 91, 0, 92, 410, 0, 107, 90, 0, 5449, 0, 0, 0, 0, 0, 0, 0, 0, 373, 3254, 148, 91, 1095, 0, 1497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6287, 5726, 6131, 6297, 5780, 6234, 6239, 6458] avg=6144.0000 max=6458 max_rank=7 min=5726 min_rank=1 imbalance_ratio=1.051107 max_load_reduction=42694 max_load_reduction_pct=0.868612 moved_to_echo=24711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3439, 0, 0, 50, 387, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 3298, 115, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 321, 0, 0, 0, 1077, 4246, 217, 0, 0, 0, 0, 0, 0, 157, 246, 0, 79, 0, 50, 116, 0, 3232, 2131, 0, 0, 0, 0, 0, 0, 264, 433, 65, 0, 3538, 0, 2425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1499, 0, 0, 0, 2141, 113, 0, 0, 1243, 906, 0, 0, 0, 0, 0, 0, 65, 0, 66, 353, 0, 50, 49, 0, 5549, 0, 0, 0, 0, 0, 0, 0, 0, 329, 3365, 285, 64, 535, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5705, 6727, 6110, 6011, 6725, 5902, 6132, 5840] avg=6144.0000 max=6727 max_rank=1 min=5705 min_rank=0 imbalance_ratio=1.094889 max_load_reduction=42425 max_load_reduction_pct=0.863139 moved_to_echo=24599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3489, 0, 1, 59, 305, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 2605, 131, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 245, 72, 299, 54, 0, 0, 1075, 4228, 213, 0, 0, 0, 0, 0, 0, 213, 258, 0, 88, 0, 59, 152, 0, 3219, 2100, 0, 0, 0, 0, 0, 0, 261, 432, 72, 0, 2575, 0, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1597, 0, 0, 0, 2205, 249, 0, 0, 1532, 889, 0, 0, 0, 0, 0, 0, 73, 0, 72, 344, 0, 61, 71, 0, 5468, 0, 0, 0, 0, 0, 0, 0, 0, 408, 3302, 277, 72, 643, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 5957, 6186, 6089, 6230, 6472, 6089, 6019] avg=6144.0000 max=6472 max_rank=5 min=5957 min_rank=1 imbalance_ratio=1.053385 max_load_reduction=42680 max_load_reduction_pct=0.868327 moved_to_echo=24810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3040, 0, 11, 57, 350, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 2192, 145, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 216, 94, 249, 75, 0, 0, 1085, 4214, 211, 0, 0, 0, 0, 0, 0, 220, 249, 0, 115, 62, 57, 184, 0, 3209, 2090, 0, 0, 0, 0, 0, 0, 106, 430, 85, 0, 1468, 0, 3401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1697, 0, 0, 0, 2186, 176, 0, 0, 1782, 881, 0, 0, 0, 0, 0, 0, 85, 0, 85, 395, 0, 59, 83, 0, 5447, 0, 0, 0, 0, 0, 0, 0, 0, 414, 3270, 146, 84, 1292, 0, 1628, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6080, 5542, 6144, 6186, 5490, 6722, 6154, 6834] avg=6144.0000 max=6834 max_rank=7 min=5490 min_rank=4 imbalance_ratio=1.112305 max_load_reduction=42318 max_load_reduction_pct=0.860962 moved_to_echo=25200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3909, 0, 0, 50, 376, 1971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 2819, 128, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 225, 37, 287, 0, 0, 0, 1105, 4217, 215, 0, 0, 0, 0, 0, 0, 259, 252, 0, 79, 0, 50, 189, 0, 3211, 2082, 0, 0, 0, 0, 0, 0, 272, 430, 78, 0, 3407, 0, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 0, 0, 0, 2019, 241, 0, 0, 1338, 897, 0, 0, 0, 0, 0, 0, 78, 0, 78, 403, 0, 50, 60, 0, 5425, 0, 0, 0, 0, 0, 0, 0, 0, 335, 3328, 298, 77, 500, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6306, 6261, 6086, 6122, 6630, 5844, 6094, 5809] avg=6144.0000 max=6630 max_rank=4 min=5809 min_rank=7 imbalance_ratio=1.079102 max_load_reduction=42522 max_load_reduction_pct=0.865112 moved_to_echo=24377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3807, 0, 0, 37, 368, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 674, 2863, 145, 2570, 0, 0, 0, 0, 0, 0, 0, 0, 255, 152, 291, 3, 1, 1, 1133, 4211, 210, 0, 0, 0, 0, 0, 0, 315, 247, 1, 127, 4, 37, 259, 0, 3206, 2073, 0, 0, 0, 0, 0, 0, 251, 430, 107, 1, 2445, 0, 2439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 0, 1, 0, 2029, 241, 0, 1, 1440, 877, 0, 0, 0, 0, 0, 0, 107, 1, 107, 466, 0, 43, 107, 1, 5401, 0, 0, 0, 0, 0, 0, 0, 0, 405, 3255, 191, 108, 651, 1, 1650, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6333, 6253, 6257, 6269, 5673, 5873, 6233, 6261] avg=6144.0000 max=6333 max_rank=0 min=5673 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=42819 max_load_reduction_pct=0.871155 moved_to_echo=24040 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3812, 0, 1, 59, 382, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 2639, 137, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 241, 81, 261, 45, 0, 0, 1157, 4199, 219, 0, 0, 0, 0, 0, 0, 213, 255, 0, 95, 0, 59, 163, 0, 3198, 2083, 0, 0, 0, 0, 0, 0, 274, 418, 77, 0, 2807, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 0, 0, 2075, 244, 0, 0, 1568, 892, 0, 0, 0, 0, 0, 0, 77, 0, 80, 409, 0, 59, 75, 0, 5424, 0, 0, 0, 0, 0, 0, 0, 0, 425, 3279, 243, 77, 658, 0, 1369, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6551, 6062, 6203, 6066, 5843, 6252, 6124, 6051] avg=6144.0000 max=6551 max_rank=0 min=5843 min_rank=4 imbalance_ratio=1.066243 max_load_reduction=42601 max_load_reduction_pct=0.866720 moved_to_echo=24270 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3439, 0, 0, 50, 387, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 3298, 115, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 321, 0, 0, 0, 1077, 4246, 217, 0, 0, 0, 0, 0, 0, 157, 246, 0, 79, 0, 50, 116, 0, 3232, 2131, 0, 0, 0, 0, 0, 0, 264, 433, 65, 0, 3538, 0, 2425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1499, 0, 0, 0, 2141, 113, 0, 0, 1243, 906, 0, 0, 0, 0, 0, 0, 65, 0, 66, 353, 0, 50, 49, 0, 5549, 0, 0, 0, 0, 0, 0, 0, 0, 329, 3365, 285, 64, 535, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5705, 6727, 6110, 6011, 6725, 5902, 6132, 5840] avg=6144.0000 max=6727 max_rank=1 min=5705 min_rank=0 imbalance_ratio=1.094889 max_load_reduction=42425 max_load_reduction_pct=0.863139 moved_to_echo=24599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3611, 0, 0, 51, 301, 1817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 3244, 125, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 204, 30, 329, 0, 0, 0, 1029, 4228, 215, 0, 0, 0, 0, 0, 0, 209, 244, 0, 123, 0, 51, 155, 0, 3219, 2111, 0, 0, 0, 0, 0, 0, 261, 431, 74, 0, 3171, 0, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1454, 0, 0, 0, 2057, 212, 0, 0, 1234, 896, 0, 0, 0, 0, 0, 0, 76, 0, 74, 328, 0, 51, 59, 0, 5499, 0, 0, 0, 0, 0, 0, 0, 0, 389, 3324, 276, 73, 510, 0, 1308, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5780, 6624, 6035, 6112, 6781, 5853, 6087, 5880] avg=6144.0000 max=6781 max_rank=4 min=5780 min_rank=0 imbalance_ratio=1.103678 max_load_reduction=42371 max_load_reduction_pct=0.862040 moved_to_echo=24601 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3812, 0, 1, 59, 382, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 2639, 137, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 241, 81, 261, 45, 0, 0, 1157, 4199, 219, 0, 0, 0, 0, 0, 0, 213, 255, 0, 95, 0, 59, 163, 0, 3198, 2083, 0, 0, 0, 0, 0, 0, 274, 418, 77, 0, 2807, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 0, 0, 2075, 244, 0, 0, 1568, 892, 0, 0, 0, 0, 0, 0, 77, 0, 80, 409, 0, 59, 75, 0, 5424, 0, 0, 0, 0, 0, 0, 0, 0, 425, 3279, 243, 77, 658, 0, 1369, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6551, 6062, 6203, 6066, 5843, 6252, 6124, 6051] avg=6144.0000 max=6551 max_rank=0 min=5843 min_rank=4 imbalance_ratio=1.066243 max_load_reduction=42601 max_load_reduction_pct=0.866720 moved_to_echo=24270 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12032, 0, 128, 0, 16640, 1664, 0, 128, 11904, 7168, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 3200, 26496, 1920, 768, 5888, 128, 11392, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 5248, 22144, 1152, 20864, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 3456, 768, 128, 20864, 0, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12032, 0, 128, 0, 16640, 1664, 0, 128, 11904, 7168, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 28544, 0, 128, 512, 2816, 17536, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 1920, 640, 2432, 256, 128, 128, 8832, 33792, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 2048, 128, 896, 256, 512, 1536, 0, 25728, 16768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 5248, 22144, 1152, 20864, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 3456, 768, 128, 20864, 0, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 128, 768, 3200, 0, 512, 640, 128, 43776, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 2048, 128, 896, 256, 512, 1536, 0, 25728, 16768, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 301, 0, 0, 0, 0, 412, 291, 1339, 0, 977, 341, 0, 90, 54, 0, 0, 0, 0, 94, 0, 0, 161, 5465, 0, 295, 0, 1836, 0, 0, 325, 2039, 0, 0, 0, 3755, 0, 727, 2478, 609, 3921, 0, 527, 1010, 2447, 0, 0, 134, 0, 788, 0, 0, 52, 0, 500, 0, 6952, 6469, 4376, 205, 0, 182] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 359, 0, 0, 0, 0, 386, 256, 1385, 0, 1021, 312, 0, 84, 50, 0, 1, 0, 0, 88, 0, 0, 173, 5481, 0, 300, 0, 1992, 0, 0, 399, 1612, 0, 0, 0, 3927, 0, 759, 2508, 545, 4096, 0, 588, 919, 2179, 0, 0, 101, 0, 830, 0, 0, 48, 0, 456, 0, 7068, 6517, 4286, 257, 0, 169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 0, 0, 0, 380, 369, 1059, 0, 889, 140, 0, 73, 88, 0, 0, 0, 0, 86, 0, 0, 207, 6224, 0, 300, 0, 1905, 0, 0, 226, 2104, 0, 0, 0, 4031, 0, 655, 3790, 265, 2788, 0, 502, 1642, 2466, 0, 0, 82, 0, 470, 0, 0, 6, 0, 560, 0, 7226, 6520, 3464, 143, 0, 147] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 369, 0, 0, 0, 0, 412, 335, 1246, 0, 755, 195, 0, 72, 53, 0, 0, 0, 0, 95, 0, 0, 219, 6548, 0, 298, 0, 1846, 0, 0, 273, 1796, 0, 0, 0, 3868, 0, 808, 3454, 364, 3232, 0, 542, 1261, 2297, 0, 0, 80, 0, 611, 0, 0, 29, 0, 573, 0, 6956, 6514, 3777, 130, 0, 144] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 0, 0, 0, 386, 275, 950, 0, 987, 171, 0, 77, 66, 0, 0, 0, 0, 82, 0, 0, 208, 6565, 0, 268, 0, 1947, 0, 0, 236, 1954, 0, 0, 0, 3776, 0, 752, 3569, 373, 3123, 0, 617, 1341, 2318, 0, 0, 86, 0, 624, 0, 0, 34, 0, 574, 0, 6978, 6469, 3696, 151, 0, 154] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 333, 0, 0, 0, 0, 373, 429, 677, 0, 997, 64, 0, 64, 71, 0, 0, 0, 0, 79, 0, 0, 207, 6821, 0, 274, 0, 1797, 0, 0, 191, 1923, 0, 0, 0, 4226, 0, 731, 4164, 201, 2719, 0, 425, 1755, 2489, 0, 0, 52, 0, 488, 0, 0, 0, 0, 521, 0, 7320, 6556, 2999, 77, 0, 129] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 301, 0, 0, 0, 0, 412, 291, 1339, 0, 977, 341, 0, 90, 54, 0, 0, 0, 0, 94, 0, 0, 161, 5465, 0, 295, 0, 1836, 0, 0, 325, 2039, 0, 0, 0, 3755, 0, 727, 2478, 609, 3921, 0, 527, 1010, 2447, 0, 0, 134, 0, 788, 0, 0, 52, 0, 500, 0, 6952, 6469, 4376, 205, 0, 182] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 1, 0, 1, 331, 189, 1058, 1, 829, 312, 1, 107, 66, 1, 0, 0, 1, 49, 0, 0, 193, 6409, 0, 277, 0, 1748, 0, 2, 299, 2293, 0, 0, 0, 3885, 1, 734, 3443, 476, 3299, 1, 595, 1197, 2293, 0, 0, 152, 1, 631, 0, 0, 66, 1, 536, 1, 6724, 6427, 3754, 208, 0, 214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 359, 0, 0, 0, 0, 386, 256, 1385, 0, 1021, 312, 0, 84, 50, 0, 1, 0, 0, 88, 0, 0, 173, 5481, 0, 300, 0, 1992, 0, 0, 399, 1612, 0, 0, 0, 3927, 0, 759, 2508, 545, 4096, 0, 588, 919, 2179, 0, 0, 101, 0, 830, 0, 0, 48, 0, 456, 0, 7068, 6517, 4286, 257, 0, 169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 355, 0, 0, 0, 0, 367, 277, 908, 0, 953, 81, 0, 77, 59, 0, 0, 0, 0, 66, 0, 0, 209, 6838, 0, 297, 0, 1581, 0, 0, 229, 2019, 0, 0, 0, 4020, 0, 746, 4229, 343, 2455, 0, 498, 1581, 2705, 0, 0, 112, 0, 533, 0, 0, 8, 0, 578, 0, 7031, 6561, 3184, 97, 0, 155] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 0, 0, 0, 386, 275, 950, 0, 987, 171, 0, 77, 66, 0, 0, 0, 0, 82, 0, 0, 208, 6565, 0, 268, 0, 1947, 0, 0, 236, 1954, 0, 0, 0, 3776, 0, 752, 3569, 373, 3123, 0, 617, 1341, 2318, 0, 0, 86, 0, 624, 0, 0, 34, 0, 574, 0, 6978, 6469, 3696, 151, 0, 154] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 369, 0, 0, 0, 0, 412, 335, 1246, 0, 755, 195, 0, 72, 53, 0, 0, 0, 0, 95, 0, 0, 219, 6548, 0, 298, 0, 1846, 0, 0, 273, 1796, 0, 0, 0, 3868, 0, 808, 3454, 364, 3232, 0, 542, 1261, 2297, 0, 0, 80, 0, 611, 0, 0, 29, 0, 573, 0, 6956, 6514, 3777, 130, 0, 144] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 0, 0, 0, 380, 369, 1059, 0, 889, 140, 0, 73, 88, 0, 0, 0, 0, 86, 0, 0, 207, 6224, 0, 300, 0, 1905, 0, 0, 226, 2104, 0, 0, 0, 4031, 0, 655, 3790, 265, 2788, 0, 502, 1642, 2466, 0, 0, 82, 0, 470, 0, 0, 6, 0, 560, 0, 7226, 6520, 3464, 143, 0, 147] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 301, 0, 0, 0, 0, 4742, 1020, 0, 0, 0, 0, 0, 0, 412, 291, 1339, 0, 977, 341, 0, 90, 2118, 735, 583, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 94, 0, 0, 5932, 0, 0, 0, 0, 0, 0, 0, 161, 3323, 0, 295, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 2039, 0, 0, 0, 3020, 0, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 1803, 0, 527, 1010, 2447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 788, 0, 0, 52, 0, 2478, 1559, 140, 0, 0, 0, 0, 0, 500, 0, 0, 1587, 4376, 205, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6886, 6080, 5615, 6111, 6396, 5151, 6850] avg=6144.0000 max=6886 max_rank=1 min=5151 min_rank=6 imbalance_ratio=1.120768 max_load_reduction=42266 max_load_reduction_pct=0.859904 moved_to_echo=24508 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 359, 0, 0, 0, 0, 4777, 1036, 0, 0, 0, 0, 0, 0, 386, 256, 1385, 0, 1021, 312, 0, 84, 2213, 769, 584, 0, 0, 0, 0, 0, 50, 0, 1, 0, 0, 88, 0, 0, 6032, 0, 0, 0, 0, 0, 0, 0, 173, 3334, 0, 300, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 1612, 0, 0, 0, 3158, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 1883, 0, 588, 919, 2179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 830, 0, 0, 48, 0, 2508, 1563, 141, 0, 0, 0, 0, 0, 456, 0, 0, 1599, 4286, 257, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6172, 7010, 6171, 5799, 5928, 6114, 5191, 6767] avg=6144.0000 max=7010 max_rank=1 min=5191 min_rank=6 imbalance_ratio=1.140951 max_load_reduction=42142 max_load_reduction_pct=0.857381 moved_to_echo=24000 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 333, 0, 0, 0, 0, 4806, 1074, 0, 0, 0, 0, 0, 0, 373, 429, 677, 0, 997, 64, 0, 64, 1469, 827, 727, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 79, 0, 0, 6246, 0, 0, 0, 0, 0, 0, 0, 207, 4148, 0, 274, 0, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 1923, 0, 0, 0, 3399, 0, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1250, 0, 425, 1755, 2489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 488, 0, 0, 0, 0, 4164, 1946, 142, 0, 0, 0, 0, 0, 521, 0, 0, 1608, 2999, 77, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6213, 5627, 6396, 6426, 6244, 6120, 6792, 5334] avg=6144.0000 max=6792 max_rank=6 min=5334 min_rank=7 imbalance_ratio=1.105469 max_load_reduction=42360 max_load_reduction_pct=0.861816 moved_to_echo=24490 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 0, 0, 0, 4779, 1060, 0, 0, 0, 0, 0, 0, 380, 369, 1059, 0, 889, 140, 0, 73, 1506, 789, 664, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 86, 0, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 207, 3785, 0, 300, 0, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 2104, 0, 0, 0, 3242, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 1282, 0, 502, 1642, 2466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 470, 0, 0, 6, 0, 3790, 1775, 141, 0, 0, 0, 0, 0, 560, 0, 0, 1600, 3464, 143, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 5869, 6340, 6197, 6227, 6157, 6264, 5914] avg=6144.0000 max=6340 max_rank=2 min=5869 min_rank=1 imbalance_ratio=1.031901 max_load_reduction=42812 max_load_reduction_pct=0.871012 moved_to_echo=24562 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 369, 0, 0, 0, 0, 4775, 1021, 0, 0, 0, 0, 0, 0, 412, 335, 1246, 0, 755, 195, 0, 72, 1746, 757, 698, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 95, 0, 0, 5935, 0, 0, 0, 0, 0, 0, 0, 219, 3982, 0, 298, 0, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 1796, 0, 0, 0, 3111, 0, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 1486, 0, 542, 1261, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 611, 0, 0, 29, 0, 3454, 1868, 141, 0, 0, 0, 0, 0, 573, 0, 0, 1598, 3777, 130, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6165, 6216, 6083, 6345, 5988, 5950, 6183, 6222] avg=6144.0000 max=6345 max_rank=3 min=5950 min_rank=5 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 1, 0, 1, 4711, 986, 0, 0, 0, 0, 0, 0, 331, 189, 1058, 1, 829, 312, 1, 107, 1782, 760, 683, 0, 0, 0, 0, 0, 66, 1, 0, 0, 1, 49, 0, 0, 5738, 0, 0, 0, 0, 0, 0, 0, 193, 3898, 0, 277, 0, 1748, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 299, 2293, 0, 0, 0, 3125, 1, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 1517, 1, 595, 1197, 2293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 1, 631, 0, 0, 66, 1, 3443, 1828, 139, 0, 0, 0, 0, 0, 536, 1, 0, 1577, 3754, 208, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6044, 6053, 5855, 6118, 6452, 6079, 6261, 6290] avg=6144.0000 max=6452 max_rank=4 min=5855 min_rank=2 imbalance_ratio=1.050130 max_load_reduction=42700 max_load_reduction_pct=0.868734 moved_to_echo=25082 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 355, 0, 0, 0, 0, 4809, 1031, 0, 0, 0, 0, 0, 0, 367, 277, 908, 0, 953, 81, 0, 77, 1326, 787, 729, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 66, 0, 0, 6000, 0, 0, 0, 0, 0, 0, 0, 209, 4159, 0, 297, 0, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 2019, 0, 0, 0, 3233, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 1129, 0, 498, 1581, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 533, 0, 0, 8, 0, 4229, 1950, 142, 0, 0, 0, 0, 0, 578, 0, 0, 1610, 3184, 97, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6195, 5505, 6125, 6246, 6227, 6256, 6974, 5624] avg=6144.0000 max=6974 max_rank=6 min=5505 min_rank=1 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=25081 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 359, 0, 0, 0, 0, 4777, 1036, 0, 0, 0, 0, 0, 0, 386, 256, 1385, 0, 1021, 312, 0, 84, 2213, 769, 584, 0, 0, 0, 0, 0, 50, 0, 1, 0, 0, 88, 0, 0, 6032, 0, 0, 0, 0, 0, 0, 0, 173, 3334, 0, 300, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 1612, 0, 0, 0, 3158, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 1883, 0, 588, 919, 2179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 830, 0, 0, 48, 0, 2508, 1563, 141, 0, 0, 0, 0, 0, 456, 0, 0, 1599, 4286, 257, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6172, 7010, 6171, 5799, 5928, 6114, 5191, 6767] avg=6144.0000 max=7010 max_rank=1 min=5191 min_rank=6 imbalance_ratio=1.140951 max_load_reduction=42142 max_load_reduction_pct=0.857381 moved_to_echo=24000 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 333, 0, 0, 0, 0, 4806, 1074, 0, 0, 0, 0, 0, 0, 373, 429, 677, 0, 997, 64, 0, 64, 1469, 827, 727, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 79, 0, 0, 6246, 0, 0, 0, 0, 0, 0, 0, 207, 4148, 0, 274, 0, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 1923, 0, 0, 0, 3399, 0, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1250, 0, 425, 1755, 2489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 488, 0, 0, 0, 0, 4164, 1946, 142, 0, 0, 0, 0, 0, 521, 0, 0, 1608, 2999, 77, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6213, 5627, 6396, 6426, 6244, 6120, 6792, 5334] avg=6144.0000 max=6792 max_rank=6 min=5334 min_rank=7 imbalance_ratio=1.105469 max_load_reduction=42360 max_load_reduction_pct=0.861816 moved_to_echo=24490 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 0, 0, 0, 4747, 1024, 0, 0, 0, 0, 0, 0, 386, 275, 950, 0, 987, 171, 0, 77, 1692, 743, 705, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 82, 0, 0, 5954, 0, 0, 0, 0, 0, 0, 0, 208, 3983, 0, 268, 0, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 1954, 0, 0, 0, 3033, 0, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 1431, 0, 617, 1341, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 624, 0, 0, 34, 0, 3569, 1877, 146, 0, 0, 0, 0, 0, 574, 0, 0, 1576, 3696, 151, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6116, 5986, 6102, 6406, 5975, 6080, 6336, 6151] avg=6144.0000 max=6406 max_rank=3 min=5975 min_rank=4 imbalance_ratio=1.042643 max_load_reduction=42746 max_load_reduction_pct=0.869670 moved_to_echo=24542 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 0, 0, 0, 4779, 1060, 0, 0, 0, 0, 0, 0, 380, 369, 1059, 0, 889, 140, 0, 73, 1506, 789, 664, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 86, 0, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 207, 3785, 0, 300, 0, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 2104, 0, 0, 0, 3242, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 1282, 0, 502, 1642, 2466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 470, 0, 0, 6, 0, 3790, 1775, 141, 0, 0, 0, 0, 0, 560, 0, 0, 1600, 3464, 143, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 5869, 6340, 6197, 6227, 6157, 6264, 5914] avg=6144.0000 max=6340 max_rank=2 min=5869 min_rank=1 imbalance_ratio=1.031901 max_load_reduction=42812 max_load_reduction_pct=0.871012 moved_to_echo=24562 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3072, 2432, 8704, 128, 7424, 1664, 128, 768, 13952, 6272, 5376, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 30720, 0, 2432, 0, 14720, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 2816, 0, 128, 0, 128, 38272, 8320, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2304, 15744, 0, 0, 0, 25344, 128, 6016, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4352, 128, 0, 12800, 29568, 1280, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 3200, 11904, 128, 4352, 10752, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 2816, 0, 128, 0, 128, 38272, 8320, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 896, 128, 4992, 0, 0, 256, 128, 27648, 14464, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 3200, 11904, 128, 4352, 10752, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 128, 128, 0, 128, 640, 0, 0, 48128, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 267, 323, 0, 6860, 0, 399, 0, 0, 13, 0, 65, 0, 0, 0, 2794, 0, 0, 0, 0, 3374, 0, 3777, 0, 0, 80, 5440, 0, 0, 0, 50, 5472, 0, 65, 2826, 0, 0, 143, 527, 193, 0, 0, 3545, 73, 0, 0, 0, 740, 0, 3175, 62, 6233, 0, 224, 0, 0, 54, 626, 606, 816, 217, 0, 113] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 304, 331, 3, 7215, 0, 408, 0, 0, 7, 0, 92, 0, 0, 0, 2941, 0, 0, 0, 0, 3138, 0, 3685, 0, 0, 30, 2889, 0, 0, 0, 66, 5735, 0, 91, 3297, 0, 0, 185, 579, 251, 0, 0, 2728, 93, 0, 0, 0, 572, 0, 3885, 90, 6045, 4, 581, 0, 1, 220, 1340, 717, 824, 583, 0, 222] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 312, 351, 0, 6934, 0, 375, 0, 0, 13, 0, 72, 0, 0, 0, 2814, 0, 0, 0, 0, 3207, 0, 3795, 0, 0, 62, 4032, 0, 0, 0, 59, 5673, 2, 72, 3551, 0, 0, 167, 540, 226, 0, 0, 2862, 95, 0, 0, 0, 690, 0, 3467, 72, 6225, 0, 415, 0, 0, 95, 939, 624, 863, 397, 0, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 292, 285, 2, 7277, 0, 415, 0, 0, 12, 0, 85, 0, 0, 0, 2987, 0, 0, 0, 0, 2866, 0, 3508, 0, 0, 27, 2845, 0, 0, 0, 57, 6072, 0, 85, 3703, 0, 0, 172, 695, 229, 0, 0, 2582, 88, 0, 0, 0, 610, 0, 4044, 83, 6116, 0, 585, 0, 0, 147, 1148, 670, 868, 419, 0, 178] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 3, 307, 328, 0, 7072, 0, 441, 0, 0, 22, 0, 77, 0, 0, 0, 2473, 0, 0, 0, 0, 3101, 0, 4298, 0, 0, 45, 4040, 0, 0, 0, 59, 5749, 0, 77, 3133, 0, 0, 159, 576, 218, 0, 0, 3116, 82, 0, 0, 0, 658, 0, 3568, 77, 6184, 0, 374, 0, 0, 117, 929, 632, 816, 258, 0, 163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 8, 311, 290, 0, 6905, 0, 470, 0, 0, 4, 0, 107, 0, 0, 0, 2438, 0, 0, 0, 1, 2714, 0, 4275, 0, 0, 68, 3631, 1, 0, 0, 37, 5651, 2, 107, 2634, 0, 1, 156, 585, 193, 1, 0, 3378, 49, 0, 1, 0, 704, 1, 4314, 107, 5659, 3, 552, 1, 0, 182, 1269, 720, 877, 490, 1, 254] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 322, 330, 0, 6566, 0, 371, 0, 0, 6, 0, 76, 0, 0, 0, 2913, 0, 0, 0, 0, 3366, 0, 3926, 0, 0, 43, 4748, 0, 0, 0, 51, 5429, 0, 74, 3688, 0, 0, 159, 576, 210, 0, 0, 3289, 84, 0, 0, 0, 822, 0, 2809, 71, 6181, 0, 320, 0, 0, 89, 721, 629, 815, 334, 0, 134] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 288, 317, 0, 6726, 0, 444, 0, 0, 5, 0, 78, 0, 0, 0, 2547, 0, 0, 0, 0, 3278, 0, 4402, 0, 0, 43, 4673, 0, 0, 0, 50, 5368, 1, 78, 3182, 0, 0, 143, 502, 193, 0, 0, 3392, 65, 0, 0, 0, 789, 0, 3205, 77, 6145, 0, 249, 0, 0, 118, 843, 642, 845, 278, 0, 186] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 312, 351, 0, 6934, 0, 375, 0, 0, 13, 0, 72, 0, 0, 0, 2814, 0, 0, 0, 0, 3207, 0, 3795, 0, 0, 62, 4032, 0, 0, 0, 59, 5673, 2, 72, 3551, 0, 0, 167, 540, 226, 0, 0, 2862, 95, 0, 0, 0, 690, 0, 3467, 72, 6225, 0, 415, 0, 0, 95, 939, 624, 863, 397, 0, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 8, 311, 290, 0, 6905, 0, 470, 0, 0, 4, 0, 107, 0, 0, 0, 2438, 0, 0, 0, 1, 2714, 0, 4275, 0, 0, 68, 3631, 1, 0, 0, 37, 5651, 2, 107, 2634, 0, 1, 156, 585, 193, 1, 0, 3378, 49, 0, 1, 0, 704, 1, 4314, 107, 5659, 3, 552, 1, 0, 182, 1269, 720, 877, 490, 1, 254] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 267, 323, 0, 6860, 0, 399, 0, 0, 13, 0, 65, 0, 0, 0, 2794, 0, 0, 0, 0, 3374, 0, 3777, 0, 0, 80, 5440, 0, 0, 0, 50, 5472, 0, 65, 2826, 0, 0, 143, 527, 193, 0, 0, 3545, 73, 0, 0, 0, 740, 0, 3175, 62, 6233, 0, 224, 0, 0, 54, 626, 606, 816, 217, 0, 113] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 3, 307, 328, 0, 7072, 0, 441, 0, 0, 22, 0, 77, 0, 0, 0, 2473, 0, 0, 0, 0, 3101, 0, 4298, 0, 0, 45, 4040, 0, 0, 0, 59, 5749, 0, 77, 3133, 0, 0, 159, 576, 218, 0, 0, 3116, 82, 0, 0, 0, 658, 0, 3568, 77, 6184, 0, 374, 0, 0, 117, 929, 632, 816, 258, 0, 163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 304, 331, 3, 5306, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 92, 0, 0, 0, 4663, 1254, 0, 0, 0, 0, 0, 0, 2941, 0, 0, 0, 0, 3138, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2889, 0, 0, 0, 66, 1909, 168, 0, 0, 0, 0, 0, 0, 2113, 0, 91, 3297, 0, 0, 185, 579, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 2728, 93, 0, 0, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 572, 0, 3885, 90, 1382, 4, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 220, 1340, 717, 824, 583, 0, 222, 2174, 671, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 6016, 6336, 5062, 6265, 5855, 6514, 6752] avg=6144.0000 max=6752 max_rank=7 min=5062 min_rank=3 imbalance_ratio=1.098958 max_load_reduction=42400 max_load_reduction_pct=0.862630 moved_to_echo=25386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 322, 330, 0, 4829, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 76, 0, 0, 0, 4768, 1336, 0, 0, 0, 0, 0, 0, 2913, 0, 0, 0, 0, 3366, 0, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4748, 0, 0, 0, 51, 1737, 159, 0, 0, 0, 0, 0, 0, 2000, 0, 74, 3688, 0, 0, 159, 576, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 3289, 84, 0, 0, 0, 2635, 0, 0, 0, 0, 0, 0, 0, 822, 0, 2809, 71, 1413, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 721, 629, 815, 334, 0, 134, 2316, 635, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5852, 6186, 6553, 6738, 6497, 6218, 5435, 5673] avg=6144.0000 max=6738 max_rank=3 min=5435 min_rank=6 imbalance_ratio=1.096680 max_load_reduction=42414 max_load_reduction_pct=0.862915 moved_to_echo=23823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 288, 317, 0, 4947, 0, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 78, 0, 0, 0, 4740, 1498, 0, 0, 0, 0, 0, 0, 2547, 0, 0, 0, 0, 3278, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4673, 0, 0, 0, 50, 1779, 157, 0, 0, 0, 0, 0, 0, 1978, 1, 78, 3182, 0, 0, 143, 502, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 3392, 65, 0, 0, 0, 2605, 0, 0, 0, 0, 0, 0, 0, 789, 0, 3205, 77, 1405, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 843, 642, 845, 278, 0, 186, 2597, 628, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6321, 6132, 6702, 5884, 6255, 5725, 6137] avg=6144.0000 max=6702 max_rank=3 min=5725 min_rank=6 imbalance_ratio=1.090820 max_load_reduction=42450 max_load_reduction_pct=0.863647 moved_to_echo=24001 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 292, 285, 2, 5352, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 85, 0, 0, 0, 4718, 1194, 0, 0, 0, 0, 0, 0, 2987, 0, 0, 0, 0, 2866, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2845, 0, 0, 0, 57, 1925, 178, 0, 0, 0, 0, 0, 0, 2236, 0, 85, 3703, 0, 0, 172, 695, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 2582, 88, 0, 0, 0, 2947, 0, 0, 0, 0, 0, 0, 0, 610, 0, 4044, 83, 1398, 0, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 1148, 670, 868, 419, 0, 178, 2069, 711, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6346, 6009, 6098, 5032, 6891, 5846, 6720, 6210] avg=6144.0000 max=6891 max_rank=4 min=5032 min_rank=3 imbalance_ratio=1.121582 max_load_reduction=42261 max_load_reduction_pct=0.859802 moved_to_echo=25667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 267, 323, 0, 5045, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 65, 0, 0, 0, 4808, 1286, 0, 0, 0, 0, 0, 0, 2794, 0, 0, 0, 0, 3374, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 5440, 0, 0, 0, 50, 1815, 160, 0, 0, 0, 0, 0, 0, 2016, 0, 65, 2826, 0, 0, 143, 527, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 3545, 73, 0, 0, 0, 2656, 0, 0, 0, 0, 0, 0, 0, 740, 0, 3175, 62, 1425, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 626, 606, 816, 217, 0, 113, 2228, 640, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6034, 6172, 6431, 7545, 5577, 6467, 5626, 5300] avg=6144.0000 max=7545 max_rank=3 min=5300 min_rank=7 imbalance_ratio=1.228027 max_load_reduction=41607 max_load_reduction_pct=0.846497 moved_to_echo=22970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 312, 351, 0, 5100, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 72, 0, 0, 0, 4802, 1292, 0, 0, 0, 0, 0, 0, 2814, 0, 0, 0, 0, 3207, 0, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4032, 0, 0, 0, 59, 1834, 166, 0, 0, 0, 0, 0, 0, 2090, 2, 72, 3551, 0, 0, 167, 540, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 2862, 95, 0, 0, 0, 2753, 0, 0, 0, 0, 0, 0, 0, 690, 0, 3467, 72, 1423, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 939, 624, 863, 397, 0, 151, 2239, 664, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6179, 6285, 6153, 6422, 5936, 6067, 5972] avg=6144.0000 max=6422 max_rank=4 min=5936 min_rank=5 imbalance_ratio=1.045247 max_load_reduction=42730 max_load_reduction_pct=0.869344 moved_to_echo=24397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 288, 317, 0, 4947, 0, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 78, 0, 0, 0, 4740, 1498, 0, 0, 0, 0, 0, 0, 2547, 0, 0, 0, 0, 3278, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4673, 0, 0, 0, 50, 1779, 157, 0, 0, 0, 0, 0, 0, 1978, 1, 78, 3182, 0, 0, 143, 502, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 3392, 65, 0, 0, 0, 2605, 0, 0, 0, 0, 0, 0, 0, 789, 0, 3205, 77, 1405, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 843, 642, 845, 278, 0, 186, 2597, 628, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6321, 6132, 6702, 5884, 6255, 5725, 6137] avg=6144.0000 max=6702 max_rank=3 min=5725 min_rank=6 imbalance_ratio=1.090820 max_load_reduction=42450 max_load_reduction_pct=0.863647 moved_to_echo=24001 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 307, 328, 0, 5198, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 77, 0, 0, 0, 4772, 1467, 0, 0, 0, 0, 0, 0, 2473, 0, 0, 0, 0, 3101, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4040, 0, 0, 0, 59, 1874, 173, 0, 0, 0, 0, 0, 0, 2104, 0, 77, 3133, 0, 0, 159, 576, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 3116, 82, 0, 0, 0, 2794, 0, 0, 0, 0, 0, 0, 0, 658, 0, 3568, 77, 1412, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 929, 632, 816, 258, 0, 163, 2539, 678, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6277, 6338, 5866, 6191, 6049, 6210, 6089, 6132] avg=6144.0000 max=6338 max_rank=1 min=5866 min_rank=2 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 311, 290, 0, 5078, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 107, 0, 0, 0, 4365, 1455, 0, 0, 0, 0, 0, 0, 2438, 0, 0, 0, 1, 2714, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 3631, 1, 0, 0, 37, 1827, 165, 0, 0, 0, 0, 0, 0, 2082, 2, 107, 2634, 0, 1, 156, 585, 0, 0, 0, 0, 0, 0, 0, 0, 193, 1, 0, 3378, 49, 0, 1, 0, 2743, 0, 0, 0, 0, 0, 0, 0, 704, 1, 4314, 107, 1294, 3, 552, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 1269, 720, 877, 490, 1, 254, 2522, 661, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 5931, 5451, 5729, 5567, 6365, 6976, 6976] avg=6144.0000 max=6976 max_rank=6 min=5451 min_rank=2 imbalance_ratio=1.135417 max_load_reduction=42176 max_load_reduction_pct=0.858073 moved_to_echo=25884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 322, 330, 0, 4829, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 76, 0, 0, 0, 4768, 1336, 0, 0, 0, 0, 0, 0, 2913, 0, 0, 0, 0, 3366, 0, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4748, 0, 0, 0, 51, 1737, 159, 0, 0, 0, 0, 0, 0, 2000, 0, 74, 3688, 0, 0, 159, 576, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 3289, 84, 0, 0, 0, 2635, 0, 0, 0, 0, 0, 0, 0, 822, 0, 2809, 71, 1413, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 721, 629, 815, 334, 0, 134, 2316, 635, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5852, 6186, 6553, 6738, 6497, 6218, 5435, 5673] avg=6144.0000 max=6738 max_rank=3 min=5435 min_rank=6 imbalance_ratio=1.096680 max_load_reduction=42414 max_load_reduction_pct=0.862915 moved_to_echo=23823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 267, 323, 0, 5045, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 65, 0, 0, 0, 4808, 1286, 0, 0, 0, 0, 0, 0, 2794, 0, 0, 0, 0, 3374, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 5440, 0, 0, 0, 50, 1815, 160, 0, 0, 0, 0, 0, 0, 2016, 0, 65, 2826, 0, 0, 143, 527, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 3545, 73, 0, 0, 0, 2656, 0, 0, 0, 0, 0, 0, 0, 740, 0, 3175, 62, 1425, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 626, 606, 816, 217, 0, 113, 2228, 640, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6034, 6172, 6431, 7545, 5577, 6467, 5626, 5300] avg=6144.0000 max=7545 max_rank=3 min=5300 min_rank=7 imbalance_ratio=1.228027 max_load_reduction=41607 max_load_reduction_pct=0.846497 moved_to_echo=22970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 307, 328, 0, 5198, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 77, 0, 0, 0, 4772, 1467, 0, 0, 0, 0, 0, 0, 2473, 0, 0, 0, 0, 3101, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4040, 0, 0, 0, 59, 1874, 173, 0, 0, 0, 0, 0, 0, 2104, 0, 77, 3133, 0, 0, 159, 576, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 3116, 82, 0, 0, 0, 2794, 0, 0, 0, 0, 0, 0, 0, 658, 0, 3568, 77, 1412, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 929, 632, 816, 258, 0, 163, 2539, 678, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6277, 6338, 5866, 6191, 6049, 6210, 6089, 6132] avg=6144.0000 max=6338 max_rank=1 min=5866 min_rank=2 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1024, 7936, 5248, 6784, 3072, 128, 1408, 18688, 5376, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[16640, 128, 768, 26112, 0, 128, 1408, 4608, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 512, 32384, 128, 0, 0, 512, 14720, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[22016, 0, 0, 0, 128, 25088, 0, 2304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[16640, 128, 768, 26112, 0, 128, 1408, 4608, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 128, 28544, 640, 11264, 128, 3328, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 128, 0, 24960, 640, 0, 128, 0, 22016, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[22016, 0, 0, 0, 128, 25088, 0, 2304, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 768, 0, 0, 0, 37760, 10880, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 2432, 2560, 128, 40960, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[785, 0, 39, 0, 59, 0, 0, 0, 0, 2694, 0, 80, 72, 1, 464, 1766, 0, 7313, 0, 72, 0, 0, 53, 0, 7530, 0, 118, 258, 0, 1480, 72, 7696, 6919, 593, 0, 0, 0, 0, 0, 0, 72, 0, 1025, 0, 0, 2367, 258, 0, 25, 0, 0, 5114, 0, 0, 0, 0, 0, 2174, 0, 0, 0, 0, 0, 53] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1196, 0, 0, 0, 50, 0, 0, 0, 0, 2488, 0, 52, 65, 2, 462, 1757, 0, 7391, 0, 65, 0, 0, 71, 0, 7541, 0, 88, 244, 0, 1455, 64, 7743, 6877, 672, 0, 0, 0, 0, 0, 0, 65, 0, 914, 0, 0, 2677, 244, 0, 4, 0, 0, 5158, 0, 0, 0, 0, 0, 1736, 0, 0, 0, 0, 0, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1101, 0, 7, 0, 50, 0, 0, 0, 0, 2390, 0, 112, 78, 1, 423, 1843, 1, 7337, 0, 79, 0, 0, 59, 0, 7461, 0, 51, 251, 0, 1429, 77, 7698, 6840, 585, 0, 0, 0, 0, 0, 0, 78, 0, 877, 0, 0, 3080, 251, 0, 8, 0, 0, 5168, 0, 0, 0, 0, 0, 1758, 0, 0, 0, 0, 0, 59] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[559, 0, 37, 0, 37, 0, 0, 0, 1, 2778, 0, 152, 107, 14, 472, 1782, 0, 7244, 1, 107, 0, 0, 67, 1, 7417, 1, 96, 245, 1, 1388, 107, 7642, 6849, 664, 1, 0, 1, 0, 0, 1, 107, 0, 1228, 1, 0, 2643, 245, 0, 209, 0, 0, 4931, 0, 0, 0, 0, 0, 1950, 0, 0, 0, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[739, 0, 43, 0, 59, 0, 0, 0, 0, 2651, 0, 86, 77, 8, 497, 1714, 0, 7293, 0, 77, 0, 0, 66, 0, 7458, 0, 73, 246, 0, 1615, 77, 7670, 6840, 701, 0, 0, 0, 0, 0, 0, 77, 0, 1033, 0, 0, 2603, 246, 0, 13, 0, 0, 4981, 0, 0, 0, 0, 0, 2143, 0, 0, 0, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[369, 0, 56, 0, 57, 0, 0, 0, 0, 3032, 0, 100, 84, 27, 474, 1689, 0, 7290, 0, 85, 0, 0, 50, 0, 7481, 0, 89, 241, 0, 1388, 84, 7571, 7094, 784, 0, 0, 0, 0, 0, 0, 85, 0, 1305, 0, 0, 2074, 241, 0, 317, 0, 0, 4542, 0, 0, 0, 0, 0, 2493, 0, 0, 0, 0, 0, 50] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[489, 0, 57, 0, 66, 0, 0, 0, 0, 3135, 0, 134, 91, 17, 460, 1708, 0, 7257, 0, 91, 0, 0, 54, 0, 7437, 0, 108, 252, 0, 1363, 91, 7560, 7182, 793, 0, 0, 0, 0, 0, 0, 91, 0, 1411, 0, 0, 1829, 251, 0, 191, 0, 0, 4705, 0, 0, 0, 0, 0, 2276, 0, 0, 0, 0, 0, 53] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[739, 0, 43, 0, 59, 0, 0, 0, 0, 2651, 0, 86, 77, 8, 497, 1714, 0, 7293, 0, 77, 0, 0, 66, 0, 7458, 0, 73, 246, 0, 1615, 77, 7670, 6840, 701, 0, 0, 0, 0, 0, 0, 77, 0, 1033, 0, 0, 2603, 246, 0, 13, 0, 0, 4981, 0, 0, 0, 0, 0, 2143, 0, 0, 0, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[785, 0, 39, 0, 59, 0, 0, 0, 0, 2694, 0, 80, 72, 1, 464, 1766, 0, 7313, 0, 72, 0, 0, 53, 0, 7530, 0, 118, 258, 0, 1480, 72, 7696, 6919, 593, 0, 0, 0, 0, 0, 0, 72, 0, 1025, 0, 0, 2367, 258, 0, 25, 0, 0, 5114, 0, 0, 0, 0, 0, 2174, 0, 0, 0, 0, 0, 53] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1196, 0, 0, 0, 50, 0, 0, 0, 0, 2488, 0, 52, 65, 2, 462, 1757, 0, 7391, 0, 65, 0, 0, 71, 0, 7541, 0, 88, 244, 0, 1455, 64, 7743, 6877, 672, 0, 0, 0, 0, 0, 0, 65, 0, 914, 0, 0, 2677, 244, 0, 4, 0, 0, 5158, 0, 0, 0, 0, 0, 1736, 0, 0, 0, 0, 0, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[816, 0, 7, 0, 51, 0, 0, 0, 0, 2680, 0, 82, 74, 3, 504, 1635, 0, 7326, 0, 74, 0, 0, 88, 0, 7514, 0, 23, 244, 0, 1779, 73, 7717, 6920, 633, 0, 0, 0, 0, 0, 0, 74, 0, 1015, 0, 0, 2270, 244, 0, 29, 0, 0, 5375, 0, 0, 0, 0, 0, 1814, 0, 0, 0, 0, 0, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[785, 0, 39, 0, 59, 0, 0, 0, 5327, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 0, 80, 72, 1, 464, 1766, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 6005, 0, 72, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4277, 0, 118, 258, 0, 1480, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5450, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1025, 0, 0, 2367, 258, 0, 1564, 715, 0, 0, 0, 0, 0, 0, 25, 0, 0, 5114, 0, 0, 0, 0, 754, 291, 0, 0, 0, 0, 0, 0, 0, 2174, 0, 0, 0, 0, 0, 53, 2369, 1689, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6210, 6094, 6130, 6205, 6043, 6001, 6184, 6285] avg=6144.0000 max=6285 max_rank=7 min=6001 min_rank=5 imbalance_ratio=1.022949 max_load_reduction=42867 max_load_reduction_pct=0.872131 moved_to_echo=24513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[369, 0, 56, 0, 57, 0, 0, 0, 5239, 0, 0, 0, 0, 0, 0, 0, 0, 3032, 0, 100, 84, 27, 474, 1689, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 5986, 0, 85, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4250, 0, 89, 241, 0, 1388, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5588, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 1305, 0, 0, 2074, 241, 0, 1553, 733, 0, 0, 0, 0, 0, 0, 317, 0, 0, 4542, 0, 0, 0, 0, 773, 290, 0, 0, 0, 0, 0, 0, 0, 2493, 0, 0, 0, 0, 0, 50, 2332, 1678, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5721, 6420, 6121, 6052, 6372, 5991, 5922, 6553] avg=6144.0000 max=6553 max_rank=7 min=5721 min_rank=0 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=24838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1101, 0, 7, 0, 50, 0, 0, 0, 5327, 0, 0, 0, 0, 0, 0, 0, 0, 2390, 0, 112, 78, 1, 423, 1843, 1021, 0, 0, 0, 0, 0, 0, 0, 1, 6025, 0, 79, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4238, 0, 51, 251, 0, 1429, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5387, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 877, 0, 0, 3080, 251, 0, 1549, 707, 0, 0, 0, 0, 0, 0, 8, 0, 0, 5168, 0, 0, 0, 0, 746, 291, 0, 0, 0, 0, 0, 0, 0, 1758, 0, 0, 0, 0, 0, 59, 2371, 1674, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6485, 5868, 6164, 6046, 5972, 6542, 6213, 5862] avg=6144.0000 max=6542 max_rank=5 min=5862 min_rank=7 imbalance_ratio=1.064779 max_load_reduction=42610 max_load_reduction_pct=0.866903 moved_to_echo=24589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[559, 0, 37, 0, 37, 0, 0, 0, 5288, 0, 0, 0, 0, 0, 0, 0, 1, 2778, 0, 152, 107, 14, 472, 1782, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 5948, 1, 107, 0, 0, 67, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4213, 1, 96, 245, 1, 1388, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5394, 664, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 1228, 1, 0, 2643, 245, 0, 1540, 708, 0, 0, 0, 0, 0, 0, 209, 0, 0, 4931, 0, 0, 0, 0, 747, 288, 0, 0, 0, 0, 0, 0, 0, 1950, 0, 0, 0, 0, 0, 66, 2354, 1664, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5921, 6314, 6124, 6051, 6061, 6472, 6175, 6034] avg=6144.0000 max=6472 max_rank=5 min=5921 min_rank=0 imbalance_ratio=1.053385 max_load_reduction=42680 max_load_reduction_pct=0.868327 moved_to_echo=24742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[489, 0, 57, 0, 66, 0, 0, 0, 5233, 0, 0, 0, 0, 0, 0, 0, 0, 3135, 0, 134, 91, 17, 460, 1708, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 5960, 0, 91, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4225, 0, 108, 252, 0, 1363, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5657, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 1411, 0, 0, 1829, 251, 0, 1544, 742, 0, 0, 0, 0, 0, 0, 191, 0, 0, 4705, 0, 0, 0, 0, 783, 288, 0, 0, 0, 0, 0, 0, 0, 2276, 0, 0, 0, 0, 0, 53, 2327, 1668, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5845, 6554, 6105, 6039, 6450, 5868, 5967, 6324] avg=6144.0000 max=6554 max_rank=1 min=5845 min_rank=0 imbalance_ratio=1.066732 max_load_reduction=42598 max_load_reduction_pct=0.866659 moved_to_echo=24609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[369, 0, 56, 0, 57, 0, 0, 0, 5239, 0, 0, 0, 0, 0, 0, 0, 0, 3032, 0, 100, 84, 27, 474, 1689, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 5986, 0, 85, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4250, 0, 89, 241, 0, 1388, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5588, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 1305, 0, 0, 2074, 241, 0, 1553, 733, 0, 0, 0, 0, 0, 0, 317, 0, 0, 4542, 0, 0, 0, 0, 773, 290, 0, 0, 0, 0, 0, 0, 0, 2493, 0, 0, 0, 0, 0, 50, 2332, 1678, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5721, 6420, 6121, 6052, 6372, 5991, 5922, 6553] avg=6144.0000 max=6553 max_rank=7 min=5721 min_rank=0 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=24838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[785, 0, 39, 0, 59, 0, 0, 0, 5327, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 0, 80, 72, 1, 464, 1766, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 6005, 0, 72, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4277, 0, 118, 258, 0, 1480, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5450, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1025, 0, 0, 2367, 258, 0, 1564, 715, 0, 0, 0, 0, 0, 0, 25, 0, 0, 5114, 0, 0, 0, 0, 754, 291, 0, 0, 0, 0, 0, 0, 0, 2174, 0, 0, 0, 0, 0, 53, 2369, 1689, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6210, 6094, 6130, 6205, 6043, 6001, 6184, 6285] avg=6144.0000 max=6285 max_rank=7 min=6001 min_rank=5 imbalance_ratio=1.022949 max_load_reduction=42867 max_load_reduction_pct=0.872131 moved_to_echo=24513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1196, 0, 0, 0, 50, 0, 0, 0, 5359, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 52, 65, 2, 462, 1757, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 6069, 0, 65, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4283, 0, 88, 244, 0, 1455, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5416, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 914, 0, 0, 2677, 244, 0, 1566, 711, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5158, 0, 0, 0, 0, 750, 294, 0, 0, 0, 0, 0, 0, 0, 1736, 0, 0, 0, 0, 0, 71, 2384, 1692, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6605, 5854, 6205, 6134, 6088, 6177, 6206, 5883] avg=6144.0000 max=6605 max_rank=0 min=5854 min_rank=1 imbalance_ratio=1.075033 max_load_reduction=42547 max_load_reduction_pct=0.865621 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[559, 0, 37, 0, 37, 0, 0, 0, 5288, 0, 0, 0, 0, 0, 0, 0, 1, 2778, 0, 152, 107, 14, 472, 1782, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 5948, 1, 107, 0, 0, 67, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4213, 1, 96, 245, 1, 1388, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5394, 664, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 1228, 1, 0, 2643, 245, 0, 1540, 708, 0, 0, 0, 0, 0, 0, 209, 0, 0, 4931, 0, 0, 0, 0, 747, 288, 0, 0, 0, 0, 0, 0, 0, 1950, 0, 0, 0, 0, 0, 66, 2354, 1664, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5921, 6314, 6124, 6051, 6061, 6472, 6175, 6034] avg=6144.0000 max=6472 max_rank=5 min=5921 min_rank=0 imbalance_ratio=1.053385 max_load_reduction=42680 max_load_reduction_pct=0.868327 moved_to_echo=24742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[739, 0, 43, 0, 59, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 0, 86, 77, 8, 497, 1714, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 5982, 0, 77, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4227, 0, 73, 246, 0, 1615, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5381, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 1033, 0, 0, 2603, 246, 0, 1554, 710, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4981, 0, 0, 0, 0, 749, 293, 0, 0, 0, 0, 0, 0, 0, 2143, 0, 0, 0, 0, 0, 66, 2361, 1677, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6051, 6125, 6238, 6082, 6223, 6036, 6247] avg=6144.0000 max=6247 max_rank=7 min=6036 min_rank=6 imbalance_ratio=1.016764 max_load_reduction=42905 max_load_reduction_pct=0.872904 moved_to_echo=24588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[816, 0, 7, 0, 51, 0, 0, 0, 5341, 0, 0, 0, 0, 0, 0, 0, 0, 2680, 0, 82, 74, 3, 504, 1635, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 6016, 0, 74, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4268, 0, 23, 244, 0, 1779, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5451, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 1015, 0, 0, 2270, 244, 0, 1560, 715, 0, 0, 0, 0, 0, 0, 29, 0, 0, 5375, 0, 0, 0, 0, 754, 291, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 0, 0, 0, 0, 88, 2376, 1686, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6215, 5997, 6178, 6387, 6084, 5878, 6449, 5964] avg=6144.0000 max=6449 max_rank=6 min=5878 min_rank=5 imbalance_ratio=1.049642 max_load_reduction=42703 max_load_reduction_pct=0.868795 moved_to_echo=24375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[739, 0, 43, 0, 59, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 0, 86, 77, 8, 497, 1714, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 5982, 0, 77, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4227, 0, 73, 246, 0, 1615, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5381, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 1033, 0, 0, 2603, 246, 0, 1554, 710, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4981, 0, 0, 0, 0, 749, 293, 0, 0, 0, 0, 0, 0, 0, 2143, 0, 0, 0, 0, 0, 66, 2361, 1677, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6051, 6125, 6238, 6082, 6223, 6036, 6247] avg=6144.0000 max=6247 max_rank=7 min=6036 min_rank=6 imbalance_ratio=1.016764 max_load_reduction=42905 max_load_reduction_pct=0.872904 moved_to_echo=24588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 0, 8832, 128, 0, 19584, 2048, 0, 12544, 5760, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 21888, 0, 896, 768, 128, 3840, 13952, 8192, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 16384, 0, 0, 0, 0, 0, 512, 18944, 13440, 0, 0, 0, 0, 0, 0] @@ -9446,28 +9446,28 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 56, 636, 0, 0, 1474, 0, 0, 4083, 4741, 0, 0, 0, 646, 0, 88, 329, 0, 355, 1, 47, 0, 0, 2886, 0, 0, 84, 0, 0, 0, 0, 50, 326, 0, 0, 4579, 161, 0, 84, 4, 0, 4929, 0, 0, 0, 0, 0, 721, 0, 83, 0, 7531, 0, 88, 72, 7052, 0, 0, 0, 6825, 645, 576, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 34, 647, 0, 0, 1730, 0, 0, 3722, 5472, 0, 1, 2, 548, 0, 49, 295, 0, 248, 2, 61, 0, 0, 2945, 0, 0, 107, 1, 0, 0, 1, 68, 352, 1, 1, 3977, 154, 0, 107, 14, 0, 4936, 0, 0, 0, 0, 1, 704, 1, 107, 0, 7545, 1, 49, 104, 7093, 0, 0, 0, 6844, 627, 601, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 556, 0, 0, 2196, 0, 0, 2617, 6154, 0, 0, 0, 690, 0, 79, 323, 0, 257, 0, 62, 0, 0, 1832, 0, 0, 64, 0, 0, 0, 0, 71, 308, 0, 0, 4807, 133, 0, 64, 28, 0, 5128, 0, 0, 0, 0, 0, 691, 0, 57, 0, 7619, 0, 79, 3, 7250, 0, 0, 0, 6943, 552, 589, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 600, 0, 0, 1892, 0, 0, 3599, 5413, 0, 0, 0, 715, 0, 95, 353, 0, 328, 0, 52, 0, 0, 2516, 0, 0, 72, 0, 0, 0, 0, 53, 330, 0, 0, 4613, 143, 0, 72, 1, 0, 4744, 0, 0, 0, 0, 0, 676, 0, 71, 0, 7583, 0, 95, 24, 7138, 0, 0, 0, 6855, 531, 588, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 46, 592, 0, 0, 1479, 0, 0, 4305, 4643, 0, 0, 0, 627, 0, 94, 346, 0, 353, 0, 54, 0, 0, 2891, 0, 0, 91, 0, 0, 0, 0, 54, 342, 0, 0, 4471, 173, 0, 91, 1, 0, 4949, 0, 0, 0, 0, 0, 652, 0, 90, 0, 7497, 0, 94, 79, 7096, 0, 0, 0, 6917, 571, 554, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 600, 0, 0, 1892, 0, 0, 3599, 5413, 0, 0, 0, 715, 0, 95, 353, 0, 328, 0, 52, 0, 0, 2516, 0, 0, 72, 0, 0, 0, 0, 53, 330, 0, 0, 4613, 143, 0, 72, 1, 0, 4744, 0, 0, 0, 0, 0, 676, 0, 71, 0, 7583, 0, 95, 24, 7138, 0, 0, 0, 6855, 531, 588, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 542, 0, 0, 2191, 0, 0, 2836, 6269, 0, 0, 0, 693, 0, 66, 317, 0, 259, 6, 52, 0, 0, 1935, 0, 0, 77, 0, 0, 0, 0, 59, 329, 0, 0, 4659, 146, 0, 77, 35, 0, 4912, 0, 0, 0, 0, 0, 726, 0, 71, 0, 7611, 0, 66, 2, 7224, 0, 0, 0, 6843, 595, 554, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 601, 0, 0, 2059, 0, 0, 2778, 6223, 0, 0, 0, 728, 0, 86, 330, 0, 277, 1, 77, 0, 0, 2003, 0, 0, 78, 0, 0, 0, 0, 88, 312, 0, 0, 4361, 158, 0, 73, 21, 0, 5409, 0, 0, 0, 0, 0, 673, 0, 65, 0, 7575, 0, 86, 6, 7148, 0, 0, 0, 6836, 527, 573, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 36, 610, 0, 0, 2031, 0, 0, 2996, 5479, 0, 0, 0, 703, 0, 82, 328, 0, 325, 1, 60, 0, 0, 2392, 0, 0, 77, 0, 0, 0, 0, 66, 323, 0, 0, 4824, 145, 0, 77, 10, 0, 4962, 0, 0, 0, 0, 0, 678, 0, 76, 0, 7549, 0, 82, 45, 7163, 0, 0, 0, 6802, 617, 613, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 56, 636, 0, 0, 1474, 0, 1601, 1745, 0, 0, 0, 0, 0, 0, 0, 4083, 1802, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 329, 0, 355, 1, 47, 0, 0, 5307, 0, 0, 0, 0, 0, 0, 0, 2886, 0, 0, 84, 0, 0, 0, 0, 2061, 1338, 0, 0, 0, 0, 0, 0, 50, 326, 0, 0, 4579, 161, 0, 84, 978, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4929, 0, 0, 0, 0, 0, 890, 237, 0, 0, 0, 0, 0, 0, 721, 0, 83, 0, 5233, 0, 88, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4957, 645, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5512, 6531, 6127, 6369, 6178, 6060, 6197, 6178] avg=6144.0000 max=6531 max_rank=1 min=5512 min_rank=0 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 600, 0, 0, 1892, 0, 1828, 1767, 0, 0, 0, 0, 0, 0, 0, 3599, 2057, 0, 0, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 353, 0, 328, 0, 52, 0, 0, 5371, 0, 0, 0, 0, 0, 0, 0, 2516, 0, 0, 72, 0, 0, 0, 0, 2075, 1528, 0, 0, 0, 0, 0, 0, 53, 330, 0, 0, 4613, 143, 0, 72, 983, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4744, 0, 0, 0, 0, 0, 894, 239, 0, 0, 0, 0, 0, 0, 676, 0, 71, 0, 5269, 0, 95, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4978, 531, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6087, 6371, 6199, 6191, 6194, 5878, 6135, 6097] avg=6144.0000 max=6371 max_rank=1 min=5878 min_rank=5 imbalance_ratio=1.036947 max_load_reduction=42781 max_load_reduction_pct=0.870382 moved_to_echo=24304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 556, 0, 0, 2196, 0, 0, 2617, 6154, 0, 0, 0, 690, 0, 79, 323, 0, 257, 0, 62, 0, 0, 1832, 0, 0, 64, 0, 0, 0, 0, 71, 308, 0, 0, 4807, 133, 0, 64, 28, 0, 5128, 0, 0, 0, 0, 0, 691, 0, 57, 0, 7619, 0, 79, 3, 7250, 0, 0, 0, 6943, 552, 589, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 34, 647, 0, 0, 1730, 0, 0, 3722, 5472, 0, 1, 2, 548, 0, 49, 295, 0, 248, 2, 61, 0, 0, 2945, 0, 0, 107, 1, 0, 0, 1, 68, 352, 1, 1, 3977, 154, 0, 107, 14, 0, 4936, 0, 0, 0, 0, 1, 704, 1, 107, 0, 7545, 1, 49, 104, 7093, 0, 0, 0, 6844, 627, 601, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 601, 0, 0, 2059, 0, 0, 2778, 6223, 0, 0, 0, 728, 0, 86, 330, 0, 277, 1, 77, 0, 0, 2003, 0, 0, 78, 0, 0, 0, 0, 88, 312, 0, 0, 4361, 158, 0, 73, 21, 0, 5409, 0, 0, 0, 0, 0, 673, 0, 65, 0, 7575, 0, 86, 6, 7148, 0, 0, 0, 6836, 527, 573, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 56, 636, 0, 0, 1474, 0, 0, 4083, 4741, 0, 0, 0, 646, 0, 88, 329, 0, 355, 1, 47, 0, 0, 2886, 0, 0, 84, 0, 0, 0, 0, 50, 326, 0, 0, 4579, 161, 0, 84, 4, 0, 4929, 0, 0, 0, 0, 0, 721, 0, 83, 0, 7531, 0, 88, 72, 7052, 0, 0, 0, 6825, 645, 576, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 46, 592, 0, 0, 1479, 0, 1568, 1756, 0, 0, 0, 0, 0, 0, 0, 4305, 1764, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 346, 0, 353, 0, 54, 0, 0, 5340, 0, 0, 0, 0, 0, 0, 0, 2891, 0, 0, 91, 0, 0, 0, 0, 2052, 1311, 0, 0, 0, 0, 0, 0, 54, 342, 0, 0, 4471, 173, 0, 91, 992, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4949, 0, 0, 0, 0, 0, 902, 236, 0, 0, 0, 0, 0, 0, 652, 0, 90, 0, 5209, 0, 94, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 571, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5441, 6696, 6187, 6345, 6123, 6088, 6124, 6148] avg=6144.0000 max=6696 max_rank=1 min=5441 min_rank=0 imbalance_ratio=1.089844 max_load_reduction=42456 max_load_reduction_pct=0.863770 moved_to_echo=24483 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 556, 0, 0, 2196, 0, 2078, 1795, 0, 0, 0, 0, 0, 0, 0, 2617, 2339, 0, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 323, 0, 257, 0, 62, 0, 0, 5455, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 0, 64, 0, 0, 0, 0, 2085, 1737, 0, 0, 0, 0, 0, 0, 71, 308, 0, 0, 4807, 133, 0, 64, 995, 0, 0, 0, 0, 0, 0, 0, 28, 0, 5128, 0, 0, 0, 0, 0, 905, 240, 0, 0, 0, 0, 0, 0, 691, 0, 57, 0, 5294, 0, 79, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5043, 552, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6625, 5646, 6176, 5718, 6378, 6301, 6124, 6184] avg=6144.0000 max=6625 max_rank=0 min=5646 min_rank=1 imbalance_ratio=1.078288 max_load_reduction=42527 max_load_reduction_pct=0.865214 moved_to_echo=24987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 34, 647, 0, 0, 1730, 0, 1848, 1755, 0, 0, 0, 0, 0, 0, 0, 3722, 2079, 0, 1, 2, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 295, 0, 248, 2, 61, 0, 0, 5338, 0, 0, 0, 0, 0, 0, 0, 2945, 0, 0, 107, 1, 0, 0, 1, 2065, 1545, 0, 0, 0, 0, 0, 0, 68, 352, 1, 1, 3977, 154, 0, 107, 981, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4936, 0, 0, 0, 0, 1, 892, 238, 0, 0, 0, 0, 0, 0, 704, 1, 107, 0, 5242, 1, 49, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4971, 627, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 6352, 5993, 6664, 5641, 6081, 6208, 6199] avg=6144.0000 max=6664 max_rank=3 min=5641 min_rank=4 imbalance_ratio=1.084635 max_load_reduction=42488 max_load_reduction_pct=0.864421 moved_to_echo=24129 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 600, 0, 0, 1892, 0, 1828, 1767, 0, 0, 0, 0, 0, 0, 0, 3599, 2057, 0, 0, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 353, 0, 328, 0, 52, 0, 0, 5371, 0, 0, 0, 0, 0, 0, 0, 2516, 0, 0, 72, 0, 0, 0, 0, 2075, 1528, 0, 0, 0, 0, 0, 0, 53, 330, 0, 0, 4613, 143, 0, 72, 983, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4744, 0, 0, 0, 0, 0, 894, 239, 0, 0, 0, 0, 0, 0, 676, 0, 71, 0, 5269, 0, 95, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4978, 531, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6087, 6371, 6199, 6191, 6194, 5878, 6135, 6097] avg=6144.0000 max=6371 max_rank=1 min=5878 min_rank=5 imbalance_ratio=1.036947 max_load_reduction=42781 max_load_reduction_pct=0.870382 moved_to_echo=24304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 542, 0, 0, 2191, 0, 2117, 1788, 0, 0, 0, 0, 0, 0, 0, 2836, 2382, 0, 0, 0, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 317, 0, 259, 6, 52, 0, 0, 5436, 0, 0, 0, 0, 0, 0, 0, 1935, 0, 0, 77, 0, 0, 0, 0, 2083, 1770, 0, 0, 0, 0, 0, 0, 59, 329, 0, 0, 4659, 146, 0, 77, 981, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4912, 0, 0, 0, 0, 0, 892, 240, 0, 0, 0, 0, 0, 0, 726, 0, 71, 0, 5288, 0, 66, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4970, 595, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6638, 5911, 6136, 5865, 6251, 6079, 6153, 6119] avg=6144.0000 max=6638 max_rank=0 min=5865 min_rank=3 imbalance_ratio=1.080404 max_load_reduction=42514 max_load_reduction_pct=0.864950 moved_to_echo=24602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 36, 610, 0, 0, 2031, 0, 1853, 1774, 0, 0, 0, 0, 0, 0, 0, 2996, 2075, 0, 0, 0, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 328, 0, 325, 1, 60, 0, 0, 5389, 0, 0, 0, 0, 0, 0, 0, 2392, 0, 0, 77, 0, 0, 0, 0, 2069, 1551, 0, 0, 0, 0, 0, 0, 66, 323, 0, 0, 4824, 145, 0, 77, 980, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4962, 0, 0, 0, 0, 0, 890, 243, 0, 0, 0, 0, 0, 0, 678, 0, 76, 0, 5237, 0, 82, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4932, 617, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6304, 5774, 6185, 6089, 6415, 6105, 6118, 6162] avg=6144.0000 max=6415 max_rank=4 min=5774 min_rank=1 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=24800 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 56, 636, 0, 0, 1474, 0, 1601, 1745, 0, 0, 0, 0, 0, 0, 0, 4083, 1802, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 329, 0, 355, 1, 47, 0, 0, 5307, 0, 0, 0, 0, 0, 0, 0, 2886, 0, 0, 84, 0, 0, 0, 0, 2061, 1338, 0, 0, 0, 0, 0, 0, 50, 326, 0, 0, 4579, 161, 0, 84, 978, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4929, 0, 0, 0, 0, 0, 890, 237, 0, 0, 0, 0, 0, 0, 721, 0, 83, 0, 5233, 0, 88, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4957, 645, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5512, 6531, 6127, 6369, 6178, 6060, 6197, 6178] avg=6144.0000 max=6531 max_rank=1 min=5512 min_rank=0 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 34, 647, 0, 0, 1730, 0, 1848, 1755, 0, 0, 0, 0, 0, 0, 0, 3722, 2079, 0, 1, 2, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 295, 0, 248, 2, 61, 0, 0, 5338, 0, 0, 0, 0, 0, 0, 0, 2945, 0, 0, 107, 1, 0, 0, 1, 2065, 1545, 0, 0, 0, 0, 0, 0, 68, 352, 1, 1, 3977, 154, 0, 107, 981, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4936, 0, 0, 0, 0, 1, 892, 238, 0, 0, 0, 0, 0, 0, 704, 1, 107, 0, 5242, 1, 49, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4971, 627, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 6352, 5993, 6664, 5641, 6081, 6208, 6199] avg=6144.0000 max=6664 max_rank=3 min=5641 min_rank=4 imbalance_ratio=1.084635 max_load_reduction=42488 max_load_reduction_pct=0.864421 moved_to_echo=24129 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 556, 0, 0, 2196, 0, 2078, 1795, 0, 0, 0, 0, 0, 0, 0, 2617, 2339, 0, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 323, 0, 257, 0, 62, 0, 0, 5455, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 0, 64, 0, 0, 0, 0, 2085, 1737, 0, 0, 0, 0, 0, 0, 71, 308, 0, 0, 4807, 133, 0, 64, 995, 0, 0, 0, 0, 0, 0, 0, 28, 0, 5128, 0, 0, 0, 0, 0, 905, 240, 0, 0, 0, 0, 0, 0, 691, 0, 57, 0, 5294, 0, 79, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5043, 552, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6625, 5646, 6176, 5718, 6378, 6301, 6124, 6184] avg=6144.0000 max=6625 max_rank=0 min=5646 min_rank=1 imbalance_ratio=1.078288 max_load_reduction=42527 max_load_reduction_pct=0.865214 moved_to_echo=24987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 601, 0, 0, 2059, 0, 2101, 1770, 0, 0, 0, 0, 0, 0, 0, 2778, 2365, 0, 0, 0, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 330, 0, 277, 1, 77, 0, 0, 5378, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 0, 78, 0, 0, 0, 0, 2073, 1757, 0, 0, 0, 0, 0, 0, 88, 312, 0, 0, 4361, 158, 0, 73, 980, 0, 0, 0, 0, 0, 0, 0, 21, 0, 5409, 0, 0, 0, 0, 0, 891, 239, 0, 0, 0, 0, 0, 0, 673, 0, 65, 0, 5263, 0, 86, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4965, 527, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 5871, 6149, 5911, 5972, 6560, 6093, 6065] avg=6144.0000 max=6560 max_rank=5 min=5871 min_rank=1 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 36, 610, 0, 0, 2031, 0, 1853, 1774, 0, 0, 0, 0, 0, 0, 0, 2996, 2075, 0, 0, 0, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 328, 0, 325, 1, 60, 0, 0, 5389, 0, 0, 0, 0, 0, 0, 0, 2392, 0, 0, 77, 0, 0, 0, 0, 2069, 1551, 0, 0, 0, 0, 0, 0, 66, 323, 0, 0, 4824, 145, 0, 77, 980, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4962, 0, 0, 0, 0, 0, 890, 243, 0, 0, 0, 0, 0, 0, 678, 0, 76, 0, 5237, 0, 82, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4932, 617, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6304, 5774, 6185, 6089, 6415, 6105, 6118, 6162] avg=6144.0000 max=6415 max_rank=4 min=5774 min_rank=1 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=24800 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 256, 4864, 0, 0, 15104, 0, 15104, 14208, 0, 0, 0, 0, 0, 0] @@ -9475,280 +9475,280 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19456, 0, 0, 768, 128, 0, 0, 128, 16640, 12544, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 40064, 0, 0, 0, 0, 128, 7168, 1920, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 2688, 128, 128, 36352, 1280, 0, 768, 7936, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5632, 128, 640, 0, 42112, 128, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 2688, 0, 2432, 128, 512, 0, 0, 43136, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 39936, 4736, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5632, 128, 640, 0, 42112, 128, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[3369, 0, 0, 65, 700, 5862, 0, 0, 1330, 61, 349, 0, 0, 0, 0, 0, 0, 65, 130, 0, 0, 64, 0, 0, 136, 0, 0, 0, 0, 65, 0, 0, 0, 0, 699, 0, 0, 0, 1885, 315, 0, 7198, 5937, 8085, 0, 0, 3882, 0, 0, 0, 0, 65, 0, 0, 0, 7021, 884, 0, 0, 0, 603, 123, 244, 15] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[3634, 0, 0, 72, 736, 5423, 0, 0, 1196, 95, 305, 0, 0, 0, 0, 0, 0, 72, 154, 0, 0, 72, 0, 0, 127, 0, 0, 0, 0, 72, 0, 0, 0, 0, 984, 0, 0, 0, 2352, 365, 0, 7090, 5639, 8039, 0, 0, 3898, 0, 0, 0, 0, 72, 0, 0, 0, 6921, 835, 0, 0, 1, 607, 133, 258, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3488, 0, 2, 107, 756, 5253, 0, 0, 1212, 49, 358, 0, 0, 0, 0, 0, 0, 107, 86, 1, 0, 107, 1, 0, 182, 0, 0, 1, 1, 107, 0, 1, 1, 0, 1085, 0, 0, 1, 2031, 399, 0, 7014, 5621, 8104, 0, 1, 4100, 0, 0, 0, 0, 107, 0, 0, 4, 6781, 1007, 2, 0, 5, 604, 218, 245, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3418, 0, 0, 91, 722, 5093, 0, 0, 1053, 94, 372, 0, 0, 0, 0, 0, 0, 90, 160, 0, 0, 90, 0, 0, 147, 0, 0, 0, 0, 91, 0, 0, 0, 0, 1185, 0, 0, 0, 2143, 446, 0, 6811, 5829, 8032, 0, 0, 4401, 0, 0, 0, 0, 91, 0, 0, 3, 6886, 892, 0, 0, 1, 569, 188, 252, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3360, 0, 0, 85, 746, 5058, 0, 0, 1141, 88, 345, 0, 0, 0, 0, 0, 0, 85, 145, 0, 0, 83, 0, 0, 137, 0, 0, 0, 0, 85, 0, 0, 0, 0, 1097, 0, 0, 0, 2411, 388, 0, 6904, 5642, 8047, 0, 0, 4444, 0, 0, 0, 0, 85, 0, 0, 0, 6867, 894, 0, 0, 0, 623, 150, 241, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[3369, 0, 0, 65, 700, 5862, 0, 0, 1330, 61, 349, 0, 0, 0, 0, 0, 0, 65, 130, 0, 0, 64, 0, 0, 136, 0, 0, 0, 0, 65, 0, 0, 0, 0, 699, 0, 0, 0, 1885, 315, 0, 7198, 5937, 8085, 0, 0, 3882, 0, 0, 0, 0, 65, 0, 0, 0, 7021, 884, 0, 0, 0, 603, 123, 244, 15] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3196, 0, 0, 78, 693, 5635, 0, 0, 1278, 63, 339, 0, 0, 0, 0, 0, 0, 78, 117, 0, 0, 77, 0, 0, 151, 0, 0, 0, 0, 78, 0, 0, 0, 0, 845, 0, 0, 0, 2162, 390, 0, 7116, 6102, 8081, 0, 0, 3807, 0, 0, 0, 0, 78, 0, 0, 0, 6812, 918, 0, 0, 0, 636, 171, 251, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[3163, 0, 0, 77, 754, 5560, 0, 0, 1030, 82, 354, 0, 0, 0, 0, 0, 0, 77, 141, 0, 0, 77, 0, 0, 151, 0, 0, 0, 0, 77, 0, 0, 0, 0, 1048, 0, 0, 0, 2039, 385, 0, 7043, 6012, 8051, 0, 0, 4047, 0, 0, 0, 0, 77, 0, 0, 0, 6871, 989, 0, 0, 0, 649, 152, 246, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3488, 0, 2, 107, 756, 5253, 0, 0, 1212, 49, 358, 0, 0, 0, 0, 0, 0, 107, 86, 1, 0, 107, 1, 0, 182, 0, 0, 1, 1, 107, 0, 1, 1, 0, 1085, 0, 0, 1, 2031, 399, 0, 7014, 5621, 8104, 0, 1, 4100, 0, 0, 0, 0, 107, 0, 0, 4, 6781, 1007, 2, 0, 5, 604, 218, 245, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[3634, 0, 0, 72, 736, 5423, 0, 0, 1196, 95, 305, 0, 0, 0, 0, 0, 0, 72, 154, 0, 0, 72, 0, 0, 127, 0, 0, 0, 0, 72, 0, 0, 0, 0, 984, 0, 0, 0, 2352, 365, 0, 7090, 5639, 8039, 0, 0, 3898, 0, 0, 0, 0, 72, 0, 0, 0, 6921, 835, 0, 0, 1, 607, 133, 258, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[3814, 0, 0, 74, 730, 5681, 0, 0, 1103, 83, 353, 0, 0, 0, 0, 0, 0, 74, 137, 0, 0, 73, 0, 0, 171, 0, 0, 0, 0, 74, 0, 0, 0, 0, 842, 0, 0, 0, 1508, 310, 0, 7130, 5984, 8058, 0, 0, 3892, 0, 0, 0, 0, 74, 0, 0, 0, 7009, 978, 0, 0, 1, 566, 170, 244, 19] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3360, 0, 0, 85, 746, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1141, 88, 345, 0, 0, 0, 0, 0, 3275, 1170, 0, 0, 0, 0, 0, 0, 0, 85, 145, 0, 0, 83, 0, 0, 3629, 2145, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 85, 0, 0, 5902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 0, 0, 0, 2411, 388, 1760, 830, 0, 0, 0, 0, 0, 0, 0, 0, 2013, 0, 0, 0, 4444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 6037, 0, 0, 0, 0, 0, 0, 0, 0, 894, 0, 0, 0, 623, 150, 241, 1, 2459, 1530, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5959, 6019, 6087, 6124, 6486, 6457, 6122, 5898] avg=6144.0000 max=6486 max_rank=4 min=5898 min_rank=7 imbalance_ratio=1.055664 max_load_reduction=42666 max_load_reduction_pct=0.868042 moved_to_echo=24963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3488, 0, 2, 107, 756, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 49, 358, 0, 0, 0, 0, 0, 3327, 1166, 0, 0, 0, 0, 0, 0, 0, 107, 86, 1, 0, 107, 1, 0, 3687, 2161, 0, 0, 0, 0, 0, 0, 182, 0, 0, 1, 1, 107, 0, 1, 5943, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1085, 0, 0, 1, 2031, 399, 1828, 820, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 0, 1, 4100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 4, 5961, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 2, 0, 5, 604, 218, 245, 3, 2449, 1589, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6189, 6112, 6150, 6235, 6165, 6107, 6072, 6122] avg=6144.0000 max=6235 max_rank=3 min=6072 min_rank=6 imbalance_ratio=1.014811 max_load_reduction=42917 max_load_reduction_pct=0.873149 moved_to_echo=24466 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3418, 0, 0, 91, 722, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 94, 372, 0, 0, 0, 0, 0, 3231, 1209, 0, 0, 0, 0, 0, 0, 0, 90, 160, 0, 0, 90, 0, 0, 3580, 2142, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 91, 0, 0, 5890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 0, 0, 0, 2143, 446, 1772, 832, 0, 0, 0, 0, 0, 0, 0, 0, 2080, 0, 0, 0, 4401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 3, 6054, 0, 0, 0, 0, 0, 0, 0, 0, 892, 0, 0, 1, 569, 188, 252, 2, 2540, 1540, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6012, 5959, 6062, 6128, 6378, 6481, 6148, 5984] avg=6144.0000 max=6481 max_rank=5 min=5959 min_rank=1 imbalance_ratio=1.054850 max_load_reduction=42671 max_load_reduction_pct=0.868144 moved_to_echo=24991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3196, 0, 0, 78, 693, 1970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1278, 63, 339, 0, 0, 0, 0, 0, 3376, 1266, 0, 0, 0, 0, 0, 0, 0, 78, 117, 0, 0, 77, 0, 0, 3740, 2155, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 78, 0, 0, 5926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 0, 0, 0, 2162, 390, 1961, 823, 0, 0, 0, 0, 0, 0, 0, 0, 2177, 0, 0, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 5989, 0, 0, 0, 0, 0, 0, 0, 0, 918, 0, 0, 0, 636, 171, 251, 0, 2659, 1704, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5937, 6322, 6167, 6155, 6181, 5984, 6067, 6339] avg=6144.0000 max=6339 max_rank=7 min=5937 min_rank=0 imbalance_ratio=1.031738 max_load_reduction=42813 max_load_reduction_pct=0.871033 moved_to_echo=24571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3360, 0, 0, 85, 746, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1141, 88, 345, 0, 0, 0, 0, 0, 3275, 1170, 0, 0, 0, 0, 0, 0, 0, 85, 145, 0, 0, 83, 0, 0, 3629, 2145, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 85, 0, 0, 5902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 0, 0, 0, 2411, 388, 1760, 830, 0, 0, 0, 0, 0, 0, 0, 0, 2013, 0, 0, 0, 4444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 6037, 0, 0, 0, 0, 0, 0, 0, 0, 894, 0, 0, 0, 623, 150, 241, 1, 2459, 1530, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5959, 6019, 6087, 6124, 6486, 6457, 6122, 5898] avg=6144.0000 max=6486 max_rank=4 min=5898 min_rank=7 imbalance_ratio=1.055664 max_load_reduction=42666 max_load_reduction_pct=0.868042 moved_to_echo=24963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3634, 0, 0, 72, 736, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1196, 95, 305, 0, 0, 0, 0, 0, 3365, 1169, 0, 0, 0, 0, 0, 0, 0, 72, 154, 0, 0, 72, 0, 0, 3725, 2144, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 72, 0, 0, 5895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 0, 0, 0, 2352, 365, 1887, 837, 0, 0, 0, 0, 0, 0, 0, 0, 2013, 0, 0, 0, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 1, 607, 133, 258, 0, 2457, 1640, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6338, 6130, 6167, 6094, 6425, 5911, 6156, 5931] avg=6144.0000 max=6425 max_rank=4 min=5911 min_rank=5 imbalance_ratio=1.045736 max_load_reduction=42727 max_load_reduction_pct=0.869283 moved_to_echo=24423 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3369, 0, 0, 65, 700, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 61, 349, 0, 0, 0, 0, 0, 3416, 1231, 0, 0, 0, 0, 0, 0, 0, 65, 130, 0, 0, 64, 0, 0, 3782, 2156, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 65, 0, 0, 5929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 699, 0, 0, 0, 1885, 315, 2040, 849, 0, 0, 0, 0, 0, 0, 0, 0, 2119, 0, 0, 0, 3882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 6172, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 0, 0, 603, 123, 244, 15, 2587, 1773, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6387, 6197, 6130, 5788, 6001, 6237, 6229] avg=6144.0000 max=6387 max_rank=1 min=5788 min_rank=4 imbalance_ratio=1.039551 max_load_reduction=42765 max_load_reduction_pct=0.870056 moved_to_echo=24255 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3196, 0, 0, 78, 693, 1970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1278, 63, 339, 0, 0, 0, 0, 0, 3376, 1266, 0, 0, 0, 0, 0, 0, 0, 78, 117, 0, 0, 77, 0, 0, 3740, 2155, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 78, 0, 0, 5926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 0, 0, 0, 2162, 390, 1961, 823, 0, 0, 0, 0, 0, 0, 0, 0, 2177, 0, 0, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 5989, 0, 0, 0, 0, 0, 0, 0, 0, 918, 0, 0, 0, 636, 171, 251, 0, 2659, 1704, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5937, 6322, 6167, 6155, 6181, 5984, 6067, 6339] avg=6144.0000 max=6339 max_rank=7 min=5937 min_rank=0 imbalance_ratio=1.031738 max_load_reduction=42813 max_load_reduction_pct=0.871033 moved_to_echo=24571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3163, 0, 0, 77, 754, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 82, 354, 0, 0, 0, 0, 0, 3342, 1251, 0, 0, 0, 0, 0, 0, 0, 77, 141, 0, 0, 77, 0, 0, 3701, 2147, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 77, 0, 0, 5904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 0, 0, 2039, 385, 1939, 834, 0, 0, 0, 0, 0, 0, 0, 0, 2137, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 6037, 0, 0, 0, 0, 0, 0, 0, 0, 989, 0, 0, 0, 649, 152, 246, 0, 2624, 1685, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 6059, 6143, 6132, 6245, 6184, 6114, 6345] avg=6144.0000 max=6345 max_rank=7 min=5930 min_rank=0 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24888 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3488, 0, 2, 107, 756, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 49, 358, 0, 0, 0, 0, 0, 3327, 1166, 0, 0, 0, 0, 0, 0, 0, 107, 86, 1, 0, 107, 1, 0, 3687, 2161, 0, 0, 0, 0, 0, 0, 182, 0, 0, 1, 1, 107, 0, 1, 5943, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1085, 0, 0, 1, 2031, 399, 1828, 820, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 0, 1, 4100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 4, 5961, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 2, 0, 5, 604, 218, 245, 3, 2449, 1589, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6189, 6112, 6150, 6235, 6165, 6107, 6072, 6122] avg=6144.0000 max=6235 max_rank=3 min=6072 min_rank=6 imbalance_ratio=1.014811 max_load_reduction=42917 max_load_reduction_pct=0.873149 moved_to_echo=24466 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3814, 0, 0, 74, 730, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 83, 353, 0, 0, 0, 0, 0, 3384, 1241, 0, 0, 0, 0, 0, 0, 0, 74, 137, 0, 0, 73, 0, 0, 3746, 2149, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 74, 0, 0, 5909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 0, 0, 0, 1508, 310, 1977, 847, 0, 0, 0, 0, 0, 0, 0, 0, 2135, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 6162, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 0, 1, 566, 170, 244, 19, 2608, 1718, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6604, 6164, 6179, 6154, 5484, 6027, 6236, 6304] avg=6144.0000 max=6604 max_rank=0 min=5484 min_rank=4 imbalance_ratio=1.074870 max_load_reduction=42548 max_load_reduction_pct=0.865641 moved_to_echo=24051 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3163, 0, 0, 77, 754, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 82, 354, 0, 0, 0, 0, 0, 3342, 1251, 0, 0, 0, 0, 0, 0, 0, 77, 141, 0, 0, 77, 0, 0, 3701, 2147, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 77, 0, 0, 5904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 0, 0, 2039, 385, 1939, 834, 0, 0, 0, 0, 0, 0, 0, 0, 2137, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 6037, 0, 0, 0, 0, 0, 0, 0, 0, 989, 0, 0, 0, 649, 152, 246, 0, 2624, 1685, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 6059, 6143, 6132, 6245, 6184, 6114, 6345] avg=6144.0000 max=6345 max_rank=7 min=5930 min_rank=0 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24888 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 128, 0, 128, 4864, 1408, 2048, 128, 20480, 13184, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[9344, 640, 2816, 0, 0, 0, 0, 0, 26752, 9728, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 128, 0, 128, 4864, 1408, 2048, 128, 20480, 13184, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 7808, 0, 0, 128, 16640, 3072, 15232, 6784, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[27520, 0, 128, 768, 5888, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 16768, 0, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 768, 1152, 128, 0, 768, 128, 0, 29696, 17280, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1280, 0, 0, 128, 128, 768, 0, 128, 47360, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 768, 1152, 128, 0, 768, 128, 0, 29696, 17280, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 768, 0, 0, 128, 48512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[1703, 845, 0, 0, 0, 0, 2179, 0, 0, 3036, 0, 78, 0, 0, 324, 0, 0, 169, 0, 0, 0, 576, 0, 0, 0, 0, 6976, 240, 1725, 0, 230, 3157, 0, 0, 0, 0, 0, 0, 0, 3502, 0, 0, 1461, 145, 7208, 90, 7052, 0, 7067, 0, 0, 169, 0, 84, 0, 0, 88, 0, 0, 0, 741, 84, 0, 223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[1808, 921, 0, 0, 0, 0, 2162, 0, 0, 3183, 0, 62, 0, 0, 334, 0, 0, 182, 1, 0, 0, 528, 0, 0, 0, 0, 7020, 252, 1661, 0, 219, 3060, 0, 0, 0, 0, 0, 0, 0, 3045, 0, 0, 1609, 160, 7225, 92, 7019, 0, 7097, 0, 0, 188, 0, 91, 0, 0, 91, 0, 0, 0, 777, 90, 0, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[1239, 864, 0, 0, 0, 0, 3160, 0, 0, 2978, 0, 12, 0, 0, 381, 0, 0, 155, 0, 0, 0, 608, 0, 0, 0, 0, 7090, 251, 1516, 0, 196, 4347, 0, 0, 0, 0, 0, 0, 0, 2283, 0, 0, 1936, 115, 7206, 78, 6681, 0, 6898, 0, 0, 161, 0, 77, 0, 0, 11, 0, 0, 0, 643, 76, 0, 190] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[1384, 813, 0, 0, 0, 0, 2956, 0, 0, 3175, 0, 51, 0, 0, 412, 0, 0, 154, 0, 0, 0, 578, 0, 0, 0, 0, 6968, 248, 1539, 0, 214, 3456, 0, 0, 0, 0, 0, 0, 0, 2875, 0, 0, 1728, 141, 7195, 77, 6830, 0, 6996, 0, 0, 178, 0, 77, 0, 0, 65, 0, 0, 0, 797, 76, 0, 169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[1155, 834, 0, 0, 0, 0, 3421, 0, 0, 2587, 0, 85, 0, 0, 188, 0, 0, 129, 0, 0, 0, 593, 0, 0, 0, 0, 7064, 244, 1164, 0, 149, 4457, 0, 0, 0, 0, 0, 0, 0, 2384, 0, 0, 2003, 129, 7311, 65, 6890, 0, 7058, 0, 0, 169, 0, 64, 0, 0, 0, 0, 0, 0, 776, 57, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[1808, 921, 0, 0, 0, 0, 2162, 0, 0, 3183, 0, 62, 0, 0, 334, 0, 0, 182, 1, 0, 0, 528, 0, 0, 0, 0, 7020, 252, 1661, 0, 219, 3060, 0, 0, 0, 0, 0, 0, 0, 3045, 0, 0, 1609, 160, 7225, 92, 7019, 0, 7097, 0, 0, 188, 0, 91, 0, 0, 91, 0, 0, 0, 777, 90, 0, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[1492, 1053, 0, 0, 0, 0, 2265, 0, 0, 3086, 0, 95, 1, 0, 348, 0, 1, 214, 2, 0, 1, 554, 0, 1, 0, 0, 6902, 245, 1511, 1, 161, 3569, 2, 0, 0, 0, 1, 0, 0, 3114, 1, 2, 1605, 86, 7314, 107, 6903, 1, 6943, 1, 0, 198, 0, 107, 0, 0, 49, 0, 0, 1, 719, 107, 0, 389] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[1323, 789, 0, 0, 0, 0, 2702, 0, 0, 3250, 0, 34, 0, 0, 378, 0, 0, 144, 0, 0, 0, 618, 0, 0, 0, 0, 7051, 258, 1639, 0, 194, 3789, 0, 0, 0, 0, 0, 0, 0, 2780, 0, 0, 1720, 154, 7211, 72, 6852, 0, 7067, 0, 0, 154, 0, 72, 0, 0, 57, 0, 0, 0, 617, 71, 0, 156] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[1165, 1065, 0, 0, 0, 0, 2903, 0, 0, 2975, 0, 15, 0, 0, 474, 0, 0, 147, 0, 0, 0, 553, 0, 0, 0, 0, 7117, 244, 1204, 0, 163, 4341, 0, 0, 0, 0, 0, 0, 0, 2711, 0, 0, 2149, 137, 7204, 75, 6498, 0, 6879, 0, 0, 187, 0, 73, 0, 0, 9, 0, 0, 0, 587, 71, 0, 206] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[1384, 813, 0, 0, 0, 0, 2956, 0, 0, 3175, 0, 51, 0, 0, 412, 0, 0, 154, 0, 0, 0, 578, 0, 0, 0, 0, 6968, 248, 1539, 0, 214, 3456, 0, 0, 0, 0, 0, 0, 0, 2875, 0, 0, 1728, 141, 7195, 77, 6830, 0, 6996, 0, 0, 178, 0, 77, 0, 0, 65, 0, 0, 0, 797, 76, 0, 169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[1323, 789, 0, 0, 0, 0, 2702, 0, 0, 3250, 0, 34, 0, 0, 378, 0, 0, 144, 0, 0, 0, 618, 0, 0, 0, 0, 7051, 258, 1639, 0, 194, 3789, 0, 0, 0, 0, 0, 0, 0, 2780, 0, 0, 1720, 154, 7211, 72, 6852, 0, 7067, 0, 0, 154, 0, 72, 0, 0, 57, 0, 0, 0, 617, 71, 0, 156] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[1155, 834, 0, 0, 0, 0, 3421, 0, 0, 2587, 0, 85, 0, 0, 188, 0, 0, 129, 0, 0, 0, 593, 0, 0, 0, 0, 7064, 244, 1164, 0, 149, 4457, 0, 0, 0, 0, 0, 0, 0, 2384, 0, 0, 2003, 129, 7311, 65, 6890, 0, 7058, 0, 0, 169, 0, 64, 0, 0, 0, 0, 0, 0, 776, 57, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1239, 864, 0, 0, 0, 0, 3160, 0, 1096, 5, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 12, 0, 0, 381, 0, 2620, 18, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 608, 0, 0, 5384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 251, 1516, 0, 196, 4347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2283, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 115, 0, 78, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5802, 0, 0, 161, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 643, 76, 0, 190, 3290, 1822, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6364, 6009, 6147, 6754, 5621, 6185, 6040, 6032] avg=6144.0000 max=6754 max_rank=3 min=5621 min_rank=4 imbalance_ratio=1.099284 max_load_reduction=42398 max_load_reduction_pct=0.862590 moved_to_echo=23878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1492, 1053, 0, 0, 0, 0, 2265, 0, 1103, 6, 0, 0, 0, 0, 0, 0, 0, 3086, 0, 95, 1, 0, 348, 0, 2707, 18, 0, 0, 0, 0, 0, 0, 1, 214, 2, 0, 1, 554, 0, 1, 5465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 245, 1511, 1, 161, 3569, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 3114, 3249, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1605, 86, 0, 107, 4190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5840, 1, 0, 198, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 1, 719, 107, 0, 389, 3203, 1849, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5919, 6255, 6238, 5919, 6366, 5992, 6146, 6317] avg=6144.0000 max=6366 max_rank=4 min=5919 min_rank=0 imbalance_ratio=1.036133 max_load_reduction=42786 max_load_reduction_pct=0.870483 moved_to_echo=24821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1808, 921, 0, 0, 0, 0, 2162, 0, 1128, 6, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 62, 0, 0, 334, 0, 2752, 18, 0, 0, 0, 0, 0, 0, 0, 182, 1, 0, 0, 528, 0, 0, 5398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 252, 1661, 0, 219, 3060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3045, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1609, 160, 0, 92, 4261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 188, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 777, 90, 0, 275, 3258, 1827, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6025, 6349, 6109, 5631, 6350, 6122, 6248, 6318] avg=6144.0000 max=6350 max_rank=4 min=5631 min_rank=3 imbalance_ratio=1.033529 max_load_reduction=42802 max_load_reduction_pct=0.870809 moved_to_echo=25038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1703, 845, 0, 0, 0, 0, 2179, 0, 1123, 6, 0, 0, 0, 0, 0, 0, 0, 3036, 0, 78, 0, 0, 324, 0, 2765, 18, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 576, 0, 0, 5385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 240, 1725, 0, 230, 3157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 145, 0, 90, 4281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5944, 0, 0, 169, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 741, 84, 0, 223, 3237, 1823, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5856, 6221, 6130, 5789, 6786, 5977, 6197, 6196] avg=6144.0000 max=6786 max_rank=4 min=5789 min_rank=3 imbalance_ratio=1.104492 max_load_reduction=42366 max_load_reduction_pct=0.861938 moved_to_echo=25156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1239, 864, 0, 0, 0, 0, 3160, 0, 1096, 5, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 12, 0, 0, 381, 0, 2620, 18, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 608, 0, 0, 5384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 251, 1516, 0, 196, 4347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2283, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 115, 0, 78, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5802, 0, 0, 161, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 643, 76, 0, 190, 3290, 1822, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6364, 6009, 6147, 6754, 5621, 6185, 6040, 6032] avg=6144.0000 max=6754 max_rank=3 min=5621 min_rank=4 imbalance_ratio=1.099284 max_load_reduction=42398 max_load_reduction_pct=0.862590 moved_to_echo=23878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1155, 834, 0, 0, 0, 0, 3421, 0, 1121, 6, 0, 0, 0, 0, 0, 0, 0, 2587, 0, 85, 0, 0, 188, 0, 2702, 18, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 593, 0, 0, 5463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 244, 1164, 0, 149, 4457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 129, 0, 65, 4182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5937, 0, 0, 169, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 57, 0, 176, 3278, 1848, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6537, 5580, 6185, 6457, 5709, 6379, 6170, 6135] avg=6144.0000 max=6537 max_rank=0 min=5580 min_rank=1 imbalance_ratio=1.063965 max_load_reduction=42615 max_load_reduction_pct=0.867004 moved_to_echo=24393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1323, 789, 0, 0, 0, 0, 2702, 0, 1123, 6, 0, 0, 0, 0, 0, 0, 0, 3250, 0, 34, 0, 0, 378, 0, 2687, 18, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 618, 0, 0, 5389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 258, 1639, 0, 194, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2780, 3319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 154, 0, 72, 4159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5944, 0, 0, 154, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 617, 71, 0, 156, 3272, 1822, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5943, 6367, 6151, 6322, 6099, 6105, 6170, 5995] avg=6144.0000 max=6367 max_rank=1 min=5943 min_rank=0 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1384, 813, 0, 0, 0, 0, 2956, 0, 1116, 8, 0, 0, 0, 0, 0, 0, 0, 3175, 0, 51, 0, 0, 412, 0, 2682, 21, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 578, 0, 0, 5377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 248, 1539, 0, 214, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2875, 3285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 141, 0, 77, 4140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5880, 0, 0, 178, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 797, 76, 0, 169, 3237, 1818, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6277, 6341, 6109, 5882, 6160, 6086, 6135, 6162] avg=6144.0000 max=6341 max_rank=1 min=5882 min_rank=3 imbalance_ratio=1.032064 max_load_reduction=42811 max_load_reduction_pct=0.870992 moved_to_echo=24543 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1492, 1053, 0, 0, 0, 0, 2265, 0, 1103, 6, 0, 0, 0, 0, 0, 0, 0, 3086, 0, 95, 1, 0, 348, 0, 2707, 18, 0, 0, 0, 0, 0, 0, 1, 214, 2, 0, 1, 554, 0, 1, 5465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 245, 1511, 1, 161, 3569, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 3114, 3249, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1605, 86, 0, 107, 4190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5840, 1, 0, 198, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 1, 719, 107, 0, 389, 3203, 1849, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5919, 6255, 6238, 5919, 6366, 5992, 6146, 6317] avg=6144.0000 max=6366 max_rank=4 min=5919 min_rank=0 imbalance_ratio=1.036133 max_load_reduction=42786 max_load_reduction_pct=0.870483 moved_to_echo=24821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1165, 1065, 0, 0, 0, 0, 2903, 0, 1093, 5, 0, 0, 0, 0, 0, 0, 0, 2975, 0, 15, 0, 0, 474, 0, 2548, 18, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 553, 0, 0, 5383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 244, 1204, 0, 163, 4341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2711, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2149, 137, 0, 75, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5786, 0, 0, 187, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 587, 71, 0, 206, 3303, 1821, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6231, 6030, 6083, 6398, 6061, 6306, 6046, 5997] avg=6144.0000 max=6398 max_rank=3 min=5997 min_rank=7 imbalance_ratio=1.041341 max_load_reduction=42754 max_load_reduction_pct=0.869832 moved_to_echo=24410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1155, 834, 0, 0, 0, 0, 3421, 0, 1121, 6, 0, 0, 0, 0, 0, 0, 0, 2587, 0, 85, 0, 0, 188, 0, 2702, 18, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 593, 0, 0, 5463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 244, 1164, 0, 149, 4457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 129, 0, 65, 4182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5937, 0, 0, 169, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 57, 0, 176, 3278, 1848, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6537, 5580, 6185, 6457, 5709, 6379, 6170, 6135] avg=6144.0000 max=6537 max_rank=0 min=5580 min_rank=1 imbalance_ratio=1.063965 max_load_reduction=42615 max_load_reduction_pct=0.867004 moved_to_echo=24393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1384, 813, 0, 0, 0, 0, 2956, 0, 1116, 8, 0, 0, 0, 0, 0, 0, 0, 3175, 0, 51, 0, 0, 412, 0, 2682, 21, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 578, 0, 0, 5377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 248, 1539, 0, 214, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2875, 3285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 141, 0, 77, 4140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5880, 0, 0, 178, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 797, 76, 0, 169, 3237, 1818, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6277, 6341, 6109, 5882, 6160, 6086, 6135, 6162] avg=6144.0000 max=6341 max_rank=1 min=5882 min_rank=3 imbalance_ratio=1.032064 max_load_reduction=42811 max_load_reduction_pct=0.870992 moved_to_echo=24543 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11392, 7296, 0, 0, 0, 0, 21760, 0, 8960, 128, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 0, 0, 128, 5760, 640, 0, 1792, 26112, 14720, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 24320, 0, 512, 128, 0, 2944, 0, 21504, 256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 22784, 26496, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 3584, 2048, 12032, 128, 1536, 30208, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 24320, 0, 512, 128, 0, 2944, 0, 21504, 256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 14336, 1152, 0, 768, 33280, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[47104, 128, 0, 1408, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 3584, 2048, 12032, 128, 1536, 30208, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1408, 128, 0, 128, 4608, 0, 128, 43264, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[47104, 128, 0, 1408, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 833, 0, 0, 0, 1443, 0, 0, 0, 141, 88, 7347, 0, 0, 7595, 85, 0, 0, 0, 0, 3313, 0, 0, 0, 0, 0, 8108, 0, 0, 7492, 0, 0, 0, 84, 0, 0, 7735, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 85, 0, 2503, 0, 0, 502, 1153, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 894, 0, 0, 0, 1583, 0, 0, 0, 144, 49, 7293, 0, 1, 7534, 107, 0, 0, 1, 0, 2720, 0, 1, 1, 0, 0, 8082, 0, 2, 7550, 0, 0, 0, 107, 0, 0, 7733, 2, 0, 1, 0, 420, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 245, 0, 107, 0, 2566, 0, 0, 534, 1472, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 1088, 0, 0, 0, 1629, 0, 0, 0, 127, 66, 7310, 0, 0, 7580, 78, 0, 0, 0, 0, 1765, 0, 0, 0, 0, 0, 8080, 0, 0, 7637, 0, 0, 0, 77, 0, 0, 7821, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 78, 0, 2180, 0, 0, 506, 2516, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 962, 0, 0, 0, 1355, 0, 0, 0, 136, 82, 7378, 0, 0, 7581, 77, 0, 0, 0, 0, 2468, 0, 0, 0, 0, 0, 8104, 0, 0, 7558, 0, 0, 0, 77, 0, 0, 7811, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 77, 0, 2451, 0, 0, 470, 1950, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 928, 0, 0, 0, 1438, 0, 0, 0, 131, 95, 7399, 0, 0, 7575, 72, 0, 0, 0, 0, 2607, 0, 0, 0, 0, 0, 8114, 0, 0, 7515, 0, 0, 0, 72, 0, 0, 7814, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 72, 0, 2249, 0, 0, 484, 1967, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 825, 0, 0, 0, 1317, 0, 0, 0, 157, 94, 7400, 0, 0, 7532, 91, 0, 0, 0, 0, 3384, 0, 0, 0, 0, 0, 8101, 0, 0, 7459, 0, 0, 0, 91, 0, 0, 7671, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 91, 0, 2532, 0, 0, 459, 1234, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 1281, 0, 0, 0, 1674, 0, 0, 0, 114, 79, 7354, 0, 0, 7633, 65, 0, 0, 0, 0, 1335, 0, 0, 0, 0, 0, 8083, 0, 0, 7656, 0, 0, 0, 64, 0, 0, 7890, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 65, 0, 1832, 0, 0, 448, 3018, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 962, 0, 0, 0, 1355, 0, 0, 0, 136, 82, 7378, 0, 0, 7581, 77, 0, 0, 0, 0, 2468, 0, 0, 0, 0, 0, 8104, 0, 0, 7558, 0, 0, 0, 77, 0, 0, 7811, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 77, 0, 2451, 0, 0, 470, 1950, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 1123, 0, 0, 0, 1546, 0, 0, 0, 124, 87, 7371, 0, 0, 7570, 74, 0, 0, 0, 0, 1599, 0, 0, 0, 0, 0, 8067, 0, 0, 7617, 0, 0, 0, 73, 0, 0, 7830, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 74, 0, 2121, 0, 0, 463, 2784, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 1088, 0, 0, 0, 1629, 0, 0, 0, 127, 66, 7310, 0, 0, 7580, 78, 0, 0, 0, 0, 1765, 0, 0, 0, 0, 0, 8080, 0, 0, 7637, 0, 0, 0, 77, 0, 0, 7821, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 78, 0, 2180, 0, 0, 506, 2516, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 833, 0, 0, 0, 1443, 0, 1713, 1425, 431, 71, 0, 0, 0, 0, 0, 0, 141, 88, 5922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 3313, 0, 0, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 6022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 3491, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 85, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 2503, 0, 0, 502, 1153, 0, 0, 1332, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5916, 6151, 7071, 6089, 6106, 6177, 6152, 5490] avg=6144.0000 max=7071 max_rank=2 min=5490 min_rank=7 imbalance_ratio=1.150879 max_load_reduction=42081 max_load_reduction_pct=0.856140 moved_to_echo=23925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 894, 0, 0, 0, 1583, 0, 0, 0, 144, 49, 7293, 0, 1, 7534, 107, 0, 0, 1, 0, 2720, 0, 1, 1, 0, 0, 8082, 0, 2, 7550, 0, 0, 0, 107, 0, 0, 7733, 2, 0, 1, 0, 420, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 245, 0, 107, 0, 2566, 0, 0, 534, 1472, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 833, 0, 0, 0, 1443, 0, 0, 0, 141, 88, 7347, 0, 0, 7595, 85, 0, 0, 0, 0, 3313, 0, 0, 0, 0, 0, 8108, 0, 0, 7492, 0, 0, 0, 84, 0, 0, 7735, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 85, 0, 2503, 0, 0, 502, 1153, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1088, 0, 0, 0, 1629, 0, 1732, 1417, 430, 72, 0, 0, 0, 0, 0, 0, 127, 66, 5893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 1765, 0, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 6089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 3484, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 78, 5807, 0, 0, 0, 0, 0, 0, 0, 0, 2180, 0, 0, 506, 2516, 0, 0, 1358, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6368, 6086, 5509, 6207, 6166, 6120, 6136, 6560] avg=6144.0000 max=6560 max_rank=7 min=5509 min_rank=2 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24982 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 825, 0, 0, 0, 1317, 0, 1699, 1435, 427, 71, 0, 0, 0, 0, 0, 0, 157, 94, 5965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 3384, 0, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 5972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 3461, 2279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 91, 5822, 0, 0, 0, 0, 0, 0, 0, 0, 2532, 0, 0, 459, 1234, 0, 0, 1326, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5774, 6216, 7119, 6062, 6063, 6203, 6164, 5551] avg=6144.0000 max=7119 max_rank=2 min=5551 min_rank=7 imbalance_ratio=1.158691 max_load_reduction=42033 max_load_reduction_pct=0.855164 moved_to_echo=23981 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1281, 0, 0, 0, 1674, 0, 1747, 1426, 433, 73, 0, 0, 0, 0, 0, 0, 114, 79, 5928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 1335, 0, 0, 3693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 3507, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 65, 5808, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 0, 448, 3018, 0, 0, 1361, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6634, 6121, 5093, 6222, 6207, 6099, 6117, 6659] avg=6144.0000 max=6659 max_rank=7 min=5093 min_rank=2 imbalance_ratio=1.083822 max_load_reduction=42493 max_load_reduction_pct=0.864522 moved_to_echo=25082 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 928, 0, 0, 0, 1438, 0, 1730, 1435, 431, 71, 0, 0, 0, 0, 0, 0, 131, 95, 5964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 2607, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 6084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 3481, 2283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 72, 5831, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, 484, 1967, 0, 0, 1336, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6033, 6190, 6342, 6108, 6156, 6126, 6161, 6036] avg=6144.0000 max=6342 max_rank=2 min=6033 min_rank=0 imbalance_ratio=1.032227 max_load_reduction=42810 max_load_reduction_pct=0.870972 moved_to_echo=24479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 894, 0, 0, 0, 1583, 0, 1712, 1414, 427, 72, 0, 0, 0, 0, 0, 0, 144, 49, 5879, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 1, 0, 2720, 0, 1, 3643, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 6021, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 420, 0, 0, 2, 0, 0, 3464, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 107, 5809, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 534, 1472, 1, 0, 1342, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 6073, 6472, 6139, 6130, 6160, 6161, 5915] avg=6144.0000 max=6472 max_rank=2 min=5915 min_rank=7 imbalance_ratio=1.053385 max_load_reduction=42680 max_load_reduction_pct=0.868327 moved_to_echo=24366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 962, 0, 0, 0, 1355, 0, 1733, 1434, 431, 76, 0, 0, 0, 0, 0, 0, 136, 82, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 2468, 0, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 6078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 3484, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 77, 5823, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 0, 0, 470, 1950, 0, 0, 1348, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5991, 6162, 6211, 6134, 6155, 6134, 6146, 6219] avg=6144.0000 max=6219 max_rank=7 min=5991 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24654 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1123, 0, 0, 0, 1546, 0, 1734, 1429, 430, 72, 0, 0, 0, 0, 0, 0, 124, 87, 5942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 1599, 0, 0, 3662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 6096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 3478, 2270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 74, 5797, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 0, 0, 463, 2784, 0, 0, 1354, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6334, 6153, 5335, 6191, 6169, 6133, 6115, 6722] avg=6144.0000 max=6722 max_rank=7 min=5335 min_rank=2 imbalance_ratio=1.094076 max_load_reduction=42430 max_load_reduction_pct=0.863241 moved_to_echo=25139 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1088, 0, 0, 0, 1629, 0, 1732, 1417, 430, 72, 0, 0, 0, 0, 0, 0, 127, 66, 5893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 1765, 0, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 6089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 3484, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 78, 5807, 0, 0, 0, 0, 0, 0, 0, 0, 2180, 0, 0, 506, 2516, 0, 0, 1358, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6368, 6086, 5509, 6207, 6166, 6120, 6136, 6560] avg=6144.0000 max=6560 max_rank=7 min=5509 min_rank=2 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24982 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 833, 0, 0, 0, 1443, 0, 1713, 1425, 431, 71, 0, 0, 0, 0, 0, 0, 141, 88, 5922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 3313, 0, 0, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 6022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 3491, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 85, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 2503, 0, 0, 502, 1153, 0, 0, 1332, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5916, 6151, 7071, 6089, 6106, 6177, 6152, 5490] avg=6144.0000 max=7071 max_rank=2 min=5490 min_rank=7 imbalance_ratio=1.150879 max_load_reduction=42081 max_load_reduction_pct=0.856140 moved_to_echo=23925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1281, 0, 0, 0, 1674, 0, 1747, 1426, 433, 73, 0, 0, 0, 0, 0, 0, 114, 79, 5928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 1335, 0, 0, 3693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 3507, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 65, 5808, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 0, 448, 3018, 0, 0, 1361, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6634, 6121, 5093, 6222, 6207, 6099, 6117, 6659] avg=6144.0000 max=6659 max_rank=7 min=5093 min_rank=2 imbalance_ratio=1.083822 max_load_reduction=42493 max_load_reduction_pct=0.864522 moved_to_echo=25082 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 962, 0, 0, 0, 1355, 0, 1733, 1434, 431, 76, 0, 0, 0, 0, 0, 0, 136, 82, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 2468, 0, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 6078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 3484, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 77, 5823, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 0, 0, 470, 1950, 0, 0, 1348, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5991, 6162, 6211, 6134, 6155, 6134, 6146, 6219] avg=6144.0000 max=6219 max_rank=7 min=5991 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24654 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 7936, 0, 0, 0, 12032, 0, 13824, 11520, 3456, 640, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[768, 0, 0, 128, 0, 19200, 0, 128, 29312, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 18560, 0, 0, 3968, 16128, 128, 0, 10880, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 2048, 0, 768, 46592, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 1152, 640, 47488, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 3200, 0, 0, 128, 0, 0, 27904, 18304, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 768, 0, 0, 48512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 3200, 0, 0, 128, 0, 0, 27904, 18304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 1152, 640, 47488, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 57, 305, 0, 2252, 0, 169, 7912, 7873, 0, 0, 0, 0, 0, 884, 0, 7719, 0, 57, 0, 7531, 0, 1, 0, 0, 4404, 7491, 85, 0, 0, 0, 0, 0, 57, 0, 50, 0, 614, 0, 188, 0, 0, 0, 0, 0, 0, 0, 1, 227, 0, 355, 331] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 610, 0, 0, 37, 408, 0, 2551, 0, 214, 7823, 7758, 0, 1, 0, 0, 0, 1157, 0, 7683, 0, 38, 0, 7641, 1, 0, 0, 1, 3558, 7523, 107, 0, 0, 0, 1, 1, 37, 0, 66, 0, 574, 1, 201, 0, 1, 0, 0, 2, 0, 0, 8, 328, 2, 452, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 50, 576, 0, 2804, 0, 155, 7924, 7866, 0, 0, 0, 0, 0, 1952, 0, 7697, 0, 50, 0, 7694, 0, 1, 0, 0, 2367, 7573, 78, 0, 0, 0, 0, 0, 48, 0, 59, 0, 552, 0, 178, 0, 0, 0, 0, 0, 0, 2, 38, 91, 0, 510, 367] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 59, 614, 0, 2516, 0, 144, 7964, 7915, 0, 0, 0, 0, 0, 1163, 0, 7758, 0, 59, 0, 7700, 0, 0, 0, 0, 3465, 7502, 72, 0, 0, 0, 0, 0, 59, 0, 53, 0, 561, 0, 175, 0, 0, 0, 0, 0, 0, 0, 13, 77, 0, 426, 303] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 51, 626, 0, 2273, 0, 147, 7962, 7875, 0, 0, 0, 0, 0, 1971, 0, 7769, 0, 51, 0, 7768, 0, 1, 0, 0, 2705, 7475, 74, 0, 0, 0, 0, 0, 50, 0, 88, 0, 652, 0, 168, 0, 0, 0, 0, 0, 0, 1, 75, 82, 0, 431, 298] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 57, 305, 0, 2252, 0, 169, 7912, 7873, 0, 0, 0, 0, 0, 884, 0, 7719, 0, 57, 0, 7531, 0, 1, 0, 0, 4404, 7491, 85, 0, 0, 0, 0, 0, 57, 0, 50, 0, 614, 0, 188, 0, 0, 0, 0, 0, 0, 0, 1, 227, 0, 355, 331] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 50, 486, 0, 2174, 0, 129, 8009, 7940, 0, 0, 0, 0, 0, 2306, 0, 7742, 0, 50, 0, 7754, 0, 2, 0, 0, 2438, 7529, 65, 0, 0, 0, 0, 0, 49, 0, 71, 0, 671, 0, 131, 0, 0, 0, 0, 0, 0, 1, 60, 146, 0, 494, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 66, 249, 0, 2175, 0, 182, 7874, 7822, 0, 0, 0, 0, 0, 838, 0, 7718, 0, 66, 0, 7569, 0, 0, 0, 0, 4577, 7498, 91, 0, 0, 0, 0, 0, 64, 0, 54, 0, 645, 0, 225, 0, 0, 0, 0, 0, 0, 0, 3, 362, 0, 319, 317] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 59, 590, 0, 2774, 0, 154, 7951, 7886, 0, 0, 0, 0, 0, 1413, 0, 7692, 0, 59, 0, 7600, 0, 1, 0, 0, 3023, 7476, 77, 0, 0, 0, 0, 0, 56, 0, 66, 0, 493, 0, 168, 0, 0, 0, 0, 0, 0, 3, 44, 122, 0, 497, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 610, 0, 0, 37, 408, 0, 2551, 0, 214, 7823, 7758, 0, 1, 0, 0, 0, 1157, 0, 7683, 0, 38, 0, 7641, 1, 0, 0, 1, 3558, 7523, 107, 0, 0, 0, 1, 1, 37, 0, 66, 0, 574, 1, 201, 0, 1, 0, 0, 2, 0, 0, 8, 328, 2, 452, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 50, 486, 0, 2174, 0, 129, 8009, 7940, 0, 0, 0, 0, 0, 2306, 0, 7742, 0, 50, 0, 7754, 0, 2, 0, 0, 2438, 7529, 65, 0, 0, 0, 0, 0, 49, 0, 71, 0, 671, 0, 131, 0, 0, 0, 0, 0, 0, 1, 60, 146, 0, 494, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 51, 626, 0, 2273, 0, 147, 7962, 7875, 0, 0, 0, 0, 0, 1971, 0, 7769, 0, 51, 0, 7768, 0, 1, 0, 0, 2705, 7475, 74, 0, 0, 0, 0, 0, 50, 0, 88, 0, 652, 0, 168, 0, 0, 0, 0, 0, 0, 1, 75, 82, 0, 431, 298] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 57, 305, 0, 3091, 1970, 0, 0, 0, 0, 0, 0, 2252, 0, 169, 0, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 4628, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4404, 2740, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 50, 4174, 1781, 0, 0, 0, 0, 0, 0, 0, 614, 0, 188, 0, 0, 0, 0, 3357, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 227, 0, 355, 331, 2800, 2357, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6012, 5967, 5569, 7230, 6062, 6110, 6071] avg=6144.0000 max=7230 max_rank=4 min=5569 min_rank=3 imbalance_ratio=1.176758 max_load_reduction=41922 max_load_reduction_pct=0.852905 moved_to_echo=25473 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 66, 249, 0, 3090, 1958, 0, 0, 0, 0, 0, 0, 2175, 0, 182, 0, 3523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 4628, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4577, 2742, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 54, 4195, 1772, 0, 0, 0, 0, 0, 0, 0, 645, 0, 225, 0, 0, 0, 0, 3374, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 362, 0, 319, 317, 2803, 2341, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 5801, 5880, 5532, 7410, 6085, 6197, 6145] avg=6144.0000 max=7410 max_rank=4 min=5532 min_rank=3 imbalance_ratio=1.206055 max_load_reduction=41742 max_load_reduction_pct=0.849243 moved_to_echo=25837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2, 0, 0, 0, 6062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 37, 408, 0, 3076, 1942, 0, 0, 0, 0, 0, 0, 2551, 0, 214, 0, 3494, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 0, 4607, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3558, 2751, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 37, 0, 66, 4235, 1761, 0, 0, 0, 0, 0, 0, 0, 574, 1, 201, 0, 1, 0, 0, 3406, 1960, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 328, 2, 452, 364, 2812, 2322, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6065, 6073, 6260, 5802, 6418, 6101, 6143, 6290] avg=6144.0000 max=6418 max_rank=4 min=5802 min_rank=3 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24952 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 50, 576, 0, 3082, 1969, 0, 0, 0, 0, 0, 0, 2804, 0, 155, 0, 3543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1952, 0, 4615, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2367, 2769, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 59, 4264, 1783, 0, 0, 0, 0, 0, 0, 0, 552, 0, 178, 0, 0, 0, 0, 3430, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 2, 38, 91, 0, 510, 367, 2831, 2354, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6197, 6502, 6617, 5215, 6154, 6133, 6193] avg=6144.0000 max=6617 max_rank=3 min=5215 min_rank=4 imbalance_ratio=1.076986 max_load_reduction=42535 max_load_reduction_pct=0.865377 moved_to_echo=23695 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 66, 249, 0, 3090, 1958, 0, 0, 0, 0, 0, 0, 2175, 0, 182, 0, 3523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 4628, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4577, 2742, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 54, 4195, 1772, 0, 0, 0, 0, 0, 0, 0, 645, 0, 225, 0, 0, 0, 0, 3374, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 362, 0, 319, 317, 2803, 2341, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 5801, 5880, 5532, 7410, 6085, 6197, 6145] avg=6144.0000 max=7410 max_rank=4 min=5532 min_rank=3 imbalance_ratio=1.206055 max_load_reduction=41742 max_load_reduction_pct=0.849243 moved_to_echo=25837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 59, 614, 0, 3106, 1981, 0, 0, 0, 0, 0, 0, 2516, 0, 144, 0, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 0, 4652, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 2744, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 53, 4268, 1791, 0, 0, 0, 0, 0, 0, 0, 561, 0, 175, 0, 0, 0, 0, 3432, 1954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 77, 0, 426, 303, 2804, 2369, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6173, 6314, 6225, 5874, 6281, 6171, 6122, 5992] avg=6144.0000 max=6314 max_rank=1 min=5874 min_rank=3 imbalance_ratio=1.027669 max_load_reduction=42838 max_load_reduction_pct=0.871541 moved_to_echo=24566 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 57, 305, 0, 3091, 1970, 0, 0, 0, 0, 0, 0, 2252, 0, 169, 0, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 4628, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4404, 2740, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 50, 4174, 1781, 0, 0, 0, 0, 0, 0, 0, 614, 0, 188, 0, 0, 0, 0, 3357, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 227, 0, 355, 331, 2800, 2357, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6012, 5967, 5569, 7230, 6062, 6110, 6071] avg=6144.0000 max=7230 max_rank=4 min=5569 min_rank=3 imbalance_ratio=1.176758 max_load_reduction=41922 max_load_reduction_pct=0.852905 moved_to_echo=25473 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 51, 626, 0, 3111, 1971, 0, 0, 0, 0, 0, 0, 2273, 0, 147, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 0, 4658, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2705, 2734, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 88, 4306, 1791, 0, 0, 0, 0, 0, 0, 0, 652, 0, 168, 0, 0, 0, 0, 3462, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 1, 75, 82, 0, 431, 298, 2794, 2357, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6171, 6318, 5967, 6680, 5514, 6235, 6229, 6038] avg=6144.0000 max=6680 max_rank=3 min=5514 min_rank=4 imbalance_ratio=1.087240 max_load_reduction=42472 max_load_reduction_pct=0.864095 moved_to_echo=24016 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 50, 486, 0, 3100, 1987, 0, 0, 0, 0, 0, 0, 2174, 0, 129, 0, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 0, 4642, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2438, 2753, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 71, 4298, 1802, 0, 0, 0, 0, 0, 0, 0, 671, 0, 131, 0, 0, 0, 0, 3456, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 1, 60, 146, 0, 494, 335, 2815, 2377, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6207, 6143, 5879, 6998, 5258, 6220, 6219, 6228] avg=6144.0000 max=6998 max_rank=3 min=5258 min_rank=4 imbalance_ratio=1.138997 max_load_reduction=42154 max_load_reduction_pct=0.857625 moved_to_echo=23925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 59, 590, 0, 3084, 1977, 0, 0, 0, 0, 0, 0, 2774, 0, 154, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1413, 0, 4608, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3023, 2725, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 66, 4213, 1789, 0, 0, 0, 0, 0, 0, 0, 493, 0, 168, 0, 0, 0, 0, 3387, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 3, 44, 122, 0, 497, 364, 2800, 2365, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6162, 6294, 6472, 6080, 5826, 6124, 5999, 6195] avg=6144.0000 max=6472 max_rank=2 min=5826 min_rank=4 imbalance_ratio=1.053385 max_load_reduction=42680 max_load_reduction_pct=0.868327 moved_to_echo=24144 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2, 0, 0, 0, 6062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 37, 408, 0, 3076, 1942, 0, 0, 0, 0, 0, 0, 2551, 0, 214, 0, 3494, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 0, 4607, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3558, 2751, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 37, 0, 66, 4235, 1761, 0, 0, 0, 0, 0, 0, 0, 574, 1, 201, 0, 1, 0, 0, 3406, 1960, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 328, 2, 452, 364, 2812, 2322, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6065, 6073, 6260, 5802, 6418, 6101, 6143, 6290] avg=6144.0000 max=6418 max_rank=4 min=5802 min_rank=3 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24952 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 51, 626, 0, 3111, 1971, 0, 0, 0, 0, 0, 0, 2273, 0, 147, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 0, 4658, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2705, 2734, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 88, 4306, 1791, 0, 0, 0, 0, 0, 0, 0, 652, 0, 168, 0, 0, 0, 0, 3462, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 1, 75, 82, 0, 431, 298, 2794, 2357, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6171, 6318, 5967, 6680, 5514, 6235, 6229, 6038] avg=6144.0000 max=6680 max_rank=3 min=5514 min_rank=4 imbalance_ratio=1.087240 max_load_reduction=42472 max_load_reduction_pct=0.864095 moved_to_echo=24016 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 256, 1536, 128, 3584, 2688, 22528, 18944, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 4480, 0, 0, 512, 3968, 0, 24832, 15872, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 256, 1536, 128, 3584, 2688, 22528, 18944, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 26624, 22016, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[19584, 0, 1408, 0, 28416, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 11776, 0, 37120, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 4864, 128, 1536, 0, 128, 0, 0, 27392, 15744, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 512, 0, 512, 34048, 14336, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1127, 0, 0, 0, 0, 0, 0, 0, 84, 0, 84, 1396, 0, 0, 0, 663, 2291, 8103, 1022, 0, 84, 7764, 0, 0, 7582, 157, 57, 0, 0, 0, 0, 84, 0, 0, 0, 0, 1436, 0, 4295, 0, 0, 0, 0, 79, 0, 0, 0, 0, 2743, 6643, 0, 471, 157, 2466, 84, 0, 0, 0, 278, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1251, 0, 0, 0, 0, 0, 0, 0, 91, 0, 90, 1161, 0, 0, 0, 637, 2351, 8100, 972, 0, 90, 7880, 0, 0, 7573, 201, 66, 0, 0, 0, 0, 91, 0, 0, 0, 0, 1392, 0, 4245, 0, 0, 0, 0, 86, 0, 0, 0, 0, 3046, 6616, 0, 336, 200, 2309, 91, 0, 0, 0, 277, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 779, 0, 0, 0, 0, 0, 0, 0, 62, 0, 65, 1329, 0, 0, 0, 238, 1928, 8075, 911, 0, 27, 7686, 0, 0, 7587, 84, 50, 0, 0, 0, 0, 70, 0, 0, 0, 0, 1132, 0, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1894, 6723, 0, 1075, 87, 3088, 64, 0, 0, 0, 218, 36, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1129, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 1195, 0, 0, 0, 416, 1818, 8113, 941, 0, 54, 7624, 0, 0, 7551, 153, 50, 0, 0, 0, 0, 80, 0, 0, 0, 0, 1034, 0, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1944, 6752, 0, 1034, 152, 2812, 77, 0, 0, 0, 222, 85, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1158, 0, 0, 0, 0, 0, 0, 0, 72, 0, 71, 1276, 0, 0, 0, 495, 2193, 8120, 962, 0, 68, 7706, 0, 0, 7633, 137, 59, 0, 0, 0, 0, 72, 0, 0, 0, 0, 954, 0, 5064, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2608, 6766, 0, 697, 139, 2527, 72, 0, 0, 0, 258, 31, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 1, 1339, 0, 0, 0, 0, 0, 0, 0, 107, 0, 107, 933, 0, 0, 0, 739, 2198, 8079, 920, 1, 107, 7852, 1, 0, 7505, 191, 37, 0, 0, 0, 2, 109, 2, 1, 0, 0, 976, 1, 4848, 0, 2, 0, 0, 29, 0, 0, 0, 0, 2384, 6677, 0, 693, 188, 2718, 108, 0, 0, 2, 246, 49, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1127, 0, 0, 0, 0, 0, 0, 0, 84, 0, 84, 1396, 0, 0, 0, 663, 2291, 8103, 1022, 0, 84, 7764, 0, 0, 7582, 157, 57, 0, 0, 0, 0, 84, 0, 0, 0, 0, 1436, 0, 4295, 0, 0, 0, 0, 79, 0, 0, 0, 0, 2743, 6643, 0, 471, 157, 2466, 84, 0, 0, 0, 278, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1156, 0, 0, 0, 0, 0, 0, 0, 77, 0, 77, 1400, 0, 0, 0, 419, 1934, 8114, 1029, 0, 64, 7635, 0, 0, 7545, 145, 59, 0, 0, 0, 0, 77, 0, 0, 0, 0, 951, 0, 5271, 0, 0, 0, 0, 19, 0, 0, 0, 0, 2596, 6604, 0, 707, 141, 2753, 77, 0, 0, 0, 256, 46, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 74, 0, 74, 1461, 0, 0, 0, 330, 2082, 8117, 856, 0, 48, 7712, 0, 0, 7607, 131, 51, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1352, 0, 5261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1897, 6552, 0, 958, 129, 2980, 73, 0, 0, 0, 256, 12, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1129, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 1195, 0, 0, 0, 416, 1818, 8113, 941, 0, 54, 7624, 0, 0, 7551, 153, 50, 0, 0, 0, 0, 80, 0, 0, 0, 0, 1034, 0, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1944, 6752, 0, 1034, 152, 2812, 77, 0, 0, 0, 222, 85, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 1, 1339, 0, 0, 0, 0, 0, 0, 0, 107, 0, 107, 933, 0, 0, 0, 739, 2198, 8079, 920, 1, 107, 7852, 1, 0, 7505, 191, 37, 0, 0, 0, 2, 109, 2, 1, 0, 0, 976, 1, 4848, 0, 2, 0, 0, 29, 0, 0, 0, 0, 2384, 6677, 0, 693, 188, 2718, 108, 0, 0, 2, 246, 49, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1127, 0, 0, 0, 0, 5038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 84, 1396, 0, 3375, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 663, 2291, 2535, 1022, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5949, 157, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 1436, 2726, 2193, 0, 0, 0, 0, 0, 0, 0, 4295, 0, 0, 0, 0, 79, 0, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2743, 2850, 0, 471, 157, 0, 0, 0, 0, 0, 0, 0, 0, 2466, 84, 0, 0, 0, 278, 2, 0, 2460, 603, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6165, 6272, 6595, 6163, 6439, 5404, 6221, 5893] avg=6144.0000 max=6595 max_rank=2 min=5404 min_rank=5 imbalance_ratio=1.073405 max_load_reduction=42557 max_load_reduction_pct=0.865824 moved_to_echo=23957 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1251, 0, 0, 0, 0, 5114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 90, 1161, 0, 3374, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 637, 2351, 2533, 972, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5941, 201, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 1392, 2766, 2193, 0, 0, 0, 0, 0, 0, 0, 4245, 0, 0, 0, 0, 86, 0, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3046, 2839, 0, 336, 200, 0, 0, 0, 0, 0, 0, 0, 0, 2309, 91, 0, 0, 0, 277, 0, 0, 2450, 603, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6365, 6043, 6583, 6208, 6442, 5360, 6421, 5730] avg=6144.0000 max=6583 max_rank=2 min=5360 min_rank=5 imbalance_ratio=1.071452 max_load_reduction=42569 max_load_reduction_pct=0.866069 moved_to_echo=23953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 779, 0, 0, 0, 0, 4988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 65, 1329, 0, 3364, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 238, 1928, 2525, 911, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5952, 84, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 1132, 2698, 2186, 0, 0, 0, 0, 0, 0, 0, 5944, 0, 0, 0, 0, 0, 0, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1894, 2884, 0, 1075, 87, 0, 0, 0, 0, 0, 0, 0, 0, 3088, 64, 0, 0, 0, 218, 36, 0, 2490, 604, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5767, 6169, 5629, 6086, 6086, 6975, 5940, 6500] avg=6144.0000 max=6975 max_rank=5 min=5629 min_rank=2 imbalance_ratio=1.135254 max_load_reduction=42177 max_load_reduction_pct=0.858093 moved_to_echo=25501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1158, 0, 0, 0, 0, 5002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 71, 1276, 0, 3382, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 495, 2193, 2540, 962, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5988, 137, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 954, 2704, 2198, 0, 0, 0, 0, 0, 0, 0, 5064, 0, 0, 0, 0, 14, 0, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2608, 2904, 0, 697, 139, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 72, 0, 0, 0, 258, 31, 0, 2505, 608, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6158, 6258, 6184, 5928, 6115, 6348, 6001] avg=6144.0000 max=6348 max_rank=6 min=5928 min_rank=4 imbalance_ratio=1.033203 max_load_reduction=42804 max_load_reduction_pct=0.870850 moved_to_echo=24392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1251, 0, 0, 0, 0, 0, 0, 0, 91, 0, 90, 1161, 0, 0, 0, 637, 2351, 8100, 972, 0, 90, 7880, 0, 0, 7573, 201, 66, 0, 0, 0, 0, 91, 0, 0, 0, 0, 1392, 0, 4245, 0, 0, 0, 0, 86, 0, 0, 0, 0, 3046, 6616, 0, 336, 200, 2309, 91, 0, 0, 0, 277, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 779, 0, 0, 0, 0, 0, 0, 0, 62, 0, 65, 1329, 0, 0, 0, 238, 1928, 8075, 911, 0, 27, 7686, 0, 0, 7587, 84, 50, 0, 0, 0, 0, 70, 0, 0, 0, 0, 1132, 0, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1894, 6723, 0, 1075, 87, 3088, 64, 0, 0, 0, 218, 36, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1129, 0, 0, 0, 0, 4947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 1195, 0, 3380, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 416, 1818, 2537, 941, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5924, 153, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 1034, 2677, 2196, 0, 0, 0, 0, 0, 0, 0, 5760, 0, 0, 0, 0, 0, 0, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1944, 2897, 0, 1034, 152, 0, 0, 0, 0, 0, 0, 0, 0, 2812, 77, 0, 0, 0, 222, 85, 0, 2500, 601, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 6086, 5766, 6127, 5987, 6786, 6027, 6297] avg=6144.0000 max=6786 max_rank=5 min=5766 min_rank=2 imbalance_ratio=1.104492 max_load_reduction=42366 max_load_reduction_pct=0.861938 moved_to_echo=25097 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1156, 0, 0, 0, 0, 4956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 77, 1400, 0, 3385, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 419, 1934, 2528, 1029, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5912, 145, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 951, 2679, 2201, 0, 0, 0, 0, 0, 0, 0, 5271, 0, 0, 0, 0, 19, 0, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 2825, 0, 707, 141, 0, 0, 0, 0, 0, 0, 0, 0, 2753, 77, 0, 0, 0, 256, 46, 0, 2450, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6268, 5974, 6116, 5908, 6318, 6269, 6187] avg=6144.0000 max=6318 max_rank=5 min=5908 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=42834 max_load_reduction_pct=0.871460 moved_to_echo=24682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1339, 0, 0, 0, 0, 5095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 107, 933, 0, 3365, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 739, 2198, 2527, 920, 1, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5888, 191, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 109, 2, 1, 0, 0, 976, 2757, 2187, 0, 0, 0, 0, 0, 0, 1, 4848, 0, 2, 0, 0, 29, 0, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 2865, 0, 693, 188, 0, 0, 0, 0, 0, 0, 0, 0, 2718, 108, 0, 0, 2, 246, 49, 0, 2472, 597, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6435, 5852, 6492, 6117, 6034, 5900, 6130, 6192] avg=6144.0000 max=6492 max_rank=2 min=5852 min_rank=1 imbalance_ratio=1.056641 max_load_reduction=42660 max_load_reduction_pct=0.867920 moved_to_echo=24256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1251, 0, 0, 0, 0, 5114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 90, 1161, 0, 3374, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 637, 2351, 2533, 972, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5941, 201, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 1392, 2766, 2193, 0, 0, 0, 0, 0, 0, 0, 4245, 0, 0, 0, 0, 86, 0, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3046, 2839, 0, 336, 200, 0, 0, 0, 0, 0, 0, 0, 0, 2309, 91, 0, 0, 0, 277, 0, 0, 2450, 603, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6365, 6043, 6583, 6208, 6442, 5360, 6421, 5730] avg=6144.0000 max=6583 max_rank=2 min=5360 min_rank=5 imbalance_ratio=1.071452 max_load_reduction=42569 max_load_reduction_pct=0.866069 moved_to_echo=23953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1158, 0, 0, 0, 0, 5002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 71, 1276, 0, 3382, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 495, 2193, 2540, 962, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5988, 137, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 954, 2704, 2198, 0, 0, 0, 0, 0, 0, 0, 5064, 0, 0, 0, 0, 14, 0, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2608, 2904, 0, 697, 139, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 72, 0, 0, 0, 258, 31, 0, 2505, 608, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6158, 6258, 6184, 5928, 6115, 6348, 6001] avg=6144.0000 max=6348 max_rank=6 min=5928 min_rank=4 imbalance_ratio=1.033203 max_load_reduction=42804 max_load_reduction_pct=0.870850 moved_to_echo=24392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1127, 0, 0, 0, 0, 5038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 84, 1396, 0, 3375, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 663, 2291, 2535, 1022, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5949, 157, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 1436, 2726, 2193, 0, 0, 0, 0, 0, 0, 0, 4295, 0, 0, 0, 0, 79, 0, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2743, 2850, 0, 471, 157, 0, 0, 0, 0, 0, 0, 0, 0, 2466, 84, 0, 0, 0, 278, 2, 0, 2460, 603, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6165, 6272, 6595, 6163, 6439, 5404, 6221, 5893] avg=6144.0000 max=6595 max_rank=2 min=5404 min_rank=5 imbalance_ratio=1.073405 max_load_reduction=42557 max_load_reduction_pct=0.865824 moved_to_echo=23957 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1066, 0, 0, 0, 0, 5006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 74, 1461, 0, 3381, 1314, 0, 0, 0, 0, 0, 0, 0, 0, 330, 2082, 2539, 856, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 131, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1352, 2706, 2197, 0, 0, 0, 0, 0, 0, 0, 5261, 0, 0, 0, 0, 0, 0, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1897, 2812, 0, 958, 129, 0, 0, 0, 0, 0, 0, 0, 0, 2980, 73, 0, 0, 0, 256, 12, 0, 2426, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 6304, 5855, 6151, 6328, 6294, 5796, 6352] avg=6144.0000 max=6352 max_rank=7 min=5796 min_rank=6 imbalance_ratio=1.033854 max_load_reduction=42800 max_load_reduction_pct=0.870768 moved_to_echo=24770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 779, 0, 0, 0, 0, 4988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 65, 1329, 0, 3364, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 238, 1928, 2525, 911, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5952, 84, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 1132, 2698, 2186, 0, 0, 0, 0, 0, 0, 0, 5944, 0, 0, 0, 0, 0, 0, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1894, 2884, 0, 1075, 87, 0, 0, 0, 0, 0, 0, 0, 0, 3088, 64, 0, 0, 0, 218, 36, 0, 2490, 604, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5767, 6169, 5629, 6086, 6086, 6975, 5940, 6500] avg=6144.0000 max=6975 max_rank=5 min=5629 min_rank=2 imbalance_ratio=1.135254 max_load_reduction=42177 max_load_reduction_pct=0.858093 moved_to_echo=25501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1156, 0, 0, 0, 0, 4956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 77, 1400, 0, 3385, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 419, 1934, 2528, 1029, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5912, 145, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 951, 2679, 2201, 0, 0, 0, 0, 0, 0, 0, 5271, 0, 0, 0, 0, 19, 0, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 2825, 0, 707, 141, 0, 0, 0, 0, 0, 0, 0, 0, 2753, 77, 0, 0, 0, 256, 46, 0, 2450, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6268, 5974, 6116, 5908, 6318, 6269, 6187] avg=6144.0000 max=6318 max_rank=5 min=5908 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=42834 max_load_reduction_pct=0.871460 moved_to_echo=24682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 768, 0, 768, 10240, 0, 27008, 10752, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 19200, 22912, 0, 6016, 1280, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 3968, 16896, 20352, 7680, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[21760, 768, 0, 0, 128, 2048, 384, 0, 19840, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 19200, 22912, 0, 6016, 1280, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 768, 128, 128, 0, 0, 9344, 21760, 17664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 9088, 0, 0, 0, 0, 40192, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[21760, 768, 0, 0, 128, 2048, 384, 0, 19840, 4864, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 40704, 0, 128, 0, 0, 256, 0, 8320, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 9088, 0, 0, 0, 0, 40192, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 47616, 1280, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 4362, 405, 1611, 7990, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 623, 0, 0, 0, 0, 253, 6873, 77, 0, 0, 0, 0, 0, 2872, 0, 0, 4328, 7438, 59, 0, 0, 402, 0, 0, 0, 0, 0, 0, 879, 0, 0, 0, 0, 82, 7858, 77, 1821, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 4914, 411, 1397, 8001, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 0, 0, 578, 0, 0, 0, 0, 283, 6811, 86, 0, 0, 0, 0, 0, 2223, 0, 0, 5552, 7140, 57, 0, 0, 247, 0, 0, 0, 0, 0, 0, 819, 0, 0, 0, 0, 88, 7840, 84, 1195, 0, 0, 15, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 4673, 406, 1500, 7978, 636, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 597, 0, 1, 572, 0, 0, 0, 0, 309, 6892, 104, 0, 1, 0, 0, 0, 2404, 0, 0, 4957, 7234, 39, 0, 0, 313, 0, 2, 1, 0, 1, 0, 1005, 1, 0, 0, 2, 49, 7795, 107, 1562, 0, 2, 3, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 4130, 394, 1460, 8006, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 649, 0, 0, 0, 0, 308, 7019, 77, 0, 0, 0, 0, 0, 3624, 0, 0, 3507, 7493, 50, 0, 0, 462, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 66, 7870, 77, 2256, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 4339, 425, 1310, 8008, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 622, 0, 0, 0, 0, 269, 6931, 72, 0, 0, 0, 0, 0, 2763, 0, 0, 5010, 7422, 59, 0, 0, 327, 0, 0, 0, 0, 0, 0, 744, 0, 0, 0, 0, 95, 7852, 72, 1627, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 4727, 435, 1405, 7981, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 0, 0, 581, 0, 0, 0, 0, 310, 6686, 103, 0, 0, 0, 0, 0, 2231, 0, 0, 5545, 7215, 66, 0, 0, 431, 0, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 94, 7784, 91, 1328, 0, 0, 65, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 3979, 387, 1354, 8007, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 798, 0, 0, 747, 0, 0, 0, 0, 223, 6973, 64, 0, 0, 0, 0, 0, 3729, 0, 0, 3586, 7635, 50, 0, 0, 145, 0, 0, 0, 0, 0, 0, 691, 0, 0, 0, 0, 79, 7933, 64, 2181, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 4041, 403, 1408, 7980, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 0, 0, 716, 0, 0, 0, 0, 269, 6885, 73, 0, 0, 0, 0, 0, 3526, 0, 0, 4387, 7525, 51, 0, 0, 226, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 0, 86, 7886, 73, 1549, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4157, 411, 1397, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 0, 0, 578, 0, 0, 3299, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 283, 5723, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 0, 3817, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 3841, 1852, 0, 0, 0, 0, 0, 0, 0, 819, 0, 0, 0, 0, 88, 5255, 0, 0, 0, 0, 0, 0, 0, 0, 84, 1195, 0, 0, 15, 0, 0, 0, 1735, 1143, 1088, 757, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6574, 6149, 6121, 6092, 6097, 5940, 6162, 6017] avg=6144.0000 max=6574 max_rank=0 min=5940 min_rank=5 imbalance_ratio=1.069987 max_load_reduction=42578 max_load_reduction_pct=0.866252 moved_to_echo=24216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 4914, 411, 1397, 8001, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 0, 0, 578, 0, 0, 0, 0, 283, 6811, 86, 0, 0, 0, 0, 0, 2223, 0, 0, 5552, 7140, 57, 0, 0, 247, 0, 0, 0, 0, 0, 0, 819, 0, 0, 0, 0, 88, 7840, 84, 1195, 0, 0, 15, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 4339, 425, 1310, 8008, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 622, 0, 0, 0, 0, 269, 6931, 72, 0, 0, 0, 0, 0, 2763, 0, 0, 5010, 7422, 59, 0, 0, 327, 0, 0, 0, 0, 0, 0, 744, 0, 0, 0, 0, 95, 7852, 72, 1627, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 4130, 394, 1460, 8006, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 649, 0, 0, 0, 0, 308, 7019, 77, 0, 0, 0, 0, 0, 3624, 0, 0, 3507, 7493, 50, 0, 0, 462, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 66, 7870, 77, 2256, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 4673, 406, 1500, 7978, 636, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 597, 0, 1, 572, 0, 0, 0, 0, 309, 6892, 104, 0, 1, 0, 0, 0, 2404, 0, 0, 4957, 7234, 39, 0, 0, 313, 0, 2, 1, 0, 1, 0, 1005, 1, 0, 0, 2, 49, 7795, 107, 1562, 0, 2, 3, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 4362, 405, 1611, 7990, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 623, 0, 0, 0, 0, 253, 6873, 77, 0, 0, 0, 0, 0, 2872, 0, 0, 4328, 7438, 59, 0, 0, 402, 0, 0, 0, 0, 0, 0, 879, 0, 0, 0, 0, 82, 7858, 77, 1821, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3999, 435, 1405, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 0, 0, 581, 0, 0, 3333, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 310, 5618, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2231, 0, 0, 3813, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 3882, 1847, 0, 0, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 94, 5218, 0, 0, 0, 0, 0, 0, 0, 0, 91, 1328, 0, 0, 65, 0, 0, 0, 1732, 1135, 1068, 728, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6429, 6134, 6012, 6031, 6110, 6160, 6129, 6147] avg=6144.0000 max=6429 max_rank=0 min=6012 min_rank=2 imbalance_ratio=1.046387 max_load_reduction=42723 max_load_reduction_pct=0.869202 moved_to_echo=24546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3494, 394, 1460, 0, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 649, 0, 0, 3462, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 308, 5898, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 2412, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 4031, 1854, 0, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 66, 5276, 0, 0, 0, 0, 0, 0, 0, 0, 77, 2256, 0, 0, 0, 0, 0, 0, 1095, 1147, 1121, 636, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5888, 6152, 5938, 6283, 6086, 6347, 6126, 6332] avg=6144.0000 max=6347 max_rank=5 min=5888 min_rank=0 imbalance_ratio=1.033040 max_load_reduction=42805 max_load_reduction_pct=0.870870 moved_to_echo=24891 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3671, 425, 1310, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 622, 0, 0, 3430, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 269, 5824, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2763, 0, 0, 3445, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 3992, 1853, 0, 0, 0, 0, 0, 0, 0, 744, 0, 0, 0, 0, 95, 5263, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1627, 0, 0, 0, 0, 0, 0, 1565, 1145, 1107, 668, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5976, 6155, 6131, 6165, 6267, 6172, 6102, 6184] avg=6144.0000 max=6267 max_rank=4 min=5976 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=42885 max_load_reduction_pct=0.872498 moved_to_echo=24725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3687, 405, 1611, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 623, 0, 0, 3437, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 253, 5773, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 2971, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 4001, 1849, 0, 0, 0, 0, 0, 0, 0, 879, 0, 0, 0, 0, 82, 5259, 0, 0, 0, 0, 0, 0, 0, 0, 77, 1821, 0, 0, 0, 0, 0, 0, 1357, 1149, 1100, 675, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6351, 6141, 6004, 6103, 5902, 6252, 6220, 6179] avg=6144.0000 max=6351 max_rank=0 min=5902 min_rank=4 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3366, 387, 1354, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 798, 0, 0, 747, 0, 0, 3528, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 223, 5859, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 0, 0, 2466, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 4107, 1853, 0, 0, 0, 0, 0, 0, 0, 691, 0, 0, 0, 0, 79, 5317, 0, 0, 0, 0, 0, 0, 0, 0, 64, 2181, 0, 0, 0, 0, 0, 0, 1120, 1157, 1114, 613, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5634, 6154, 6532, 6146, 6245, 6105, 6087, 6249] avg=6144.0000 max=6532 max_rank=2 min=5634 min_rank=0 imbalance_ratio=1.063151 max_load_reduction=42620 max_load_reduction_pct=0.867106 moved_to_echo=24686 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3671, 425, 1310, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 622, 0, 0, 3430, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 269, 5824, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2763, 0, 0, 3445, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 3992, 1853, 0, 0, 0, 0, 0, 0, 0, 744, 0, 0, 0, 0, 95, 5263, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1627, 0, 0, 0, 0, 0, 0, 1565, 1145, 1107, 668, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5976, 6155, 6131, 6165, 6267, 6172, 6102, 6184] avg=6144.0000 max=6267 max_rank=4 min=5976 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=42885 max_load_reduction_pct=0.872498 moved_to_echo=24725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4157, 411, 1397, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 0, 0, 578, 0, 0, 3299, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 283, 5723, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 0, 3817, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 3841, 1852, 0, 0, 0, 0, 0, 0, 0, 819, 0, 0, 0, 0, 88, 5255, 0, 0, 0, 0, 0, 0, 0, 0, 84, 1195, 0, 0, 15, 0, 0, 0, 1735, 1143, 1088, 757, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6574, 6149, 6121, 6092, 6097, 5940, 6162, 6017] avg=6144.0000 max=6574 max_rank=0 min=5940 min_rank=5 imbalance_ratio=1.069987 max_load_reduction=42578 max_load_reduction_pct=0.866252 moved_to_echo=24216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3494, 394, 1460, 0, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 649, 0, 0, 3462, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 308, 5898, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 2412, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 4031, 1854, 0, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 66, 5276, 0, 0, 0, 0, 0, 0, 0, 0, 77, 2256, 0, 0, 0, 0, 0, 0, 1095, 1147, 1121, 636, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5888, 6152, 5938, 6283, 6086, 6347, 6126, 6332] avg=6144.0000 max=6347 max_rank=5 min=5888 min_rank=0 imbalance_ratio=1.033040 max_load_reduction=42805 max_load_reduction_pct=0.870870 moved_to_echo=24891 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3954, 406, 1500, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 6131, 0, 0, 0, 0, 0, 0, 0, 2, 0, 597, 0, 1, 572, 0, 0, 3342, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 309, 5791, 104, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2404, 0, 0, 3408, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 2, 1, 0, 1, 3892, 1847, 0, 0, 0, 0, 0, 0, 0, 1005, 1, 0, 0, 2, 49, 5226, 0, 0, 0, 0, 0, 0, 0, 0, 107, 1562, 0, 2, 3, 0, 0, 1, 1549, 1136, 1101, 719, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6496, 6134, 5947, 6205, 5851, 6056, 6283, 6180] avg=6144.0000 max=6496 max_rank=0 min=5851 min_rank=4 imbalance_ratio=1.057292 max_load_reduction=42656 max_load_reduction_pct=0.867839 moved_to_echo=24370 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3419, 403, 1408, 0, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 0, 0, 716, 0, 0, 3478, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 269, 5785, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3526, 0, 0, 3017, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 4047, 1847, 0, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 0, 86, 5286, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1549, 0, 0, 0, 0, 0, 0, 1370, 1150, 1100, 622, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5804, 6133, 6467, 6127, 6594, 6120, 6043, 5864] avg=6144.0000 max=6594 max_rank=4 min=5804 min_rank=0 imbalance_ratio=1.073242 max_load_reduction=42558 max_load_reduction_pct=0.865845 moved_to_echo=24621 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3687, 405, 1611, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 623, 0, 0, 3437, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 253, 5773, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 2971, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 4001, 1849, 0, 0, 0, 0, 0, 0, 0, 879, 0, 0, 0, 0, 82, 5259, 0, 0, 0, 0, 0, 0, 0, 0, 77, 1821, 0, 0, 0, 0, 0, 0, 1357, 1149, 1100, 675, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6351, 6141, 6004, 6103, 5902, 6252, 6220, 6179] avg=6144.0000 max=6351 max_rank=0 min=5902 min_rank=4 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 13568, 0, 128, 128, 0, 0, 128, 11648, 9216, 8832, 5504, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 5248, 0, 128, 5120, 0, 0, 27392, 11648, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 2560, 0, 128, 128, 0, 128, 31872, 14848, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 5248, 0, 128, 5120, 0, 0, 27392, 11648, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 23424, 0, 0, 25472, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 29824, 3328, 11520, 0, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 6528, 128, 0, 0, 128, 640, 42112, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 2304, 46336, 768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[7701, 0, 0, 0, 0, 0, 0, 1380, 503, 0, 360, 0, 0, 0, 0, 906, 0, 0, 0, 5380, 282, 396, 0, 5097, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 7349, 0, 0, 0, 7324, 0, 0, 51, 0, 0, 0, 0, 2389, 0, 7849, 0, 0, 0, 0, 0, 1309, 172, 0, 0, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[7809, 0, 0, 0, 0, 0, 0, 1033, 443, 0, 582, 0, 0, 0, 0, 829, 0, 0, 0, 3429, 191, 380, 0, 5962, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 7829, 0, 0, 0, 7310, 0, 0, 71, 0, 0, 0, 0, 2816, 0, 7544, 0, 0, 0, 0, 0, 2141, 143, 0, 0, 78] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[7811, 0, 0, 0, 0, 0, 0, 1223, 508, 0, 491, 0, 0, 0, 0, 880, 0, 0, 0, 4167, 244, 384, 0, 5492, 0, 0, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 7545, 0, 0, 0, 7305, 0, 0, 69, 0, 0, 0, 0, 2675, 0, 7812, 0, 0, 0, 0, 0, 1670, 159, 0, 0, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[7628, 0, 0, 0, 0, 0, 0, 1199, 479, 0, 416, 0, 0, 0, 0, 878, 0, 0, 0, 5428, 244, 449, 0, 5465, 0, 0, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 7212, 0, 0, 0, 7395, 0, 0, 54, 0, 0, 0, 0, 2340, 0, 7800, 0, 0, 0, 0, 0, 1216, 185, 0, 0, 91] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[7814, 0, 0, 0, 0, 0, 0, 976, 481, 0, 538, 0, 0, 0, 0, 861, 0, 0, 0, 3856, 278, 393, 0, 6020, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 7525, 0, 0, 0, 7316, 0, 0, 54, 0, 0, 0, 0, 2778, 0, 7841, 0, 0, 0, 0, 0, 1616, 167, 0, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[7882, 0, 0, 0, 0, 0, 0, 879, 452, 0, 603, 0, 0, 0, 0, 805, 0, 0, 0, 3238, 209, 270, 0, 6251, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 7814, 0, 0, 0, 7325, 0, 0, 83, 0, 0, 0, 0, 2804, 0, 7568, 0, 0, 0, 0, 0, 2301, 143, 0, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[7722, 1, 0, 0, 0, 0, 0, 944, 512, 2, 522, 0, 0, 0, 0, 933, 0, 2, 0, 4703, 283, 434, 0, 5508, 0, 0, 0, 0, 0, 0, 0, 0, 656, 1, 0, 0, 0, 0, 0, 7713, 0, 2, 0, 7311, 0, 2, 68, 1, 0, 0, 0, 2146, 1, 7783, 0, 1, 0, 0, 0, 1637, 157, 0, 0, 107] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[7628, 0, 0, 0, 0, 0, 0, 1199, 479, 0, 416, 0, 0, 0, 0, 878, 0, 0, 0, 5428, 244, 449, 0, 5465, 0, 0, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 7212, 0, 0, 0, 7395, 0, 0, 54, 0, 0, 0, 0, 2340, 0, 7800, 0, 0, 0, 0, 0, 1216, 185, 0, 0, 91] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[7826, 0, 0, 0, 0, 0, 0, 832, 490, 0, 510, 0, 0, 0, 0, 870, 0, 0, 0, 3224, 240, 355, 0, 6564, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 7522, 0, 0, 0, 7309, 0, 0, 91, 0, 0, 0, 0, 2887, 0, 7622, 0, 0, 0, 0, 0, 2069, 159, 0, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5038, 0, 0, 0, 0, 0, 0, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 360, 0, 0, 0, 0, 906, 2541, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5380, 282, 396, 0, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 1712, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 5401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6076, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2389, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 172, 0, 0, 85, 1948, 1248, 895, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6418, 6078, 7234, 5633, 6020, 6127, 5985, 5657] avg=6144.0000 max=7234 max_rank=2 min=5633 min_rank=3 imbalance_ratio=1.177409 max_load_reduction=41918 max_load_reduction_pct=0.852824 moved_to_echo=23789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[7882, 0, 0, 0, 0, 0, 0, 879, 452, 0, 603, 0, 0, 0, 0, 805, 0, 0, 0, 3238, 209, 270, 0, 6251, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 7814, 0, 0, 0, 7325, 0, 0, 83, 0, 0, 0, 0, 2804, 0, 7568, 0, 0, 0, 0, 0, 2301, 143, 0, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[7701, 0, 0, 0, 0, 0, 0, 1380, 503, 0, 360, 0, 0, 0, 0, 906, 0, 0, 0, 5380, 282, 396, 0, 5097, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 7349, 0, 0, 0, 7324, 0, 0, 51, 0, 0, 0, 0, 2389, 0, 7849, 0, 0, 0, 0, 0, 1309, 172, 0, 0, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[7811, 0, 0, 0, 0, 0, 0, 1223, 508, 0, 491, 0, 0, 0, 0, 880, 0, 0, 0, 4167, 244, 384, 0, 5492, 0, 0, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 7545, 0, 0, 0, 7305, 0, 0, 69, 0, 0, 0, 0, 2675, 0, 7812, 0, 0, 0, 0, 0, 1670, 159, 0, 0, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5109, 0, 0, 0, 0, 0, 0, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 582, 0, 0, 0, 0, 829, 2442, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3429, 191, 380, 0, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4586, 1645, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 5754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6064, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2141, 143, 0, 0, 78, 2075, 1246, 907, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6089, 5376, 6231, 6316, 6135, 6273, 6590] avg=6144.0000 max=6590 max_rank=7 min=5376 min_rank=2 imbalance_ratio=1.072591 max_load_reduction=42562 max_load_reduction_pct=0.865926 moved_to_echo=25314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5112, 0, 0, 0, 0, 0, 0, 976, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 538, 0, 0, 0, 0, 861, 2539, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 278, 393, 0, 1389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4631, 1710, 0, 0, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 5530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6069, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2778, 0, 3592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 167, 0, 0, 72, 1995, 1247, 908, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6088, 6213, 5916, 6341, 6096, 6123, 6370, 6005] avg=6144.0000 max=6370 max_rank=6 min=5916 min_rank=2 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5052, 1, 0, 0, 0, 0, 0, 944, 0, 0, 0, 0, 0, 0, 0, 0, 512, 2, 522, 0, 0, 0, 0, 933, 2520, 1773, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4703, 283, 434, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4237, 1697, 0, 0, 0, 0, 0, 0, 656, 1, 0, 0, 0, 0, 0, 5669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6065, 0, 2, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 1, 3566, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1637, 157, 0, 0, 107, 2044, 1246, 897, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5997, 6262, 6693, 5934, 6326, 6138, 5714, 6088] avg=6144.0000 max=6693 max_rank=2 min=5714 min_rank=6 imbalance_ratio=1.089355 max_load_reduction=42459 max_load_reduction_pct=0.863831 moved_to_echo=24266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4991, 0, 0, 0, 0, 0, 0, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 416, 0, 0, 0, 0, 878, 2525, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5428, 244, 449, 0, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4204, 1701, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 5300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2340, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 185, 0, 0, 91, 1912, 1260, 886, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6190, 6049, 7382, 5905, 5973, 6189, 5914, 5550] avg=6144.0000 max=7382 max_rank=2 min=5550 min_rank=7 imbalance_ratio=1.201497 max_load_reduction=41770 max_load_reduction_pct=0.849813 moved_to_echo=23626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5112, 0, 0, 0, 0, 0, 0, 976, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 538, 0, 0, 0, 0, 861, 2539, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 278, 393, 0, 1389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4631, 1710, 0, 0, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 5530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6069, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2778, 0, 3592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 167, 0, 0, 72, 1995, 1247, 908, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6088, 6213, 5916, 6341, 6096, 6123, 6370, 6005] avg=6144.0000 max=6370 max_rank=6 min=5916 min_rank=2 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5121, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 510, 0, 0, 0, 0, 870, 2468, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 240, 355, 0, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5049, 1662, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 5528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2887, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2069, 159, 0, 0, 74, 1994, 1246, 909, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5953, 6134, 5334, 6711, 6036, 6154, 6379, 6451] avg=6144.0000 max=6711 max_rank=3 min=5334 min_rank=2 imbalance_ratio=1.092285 max_load_reduction=42441 max_load_reduction_pct=0.863464 moved_to_echo=25020 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5157, 0, 0, 0, 0, 0, 0, 879, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 603, 0, 0, 0, 0, 805, 2450, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3238, 209, 270, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4809, 1650, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 5743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6077, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2804, 0, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2301, 143, 0, 0, 65, 2071, 1248, 916, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6036, 6119, 5159, 6459, 6203, 6160, 6272, 6744] avg=6144.0000 max=6744 max_rank=7 min=5159 min_rank=2 imbalance_ratio=1.097656 max_load_reduction=42408 max_load_reduction_pct=0.862793 moved_to_echo=25379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5052, 1, 0, 0, 0, 0, 0, 944, 0, 0, 0, 0, 0, 0, 0, 0, 512, 2, 522, 0, 0, 0, 0, 933, 2520, 1773, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4703, 283, 434, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4237, 1697, 0, 0, 0, 0, 0, 0, 656, 1, 0, 0, 0, 0, 0, 5669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6065, 0, 2, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 1, 3566, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1637, 157, 0, 0, 107, 2044, 1246, 897, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5997, 6262, 6693, 5934, 6326, 6138, 5714, 6088] avg=6144.0000 max=6693 max_rank=2 min=5714 min_rank=6 imbalance_ratio=1.089355 max_load_reduction=42459 max_load_reduction_pct=0.863831 moved_to_echo=24266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5038, 0, 0, 0, 0, 0, 0, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 360, 0, 0, 0, 0, 906, 2541, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5380, 282, 396, 0, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 1712, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 5401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6076, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2389, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 172, 0, 0, 85, 1948, 1248, 895, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6418, 6078, 7234, 5633, 6020, 6127, 5985, 5657] avg=6144.0000 max=7234 max_rank=2 min=5633 min_rank=3 imbalance_ratio=1.177409 max_load_reduction=41918 max_load_reduction_pct=0.852824 moved_to_echo=23789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5105, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 491, 0, 0, 0, 0, 880, 2533, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4167, 244, 384, 0, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4229, 1709, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 5542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1670, 159, 0, 0, 77, 2003, 1248, 910, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 6208, 6058, 5938, 6182, 6126, 6245, 6067] avg=6144.0000 max=6328 max_rank=0 min=5938 min_rank=3 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5121, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 510, 0, 0, 0, 0, 870, 2468, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 240, 355, 0, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5049, 1662, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 5528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2887, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2069, 159, 0, 0, 74, 1994, 1246, 909, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5953, 6134, 5334, 6711, 6036, 6154, 6379, 6451] avg=6144.0000 max=6711 max_rank=3 min=5334 min_rank=2 imbalance_ratio=1.092285 max_load_reduction=42441 max_load_reduction_pct=0.863464 moved_to_echo=25020 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3968, 128, 4096, 0, 0, 0, 0, 7040, 20096, 14336, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 0, 14080, 1408, 0, 0, 768, 16128, 10112, 7296, 0, 0, 0, 0, 0] @@ -9767,428 +9767,428 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7817, 0, 0, 0, 0, 0, 2, 13, 0, 0, 8081, 0, 446, 0, 1, 363, 37, 0, 0, 0, 37, 0, 228, 0, 1, 0, 0, 0, 0, 7000, 6560, 0, 107, 0, 2, 0, 108, 0, 181, 444, 0, 0, 2, 6969, 8046, 0, 2, 106, 0, 0, 245, 0, 107, 0, 0, 2, 2243, 1, 1, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 429, 0, 0, 331, 57, 0, 0, 0, 57, 0, 189, 0, 0, 0, 0, 0, 0, 7474, 6990, 0, 84, 0, 0, 0, 90, 0, 203, 379, 0, 0, 0, 6868, 8051, 0, 0, 85, 0, 0, 241, 0, 85, 0, 0, 0, 1671, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 433, 0, 1, 303, 59, 0, 0, 0, 59, 0, 187, 0, 0, 0, 0, 0, 0, 7156, 6732, 0, 72, 0, 0, 0, 86, 0, 185, 376, 0, 0, 0, 6938, 8061, 0, 0, 71, 0, 0, 258, 0, 72, 0, 0, 0, 2238, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 480, 0, 2, 317, 66, 0, 0, 0, 66, 0, 229, 0, 0, 0, 0, 0, 0, 7489, 6861, 0, 92, 0, 0, 0, 74, 0, 222, 410, 0, 0, 0, 6842, 8035, 0, 0, 88, 0, 0, 252, 0, 91, 0, 0, 0, 1684, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 414, 0, 1, 364, 59, 0, 0, 0, 59, 0, 187, 0, 0, 0, 0, 0, 0, 7050, 6683, 0, 77, 0, 0, 0, 99, 0, 188, 393, 0, 0, 0, 7061, 8056, 0, 0, 76, 0, 0, 246, 0, 77, 0, 0, 0, 2175, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7799, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8114, 0, 429, 0, 0, 367, 50, 0, 0, 0, 50, 0, 237, 0, 0, 0, 0, 0, 0, 6992, 6267, 0, 77, 0, 0, 0, 85, 0, 195, 389, 0, 0, 0, 7076, 8065, 0, 0, 78, 0, 0, 251, 0, 78, 0, 0, 0, 2552, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 412, 0, 0, 298, 51, 0, 0, 0, 51, 0, 180, 0, 0, 0, 0, 0, 0, 7012, 6578, 0, 73, 0, 0, 0, 147, 0, 173, 340, 0, 0, 0, 6990, 8068, 0, 0, 74, 0, 0, 244, 0, 74, 0, 0, 0, 2523, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 433, 0, 1, 303, 59, 0, 0, 0, 59, 0, 187, 0, 0, 0, 0, 0, 0, 7156, 6732, 0, 72, 0, 0, 0, 86, 0, 185, 376, 0, 0, 0, 6938, 8061, 0, 0, 71, 0, 0, 258, 0, 72, 0, 0, 0, 2238, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7799, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8114, 0, 429, 0, 0, 367, 50, 0, 0, 0, 50, 0, 237, 0, 0, 0, 0, 0, 0, 6992, 6267, 0, 77, 0, 0, 0, 85, 0, 195, 389, 0, 0, 0, 7076, 8065, 0, 0, 78, 0, 0, 251, 0, 78, 0, 0, 0, 2552, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 414, 0, 1, 364, 59, 0, 0, 0, 59, 0, 187, 0, 0, 0, 0, 0, 0, 7050, 6683, 0, 77, 0, 0, 0, 99, 0, 188, 393, 0, 0, 0, 7061, 8056, 0, 0, 76, 0, 0, 246, 0, 77, 0, 0, 0, 2175, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7761, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8127, 0, 375, 0, 0, 335, 50, 0, 0, 0, 50, 0, 193, 0, 0, 0, 0, 0, 0, 7033, 6287, 0, 64, 0, 0, 0, 129, 0, 192, 369, 0, 0, 0, 7059, 8078, 0, 0, 65, 0, 0, 244, 0, 65, 0, 0, 0, 2674, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5713, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 57, 0, 0, 0, 57, 0, 2394, 2321, 981, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 6209, 0, 0, 0, 0, 0, 0, 0, 0, 6295, 0, 84, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 379, 0, 0, 0, 5479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 241, 0, 85, 0, 5730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 0, 0, 0, 0, 0, 1389, 1265, 695, 641, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6142, 6141, 6398, 6469, 6061, 6141, 5661] avg=6144.0000 max=6469 max_rank=4 min=5661 min_rank=7 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 429, 0, 0, 331, 57, 0, 0, 0, 57, 0, 189, 0, 0, 0, 0, 0, 0, 7474, 6990, 0, 84, 0, 0, 0, 90, 0, 203, 379, 0, 0, 0, 6868, 8051, 0, 0, 85, 0, 0, 241, 0, 85, 0, 0, 0, 1671, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 0, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 303, 59, 0, 0, 0, 59, 0, 2398, 2324, 979, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 5945, 0, 0, 0, 0, 0, 0, 0, 0, 6062, 0, 72, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 376, 0, 0, 0, 5535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 258, 0, 72, 0, 5737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2238, 0, 0, 0, 0, 0, 1403, 1211, 670, 640, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6155, 6123, 6132, 6220, 6096, 6138, 6162] avg=6144.0000 max=6220 max_rank=4 min=6096 min_rank=5 imbalance_ratio=1.012370 max_load_reduction=42932 max_load_reduction_pct=0.873454 moved_to_echo=24616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5708, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 317, 66, 0, 0, 0, 66, 0, 2393, 2316, 980, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 0, 0, 0, 0, 0, 6178, 0, 92, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 410, 0, 0, 0, 5458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 252, 0, 91, 0, 5719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1684, 0, 0, 0, 0, 0, 1384, 1267, 683, 640, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6188, 6140, 6451, 6344, 6090, 6150, 5658] avg=6144.0000 max=6451 max_rank=3 min=5658 min_rank=7 imbalance_ratio=1.049967 max_load_reduction=42701 max_load_reduction_pct=0.868754 moved_to_echo=24242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5718, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 50, 0, 0, 0, 50, 0, 2396, 2325, 986, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 5809, 0, 0, 0, 0, 0, 0, 0, 0, 5644, 0, 77, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 389, 0, 0, 0, 5645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 251, 0, 78, 0, 5740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 0, 0, 0, 1431, 1183, 623, 644, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6169, 6148, 6174, 6046, 5806, 6229, 6147, 6433] avg=6144.0000 max=6433 max_rank=7 min=5806 min_rank=4 imbalance_ratio=1.047038 max_load_reduction=42719 max_load_reduction_pct=0.869120 moved_to_echo=24615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 0, 5694, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 363, 37, 0, 0, 0, 37, 0, 2387, 2319, 988, 0, 0, 0, 0, 0, 228, 0, 1, 0, 0, 0, 0, 5815, 0, 0, 0, 0, 0, 0, 0, 0, 5907, 0, 107, 0, 2, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 444, 0, 0, 2, 5559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 106, 0, 0, 245, 0, 107, 0, 5727, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2243, 1, 1, 0, 0, 0, 1410, 1185, 653, 646, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6155, 6132, 6044, 6124, 6186, 6187, 6141] avg=6144.0000 max=6187 max_rank=6 min=6044 min_rank=3 imbalance_ratio=1.006999 max_load_reduction=42965 max_load_reduction_pct=0.874125 moved_to_echo=24638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5720, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 51, 0, 0, 0, 51, 0, 2398, 2326, 979, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 5825, 0, 0, 0, 0, 0, 0, 0, 0, 5924, 0, 73, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 340, 0, 0, 0, 5576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 244, 0, 74, 0, 5742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 0, 0, 0, 0, 0, 1414, 1187, 654, 640, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6132, 6103, 6005, 6144, 6089, 6134, 6418] avg=6144.0000 max=6418 max_rank=7 min=6005 min_rank=3 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5713, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 57, 0, 0, 0, 57, 0, 2394, 2321, 981, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 6209, 0, 0, 0, 0, 0, 0, 0, 0, 6295, 0, 84, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 379, 0, 0, 0, 5479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 241, 0, 85, 0, 5730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 0, 0, 0, 0, 0, 1389, 1265, 695, 641, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6142, 6141, 6398, 6469, 6061, 6141, 5661] avg=6144.0000 max=6469 max_rank=4 min=5661 min_rank=7 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5727, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 50, 0, 0, 0, 50, 0, 2400, 2329, 981, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 5843, 0, 0, 0, 0, 0, 0, 0, 0, 5662, 0, 64, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 369, 0, 0, 0, 5631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 244, 0, 65, 0, 5749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2674, 0, 0, 0, 0, 0, 1428, 1190, 625, 641, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6104, 6145, 6036, 5855, 6192, 6123, 6558] avg=6144.0000 max=6558 max_rank=7 min=5855 min_rank=4 imbalance_ratio=1.067383 max_load_reduction=42594 max_load_reduction_pct=0.866577 moved_to_echo=24728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 0, 5694, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 363, 37, 0, 0, 0, 37, 0, 2387, 2319, 988, 0, 0, 0, 0, 0, 228, 0, 1, 0, 0, 0, 0, 5815, 0, 0, 0, 0, 0, 0, 0, 0, 5907, 0, 107, 0, 2, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 444, 0, 0, 2, 5559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 106, 0, 0, 245, 0, 107, 0, 5727, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2243, 1, 1, 0, 0, 0, 1410, 1185, 653, 646, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6155, 6132, 6044, 6124, 6186, 6187, 6141] avg=6144.0000 max=6187 max_rank=6 min=6044 min_rank=3 imbalance_ratio=1.006999 max_load_reduction=42965 max_load_reduction_pct=0.874125 moved_to_echo=24638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5714, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 364, 59, 0, 0, 0, 59, 0, 2401, 2323, 987, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 5853, 0, 0, 0, 0, 0, 0, 0, 0, 6014, 0, 77, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 393, 0, 0, 0, 5628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 246, 0, 77, 0, 5733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2175, 0, 0, 0, 0, 0, 1433, 1197, 669, 647, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6128, 6194, 6040, 6190, 6209, 6132, 6121] avg=6144.0000 max=6209 max_rank=5 min=6040 min_rank=3 imbalance_ratio=1.010579 max_load_reduction=42943 max_load_reduction_pct=0.873678 moved_to_echo=24652 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5720, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 51, 0, 0, 0, 51, 0, 2398, 2326, 979, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 5825, 0, 0, 0, 0, 0, 0, 0, 0, 5924, 0, 73, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 340, 0, 0, 0, 5576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 244, 0, 74, 0, 5742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 0, 0, 0, 0, 0, 1414, 1187, 654, 640, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6132, 6103, 6005, 6144, 6089, 6134, 6418] avg=6144.0000 max=6418 max_rank=7 min=6005 min_rank=3 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2688, 512, 0, 0, 0, 512, 0, 19200, 18688, 7936, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 17792, 128, 128, 0, 0, 0, 11392, 9728, 5376, 5248, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 45824, 0, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1664, 3200, 0, 0, 128, 44544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[47744, 0, 768, 0, 128, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[1664, 0, 128, 0, 0, 0, 0, 47616, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 768, 0, 0, 2048, 0, 768, 0, 45952, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1664, 3200, 0, 0, 128, 44544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[1664, 0, 128, 0, 0, 0, 0, 47616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 45824, 0, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[47744, 0, 768, 0, 128, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49152 observed_loss=0 tokens_per_expert=[0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49152 slack_slots=49152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 350, 0, 0, 7518, 6185, 0, 0, 77, 0, 0, 0, 0, 0, 50, 0, 530, 0, 0, 0, 0, 0, 0, 1357, 7463, 0, 0, 0, 0, 0, 0, 757, 7292, 0, 8115, 78, 0, 0, 805, 77, 0, 149, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 329, 7834, 0, 0, 0, 64] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 342, 0, 0, 7443, 5776, 0, 0, 84, 0, 0, 0, 0, 0, 57, 0, 593, 0, 0, 0, 0, 0, 0, 1309, 7742, 0, 0, 0, 0, 0, 0, 826, 7257, 0, 8108, 85, 0, 0, 906, 84, 0, 85, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 326, 7819, 0, 0, 0, 134] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 4, 1, 2, 2, 1, 317, 0, 0, 7490, 5916, 0, 0, 107, 0, 1, 0, 0, 0, 36, 2, 538, 2, 0, 0, 0, 0, 0, 1360, 7675, 0, 0, 0, 2, 0, 0, 802, 7155, 0, 8083, 107, 0, 0, 853, 111, 0, 112, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 0, 352, 7781, 0, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 342, 0, 0, 7434, 6082, 0, 0, 72, 0, 0, 0, 0, 0, 59, 0, 529, 0, 0, 0, 0, 0, 0, 1271, 7663, 0, 0, 0, 0, 0, 0, 826, 7294, 0, 8120, 72, 0, 0, 875, 72, 0, 78, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 330, 7839, 0, 0, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 290, 0, 0, 7456, 5892, 0, 0, 91, 0, 0, 0, 0, 0, 66, 0, 467, 0, 0, 0, 0, 0, 0, 1299, 7761, 0, 0, 0, 0, 0, 0, 835, 7225, 0, 8101, 91, 0, 0, 875, 91, 0, 97, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 343, 7852, 0, 0, 0, 100] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 319, 0, 0, 7481, 6206, 0, 0, 64, 0, 0, 0, 0, 0, 50, 0, 540, 0, 0, 0, 0, 0, 0, 1345, 7468, 0, 0, 0, 0, 0, 0, 793, 7329, 0, 8128, 65, 0, 0, 825, 64, 0, 158, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 309, 7874, 0, 0, 0, 79] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 350, 0, 0, 7518, 6185, 0, 0, 77, 0, 0, 0, 0, 0, 50, 0, 530, 0, 0, 0, 0, 0, 0, 1357, 7463, 0, 0, 0, 0, 0, 0, 757, 7292, 0, 8115, 78, 0, 0, 805, 77, 0, 149, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 329, 7834, 0, 0, 0, 64] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 4, 1, 2, 2, 1, 317, 0, 0, 7490, 5916, 0, 0, 107, 0, 1, 0, 0, 0, 36, 2, 538, 2, 0, 0, 0, 0, 0, 1360, 7675, 0, 0, 0, 2, 0, 0, 802, 7155, 0, 8083, 107, 0, 0, 853, 111, 0, 112, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 0, 352, 7781, 0, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 321, 0, 0, 7433, 6085, 0, 0, 77, 0, 0, 0, 0, 0, 59, 0, 499, 0, 0, 0, 0, 0, 0, 1438, 7573, 0, 0, 0, 0, 0, 0, 839, 7268, 0, 8115, 77, 0, 0, 877, 77, 0, 85, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 323, 7835, 0, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 342, 0, 0, 7434, 6082, 0, 0, 72, 0, 0, 0, 0, 0, 59, 0, 529, 0, 0, 0, 0, 0, 0, 1271, 7663, 0, 0, 0, 0, 0, 0, 826, 7294, 0, 8120, 72, 0, 0, 875, 72, 0, 78, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 330, 7839, 0, 0, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 324, 0, 0, 7454, 6054, 0, 0, 73, 0, 0, 0, 0, 0, 51, 0, 498, 0, 0, 0, 0, 0, 0, 1389, 7602, 0, 0, 0, 0, 0, 0, 746, 7302, 0, 8119, 74, 0, 0, 890, 73, 0, 112, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 318, 7863, 0, 0, 0, 100] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 342, 0, 3694, 2110, 0, 0, 0, 0, 0, 0, 0, 40, 5776, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 593, 0, 0, 3709, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 4877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 5335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 906, 84, 0, 85, 0, 2105, 1922, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 5998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 5714, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 5900, 6234, 6186, 6161, 6177, 6172, 6174] avg=6144.0000 max=6234 max_rank=2 min=5900 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=42918 max_load_reduction_pct=0.873169 moved_to_echo=24684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 342, 0, 3690, 2114, 0, 0, 0, 0, 0, 0, 0, 39, 6082, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 529, 0, 0, 3705, 1856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 4827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 5362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 875, 72, 0, 78, 0, 2110, 1932, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 6006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 5729, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6193, 6149, 6098, 6188, 6119, 6130, 6129] avg=6144.0000 max=6193 max_rank=1 min=6098 min_rank=3 imbalance_ratio=1.007975 max_load_reduction=42959 max_load_reduction_pct=0.874003 moved_to_echo=24566 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 350, 0, 3731, 2112, 0, 0, 0, 0, 0, 0, 0, 41, 6185, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 530, 0, 0, 3746, 1808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 4701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 5361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 805, 77, 0, 149, 0, 2109, 1931, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 6003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 5725, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6193, 6303, 6134, 6058, 6118, 6103, 6125, 6118] avg=6144.0000 max=6303 max_rank=1 min=6058 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 319, 0, 0, 7481, 6206, 0, 0, 64, 0, 0, 0, 0, 0, 50, 0, 540, 0, 0, 0, 0, 0, 0, 1345, 7468, 0, 0, 0, 0, 0, 0, 793, 7329, 0, 8128, 65, 0, 0, 825, 64, 0, 158, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 309, 7874, 0, 0, 0, 79] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 290, 0, 3701, 2108, 0, 0, 0, 0, 0, 0, 0, 39, 5892, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 467, 0, 0, 3716, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1299, 4889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 5312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 875, 91, 0, 97, 0, 2114, 1913, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 5993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 5738, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 6022, 6129, 6188, 6147, 6173, 6207, 6181] avg=6144.0000 max=6207 max_rank=6 min=6022 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=42945 max_load_reduction_pct=0.873718 moved_to_echo=24708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 319, 0, 3713, 2115, 0, 0, 0, 0, 0, 0, 0, 40, 6206, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 540, 0, 0, 3728, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 4704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 5388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 825, 64, 0, 158, 0, 2119, 1941, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 6013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 5755, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6310, 6127, 6049, 6181, 6127, 6068, 6143] avg=6144.0000 max=6310 max_rank=1 min=6049 min_rank=3 imbalance_ratio=1.027018 max_load_reduction=42842 max_load_reduction_pct=0.871623 moved_to_echo=24519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 350, 0, 3731, 2112, 0, 0, 0, 0, 0, 0, 0, 41, 6185, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 530, 0, 0, 3746, 1808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 4701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 5361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 805, 77, 0, 149, 0, 2109, 1931, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 6003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 5725, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6193, 6303, 6134, 6058, 6118, 6103, 6125, 6118] avg=6144.0000 max=6303 max_rank=1 min=6058 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 1, 2, 2, 1, 317, 0, 3718, 2103, 0, 0, 0, 0, 0, 0, 0, 39, 5916, 0, 0, 107, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 2, 538, 2, 0, 3733, 1859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 4835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 802, 5260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 853, 111, 0, 112, 0, 2094, 1895, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 5980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 5687, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6063, 6170, 6195, 6064, 6153, 6228, 6131] avg=6144.0000 max=6228 max_rank=6 min=6063 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=42924 max_load_reduction_pct=0.873291 moved_to_echo=24576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 321, 0, 3693, 2113, 0, 0, 0, 0, 0, 0, 0, 33, 6085, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 499, 0, 0, 3707, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 4762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 5340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 877, 77, 0, 85, 0, 2113, 1928, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 6002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 5722, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6195, 6104, 6200, 6179, 6129, 6106, 6111] avg=6144.0000 max=6200 max_rank=3 min=6104 min_rank=2 imbalance_ratio=1.009115 max_load_reduction=42952 max_load_reduction_pct=0.873861 moved_to_echo=24525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 342, 0, 3694, 2110, 0, 0, 0, 0, 0, 0, 0, 40, 5776, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 593, 0, 0, 3709, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 4877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 5335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 906, 84, 0, 85, 0, 2105, 1922, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 5998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 5714, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 5900, 6234, 6186, 6161, 6177, 6172, 6174] avg=6144.0000 max=6234 max_rank=2 min=5900 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=42918 max_load_reduction_pct=0.873169 moved_to_echo=24684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 342, 0, 3690, 2114, 0, 0, 0, 0, 0, 0, 0, 39, 6082, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 529, 0, 0, 3705, 1856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 4827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 5362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 875, 72, 0, 78, 0, 2110, 1932, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 6006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 5729, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6193, 6149, 6098, 6188, 6119, 6130, 6129] avg=6144.0000 max=6193 max_rank=1 min=6098 min_rank=3 imbalance_ratio=1.007975 max_load_reduction=42959 max_load_reduction_pct=0.874003 moved_to_echo=24566 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 324, 0, 3700, 2113, 0, 0, 0, 0, 0, 0, 0, 39, 6054, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 498, 0, 0, 3715, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1389, 4789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 5368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 890, 73, 0, 112, 0, 2116, 1934, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 6006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 5747, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6166, 6105, 6178, 6114, 6171, 6116, 6165] avg=6144.0000 max=6178 max_rank=3 min=6105 min_rank=2 imbalance_ratio=1.005534 max_load_reduction=42974 max_load_reduction_pct=0.874308 moved_to_echo=24566 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 321, 0, 3693, 2113, 0, 0, 0, 0, 0, 0, 0, 33, 6085, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 499, 0, 0, 3707, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 4762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 5340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 877, 77, 0, 85, 0, 2113, 1928, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 6002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 5722, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6195, 6104, 6200, 6179, 6129, 6106, 6111] avg=6144.0000 max=6200 max_rank=3 min=6104 min_rank=2 imbalance_ratio=1.009115 max_load_reduction=42952 max_load_reduction_pct=0.873861 moved_to_echo=24525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 319, 0, 3713, 2115, 0, 0, 0, 0, 0, 0, 0, 40, 6206, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 540, 0, 0, 3728, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 4704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 5388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 825, 64, 0, 158, 0, 2119, 1941, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 6013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 5755, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6310, 6127, 6049, 6181, 6127, 6068, 6143] avg=6144.0000 max=6310 max_rank=1 min=6049 min_rank=3 imbalance_ratio=1.027018 max_load_reduction=42842 max_load_reduction_pct=0.871623 moved_to_echo=24519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[768, 0, 0, 6912, 768, 0, 896, 0, 16896, 15488, 7808, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 2688, 0, 29696, 16896, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 10880, 38400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 512, 128, 4224, 128, 0, 29824, 14848, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 2688, 45824, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 6528, 42752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 512, 128, 4224, 128, 0, 29824, 14848, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 1152, 0, 0, 48128, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 384, 48256, 0, 0, 768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True - [2026-08-26 15:39:22] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 805.7 | throughput per GPU (TFLOP/s/GPU): 158.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.613759E+00 | loss scale: 1.0 | grad norm: 3.002 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:35:25] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 814.7 | throughput per GPU (TFLOP/s/GPU): 156.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.613759E+00 | loss scale: 1.0 | grad norm: 3.002 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1331, 307, 753, 989, 292, 902, 715, 1115, 1370, 239, 613, 373, 388, 1296, 807, 1140, 634, 966, 879, 518, 371, 532, 311, 1424, 490, 679, 781, 355, 722, 80, 823, 1456, 882, 803, 1018, 1081, 526, 191, 529, 907, 666, 629, 498, 891, 914, 921, 550, 590, 766, 538, 798, 561, 1042, 1233, 866, 894, 1530, 470, 1477, 750, 193, 670, 1156, 961] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1312, 232, 788, 954, 286, 746, 791, 1137, 1314, 188, 572, 329, 375, 1308, 970, 1184, 670, 985, 843, 648, 278, 529, 254, 1508, 573, 692, 733, 384, 775, 80, 773, 1402, 781, 841, 951, 1004, 421, 198, 553, 1060, 599, 568, 524, 916, 910, 905, 565, 687, 799, 555, 857, 472, 1160, 1226, 946, 919, 1490, 412, 1423, 686, 159, 803, 1181, 968] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1401, 272, 734, 891, 283, 907, 760, 976, 1427, 182, 611, 358, 336, 1473, 855, 1128, 618, 945, 786, 618, 294, 512, 254, 1356, 474, 720, 765, 358, 679, 74, 822, 1072, 851, 827, 1028, 1146, 774, 223, 589, 945, 605, 589, 582, 817, 923, 970, 411, 636, 686, 575, 834, 583, 1024, 1394, 869, 806, 1600, 481, 1602, 768, 187, 737, 1209, 940] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1375, 259, 729, 849, 290, 1081, 856, 912, 1342, 258, 685, 337, 291, 1483, 854, 1132, 769, 966, 754, 602, 347, 548, 288, 1415, 342, 638, 843, 384, 704, 105, 836, 1410, 823, 785, 1062, 1165, 432, 253, 487, 762, 624, 625, 582, 826, 990, 895, 429, 571, 886, 553, 787, 624, 1012, 1346, 809, 884, 1521, 310, 1544, 756, 195, 815, 1270, 845] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1350, 333, 739, 968, 302, 812, 788, 1134, 1337, 198, 642, 336, 417, 1296, 839, 984, 694, 1002, 734, 593, 307, 514, 283, 1393, 474, 685, 783, 390, 713, 81, 915, 1339, 858, 878, 1082, 1102, 526, 212, 505, 893, 718, 657, 591, 923, 819, 933, 622, 611, 720, 551, 835, 546, 1081, 1225, 828, 928, 1538, 444, 1421, 730, 161, 639, 1120, 1080] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1325, 295, 774, 906, 337, 1041, 838, 1036, 1355, 242, 663, 315, 347, 1334, 963, 1159, 681, 988, 824, 611, 383, 521, 255, 1401, 501, 645, 771, 402, 734, 115, 777, 1353, 809, 783, 1008, 1087, 513, 217, 485, 1005, 644, 673, 511, 822, 852, 917, 454, 572, 861, 545, 812, 556, 1073, 1280, 893, 762, 1530, 265, 1415, 808, 243, 762, 1180, 923] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1283, 288, 816, 883, 280, 932, 861, 1057, 1405, 233, 676, 322, 362, 1464, 1054, 984, 728, 1007, 796, 662, 335, 521, 268, 1431, 483, 576, 750, 365, 778, 82, 855, 1339, 793, 789, 986, 1115, 508, 232, 527, 911, 653, 611, 506, 853, 848, 937, 422, 633, 867, 623, 786, 562, 1034, 1263, 824, 832, 1442, 510, 1566, 662, 167, 758, 1162, 894] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1331, 307, 753, 989, 292, 902, 715, 1115, 1370, 239, 613, 373, 388, 1296, 807, 1140, 634, 966, 879, 518, 371, 532, 311, 1424, 490, 679, 781, 355, 722, 80, 823, 1456, 882, 803, 1018, 1081, 526, 191, 529, 907, 666, 629, 498, 891, 914, 921, 550, 590, 766, 538, 798, 561, 1042, 1233, 866, 894, 1530, 470, 1477, 750, 193, 670, 1156, 961] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1280, 260, 718, 924, 316, 996, 807, 1063, 1290, 190, 669, 372, 366, 1293, 955, 1123, 681, 967, 838, 618, 340, 526, 285, 1362, 534, 673, 773, 390, 738, 162, 914, 1153, 882, 755, 1110, 1116, 693, 209, 526, 967, 638, 607, 556, 858, 915, 938, 535, 664, 710, 549, 772, 518, 1058, 1288, 838, 890, 1508, 267, 1446, 736, 191, 684, 1251, 901] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1163, 272, 734, 891, 283, 907, 760, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1427, 182, 611, 358, 336, 1306, 855, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 618, 945, 786, 618, 294, 512, 254, 1356, 310, 238, 10, 0, 0, 0, 0, 0, 474, 720, 765, 358, 679, 74, 822, 1072, 905, 0, 0, 0, 0, 0, 0, 0, 851, 827, 1028, 1146, 774, 223, 589, 945, 157, 0, 0, 0, 0, 0, 0, 0, 605, 589, 582, 817, 923, 970, 411, 636, 377, 194, 0, 0, 0, 0, 0, 0, 686, 575, 834, 583, 1024, 707, 869, 806, 0, 0, 0, 0, 0, 0, 0, 0, 501, 481, 1602, 768, 187, 737, 1209, 940, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5986, 6203, 5941, 5869, 6540, 6104, 6084, 6425] avg=6144.0000 max=6540 max_rank=4 min=5869 min_rank=3 imbalance_ratio=1.064453 max_load_reduction=42612 max_load_reduction_pct=0.866943 moved_to_echo=25153 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1089, 232, 788, 954, 286, 746, 791, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 1314, 188, 572, 329, 375, 1159, 970, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 670, 985, 843, 648, 278, 529, 254, 1508, 273, 223, 9, 0, 0, 0, 0, 0, 573, 692, 733, 384, 775, 80, 773, 1402, 843, 0, 0, 0, 0, 0, 0, 0, 781, 841, 951, 1004, 421, 198, 553, 1060, 140, 0, 0, 0, 0, 0, 0, 0, 599, 568, 524, 916, 910, 905, 565, 687, 331, 180, 0, 0, 0, 0, 0, 0, 799, 555, 857, 472, 1160, 622, 946, 919, 0, 0, 0, 0, 0, 0, 0, 0, 467, 412, 1423, 686, 159, 803, 1181, 968, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 6091, 6220, 6255, 5949, 6185, 6330, 6099] avg=6144.0000 max=6330 max_rank=6 min=5949 min_rank=4 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24563 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1350, 333, 739, 968, 302, 812, 788, 1134, 1337, 198, 642, 336, 417, 1296, 839, 984, 694, 1002, 734, 593, 307, 514, 283, 1393, 474, 685, 783, 390, 713, 81, 915, 1339, 858, 878, 1082, 1102, 526, 212, 505, 893, 718, 657, 591, 923, 819, 933, 622, 611, 720, 551, 835, 546, 1081, 1225, 828, 928, 1538, 444, 1421, 730, 161, 639, 1120, 1080] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1325, 295, 774, 906, 337, 1041, 838, 1036, 1355, 242, 663, 315, 347, 1334, 963, 1159, 681, 988, 824, 611, 383, 521, 255, 1401, 501, 645, 771, 402, 734, 115, 777, 1353, 809, 783, 1008, 1087, 513, 217, 485, 1005, 644, 673, 511, 822, 852, 917, 454, 572, 861, 545, 812, 556, 1073, 1280, 893, 762, 1530, 265, 1415, 808, 243, 762, 1180, 923] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1401, 272, 734, 891, 283, 907, 760, 976, 1427, 182, 611, 358, 336, 1473, 855, 1128, 618, 945, 786, 618, 294, 512, 254, 1356, 474, 720, 765, 358, 679, 74, 822, 1072, 851, 827, 1028, 1146, 774, 223, 589, 945, 605, 589, 582, 817, 923, 970, 411, 636, 686, 575, 834, 583, 1024, 1394, 869, 806, 1600, 481, 1602, 768, 187, 737, 1209, 940] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1105, 307, 753, 989, 292, 902, 715, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 1370, 239, 613, 373, 388, 1149, 807, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 634, 966, 879, 518, 371, 532, 311, 1424, 274, 226, 9, 0, 0, 0, 0, 0, 490, 679, 781, 355, 722, 80, 823, 1456, 866, 0, 0, 0, 0, 0, 0, 0, 882, 803, 1018, 1081, 526, 191, 529, 907, 138, 0, 0, 0, 0, 0, 0, 0, 666, 629, 498, 891, 914, 921, 550, 590, 333, 185, 0, 0, 0, 0, 0, 0, 766, 538, 798, 561, 1042, 626, 866, 894, 0, 0, 0, 0, 0, 0, 0, 0, 479, 470, 1477, 750, 193, 670, 1156, 961, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6079, 6144, 6252, 6075, 6177, 6091, 6156] avg=6144.0000 max=6252 max_rank=3 min=6075 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=42900 max_load_reduction_pct=0.872803 moved_to_echo=24499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1141, 259, 729, 849, 290, 1081, 856, 912, 0, 0, 0, 0, 0, 0, 0, 0, 1342, 258, 685, 337, 291, 1315, 854, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 769, 966, 754, 602, 347, 548, 288, 1415, 300, 234, 10, 0, 0, 0, 0, 0, 342, 638, 843, 384, 704, 105, 836, 1410, 861, 0, 0, 0, 0, 0, 0, 0, 823, 785, 1062, 1165, 432, 253, 487, 762, 158, 0, 0, 0, 0, 0, 0, 0, 624, 625, 582, 826, 990, 895, 429, 571, 364, 184, 0, 0, 0, 0, 0, 0, 886, 553, 787, 624, 1012, 682, 809, 884, 0, 0, 0, 0, 0, 0, 0, 0, 476, 310, 1544, 756, 195, 815, 1270, 845, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6117, 6214, 6233, 6123, 5927, 6090, 6237, 6211] avg=6144.0000 max=6237 max_rank=6 min=5927 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=42915 max_load_reduction_pct=0.873108 moved_to_echo=24465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1100, 295, 774, 906, 337, 1041, 838, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1355, 242, 663, 315, 347, 1183, 963, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 681, 988, 824, 611, 383, 521, 255, 1401, 285, 225, 9, 0, 0, 0, 0, 0, 501, 645, 771, 402, 734, 115, 777, 1353, 866, 0, 0, 0, 0, 0, 0, 0, 809, 783, 1008, 1087, 513, 217, 485, 1005, 142, 0, 0, 0, 0, 0, 0, 0, 644, 673, 511, 822, 852, 917, 454, 572, 346, 185, 0, 0, 0, 0, 0, 0, 861, 545, 812, 556, 1073, 649, 893, 762, 0, 0, 0, 0, 0, 0, 0, 0, 479, 265, 1415, 808, 243, 762, 1180, 923, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6327, 6227, 6183, 6164, 6049, 5976, 6151, 6075] avg=6144.0000 max=6327 max_rank=0 min=5976 min_rank=5 imbalance_ratio=1.029785 max_load_reduction=42825 max_load_reduction_pct=0.871277 moved_to_echo=24251 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1116, 333, 739, 968, 302, 812, 788, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 198, 642, 336, 417, 1140, 839, 984, 0, 0, 0, 0, 0, 0, 0, 0, 694, 1002, 734, 593, 307, 514, 283, 1393, 277, 234, 13, 0, 0, 0, 0, 0, 474, 685, 783, 390, 713, 81, 915, 1339, 875, 0, 0, 0, 0, 0, 0, 0, 858, 878, 1082, 1102, 526, 212, 505, 893, 143, 0, 0, 0, 0, 0, 0, 0, 718, 657, 591, 923, 819, 933, 622, 611, 335, 190, 0, 0, 0, 0, 0, 0, 720, 551, 835, 546, 1081, 613, 828, 928, 0, 0, 0, 0, 0, 0, 0, 0, 473, 444, 1421, 730, 161, 639, 1120, 1080, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6192, 5893, 6044, 6255, 6199, 6399, 6102, 6068] avg=6144.0000 max=6399 max_rank=5 min=5893 min_rank=1 imbalance_ratio=1.041504 max_load_reduction=42753 max_load_reduction_pct=0.869812 moved_to_echo=24768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1089, 232, 788, 954, 286, 746, 791, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 1314, 188, 572, 329, 375, 1159, 970, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 670, 985, 843, 648, 278, 529, 254, 1508, 273, 223, 9, 0, 0, 0, 0, 0, 573, 692, 733, 384, 775, 80, 773, 1402, 843, 0, 0, 0, 0, 0, 0, 0, 781, 841, 951, 1004, 421, 198, 553, 1060, 140, 0, 0, 0, 0, 0, 0, 0, 599, 568, 524, 916, 910, 905, 565, 687, 331, 180, 0, 0, 0, 0, 0, 0, 799, 555, 857, 472, 1160, 622, 946, 919, 0, 0, 0, 0, 0, 0, 0, 0, 467, 412, 1423, 686, 159, 803, 1181, 968, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 6091, 6220, 6255, 5949, 6185, 6330, 6099] avg=6144.0000 max=6330 max_rank=6 min=5949 min_rank=4 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24563 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1065, 288, 816, 883, 280, 932, 861, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1405, 233, 676, 322, 362, 1298, 1054, 984, 0, 0, 0, 0, 0, 0, 0, 0, 728, 1007, 796, 662, 335, 521, 268, 1431, 281, 218, 10, 0, 0, 0, 0, 0, 483, 576, 750, 365, 778, 82, 855, 1339, 816, 0, 0, 0, 0, 0, 0, 0, 793, 789, 986, 1115, 508, 232, 527, 911, 156, 0, 0, 0, 0, 0, 0, 0, 653, 611, 506, 853, 848, 937, 422, 633, 341, 175, 0, 0, 0, 0, 0, 0, 867, 623, 786, 562, 1034, 641, 824, 832, 0, 0, 0, 0, 0, 0, 0, 0, 451, 510, 1566, 662, 167, 758, 1162, 894, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6182, 6334, 6257, 6044, 6017, 5979, 6169, 6170] avg=6144.0000 max=6334 max_rank=1 min=5979 min_rank=5 imbalance_ratio=1.030924 max_load_reduction=42818 max_load_reduction_pct=0.871134 moved_to_echo=24335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1105, 307, 753, 989, 292, 902, 715, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 1370, 239, 613, 373, 388, 1149, 807, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 634, 966, 879, 518, 371, 532, 311, 1424, 274, 226, 9, 0, 0, 0, 0, 0, 490, 679, 781, 355, 722, 80, 823, 1456, 866, 0, 0, 0, 0, 0, 0, 0, 882, 803, 1018, 1081, 526, 191, 529, 907, 138, 0, 0, 0, 0, 0, 0, 0, 666, 629, 498, 891, 914, 921, 550, 590, 333, 185, 0, 0, 0, 0, 0, 0, 766, 538, 798, 561, 1042, 626, 866, 894, 0, 0, 0, 0, 0, 0, 0, 0, 479, 470, 1477, 750, 193, 670, 1156, 961, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6079, 6144, 6252, 6075, 6177, 6091, 6156] avg=6144.0000 max=6252 max_rank=3 min=6075 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=42900 max_load_reduction_pct=0.872803 moved_to_echo=24499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1163, 272, 734, 891, 283, 907, 760, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1427, 182, 611, 358, 336, 1306, 855, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 618, 945, 786, 618, 294, 512, 254, 1356, 310, 238, 10, 0, 0, 0, 0, 0, 474, 720, 765, 358, 679, 74, 822, 1072, 905, 0, 0, 0, 0, 0, 0, 0, 851, 827, 1028, 1146, 774, 223, 589, 945, 157, 0, 0, 0, 0, 0, 0, 0, 605, 589, 582, 817, 923, 970, 411, 636, 377, 194, 0, 0, 0, 0, 0, 0, 686, 575, 834, 583, 1024, 707, 869, 806, 0, 0, 0, 0, 0, 0, 0, 0, 501, 481, 1602, 768, 187, 737, 1209, 940, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5986, 6203, 5941, 5869, 6540, 6104, 6084, 6425] avg=6144.0000 max=6540 max_rank=4 min=5869 min_rank=3 imbalance_ratio=1.064453 max_load_reduction=42612 max_load_reduction_pct=0.866943 moved_to_echo=25153 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1063, 260, 718, 924, 316, 996, 807, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1290, 190, 669, 372, 366, 1146, 955, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 681, 967, 838, 618, 340, 526, 285, 1362, 287, 217, 9, 0, 0, 0, 0, 0, 534, 673, 773, 390, 738, 162, 914, 1153, 853, 0, 0, 0, 0, 0, 0, 0, 882, 755, 1110, 1116, 693, 209, 526, 967, 138, 0, 0, 0, 0, 0, 0, 0, 638, 607, 556, 858, 915, 938, 535, 664, 348, 183, 0, 0, 0, 0, 0, 0, 710, 549, 772, 518, 1058, 653, 838, 890, 0, 0, 0, 0, 0, 0, 0, 0, 472, 267, 1446, 736, 191, 684, 1251, 901, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6111, 6130, 6190, 6396, 6242, 5988, 5948] avg=6144.0000 max=6396 max_rank=4 min=5948 min_rank=7 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=24574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1100, 295, 774, 906, 337, 1041, 838, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1355, 242, 663, 315, 347, 1183, 963, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 681, 988, 824, 611, 383, 521, 255, 1401, 285, 225, 9, 0, 0, 0, 0, 0, 501, 645, 771, 402, 734, 115, 777, 1353, 866, 0, 0, 0, 0, 0, 0, 0, 809, 783, 1008, 1087, 513, 217, 485, 1005, 142, 0, 0, 0, 0, 0, 0, 0, 644, 673, 511, 822, 852, 917, 454, 572, 346, 185, 0, 0, 0, 0, 0, 0, 861, 545, 812, 556, 1073, 649, 893, 762, 0, 0, 0, 0, 0, 0, 0, 0, 479, 265, 1415, 808, 243, 762, 1180, 923, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6327, 6227, 6183, 6164, 6049, 5976, 6151, 6075] avg=6144.0000 max=6327 max_rank=0 min=5976 min_rank=5 imbalance_ratio=1.029785 max_load_reduction=42825 max_load_reduction_pct=0.871277 moved_to_echo=24251 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1116, 333, 739, 968, 302, 812, 788, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 198, 642, 336, 417, 1140, 839, 984, 0, 0, 0, 0, 0, 0, 0, 0, 694, 1002, 734, 593, 307, 514, 283, 1393, 277, 234, 13, 0, 0, 0, 0, 0, 474, 685, 783, 390, 713, 81, 915, 1339, 875, 0, 0, 0, 0, 0, 0, 0, 858, 878, 1082, 1102, 526, 212, 505, 893, 143, 0, 0, 0, 0, 0, 0, 0, 718, 657, 591, 923, 819, 933, 622, 611, 335, 190, 0, 0, 0, 0, 0, 0, 720, 551, 835, 546, 1081, 613, 828, 928, 0, 0, 0, 0, 0, 0, 0, 0, 473, 444, 1421, 730, 161, 639, 1120, 1080, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6192, 5893, 6044, 6255, 6199, 6399, 6102, 6068] avg=6144.0000 max=6399 max_rank=5 min=5893 min_rank=1 imbalance_ratio=1.041504 max_load_reduction=42753 max_load_reduction_pct=0.869812 moved_to_echo=24768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8960, 2304, 6144, 7424, 2432, 7424, 6528, 8448, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6784, 6528, 8320, 8832, 4480, 1792, 4224, 7552, 1280, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5504, 7936, 6528, 4992, 2688, 4224, 2304, 11392, 2304, 1920, 128, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5248, 4992, 4352, 6912, 7296, 7424, 4096, 4992, 2816, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 4608, 6528, 4480, 8576, 5248, 6912, 7040, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6784, 6528, 8320, 8832, 4480, 1792, 4224, 7552, 1280, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10880, 1792, 5248, 2816, 2944, 9728, 7424, 8960, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3840, 3200, 11904, 6016, 1536, 5888, 9600, 7552, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 4608, 6528, 4480, 8576, 5248, 6912, 7040, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3968, 5376, 6272, 3072, 5888, 896, 6784, 10624, 6912, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[697, 840, 786, 346, 963, 262, 322, 1057, 86, 926, 1414, 399, 614, 969, 839, 1496, 1029, 1234, 992, 410, 1426, 174, 2014, 1096, 690, 644, 689, 824, 83, 300, 172, 102, 519, 311, 447, 135, 1115, 1750, 1194, 1026, 2049, 1548, 1143, 1290, 386, 1041, 70, 202, 805, 1250, 860, 20, 173, 1631, 261, 1283, 947, 370, 666, 575, 937, 590, 256, 407] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[758, 812, 974, 432, 893, 242, 271, 1115, 91, 962, 1089, 308, 540, 940, 1055, 1387, 1375, 1185, 891, 437, 1544, 121, 1678, 1050, 558, 686, 659, 787, 94, 411, 167, 68, 453, 232, 561, 203, 1156, 2011, 1059, 1044, 2251, 1524, 1395, 1584, 472, 688, 29, 183, 624, 1365, 682, 26, 169, 1716, 203, 1224, 674, 322, 864, 584, 1017, 573, 282, 402] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[780, 742, 961, 410, 888, 295, 402, 1093, 108, 1059, 1403, 346, 600, 1055, 878, 1481, 1197, 1250, 835, 412, 1417, 135, 1630, 1004, 592, 645, 654, 801, 95, 299, 262, 133, 488, 295, 533, 245, 1146, 1815, 1074, 1132, 2194, 1516, 1292, 1372, 438, 770, 59, 207, 679, 1273, 689, 32, 195, 1628, 219, 1399, 766, 301, 722, 585, 1042, 556, 292, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[717, 806, 893, 481, 992, 260, 296, 1149, 82, 965, 1411, 378, 577, 1012, 886, 1655, 1154, 1150, 968, 400, 1352, 147, 1752, 1111, 631, 742, 779, 885, 103, 261, 231, 123, 344, 209, 446, 123, 1154, 1657, 1016, 944, 2181, 1487, 1066, 1309, 405, 1021, 41, 212, 901, 1198, 1030, 14, 125, 1475, 186, 1261, 924, 334, 890, 606, 859, 635, 348, 402] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[648, 899, 923, 463, 874, 217, 256, 1088, 80, 1004, 1314, 335, 550, 945, 840, 1391, 1197, 1144, 964, 524, 1313, 161, 1695, 1098, 634, 725, 632, 819, 117, 494, 200, 115, 480, 364, 518, 137, 1012, 1830, 1191, 1072, 2178, 1537, 1221, 1491, 445, 893, 52, 219, 753, 1127, 891, 14, 199, 1625, 219, 1328, 768, 340, 804, 570, 976, 545, 288, 406] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[758, 812, 974, 432, 893, 242, 271, 1115, 91, 962, 1089, 308, 540, 940, 1055, 1387, 1375, 1185, 891, 437, 1544, 121, 1678, 1050, 558, 686, 659, 787, 94, 411, 167, 68, 453, 232, 561, 203, 1156, 2011, 1059, 1044, 2251, 1524, 1395, 1584, 472, 688, 29, 183, 624, 1365, 682, 26, 169, 1716, 203, 1224, 674, 322, 864, 584, 1017, 573, 282, 402] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[685, 945, 923, 432, 963, 258, 334, 1194, 112, 960, 1366, 367, 547, 1047, 864, 1429, 1163, 1196, 853, 444, 1401, 189, 1544, 1121, 625, 709, 707, 840, 119, 412, 269, 142, 393, 275, 482, 124, 1096, 1859, 1100, 973, 2058, 1560, 1124, 1378, 509, 950, 36, 174, 671, 1260, 860, 21, 179, 1618, 237, 1249, 930, 316, 765, 606, 877, 634, 298, 380] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[717, 806, 893, 481, 992, 260, 296, 1149, 82, 965, 1411, 378, 577, 1012, 886, 1655, 1154, 1150, 968, 400, 1352, 147, 1752, 1111, 631, 742, 779, 885, 103, 261, 231, 123, 344, 209, 446, 123, 1154, 1657, 1016, 944, 2181, 1487, 1066, 1309, 405, 1021, 41, 212, 901, 1198, 1030, 14, 125, 1475, 186, 1261, 924, 334, 890, 606, 859, 635, 348, 402] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[644, 785, 1014, 473, 1024, 235, 315, 1093, 150, 1157, 1192, 362, 669, 1064, 987, 1460, 1260, 1260, 864, 338, 1367, 123, 1724, 1002, 624, 564, 588, 756, 104, 303, 222, 103, 460, 238, 588, 271, 1053, 1902, 1018, 1086, 2283, 1516, 1255, 1444, 429, 756, 28, 217, 732, 1296, 704, 45, 188, 1564, 197, 1268, 784, 335, 807, 592, 1015, 576, 329, 350] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[780, 742, 961, 410, 888, 295, 402, 1093, 108, 1059, 1403, 346, 600, 1055, 878, 1481, 1197, 1250, 835, 412, 1417, 135, 1630, 1004, 592, 645, 654, 801, 95, 299, 262, 133, 488, 295, 533, 245, 1146, 1815, 1074, 1132, 2194, 1516, 1292, 1372, 438, 770, 59, 207, 679, 1273, 689, 32, 195, 1628, 219, 1399, 766, 301, 722, 585, 1042, 556, 292, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[739, 860, 902, 420, 1037, 209, 301, 1241, 125, 1004, 1340, 378, 593, 1014, 937, 1528, 1172, 1166, 809, 439, 1268, 140, 1568, 1195, 598, 765, 632, 801, 135, 372, 204, 130, 441, 236, 530, 134, 1041, 1766, 1068, 941, 2171, 1372, 1222, 1357, 463, 880, 38, 203, 864, 1274, 989, 20, 152, 1561, 220, 1158, 825, 359, 962, 643, 869, 633, 298, 440] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[780, 742, 961, 410, 888, 295, 402, 1093, 294, 184, 127, 0, 0, 0, 0, 0, 108, 1059, 1403, 346, 600, 1055, 878, 859, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 1250, 835, 412, 1224, 135, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 592, 645, 654, 801, 95, 299, 262, 133, 1734, 813, 0, 0, 0, 0, 0, 0, 488, 295, 533, 245, 1146, 1521, 1074, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 460, 1516, 1292, 1372, 438, 770, 59, 207, 0, 0, 0, 0, 0, 0, 0, 0, 679, 1273, 689, 32, 195, 1628, 219, 1399, 9, 0, 0, 0, 0, 0, 0, 0, 766, 301, 722, 585, 1042, 556, 292, 336, 817, 495, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 6308, 6057, 6028, 6434, 6114, 6123, 5912] avg=6144.0000 max=6434 max_rank=4 min=5912 min_rank=7 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[758, 812, 974, 432, 893, 242, 271, 1115, 325, 200, 119, 0, 0, 0, 0, 0, 91, 962, 1089, 308, 540, 940, 1055, 805, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 1185, 891, 437, 1334, 121, 0, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 558, 686, 659, 787, 94, 411, 167, 68, 1780, 838, 0, 0, 0, 0, 0, 0, 453, 232, 561, 203, 1156, 1686, 1059, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 471, 1524, 1395, 1584, 472, 688, 29, 183, 0, 0, 0, 0, 0, 0, 0, 0, 624, 1365, 682, 26, 169, 1716, 203, 1224, 10, 0, 0, 0, 0, 0, 0, 0, 674, 322, 864, 584, 1017, 573, 282, 402, 840, 463, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 5790, 6393, 6048, 6394, 6346, 6019, 6021] avg=6144.0000 max=6394 max_rank=4 min=5790 min_rank=1 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[717, 806, 893, 481, 992, 260, 296, 1149, 268, 175, 142, 0, 0, 0, 0, 0, 82, 965, 1411, 378, 577, 1012, 886, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 1150, 968, 400, 1169, 147, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 631, 742, 779, 885, 103, 261, 231, 123, 1724, 874, 0, 0, 0, 0, 0, 0, 344, 209, 446, 123, 1154, 1389, 1016, 944, 0, 0, 0, 0, 0, 0, 0, 0, 457, 1487, 1066, 1309, 405, 1021, 41, 212, 0, 0, 0, 0, 0, 0, 0, 0, 901, 1198, 1030, 14, 125, 1475, 186, 1261, 8, 0, 0, 0, 0, 0, 0, 0, 924, 334, 890, 606, 859, 635, 348, 402, 878, 553, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6179, 6271, 6099, 6353, 5625, 5998, 6198, 6429] avg=6144.0000 max=6429 max_rank=7 min=5625 min_rank=4 imbalance_ratio=1.046387 max_load_reduction=42723 max_load_reduction_pct=0.869202 moved_to_echo=24250 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[648, 899, 923, 463, 874, 217, 256, 1088, 296, 170, 119, 0, 0, 0, 0, 0, 80, 1004, 1314, 335, 550, 945, 840, 808, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 1144, 964, 524, 1135, 161, 0, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 634, 725, 632, 819, 117, 494, 200, 115, 1722, 845, 0, 0, 0, 0, 0, 0, 480, 364, 518, 137, 1012, 1534, 1191, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 456, 1537, 1221, 1491, 445, 893, 52, 219, 0, 0, 0, 0, 0, 0, 0, 0, 753, 1127, 891, 14, 199, 1625, 219, 1328, 8, 0, 0, 0, 0, 0, 0, 0, 768, 340, 804, 570, 976, 545, 288, 406, 850, 464, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5953, 5876, 6223, 6303, 6308, 6314, 6164, 6011] avg=6144.0000 max=6314 max_rank=5 min=5876 min_rank=1 imbalance_ratio=1.027669 max_load_reduction=42838 max_load_reduction_pct=0.871541 moved_to_echo=24797 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[697, 840, 786, 346, 963, 262, 322, 1057, 283, 185, 128, 0, 0, 0, 0, 0, 86, 926, 1414, 399, 614, 969, 839, 868, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1234, 992, 410, 1232, 174, 0, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 690, 644, 689, 824, 83, 300, 172, 102, 1620, 1005, 0, 0, 0, 0, 0, 0, 519, 311, 447, 135, 1115, 1467, 1194, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 429, 1548, 1143, 1290, 386, 1041, 70, 202, 0, 0, 0, 0, 0, 0, 0, 0, 805, 1250, 860, 20, 173, 1631, 261, 1283, 9, 0, 0, 0, 0, 0, 0, 0, 947, 370, 666, 575, 937, 590, 256, 407, 1009, 500, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5869, 6115, 6167, 6129, 6214, 6109, 6292, 6257] avg=6144.0000 max=6292 max_rank=6 min=5869 min_rank=0 imbalance_ratio=1.024089 max_load_reduction=42860 max_load_reduction_pct=0.871989 moved_to_echo=24872 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[780, 742, 961, 410, 888, 295, 402, 1093, 294, 184, 127, 0, 0, 0, 0, 0, 108, 1059, 1403, 346, 600, 1055, 878, 859, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 1250, 835, 412, 1224, 135, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 592, 645, 654, 801, 95, 299, 262, 133, 1734, 813, 0, 0, 0, 0, 0, 0, 488, 295, 533, 245, 1146, 1521, 1074, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 460, 1516, 1292, 1372, 438, 770, 59, 207, 0, 0, 0, 0, 0, 0, 0, 0, 679, 1273, 689, 32, 195, 1628, 219, 1399, 9, 0, 0, 0, 0, 0, 0, 0, 766, 301, 722, 585, 1042, 556, 292, 336, 817, 495, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 6308, 6057, 6028, 6434, 6114, 6123, 5912] avg=6144.0000 max=6434 max_rank=4 min=5912 min_rank=7 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[685, 945, 923, 432, 963, 258, 334, 1194, 301, 182, 122, 0, 0, 0, 0, 0, 112, 960, 1366, 367, 547, 1047, 864, 830, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 1196, 853, 444, 1210, 189, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 625, 709, 707, 840, 119, 412, 269, 142, 1627, 771, 0, 0, 0, 0, 0, 0, 393, 275, 482, 124, 1096, 1558, 1100, 973, 0, 0, 0, 0, 0, 0, 0, 0, 431, 1560, 1124, 1378, 509, 950, 36, 174, 0, 0, 0, 0, 0, 0, 0, 0, 671, 1260, 860, 21, 179, 1618, 237, 1249, 9, 0, 0, 0, 0, 0, 0, 0, 930, 316, 765, 606, 877, 634, 298, 380, 773, 477, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6339, 6093, 6176, 6221, 6001, 6162, 6104, 6056] avg=6144.0000 max=6339 max_rank=0 min=6001 min_rank=4 imbalance_ratio=1.031738 max_load_reduction=42813 max_load_reduction_pct=0.871033 moved_to_echo=24323 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[644, 785, 1014, 473, 1024, 235, 315, 1093, 311, 182, 129, 0, 0, 0, 0, 0, 150, 1157, 1192, 362, 669, 1064, 987, 840, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 1260, 864, 338, 1173, 123, 0, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 624, 564, 588, 756, 104, 303, 222, 103, 1809, 861, 0, 0, 0, 0, 0, 0, 460, 238, 588, 271, 1053, 1591, 1018, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 474, 1516, 1255, 1444, 429, 756, 28, 217, 0, 0, 0, 0, 0, 0, 0, 0, 732, 1296, 704, 45, 188, 1564, 197, 1268, 12, 0, 0, 0, 0, 0, 0, 0, 784, 335, 807, 592, 1015, 576, 329, 350, 863, 491, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6421, 6020, 5934, 6305, 6119, 6006, 6142] avg=6144.0000 max=6421 max_rank=1 min=5934 min_rank=3 imbalance_ratio=1.045085 max_load_reduction=42731 max_load_reduction_pct=0.869364 moved_to_echo=24572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[739, 860, 902, 420, 1037, 209, 301, 1241, 286, 164, 131, 0, 0, 0, 0, 0, 125, 1004, 1340, 378, 593, 1014, 937, 887, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 1166, 809, 439, 1096, 140, 0, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 598, 765, 632, 801, 135, 372, 204, 130, 1716, 783, 0, 0, 0, 0, 0, 0, 441, 236, 530, 134, 1041, 1480, 1068, 941, 0, 0, 0, 0, 0, 0, 0, 0, 455, 1372, 1222, 1357, 463, 880, 38, 203, 0, 0, 0, 0, 0, 0, 0, 0, 864, 1274, 989, 20, 152, 1561, 220, 1158, 8, 0, 0, 0, 0, 0, 0, 0, 825, 359, 962, 643, 869, 633, 298, 440, 785, 510, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6290, 6278, 6017, 6136, 5871, 5990, 6246, 6324] avg=6144.0000 max=6324 max_rank=7 min=5871 min_rank=4 imbalance_ratio=1.029297 max_load_reduction=42828 max_load_reduction_pct=0.871338 moved_to_echo=24431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[644, 785, 1014, 473, 1024, 235, 315, 1093, 311, 182, 129, 0, 0, 0, 0, 0, 150, 1157, 1192, 362, 669, 1064, 987, 840, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 1260, 864, 338, 1173, 123, 0, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 624, 564, 588, 756, 104, 303, 222, 103, 1809, 861, 0, 0, 0, 0, 0, 0, 460, 238, 588, 271, 1053, 1591, 1018, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 474, 1516, 1255, 1444, 429, 756, 28, 217, 0, 0, 0, 0, 0, 0, 0, 0, 732, 1296, 704, 45, 188, 1564, 197, 1268, 12, 0, 0, 0, 0, 0, 0, 0, 784, 335, 807, 592, 1015, 576, 329, 350, 863, 491, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6421, 6020, 5934, 6305, 6119, 6006, 6142] avg=6144.0000 max=6421 max_rank=1 min=5934 min_rank=3 imbalance_ratio=1.045085 max_load_reduction=42731 max_load_reduction_pct=0.869364 moved_to_echo=24572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[758, 812, 974, 432, 893, 242, 271, 1115, 325, 200, 119, 0, 0, 0, 0, 0, 91, 962, 1089, 308, 540, 940, 1055, 805, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 1185, 891, 437, 1334, 121, 0, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 558, 686, 659, 787, 94, 411, 167, 68, 1780, 838, 0, 0, 0, 0, 0, 0, 453, 232, 561, 203, 1156, 1686, 1059, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 471, 1524, 1395, 1584, 472, 688, 29, 183, 0, 0, 0, 0, 0, 0, 0, 0, 624, 1365, 682, 26, 169, 1716, 203, 1224, 10, 0, 0, 0, 0, 0, 0, 0, 674, 322, 864, 584, 1017, 573, 282, 402, 840, 463, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 5790, 6393, 6048, 6394, 6346, 6019, 6021] avg=6144.0000 max=6394 max_rank=4 min=5790 min_rank=1 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3712, 12160, 9728, 11264, 3584, 7040, 384, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 5504, 5376, 6528, 896, 2944, 1792, 1024, 13824, 6912, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 8064, 10624, 2944, 4736, 8064, 7296, 6912, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5760, 6784, 7424, 3584, 7680, 2048, 2560, 9088, 2432, 1536, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 5504, 5376, 6528, 896, 2944, 1792, 1024, 13824, 6912, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 2176, 4224, 1408, 8832, 12288, 8832, 8320, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6656, 2688, 6528, 4864, 7680, 4864, 2432, 3200, 6912, 3968, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6144, 10112, 6784, 256, 1408, 12928, 1792, 10240, 128, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9600, 9600, 7296, 3456, 9600, 1280, 0, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6144, 10112, 6784, 256, 1408, 12928, 1792, 10240, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6656, 2688, 6528, 4864, 7680, 4864, 2432, 3200, 6912, 3968, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[362, 852, 513, 308, 148, 1242, 1169, 286, 789, 357, 946, 977, 1525, 1396, 930, 1850, 887, 1134, 312, 423, 84, 479, 1106, 951, 225, 711, 916, 552, 394, 563, 832, 556, 470, 849, 1017, 682, 1573, 1349, 1873, 543, 392, 315, 308, 239, 1609, 1068, 190, 1711, 1042, 367, 974, 694, 178, 279, 273, 852, 1341, 327, 109, 858, 149, 1307, 716, 1723] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[226, 836, 553, 366, 89, 1197, 1057, 229, 896, 317, 991, 1117, 1489, 1244, 819, 1723, 905, 1310, 304, 375, 106, 438, 1303, 1042, 328, 698, 1024, 674, 426, 510, 803, 330, 387, 888, 1029, 628, 1759, 1304, 1896, 492, 478, 322, 385, 262, 1637, 1107, 225, 1516, 1030, 327, 916, 878, 101, 270, 390, 813, 1248, 298, 75, 678, 292, 1379, 541, 1876] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[208, 952, 557, 364, 86, 1365, 1051, 252, 865, 327, 1022, 1207, 1632, 1110, 680, 1681, 1000, 1343, 318, 366, 93, 336, 1169, 1037, 365, 680, 1075, 678, 372, 484, 803, 311, 452, 964, 1128, 564, 1702, 1586, 1884, 516, 462, 270, 328, 251, 1530, 1164, 193, 1415, 1017, 250, 1016, 891, 128, 324, 304, 751, 1285, 360, 69, 656, 202, 1404, 482, 1815] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[364, 783, 611, 331, 165, 1279, 1040, 273, 832, 312, 838, 1070, 1422, 1218, 1038, 1930, 834, 1100, 238, 363, 105, 595, 1213, 797, 182, 690, 784, 504, 370, 516, 785, 412, 416, 1020, 1126, 678, 1571, 1414, 2110, 590, 327, 316, 341, 156, 1700, 1171, 205, 1559, 1182, 328, 909, 736, 231, 444, 331, 984, 1258, 364, 74, 707, 231, 1200, 665, 1814] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[77, 861, 628, 310, 50, 1232, 954, 235, 691, 383, 984, 1323, 1634, 961, 141, 1433, 1027, 1562, 383, 354, 106, 213, 1179, 840, 497, 725, 1033, 897, 276, 497, 878, 220, 448, 1068, 1198, 571, 2281, 1880, 1840, 512, 457, 185, 272, 331, 1544, 1242, 166, 1523, 1030, 146, 891, 1019, 65, 155, 209, 757, 1539, 487, 67, 555, 99, 1413, 236, 2382] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[182, 927, 481, 386, 75, 1159, 979, 275, 628, 365, 1043, 1159, 1589, 1256, 491, 1560, 980, 1272, 320, 440, 96, 330, 1225, 1023, 272, 789, 1135, 826, 442, 546, 881, 267, 381, 1074, 985, 515, 1674, 1453, 1821, 547, 561, 285, 392, 288, 1741, 1231, 174, 1520, 1177, 287, 651, 836, 105, 183, 307, 718, 1366, 305, 104, 893, 140, 1540, 447, 2052] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[226, 836, 553, 366, 89, 1197, 1057, 229, 896, 317, 991, 1117, 1489, 1244, 819, 1723, 905, 1310, 304, 375, 106, 438, 1303, 1042, 328, 698, 1024, 674, 426, 510, 803, 330, 387, 888, 1029, 628, 1759, 1304, 1896, 492, 478, 322, 385, 262, 1637, 1107, 225, 1516, 1030, 327, 916, 878, 101, 270, 390, 813, 1248, 298, 75, 678, 292, 1379, 541, 1876] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[161, 916, 686, 341, 95, 1298, 891, 207, 902, 346, 1016, 1146, 1677, 972, 329, 1329, 1030, 1299, 295, 410, 87, 298, 1026, 915, 494, 742, 1168, 783, 364, 452, 859, 304, 420, 1084, 1253, 614, 1929, 1767, 1881, 479, 412, 298, 371, 206, 1551, 1378, 173, 1479, 1001, 219, 936, 824, 190, 240, 267, 727, 1489, 510, 89, 536, 179, 1298, 407, 2107] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[264, 907, 628, 330, 137, 1266, 931, 256, 854, 301, 922, 1073, 1679, 1322, 873, 1732, 929, 1213, 249, 465, 78, 426, 1068, 908, 237, 715, 910, 495, 347, 534, 774, 491, 360, 986, 1102, 732, 1663, 1564, 2094, 684, 327, 317, 377, 195, 1702, 1101, 175, 1521, 880, 310, 894, 935, 152, 312, 236, 913, 1286, 391, 77, 715, 146, 1182, 585, 1924] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[226, 836, 553, 366, 89, 1197, 1057, 229, 1156, 372, 0, 0, 0, 0, 0, 0, 896, 317, 991, 1117, 986, 1244, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 905, 1310, 304, 375, 106, 438, 1303, 1042, 503, 46, 46, 0, 0, 0, 0, 0, 328, 698, 1024, 674, 426, 510, 803, 330, 794, 567, 0, 0, 0, 0, 0, 0, 387, 888, 1029, 628, 919, 1304, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 478, 322, 385, 262, 1637, 1107, 225, 1516, 299, 0, 0, 0, 0, 0, 0, 0, 1030, 327, 916, 878, 101, 270, 390, 813, 1524, 0, 0, 0, 0, 0, 0, 0, 1248, 298, 75, 678, 292, 1379, 541, 1531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6370, 6378, 6154, 5647, 6231, 6249, 6042] avg=6144.0000 max=6378 max_rank=2 min=5647 min_rank=4 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=24169 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[208, 952, 557, 364, 86, 1365, 1051, 252, 1129, 371, 0, 0, 0, 0, 0, 0, 865, 327, 1022, 1207, 1081, 1110, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1343, 318, 366, 93, 336, 1169, 1037, 551, 45, 44, 0, 0, 0, 0, 0, 365, 680, 1075, 678, 372, 484, 803, 311, 769, 552, 0, 0, 0, 0, 0, 0, 452, 964, 1128, 564, 889, 1586, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 462, 270, 328, 251, 1530, 1164, 193, 1415, 289, 0, 0, 0, 0, 0, 0, 0, 1017, 250, 1016, 891, 128, 324, 304, 751, 1513, 0, 0, 0, 0, 0, 0, 0, 1285, 360, 69, 656, 202, 1404, 482, 1481, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6335, 6292, 6302, 6089, 6099, 5902, 6194, 5939] avg=6144.0000 max=6335 max_rank=0 min=5902 min_rank=5 imbalance_ratio=1.031087 max_load_reduction=42817 max_load_reduction_pct=0.871114 moved_to_echo=24134 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[182, 927, 481, 386, 75, 1159, 979, 275, 628, 365, 1043, 1159, 1589, 1256, 491, 1560, 980, 1272, 320, 440, 96, 330, 1225, 1023, 272, 789, 1135, 826, 442, 546, 881, 267, 381, 1074, 985, 515, 1674, 1453, 1821, 547, 561, 285, 392, 288, 1741, 1231, 174, 1520, 1177, 287, 651, 836, 105, 183, 307, 718, 1366, 305, 104, 893, 140, 1540, 447, 2052] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[208, 952, 557, 364, 86, 1365, 1051, 252, 865, 327, 1022, 1207, 1632, 1110, 680, 1681, 1000, 1343, 318, 366, 93, 336, 1169, 1037, 365, 680, 1075, 678, 372, 484, 803, 311, 452, 964, 1128, 564, 1702, 1586, 1884, 516, 462, 270, 328, 251, 1530, 1164, 193, 1415, 1017, 250, 1016, 891, 128, 324, 304, 751, 1285, 360, 69, 656, 202, 1404, 482, 1815] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[362, 852, 513, 308, 148, 1242, 1169, 286, 789, 357, 946, 977, 1525, 1396, 930, 1850, 887, 1134, 312, 423, 84, 479, 1106, 951, 225, 711, 916, 552, 394, 563, 832, 556, 470, 849, 1017, 682, 1573, 1349, 1873, 543, 392, 315, 308, 239, 1609, 1068, 190, 1711, 1042, 367, 974, 694, 178, 279, 273, 852, 1341, 327, 109, 858, 149, 1307, 716, 1723] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[77, 861, 628, 310, 50, 1232, 954, 235, 691, 383, 984, 1323, 1634, 961, 141, 1433, 1027, 1562, 383, 354, 106, 213, 1179, 840, 497, 725, 1033, 897, 276, 497, 878, 220, 448, 1068, 1198, 571, 2281, 1880, 1840, 512, 457, 185, 272, 331, 1544, 1242, 166, 1523, 1030, 146, 891, 1019, 65, 155, 209, 757, 1539, 487, 67, 555, 99, 1413, 236, 2382] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[364, 783, 611, 331, 165, 1279, 1040, 273, 1296, 416, 0, 0, 0, 0, 0, 0, 832, 312, 838, 1070, 942, 1218, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 1100, 238, 363, 105, 595, 1213, 797, 480, 45, 41, 0, 0, 0, 0, 0, 182, 690, 784, 504, 370, 516, 785, 412, 709, 634, 0, 0, 0, 0, 0, 0, 416, 1020, 1126, 678, 821, 1414, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 327, 316, 341, 156, 1700, 1171, 205, 1559, 289, 0, 0, 0, 0, 0, 0, 0, 1182, 328, 909, 736, 231, 444, 331, 984, 1694, 0, 0, 0, 0, 0, 0, 0, 1258, 364, 74, 707, 231, 1200, 665, 1480, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6558, 6250, 5811, 5586, 6065, 6064, 6839, 5979] avg=6144.0000 max=6839 max_rank=6 min=5586 min_rank=3 imbalance_ratio=1.113118 max_load_reduction=42313 max_load_reduction_pct=0.860860 moved_to_echo=24947 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[77, 861, 628, 310, 50, 1232, 954, 235, 961, 362, 0, 0, 0, 0, 0, 0, 691, 383, 984, 1323, 1082, 961, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1562, 383, 354, 106, 213, 1179, 840, 552, 59, 60, 0, 0, 0, 0, 0, 497, 725, 1033, 897, 276, 497, 878, 220, 1030, 472, 0, 0, 0, 0, 0, 0, 448, 1068, 1198, 571, 1191, 1880, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 457, 185, 272, 331, 1544, 1242, 166, 1523, 380, 0, 0, 0, 0, 0, 0, 0, 1030, 146, 891, 1019, 65, 155, 209, 757, 1478, 0, 0, 0, 0, 0, 0, 0, 1539, 487, 67, 555, 99, 1413, 236, 1943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5670, 5565, 6335, 6525, 6868, 6100, 5750, 6339] avg=6144.0000 max=6868 max_rank=4 min=5565 min_rank=1 imbalance_ratio=1.117839 max_load_reduction=42284 max_load_reduction_pct=0.860270 moved_to_echo=25057 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[264, 907, 628, 330, 137, 1266, 931, 256, 1163, 412, 0, 0, 0, 0, 0, 0, 854, 301, 922, 1073, 1108, 1322, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 1213, 249, 465, 78, 426, 1068, 908, 571, 51, 48, 0, 0, 0, 0, 0, 237, 715, 910, 495, 347, 534, 774, 491, 756, 569, 0, 0, 0, 0, 0, 0, 360, 986, 1102, 732, 859, 1564, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 327, 317, 377, 195, 1702, 1101, 175, 1521, 310, 0, 0, 0, 0, 0, 0, 0, 880, 310, 894, 935, 152, 312, 236, 913, 1682, 0, 0, 0, 0, 0, 0, 0, 1286, 391, 77, 715, 146, 1182, 585, 1563, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6294, 6453, 6006, 5828, 6287, 6025, 6314, 5945] avg=6144.0000 max=6453 max_rank=1 min=5828 min_rank=3 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[226, 836, 553, 366, 89, 1197, 1057, 229, 1156, 372, 0, 0, 0, 0, 0, 0, 896, 317, 991, 1117, 986, 1244, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 905, 1310, 304, 375, 106, 438, 1303, 1042, 503, 46, 46, 0, 0, 0, 0, 0, 328, 698, 1024, 674, 426, 510, 803, 330, 794, 567, 0, 0, 0, 0, 0, 0, 387, 888, 1029, 628, 919, 1304, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 478, 322, 385, 262, 1637, 1107, 225, 1516, 299, 0, 0, 0, 0, 0, 0, 0, 1030, 327, 916, 878, 101, 270, 390, 813, 1524, 0, 0, 0, 0, 0, 0, 0, 1248, 298, 75, 678, 292, 1379, 541, 1531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6370, 6378, 6154, 5647, 6231, 6249, 6042] avg=6144.0000 max=6378 max_rank=2 min=5647 min_rank=4 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=24169 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[182, 927, 481, 386, 75, 1159, 979, 275, 1046, 358, 0, 0, 0, 0, 0, 0, 628, 365, 1043, 1159, 1053, 1256, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1272, 320, 440, 96, 330, 1225, 1023, 536, 51, 44, 0, 0, 0, 0, 0, 272, 789, 1135, 826, 442, 546, 881, 267, 756, 514, 0, 0, 0, 0, 0, 0, 381, 1074, 985, 515, 874, 1453, 0, 547, 0, 0, 0, 0, 0, 0, 0, 0, 561, 285, 392, 288, 1741, 1231, 174, 1520, 327, 0, 0, 0, 0, 0, 0, 0, 1177, 287, 651, 836, 105, 183, 307, 718, 1463, 0, 0, 0, 0, 0, 0, 0, 1366, 305, 104, 893, 140, 1540, 447, 1674, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5868, 5995, 6317, 6428, 5829, 6519, 5727, 6469] avg=6144.0000 max=6519 max_rank=5 min=5727 min_rank=6 imbalance_ratio=1.061035 max_load_reduction=42633 max_load_reduction_pct=0.867371 moved_to_echo=24544 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[161, 916, 686, 341, 95, 1298, 891, 207, 892, 370, 0, 0, 0, 0, 0, 0, 902, 346, 1016, 1146, 1111, 972, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1299, 295, 410, 87, 298, 1026, 915, 566, 52, 50, 0, 0, 0, 0, 0, 494, 742, 1168, 783, 364, 452, 859, 304, 871, 437, 0, 0, 0, 0, 0, 0, 420, 1084, 1253, 614, 1008, 1767, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 412, 298, 371, 206, 1551, 1378, 173, 1479, 336, 0, 0, 0, 0, 0, 0, 0, 1001, 219, 936, 824, 190, 240, 267, 727, 1511, 0, 0, 0, 0, 0, 0, 0, 1489, 510, 89, 536, 179, 1298, 407, 1719, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5857, 5822, 6028, 6474, 6625, 6204, 5915, 6227] avg=6144.0000 max=6625 max_rank=4 min=5822 min_rank=1 imbalance_ratio=1.078288 max_load_reduction=42527 max_load_reduction_pct=0.865214 moved_to_echo=24971 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[362, 852, 513, 308, 148, 1242, 1169, 286, 1241, 368, 0, 0, 0, 0, 0, 0, 789, 357, 946, 977, 1010, 1396, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 1134, 312, 423, 84, 479, 1106, 951, 515, 43, 41, 0, 0, 0, 0, 0, 225, 711, 916, 552, 394, 563, 832, 556, 710, 609, 0, 0, 0, 0, 0, 0, 470, 849, 1017, 682, 822, 1349, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 392, 315, 308, 239, 1609, 1068, 190, 1711, 275, 0, 0, 0, 0, 0, 0, 0, 1042, 367, 974, 694, 178, 279, 273, 852, 1505, 0, 0, 0, 0, 0, 0, 0, 1341, 327, 109, 858, 149, 1307, 716, 1405, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6489, 6405, 5975, 6068, 5732, 6107, 6164, 6212] avg=6144.0000 max=6489 max_rank=0 min=5732 min_rank=4 imbalance_ratio=1.056152 max_load_reduction=42663 max_load_reduction_pct=0.867981 moved_to_echo=24215 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[208, 952, 557, 364, 86, 1365, 1051, 252, 1129, 371, 0, 0, 0, 0, 0, 0, 865, 327, 1022, 1207, 1081, 1110, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1343, 318, 366, 93, 336, 1169, 1037, 551, 45, 44, 0, 0, 0, 0, 0, 365, 680, 1075, 678, 372, 484, 803, 311, 769, 552, 0, 0, 0, 0, 0, 0, 452, 964, 1128, 564, 889, 1586, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 462, 270, 328, 251, 1530, 1164, 193, 1415, 289, 0, 0, 0, 0, 0, 0, 0, 1017, 250, 1016, 891, 128, 324, 304, 751, 1513, 0, 0, 0, 0, 0, 0, 0, 1285, 360, 69, 656, 202, 1404, 482, 1481, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6335, 6292, 6302, 6089, 6099, 5902, 6194, 5939] avg=6144.0000 max=6335 max_rank=0 min=5902 min_rank=5 imbalance_ratio=1.031087 max_load_reduction=42817 max_load_reduction_pct=0.871114 moved_to_echo=24134 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[264, 907, 628, 330, 137, 1266, 931, 256, 1163, 412, 0, 0, 0, 0, 0, 0, 854, 301, 922, 1073, 1108, 1322, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 1213, 249, 465, 78, 426, 1068, 908, 571, 51, 48, 0, 0, 0, 0, 0, 237, 715, 910, 495, 347, 534, 774, 491, 756, 569, 0, 0, 0, 0, 0, 0, 360, 986, 1102, 732, 859, 1564, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 327, 317, 377, 195, 1702, 1101, 175, 1521, 310, 0, 0, 0, 0, 0, 0, 0, 880, 310, 894, 935, 152, 312, 236, 913, 1682, 0, 0, 0, 0, 0, 0, 0, 1286, 391, 77, 715, 146, 1182, 585, 1563, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6294, 6453, 6006, 5828, 6287, 6025, 6314, 5945] avg=6144.0000 max=6453 max_rank=1 min=5828 min_rank=3 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[77, 861, 628, 310, 50, 1232, 954, 235, 961, 362, 0, 0, 0, 0, 0, 0, 691, 383, 984, 1323, 1082, 961, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1562, 383, 354, 106, 213, 1179, 840, 552, 59, 60, 0, 0, 0, 0, 0, 497, 725, 1033, 897, 276, 497, 878, 220, 1030, 472, 0, 0, 0, 0, 0, 0, 448, 1068, 1198, 571, 1191, 1880, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 457, 185, 272, 331, 1544, 1242, 166, 1523, 380, 0, 0, 0, 0, 0, 0, 0, 1030, 146, 891, 1019, 65, 155, 209, 757, 1478, 0, 0, 0, 0, 0, 0, 0, 1539, 487, 67, 555, 99, 1413, 236, 1943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5670, 5565, 6335, 6525, 6868, 6100, 5750, 6339] avg=6144.0000 max=6868 max_rank=4 min=5565 min_rank=1 imbalance_ratio=1.117839 max_load_reduction=42284 max_load_reduction_pct=0.860270 moved_to_echo=25057 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3456, 7936, 8960, 4992, 7424, 12416, 0, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3456, 2432, 2816, 2048, 13056, 9472, 1536, 12288, 2560, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 7040, 4736, 2816, 896, 10112, 8192, 2048, 8960, 3072, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6528, 2816, 7808, 9088, 8448, 9600, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10880, 3072, 768, 5632, 1536, 10752, 4096, 12800, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8448, 2304, 7296, 6912, 1152, 2304, 2432, 6528, 12416, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 5760, 8064, 5504, 3072, 4224, 6656, 2944, 6400, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3456, 2432, 2816, 2048, 13056, 9472, 1536, 12288, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10880, 3072, 768, 5632, 1536, 10752, 4096, 12800, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6528, 2816, 7808, 9088, 8448, 9600, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 10240, 2432, 3200, 768, 3200, 9344, 7552, 4352, 512, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 5760, 8064, 5504, 3072, 4224, 6656, 2944, 6400, 4480, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[582, 1150, 1181, 614, 398, 448, 787, 655, 1183, 2155, 276, 363, 829, 491, 899, 351, 1567, 85, 2029, 604, 537, 317, 60, 844, 576, 613, 699, 673, 781, 756, 1352, 946, 555, 689, 295, 370, 508, 708, 1729, 765, 580, 1941, 149, 563, 440, 879, 677, 3088, 353, 464, 426, 928, 648, 1091, 385, 731, 189, 407, 1414, 620, 1073, 700, 660, 326] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[281, 577, 1582, 314, 648, 582, 734, 731, 2020, 2974, 352, 301, 1349, 449, 1356, 371, 2046, 45, 2679, 464, 602, 395, 8, 683, 910, 881, 653, 491, 656, 872, 1245, 717, 323, 620, 219, 243, 726, 467, 1334, 670, 310, 2753, 94, 497, 229, 531, 415, 3151, 34, 221, 187, 1131, 355, 874, 173, 247, 236, 311, 1370, 627, 918, 869, 523, 526] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[719, 1257, 1265, 556, 402, 447, 765, 588, 1075, 2261, 240, 440, 1038, 481, 811, 419, 1534, 61, 2149, 609, 528, 373, 46, 901, 929, 736, 633, 674, 581, 618, 1339, 907, 220, 740, 238, 363, 668, 556, 1787, 686, 715, 2055, 150, 513, 304, 683, 692, 3549, 320, 294, 296, 916, 530, 1180, 378, 587, 210, 316, 1402, 623, 1078, 746, 645, 330] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[477, 950, 1358, 516, 485, 539, 736, 712, 1505, 2361, 275, 373, 858, 633, 1023, 375, 1486, 70, 2198, 590, 482, 353, 15, 715, 798, 727, 628, 510, 698, 689, 1295, 1063, 491, 747, 297, 341, 653, 610, 1779, 700, 442, 2129, 121, 511, 393, 695, 624, 3147, 303, 316, 322, 985, 479, 991, 529, 676, 285, 473, 1204, 596, 940, 809, 656, 415] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[425, 1063, 1398, 564, 507, 497, 725, 567, 1173, 2292, 214, 469, 914, 556, 1019, 406, 1969, 83, 2207, 474, 470, 400, 27, 766, 1087, 679, 616, 580, 640, 848, 1540, 972, 223, 712, 235, 361, 695, 579, 2081, 745, 391, 1825, 162, 519, 322, 612, 548, 3237, 220, 315, 247, 1021, 657, 946, 281, 515, 391, 401, 1215, 590, 1026, 817, 692, 424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[498, 819, 1478, 317, 621, 529, 717, 820, 1445, 2688, 323, 385, 1030, 596, 1095, 286, 1640, 134, 2225, 606, 645, 312, 18, 797, 906, 762, 657, 487, 664, 776, 1188, 884, 503, 707, 228, 348, 677, 558, 1452, 692, 552, 2393, 128, 479, 313, 690, 418, 3243, 128, 273, 230, 1102, 437, 1064, 325, 488, 282, 355, 1396, 596, 964, 842, 569, 372] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[708, 1451, 1197, 560, 365, 386, 723, 574, 1138, 2186, 226, 370, 825, 538, 838, 412, 1404, 57, 1882, 548, 458, 247, 36, 761, 669, 461, 652, 589, 593, 597, 1268, 1072, 276, 827, 385, 414, 521, 645, 1858, 734, 861, 1933, 158, 559, 368, 1105, 978, 2909, 628, 424, 575, 789, 535, 1132, 657, 963, 228, 468, 1232, 651, 940, 674, 619, 315] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[719, 1257, 1265, 556, 402, 447, 765, 588, 1075, 2261, 240, 440, 1038, 481, 811, 419, 1534, 61, 2149, 609, 528, 373, 46, 901, 929, 736, 633, 674, 581, 618, 1339, 907, 220, 740, 238, 363, 668, 556, 1787, 686, 715, 2055, 150, 513, 304, 683, 692, 3549, 320, 294, 296, 916, 530, 1180, 378, 587, 210, 316, 1402, 623, 1078, 746, 645, 330] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[498, 819, 1478, 317, 621, 529, 717, 820, 1445, 2688, 323, 385, 1030, 596, 1095, 286, 1640, 134, 2225, 606, 645, 312, 18, 797, 906, 762, 657, 487, 664, 776, 1188, 884, 503, 707, 228, 348, 677, 558, 1452, 692, 552, 2393, 128, 479, 313, 690, 418, 3243, 128, 273, 230, 1102, 437, 1064, 325, 488, 282, 355, 1396, 596, 964, 842, 569, 372] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[425, 1063, 1398, 564, 507, 497, 725, 567, 1173, 2292, 214, 469, 914, 556, 1019, 406, 1969, 83, 2207, 474, 470, 400, 27, 766, 1087, 679, 616, 580, 640, 848, 1540, 972, 223, 712, 235, 361, 695, 579, 2081, 745, 391, 1825, 162, 519, 322, 612, 548, 3237, 220, 315, 247, 1021, 657, 946, 281, 515, 391, 401, 1215, 590, 1026, 817, 692, 424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[446, 736, 1742, 286, 553, 720, 796, 860, 1807, 2796, 408, 346, 1299, 515, 1324, 199, 1645, 68, 2282, 541, 593, 336, 19, 749, 809, 965, 620, 446, 487, 851, 1230, 741, 556, 631, 225, 269, 649, 526, 1201, 673, 474, 2964, 126, 476, 264, 573, 431, 3161, 69, 243, 254, 1090, 369, 943, 274, 341, 205, 332, 1375, 576, 884, 911, 482, 390] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[582, 1150, 1181, 614, 398, 448, 787, 655, 236, 37, 34, 0, 0, 0, 0, 0, 1183, 970, 276, 363, 829, 491, 899, 351, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 85, 1992, 604, 537, 317, 60, 844, 0, 0, 0, 0, 0, 0, 0, 0, 576, 613, 699, 673, 781, 756, 1116, 946, 0, 0, 0, 0, 0, 0, 0, 0, 555, 689, 295, 370, 508, 708, 1729, 765, 447, 336, 0, 0, 0, 0, 0, 0, 580, 1941, 149, 563, 440, 879, 677, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 353, 464, 426, 928, 648, 1091, 385, 731, 1728, 0, 0, 0, 0, 0, 0, 0, 189, 407, 1414, 620, 1073, 700, 660, 326, 704, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 5362, 6006, 6160, 6402, 6253, 6754, 6093] avg=6144.0000 max=6754 max_rank=6 min=5362 min_rank=1 imbalance_ratio=1.099284 max_load_reduction=42398 max_load_reduction_pct=0.862590 moved_to_echo=25502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[582, 1150, 1181, 614, 398, 448, 787, 655, 1183, 2155, 276, 363, 829, 491, 899, 351, 1567, 85, 2029, 604, 537, 317, 60, 844, 576, 613, 699, 673, 781, 756, 1352, 946, 555, 689, 295, 370, 508, 708, 1729, 765, 580, 1941, 149, 563, 440, 879, 677, 3088, 353, 464, 426, 928, 648, 1091, 385, 731, 189, 407, 1414, 620, 1073, 700, 660, 326] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[281, 577, 1582, 314, 648, 582, 734, 731, 217, 48, 47, 0, 0, 0, 0, 0, 2020, 1339, 352, 301, 1349, 449, 1356, 371, 0, 0, 0, 0, 0, 0, 0, 0, 2046, 45, 2631, 464, 602, 395, 8, 683, 0, 0, 0, 0, 0, 0, 0, 0, 910, 881, 653, 491, 656, 872, 1028, 717, 0, 0, 0, 0, 0, 0, 0, 0, 323, 620, 219, 243, 726, 467, 1334, 670, 617, 342, 0, 0, 0, 0, 0, 0, 310, 2753, 94, 497, 229, 531, 415, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 34, 221, 187, 1131, 355, 874, 173, 247, 1763, 0, 0, 0, 0, 0, 0, 0, 236, 311, 1370, 627, 918, 869, 523, 526, 971, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5761, 7537, 6874, 6208, 5561, 5875, 4985, 6351] avg=6144.0000 max=7537 max_rank=1 min=4985 min_rank=6 imbalance_ratio=1.226725 max_load_reduction=41615 max_load_reduction_pct=0.846659 moved_to_echo=22772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[708, 1451, 1197, 560, 365, 386, 723, 574, 221, 34, 35, 0, 0, 0, 0, 0, 1138, 983, 226, 370, 825, 538, 838, 412, 0, 0, 0, 0, 0, 0, 0, 0, 1404, 57, 1848, 548, 458, 247, 36, 761, 0, 0, 0, 0, 0, 0, 0, 0, 669, 461, 652, 589, 593, 597, 1047, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 276, 827, 385, 414, 521, 645, 1858, 734, 454, 316, 0, 0, 0, 0, 0, 0, 861, 1933, 158, 559, 368, 1105, 978, 965, 0, 0, 0, 0, 0, 0, 0, 0, 628, 424, 575, 789, 535, 1132, 657, 963, 1628, 0, 0, 0, 0, 0, 0, 0, 228, 468, 1232, 651, 940, 674, 619, 315, 714, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6254, 5330, 5359, 5680, 6430, 6927, 7331, 5841] avg=6144.0000 max=7331 max_rank=6 min=5330 min_rank=1 imbalance_ratio=1.193197 max_load_reduction=41821 max_load_reduction_pct=0.850850 moved_to_echo=26529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[477, 950, 1358, 516, 485, 539, 736, 712, 226, 40, 37, 0, 0, 0, 0, 0, 1505, 1063, 275, 373, 858, 633, 1023, 375, 0, 0, 0, 0, 0, 0, 0, 0, 1486, 70, 2158, 590, 482, 353, 15, 715, 0, 0, 0, 0, 0, 0, 0, 0, 798, 727, 628, 510, 698, 689, 1069, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 491, 747, 297, 341, 653, 610, 1779, 700, 490, 342, 0, 0, 0, 0, 0, 0, 442, 2129, 121, 511, 393, 695, 624, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 303, 316, 322, 985, 479, 991, 529, 676, 1761, 0, 0, 0, 0, 0, 0, 0, 285, 473, 1204, 596, 940, 809, 656, 415, 771, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 6105, 5869, 6182, 6450, 5959, 6362, 6149] avg=6144.0000 max=6450 max_rank=4 min=5869 min_rank=2 imbalance_ratio=1.049805 max_load_reduction=42702 max_load_reduction_pct=0.868774 moved_to_echo=24920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[582, 1150, 1181, 614, 398, 448, 787, 655, 236, 37, 34, 0, 0, 0, 0, 0, 1183, 970, 276, 363, 829, 491, 899, 351, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 85, 1992, 604, 537, 317, 60, 844, 0, 0, 0, 0, 0, 0, 0, 0, 576, 613, 699, 673, 781, 756, 1116, 946, 0, 0, 0, 0, 0, 0, 0, 0, 555, 689, 295, 370, 508, 708, 1729, 765, 447, 336, 0, 0, 0, 0, 0, 0, 580, 1941, 149, 563, 440, 879, 677, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 353, 464, 426, 928, 648, 1091, 385, 731, 1728, 0, 0, 0, 0, 0, 0, 0, 189, 407, 1414, 620, 1073, 700, 660, 326, 704, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 5362, 6006, 6160, 6402, 6253, 6754, 6093] avg=6144.0000 max=6754 max_rank=6 min=5362 min_rank=1 imbalance_ratio=1.099284 max_load_reduction=42398 max_load_reduction_pct=0.862590 moved_to_echo=25502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 819, 1478, 317, 621, 529, 717, 820, 207, 40, 43, 0, 0, 0, 0, 0, 1445, 1209, 323, 385, 1030, 596, 1095, 286, 0, 0, 0, 0, 0, 0, 0, 0, 1640, 134, 2185, 606, 645, 312, 18, 797, 0, 0, 0, 0, 0, 0, 0, 0, 906, 762, 657, 487, 664, 776, 981, 884, 0, 0, 0, 0, 0, 0, 0, 0, 503, 707, 228, 348, 677, 558, 1452, 692, 558, 352, 0, 0, 0, 0, 0, 0, 552, 2393, 128, 479, 313, 690, 418, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 128, 273, 230, 1102, 437, 1064, 325, 488, 1815, 0, 0, 0, 0, 0, 0, 0, 282, 355, 1396, 596, 964, 842, 569, 372, 878, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6089, 6369, 6337, 6117, 6075, 6049, 5862, 6254] avg=6144.0000 max=6369 max_rank=1 min=5862 min_rank=6 imbalance_ratio=1.036621 max_load_reduction=42783 max_load_reduction_pct=0.870422 moved_to_echo=24240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[708, 1451, 1197, 560, 365, 386, 723, 574, 221, 34, 35, 0, 0, 0, 0, 0, 1138, 983, 226, 370, 825, 538, 838, 412, 0, 0, 0, 0, 0, 0, 0, 0, 1404, 57, 1848, 548, 458, 247, 36, 761, 0, 0, 0, 0, 0, 0, 0, 0, 669, 461, 652, 589, 593, 597, 1047, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 276, 827, 385, 414, 521, 645, 1858, 734, 454, 316, 0, 0, 0, 0, 0, 0, 861, 1933, 158, 559, 368, 1105, 978, 965, 0, 0, 0, 0, 0, 0, 0, 0, 628, 424, 575, 789, 535, 1132, 657, 963, 1628, 0, 0, 0, 0, 0, 0, 0, 228, 468, 1232, 651, 940, 674, 619, 315, 714, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6254, 5330, 5359, 5680, 6430, 6927, 7331, 5841] avg=6144.0000 max=7331 max_rank=6 min=5330 min_rank=1 imbalance_ratio=1.193197 max_load_reduction=41821 max_load_reduction_pct=0.850850 moved_to_echo=26529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[281, 577, 1582, 314, 648, 582, 734, 731, 217, 48, 47, 0, 0, 0, 0, 0, 2020, 1339, 352, 301, 1349, 449, 1356, 371, 0, 0, 0, 0, 0, 0, 0, 0, 2046, 45, 2631, 464, 602, 395, 8, 683, 0, 0, 0, 0, 0, 0, 0, 0, 910, 881, 653, 491, 656, 872, 1028, 717, 0, 0, 0, 0, 0, 0, 0, 0, 323, 620, 219, 243, 726, 467, 1334, 670, 617, 342, 0, 0, 0, 0, 0, 0, 310, 2753, 94, 497, 229, 531, 415, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 34, 221, 187, 1131, 355, 874, 173, 247, 1763, 0, 0, 0, 0, 0, 0, 0, 236, 311, 1370, 627, 918, 869, 523, 526, 971, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5761, 7537, 6874, 6208, 5561, 5875, 4985, 6351] avg=6144.0000 max=7537 max_rank=1 min=4985 min_rank=6 imbalance_ratio=1.226725 max_load_reduction=41615 max_load_reduction_pct=0.846659 moved_to_echo=22772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[719, 1257, 1265, 556, 402, 447, 765, 588, 239, 42, 40, 0, 0, 0, 0, 0, 1075, 1005, 240, 440, 1038, 481, 811, 419, 0, 0, 0, 0, 0, 0, 0, 0, 1534, 61, 2107, 609, 528, 373, 46, 901, 0, 0, 0, 0, 0, 0, 0, 0, 929, 736, 633, 674, 581, 618, 1100, 907, 0, 0, 0, 0, 0, 0, 0, 0, 220, 740, 238, 363, 668, 556, 1787, 686, 474, 390, 0, 0, 0, 0, 0, 0, 715, 2055, 150, 513, 304, 683, 692, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 320, 294, 296, 916, 530, 1180, 378, 587, 1991, 0, 0, 0, 0, 0, 0, 0, 210, 316, 1402, 623, 1078, 746, 645, 330, 742, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6320, 5509, 6159, 6178, 6122, 6280, 6492, 6092] avg=6144.0000 max=6492 max_rank=6 min=5509 min_rank=1 imbalance_ratio=1.056641 max_load_reduction=42660 max_load_reduction_pct=0.867920 moved_to_echo=24986 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[425, 1063, 1398, 564, 507, 497, 725, 567, 269, 40, 36, 0, 0, 0, 0, 0, 1173, 1032, 214, 469, 914, 556, 1019, 406, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 83, 2167, 474, 470, 400, 27, 766, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 679, 616, 580, 640, 848, 1271, 972, 0, 0, 0, 0, 0, 0, 0, 0, 223, 712, 235, 361, 695, 579, 2081, 745, 476, 352, 0, 0, 0, 0, 0, 0, 391, 1825, 162, 519, 322, 612, 548, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 220, 315, 247, 1021, 657, 946, 281, 515, 1812, 0, 0, 0, 0, 0, 0, 0, 391, 401, 1215, 590, 1026, 817, 692, 424, 748, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 5783, 6356, 6693, 6459, 5452, 6014, 6304] avg=6144.0000 max=6693 max_rank=3 min=5452 min_rank=5 imbalance_ratio=1.089355 max_load_reduction=42459 max_load_reduction_pct=0.863831 moved_to_echo=24229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[446, 736, 1742, 286, 553, 720, 796, 860, 215, 41, 44, 0, 0, 0, 0, 0, 1807, 1259, 408, 346, 1299, 515, 1324, 199, 0, 0, 0, 0, 0, 0, 0, 0, 1645, 68, 2241, 541, 593, 336, 19, 749, 0, 0, 0, 0, 0, 0, 0, 0, 809, 965, 620, 446, 487, 851, 1015, 741, 0, 0, 0, 0, 0, 0, 0, 0, 556, 631, 225, 269, 649, 526, 1201, 673, 580, 343, 0, 0, 0, 0, 0, 0, 474, 2964, 126, 476, 264, 573, 431, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 69, 243, 254, 1090, 369, 943, 274, 341, 1769, 0, 0, 0, 0, 0, 0, 0, 205, 332, 1375, 576, 884, 911, 482, 390, 913, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6439, 7157, 6192, 5934, 5653, 6357, 5352, 6068] avg=6144.0000 max=7157 max_rank=1 min=5352 min_rank=6 imbalance_ratio=1.164876 max_load_reduction=41995 max_load_reduction_pct=0.854390 moved_to_echo=23430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[425, 1063, 1398, 564, 507, 497, 725, 567, 269, 40, 36, 0, 0, 0, 0, 0, 1173, 1032, 214, 469, 914, 556, 1019, 406, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 83, 2167, 474, 470, 400, 27, 766, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 679, 616, 580, 640, 848, 1271, 972, 0, 0, 0, 0, 0, 0, 0, 0, 223, 712, 235, 361, 695, 579, 2081, 745, 476, 352, 0, 0, 0, 0, 0, 0, 391, 1825, 162, 519, 322, 612, 548, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 220, 315, 247, 1021, 657, 946, 281, 515, 1812, 0, 0, 0, 0, 0, 0, 0, 391, 401, 1215, 590, 1026, 817, 692, 424, 748, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 5783, 6356, 6693, 6459, 5452, 6014, 6304] avg=6144.0000 max=6693 max_rank=3 min=5452 min_rank=5 imbalance_ratio=1.089355 max_load_reduction=42459 max_load_reduction_pct=0.863831 moved_to_echo=24229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[719, 1257, 1265, 556, 402, 447, 765, 588, 239, 42, 40, 0, 0, 0, 0, 0, 1075, 1005, 240, 440, 1038, 481, 811, 419, 0, 0, 0, 0, 0, 0, 0, 0, 1534, 61, 2107, 609, 528, 373, 46, 901, 0, 0, 0, 0, 0, 0, 0, 0, 929, 736, 633, 674, 581, 618, 1100, 907, 0, 0, 0, 0, 0, 0, 0, 0, 220, 740, 238, 363, 668, 556, 1787, 686, 474, 390, 0, 0, 0, 0, 0, 0, 715, 2055, 150, 513, 304, 683, 692, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 320, 294, 296, 916, 530, 1180, 378, 587, 1991, 0, 0, 0, 0, 0, 0, 0, 210, 316, 1402, 623, 1078, 746, 645, 330, 742, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6320, 5509, 6159, 6178, 6122, 6280, 6492, 6092] avg=6144.0000 max=6492 max_rank=6 min=5509 min_rank=1 imbalance_ratio=1.056641 max_load_reduction=42660 max_load_reduction_pct=0.867920 moved_to_echo=24986 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 5760, 2176, 2816, 5120, 4736, 13312, 5760, 4096, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 3072, 10624, 4992, 7936, 6400, 4864, 3200, 6528, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[13312, 640, 17408, 4480, 4352, 2816, 256, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 3072, 10624, 4992, 7936, 6400, 4864, 3200, 6528, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 18048, 1152, 4224, 2688, 5888, 4864, 8448, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2176, 2560, 2560, 8064, 4096, 8320, 3072, 4608, 14336, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[4224, 8064, 11264, 3840, 4096, 4224, 6016, 5632, 1920, 384, 384, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11392, 8960, 2432, 3072, 8192, 4352, 8448, 2944, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 5888, 5248, 4480, 5120, 6016, 8704, 7424, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2176, 2560, 2560, 8064, 4096, 8320, 3072, 4608, 14336, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 5760, 2176, 2816, 5120, 4736, 13312, 5760, 4096, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11392, 8960, 2432, 3072, 8192, 4352, 8448, 2944, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[33, 1529, 1515, 580, 1124, 1242, 1192, 763, 475, 480, 1480, 249, 512, 394, 430, 1466, 1392, 569, 164, 192, 303, 265, 184, 225, 395, 404, 1507, 620, 1477, 1116, 949, 196, 548, 217, 423, 523, 965, 996, 341, 989, 433, 309, 1124, 874, 974, 82, 1781, 46, 439, 350, 277, 2275, 568, 815, 792, 1211, 1084, 444, 387, 1067, 1208, 1656, 1399, 1133] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[12, 1348, 1466, 677, 1244, 1143, 793, 1265, 343, 614, 2138, 155, 207, 442, 379, 972, 1313, 172, 181, 90, 191, 416, 337, 278, 574, 367, 2054, 458, 2138, 997, 714, 84, 357, 20, 74, 975, 1113, 464, 171, 1221, 325, 439, 338, 897, 817, 9, 2408, 8, 210, 672, 271, 1844, 430, 1304, 965, 1289, 878, 501, 260, 1441, 1065, 2012, 1668, 1144] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[35, 1429, 1590, 601, 1139, 1076, 952, 902, 422, 532, 1614, 178, 301, 389, 430, 1495, 1441, 228, 274, 165, 238, 363, 218, 249, 561, 372, 1989, 620, 1419, 951, 693, 130, 457, 67, 179, 679, 1414, 796, 265, 1277, 385, 302, 706, 921, 936, 41, 1917, 27, 391, 359, 307, 2147, 586, 900, 868, 1251, 961, 430, 409, 1242, 962, 2294, 1469, 1211] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[30, 1716, 1707, 723, 1109, 1014, 1000, 776, 643, 579, 1541, 179, 343, 374, 427, 1323, 1836, 422, 145, 148, 211, 361, 172, 192, 696, 393, 1438, 627, 1274, 897, 892, 200, 449, 110, 305, 546, 1415, 901, 295, 984, 375, 594, 814, 849, 962, 42, 1961, 56, 396, 640, 265, 1999, 567, 923, 825, 1071, 1133, 386, 325, 1048, 1025, 1965, 1488, 1050] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[105, 1752, 1537, 416, 975, 976, 1352, 650, 523, 395, 1177, 162, 429, 351, 784, 1212, 1573, 499, 291, 200, 282, 281, 210, 190, 333, 447, 817, 616, 947, 1020, 878, 745, 778, 605, 518, 414, 1135, 1219, 378, 1189, 412, 302, 1656, 873, 982, 181, 1733, 299, 678, 193, 197, 2567, 664, 812, 640, 858, 945, 317, 435, 815, 887, 1902, 1018, 1425] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 1020, 1513, 1084, 1077, 1113, 706, 1714, 321, 826, 2500, 97, 171, 474, 349, 770, 1832, 81, 107, 62, 124, 580, 317, 156, 510, 359, 2420, 284, 2431, 717, 576, 60, 302, 1, 14, 1267, 841, 172, 101, 1059, 327, 439, 64, 916, 593, 0, 2704, 0, 56, 914, 219, 1742, 464, 1517, 1406, 1076, 655, 515, 159, 1589, 1180, 1875, 1322, 1312] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[34, 1959, 1536, 398, 1110, 1364, 1027, 823, 641, 448, 1449, 326, 306, 424, 471, 1135, 1868, 260, 177, 220, 196, 300, 255, 206, 415, 376, 1822, 608, 1681, 1131, 870, 126, 386, 94, 162, 756, 1083, 844, 307, 1037, 421, 604, 1002, 912, 595, 20, 1802, 19, 340, 620, 249, 1762, 544, 987, 878, 1198, 1062, 484, 282, 1259, 1015, 2071, 1377, 1018] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[35, 1429, 1590, 601, 1139, 1076, 952, 902, 422, 532, 1614, 178, 301, 389, 430, 1495, 1441, 228, 274, 165, 238, 363, 218, 249, 561, 372, 1989, 620, 1419, 951, 693, 130, 457, 67, 179, 679, 1414, 796, 265, 1277, 385, 302, 706, 921, 936, 41, 1917, 27, 391, 359, 307, 2147, 586, 900, 868, 1251, 961, 430, 409, 1242, 962, 2294, 1469, 1211] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 713, 1538, 1185, 1120, 980, 708, 1559, 287, 768, 2549, 139, 164, 468, 361, 646, 1510, 121, 190, 48, 131, 635, 402, 224, 594, 304, 2120, 212, 2378, 790, 524, 77, 378, 4, 56, 1315, 938, 309, 132, 1246, 298, 383, 114, 901, 669, 0, 2571, 1, 112, 971, 218, 1779, 449, 1635, 1258, 1190, 833, 389, 212, 1562, 1034, 1887, 1403, 1460] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[105, 1752, 1537, 416, 975, 976, 1352, 650, 523, 395, 1177, 162, 429, 351, 784, 1212, 1573, 499, 291, 200, 282, 281, 210, 190, 333, 447, 817, 616, 947, 1020, 878, 745, 778, 605, 518, 414, 1135, 1219, 378, 1189, 412, 302, 1656, 873, 982, 181, 1733, 299, 678, 193, 197, 2567, 664, 812, 640, 858, 945, 317, 435, 815, 887, 1902, 1018, 1425] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[12, 1348, 1466, 677, 1244, 1143, 793, 1265, 343, 614, 2138, 155, 207, 442, 379, 972, 1313, 172, 181, 90, 191, 416, 337, 278, 574, 367, 2054, 458, 2138, 997, 714, 84, 357, 20, 74, 975, 1113, 464, 171, 1221, 325, 439, 338, 897, 817, 9, 2408, 8, 210, 672, 271, 1844, 430, 1304, 965, 1289, 878, 501, 260, 1441, 1065, 2012, 1668, 1144] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 1020, 1513, 1084, 1077, 1113, 706, 1714, 321, 826, 2500, 97, 171, 474, 349, 770, 1832, 81, 107, 62, 124, 580, 317, 156, 510, 359, 2420, 284, 2431, 717, 576, 60, 302, 1, 14, 1267, 841, 172, 101, 1059, 327, 439, 64, 916, 593, 0, 2704, 0, 56, 914, 219, 1742, 464, 1517, 1406, 1076, 655, 515, 159, 1589, 1180, 1875, 1322, 1312] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[34, 1581, 0, 398, 1110, 1364, 1027, 823, 0, 0, 0, 0, 0, 0, 0, 0, 641, 448, 1449, 326, 306, 424, 471, 1135, 673, 114, 29, 0, 0, 0, 0, 0, 1868, 260, 177, 220, 196, 300, 255, 206, 2071, 870, 0, 0, 0, 0, 0, 0, 415, 376, 1149, 608, 1681, 1131, 870, 126, 0, 0, 0, 0, 0, 0, 0, 0, 386, 94, 162, 756, 1083, 844, 307, 1037, 666, 568, 0, 0, 0, 0, 0, 0, 421, 604, 1002, 912, 595, 20, 1802, 19, 419, 378, 0, 0, 0, 0, 0, 0, 340, 620, 249, 1080, 544, 987, 878, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 484, 282, 1259, 1015, 0, 929, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6337, 6016, 6423, 6356, 5903, 6172, 5896, 6049] avg=6144.0000 max=6423 max_rank=2 min=5896 min_rank=6 imbalance_ratio=1.045410 max_load_reduction=42729 max_load_reduction_pct=0.869324 moved_to_echo=24020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[33, 1234, 0, 580, 1124, 1242, 1192, 763, 0, 0, 0, 0, 0, 0, 0, 0, 475, 480, 1480, 249, 512, 394, 430, 1466, 557, 147, 29, 0, 0, 0, 0, 0, 1392, 569, 164, 192, 303, 265, 184, 225, 1656, 858, 0, 0, 0, 0, 0, 0, 395, 404, 950, 620, 1477, 1116, 949, 196, 0, 0, 0, 0, 0, 0, 0, 0, 548, 217, 423, 523, 965, 996, 341, 989, 657, 734, 0, 0, 0, 0, 0, 0, 433, 309, 1124, 874, 974, 82, 1781, 46, 425, 295, 0, 0, 0, 0, 0, 0, 439, 350, 277, 1394, 568, 815, 792, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 444, 387, 1067, 1208, 0, 945, 1133, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 6219, 5808, 6107, 6393, 6343, 5846, 6268] avg=6144.0000 max=6393 max_rank=4 min=5808 min_rank=2 imbalance_ratio=1.040527 max_load_reduction=42759 max_load_reduction_pct=0.869934 moved_to_echo=24850 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[35, 1153, 0, 601, 1139, 1076, 952, 902, 0, 0, 0, 0, 0, 0, 0, 0, 422, 532, 1614, 178, 301, 389, 430, 1495, 735, 139, 31, 0, 0, 0, 0, 0, 1441, 228, 274, 165, 238, 363, 218, 249, 2294, 900, 0, 0, 0, 0, 0, 0, 561, 372, 1254, 620, 1419, 951, 693, 130, 0, 0, 0, 0, 0, 0, 0, 0, 457, 67, 179, 679, 1414, 796, 265, 1277, 690, 692, 0, 0, 0, 0, 0, 0, 385, 302, 706, 921, 936, 41, 1917, 27, 447, 276, 0, 0, 0, 0, 0, 0, 391, 359, 307, 1316, 586, 900, 868, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 961, 430, 409, 1242, 962, 0, 991, 1211, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5858, 6266, 6370, 6000, 6516, 5958, 5978, 6206] avg=6144.0000 max=6516 max_rank=4 min=5858 min_rank=0 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 1088, 0, 677, 1244, 1143, 793, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 343, 614, 2138, 155, 207, 442, 379, 972, 759, 119, 35, 0, 0, 0, 0, 0, 1313, 172, 181, 90, 191, 416, 337, 278, 2012, 831, 0, 0, 0, 0, 0, 0, 574, 367, 1295, 458, 2138, 997, 714, 84, 0, 0, 0, 0, 0, 0, 0, 0, 357, 20, 74, 975, 1113, 464, 171, 1221, 635, 595, 0, 0, 0, 0, 0, 0, 325, 439, 338, 897, 817, 9, 2408, 8, 507, 260, 0, 0, 0, 0, 0, 0, 210, 672, 271, 1130, 430, 1304, 965, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 878, 501, 260, 1441, 1065, 0, 1126, 1144, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6222, 6163, 5821, 6627, 5625, 6008, 6271, 6415] avg=6144.0000 max=6627 max_rank=3 min=5625 min_rank=4 imbalance_ratio=1.078613 max_load_reduction=42525 max_load_reduction_pct=0.865173 moved_to_echo=24319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[105, 1414, 0, 416, 975, 976, 1352, 650, 0, 0, 0, 0, 0, 0, 0, 0, 523, 395, 1177, 162, 429, 351, 784, 1212, 302, 166, 21, 0, 0, 0, 0, 0, 1573, 499, 291, 200, 282, 281, 210, 190, 1902, 871, 0, 0, 0, 0, 0, 0, 333, 447, 515, 616, 947, 1020, 878, 745, 0, 0, 0, 0, 0, 0, 0, 0, 778, 605, 518, 414, 1135, 1219, 378, 1189, 666, 828, 0, 0, 0, 0, 0, 0, 412, 302, 1656, 873, 982, 181, 1733, 299, 309, 338, 0, 0, 0, 0, 0, 0, 678, 193, 197, 1573, 664, 812, 640, 858, 0, 0, 0, 0, 0, 0, 0, 0, 945, 317, 435, 815, 887, 0, 688, 1425, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5888, 5522, 6299, 5501, 7730, 7085, 5615, 5512] avg=6144.0000 max=7730 max_rank=4 min=5501 min_rank=3 imbalance_ratio=1.258138 max_load_reduction=41422 max_load_reduction_pct=0.842733 moved_to_echo=25942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 823, 0, 1084, 1077, 1113, 706, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 321, 826, 2500, 97, 171, 474, 349, 770, 894, 113, 28, 0, 0, 0, 0, 0, 1832, 81, 107, 62, 124, 580, 317, 156, 1875, 857, 0, 0, 0, 0, 0, 0, 510, 359, 1526, 284, 2431, 717, 576, 60, 0, 0, 0, 0, 0, 0, 0, 0, 302, 1, 14, 1267, 841, 172, 101, 1059, 656, 562, 0, 0, 0, 0, 0, 0, 327, 439, 64, 916, 593, 0, 2704, 0, 402, 197, 0, 0, 0, 0, 0, 0, 56, 914, 219, 1067, 464, 1517, 1406, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 655, 515, 159, 1589, 1180, 0, 892, 1312, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6517, 6543, 5991, 6463, 4975, 5642, 6719, 6302] avg=6144.0000 max=6719 max_rank=6 min=4975 min_rank=4 imbalance_ratio=1.093587 max_load_reduction=42433 max_load_reduction_pct=0.863302 moved_to_echo=23638 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[34, 1581, 0, 398, 1110, 1364, 1027, 823, 0, 0, 0, 0, 0, 0, 0, 0, 641, 448, 1449, 326, 306, 424, 471, 1135, 673, 114, 29, 0, 0, 0, 0, 0, 1868, 260, 177, 220, 196, 300, 255, 206, 2071, 870, 0, 0, 0, 0, 0, 0, 415, 376, 1149, 608, 1681, 1131, 870, 126, 0, 0, 0, 0, 0, 0, 0, 0, 386, 94, 162, 756, 1083, 844, 307, 1037, 666, 568, 0, 0, 0, 0, 0, 0, 421, 604, 1002, 912, 595, 20, 1802, 19, 419, 378, 0, 0, 0, 0, 0, 0, 340, 620, 249, 1080, 544, 987, 878, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 484, 282, 1259, 1015, 0, 929, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6337, 6016, 6423, 6356, 5903, 6172, 5896, 6049] avg=6144.0000 max=6423 max_rank=2 min=5896 min_rank=6 imbalance_ratio=1.045410 max_load_reduction=42729 max_load_reduction_pct=0.869324 moved_to_echo=24020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[30, 1382, 0, 723, 1109, 1014, 1000, 776, 0, 0, 0, 0, 0, 0, 0, 0, 643, 579, 1541, 179, 343, 374, 427, 1323, 535, 134, 36, 0, 0, 0, 0, 0, 1836, 422, 145, 148, 211, 361, 172, 192, 1965, 968, 0, 0, 0, 0, 0, 0, 696, 393, 903, 627, 1274, 897, 892, 200, 0, 0, 0, 0, 0, 0, 0, 0, 449, 110, 305, 546, 1415, 901, 295, 984, 739, 648, 0, 0, 0, 0, 0, 0, 375, 594, 814, 849, 962, 42, 1961, 56, 456, 334, 0, 0, 0, 0, 0, 0, 396, 640, 265, 1217, 567, 923, 825, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 386, 325, 1048, 1025, 0, 996, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6034, 6114, 6420, 5882, 6392, 6443, 5904, 5963] avg=6144.0000 max=6443 max_rank=5 min=5882 min_rank=3 imbalance_ratio=1.048665 max_load_reduction=42709 max_load_reduction_pct=0.868917 moved_to_echo=24702 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 576, 0, 1185, 1120, 980, 708, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 287, 768, 2549, 139, 164, 468, 361, 646, 783, 115, 29, 0, 0, 0, 0, 0, 1510, 121, 190, 48, 131, 635, 402, 224, 1887, 872, 0, 0, 0, 0, 0, 0, 594, 304, 1337, 212, 2378, 790, 524, 77, 0, 0, 0, 0, 0, 0, 0, 0, 378, 4, 56, 1315, 938, 309, 132, 1246, 666, 574, 0, 0, 0, 0, 0, 0, 298, 383, 114, 901, 669, 0, 2571, 1, 427, 137, 0, 0, 0, 0, 0, 0, 112, 971, 218, 1090, 449, 1635, 1258, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 833, 389, 212, 1562, 1034, 0, 947, 1460, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6309, 6020, 6216, 5618, 5501, 6923, 6437] avg=6144.0000 max=6923 max_rank=6 min=5501 min_rank=5 imbalance_ratio=1.126790 max_load_reduction=42229 max_load_reduction_pct=0.859151 moved_to_echo=24479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 1088, 0, 677, 1244, 1143, 793, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 343, 614, 2138, 155, 207, 442, 379, 972, 759, 119, 35, 0, 0, 0, 0, 0, 1313, 172, 181, 90, 191, 416, 337, 278, 2012, 831, 0, 0, 0, 0, 0, 0, 574, 367, 1295, 458, 2138, 997, 714, 84, 0, 0, 0, 0, 0, 0, 0, 0, 357, 20, 74, 975, 1113, 464, 171, 1221, 635, 595, 0, 0, 0, 0, 0, 0, 325, 439, 338, 897, 817, 9, 2408, 8, 507, 260, 0, 0, 0, 0, 0, 0, 210, 672, 271, 1130, 430, 1304, 965, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 878, 501, 260, 1441, 1065, 0, 1126, 1144, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6222, 6163, 5821, 6627, 5625, 6008, 6271, 6415] avg=6144.0000 max=6627 max_rank=3 min=5625 min_rank=4 imbalance_ratio=1.078613 max_load_reduction=42525 max_load_reduction_pct=0.865173 moved_to_echo=24319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[30, 1382, 0, 723, 1109, 1014, 1000, 776, 0, 0, 0, 0, 0, 0, 0, 0, 643, 579, 1541, 179, 343, 374, 427, 1323, 535, 134, 36, 0, 0, 0, 0, 0, 1836, 422, 145, 148, 211, 361, 172, 192, 1965, 968, 0, 0, 0, 0, 0, 0, 696, 393, 903, 627, 1274, 897, 892, 200, 0, 0, 0, 0, 0, 0, 0, 0, 449, 110, 305, 546, 1415, 901, 295, 984, 739, 648, 0, 0, 0, 0, 0, 0, 375, 594, 814, 849, 962, 42, 1961, 56, 456, 334, 0, 0, 0, 0, 0, 0, 396, 640, 265, 1217, 567, 923, 825, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 386, 325, 1048, 1025, 0, 996, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6034, 6114, 6420, 5882, 6392, 6443, 5904, 5963] avg=6144.0000 max=6443 max_rank=5 min=5882 min_rank=3 imbalance_ratio=1.048665 max_load_reduction=42709 max_load_reduction_pct=0.868917 moved_to_echo=24702 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 823, 0, 1084, 1077, 1113, 706, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 321, 826, 2500, 97, 171, 474, 349, 770, 894, 113, 28, 0, 0, 0, 0, 0, 1832, 81, 107, 62, 124, 580, 317, 156, 1875, 857, 0, 0, 0, 0, 0, 0, 510, 359, 1526, 284, 2431, 717, 576, 60, 0, 0, 0, 0, 0, 0, 0, 0, 302, 1, 14, 1267, 841, 172, 101, 1059, 656, 562, 0, 0, 0, 0, 0, 0, 327, 439, 64, 916, 593, 0, 2704, 0, 402, 197, 0, 0, 0, 0, 0, 0, 56, 914, 219, 1067, 464, 1517, 1406, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 655, 515, 159, 1589, 1180, 0, 892, 1312, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6517, 6543, 5991, 6463, 4975, 5642, 6719, 6302] avg=6144.0000 max=6719 max_rank=6 min=4975 min_rank=4 imbalance_ratio=1.093587 max_load_reduction=42433 max_load_reduction_pct=0.863302 moved_to_echo=23638 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7552, 3584, 2560, 10112, 8448, 0, 7552, 9856, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 9344, 0, 5760, 8960, 8960, 7808, 8576, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 4736, 14464, 1536, 2560, 3328, 3712, 9088, 5248, 1152, 256, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3712, 1152, 1792, 6528, 8960, 5760, 2048, 9088, 5376, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4096, 3072, 8960, 4096, 13824, 7680, 6144, 1664, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 4736, 2048, 9984, 4352, 8960, 7680, 9216, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12800, 2432, 1536, 1152, 1792, 3328, 2176, 1792, 15744, 7040, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 3456, 5888, 7168, 6528, 384, 16896, 512, 3456, 2304, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4096, 3072, 8960, 4096, 13824, 7680, 6144, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12800, 2432, 1536, 1152, 1792, 3328, 2176, 1792, 15744, 7040, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[426, 1718, 861, 404, 85, 221, 2581, 549, 1, 0, 757, 690, 2457, 0, 1, 630, 488, 964, 3702, 16, 112, 1162, 366, 0, 112, 2771, 781, 101, 249, 284, 3976, 278, 1905, 471, 129, 592, 738, 230, 1465, 784, 36, 0, 20, 1663, 60, 514, 1722, 577, 872, 1289, 1454, 1018, 44, 554, 0, 1155, 1216, 424, 476, 648, 128, 293, 852, 1080] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[654, 1450, 910, 316, 217, 169, 2952, 475, 8, 5, 884, 807, 1668, 0, 16, 795, 575, 1008, 3565, 31, 225, 505, 288, 2, 188, 2295, 807, 147, 347, 249, 3542, 440, 1850, 575, 185, 762, 811, 298, 1357, 990, 105, 4, 117, 2080, 92, 516, 1490, 456, 970, 1239, 1124, 906, 85, 635, 5, 1259, 1085, 401, 450, 837, 209, 483, 1120, 1116] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[922, 942, 1582, 252, 611, 180, 1523, 258, 167, 265, 1353, 967, 867, 4, 112, 1937, 722, 723, 1513, 67, 493, 121, 252, 813, 486, 1042, 641, 198, 741, 471, 2810, 391, 1139, 968, 692, 2511, 906, 704, 1609, 793, 302, 186, 378, 1916, 338, 359, 541, 607, 1138, 1077, 659, 353, 312, 713, 57, 1526, 449, 544, 514, 1259, 218, 351, 1879, 728] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[718, 1196, 1246, 326, 583, 230, 2579, 264, 66, 75, 1180, 970, 1225, 0, 66, 1229, 688, 952, 2854, 88, 296, 286, 334, 38, 187, 1662, 793, 234, 529, 283, 2939, 438, 1448, 741, 316, 1214, 1041, 365, 1647, 881, 301, 13, 170, 2052, 157, 421, 1021, 440, 1032, 1288, 893, 670, 152, 801, 0, 1292, 766, 507, 466, 868, 307, 508, 1560, 1260] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[426, 1718, 861, 404, 85, 221, 2581, 549, 1, 0, 757, 690, 2457, 0, 1, 630, 488, 964, 3702, 16, 112, 1162, 366, 0, 112, 2771, 781, 101, 249, 284, 3976, 278, 1905, 471, 129, 592, 738, 230, 1465, 784, 36, 0, 20, 1663, 60, 514, 1722, 577, 872, 1289, 1454, 1018, 44, 554, 0, 1155, 1216, 424, 476, 648, 128, 293, 852, 1080] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[773, 1614, 1206, 321, 399, 141, 3227, 445, 38, 3, 1012, 887, 1323, 0, 31, 1154, 714, 959, 2985, 63, 257, 335, 261, 8, 283, 2057, 849, 145, 483, 270, 3292, 449, 1682, 627, 217, 1095, 741, 295, 1455, 1098, 201, 1, 156, 2030, 140, 361, 1153, 473, 944, 1176, 977, 723, 82, 704, 0, 1274, 829, 462, 477, 843, 217, 452, 1247, 1036] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[788, 1871, 1236, 412, 502, 254, 2385, 316, 46, 11, 995, 905, 1703, 0, 33, 1554, 653, 805, 2687, 61, 279, 364, 359, 26, 174, 1536, 852, 216, 485, 298, 2836, 427, 1753, 789, 266, 1362, 680, 334, 1966, 871, 327, 6, 62, 1779, 234, 344, 1215, 452, 1023, 1482, 996, 509, 140, 631, 0, 1422, 865, 489, 447, 829, 309, 402, 1278, 821] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[922, 942, 1582, 252, 611, 180, 1523, 258, 167, 265, 1353, 967, 867, 4, 112, 1937, 722, 723, 1513, 67, 493, 121, 252, 813, 486, 1042, 641, 198, 741, 471, 2810, 391, 1139, 968, 692, 2511, 906, 704, 1609, 793, 302, 186, 378, 1916, 338, 359, 541, 607, 1138, 1077, 659, 353, 312, 713, 57, 1526, 449, 544, 514, 1259, 218, 351, 1879, 728] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[620, 1650, 1221, 386, 432, 217, 2386, 251, 55, 60, 1053, 891, 1428, 1, 50, 1059, 765, 814, 2400, 55, 372, 271, 308, 42, 167, 1466, 792, 126, 614, 252, 3368, 282, 1419, 674, 299, 1448, 816, 309, 2098, 1223, 172, 18, 207, 2355, 160, 338, 1410, 421, 1020, 1397, 840, 743, 133, 739, 6, 1241, 754, 504, 462, 1121, 288, 440, 1236, 1007] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[773, 1614, 1206, 321, 399, 141, 3227, 445, 38, 3, 1012, 887, 1323, 0, 31, 1154, 714, 959, 2985, 63, 257, 335, 261, 8, 283, 2057, 849, 145, 483, 270, 3292, 449, 1682, 627, 217, 1095, 741, 295, 1455, 1098, 201, 1, 156, 2030, 140, 361, 1153, 473, 944, 1176, 977, 723, 82, 704, 0, 1274, 829, 462, 477, 843, 217, 452, 1247, 1036] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[654, 1450, 910, 316, 217, 169, 2952, 475, 8, 5, 884, 807, 1668, 0, 16, 795, 575, 1008, 3565, 31, 225, 505, 288, 2, 188, 2295, 807, 147, 347, 249, 3542, 440, 1850, 575, 185, 762, 811, 298, 1357, 990, 105, 4, 117, 2080, 92, 516, 1490, 456, 970, 1239, 1124, 906, 85, 635, 5, 1259, 1085, 401, 450, 837, 209, 483, 1120, 1116] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[481, 1608, 822, 285, 154, 111, 2776, 541, 3, 0, 786, 671, 1947, 0, 4, 703, 563, 917, 3417, 26, 133, 886, 285, 0, 191, 2608, 788, 101, 256, 196, 4023, 303, 2137, 462, 161, 554, 881, 223, 1214, 1194, 53, 0, 51, 1755, 55, 639, 1594, 565, 857, 1151, 1297, 1341, 92, 612, 0, 1157, 1187, 379, 392, 726, 147, 350, 981, 1360] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[426, 1718, 861, 404, 85, 221, 1561, 549, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 757, 690, 2457, 0, 1, 630, 1569, 71, 0, 0, 0, 0, 0, 0, 488, 964, 3702, 16, 112, 1162, 366, 0, 484, 0, 0, 0, 0, 0, 0, 0, 112, 2771, 781, 101, 249, 284, 2278, 278, 0, 0, 0, 0, 0, 0, 0, 0, 290, 471, 129, 592, 738, 230, 1465, 784, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 20, 1663, 60, 514, 1722, 577, 1698, 46, 0, 0, 0, 0, 0, 0, 872, 1289, 1454, 1018, 44, 554, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 424, 476, 648, 128, 293, 852, 1080, 465, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5825, 6176, 7294, 6854, 4699, 6336, 6372, 5596] avg=6144.0000 max=7294 max_rank=2 min=4699 min_rank=4 imbalance_ratio=1.187174 max_load_reduction=41858 max_load_reduction_pct=0.851603 moved_to_echo=23003 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[654, 1450, 910, 316, 217, 169, 1785, 475, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 884, 807, 1668, 0, 16, 795, 1523, 81, 0, 0, 0, 0, 0, 0, 575, 1008, 3565, 31, 225, 505, 288, 2, 554, 0, 0, 0, 0, 0, 0, 0, 188, 2295, 807, 147, 347, 249, 2029, 440, 0, 0, 0, 0, 0, 0, 0, 0, 282, 575, 185, 762, 811, 298, 1357, 990, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4, 117, 2080, 92, 516, 1490, 456, 1513, 45, 0, 0, 0, 0, 0, 0, 970, 1239, 1124, 906, 85, 635, 5, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 401, 450, 837, 209, 483, 1120, 1116, 532, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5976, 5787, 6753, 6502, 5260, 6418, 6208, 6248] avg=6144.0000 max=6753 max_rank=2 min=5260 min_rank=4 imbalance_ratio=1.099121 max_load_reduction=42399 max_load_reduction_pct=0.862610 moved_to_echo=24134 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[718, 1196, 1246, 326, 583, 230, 1559, 264, 0, 0, 0, 0, 0, 0, 0, 0, 66, 75, 1180, 970, 1225, 0, 66, 1229, 1192, 71, 0, 0, 0, 0, 0, 0, 688, 952, 2854, 88, 296, 286, 334, 38, 484, 0, 0, 0, 0, 0, 0, 0, 187, 1662, 793, 234, 529, 283, 1684, 438, 0, 0, 0, 0, 0, 0, 0, 0, 221, 741, 316, 1214, 1041, 365, 1647, 881, 0, 0, 0, 0, 0, 0, 0, 0, 301, 13, 170, 2052, 157, 421, 1021, 440, 1255, 35, 0, 0, 0, 0, 0, 0, 1032, 1288, 893, 670, 152, 801, 0, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 766, 507, 466, 868, 307, 508, 1560, 1260, 465, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6074, 6020, 5810, 6426, 5865, 6112, 6723] avg=6144.0000 max=6723 max_rank=7 min=5810 min_rank=3 imbalance_ratio=1.094238 max_load_reduction=42429 max_load_reduction_pct=0.863220 moved_to_echo=25126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[922, 942, 1582, 252, 611, 180, 922, 258, 0, 0, 0, 0, 0, 0, 0, 0, 167, 265, 1353, 967, 867, 4, 112, 1937, 938, 41, 0, 0, 0, 0, 0, 0, 722, 723, 1513, 67, 493, 121, 252, 813, 286, 0, 0, 0, 0, 0, 0, 0, 486, 1042, 641, 198, 741, 471, 1610, 391, 0, 0, 0, 0, 0, 0, 0, 0, 174, 968, 692, 2511, 906, 704, 1609, 793, 0, 0, 0, 0, 0, 0, 0, 0, 302, 186, 378, 1916, 338, 359, 541, 607, 1200, 27, 0, 0, 0, 0, 0, 0, 1138, 1077, 659, 353, 312, 713, 57, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 449, 544, 514, 1259, 218, 351, 1879, 728, 274, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5669, 6651, 4990, 5580, 8357, 5854, 5816, 6235] avg=6144.0000 max=8357 max_rank=4 min=4990 min_rank=2 imbalance_ratio=1.360189 max_load_reduction=40795 max_load_reduction_pct=0.829976 moved_to_echo=26262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[773, 1614, 1206, 321, 399, 141, 1951, 445, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 1012, 887, 1323, 0, 31, 1154, 1385, 88, 0, 0, 0, 0, 0, 0, 714, 959, 2985, 63, 257, 335, 261, 8, 606, 0, 0, 0, 0, 0, 0, 0, 283, 2057, 849, 145, 483, 270, 1886, 449, 0, 0, 0, 0, 0, 0, 0, 0, 256, 627, 217, 1095, 741, 295, 1455, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1, 156, 2030, 140, 361, 1153, 473, 1406, 41, 0, 0, 0, 0, 0, 0, 944, 1176, 977, 723, 82, 704, 0, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 829, 462, 477, 843, 217, 452, 1247, 1036, 582, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6850, 5921, 6188, 6422, 5784, 5962, 5865, 6160] avg=6144.0000 max=6850 max_rank=0 min=5784 min_rank=4 imbalance_ratio=1.114909 max_load_reduction=42302 max_load_reduction_pct=0.860636 moved_to_echo=23771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[788, 1871, 1236, 412, 502, 254, 1443, 316, 0, 0, 0, 0, 0, 0, 0, 0, 46, 11, 995, 905, 1703, 0, 33, 1554, 1443, 65, 0, 0, 0, 0, 0, 0, 653, 805, 2687, 61, 279, 364, 359, 26, 447, 0, 0, 0, 0, 0, 0, 0, 174, 1536, 852, 216, 485, 298, 1625, 427, 0, 0, 0, 0, 0, 0, 0, 0, 268, 789, 266, 1362, 680, 334, 1966, 871, 0, 0, 0, 0, 0, 0, 0, 0, 327, 6, 62, 1779, 234, 344, 1215, 452, 1211, 42, 0, 0, 0, 0, 0, 0, 1023, 1482, 996, 509, 140, 631, 0, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 865, 489, 447, 829, 309, 402, 1278, 821, 430, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6822, 6755, 5681, 5613, 6536, 5672, 6186, 5887] avg=6144.0000 max=6822 max_rank=0 min=5613 min_rank=3 imbalance_ratio=1.110352 max_load_reduction=42330 max_load_reduction_pct=0.861206 moved_to_echo=24281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[620, 1650, 1221, 386, 432, 217, 1431, 251, 0, 0, 0, 0, 0, 0, 0, 0, 55, 60, 1053, 891, 1428, 1, 50, 1059, 1172, 69, 0, 0, 0, 0, 0, 0, 765, 814, 2400, 55, 372, 271, 308, 42, 451, 0, 0, 0, 0, 0, 0, 0, 167, 1466, 792, 126, 614, 252, 1924, 282, 0, 0, 0, 0, 0, 0, 0, 0, 209, 674, 299, 1448, 816, 309, 2098, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 172, 18, 207, 2355, 160, 338, 1410, 421, 1444, 38, 0, 0, 0, 0, 0, 0, 1020, 1397, 840, 743, 133, 739, 6, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 754, 504, 462, 1121, 288, 440, 1236, 1007, 435, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6208, 5838, 5478, 5623, 7076, 6563, 6100, 6266] avg=6144.0000 max=7076 max_rank=4 min=5478 min_rank=2 imbalance_ratio=1.151693 max_load_reduction=42076 max_load_reduction_pct=0.856038 moved_to_echo=26005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[718, 1196, 1246, 326, 583, 230, 1559, 264, 0, 0, 0, 0, 0, 0, 0, 0, 66, 75, 1180, 970, 1225, 0, 66, 1229, 1192, 71, 0, 0, 0, 0, 0, 0, 688, 952, 2854, 88, 296, 286, 334, 38, 484, 0, 0, 0, 0, 0, 0, 0, 187, 1662, 793, 234, 529, 283, 1684, 438, 0, 0, 0, 0, 0, 0, 0, 0, 221, 741, 316, 1214, 1041, 365, 1647, 881, 0, 0, 0, 0, 0, 0, 0, 0, 301, 13, 170, 2052, 157, 421, 1021, 440, 1255, 35, 0, 0, 0, 0, 0, 0, 1032, 1288, 893, 670, 152, 801, 0, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 766, 507, 466, 868, 307, 508, 1560, 1260, 465, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6074, 6020, 5810, 6426, 5865, 6112, 6723] avg=6144.0000 max=6723 max_rank=7 min=5810 min_rank=3 imbalance_ratio=1.094238 max_load_reduction=42429 max_load_reduction_pct=0.863220 moved_to_echo=25126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[773, 1614, 1206, 321, 399, 141, 1951, 445, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 1012, 887, 1323, 0, 31, 1154, 1385, 88, 0, 0, 0, 0, 0, 0, 714, 959, 2985, 63, 257, 335, 261, 8, 606, 0, 0, 0, 0, 0, 0, 0, 283, 2057, 849, 145, 483, 270, 1886, 449, 0, 0, 0, 0, 0, 0, 0, 0, 256, 627, 217, 1095, 741, 295, 1455, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1, 156, 2030, 140, 361, 1153, 473, 1406, 41, 0, 0, 0, 0, 0, 0, 944, 1176, 977, 723, 82, 704, 0, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 829, 462, 477, 843, 217, 452, 1247, 1036, 582, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6850, 5921, 6188, 6422, 5784, 5962, 5865, 6160] avg=6144.0000 max=6850 max_rank=0 min=5784 min_rank=4 imbalance_ratio=1.114909 max_load_reduction=42302 max_load_reduction_pct=0.860636 moved_to_echo=23771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[922, 942, 1582, 252, 611, 180, 922, 258, 0, 0, 0, 0, 0, 0, 0, 0, 167, 265, 1353, 967, 867, 4, 112, 1937, 938, 41, 0, 0, 0, 0, 0, 0, 722, 723, 1513, 67, 493, 121, 252, 813, 286, 0, 0, 0, 0, 0, 0, 0, 486, 1042, 641, 198, 741, 471, 1610, 391, 0, 0, 0, 0, 0, 0, 0, 0, 174, 968, 692, 2511, 906, 704, 1609, 793, 0, 0, 0, 0, 0, 0, 0, 0, 302, 186, 378, 1916, 338, 359, 541, 607, 1200, 27, 0, 0, 0, 0, 0, 0, 1138, 1077, 659, 353, 312, 713, 57, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 449, 544, 514, 1259, 218, 351, 1879, 728, 274, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5669, 6651, 4990, 5580, 8357, 5854, 5816, 6235] avg=6144.0000 max=8357 max_rank=4 min=4990 min_rank=2 imbalance_ratio=1.360189 max_load_reduction=40795 max_load_reduction_pct=0.829976 moved_to_echo=26262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[654, 1450, 910, 316, 217, 169, 1785, 475, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 884, 807, 1668, 0, 16, 795, 1523, 81, 0, 0, 0, 0, 0, 0, 575, 1008, 3565, 31, 225, 505, 288, 2, 554, 0, 0, 0, 0, 0, 0, 0, 188, 2295, 807, 147, 347, 249, 2029, 440, 0, 0, 0, 0, 0, 0, 0, 0, 282, 575, 185, 762, 811, 298, 1357, 990, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4, 117, 2080, 92, 516, 1490, 456, 1513, 45, 0, 0, 0, 0, 0, 0, 970, 1239, 1124, 906, 85, 635, 5, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 401, 450, 837, 209, 483, 1120, 1116, 532, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5976, 5787, 6753, 6502, 5260, 6418, 6208, 6248] avg=6144.0000 max=6753 max_rank=2 min=5260 min_rank=4 imbalance_ratio=1.099121 max_load_reduction=42399 max_load_reduction_pct=0.862610 moved_to_echo=24134 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[481, 1608, 822, 285, 154, 111, 1678, 541, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 786, 671, 1947, 0, 4, 703, 1760, 76, 0, 0, 0, 0, 0, 0, 563, 917, 3417, 26, 133, 886, 285, 0, 521, 0, 0, 0, 0, 0, 0, 0, 191, 2608, 788, 101, 256, 196, 2305, 303, 0, 0, 0, 0, 0, 0, 0, 0, 325, 462, 161, 554, 881, 223, 1214, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 51, 1755, 55, 639, 1594, 565, 1718, 52, 0, 0, 0, 0, 0, 0, 857, 1151, 1297, 1341, 92, 612, 0, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 379, 392, 726, 147, 350, 981, 1360, 501, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5680, 5950, 6748, 6748, 5014, 6482, 6493, 6037] avg=6144.0000 max=6748 max_rank=2 min=5014 min_rank=4 imbalance_ratio=1.098307 max_load_reduction=42404 max_load_reduction_pct=0.862712 moved_to_echo=24026 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[620, 1650, 1221, 386, 432, 217, 1431, 251, 0, 0, 0, 0, 0, 0, 0, 0, 55, 60, 1053, 891, 1428, 1, 50, 1059, 1172, 69, 0, 0, 0, 0, 0, 0, 765, 814, 2400, 55, 372, 271, 308, 42, 451, 0, 0, 0, 0, 0, 0, 0, 167, 1466, 792, 126, 614, 252, 1924, 282, 0, 0, 0, 0, 0, 0, 0, 0, 209, 674, 299, 1448, 816, 309, 2098, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 172, 18, 207, 2355, 160, 338, 1410, 421, 1444, 38, 0, 0, 0, 0, 0, 0, 1020, 1397, 840, 743, 133, 739, 6, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 754, 504, 462, 1121, 288, 440, 1236, 1007, 435, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6208, 5838, 5478, 5623, 7076, 6563, 6100, 6266] avg=6144.0000 max=7076 max_rank=4 min=5478 min_rank=2 imbalance_ratio=1.151693 max_load_reduction=42076 max_load_reduction_pct=0.856038 moved_to_echo=26005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1536, 256, 1280, 15744, 1280, 3584, 10240, 4096, 11520, 384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5504, 12160, 9088, 2816, 3072, 1536, 12416, 3200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1536, 256, 1280, 15744, 1280, 3584, 10240, 4096, 11520, 384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7936, 10112, 8320, 6272, 1152, 5504, 128, 10240, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 5376, 2304, 9600, 6656, 2816, 12928, 7936, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[1792, 15488, 6400, 1280, 3712, 2304, 15360, 3072, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5248, 7168, 23168, 512, 2176, 3968, 2560, 1024, 3840, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7168, 3712, 3712, 7168, 1920, 3328, 10240, 8448, 3712, 256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 512, 8064, 6912, 12672, 128, 384, 9088, 11008, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[1792, 15488, 6400, 1280, 3712, 2304, 15360, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5248, 7168, 23168, 512, 2176, 3968, 2560, 1024, 3840, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[653, 120, 0, 4574, 78, 0, 209, 97, 0, 456, 582, 532, 1043, 526, 440, 1018, 0, 0, 601, 4, 3, 447, 404, 1434, 102, 59, 528, 0, 177, 441, 0, 0, 97, 2033, 170, 1, 0, 0, 0, 3079, 1078, 268, 209, 1606, 5531, 87, 963, 4555, 422, 96, 0, 1647, 325, 1619, 6345, 0, 3981, 0, 0, 0, 4, 0, 364, 144] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[623, 91, 0, 4569, 83, 1, 183, 74, 0, 453, 731, 557, 971, 777, 389, 946, 0, 1, 553, 0, 26, 430, 370, 1780, 96, 74, 572, 0, 276, 227, 0, 0, 74, 1956, 205, 1, 0, 0, 0, 2827, 1098, 254, 178, 1664, 5559, 132, 876, 4278, 379, 100, 0, 1630, 297, 1505, 6332, 0, 4285, 0, 0, 5, 146, 0, 383, 135] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[808, 294, 2, 3444, 76, 67, 279, 126, 0, 583, 1524, 935, 1087, 550, 431, 1610, 3, 31, 827, 21, 852, 549, 459, 2407, 101, 328, 629, 5, 673, 478, 0, 29, 109, 1891, 400, 41, 0, 0, 0, 2031, 1283, 259, 239, 1592, 4851, 374, 951, 3010, 403, 642, 4, 1066, 307, 850, 5242, 0, 3502, 5, 1, 42, 343, 0, 294, 212] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[886, 260, 2, 2910, 76, 55, 261, 118, 2, 691, 1411, 917, 1107, 429, 450, 1541, 0, 35, 853, 30, 288, 561, 515, 2492, 140, 244, 715, 2, 683, 285, 0, 26, 104, 1683, 493, 37, 2, 0, 0, 1678, 1281, 314, 241, 1978, 4805, 357, 843, 3087, 364, 1171, 4, 737, 380, 1155, 5796, 0, 3699, 16, 1, 45, 363, 0, 373, 160] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[711, 151, 0, 3366, 100, 29, 251, 67, 0, 519, 1684, 950, 1081, 528, 421, 1396, 4, 8, 769, 7, 349, 509, 474, 2426, 113, 63, 942, 0, 547, 295, 0, 7, 69, 1845, 303, 18, 0, 0, 0, 1752, 1259, 233, 221, 1828, 5122, 332, 802, 3725, 380, 566, 0, 982, 383, 976, 5883, 0, 3735, 4, 0, 7, 346, 0, 460, 154] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[702, 150, 0, 4230, 80, 3, 217, 89, 0, 512, 1127, 746, 1023, 693, 421, 1216, 1, 6, 756, 3, 73, 510, 393, 2126, 136, 66, 663, 0, 366, 171, 0, 5, 91, 1864, 205, 5, 5, 4, 6, 2433, 1157, 254, 185, 1667, 5443, 128, 1056, 3863, 375, 143, 1, 1568, 329, 1025, 5984, 2, 4047, 0, 0, 26, 281, 1, 382, 138] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[653, 120, 0, 4574, 78, 0, 209, 97, 0, 456, 582, 532, 1043, 526, 440, 1018, 0, 0, 601, 4, 3, 447, 404, 1434, 102, 59, 528, 0, 177, 441, 0, 0, 97, 2033, 170, 1, 0, 0, 0, 3079, 1078, 268, 209, 1606, 5531, 87, 963, 4555, 422, 96, 0, 1647, 325, 1619, 6345, 0, 3981, 0, 0, 0, 4, 0, 364, 144] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[623, 91, 0, 4569, 83, 1, 183, 74, 0, 453, 731, 557, 971, 777, 389, 946, 0, 1, 553, 0, 26, 430, 370, 1780, 96, 74, 572, 0, 276, 227, 0, 0, 74, 1956, 205, 1, 0, 0, 0, 2827, 1098, 254, 178, 1664, 5559, 132, 876, 4278, 379, 100, 0, 1630, 297, 1505, 6332, 0, 4285, 0, 0, 5, 146, 0, 383, 135] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[976, 515, 100, 832, 172, 693, 390, 140, 166, 942, 1015, 766, 1028, 185, 447, 1304, 17, 259, 913, 92, 2183, 683, 890, 1295, 188, 1095, 566, 316, 1494, 535, 35, 185, 231, 1655, 557, 273, 21, 340, 69, 2573, 1159, 275, 840, 1538, 2403, 1331, 364, 1352, 433, 2798, 243, 294, 786, 607, 3690, 305, 1842, 747, 74, 281, 1042, 44, 315, 253] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[880, 311, 1, 3013, 76, 33, 303, 137, 2, 546, 1437, 751, 1064, 583, 413, 1548, 1, 17, 777, 37, 344, 476, 479, 2582, 114, 199, 989, 1, 712, 230, 0, 19, 71, 1926, 548, 11, 0, 5, 4, 1944, 1152, 250, 216, 1726, 5082, 396, 784, 3144, 388, 807, 1, 839, 327, 910, 6004, 5, 3693, 7, 3, 32, 378, 0, 302, 122] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[886, 260, 2, 2910, 76, 55, 261, 118, 2, 691, 1411, 917, 1107, 429, 450, 1541, 0, 35, 853, 30, 288, 561, 515, 2492, 140, 244, 715, 2, 683, 285, 0, 26, 104, 1683, 493, 37, 2, 0, 0, 1678, 1281, 314, 241, 1978, 4805, 357, 843, 3087, 364, 1171, 4, 737, 380, 1155, 5796, 0, 3699, 16, 1, 45, 363, 0, 373, 160] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[808, 294, 2, 3444, 76, 67, 279, 126, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 583, 1524, 935, 1087, 550, 431, 1610, 217, 0, 0, 0, 0, 0, 0, 0, 3, 31, 827, 21, 852, 549, 459, 2407, 825, 823, 0, 0, 0, 0, 0, 0, 101, 328, 629, 5, 673, 478, 0, 29, 4028, 0, 0, 0, 0, 0, 0, 0, 109, 1891, 400, 41, 0, 0, 0, 2031, 884, 420, 0, 0, 0, 0, 0, 0, 1283, 259, 239, 1592, 0, 374, 951, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 403, 642, 4, 1066, 307, 850, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 5, 1, 42, 343, 0, 294, 212, 1381, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6937, 6797, 6271, 5776, 6022, 5424, 5780] avg=6144.0000 max=6937 max_rank=1 min=5424 min_rank=6 imbalance_ratio=1.129069 max_load_reduction=42215 max_load_reduction_pct=0.858866 moved_to_echo=23002 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[976, 515, 100, 832, 172, 693, 390, 140, 166, 942, 1015, 766, 1028, 185, 447, 1304, 17, 259, 913, 92, 2183, 683, 890, 1295, 188, 1095, 566, 316, 1494, 535, 35, 185, 231, 1655, 557, 273, 21, 340, 69, 2573, 1159, 275, 840, 1538, 2403, 1331, 364, 1352, 433, 2798, 243, 294, 786, 607, 3690, 305, 1842, 747, 74, 281, 1042, 44, 315, 253] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[653, 120, 0, 4574, 78, 0, 209, 97, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 456, 582, 532, 1043, 526, 440, 1018, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 4, 3, 447, 404, 1434, 999, 939, 0, 0, 0, 0, 0, 0, 102, 59, 528, 0, 177, 441, 0, 0, 4592, 0, 0, 0, 0, 0, 0, 0, 97, 2033, 170, 1, 0, 0, 0, 3079, 1070, 636, 0, 0, 0, 0, 0, 0, 1078, 268, 209, 1606, 0, 87, 963, 2004, 0, 0, 0, 0, 0, 0, 0, 0, 422, 96, 0, 1647, 325, 1619, 2604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 0, 0, 4, 0, 364, 144, 1672, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7318, 4925, 4831, 5899, 7086, 6215, 6713, 6165] avg=6144.0000 max=7318 max_rank=0 min=4831 min_rank=2 imbalance_ratio=1.191081 max_load_reduction=41834 max_load_reduction_pct=0.851115 moved_to_echo=26179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[976, 515, 100, 832, 172, 693, 390, 140, 471, 0, 0, 0, 0, 0, 0, 0, 166, 942, 1015, 766, 1028, 185, 447, 1304, 97, 0, 0, 0, 0, 0, 0, 0, 17, 259, 913, 92, 2183, 683, 890, 1295, 581, 408, 0, 0, 0, 0, 0, 0, 188, 1095, 566, 316, 1494, 535, 35, 185, 1995, 0, 0, 0, 0, 0, 0, 0, 231, 1655, 557, 273, 21, 340, 69, 2573, 622, 188, 0, 0, 0, 0, 0, 0, 1159, 275, 840, 1538, 0, 1331, 364, 596, 0, 0, 0, 0, 0, 0, 0, 0, 433, 2798, 243, 294, 786, 607, 1515, 305, 0, 0, 0, 0, 0, 0, 0, 0, 1842, 747, 74, 281, 1042, 44, 315, 253, 972, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4289, 5950, 7321, 6409, 6529, 6103, 6981, 5570] avg=6144.0000 max=7321 max_rank=2 min=4289 min_rank=0 imbalance_ratio=1.191569 max_load_reduction=41831 max_load_reduction_pct=0.851054 moved_to_echo=25183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[623, 91, 0, 4569, 83, 1, 183, 74, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 453, 731, 557, 971, 777, 389, 946, 308, 0, 0, 0, 0, 0, 0, 0, 0, 1, 553, 0, 26, 430, 370, 1780, 997, 944, 0, 0, 0, 0, 0, 0, 96, 74, 572, 0, 276, 227, 0, 0, 4615, 0, 0, 0, 0, 0, 0, 0, 74, 1956, 205, 1, 0, 0, 0, 2827, 1067, 597, 0, 0, 0, 0, 0, 0, 1098, 254, 178, 1664, 0, 132, 876, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 379, 100, 0, 1630, 297, 1505, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4285, 0, 0, 5, 146, 0, 383, 135, 1668, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7114, 5132, 5101, 5860, 6727, 6085, 6511, 6622] avg=6144.0000 max=7114 max_rank=0 min=5101 min_rank=2 imbalance_ratio=1.157878 max_load_reduction=42038 max_load_reduction_pct=0.855265 moved_to_echo=25945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[702, 150, 0, 4230, 80, 3, 217, 89, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 512, 1127, 746, 1023, 693, 421, 1216, 278, 0, 0, 0, 0, 0, 0, 0, 1, 6, 756, 3, 73, 510, 393, 2126, 942, 925, 0, 0, 0, 0, 0, 0, 136, 66, 663, 0, 366, 171, 0, 5, 4518, 0, 0, 0, 0, 0, 0, 0, 91, 1864, 205, 5, 5, 4, 6, 2433, 1009, 539, 0, 0, 0, 0, 0, 0, 1157, 254, 185, 1667, 0, 128, 1056, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 375, 143, 1, 1568, 329, 1025, 2456, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 26, 281, 1, 382, 138, 1577, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6817, 6016, 5735, 5925, 6161, 6147, 5899, 6452] avg=6144.0000 max=6817 max_rank=0 min=5735 min_rank=2 imbalance_ratio=1.109538 max_load_reduction=42335 max_load_reduction_pct=0.861308 moved_to_echo=24659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[711, 151, 0, 3366, 100, 29, 251, 67, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 519, 1684, 950, 1081, 528, 421, 1396, 268, 0, 0, 0, 0, 0, 0, 0, 4, 8, 769, 7, 349, 509, 474, 2426, 926, 869, 0, 0, 0, 0, 0, 0, 113, 63, 942, 0, 547, 295, 0, 7, 4253, 0, 0, 0, 0, 0, 0, 0, 69, 1845, 303, 18, 0, 0, 0, 1752, 992, 520, 0, 0, 0, 0, 0, 0, 1259, 233, 221, 1828, 0, 332, 802, 1639, 0, 0, 0, 0, 0, 0, 0, 0, 380, 566, 0, 982, 383, 976, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 4, 0, 7, 346, 0, 460, 154, 1550, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5973, 6847, 6341, 6220, 5499, 6314, 5702, 6256] avg=6144.0000 max=6847 max_rank=1 min=5499 min_rank=4 imbalance_ratio=1.114421 max_load_reduction=42305 max_load_reduction_pct=0.860697 moved_to_echo=23771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[976, 515, 100, 832, 172, 693, 390, 140, 471, 0, 0, 0, 0, 0, 0, 0, 166, 942, 1015, 766, 1028, 185, 447, 1304, 97, 0, 0, 0, 0, 0, 0, 0, 17, 259, 913, 92, 2183, 683, 890, 1295, 581, 408, 0, 0, 0, 0, 0, 0, 188, 1095, 566, 316, 1494, 535, 35, 185, 1995, 0, 0, 0, 0, 0, 0, 0, 231, 1655, 557, 273, 21, 340, 69, 2573, 622, 188, 0, 0, 0, 0, 0, 0, 1159, 275, 840, 1538, 0, 1331, 364, 596, 0, 0, 0, 0, 0, 0, 0, 0, 433, 2798, 243, 294, 786, 607, 1515, 305, 0, 0, 0, 0, 0, 0, 0, 0, 1842, 747, 74, 281, 1042, 44, 315, 253, 972, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4289, 5950, 7321, 6409, 6529, 6103, 6981, 5570] avg=6144.0000 max=7321 max_rank=2 min=4289 min_rank=0 imbalance_ratio=1.191569 max_load_reduction=41831 max_load_reduction_pct=0.851054 moved_to_echo=25183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[880, 311, 1, 3013, 76, 33, 303, 137, 1099, 0, 0, 0, 0, 0, 0, 0, 2, 546, 1437, 751, 1064, 583, 413, 1548, 231, 0, 0, 0, 0, 0, 0, 0, 1, 17, 777, 37, 344, 476, 479, 2582, 949, 863, 0, 0, 0, 0, 0, 0, 114, 199, 989, 1, 712, 230, 0, 19, 4219, 0, 0, 0, 0, 0, 0, 0, 71, 1926, 548, 11, 0, 5, 4, 1944, 1015, 445, 0, 0, 0, 0, 0, 0, 1152, 250, 216, 1726, 0, 396, 784, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 388, 807, 1, 839, 327, 910, 2454, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 7, 3, 32, 378, 0, 302, 122, 1586, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5853, 6575, 6525, 6483, 5969, 5893, 5731, 6123] avg=6144.0000 max=6575 max_rank=1 min=5731 min_rank=6 imbalance_ratio=1.070150 max_load_reduction=42577 max_load_reduction_pct=0.866231 moved_to_echo=23716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[886, 260, 2, 2910, 76, 55, 261, 118, 1075, 0, 0, 0, 0, 0, 0, 0, 2, 691, 1411, 917, 1107, 429, 450, 1541, 222, 0, 0, 0, 0, 0, 0, 0, 0, 35, 853, 30, 288, 561, 515, 2492, 912, 815, 0, 0, 0, 0, 0, 0, 140, 244, 715, 2, 683, 285, 0, 26, 3990, 0, 0, 0, 0, 0, 0, 0, 104, 1683, 493, 37, 2, 0, 0, 1678, 977, 431, 0, 0, 0, 0, 0, 0, 1281, 314, 241, 1978, 0, 357, 843, 1359, 0, 0, 0, 0, 0, 0, 0, 0, 364, 1171, 4, 737, 380, 1155, 2380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 16, 1, 45, 363, 0, 373, 160, 1527, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5643, 6770, 6501, 6085, 5405, 6373, 6191, 6184] avg=6144.0000 max=6770 max_rank=1 min=5405 min_rank=4 imbalance_ratio=1.101888 max_load_reduction=42382 max_load_reduction_pct=0.862264 moved_to_echo=24153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[623, 91, 0, 4569, 83, 1, 183, 74, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 453, 731, 557, 971, 777, 389, 946, 308, 0, 0, 0, 0, 0, 0, 0, 0, 1, 553, 0, 26, 430, 370, 1780, 997, 944, 0, 0, 0, 0, 0, 0, 96, 74, 572, 0, 276, 227, 0, 0, 4615, 0, 0, 0, 0, 0, 0, 0, 74, 1956, 205, 1, 0, 0, 0, 2827, 1067, 597, 0, 0, 0, 0, 0, 0, 1098, 254, 178, 1664, 0, 132, 876, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 379, 100, 0, 1630, 297, 1505, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4285, 0, 0, 5, 146, 0, 383, 135, 1668, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7114, 5132, 5101, 5860, 6727, 6085, 6511, 6622] avg=6144.0000 max=7114 max_rank=0 min=5101 min_rank=2 imbalance_ratio=1.157878 max_load_reduction=42038 max_load_reduction_pct=0.855265 moved_to_echo=25945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[880, 311, 1, 3013, 76, 33, 303, 137, 1099, 0, 0, 0, 0, 0, 0, 0, 2, 546, 1437, 751, 1064, 583, 413, 1548, 231, 0, 0, 0, 0, 0, 0, 0, 1, 17, 777, 37, 344, 476, 479, 2582, 949, 863, 0, 0, 0, 0, 0, 0, 114, 199, 989, 1, 712, 230, 0, 19, 4219, 0, 0, 0, 0, 0, 0, 0, 71, 1926, 548, 11, 0, 5, 4, 1944, 1015, 445, 0, 0, 0, 0, 0, 0, 1152, 250, 216, 1726, 0, 396, 784, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 388, 807, 1, 839, 327, 910, 2454, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 7, 3, 32, 378, 0, 302, 122, 1586, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5853, 6575, 6525, 6483, 5969, 5893, 5731, 6123] avg=6144.0000 max=6575 max_rank=1 min=5731 min_rank=6 imbalance_ratio=1.070150 max_load_reduction=42577 max_load_reduction_pct=0.866231 moved_to_echo=23716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[808, 294, 2, 3444, 76, 67, 279, 126, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 583, 1524, 935, 1087, 550, 431, 1610, 217, 0, 0, 0, 0, 0, 0, 0, 3, 31, 827, 21, 852, 549, 459, 2407, 825, 823, 0, 0, 0, 0, 0, 0, 101, 328, 629, 5, 673, 478, 0, 29, 4028, 0, 0, 0, 0, 0, 0, 0, 109, 1891, 400, 41, 0, 0, 0, 2031, 884, 420, 0, 0, 0, 0, 0, 0, 1283, 259, 239, 1592, 0, 374, 951, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 403, 642, 4, 1066, 307, 850, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 5, 1, 42, 343, 0, 294, 212, 1381, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6937, 6797, 6271, 5776, 6022, 5424, 5780] avg=6144.0000 max=6937 max_rank=1 min=5424 min_rank=6 imbalance_ratio=1.129069 max_load_reduction=42215 max_load_reduction_pct=0.858866 moved_to_echo=23002 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 384, 6144, 256, 4224, 4224, 4096, 16640, 7168, 6656, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9472, 2176, 2432, 13696, 0, 3200, 6656, 11904, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 4736, 9600, 6272, 8448, 4352, 3456, 10624, 2048, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 6400, 256, 8832, 3200, 8704, 18688, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9472, 2176, 2432, 13696, 0, 3200, 6656, 11904, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6272, 1920, 128, 27008, 768, 896, 2176, 896, 9472, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 6400, 256, 8832, 3200, 8704, 18688, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 14976, 2944, 512, 128, 384, 128, 18432, 7680, 3840, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[28800, 896, 128, 512, 2944, 128, 2944, 1408, 12032, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1024, 2176, 5632, 384, 4992, 2688, 128, 384, 32256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1611, 3495, 1, 372, 1236, 185, 77, 107, 1, 401, 190, 265, 52, 325, 528, 641, 372, 921, 0, 11, 852, 112, 3083, 1304, 2617, 3714, 1198, 366, 0, 305, 122, 464, 0, 272, 2498, 0, 74, 1398, 810, 251, 665, 115, 0, 71, 1958, 5401, 628, 0, 5439, 0, 1843, 572, 461, 105, 356, 9, 13, 389, 0, 0, 0, 455, 132, 309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[2258, 6890, 0, 43, 553, 0, 68, 0, 0, 335, 98, 206, 79, 343, 581, 676, 172, 1829, 0, 0, 194, 107, 1242, 1203, 4435, 3885, 2232, 336, 1, 0, 82, 261, 0, 339, 581, 0, 80, 1648, 633, 93, 700, 0, 0, 59, 801, 6185, 75, 0, 5425, 0, 842, 874, 500, 65, 216, 0, 0, 424, 0, 0, 0, 972, 253, 278] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[988, 1028, 249, 607, 3213, 766, 798, 482, 11, 534, 1401, 732, 169, 617, 681, 787, 432, 1543, 248, 642, 754, 133, 2191, 1872, 1878, 855, 768, 155, 241, 414, 336, 787, 434, 560, 1919, 141, 94, 1113, 1288, 191, 807, 407, 68, 609, 1159, 1930, 211, 48, 3101, 738, 2893, 481, 558, 288, 362, 546, 243, 816, 153, 63, 26, 94, 860, 639] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1545, 4404, 0, 211, 1099, 132, 57, 35, 0, 290, 145, 236, 160, 313, 556, 714, 361, 922, 0, 6, 600, 109, 2873, 1172, 2844, 3576, 1130, 361, 0, 304, 144, 392, 0, 241, 1815, 0, 100, 1357, 757, 205, 746, 60, 0, 54, 1997, 5843, 650, 0, 5469, 0, 1973, 739, 476, 124, 350, 5, 10, 374, 0, 0, 0, 545, 144, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1696, 3678, 8, 246, 1209, 190, 52, 82, 0, 275, 171, 392, 65, 322, 556, 814, 394, 969, 7, 15, 866, 93, 3203, 1698, 2485, 3422, 1325, 277, 1, 369, 93, 391, 8, 259, 1689, 0, 73, 1425, 813, 205, 696, 77, 0, 78, 1536, 5395, 516, 5, 5642, 8, 2478, 581, 457, 109, 374, 10, 15, 403, 5, 4, 3, 390, 166, 368] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1916, 6896, 0, 102, 581, 18, 53, 0, 0, 296, 85, 216, 130, 339, 474, 729, 159, 1471, 0, 0, 225, 79, 1887, 1219, 4052, 3921, 2145, 306, 0, 0, 157, 262, 0, 358, 606, 0, 90, 1567, 597, 116, 696, 9, 0, 72, 797, 6227, 82, 0, 5278, 0, 1320, 859, 466, 83, 248, 0, 0, 359, 0, 0, 0, 1029, 227, 348] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1705, 3807, 0, 243, 1283, 178, 70, 79, 2, 466, 215, 533, 82, 334, 584, 743, 407, 824, 0, 41, 799, 98, 2393, 1478, 2625, 3495, 1186, 258, 6, 347, 184, 434, 0, 246, 1710, 0, 76, 1830, 936, 296, 722, 81, 0, 68, 1842, 5261, 809, 0, 5567, 0, 1715, 602, 437, 131, 368, 3, 13, 383, 0, 0, 0, 508, 250, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[2258, 6890, 0, 43, 553, 0, 68, 0, 0, 335, 98, 206, 79, 343, 581, 676, 172, 1829, 0, 0, 194, 107, 1242, 1203, 4435, 3885, 2232, 336, 1, 0, 82, 261, 0, 339, 581, 0, 80, 1648, 633, 93, 700, 0, 0, 59, 801, 6185, 75, 0, 5425, 0, 842, 874, 500, 65, 216, 0, 0, 424, 0, 0, 0, 972, 253, 278] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1611, 3495, 1, 372, 1236, 185, 77, 107, 1, 401, 190, 265, 52, 325, 528, 641, 372, 921, 0, 11, 852, 112, 3083, 1304, 2617, 3714, 1198, 366, 0, 305, 122, 464, 0, 272, 2498, 0, 74, 1398, 810, 251, 665, 115, 0, 71, 1958, 5401, 628, 0, 5439, 0, 1843, 572, 461, 105, 356, 9, 13, 389, 0, 0, 0, 455, 132, 309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1864, 6042, 11, 143, 635, 39, 61, 9, 0, 306, 120, 225, 117, 331, 439, 681, 265, 1243, 6, 3, 509, 109, 2336, 1264, 3358, 4070, 1697, 321, 1, 62, 133, 340, 0, 315, 1078, 0, 76, 1674, 671, 129, 665, 8, 0, 63, 1271, 6110, 206, 7, 5509, 11, 1247, 799, 453, 81, 286, 1, 4, 374, 5, 1, 2, 835, 196, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[988, 1028, 249, 607, 3213, 766, 798, 482, 11, 534, 1401, 732, 169, 617, 681, 787, 432, 1543, 248, 642, 754, 133, 2191, 1872, 1878, 855, 768, 155, 241, 414, 336, 787, 434, 560, 1919, 141, 94, 1113, 1288, 191, 807, 407, 68, 609, 1159, 1930, 211, 48, 3101, 738, 2893, 481, 558, 288, 362, 546, 243, 816, 153, 63, 26, 94, 860, 639] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1916, 6896, 0, 102, 581, 18, 53, 0, 0, 296, 85, 216, 130, 339, 474, 729, 159, 1471, 0, 0, 225, 79, 1887, 1219, 4052, 3921, 2145, 306, 0, 0, 157, 262, 0, 358, 606, 0, 90, 1567, 597, 116, 696, 9, 0, 72, 797, 6227, 82, 0, 5278, 0, 1320, 859, 466, 83, 248, 0, 0, 359, 0, 0, 0, 1029, 227, 348] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1611, 1928, 1, 372, 1236, 185, 77, 107, 0, 0, 0, 0, 0, 0, 0, 0, 1, 401, 190, 265, 52, 325, 528, 641, 1567, 1134, 273, 0, 0, 0, 0, 0, 372, 921, 0, 11, 852, 112, 3057, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 575, 1198, 366, 0, 305, 122, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 2498, 0, 74, 1398, 810, 251, 1689, 26, 0, 0, 0, 0, 0, 0, 665, 115, 0, 71, 1958, 3439, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2712, 0, 1843, 572, 461, 105, 356, 9, 0, 0, 0, 0, 0, 0, 0, 0, 13, 389, 0, 0, 0, 455, 132, 309, 3139, 1593, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5517, 5377, 6629, 5647, 7018, 6876, 6058, 6030] avg=6144.0000 max=7018 max_rank=4 min=5377 min_rank=1 imbalance_ratio=1.142253 max_load_reduction=42134 max_load_reduction_pct=0.857218 moved_to_echo=25982 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2258, 3800, 0, 43, 553, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 98, 206, 79, 343, 581, 676, 3090, 1131, 313, 0, 0, 0, 0, 0, 172, 1829, 0, 0, 194, 107, 1232, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 4435, 601, 2232, 336, 1, 0, 82, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 581, 0, 80, 1648, 633, 93, 1934, 10, 0, 0, 0, 0, 0, 0, 700, 0, 0, 59, 801, 3938, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 842, 874, 500, 65, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 972, 253, 278, 3284, 1589, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6722, 6852, 4737, 7948, 5318, 5573, 5202, 6800] avg=6144.0000 max=7948 max_rank=3 min=4737 min_rank=2 imbalance_ratio=1.293620 max_load_reduction=41204 max_load_reduction_pct=0.838298 moved_to_echo=22893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1696, 3678, 8, 246, 1209, 190, 52, 82, 0, 275, 171, 392, 65, 322, 556, 814, 394, 969, 7, 15, 866, 93, 3203, 1698, 2485, 3422, 1325, 277, 1, 369, 93, 391, 8, 259, 1689, 0, 73, 1425, 813, 205, 696, 77, 0, 78, 1536, 5395, 516, 5, 5642, 8, 2478, 581, 457, 109, 374, 10, 15, 403, 5, 4, 3, 390, 166, 368] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1545, 2429, 0, 211, 1099, 132, 57, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 145, 236, 160, 313, 556, 714, 1975, 1141, 295, 0, 0, 0, 0, 0, 361, 922, 0, 6, 600, 109, 2848, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 2844, 553, 1130, 361, 0, 304, 144, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 1815, 0, 100, 1357, 757, 205, 1827, 25, 0, 0, 0, 0, 0, 0, 746, 60, 0, 54, 1997, 3721, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2726, 0, 1973, 739, 476, 124, 350, 5, 0, 0, 0, 0, 0, 0, 0, 0, 10, 374, 0, 0, 0, 545, 144, 427, 3023, 1602, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5508, 5825, 6018, 5728, 6327, 7228, 6393, 6125] avg=6144.0000 max=7228 max_rank=5 min=5508 min_rank=0 imbalance_ratio=1.176432 max_load_reduction=41924 max_load_reduction_pct=0.852946 moved_to_echo=26073 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1864, 3333, 11, 143, 635, 39, 61, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 120, 225, 117, 331, 439, 681, 2709, 1149, 309, 0, 0, 0, 0, 0, 265, 1243, 6, 3, 509, 109, 2316, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 630, 1697, 321, 1, 62, 133, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 1078, 0, 76, 1674, 671, 129, 1911, 20, 0, 0, 0, 0, 0, 0, 665, 8, 0, 63, 1271, 3890, 206, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2747, 11, 1247, 799, 453, 81, 286, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 374, 5, 1, 2, 835, 196, 335, 3440, 1613, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6386, 5715, 6542, 5874, 6110, 5625, 6805] avg=6144.0000 max=6805 max_rank=7 min=5625 min_rank=6 imbalance_ratio=1.107585 max_load_reduction=42347 max_load_reduction_pct=0.861552 moved_to_echo=24414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[988, 567, 249, 607, 3213, 766, 798, 482, 0, 0, 0, 0, 0, 0, 0, 0, 11, 534, 1401, 732, 169, 617, 681, 787, 461, 646, 97, 0, 0, 0, 0, 0, 432, 1543, 248, 642, 754, 133, 2172, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 133, 768, 155, 241, 414, 336, 787, 0, 0, 0, 0, 0, 0, 0, 0, 434, 560, 1919, 141, 94, 1113, 1288, 191, 603, 19, 0, 0, 0, 0, 0, 0, 807, 407, 68, 609, 1159, 1230, 211, 48, 0, 0, 0, 0, 0, 0, 0, 0, 1547, 738, 2893, 481, 558, 288, 362, 546, 0, 0, 0, 0, 0, 0, 0, 0, 243, 816, 153, 63, 26, 94, 860, 639, 722, 908, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7670, 6136, 7796, 4712, 6362, 4539, 7413, 4524] avg=6144.0000 max=7796 max_rank=2 min=4524 min_rank=7 imbalance_ratio=1.268880 max_load_reduction=41356 max_load_reduction_pct=0.841390 moved_to_echo=22838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1696, 2025, 8, 246, 1209, 190, 52, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 171, 392, 65, 322, 556, 814, 1653, 1181, 277, 0, 0, 0, 0, 0, 394, 969, 7, 15, 866, 93, 3171, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 2485, 524, 1325, 277, 1, 369, 93, 391, 0, 0, 0, 0, 0, 0, 0, 0, 8, 259, 1689, 0, 73, 1425, 813, 205, 1693, 32, 0, 0, 0, 0, 0, 0, 696, 77, 0, 78, 1536, 3425, 516, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2805, 8, 2478, 581, 457, 109, 374, 10, 0, 0, 0, 0, 0, 0, 0, 0, 15, 403, 5, 4, 3, 390, 166, 368, 2898, 1656, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5508, 5706, 7213, 5465, 6197, 6333, 6822, 5908] avg=6144.0000 max=7213 max_rank=2 min=5465 min_rank=3 imbalance_ratio=1.173991 max_load_reduction=41939 max_load_reduction_pct=0.853251 moved_to_echo=25260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1705, 2100, 0, 243, 1283, 178, 70, 79, 0, 0, 0, 0, 0, 0, 0, 0, 2, 466, 215, 533, 82, 334, 584, 743, 1707, 1161, 266, 0, 0, 0, 0, 0, 407, 824, 0, 41, 799, 98, 2373, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 2625, 541, 1186, 258, 6, 347, 184, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1710, 0, 76, 1830, 936, 296, 1645, 20, 0, 0, 0, 0, 0, 0, 722, 81, 0, 68, 1842, 3350, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2776, 0, 1715, 602, 437, 131, 368, 3, 0, 0, 0, 0, 0, 0, 0, 0, 13, 383, 0, 0, 0, 508, 250, 399, 2954, 1630, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5658, 6093, 6020, 5581, 6759, 6872, 6032, 6137] avg=6144.0000 max=6872 max_rank=5 min=5581 min_rank=3 imbalance_ratio=1.118490 max_load_reduction=42280 max_load_reduction_pct=0.860189 moved_to_echo=25800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2258, 3800, 0, 43, 553, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 98, 206, 79, 343, 581, 676, 3090, 1131, 313, 0, 0, 0, 0, 0, 172, 1829, 0, 0, 194, 107, 1232, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 4435, 601, 2232, 336, 1, 0, 82, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 581, 0, 80, 1648, 633, 93, 1934, 10, 0, 0, 0, 0, 0, 0, 700, 0, 0, 59, 801, 3938, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 842, 874, 500, 65, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 972, 253, 278, 3284, 1589, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6722, 6852, 4737, 7948, 5318, 5573, 5202, 6800] avg=6144.0000 max=7948 max_rank=3 min=4737 min_rank=2 imbalance_ratio=1.293620 max_load_reduction=41204 max_load_reduction_pct=0.838298 moved_to_echo=22893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1611, 1928, 1, 372, 1236, 185, 77, 107, 0, 0, 0, 0, 0, 0, 0, 0, 1, 401, 190, 265, 52, 325, 528, 641, 1567, 1134, 273, 0, 0, 0, 0, 0, 372, 921, 0, 11, 852, 112, 3057, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 575, 1198, 366, 0, 305, 122, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 2498, 0, 74, 1398, 810, 251, 1689, 26, 0, 0, 0, 0, 0, 0, 665, 115, 0, 71, 1958, 3439, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2712, 0, 1843, 572, 461, 105, 356, 9, 0, 0, 0, 0, 0, 0, 0, 0, 13, 389, 0, 0, 0, 455, 132, 309, 3139, 1593, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5517, 5377, 6629, 5647, 7018, 6876, 6058, 6030] avg=6144.0000 max=7018 max_rank=4 min=5377 min_rank=1 imbalance_ratio=1.142253 max_load_reduction=42134 max_load_reduction_pct=0.857218 moved_to_echo=25982 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1916, 3804, 0, 102, 581, 18, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 85, 216, 130, 339, 474, 729, 3092, 1101, 315, 0, 0, 0, 0, 0, 159, 1471, 0, 0, 225, 79, 1871, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 607, 2145, 306, 0, 0, 157, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 606, 0, 90, 1567, 597, 116, 1947, 16, 0, 0, 0, 0, 0, 0, 696, 9, 0, 72, 797, 3965, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 0, 1320, 859, 466, 83, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 1029, 227, 348, 3314, 1546, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6474, 6777, 5024, 7529, 5297, 5621, 5607, 6823] avg=6144.0000 max=7529 max_rank=3 min=5024 min_rank=2 imbalance_ratio=1.225423 max_load_reduction=41623 max_load_reduction_pct=0.846822 moved_to_echo=23348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1864, 3333, 11, 143, 635, 39, 61, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 120, 225, 117, 331, 439, 681, 2709, 1149, 309, 0, 0, 0, 0, 0, 265, 1243, 6, 3, 509, 109, 2316, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 630, 1697, 321, 1, 62, 133, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 1078, 0, 76, 1674, 671, 129, 1911, 20, 0, 0, 0, 0, 0, 0, 665, 8, 0, 63, 1271, 3890, 206, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2747, 11, 1247, 799, 453, 81, 286, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 374, 5, 1, 2, 835, 196, 335, 3440, 1613, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6386, 5715, 6542, 5874, 6110, 5625, 6805] avg=6144.0000 max=6805 max_rank=7 min=5625 min_rank=6 imbalance_ratio=1.107585 max_load_reduction=42347 max_load_reduction_pct=0.861552 moved_to_echo=24414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1696, 2025, 8, 246, 1209, 190, 52, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 171, 392, 65, 322, 556, 814, 1653, 1181, 277, 0, 0, 0, 0, 0, 394, 969, 7, 15, 866, 93, 3171, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 2485, 524, 1325, 277, 1, 369, 93, 391, 0, 0, 0, 0, 0, 0, 0, 0, 8, 259, 1689, 0, 73, 1425, 813, 205, 1693, 32, 0, 0, 0, 0, 0, 0, 696, 77, 0, 78, 1536, 3425, 516, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2805, 8, 2478, 581, 457, 109, 374, 10, 0, 0, 0, 0, 0, 0, 0, 0, 15, 403, 5, 4, 3, 390, 166, 368, 2898, 1656, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5508, 5706, 7213, 5465, 6197, 6333, 6822, 5908] avg=6144.0000 max=7213 max_rank=2 min=5465 min_rank=3 imbalance_ratio=1.173991 max_load_reduction=41939 max_load_reduction_pct=0.853251 moved_to_echo=25260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2944, 2432, 2816, 896, 2944, 4480, 5888, 16256, 8704, 2176, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 3584, 256, 128, 128, 4864, 2304, 3200, 22784, 12160, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13696, 20096, 384, 2048, 9856, 1536, 1280, 896, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 9728, 384, 768, 4864, 896, 19072, 11264, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5760, 768, 128, 1152, 11392, 27008, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[20736, 768, 14336, 5632, 3840, 1024, 2560, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 2688, 11904, 256, 768, 12032, 6528, 1536, 13312, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13696, 20096, 384, 2048, 9856, 1536, 1280, 896, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[24320, 4224, 11776, 2432, 256, 1920, 1280, 3456, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 2688, 11904, 256, 768, 12032, 6528, 1536, 13312, 256, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[587, 530, 4055, 3631, 1269, 0, 6341, 5915, 0, 148, 0, 655, 0, 1095, 0, 277, 256, 40, 1729, 28, 846, 71, 401, 161, 70, 102, 1134, 0, 4, 145, 0, 174, 5458, 789, 0, 92, 260, 763, 721, 0, 386, 0, 125, 0, 27, 0, 363, 93, 2251, 0, 217, 194, 129, 1942, 69, 0, 2560, 446, 75, 855, 1408, 1, 264, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[543, 449, 5080, 2586, 2212, 2, 6099, 5779, 7, 153, 4, 787, 9, 751, 0, 270, 287, 0, 2439, 45, 935, 55, 363, 135, 64, 76, 942, 8, 0, 137, 6, 171, 5643, 668, 0, 76, 232, 867, 720, 1, 373, 1, 10, 4, 0, 5, 325, 21, 2210, 5, 8, 133, 15, 1563, 207, 5, 2220, 454, 22, 1275, 1656, 0, 39, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[669, 2263, 2130, 1143, 422, 332, 3669, 1939, 135, 204, 222, 611, 277, 1276, 140, 1071, 502, 359, 1522, 75, 804, 125, 1356, 1493, 63, 585, 2011, 144, 1014, 200, 421, 1208, 3141, 697, 169, 167, 466, 1709, 768, 226, 511, 917, 1363, 1187, 307, 257, 620, 323, 1086, 58, 273, 407, 973, 553, 308, 221, 877, 738, 360, 204, 956, 13, 784, 128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[628, 590, 4148, 3267, 1550, 7, 6140, 5996, 4, 144, 9, 684, 7, 1086, 0, 322, 260, 1, 1654, 60, 840, 54, 433, 159, 47, 85, 1417, 0, 9, 122, 12, 194, 5488, 737, 1, 78, 235, 740, 628, 4, 387, 3, 75, 3, 44, 6, 370, 21, 2245, 0, 162, 234, 147, 2034, 95, 3, 3239, 547, 111, 590, 675, 0, 317, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[541, 336, 4705, 2490, 1931, 0, 6251, 5189, 0, 160, 0, 824, 0, 1016, 0, 370, 277, 0, 2393, 28, 967, 46, 85, 148, 68, 80, 756, 0, 0, 148, 0, 190, 5635, 709, 0, 80, 203, 1415, 758, 0, 422, 0, 7, 0, 0, 0, 72, 3, 2641, 0, 34, 102, 3, 1566, 443, 0, 1988, 509, 18, 1494, 2020, 6, 25, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[584, 527, 5017, 2718, 1925, 0, 6147, 6048, 0, 199, 0, 808, 0, 961, 0, 222, 279, 0, 1997, 45, 922, 50, 346, 123, 55, 112, 1155, 0, 0, 155, 0, 165, 5460, 765, 0, 138, 244, 748, 749, 0, 359, 0, 22, 0, 0, 0, 345, 144, 2473, 0, 112, 201, 5, 1281, 93, 0, 2733, 478, 87, 843, 1162, 3, 146, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[603, 630, 3877, 3372, 1973, 0, 6245, 6351, 0, 146, 0, 616, 0, 1126, 0, 196, 244, 17, 2234, 51, 869, 77, 359, 171, 64, 128, 1326, 0, 10, 138, 0, 167, 5479, 733, 0, 96, 214, 648, 589, 0, 370, 0, 76, 0, 125, 0, 330, 31, 2412, 0, 44, 215, 8, 1228, 106, 0, 2777, 468, 80, 803, 1047, 3, 278, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[628, 590, 4148, 3267, 1550, 7, 6140, 5996, 4, 144, 9, 684, 7, 1086, 0, 322, 260, 1, 1654, 60, 840, 54, 433, 159, 47, 85, 1417, 0, 9, 122, 12, 194, 5488, 737, 1, 78, 235, 740, 628, 4, 387, 3, 75, 3, 44, 6, 370, 21, 2245, 0, 162, 234, 147, 2034, 95, 3, 3239, 547, 111, 590, 675, 0, 317, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[669, 2263, 2130, 1143, 422, 332, 3669, 1939, 135, 204, 222, 611, 277, 1276, 140, 1071, 502, 359, 1522, 75, 804, 125, 1356, 1493, 63, 585, 2011, 144, 1014, 200, 421, 1208, 3141, 697, 169, 167, 466, 1709, 768, 226, 511, 917, 1363, 1187, 307, 257, 620, 323, 1086, 58, 273, 407, 973, 553, 308, 221, 877, 738, 360, 204, 956, 13, 784, 128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[587, 530, 4055, 3631, 1269, 0, 6341, 5915, 0, 148, 0, 655, 0, 1095, 0, 277, 256, 40, 1729, 28, 846, 71, 401, 161, 70, 102, 1134, 0, 4, 145, 0, 174, 5458, 789, 0, 92, 260, 763, 721, 0, 386, 0, 125, 0, 27, 0, 363, 93, 2251, 0, 217, 194, 129, 1942, 69, 0, 2560, 446, 75, 855, 1408, 1, 264, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[583, 444, 5051, 2191, 2400, 0, 5900, 5125, 0, 178, 0, 899, 0, 994, 0, 240, 278, 0, 2654, 23, 891, 46, 131, 136, 53, 89, 781, 0, 0, 143, 0, 218, 5770, 654, 0, 90, 206, 1063, 710, 0, 342, 5, 5, 0, 0, 0, 192, 22, 2308, 0, 1, 142, 9, 1779, 482, 0, 2469, 499, 30, 1382, 1535, 0, 9, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[603, 630, 3877, 3372, 1973, 0, 6245, 6351, 0, 146, 0, 616, 0, 1126, 0, 196, 244, 17, 2234, 51, 869, 77, 359, 171, 64, 128, 1326, 0, 10, 138, 0, 167, 5479, 733, 0, 96, 214, 648, 589, 0, 370, 0, 76, 0, 125, 0, 330, 31, 2412, 0, 44, 215, 8, 1228, 106, 0, 2777, 468, 80, 803, 1047, 3, 278, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[543, 449, 5080, 2586, 2212, 2, 6099, 5779, 7, 153, 4, 787, 9, 751, 0, 270, 287, 0, 2439, 45, 935, 55, 363, 135, 64, 76, 942, 8, 0, 137, 6, 171, 5643, 668, 0, 76, 232, 867, 720, 1, 373, 1, 10, 4, 0, 5, 325, 21, 2210, 5, 8, 133, 15, 1563, 207, 5, 2220, 454, 22, 1275, 1656, 0, 39, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[541, 336, 446, 2490, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 824, 0, 1016, 0, 370, 2254, 1582, 0, 0, 0, 0, 0, 0, 277, 0, 2393, 28, 967, 46, 85, 148, 2179, 0, 0, 0, 0, 0, 0, 0, 68, 80, 756, 0, 0, 148, 0, 190, 2935, 1154, 0, 0, 0, 0, 0, 0, 3554, 709, 0, 80, 203, 1415, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 7, 0, 0, 0, 72, 3, 5097, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 34, 102, 3, 1566, 443, 0, 1353, 498, 0, 0, 0, 0, 0, 0, 1988, 509, 18, 1494, 2020, 6, 25, 0, 728, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5744, 6206, 6123, 5331, 6719, 5601, 6640, 6788] avg=6144.0000 max=6788 max_rank=7 min=5331 min_rank=3 imbalance_ratio=1.104818 max_load_reduction=42364 max_load_reduction_pct=0.861898 moved_to_echo=25748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[669, 2263, 203, 1143, 422, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 204, 222, 611, 277, 1276, 140, 1071, 843, 716, 0, 0, 0, 0, 0, 0, 502, 359, 1522, 75, 804, 125, 1356, 1493, 986, 0, 0, 0, 0, 0, 0, 0, 63, 585, 2011, 144, 1014, 200, 421, 1208, 1096, 678, 0, 0, 0, 0, 0, 0, 1981, 697, 169, 167, 466, 1709, 768, 226, 0, 0, 0, 0, 0, 0, 0, 0, 511, 917, 1363, 1187, 307, 257, 620, 323, 2991, 0, 0, 0, 0, 0, 0, 0, 1086, 58, 273, 407, 973, 553, 308, 221, 754, 225, 0, 0, 0, 0, 0, 0, 877, 738, 360, 204, 956, 13, 784, 128, 406, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5032, 5495, 7222, 7420, 6183, 8476, 4858, 4466] avg=6144.0000 max=8476 max_rank=5 min=4466 min_rank=7 imbalance_ratio=1.379557 max_load_reduction=40676 max_load_reduction_pct=0.827555 moved_to_echo=23983 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[584, 527, 476, 2718, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 808, 0, 961, 0, 222, 2627, 1687, 0, 0, 0, 0, 0, 0, 279, 0, 1997, 45, 922, 50, 346, 123, 2323, 0, 0, 0, 0, 0, 0, 0, 55, 112, 1155, 0, 0, 155, 0, 165, 3421, 1134, 0, 0, 0, 0, 0, 0, 3444, 765, 0, 138, 244, 748, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 22, 0, 0, 0, 345, 144, 5013, 0, 0, 0, 0, 0, 0, 0, 2473, 0, 112, 201, 5, 1281, 93, 0, 1311, 531, 0, 0, 0, 0, 0, 0, 2733, 478, 87, 843, 1162, 3, 146, 1, 705, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6230, 6504, 6085, 6197, 6088, 5883, 6007, 6158] avg=6144.0000 max=6504 max_rank=1 min=5883 min_rank=5 imbalance_ratio=1.058594 max_load_reduction=42648 max_load_reduction_pct=0.867676 moved_to_echo=24136 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[543, 449, 482, 2586, 2212, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 153, 4, 787, 9, 751, 0, 270, 2511, 1708, 0, 0, 0, 0, 0, 0, 287, 0, 2439, 45, 935, 55, 363, 135, 2352, 0, 0, 0, 0, 0, 0, 0, 64, 76, 942, 8, 0, 137, 6, 171, 3268, 1127, 0, 0, 0, 0, 0, 0, 3559, 668, 0, 76, 232, 867, 720, 1, 0, 0, 0, 0, 0, 0, 0, 0, 373, 1, 10, 4, 0, 5, 325, 21, 4972, 0, 0, 0, 0, 0, 0, 0, 2210, 5, 8, 133, 15, 1563, 207, 5, 1355, 538, 0, 0, 0, 0, 0, 0, 2220, 454, 22, 1275, 1656, 0, 39, 0, 729, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 6200, 6611, 5799, 6123, 5711, 6039, 6395] avg=6144.0000 max=6611 max_rank=2 min=5711 min_rank=5 imbalance_ratio=1.076009 max_load_reduction=42541 max_load_reduction_pct=0.865499 moved_to_echo=24268 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[603, 630, 369, 3372, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 616, 0, 1126, 0, 196, 2759, 1303, 0, 0, 0, 0, 0, 0, 244, 17, 2234, 51, 869, 77, 359, 171, 1795, 0, 0, 0, 0, 0, 0, 0, 64, 128, 1326, 0, 10, 138, 0, 167, 3592, 1152, 0, 0, 0, 0, 0, 0, 3455, 733, 0, 96, 214, 648, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 76, 0, 125, 0, 330, 31, 5093, 0, 0, 0, 0, 0, 0, 0, 2412, 0, 44, 215, 8, 1228, 106, 0, 1316, 410, 0, 0, 0, 0, 0, 0, 2777, 468, 80, 803, 1047, 3, 278, 2, 708, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6947, 6146, 5817, 6577, 5735, 6025, 5739, 6166] avg=6144.0000 max=6947 max_rank=0 min=5735 min_rank=4 imbalance_ratio=1.130697 max_load_reduction=42205 max_load_reduction_pct=0.858663 moved_to_echo=23665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[587, 530, 385, 3631, 1269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 655, 0, 1095, 0, 277, 2569, 1363, 0, 0, 0, 0, 0, 0, 256, 40, 1729, 28, 846, 71, 401, 161, 1878, 0, 0, 0, 0, 0, 0, 0, 70, 102, 1134, 0, 4, 145, 0, 174, 3346, 1170, 0, 0, 0, 0, 0, 0, 3442, 789, 0, 92, 260, 763, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 0, 125, 0, 27, 0, 363, 93, 5171, 0, 0, 0, 0, 0, 0, 0, 2251, 0, 217, 194, 129, 1942, 69, 0, 1311, 429, 0, 0, 0, 0, 0, 0, 2560, 446, 75, 855, 1408, 1, 264, 0, 705, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6402, 6107, 5410, 6145, 6067, 6165, 6542, 6314] avg=6144.0000 max=6542 max_rank=6 min=5410 min_rank=2 imbalance_ratio=1.064779 max_load_reduction=42610 max_load_reduction_pct=0.866903 moved_to_echo=25088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[669, 2263, 203, 1143, 422, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 204, 222, 611, 277, 1276, 140, 1071, 843, 716, 0, 0, 0, 0, 0, 0, 502, 359, 1522, 75, 804, 125, 1356, 1493, 986, 0, 0, 0, 0, 0, 0, 0, 63, 585, 2011, 144, 1014, 200, 421, 1208, 1096, 678, 0, 0, 0, 0, 0, 0, 1981, 697, 169, 167, 466, 1709, 768, 226, 0, 0, 0, 0, 0, 0, 0, 0, 511, 917, 1363, 1187, 307, 257, 620, 323, 2991, 0, 0, 0, 0, 0, 0, 0, 1086, 58, 273, 407, 973, 553, 308, 221, 754, 225, 0, 0, 0, 0, 0, 0, 877, 738, 360, 204, 956, 13, 784, 128, 406, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5032, 5495, 7222, 7420, 6183, 8476, 4858, 4466] avg=6144.0000 max=8476 max_rank=5 min=4466 min_rank=7 imbalance_ratio=1.379557 max_load_reduction=40676 max_load_reduction_pct=0.827555 moved_to_echo=23983 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[603, 630, 369, 3372, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 616, 0, 1126, 0, 196, 2759, 1303, 0, 0, 0, 0, 0, 0, 244, 17, 2234, 51, 869, 77, 359, 171, 1795, 0, 0, 0, 0, 0, 0, 0, 64, 128, 1326, 0, 10, 138, 0, 167, 3592, 1152, 0, 0, 0, 0, 0, 0, 3455, 733, 0, 96, 214, 648, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 76, 0, 125, 0, 330, 31, 5093, 0, 0, 0, 0, 0, 0, 0, 2412, 0, 44, 215, 8, 1228, 106, 0, 1316, 410, 0, 0, 0, 0, 0, 0, 2777, 468, 80, 803, 1047, 3, 278, 2, 708, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6947, 6146, 5817, 6577, 5735, 6025, 5739, 6166] avg=6144.0000 max=6947 max_rank=0 min=5735 min_rank=4 imbalance_ratio=1.130697 max_load_reduction=42205 max_load_reduction_pct=0.858663 moved_to_echo=23665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[628, 590, 384, 3267, 1550, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 144, 9, 684, 7, 1086, 0, 322, 2605, 1397, 0, 0, 0, 0, 0, 0, 260, 1, 1654, 60, 840, 54, 433, 159, 1925, 0, 0, 0, 0, 0, 0, 0, 47, 85, 1417, 0, 9, 122, 12, 194, 3391, 1134, 0, 0, 0, 0, 0, 0, 3452, 737, 1, 78, 235, 740, 628, 4, 0, 0, 0, 0, 0, 0, 0, 0, 387, 3, 75, 3, 44, 6, 370, 21, 5006, 0, 0, 0, 0, 0, 0, 0, 2245, 0, 162, 234, 147, 2034, 95, 3, 1323, 442, 0, 0, 0, 0, 0, 0, 3239, 547, 111, 590, 675, 0, 317, 4, 713, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6426, 6258, 5386, 6411, 5875, 5915, 6685, 6196] avg=6144.0000 max=6685 max_rank=6 min=5386 min_rank=2 imbalance_ratio=1.088053 max_load_reduction=42467 max_load_reduction_pct=0.863993 moved_to_echo=24671 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[543, 449, 482, 2586, 2212, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 153, 4, 787, 9, 751, 0, 270, 2511, 1708, 0, 0, 0, 0, 0, 0, 287, 0, 2439, 45, 935, 55, 363, 135, 2352, 0, 0, 0, 0, 0, 0, 0, 64, 76, 942, 8, 0, 137, 6, 171, 3268, 1127, 0, 0, 0, 0, 0, 0, 3559, 668, 0, 76, 232, 867, 720, 1, 0, 0, 0, 0, 0, 0, 0, 0, 373, 1, 10, 4, 0, 5, 325, 21, 4972, 0, 0, 0, 0, 0, 0, 0, 2210, 5, 8, 133, 15, 1563, 207, 5, 1355, 538, 0, 0, 0, 0, 0, 0, 2220, 454, 22, 1275, 1656, 0, 39, 0, 729, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 6200, 6611, 5799, 6123, 5711, 6039, 6395] avg=6144.0000 max=6611 max_rank=2 min=5711 min_rank=5 imbalance_ratio=1.076009 max_load_reduction=42541 max_load_reduction_pct=0.865499 moved_to_echo=24268 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[583, 444, 479, 2191, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 899, 0, 994, 0, 240, 2227, 1698, 0, 0, 0, 0, 0, 0, 278, 0, 2654, 23, 891, 46, 131, 136, 2339, 0, 0, 0, 0, 0, 0, 0, 53, 89, 781, 0, 0, 143, 0, 218, 2898, 1090, 0, 0, 0, 0, 0, 0, 3639, 654, 0, 90, 206, 1063, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 5, 5, 0, 0, 0, 192, 22, 4810, 0, 0, 0, 0, 0, 0, 0, 2308, 0, 1, 142, 9, 1779, 482, 0, 1386, 535, 0, 0, 0, 0, 0, 0, 2469, 499, 30, 1382, 1535, 0, 9, 0, 745, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6236, 6498, 5272, 6362, 5376, 6642, 6669] avg=6144.0000 max=6669 max_rank=7 min=5272 min_rank=3 imbalance_ratio=1.085449 max_load_reduction=42483 max_load_reduction_pct=0.864319 moved_to_echo=25049 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 1408, 256, 5888, 384, 8320, 256, 3072, 18432, 11520, 0, 0, 0, 0, 0, 0] @@ -10201,235 +10201,235 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3200, 1024, 1792, 1280, 512, 384, 2688, 768, 38272, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 6715, 1873, 7, 7, 64, 494, 5195, 0, 2, 0, 0, 0, 512, 2204, 161, 324, 353, 52, 240, 85, 0, 211, 991, 431, 313, 27, 166, 237, 82, 128, 2, 53, 7361, 105, 0, 737, 0, 1608, 0, 1047, 0, 0, 0, 2327, 149, 0, 0, 53, 0, 77, 822, 71, 534, 73, 0, 0, 377, 6580, 131, 98, 2030, 21, 4022] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 7477, 2701, 0, 3, 68, 580, 4156, 0, 0, 0, 0, 0, 686, 2747, 148, 0, 464, 75, 247, 91, 0, 8, 943, 312, 327, 0, 119, 181, 75, 164, 1, 141, 7579, 80, 0, 2317, 0, 1206, 0, 1751, 0, 0, 0, 2210, 68, 0, 0, 80, 0, 90, 591, 0, 421, 80, 0, 0, 420, 6829, 239, 109, 1559, 0, 1809] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 7070, 2217, 0, 2, 55, 580, 4998, 0, 0, 0, 0, 0, 582, 2244, 162, 252, 333, 160, 247, 67, 0, 137, 964, 593, 346, 0, 135, 232, 69, 262, 0, 91, 7399, 109, 0, 809, 0, 1704, 0, 1047, 0, 0, 0, 2347, 112, 0, 0, 95, 0, 111, 846, 54, 495, 99, 0, 0, 361, 6405, 81, 113, 1869, 5, 3293] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 6875, 2428, 20, 9, 72, 584, 4135, 0, 0, 0, 0, 0, 473, 2229, 164, 361, 417, 86, 252, 84, 0, 249, 962, 491, 325, 2, 175, 250, 91, 157, 4, 56, 7353, 85, 0, 771, 0, 1781, 0, 863, 0, 0, 0, 2187, 211, 0, 0, 62, 0, 75, 1010, 32, 550, 74, 0, 0, 359, 6622, 91, 85, 1931, 15, 4044] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 6986, 2390, 1, 8, 50, 580, 4786, 0, 7, 3, 2, 0, 513, 2553, 127, 348, 357, 64, 255, 64, 7, 144, 984, 272, 331, 16, 182, 256, 65, 140, 0, 58, 7334, 86, 10, 759, 7, 1593, 0, 901, 2, 10, 5, 1799, 194, 3, 11, 58, 11, 84, 855, 35, 532, 72, 10, 1, 385, 6472, 75, 102, 2519, 23, 3655] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 7477, 2701, 0, 3, 68, 580, 4156, 0, 0, 0, 0, 0, 686, 2747, 148, 0, 464, 75, 247, 91, 0, 8, 943, 312, 327, 0, 119, 181, 75, 164, 1, 141, 7579, 80, 0, 2317, 0, 1206, 0, 1751, 0, 0, 0, 2210, 68, 0, 0, 80, 0, 90, 591, 0, 421, 80, 0, 0, 420, 6829, 239, 109, 1559, 0, 1809] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 7327, 2161, 0, 7, 64, 567, 4689, 0, 18, 1, 0, 0, 629, 2273, 140, 44, 413, 116, 239, 88, 8, 53, 889, 460, 315, 0, 137, 236, 61, 192, 0, 131, 7581, 81, 6, 1696, 0, 1745, 1, 1383, 0, 11, 0, 2446, 99, 0, 11, 77, 9, 82, 648, 10, 510, 76, 9, 2, 374, 6703, 165, 94, 1630, 0, 2445] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[63, 3364, 4198, 607, 164, 278, 346, 485, 213, 1230, 357, 260, 151, 780, 2264, 257, 433, 218, 1528, 15, 291, 412, 253, 985, 225, 865, 550, 452, 260, 70, 915, 24, 799, 2336, 474, 216, 319, 1426, 1857, 31, 299, 146, 822, 411, 411, 388, 64, 292, 145, 422, 70, 674, 808, 953, 876, 711, 1063, 579, 2223, 129, 461, 2627, 2534, 2073] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 6875, 2428, 20, 9, 72, 584, 4135, 0, 0, 0, 0, 0, 473, 2229, 164, 361, 417, 86, 252, 84, 0, 249, 962, 491, 325, 2, 175, 250, 91, 157, 4, 56, 7353, 85, 0, 771, 0, 1781, 0, 863, 0, 0, 0, 2187, 211, 0, 0, 62, 0, 75, 1010, 32, 550, 74, 0, 0, 359, 6622, 91, 85, 1931, 15, 4044] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 7439, 2204, 0, 2, 53, 554, 4514, 0, 3, 0, 0, 0, 685, 2770, 145, 0, 392, 116, 265, 70, 0, 4, 884, 389, 352, 0, 156, 218, 53, 227, 1, 97, 7640, 90, 0, 2128, 0, 1504, 0, 1624, 0, 0, 0, 2409, 74, 0, 0, 90, 0, 106, 672, 0, 399, 85, 0, 0, 350, 6765, 167, 116, 1555, 0, 1785] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1873, 7, 7, 64, 494, 3640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 512, 2204, 161, 2838, 55, 0, 0, 0, 0, 0, 0, 324, 353, 52, 240, 85, 0, 211, 991, 4137, 0, 0, 0, 0, 0, 0, 0, 431, 313, 27, 166, 237, 82, 128, 2, 4543, 0, 0, 0, 0, 0, 0, 0, 53, 3020, 105, 0, 737, 0, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 2327, 149, 0, 0, 1503, 1555, 0, 0, 0, 0, 0, 0, 53, 0, 77, 822, 71, 534, 73, 0, 2172, 2227, 0, 0, 0, 0, 0, 0, 0, 377, 161, 131, 98, 2030, 21, 4022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 5772, 6393, 5929, 5523, 6581, 6029, 6840] avg=6144.0000 max=6840 max_rank=7 min=5523 min_rank=4 imbalance_ratio=1.113281 max_load_reduction=42312 max_load_reduction_pct=0.860840 moved_to_echo=24973 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 6715, 1873, 7, 7, 64, 494, 5195, 0, 2, 0, 0, 0, 512, 2204, 161, 324, 353, 52, 240, 85, 0, 211, 991, 431, 313, 27, 166, 237, 82, 128, 2, 53, 7361, 105, 0, 737, 0, 1608, 0, 1047, 0, 0, 0, 2327, 149, 0, 0, 53, 0, 77, 822, 71, 534, 73, 0, 0, 377, 6580, 131, 98, 2030, 21, 4022] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[63, 3364, 4198, 607, 164, 278, 346, 485, 213, 1230, 357, 260, 151, 780, 2264, 257, 433, 218, 1528, 15, 291, 412, 253, 985, 225, 865, 550, 452, 260, 70, 915, 24, 799, 2336, 474, 216, 319, 1426, 1857, 31, 299, 146, 822, 411, 411, 388, 64, 292, 145, 422, 70, 674, 808, 953, 876, 711, 1063, 579, 2223, 129, 461, 2627, 2534, 2073] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 6986, 2390, 1, 8, 50, 580, 4786, 0, 7, 3, 2, 0, 513, 2553, 127, 348, 357, 64, 255, 64, 7, 144, 984, 272, 331, 16, 182, 256, 65, 140, 0, 58, 7334, 86, 10, 759, 7, 1593, 0, 901, 2, 10, 5, 1799, 194, 3, 11, 58, 11, 84, 855, 35, 532, 72, 10, 1, 385, 6472, 75, 102, 2519, 23, 3655] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2701, 0, 3, 68, 580, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 2747, 148, 2922, 57, 0, 0, 0, 0, 0, 0, 0, 464, 75, 247, 91, 0, 8, 943, 4293, 0, 0, 0, 0, 0, 0, 0, 312, 327, 0, 119, 181, 75, 164, 1, 5059, 0, 0, 0, 0, 0, 0, 0, 141, 3109, 80, 0, 2317, 0, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1751, 0, 0, 0, 2210, 68, 0, 0, 1548, 1244, 0, 0, 0, 0, 0, 0, 80, 0, 90, 591, 0, 421, 80, 0, 2418, 2311, 0, 0, 0, 0, 0, 0, 0, 420, 168, 239, 109, 1559, 0, 1809, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 6560, 6121, 6238, 6853, 6821, 5991, 4304] avg=6144.0000 max=6853 max_rank=4 min=4304 min_rank=7 imbalance_ratio=1.115397 max_load_reduction=42299 max_load_reduction_pct=0.860575 moved_to_echo=23969 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2217, 0, 2, 55, 580, 3502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 2244, 162, 2853, 54, 0, 0, 0, 0, 0, 0, 252, 333, 160, 247, 67, 0, 137, 964, 4027, 0, 0, 0, 0, 0, 0, 0, 593, 346, 0, 135, 232, 69, 262, 0, 4784, 0, 0, 0, 0, 0, 0, 0, 91, 3035, 109, 0, 809, 0, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 2347, 112, 0, 0, 1511, 1496, 0, 0, 0, 0, 0, 0, 95, 0, 111, 846, 54, 495, 99, 0, 2286, 2167, 0, 0, 0, 0, 0, 0, 0, 361, 157, 81, 113, 1869, 5, 3293, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6356, 5895, 6187, 6421, 5748, 6513, 6153, 5879] avg=6144.0000 max=6513 max_rank=5 min=5748 min_rank=4 imbalance_ratio=1.060059 max_load_reduction=42639 max_load_reduction_pct=0.867493 moved_to_echo=24293 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2161, 0, 7, 64, 567, 3286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 629, 2273, 140, 2923, 56, 0, 0, 0, 0, 0, 0, 44, 413, 116, 239, 88, 8, 53, 889, 4214, 0, 0, 0, 0, 0, 0, 0, 460, 315, 0, 137, 236, 61, 192, 0, 4958, 0, 0, 0, 0, 0, 0, 0, 131, 3110, 81, 6, 1696, 0, 1745, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 0, 11, 0, 2446, 99, 0, 11, 1548, 1403, 0, 0, 0, 0, 0, 0, 77, 9, 82, 648, 10, 510, 76, 9, 2369, 2268, 0, 0, 0, 0, 0, 0, 2, 374, 165, 165, 94, 1630, 0, 2445, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 6040, 6064, 6359, 6770, 6901, 6058, 4875] avg=6144.0000 max=6901 max_rank=5 min=4875 min_rank=7 imbalance_ratio=1.123210 max_load_reduction=42251 max_load_reduction_pct=0.859599 moved_to_echo=24604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[63, 0, 4198, 607, 164, 278, 346, 340, 0, 0, 0, 0, 0, 0, 0, 0, 213, 1230, 357, 260, 151, 780, 2264, 257, 900, 18, 0, 0, 0, 0, 0, 0, 433, 218, 1528, 15, 291, 412, 253, 985, 1397, 0, 0, 0, 0, 0, 0, 0, 225, 865, 550, 452, 260, 70, 915, 24, 2277, 0, 0, 0, 0, 0, 0, 0, 799, 959, 474, 216, 319, 1426, 1857, 31, 0, 0, 0, 0, 0, 0, 0, 0, 299, 146, 822, 411, 411, 388, 64, 292, 477, 145, 0, 0, 0, 0, 0, 0, 145, 422, 70, 674, 808, 953, 876, 711, 1087, 752, 0, 0, 0, 0, 0, 0, 1063, 579, 56, 129, 461, 2627, 2534, 2073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6430, 5532, 5638, 6081, 3455, 6498, 9522] avg=6144.0000 max=9522 max_rank=7 min=3455 min_rank=5 imbalance_ratio=1.549805 max_load_reduction=39630 max_load_reduction_pct=0.806274 moved_to_echo=25556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2428, 20, 9, 72, 584, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 2229, 164, 2835, 55, 0, 0, 0, 0, 0, 0, 361, 417, 86, 252, 84, 0, 249, 962, 4163, 0, 0, 0, 0, 0, 0, 0, 491, 325, 2, 175, 250, 91, 157, 4, 4652, 0, 0, 0, 0, 0, 0, 0, 56, 3016, 85, 0, 771, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 0, 0, 0, 2187, 211, 0, 0, 1502, 1237, 0, 0, 0, 0, 0, 0, 62, 0, 75, 1010, 32, 550, 74, 0, 2223, 2241, 0, 0, 0, 0, 0, 0, 0, 359, 163, 91, 85, 1931, 15, 4044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6011, 5756, 6574, 6147, 5709, 6000, 6267, 6688] avg=6144.0000 max=6688 max_rank=7 min=5709 min_rank=4 imbalance_ratio=1.088542 max_load_reduction=42464 max_load_reduction_pct=0.863932 moved_to_echo=24664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2390, 1, 8, 50, 580, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 2, 0, 513, 2553, 127, 2831, 58, 0, 0, 0, 0, 0, 0, 348, 357, 64, 255, 64, 7, 144, 984, 4074, 0, 0, 0, 0, 0, 0, 0, 272, 331, 16, 182, 256, 65, 140, 0, 4728, 0, 0, 0, 0, 0, 0, 0, 58, 3001, 86, 10, 759, 7, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 901, 2, 10, 5, 1799, 194, 3, 11, 1502, 1436, 0, 0, 0, 0, 0, 0, 58, 11, 84, 855, 35, 532, 72, 10, 2258, 2195, 0, 0, 0, 0, 0, 0, 1, 385, 145, 75, 102, 2519, 23, 3655, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6379, 6094, 6297, 5990, 5514, 5863, 6110, 6905] avg=6144.0000 max=6905 max_rank=7 min=5514 min_rank=4 imbalance_ratio=1.123861 max_load_reduction=42247 max_load_reduction_pct=0.859517 moved_to_echo=24392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1873, 7, 7, 64, 494, 3640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 512, 2204, 161, 2838, 55, 0, 0, 0, 0, 0, 0, 324, 353, 52, 240, 85, 0, 211, 991, 4137, 0, 0, 0, 0, 0, 0, 0, 431, 313, 27, 166, 237, 82, 128, 2, 4543, 0, 0, 0, 0, 0, 0, 0, 53, 3020, 105, 0, 737, 0, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 2327, 149, 0, 0, 1503, 1555, 0, 0, 0, 0, 0, 0, 53, 0, 77, 822, 71, 534, 73, 0, 2172, 2227, 0, 0, 0, 0, 0, 0, 0, 377, 161, 131, 98, 2030, 21, 4022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 5772, 6393, 5929, 5523, 6581, 6029, 6840] avg=6144.0000 max=6840 max_rank=7 min=5523 min_rank=4 imbalance_ratio=1.113281 max_load_reduction=42312 max_load_reduction_pct=0.860840 moved_to_echo=24973 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[63, 0, 4198, 607, 164, 278, 346, 340, 0, 0, 0, 0, 0, 0, 0, 0, 213, 1230, 357, 260, 151, 780, 2264, 257, 900, 18, 0, 0, 0, 0, 0, 0, 433, 218, 1528, 15, 291, 412, 253, 985, 1397, 0, 0, 0, 0, 0, 0, 0, 225, 865, 550, 452, 260, 70, 915, 24, 2277, 0, 0, 0, 0, 0, 0, 0, 799, 959, 474, 216, 319, 1426, 1857, 31, 0, 0, 0, 0, 0, 0, 0, 0, 299, 146, 822, 411, 411, 388, 64, 292, 477, 145, 0, 0, 0, 0, 0, 0, 145, 422, 70, 674, 808, 953, 876, 711, 1087, 752, 0, 0, 0, 0, 0, 0, 1063, 579, 56, 129, 461, 2627, 2534, 2073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6430, 5532, 5638, 6081, 3455, 6498, 9522] avg=6144.0000 max=9522 max_rank=7 min=3455 min_rank=5 imbalance_ratio=1.549805 max_load_reduction=39630 max_load_reduction_pct=0.806274 moved_to_echo=25556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2161, 0, 7, 64, 567, 3286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 629, 2273, 140, 2923, 56, 0, 0, 0, 0, 0, 0, 44, 413, 116, 239, 88, 8, 53, 889, 4214, 0, 0, 0, 0, 0, 0, 0, 460, 315, 0, 137, 236, 61, 192, 0, 4958, 0, 0, 0, 0, 0, 0, 0, 131, 3110, 81, 6, 1696, 0, 1745, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 0, 11, 0, 2446, 99, 0, 11, 1548, 1403, 0, 0, 0, 0, 0, 0, 77, 9, 82, 648, 10, 510, 76, 9, 2369, 2268, 0, 0, 0, 0, 0, 0, 2, 374, 165, 165, 94, 1630, 0, 2445, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 6040, 6064, 6359, 6770, 6901, 6058, 4875] avg=6144.0000 max=6901 max_rank=5 min=4875 min_rank=7 imbalance_ratio=1.123210 max_load_reduction=42251 max_load_reduction_pct=0.859599 moved_to_echo=24604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2204, 0, 2, 53, 554, 3163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 685, 2770, 145, 2945, 57, 0, 0, 0, 0, 0, 0, 0, 392, 116, 265, 70, 0, 4, 884, 4253, 0, 0, 0, 0, 0, 0, 0, 389, 352, 0, 156, 218, 53, 227, 1, 5034, 0, 0, 0, 0, 0, 0, 0, 97, 3135, 90, 0, 2128, 0, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1624, 0, 0, 0, 2409, 74, 0, 0, 1560, 1351, 0, 0, 0, 0, 0, 0, 90, 0, 106, 672, 0, 399, 85, 0, 2405, 2289, 0, 0, 0, 0, 0, 0, 0, 350, 166, 167, 116, 1555, 0, 1785, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5976, 6605, 5984, 6430, 6954, 7018, 6046, 4139] avg=6144.0000 max=7018 max_rank=5 min=4139 min_rank=7 imbalance_ratio=1.142253 max_load_reduction=42134 max_load_reduction_pct=0.857218 moved_to_echo=24157 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2390, 1, 8, 50, 580, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 2, 0, 513, 2553, 127, 2831, 58, 0, 0, 0, 0, 0, 0, 348, 357, 64, 255, 64, 7, 144, 984, 4074, 0, 0, 0, 0, 0, 0, 0, 272, 331, 16, 182, 256, 65, 140, 0, 4728, 0, 0, 0, 0, 0, 0, 0, 58, 3001, 86, 10, 759, 7, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 901, 2, 10, 5, 1799, 194, 3, 11, 1502, 1436, 0, 0, 0, 0, 0, 0, 58, 11, 84, 855, 35, 532, 72, 10, 2258, 2195, 0, 0, 0, 0, 0, 0, 1, 385, 145, 75, 102, 2519, 23, 3655, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6379, 6094, 6297, 5990, 5514, 5863, 6110, 6905] avg=6144.0000 max=6905 max_rank=7 min=5514 min_rank=4 imbalance_ratio=1.123861 max_load_reduction=42247 max_load_reduction_pct=0.859517 moved_to_echo=24392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 512, 768, 6144, 1024, 4480, 1536, 768, 17280, 16512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 3328, 1280, 1152, 1280, 15744, 2688, 23168, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1536, 22400, 1152, 256, 9600, 1536, 13056, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 20224, 640, 256, 768, 4352, 23168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 3328, 1280, 1152, 1280, 15744, 2688, 23168, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[8960, 256, 896, 512, 16256, 1408, 128, 384, 11264, 9984, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 20224, 640, 256, 768, 4352, 23168, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 1280, 384, 384, 256, 4864, 19328, 1408, 21120, 512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 3072, 2304, 1792, 896, 512, 1152, 7680, 30592, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 3200, 640, 1536, 1920, 640, 2304, 128, 36096, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 3072, 2304, 1792, 896, 512, 1152, 7680, 30592, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 414, 0, 0, 0, 0, 389, 153, 986, 0, 1122, 313, 0, 73, 72, 0, 17, 0, 0, 85, 0, 0, 24, 4325, 0, 298, 170, 976, 0, 0, 1429, 823, 0, 0, 0, 2231, 0, 862, 1299, 1502, 5996, 0, 565, 545, 521, 0, 0, 160, 0, 4504, 0, 0, 71, 0, 694, 0, 5429, 6077, 5125, 1754, 0, 148] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 401, 0, 0, 0, 0, 405, 130, 1374, 0, 1140, 325, 0, 83, 71, 0, 13, 0, 0, 84, 0, 0, 24, 4477, 0, 283, 237, 975, 0, 1, 951, 1358, 0, 0, 0, 2753, 0, 954, 1569, 1979, 5602, 0, 550, 524, 717, 0, 0, 160, 0, 4246, 0, 0, 68, 0, 741, 0, 4623, 5949, 4285, 1951, 0, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 257, 0, 0, 0, 0, 412, 403, 2124, 0, 867, 322, 0, 80, 59, 0, 1, 0, 0, 97, 0, 0, 3, 5125, 0, 309, 206, 1683, 0, 0, 422, 1931, 0, 0, 0, 2451, 0, 871, 2878, 595, 4479, 0, 629, 1180, 1519, 0, 0, 81, 0, 2095, 0, 0, 59, 0, 692, 0, 6384, 6318, 3796, 664, 0, 160] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 387, 0, 0, 0, 0, 369, 247, 1610, 0, 922, 346, 0, 99, 54, 0, 3, 0, 0, 67, 0, 0, 5, 4883, 0, 282, 178, 1322, 0, 0, 1224, 1182, 0, 0, 0, 2266, 0, 860, 1643, 1318, 5004, 0, 553, 652, 754, 0, 0, 264, 0, 3631, 0, 0, 54, 0, 651, 0, 5634, 6174, 4819, 1496, 0, 199] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[817, 497, 10, 974, 1038, 207, 79, 734, 581, 176, 395, 690, 2038, 879, 485, 196, 979, 644, 76, 15, 855, 233, 132, 1118, 289, 2824, 165, 647, 862, 1811, 594, 936, 222, 473, 11, 525, 30, 1515, 149, 1193, 2044, 3725, 2531, 318, 825, 450, 815, 254, 9, 208, 649, 1272, 332, 1006, 657, 238, 889, 555, 2543, 1274, 1058, 1193, 28, 185] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 401, 0, 0, 0, 0, 405, 130, 1374, 0, 1140, 325, 0, 83, 71, 0, 13, 0, 0, 84, 0, 0, 24, 4477, 0, 283, 237, 975, 0, 1, 951, 1358, 0, 0, 0, 2753, 0, 954, 1569, 1979, 5602, 0, 550, 524, 717, 0, 0, 160, 0, 4246, 0, 0, 68, 0, 741, 0, 4623, 5949, 4285, 1951, 0, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1, 2, 3, 362, 2, 0, 2, 10, 367, 113, 1122, 12, 1057, 330, 4, 75, 77, 3, 5, 0, 7, 64, 0, 6, 33, 4212, 2, 238, 193, 1063, 2, 10, 949, 971, 0, 5, 1, 2810, 2, 886, 1427, 1512, 5626, 2, 558, 389, 785, 1, 0, 171, 12, 4003, 0, 9, 76, 3, 675, 10, 5689, 6185, 4779, 2089, 1, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 242, 0, 0, 0, 0, 392, 397, 1914, 0, 857, 354, 0, 89, 72, 0, 2, 0, 0, 74, 0, 0, 14, 4974, 0, 281, 165, 1670, 0, 0, 559, 1511, 0, 0, 0, 2552, 0, 899, 2743, 623, 4569, 0, 555, 1206, 1194, 1, 0, 139, 0, 2204, 0, 0, 72, 0, 560, 0, 6505, 6414, 4354, 817, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 361, 1, 0, 1, 9, 388, 422, 1570, 8, 909, 317, 3, 76, 63, 0, 4, 0, 8, 90, 0, 3, 20, 4952, 2, 313, 188, 1238, 9, 7, 805, 1254, 0, 1, 0, 2260, 5, 866, 2281, 1044, 5432, 2, 556, 1030, 958, 1, 0, 134, 13, 2807, 1, 2, 63, 0, 531, 10, 6002, 6292, 4748, 940, 0, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 257, 0, 0, 0, 0, 6211, 96, 0, 0, 0, 0, 0, 0, 412, 403, 2124, 0, 867, 322, 0, 80, 2095, 41, 0, 0, 0, 0, 0, 0, 59, 0, 1, 0, 0, 97, 0, 0, 6222, 0, 0, 0, 0, 0, 0, 0, 3, 4503, 0, 309, 206, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 1931, 0, 0, 0, 2451, 0, 871, 581, 238, 0, 0, 0, 0, 0, 0, 2878, 595, 270, 0, 629, 1180, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 2095, 0, 0, 59, 0, 2114, 173, 0, 0, 0, 0, 0, 0, 692, 0, 0, 0, 3558, 664, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6564, 6344, 6379, 6704, 6494, 7071, 4522, 5074] avg=6144.0000 max=7071 max_rank=5 min=4522 min_rank=6 imbalance_ratio=1.150879 max_load_reduction=42081 max_load_reduction_pct=0.856140 moved_to_echo=23161 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 414, 0, 0, 0, 0, 5282, 93, 0, 0, 0, 0, 0, 0, 389, 153, 986, 0, 1122, 313, 0, 73, 2804, 34, 0, 0, 0, 0, 0, 0, 72, 0, 17, 0, 0, 85, 0, 0, 5984, 0, 0, 0, 0, 0, 0, 0, 24, 3801, 0, 298, 170, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 823, 0, 0, 0, 2231, 0, 862, 490, 321, 0, 0, 0, 0, 0, 0, 1299, 1502, 362, 0, 565, 545, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 4504, 0, 0, 71, 0, 2830, 147, 0, 0, 0, 0, 0, 0, 694, 0, 0, 0, 4804, 1754, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5789, 5874, 6158, 5269, 6156, 4794, 7712, 7400] avg=6144.0000 max=7712 max_rank=6 min=4794 min_rank=5 imbalance_ratio=1.255208 max_load_reduction=41440 max_load_reduction_pct=0.843099 moved_to_echo=26062 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[817, 497, 10, 974, 1038, 207, 79, 734, 2475, 20, 0, 0, 0, 0, 0, 0, 581, 176, 395, 690, 2038, 879, 485, 196, 1183, 22, 0, 0, 0, 0, 0, 0, 979, 644, 76, 15, 855, 233, 132, 1118, 1254, 0, 0, 0, 0, 0, 0, 0, 289, 2482, 165, 647, 862, 1811, 594, 936, 0, 0, 0, 0, 0, 0, 0, 0, 222, 473, 11, 525, 30, 1515, 149, 1193, 320, 66, 0, 0, 0, 0, 0, 0, 2044, 3725, 154, 318, 825, 450, 815, 254, 0, 0, 0, 0, 0, 0, 0, 0, 9, 208, 649, 1272, 332, 1006, 657, 238, 1194, 68, 0, 0, 0, 0, 0, 0, 889, 555, 0, 0, 992, 1193, 28, 185, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6851, 6645, 5306, 7786, 4504, 8585, 5633, 3842] avg=6144.0000 max=8585 max_rank=5 min=3842 min_rank=7 imbalance_ratio=1.397298 max_load_reduction=40567 max_load_reduction_pct=0.825338 moved_to_echo=22564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 387, 0, 0, 0, 0, 5481, 94, 0, 0, 0, 0, 0, 0, 369, 247, 1610, 0, 922, 346, 0, 99, 2340, 39, 0, 0, 0, 0, 0, 0, 54, 0, 3, 0, 0, 67, 0, 0, 6080, 0, 0, 0, 0, 0, 0, 0, 5, 4290, 0, 282, 178, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 1182, 0, 0, 0, 2266, 0, 860, 554, 302, 0, 0, 0, 0, 0, 0, 1643, 1318, 302, 0, 553, 652, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 3631, 0, 0, 54, 0, 2362, 153, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 4517, 1496, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5962, 5972, 6204, 6077, 6388, 5222, 6464, 6863] avg=6144.0000 max=6863 max_rank=7 min=5222 min_rank=5 imbalance_ratio=1.117025 max_load_reduction=42289 max_load_reduction_pct=0.860372 moved_to_echo=24937 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[817, 497, 10, 974, 1038, 207, 79, 734, 581, 176, 395, 690, 2038, 879, 485, 196, 979, 644, 76, 15, 855, 233, 132, 1118, 289, 2824, 165, 647, 862, 1811, 594, 936, 222, 473, 11, 525, 30, 1515, 149, 1193, 2044, 3725, 2531, 318, 825, 450, 815, 254, 9, 208, 649, 1272, 332, 1006, 657, 238, 889, 555, 2543, 1274, 1058, 1193, 28, 185] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 257, 0, 0, 0, 0, 412, 403, 2124, 0, 867, 322, 0, 80, 59, 0, 1, 0, 0, 97, 0, 0, 3, 5125, 0, 309, 206, 1683, 0, 0, 422, 1931, 0, 0, 0, 2451, 0, 871, 2878, 595, 4479, 0, 629, 1180, 1519, 0, 0, 81, 0, 2095, 0, 0, 59, 0, 692, 0, 6384, 6318, 3796, 664, 0, 160] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 414, 0, 0, 0, 0, 389, 153, 986, 0, 1122, 313, 0, 73, 72, 0, 17, 0, 0, 85, 0, 0, 24, 4325, 0, 298, 170, 976, 0, 0, 1429, 823, 0, 0, 0, 2231, 0, 862, 1299, 1502, 5996, 0, 565, 545, 521, 0, 0, 160, 0, 4504, 0, 0, 71, 0, 694, 0, 5429, 6077, 5125, 1754, 0, 148] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 242, 0, 0, 0, 0, 392, 397, 1914, 0, 857, 354, 0, 89, 72, 0, 2, 0, 0, 74, 0, 0, 14, 4974, 0, 281, 165, 1670, 0, 0, 559, 1511, 0, 0, 0, 2552, 0, 899, 2743, 623, 4569, 0, 555, 1206, 1194, 1, 0, 139, 0, 2204, 0, 0, 72, 0, 560, 0, 6505, 6414, 4354, 817, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 401, 0, 0, 0, 0, 4498, 91, 0, 0, 0, 0, 0, 0, 405, 130, 1374, 0, 1140, 325, 0, 83, 2620, 36, 0, 0, 0, 0, 0, 0, 71, 0, 13, 0, 0, 84, 0, 0, 5858, 0, 0, 0, 0, 0, 0, 0, 24, 3933, 0, 283, 237, 975, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1358, 0, 0, 0, 2753, 0, 954, 508, 269, 0, 0, 0, 0, 0, 0, 1569, 1979, 338, 0, 550, 524, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 4246, 0, 0, 68, 0, 2644, 125, 0, 0, 0, 0, 0, 0, 741, 0, 0, 0, 4016, 1951, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4990, 6113, 6026, 5453, 6793, 5677, 7243, 6857] avg=6144.0000 max=7243 max_rank=6 min=4990 min_rank=0 imbalance_ratio=1.178874 max_load_reduction=41909 max_load_reduction_pct=0.852641 moved_to_echo=26570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 387, 0, 0, 0, 0, 5481, 94, 0, 0, 0, 0, 0, 0, 369, 247, 1610, 0, 922, 346, 0, 99, 2340, 39, 0, 0, 0, 0, 0, 0, 54, 0, 3, 0, 0, 67, 0, 0, 6080, 0, 0, 0, 0, 0, 0, 0, 5, 4290, 0, 282, 178, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 1182, 0, 0, 0, 2266, 0, 860, 554, 302, 0, 0, 0, 0, 0, 0, 1643, 1318, 302, 0, 553, 652, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 3631, 0, 0, 54, 0, 2362, 153, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 4517, 1496, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5962, 5972, 6204, 6077, 6388, 5222, 6464, 6863] avg=6144.0000 max=6863 max_rank=7 min=5222 min_rank=5 imbalance_ratio=1.117025 max_load_reduction=42289 max_load_reduction_pct=0.860372 moved_to_echo=24937 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[817, 497, 10, 974, 1038, 207, 79, 734, 2475, 20, 0, 0, 0, 0, 0, 0, 581, 176, 395, 690, 2038, 879, 485, 196, 1183, 22, 0, 0, 0, 0, 0, 0, 979, 644, 76, 15, 855, 233, 132, 1118, 1254, 0, 0, 0, 0, 0, 0, 0, 289, 2482, 165, 647, 862, 1811, 594, 936, 0, 0, 0, 0, 0, 0, 0, 0, 222, 473, 11, 525, 30, 1515, 149, 1193, 320, 66, 0, 0, 0, 0, 0, 0, 2044, 3725, 154, 318, 825, 450, 815, 254, 0, 0, 0, 0, 0, 0, 0, 0, 9, 208, 649, 1272, 332, 1006, 657, 238, 1194, 68, 0, 0, 0, 0, 0, 0, 889, 555, 0, 0, 992, 1193, 28, 185, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6851, 6645, 5306, 7786, 4504, 8585, 5633, 3842] avg=6144.0000 max=8585 max_rank=5 min=3842 min_rank=7 imbalance_ratio=1.397298 max_load_reduction=40567 max_load_reduction_pct=0.825338 moved_to_echo=22564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 257, 0, 0, 0, 0, 6211, 96, 0, 0, 0, 0, 0, 0, 412, 403, 2124, 0, 867, 322, 0, 80, 2095, 41, 0, 0, 0, 0, 0, 0, 59, 0, 1, 0, 0, 97, 0, 0, 6222, 0, 0, 0, 0, 0, 0, 0, 3, 4503, 0, 309, 206, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 1931, 0, 0, 0, 2451, 0, 871, 581, 238, 0, 0, 0, 0, 0, 0, 2878, 595, 270, 0, 629, 1180, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 2095, 0, 0, 59, 0, 2114, 173, 0, 0, 0, 0, 0, 0, 692, 0, 0, 0, 3558, 664, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6564, 6344, 6379, 6704, 6494, 7071, 4522, 5074] avg=6144.0000 max=7071 max_rank=5 min=4522 min_rank=6 imbalance_ratio=1.150879 max_load_reduction=42081 max_load_reduction_pct=0.856140 moved_to_echo=23161 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 361, 1, 0, 1, 9, 5841, 96, 0, 0, 0, 0, 0, 0, 388, 422, 1570, 8, 909, 317, 3, 76, 2540, 39, 0, 0, 0, 0, 0, 0, 63, 0, 4, 0, 8, 90, 0, 3, 6196, 0, 0, 0, 0, 0, 0, 0, 20, 4351, 2, 313, 188, 1238, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 805, 1254, 0, 1, 0, 2260, 5, 866, 562, 298, 0, 0, 0, 0, 0, 0, 2281, 1044, 328, 2, 556, 1030, 958, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 13, 2807, 1, 2, 63, 0, 2564, 161, 0, 0, 0, 0, 0, 0, 531, 10, 0, 0, 4450, 940, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6310, 6272, 6364, 6128, 6051, 6200, 5745, 6082] avg=6144.0000 max=6364 max_rank=2 min=5745 min_rank=6 imbalance_ratio=1.035807 max_load_reduction=42788 max_load_reduction_pct=0.870524 moved_to_echo=24078 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 3, 362, 2, 0, 2, 10, 5536, 95, 0, 0, 0, 0, 0, 0, 367, 113, 1122, 12, 1057, 330, 4, 75, 2635, 37, 0, 0, 0, 0, 0, 0, 77, 3, 5, 0, 7, 64, 0, 6, 6090, 0, 0, 0, 0, 0, 0, 0, 33, 3694, 2, 238, 193, 1063, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 949, 971, 0, 5, 1, 2810, 2, 886, 481, 303, 0, 0, 0, 0, 0, 0, 1427, 1512, 331, 2, 558, 389, 785, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 12, 4003, 0, 9, 76, 3, 2660, 153, 0, 0, 0, 0, 0, 0, 675, 10, 0, 0, 4476, 2089, 1, 149, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6013, 5752, 6252, 5235, 6408, 5005, 7087, 7400] avg=6144.0000 max=7400 max_rank=7 min=5005 min_rank=5 imbalance_ratio=1.204427 max_load_reduction=41752 max_load_reduction_pct=0.849447 moved_to_echo=25900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 242, 0, 0, 0, 0, 6330, 99, 0, 0, 0, 0, 0, 0, 392, 397, 1914, 0, 857, 354, 0, 89, 2137, 40, 0, 0, 0, 0, 0, 0, 72, 0, 2, 0, 0, 74, 0, 0, 6315, 0, 0, 0, 0, 0, 0, 0, 14, 4370, 0, 281, 165, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 1511, 0, 0, 0, 2552, 0, 899, 564, 273, 0, 0, 0, 0, 0, 0, 2743, 623, 276, 0, 555, 1206, 1194, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 2204, 0, 0, 72, 0, 2156, 175, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 4081, 817, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6673, 6180, 6463, 6500, 6358, 6598, 4746, 5634] avg=6144.0000 max=6673 max_rank=0 min=4746 min_rank=6 imbalance_ratio=1.086100 max_load_reduction=42479 max_load_reduction_pct=0.864237 moved_to_echo=23336 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 414, 0, 0, 0, 0, 5282, 93, 0, 0, 0, 0, 0, 0, 389, 153, 986, 0, 1122, 313, 0, 73, 2804, 34, 0, 0, 0, 0, 0, 0, 72, 0, 17, 0, 0, 85, 0, 0, 5984, 0, 0, 0, 0, 0, 0, 0, 24, 3801, 0, 298, 170, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 823, 0, 0, 0, 2231, 0, 862, 490, 321, 0, 0, 0, 0, 0, 0, 1299, 1502, 362, 0, 565, 545, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 4504, 0, 0, 71, 0, 2830, 147, 0, 0, 0, 0, 0, 0, 694, 0, 0, 0, 4804, 1754, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5789, 5874, 6158, 5269, 6156, 4794, 7712, 7400] avg=6144.0000 max=7712 max_rank=6 min=4794 min_rank=5 imbalance_ratio=1.255208 max_load_reduction=41440 max_load_reduction_pct=0.843099 moved_to_echo=26062 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 3, 362, 2, 0, 2, 10, 5536, 95, 0, 0, 0, 0, 0, 0, 367, 113, 1122, 12, 1057, 330, 4, 75, 2635, 37, 0, 0, 0, 0, 0, 0, 77, 3, 5, 0, 7, 64, 0, 6, 6090, 0, 0, 0, 0, 0, 0, 0, 33, 3694, 2, 238, 193, 1063, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 949, 971, 0, 5, 1, 2810, 2, 886, 481, 303, 0, 0, 0, 0, 0, 0, 1427, 1512, 331, 2, 558, 389, 785, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 12, 4003, 0, 9, 76, 3, 2660, 153, 0, 0, 0, 0, 0, 0, 675, 10, 0, 0, 4476, 2089, 1, 149, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6013, 5752, 6252, 5235, 6408, 5005, 7087, 7400] avg=6144.0000 max=7400 max_rank=7 min=5005 min_rank=5 imbalance_ratio=1.204427 max_load_reduction=41752 max_load_reduction_pct=0.849447 moved_to_echo=25900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 2048, 11136, 768, 8960, 3200, 512, 896, 18432, 384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[16000, 12416, 2432, 384, 4864, 6016, 7296, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5504, 640, 0, 0, 30976, 11008, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1408, 768, 24832, 384, 1024, 1152, 256, 18560, 1280, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[6656, 9600, 128, 640, 128, 18944, 256, 7424, 4096, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5504, 640, 0, 0, 30976, 11008, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 31488, 256, 2688, 2304, 10752, 640, 1024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1408, 768, 24832, 384, 1024, 1152, 256, 18560, 1280, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 512, 128, 3456, 1152, 256, 128, 768, 41728, 768, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 768, 128, 128, 896, 896, 256, 1152, 44032, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[505, 412, 872, 481, 815, 1700, 1067, 879, 88, 266, 52, 357, 727, 1556, 194, 431, 81, 0, 99, 27, 86, 636, 368, 1826, 272, 637, 792, 573, 708, 237, 69, 456, 3696, 805, 160, 1183, 68, 953, 189, 1244, 333, 1035, 2165, 1035, 155, 183, 1417, 489, 418, 682, 1414, 233, 856, 55, 2667, 76, 469, 2625, 2294, 685, 873, 2338, 363, 725] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 64, 311, 300, 264, 6915, 0, 418, 0, 0, 23, 0, 74, 0, 0, 0, 1555, 0, 0, 0, 0, 2733, 0, 1889, 0, 52, 15, 609, 0, 0, 0, 64, 5972, 44, 74, 4047, 0, 0, 158, 1522, 299, 0, 0, 1365, 85, 0, 0, 207, 391, 50, 3830, 73, 4734, 64, 3941, 0, 27, 1453, 2884, 601, 1437, 534, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 55, 301, 314, 132, 6661, 0, 461, 0, 0, 14, 0, 100, 0, 0, 0, 1257, 0, 0, 0, 0, 2559, 0, 2111, 0, 140, 14, 594, 0, 0, 0, 55, 5790, 0, 101, 4735, 0, 0, 166, 1081, 306, 0, 0, 1427, 67, 0, 0, 3, 350, 0, 3523, 99, 5140, 54, 4263, 0, 33, 1163, 3431, 702, 1300, 530, 0, 120] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 67, 306, 344, 24, 6289, 0, 460, 0, 0, 15, 0, 80, 0, 0, 0, 1996, 0, 0, 0, 0, 3131, 0, 2695, 0, 0, 66, 1800, 0, 0, 0, 68, 5854, 0, 80, 4315, 0, 0, 177, 699, 245, 0, 0, 2382, 96, 0, 0, 0, 425, 0, 3723, 80, 5756, 37, 2802, 0, 0, 548, 2354, 652, 996, 429, 0, 161] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 55, 301, 314, 132, 6661, 0, 461, 0, 0, 14, 0, 100, 0, 0, 0, 1257, 0, 0, 0, 0, 2559, 0, 2111, 0, 140, 14, 594, 0, 0, 0, 55, 5790, 0, 101, 4735, 0, 0, 166, 1081, 306, 0, 0, 1427, 67, 0, 0, 3, 350, 0, 3523, 99, 5140, 54, 4263, 0, 33, 1163, 3431, 702, 1300, 530, 0, 120] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 53, 336, 339, 39, 6471, 0, 432, 0, 0, 19, 0, 90, 0, 0, 0, 1653, 0, 0, 0, 0, 2953, 0, 2320, 0, 0, 35, 1544, 0, 0, 0, 53, 5818, 5, 90, 4780, 0, 0, 163, 695, 223, 0, 0, 2213, 74, 0, 0, 0, 410, 0, 3411, 87, 5839, 41, 2894, 0, 12, 653, 2948, 689, 1090, 454, 0, 226] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 64, 311, 300, 264, 6915, 0, 418, 0, 0, 23, 0, 74, 0, 0, 0, 1555, 0, 0, 0, 0, 2733, 0, 1889, 0, 52, 15, 609, 0, 0, 0, 64, 5972, 44, 74, 4047, 0, 0, 158, 1522, 299, 0, 0, 1365, 85, 0, 0, 207, 391, 50, 3830, 73, 4734, 64, 3941, 0, 27, 1453, 2884, 601, 1437, 534, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[505, 412, 872, 481, 815, 1700, 1067, 879, 88, 266, 52, 357, 727, 1556, 194, 431, 81, 0, 99, 27, 86, 636, 368, 1826, 272, 637, 792, 573, 708, 237, 69, 456, 3696, 805, 160, 1183, 68, 953, 189, 1244, 333, 1035, 2165, 1035, 155, 183, 1417, 489, 418, 682, 1414, 233, 856, 55, 2667, 76, 469, 2625, 2294, 685, 873, 2338, 363, 725] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 47, 334, 320, 194, 7252, 4, 440, 0, 3, 24, 0, 74, 13, 3, 4, 1563, 1, 0, 0, 4, 1928, 4, 2072, 1, 63, 12, 489, 3, 0, 5, 50, 6276, 20, 74, 3575, 0, 5, 136, 1257, 259, 14, 17, 1506, 64, 2, 14, 13, 359, 2, 4193, 73, 4722, 47, 4057, 2, 51, 1818, 2654, 704, 1526, 726, 7, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 70, 319, 335, 110, 6791, 0, 438, 0, 0, 17, 0, 75, 0, 0, 0, 1323, 0, 0, 0, 0, 2593, 0, 2973, 0, 82, 13, 463, 0, 0, 0, 70, 5953, 0, 75, 3494, 0, 0, 158, 1363, 302, 0, 0, 1886, 84, 0, 0, 119, 386, 3, 4473, 74, 4332, 70, 3983, 0, 39, 1186, 2639, 644, 1507, 635, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 53, 336, 339, 39, 6471, 0, 432, 0, 0, 19, 0, 90, 0, 0, 0, 1653, 0, 0, 0, 0, 2953, 0, 2320, 0, 0, 35, 1544, 0, 0, 0, 53, 5818, 5, 90, 4780, 0, 0, 163, 695, 223, 0, 0, 2213, 74, 0, 0, 0, 410, 0, 3411, 87, 5839, 41, 2894, 0, 12, 653, 2948, 689, 1090, 454, 0, 226] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 61, 305, 334, 72, 6277, 4, 394, 0, 2, 40, 4, 78, 8, 0, 0, 1677, 3, 0, 0, 2, 2808, 3, 2073, 0, 27, 48, 1128, 0, 0, 0, 66, 5985, 24, 78, 4971, 0, 10, 164, 829, 239, 10, 15, 1654, 88, 2, 13, 0, 388, 0, 3584, 76, 5827, 61, 3609, 4, 22, 876, 2888, 628, 1074, 447, 6, 166] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 64, 311, 300, 264, 5052, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 74, 0, 0, 0, 5262, 605, 0, 0, 0, 0, 0, 0, 1555, 0, 0, 0, 0, 2733, 0, 1889, 19, 0, 0, 0, 0, 0, 0, 0, 0, 52, 15, 609, 0, 0, 0, 64, 4129, 762, 0, 0, 0, 0, 0, 0, 710, 44, 74, 4047, 0, 0, 158, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 1365, 85, 0, 0, 207, 1729, 1101, 908, 0, 0, 0, 0, 0, 391, 50, 3830, 73, 0, 64, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1453, 1957, 601, 1437, 534, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6409, 5964, 6196, 5631, 6555, 5694, 6620, 6083] avg=6144.0000 max=6620 max_rank=6 min=5631 min_rank=3 imbalance_ratio=1.077474 max_load_reduction=42532 max_load_reduction_pct=0.865316 moved_to_echo=24952 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[505, 412, 872, 481, 815, 1243, 1067, 879, 0, 0, 0, 0, 0, 0, 0, 0, 88, 266, 52, 357, 727, 1556, 194, 431, 3257, 110, 0, 0, 0, 0, 0, 0, 81, 0, 99, 27, 86, 636, 368, 1826, 15, 0, 0, 0, 0, 0, 0, 0, 272, 637, 792, 573, 708, 237, 69, 456, 746, 187, 0, 0, 0, 0, 0, 0, 439, 805, 160, 1183, 68, 953, 189, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 333, 1035, 2165, 1035, 155, 183, 1417, 489, 1170, 270, 722, 0, 0, 0, 0, 0, 418, 682, 1414, 233, 0, 55, 1497, 76, 0, 0, 0, 0, 0, 0, 0, 0, 469, 2625, 1557, 685, 873, 2338, 363, 725, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 7038, 3138, 4677, 5041, 8974, 4375, 9635] avg=6144.0000 max=9635 max_rank=7 min=3138 min_rank=2 imbalance_ratio=1.568197 max_load_reduction=39517 max_load_reduction_pct=0.803975 moved_to_echo=28025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 67, 306, 344, 24, 4595, 0, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 80, 0, 0, 0, 5158, 737, 0, 0, 0, 0, 0, 0, 1996, 0, 0, 0, 0, 3131, 0, 2695, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1800, 0, 0, 0, 68, 5019, 693, 0, 0, 0, 0, 0, 0, 696, 0, 80, 4315, 0, 0, 177, 699, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 2382, 96, 0, 0, 0, 1229, 1001, 741, 0, 0, 0, 0, 0, 425, 0, 3723, 80, 0, 37, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 1598, 652, 996, 429, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5796, 5990, 7837, 7646, 5967, 5694, 5838, 4384] avg=6144.0000 max=7837 max_rank=2 min=4384 min_rank=7 imbalance_ratio=1.275553 max_load_reduction=41315 max_load_reduction_pct=0.840556 moved_to_echo=21883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 55, 301, 314, 132, 4867, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 100, 0, 0, 0, 5102, 657, 0, 0, 0, 0, 0, 0, 1257, 0, 0, 0, 0, 2559, 0, 2111, 23, 0, 0, 0, 0, 0, 0, 0, 0, 140, 14, 594, 0, 0, 0, 55, 4483, 734, 0, 0, 0, 0, 0, 0, 688, 0, 101, 4735, 0, 0, 166, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 0, 1427, 67, 0, 0, 3, 1871, 1060, 1081, 0, 0, 0, 0, 0, 350, 0, 3523, 99, 0, 54, 2392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1163, 2327, 702, 1300, 530, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 5873, 5950, 6020, 6771, 5815, 6418, 6175] avg=6144.0000 max=6771 max_rank=4 min=5815 min_rank=5 imbalance_ratio=1.102051 max_load_reduction=42381 max_load_reduction_pct=0.862244 moved_to_echo=25179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 61, 305, 334, 72, 4586, 4, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 40, 4, 78, 8, 0, 0, 5274, 746, 0, 0, 0, 0, 0, 0, 1677, 3, 0, 0, 2, 2808, 3, 2073, 19, 0, 0, 0, 0, 0, 0, 0, 0, 27, 48, 1128, 0, 0, 0, 66, 5081, 692, 0, 0, 0, 0, 0, 0, 711, 24, 78, 4971, 0, 10, 164, 829, 0, 0, 0, 0, 0, 0, 0, 0, 239, 10, 15, 1654, 88, 2, 13, 0, 1584, 999, 909, 0, 0, 0, 0, 0, 388, 0, 3584, 76, 0, 61, 2025, 4, 0, 0, 0, 0, 0, 0, 0, 0, 22, 876, 1960, 628, 1074, 447, 6, 166, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5756, 6152, 6585, 7042, 6787, 5513, 6138, 5179] avg=6144.0000 max=7042 max_rank=3 min=5179 min_rank=7 imbalance_ratio=1.146159 max_load_reduction=42110 max_load_reduction_pct=0.856730 moved_to_echo=23617 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 47, 334, 320, 194, 5288, 4, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 24, 0, 74, 13, 3, 4, 5533, 605, 0, 0, 0, 0, 0, 0, 1563, 1, 0, 0, 4, 1928, 4, 2072, 22, 0, 0, 0, 0, 0, 0, 0, 1, 63, 12, 489, 3, 0, 5, 50, 4117, 805, 0, 0, 0, 0, 0, 0, 743, 20, 74, 3575, 0, 5, 136, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 259, 14, 17, 1506, 64, 2, 14, 13, 1784, 1159, 841, 0, 0, 0, 0, 0, 359, 2, 4193, 73, 0, 47, 2273, 2, 0, 0, 0, 0, 0, 0, 0, 0, 51, 1818, 1791, 704, 1526, 726, 7, 72, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6627, 6259, 5594, 5545, 5810, 5673, 6949, 6695] avg=6144.0000 max=6949 max_rank=6 min=5545 min_rank=3 imbalance_ratio=1.131022 max_load_reduction=42203 max_load_reduction_pct=0.858622 moved_to_echo=25127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 70, 319, 335, 110, 4961, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 75, 0, 0, 0, 5246, 554, 0, 0, 0, 0, 0, 0, 1323, 0, 0, 0, 0, 2593, 0, 2973, 17, 0, 0, 0, 0, 0, 0, 0, 0, 82, 13, 463, 0, 0, 0, 70, 3778, 749, 0, 0, 0, 0, 0, 0, 707, 0, 75, 3494, 0, 0, 158, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 1886, 84, 0, 0, 119, 1748, 1081, 831, 0, 0, 0, 0, 0, 386, 3, 4473, 74, 0, 70, 2235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1186, 1791, 644, 1507, 635, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6233, 5892, 6906, 5155, 5797, 6051, 7241, 5877] avg=6144.0000 max=7241 max_rank=6 min=5155 min_rank=3 imbalance_ratio=1.178548 max_load_reduction=41911 max_load_reduction_pct=0.852681 moved_to_echo=24966 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 64, 311, 300, 264, 5052, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 74, 0, 0, 0, 5262, 605, 0, 0, 0, 0, 0, 0, 1555, 0, 0, 0, 0, 2733, 0, 1889, 19, 0, 0, 0, 0, 0, 0, 0, 0, 52, 15, 609, 0, 0, 0, 64, 4129, 762, 0, 0, 0, 0, 0, 0, 710, 44, 74, 4047, 0, 0, 158, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 1365, 85, 0, 0, 207, 1729, 1101, 908, 0, 0, 0, 0, 0, 391, 50, 3830, 73, 0, 64, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1453, 1957, 601, 1437, 534, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6409, 5964, 6196, 5631, 6555, 5694, 6620, 6083] avg=6144.0000 max=6620 max_rank=6 min=5631 min_rank=3 imbalance_ratio=1.077474 max_load_reduction=42532 max_load_reduction_pct=0.865316 moved_to_echo=24952 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 53, 336, 339, 39, 4728, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 90, 0, 0, 0, 5127, 748, 0, 0, 0, 0, 0, 0, 1653, 0, 0, 0, 0, 2953, 0, 2320, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1544, 0, 0, 0, 53, 5091, 713, 0, 0, 0, 0, 0, 0, 691, 5, 90, 4780, 0, 0, 163, 695, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 2213, 74, 0, 0, 0, 1270, 1030, 928, 0, 0, 0, 0, 0, 410, 0, 3411, 87, 0, 41, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 653, 2000, 689, 1090, 454, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5927, 5984, 6946, 7436, 6424, 5738, 5573, 5124] avg=6144.0000 max=7436 max_rank=3 min=5124 min_rank=7 imbalance_ratio=1.210286 max_load_reduction=41716 max_load_reduction_pct=0.848714 moved_to_echo=22859 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[505, 412, 872, 481, 815, 1243, 1067, 879, 0, 0, 0, 0, 0, 0, 0, 0, 88, 266, 52, 357, 727, 1556, 194, 431, 3257, 110, 0, 0, 0, 0, 0, 0, 81, 0, 99, 27, 86, 636, 368, 1826, 15, 0, 0, 0, 0, 0, 0, 0, 272, 637, 792, 573, 708, 237, 69, 456, 746, 187, 0, 0, 0, 0, 0, 0, 439, 805, 160, 1183, 68, 953, 189, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 333, 1035, 2165, 1035, 155, 183, 1417, 489, 1170, 270, 722, 0, 0, 0, 0, 0, 418, 682, 1414, 233, 0, 55, 1497, 76, 0, 0, 0, 0, 0, 0, 0, 0, 469, 2625, 1557, 685, 873, 2338, 363, 725, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 7038, 3138, 4677, 5041, 8974, 4375, 9635] avg=6144.0000 max=9635 max_rank=7 min=3138 min_rank=2 imbalance_ratio=1.568197 max_load_reduction=39517 max_load_reduction_pct=0.803975 moved_to_echo=28025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 61, 305, 334, 72, 4586, 4, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 40, 4, 78, 8, 0, 0, 5274, 746, 0, 0, 0, 0, 0, 0, 1677, 3, 0, 0, 2, 2808, 3, 2073, 19, 0, 0, 0, 0, 0, 0, 0, 0, 27, 48, 1128, 0, 0, 0, 66, 5081, 692, 0, 0, 0, 0, 0, 0, 711, 24, 78, 4971, 0, 10, 164, 829, 0, 0, 0, 0, 0, 0, 0, 0, 239, 10, 15, 1654, 88, 2, 13, 0, 1584, 999, 909, 0, 0, 0, 0, 0, 388, 0, 3584, 76, 0, 61, 2025, 4, 0, 0, 0, 0, 0, 0, 0, 0, 22, 876, 1960, 628, 1074, 447, 6, 166, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5756, 6152, 6585, 7042, 6787, 5513, 6138, 5179] avg=6144.0000 max=7042 max_rank=3 min=5179 min_rank=7 imbalance_ratio=1.146159 max_load_reduction=42110 max_load_reduction_pct=0.856730 moved_to_echo=23617 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 47, 334, 320, 194, 5288, 4, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 24, 0, 74, 13, 3, 4, 5533, 605, 0, 0, 0, 0, 0, 0, 1563, 1, 0, 0, 4, 1928, 4, 2072, 22, 0, 0, 0, 0, 0, 0, 0, 1, 63, 12, 489, 3, 0, 5, 50, 4117, 805, 0, 0, 0, 0, 0, 0, 743, 20, 74, 3575, 0, 5, 136, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 259, 14, 17, 1506, 64, 2, 14, 13, 1784, 1159, 841, 0, 0, 0, 0, 0, 359, 2, 4193, 73, 0, 47, 2273, 2, 0, 0, 0, 0, 0, 0, 0, 0, 51, 1818, 1791, 704, 1526, 726, 7, 72, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6627, 6259, 5594, 5545, 5810, 5673, 6949, 6695] avg=6144.0000 max=6949 max_rank=6 min=5545 min_rank=3 imbalance_ratio=1.131022 max_load_reduction=42203 max_load_reduction_pct=0.858622 moved_to_echo=25127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 10368, 15104, 5376, 9856, 6144, 384, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2304, 1152, 2304, 13568, 768, 256, 1536, 896, 12416, 7808, 7040, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 10368, 15104, 5376, 9856, 6144, 384, 1664, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3200, 768, 28160, 896, 0, 512, 15872, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11136, 128, 128, 128, 128, 19456, 384, 18048, 256, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5504, 1024, 768, 31104, 128, 1024, 1408, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 896, 3200, 2816, 1664, 35328, 1152, 3968, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 384, 256, 384, 1408, 1664, 256, 512, 40064, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 896, 3200, 2816, 1664, 35328, 1152, 3968, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[384, 1024, 1024, 7296, 768, 256, 128, 896, 32512, 5376, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[158, 187, 536, 154, 91, 253, 146, 10, 428, 1379, 51, 196, 267, 2005, 471, 702, 35, 2953, 630, 480, 205, 22, 1644, 238, 2572, 953, 254, 306, 228, 2426, 233, 3476, 3887, 783, 1507, 280, 312, 484, 7, 747, 686, 483, 1819, 1506, 108, 1694, 1036, 2279, 288, 541, 115, 1200, 278, 148, 135, 220, 541, 1295, 20, 190, 535, 1610, 98, 631] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 70, 0, 75, 0, 0, 0, 0, 1606, 0, 167, 75, 525, 366, 1609, 0, 7319, 0, 437, 0, 0, 68, 0, 7343, 0, 124, 251, 0, 676, 74, 6691, 7530, 2669, 0, 9, 0, 0, 0, 0, 75, 68, 3228, 0, 0, 2157, 251, 0, 424, 0, 0, 2344, 0, 0, 0, 0, 0, 2853, 0, 0, 0, 0, 0, 68] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[12, 0, 55, 0, 91, 0, 0, 0, 0, 2125, 0, 266, 100, 311, 378, 1744, 0, 7211, 0, 442, 0, 0, 54, 0, 7393, 0, 69, 247, 0, 736, 99, 6841, 7517, 2593, 0, 0, 0, 0, 0, 0, 100, 53, 1960, 0, 0, 2086, 247, 0, 417, 0, 0, 2750, 0, 0, 0, 0, 0, 3201, 0, 0, 0, 0, 0, 54] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[17, 0, 64, 0, 92, 0, 0, 0, 0, 1765, 0, 163, 74, 419, 392, 1562, 0, 7323, 0, 433, 0, 0, 71, 0, 7372, 0, 4, 240, 0, 651, 73, 6359, 7555, 3464, 0, 0, 0, 0, 0, 0, 74, 66, 2783, 0, 0, 2039, 240, 0, 381, 0, 0, 1891, 0, 0, 0, 0, 0, 3514, 0, 0, 0, 0, 0, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[124, 0, 61, 0, 83, 0, 0, 0, 2, 2525, 0, 202, 76, 99, 381, 1564, 1, 7300, 6, 449, 0, 0, 63, 1, 7410, 12, 142, 239, 5, 837, 76, 6896, 7391, 2516, 2, 9, 12, 10, 0, 0, 77, 57, 1344, 7, 0, 2150, 245, 10, 365, 1, 0, 3232, 0, 0, 1, 1, 4, 3075, 9, 0, 3, 12, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[8, 0, 47, 1, 70, 1, 0, 0, 5, 1979, 0, 147, 75, 423, 393, 1476, 0, 7303, 9, 455, 0, 0, 83, 6, 7471, 15, 94, 256, 8, 749, 72, 6552, 7520, 2510, 12, 3, 5, 0, 0, 1, 78, 79, 2978, 9, 0, 1906, 261, 14, 413, 0, 0, 2166, 1, 0, 0, 1, 8, 3393, 0, 0, 1, 15, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[201, 0, 68, 0, 72, 0, 0, 0, 0, 2526, 0, 130, 80, 93, 443, 1545, 0, 7295, 0, 99, 0, 0, 59, 0, 7414, 0, 107, 247, 0, 1274, 80, 7292, 7044, 2381, 0, 0, 0, 0, 0, 0, 80, 37, 1243, 0, 0, 2005, 247, 0, 365, 0, 0, 3756, 0, 0, 0, 0, 0, 2910, 0, 0, 0, 0, 0, 59] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[124, 0, 61, 0, 83, 0, 0, 0, 2, 2525, 0, 202, 76, 99, 381, 1564, 1, 7300, 6, 449, 0, 0, 63, 1, 7410, 12, 142, 239, 5, 837, 76, 6896, 7391, 2516, 2, 9, 12, 10, 0, 0, 77, 57, 1344, 7, 0, 2150, 245, 10, 365, 1, 0, 3232, 0, 0, 1, 1, 4, 3075, 9, 0, 3, 12, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[17, 0, 64, 0, 92, 0, 0, 0, 0, 1765, 0, 163, 74, 419, 392, 1562, 0, 7323, 0, 433, 0, 0, 71, 0, 7372, 0, 4, 240, 0, 651, 73, 6359, 7555, 3464, 0, 0, 0, 0, 0, 0, 74, 66, 2783, 0, 0, 2039, 240, 0, 381, 0, 0, 1891, 0, 0, 0, 0, 0, 3514, 0, 0, 0, 0, 0, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[114, 0, 52, 0, 60, 0, 0, 0, 0, 3042, 0, 208, 88, 54, 387, 1548, 0, 7268, 0, 130, 0, 0, 72, 0, 7394, 0, 54, 265, 0, 866, 89, 7158, 7366, 2587, 0, 1, 0, 0, 0, 0, 90, 62, 1032, 0, 0, 2006, 265, 0, 340, 0, 0, 3377, 0, 0, 0, 0, 0, 3105, 0, 0, 0, 0, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[12, 0, 55, 0, 91, 0, 0, 0, 0, 2125, 0, 266, 100, 311, 378, 1744, 0, 7211, 0, 442, 0, 0, 54, 0, 7393, 0, 69, 247, 0, 736, 99, 6841, 7517, 2593, 0, 0, 0, 0, 0, 0, 100, 53, 1960, 0, 0, 2086, 247, 0, 417, 0, 0, 2750, 0, 0, 0, 0, 0, 3201, 0, 0, 0, 0, 0, 54] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[8, 0, 47, 1, 70, 1, 0, 0, 5, 1979, 0, 147, 75, 423, 393, 1476, 0, 7303, 9, 455, 0, 0, 83, 6, 7471, 15, 94, 256, 8, 749, 72, 6552, 7520, 2510, 12, 3, 5, 0, 0, 1, 78, 79, 2978, 9, 0, 1906, 261, 14, 413, 0, 0, 2166, 1, 0, 0, 1, 8, 3393, 0, 0, 1, 15, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[158, 187, 536, 154, 91, 253, 146, 10, 428, 1379, 51, 196, 267, 2005, 471, 702, 35, 2953, 630, 480, 205, 22, 1644, 238, 2572, 953, 254, 306, 228, 2426, 233, 3476, 3887, 783, 1507, 280, 312, 484, 7, 747, 686, 483, 1819, 1506, 108, 1694, 1036, 2279, 288, 541, 115, 1200, 278, 148, 135, 220, 541, 1295, 20, 190, 535, 1610, 98, 631] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 70, 0, 75, 0, 0, 0, 0, 1606, 0, 167, 75, 525, 366, 1609, 0, 7319, 0, 437, 0, 0, 68, 0, 7343, 0, 124, 251, 0, 676, 74, 6691, 7530, 2669, 0, 9, 0, 0, 0, 0, 75, 68, 3228, 0, 0, 2157, 251, 0, 424, 0, 0, 2344, 0, 0, 0, 0, 0, 2853, 0, 0, 0, 0, 0, 68] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[201, 0, 68, 0, 72, 0, 0, 0, 6297, 0, 0, 0, 0, 0, 0, 0, 0, 2526, 0, 130, 80, 93, 443, 1545, 1043, 429, 0, 0, 0, 0, 0, 0, 0, 5808, 0, 99, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 247, 0, 1274, 80, 5125, 0, 0, 0, 0, 0, 0, 0, 0, 3314, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 37, 1243, 0, 0, 2005, 247, 0, 1058, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 3756, 0, 0, 0, 0, 1995, 1117, 0, 0, 0, 0, 0, 0, 0, 2910, 0, 0, 0, 0, 0, 59, 1735, 1124, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6638, 6289, 5966, 6833, 5695, 4670, 7233, 5828] avg=6144.0000 max=7233 max_rank=6 min=4670 min_rank=5 imbalance_ratio=1.177246 max_load_reduction=41919 max_load_reduction_pct=0.852844 moved_to_echo=23426 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 64, 0, 92, 0, 0, 0, 6261, 0, 0, 0, 0, 0, 0, 0, 0, 1765, 0, 163, 74, 419, 392, 1562, 909, 431, 0, 0, 0, 0, 0, 0, 0, 5830, 0, 433, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 240, 0, 651, 73, 4469, 0, 0, 0, 0, 0, 0, 0, 0, 3554, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 66, 2783, 0, 0, 2039, 240, 0, 1062, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 1891, 0, 0, 0, 0, 2140, 1111, 0, 0, 0, 0, 0, 0, 0, 3514, 0, 0, 0, 0, 0, 71, 1861, 981, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6434, 5715, 6334, 5437, 7018, 6264, 5523, 6427] avg=6144.0000 max=7018 max_rank=4 min=5437 min_rank=3 imbalance_ratio=1.142253 max_load_reduction=42134 max_load_reduction_pct=0.857218 moved_to_echo=25232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[158, 187, 536, 154, 91, 253, 146, 10, 2185, 0, 0, 0, 0, 0, 0, 0, 428, 1379, 51, 196, 267, 2005, 471, 702, 497, 173, 0, 0, 0, 0, 0, 0, 35, 2352, 630, 480, 205, 22, 1644, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 254, 306, 228, 2426, 233, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 783, 1507, 280, 312, 484, 7, 747, 0, 0, 0, 0, 0, 0, 0, 0, 686, 483, 1819, 1506, 108, 1694, 1036, 2279, 428, 0, 0, 0, 0, 0, 0, 0, 288, 541, 115, 1200, 278, 148, 135, 220, 1101, 387, 0, 0, 0, 0, 0, 0, 541, 1295, 20, 190, 535, 1610, 98, 631, 957, 536, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3720, 6169, 5606, 6843, 5949, 10039, 4413, 6413] avg=6144.0000 max=10039 max_rank=5 min=3720 min_rank=0 imbalance_ratio=1.633952 max_load_reduction=39113 max_load_reduction_pct=0.795756 moved_to_echo=26814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 55, 0, 91, 0, 0, 0, 6279, 0, 0, 0, 0, 0, 0, 0, 0, 2125, 0, 266, 100, 311, 378, 1744, 978, 424, 0, 0, 0, 0, 0, 0, 0, 5741, 0, 442, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 247, 0, 736, 99, 4808, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 53, 1960, 0, 0, 2086, 247, 0, 1046, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 2750, 0, 0, 0, 0, 2129, 1114, 0, 0, 0, 0, 0, 0, 0, 3201, 0, 0, 0, 0, 0, 54, 1852, 1055, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6437, 6326, 6237, 5959, 6129, 5492, 6410, 6162] avg=6144.0000 max=6437 max_rank=0 min=5492 min_rank=5 imbalance_ratio=1.047689 max_load_reduction=42715 max_load_reduction_pct=0.869039 moved_to_echo=24193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[124, 0, 61, 0, 83, 0, 0, 0, 6294, 0, 0, 0, 0, 0, 0, 0, 2, 2525, 0, 202, 76, 99, 381, 1564, 986, 430, 0, 0, 0, 0, 0, 0, 1, 5811, 6, 449, 0, 0, 63, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 142, 239, 5, 837, 76, 4847, 0, 0, 0, 0, 0, 0, 0, 0, 3477, 2516, 2, 9, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 57, 1344, 7, 0, 2150, 245, 10, 1059, 0, 0, 0, 0, 0, 0, 0, 365, 1, 0, 3232, 0, 0, 1, 1, 2093, 1116, 0, 0, 0, 0, 0, 0, 4, 3075, 9, 0, 3, 12, 0, 65, 1821, 1063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6562, 6265, 6331, 6158, 6026, 4949, 6809, 6052] avg=6144.0000 max=6809 max_rank=6 min=4949 min_rank=5 imbalance_ratio=1.108236 max_load_reduction=42343 max_load_reduction_pct=0.861471 moved_to_echo=23836 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[201, 0, 68, 0, 72, 0, 0, 0, 6297, 0, 0, 0, 0, 0, 0, 0, 0, 2526, 0, 130, 80, 93, 443, 1545, 1043, 429, 0, 0, 0, 0, 0, 0, 0, 5808, 0, 99, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 247, 0, 1274, 80, 5125, 0, 0, 0, 0, 0, 0, 0, 0, 3314, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 37, 1243, 0, 0, 2005, 247, 0, 1058, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 3756, 0, 0, 0, 0, 1995, 1117, 0, 0, 0, 0, 0, 0, 0, 2910, 0, 0, 0, 0, 0, 59, 1735, 1124, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6638, 6289, 5966, 6833, 5695, 4670, 7233, 5828] avg=6144.0000 max=7233 max_rank=6 min=4670 min_rank=5 imbalance_ratio=1.177246 max_load_reduction=41919 max_load_reduction_pct=0.852844 moved_to_echo=23426 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[114, 0, 52, 0, 60, 0, 0, 0, 6281, 0, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 208, 88, 54, 387, 1548, 1024, 428, 0, 0, 0, 0, 0, 0, 0, 5786, 0, 130, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 265, 0, 866, 89, 5030, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 2587, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 62, 1032, 0, 0, 2006, 265, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 3377, 0, 0, 0, 0, 2086, 1113, 0, 0, 0, 0, 0, 0, 0, 3105, 0, 0, 0, 0, 0, 72, 1814, 1104, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6507, 6779, 5988, 6304, 6054, 4509, 6916, 6095] avg=6144.0000 max=6916 max_rank=6 min=4509 min_rank=5 imbalance_ratio=1.125651 max_load_reduction=42236 max_load_reduction_pct=0.859294 moved_to_echo=23574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 70, 0, 75, 0, 0, 0, 6236, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 0, 167, 75, 525, 366, 1609, 957, 431, 0, 0, 0, 0, 0, 0, 0, 5826, 0, 437, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 251, 0, 676, 74, 4702, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 2669, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 68, 3228, 0, 0, 2157, 251, 0, 1062, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 2344, 0, 0, 0, 0, 2133, 1107, 0, 0, 0, 0, 0, 0, 0, 2853, 0, 0, 0, 0, 0, 68, 1855, 1032, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6381, 5736, 6331, 5827, 6220, 6841, 6008, 5808] avg=6144.0000 max=6841 max_rank=5 min=5736 min_rank=1 imbalance_ratio=1.113444 max_load_reduction=42311 max_load_reduction_pct=0.860819 moved_to_echo=24877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 47, 1, 70, 1, 0, 0, 6346, 0, 0, 0, 0, 0, 0, 0, 5, 1979, 0, 147, 75, 423, 393, 1476, 941, 434, 0, 0, 0, 0, 0, 0, 0, 5806, 9, 455, 0, 0, 83, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 94, 256, 8, 749, 72, 4597, 0, 0, 0, 0, 0, 0, 0, 0, 3530, 2510, 12, 3, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 78, 79, 2978, 9, 0, 1906, 261, 14, 1063, 0, 0, 0, 0, 0, 0, 0, 413, 0, 0, 2166, 1, 0, 0, 1, 2134, 1125, 0, 0, 0, 0, 0, 0, 8, 3393, 0, 0, 1, 15, 0, 80, 1856, 1014, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6473, 5873, 6359, 5791, 6061, 6388, 5840, 6367] avg=6144.0000 max=6473 max_rank=0 min=5791 min_rank=3 imbalance_ratio=1.053548 max_load_reduction=42679 max_load_reduction_pct=0.868306 moved_to_echo=24656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[114, 0, 52, 0, 60, 0, 0, 0, 6281, 0, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 208, 88, 54, 387, 1548, 1024, 428, 0, 0, 0, 0, 0, 0, 0, 5786, 0, 130, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 265, 0, 866, 89, 5030, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 2587, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 62, 1032, 0, 0, 2006, 265, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 3377, 0, 0, 0, 0, 2086, 1113, 0, 0, 0, 0, 0, 0, 0, 3105, 0, 0, 0, 0, 0, 72, 1814, 1104, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6507, 6779, 5988, 6304, 6054, 4509, 6916, 6095] avg=6144.0000 max=6916 max_rank=6 min=4509 min_rank=5 imbalance_ratio=1.125651 max_load_reduction=42236 max_load_reduction_pct=0.859294 moved_to_echo=23574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 17024, 128, 1536, 896, 3968, 3328, 11776, 7424, 3200, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 23424, 128, 256, 640, 1664, 128, 1152, 13824, 7936, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 1024, 16512, 1536, 128, 16128, 2816, 2304, 7936, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 640, 128, 20736, 384, 256, 256, 256, 15872, 8192, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[26368, 19584, 1536, 384, 384, 512, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 640, 128, 20736, 384, 256, 256, 256, 15872, 8192, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 1024, 16512, 1536, 128, 16128, 2816, 2304, 7936, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 1024, 896, 2176, 256, 8320, 896, 36096, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 43008, 768, 2944, 256, 128, 2176, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 256, 1024, 256, 640, 256, 256, 128, 46208, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 118, 1054, 0, 1, 1402, 0, 0, 3806, 2419, 1, 7, 16, 615, 0, 88, 327, 8, 823, 15, 60, 2, 0, 4825, 0, 0, 85, 2, 0, 0, 5, 65, 316, 6, 4, 3976, 150, 1, 76, 15, 9, 5341, 0, 1, 1, 1, 13, 671, 15, 76, 0, 7297, 13, 90, 77, 7138, 0, 0, 0, 6823, 675, 623, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 82, 1922, 0, 0, 929, 0, 0, 5276, 2091, 0, 0, 0, 614, 0, 84, 335, 0, 1109, 0, 67, 0, 0, 5850, 35, 0, 74, 0, 0, 0, 0, 68, 326, 0, 0, 2835, 213, 0, 74, 0, 0, 4250, 0, 0, 0, 0, 0, 685, 0, 74, 0, 6926, 0, 48, 75, 7021, 0, 0, 0, 6686, 718, 685, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 81, 909, 0, 0, 1805, 0, 0, 2880, 3581, 0, 0, 0, 669, 0, 97, 344, 0, 655, 0, 59, 0, 0, 4066, 0, 0, 80, 0, 0, 0, 0, 59, 327, 0, 0, 3695, 174, 0, 80, 0, 0, 5474, 0, 0, 0, 0, 0, 671, 0, 80, 0, 7469, 0, 96, 79, 7249, 0, 0, 0, 6981, 770, 722, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 52, 1783, 0, 0, 1029, 0, 0, 4930, 1723, 0, 0, 0, 628, 0, 85, 325, 0, 1072, 0, 68, 0, 0, 5461, 78, 0, 73, 0, 0, 0, 0, 71, 314, 0, 0, 3841, 180, 0, 73, 0, 0, 4463, 0, 0, 0, 0, 0, 682, 0, 73, 0, 6760, 0, 7, 77, 7094, 0, 0, 0, 6872, 694, 644, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[106, 217, 828, 600, 660, 527, 1008, 2, 0, 2044, 2184, 76, 1710, 2326, 396, 23, 82, 451, 1052, 1412, 365, 72, 3, 26, 1610, 767, 200, 270, 307, 129, 251, 610, 251, 234, 2461, 1170, 552, 464, 410, 115, 834, 675, 2610, 70, 517, 268, 768, 2492, 576, 664, 138, 80, 4245, 837, 390, 340, 1662, 103, 857, 397, 2516, 743, 1118, 281] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 160, 1397, 0, 0, 933, 0, 0, 4958, 1997, 0, 0, 0, 625, 0, 67, 314, 0, 1024, 0, 54, 0, 0, 5137, 59, 0, 99, 0, 0, 0, 0, 54, 347, 0, 0, 3742, 164, 0, 99, 0, 0, 4916, 0, 0, 0, 1, 0, 636, 0, 98, 0, 6948, 0, 8, 99, 7140, 0, 0, 0, 6715, 699, 662, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1, 1, 63, 1484, 3, 3, 868, 0, 0, 5029, 1975, 0, 15, 19, 640, 1, 64, 328, 5, 936, 15, 74, 0, 1, 5726, 41, 0, 75, 0, 1, 2, 4, 77, 342, 12, 5, 3149, 252, 12, 72, 3, 9, 4883, 0, 4, 3, 10, 18, 656, 15, 73, 0, 6849, 4, 28, 74, 7065, 0, 2, 0, 6816, 707, 638, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 118, 1054, 0, 1, 1402, 0, 0, 3806, 2419, 1, 7, 16, 615, 0, 88, 327, 8, 823, 15, 60, 2, 0, 4825, 0, 0, 85, 2, 0, 0, 5, 65, 316, 6, 4, 3976, 150, 1, 76, 15, 9, 5341, 0, 1, 1, 1, 13, 671, 15, 76, 0, 7297, 13, 90, 77, 7138, 0, 0, 0, 6823, 675, 623, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 52, 1783, 0, 0, 1029, 0, 0, 4930, 1723, 0, 0, 0, 628, 0, 85, 325, 0, 1072, 0, 68, 0, 0, 5461, 78, 0, 73, 0, 0, 0, 0, 71, 314, 0, 0, 3841, 180, 0, 73, 0, 0, 4463, 0, 0, 0, 0, 0, 682, 0, 73, 0, 6760, 0, 7, 77, 7094, 0, 0, 0, 6872, 694, 644, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 136, 943, 0, 0, 1566, 0, 0, 3334, 2953, 0, 0, 0, 649, 0, 75, 339, 0, 693, 0, 72, 0, 0, 4314, 0, 0, 89, 0, 0, 0, 0, 72, 355, 0, 0, 3777, 186, 0, 89, 4, 0, 5728, 0, 0, 0, 0, 0, 624, 0, 87, 0, 7278, 5, 74, 85, 7281, 0, 0, 0, 7020, 685, 639, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 82, 1922, 0, 0, 929, 0, 0, 5276, 2091, 0, 0, 0, 614, 0, 84, 335, 0, 1109, 0, 67, 0, 0, 5850, 35, 0, 74, 0, 0, 0, 0, 68, 326, 0, 0, 2835, 213, 0, 74, 0, 0, 4250, 0, 0, 0, 0, 0, 685, 0, 74, 0, 6926, 0, 48, 75, 7021, 0, 0, 0, 6686, 718, 685, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1, 1, 63, 1484, 3, 3, 868, 0, 0, 5029, 1975, 0, 15, 19, 640, 1, 64, 328, 5, 936, 15, 74, 0, 1, 5726, 41, 0, 75, 0, 1, 2, 4, 77, 342, 12, 5, 3149, 252, 12, 72, 3, 9, 4883, 0, 4, 3, 10, 18, 656, 15, 73, 0, 6849, 4, 28, 74, 7065, 0, 2, 0, 6816, 707, 638, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 81, 909, 0, 0, 1805, 0, 2195, 1499, 0, 0, 0, 0, 0, 0, 0, 1893, 3581, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 344, 0, 655, 0, 59, 0, 0, 5054, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 80, 0, 0, 0, 0, 472, 491, 0, 0, 0, 0, 0, 0, 59, 327, 0, 0, 3695, 174, 0, 80, 1425, 485, 0, 0, 0, 0, 0, 0, 0, 0, 5474, 0, 0, 0, 0, 0, 515, 0, 0, 0, 0, 0, 0, 0, 671, 0, 80, 0, 5553, 0, 96, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4997, 770, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6489, 6143, 6209, 5109, 6245, 5989, 6479, 6489] avg=6144.0000 max=6489 max_rank=0 min=5109 min_rank=3 imbalance_ratio=1.056152 max_load_reduction=42663 max_load_reduction_pct=0.867981 moved_to_echo=25202 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 82, 1922, 0, 0, 929, 0, 2126, 1436, 0, 0, 0, 0, 0, 0, 0, 3469, 2091, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 335, 0, 1109, 0, 67, 0, 0, 4895, 0, 0, 0, 0, 0, 0, 0, 5850, 35, 0, 74, 0, 0, 0, 0, 864, 455, 0, 0, 0, 0, 0, 0, 68, 326, 0, 0, 2835, 213, 0, 74, 1321, 464, 0, 0, 0, 0, 0, 0, 0, 0, 4250, 0, 0, 0, 0, 0, 943, 0, 0, 0, 0, 0, 0, 0, 685, 0, 74, 0, 5150, 0, 48, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4786, 718, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6495, 6174, 6490, 7278, 5301, 5193, 6032, 6189] avg=6144.0000 max=7278 max_rank=3 min=5193 min_rank=5 imbalance_ratio=1.184570 max_load_reduction=41874 max_load_reduction_pct=0.851929 moved_to_echo=22715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 160, 1397, 0, 0, 933, 0, 2162, 1442, 0, 0, 0, 0, 0, 0, 0, 3259, 1997, 0, 0, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 314, 0, 1024, 0, 54, 0, 0, 4978, 0, 0, 0, 0, 0, 0, 0, 5137, 59, 0, 99, 0, 0, 0, 0, 812, 457, 0, 0, 0, 0, 0, 0, 54, 347, 0, 0, 3742, 164, 0, 99, 1325, 466, 0, 0, 0, 0, 0, 0, 0, 0, 4916, 0, 0, 0, 1, 0, 887, 0, 0, 0, 0, 0, 0, 0, 636, 0, 98, 0, 5166, 0, 8, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4807, 699, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 5881, 6437, 6564, 6197, 5804, 6007, 6168] avg=6144.0000 max=6564 max_rank=3 min=5804 min_rank=5 imbalance_ratio=1.068359 max_load_reduction=42588 max_load_reduction_pct=0.866455 moved_to_echo=24176 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[106, 217, 828, 600, 660, 527, 1008, 2, 504, 540, 0, 0, 0, 0, 0, 0, 0, 1344, 2184, 76, 1710, 2326, 396, 23, 0, 0, 0, 0, 0, 0, 0, 0, 82, 451, 1052, 1412, 365, 72, 3, 26, 1158, 0, 0, 0, 0, 0, 0, 0, 1610, 767, 200, 270, 307, 129, 251, 610, 335, 279, 0, 0, 0, 0, 0, 0, 251, 234, 2461, 1170, 552, 464, 410, 115, 809, 174, 0, 0, 0, 0, 0, 0, 834, 675, 2610, 70, 517, 268, 768, 2492, 365, 0, 0, 0, 0, 0, 0, 0, 576, 664, 138, 80, 3157, 837, 390, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 857, 397, 1802, 743, 1118, 281, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4992, 8059, 4621, 4758, 6640, 8599, 6182, 5301] avg=6144.0000 max=8599 max_rank=5 min=4621 min_rank=2 imbalance_ratio=1.399577 max_load_reduction=40553 max_load_reduction_pct=0.825053 moved_to_echo=26722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 52, 1783, 0, 0, 1029, 0, 2148, 1476, 0, 0, 0, 0, 0, 0, 0, 3240, 1723, 0, 0, 0, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 325, 0, 1072, 0, 68, 0, 0, 4946, 0, 0, 0, 0, 0, 0, 0, 5461, 78, 0, 73, 0, 0, 0, 0, 808, 445, 0, 0, 0, 0, 0, 0, 71, 314, 0, 0, 3841, 180, 0, 73, 1289, 477, 0, 0, 0, 0, 0, 0, 0, 0, 4463, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 0, 682, 0, 73, 0, 5026, 0, 7, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4919, 694, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6488, 5591, 6496, 6865, 6245, 5345, 5865, 6257] avg=6144.0000 max=6865 max_rank=3 min=5345 min_rank=5 imbalance_ratio=1.117350 max_load_reduction=42287 max_load_reduction_pct=0.860331 moved_to_echo=23712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 118, 1054, 0, 1, 1402, 0, 2163, 1465, 0, 0, 0, 0, 0, 0, 0, 2503, 2419, 1, 7, 16, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 327, 8, 823, 15, 60, 2, 0, 4975, 0, 0, 0, 0, 0, 0, 0, 4825, 0, 0, 85, 2, 0, 0, 5, 623, 480, 0, 0, 0, 0, 0, 0, 65, 316, 6, 4, 3976, 150, 1, 76, 1392, 474, 0, 0, 0, 0, 0, 0, 15, 9, 5341, 0, 1, 1, 1, 13, 680, 0, 0, 0, 0, 0, 0, 0, 671, 15, 76, 0, 5425, 13, 90, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4884, 675, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6203, 5561, 6298, 6020, 6460, 6061, 6367, 6182] avg=6144.0000 max=6460 max_rank=4 min=5561 min_rank=1 imbalance_ratio=1.051432 max_load_reduction=42692 max_load_reduction_pct=0.868571 moved_to_echo=25070 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[106, 217, 828, 600, 660, 527, 1008, 2, 504, 540, 0, 0, 0, 0, 0, 0, 0, 1344, 2184, 76, 1710, 2326, 396, 23, 0, 0, 0, 0, 0, 0, 0, 0, 82, 451, 1052, 1412, 365, 72, 3, 26, 1158, 0, 0, 0, 0, 0, 0, 0, 1610, 767, 200, 270, 307, 129, 251, 610, 335, 279, 0, 0, 0, 0, 0, 0, 251, 234, 2461, 1170, 552, 464, 410, 115, 809, 174, 0, 0, 0, 0, 0, 0, 834, 675, 2610, 70, 517, 268, 768, 2492, 365, 0, 0, 0, 0, 0, 0, 0, 576, 664, 138, 80, 3157, 837, 390, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 857, 397, 1802, 743, 1118, 281, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4992, 8059, 4621, 4758, 6640, 8599, 6182, 5301] avg=6144.0000 max=8599 max_rank=5 min=4621 min_rank=2 imbalance_ratio=1.399577 max_load_reduction=40553 max_load_reduction_pct=0.825053 moved_to_echo=26722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 160, 1397, 0, 0, 933, 0, 2162, 1442, 0, 0, 0, 0, 0, 0, 0, 3259, 1997, 0, 0, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 314, 0, 1024, 0, 54, 0, 0, 4978, 0, 0, 0, 0, 0, 0, 0, 5137, 59, 0, 99, 0, 0, 0, 0, 812, 457, 0, 0, 0, 0, 0, 0, 54, 347, 0, 0, 3742, 164, 0, 99, 1325, 466, 0, 0, 0, 0, 0, 0, 0, 0, 4916, 0, 0, 0, 1, 0, 887, 0, 0, 0, 0, 0, 0, 0, 636, 0, 98, 0, 5166, 0, 8, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4807, 699, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 5881, 6437, 6564, 6197, 5804, 6007, 6168] avg=6144.0000 max=6564 max_rank=3 min=5804 min_rank=5 imbalance_ratio=1.068359 max_load_reduction=42588 max_load_reduction_pct=0.866455 moved_to_echo=24176 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 63, 1484, 3, 3, 868, 0, 2141, 1468, 0, 0, 0, 0, 0, 0, 0, 3299, 1975, 0, 15, 19, 640, 1, 0, 0, 0, 0, 0, 0, 0, 0, 64, 328, 5, 936, 15, 74, 0, 1, 4924, 0, 0, 0, 0, 0, 0, 0, 5726, 41, 0, 75, 0, 1, 2, 4, 827, 454, 0, 0, 0, 0, 0, 0, 77, 342, 12, 5, 3149, 252, 12, 72, 1311, 478, 0, 0, 0, 0, 0, 0, 3, 9, 4883, 0, 4, 3, 10, 18, 903, 0, 0, 0, 0, 0, 0, 0, 656, 15, 73, 0, 5084, 4, 28, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4870, 707, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6032, 5949, 6347, 7130, 5710, 5833, 5934, 6217] avg=6144.0000 max=7130 max_rank=3 min=5710 min_rank=4 imbalance_ratio=1.160482 max_load_reduction=42022 max_load_reduction_pct=0.854940 moved_to_echo=23694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 136, 943, 0, 0, 1566, 0, 2206, 1508, 0, 0, 0, 0, 0, 0, 0, 2192, 2953, 0, 0, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 339, 0, 693, 0, 72, 0, 0, 5075, 0, 0, 0, 0, 0, 0, 0, 4314, 0, 0, 89, 0, 0, 0, 0, 546, 479, 0, 0, 0, 0, 0, 0, 72, 355, 0, 0, 3777, 186, 0, 89, 1388, 487, 0, 0, 0, 0, 0, 0, 4, 0, 5728, 0, 0, 0, 0, 0, 596, 0, 0, 0, 0, 0, 0, 0, 624, 0, 87, 0, 5411, 5, 74, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5025, 685, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6359, 5794, 6254, 5428, 6354, 6328, 6286, 6349] avg=6144.0000 max=6359 max_rank=0 min=5428 min_rank=3 imbalance_ratio=1.034993 max_load_reduction=42793 max_load_reduction_pct=0.870626 moved_to_echo=25317 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 82, 1922, 0, 0, 929, 0, 2126, 1436, 0, 0, 0, 0, 0, 0, 0, 3469, 2091, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 335, 0, 1109, 0, 67, 0, 0, 4895, 0, 0, 0, 0, 0, 0, 0, 5850, 35, 0, 74, 0, 0, 0, 0, 864, 455, 0, 0, 0, 0, 0, 0, 68, 326, 0, 0, 2835, 213, 0, 74, 1321, 464, 0, 0, 0, 0, 0, 0, 0, 0, 4250, 0, 0, 0, 0, 0, 943, 0, 0, 0, 0, 0, 0, 0, 685, 0, 74, 0, 5150, 0, 48, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4786, 718, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6495, 6174, 6490, 7278, 5301, 5193, 6032, 6189] avg=6144.0000 max=7278 max_rank=3 min=5193 min_rank=5 imbalance_ratio=1.184570 max_load_reduction=41874 max_load_reduction_pct=0.851929 moved_to_echo=22715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 63, 1484, 3, 3, 868, 0, 2141, 1468, 0, 0, 0, 0, 0, 0, 0, 3299, 1975, 0, 15, 19, 640, 1, 0, 0, 0, 0, 0, 0, 0, 0, 64, 328, 5, 936, 15, 74, 0, 1, 4924, 0, 0, 0, 0, 0, 0, 0, 5726, 41, 0, 75, 0, 1, 2, 4, 827, 454, 0, 0, 0, 0, 0, 0, 77, 342, 12, 5, 3149, 252, 12, 72, 1311, 478, 0, 0, 0, 0, 0, 0, 3, 9, 4883, 0, 4, 3, 10, 18, 903, 0, 0, 0, 0, 0, 0, 0, 656, 15, 73, 0, 5084, 4, 28, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4870, 707, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6032, 5949, 6347, 7130, 5710, 5833, 5934, 6217] avg=6144.0000 max=7130 max_rank=3 min=5710 min_rank=4 imbalance_ratio=1.160482 max_load_reduction=42022 max_load_reduction_pct=0.854940 moved_to_echo=23694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 256, 1536, 10112, 768, 640, 9600, 128, 15744, 10880, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 2688, 2560, 1280, 25600, 1792, 512, 768, 10368, 3584, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 21248, 18944, 128, 1792, 2432, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 768, 37760, 128, 640, 384, 896, 2560, 5888, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[768, 2816, 1152, 7808, 512, 640, 128, 128, 36096, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[36992, 1024, 256, 896, 384, 256, 256, 640, 5376, 3584, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5248, 768, 768, 128, 40064, 896, 768, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 768, 37760, 128, 640, 384, 896, 2560, 5888, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 896, 512, 36096, 5760, 5760, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2579, 0, 18, 76, 689, 4021, 1, 0, 1363, 92, 467, 0, 0, 4, 1, 1, 0, 76, 161, 15, 3, 76, 9, 0, 53, 0, 0, 9, 0, 76, 0, 0, 3, 0, 4395, 0, 0, 5, 3627, 506, 0, 7189, 2775, 7950, 0, 11, 4839, 66, 5, 2, 6, 76, 0, 1, 10, 6187, 321, 14, 0, 308, 587, 179, 240, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3287, 0, 0, 101, 692, 2442, 0, 0, 1641, 84, 694, 0, 0, 0, 0, 0, 0, 75, 150, 0, 0, 74, 0, 0, 16, 0, 0, 0, 60, 75, 0, 0, 0, 0, 4911, 0, 0, 0, 4165, 494, 0, 7006, 1823, 8027, 0, 0, 5026, 74, 0, 0, 0, 75, 0, 0, 23, 6446, 285, 0, 0, 362, 642, 150, 251, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[2646, 0, 0, 112, 724, 2864, 0, 0, 1207, 67, 585, 0, 0, 0, 0, 0, 0, 100, 122, 0, 0, 99, 0, 0, 19, 0, 0, 0, 5, 100, 0, 0, 0, 0, 4987, 0, 0, 0, 4126, 569, 0, 7140, 2529, 7981, 0, 0, 4989, 96, 0, 0, 0, 100, 0, 0, 45, 6240, 268, 0, 0, 356, 609, 215, 247, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2977, 0, 0, 147, 704, 1949, 0, 0, 1488, 85, 771, 0, 0, 0, 0, 0, 0, 74, 132, 0, 0, 73, 0, 0, 10, 0, 0, 0, 23, 74, 0, 0, 0, 0, 5470, 0, 0, 0, 4364, 727, 0, 7146, 1701, 8025, 0, 0, 5142, 71, 0, 0, 0, 74, 0, 0, 118, 6195, 299, 0, 0, 329, 611, 124, 240, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2579, 0, 18, 76, 689, 4021, 1, 0, 1363, 92, 467, 0, 0, 4, 1, 1, 0, 76, 161, 15, 3, 76, 9, 0, 53, 0, 0, 9, 0, 76, 0, 0, 3, 0, 4395, 0, 0, 5, 3627, 506, 0, 7189, 2775, 7950, 0, 11, 4839, 66, 5, 2, 6, 76, 0, 1, 10, 6187, 321, 14, 0, 308, 587, 179, 240, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[2589, 0, 0, 80, 747, 4999, 0, 0, 916, 96, 353, 0, 0, 0, 0, 0, 0, 80, 164, 0, 0, 80, 0, 0, 113, 0, 0, 0, 0, 80, 0, 0, 0, 0, 3965, 0, 0, 0, 2979, 445, 0, 7171, 3623, 8009, 0, 0, 4273, 46, 0, 0, 0, 80, 0, 0, 0, 6355, 440, 0, 0, 379, 686, 140, 247, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[2720, 0, 0, 90, 732, 4530, 0, 0, 913, 74, 367, 0, 0, 0, 0, 0, 0, 90, 128, 0, 0, 88, 0, 0, 123, 0, 0, 0, 0, 90, 0, 0, 0, 0, 3879, 0, 0, 0, 2958, 501, 0, 7185, 3558, 8002, 0, 0, 4793, 65, 0, 0, 0, 90, 0, 0, 0, 6392, 466, 0, 0, 223, 595, 203, 265, 32] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[433, 72, 3175, 247, 668, 3361, 1101, 611, 1360, 270, 209, 0, 358, 85, 275, 238, 14, 43, 594, 2152, 285, 105, 1279, 603, 163, 141, 724, 323, 126, 192, 279, 375, 1749, 243, 1148, 62, 391, 660, 2156, 675, 143, 2868, 1486, 1089, 176, 1241, 2863, 621, 301, 1611, 337, 134, 84, 306, 548, 1275, 1257, 2924, 97, 808, 591, 129, 1023, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3287, 0, 0, 101, 692, 2442, 0, 0, 1641, 84, 694, 0, 0, 0, 0, 0, 0, 75, 150, 0, 0, 74, 0, 0, 16, 0, 0, 0, 60, 75, 0, 0, 0, 0, 4911, 0, 0, 0, 4165, 494, 0, 7006, 1823, 8027, 0, 0, 5026, 74, 0, 0, 0, 75, 0, 0, 23, 6446, 285, 0, 0, 362, 642, 150, 251, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2701, 0, 24, 131, 712, 3234, 6, 0, 1327, 72, 597, 0, 1, 1, 0, 5, 0, 72, 112, 17, 6, 72, 8, 5, 7, 0, 0, 5, 1, 74, 0, 2, 9, 0, 5045, 0, 0, 2, 3723, 496, 2, 7164, 2075, 7822, 1, 15, 5270, 72, 2, 5, 1, 72, 0, 5, 19, 6363, 392, 19, 0, 368, 605, 137, 256, 20] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2579, 0, 18, 76, 689, 2577, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1363, 92, 467, 0, 0, 4, 1, 1, 2998, 1435, 0, 0, 0, 0, 0, 0, 0, 76, 161, 15, 3, 76, 9, 0, 4191, 1550, 0, 0, 0, 0, 0, 0, 53, 0, 0, 9, 0, 76, 0, 0, 6400, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1759, 0, 0, 5, 3627, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 11, 3498, 66, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 6, 76, 0, 1, 10, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 321, 14, 0, 308, 587, 179, 240, 60, 1341, 1444, 1201, 58, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5940, 6361, 6081, 6538, 5900, 6350, 6229, 5753] avg=6144.0000 max=6538 max_rank=3 min=5753 min_rank=7 imbalance_ratio=1.064128 max_load_reduction=42614 max_load_reduction_pct=0.866984 moved_to_echo=24232 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2977, 0, 0, 147, 704, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1488, 85, 771, 0, 0, 0, 0, 0, 2979, 1786, 0, 0, 0, 0, 0, 0, 0, 74, 132, 0, 0, 73, 0, 0, 4167, 1564, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 23, 74, 0, 0, 6461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 0, 0, 4364, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1701, 0, 0, 0, 3717, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 118, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 329, 611, 124, 240, 9, 1425, 699, 1495, 58, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5078, 7109, 6010, 6568, 7280, 5489, 6329, 5289] avg=6144.0000 max=7280 max_rank=4 min=5078 min_rank=0 imbalance_ratio=1.184896 max_load_reduction=41872 max_load_reduction_pct=0.851888 moved_to_echo=24387 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[433, 72, 3175, 247, 668, 3361, 1101, 611, 1360, 270, 209, 0, 358, 85, 275, 238, 14, 43, 594, 2152, 285, 105, 1279, 603, 163, 141, 724, 323, 126, 192, 279, 375, 1749, 243, 1148, 62, 391, 660, 2156, 675, 143, 2868, 1486, 1089, 176, 1241, 2863, 621, 301, 1611, 337, 134, 84, 306, 548, 1275, 1257, 2924, 97, 808, 591, 129, 1023, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[2720, 0, 0, 90, 732, 4530, 0, 0, 913, 74, 367, 0, 0, 0, 0, 0, 0, 90, 128, 0, 0, 88, 0, 0, 123, 0, 0, 0, 0, 90, 0, 0, 0, 0, 3879, 0, 0, 0, 2958, 501, 0, 7185, 3558, 8002, 0, 0, 4793, 65, 0, 0, 0, 90, 0, 0, 0, 6392, 466, 0, 0, 223, 595, 203, 265, 32] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2646, 0, 0, 112, 724, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1207, 67, 585, 0, 0, 0, 0, 0, 2977, 1629, 0, 0, 0, 0, 0, 0, 0, 100, 122, 0, 0, 99, 0, 0, 4163, 1555, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 5, 100, 0, 0, 6426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1995, 0, 0, 0, 4126, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2529, 0, 0, 0, 3606, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 45, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 356, 609, 215, 247, 5, 1383, 1028, 1363, 59, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5318, 6465, 6039, 6550, 6690, 6231, 6326, 5533] avg=6144.0000 max=6690 max_rank=4 min=5318 min_rank=0 imbalance_ratio=1.088867 max_load_reduction=42462 max_load_reduction_pct=0.863892 moved_to_echo=24780 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2589, 0, 0, 80, 747, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 96, 353, 0, 0, 0, 0, 0, 2989, 1295, 0, 0, 0, 0, 0, 0, 0, 80, 164, 0, 0, 80, 0, 0, 4182, 1560, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 80, 0, 0, 6449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1586, 0, 0, 0, 2979, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 0, 0, 3089, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 6295, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 379, 686, 140, 247, 17, 1184, 1795, 1084, 60, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6620, 5649, 6066, 6642, 5010, 6758, 6375, 6032] avg=6144.0000 max=6758 max_rank=5 min=5010 min_rank=4 imbalance_ratio=1.099935 max_load_reduction=42394 max_load_reduction_pct=0.862508 moved_to_echo=24175 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[433, 72, 3175, 247, 668, 2154, 1101, 611, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 270, 209, 0, 358, 85, 275, 238, 1196, 375, 0, 0, 0, 0, 0, 0, 14, 43, 594, 2152, 285, 105, 1279, 603, 1672, 213, 0, 0, 0, 0, 0, 0, 163, 141, 724, 323, 126, 192, 279, 375, 876, 0, 0, 0, 0, 0, 0, 0, 1749, 243, 460, 62, 391, 660, 2156, 675, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 1486, 0, 176, 1241, 2070, 621, 0, 0, 0, 0, 0, 0, 0, 0, 301, 1611, 337, 134, 84, 306, 548, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 2924, 97, 808, 591, 129, 1023, 295, 793, 1207, 313, 12, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8461, 4366, 6960, 3199, 6396, 5737, 4584, 9449] avg=6144.0000 max=9449 max_rank=7 min=3199 min_rank=3 imbalance_ratio=1.537923 max_load_reduction=39703 max_load_reduction_pct=0.807760 moved_to_echo=26166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3287, 0, 0, 101, 692, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 84, 694, 0, 0, 0, 0, 0, 2921, 1604, 0, 0, 0, 0, 0, 0, 0, 75, 150, 0, 0, 74, 0, 0, 4085, 1564, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 60, 75, 0, 0, 6463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 0, 0, 0, 4165, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1823, 0, 0, 0, 3633, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 23, 6385, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 362, 642, 150, 251, 1, 1393, 876, 1342, 61, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5646, 6944, 5948, 6614, 6624, 5530, 6483, 5363] avg=6144.0000 max=6944 max_rank=1 min=5363 min_rank=7 imbalance_ratio=1.130208 max_load_reduction=42208 max_load_reduction_pct=0.858724 moved_to_echo=24000 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2720, 0, 0, 90, 732, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 74, 367, 0, 0, 0, 0, 0, 2996, 1267, 0, 0, 0, 0, 0, 0, 0, 90, 128, 0, 0, 88, 0, 0, 4189, 1560, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 90, 0, 0, 6442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 0, 0, 0, 2958, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 0, 0, 3465, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 6332, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 223, 595, 203, 265, 32, 1328, 1626, 1060, 60, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6446, 5617, 6055, 6655, 5011, 7088, 6422, 5858] avg=6144.0000 max=7088 max_rank=5 min=5011 min_rank=4 imbalance_ratio=1.153646 max_load_reduction=42064 max_load_reduction_pct=0.855794 moved_to_echo=24379 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[433, 72, 3175, 247, 668, 2154, 1101, 611, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 270, 209, 0, 358, 85, 275, 238, 1196, 375, 0, 0, 0, 0, 0, 0, 14, 43, 594, 2152, 285, 105, 1279, 603, 1672, 213, 0, 0, 0, 0, 0, 0, 163, 141, 724, 323, 126, 192, 279, 375, 876, 0, 0, 0, 0, 0, 0, 0, 1749, 243, 460, 62, 391, 660, 2156, 675, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 1486, 0, 176, 1241, 2070, 621, 0, 0, 0, 0, 0, 0, 0, 0, 301, 1611, 337, 134, 84, 306, 548, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 2924, 97, 808, 591, 129, 1023, 295, 793, 1207, 313, 12, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8461, 4366, 6960, 3199, 6396, 5737, 4584, 9449] avg=6144.0000 max=9449 max_rank=7 min=3199 min_rank=3 imbalance_ratio=1.537923 max_load_reduction=39703 max_load_reduction_pct=0.807760 moved_to_echo=26166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2977, 0, 0, 147, 704, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1488, 85, 771, 0, 0, 0, 0, 0, 2979, 1786, 0, 0, 0, 0, 0, 0, 0, 74, 132, 0, 0, 73, 0, 0, 4167, 1564, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 23, 74, 0, 0, 6461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 0, 0, 4364, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1701, 0, 0, 0, 3717, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 118, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 329, 611, 124, 240, 9, 1425, 699, 1495, 58, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5078, 7109, 6010, 6568, 7280, 5489, 6329, 5289] avg=6144.0000 max=7280 max_rank=4 min=5078 min_rank=0 imbalance_ratio=1.184896 max_load_reduction=41872 max_load_reduction_pct=0.851888 moved_to_echo=24387 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2579, 0, 18, 76, 689, 2577, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1363, 92, 467, 0, 0, 4, 1, 1, 2998, 1435, 0, 0, 0, 0, 0, 0, 0, 76, 161, 15, 3, 76, 9, 0, 4191, 1550, 0, 0, 0, 0, 0, 0, 53, 0, 0, 9, 0, 76, 0, 0, 6400, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1759, 0, 0, 5, 3627, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 11, 3498, 66, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 6, 76, 0, 1, 10, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 321, 14, 0, 308, 587, 179, 240, 60, 1341, 1444, 1201, 58, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5940, 6361, 6081, 6538, 5900, 6350, 6229, 5753] avg=6144.0000 max=6538 max_rank=3 min=5753 min_rank=7 imbalance_ratio=1.064128 max_load_reduction=42614 max_load_reduction_pct=0.866984 moved_to_echo=24232 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2701, 0, 24, 131, 712, 2069, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 72, 597, 0, 1, 1, 0, 5, 2988, 1650, 0, 0, 0, 0, 0, 0, 0, 72, 112, 17, 6, 72, 8, 5, 4176, 1525, 0, 0, 0, 0, 0, 0, 7, 0, 0, 5, 1, 74, 0, 2, 6297, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2011, 0, 0, 2, 3723, 496, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2075, 0, 1, 15, 3804, 72, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 1, 72, 0, 5, 19, 6300, 0, 0, 0, 0, 0, 0, 0, 0, 392, 19, 0, 368, 605, 137, 256, 20, 1466, 1165, 1384, 63, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5643, 6641, 5993, 6386, 6241, 5969, 6404, 5875] avg=6144.0000 max=6641 max_rank=1 min=5643 min_rank=0 imbalance_ratio=1.080892 max_load_reduction=42511 max_load_reduction_pct=0.864889 moved_to_echo=24489 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2589, 0, 0, 80, 747, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 96, 353, 0, 0, 0, 0, 0, 2989, 1295, 0, 0, 0, 0, 0, 0, 0, 80, 164, 0, 0, 80, 0, 0, 4182, 1560, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 80, 0, 0, 6449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1586, 0, 0, 0, 2979, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 0, 0, 3089, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 6295, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 379, 686, 140, 247, 17, 1184, 1795, 1084, 60, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6620, 5649, 6066, 6642, 5010, 6758, 6375, 6032] avg=6144.0000 max=6758 max_rank=5 min=5010 min_rank=4 imbalance_ratio=1.099935 max_load_reduction=42394 max_load_reduction_pct=0.862508 moved_to_echo=24175 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2720, 0, 0, 90, 732, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 74, 367, 0, 0, 0, 0, 0, 2996, 1267, 0, 0, 0, 0, 0, 0, 0, 90, 128, 0, 0, 88, 0, 0, 4189, 1560, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 90, 0, 0, 6442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 0, 0, 0, 2958, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 0, 0, 3465, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 6332, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 223, 595, 203, 265, 32, 1328, 1626, 1060, 60, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6446, 5617, 6055, 6655, 5011, 7088, 6422, 5858] avg=6144.0000 max=7088 max_rank=5 min=5011 min_rank=4 imbalance_ratio=1.153646 max_load_reduction=42064 max_load_reduction_pct=0.855794 moved_to_echo=24379 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10240, 896, 4096, 0, 384, 128, 384, 256, 22144, 11136, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3840, 3072, 128, 3200, 4992, 1280, 2816, 512, 10368, 9856, 9344, 512, 0, 0, 0, 0] @@ -10439,318 +10439,318 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 640, 1664, 2304, 384, 768, 1408, 640, 30848, 11136, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 1664, 384, 768, 128, 384, 768, 45056, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 256, 768, 384, 256, 768, 384, 384, 45824, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[2744, 914, 0, 0, 0, 0, 2413, 0, 0, 3430, 0, 392, 0, 0, 334, 0, 0, 160, 432, 0, 0, 621, 0, 0, 0, 0, 6399, 652, 2066, 0, 302, 1573, 0, 0, 0, 0, 0, 0, 0, 3346, 0, 0, 1090, 163, 6436, 100, 6266, 0, 6489, 0, 0, 143, 0, 80, 0, 0, 87, 0, 0, 0, 1762, 80, 0, 678] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[3158, 439, 0, 0, 0, 0, 1826, 3, 0, 5762, 0, 452, 154, 0, 748, 0, 0, 147, 788, 0, 0, 525, 6, 0, 0, 0, 4704, 624, 1488, 0, 233, 639, 0, 0, 0, 0, 0, 0, 0, 5274, 0, 0, 233, 131, 5413, 219, 6875, 0, 6771, 12, 0, 181, 0, 73, 0, 0, 85, 0, 0, 0, 1500, 74, 0, 615] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[3029, 604, 0, 0, 0, 0, 1953, 1, 0, 4667, 0, 547, 129, 0, 754, 0, 0, 199, 625, 0, 0, 523, 0, 0, 0, 0, 5210, 581, 2003, 0, 375, 820, 0, 0, 0, 0, 0, 0, 0, 4312, 0, 0, 350, 122, 6113, 261, 6677, 0, 6605, 0, 0, 166, 0, 99, 0, 0, 67, 0, 0, 0, 1486, 100, 0, 774] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[2413, 537, 0, 0, 0, 0, 1734, 0, 0, 5444, 0, 589, 189, 0, 797, 0, 0, 149, 711, 1, 0, 487, 0, 0, 0, 0, 4739, 607, 1467, 0, 262, 669, 0, 0, 0, 0, 0, 0, 0, 5728, 0, 0, 203, 141, 5797, 216, 6906, 0, 6817, 14, 0, 202, 0, 74, 0, 0, 84, 0, 0, 0, 1437, 75, 0, 663] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[1529, 944, 275, 16, 226, 1026, 554, 198, 514, 1223, 1258, 522, 724, 284, 91, 64, 622, 223, 408, 811, 497, 1193, 169, 446, 60, 520, 750, 782, 1204, 985, 327, 737, 3251, 883, 525, 44, 2954, 80, 287, 1596, 346, 1703, 1801, 59, 1936, 213, 1630, 1623, 2472, 345, 189, 298, 0, 71, 47, 315, 59, 241, 80, 3407, 1943, 264, 844, 464] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[3029, 604, 0, 0, 0, 0, 1953, 1, 0, 4667, 0, 547, 129, 0, 754, 0, 0, 199, 625, 0, 0, 523, 0, 0, 0, 0, 5210, 581, 2003, 0, 375, 820, 0, 0, 0, 0, 0, 0, 0, 4312, 0, 0, 350, 122, 6113, 261, 6677, 0, 6605, 0, 0, 166, 0, 99, 0, 0, 67, 0, 0, 0, 1486, 100, 0, 774] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[2922, 653, 0, 0, 0, 1, 2069, 10, 0, 4005, 1, 429, 57, 0, 598, 0, 3, 152, 550, 0, 7, 573, 0, 4, 1, 0, 5862, 610, 2473, 8, 326, 1222, 17, 16, 0, 0, 15, 1, 0, 3559, 0, 15, 717, 149, 6309, 138, 6518, 11, 6529, 0, 1, 175, 0, 76, 0, 2, 88, 0, 0, 19, 1534, 76, 2, 649] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[3019, 534, 0, 0, 3, 2, 1667, 6, 2, 4317, 12, 700, 172, 2, 908, 0, 7, 144, 660, 10, 1, 520, 1, 2, 5, 1, 5133, 639, 1585, 6, 232, 653, 24, 12, 2, 0, 22, 4, 0, 5417, 1, 15, 376, 109, 5745, 239, 6735, 14, 6858, 2, 0, 176, 0, 72, 0, 12, 64, 1, 0, 22, 1560, 72, 4, 651] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[2744, 914, 0, 0, 0, 0, 2413, 0, 0, 3430, 0, 392, 0, 0, 334, 0, 0, 160, 432, 0, 0, 621, 0, 0, 0, 0, 6399, 652, 2066, 0, 302, 1573, 0, 0, 0, 0, 0, 0, 0, 3346, 0, 0, 1090, 163, 6436, 100, 6266, 0, 6489, 0, 0, 143, 0, 80, 0, 0, 87, 0, 0, 0, 1762, 80, 0, 678] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[2922, 653, 0, 0, 0, 1, 2069, 10, 0, 4005, 1, 429, 57, 0, 598, 0, 3, 152, 550, 0, 7, 573, 0, 4, 1, 0, 5862, 610, 2473, 8, 326, 1222, 17, 16, 0, 0, 15, 1, 0, 3559, 0, 15, 717, 149, 6309, 138, 6518, 11, 6529, 0, 1, 175, 0, 76, 0, 2, 88, 0, 0, 19, 1534, 76, 2, 649] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[3158, 439, 0, 0, 0, 0, 1826, 3, 0, 5762, 0, 452, 154, 0, 748, 0, 0, 147, 788, 0, 0, 525, 6, 0, 0, 0, 4704, 624, 1488, 0, 233, 639, 0, 0, 0, 0, 0, 0, 0, 5274, 0, 0, 233, 131, 5413, 219, 6875, 0, 6771, 12, 0, 181, 0, 73, 0, 0, 85, 0, 0, 0, 1500, 74, 0, 615] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[2833, 1038, 0, 0, 0, 0, 2258, 4, 0, 3560, 0, 402, 7, 0, 407, 0, 0, 179, 391, 0, 0, 534, 0, 0, 0, 0, 6584, 548, 2273, 0, 338, 1223, 0, 0, 0, 0, 0, 0, 0, 3222, 0, 0, 1063, 119, 6440, 133, 6282, 0, 6596, 0, 0, 179, 0, 89, 0, 0, 68, 0, 0, 0, 1566, 88, 0, 728] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3029, 604, 0, 0, 0, 0, 1953, 1, 774, 0, 0, 0, 0, 0, 0, 0, 0, 4667, 0, 547, 129, 0, 754, 0, 413, 197, 0, 0, 0, 0, 0, 0, 0, 199, 625, 0, 0, 523, 0, 0, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 581, 2003, 0, 375, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4312, 1121, 11, 0, 0, 0, 0, 0, 0, 0, 0, 350, 122, 5131, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6192, 0, 0, 166, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 1486, 100, 0, 774, 1703, 1704, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6361, 6707, 6320, 6165, 5444, 5864, 6457, 5834] avg=6144.0000 max=6707 max_rank=1 min=5444 min_rank=4 imbalance_ratio=1.091634 max_load_reduction=42445 max_load_reduction_pct=0.863546 moved_to_echo=23599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2413, 537, 0, 0, 0, 0, 1734, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 5444, 0, 589, 189, 0, 797, 0, 427, 187, 0, 0, 0, 0, 0, 0, 0, 149, 711, 1, 0, 487, 0, 0, 5144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2171, 607, 1467, 0, 262, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5728, 1019, 11, 0, 0, 0, 0, 0, 0, 0, 0, 203, 141, 4865, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6390, 14, 0, 202, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 1437, 75, 0, 663, 1549, 1762, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5418, 7633, 6492, 5176, 6758, 5425, 6680, 5570] avg=6144.0000 max=7633 max_rank=1 min=5176 min_rank=3 imbalance_ratio=1.242350 max_load_reduction=41519 max_load_reduction_pct=0.844706 moved_to_echo=24433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2744, 914, 0, 0, 0, 0, 2413, 0, 815, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 0, 392, 0, 0, 334, 0, 406, 207, 0, 0, 0, 0, 0, 0, 0, 160, 432, 0, 0, 621, 0, 0, 4667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 652, 2066, 0, 302, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3346, 1377, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 163, 5402, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6083, 0, 0, 143, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 1762, 80, 0, 678, 2091, 1599, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6886, 4769, 5880, 7524, 4735, 6755, 6306, 6297] avg=6144.0000 max=7524 max_rank=3 min=4735 min_rank=4 imbalance_ratio=1.224609 max_load_reduction=41628 max_load_reduction_pct=0.846924 moved_to_echo=24093 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3158, 439, 0, 0, 0, 0, 1826, 3, 685, 0, 0, 0, 0, 0, 0, 0, 0, 5762, 0, 452, 154, 0, 748, 0, 424, 174, 0, 0, 0, 0, 0, 0, 0, 147, 788, 0, 0, 525, 6, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2155, 624, 1488, 0, 233, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5274, 1012, 10, 0, 0, 0, 0, 0, 0, 0, 0, 233, 131, 4544, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6347, 12, 0, 181, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 1500, 74, 0, 615, 1537, 1755, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 7714, 6586, 5139, 6296, 5127, 6613, 5566] avg=6144.0000 max=7714 max_rank=1 min=5127 min_rank=5 imbalance_ratio=1.255534 max_load_reduction=41438 max_load_reduction_pct=0.843058 moved_to_echo=23602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1529, 944, 275, 16, 226, 1026, 554, 198, 245, 0, 0, 0, 0, 0, 0, 0, 514, 1223, 1258, 522, 724, 284, 91, 64, 154, 62, 0, 0, 0, 0, 0, 0, 622, 223, 408, 811, 497, 1193, 169, 446, 1215, 0, 0, 0, 0, 0, 0, 0, 60, 520, 344, 782, 1204, 985, 327, 737, 0, 0, 0, 0, 0, 0, 0, 0, 3251, 883, 525, 44, 2954, 80, 287, 1596, 161, 3, 0, 0, 0, 0, 0, 0, 346, 1703, 1801, 59, 1626, 213, 0, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 345, 189, 298, 0, 71, 47, 315, 0, 0, 0, 0, 0, 0, 0, 0, 59, 241, 80, 3407, 1943, 264, 844, 464, 245, 415, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5013, 4896, 5584, 4959, 9784, 7371, 3583, 7962] avg=6144.0000 max=9784 max_rank=4 min=3583 min_rank=6 imbalance_ratio=1.592448 max_load_reduction=39368 max_load_reduction_pct=0.800944 moved_to_echo=28700 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3029, 604, 0, 0, 0, 0, 1953, 1, 774, 0, 0, 0, 0, 0, 0, 0, 0, 4667, 0, 547, 129, 0, 754, 0, 413, 197, 0, 0, 0, 0, 0, 0, 0, 199, 625, 0, 0, 523, 0, 0, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 581, 2003, 0, 375, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4312, 1121, 11, 0, 0, 0, 0, 0, 0, 0, 0, 350, 122, 5131, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6192, 0, 0, 166, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 1486, 100, 0, 774, 1703, 1704, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6361, 6707, 6320, 6165, 5444, 5864, 6457, 5834] avg=6144.0000 max=6707 max_rank=1 min=5444 min_rank=4 imbalance_ratio=1.091634 max_load_reduction=42445 max_load_reduction_pct=0.863546 moved_to_echo=23599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3158, 439, 0, 0, 0, 0, 1826, 3, 685, 0, 0, 0, 0, 0, 0, 0, 0, 5762, 0, 452, 154, 0, 748, 0, 424, 174, 0, 0, 0, 0, 0, 0, 0, 147, 788, 0, 0, 525, 6, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2155, 624, 1488, 0, 233, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5274, 1012, 10, 0, 0, 0, 0, 0, 0, 0, 0, 233, 131, 4544, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6347, 12, 0, 181, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 1500, 74, 0, 615, 1537, 1755, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 7714, 6586, 5139, 6296, 5127, 6613, 5566] avg=6144.0000 max=7714 max_rank=1 min=5127 min_rank=5 imbalance_ratio=1.255534 max_load_reduction=41438 max_load_reduction_pct=0.843058 moved_to_echo=23602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2922, 653, 0, 0, 0, 1, 2069, 10, 799, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 1, 429, 57, 0, 598, 0, 408, 203, 0, 0, 0, 0, 0, 0, 3, 152, 550, 0, 7, 573, 0, 4, 4855, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2685, 610, 2473, 8, 326, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 17, 16, 0, 0, 15, 1, 0, 3559, 1261, 12, 0, 0, 0, 0, 0, 0, 0, 15, 717, 149, 5295, 138, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 6121, 0, 1, 175, 0, 76, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 19, 1534, 76, 2, 649, 1916, 1663, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6454, 5701, 6144, 7325, 4881, 6325, 6375, 5947] avg=6144.0000 max=7325 max_rank=3 min=4881 min_rank=4 imbalance_ratio=1.192220 max_load_reduction=41827 max_load_reduction_pct=0.850972 moved_to_echo=23528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2413, 537, 0, 0, 0, 0, 1734, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 5444, 0, 589, 189, 0, 797, 0, 427, 187, 0, 0, 0, 0, 0, 0, 0, 149, 711, 1, 0, 487, 0, 0, 5144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2171, 607, 1467, 0, 262, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5728, 1019, 11, 0, 0, 0, 0, 0, 0, 0, 0, 203, 141, 4865, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6390, 14, 0, 202, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 1437, 75, 0, 663, 1549, 1762, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5418, 7633, 6492, 5176, 6758, 5425, 6680, 5570] avg=6144.0000 max=7633 max_rank=1 min=5176 min_rank=3 imbalance_ratio=1.242350 max_load_reduction=41519 max_load_reduction_pct=0.844706 moved_to_echo=24433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3019, 534, 0, 0, 3, 2, 1667, 6, 730, 0, 0, 0, 0, 0, 0, 0, 2, 4317, 12, 700, 172, 2, 908, 0, 433, 190, 0, 0, 0, 0, 0, 0, 7, 144, 660, 10, 1, 520, 1, 2, 5017, 0, 0, 0, 0, 0, 0, 0, 5, 1, 2345, 639, 1585, 6, 232, 653, 0, 0, 0, 0, 0, 0, 0, 0, 24, 12, 2, 0, 22, 4, 0, 5417, 1108, 15, 0, 0, 0, 0, 0, 0, 1, 15, 376, 109, 4810, 239, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 6425, 2, 0, 176, 0, 72, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 22, 1560, 72, 4, 651, 1680, 1718, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5961, 6736, 6362, 5466, 6604, 5564, 6687, 5772] avg=6144.0000 max=6736 max_rank=1 min=5466 min_rank=3 imbalance_ratio=1.096354 max_load_reduction=42416 max_load_reduction_pct=0.862956 moved_to_echo=24627 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2833, 1038, 0, 0, 0, 0, 2258, 4, 815, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 402, 7, 0, 407, 0, 413, 207, 0, 0, 0, 0, 0, 0, 0, 179, 391, 0, 0, 534, 0, 0, 4680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3016, 548, 2273, 0, 338, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3222, 1416, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 119, 5406, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6183, 0, 0, 179, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 1566, 88, 0, 728, 2152, 1602, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6948, 4996, 5784, 7398, 4650, 6721, 6451, 6204] avg=6144.0000 max=7398 max_rank=3 min=4650 min_rank=4 imbalance_ratio=1.204102 max_load_reduction=41754 max_load_reduction_pct=0.849487 moved_to_echo=24026 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3019, 534, 0, 0, 3, 2, 1667, 6, 730, 0, 0, 0, 0, 0, 0, 0, 2, 4317, 12, 700, 172, 2, 908, 0, 433, 190, 0, 0, 0, 0, 0, 0, 7, 144, 660, 10, 1, 520, 1, 2, 5017, 0, 0, 0, 0, 0, 0, 0, 5, 1, 2345, 639, 1585, 6, 232, 653, 0, 0, 0, 0, 0, 0, 0, 0, 24, 12, 2, 0, 22, 4, 0, 5417, 1108, 15, 0, 0, 0, 0, 0, 0, 1, 15, 376, 109, 4810, 239, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 6425, 2, 0, 176, 0, 72, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 22, 1560, 72, 4, 651, 1680, 1718, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5961, 6736, 6362, 5466, 6604, 5564, 6687, 5772] avg=6144.0000 max=6736 max_rank=1 min=5466 min_rank=3 imbalance_ratio=1.096354 max_load_reduction=42416 max_load_reduction_pct=0.862956 moved_to_echo=24627 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[21760, 5760, 384, 128, 256, 1152, 14592, 256, 5632, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 256, 128, 3456, 12800, 896, 896, 5248, 12928, 12288, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[640, 32512, 1280, 4096, 1536, 384, 4736, 128, 3200, 1536, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 640, 18048, 5120, 14592, 1024, 2432, 7552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[640, 32512, 1280, 4096, 1536, 384, 4736, 128, 3200, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 256, 128, 3456, 12800, 896, 896, 5248, 12928, 12288, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3328, 1024, 640, 128, 3072, 128, 384, 32512, 8576, 128, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 1408, 4608, 896, 512, 4992, 256, 512, 35712, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[384, 1792, 5888, 1024, 37120, 1536, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[46080, 384, 256, 1536, 0, 640, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 58, 644, 0, 0, 0, 2577, 0, 0, 0, 145, 104, 7491, 0, 0, 7593, 2, 0, 0, 0, 0, 4890, 0, 0, 0, 0, 0, 8116, 0, 0, 3002, 0, 0, 0, 73, 0, 0, 6802, 0, 0, 0, 0, 475, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 257, 0, 74, 0, 5790, 0, 0, 829, 225, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 30, 778, 0, 0, 0, 2584, 0, 0, 0, 158, 81, 7422, 0, 0, 7504, 61, 0, 0, 0, 0, 4241, 0, 0, 0, 0, 0, 8091, 0, 0, 3559, 0, 0, 0, 99, 0, 0, 7204, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 100, 0, 5398, 0, 0, 818, 225, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 62, 555, 0, 0, 0, 2791, 0, 0, 0, 148, 97, 7423, 0, 0, 7556, 13, 0, 0, 0, 0, 4689, 0, 0, 0, 0, 0, 8118, 0, 0, 3398, 0, 7, 0, 74, 0, 0, 6979, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 251, 0, 75, 0, 5517, 0, 0, 833, 94, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 1791, 0, 0, 0, 2119, 0, 0, 0, 148, 131, 7289, 0, 0, 7548, 78, 0, 0, 0, 0, 2380, 0, 0, 0, 0, 0, 8062, 0, 0, 5314, 0, 0, 0, 80, 0, 0, 7684, 0, 0, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 80, 0, 4238, 0, 0, 509, 1030, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[83, 473, 1786, 451, 21, 97, 975, 4, 2, 215, 95, 54, 3014, 4, 2275, 2301, 247, 86, 14, 664, 764, 378, 230, 2208, 151, 706, 94, 2611, 79, 3574, 1193, 551, 213, 4, 106, 63, 138, 1495, 1437, 326, 659, 0, 86, 0, 1424, 3950, 1111, 129, 0, 450, 91, 471, 698, 1, 2342, 183, 152, 832, 0, 343, 690, 2553, 3345, 460] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 58, 644, 0, 0, 0, 2577, 0, 0, 0, 145, 104, 7491, 0, 0, 7593, 2, 0, 0, 0, 0, 4890, 0, 0, 0, 0, 0, 8116, 0, 0, 3002, 0, 0, 0, 73, 0, 0, 6802, 0, 0, 0, 0, 475, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 257, 0, 74, 0, 5790, 0, 0, 829, 225, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[3, 4, 1273, 1, 0, 0, 2073, 0, 0, 0, 137, 103, 7352, 0, 18, 7506, 64, 1, 0, 0, 0, 3429, 7, 12, 4, 0, 0, 8065, 0, 19, 4135, 0, 0, 1, 76, 0, 1, 7584, 5, 7, 3, 0, 531, 0, 2, 19, 13, 0, 0, 12, 0, 0, 0, 251, 14, 76, 1, 4785, 0, 0, 747, 811, 7, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 30, 778, 0, 0, 0, 2584, 0, 0, 0, 158, 81, 7422, 0, 0, 7504, 61, 0, 0, 0, 0, 4241, 0, 0, 0, 0, 0, 8091, 0, 0, 3559, 0, 0, 0, 99, 0, 0, 7204, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 100, 0, 5398, 0, 0, 818, 225, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[5, 71, 893, 1, 0, 0, 2266, 0, 0, 0, 125, 81, 7457, 2, 25, 7577, 5, 0, 0, 0, 3, 4278, 2, 16, 2, 2, 0, 8072, 2, 26, 3609, 2, 8, 0, 73, 0, 0, 7120, 3, 4, 0, 0, 422, 0, 3, 28, 15, 0, 0, 6, 17, 1, 1, 255, 18, 72, 3, 5542, 0, 5, 785, 237, 10, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 1747, 0, 0, 0, 1898, 0, 0, 0, 142, 92, 7339, 0, 0, 7491, 25, 0, 0, 0, 0, 2666, 0, 0, 0, 0, 0, 8087, 0, 0, 5144, 0, 0, 0, 89, 0, 0, 7639, 0, 2, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 90, 0, 4536, 0, 0, 513, 854, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 58, 644, 0, 0, 0, 2577, 0, 2591, 291, 0, 0, 0, 0, 0, 0, 0, 0, 145, 104, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4890, 0, 0, 1078, 808, 563, 0, 0, 0, 0, 0, 0, 0, 0, 2000, 0, 0, 3002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 5994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 0, 0, 3525, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 257, 0, 74, 5849, 0, 0, 0, 0, 0, 0, 0, 0, 5227, 0, 0, 829, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6161, 6371, 7341, 5002, 6067, 5744, 6185, 6281] avg=6144.0000 max=7341 max_rank=2 min=5002 min_rank=3 imbalance_ratio=1.194824 max_load_reduction=41811 max_load_reduction_pct=0.850647 moved_to_echo=24277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[83, 473, 1786, 451, 21, 97, 975, 4, 2, 215, 95, 54, 3014, 4, 2275, 2301, 247, 86, 14, 664, 764, 378, 230, 2208, 151, 706, 94, 2611, 79, 3574, 1193, 551, 213, 4, 106, 63, 138, 1495, 1437, 326, 659, 0, 86, 0, 1424, 3950, 1111, 129, 0, 450, 91, 471, 698, 1, 2342, 183, 152, 832, 0, 343, 690, 2553, 3345, 460] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[5, 71, 893, 1, 0, 0, 2266, 0, 0, 0, 125, 81, 7457, 2, 25, 7577, 5, 0, 0, 0, 3, 4278, 2, 16, 2, 2, 0, 8072, 2, 26, 3609, 2, 8, 0, 73, 0, 0, 7120, 3, 4, 0, 0, 422, 0, 3, 28, 15, 0, 0, 6, 17, 1, 1, 255, 18, 72, 3, 5542, 0, 5, 785, 237, 10, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 555, 0, 0, 0, 2791, 0, 2591, 288, 0, 0, 0, 0, 0, 0, 0, 0, 148, 97, 6067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4689, 0, 0, 1068, 829, 537, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 0, 0, 3398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 74, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 3526, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 251, 0, 75, 5820, 0, 0, 0, 0, 0, 0, 0, 0, 4980, 0, 0, 833, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6287, 6312, 7136, 5399, 6231, 5730, 6150, 5907] avg=6144.0000 max=7136 max_rank=2 min=5399 min_rank=3 imbalance_ratio=1.161458 max_load_reduction=42016 max_load_reduction_pct=0.854818 moved_to_echo=24018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 778, 0, 0, 0, 2584, 0, 2583, 288, 0, 0, 0, 0, 0, 0, 0, 0, 158, 81, 6066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 4241, 0, 0, 1068, 856, 525, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 6348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 3514, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 100, 5780, 0, 0, 0, 0, 0, 0, 0, 0, 4873, 0, 0, 818, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 6305, 6751, 5553, 6447, 5781, 6136, 5916] avg=6144.0000 max=6751 max_rank=2 min=5553 min_rank=3 imbalance_ratio=1.098796 max_load_reduction=42401 max_load_reduction_pct=0.862651 moved_to_echo=24280 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1791, 0, 0, 0, 2119, 0, 2574, 283, 0, 0, 0, 0, 0, 0, 0, 0, 148, 131, 5957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 2380, 0, 0, 1049, 913, 412, 0, 0, 0, 0, 0, 0, 0, 0, 1986, 0, 0, 5314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 6771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 0, 0, 0, 0, 3502, 1734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 80, 5814, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 0, 0, 509, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6767, 6236, 4832, 7300, 6851, 5658, 6143, 5365] avg=6144.0000 max=7300 max_rank=3 min=4832 min_rank=2 imbalance_ratio=1.188151 max_load_reduction=41852 max_load_reduction_pct=0.851481 moved_to_echo=24017 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 58, 644, 0, 0, 0, 2577, 0, 2591, 291, 0, 0, 0, 0, 0, 0, 0, 0, 145, 104, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4890, 0, 0, 1078, 808, 563, 0, 0, 0, 0, 0, 0, 0, 0, 2000, 0, 0, 3002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 5994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 0, 0, 3525, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 257, 0, 74, 5849, 0, 0, 0, 0, 0, 0, 0, 0, 5227, 0, 0, 829, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6161, 6371, 7341, 5002, 6067, 5744, 6185, 6281] avg=6144.0000 max=7341 max_rank=2 min=5002 min_rank=3 imbalance_ratio=1.194824 max_load_reduction=41811 max_load_reduction_pct=0.850647 moved_to_echo=24277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1747, 0, 0, 0, 1898, 0, 2582, 285, 0, 0, 0, 0, 0, 0, 0, 0, 142, 92, 5998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 2666, 0, 0, 1056, 908, 441, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 0, 0, 5144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 6731, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 3513, 1722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 90, 5769, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 513, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6512, 6232, 5096, 7136, 6822, 5703, 6189, 5462] avg=6144.0000 max=7136 max_rank=3 min=5096 min_rank=2 imbalance_ratio=1.161458 max_load_reduction=42016 max_load_reduction_pct=0.854818 moved_to_echo=24176 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[3, 4, 1273, 1, 0, 0, 2073, 0, 2575, 285, 0, 0, 0, 0, 0, 0, 0, 0, 137, 103, 6009, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 0, 0, 3429, 7, 12, 1058, 901, 465, 0, 0, 0, 0, 0, 4, 0, 0, 1987, 0, 19, 4135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 76, 0, 1, 6683, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 531, 0, 2, 19, 13, 0, 3503, 1724, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 251, 14, 76, 5782, 0, 0, 0, 0, 0, 0, 0, 1, 4320, 0, 0, 747, 811, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 6267, 5937, 6145, 6773, 5795, 6135, 5886] avg=6144.0000 max=6773 max_rank=4 min=5795 min_rank=5 imbalance_ratio=1.102376 max_load_reduction=42379 max_load_reduction_pct=0.862203 moved_to_echo=24589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[83, 473, 1786, 451, 21, 97, 975, 4, 833, 117, 0, 0, 0, 0, 0, 0, 2, 215, 95, 54, 2464, 4, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 86, 14, 664, 764, 378, 230, 2208, 433, 177, 81, 0, 0, 0, 0, 0, 151, 706, 94, 644, 79, 3574, 1193, 551, 0, 0, 0, 0, 0, 0, 0, 0, 213, 4, 106, 63, 138, 1318, 1437, 326, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 86, 0, 1424, 3950, 1111, 129, 1134, 529, 0, 0, 0, 0, 0, 0, 0, 450, 91, 471, 698, 1, 2342, 183, 1772, 0, 0, 0, 0, 0, 0, 0, 152, 751, 0, 343, 690, 2553, 3345, 460, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4840, 5109, 5282, 6992, 3605, 9022, 6008, 8294] avg=6144.0000 max=9022 max_rank=5 min=3605 min_rank=4 imbalance_ratio=1.468424 max_load_reduction=40130 max_load_reduction_pct=0.816447 moved_to_echo=26929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 555, 0, 0, 0, 2791, 0, 2591, 288, 0, 0, 0, 0, 0, 0, 0, 0, 148, 97, 6067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4689, 0, 0, 1068, 829, 537, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 0, 0, 3398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 74, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 3526, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 251, 0, 75, 5820, 0, 0, 0, 0, 0, 0, 0, 0, 4980, 0, 0, 833, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6287, 6312, 7136, 5399, 6231, 5730, 6150, 5907] avg=6144.0000 max=7136 max_rank=2 min=5399 min_rank=3 imbalance_ratio=1.161458 max_load_reduction=42016 max_load_reduction_pct=0.854818 moved_to_echo=24018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 778, 0, 0, 0, 2584, 0, 2583, 288, 0, 0, 0, 0, 0, 0, 0, 0, 158, 81, 6066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 4241, 0, 0, 1068, 856, 525, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 6348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 3514, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 100, 5780, 0, 0, 0, 0, 0, 0, 0, 0, 4873, 0, 0, 818, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 6305, 6751, 5553, 6447, 5781, 6136, 5916] avg=6144.0000 max=6751 max_rank=2 min=5553 min_rank=3 imbalance_ratio=1.098796 max_load_reduction=42401 max_load_reduction_pct=0.862651 moved_to_echo=24280 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[5, 71, 893, 1, 0, 0, 2266, 0, 2580, 292, 0, 0, 0, 0, 0, 0, 0, 0, 125, 81, 6087, 2, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 4278, 2, 16, 1078, 852, 543, 0, 0, 0, 0, 0, 2, 2, 0, 1982, 2, 26, 3609, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 73, 0, 0, 6268, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 3, 28, 15, 0, 3510, 1741, 0, 0, 0, 0, 0, 0, 0, 6, 17, 1, 1, 255, 18, 72, 5836, 0, 0, 0, 0, 0, 0, 0, 3, 4999, 0, 5, 785, 237, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6108, 6320, 6777, 5625, 6356, 5719, 6206, 6041] avg=6144.0000 max=6777 max_rank=2 min=5625 min_rank=3 imbalance_ratio=1.103027 max_load_reduction=42375 max_load_reduction_pct=0.862122 moved_to_echo=24322 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1747, 0, 0, 0, 1898, 0, 2582, 285, 0, 0, 0, 0, 0, 0, 0, 0, 142, 92, 5998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 2666, 0, 0, 1056, 908, 441, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 0, 0, 5144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 6731, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 3513, 1722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 90, 5769, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 513, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6512, 6232, 5096, 7136, 6822, 5703, 6189, 5462] avg=6144.0000 max=7136 max_rank=3 min=5096 min_rank=2 imbalance_ratio=1.161458 max_load_reduction=42016 max_load_reduction_pct=0.854818 moved_to_echo=24176 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 768, 9472, 512, 128, 128, 17408, 128, 18944, 2176, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 128, 128, 768, 768, 27008, 256, 2304, 7936, 6272, 3584, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 768, 128, 14592, 128, 3712, 29440, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 0, 3456, 0, 1536, 4096, 1152, 256, 25728, 12672, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 33152, 0, 384, 5760, 6144, 3456, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 0, 3456, 0, 1536, 4096, 1152, 256, 25728, 12672, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 256, 1152, 768, 44800, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 128, 768, 128, 256, 46336, 1536, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 512, 128, 512, 768, 1920, 2432, 768, 42496, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 128, 768, 128, 256, 46336, 1536, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 69, 0, 0, 0, 0, 175, 0, 0, 64, 323, 0, 3847, 0, 78, 7740, 7727, 0, 0, 0, 0, 55, 594, 0, 7708, 0, 64, 0, 4112, 0, 0, 0, 176, 6061, 7510, 74, 0, 0, 0, 0, 0, 26, 0, 125, 0, 1377, 0, 137, 0, 0, 0, 0, 0, 0, 0, 7, 430, 0, 318, 345] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 73, 0, 0, 0, 0, 190, 0, 0, 70, 629, 0, 3946, 0, 115, 7687, 7622, 0, 0, 0, 0, 1, 581, 0, 7683, 0, 70, 0, 4429, 0, 0, 0, 240, 5056, 7532, 75, 0, 0, 0, 0, 0, 64, 0, 123, 0, 1657, 0, 163, 0, 0, 0, 0, 0, 0, 0, 2, 433, 0, 339, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 92, 0, 0, 0, 0, 221, 0, 0, 55, 728, 0, 3754, 0, 198, 7525, 7617, 0, 0, 0, 0, 0, 630, 0, 7675, 0, 55, 0, 4569, 0, 0, 0, 148, 5399, 7539, 100, 0, 0, 0, 0, 0, 55, 0, 136, 0, 1260, 0, 227, 0, 0, 0, 0, 0, 0, 0, 9, 454, 0, 336, 362] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 68, 969, 0, 3397, 0, 160, 7626, 7869, 0, 0, 0, 0, 0, 1397, 0, 7650, 0, 68, 0, 6722, 0, 5, 0, 0, 2986, 7464, 80, 0, 0, 0, 0, 0, 68, 0, 59, 0, 745, 0, 177, 0, 0, 0, 0, 0, 0, 0, 160, 468, 0, 358, 380] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[29, 0, 13, 0, 19, 47, 1, 1, 0, 0, 158, 6, 0, 63, 896, 0, 3285, 15, 153, 7586, 7793, 0, 10, 5, 0, 0, 925, 0, 7696, 1, 68, 0, 5658, 8, 11, 1, 2, 4610, 7468, 76, 0, 5, 0, 16, 3, 61, 0, 63, 0, 983, 0, 210, 0, 17, 0, 0, 2, 0, 21, 25, 448, 18, 344, 332] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 69, 0, 0, 0, 0, 175, 0, 0, 64, 323, 0, 3847, 0, 78, 7740, 7727, 0, 0, 0, 0, 55, 594, 0, 7708, 0, 64, 0, 4112, 0, 0, 0, 176, 6061, 7510, 74, 0, 0, 0, 0, 0, 26, 0, 125, 0, 1377, 0, 137, 0, 0, 0, 0, 0, 0, 0, 7, 430, 0, 318, 345] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[15, 30, 3762, 1, 4175, 131, 364, 865, 104, 9, 875, 402, 21, 333, 154, 300, 1068, 897, 208, 1797, 1333, 28, 1305, 882, 0, 737, 711, 0, 1878, 2132, 889, 2, 878, 194, 691, 480, 196, 198, 1588, 69, 14, 698, 6, 2964, 47, 162, 0, 36, 241, 42, 1620, 1589, 0, 3107, 0, 41, 1113, 91, 1166, 262, 409, 3674, 1753, 415] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 21, 0, 27, 72, 12, 5, 0, 0, 172, 1, 0, 61, 520, 0, 3787, 13, 145, 7645, 7578, 0, 14, 4, 0, 5, 625, 0, 7668, 10, 49, 0, 4244, 7, 5, 1, 254, 5742, 7494, 72, 0, 1, 0, 19, 3, 47, 0, 119, 0, 1315, 2, 143, 0, 24, 0, 0, 16, 0, 14, 43, 458, 27, 301, 367] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 73, 0, 0, 0, 0, 190, 0, 0, 70, 629, 0, 3946, 0, 115, 7687, 7622, 0, 0, 0, 0, 1, 581, 0, 7683, 0, 70, 0, 4429, 0, 0, 0, 240, 5056, 7532, 75, 0, 0, 0, 0, 0, 64, 0, 123, 0, 1657, 0, 163, 0, 0, 0, 0, 0, 0, 0, 2, 433, 0, 339, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[52, 0, 0, 0, 0, 22, 0, 0, 0, 0, 206, 0, 0, 53, 961, 0, 3106, 0, 179, 7539, 7797, 0, 0, 0, 0, 0, 1225, 0, 7693, 0, 53, 0, 6475, 0, 6, 0, 0, 3781, 7525, 90, 0, 0, 0, 0, 0, 48, 0, 72, 0, 852, 0, 212, 0, 0, 0, 0, 0, 0, 5, 68, 468, 0, 321, 343] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 69, 0, 0, 3493, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 64, 323, 0, 4017, 1567, 0, 0, 0, 0, 0, 0, 3847, 0, 78, 2605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 594, 0, 5292, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 0, 0, 0, 176, 6061, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 125, 6160, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 137, 0, 0, 0, 0, 3174, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 430, 0, 318, 345, 2416, 1600, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5533, 6146, 6530, 6005, 7671, 6311, 5840, 5116] avg=6144.0000 max=7671 max_rank=4 min=5116 min_rank=7 imbalance_ratio=1.248535 max_load_reduction=41481 max_load_reduction_pct=0.843933 moved_to_echo=24938 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[29, 0, 13, 0, 19, 47, 1, 1, 0, 0, 158, 6, 0, 63, 896, 0, 3285, 15, 153, 7586, 7793, 0, 10, 5, 0, 0, 925, 0, 7696, 1, 68, 0, 5658, 8, 11, 1, 2, 4610, 7468, 76, 0, 5, 0, 16, 3, 61, 0, 63, 0, 983, 0, 210, 0, 17, 0, 0, 2, 0, 21, 25, 448, 18, 344, 332] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 21, 0, 27, 72, 12, 5, 0, 0, 172, 1, 0, 61, 520, 0, 3787, 13, 145, 7645, 7578, 0, 14, 4, 0, 5, 625, 0, 7668, 10, 49, 0, 4244, 7, 5, 1, 254, 5742, 7494, 72, 0, 1, 0, 19, 3, 47, 0, 119, 0, 1315, 2, 143, 0, 24, 0, 0, 16, 0, 14, 43, 458, 27, 301, 367] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[80, 0, 0, 0, 0, 0, 0, 0, 3473, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 68, 969, 0, 3991, 1596, 0, 0, 0, 0, 0, 0, 3397, 0, 160, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 0, 5252, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 5, 0, 0, 2986, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 59, 6273, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 177, 0, 0, 0, 0, 3127, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 468, 0, 358, 380, 2398, 2616, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5485, 6820, 6124, 6717, 5294, 6400, 5932, 6380] avg=6144.0000 max=6820 max_rank=1 min=5294 min_rank=4 imbalance_ratio=1.110026 max_load_reduction=42332 max_load_reduction_pct=0.861247 moved_to_echo=24006 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 13, 0, 19, 47, 1, 1, 3474, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 158, 6, 0, 63, 896, 0, 3994, 1581, 0, 0, 0, 0, 0, 0, 3285, 15, 153, 2553, 0, 0, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 925, 0, 5284, 1, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1871, 8, 11, 1, 2, 4610, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 16, 3, 61, 0, 63, 6212, 0, 0, 0, 0, 0, 0, 0, 0, 983, 0, 210, 0, 17, 0, 0, 3111, 1585, 0, 0, 0, 0, 0, 0, 2, 0, 21, 25, 448, 18, 344, 332, 2412, 2202, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5506, 6698, 6021, 6278, 6579, 6360, 5906, 5804] avg=6144.0000 max=6698 max_rank=1 min=5506 min_rank=0 imbalance_ratio=1.090169 max_load_reduction=42454 max_load_reduction_pct=0.863729 moved_to_echo=24649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 92, 0, 0, 3506, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 55, 728, 0, 4033, 1544, 0, 0, 0, 0, 0, 0, 3754, 0, 198, 2533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 5269, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 0, 0, 0, 148, 5399, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 136, 6073, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 0, 227, 0, 0, 0, 0, 3086, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 454, 0, 336, 362, 2406, 1778, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5512, 6581, 6485, 5954, 7158, 6264, 5853, 5345] avg=6144.0000 max=7158 max_rank=4 min=5345 min_rank=7 imbalance_ratio=1.165039 max_load_reduction=41994 max_load_reduction_pct=0.854370 moved_to_echo=24620 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[15, 30, 3762, 1, 4175, 131, 364, 865, 739, 455, 0, 0, 0, 0, 0, 0, 104, 9, 875, 402, 21, 333, 154, 300, 849, 271, 0, 0, 0, 0, 0, 0, 1068, 897, 208, 605, 0, 28, 1305, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 737, 711, 0, 1290, 2132, 889, 2, 0, 0, 0, 0, 0, 0, 0, 0, 291, 194, 691, 480, 196, 198, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 14, 698, 6, 2964, 47, 162, 0, 36, 1062, 0, 0, 0, 0, 0, 0, 0, 241, 42, 1620, 1589, 0, 3107, 0, 41, 737, 246, 0, 0, 0, 0, 0, 0, 1113, 91, 1166, 262, 409, 3674, 1753, 415, 588, 341, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10537, 3318, 4993, 5761, 2119, 4989, 7623, 9812] avg=6144.0000 max=10537 max_rank=0 min=2119 min_rank=4 imbalance_ratio=1.715007 max_load_reduction=38615 max_load_reduction_pct=0.785624 moved_to_echo=24543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 73, 0, 0, 3503, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 70, 629, 0, 4029, 1545, 0, 0, 0, 0, 0, 0, 3946, 0, 115, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 581, 0, 5275, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 0, 0, 0, 240, 5056, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 123, 6077, 0, 0, 0, 0, 0, 0, 0, 0, 1657, 0, 163, 0, 0, 0, 0, 3153, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 433, 0, 339, 364, 2408, 1724, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5531, 6463, 6648, 5927, 6835, 6264, 6214, 5270] avg=6144.0000 max=6835 max_rank=4 min=5270 min_rank=7 imbalance_ratio=1.112467 max_load_reduction=42317 max_load_reduction_pct=0.860942 moved_to_echo=24583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 21, 0, 27, 72, 12, 5, 3486, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1, 0, 61, 520, 0, 4008, 1538, 0, 0, 0, 0, 0, 0, 3787, 13, 145, 2566, 0, 0, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 625, 0, 5261, 10, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 7, 5, 1, 254, 5742, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 3, 47, 0, 119, 6040, 0, 0, 0, 0, 0, 0, 0, 0, 1315, 2, 143, 0, 24, 0, 0, 3139, 1191, 0, 0, 0, 0, 0, 0, 16, 0, 14, 43, 458, 27, 301, 367, 2407, 1656, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5563, 6300, 6529, 5950, 7478, 6229, 5814, 5289] avg=6144.0000 max=7478 max_rank=4 min=5289 min_rank=7 imbalance_ratio=1.217122 max_load_reduction=41674 max_load_reduction_pct=0.847860 moved_to_echo=24810 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 69, 0, 0, 3493, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 64, 323, 0, 4017, 1567, 0, 0, 0, 0, 0, 0, 3847, 0, 78, 2605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 594, 0, 5292, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 0, 0, 0, 176, 6061, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 125, 6160, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 137, 0, 0, 0, 0, 3174, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 430, 0, 318, 345, 2416, 1600, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5533, 6146, 6530, 6005, 7671, 6311, 5840, 5116] avg=6144.0000 max=7671 max_rank=4 min=5116 min_rank=7 imbalance_ratio=1.248535 max_load_reduction=41481 max_load_reduction_pct=0.843933 moved_to_echo=24938 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[15, 30, 3762, 1, 4175, 131, 364, 865, 739, 455, 0, 0, 0, 0, 0, 0, 104, 9, 875, 402, 21, 333, 154, 300, 849, 271, 0, 0, 0, 0, 0, 0, 1068, 897, 208, 605, 0, 28, 1305, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 737, 711, 0, 1290, 2132, 889, 2, 0, 0, 0, 0, 0, 0, 0, 0, 291, 194, 691, 480, 196, 198, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 14, 698, 6, 2964, 47, 162, 0, 36, 1062, 0, 0, 0, 0, 0, 0, 0, 241, 42, 1620, 1589, 0, 3107, 0, 41, 737, 246, 0, 0, 0, 0, 0, 0, 1113, 91, 1166, 262, 409, 3674, 1753, 415, 588, 341, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10537, 3318, 4993, 5761, 2119, 4989, 7623, 9812] avg=6144.0000 max=10537 max_rank=0 min=2119 min_rank=4 imbalance_ratio=1.715007 max_load_reduction=38615 max_load_reduction_pct=0.785624 moved_to_echo=24543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[52, 0, 0, 0, 0, 22, 0, 0, 3501, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 53, 961, 0, 4024, 1582, 0, 0, 0, 0, 0, 0, 3106, 0, 179, 2537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 5282, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2141, 0, 6, 0, 0, 3781, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 72, 6215, 0, 0, 0, 0, 0, 0, 0, 0, 852, 0, 212, 0, 0, 0, 0, 3092, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 5, 68, 468, 0, 321, 343, 2411, 2520, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5485, 6826, 5822, 6560, 6018, 6335, 5970, 6136] avg=6144.0000 max=6826 max_rank=1 min=5485 min_rank=0 imbalance_ratio=1.111003 max_load_reduction=42326 max_load_reduction_pct=0.861125 moved_to_echo=24459 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 13, 0, 19, 47, 1, 1, 3474, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 158, 6, 0, 63, 896, 0, 3994, 1581, 0, 0, 0, 0, 0, 0, 3285, 15, 153, 2553, 0, 0, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 925, 0, 5284, 1, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1871, 8, 11, 1, 2, 4610, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 16, 3, 61, 0, 63, 6212, 0, 0, 0, 0, 0, 0, 0, 0, 983, 0, 210, 0, 17, 0, 0, 3111, 1585, 0, 0, 0, 0, 0, 0, 2, 0, 21, 25, 448, 18, 344, 332, 2412, 2202, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5506, 6698, 6021, 6278, 6579, 6360, 5906, 5804] avg=6144.0000 max=6698 max_rank=1 min=5506 min_rank=0 imbalance_ratio=1.090169 max_load_reduction=42454 max_load_reduction_pct=0.863729 moved_to_echo=24649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 21, 0, 27, 72, 12, 5, 3486, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1, 0, 61, 520, 0, 4008, 1538, 0, 0, 0, 0, 0, 0, 3787, 13, 145, 2566, 0, 0, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 625, 0, 5261, 10, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 7, 5, 1, 254, 5742, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 3, 47, 0, 119, 6040, 0, 0, 0, 0, 0, 0, 0, 0, 1315, 2, 143, 0, 24, 0, 0, 3139, 1191, 0, 0, 0, 0, 0, 0, 16, 0, 14, 43, 458, 27, 301, 367, 2407, 1656, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5563, 6300, 6529, 5950, 7478, 6229, 5814, 5289] avg=6144.0000 max=7478 max_rank=4 min=5289 min_rank=7 imbalance_ratio=1.217122 max_load_reduction=41674 max_load_reduction_pct=0.847860 moved_to_echo=24810 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 128, 3840, 128, 4224, 512, 384, 896, 25216, 14080, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[26240, 1024, 1280, 18560, 0, 128, 1408, 896, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[12288, 256, 768, 512, 1024, 33920, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1152, 128, 1280, 640, 3584, 3840, 4096, 2944, 17536, 14464, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 8320, 1664, 2944, 0, 3200, 0, 128, 22656, 10496, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 2304, 512, 128, 768, 5248, 384, 29056, 11264, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 896, 6784, 0, 38272, 2176, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 2304, 512, 128, 768, 5248, 384, 29056, 11264, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 768, 128, 3072, 128, 640, 0, 768, 44160, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2219, 0, 0, 0, 0, 0, 0, 6, 74, 0, 74, 1774, 0, 0, 0, 3070, 1717, 8115, 825, 0, 74, 7636, 0, 0, 7518, 466, 64, 0, 0, 0, 0, 73, 0, 32, 0, 0, 1941, 0, 1267, 0, 0, 0, 0, 71, 0, 0, 0, 0, 2843, 4784, 0, 662, 85, 1401, 73, 0, 0, 0, 2136, 152, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2241, 0, 0, 0, 0, 0, 0, 0, 80, 0, 80, 1814, 0, 0, 0, 1500, 1536, 8085, 709, 0, 80, 7584, 0, 0, 7434, 257, 68, 0, 0, 0, 0, 94, 0, 0, 0, 0, 2496, 0, 2551, 0, 0, 0, 0, 77, 0, 0, 0, 0, 1775, 5774, 0, 1562, 151, 2111, 80, 0, 0, 0, 912, 98, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2305, 0, 0, 0, 0, 0, 0, 5, 75, 0, 75, 1587, 0, 0, 0, 3054, 1931, 8117, 906, 0, 75, 7613, 0, 0, 7518, 526, 70, 0, 0, 0, 0, 72, 0, 39, 0, 0, 1696, 0, 1144, 0, 0, 0, 0, 74, 0, 0, 0, 0, 2527, 5018, 0, 698, 113, 1296, 74, 0, 0, 0, 2364, 180, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2163, 0, 0, 0, 0, 0, 0, 0, 100, 0, 100, 1642, 0, 0, 0, 2796, 1738, 8092, 903, 0, 100, 7576, 0, 0, 7477, 550, 55, 0, 0, 0, 0, 99, 0, 6, 0, 0, 2290, 0, 1543, 0, 0, 0, 0, 96, 0, 0, 0, 0, 2544, 5069, 0, 794, 211, 1475, 99, 0, 0, 0, 1490, 141, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[152, 8, 2599, 2526, 179, 774, 21, 0, 39, 89, 366, 30, 1816, 0, 607, 1079, 90, 1137, 74, 279, 512, 379, 43, 92, 1936, 673, 23, 557, 268, 88, 88, 71, 0, 2875, 795, 4325, 58, 1, 1, 942, 2391, 158, 911, 3556, 45, 0, 110, 0, 465, 753, 526, 1022, 3548, 695, 2215, 373, 1050, 403, 409, 36, 4184, 137, 298, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 15, 2309, 3, 17, 0, 0, 0, 0, 0, 76, 7, 75, 1959, 10, 0, 0, 2208, 1559, 8053, 738, 4, 76, 7604, 8, 0, 7396, 429, 66, 0, 0, 0, 16, 115, 17, 0, 0, 0, 2694, 10, 1925, 0, 18, 1, 0, 71, 0, 4, 0, 3, 2079, 5465, 2, 1276, 185, 1426, 78, 10, 0, 18, 1069, 53, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 18, 2302, 7, 20, 0, 0, 1, 0, 7, 72, 10, 72, 1692, 5, 0, 0, 2486, 1783, 8019, 953, 4, 72, 7594, 13, 0, 7455, 484, 47, 0, 0, 0, 22, 81, 30, 26, 0, 0, 2292, 13, 1286, 3, 25, 1, 0, 73, 0, 0, 4, 11, 2641, 5237, 2, 620, 88, 1322, 74, 0, 0, 27, 2018, 137, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2240, 0, 0, 0, 0, 0, 0, 0, 89, 0, 88, 1875, 0, 0, 0, 1764, 1746, 8065, 732, 0, 87, 7617, 0, 0, 7496, 392, 53, 0, 0, 0, 0, 109, 0, 0, 0, 0, 2787, 0, 2240, 0, 0, 0, 0, 75, 0, 0, 0, 0, 1850, 5462, 0, 1327, 190, 1816, 89, 0, 0, 0, 878, 76, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2163, 0, 0, 0, 0, 0, 0, 0, 100, 0, 100, 1642, 0, 0, 0, 2796, 1738, 8092, 903, 0, 100, 7576, 0, 0, 7477, 550, 55, 0, 0, 0, 0, 99, 0, 6, 0, 0, 2290, 0, 1543, 0, 0, 0, 0, 96, 0, 0, 0, 0, 2544, 5069, 0, 794, 211, 1475, 99, 0, 0, 0, 1490, 141, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 18, 2302, 7, 20, 0, 0, 1, 0, 7, 72, 10, 72, 1692, 5, 0, 0, 2486, 1783, 8019, 953, 4, 72, 7594, 13, 0, 7455, 484, 47, 0, 0, 0, 22, 81, 30, 26, 0, 0, 2292, 13, 1286, 3, 25, 1, 0, 73, 0, 0, 4, 11, 2641, 5237, 2, 620, 88, 1322, 74, 0, 0, 27, 2018, 137, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 15, 2309, 3, 17, 0, 0, 0, 0, 0, 76, 7, 75, 1959, 10, 0, 0, 2208, 1559, 8053, 738, 4, 76, 7604, 8, 0, 7396, 429, 66, 0, 0, 0, 16, 115, 17, 0, 0, 0, 2694, 10, 1925, 0, 18, 1, 0, 71, 0, 4, 0, 3, 2079, 5465, 2, 1276, 185, 1426, 78, 10, 0, 18, 1069, 53, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2305, 0, 0, 0, 0, 0, 0, 5, 75, 0, 75, 1587, 0, 0, 0, 3054, 1931, 8117, 906, 0, 75, 7613, 0, 0, 7518, 526, 70, 0, 0, 0, 0, 72, 0, 39, 0, 0, 1696, 0, 1144, 0, 0, 0, 0, 74, 0, 0, 0, 0, 2527, 5018, 0, 698, 113, 1296, 74, 0, 0, 0, 2364, 180, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2219, 0, 0, 0, 0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 74, 0, 74, 1774, 0, 4374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3070, 1717, 1684, 825, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6315, 466, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 32, 0, 0, 1941, 1754, 1279, 0, 0, 0, 0, 0, 0, 0, 1267, 0, 0, 0, 0, 71, 0, 3262, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2843, 2236, 0, 662, 85, 0, 0, 0, 0, 0, 0, 0, 0, 1401, 73, 0, 0, 0, 2136, 152, 0, 1269, 1203, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6302, 7370, 6845, 5079, 5433, 5826, 6234] avg=6144.0000 max=7370 max_rank=2 min=5079 min_rank=4 imbalance_ratio=1.199544 max_load_reduction=41782 max_load_reduction_pct=0.850057 moved_to_echo=22572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[152, 8, 2599, 2526, 179, 774, 21, 0, 242, 0, 0, 0, 0, 0, 0, 0, 39, 89, 366, 30, 1816, 0, 607, 1079, 1110, 0, 0, 0, 0, 0, 0, 0, 90, 1137, 74, 279, 108, 379, 43, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 23, 468, 268, 88, 88, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2875, 795, 4325, 58, 1, 1, 942, 110, 949, 0, 0, 0, 0, 0, 0, 2391, 158, 911, 3556, 45, 0, 110, 0, 826, 52, 0, 0, 0, 0, 0, 0, 465, 753, 526, 1022, 1658, 695, 2215, 373, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 403, 409, 36, 4184, 137, 298, 275, 941, 89, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6501, 5136, 2202, 1679, 10056, 8049, 7707, 7822] avg=6144.0000 max=10056 max_rank=4 min=1679 min_rank=3 imbalance_ratio=1.636719 max_load_reduction=39096 max_load_reduction_pct=0.795410 moved_to_echo=33634 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2241, 0, 0, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 80, 1814, 0, 4345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1500, 1536, 1678, 709, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6245, 257, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 2496, 1747, 1544, 0, 0, 0, 0, 0, 0, 0, 2551, 0, 0, 0, 0, 77, 0, 3239, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 2699, 0, 1562, 151, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 80, 0, 0, 0, 912, 98, 3, 1531, 1189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6071, 6319, 5503, 6570, 5881, 6697, 6187, 5924] avg=6144.0000 max=6697 max_rank=5 min=5503 min_rank=2 imbalance_ratio=1.090007 max_load_reduction=42455 max_load_reduction_pct=0.863749 moved_to_echo=24689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2163, 0, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 100, 1642, 0, 4340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2796, 1738, 1680, 903, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6281, 550, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 6, 0, 0, 2290, 1749, 1355, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 0, 0, 0, 96, 0, 3236, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2544, 2370, 0, 794, 211, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 99, 0, 0, 0, 1490, 141, 3, 1344, 1196, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6182, 7217, 6886, 5499, 5705, 5919, 5748] avg=6144.0000 max=7217 max_rank=2 min=5499 min_rank=4 imbalance_ratio=1.174642 max_load_reduction=41935 max_load_reduction_pct=0.853170 moved_to_echo=22871 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[152, 8, 2599, 2526, 179, 774, 21, 0, 242, 0, 0, 0, 0, 0, 0, 0, 39, 89, 366, 30, 1816, 0, 607, 1079, 1110, 0, 0, 0, 0, 0, 0, 0, 90, 1137, 74, 279, 108, 379, 43, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 23, 468, 268, 88, 88, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2875, 795, 4325, 58, 1, 1, 942, 110, 949, 0, 0, 0, 0, 0, 0, 2391, 158, 911, 3556, 45, 0, 110, 0, 826, 52, 0, 0, 0, 0, 0, 0, 465, 753, 526, 1022, 1658, 695, 2215, 373, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 403, 409, 36, 4184, 137, 298, 275, 941, 89, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6501, 5136, 2202, 1679, 10056, 8049, 7707, 7822] avg=6144.0000 max=10056 max_rank=4 min=1679 min_rank=3 imbalance_ratio=1.636719 max_load_reduction=39096 max_load_reduction_pct=0.795410 moved_to_echo=33634 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2240, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 88, 1875, 0, 4364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 1746, 1673, 732, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6297, 392, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 2787, 1743, 1461, 0, 0, 0, 0, 0, 0, 0, 2240, 0, 0, 0, 0, 75, 0, 3253, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 2553, 0, 1327, 190, 0, 0, 0, 0, 0, 0, 0, 0, 1816, 89, 0, 0, 0, 878, 76, 9, 1448, 1199, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6061, 6416, 6002, 6742, 6100, 6396, 5920, 5515] avg=6144.0000 max=6742 max_rank=3 min=5515 min_rank=7 imbalance_ratio=1.097331 max_load_reduction=42410 max_load_reduction_pct=0.862834 moved_to_echo=23931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 2309, 3, 17, 0, 0, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 7, 75, 1959, 10, 4357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2208, 1559, 1672, 738, 4, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6213, 429, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 115, 17, 0, 0, 0, 2694, 1740, 1461, 0, 0, 0, 0, 0, 0, 10, 1925, 0, 18, 1, 0, 71, 0, 3247, 826, 0, 0, 0, 0, 0, 0, 4, 0, 3, 2079, 2555, 2, 1276, 185, 0, 0, 0, 0, 0, 0, 0, 0, 1426, 78, 10, 0, 18, 1069, 53, 5, 1449, 1183, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 6484, 6257, 6716, 6043, 6098, 6104, 5291] avg=6144.0000 max=6716 max_rank=3 min=5291 min_rank=7 imbalance_ratio=1.093099 max_load_reduction=42436 max_load_reduction_pct=0.863363 moved_to_echo=23536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2305, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 75, 0, 75, 1587, 0, 4361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3054, 1931, 1685, 906, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6315, 526, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 39, 0, 0, 1696, 1754, 1342, 0, 0, 0, 0, 0, 0, 0, 1144, 0, 0, 0, 0, 74, 0, 3252, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 2345, 0, 698, 113, 0, 0, 0, 0, 0, 0, 0, 0, 1296, 74, 0, 0, 0, 2364, 180, 0, 1331, 1203, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6103, 7651, 6911, 4903, 5303, 5683, 6448] avg=6144.0000 max=7651 max_rank=2 min=4903 min_rank=4 imbalance_ratio=1.245280 max_load_reduction=41501 max_load_reduction_pct=0.844340 moved_to_echo=22337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2163, 0, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 100, 1642, 0, 4340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2796, 1738, 1680, 903, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6281, 550, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 6, 0, 0, 2290, 1749, 1355, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 0, 0, 0, 96, 0, 3236, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2544, 2370, 0, 794, 211, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 99, 0, 0, 0, 1490, 141, 3, 1344, 1196, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6182, 7217, 6886, 5499, 5705, 5919, 5748] avg=6144.0000 max=7217 max_rank=2 min=5499 min_rank=4 imbalance_ratio=1.174642 max_load_reduction=41935 max_load_reduction_pct=0.853170 moved_to_echo=22871 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 2302, 7, 20, 0, 0, 3804, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 72, 10, 72, 1692, 5, 4351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 1783, 1652, 953, 4, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 6259, 484, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 81, 30, 26, 0, 0, 2292, 1736, 1404, 0, 0, 0, 0, 0, 0, 13, 1286, 3, 25, 1, 0, 73, 0, 3243, 827, 0, 0, 0, 0, 0, 0, 0, 4, 11, 2641, 2440, 2, 620, 88, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 74, 0, 0, 27, 2018, 137, 3, 1393, 1196, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 6210, 6950, 6803, 5591, 5471, 5806, 6170] avg=6144.0000 max=6950 max_rank=2 min=5471 min_rank=5 imbalance_ratio=1.131185 max_load_reduction=42202 max_load_reduction_pct=0.858602 moved_to_echo=23038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2240, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 88, 1875, 0, 4364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 1746, 1673, 732, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6297, 392, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 2787, 1743, 1461, 0, 0, 0, 0, 0, 0, 0, 2240, 0, 0, 0, 0, 75, 0, 3253, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 2553, 0, 1327, 190, 0, 0, 0, 0, 0, 0, 0, 0, 1816, 89, 0, 0, 0, 878, 76, 9, 1448, 1199, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6061, 6416, 6002, 6742, 6100, 6396, 5920, 5515] avg=6144.0000 max=6742 max_rank=3 min=5515 min_rank=7 imbalance_ratio=1.097331 max_load_reduction=42410 max_load_reduction_pct=0.862834 moved_to_echo=23931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 768, 640, 17408, 18944, 768, 9216, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2432, 12160, 1024, 3712, 128, 0, 768, 0, 23680, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 768, 640, 17408, 18944, 768, 9216, 1408, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1152, 17024, 12416, 11904, 6272, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11904, 1024, 512, 128, 4352, 11008, 1152, 384, 10752, 8576, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 2944, 1536, 4480, 256, 128, 128, 17152, 12416, 10880, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 2688, 18432, 256, 896, 128, 0, 27136, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11904, 1024, 512, 128, 4352, 11008, 1152, 384, 10752, 8576, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 384, 640, 1920, 640, 13056, 1152, 31616, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 768, 128, 44416, 3456, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 6014, 529, 1333, 7555, 4937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 443, 0, 0, 0, 0, 260, 2647, 73, 0, 0, 0, 0, 0, 410, 0, 0, 6514, 4443, 149, 0, 0, 722, 0, 0, 0, 0, 0, 0, 1212, 1, 0, 0, 0, 85, 7740, 135, 3732, 0, 0, 73, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 4671, 580, 1219, 7807, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 596, 0, 0, 0, 0, 281, 4905, 80, 0, 0, 0, 0, 0, 783, 0, 0, 5509, 6402, 164, 0, 0, 651, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 0, 97, 7082, 80, 4377, 0, 0, 80, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 5642, 528, 1206, 7435, 5207, 4, 0, 1, 0, 0, 2, 3, 4, 11, 10, 26, 8, 156, 0, 12, 495, 0, 0, 0, 0, 246, 3036, 72, 0, 17, 0, 0, 1, 597, 0, 0, 6485, 4073, 111, 2, 0, 674, 0, 16, 2, 2, 16, 14, 1290, 7, 0, 0, 30, 65, 7723, 148, 3662, 0, 27, 72, 1, 0, 13] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 5435, 685, 1325, 7507, 4256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 476, 0, 0, 0, 0, 330, 3262, 99, 0, 0, 0, 0, 0, 571, 0, 0, 6189, 4798, 122, 0, 0, 738, 0, 0, 0, 0, 0, 0, 1221, 0, 0, 0, 0, 67, 7500, 145, 4203, 0, 0, 99, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[1, 381, 212, 466, 773, 1868, 195, 0, 137, 211, 350, 141, 330, 1512, 1178, 1194, 4005, 873, 1, 27, 3677, 827, 25, 3, 59, 0, 443, 1396, 83, 20, 1478, 162, 251, 135, 325, 64, 26, 1277, 518, 18, 1068, 79, 279, 471, 2446, 345, 153, 2077, 1923, 557, 896, 0, 64, 4688, 245, 924, 141, 1557, 258, 3948, 43, 236, 258, 1854] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 4835, 592, 1211, 7624, 2809, 5, 0, 1, 0, 6, 1, 3, 0, 12, 9, 19, 1, 199, 0, 9, 622, 0, 0, 0, 0, 328, 4171, 76, 0, 16, 0, 0, 0, 788, 0, 0, 5898, 5787, 149, 1, 0, 690, 0, 14, 0, 2, 24, 0, 1033, 22, 0, 0, 21, 88, 7313, 87, 4584, 5, 15, 76, 1, 1, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 5930, 568, 1068, 7459, 5570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 503, 0, 0, 0, 0, 333, 3616, 74, 0, 0, 0, 0, 0, 499, 0, 0, 6419, 3720, 154, 0, 0, 683, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 84, 7646, 140, 3148, 0, 0, 74, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 4671, 580, 1219, 7807, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 596, 0, 0, 0, 0, 281, 4905, 80, 0, 0, 0, 0, 0, 783, 0, 0, 5509, 6402, 164, 0, 0, 651, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 0, 97, 7082, 80, 4377, 0, 0, 80, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 4867, 655, 1288, 7752, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 596, 0, 0, 0, 0, 307, 4513, 88, 0, 0, 0, 0, 0, 599, 0, 0, 5627, 6135, 127, 0, 0, 658, 0, 0, 0, 0, 0, 0, 1068, 0, 0, 0, 0, 74, 7171, 89, 4369, 0, 0, 88, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 854, 529, 1333, 0, 4937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 443, 0, 0, 4061, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 260, 2647, 73, 0, 0, 0, 1425, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 1074, 4443, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 4073, 1384, 0, 0, 0, 0, 0, 0, 0, 1212, 1, 0, 0, 0, 85, 4664, 0, 0, 0, 0, 0, 0, 0, 0, 135, 3732, 0, 0, 73, 0, 0, 0, 1651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7653, 6171, 6016, 5504, 6076, 6179, 5962, 5591] avg=6144.0000 max=7653 max_rank=0 min=5504 min_rank=3 imbalance_ratio=1.245605 max_load_reduction=41499 max_load_reduction_pct=0.844299 moved_to_echo=23808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 6014, 529, 1333, 7555, 4937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 443, 0, 0, 0, 0, 260, 2647, 73, 0, 0, 0, 0, 0, 410, 0, 0, 6514, 4443, 149, 0, 0, 722, 0, 0, 0, 0, 0, 0, 1212, 1, 0, 0, 0, 85, 7740, 135, 3732, 0, 0, 73, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 5930, 568, 1068, 7459, 5570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 503, 0, 0, 0, 0, 333, 3616, 74, 0, 0, 0, 0, 0, 499, 0, 0, 6419, 3720, 154, 0, 0, 683, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 84, 7646, 140, 3148, 0, 0, 74, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 5642, 528, 1206, 7435, 5207, 4, 0, 1, 0, 0, 2, 3, 4, 11, 10, 26, 8, 156, 0, 12, 495, 0, 0, 0, 0, 246, 3036, 72, 0, 17, 0, 0, 1, 597, 0, 0, 6485, 4073, 111, 2, 0, 674, 0, 16, 2, 2, 16, 14, 1290, 7, 0, 0, 30, 65, 7723, 148, 3662, 0, 27, 72, 1, 0, 13] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 4835, 592, 1211, 7624, 2809, 5, 0, 1, 0, 6, 1, 3, 0, 12, 9, 19, 1, 199, 0, 9, 622, 0, 0, 0, 0, 328, 4171, 76, 0, 16, 0, 0, 0, 788, 0, 0, 5898, 5787, 149, 1, 0, 690, 0, 14, 0, 2, 24, 0, 1033, 22, 0, 0, 21, 88, 7313, 87, 4584, 5, 15, 76, 1, 1, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 772, 685, 1325, 0, 4256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 476, 0, 0, 3670, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 330, 3262, 99, 0, 0, 0, 1380, 993, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 1020, 4798, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 0, 0, 0, 0, 0, 3870, 1376, 0, 0, 0, 0, 0, 0, 0, 1221, 0, 0, 0, 0, 67, 4520, 0, 0, 0, 0, 0, 0, 0, 0, 145, 4203, 0, 0, 99, 0, 0, 0, 1600, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7038, 6131, 5569, 6064, 6511, 5984, 5808, 6047] avg=6144.0000 max=7038 max_rank=0 min=5569 min_rank=2 imbalance_ratio=1.145508 max_load_reduction=42114 max_load_reduction_pct=0.856812 moved_to_echo=24350 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 843, 568, 1068, 0, 5570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 503, 0, 0, 4004, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 333, 3616, 74, 0, 0, 0, 1407, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 499, 0, 0, 1059, 3720, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 0, 0, 0, 0, 0, 4013, 1367, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 84, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 140, 3148, 0, 0, 74, 0, 0, 0, 1631, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 6092, 6001, 6513, 5432, 6063, 6009, 4993] avg=6144.0000 max=8049 max_rank=0 min=4993 min_rank=7 imbalance_ratio=1.310059 max_load_reduction=41103 max_load_reduction_pct=0.836243 moved_to_echo=22497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 664, 580, 1219, 0, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 596, 0, 0, 3154, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 281, 4905, 80, 0, 0, 0, 1303, 853, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 0, 909, 6402, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 0, 0, 3444, 1431, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 0, 97, 4269, 0, 0, 0, 0, 0, 0, 0, 0, 80, 4377, 0, 0, 80, 0, 0, 0, 1510, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5054, 6376, 5113, 7422, 8258, 5526, 5356, 6047] avg=6144.0000 max=8258 max_rank=4 min=5054 min_rank=0 imbalance_ratio=1.344076 max_load_reduction=40894 max_load_reduction_pct=0.831991 moved_to_echo=25187 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 854, 529, 1333, 0, 4937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 443, 0, 0, 4061, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 260, 2647, 73, 0, 0, 0, 1425, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 1074, 4443, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 4073, 1384, 0, 0, 0, 0, 0, 0, 0, 1212, 1, 0, 0, 0, 85, 4664, 0, 0, 0, 0, 0, 0, 0, 0, 135, 3732, 0, 0, 73, 0, 0, 0, 1651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7653, 6171, 6016, 5504, 6076, 6179, 5962, 5591] avg=6144.0000 max=7653 max_rank=0 min=5504 min_rank=3 imbalance_ratio=1.245605 max_load_reduction=41499 max_load_reduction_pct=0.844299 moved_to_echo=23808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1, 55, 212, 466, 0, 1868, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 211, 350, 141, 330, 1512, 1178, 1194, 632, 0, 0, 0, 0, 0, 0, 0, 4005, 873, 1, 27, 3677, 827, 25, 3, 257, 268, 0, 0, 0, 0, 0, 0, 59, 0, 443, 1396, 83, 20, 1478, 162, 170, 69, 0, 0, 0, 0, 0, 0, 251, 135, 325, 64, 26, 211, 518, 18, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 79, 279, 471, 2446, 345, 153, 2077, 798, 141, 0, 0, 0, 0, 0, 0, 1923, 557, 896, 0, 64, 4688, 245, 557, 0, 0, 0, 0, 0, 0, 0, 0, 141, 1557, 258, 3948, 43, 236, 258, 1854, 197, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2797, 5685, 9963, 3880, 1548, 7857, 8930, 8492] avg=6144.0000 max=9963 max_rank=2 min=1548 min_rank=4 imbalance_ratio=1.621582 max_load_reduction=39189 max_load_reduction_pct=0.797302 moved_to_echo=26827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 772, 685, 1325, 0, 4256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 476, 0, 0, 3670, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 330, 3262, 99, 0, 0, 0, 1380, 993, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 1020, 4798, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 0, 0, 0, 0, 0, 3870, 1376, 0, 0, 0, 0, 0, 0, 0, 1221, 0, 0, 0, 0, 67, 4520, 0, 0, 0, 0, 0, 0, 0, 0, 145, 4203, 0, 0, 99, 0, 0, 0, 1600, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7038, 6131, 5569, 6064, 6511, 5984, 5808, 6047] avg=6144.0000 max=7038 max_rank=0 min=5569 min_rank=2 imbalance_ratio=1.145508 max_load_reduction=42114 max_load_reduction_pct=0.856812 moved_to_echo=24350 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 692, 655, 1288, 0, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 596, 0, 0, 3286, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 307, 4513, 88, 0, 0, 0, 1320, 889, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 928, 6135, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 0, 0, 0, 0, 0, 3518, 1420, 0, 0, 0, 0, 0, 0, 0, 1068, 0, 0, 0, 0, 74, 4322, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4369, 0, 0, 88, 0, 0, 0, 1529, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5516, 6332, 5263, 7117, 7789, 5596, 5464, 6075] avg=6144.0000 max=7789 max_rank=4 min=5263 min_rank=2 imbalance_ratio=1.267741 max_load_reduction=41363 max_load_reduction_pct=0.841532 moved_to_echo=24924 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 688, 592, 1211, 0, 2809, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 1, 3, 0, 12, 9, 6228, 0, 0, 0, 0, 0, 0, 0, 19, 1, 199, 0, 9, 622, 0, 0, 3264, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 328, 4171, 76, 0, 16, 0, 1346, 883, 0, 0, 0, 0, 0, 0, 0, 0, 788, 0, 0, 972, 5787, 149, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 690, 0, 14, 0, 2, 24, 3688, 1396, 0, 0, 0, 0, 0, 0, 0, 1033, 22, 0, 0, 21, 88, 4407, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4584, 5, 15, 76, 1, 1, 4, 1560, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5305, 6260, 5352, 6820, 7696, 5815, 5571, 6333] avg=6144.0000 max=7696 max_rank=4 min=5305 min_rank=0 imbalance_ratio=1.252604 max_load_reduction=41456 max_load_reduction_pct=0.843424 moved_to_echo=25415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 795, 528, 1206, 0, 5207, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 3, 4, 11, 10, 6074, 0, 0, 0, 0, 0, 0, 0, 26, 8, 156, 0, 12, 495, 0, 0, 3812, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 246, 3036, 72, 0, 17, 0, 1426, 1035, 0, 0, 0, 0, 0, 0, 0, 1, 597, 0, 0, 1060, 4073, 111, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 674, 0, 16, 2, 2, 16, 4059, 1361, 0, 0, 0, 0, 0, 0, 14, 1290, 7, 0, 0, 30, 65, 4646, 0, 0, 0, 0, 0, 0, 0, 0, 148, 3662, 0, 27, 72, 1, 0, 13, 1651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7740, 6105, 5875, 5832, 5842, 6132, 6052, 5574] avg=6144.0000 max=7740 max_rank=0 min=5574 min_rank=7 imbalance_ratio=1.259766 max_load_reduction=41412 max_load_reduction_pct=0.842529 moved_to_echo=23600 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 843, 568, 1068, 0, 5570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 503, 0, 0, 4004, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 333, 3616, 74, 0, 0, 0, 1407, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 499, 0, 0, 1059, 3720, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 0, 0, 0, 0, 0, 4013, 1367, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 84, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 140, 3148, 0, 0, 74, 0, 0, 0, 1631, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 6092, 6001, 6513, 5432, 6063, 6009, 4993] avg=6144.0000 max=8049 max_rank=0 min=4993 min_rank=7 imbalance_ratio=1.310059 max_load_reduction=41103 max_load_reduction_pct=0.836243 moved_to_echo=22497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 692, 655, 1288, 0, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 596, 0, 0, 3286, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 307, 4513, 88, 0, 0, 0, 1320, 889, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 928, 6135, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 0, 0, 0, 0, 0, 3518, 1420, 0, 0, 0, 0, 0, 0, 0, 1068, 0, 0, 0, 0, 74, 4322, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4369, 0, 0, 88, 0, 0, 0, 1529, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5516, 6332, 5263, 7117, 7789, 5596, 5464, 6075] avg=6144.0000 max=7789 max_rank=4 min=5263 min_rank=2 imbalance_ratio=1.267741 max_load_reduction=41363 max_load_reduction_pct=0.841532 moved_to_echo=24924 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4096, 896, 1280, 128, 3712, 4608, 128, 128, 25600, 9344, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 5376, 4352, 9216, 0, 30208, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 2560, 27648, 768, 128, 1536, 256, 9856, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 29696, 384, 4096, 640, 256, 384, 1920, 11392, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 256, 4608, 128, 128, 7296, 35968, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 29696, 384, 4096, 640, 256, 384, 1920, 11392, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 8704, 1024, 0, 128, 4864, 896, 32000, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 128, 5120, 512, 2560, 384, 256, 2176, 27520, 9984, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 256, 384, 256, 384, 1536, 1280, 1280, 44160, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[6390, 0, 0, 0, 0, 0, 0, 2230, 547, 0, 148, 0, 0, 0, 5, 2316, 0, 0, 0, 6282, 241, 688, 0, 766, 0, 0, 0, 0, 0, 0, 0, 0, 2333, 0, 0, 0, 0, 0, 0, 6633, 0, 0, 0, 7642, 0, 0, 144, 0, 0, 0, 0, 1712, 0, 8029, 0, 0, 0, 0, 0, 2814, 158, 0, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[6859, 0, 0, 0, 0, 0, 0, 1909, 544, 0, 182, 0, 0, 0, 6, 2464, 0, 0, 0, 5799, 244, 611, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 6666, 0, 0, 0, 7502, 0, 0, 129, 0, 0, 0, 0, 2562, 0, 7916, 0, 0, 0, 0, 0, 2973, 166, 0, 0, 100] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[7658, 0, 0, 0, 0, 0, 0, 1836, 584, 0, 285, 0, 0, 0, 4, 1746, 0, 0, 0, 4339, 164, 414, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 6957, 0, 0, 0, 7310, 0, 0, 184, 0, 0, 0, 0, 3816, 0, 7784, 0, 0, 0, 0, 0, 3817, 176, 0, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[6442, 0, 0, 0, 0, 0, 0, 2398, 522, 0, 116, 0, 0, 0, 0, 2438, 0, 0, 0, 6353, 256, 633, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 0, 0, 0, 6678, 0, 0, 0, 7552, 0, 0, 144, 0, 0, 0, 0, 1368, 0, 8034, 0, 0, 0, 0, 0, 3340, 158, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[6859, 0, 0, 0, 0, 0, 0, 1909, 544, 0, 182, 0, 0, 0, 6, 2464, 0, 0, 0, 5799, 244, 611, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 6666, 0, 0, 0, 7502, 0, 0, 129, 0, 0, 0, 0, 2562, 0, 7916, 0, 0, 0, 0, 0, 2973, 166, 0, 0, 100] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[7415, 18, 0, 1, 0, 0, 1, 1866, 538, 21, 349, 1, 0, 0, 41, 2103, 1, 7, 1, 4611, 155, 449, 1, 971, 0, 0, 0, 0, 0, 2, 0, 7, 1449, 13, 1, 0, 0, 0, 0, 6733, 0, 19, 0, 7288, 7, 18, 195, 0, 0, 0, 0, 3351, 9, 7746, 0, 15, 0, 6, 0, 3501, 164, 2, 0, 76] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[2080, 3727, 100, 119, 82, 3, 1285, 140, 140, 4858, 2223, 265, 1160, 1, 117, 393, 23, 1277, 75, 297, 209, 52, 234, 251, 124, 101, 0, 327, 199, 530, 4, 167, 1961, 5691, 233, 0, 86, 0, 10, 351, 0, 4826, 24, 40, 2408, 3975, 59, 59, 95, 137, 29, 597, 441, 1033, 0, 2110, 226, 852, 283, 2197, 425, 346, 65, 30] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[6390, 0, 0, 0, 0, 0, 0, 2230, 547, 0, 148, 0, 0, 0, 5, 2316, 0, 0, 0, 6282, 241, 688, 0, 766, 0, 0, 0, 0, 0, 0, 0, 0, 2333, 0, 0, 0, 0, 0, 0, 6633, 0, 0, 0, 7642, 0, 0, 144, 0, 0, 0, 0, 1712, 0, 8029, 0, 0, 0, 0, 0, 2814, 158, 0, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[7533, 0, 0, 0, 0, 0, 0, 1831, 531, 0, 311, 0, 0, 0, 38, 1686, 0, 0, 0, 4759, 205, 448, 0, 975, 0, 0, 0, 0, 0, 0, 0, 0, 942, 0, 0, 0, 0, 0, 0, 6893, 0, 0, 0, 7354, 0, 0, 184, 0, 0, 0, 0, 4017, 0, 7753, 0, 0, 0, 0, 0, 3439, 163, 0, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[7658, 0, 0, 0, 0, 0, 0, 1836, 584, 0, 285, 0, 0, 0, 4, 1746, 0, 0, 0, 4339, 164, 414, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 6957, 0, 0, 0, 7310, 0, 0, 184, 0, 0, 0, 0, 3816, 0, 7784, 0, 0, 0, 0, 0, 3817, 176, 0, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[6559, 19, 0, 0, 1, 0, 3, 2287, 519, 31, 216, 2, 4, 0, 2, 2444, 0, 12, 7, 6282, 242, 566, 1, 827, 0, 0, 0, 0, 0, 5, 0, 3, 1809, 26, 2, 0, 1, 0, 0, 6517, 0, 30, 0, 7433, 11, 27, 118, 0, 0, 0, 0, 1908, 8, 7920, 0, 19, 0, 2, 1, 3073, 136, 4, 3, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[2080, 3727, 100, 119, 82, 3, 1285, 140, 140, 4858, 2223, 265, 1160, 1, 117, 393, 23, 1277, 75, 297, 209, 52, 234, 251, 124, 101, 0, 327, 199, 530, 4, 167, 1961, 5691, 233, 0, 86, 0, 10, 351, 0, 4826, 24, 40, 2408, 3975, 59, 59, 95, 137, 29, 597, 441, 1033, 0, 2110, 226, 852, 283, 2197, 425, 346, 65, 30] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3706, 0, 0, 0, 0, 0, 0, 2398, 0, 0, 0, 0, 0, 0, 0, 0, 522, 0, 116, 0, 0, 0, 0, 2438, 2250, 555, 47, 0, 0, 0, 0, 0, 0, 0, 0, 5798, 256, 633, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4181, 2330, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 0, 0, 0, 4259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5302, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1368, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 158, 0, 0, 75, 2372, 406, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6104, 5928, 7505, 6511, 6086, 5446, 5221, 6351] avg=6144.0000 max=7505 max_rank=2 min=5221 min_rank=6 imbalance_ratio=1.221517 max_load_reduction=41647 max_load_reduction_pct=0.847310 moved_to_echo=23104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4405, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 285, 0, 0, 0, 4, 1746, 2178, 379, 49, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 164, 414, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 2770, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 4437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5132, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 176, 0, 0, 80, 2471, 483, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6241, 5225, 5674, 6821, 5299, 5316, 7549, 7027] avg=6144.0000 max=7549 max_rank=6 min=5225 min_rank=1 imbalance_ratio=1.228678 max_load_reduction=41603 max_load_reduction_pct=0.846415 moved_to_echo=25191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3946, 0, 0, 0, 0, 0, 0, 1909, 0, 0, 0, 0, 0, 0, 0, 0, 544, 0, 182, 0, 0, 0, 6, 2464, 2235, 506, 47, 0, 0, 0, 0, 0, 0, 0, 0, 5293, 244, 611, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4119, 2481, 0, 0, 0, 0, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 4251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5267, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2973, 166, 0, 0, 100, 2368, 432, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 5984, 7001, 6600, 5918, 5396, 6359, 6039] avg=6144.0000 max=7001 max_rank=2 min=5396 min_rank=5 imbalance_ratio=1.139486 max_load_reduction=42151 max_load_reduction_pct=0.857564 moved_to_echo=23712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3676, 0, 0, 0, 0, 0, 0, 2230, 0, 0, 0, 0, 0, 0, 0, 0, 547, 0, 148, 0, 0, 0, 5, 2316, 2277, 548, 47, 0, 0, 0, 0, 0, 0, 0, 0, 5734, 241, 688, 0, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4178, 2311, 0, 0, 0, 0, 0, 0, 2333, 0, 0, 0, 0, 0, 0, 4230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5365, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1712, 0, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 158, 0, 0, 74, 2356, 403, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5906, 5888, 7429, 6489, 6563, 5509, 5563, 5805] avg=6144.0000 max=7429 max_rank=2 min=5509 min_rank=5 imbalance_ratio=1.209147 max_load_reduction=41723 max_load_reduction_pct=0.848857 moved_to_echo=23440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4333, 0, 0, 0, 0, 0, 0, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 311, 0, 0, 0, 38, 1686, 2191, 415, 49, 0, 0, 0, 0, 0, 0, 0, 0, 4344, 205, 448, 0, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 2725, 0, 0, 0, 0, 0, 0, 942, 0, 0, 0, 0, 0, 0, 4395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5163, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 163, 0, 0, 90, 2449, 475, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 5221, 5972, 6760, 5337, 5347, 7735, 6616] avg=6144.0000 max=7735 max_rank=6 min=5221 min_rank=1 imbalance_ratio=1.258952 max_load_reduction=41417 max_load_reduction_pct=0.842631 moved_to_echo=25035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4266, 18, 0, 1, 0, 0, 1, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 538, 21, 349, 1, 0, 0, 41, 2103, 2171, 402, 47, 0, 0, 0, 0, 0, 1, 7, 1, 4209, 155, 449, 1, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 4031, 2682, 0, 0, 0, 0, 0, 0, 1449, 13, 1, 0, 0, 0, 0, 4294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 5117, 7, 18, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3351, 9, 3715, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3501, 164, 2, 0, 76, 2392, 467, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 5673, 5794, 6722, 5757, 5356, 7090, 6608] avg=6144.0000 max=7090 max_rank=6 min=5356 min_rank=5 imbalance_ratio=1.153971 max_load_reduction=42062 max_load_reduction_pct=0.855754 moved_to_echo=24811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3946, 0, 0, 0, 0, 0, 0, 1909, 0, 0, 0, 0, 0, 0, 0, 0, 544, 0, 182, 0, 0, 0, 6, 2464, 2235, 506, 47, 0, 0, 0, 0, 0, 0, 0, 0, 5293, 244, 611, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4119, 2481, 0, 0, 0, 0, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 4251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5267, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2973, 166, 0, 0, 100, 2368, 432, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 5984, 7001, 6600, 5918, 5396, 6359, 6039] avg=6144.0000 max=7001 max_rank=2 min=5396 min_rank=5 imbalance_ratio=1.139486 max_load_reduction=42151 max_load_reduction_pct=0.857564 moved_to_echo=23712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1197, 3727, 100, 119, 82, 3, 1285, 140, 0, 0, 0, 0, 0, 0, 0, 0, 140, 4858, 2223, 265, 1160, 1, 117, 393, 11, 25, 2, 0, 0, 0, 0, 0, 23, 1277, 75, 272, 209, 52, 234, 251, 0, 0, 0, 0, 0, 0, 0, 0, 124, 101, 0, 327, 199, 530, 4, 167, 537, 752, 0, 0, 0, 0, 0, 0, 1961, 5691, 233, 0, 86, 0, 10, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4826, 24, 29, 2408, 3975, 59, 59, 0, 0, 0, 0, 0, 0, 0, 0, 95, 137, 29, 597, 441, 496, 0, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 226, 852, 283, 2197, 425, 346, 65, 30, 124, 131, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6653, 9195, 2393, 2741, 8206, 11380, 3905, 4679] avg=6144.0000 max=11380 max_rank=5 min=2393 min_rank=2 imbalance_ratio=1.852214 max_load_reduction=37772 max_load_reduction_pct=0.768473 moved_to_echo=28170 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4405, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 285, 0, 0, 0, 4, 1746, 2178, 379, 49, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 164, 414, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 2770, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 4437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5132, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 176, 0, 0, 80, 2471, 483, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6241, 5225, 5674, 6821, 5299, 5316, 7549, 7027] avg=6144.0000 max=7549 max_rank=6 min=5225 min_rank=1 imbalance_ratio=1.228678 max_load_reduction=41603 max_load_reduction_pct=0.846415 moved_to_echo=25191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3706, 0, 0, 0, 0, 0, 0, 2398, 0, 0, 0, 0, 0, 0, 0, 0, 522, 0, 116, 0, 0, 0, 0, 2438, 2250, 555, 47, 0, 0, 0, 0, 0, 0, 0, 0, 5798, 256, 633, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4181, 2330, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 0, 0, 0, 4259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5302, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1368, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 158, 0, 0, 75, 2372, 406, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6104, 5928, 7505, 6511, 6086, 5446, 5221, 6351] avg=6144.0000 max=7505 max_rank=2 min=5221 min_rank=6 imbalance_ratio=1.221517 max_load_reduction=41647 max_load_reduction_pct=0.847310 moved_to_echo=23104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3767, 19, 0, 0, 1, 0, 3, 2287, 0, 0, 0, 0, 0, 0, 0, 0, 519, 31, 216, 2, 4, 0, 2, 2444, 2217, 553, 50, 0, 0, 0, 0, 0, 0, 12, 7, 5729, 242, 566, 1, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 4125, 2375, 0, 0, 0, 0, 0, 0, 1809, 26, 2, 0, 1, 0, 0, 4148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 5216, 11, 27, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 8, 3795, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3073, 136, 4, 3, 72, 2319, 417, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6077, 6038, 7384, 6508, 5986, 5402, 5730, 6027] avg=6144.0000 max=7384 max_rank=2 min=5402 min_rank=5 imbalance_ratio=1.201823 max_load_reduction=41768 max_load_reduction_pct=0.849772 moved_to_echo=23145 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4333, 0, 0, 0, 0, 0, 0, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 311, 0, 0, 0, 38, 1686, 2191, 415, 49, 0, 0, 0, 0, 0, 0, 0, 0, 4344, 205, 448, 0, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 2725, 0, 0, 0, 0, 0, 0, 942, 0, 0, 0, 0, 0, 0, 4395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5163, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 163, 0, 0, 90, 2449, 475, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 5221, 5972, 6760, 5337, 5347, 7735, 6616] avg=6144.0000 max=7735 max_rank=6 min=5221 min_rank=1 imbalance_ratio=1.258952 max_load_reduction=41417 max_load_reduction_pct=0.842631 moved_to_echo=25035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 384, 256, 640, 128, 256, 29312, 18432, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 4992, 3840, 384, 1280, 128, 256, 15616, 15616, 3456, 384, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 896, 384, 25216, 1664, 384, 128, 640, 16896, 3328, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 256, 128, 19456, 512, 27008, 0, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 896, 384, 25216, 1664, 384, 128, 640, 16896, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12928, 5760, 256, 0, 128, 0, 128, 30336, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[29312, 3840, 128, 128, 128, 128, 1408, 14592, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 4992, 128, 36608, 2432, 4096, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12928, 5760, 256, 0, 128, 0, 128, 30336, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1408, 128, 35456, 1792, 3968, 256, 6656, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[491, 762, 868, 493, 47, 1, 117, 89, 3448, 60, 331, 24, 291, 13, 379, 232, 294, 972, 70, 63, 0, 2, 1, 203, 262, 578, 192, 42, 1475, 6, 0, 166, 39, 86, 58, 0, 3777, 86, 858, 1, 1934, 1630, 286, 2897, 5041, 691, 246, 28, 5286, 212, 198, 1436, 1595, 0, 123, 184, 3, 5153, 396, 4440, 290, 146, 16, 44] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[1, 1, 6808, 6, 3, 1, 3, 0, 13, 257, 0, 0, 7801, 0, 632, 1, 10, 379, 49, 0, 0, 0, 70, 0, 162, 4, 6, 1, 0, 1, 1, 7231, 6911, 0, 222, 0, 27, 0, 149, 0, 913, 852, 2, 20, 32, 6632, 8028, 0, 33, 3, 0, 1, 271, 0, 76, 0, 0, 33, 1474, 29, 1, 2, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7006, 0, 0, 0, 0, 0, 0, 204, 0, 0, 7845, 0, 654, 0, 6, 362, 55, 0, 0, 0, 75, 0, 263, 0, 0, 0, 0, 0, 0, 7267, 6893, 0, 248, 0, 0, 0, 153, 0, 1033, 680, 0, 0, 0, 6503, 8038, 0, 0, 0, 0, 0, 247, 0, 100, 0, 0, 0, 1520, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7414, 0, 0, 0, 0, 0, 0, 79, 0, 0, 7859, 0, 517, 0, 34, 380, 68, 0, 0, 0, 245, 0, 354, 0, 0, 0, 0, 0, 0, 6857, 6043, 0, 171, 0, 0, 0, 139, 0, 1155, 769, 0, 0, 0, 6711, 8044, 0, 0, 5, 0, 0, 247, 0, 80, 0, 0, 0, 1981, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7122, 0, 0, 0, 0, 0, 0, 218, 0, 0, 7878, 0, 570, 0, 1, 345, 64, 0, 0, 0, 86, 0, 176, 0, 0, 0, 0, 0, 0, 7330, 7158, 0, 288, 0, 0, 0, 144, 0, 692, 750, 0, 0, 0, 6582, 8055, 0, 0, 0, 0, 0, 240, 0, 87, 0, 0, 0, 1366, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7019, 0, 0, 0, 0, 0, 0, 246, 0, 0, 7867, 0, 687, 0, 0, 365, 70, 0, 0, 0, 71, 0, 170, 0, 0, 0, 0, 0, 0, 7316, 7020, 0, 229, 0, 0, 0, 142, 0, 626, 687, 0, 0, 0, 6685, 8048, 0, 0, 0, 0, 0, 251, 0, 76, 0, 0, 0, 1577, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 2, 7162, 6, 0, 0, 2, 0, 17, 207, 0, 0, 7833, 0, 557, 0, 22, 333, 61, 0, 0, 0, 184, 1, 403, 1, 9, 0, 0, 0, 0, 6869, 6513, 0, 191, 0, 19, 0, 139, 0, 1170, 725, 1, 5, 20, 6722, 8028, 1, 23, 6, 0, 1, 242, 0, 77, 0, 0, 22, 1561, 17, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7414, 0, 0, 0, 0, 0, 0, 79, 0, 0, 7859, 0, 517, 0, 34, 380, 68, 0, 0, 0, 245, 0, 354, 0, 0, 0, 0, 0, 0, 6857, 6043, 0, 171, 0, 0, 0, 139, 0, 1155, 769, 0, 0, 0, 6711, 8044, 0, 0, 5, 0, 0, 247, 0, 80, 0, 0, 0, 1981, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7359, 0, 0, 0, 0, 0, 0, 154, 0, 0, 7840, 0, 602, 0, 19, 343, 53, 0, 0, 0, 171, 0, 377, 0, 0, 0, 0, 0, 0, 6907, 6111, 0, 225, 0, 0, 0, 161, 0, 1191, 762, 0, 0, 0, 6563, 8050, 0, 0, 6, 0, 0, 265, 0, 90, 0, 0, 0, 1903, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[491, 762, 806, 493, 47, 1, 117, 89, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 60, 331, 24, 207, 13, 379, 232, 0, 0, 0, 0, 0, 0, 0, 0, 294, 972, 70, 63, 0, 2, 1, 203, 190, 0, 0, 0, 0, 0, 0, 0, 262, 578, 192, 42, 1475, 6, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 35, 86, 58, 0, 3777, 86, 858, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1934, 1630, 286, 2897, 5041, 371, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 5286, 212, 198, 1436, 1595, 0, 123, 184, 320, 56, 15, 0, 0, 0, 0, 0, 3, 5153, 396, 4440, 290, 146, 16, 44, 69, 17, 4, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2806, 4694, 1795, 2704, 4901, 12187, 9425, 10640] avg=6144.0000 max=12187 max_rank=5 min=1795 min_rank=2 imbalance_ratio=1.983561 max_load_reduction=36965 max_load_reduction_pct=0.752055 moved_to_echo=37153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 5559, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 380, 68, 0, 0, 0, 245, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 5390, 0, 171, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 769, 0, 0, 0, 3599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 247, 0, 80, 0, 3112, 1837, 418, 0, 0, 0, 0, 0, 0, 0, 1981, 0, 0, 0, 0, 0, 1882, 724, 653, 530, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6884, 6155, 6934, 6487, 5700, 5523, 5699, 5770] avg=6144.0000 max=6934 max_rank=2 min=5523 min_rank=5 imbalance_ratio=1.128581 max_load_reduction=42218 max_load_reduction_pct=0.858927 moved_to_echo=22692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7006, 0, 0, 0, 0, 0, 0, 204, 0, 0, 7845, 0, 654, 0, 6, 362, 55, 0, 0, 0, 75, 0, 263, 0, 0, 0, 0, 0, 0, 7267, 6893, 0, 248, 0, 0, 0, 153, 0, 1033, 680, 0, 0, 0, 6503, 8038, 0, 0, 0, 0, 0, 247, 0, 100, 0, 0, 0, 1520, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[491, 762, 868, 493, 47, 1, 117, 89, 3448, 60, 331, 24, 291, 13, 379, 232, 294, 972, 70, 63, 0, 2, 1, 203, 262, 578, 192, 42, 1475, 6, 0, 166, 39, 86, 58, 0, 3777, 86, 858, 1, 1934, 1630, 286, 2897, 5041, 691, 246, 28, 5286, 212, 198, 1436, 1595, 0, 123, 184, 3, 5153, 396, 4440, 290, 146, 16, 44] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 5573, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 345, 64, 0, 0, 0, 86, 0, 6215, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 6556, 0, 0, 0, 0, 0, 0, 0, 0, 6384, 0, 288, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 692, 750, 0, 0, 0, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 87, 0, 3052, 1840, 419, 0, 0, 0, 0, 0, 0, 0, 1366, 0, 0, 0, 0, 0, 1886, 774, 774, 509, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6613, 6361, 6711, 6732, 6816, 4972, 5638, 5309] avg=6144.0000 max=6816 max_rank=4 min=4972 min_rank=5 imbalance_ratio=1.109375 max_load_reduction=42336 max_load_reduction_pct=0.861328 moved_to_echo=22735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 5550, 0, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 362, 55, 0, 0, 0, 75, 0, 6202, 0, 0, 0, 0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 6500, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 248, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 680, 0, 0, 0, 3488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 100, 0, 3015, 1836, 417, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 0, 0, 0, 0, 1878, 767, 745, 501, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6505, 6408, 6700, 6763, 6549, 5201, 5615, 5411] avg=6144.0000 max=6763 max_rank=3 min=5201 min_rank=5 imbalance_ratio=1.100749 max_load_reduction=42389 max_load_reduction_pct=0.862406 moved_to_echo=22776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 6650, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 207, 0, 0, 5540, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 333, 61, 0, 0, 0, 184, 1, 6195, 0, 0, 0, 0, 0, 0, 0, 403, 1, 9, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 5809, 0, 191, 0, 19, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 725, 1, 5, 20, 3605, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6, 0, 1, 242, 0, 77, 0, 3117, 1833, 417, 0, 0, 0, 0, 0, 0, 22, 1561, 17, 0, 0, 0, 0, 1876, 725, 704, 512, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6660, 6321, 6796, 6557, 6158, 5527, 5716, 5417] avg=6144.0000 max=6796 max_rank=2 min=5417 min_rank=7 imbalance_ratio=1.106120 max_load_reduction=42356 max_load_reduction_pct=0.861735 moved_to_echo=22818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 5559, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 380, 68, 0, 0, 0, 245, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 5390, 0, 171, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 769, 0, 0, 0, 3599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 247, 0, 80, 0, 3112, 1837, 418, 0, 0, 0, 0, 0, 0, 0, 1981, 0, 0, 0, 0, 0, 1882, 724, 653, 530, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6884, 6155, 6934, 6487, 5700, 5523, 5699, 5770] avg=6144.0000 max=6934 max_rank=2 min=5523 min_rank=5 imbalance_ratio=1.128581 max_load_reduction=42218 max_load_reduction_pct=0.858927 moved_to_echo=22692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 5565, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 70, 0, 0, 0, 71, 0, 6209, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 6544, 0, 0, 0, 0, 0, 0, 0, 0, 6261, 0, 229, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 687, 0, 0, 0, 3585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 76, 0, 3100, 1839, 418, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 0, 0, 1884, 772, 759, 502, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6517, 6498, 6715, 6714, 6632, 4898, 5684, 5494] avg=6144.0000 max=6715 max_rank=2 min=4898 min_rank=5 imbalance_ratio=1.092936 max_load_reduction=42437 max_load_reduction_pct=0.863383 moved_to_echo=22708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 6319, 6, 3, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 257, 0, 0, 5510, 0, 632, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 379, 49, 0, 0, 0, 70, 0, 6195, 0, 0, 0, 0, 0, 0, 0, 162, 4, 6, 1, 0, 1, 1, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 222, 0, 27, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 852, 2, 20, 32, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3, 0, 1, 271, 0, 76, 0, 3080, 1833, 419, 0, 0, 0, 0, 0, 0, 33, 1474, 29, 1, 2, 0, 0, 1872, 766, 750, 489, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6334, 6413, 6703, 6640, 6559, 5371, 5716, 5416] avg=6144.0000 max=6703 max_rank=2 min=5371 min_rank=5 imbalance_ratio=1.090983 max_load_reduction=42449 max_load_reduction_pct=0.863627 moved_to_echo=23062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 6650, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 207, 0, 0, 5540, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 333, 61, 0, 0, 0, 184, 1, 6195, 0, 0, 0, 0, 0, 0, 0, 403, 1, 9, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 5809, 0, 191, 0, 19, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 725, 1, 5, 20, 3605, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6, 0, 1, 242, 0, 77, 0, 3117, 1833, 417, 0, 0, 0, 0, 0, 0, 22, 1561, 17, 0, 0, 0, 0, 1876, 725, 704, 512, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6660, 6321, 6796, 6557, 6158, 5527, 5716, 5417] avg=6144.0000 max=6796 max_rank=2 min=5417 min_rank=7 imbalance_ratio=1.106120 max_load_reduction=42356 max_load_reduction_pct=0.861735 moved_to_echo=22818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 5550, 0, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 362, 55, 0, 0, 0, 75, 0, 6202, 0, 0, 0, 0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 6500, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 248, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 680, 0, 0, 0, 3488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 100, 0, 3015, 1836, 417, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 0, 0, 0, 0, 1878, 767, 745, 501, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6505, 6408, 6700, 6763, 6549, 5201, 5615, 5411] avg=6144.0000 max=6763 max_rank=3 min=5201 min_rank=5 imbalance_ratio=1.100749 max_load_reduction=42389 max_load_reduction_pct=0.862406 moved_to_echo=22776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 5546, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 343, 53, 0, 0, 0, 171, 0, 6212, 0, 0, 0, 0, 0, 0, 0, 377, 0, 0, 0, 0, 0, 0, 6178, 0, 0, 0, 0, 0, 0, 0, 0, 5451, 0, 225, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1191, 762, 0, 0, 0, 3520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 265, 0, 90, 0, 3043, 1838, 417, 0, 0, 0, 0, 0, 0, 0, 1903, 0, 0, 0, 0, 0, 1877, 729, 660, 526, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6833, 6302, 6798, 6555, 5837, 5473, 5659, 5695] avg=6144.0000 max=6833 max_rank=0 min=5473 min_rank=5 imbalance_ratio=1.112142 max_load_reduction=42319 max_load_reduction_pct=0.860982 moved_to_echo=22664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[491, 762, 806, 493, 47, 1, 117, 89, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 60, 331, 24, 207, 13, 379, 232, 0, 0, 0, 0, 0, 0, 0, 0, 294, 972, 70, 63, 0, 2, 1, 203, 190, 0, 0, 0, 0, 0, 0, 0, 262, 578, 192, 42, 1475, 6, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 35, 86, 58, 0, 3777, 86, 858, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1934, 1630, 286, 2897, 5041, 371, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 5286, 212, 198, 1436, 1595, 0, 123, 184, 320, 56, 15, 0, 0, 0, 0, 0, 3, 5153, 396, 4440, 290, 146, 16, 44, 69, 17, 4, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2806, 4694, 1795, 2704, 4901, 12187, 9425, 10640] avg=6144.0000 max=12187 max_rank=5 min=1795 min_rank=2 imbalance_ratio=1.983561 max_load_reduction=36965 max_load_reduction_pct=0.752055 moved_to_echo=37153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[1, 1, 6808, 6, 3, 1, 3, 0, 13, 257, 0, 0, 7801, 0, 632, 1, 10, 379, 49, 0, 0, 0, 70, 0, 162, 4, 6, 1, 0, 1, 1, 7231, 6911, 0, 222, 0, 27, 0, 149, 0, 913, 852, 2, 20, 32, 6632, 8028, 0, 33, 3, 0, 1, 271, 0, 76, 0, 0, 33, 1474, 29, 1, 2, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 6319, 6, 3, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 257, 0, 0, 5510, 0, 632, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 379, 49, 0, 0, 0, 70, 0, 6195, 0, 0, 0, 0, 0, 0, 0, 162, 4, 6, 1, 0, 1, 1, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 222, 0, 27, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 852, 2, 20, 32, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3, 0, 1, 271, 0, 76, 0, 3080, 1833, 419, 0, 0, 0, 0, 0, 0, 33, 1474, 29, 1, 2, 0, 0, 1872, 766, 750, 489, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6334, 6413, 6703, 6640, 6559, 5371, 5716, 5416] avg=6144.0000 max=6703 max_rank=2 min=5371 min_rank=5 imbalance_ratio=1.090983 max_load_reduction=42449 max_load_reduction_pct=0.863627 moved_to_echo=23062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50304 observed_loss=-1152 tokens_per_expert=[128, 5248, 11904, 4608, 384, 256, 128, 128, 13312, 5376, 5120, 3712, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8832, 6912, 384, 2944, 5120, 25344, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 256, 256, 1536, 3456, 0, 768, 256, 21888, 12928, 2944, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 1536, 384, 128, 39168, 128, 4608, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 3584, 512, 128, 0, 128, 1024, 256, 43648, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[41728, 128, 1664, 0, 3840, 128, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 3584, 512, 128, 0, 128, 1024, 256, 43648, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 640, 256, 128, 1536, 128, 128, 44672, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 768, 47232, 512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50304 slack_slots=48000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 55, 0, 0, 0, 0, 450, 0, 95, 7501, 2354, 0, 0, 99, 0, 0, 0, 0, 0, 55, 0, 360, 0, 0, 0, 0, 0, 0, 945, 7739, 0, 0, 0, 0, 0, 0, 497, 7130, 0, 8093, 100, 0, 0, 2951, 133, 0, 73, 0, 0, 0, 0, 0, 0, 2215, 0, 0, 0, 0, 347, 7812, 0, 0, 0, 148] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 495, 0, 67, 7479, 2097, 0, 0, 73, 0, 0, 0, 0, 0, 64, 0, 396, 0, 0, 0, 0, 0, 0, 830, 7728, 0, 0, 0, 0, 0, 0, 536, 7270, 0, 8119, 74, 0, 0, 2798, 113, 0, 29, 0, 0, 0, 0, 0, 0, 2630, 0, 0, 0, 0, 314, 7864, 0, 0, 0, 112] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 351, 0, 27, 7428, 3787, 0, 0, 80, 0, 0, 0, 0, 0, 68, 0, 388, 0, 0, 0, 0, 0, 0, 1048, 7635, 0, 0, 0, 0, 0, 0, 646, 7198, 0, 8112, 80, 0, 0, 2351, 84, 0, 100, 0, 0, 0, 0, 0, 0, 1239, 0, 0, 0, 0, 327, 7800, 0, 0, 0, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 61, 8, 21, 23, 14, 378, 9, 73, 7464, 3298, 0, 0, 77, 1, 0, 1, 0, 0, 66, 21, 397, 17, 0, 0, 0, 0, 0, 1006, 7692, 15, 0, 0, 20, 0, 0, 680, 7129, 1, 8094, 76, 0, 1, 2651, 129, 2, 152, 0, 6, 1, 0, 1, 0, 1277, 0, 0, 0, 0, 317, 7752, 0, 0, 0, 221] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[25, 41, 3909, 4997, 4764, 3087, 53, 15, 84, 821, 64, 0, 0, 2287, 27, 140, 22, 833, 0, 9, 4799, 56, 1884, 0, 25, 1184, 2, 114, 274, 422, 2939, 0, 136, 5364, 3, 67, 1715, 55, 1172, 2319, 25, 94, 5, 183, 1910, 448, 1446, 106, 205, 42, 168, 14, 31, 1, 0, 0, 0, 5, 321, 34, 92, 67, 146, 101] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 47, 21, 32, 32, 18, 462, 2, 72, 7445, 1852, 0, 0, 78, 5, 2, 1, 6, 0, 48, 32, 430, 20, 0, 0, 3, 0, 0, 895, 7662, 20, 0, 1, 32, 0, 1, 631, 7157, 2, 8084, 72, 1, 0, 3317, 186, 3, 112, 0, 3, 0, 0, 0, 0, 2147, 0, 0, 0, 0, 341, 7722, 0, 3, 2, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 55, 0, 0, 0, 0, 450, 0, 95, 7501, 2354, 0, 0, 99, 0, 0, 0, 0, 0, 55, 0, 360, 0, 0, 0, 0, 0, 0, 945, 7739, 0, 0, 0, 0, 0, 0, 497, 7130, 0, 8093, 100, 0, 0, 2951, 133, 0, 73, 0, 0, 0, 0, 0, 0, 2215, 0, 0, 0, 0, 347, 7812, 0, 0, 0, 148] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 61, 8, 21, 23, 14, 378, 9, 73, 7464, 3298, 0, 0, 77, 1, 0, 1, 0, 0, 66, 21, 397, 17, 0, 0, 0, 0, 0, 1006, 7692, 15, 0, 0, 20, 0, 0, 680, 7129, 1, 8094, 76, 0, 1, 2651, 129, 2, 152, 0, 6, 1, 0, 1, 0, 1277, 0, 0, 0, 0, 317, 7752, 0, 0, 0, 221] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 49, 0, 0, 0, 0, 312, 0, 64, 7476, 3596, 0, 0, 89, 0, 0, 0, 0, 0, 53, 0, 358, 0, 0, 0, 0, 0, 0, 1051, 7712, 0, 0, 0, 0, 0, 0, 562, 7141, 0, 8103, 90, 0, 0, 2146, 119, 0, 110, 0, 0, 0, 0, 0, 0, 1669, 0, 0, 0, 0, 355, 7822, 0, 0, 0, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 70, 0, 0, 0, 0, 543, 0, 74, 7494, 2229, 0, 0, 74, 0, 0, 0, 0, 0, 70, 0, 488, 0, 0, 0, 0, 0, 0, 839, 7688, 0, 0, 0, 0, 0, 0, 487, 7276, 0, 8118, 75, 0, 0, 2689, 88, 0, 41, 0, 0, 0, 0, 0, 0, 2523, 0, 0, 0, 0, 326, 7849, 0, 0, 0, 111] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 47, 21, 32, 32, 18, 462, 2, 72, 7445, 1852, 0, 0, 78, 5, 2, 1, 6, 0, 48, 32, 430, 20, 0, 0, 3, 0, 0, 895, 7662, 20, 0, 1, 32, 0, 1, 631, 7157, 2, 8084, 72, 1, 0, 3317, 186, 3, 112, 0, 3, 0, 0, 0, 0, 2147, 0, 0, 0, 0, 341, 7722, 0, 3, 2, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 351, 0, 27, 7428, 3787, 0, 0, 80, 0, 0, 0, 0, 0, 68, 0, 388, 0, 0, 0, 0, 0, 0, 1048, 7635, 0, 0, 0, 0, 0, 0, 646, 7198, 0, 8112, 80, 0, 0, 2351, 84, 0, 100, 0, 0, 0, 0, 0, 0, 1239, 0, 0, 0, 0, 327, 7800, 0, 0, 0, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 495, 0, 2279, 1784, 0, 0, 0, 0, 0, 0, 67, 3703, 2097, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 396, 0, 0, 5254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 5406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 5284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 2798, 113, 0, 29, 0, 1986, 1427, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2630, 0, 0, 2865, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 314, 6437, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4622, 5940, 5714, 6236, 5820, 6470, 7487, 6863] avg=6144.0000 max=7487 max_rank=6 min=4622 min_rank=0 imbalance_ratio=1.218587 max_load_reduction=41665 max_load_reduction_pct=0.847677 moved_to_echo=26640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 351, 0, 2251, 1772, 0, 0, 0, 0, 0, 0, 27, 3678, 3787, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 388, 0, 0, 5250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 5341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 5232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 2351, 84, 0, 100, 0, 1966, 1416, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 0, 0, 2862, 1978, 0, 0, 0, 0, 0, 0, 0, 0, 327, 6384, 0, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4438, 7572, 5706, 6389, 5878, 6040, 6079, 7050] avg=6144.0000 max=7572 max_rank=1 min=4438 min_rank=0 imbalance_ratio=1.232422 max_load_reduction=41580 max_load_reduction_pct=0.845947 moved_to_echo=25047 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[25, 41, 3909, 4997, 4764, 3087, 53, 15, 124, 195, 0, 0, 0, 0, 0, 0, 84, 408, 64, 0, 0, 2287, 27, 140, 0, 0, 0, 0, 0, 0, 0, 0, 22, 833, 0, 9, 4799, 56, 1884, 0, 1501, 0, 0, 0, 0, 0, 0, 0, 25, 1184, 2, 114, 274, 296, 2939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 5364, 3, 67, 1715, 40, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 94, 5, 183, 1910, 448, 1446, 106, 15, 6, 2, 0, 0, 0, 0, 0, 205, 42, 168, 14, 31, 1, 0, 0, 818, 218, 0, 0, 0, 0, 0, 0, 0, 5, 321, 28, 92, 67, 146, 101, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17210, 3010, 9104, 4834, 8497, 4240, 1497, 760] avg=6144.0000 max=17210 max_rank=0 min=760 min_rank=7 imbalance_ratio=2.801107 max_load_reduction=31942 max_load_reduction_pct=0.649862 moved_to_echo=14994 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 55, 0, 0, 0, 0, 450, 0, 2282, 1789, 0, 0, 0, 0, 0, 0, 95, 3714, 2354, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 360, 0, 0, 5237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 945, 5413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 5183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 2951, 133, 0, 73, 0, 1947, 1418, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2215, 0, 0, 2856, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 347, 6394, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4576, 6262, 5652, 6358, 5680, 6666, 7069, 6889] avg=6144.0000 max=7069 max_rank=6 min=4576 min_rank=0 imbalance_ratio=1.150553 max_load_reduction=42083 max_load_reduction_pct=0.856181 moved_to_echo=26304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 61, 8, 21, 23, 14, 378, 9, 2268, 1781, 0, 0, 0, 0, 0, 0, 73, 3695, 3298, 0, 0, 77, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 66, 21, 397, 17, 0, 5239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 5381, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 680, 5182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 1, 2651, 129, 2, 152, 0, 1947, 1407, 43, 0, 0, 0, 0, 0, 6, 1, 0, 1, 0, 1277, 0, 0, 2855, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 317, 6345, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4563, 7144, 5741, 6402, 5883, 6408, 6128, 6883] avg=6144.0000 max=7144 max_rank=1 min=4563 min_rank=0 imbalance_ratio=1.162760 max_load_reduction=42008 max_load_reduction_pct=0.854655 moved_to_echo=25302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 70, 0, 0, 0, 0, 543, 0, 2267, 1788, 0, 0, 0, 0, 0, 0, 74, 3710, 2229, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 488, 0, 0, 5253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 5378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 5289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 2689, 88, 0, 41, 0, 1987, 1425, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 0, 0, 2865, 1996, 0, 0, 0, 0, 0, 0, 0, 0, 326, 6424, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4668, 6087, 5811, 6217, 5776, 6348, 7384, 6861] avg=6144.0000 max=7384 max_rank=6 min=4668 min_rank=0 imbalance_ratio=1.201823 max_load_reduction=41768 max_load_reduction_pct=0.849772 moved_to_echo=26369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 47, 21, 32, 32, 18, 462, 2, 2262, 1781, 0, 0, 0, 0, 0, 0, 72, 3677, 1852, 0, 0, 78, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 48, 32, 430, 20, 0, 5232, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 895, 5352, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 0, 1, 631, 5198, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1, 0, 3317, 186, 3, 112, 0, 1959, 1404, 48, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2147, 0, 0, 2852, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 341, 6318, 0, 3, 2, 150, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4657, 5686, 5769, 6270, 5865, 7102, 6989, 6814] avg=6144.0000 max=7102 max_rank=5 min=4657 min_rank=0 imbalance_ratio=1.155924 max_load_reduction=42050 max_load_reduction_pct=0.855509 moved_to_echo=26770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 351, 0, 2251, 1772, 0, 0, 0, 0, 0, 0, 27, 3678, 3787, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 388, 0, 0, 5250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 5341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 5232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 2351, 84, 0, 100, 0, 1966, 1416, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 0, 0, 2862, 1978, 0, 0, 0, 0, 0, 0, 0, 0, 327, 6384, 0, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4438, 7572, 5706, 6389, 5878, 6040, 6079, 7050] avg=6144.0000 max=7572 max_rank=1 min=4438 min_rank=0 imbalance_ratio=1.232422 max_load_reduction=41580 max_load_reduction_pct=0.845947 moved_to_echo=25047 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 61, 8, 21, 23, 14, 378, 9, 2268, 1781, 0, 0, 0, 0, 0, 0, 73, 3695, 3298, 0, 0, 77, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 66, 21, 397, 17, 0, 5239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 5381, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 680, 5182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 1, 2651, 129, 2, 152, 0, 1947, 1407, 43, 0, 0, 0, 0, 0, 6, 1, 0, 1, 0, 1277, 0, 0, 2855, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 317, 6345, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4563, 7144, 5741, 6402, 5883, 6408, 6128, 6883] avg=6144.0000 max=7144 max_rank=1 min=4563 min_rank=0 imbalance_ratio=1.162760 max_load_reduction=42008 max_load_reduction_pct=0.854655 moved_to_echo=25302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 49, 0, 0, 0, 0, 312, 0, 2274, 1783, 0, 0, 0, 0, 0, 0, 64, 3702, 3596, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 358, 0, 0, 5244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 5395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 5191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 2146, 119, 0, 110, 0, 1950, 1420, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1669, 0, 0, 2859, 1991, 0, 0, 0, 0, 0, 0, 0, 0, 355, 6402, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4418, 7451, 5655, 6446, 5753, 5878, 6519, 7032] avg=6144.0000 max=7451 max_rank=1 min=4418 min_rank=0 imbalance_ratio=1.212728 max_load_reduction=41701 max_load_reduction_pct=0.848409 moved_to_echo=25182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[25, 41, 3909, 4997, 4764, 3087, 53, 15, 124, 195, 0, 0, 0, 0, 0, 0, 84, 408, 64, 0, 0, 2287, 27, 140, 0, 0, 0, 0, 0, 0, 0, 0, 22, 833, 0, 9, 4799, 56, 1884, 0, 1501, 0, 0, 0, 0, 0, 0, 0, 25, 1184, 2, 114, 274, 296, 2939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 5364, 3, 67, 1715, 40, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 94, 5, 183, 1910, 448, 1446, 106, 15, 6, 2, 0, 0, 0, 0, 0, 205, 42, 168, 14, 31, 1, 0, 0, 818, 218, 0, 0, 0, 0, 0, 0, 0, 5, 321, 28, 92, 67, 146, 101, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17210, 3010, 9104, 4834, 8497, 4240, 1497, 760] avg=6144.0000 max=17210 max_rank=0 min=760 min_rank=7 imbalance_ratio=2.801107 max_load_reduction=31942 max_load_reduction_pct=0.649862 moved_to_echo=14994 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 47, 21, 32, 32, 18, 462, 2, 2262, 1781, 0, 0, 0, 0, 0, 0, 72, 3677, 1852, 0, 0, 78, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 48, 32, 430, 20, 0, 5232, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 895, 5352, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 0, 1, 631, 5198, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1, 0, 3317, 186, 3, 112, 0, 1959, 1404, 48, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2147, 0, 0, 2852, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 341, 6318, 0, 3, 2, 150, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4657, 5686, 5769, 6270, 5865, 7102, 6989, 6814] avg=6144.0000 max=7102 max_rank=5 min=4657 min_rank=0 imbalance_ratio=1.155924 max_load_reduction=42050 max_load_reduction_pct=0.855509 moved_to_echo=26770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 512, 3968, 5120, 4864, 3200, 3072, 128, 16128, 12800, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 128, 128, 19200, 2816, 512, 2176, 128, 13824, 9984, 384, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 26368, 19328, 0, 0, 2944, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] @@ -10767,790 +10767,790 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True - [2026-08-26 15:39:23] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 805.2 | throughput per GPU (TFLOP/s/GPU): 158.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.899922E+00 | loss scale: 1.0 | grad norm: 1155.511 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:35:26] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 824.4 | throughput per GPU (TFLOP/s/GPU): 154.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.899922E+00 | loss scale: 1.0 | grad norm: 1155.511 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1426, 336, 755, 935, 309, 1151, 881, 797, 1237, 321, 758, 313, 288, 1669, 822, 912, 714, 964, 829, 647, 324, 630, 278, 1300, 376, 663, 837, 445, 699, 165, 740, 1265, 834, 797, 1100, 1187, 534, 191, 578, 970, 543, 594, 622, 733, 922, 1004, 465, 456, 749, 608, 755, 560, 923, 1435, 867, 778, 1432, 317, 1533, 700, 196, 768, 1315, 900] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1347, 270, 813, 863, 299, 826, 790, 1071, 1234, 200, 536, 307, 416, 1311, 907, 1084, 743, 940, 790, 605, 312, 499, 307, 1391, 561, 654, 771, 357, 681, 77, 917, 1366, 844, 867, 978, 1135, 437, 233, 501, 924, 691, 601, 538, 891, 921, 943, 585, 779, 791, 582, 839, 483, 1003, 1232, 828, 910, 1548, 497, 1466, 701, 174, 754, 1202, 1029] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1348, 327, 818, 924, 322, 836, 830, 1074, 1242, 215, 611, 343, 330, 1383, 933, 1152, 694, 937, 848, 647, 244, 523, 293, 1401, 532, 711, 772, 393, 695, 80, 900, 1413, 791, 841, 1057, 1065, 456, 223, 515, 957, 639, 630, 582, 840, 904, 933, 542, 642, 773, 549, 885, 488, 990, 1301, 738, 879, 1561, 363, 1410, 720, 135, 727, 1222, 1023] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1337, 263, 856, 910, 310, 927, 839, 1023, 1350, 225, 589, 330, 369, 1444, 968, 1076, 723, 1029, 846, 675, 293, 536, 265, 1351, 557, 671, 724, 377, 727, 95, 838, 1290, 790, 831, 961, 1010, 504, 206, 574, 1022, 655, 566, 556, 754, 884, 998, 534, 678, 795, 522, 910, 444, 1003, 1356, 880, 863, 1585, 308, 1500, 650, 159, 771, 1168, 902] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1225, 257, 788, 939, 315, 1027, 844, 924, 1242, 220, 550, 357, 385, 1306, 926, 1191, 744, 1060, 788, 527, 342, 513, 242, 1489, 428, 642, 858, 311, 680, 136, 956, 1449, 815, 810, 1059, 1145, 362, 212, 567, 939, 578, 651, 590, 894, 905, 868, 569, 639, 692, 521, 756, 532, 1119, 1228, 964, 897, 1529, 378, 1492, 793, 177, 725, 1165, 920] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1347, 270, 813, 863, 299, 826, 790, 1071, 1234, 200, 536, 307, 416, 1311, 907, 1084, 743, 940, 790, 605, 312, 499, 307, 1391, 561, 654, 771, 357, 681, 77, 917, 1366, 844, 867, 978, 1135, 437, 233, 501, 924, 691, 601, 538, 891, 921, 943, 585, 779, 791, 582, 839, 483, 1003, 1232, 828, 910, 1548, 497, 1466, 701, 174, 754, 1202, 1029] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1340, 281, 722, 829, 273, 885, 741, 1106, 1377, 196, 599, 331, 366, 1344, 915, 1176, 661, 928, 766, 587, 278, 508, 264, 1501, 542, 683, 741, 388, 713, 101, 879, 1138, 868, 795, 1119, 1017, 730, 223, 478, 1118, 642, 596, 555, 980, 882, 984, 608, 659, 687, 509, 820, 576, 1028, 1363, 835, 880, 1504, 379, 1403, 713, 120, 721, 1152, 1049] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1341, 263, 746, 867, 265, 792, 905, 916, 1315, 257, 582, 338, 295, 1451, 903, 1085, 701, 1019, 805, 671, 267, 536, 284, 1334, 455, 589, 777, 457, 798, 95, 801, 1106, 840, 795, 1306, 1139, 658, 210, 636, 908, 664, 527, 552, 892, 962, 981, 492, 595, 768, 603, 736, 561, 983, 1409, 814, 943, 1491, 375, 1567, 666, 183, 789, 1266, 825] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1337, 263, 856, 910, 310, 927, 839, 1023, 1350, 225, 589, 330, 369, 1444, 968, 1076, 723, 1029, 846, 675, 293, 536, 265, 1351, 557, 671, 724, 377, 727, 95, 838, 1290, 790, 831, 961, 1010, 504, 206, 574, 1022, 655, 566, 556, 754, 884, 998, 534, 678, 795, 522, 910, 444, 1003, 1356, 880, 863, 1585, 308, 1500, 650, 159, 771, 1168, 902] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1309, 301, 768, 889, 249, 987, 843, 998, 1264, 231, 658, 318, 382, 1461, 946, 1022, 713, 1016, 769, 600, 308, 528, 269, 1374, 447, 653, 863, 361, 730, 124, 850, 1453, 894, 833, 1060, 1135, 370, 195, 650, 878, 613, 600, 569, 899, 967, 928, 498, 634, 760, 612, 756, 538, 1020, 1246, 877, 912, 1539, 265, 1450, 738, 187, 737, 1223, 885] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1426, 336, 755, 935, 309, 1151, 881, 797, 1237, 321, 758, 313, 288, 1669, 822, 912, 714, 964, 829, 647, 324, 630, 278, 1300, 376, 663, 837, 445, 699, 165, 740, 1265, 834, 797, 1100, 1187, 534, 191, 578, 970, 543, 594, 622, 733, 922, 1004, 465, 456, 749, 608, 755, 560, 923, 1435, 867, 778, 1432, 317, 1533, 700, 196, 768, 1315, 900] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1341, 263, 746, 867, 265, 792, 905, 916, 1315, 257, 582, 338, 295, 1451, 903, 1085, 701, 1019, 805, 671, 267, 536, 284, 1334, 455, 589, 777, 457, 798, 95, 801, 1106, 840, 795, 1306, 1139, 658, 210, 636, 908, 664, 527, 552, 892, 962, 981, 492, 595, 768, 603, 736, 561, 983, 1409, 814, 943, 1491, 375, 1567, 666, 183, 789, 1266, 825] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1147, 327, 818, 924, 322, 836, 830, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1242, 215, 611, 343, 330, 1298, 933, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 694, 937, 848, 647, 244, 523, 293, 1401, 343, 178, 0, 0, 0, 0, 0, 0, 532, 711, 772, 393, 695, 80, 900, 1413, 845, 0, 0, 0, 0, 0, 0, 0, 791, 841, 1057, 1065, 456, 223, 515, 957, 57, 0, 0, 0, 0, 0, 0, 0, 639, 630, 582, 840, 904, 933, 542, 642, 211, 201, 28, 0, 0, 0, 0, 0, 773, 549, 885, 488, 990, 747, 738, 879, 0, 0, 0, 0, 0, 0, 0, 0, 538, 363, 1410, 720, 135, 727, 1222, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 6124, 6108, 6341, 5962, 6152, 6049, 6138] avg=6144.0000 max=6341 max_rank=3 min=5962 min_rank=4 imbalance_ratio=1.032064 max_load_reduction=42811 max_load_reduction_pct=0.870992 moved_to_echo=24301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1146, 270, 813, 863, 299, 826, 790, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 200, 536, 307, 416, 1230, 907, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 743, 940, 790, 605, 312, 499, 307, 1391, 325, 176, 0, 0, 0, 0, 0, 0, 561, 654, 771, 357, 681, 77, 917, 1366, 838, 0, 0, 0, 0, 0, 0, 0, 844, 867, 978, 1135, 437, 233, 501, 924, 54, 0, 0, 0, 0, 0, 0, 0, 691, 601, 538, 891, 921, 943, 585, 779, 200, 201, 27, 0, 0, 0, 0, 0, 791, 582, 839, 483, 1003, 707, 828, 910, 0, 0, 0, 0, 0, 0, 0, 0, 534, 497, 1466, 701, 174, 754, 1202, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6078, 5914, 6088, 6222, 5973, 6377, 6143, 6357] avg=6144.0000 max=6377 max_rank=5 min=5914 min_rank=1 imbalance_ratio=1.037923 max_load_reduction=42775 max_load_reduction_pct=0.870260 moved_to_echo=24850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1213, 336, 755, 935, 309, 1151, 881, 797, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 321, 758, 313, 288, 1566, 822, 912, 0, 0, 0, 0, 0, 0, 0, 0, 714, 964, 829, 647, 324, 630, 278, 1300, 378, 163, 0, 0, 0, 0, 0, 0, 376, 663, 837, 445, 699, 165, 740, 1265, 775, 0, 0, 0, 0, 0, 0, 0, 834, 797, 1100, 1187, 534, 191, 578, 970, 69, 0, 0, 0, 0, 0, 0, 0, 543, 594, 622, 733, 922, 1004, 465, 456, 233, 213, 34, 0, 0, 0, 0, 0, 749, 608, 755, 560, 923, 824, 867, 778, 0, 0, 0, 0, 0, 0, 0, 0, 494, 317, 1533, 700, 196, 768, 1315, 900, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6377, 6217, 6227, 5965, 6260, 5819, 6064, 6223] avg=6144.0000 max=6377 max_rank=0 min=5819 min_rank=5 imbalance_ratio=1.037923 max_load_reduction=42775 max_load_reduction_pct=0.870260 moved_to_echo=24366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1042, 257, 788, 939, 315, 1027, 844, 924, 0, 0, 0, 0, 0, 0, 0, 0, 1242, 220, 550, 357, 385, 1225, 926, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 744, 1060, 788, 527, 342, 513, 242, 1489, 324, 174, 0, 0, 0, 0, 0, 0, 428, 642, 858, 311, 680, 136, 956, 1449, 827, 0, 0, 0, 0, 0, 0, 0, 815, 810, 1059, 1145, 362, 212, 567, 939, 54, 0, 0, 0, 0, 0, 0, 0, 578, 651, 590, 894, 905, 868, 569, 639, 199, 183, 27, 0, 0, 0, 0, 0, 692, 521, 756, 532, 1119, 705, 964, 897, 0, 0, 0, 0, 0, 0, 0, 0, 528, 378, 1492, 793, 177, 725, 1165, 920, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6096, 6203, 6287, 5963, 6103, 6186, 6178] avg=6144.0000 max=6287 max_rank=3 min=5963 min_rank=4 imbalance_ratio=1.023275 max_load_reduction=42865 max_load_reduction_pct=0.872091 moved_to_echo=24430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1138, 263, 856, 910, 310, 927, 839, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1350, 225, 589, 330, 369, 1354, 968, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 723, 1029, 846, 675, 293, 536, 265, 1351, 358, 180, 0, 0, 0, 0, 0, 0, 557, 671, 724, 377, 727, 95, 838, 1290, 858, 0, 0, 0, 0, 0, 0, 0, 790, 831, 961, 1010, 504, 206, 574, 1022, 60, 0, 0, 0, 0, 0, 0, 0, 655, 566, 556, 754, 884, 998, 534, 678, 220, 199, 30, 0, 0, 0, 0, 0, 795, 522, 910, 444, 1003, 778, 880, 863, 0, 0, 0, 0, 0, 0, 0, 0, 547, 308, 1500, 650, 159, 771, 1168, 902, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6266, 6261, 6256, 6137, 5958, 6074, 6195, 6005] avg=6144.0000 max=6266 max_rank=0 min=5958 min_rank=4 imbalance_ratio=1.019857 max_load_reduction=42886 max_load_reduction_pct=0.872518 moved_to_echo=24232 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1140, 281, 722, 829, 273, 885, 741, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 196, 599, 331, 366, 1260, 915, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 661, 928, 766, 587, 278, 508, 264, 1501, 359, 171, 0, 0, 0, 0, 0, 0, 542, 683, 741, 388, 713, 101, 879, 1138, 814, 0, 0, 0, 0, 0, 0, 0, 868, 795, 1119, 1017, 730, 223, 478, 1118, 56, 0, 0, 0, 0, 0, 0, 0, 642, 596, 555, 980, 882, 984, 608, 659, 221, 200, 28, 0, 0, 0, 0, 0, 687, 509, 820, 576, 1028, 783, 835, 880, 0, 0, 0, 0, 0, 0, 0, 0, 519, 379, 1403, 713, 120, 721, 1152, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6220, 6023, 5999, 6404, 6355, 6118, 6056] avg=6144.0000 max=6404 max_rank=4 min=5977 min_rank=0 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1114, 301, 768, 889, 249, 987, 843, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 231, 658, 318, 382, 1370, 946, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 713, 1016, 769, 600, 308, 528, 269, 1374, 329, 175, 0, 0, 0, 0, 0, 0, 447, 653, 863, 361, 730, 124, 850, 1453, 833, 0, 0, 0, 0, 0, 0, 0, 894, 833, 1060, 1135, 370, 195, 650, 878, 61, 0, 0, 0, 0, 0, 0, 0, 613, 600, 569, 899, 967, 928, 498, 634, 202, 195, 30, 0, 0, 0, 0, 0, 760, 612, 756, 538, 1020, 715, 877, 912, 0, 0, 0, 0, 0, 0, 0, 0, 531, 265, 1450, 738, 187, 737, 1223, 885, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6191, 6081, 6314, 6076, 6135, 6190, 6016] avg=6144.0000 max=6314 max_rank=3 min=6016 min_rank=7 imbalance_ratio=1.027669 max_load_reduction=42838 max_load_reduction_pct=0.871541 moved_to_echo=24417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1042, 257, 788, 939, 315, 1027, 844, 924, 0, 0, 0, 0, 0, 0, 0, 0, 1242, 220, 550, 357, 385, 1225, 926, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 744, 1060, 788, 527, 342, 513, 242, 1489, 324, 174, 0, 0, 0, 0, 0, 0, 428, 642, 858, 311, 680, 136, 956, 1449, 827, 0, 0, 0, 0, 0, 0, 0, 815, 810, 1059, 1145, 362, 212, 567, 939, 54, 0, 0, 0, 0, 0, 0, 0, 578, 651, 590, 894, 905, 868, 569, 639, 199, 183, 27, 0, 0, 0, 0, 0, 692, 521, 756, 532, 1119, 705, 964, 897, 0, 0, 0, 0, 0, 0, 0, 0, 528, 378, 1492, 793, 177, 725, 1165, 920, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6096, 6203, 6287, 5963, 6103, 6186, 6178] avg=6144.0000 max=6287 max_rank=3 min=5963 min_rank=4 imbalance_ratio=1.023275 max_load_reduction=42865 max_load_reduction_pct=0.872091 moved_to_echo=24430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1146, 270, 813, 863, 299, 826, 790, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 200, 536, 307, 416, 1230, 907, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 743, 940, 790, 605, 312, 499, 307, 1391, 325, 176, 0, 0, 0, 0, 0, 0, 561, 654, 771, 357, 681, 77, 917, 1366, 838, 0, 0, 0, 0, 0, 0, 0, 844, 867, 978, 1135, 437, 233, 501, 924, 54, 0, 0, 0, 0, 0, 0, 0, 691, 601, 538, 891, 921, 943, 585, 779, 200, 201, 27, 0, 0, 0, 0, 0, 791, 582, 839, 483, 1003, 707, 828, 910, 0, 0, 0, 0, 0, 0, 0, 0, 534, 497, 1466, 701, 174, 754, 1202, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6078, 5914, 6088, 6222, 5973, 6377, 6143, 6357] avg=6144.0000 max=6377 max_rank=5 min=5914 min_rank=1 imbalance_ratio=1.037923 max_load_reduction=42775 max_load_reduction_pct=0.870260 moved_to_echo=24850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1140, 281, 722, 829, 273, 885, 741, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 196, 599, 331, 366, 1260, 915, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 661, 928, 766, 587, 278, 508, 264, 1501, 359, 171, 0, 0, 0, 0, 0, 0, 542, 683, 741, 388, 713, 101, 879, 1138, 814, 0, 0, 0, 0, 0, 0, 0, 868, 795, 1119, 1017, 730, 223, 478, 1118, 56, 0, 0, 0, 0, 0, 0, 0, 642, 596, 555, 980, 882, 984, 608, 659, 221, 200, 28, 0, 0, 0, 0, 0, 687, 509, 820, 576, 1028, 783, 835, 880, 0, 0, 0, 0, 0, 0, 0, 0, 519, 379, 1403, 713, 120, 721, 1152, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6220, 6023, 5999, 6404, 6355, 6118, 6056] avg=6144.0000 max=6404 max_rank=4 min=5977 min_rank=0 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1137, 263, 746, 867, 265, 792, 905, 916, 0, 0, 0, 0, 0, 0, 0, 0, 1315, 257, 582, 338, 295, 1354, 903, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 701, 1019, 805, 671, 267, 536, 284, 1334, 375, 174, 0, 0, 0, 0, 0, 0, 455, 589, 777, 457, 798, 95, 801, 1106, 809, 0, 0, 0, 0, 0, 0, 0, 840, 795, 1306, 1139, 658, 210, 636, 908, 64, 0, 0, 0, 0, 0, 0, 0, 664, 527, 552, 892, 962, 981, 492, 595, 235, 204, 33, 0, 0, 0, 0, 0, 768, 603, 736, 561, 983, 799, 814, 943, 0, 0, 0, 0, 0, 0, 0, 0, 508, 375, 1567, 666, 183, 789, 1266, 825, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5891, 6129, 6166, 5887, 6556, 6137, 6207, 6179] avg=6144.0000 max=6556 max_rank=4 min=5887 min_rank=3 imbalance_ratio=1.067057 max_load_reduction=42596 max_load_reduction_pct=0.866618 moved_to_echo=25079 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1213, 336, 755, 935, 309, 1151, 881, 797, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 321, 758, 313, 288, 1566, 822, 912, 0, 0, 0, 0, 0, 0, 0, 0, 714, 964, 829, 647, 324, 630, 278, 1300, 378, 163, 0, 0, 0, 0, 0, 0, 376, 663, 837, 445, 699, 165, 740, 1265, 775, 0, 0, 0, 0, 0, 0, 0, 834, 797, 1100, 1187, 534, 191, 578, 970, 69, 0, 0, 0, 0, 0, 0, 0, 543, 594, 622, 733, 922, 1004, 465, 456, 233, 213, 34, 0, 0, 0, 0, 0, 749, 608, 755, 560, 923, 824, 867, 778, 0, 0, 0, 0, 0, 0, 0, 0, 494, 317, 1533, 700, 196, 768, 1315, 900, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6377, 6217, 6227, 5965, 6260, 5819, 6064, 6223] avg=6144.0000 max=6377 max_rank=0 min=5819 min_rank=5 imbalance_ratio=1.037923 max_load_reduction=42775 max_load_reduction_pct=0.870260 moved_to_echo=24366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6784, 6656, 8704, 8960, 4096, 1792, 4608, 7808, 512, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9088, 2304, 6272, 7168, 2432, 7552, 6784, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 7936, 6528, 4992, 2432, 4352, 2304, 11264, 2816, 1408, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 4864, 4608, 6912, 7424, 7680, 4352, 5120, 1792, 1664, 256, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10368, 1920, 4992, 2688, 2944, 10752, 7424, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4224, 2944, 11904, 5760, 1408, 6016, 9728, 7552, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 7936, 6528, 4992, 2432, 4352, 2304, 11264, 2816, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9088, 2304, 6272, 7168, 2432, 7552, 6784, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6784, 6656, 8704, 8960, 4096, 1792, 4608, 7808, 512, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 4608, 6528, 4224, 8192, 6144, 6912, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4224, 2944, 11904, 5760, 1408, 6016, 9728, 7552, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3968, 5376, 6400, 3200, 5760, 896, 6912, 10496, 6656, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[674, 826, 1091, 440, 990, 236, 307, 1111, 90, 941, 1367, 410, 606, 1049, 953, 1475, 1120, 1082, 1030, 416, 1421, 143, 1670, 1064, 698, 641, 667, 826, 108, 354, 173, 108, 429, 271, 490, 194, 1089, 1809, 1090, 1048, 2163, 1532, 1150, 1320, 431, 968, 22, 192, 744, 1248, 829, 14, 160, 1431, 238, 1237, 972, 358, 844, 567, 959, 575, 288, 403] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[728, 825, 1014, 496, 905, 231, 322, 1193, 97, 983, 1271, 302, 553, 1014, 998, 1476, 1189, 1306, 887, 424, 1496, 188, 1852, 1069, 587, 669, 609, 750, 103, 373, 215, 120, 488, 310, 574, 186, 1025, 1847, 980, 1009, 2279, 1432, 1274, 1192, 437, 945, 55, 190, 743, 1289, 750, 24, 130, 1629, 228, 1357, 797, 315, 695, 534, 979, 555, 266, 393] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[744, 849, 893, 406, 1009, 257, 234, 1225, 85, 911, 1584, 375, 634, 994, 804, 1560, 1134, 1161, 895, 466, 1330, 162, 1807, 1055, 687, 651, 696, 899, 81, 300, 239, 125, 426, 287, 448, 114, 1231, 1539, 1055, 1051, 2211, 1542, 1014, 1255, 410, 959, 56, 216, 902, 1238, 946, 9, 150, 1711, 207, 1251, 871, 323, 755, 520, 972, 517, 305, 409] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[695, 746, 1037, 557, 875, 173, 337, 1287, 108, 1077, 1218, 342, 565, 998, 969, 1467, 1202, 1188, 832, 425, 1315, 163, 1537, 1298, 594, 798, 656, 879, 157, 383, 228, 106, 399, 254, 561, 144, 1251, 1661, 1029, 924, 2231, 1512, 1155, 1473, 415, 710, 30, 183, 786, 1203, 1064, 15, 124, 1526, 240, 1083, 816, 361, 861, 593, 920, 590, 351, 475] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[663, 836, 1022, 425, 1036, 268, 320, 1055, 148, 959, 1346, 341, 713, 1088, 879, 1473, 1176, 1245, 931, 412, 1460, 114, 1661, 976, 616, 643, 687, 837, 111, 292, 233, 126, 379, 203, 527, 227, 1205, 1749, 1081, 1123, 2132, 1553, 1327, 1413, 485, 733, 26, 180, 661, 1234, 898, 12, 151, 1453, 246, 1227, 746, 305, 802, 591, 1017, 568, 334, 472] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[641, 789, 1064, 477, 917, 255, 304, 1082, 112, 922, 1209, 372, 694, 1114, 858, 1480, 1219, 1211, 920, 436, 1565, 170, 1631, 1080, 647, 665, 763, 821, 96, 355, 229, 77, 435, 240, 509, 189, 1214, 1750, 1158, 1060, 2097, 1507, 1098, 1452, 427, 707, 33, 159, 739, 1190, 883, 23, 176, 1601, 207, 1223, 860, 378, 939, 591, 876, 593, 264, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[744, 849, 893, 406, 1009, 257, 234, 1225, 85, 911, 1584, 375, 634, 994, 804, 1560, 1134, 1161, 895, 466, 1330, 162, 1807, 1055, 687, 651, 696, 899, 81, 300, 239, 125, 426, 287, 448, 114, 1231, 1539, 1055, 1051, 2211, 1542, 1014, 1255, 410, 959, 56, 216, 902, 1238, 946, 9, 150, 1711, 207, 1251, 871, 323, 755, 520, 972, 517, 305, 409] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[663, 836, 1022, 425, 1036, 268, 320, 1055, 148, 959, 1346, 341, 713, 1088, 879, 1473, 1176, 1245, 931, 412, 1460, 114, 1661, 976, 616, 643, 687, 837, 111, 292, 233, 126, 379, 203, 527, 227, 1205, 1749, 1081, 1123, 2132, 1553, 1327, 1413, 485, 733, 26, 180, 661, 1234, 898, 12, 151, 1453, 246, 1227, 746, 305, 802, 591, 1017, 568, 334, 472] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[750, 909, 800, 384, 1025, 224, 272, 1212, 108, 922, 1534, 397, 517, 807, 778, 1589, 998, 1381, 884, 475, 1393, 254, 1847, 1181, 755, 796, 796, 911, 129, 330, 209, 114, 438, 329, 415, 98, 1175, 1329, 1091, 889, 2172, 1498, 894, 1188, 447, 878, 78, 210, 1042, 1180, 1069, 18, 152, 1524, 229, 1289, 938, 317, 775, 537, 1005, 560, 341, 366] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[674, 826, 1091, 440, 990, 236, 307, 1111, 90, 941, 1367, 410, 606, 1049, 953, 1475, 1120, 1082, 1030, 416, 1421, 143, 1670, 1064, 698, 641, 667, 826, 108, 354, 173, 108, 429, 271, 490, 194, 1089, 1809, 1090, 1048, 2163, 1532, 1150, 1320, 431, 968, 22, 192, 744, 1248, 829, 14, 160, 1431, 238, 1237, 972, 358, 844, 567, 959, 575, 288, 403] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[690, 798, 856, 535, 1083, 186, 312, 1428, 97, 1012, 1413, 414, 597, 1094, 820, 1574, 1122, 1110, 863, 383, 1186, 202, 1609, 1379, 737, 636, 754, 909, 106, 391, 223, 125, 403, 254, 451, 99, 1157, 1330, 889, 896, 2280, 1469, 940, 1229, 374, 973, 45, 251, 1002, 1309, 1140, 11, 124, 1503, 229, 1095, 955, 342, 904, 572, 830, 635, 358, 459] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[695, 746, 1037, 557, 875, 173, 337, 1287, 239, 150, 45, 23, 0, 0, 0, 0, 108, 1077, 1218, 342, 565, 998, 969, 800, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 1188, 832, 425, 1076, 163, 0, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 594, 798, 656, 879, 157, 383, 228, 106, 1537, 754, 0, 0, 0, 0, 0, 0, 399, 254, 561, 144, 1251, 1616, 1029, 924, 0, 0, 0, 0, 0, 0, 0, 0, 700, 1512, 1155, 1473, 415, 710, 30, 183, 0, 0, 0, 0, 0, 0, 0, 0, 786, 1203, 1064, 15, 124, 1503, 240, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 816, 361, 861, 593, 920, 590, 351, 475, 777, 517, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 6077, 6184, 6092, 6178, 6178, 6018, 6261] avg=6144.0000 max=6261 max_rank=7 min=6018 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[728, 825, 1014, 496, 905, 231, 322, 1193, 272, 151, 50, 25, 0, 0, 0, 0, 97, 983, 1271, 302, 553, 1014, 998, 805, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 1306, 887, 424, 1224, 188, 0, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 587, 669, 609, 750, 103, 373, 215, 120, 1852, 770, 0, 0, 0, 0, 0, 0, 488, 310, 574, 186, 1025, 1797, 980, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 716, 1432, 1274, 1192, 437, 945, 55, 190, 0, 0, 0, 0, 0, 0, 0, 0, 743, 1289, 750, 24, 130, 1604, 228, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 797, 315, 695, 534, 979, 555, 266, 393, 793, 520, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6212, 6023, 6287, 6048, 6369, 6241, 6125, 5847] avg=6144.0000 max=6369 max_rank=4 min=5847 min_rank=7 imbalance_ratio=1.036621 max_load_reduction=42783 max_load_reduction_pct=0.870422 moved_to_echo=24582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[674, 826, 1091, 440, 990, 236, 307, 1111, 258, 151, 49, 22, 0, 0, 0, 0, 90, 941, 1367, 410, 606, 1049, 953, 804, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 1082, 1030, 416, 1163, 143, 0, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 698, 641, 667, 826, 108, 354, 173, 108, 1670, 731, 0, 0, 0, 0, 0, 0, 429, 271, 490, 194, 1089, 1760, 1090, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 679, 1532, 1150, 1320, 431, 968, 22, 192, 0, 0, 0, 0, 0, 0, 0, 0, 744, 1248, 829, 14, 160, 1409, 238, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 972, 358, 844, 567, 959, 575, 288, 403, 753, 520, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6220, 6018, 5976, 6371, 6294, 5879, 6239] avg=6144.0000 max=6371 max_rank=4 min=5879 min_rank=6 imbalance_ratio=1.036947 max_load_reduction=42781 max_load_reduction_pct=0.870382 moved_to_echo=24783 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[641, 789, 1064, 477, 917, 255, 304, 1082, 285, 151, 47, 25, 0, 0, 0, 0, 112, 922, 1209, 372, 694, 1114, 858, 807, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 1211, 920, 436, 1280, 170, 0, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 647, 665, 763, 821, 96, 355, 229, 77, 1631, 709, 0, 0, 0, 0, 0, 0, 435, 240, 509, 189, 1214, 1703, 1158, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 658, 1507, 1098, 1452, 427, 707, 33, 159, 0, 0, 0, 0, 0, 0, 0, 0, 739, 1190, 883, 23, 176, 1576, 207, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 860, 378, 939, 591, 876, 593, 264, 399, 730, 522, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6037, 6088, 6316, 5993, 6508, 6041, 6017, 6152] avg=6144.0000 max=6508 max_rank=4 min=5993 min_rank=3 imbalance_ratio=1.059245 max_load_reduction=42644 max_load_reduction_pct=0.867594 moved_to_echo=24718 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[744, 849, 893, 406, 1009, 257, 234, 1225, 242, 159, 41, 26, 0, 0, 0, 0, 85, 911, 1584, 375, 634, 994, 804, 851, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 1161, 895, 466, 1088, 162, 0, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 687, 651, 696, 899, 81, 300, 239, 125, 1807, 747, 0, 0, 0, 0, 0, 0, 426, 287, 448, 114, 1231, 1498, 1055, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 694, 1542, 1014, 1255, 410, 959, 56, 216, 0, 0, 0, 0, 0, 0, 0, 0, 902, 1238, 946, 9, 150, 1685, 207, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 871, 323, 755, 520, 972, 517, 305, 409, 770, 550, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 6238, 5961, 6232, 6110, 6146, 6388, 5992] avg=6144.0000 max=6388 max_rank=6 min=5961 min_rank=2 imbalance_ratio=1.039714 max_load_reduction=42764 max_load_reduction_pct=0.870036 moved_to_echo=24636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[695, 746, 1037, 557, 875, 173, 337, 1287, 239, 150, 45, 23, 0, 0, 0, 0, 108, 1077, 1218, 342, 565, 998, 969, 800, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 1188, 832, 425, 1076, 163, 0, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 594, 798, 656, 879, 157, 383, 228, 106, 1537, 754, 0, 0, 0, 0, 0, 0, 399, 254, 561, 144, 1251, 1616, 1029, 924, 0, 0, 0, 0, 0, 0, 0, 0, 700, 1512, 1155, 1473, 415, 710, 30, 183, 0, 0, 0, 0, 0, 0, 0, 0, 786, 1203, 1064, 15, 124, 1503, 240, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 816, 361, 861, 593, 920, 590, 351, 475, 777, 517, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 6077, 6184, 6092, 6178, 6178, 6018, 6261] avg=6144.0000 max=6261 max_rank=7 min=6018 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[750, 909, 800, 384, 1025, 224, 272, 1212, 253, 162, 36, 23, 0, 0, 0, 0, 108, 922, 1534, 397, 517, 807, 778, 867, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1381, 884, 475, 1140, 254, 0, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 755, 796, 796, 911, 129, 330, 209, 114, 1847, 734, 0, 0, 0, 0, 0, 0, 438, 329, 415, 98, 1175, 1293, 1091, 889, 0, 0, 0, 0, 0, 0, 0, 0, 682, 1498, 894, 1188, 447, 878, 78, 210, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1180, 1069, 18, 152, 1501, 229, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 938, 317, 775, 537, 1005, 560, 341, 366, 756, 560, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6050, 5930, 6313, 6621, 5728, 5875, 6480, 6155] avg=6144.0000 max=6621 max_rank=3 min=5728 min_rank=4 imbalance_ratio=1.077637 max_load_reduction=42531 max_load_reduction_pct=0.865295 moved_to_echo=24238 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[663, 836, 1022, 425, 1036, 268, 320, 1055, 266, 150, 47, 22, 0, 0, 0, 0, 148, 959, 1346, 341, 713, 1088, 879, 804, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 1245, 931, 412, 1194, 114, 0, 976, 0, 0, 0, 0, 0, 0, 0, 0, 616, 643, 687, 837, 111, 292, 233, 126, 1661, 720, 0, 0, 0, 0, 0, 0, 379, 203, 527, 227, 1205, 1702, 1081, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 670, 1553, 1327, 1413, 485, 733, 26, 180, 0, 0, 0, 0, 0, 0, 0, 0, 661, 1234, 898, 12, 151, 1431, 246, 1227, 0, 0, 0, 0, 0, 0, 0, 0, 746, 305, 802, 591, 1017, 568, 334, 472, 742, 519, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 6278, 6048, 5926, 6447, 6387, 5860, 6096] avg=6144.0000 max=6447 max_rank=4 min=5860 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=42705 max_load_reduction_pct=0.868835 moved_to_echo=24790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[690, 798, 856, 535, 1083, 186, 312, 1428, 220, 165, 38, 28, 0, 0, 0, 0, 97, 1012, 1413, 414, 597, 1094, 820, 851, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 1110, 863, 383, 966, 202, 0, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 737, 636, 754, 909, 106, 391, 223, 125, 1609, 774, 0, 0, 0, 0, 0, 0, 403, 254, 451, 99, 1157, 1292, 889, 896, 0, 0, 0, 0, 0, 0, 0, 0, 709, 1469, 940, 1229, 374, 973, 45, 251, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1309, 1140, 11, 124, 1475, 229, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 955, 342, 904, 572, 830, 635, 358, 459, 797, 558, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6339, 6298, 6025, 6264, 5441, 5990, 6385, 6410] avg=6144.0000 max=6410 max_rank=7 min=5441 min_rank=4 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[674, 826, 1091, 440, 990, 236, 307, 1111, 258, 151, 49, 22, 0, 0, 0, 0, 90, 941, 1367, 410, 606, 1049, 953, 804, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 1082, 1030, 416, 1163, 143, 0, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 698, 641, 667, 826, 108, 354, 173, 108, 1670, 731, 0, 0, 0, 0, 0, 0, 429, 271, 490, 194, 1089, 1760, 1090, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 679, 1532, 1150, 1320, 431, 968, 22, 192, 0, 0, 0, 0, 0, 0, 0, 0, 744, 1248, 829, 14, 160, 1409, 238, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 972, 358, 844, 567, 959, 575, 288, 403, 753, 520, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6220, 6018, 5976, 6371, 6294, 5879, 6239] avg=6144.0000 max=6371 max_rank=4 min=5879 min_rank=6 imbalance_ratio=1.036947 max_load_reduction=42781 max_load_reduction_pct=0.870382 moved_to_echo=24783 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[728, 825, 1014, 496, 905, 231, 322, 1193, 272, 151, 50, 25, 0, 0, 0, 0, 97, 983, 1271, 302, 553, 1014, 998, 805, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 1306, 887, 424, 1224, 188, 0, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 587, 669, 609, 750, 103, 373, 215, 120, 1852, 770, 0, 0, 0, 0, 0, 0, 488, 310, 574, 186, 1025, 1797, 980, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 716, 1432, 1274, 1192, 437, 945, 55, 190, 0, 0, 0, 0, 0, 0, 0, 0, 743, 1289, 750, 24, 130, 1604, 228, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 797, 315, 695, 534, 979, 555, 266, 393, 793, 520, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6212, 6023, 6287, 6048, 6369, 6241, 6125, 5847] avg=6144.0000 max=6369 max_rank=4 min=5847 min_rank=7 imbalance_ratio=1.036621 max_load_reduction=42783 max_load_reduction_pct=0.870422 moved_to_echo=24582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[750, 909, 800, 384, 1025, 224, 272, 1212, 253, 162, 36, 23, 0, 0, 0, 0, 108, 922, 1534, 397, 517, 807, 778, 867, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1381, 884, 475, 1140, 254, 0, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 755, 796, 796, 911, 129, 330, 209, 114, 1847, 734, 0, 0, 0, 0, 0, 0, 438, 329, 415, 98, 1175, 1293, 1091, 889, 0, 0, 0, 0, 0, 0, 0, 0, 682, 1498, 894, 1188, 447, 878, 78, 210, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1180, 1069, 18, 152, 1501, 229, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 938, 317, 775, 537, 1005, 560, 341, 366, 756, 560, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6050, 5930, 6313, 6621, 5728, 5875, 6480, 6155] avg=6144.0000 max=6621 max_rank=3 min=5728 min_rank=4 imbalance_ratio=1.077637 max_load_reduction=42531 max_load_reduction_pct=0.865295 moved_to_echo=24238 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 12160, 8960, 10624, 3456, 6912, 384, 1664, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 7808, 11008, 3072, 4992, 8192, 7168, 6656, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 5504, 5632, 6912, 896, 2816, 1792, 1024, 13696, 6016, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[7040, 2816, 6656, 4608, 7680, 4608, 2560, 3456, 6144, 4352, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 6656, 7808, 3840, 7936, 1920, 2432, 9600, 2048, 1280, 384, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[7040, 2816, 6656, 4608, 7680, 4608, 2560, 3456, 6144, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 5504, 5632, 6912, 896, 2816, 1792, 1024, 13696, 6016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3456, 2176, 4096, 1280, 9472, 12672, 8448, 8064, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9216, 9728, 7296, 3456, 9216, 1408, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6656, 9984, 7680, 128, 1280, 12288, 1920, 9856, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3456, 2176, 4096, 1280, 9472, 12672, 8448, 8064, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[298, 1004, 436, 436, 96, 1138, 862, 261, 822, 362, 1081, 1241, 1705, 1490, 785, 1461, 951, 1286, 281, 466, 82, 357, 1334, 972, 306, 666, 849, 434, 499, 525, 863, 350, 344, 977, 1059, 710, 1694, 1587, 1811, 558, 528, 273, 338, 252, 1617, 1150, 188, 1360, 1006, 321, 916, 737, 143, 338, 335, 864, 1364, 360, 72, 604, 191, 1344, 505, 1907] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[245, 949, 465, 478, 135, 1193, 976, 237, 849, 357, 1063, 1146, 1720, 1343, 715, 1558, 910, 1229, 267, 472, 72, 385, 1213, 854, 389, 693, 807, 547, 386, 600, 942, 377, 371, 1044, 1099, 742, 1851, 1667, 1921, 527, 489, 259, 321, 295, 1609, 1176, 151, 1341, 925, 289, 890, 823, 109, 339, 288, 724, 1397, 419, 55, 650, 135, 1308, 440, 1926] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[101, 1002, 503, 344, 81, 1175, 1011, 199, 736, 332, 1207, 1230, 1505, 1121, 254, 1364, 954, 1608, 306, 458, 223, 278, 1296, 1016, 344, 733, 1187, 784, 416, 476, 955, 196, 372, 971, 1005, 538, 2219, 1561, 1677, 501, 575, 244, 363, 266, 1525, 1423, 208, 1534, 1045, 209, 712, 820, 93, 144, 280, 706, 1248, 372, 80, 658, 156, 1692, 311, 2249] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[220, 878, 433, 421, 100, 1069, 1108, 231, 682, 247, 1026, 1022, 1539, 1567, 867, 1770, 969, 1308, 236, 411, 225, 396, 1121, 965, 272, 801, 1054, 605, 384, 503, 793, 336, 396, 986, 982, 659, 1791, 1423, 1884, 545, 511, 342, 469, 260, 1570, 1128, 206, 1728, 1154, 321, 606, 826, 135, 301, 319, 795, 1192, 328, 91, 825, 193, 1202, 618, 1807] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[182, 816, 547, 338, 88, 1101, 888, 233, 717, 352, 1084, 1198, 1647, 1219, 594, 1603, 951, 1329, 287, 394, 96, 382, 1252, 1041, 329, 681, 944, 600, 355, 539, 962, 309, 334, 951, 1124, 707, 1934, 1712, 1923, 565, 403, 312, 468, 270, 1607, 1129, 174, 1389, 999, 239, 890, 892, 95, 274, 308, 783, 1404, 463, 57, 601, 170, 1385, 427, 2105] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[202, 851, 584, 461, 61, 1419, 983, 253, 860, 306, 1053, 1074, 1658, 1161, 380, 1462, 1049, 1295, 302, 403, 98, 332, 1049, 1019, 450, 629, 911, 684, 367, 516, 965, 317, 383, 1125, 1177, 680, 1725, 1728, 1969, 527, 473, 231, 309, 259, 1589, 1203, 173, 1378, 1193, 253, 927, 847, 209, 393, 287, 818, 1376, 402, 78, 422, 180, 1135, 423, 2126] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[220, 878, 433, 421, 100, 1069, 1108, 231, 682, 247, 1026, 1022, 1539, 1567, 867, 1770, 969, 1308, 236, 411, 225, 396, 1121, 965, 272, 801, 1054, 605, 384, 503, 793, 336, 396, 986, 982, 659, 1791, 1423, 1884, 545, 511, 342, 469, 260, 1570, 1128, 206, 1728, 1154, 321, 606, 826, 135, 301, 319, 795, 1192, 328, 91, 825, 193, 1202, 618, 1807] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[245, 949, 465, 478, 135, 1193, 976, 237, 849, 357, 1063, 1146, 1720, 1343, 715, 1558, 910, 1229, 267, 472, 72, 385, 1213, 854, 389, 693, 807, 547, 386, 600, 942, 377, 371, 1044, 1099, 742, 1851, 1667, 1921, 527, 489, 259, 321, 295, 1609, 1176, 151, 1341, 925, 289, 890, 823, 109, 339, 288, 724, 1397, 419, 55, 650, 135, 1308, 440, 1926] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[92, 1065, 611, 387, 35, 1408, 847, 249, 659, 392, 1147, 1227, 1785, 835, 126, 992, 1109, 1475, 472, 381, 124, 203, 1088, 1050, 631, 782, 1173, 1052, 337, 479, 884, 203, 392, 1159, 1301, 497, 2461, 1991, 1608, 466, 647, 222, 319, 333, 1449, 1280, 202, 1218, 974, 125, 779, 733, 103, 186, 271, 692, 1499, 423, 79, 396, 144, 1528, 161, 2214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[182, 816, 547, 338, 88, 1101, 888, 233, 717, 352, 1084, 1198, 1647, 1219, 594, 1603, 951, 1329, 287, 394, 96, 382, 1252, 1041, 329, 681, 944, 600, 355, 539, 962, 309, 334, 951, 1124, 707, 1934, 1712, 1923, 565, 403, 312, 468, 270, 1607, 1129, 174, 1389, 999, 239, 890, 892, 95, 274, 308, 783, 1404, 463, 57, 601, 170, 1385, 427, 2105] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[80, 1122, 437, 422, 33, 1260, 1033, 228, 641, 311, 1314, 1373, 1328, 1031, 243, 1372, 1052, 1361, 586, 341, 156, 239, 1234, 1115, 483, 686, 1189, 855, 398, 499, 1011, 195, 403, 1063, 1100, 425, 2425, 1573, 1607, 523, 687, 234, 360, 406, 1435, 1297, 165, 1468, 1097, 167, 492, 822, 87, 165, 264, 650, 1334, 306, 124, 614, 120, 1642, 286, 2183] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[298, 1004, 436, 436, 96, 1138, 862, 261, 1492, 155, 0, 0, 0, 0, 0, 0, 822, 362, 1081, 1241, 0, 1490, 785, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1286, 281, 466, 82, 357, 1334, 972, 376, 0, 0, 0, 0, 0, 0, 0, 306, 666, 849, 434, 499, 525, 863, 350, 978, 213, 34, 0, 0, 0, 0, 0, 344, 977, 1059, 710, 0, 1587, 833, 558, 0, 0, 0, 0, 0, 0, 0, 0, 528, 273, 338, 252, 1617, 1150, 188, 1360, 271, 1, 0, 0, 0, 0, 0, 0, 1006, 321, 916, 737, 143, 338, 335, 864, 1539, 0, 0, 0, 0, 0, 0, 0, 1364, 360, 72, 604, 191, 1344, 505, 1636, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6831, 6105, 5717, 6068, 5978, 6199, 6076] avg=6144.0000 max=6831 max_rank=1 min=5717 min_rank=3 imbalance_ratio=1.111816 max_load_reduction=42321 max_load_reduction_pct=0.861023 moved_to_echo=24321 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[245, 949, 465, 478, 135, 1193, 976, 237, 1505, 169, 0, 0, 0, 0, 0, 0, 849, 357, 1063, 1146, 0, 1343, 715, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 910, 1229, 267, 472, 72, 385, 1213, 854, 401, 0, 0, 0, 0, 0, 0, 0, 389, 693, 807, 547, 386, 600, 942, 377, 1037, 215, 37, 0, 0, 0, 0, 0, 371, 1044, 1099, 742, 0, 1667, 884, 527, 0, 0, 0, 0, 0, 0, 0, 0, 489, 259, 321, 295, 1609, 1176, 151, 1341, 274, 1, 0, 0, 0, 0, 0, 0, 925, 289, 890, 823, 109, 339, 288, 724, 1682, 0, 0, 0, 0, 0, 0, 0, 1397, 419, 55, 650, 135, 1308, 440, 1652, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 6592, 5803, 6030, 6334, 5916, 6069, 6056] avg=6144.0000 max=6592 max_rank=1 min=5803 min_rank=2 imbalance_ratio=1.072917 max_load_reduction=42560 max_load_reduction_pct=0.865885 moved_to_echo=24375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[220, 878, 433, 421, 100, 1069, 1108, 231, 1347, 164, 0, 0, 0, 0, 0, 0, 682, 247, 1026, 1022, 0, 1567, 867, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1308, 236, 411, 225, 396, 1121, 965, 456, 0, 0, 0, 0, 0, 0, 0, 272, 801, 1054, 605, 384, 503, 793, 336, 1017, 192, 42, 0, 0, 0, 0, 0, 396, 986, 982, 659, 0, 1423, 867, 545, 0, 0, 0, 0, 0, 0, 0, 0, 511, 342, 469, 260, 1570, 1128, 206, 1728, 257, 1, 0, 0, 0, 0, 0, 0, 1154, 321, 606, 826, 135, 301, 319, 795, 1627, 0, 0, 0, 0, 0, 0, 0, 1192, 328, 91, 825, 193, 1202, 618, 1550, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5971, 6682, 6087, 5999, 5858, 6472, 6084, 5999] avg=6144.0000 max=6682 max_rank=1 min=5858 min_rank=4 imbalance_ratio=1.087565 max_load_reduction=42470 max_load_reduction_pct=0.864054 moved_to_echo=24413 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[101, 1002, 503, 344, 81, 1175, 1011, 199, 1317, 204, 0, 0, 0, 0, 0, 0, 736, 332, 1207, 1230, 0, 1121, 254, 980, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1608, 306, 458, 223, 278, 1296, 1016, 351, 0, 0, 0, 0, 0, 0, 0, 344, 733, 1187, 784, 416, 476, 955, 196, 905, 188, 32, 0, 0, 0, 0, 0, 372, 971, 1005, 538, 0, 1561, 772, 501, 0, 0, 0, 0, 0, 0, 0, 0, 575, 244, 363, 266, 1525, 1423, 208, 1534, 320, 1, 0, 0, 0, 0, 0, 0, 1045, 209, 712, 820, 93, 144, 280, 706, 2015, 0, 0, 0, 0, 0, 0, 0, 1248, 372, 80, 658, 156, 1692, 311, 1929, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5937, 5860, 6490, 6216, 5720, 6459, 6024, 6446] avg=6144.0000 max=6490 max_rank=2 min=5720 min_rank=4 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[202, 851, 584, 461, 61, 1419, 983, 253, 1450, 158, 0, 0, 0, 0, 0, 0, 860, 306, 1053, 1074, 0, 1161, 380, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1295, 302, 403, 98, 332, 1049, 1019, 376, 0, 0, 0, 0, 0, 0, 0, 450, 629, 911, 684, 367, 516, 965, 317, 1063, 208, 34, 0, 0, 0, 0, 0, 383, 1125, 1177, 680, 0, 1728, 906, 527, 0, 0, 0, 0, 0, 0, 0, 0, 473, 231, 309, 259, 1589, 1203, 173, 1378, 302, 1, 0, 0, 0, 0, 0, 0, 1193, 253, 927, 847, 209, 393, 287, 818, 1567, 0, 0, 0, 0, 0, 0, 0, 1376, 402, 78, 422, 180, 1135, 423, 1824, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6422, 5885, 5923, 6144, 6526, 5918, 6494, 5840] avg=6144.0000 max=6526 max_rank=4 min=5840 min_rank=7 imbalance_ratio=1.062174 max_load_reduction=42626 max_load_reduction_pct=0.867228 moved_to_echo=24778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[182, 816, 547, 338, 88, 1101, 888, 233, 1441, 177, 0, 0, 0, 0, 0, 0, 717, 352, 1084, 1198, 0, 1219, 594, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1329, 287, 394, 96, 382, 1252, 1041, 413, 0, 0, 0, 0, 0, 0, 0, 329, 681, 944, 600, 355, 539, 962, 309, 1038, 206, 38, 0, 0, 0, 0, 0, 334, 951, 1124, 707, 0, 1712, 885, 565, 0, 0, 0, 0, 0, 0, 0, 0, 403, 312, 468, 270, 1607, 1129, 174, 1389, 300, 1, 0, 0, 0, 0, 0, 0, 999, 239, 890, 892, 95, 274, 308, 783, 1757, 0, 0, 0, 0, 0, 0, 0, 1404, 463, 57, 601, 170, 1385, 427, 1805, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5811, 6315, 6145, 6001, 6278, 6053, 6237, 6312] avg=6144.0000 max=6315 max_rank=1 min=5811 min_rank=0 imbalance_ratio=1.027832 max_load_reduction=42837 max_load_reduction_pct=0.871521 moved_to_echo=24880 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[92, 1065, 611, 387, 35, 1408, 847, 249, 1562, 226, 0, 0, 0, 0, 0, 0, 659, 392, 1147, 1227, 0, 835, 126, 702, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 1475, 472, 381, 124, 203, 1088, 1050, 260, 0, 0, 0, 0, 0, 0, 0, 631, 782, 1173, 1052, 337, 479, 884, 203, 872, 223, 27, 0, 0, 0, 0, 0, 392, 1159, 1301, 497, 0, 1991, 736, 466, 0, 0, 0, 0, 0, 0, 0, 0, 647, 222, 319, 333, 1449, 1280, 202, 1218, 319, 3, 0, 0, 0, 0, 0, 0, 974, 125, 779, 733, 103, 186, 271, 692, 2235, 0, 0, 0, 0, 0, 0, 0, 1499, 423, 79, 396, 144, 1528, 161, 1895, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6482, 5088, 6162, 6663, 6542, 5992, 6098, 6125] avg=6144.0000 max=6663 max_rank=3 min=5088 min_rank=1 imbalance_ratio=1.084473 max_load_reduction=42489 max_load_reduction_pct=0.864441 moved_to_echo=24757 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[202, 851, 584, 461, 61, 1419, 983, 253, 1450, 158, 0, 0, 0, 0, 0, 0, 860, 306, 1053, 1074, 0, 1161, 380, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1295, 302, 403, 98, 332, 1049, 1019, 376, 0, 0, 0, 0, 0, 0, 0, 450, 629, 911, 684, 367, 516, 965, 317, 1063, 208, 34, 0, 0, 0, 0, 0, 383, 1125, 1177, 680, 0, 1728, 906, 527, 0, 0, 0, 0, 0, 0, 0, 0, 473, 231, 309, 259, 1589, 1203, 173, 1378, 302, 1, 0, 0, 0, 0, 0, 0, 1193, 253, 927, 847, 209, 393, 287, 818, 1567, 0, 0, 0, 0, 0, 0, 0, 1376, 402, 78, 422, 180, 1135, 423, 1824, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6422, 5885, 5923, 6144, 6526, 5918, 6494, 5840] avg=6144.0000 max=6526 max_rank=4 min=5840 min_rank=7 imbalance_ratio=1.062174 max_load_reduction=42626 max_load_reduction_pct=0.867228 moved_to_echo=24778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[245, 949, 465, 478, 135, 1193, 976, 237, 1505, 169, 0, 0, 0, 0, 0, 0, 849, 357, 1063, 1146, 0, 1343, 715, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 910, 1229, 267, 472, 72, 385, 1213, 854, 401, 0, 0, 0, 0, 0, 0, 0, 389, 693, 807, 547, 386, 600, 942, 377, 1037, 215, 37, 0, 0, 0, 0, 0, 371, 1044, 1099, 742, 0, 1667, 884, 527, 0, 0, 0, 0, 0, 0, 0, 0, 489, 259, 321, 295, 1609, 1176, 151, 1341, 274, 1, 0, 0, 0, 0, 0, 0, 925, 289, 890, 823, 109, 339, 288, 724, 1682, 0, 0, 0, 0, 0, 0, 0, 1397, 419, 55, 650, 135, 1308, 440, 1652, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 6592, 5803, 6030, 6334, 5916, 6069, 6056] avg=6144.0000 max=6592 max_rank=1 min=5803 min_rank=2 imbalance_ratio=1.072917 max_load_reduction=42560 max_load_reduction_pct=0.865885 moved_to_echo=24375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[80, 1122, 437, 422, 33, 1260, 1033, 228, 1162, 222, 0, 0, 0, 0, 0, 0, 641, 311, 1314, 1373, 0, 1031, 243, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1361, 586, 341, 156, 239, 1234, 1115, 353, 0, 0, 0, 0, 0, 0, 0, 483, 686, 1189, 855, 398, 499, 1011, 195, 868, 166, 32, 0, 0, 0, 0, 0, 403, 1063, 1100, 425, 0, 1573, 739, 523, 0, 0, 0, 0, 0, 0, 0, 0, 687, 234, 360, 406, 1435, 1297, 165, 1468, 311, 1, 0, 0, 0, 0, 0, 0, 1097, 167, 492, 822, 87, 165, 264, 650, 2203, 0, 0, 0, 0, 0, 0, 0, 1334, 306, 124, 614, 120, 1642, 286, 1872, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 5899, 6437, 6382, 5826, 6364, 5947, 6298] avg=6144.0000 max=6437 max_rank=2 min=5826 min_rank=4 imbalance_ratio=1.047689 max_load_reduction=42715 max_load_reduction_pct=0.869039 moved_to_echo=24435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[92, 1065, 611, 387, 35, 1408, 847, 249, 1562, 226, 0, 0, 0, 0, 0, 0, 659, 392, 1147, 1227, 0, 835, 126, 702, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 1475, 472, 381, 124, 203, 1088, 1050, 260, 0, 0, 0, 0, 0, 0, 0, 631, 782, 1173, 1052, 337, 479, 884, 203, 872, 223, 27, 0, 0, 0, 0, 0, 392, 1159, 1301, 497, 0, 1991, 736, 466, 0, 0, 0, 0, 0, 0, 0, 0, 647, 222, 319, 333, 1449, 1280, 202, 1218, 319, 3, 0, 0, 0, 0, 0, 0, 974, 125, 779, 733, 103, 186, 271, 692, 2235, 0, 0, 0, 0, 0, 0, 0, 1499, 423, 79, 396, 144, 1528, 161, 1895, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6482, 5088, 6162, 6663, 6542, 5992, 6098, 6125] avg=6144.0000 max=6663 max_rank=3 min=5088 min_rank=1 imbalance_ratio=1.084473 max_load_reduction=42489 max_load_reduction_pct=0.864441 moved_to_echo=24757 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8448, 2048, 6272, 6528, 1024, 2176, 2432, 6144, 14720, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4352, 2176, 3072, 2432, 12416, 9856, 1536, 11520, 2432, 128, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3072, 8320, 8960, 4992, 0, 13312, 6656, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4352, 2176, 3072, 2432, 12416, 9856, 1536, 11520, 2432, 128, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6016, 2688, 9088, 9600, 0, 9856, 3968, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8448, 2048, 6272, 6528, 1024, 2176, 2432, 6144, 14720, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3328, 5760, 8192, 5632, 3200, 4224, 7424, 2304, 7808, 1664, 384, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1536, 7808, 4096, 3328, 640, 9856, 7808, 1920, 11392, 1536, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8064, 11008, 2816, 3328, 1152, 2688, 9600, 8064, 3072, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10880, 3200, 640, 4864, 1408, 11264, 3200, 14208, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1536, 7808, 4096, 3328, 640, 9856, 7808, 1920, 11392, 1536, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[481, 812, 1368, 405, 516, 557, 769, 649, 1422, 2582, 248, 428, 1246, 555, 978, 392, 1785, 71, 2425, 508, 547, 347, 15, 727, 964, 903, 707, 522, 588, 814, 1233, 909, 221, 728, 251, 305, 844, 605, 1700, 613, 437, 2328, 139, 583, 348, 561, 473, 3161, 110, 310, 216, 909, 545, 1034, 329, 537, 342, 414, 1321, 675, 887, 667, 583, 503] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[523, 1185, 1291, 652, 542, 572, 734, 685, 1159, 2127, 202, 416, 1083, 653, 759, 390, 1456, 75, 2175, 470, 544, 287, 18, 786, 1031, 655, 543, 504, 674, 623, 1520, 1049, 253, 724, 324, 401, 730, 568, 2074, 683, 500, 1948, 150, 460, 388, 840, 681, 3024, 220, 292, 389, 960, 572, 1040, 350, 578, 311, 542, 1386, 529, 953, 821, 675, 403] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[466, 937, 1286, 399, 562, 474, 782, 710, 1359, 2376, 201, 384, 1159, 600, 1082, 391, 1484, 102, 2149, 529, 536, 328, 28, 786, 1160, 713, 682, 509, 539, 860, 1325, 1011, 216, 768, 294, 355, 728, 594, 1857, 642, 421, 2332, 165, 550, 355, 560, 514, 3168, 240, 355, 259, 958, 671, 1080, 430, 614, 275, 310, 1350, 629, 918, 719, 596, 320] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[279, 466, 1649, 358, 739, 790, 727, 909, 2250, 3074, 398, 390, 1115, 601, 1348, 143, 2043, 62, 1918, 493, 679, 459, 6, 617, 980, 990, 648, 447, 610, 939, 1406, 716, 319, 613, 240, 297, 751, 556, 1252, 637, 224, 2807, 170, 480, 238, 560, 365, 2866, 10, 243, 193, 1029, 364, 972, 213, 259, 322, 220, 1408, 570, 844, 915, 461, 505] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[466, 937, 1286, 399, 562, 474, 782, 710, 1359, 2376, 201, 384, 1159, 600, 1082, 391, 1484, 102, 2149, 529, 536, 328, 28, 786, 1160, 713, 682, 509, 539, 860, 1325, 1011, 216, 768, 294, 355, 728, 594, 1857, 642, 421, 2332, 165, 550, 355, 560, 514, 3168, 240, 355, 259, 958, 671, 1080, 430, 614, 275, 310, 1350, 629, 918, 719, 596, 320] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[523, 1185, 1291, 652, 542, 572, 734, 685, 1159, 2127, 202, 416, 1083, 653, 759, 390, 1456, 75, 2175, 470, 544, 287, 18, 786, 1031, 655, 543, 504, 674, 623, 1520, 1049, 253, 724, 324, 401, 730, 568, 2074, 683, 500, 1948, 150, 460, 388, 840, 681, 3024, 220, 292, 389, 960, 572, 1040, 350, 578, 311, 542, 1386, 529, 953, 821, 675, 403] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[318, 605, 1663, 352, 575, 584, 749, 924, 1626, 2901, 263, 413, 1143, 443, 1119, 300, 1664, 87, 2257, 451, 721, 429, 4, 677, 1140, 894, 632, 486, 639, 941, 1286, 770, 296, 697, 197, 326, 861, 652, 1829, 791, 245, 2491, 143, 473, 337, 499, 366, 3134, 26, 215, 204, 1148, 526, 921, 192, 288, 261, 251, 1299, 579, 1031, 856, 519, 443] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[481, 812, 1368, 405, 516, 557, 769, 649, 1422, 2582, 248, 428, 1246, 555, 978, 392, 1785, 71, 2425, 508, 547, 347, 15, 727, 964, 903, 707, 522, 588, 814, 1233, 909, 221, 728, 251, 305, 844, 605, 1700, 613, 437, 2328, 139, 583, 348, 561, 473, 3161, 110, 310, 216, 909, 545, 1034, 329, 537, 342, 414, 1321, 675, 887, 667, 583, 503] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[527, 800, 1477, 369, 600, 563, 818, 745, 1701, 2581, 280, 363, 1206, 640, 1099, 318, 1327, 72, 2236, 497, 570, 409, 11, 762, 1179, 1069, 676, 489, 540, 821, 1273, 845, 252, 625, 241, 327, 1038, 554, 1628, 752, 376, 2473, 162, 568, 355, 496, 498, 3272, 90, 224, 209, 1044, 426, 959, 300, 425, 272, 276, 1117, 656, 1025, 753, 574, 322] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[456, 858, 1414, 437, 673, 547, 739, 682, 1583, 2739, 328, 416, 1122, 643, 1020, 363, 1589, 69, 1925, 546, 639, 269, 20, 656, 1202, 836, 623, 451, 605, 827, 1388, 951, 262, 664, 328, 339, 734, 702, 1440, 668, 554, 2642, 163, 473, 233, 595, 497, 2950, 93, 295, 339, 943, 437, 902, 427, 525, 342, 397, 1317, 600, 791, 918, 558, 408] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[318, 605, 1663, 352, 575, 584, 749, 924, 1626, 2901, 263, 413, 1143, 443, 1119, 300, 1664, 87, 2257, 451, 721, 429, 4, 677, 1140, 894, 632, 486, 639, 941, 1286, 770, 296, 697, 197, 326, 861, 652, 1829, 791, 245, 2491, 143, 473, 337, 499, 366, 3134, 26, 215, 204, 1148, 526, 921, 192, 288, 261, 251, 1299, 579, 1031, 856, 519, 443] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[257, 689, 1625, 461, 640, 560, 764, 671, 1700, 2935, 256, 440, 1063, 501, 1298, 415, 1965, 100, 2005, 459, 674, 318, 17, 697, 1102, 822, 560, 515, 672, 833, 1604, 917, 272, 760, 271, 329, 901, 470, 1736, 791, 272, 2101, 154, 401, 249, 596, 374, 2748, 41, 271, 206, 956, 663, 936, 299, 327, 466, 404, 1324, 509, 969, 898, 496, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[466, 937, 1286, 399, 562, 474, 782, 710, 328, 0, 0, 0, 0, 0, 0, 0, 1359, 640, 201, 384, 1159, 600, 1082, 391, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 102, 2149, 529, 536, 328, 28, 786, 69, 0, 0, 0, 0, 0, 0, 0, 1160, 713, 682, 509, 539, 860, 700, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 216, 768, 294, 355, 728, 594, 1857, 642, 570, 228, 0, 0, 0, 0, 0, 0, 421, 2332, 165, 550, 355, 560, 514, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 240, 355, 259, 958, 671, 1080, 430, 614, 1736, 317, 0, 0, 0, 0, 0, 0, 275, 310, 1350, 629, 918, 719, 596, 320, 875, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5944, 5816, 6011, 6174, 6252, 6303, 6660, 5992] avg=6144.0000 max=6660 max_rank=6 min=5816 min_rank=1 imbalance_ratio=1.083984 max_load_reduction=42492 max_load_reduction_pct=0.864502 moved_to_echo=25207 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 1185, 1291, 652, 542, 572, 734, 685, 376, 0, 0, 0, 0, 0, 0, 0, 1159, 573, 202, 416, 1083, 653, 759, 390, 0, 0, 0, 0, 0, 0, 0, 0, 1456, 75, 2175, 470, 544, 287, 18, 786, 79, 0, 0, 0, 0, 0, 0, 0, 1031, 655, 543, 504, 674, 623, 804, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 253, 724, 324, 401, 730, 568, 2074, 683, 544, 261, 0, 0, 0, 0, 0, 0, 500, 1948, 150, 460, 388, 840, 681, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 220, 292, 389, 960, 572, 1040, 350, 578, 1554, 302, 0, 0, 0, 0, 0, 0, 311, 542, 1386, 529, 953, 821, 675, 403, 835, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6560, 5235, 5890, 5883, 6562, 6310, 6257, 6455] avg=6144.0000 max=6562 max_rank=4 min=5235 min_rank=1 imbalance_ratio=1.068034 max_load_reduction=42590 max_load_reduction_pct=0.866496 moved_to_echo=25584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[481, 812, 1368, 405, 516, 557, 769, 649, 305, 0, 0, 0, 0, 0, 0, 0, 1422, 696, 248, 428, 1246, 555, 978, 392, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 71, 2425, 508, 547, 347, 15, 727, 64, 0, 0, 0, 0, 0, 0, 0, 964, 903, 707, 522, 588, 814, 652, 909, 0, 0, 0, 0, 0, 0, 0, 0, 221, 728, 251, 305, 844, 605, 1700, 613, 568, 212, 0, 0, 0, 0, 0, 0, 437, 2328, 139, 583, 348, 561, 473, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 110, 310, 216, 909, 545, 1034, 329, 537, 1886, 316, 0, 0, 0, 0, 0, 0, 342, 414, 1321, 675, 887, 667, 583, 503, 873, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5862, 5965, 6489, 6059, 6047, 6273, 6192, 6265] avg=6144.0000 max=6489 max_rank=2 min=5862 min_rank=0 imbalance_ratio=1.056152 max_load_reduction=42663 max_load_reduction_pct=0.867981 moved_to_echo=24777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[456, 858, 1414, 437, 673, 547, 739, 682, 344, 0, 0, 0, 0, 0, 0, 0, 1583, 738, 328, 416, 1122, 643, 1020, 363, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 69, 1925, 546, 639, 269, 20, 656, 72, 0, 0, 0, 0, 0, 0, 0, 1202, 836, 623, 451, 605, 827, 734, 951, 0, 0, 0, 0, 0, 0, 0, 0, 262, 664, 328, 339, 734, 702, 1440, 668, 530, 238, 0, 0, 0, 0, 0, 0, 554, 2642, 163, 473, 233, 595, 497, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 93, 295, 339, 943, 437, 902, 427, 525, 2001, 295, 0, 0, 0, 0, 0, 0, 342, 397, 1317, 600, 791, 918, 558, 408, 815, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6213, 5785, 6229, 5905, 6467, 6257, 6146] avg=6144.0000 max=6467 max_rank=5 min=5785 min_rank=2 imbalance_ratio=1.052572 max_load_reduction=42685 max_load_reduction_pct=0.868429 moved_to_echo=24775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[318, 605, 1663, 352, 575, 584, 749, 924, 318, 0, 0, 0, 0, 0, 0, 0, 1626, 781, 263, 413, 1143, 443, 1119, 300, 0, 0, 0, 0, 0, 0, 0, 0, 1664, 87, 2257, 451, 721, 429, 4, 677, 67, 0, 0, 0, 0, 0, 0, 0, 1140, 894, 632, 486, 639, 941, 680, 770, 0, 0, 0, 0, 0, 0, 0, 0, 296, 697, 197, 326, 861, 652, 1829, 791, 563, 221, 0, 0, 0, 0, 0, 0, 245, 2491, 143, 473, 337, 499, 366, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 26, 215, 204, 1148, 526, 921, 192, 288, 2120, 314, 0, 0, 0, 0, 0, 0, 261, 251, 1299, 579, 1031, 856, 519, 443, 865, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6088, 6088, 6357, 6182, 6433, 5946, 5954, 6104] avg=6144.0000 max=6433 max_rank=4 min=5946 min_rank=5 imbalance_ratio=1.047038 max_load_reduction=42719 max_load_reduction_pct=0.869120 moved_to_echo=24437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[527, 800, 1477, 369, 600, 563, 818, 745, 315, 0, 0, 0, 0, 0, 0, 0, 1701, 695, 280, 363, 1206, 640, 1099, 318, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 72, 2236, 497, 570, 409, 11, 762, 66, 0, 0, 0, 0, 0, 0, 0, 1179, 1069, 676, 489, 540, 821, 673, 845, 0, 0, 0, 0, 0, 0, 0, 0, 252, 625, 241, 327, 1038, 554, 1628, 752, 588, 219, 0, 0, 0, 0, 0, 0, 376, 2473, 162, 568, 355, 496, 498, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 90, 224, 209, 1044, 426, 959, 300, 425, 1886, 327, 0, 0, 0, 0, 0, 0, 272, 276, 1117, 656, 1025, 753, 574, 322, 903, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 6302, 5950, 6292, 6224, 6382, 5890, 5898] avg=6144.0000 max=6382 max_rank=5 min=5890 min_rank=6 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[279, 466, 1649, 358, 739, 790, 727, 909, 353, 0, 0, 0, 0, 0, 0, 0, 2250, 824, 398, 390, 1115, 601, 1348, 143, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 62, 1918, 493, 679, 459, 6, 617, 75, 0, 0, 0, 0, 0, 0, 0, 980, 990, 648, 447, 610, 939, 734, 716, 0, 0, 0, 0, 0, 0, 0, 0, 319, 613, 240, 297, 751, 556, 1252, 637, 520, 244, 0, 0, 0, 0, 0, 0, 224, 2807, 170, 480, 238, 560, 365, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 10, 243, 193, 1029, 364, 972, 213, 259, 2250, 291, 0, 0, 0, 0, 0, 0, 322, 220, 1408, 570, 844, 915, 461, 505, 795, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6270, 7069, 6352, 6064, 5429, 6104, 5824, 6040] avg=6144.0000 max=7069 max_rank=1 min=5429 min_rank=4 imbalance_ratio=1.150553 max_load_reduction=42083 max_load_reduction_pct=0.856181 moved_to_echo=23397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[481, 812, 1368, 405, 516, 557, 769, 649, 305, 0, 0, 0, 0, 0, 0, 0, 1422, 696, 248, 428, 1246, 555, 978, 392, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 71, 2425, 508, 547, 347, 15, 727, 64, 0, 0, 0, 0, 0, 0, 0, 964, 903, 707, 522, 588, 814, 652, 909, 0, 0, 0, 0, 0, 0, 0, 0, 221, 728, 251, 305, 844, 605, 1700, 613, 568, 212, 0, 0, 0, 0, 0, 0, 437, 2328, 139, 583, 348, 561, 473, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 110, 310, 216, 909, 545, 1034, 329, 537, 1886, 316, 0, 0, 0, 0, 0, 0, 342, 414, 1321, 675, 887, 667, 583, 503, 873, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5862, 5965, 6489, 6059, 6047, 6273, 6192, 6265] avg=6144.0000 max=6489 max_rank=2 min=5862 min_rank=0 imbalance_ratio=1.056152 max_load_reduction=42663 max_load_reduction_pct=0.867981 moved_to_echo=24777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[257, 689, 1625, 461, 640, 560, 764, 671, 397, 0, 0, 0, 0, 0, 0, 0, 1700, 791, 256, 440, 1063, 501, 1298, 415, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 100, 2005, 459, 674, 318, 17, 697, 83, 0, 0, 0, 0, 0, 0, 0, 1102, 822, 560, 515, 672, 833, 848, 917, 0, 0, 0, 0, 0, 0, 0, 0, 272, 760, 271, 329, 901, 470, 1736, 791, 494, 276, 0, 0, 0, 0, 0, 0, 272, 2101, 154, 401, 249, 596, 374, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 41, 271, 206, 956, 663, 936, 299, 327, 2144, 275, 0, 0, 0, 0, 0, 0, 466, 404, 1324, 509, 969, 898, 496, 427, 759, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6064, 6464, 6318, 6269, 6300, 5367, 6118, 6252] avg=6144.0000 max=6464 max_rank=1 min=5367 min_rank=5 imbalance_ratio=1.052083 max_load_reduction=42688 max_load_reduction_pct=0.868490 moved_to_echo=24037 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[279, 466, 1649, 358, 739, 790, 727, 909, 353, 0, 0, 0, 0, 0, 0, 0, 2250, 824, 398, 390, 1115, 601, 1348, 143, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 62, 1918, 493, 679, 459, 6, 617, 75, 0, 0, 0, 0, 0, 0, 0, 980, 990, 648, 447, 610, 939, 734, 716, 0, 0, 0, 0, 0, 0, 0, 0, 319, 613, 240, 297, 751, 556, 1252, 637, 520, 244, 0, 0, 0, 0, 0, 0, 224, 2807, 170, 480, 238, 560, 365, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 10, 243, 193, 1029, 364, 972, 213, 259, 2250, 291, 0, 0, 0, 0, 0, 0, 322, 220, 1408, 570, 844, 915, 461, 505, 795, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6270, 7069, 6352, 6064, 5429, 6104, 5824, 6040] avg=6144.0000 max=7069 max_rank=1 min=5429 min_rank=4 imbalance_ratio=1.150553 max_load_reduction=42083 max_load_reduction_pct=0.856181 moved_to_echo=23397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[456, 858, 1414, 437, 673, 547, 739, 682, 344, 0, 0, 0, 0, 0, 0, 0, 1583, 738, 328, 416, 1122, 643, 1020, 363, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 69, 1925, 546, 639, 269, 20, 656, 72, 0, 0, 0, 0, 0, 0, 0, 1202, 836, 623, 451, 605, 827, 734, 951, 0, 0, 0, 0, 0, 0, 0, 0, 262, 664, 328, 339, 734, 702, 1440, 668, 530, 238, 0, 0, 0, 0, 0, 0, 554, 2642, 163, 473, 233, 595, 497, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 93, 295, 339, 943, 437, 902, 427, 525, 2001, 295, 0, 0, 0, 0, 0, 0, 342, 397, 1317, 600, 791, 918, 558, 408, 815, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6213, 5785, 6229, 5905, 6467, 6257, 6146] avg=6144.0000 max=6467 max_rank=5 min=5785 min_rank=2 imbalance_ratio=1.052572 max_load_reduction=42685 max_load_reduction_pct=0.868429 moved_to_echo=24775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13440, 640, 17152, 3968, 4992, 2944, 128, 5760, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 5632, 2176, 2688, 6656, 4736, 13568, 5632, 4480, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 19200, 1280, 4096, 2560, 4736, 3840, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13440, 640, 17152, 3968, 4992, 2944, 128, 5760, 640, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 2304, 2048, 8064, 4224, 7936, 2560, 3584, 15616, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8832, 6912, 5120, 3968, 4992, 6784, 5888, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 19200, 1280, 4096, 2560, 4736, 3840, 10880, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3328, 6400, 11776, 3456, 4864, 4736, 6144, 6016, 2816, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12800, 5760, 2176, 3328, 9216, 4736, 8704, 2816, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 2816, 10624, 4864, 7424, 6656, 4480, 3456, 6784, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8832, 6912, 5120, 3968, 4992, 6784, 5888, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12800, 5760, 2176, 3328, 9216, 4736, 8704, 2816, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1384, 1490, 965, 1176, 1200, 667, 1424, 447, 648, 2326, 124, 156, 396, 342, 926, 1565, 39, 190, 88, 169, 531, 278, 249, 600, 374, 2501, 316, 2316, 763, 585, 65, 290, 3, 16, 1382, 877, 278, 90, 1258, 286, 317, 139, 959, 671, 0, 2244, 0, 107, 959, 256, 1539, 414, 1235, 1232, 1238, 706, 583, 179, 1560, 1063, 2155, 1621, 1195] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[20, 1888, 1488, 544, 1193, 1648, 689, 1136, 526, 718, 2105, 130, 162, 536, 397, 850, 1545, 47, 185, 139, 259, 428, 285, 224, 447, 335, 1935, 487, 2008, 1054, 707, 91, 382, 40, 32, 1367, 785, 557, 163, 1332, 374, 749, 455, 1032, 606, 26, 1954, 43, 230, 630, 261, 1242, 440, 1297, 1052, 1061, 694, 542, 224, 1210, 1140, 2301, 1365, 1360] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[3, 1819, 1640, 724, 1208, 1084, 744, 1186, 279, 790, 2240, 152, 168, 421, 352, 936, 1323, 120, 176, 76, 138, 416, 235, 362, 766, 418, 1967, 490, 1942, 775, 741, 98, 315, 11, 40, 1185, 1111, 335, 118, 1406, 389, 419, 250, 936, 708, 1, 2292, 4, 214, 761, 279, 1262, 509, 1299, 874, 1280, 946, 458, 206, 1230, 1203, 2509, 1563, 1250] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[2, 1764, 1648, 736, 1173, 1117, 780, 1004, 477, 787, 1979, 110, 185, 417, 370, 1105, 1499, 148, 147, 89, 189, 389, 223, 217, 564, 436, 1890, 504, 1593, 1115, 826, 106, 371, 13, 46, 930, 1093, 537, 197, 1315, 336, 619, 335, 904, 744, 2, 2011, 0, 329, 654, 257, 1456, 512, 1176, 901, 1315, 756, 588, 198, 1210, 1269, 2572, 1533, 1384] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[31, 2025, 1578, 577, 1148, 1267, 923, 863, 516, 514, 1683, 245, 298, 388, 775, 1348, 1332, 92, 205, 219, 212, 317, 218, 290, 613, 330, 1383, 707, 1293, 1255, 864, 172, 463, 97, 143, 847, 1408, 928, 307, 1141, 323, 316, 784, 864, 900, 26, 1666, 47, 573, 353, 305, 1665, 512, 888, 697, 1131, 978, 463, 306, 889, 1105, 2490, 1426, 1430] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[7, 1675, 1601, 659, 1210, 1103, 880, 694, 398, 679, 1837, 148, 188, 406, 369, 1171, 1367, 99, 196, 115, 191, 369, 228, 371, 684, 412, 1540, 678, 1488, 1228, 706, 92, 487, 43, 58, 716, 1298, 587, 188, 1397, 435, 699, 497, 1053, 790, 15, 1794, 13, 451, 531, 297, 1797, 513, 1059, 843, 1276, 956, 570, 292, 1110, 1171, 2608, 1457, 1362] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[3, 1536, 1707, 794, 1168, 1016, 733, 1307, 335, 847, 2161, 73, 143, 519, 382, 869, 1408, 169, 215, 83, 191, 451, 289, 307, 699, 404, 1656, 523, 1676, 995, 627, 80, 497, 21, 41, 1131, 1094, 669, 163, 1423, 312, 620, 400, 964, 620, 4, 2002, 15, 197, 687, 251, 1225, 456, 1268, 890, 1127, 919, 543, 233, 1078, 1077, 2554, 1535, 1770] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[31, 2025, 1578, 577, 1148, 1267, 923, 863, 516, 514, 1683, 245, 298, 388, 775, 1348, 1332, 92, 205, 219, 212, 317, 218, 290, 613, 330, 1383, 707, 1293, 1255, 864, 172, 463, 97, 143, 847, 1408, 928, 307, 1141, 323, 316, 784, 864, 900, 26, 1666, 47, 573, 353, 305, 1665, 512, 888, 697, 1131, 978, 463, 306, 889, 1105, 2490, 1426, 1430] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[3, 1819, 1640, 724, 1208, 1084, 744, 1186, 279, 790, 2240, 152, 168, 421, 352, 936, 1323, 120, 176, 76, 138, 416, 235, 362, 766, 418, 1967, 490, 1942, 775, 741, 98, 315, 11, 40, 1185, 1111, 335, 118, 1406, 389, 419, 250, 936, 708, 1, 2292, 4, 214, 761, 279, 1262, 509, 1299, 874, 1280, 946, 458, 206, 1230, 1203, 2509, 1563, 1250] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1384, 1490, 965, 1176, 1200, 667, 1424, 447, 648, 2326, 124, 156, 396, 342, 926, 1565, 39, 190, 88, 169, 531, 278, 249, 600, 374, 2501, 316, 2316, 763, 585, 65, 290, 3, 16, 1382, 877, 278, 90, 1258, 286, 317, 139, 959, 671, 0, 2244, 0, 107, 959, 256, 1539, 414, 1235, 1232, 1238, 706, 583, 179, 1560, 1063, 2155, 1621, 1195] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 648, 1435, 1153, 1046, 1166, 647, 1664, 487, 798, 2839, 74, 136, 540, 323, 723, 1429, 36, 156, 39, 149, 545, 393, 201, 388, 360, 2300, 244, 2712, 672, 428, 92, 333, 2, 2, 1957, 758, 227, 51, 1294, 261, 394, 18, 833, 517, 0, 2824, 0, 51, 926, 201, 1603, 465, 1674, 1440, 1123, 525, 514, 148, 1557, 1008, 1977, 1184, 1462] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[20, 1888, 1488, 544, 1193, 1648, 689, 1136, 526, 718, 2105, 130, 162, 536, 397, 850, 1545, 47, 185, 139, 259, 428, 285, 224, 447, 335, 1935, 487, 2008, 1054, 707, 91, 382, 40, 32, 1367, 785, 557, 163, 1332, 374, 749, 455, 1032, 606, 26, 1954, 43, 230, 630, 261, 1242, 440, 1297, 1052, 1061, 694, 542, 224, 1210, 1140, 2301, 1365, 1360] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1177, 794, 1168, 1016, 733, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 335, 847, 2161, 73, 143, 519, 382, 869, 373, 242, 0, 0, 0, 0, 0, 0, 1408, 169, 215, 83, 191, 451, 289, 307, 2554, 739, 0, 0, 0, 0, 0, 0, 699, 404, 897, 523, 1676, 995, 627, 80, 0, 0, 0, 0, 0, 0, 0, 0, 497, 21, 41, 1131, 1094, 669, 163, 1423, 797, 535, 0, 0, 0, 0, 0, 0, 312, 620, 400, 964, 620, 4, 2002, 15, 676, 288, 224, 0, 0, 0, 0, 0, 197, 687, 251, 852, 456, 1268, 890, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 919, 543, 233, 1078, 1077, 0, 859, 1770, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6198, 5944, 6406, 5901, 6371, 6125, 5728, 6479] avg=6144.0000 max=6479 max_rank=7 min=5728 min_rank=6 imbalance_ratio=1.054525 max_load_reduction=42673 max_load_reduction_pct=0.868184 moved_to_echo=24703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 1104, 659, 1210, 1103, 880, 694, 0, 0, 0, 0, 0, 0, 0, 0, 398, 679, 1837, 148, 188, 406, 369, 1171, 547, 227, 0, 0, 0, 0, 0, 0, 1367, 99, 196, 115, 191, 369, 228, 371, 2608, 806, 0, 0, 0, 0, 0, 0, 684, 412, 834, 678, 1488, 1228, 706, 92, 0, 0, 0, 0, 0, 0, 0, 0, 487, 43, 58, 716, 1298, 587, 188, 1397, 869, 497, 0, 0, 0, 0, 0, 0, 435, 699, 497, 1053, 790, 15, 1794, 13, 642, 270, 209, 0, 0, 0, 0, 0, 451, 531, 297, 1250, 513, 1059, 843, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 956, 570, 292, 1110, 1171, 0, 815, 1362, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5657, 5970, 6350, 6122, 6140, 6417, 6220, 6276] avg=6144.0000 max=6417 max_rank=5 min=5657 min_rank=0 imbalance_ratio=1.044434 max_load_reduction=42735 max_load_reduction_pct=0.869446 moved_to_echo=25053 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1136, 736, 1173, 1117, 780, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 477, 787, 1979, 110, 185, 417, 370, 1105, 443, 234, 0, 0, 0, 0, 0, 0, 1499, 148, 147, 89, 189, 389, 223, 217, 2572, 850, 0, 0, 0, 0, 0, 0, 564, 436, 1024, 504, 1593, 1115, 826, 106, 0, 0, 0, 0, 0, 0, 0, 0, 371, 13, 46, 930, 1093, 537, 197, 1315, 914, 610, 0, 0, 0, 0, 0, 0, 336, 619, 335, 904, 744, 2, 2011, 0, 675, 278, 256, 0, 0, 0, 0, 0, 329, 654, 257, 1013, 512, 1176, 901, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 756, 588, 198, 1210, 1269, 0, 858, 1384, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5948, 6107, 6323, 6168, 6026, 6160, 6157, 6263] avg=6144.0000 max=6323 max_rank=2 min=5948 min_rank=0 imbalance_ratio=1.029134 max_load_reduction=42829 max_load_reduction_pct=0.871358 moved_to_echo=24606 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 1088, 577, 1148, 1267, 923, 863, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 1683, 245, 298, 388, 775, 1348, 507, 224, 0, 0, 0, 0, 0, 0, 1332, 92, 205, 219, 212, 317, 218, 290, 2490, 975, 0, 0, 0, 0, 0, 0, 613, 330, 750, 707, 1293, 1255, 864, 172, 0, 0, 0, 0, 0, 0, 0, 0, 463, 97, 143, 847, 1408, 928, 307, 1141, 1050, 446, 0, 0, 0, 0, 0, 0, 323, 316, 784, 864, 900, 26, 1666, 47, 628, 266, 187, 0, 0, 0, 0, 0, 573, 353, 305, 1158, 512, 888, 697, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 978, 463, 306, 889, 1105, 0, 798, 1430, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5897, 6498, 6350, 5984, 6830, 6007, 5617, 5969] avg=6144.0000 max=6830 max_rank=4 min=5617 min_rank=6 imbalance_ratio=1.111654 max_load_reduction=42322 max_load_reduction_pct=0.861043 moved_to_echo=24423 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1028, 965, 1176, 1200, 667, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 447, 648, 2326, 124, 156, 396, 342, 926, 469, 211, 0, 0, 0, 0, 0, 0, 1565, 39, 190, 88, 169, 531, 278, 249, 2155, 667, 0, 0, 0, 0, 0, 0, 600, 374, 1354, 316, 2316, 763, 585, 65, 0, 0, 0, 0, 0, 0, 0, 0, 290, 3, 16, 1382, 877, 278, 90, 1258, 717, 808, 0, 0, 0, 0, 0, 0, 286, 317, 139, 959, 671, 0, 2244, 0, 714, 251, 339, 0, 0, 0, 0, 0, 107, 959, 256, 1070, 414, 1235, 1232, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 706, 583, 179, 1560, 1063, 0, 907, 1195, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6460, 6045, 5931, 6373, 5719, 5920, 6511, 6193] avg=6144.0000 max=6511 max_rank=6 min=5719 min_rank=4 imbalance_ratio=1.059733 max_load_reduction=42641 max_load_reduction_pct=0.867533 moved_to_echo=24343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 1104, 659, 1210, 1103, 880, 694, 0, 0, 0, 0, 0, 0, 0, 0, 398, 679, 1837, 148, 188, 406, 369, 1171, 547, 227, 0, 0, 0, 0, 0, 0, 1367, 99, 196, 115, 191, 369, 228, 371, 2608, 806, 0, 0, 0, 0, 0, 0, 684, 412, 834, 678, 1488, 1228, 706, 92, 0, 0, 0, 0, 0, 0, 0, 0, 487, 43, 58, 716, 1298, 587, 188, 1397, 869, 497, 0, 0, 0, 0, 0, 0, 435, 699, 497, 1053, 790, 15, 1794, 13, 642, 270, 209, 0, 0, 0, 0, 0, 451, 531, 297, 1250, 513, 1059, 843, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 956, 570, 292, 1110, 1171, 0, 815, 1362, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5657, 5970, 6350, 6122, 6140, 6417, 6220, 6276] avg=6144.0000 max=6417 max_rank=5 min=5657 min_rank=0 imbalance_ratio=1.044434 max_load_reduction=42735 max_load_reduction_pct=0.869446 moved_to_echo=25053 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1130, 724, 1208, 1084, 744, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 279, 790, 2240, 152, 168, 421, 352, 936, 384, 233, 0, 0, 0, 0, 0, 0, 1323, 120, 176, 76, 138, 416, 235, 362, 2509, 876, 0, 0, 0, 0, 0, 0, 766, 418, 1065, 490, 1942, 775, 741, 98, 0, 0, 0, 0, 0, 0, 0, 0, 315, 11, 40, 1185, 1111, 335, 118, 1406, 943, 635, 0, 0, 0, 0, 0, 0, 389, 419, 250, 936, 708, 1, 2292, 4, 689, 277, 267, 0, 0, 0, 0, 0, 214, 761, 279, 878, 509, 1299, 874, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 946, 458, 206, 1230, 1203, 0, 874, 1250, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6079, 5955, 6231, 6295, 6099, 6232, 6094, 6167] avg=6144.0000 max=6295 max_rank=3 min=5955 min_rank=1 imbalance_ratio=1.024577 max_load_reduction=42857 max_load_reduction_pct=0.871928 moved_to_echo=24592 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 981, 1153, 1046, 1166, 647, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 487, 798, 2839, 74, 136, 540, 323, 723, 493, 207, 0, 0, 0, 0, 0, 0, 1429, 36, 156, 39, 149, 545, 393, 201, 1977, 313, 0, 0, 0, 0, 0, 0, 388, 360, 1236, 244, 2712, 672, 428, 92, 0, 0, 0, 0, 0, 0, 0, 0, 333, 2, 2, 1957, 758, 227, 51, 1294, 335, 747, 0, 0, 0, 0, 0, 0, 261, 394, 18, 833, 517, 0, 2824, 0, 527, 247, 317, 0, 0, 0, 0, 0, 51, 926, 201, 1110, 465, 1674, 1440, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 148, 1557, 1008, 0, 657, 1462, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6657, 6620, 5238, 6132, 5706, 5938, 6990, 5871] avg=6144.0000 max=6990 max_rank=6 min=5238 min_rank=2 imbalance_ratio=1.137695 max_load_reduction=42162 max_load_reduction_pct=0.857788 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 1088, 577, 1148, 1267, 923, 863, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 1683, 245, 298, 388, 775, 1348, 507, 224, 0, 0, 0, 0, 0, 0, 1332, 92, 205, 219, 212, 317, 218, 290, 2490, 975, 0, 0, 0, 0, 0, 0, 613, 330, 750, 707, 1293, 1255, 864, 172, 0, 0, 0, 0, 0, 0, 0, 0, 463, 97, 143, 847, 1408, 928, 307, 1141, 1050, 446, 0, 0, 0, 0, 0, 0, 323, 316, 784, 864, 900, 26, 1666, 47, 628, 266, 187, 0, 0, 0, 0, 0, 573, 353, 305, 1158, 512, 888, 697, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 978, 463, 306, 889, 1105, 0, 798, 1430, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5897, 6498, 6350, 5984, 6830, 6007, 5617, 5969] avg=6144.0000 max=6830 max_rank=4 min=5617 min_rank=6 imbalance_ratio=1.111654 max_load_reduction=42322 max_load_reduction_pct=0.861043 moved_to_echo=24423 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 1026, 544, 1193, 1648, 689, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 526, 718, 2105, 130, 162, 536, 397, 850, 378, 211, 0, 0, 0, 0, 0, 0, 1545, 47, 185, 139, 259, 428, 285, 224, 2301, 910, 0, 0, 0, 0, 0, 0, 447, 335, 1048, 487, 2008, 1054, 707, 91, 0, 0, 0, 0, 0, 0, 0, 0, 382, 40, 32, 1367, 785, 557, 163, 1332, 978, 625, 0, 0, 0, 0, 0, 0, 374, 749, 455, 1032, 606, 26, 1954, 43, 601, 251, 262, 0, 0, 0, 0, 0, 230, 630, 261, 864, 440, 1297, 1052, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 694, 542, 224, 1210, 1140, 0, 764, 1360, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6256, 6013, 6323, 6177, 6261, 6353, 5835, 5934] avg=6144.0000 max=6353 max_rank=5 min=5835 min_rank=6 imbalance_ratio=1.034017 max_load_reduction=42799 max_load_reduction_pct=0.870748 moved_to_echo=24383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1136, 736, 1173, 1117, 780, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 477, 787, 1979, 110, 185, 417, 370, 1105, 443, 234, 0, 0, 0, 0, 0, 0, 1499, 148, 147, 89, 189, 389, 223, 217, 2572, 850, 0, 0, 0, 0, 0, 0, 564, 436, 1024, 504, 1593, 1115, 826, 106, 0, 0, 0, 0, 0, 0, 0, 0, 371, 13, 46, 930, 1093, 537, 197, 1315, 914, 610, 0, 0, 0, 0, 0, 0, 336, 619, 335, 904, 744, 2, 2011, 0, 675, 278, 256, 0, 0, 0, 0, 0, 329, 654, 257, 1013, 512, 1176, 901, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 756, 588, 198, 1210, 1269, 0, 858, 1384, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5948, 6107, 6323, 6168, 6026, 6160, 6157, 6263] avg=6144.0000 max=6323 max_rank=2 min=5948 min_rank=0 imbalance_ratio=1.029134 max_load_reduction=42829 max_load_reduction_pct=0.871358 moved_to_echo=24606 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1177, 794, 1168, 1016, 733, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 335, 847, 2161, 73, 143, 519, 382, 869, 373, 242, 0, 0, 0, 0, 0, 0, 1408, 169, 215, 83, 191, 451, 289, 307, 2554, 739, 0, 0, 0, 0, 0, 0, 699, 404, 897, 523, 1676, 995, 627, 80, 0, 0, 0, 0, 0, 0, 0, 0, 497, 21, 41, 1131, 1094, 669, 163, 1423, 797, 535, 0, 0, 0, 0, 0, 0, 312, 620, 400, 964, 620, 4, 2002, 15, 676, 288, 224, 0, 0, 0, 0, 0, 197, 687, 251, 852, 456, 1268, 890, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 919, 543, 233, 1078, 1077, 0, 859, 1770, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6198, 5944, 6406, 5901, 6371, 6125, 5728, 6479] avg=6144.0000 max=6479 max_rank=7 min=5728 min_rank=6 imbalance_ratio=1.054525 max_load_reduction=42673 max_load_reduction_pct=0.868184 moved_to_echo=24703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 981, 1153, 1046, 1166, 647, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 487, 798, 2839, 74, 136, 540, 323, 723, 493, 207, 0, 0, 0, 0, 0, 0, 1429, 36, 156, 39, 149, 545, 393, 201, 1977, 313, 0, 0, 0, 0, 0, 0, 388, 360, 1236, 244, 2712, 672, 428, 92, 0, 0, 0, 0, 0, 0, 0, 0, 333, 2, 2, 1957, 758, 227, 51, 1294, 335, 747, 0, 0, 0, 0, 0, 0, 261, 394, 18, 833, 517, 0, 2824, 0, 527, 247, 317, 0, 0, 0, 0, 0, 51, 926, 201, 1110, 465, 1674, 1440, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 148, 1557, 1008, 0, 657, 1462, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6657, 6620, 5238, 6132, 5706, 5938, 6990, 5871] avg=6144.0000 max=6990 max_rank=6 min=5238 min_rank=2 imbalance_ratio=1.137695 max_load_reduction=42162 max_load_reduction_pct=0.857788 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 8704, 6272, 9344, 9728, 6144, 9344, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6528, 4352, 1792, 9856, 9088, 0, 6656, 11264, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2176, 5504, 2176, 8320, 3840, 9984, 7936, 9600, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 5888, 17280, 1152, 1536, 3712, 3328, 7936, 3712, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6528, 4352, 1792, 9856, 9088, 0, 6656, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 4224, 2944, 7552, 5632, 128, 16896, 128, 5248, 2176, 2176, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4864, 3072, 8320, 3968, 15104, 7936, 5504, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 256, 384, 9600, 8448, 4224, 1280, 10624, 6656, 4992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 5888, 17280, 1152, 1536, 3712, 3328, 7936, 3712, 1792, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11520, 768, 1536, 896, 1536, 3456, 2176, 2304, 19200, 6144, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 4224, 2944, 7552, 5632, 128, 16896, 128, 5248, 2176, 2176, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 256, 384, 9600, 8448, 4224, 1280, 10624, 6656, 4992, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[827, 1707, 1183, 260, 244, 255, 2529, 369, 44, 1, 894, 944, 1370, 0, 27, 1174, 787, 992, 3024, 96, 344, 456, 215, 8, 197, 1514, 728, 135, 383, 216, 3058, 393, 1367, 490, 267, 1527, 567, 342, 1791, 1064, 336, 0, 127, 2154, 158, 508, 1536, 449, 994, 1373, 1265, 824, 152, 670, 0, 1507, 1060, 463, 487, 850, 207, 525, 947, 771] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[714, 1604, 1016, 274, 174, 171, 2651, 397, 26, 2, 844, 757, 1516, 1, 30, 909, 725, 881, 3186, 58, 223, 571, 273, 14, 219, 1982, 676, 133, 388, 209, 3775, 471, 1754, 637, 207, 1641, 695, 315, 1522, 1183, 79, 4, 131, 1856, 135, 518, 1567, 547, 832, 1077, 1287, 937, 137, 602, 5, 1507, 1192, 484, 489, 746, 156, 348, 864, 828] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[589, 1799, 1031, 278, 212, 243, 2662, 341, 22, 0, 803, 924, 1487, 0, 6, 925, 811, 881, 3268, 48, 223, 603, 245, 2, 150, 1851, 733, 124, 350, 233, 3633, 409, 1520, 549, 256, 1392, 699, 303, 1758, 1029, 165, 0, 122, 2129, 88, 518, 1602, 465, 950, 1473, 1215, 830, 69, 647, 0, 1289, 1125, 466, 509, 698, 219, 409, 945, 827] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[699, 2022, 1039, 255, 145, 228, 2961, 440, 40, 0, 896, 892, 1293, 0, 7, 861, 731, 896, 2924, 34, 247, 683, 282, 0, 176, 2187, 849, 98, 304, 201, 3692, 332, 1435, 479, 204, 1082, 727, 309, 1564, 1134, 205, 0, 145, 1997, 94, 536, 1396, 465, 855, 1376, 1441, 867, 118, 643, 0, 1252, 1277, 412, 493, 677, 209, 438, 901, 1007] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[747, 1635, 1426, 318, 536, 239, 2571, 336, 55, 3, 915, 884, 1439, 0, 36, 1292, 969, 918, 2909, 72, 338, 393, 267, 5, 213, 1513, 825, 144, 428, 243, 3139, 615, 1472, 472, 240, 1628, 819, 303, 1960, 1039, 338, 1, 102, 2069, 121, 400, 1390, 441, 833, 1299, 849, 638, 104, 690, 0, 1301, 908, 475, 413, 658, 360, 478, 1120, 808] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[530, 1747, 812, 276, 107, 139, 2485, 662, 4, 0, 809, 673, 2368, 0, 0, 703, 524, 832, 3378, 16, 88, 1434, 281, 0, 133, 2366, 944, 132, 176, 182, 3910, 366, 1893, 394, 183, 1096, 618, 233, 1215, 1217, 23, 0, 22, 1253, 77, 846, 1383, 616, 875, 1252, 1616, 1736, 100, 512, 0, 1329, 1231, 451, 422, 516, 132, 251, 755, 828] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[647, 2023, 908, 270, 83, 174, 2643, 528, 9, 0, 735, 775, 1977, 0, 1, 788, 573, 939, 3614, 37, 141, 1193, 311, 0, 177, 2195, 833, 106, 163, 175, 3531, 358, 1932, 454, 154, 1009, 704, 310, 1407, 1004, 84, 0, 24, 1828, 87, 633, 1424, 479, 977, 1248, 1428, 1115, 101, 543, 0, 1299, 1333, 370, 457, 592, 197, 388, 800, 864] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[699, 2022, 1039, 255, 145, 228, 2961, 440, 40, 0, 896, 892, 1293, 0, 7, 861, 731, 896, 2924, 34, 247, 683, 282, 0, 176, 2187, 849, 98, 304, 201, 3692, 332, 1435, 479, 204, 1082, 727, 309, 1564, 1134, 205, 0, 145, 1997, 94, 536, 1396, 465, 855, 1376, 1441, 867, 118, 643, 0, 1252, 1277, 412, 493, 677, 209, 438, 901, 1007] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[781, 2043, 958, 336, 189, 243, 2605, 530, 28, 14, 725, 792, 1944, 0, 5, 1283, 717, 859, 3178, 26, 264, 853, 327, 25, 182, 1912, 808, 150, 309, 248, 2941, 441, 1771, 507, 249, 1937, 642, 379, 1750, 943, 122, 43, 51, 1249, 146, 457, 1386, 582, 988, 1171, 1471, 1010, 224, 521, 23, 1575, 1059, 455, 414, 437, 177, 229, 831, 637] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[714, 1604, 1016, 274, 174, 171, 1434, 397, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2, 844, 757, 1516, 1, 30, 909, 1413, 492, 0, 0, 0, 0, 0, 0, 725, 881, 3179, 58, 223, 571, 273, 14, 0, 0, 0, 0, 0, 0, 0, 0, 219, 1982, 676, 133, 388, 209, 2343, 471, 0, 0, 0, 0, 0, 0, 0, 0, 341, 637, 207, 1641, 695, 315, 1522, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 79, 4, 131, 1856, 135, 518, 1567, 547, 940, 571, 0, 0, 0, 0, 0, 0, 832, 1077, 1287, 937, 137, 602, 5, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 484, 489, 746, 156, 348, 864, 828, 646, 527, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5784, 5990, 5924, 6421, 6541, 6348, 5857, 6287] avg=6144.0000 max=6541 max_rank=4 min=5784 min_rank=0 imbalance_ratio=1.064616 max_load_reduction=42611 max_load_reduction_pct=0.866923 moved_to_echo=25033 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[827, 1707, 1183, 260, 244, 255, 1369, 369, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1, 894, 944, 1370, 0, 27, 1174, 1101, 398, 0, 0, 0, 0, 0, 0, 787, 992, 3017, 96, 344, 456, 215, 8, 0, 0, 0, 0, 0, 0, 0, 0, 197, 1514, 728, 135, 383, 216, 1899, 393, 0, 0, 0, 0, 0, 0, 0, 0, 266, 490, 267, 1527, 567, 342, 1791, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 127, 2154, 158, 508, 1536, 449, 761, 544, 0, 0, 0, 0, 0, 0, 994, 1373, 1265, 824, 152, 670, 0, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 463, 487, 850, 207, 525, 947, 771, 616, 527, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 5953, 5915, 5465, 6314, 6573, 6258, 6460] avg=6144.0000 max=6573 max_rank=5 min=5465 min_rank=3 imbalance_ratio=1.069824 max_load_reduction=42579 max_load_reduction_pct=0.866272 moved_to_echo=25605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[589, 1799, 1031, 278, 212, 243, 1440, 341, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 803, 924, 1487, 0, 6, 925, 1224, 473, 0, 0, 0, 0, 0, 0, 811, 881, 3261, 48, 223, 603, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 150, 1851, 733, 124, 350, 233, 2255, 409, 0, 0, 0, 0, 0, 0, 0, 0, 296, 549, 256, 1392, 699, 303, 1758, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 122, 2129, 88, 518, 1602, 465, 905, 573, 0, 0, 0, 0, 0, 0, 950, 1473, 1215, 830, 69, 647, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 466, 509, 698, 219, 409, 945, 827, 649, 451, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5933, 5864, 6074, 6105, 6282, 6567, 6022, 6305] avg=6144.0000 max=6567 max_rank=5 min=5864 min_rank=1 imbalance_ratio=1.068848 max_load_reduction=42585 max_load_reduction_pct=0.866394 moved_to_echo=25176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[714, 1604, 1016, 274, 174, 171, 2651, 397, 26, 2, 844, 757, 1516, 1, 30, 909, 725, 881, 3186, 58, 223, 571, 273, 14, 219, 1982, 676, 133, 388, 209, 3775, 471, 1754, 637, 207, 1641, 695, 315, 1522, 1183, 79, 4, 131, 1856, 135, 518, 1567, 547, 832, 1077, 1287, 937, 137, 602, 5, 1507, 1192, 484, 489, 746, 156, 348, 864, 828] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[530, 1747, 812, 276, 107, 139, 2485, 662, 4, 0, 809, 673, 2368, 0, 0, 703, 524, 832, 3378, 16, 88, 1434, 281, 0, 133, 2366, 944, 132, 176, 182, 3910, 366, 1893, 394, 183, 1096, 618, 233, 1215, 1217, 23, 0, 22, 1253, 77, 846, 1383, 616, 875, 1252, 1616, 1736, 100, 512, 0, 1329, 1231, 451, 422, 516, 132, 251, 755, 828] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[589, 1799, 1031, 278, 212, 243, 2662, 341, 22, 0, 803, 924, 1487, 0, 6, 925, 811, 881, 3268, 48, 223, 603, 245, 2, 150, 1851, 733, 124, 350, 233, 3633, 409, 1520, 549, 256, 1392, 699, 303, 1758, 1029, 165, 0, 122, 2129, 88, 518, 1602, 465, 950, 1473, 1215, 830, 69, 647, 0, 1289, 1125, 466, 509, 698, 219, 409, 945, 827] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[747, 1635, 1426, 318, 536, 239, 1392, 336, 0, 0, 0, 0, 0, 0, 0, 0, 55, 3, 915, 884, 1439, 0, 36, 1292, 1186, 409, 0, 0, 0, 0, 0, 0, 969, 918, 2903, 72, 338, 393, 267, 5, 0, 0, 0, 0, 0, 0, 0, 0, 213, 1513, 825, 144, 428, 243, 1948, 615, 0, 0, 0, 0, 0, 0, 0, 0, 286, 472, 240, 1628, 819, 303, 1960, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 338, 1, 102, 2069, 121, 400, 1390, 441, 782, 553, 0, 0, 0, 0, 0, 0, 833, 1299, 849, 638, 104, 690, 0, 846, 0, 0, 0, 0, 0, 0, 0, 0, 908, 475, 413, 658, 360, 478, 1120, 808, 626, 455, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6629, 6219, 5865, 5929, 6747, 6197, 5259, 6307] avg=6144.0000 max=6747 max_rank=4 min=5259 min_rank=6 imbalance_ratio=1.098145 max_load_reduction=42405 max_load_reduction_pct=0.862732 moved_to_echo=24510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[647, 2023, 908, 270, 83, 174, 1430, 528, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 735, 775, 1977, 0, 1, 788, 1556, 460, 0, 0, 0, 0, 0, 0, 573, 939, 3606, 37, 141, 1193, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 2195, 833, 106, 163, 175, 2192, 358, 0, 0, 0, 0, 0, 0, 0, 0, 376, 454, 154, 1009, 704, 310, 1407, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 24, 1828, 87, 633, 1424, 479, 879, 569, 0, 0, 0, 0, 0, 0, 977, 1248, 1428, 1115, 101, 543, 0, 845, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 370, 457, 592, 197, 388, 800, 864, 644, 454, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6301, 6800, 6199, 5418, 6007, 6257, 6107] avg=6144.0000 max=6800 max_rank=2 min=5418 min_rank=4 imbalance_ratio=1.106771 max_load_reduction=42352 max_load_reduction_pct=0.861654 moved_to_echo=23789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[699, 2022, 1039, 255, 145, 228, 1602, 440, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 896, 892, 1293, 0, 7, 861, 1156, 481, 0, 0, 0, 0, 0, 0, 731, 896, 2918, 34, 247, 683, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 2187, 849, 98, 304, 201, 2292, 332, 0, 0, 0, 0, 0, 0, 0, 0, 279, 479, 204, 1082, 727, 309, 1564, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 145, 1997, 94, 536, 1396, 465, 919, 637, 0, 0, 0, 0, 0, 0, 855, 1376, 1441, 867, 118, 643, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 1277, 412, 493, 677, 209, 438, 901, 1007, 722, 438, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6430, 5626, 5791, 6439, 5778, 6394, 6114, 6580] avg=6144.0000 max=6580 max_rank=7 min=5626 min_rank=1 imbalance_ratio=1.070964 max_load_reduction=42572 max_load_reduction_pct=0.866130 moved_to_echo=24866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[530, 1747, 812, 276, 107, 139, 1337, 662, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 809, 673, 2368, 0, 0, 703, 1528, 513, 0, 0, 0, 0, 0, 0, 524, 832, 3366, 16, 88, 1434, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 2366, 944, 132, 176, 182, 2419, 366, 0, 0, 0, 0, 0, 0, 0, 0, 365, 394, 183, 1096, 618, 233, 1215, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 22, 1253, 77, 846, 1383, 616, 978, 539, 0, 0, 0, 0, 0, 0, 875, 1252, 1616, 1736, 100, 512, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 451, 422, 516, 132, 251, 755, 828, 609, 467, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5610, 6598, 6541, 6718, 5321, 5737, 6953, 5674] avg=6144.0000 max=6953 max_rank=6 min=5321 min_rank=4 imbalance_ratio=1.131673 max_load_reduction=42199 max_load_reduction_pct=0.858541 moved_to_echo=23685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[781, 2043, 958, 336, 189, 243, 1409, 530, 0, 0, 0, 0, 0, 0, 0, 0, 28, 14, 725, 792, 1944, 0, 5, 1283, 1427, 383, 0, 0, 0, 0, 0, 0, 717, 859, 3171, 26, 264, 853, 327, 25, 0, 0, 0, 0, 0, 0, 0, 0, 182, 1912, 808, 150, 309, 248, 1826, 441, 0, 0, 0, 0, 0, 0, 0, 0, 344, 507, 249, 1937, 642, 379, 1750, 943, 0, 0, 0, 0, 0, 0, 0, 0, 122, 43, 51, 1249, 146, 457, 1386, 582, 732, 561, 0, 0, 0, 0, 0, 0, 988, 1171, 1471, 1010, 224, 521, 23, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 455, 414, 437, 177, 229, 831, 637, 635, 551, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6489, 6601, 6242, 5876, 6751, 5329, 6432, 5432] avg=6144.0000 max=6751 max_rank=4 min=5329 min_rank=5 imbalance_ratio=1.098796 max_load_reduction=42401 max_load_reduction_pct=0.862651 moved_to_echo=23944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[714, 1604, 1016, 274, 174, 171, 1434, 397, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2, 844, 757, 1516, 1, 30, 909, 1413, 492, 0, 0, 0, 0, 0, 0, 725, 881, 3179, 58, 223, 571, 273, 14, 0, 0, 0, 0, 0, 0, 0, 0, 219, 1982, 676, 133, 388, 209, 2343, 471, 0, 0, 0, 0, 0, 0, 0, 0, 341, 637, 207, 1641, 695, 315, 1522, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 79, 4, 131, 1856, 135, 518, 1567, 547, 940, 571, 0, 0, 0, 0, 0, 0, 832, 1077, 1287, 937, 137, 602, 5, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 484, 489, 746, 156, 348, 864, 828, 646, 527, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5784, 5990, 5924, 6421, 6541, 6348, 5857, 6287] avg=6144.0000 max=6541 max_rank=4 min=5784 min_rank=0 imbalance_ratio=1.064616 max_load_reduction=42611 max_load_reduction_pct=0.866923 moved_to_echo=25033 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[827, 1707, 1183, 260, 244, 255, 1369, 369, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1, 894, 944, 1370, 0, 27, 1174, 1101, 398, 0, 0, 0, 0, 0, 0, 787, 992, 3017, 96, 344, 456, 215, 8, 0, 0, 0, 0, 0, 0, 0, 0, 197, 1514, 728, 135, 383, 216, 1899, 393, 0, 0, 0, 0, 0, 0, 0, 0, 266, 490, 267, 1527, 567, 342, 1791, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 127, 2154, 158, 508, 1536, 449, 761, 544, 0, 0, 0, 0, 0, 0, 994, 1373, 1265, 824, 152, 670, 0, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 463, 487, 850, 207, 525, 947, 771, 616, 527, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 5953, 5915, 5465, 6314, 6573, 6258, 6460] avg=6144.0000 max=6573 max_rank=5 min=5465 min_rank=3 imbalance_ratio=1.069824 max_load_reduction=42579 max_load_reduction_pct=0.866272 moved_to_echo=25605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[530, 1747, 812, 276, 107, 139, 1337, 662, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 809, 673, 2368, 0, 0, 703, 1528, 513, 0, 0, 0, 0, 0, 0, 524, 832, 3366, 16, 88, 1434, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 2366, 944, 132, 176, 182, 2419, 366, 0, 0, 0, 0, 0, 0, 0, 0, 365, 394, 183, 1096, 618, 233, 1215, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 22, 1253, 77, 846, 1383, 616, 978, 539, 0, 0, 0, 0, 0, 0, 875, 1252, 1616, 1736, 100, 512, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 451, 422, 516, 132, 251, 755, 828, 609, 467, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5610, 6598, 6541, 6718, 5321, 5737, 6953, 5674] avg=6144.0000 max=6953 max_rank=6 min=5321 min_rank=4 imbalance_ratio=1.131673 max_load_reduction=42199 max_load_reduction_pct=0.858541 moved_to_echo=23685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[589, 1799, 1031, 278, 212, 243, 1440, 341, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 803, 924, 1487, 0, 6, 925, 1224, 473, 0, 0, 0, 0, 0, 0, 811, 881, 3261, 48, 223, 603, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 150, 1851, 733, 124, 350, 233, 2255, 409, 0, 0, 0, 0, 0, 0, 0, 0, 296, 549, 256, 1392, 699, 303, 1758, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 122, 2129, 88, 518, 1602, 465, 905, 573, 0, 0, 0, 0, 0, 0, 950, 1473, 1215, 830, 69, 647, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 466, 509, 698, 219, 409, 945, 827, 649, 451, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5933, 5864, 6074, 6105, 6282, 6567, 6022, 6305] avg=6144.0000 max=6567 max_rank=5 min=5864 min_rank=1 imbalance_ratio=1.068848 max_load_reduction=42585 max_load_reduction_pct=0.866394 moved_to_echo=25176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 14592, 8448, 2304, 1792, 1792, 11520, 3712, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 128, 768, 14592, 1024, 4480, 11776, 4096, 6912, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 3584, 3712, 5248, 1664, 3072, 7168, 6656, 5248, 3968, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 128, 768, 14592, 1024, 4480, 11776, 4096, 6912, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 14592, 8448, 2304, 1792, 1792, 11520, 3712, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 10368, 10624, 8064, 1024, 4864, 128, 7296, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 128, 6656, 6656, 13440, 128, 128, 7936, 10624, 3712, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5888, 7296, 25472, 512, 1920, 6272, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 4096, 1792, 11392, 5504, 2560, 13056, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 128, 6656, 6656, 13440, 128, 128, 7936, 10624, 3712, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1536, 15616, 6400, 1024, 2560, 1792, 17280, 3456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 4096, 1792, 11392, 5504, 2560, 13056, 8704, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[681, 92, 3, 3722, 91, 3, 224, 64, 2, 894, 1219, 786, 1037, 695, 434, 833, 0, 7, 804, 3, 71, 530, 436, 1675, 127, 56, 1206, 3, 401, 119, 6, 10, 71, 1657, 165, 18, 0, 3, 12, 2591, 1250, 246, 193, 1780, 5266, 148, 945, 3712, 421, 238, 7, 1828, 399, 1065, 5848, 6, 4115, 3, 2, 38, 396, 4, 351, 140] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[772, 127, 0, 3397, 80, 14, 234, 62, 0, 745, 1319, 956, 1092, 621, 464, 780, 1, 7, 820, 16, 117, 489, 440, 2627, 116, 83, 1181, 4, 486, 192, 0, 2, 62, 1669, 273, 10, 0, 0, 1, 2091, 1369, 274, 206, 1935, 5230, 260, 933, 3321, 408, 560, 2, 1262, 353, 866, 5778, 0, 4182, 4, 0, 21, 394, 3, 307, 134] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[718, 116, 0, 3879, 63, 1, 236, 62, 0, 869, 1082, 897, 1033, 669, 495, 926, 0, 0, 788, 1, 42, 482, 414, 2177, 121, 91, 1030, 0, 373, 178, 0, 0, 62, 1699, 251, 8, 0, 0, 0, 2396, 1339, 276, 186, 1747, 5400, 104, 1052, 3558, 449, 183, 0, 1431, 331, 808, 5944, 0, 4429, 0, 0, 25, 297, 0, 319, 115] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[645, 96, 0, 3947, 98, 0, 177, 82, 0, 628, 988, 721, 1042, 479, 400, 1312, 0, 0, 728, 1, 8, 435, 427, 2026, 105, 61, 656, 0, 205, 157, 0, 0, 82, 1680, 177, 1, 0, 0, 0, 3156, 1139, 239, 176, 1774, 5524, 112, 973, 4257, 382, 459, 0, 1317, 333, 1130, 6110, 0, 4189, 0, 0, 5, 0, 0, 354, 159] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[667, 88, 0, 3906, 78, 0, 223, 70, 0, 768, 921, 802, 1067, 637, 447, 880, 0, 0, 713, 6, 35, 423, 443, 2277, 94, 86, 827, 0, 371, 186, 0, 0, 70, 1700, 216, 8, 0, 0, 0, 2743, 1241, 288, 203, 1771, 5449, 115, 1066, 3709, 418, 320, 0, 1295, 339, 916, 6121, 0, 4523, 0, 0, 8, 181, 0, 308, 129] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[724, 122, 1, 2651, 88, 6, 241, 108, 0, 700, 1733, 977, 1025, 609, 390, 1341, 0, 25, 824, 8, 290, 499, 494, 2739, 116, 66, 1129, 1, 523, 262, 0, 14, 75, 1472, 389, 19, 0, 0, 0, 1865, 1375, 219, 210, 2026, 5116, 361, 1065, 3145, 328, 818, 0, 1349, 360, 741, 5701, 0, 3983, 0, 0, 39, 317, 0, 323, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[772, 127, 0, 3397, 80, 14, 234, 62, 0, 745, 1319, 956, 1092, 621, 464, 780, 1, 7, 820, 16, 117, 489, 440, 2627, 116, 83, 1181, 4, 486, 192, 0, 2, 62, 1669, 273, 10, 0, 0, 1, 2091, 1369, 274, 206, 1935, 5230, 260, 933, 3321, 408, 560, 2, 1262, 353, 866, 5778, 0, 4182, 4, 0, 21, 394, 3, 307, 134] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[606, 82, 0, 4370, 89, 0, 180, 69, 0, 1027, 574, 662, 952, 511, 403, 548, 0, 0, 662, 0, 5, 498, 384, 1225, 127, 95, 507, 0, 262, 397, 0, 0, 69, 1632, 136, 1, 0, 0, 0, 3685, 1338, 281, 180, 1645, 5550, 92, 813, 4506, 376, 138, 0, 1673, 375, 1410, 6225, 0, 4169, 0, 0, 3, 2, 0, 469, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[724, 122, 1, 2651, 88, 6, 241, 108, 0, 700, 1733, 977, 1025, 609, 390, 1341, 0, 25, 824, 8, 290, 499, 494, 2739, 116, 66, 1129, 1, 523, 262, 0, 14, 75, 1472, 389, 19, 0, 0, 0, 1865, 1375, 219, 210, 2026, 5116, 361, 1065, 3145, 328, 818, 0, 1349, 360, 741, 5701, 0, 3983, 0, 0, 39, 317, 0, 323, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[718, 116, 0, 3879, 63, 1, 236, 62, 0, 869, 1082, 897, 1033, 669, 495, 926, 0, 0, 788, 1, 42, 482, 414, 2177, 121, 91, 1030, 0, 373, 178, 0, 0, 62, 1699, 251, 8, 0, 0, 0, 2396, 1339, 276, 186, 1747, 5400, 104, 1052, 3558, 449, 183, 0, 1431, 331, 808, 5944, 0, 4429, 0, 0, 25, 297, 0, 319, 115] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[763, 127, 15, 3389, 107, 25, 199, 93, 21, 997, 844, 911, 942, 370, 442, 752, 3, 30, 852, 30, 108, 627, 400, 1406, 123, 91, 757, 0, 347, 208, 3, 13, 103, 1609, 140, 31, 12, 60, 35, 2758, 1520, 235, 252, 2066, 5324, 228, 940, 4020, 326, 822, 24, 1379, 456, 1125, 5730, 90, 4084, 52, 36, 52, 127, 1, 394, 126] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[681, 92, 3, 3722, 91, 3, 224, 64, 766, 510, 0, 0, 0, 0, 0, 0, 2, 894, 1219, 786, 1037, 695, 434, 833, 427, 0, 0, 0, 0, 0, 0, 0, 0, 7, 804, 3, 71, 530, 436, 1675, 1391, 894, 0, 0, 0, 0, 0, 0, 127, 56, 1206, 3, 401, 119, 6, 10, 4372, 0, 0, 0, 0, 0, 0, 0, 71, 1657, 165, 18, 0, 3, 12, 2591, 1502, 0, 0, 0, 0, 0, 0, 0, 1250, 246, 193, 1780, 0, 148, 945, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 421, 238, 7, 1828, 399, 1065, 2445, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4115, 3, 2, 38, 396, 4, 351, 140, 1171, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 6327, 5811, 6300, 6019, 5910, 6409, 6220] avg=6144.0000 max=6409 max_rank=6 min=5811 min_rank=2 imbalance_ratio=1.043132 max_load_reduction=42743 max_load_reduction_pct=0.869609 moved_to_echo=24558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[681, 92, 3, 3722, 91, 3, 224, 64, 2, 894, 1219, 786, 1037, 695, 434, 833, 0, 7, 804, 3, 71, 530, 436, 1675, 127, 56, 1206, 3, 401, 119, 6, 10, 71, 1657, 165, 18, 0, 3, 12, 2591, 1250, 246, 193, 1780, 5266, 148, 945, 3712, 421, 238, 7, 1828, 399, 1065, 5848, 6, 4115, 3, 2, 38, 396, 4, 351, 140] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[645, 96, 0, 3947, 98, 0, 177, 82, 0, 628, 988, 721, 1042, 479, 400, 1312, 0, 0, 728, 1, 8, 435, 427, 2026, 105, 61, 656, 0, 205, 157, 0, 0, 82, 1680, 177, 1, 0, 0, 0, 3156, 1139, 239, 176, 1774, 5524, 112, 973, 4257, 382, 459, 0, 1317, 333, 1130, 6110, 0, 4189, 0, 0, 5, 0, 0, 354, 159] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[667, 88, 0, 3906, 78, 0, 223, 70, 765, 533, 0, 0, 0, 0, 0, 0, 0, 768, 921, 802, 1067, 637, 447, 880, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 6, 35, 423, 443, 2277, 1456, 925, 0, 0, 0, 0, 0, 0, 94, 86, 827, 0, 371, 186, 0, 0, 4524, 0, 0, 0, 0, 0, 0, 0, 70, 1700, 216, 8, 0, 0, 0, 2743, 1572, 0, 0, 0, 0, 0, 0, 0, 1241, 288, 203, 1771, 0, 115, 1066, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 418, 320, 0, 1295, 339, 916, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 0, 8, 181, 0, 308, 129, 1170, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6330, 5949, 6278, 6088, 6309, 6031, 5848, 6319] avg=6144.0000 max=6330 max_rank=0 min=5848 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24507 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[772, 127, 0, 3397, 80, 14, 234, 62, 685, 503, 0, 0, 0, 0, 0, 0, 0, 745, 1319, 956, 1092, 621, 464, 780, 382, 0, 0, 0, 0, 0, 0, 0, 1, 7, 820, 16, 117, 489, 440, 2627, 1374, 888, 0, 0, 0, 0, 0, 0, 116, 83, 1181, 4, 486, 192, 0, 2, 4342, 0, 0, 0, 0, 0, 0, 0, 62, 1669, 273, 10, 0, 0, 1, 2091, 1484, 0, 0, 0, 0, 0, 0, 0, 1369, 274, 206, 1935, 0, 260, 933, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 408, 560, 2, 1262, 353, 866, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4182, 4, 0, 21, 394, 3, 307, 134, 1048, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5874, 6359, 6779, 6406, 5590, 6183, 5868, 6093] avg=6144.0000 max=6779 max_rank=2 min=5590 min_rank=4 imbalance_ratio=1.103353 max_load_reduction=42373 max_load_reduction_pct=0.862081 moved_to_echo=23734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[724, 122, 1, 2651, 88, 6, 241, 108, 649, 497, 0, 0, 0, 0, 0, 0, 0, 700, 1733, 977, 1025, 609, 390, 1341, 362, 0, 0, 0, 0, 0, 0, 0, 0, 25, 824, 8, 290, 499, 494, 2739, 1356, 869, 0, 0, 0, 0, 0, 0, 116, 66, 1129, 1, 523, 262, 0, 14, 4247, 0, 0, 0, 0, 0, 0, 0, 75, 1472, 389, 19, 0, 0, 0, 1865, 1464, 0, 0, 0, 0, 0, 0, 0, 1375, 219, 210, 2026, 0, 361, 1065, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 328, 818, 0, 1349, 360, 741, 2384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 39, 317, 0, 323, 150, 992, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5087, 7137, 7104, 6358, 5284, 6398, 5980, 5804] avg=6144.0000 max=7137 max_rank=1 min=5087 min_rank=0 imbalance_ratio=1.161621 max_load_reduction=42015 max_load_reduction_pct=0.854797 moved_to_echo=23466 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[645, 96, 0, 3947, 98, 0, 177, 82, 878, 532, 0, 0, 0, 0, 0, 0, 0, 628, 988, 721, 1042, 479, 400, 1312, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 1, 8, 435, 427, 2026, 1453, 939, 0, 0, 0, 0, 0, 0, 105, 61, 656, 0, 205, 157, 0, 0, 4585, 0, 0, 0, 0, 0, 0, 0, 82, 1680, 177, 1, 0, 0, 0, 3156, 1569, 0, 0, 0, 0, 0, 0, 0, 1139, 239, 176, 1774, 0, 112, 973, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 382, 459, 0, 1317, 333, 1130, 2556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4189, 0, 0, 5, 0, 0, 354, 159, 1343, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6455, 6060, 6017, 5769, 6665, 5959, 6177, 6050] avg=6144.0000 max=6665 max_rank=4 min=5769 min_rank=3 imbalance_ratio=1.084798 max_load_reduction=42487 max_load_reduction_pct=0.864400 moved_to_echo=24851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[763, 127, 15, 3389, 107, 25, 199, 93, 829, 499, 0, 0, 0, 0, 0, 0, 21, 997, 844, 911, 942, 370, 442, 752, 462, 0, 0, 0, 0, 0, 0, 0, 3, 30, 852, 30, 108, 627, 400, 1406, 1363, 905, 0, 0, 0, 0, 0, 0, 123, 91, 757, 0, 347, 208, 3, 13, 4419, 0, 0, 0, 0, 0, 0, 0, 103, 1609, 140, 31, 12, 60, 35, 2758, 1471, 0, 0, 0, 0, 0, 0, 0, 1520, 235, 252, 2066, 0, 228, 940, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 326, 822, 24, 1379, 456, 1125, 2397, 90, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 52, 36, 52, 127, 1, 394, 126, 1268, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 5741, 5724, 5961, 6219, 6702, 6619, 6140] avg=6144.0000 max=6702 max_rank=5 min=5724 min_rank=2 imbalance_ratio=1.090820 max_load_reduction=42450 max_load_reduction_pct=0.863647 moved_to_echo=25680 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[681, 92, 3, 3722, 91, 3, 224, 64, 766, 510, 0, 0, 0, 0, 0, 0, 2, 894, 1219, 786, 1037, 695, 434, 833, 427, 0, 0, 0, 0, 0, 0, 0, 0, 7, 804, 3, 71, 530, 436, 1675, 1391, 894, 0, 0, 0, 0, 0, 0, 127, 56, 1206, 3, 401, 119, 6, 10, 4372, 0, 0, 0, 0, 0, 0, 0, 71, 1657, 165, 18, 0, 3, 12, 2591, 1502, 0, 0, 0, 0, 0, 0, 0, 1250, 246, 193, 1780, 0, 148, 945, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 421, 238, 7, 1828, 399, 1065, 2445, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4115, 3, 2, 38, 396, 4, 351, 140, 1171, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 6327, 5811, 6300, 6019, 5910, 6409, 6220] avg=6144.0000 max=6409 max_rank=6 min=5811 min_rank=2 imbalance_ratio=1.043132 max_load_reduction=42743 max_load_reduction_pct=0.869609 moved_to_echo=24558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[718, 116, 0, 3879, 63, 1, 236, 62, 734, 518, 0, 0, 0, 0, 0, 0, 0, 869, 1082, 897, 1033, 669, 495, 926, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 1, 42, 482, 414, 2177, 1414, 918, 0, 0, 0, 0, 0, 0, 121, 91, 1030, 0, 373, 178, 0, 0, 4482, 0, 0, 0, 0, 0, 0, 0, 62, 1699, 251, 8, 0, 0, 0, 2396, 1526, 0, 0, 0, 0, 0, 0, 0, 1339, 276, 186, 1747, 0, 104, 1052, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 449, 183, 0, 1431, 331, 808, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4429, 0, 0, 25, 297, 0, 319, 115, 1123, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6327, 6380, 6236, 6275, 5942, 5996, 5688, 6308] avg=6144.0000 max=6380 max_rank=1 min=5688 min_rank=6 imbalance_ratio=1.038411 max_load_reduction=42772 max_load_reduction_pct=0.870199 moved_to_echo=23934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[724, 122, 1, 2651, 88, 6, 241, 108, 649, 497, 0, 0, 0, 0, 0, 0, 0, 700, 1733, 977, 1025, 609, 390, 1341, 362, 0, 0, 0, 0, 0, 0, 0, 0, 25, 824, 8, 290, 499, 494, 2739, 1356, 869, 0, 0, 0, 0, 0, 0, 116, 66, 1129, 1, 523, 262, 0, 14, 4247, 0, 0, 0, 0, 0, 0, 0, 75, 1472, 389, 19, 0, 0, 0, 1865, 1464, 0, 0, 0, 0, 0, 0, 0, 1375, 219, 210, 2026, 0, 361, 1065, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 328, 818, 0, 1349, 360, 741, 2384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 39, 317, 0, 323, 150, 992, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5087, 7137, 7104, 6358, 5284, 6398, 5980, 5804] avg=6144.0000 max=7137 max_rank=1 min=5087 min_rank=0 imbalance_ratio=1.161621 max_load_reduction=42015 max_load_reduction_pct=0.854797 moved_to_echo=23466 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[606, 82, 0, 4370, 89, 0, 180, 69, 934, 547, 0, 0, 0, 0, 0, 0, 0, 1027, 574, 662, 952, 511, 403, 548, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 5, 498, 384, 1225, 1486, 943, 0, 0, 0, 0, 0, 0, 127, 95, 507, 0, 262, 397, 0, 0, 4607, 0, 0, 0, 0, 0, 0, 0, 69, 1632, 136, 1, 0, 0, 0, 3685, 1601, 0, 0, 0, 0, 0, 0, 0, 1338, 281, 180, 1645, 0, 92, 813, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 376, 138, 0, 1673, 375, 1410, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4169, 0, 0, 3, 2, 0, 469, 149, 1426, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6877, 5199, 5203, 5995, 7124, 5973, 6563, 6218] avg=6144.0000 max=7124 max_rank=4 min=5199 min_rank=1 imbalance_ratio=1.159505 max_load_reduction=42028 max_load_reduction_pct=0.855062 moved_to_echo=25878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[667, 88, 0, 3906, 78, 0, 223, 70, 765, 533, 0, 0, 0, 0, 0, 0, 0, 768, 921, 802, 1067, 637, 447, 880, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 6, 35, 423, 443, 2277, 1456, 925, 0, 0, 0, 0, 0, 0, 94, 86, 827, 0, 371, 186, 0, 0, 4524, 0, 0, 0, 0, 0, 0, 0, 70, 1700, 216, 8, 0, 0, 0, 2743, 1572, 0, 0, 0, 0, 0, 0, 0, 1241, 288, 203, 1771, 0, 115, 1066, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 418, 320, 0, 1295, 339, 916, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 0, 8, 181, 0, 308, 129, 1170, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6330, 5949, 6278, 6088, 6309, 6031, 5848, 6319] avg=6144.0000 max=6330 max_rank=0 min=5848 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24507 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[763, 127, 15, 3389, 107, 25, 199, 93, 829, 499, 0, 0, 0, 0, 0, 0, 21, 997, 844, 911, 942, 370, 442, 752, 462, 0, 0, 0, 0, 0, 0, 0, 3, 30, 852, 30, 108, 627, 400, 1406, 1363, 905, 0, 0, 0, 0, 0, 0, 123, 91, 757, 0, 347, 208, 3, 13, 4419, 0, 0, 0, 0, 0, 0, 0, 103, 1609, 140, 31, 12, 60, 35, 2758, 1471, 0, 0, 0, 0, 0, 0, 0, 1520, 235, 252, 2066, 0, 228, 940, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 326, 822, 24, 1379, 456, 1125, 2397, 90, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 52, 36, 52, 127, 1, 394, 126, 1268, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 5741, 5724, 5961, 6219, 6702, 6619, 6140] avg=6144.0000 max=6702 max_rank=5 min=5724 min_rank=2 imbalance_ratio=1.090820 max_load_reduction=42450 max_load_reduction_pct=0.863647 moved_to_echo=25680 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 6272, 128, 768, 4096, 3456, 16256, 11392, 7296, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10624, 2176, 1664, 14848, 0, 1536, 7808, 11008, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 13184, 1792, 128, 128, 128, 128, 21376, 12288, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 6272, 128, 768, 4096, 3456, 16256, 11392, 7296, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 6656, 8704, 6784, 8192, 4608, 3584, 7424, 3584, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 896, 128, 29312, 768, 128, 1792, 640, 6272, 4224, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 3584, 128, 11648, 3072, 8064, 19840, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 13184, 1792, 128, 128, 128, 128, 21376, 12288, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[33792, 128, 128, 256, 1792, 128, 2944, 1152, 9600, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 640, 7296, 128, 3072, 1792, 128, 128, 35584, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1981, 4002, 0, 299, 952, 190, 67, 121, 0, 347, 205, 269, 104, 326, 583, 644, 411, 705, 0, 8, 677, 88, 2843, 1564, 2071, 3566, 1093, 262, 1, 345, 234, 414, 0, 221, 1926, 0, 91, 1651, 779, 198, 723, 71, 0, 57, 2390, 5574, 922, 0, 5392, 0, 2040, 493, 413, 115, 374, 1, 9, 346, 0, 0, 0, 484, 114, 396] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2454, 5344, 105, 166, 928, 37, 96, 65, 1, 487, 253, 297, 99, 371, 576, 531, 378, 1711, 95, 76, 277, 88, 1515, 1264, 3130, 3107, 1912, 219, 59, 230, 208, 386, 40, 450, 667, 9, 79, 2154, 1064, 159, 625, 14, 31, 74, 1983, 5521, 511, 63, 4497, 117, 1463, 516, 429, 93, 258, 12, 75, 355, 80, 26, 13, 609, 347, 353] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[2193, 4356, 0, 181, 849, 59, 52, 62, 0, 452, 177, 278, 108, 304, 558, 690, 376, 1053, 0, 5, 484, 85, 2575, 1304, 2647, 3756, 1303, 271, 0, 354, 176, 355, 0, 233, 1605, 0, 81, 1811, 684, 181, 722, 21, 0, 72, 2402, 5758, 802, 0, 5004, 0, 1692, 484, 489, 96, 349, 0, 37, 432, 0, 0, 0, 666, 132, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2313, 5459, 0, 143, 673, 47, 54, 33, 0, 288, 131, 197, 89, 302, 431, 693, 318, 1103, 0, 0, 295, 99, 2335, 1383, 2864, 3704, 1753, 208, 2, 212, 156, 354, 0, 269, 1167, 0, 63, 1981, 610, 119, 704, 15, 0, 88, 2007, 6075, 434, 0, 5196, 0, 1497, 652, 507, 65, 339, 0, 12, 463, 0, 0, 0, 763, 152, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2748, 6014, 0, 139, 638, 35, 51, 13, 0, 325, 122, 181, 98, 313, 459, 704, 237, 994, 0, 1, 167, 81, 2248, 1059, 3307, 3622, 1378, 300, 1, 39, 89, 362, 0, 258, 1071, 0, 78, 2341, 651, 133, 754, 18, 0, 84, 1986, 5922, 433, 0, 4787, 0, 1299, 693, 495, 87, 333, 0, 3, 437, 0, 0, 0, 1000, 222, 342] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2760, 6590, 0, 89, 538, 19, 64, 6, 0, 289, 104, 202, 110, 313, 507, 629, 178, 1415, 0, 0, 184, 87, 1388, 1110, 4107, 3282, 2108, 290, 0, 0, 133, 386, 0, 290, 611, 0, 99, 2454, 633, 138, 695, 4, 0, 60, 1696, 6100, 262, 0, 4598, 0, 961, 754, 477, 98, 190, 0, 4, 391, 0, 0, 0, 1166, 268, 315] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[2410, 5737, 20, 160, 737, 52, 53, 29, 0, 357, 166, 184, 133, 316, 501, 735, 402, 1124, 15, 6, 227, 80, 2232, 1470, 2482, 3559, 1793, 222, 2, 217, 261, 314, 8, 404, 854, 1, 88, 1624, 872, 138, 701, 13, 0, 68, 1692, 6148, 392, 4, 5368, 11, 1718, 539, 492, 91, 262, 1, 21, 422, 5, 4, 1, 659, 198, 357] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2760, 6590, 0, 89, 538, 19, 64, 6, 0, 289, 104, 202, 110, 313, 507, 629, 178, 1415, 0, 0, 184, 87, 1388, 1110, 4107, 3282, 2108, 290, 0, 0, 133, 386, 0, 290, 611, 0, 99, 2454, 633, 138, 695, 4, 0, 60, 1696, 6100, 262, 0, 4598, 0, 961, 754, 477, 98, 190, 0, 4, 391, 0, 0, 0, 1166, 268, 315] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2454, 5344, 105, 166, 928, 37, 96, 65, 1, 487, 253, 297, 99, 371, 576, 531, 378, 1711, 95, 76, 277, 88, 1515, 1264, 3130, 3107, 1912, 219, 59, 230, 208, 386, 40, 450, 667, 9, 79, 2154, 1064, 159, 625, 14, 31, 74, 1983, 5521, 511, 63, 4497, 117, 1463, 516, 429, 93, 258, 12, 75, 355, 80, 26, 13, 609, 347, 353] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2313, 5459, 0, 143, 673, 47, 54, 33, 0, 288, 131, 197, 89, 302, 431, 693, 318, 1103, 0, 0, 295, 99, 2335, 1383, 2864, 3704, 1753, 208, 2, 212, 156, 354, 0, 269, 1167, 0, 63, 1981, 610, 119, 704, 15, 0, 88, 2007, 6075, 434, 0, 5196, 0, 1497, 652, 507, 65, 339, 0, 12, 463, 0, 0, 0, 763, 152, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[3047, 6905, 1, 47, 541, 0, 61, 0, 0, 310, 65, 166, 124, 321, 574, 536, 155, 1740, 0, 0, 139, 75, 1100, 1027, 4378, 3326, 2083, 276, 0, 13, 87, 332, 0, 424, 458, 1, 90, 2512, 828, 99, 757, 5, 0, 95, 999, 6241, 136, 0, 4547, 0, 844, 713, 465, 73, 123, 0, 7, 374, 0, 0, 0, 1195, 371, 366] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2193, 2039, 0, 181, 849, 59, 52, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 177, 278, 108, 304, 558, 690, 1597, 1731, 0, 0, 0, 0, 0, 0, 376, 1053, 0, 5, 484, 85, 2575, 1304, 901, 0, 0, 0, 0, 0, 0, 0, 2647, 400, 1303, 271, 0, 354, 176, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 1605, 0, 81, 1811, 684, 181, 720, 807, 0, 0, 0, 0, 0, 0, 722, 21, 0, 72, 2402, 2867, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3296, 0, 1692, 484, 489, 96, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 432, 0, 0, 0, 666, 132, 336, 3356, 1160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5435, 5895, 6783, 5506, 6122, 6886, 6406, 6119] avg=6144.0000 max=6886 max_rank=5 min=5435 min_rank=0 imbalance_ratio=1.120768 max_load_reduction=42266 max_load_reduction_pct=0.859904 moved_to_echo=25533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1981, 1873, 0, 299, 952, 190, 67, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 205, 269, 104, 326, 583, 644, 1467, 1675, 0, 0, 0, 0, 0, 0, 411, 705, 0, 8, 677, 88, 2843, 1564, 971, 0, 0, 0, 0, 0, 0, 0, 2071, 380, 1093, 262, 1, 345, 234, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 1926, 0, 91, 1651, 779, 198, 662, 869, 0, 0, 0, 0, 0, 0, 723, 71, 0, 57, 2390, 2776, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3552, 0, 2040, 493, 413, 115, 374, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 346, 0, 0, 0, 484, 114, 396, 3186, 1123, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5483, 5620, 7267, 4800, 6397, 6939, 6988, 5658] avg=6144.0000 max=7267 max_rank=2 min=4800 min_rank=3 imbalance_ratio=1.182780 max_load_reduction=41885 max_load_reduction_pct=0.852153 moved_to_echo=25982 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2193, 2039, 0, 181, 849, 59, 52, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 177, 278, 108, 304, 558, 690, 1597, 1731, 0, 0, 0, 0, 0, 0, 376, 1053, 0, 5, 484, 85, 2575, 1304, 901, 0, 0, 0, 0, 0, 0, 0, 2647, 400, 1303, 271, 0, 354, 176, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 1605, 0, 81, 1811, 684, 181, 720, 807, 0, 0, 0, 0, 0, 0, 722, 21, 0, 72, 2402, 2867, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3296, 0, 1692, 484, 489, 96, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 432, 0, 0, 0, 666, 132, 336, 3356, 1160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5435, 5895, 6783, 5506, 6122, 6886, 6406, 6119] avg=6144.0000 max=6886 max_rank=5 min=5435 min_rank=0 imbalance_ratio=1.120768 max_load_reduction=42266 max_load_reduction_pct=0.859904 moved_to_echo=25533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2748, 2814, 0, 139, 638, 35, 51, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 122, 181, 98, 313, 459, 704, 2205, 1780, 0, 0, 0, 0, 0, 0, 237, 994, 0, 1, 167, 81, 2248, 1059, 862, 0, 0, 0, 0, 0, 0, 0, 3307, 386, 1378, 300, 1, 39, 89, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 1071, 0, 78, 2341, 651, 133, 995, 772, 0, 0, 0, 0, 0, 0, 754, 18, 0, 84, 1986, 2949, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3153, 0, 1299, 693, 495, 87, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 437, 0, 0, 0, 1000, 222, 342, 3236, 1193, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6438, 6187, 5649, 5862, 6299, 6224, 6060, 6433] avg=6144.0000 max=6438 max_rank=0 min=5649 min_rank=2 imbalance_ratio=1.047852 max_load_reduction=42714 max_load_reduction_pct=0.869019 moved_to_echo=25016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2410, 2685, 20, 160, 737, 52, 53, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 166, 184, 133, 316, 501, 735, 2103, 1848, 0, 0, 0, 0, 0, 0, 402, 1124, 15, 6, 227, 80, 2232, 1470, 967, 0, 0, 0, 0, 0, 0, 0, 2482, 379, 1793, 222, 2, 217, 261, 314, 0, 0, 0, 0, 0, 0, 0, 0, 8, 404, 854, 1, 88, 1624, 872, 138, 949, 865, 0, 0, 0, 0, 0, 0, 701, 13, 0, 68, 1692, 3062, 392, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 11, 1718, 539, 492, 91, 262, 1, 0, 0, 0, 0, 0, 0, 0, 0, 21, 422, 5, 4, 1, 659, 198, 357, 3180, 1238, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6343, 6523, 5670, 5803, 5932, 6650, 6085] avg=6144.0000 max=6650 max_rank=6 min=5670 min_rank=3 imbalance_ratio=1.082357 max_load_reduction=42502 max_load_reduction_pct=0.864705 moved_to_echo=24470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2313, 2555, 0, 143, 673, 47, 54, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 131, 197, 89, 302, 431, 693, 2001, 1826, 0, 0, 0, 0, 0, 0, 318, 1103, 0, 0, 295, 99, 2335, 1383, 936, 0, 0, 0, 0, 0, 0, 0, 2864, 395, 1753, 208, 2, 212, 156, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 1167, 0, 63, 1981, 610, 119, 903, 838, 0, 0, 0, 0, 0, 0, 704, 15, 0, 88, 2007, 3025, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3422, 0, 1497, 652, 507, 65, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 463, 0, 0, 0, 763, 152, 335, 3309, 1224, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5818, 5958, 6469, 5944, 5950, 6273, 6482, 6258] avg=6144.0000 max=6482 max_rank=6 min=5818 min_rank=0 imbalance_ratio=1.055013 max_load_reduction=42670 max_load_reduction_pct=0.868123 moved_to_echo=24963 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2760, 3084, 0, 89, 538, 19, 64, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 104, 202, 110, 313, 507, 629, 2416, 1833, 0, 0, 0, 0, 0, 0, 178, 1415, 0, 0, 184, 87, 1388, 1110, 828, 0, 0, 0, 0, 0, 0, 0, 4107, 350, 2108, 290, 0, 0, 133, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 611, 0, 99, 2454, 633, 138, 1090, 741, 0, 0, 0, 0, 0, 0, 695, 4, 0, 60, 1696, 3038, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 961, 754, 477, 98, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 391, 0, 0, 0, 1166, 268, 315, 2932, 1229, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6560, 6403, 5190, 7374, 6056, 5755, 5509, 6305] avg=6144.0000 max=7374 max_rank=3 min=5190 min_rank=2 imbalance_ratio=1.200195 max_load_reduction=41778 max_load_reduction_pct=0.849976 moved_to_echo=23625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2748, 2814, 0, 139, 638, 35, 51, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 122, 181, 98, 313, 459, 704, 2205, 1780, 0, 0, 0, 0, 0, 0, 237, 994, 0, 1, 167, 81, 2248, 1059, 862, 0, 0, 0, 0, 0, 0, 0, 3307, 386, 1378, 300, 1, 39, 89, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 1071, 0, 78, 2341, 651, 133, 995, 772, 0, 0, 0, 0, 0, 0, 754, 18, 0, 84, 1986, 2949, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3153, 0, 1299, 693, 495, 87, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 437, 0, 0, 0, 1000, 222, 342, 3236, 1193, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6438, 6187, 5649, 5862, 6299, 6224, 6060, 6433] avg=6144.0000 max=6438 max_rank=0 min=5649 min_rank=2 imbalance_ratio=1.047852 max_load_reduction=42714 max_load_reduction_pct=0.869019 moved_to_echo=25016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2454, 2501, 105, 166, 928, 37, 96, 65, 0, 0, 0, 0, 0, 0, 0, 0, 1, 487, 253, 297, 99, 371, 576, 531, 1959, 1659, 0, 0, 0, 0, 0, 0, 378, 1711, 95, 76, 277, 88, 1515, 1264, 810, 0, 0, 0, 0, 0, 0, 0, 3130, 331, 1912, 219, 59, 230, 208, 386, 0, 0, 0, 0, 0, 0, 0, 0, 40, 450, 667, 9, 79, 2154, 1064, 159, 884, 725, 0, 0, 0, 0, 0, 0, 625, 14, 31, 74, 1983, 2750, 511, 63, 0, 0, 0, 0, 0, 0, 0, 0, 2962, 117, 1463, 516, 429, 93, 258, 12, 0, 0, 0, 0, 0, 0, 0, 0, 75, 355, 80, 26, 13, 609, 347, 353, 2776, 1112, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 6233, 6214, 6475, 6231, 6051, 5850, 5746] avg=6144.0000 max=6475 max_rank=3 min=5746 min_rank=7 imbalance_ratio=1.053874 max_load_reduction=42677 max_load_reduction_pct=0.868266 moved_to_echo=23878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2313, 2555, 0, 143, 673, 47, 54, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 131, 197, 89, 302, 431, 693, 2001, 1826, 0, 0, 0, 0, 0, 0, 318, 1103, 0, 0, 295, 99, 2335, 1383, 936, 0, 0, 0, 0, 0, 0, 0, 2864, 395, 1753, 208, 2, 212, 156, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 1167, 0, 63, 1981, 610, 119, 903, 838, 0, 0, 0, 0, 0, 0, 704, 15, 0, 88, 2007, 3025, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3422, 0, 1497, 652, 507, 65, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 463, 0, 0, 0, 763, 152, 335, 3309, 1224, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5818, 5958, 6469, 5944, 5950, 6273, 6482, 6258] avg=6144.0000 max=6482 max_rank=6 min=5818 min_rank=0 imbalance_ratio=1.055013 max_load_reduction=42670 max_load_reduction_pct=0.868123 moved_to_echo=24963 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3047, 3223, 1, 47, 541, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 65, 166, 124, 321, 574, 536, 2537, 1880, 0, 0, 0, 0, 0, 0, 155, 1740, 0, 0, 139, 75, 1100, 1027, 821, 0, 0, 0, 0, 0, 0, 0, 4378, 352, 2083, 276, 0, 13, 87, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 458, 1, 90, 2512, 828, 99, 1145, 737, 0, 0, 0, 0, 0, 0, 757, 5, 0, 95, 999, 3100, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2989, 0, 844, 713, 465, 73, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 374, 0, 0, 0, 1195, 371, 366, 2974, 1261, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6920, 6513, 5057, 7521, 6294, 5092, 5207, 6548] avg=6144.0000 max=7521 max_rank=3 min=5057 min_rank=2 imbalance_ratio=1.224121 max_load_reduction=41631 max_load_reduction_pct=0.846985 moved_to_echo=23141 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 2944, 1280, 1792, 896, 2688, 4224, 5248, 16384, 14336, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 256, 128, 640, 15232, 23680, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[25984, 128, 11520, 4864, 3840, 768, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 9856, 128, 128, 2560, 768, 16256, 10240, 7168, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 3328, 128, 128, 128, 6656, 1920, 2816, 24960, 9600, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[25088, 3072, 13440, 2048, 128, 1536, 1408, 2944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[25984, 128, 11520, 4864, 3840, 768, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[19968, 20864, 128, 1280, 5888, 512, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 9856, 128, 128, 2560, 768, 16256, 10240, 7168, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 2560, 8448, 128, 768, 16640, 6144, 1280, 7424, 6400, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[25088, 3072, 13440, 2048, 128, 1536, 1408, 2944, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[567, 564, 4717, 3370, 1505, 7, 5318, 4673, 1, 177, 10, 880, 18, 1542, 4, 196, 254, 1, 1857, 24, 790, 69, 443, 126, 51, 94, 1341, 0, 0, 140, 11, 187, 5615, 929, 0, 89, 253, 1001, 659, 6, 394, 7, 71, 17, 2, 12, 358, 120, 1865, 2, 253, 312, 4, 1715, 399, 8, 2099, 500, 81, 1645, 1584, 0, 214, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[571, 508, 4951, 3862, 1274, 0, 5594, 4907, 0, 156, 0, 836, 0, 1529, 0, 155, 321, 1, 1770, 28, 894, 43, 333, 139, 51, 78, 1284, 0, 0, 129, 0, 204, 5687, 946, 0, 104, 208, 1213, 623, 0, 392, 0, 14, 0, 0, 0, 354, 107, 1745, 0, 164, 322, 9, 1519, 496, 0, 2096, 552, 72, 1440, 1449, 0, 22, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[575, 456, 4500, 3601, 1318, 0, 5775, 4732, 0, 195, 0, 825, 0, 1470, 0, 159, 261, 0, 2059, 22, 710, 89, 349, 130, 63, 97, 1154, 0, 0, 162, 0, 197, 5698, 867, 0, 98, 256, 1496, 653, 0, 410, 0, 13, 0, 0, 0, 330, 11, 1786, 0, 74, 195, 91, 1665, 542, 0, 1371, 479, 37, 2089, 2001, 0, 91, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[567, 564, 4717, 3370, 1505, 7, 5318, 4673, 1, 177, 10, 880, 18, 1542, 4, 196, 254, 1, 1857, 24, 790, 69, 443, 126, 51, 94, 1341, 0, 0, 140, 11, 187, 5615, 929, 0, 89, 253, 1001, 659, 6, 394, 7, 71, 17, 2, 12, 358, 120, 1865, 2, 253, 312, 4, 1715, 399, 8, 2099, 500, 81, 1645, 1584, 0, 214, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[590, 467, 4726, 3154, 1277, 0, 5418, 4024, 0, 179, 0, 1035, 0, 1657, 0, 167, 267, 0, 1516, 16, 644, 65, 264, 151, 61, 89, 838, 0, 0, 151, 0, 210, 5817, 851, 0, 90, 309, 1496, 718, 0, 410, 0, 6, 0, 0, 0, 341, 59, 2012, 0, 235, 205, 34, 1788, 734, 0, 1644, 465, 57, 2366, 2463, 0, 86, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[547, 554, 4643, 3626, 857, 72, 5511, 4087, 36, 161, 65, 673, 137, 1723, 15, 301, 258, 6, 1639, 41, 490, 99, 584, 163, 55, 122, 1197, 27, 28, 140, 116, 189, 5495, 898, 39, 117, 284, 1407, 611, 28, 420, 58, 312, 82, 11, 116, 378, 104, 1779, 15, 184, 142, 329, 1627, 307, 115, 1356, 416, 137, 1682, 2183, 0, 338, 20] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[600, 639, 4203, 4252, 904, 0, 5807, 5833, 0, 176, 0, 736, 0, 1445, 0, 169, 260, 5, 1786, 19, 752, 75, 678, 140, 62, 88, 1723, 0, 0, 150, 0, 175, 5487, 863, 0, 135, 259, 780, 590, 0, 255, 0, 95, 0, 144, 0, 366, 10, 1785, 0, 250, 316, 18, 1679, 85, 0, 1859, 452, 113, 1374, 1179, 0, 380, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[557, 562, 4485, 3791, 1487, 0, 5796, 5501, 0, 160, 0, 748, 0, 1465, 0, 190, 244, 152, 1754, 51, 914, 52, 561, 133, 52, 80, 1443, 0, 1, 132, 0, 188, 5515, 886, 0, 81, 202, 856, 586, 0, 384, 3, 81, 0, 19, 0, 355, 106, 1402, 0, 212, 293, 27, 1787, 90, 0, 2215, 490, 84, 1445, 1296, 0, 238, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[567, 468, 4642, 3829, 1405, 0, 5530, 5164, 0, 124, 0, 747, 0, 1549, 0, 147, 264, 8, 1893, 47, 886, 57, 144, 148, 54, 61, 1308, 0, 0, 117, 0, 175, 5615, 893, 0, 63, 250, 1043, 623, 0, 364, 0, 23, 0, 0, 0, 354, 45, 1982, 0, 180, 322, 10, 1611, 468, 0, 2167, 521, 86, 1688, 1323, 0, 187, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[600, 639, 4203, 4252, 904, 0, 5807, 5833, 0, 176, 0, 736, 0, 1445, 0, 169, 260, 5, 1786, 19, 752, 75, 678, 140, 62, 88, 1723, 0, 0, 150, 0, 175, 5487, 863, 0, 135, 259, 780, 590, 0, 255, 0, 95, 0, 144, 0, 366, 10, 1785, 0, 250, 316, 18, 1679, 85, 0, 1859, 452, 113, 1374, 1179, 0, 380, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[547, 554, 4643, 3626, 857, 72, 5511, 4087, 36, 161, 65, 673, 137, 1723, 15, 301, 258, 6, 1639, 41, 490, 99, 584, 163, 55, 122, 1197, 27, 28, 140, 116, 189, 5495, 898, 39, 117, 284, 1407, 611, 28, 420, 58, 312, 82, 11, 116, 378, 104, 1779, 15, 184, 142, 329, 1627, 307, 115, 1356, 416, 137, 1682, 2183, 0, 338, 20] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[567, 564, 99, 3370, 1505, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 177, 10, 880, 18, 1542, 4, 196, 2564, 858, 0, 0, 0, 0, 0, 0, 254, 1, 1857, 24, 790, 69, 443, 126, 2054, 644, 0, 0, 0, 0, 0, 0, 51, 94, 1341, 0, 0, 140, 11, 187, 3815, 369, 0, 0, 0, 0, 0, 0, 3099, 929, 0, 89, 253, 1001, 659, 6, 0, 0, 0, 0, 0, 0, 0, 0, 394, 7, 71, 17, 2, 12, 358, 120, 4949, 0, 0, 0, 0, 0, 0, 0, 1865, 2, 253, 312, 4, 1715, 399, 8, 1745, 0, 0, 0, 0, 0, 0, 0, 2099, 500, 81, 1645, 1584, 0, 214, 1, 127, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6250, 6262, 6008, 6036, 5930, 6303, 6251] avg=6144.0000 max=6303 max_rank=6 min=5930 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[575, 456, 94, 3601, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 825, 0, 1470, 0, 159, 2446, 870, 0, 0, 0, 0, 0, 0, 261, 0, 2059, 22, 710, 89, 349, 130, 1960, 653, 0, 0, 0, 0, 0, 0, 63, 97, 1154, 0, 0, 162, 0, 197, 3862, 402, 0, 0, 0, 0, 0, 0, 3146, 867, 0, 98, 256, 1496, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 13, 0, 0, 0, 330, 11, 5373, 0, 0, 0, 0, 0, 0, 0, 1786, 0, 74, 195, 91, 1665, 542, 0, 1770, 0, 0, 0, 0, 0, 0, 0, 1371, 479, 37, 2089, 2001, 0, 91, 0, 129, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6044, 5965, 6233, 5937, 6516, 6137, 6123, 6197] avg=6144.0000 max=6516 max_rank=4 min=5937 min_rank=3 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[575, 456, 4500, 3601, 1318, 0, 5775, 4732, 0, 195, 0, 825, 0, 1470, 0, 159, 261, 0, 2059, 22, 710, 89, 349, 130, 63, 97, 1154, 0, 0, 162, 0, 197, 5698, 867, 0, 98, 256, 1496, 653, 0, 410, 0, 13, 0, 0, 0, 330, 11, 1786, 0, 74, 195, 91, 1665, 542, 0, 1371, 479, 37, 2089, 2001, 0, 91, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[571, 508, 104, 3862, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 836, 0, 1529, 0, 155, 2691, 901, 0, 0, 0, 0, 0, 0, 321, 1, 1770, 28, 894, 43, 333, 139, 2156, 652, 0, 0, 0, 0, 0, 0, 51, 78, 1284, 0, 0, 129, 0, 204, 4006, 388, 0, 0, 0, 0, 0, 0, 3140, 946, 0, 104, 208, 1213, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 14, 0, 0, 0, 354, 107, 5206, 0, 0, 0, 0, 0, 0, 0, 1745, 0, 164, 322, 9, 1519, 496, 0, 1767, 0, 0, 0, 0, 0, 0, 0, 2096, 552, 72, 1440, 1449, 0, 22, 0, 128, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6319, 6268, 6337, 6140, 6234, 6073, 6022, 5759] avg=6144.0000 max=6337 max_rank=2 min=5759 min_rank=7 imbalance_ratio=1.031413 max_load_reduction=42815 max_load_reduction_pct=0.871073 moved_to_echo=24088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[557, 562, 95, 3791, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 748, 0, 1465, 0, 190, 2437, 1011, 0, 0, 0, 0, 0, 0, 244, 152, 1754, 51, 914, 52, 561, 133, 1953, 632, 0, 0, 0, 0, 0, 0, 52, 80, 1443, 0, 1, 132, 0, 188, 4490, 402, 0, 0, 0, 0, 0, 0, 3046, 886, 0, 81, 202, 856, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 3, 81, 0, 19, 0, 355, 106, 5394, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 212, 293, 27, 1787, 90, 0, 1713, 0, 0, 0, 0, 0, 0, 0, 2215, 490, 84, 1445, 1296, 0, 238, 0, 124, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6492, 6011, 6446, 6788, 5657, 6342, 5524, 5892] avg=6144.0000 max=6788 max_rank=3 min=5524 min_rank=6 imbalance_ratio=1.104818 max_load_reduction=42364 max_load_reduction_pct=0.861898 moved_to_echo=23415 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[567, 564, 99, 3370, 1505, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 177, 10, 880, 18, 1542, 4, 196, 2564, 858, 0, 0, 0, 0, 0, 0, 254, 1, 1857, 24, 790, 69, 443, 126, 2054, 644, 0, 0, 0, 0, 0, 0, 51, 94, 1341, 0, 0, 140, 11, 187, 3815, 369, 0, 0, 0, 0, 0, 0, 3099, 929, 0, 89, 253, 1001, 659, 6, 0, 0, 0, 0, 0, 0, 0, 0, 394, 7, 71, 17, 2, 12, 358, 120, 4949, 0, 0, 0, 0, 0, 0, 0, 1865, 2, 253, 312, 4, 1715, 399, 8, 1745, 0, 0, 0, 0, 0, 0, 0, 2099, 500, 81, 1645, 1584, 0, 214, 1, 127, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6250, 6262, 6008, 6036, 5930, 6303, 6251] avg=6144.0000 max=6303 max_rank=6 min=5930 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[600, 639, 89, 4252, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 736, 0, 1445, 0, 169, 2284, 1072, 0, 0, 0, 0, 0, 0, 260, 5, 1786, 19, 752, 75, 678, 140, 1830, 629, 0, 0, 0, 0, 0, 0, 62, 88, 1723, 0, 0, 150, 0, 175, 4761, 403, 0, 0, 0, 0, 0, 0, 3029, 863, 0, 135, 259, 780, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 95, 0, 144, 0, 366, 10, 5404, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 250, 316, 18, 1679, 85, 0, 1705, 0, 0, 0, 0, 0, 0, 0, 1859, 452, 113, 1374, 1179, 0, 380, 1, 124, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6484, 5882, 6174, 7362, 5656, 6274, 5838, 5482] avg=6144.0000 max=7362 max_rank=3 min=5482 min_rank=7 imbalance_ratio=1.198242 max_load_reduction=41790 max_load_reduction_pct=0.850220 moved_to_echo=23250 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[571, 508, 104, 3862, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 836, 0, 1529, 0, 155, 2691, 901, 0, 0, 0, 0, 0, 0, 321, 1, 1770, 28, 894, 43, 333, 139, 2156, 652, 0, 0, 0, 0, 0, 0, 51, 78, 1284, 0, 0, 129, 0, 204, 4006, 388, 0, 0, 0, 0, 0, 0, 3140, 946, 0, 104, 208, 1213, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 14, 0, 0, 0, 354, 107, 5206, 0, 0, 0, 0, 0, 0, 0, 1745, 0, 164, 322, 9, 1519, 496, 0, 1767, 0, 0, 0, 0, 0, 0, 0, 2096, 552, 72, 1440, 1449, 0, 22, 0, 128, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6319, 6268, 6337, 6140, 6234, 6073, 6022, 5759] avg=6144.0000 max=6337 max_rank=2 min=5759 min_rank=7 imbalance_ratio=1.031413 max_load_reduction=42815 max_load_reduction_pct=0.871073 moved_to_echo=24088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[567, 468, 98, 3829, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 747, 0, 1549, 0, 147, 2523, 950, 0, 0, 0, 0, 0, 0, 264, 8, 1893, 47, 886, 57, 144, 148, 2021, 644, 0, 0, 0, 0, 0, 0, 54, 61, 1308, 0, 0, 117, 0, 175, 4214, 385, 0, 0, 0, 0, 0, 0, 3099, 893, 0, 63, 250, 1043, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 23, 0, 0, 0, 354, 45, 5145, 0, 0, 0, 0, 0, 0, 0, 1982, 0, 180, 322, 10, 1611, 468, 0, 1745, 0, 0, 0, 0, 0, 0, 0, 2167, 521, 86, 1688, 1323, 0, 187, 0, 127, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6367, 6040, 6112, 6314, 5971, 5931, 6318, 6099] avg=6144.0000 max=6367 max_rank=0 min=5931 min_rank=5 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[590, 467, 92, 3154, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 1035, 0, 1657, 0, 167, 2572, 740, 0, 0, 0, 0, 0, 0, 267, 0, 1516, 16, 644, 65, 264, 151, 2062, 671, 0, 0, 0, 0, 0, 0, 61, 89, 838, 0, 0, 151, 0, 210, 3284, 377, 0, 0, 0, 0, 0, 0, 3200, 851, 0, 90, 309, 1496, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 6, 0, 0, 0, 341, 59, 5041, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 235, 205, 34, 1788, 734, 0, 1811, 0, 0, 0, 0, 0, 0, 0, 1644, 465, 57, 2366, 2463, 0, 86, 0, 135, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5580, 6350, 5656, 5010, 6664, 5857, 6819, 7216] avg=6144.0000 max=7216 max_rank=7 min=5010 min_rank=3 imbalance_ratio=1.174479 max_load_reduction=41936 max_load_reduction_pct=0.853190 moved_to_echo=26556 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[547, 554, 98, 3626, 857, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 161, 65, 673, 137, 1723, 15, 301, 2523, 752, 0, 0, 0, 0, 0, 0, 258, 6, 1639, 41, 490, 99, 584, 163, 2022, 630, 0, 0, 0, 0, 0, 0, 55, 122, 1197, 27, 28, 140, 116, 189, 3335, 384, 0, 0, 0, 0, 0, 0, 3034, 898, 39, 117, 284, 1407, 611, 28, 0, 0, 0, 0, 0, 0, 0, 0, 420, 58, 312, 82, 11, 116, 378, 104, 5127, 0, 0, 0, 0, 0, 0, 0, 1779, 15, 184, 142, 329, 1627, 307, 115, 1707, 0, 0, 0, 0, 0, 0, 0, 1356, 416, 137, 1682, 2183, 0, 338, 20, 124, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5754, 6386, 5932, 5593, 6418, 6608, 6205, 6256] avg=6144.0000 max=6608 max_rank=5 min=5593 min_rank=3 imbalance_ratio=1.075521 max_load_reduction=42544 max_load_reduction_pct=0.865560 moved_to_echo=25487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[590, 467, 92, 3154, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 1035, 0, 1657, 0, 167, 2572, 740, 0, 0, 0, 0, 0, 0, 267, 0, 1516, 16, 644, 65, 264, 151, 2062, 671, 0, 0, 0, 0, 0, 0, 61, 89, 838, 0, 0, 151, 0, 210, 3284, 377, 0, 0, 0, 0, 0, 0, 3200, 851, 0, 90, 309, 1496, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 6, 0, 0, 0, 341, 59, 5041, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 235, 205, 34, 1788, 734, 0, 1811, 0, 0, 0, 0, 0, 0, 0, 1644, 465, 57, 2366, 2463, 0, 86, 0, 135, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5580, 6350, 5656, 5010, 6664, 5857, 6819, 7216] avg=6144.0000 max=7216 max_rank=7 min=5010 min_rank=3 imbalance_ratio=1.174479 max_load_reduction=41936 max_load_reduction_pct=0.853190 moved_to_echo=26556 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[567, 468, 98, 3829, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 747, 0, 1549, 0, 147, 2523, 950, 0, 0, 0, 0, 0, 0, 264, 8, 1893, 47, 886, 57, 144, 148, 2021, 644, 0, 0, 0, 0, 0, 0, 54, 61, 1308, 0, 0, 117, 0, 175, 4214, 385, 0, 0, 0, 0, 0, 0, 3099, 893, 0, 63, 250, 1043, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 23, 0, 0, 0, 354, 45, 5145, 0, 0, 0, 0, 0, 0, 0, 1982, 0, 180, 322, 10, 1611, 468, 0, 1745, 0, 0, 0, 0, 0, 0, 0, 2167, 521, 86, 1688, 1323, 0, 187, 0, 127, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6367, 6040, 6112, 6314, 5971, 5931, 6318, 6099] avg=6144.0000 max=6367 max_rank=0 min=5931 min_rank=5 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[575, 456, 94, 3601, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 825, 0, 1470, 0, 159, 2446, 870, 0, 0, 0, 0, 0, 0, 261, 0, 2059, 22, 710, 89, 349, 130, 1960, 653, 0, 0, 0, 0, 0, 0, 63, 97, 1154, 0, 0, 162, 0, 197, 3862, 402, 0, 0, 0, 0, 0, 0, 3146, 867, 0, 98, 256, 1496, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 13, 0, 0, 0, 330, 11, 5373, 0, 0, 0, 0, 0, 0, 0, 1786, 0, 74, 195, 91, 1665, 542, 0, 1770, 0, 0, 0, 0, 0, 0, 0, 1371, 479, 37, 2089, 2001, 0, 91, 0, 129, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6044, 5965, 6233, 5937, 6516, 6137, 6123, 6197] avg=6144.0000 max=6516 max_rank=4 min=5937 min_rank=3 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[24832, 7168, 128, 896, 2048, 9344, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[14464, 128, 1664, 2176, 640, 13440, 3200, 128, 14080, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1408, 128, 6528, 256, 12416, 128, 1536, 20096, 7168, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[14464, 128, 1664, 2176, 640, 13440, 3200, 128, 14080, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14848, 3968, 768, 13824, 13568, 0, 1664, 128, 1024, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2176, 256, 14336, 256, 6144, 640, 3456, 1152, 16128, 5248, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4608, 4224, 896, 29568, 10112, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 768, 10368, 128, 128, 1152, 128, 1536, 31872, 3200, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 128, 640, 128, 256, 128, 2944, 640, 41728, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 7322, 1803, 4, 0, 64, 705, 3854, 0, 4, 0, 0, 0, 640, 2532, 162, 0, 693, 110, 231, 82, 0, 169, 710, 758, 328, 0, 141, 223, 80, 209, 0, 142, 7537, 101, 0, 2583, 0, 725, 0, 1560, 0, 0, 0, 2246, 202, 0, 0, 98, 0, 100, 692, 8, 464, 97, 0, 0, 381, 6544, 199, 287, 1938, 0, 2424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1, 7144, 2187, 11, 1, 56, 662, 4216, 1, 37, 12, 3, 1, 649, 2628, 143, 67, 606, 139, 246, 67, 3, 154, 769, 421, 343, 33, 157, 251, 67, 230, 1, 106, 7333, 92, 6, 1560, 16, 1063, 1, 1250, 5, 29, 7, 2263, 252, 3, 18, 81, 20, 165, 1105, 34, 519, 105, 19, 18, 401, 6375, 100, 249, 2183, 24, 2444] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 7188, 1909, 12, 1, 54, 710, 4508, 0, 1, 0, 0, 0, 686, 2389, 124, 93, 471, 89, 242, 62, 0, 153, 806, 558, 303, 13, 177, 239, 57, 150, 0, 108, 7485, 67, 0, 1647, 0, 850, 0, 1458, 0, 0, 0, 2328, 164, 0, 0, 63, 0, 64, 793, 22, 514, 61, 0, 0, 446, 6617, 140, 222, 2165, 0, 2943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 7322, 1803, 4, 0, 64, 705, 3854, 0, 4, 0, 0, 0, 640, 2532, 162, 0, 693, 110, 231, 82, 0, 169, 710, 758, 328, 0, 141, 223, 80, 209, 0, 142, 7537, 101, 0, 2583, 0, 725, 0, 1560, 0, 0, 0, 2246, 202, 0, 0, 98, 0, 100, 692, 8, 464, 97, 0, 0, 381, 6544, 199, 287, 1938, 0, 2424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7170, 1924, 1, 2, 53, 720, 4389, 0, 5, 0, 0, 0, 700, 2464, 131, 1, 484, 98, 244, 70, 0, 180, 781, 698, 322, 0, 155, 244, 53, 179, 4, 115, 7459, 83, 0, 1799, 0, 747, 0, 1409, 0, 0, 0, 2429, 243, 0, 0, 78, 0, 84, 817, 25, 491, 78, 0, 0, 417, 6627, 147, 277, 1988, 0, 2767] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 6632, 1853, 1, 1, 65, 684, 4820, 0, 0, 0, 0, 0, 515, 2113, 160, 362, 433, 104, 257, 72, 0, 202, 864, 673, 345, 64, 140, 256, 71, 199, 0, 58, 7381, 123, 0, 894, 0, 1064, 0, 957, 0, 0, 0, 2308, 170, 0, 0, 15, 0, 90, 1175, 79, 506, 89, 0, 0, 325, 6427, 82, 124, 2398, 73, 3928] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 6960, 1997, 17, 1, 60, 639, 4485, 0, 4, 0, 0, 0, 573, 2276, 140, 297, 475, 108, 241, 62, 0, 183, 852, 644, 322, 5, 144, 242, 53, 189, 1, 104, 7353, 102, 0, 1329, 0, 944, 0, 1129, 0, 0, 0, 2384, 136, 0, 0, 67, 0, 80, 1081, 54, 529, 80, 0, 0, 409, 6419, 101, 179, 2228, 13, 3461] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 7188, 1909, 12, 1, 54, 710, 4508, 0, 1, 0, 0, 0, 686, 2389, 124, 93, 471, 89, 242, 62, 0, 153, 806, 558, 303, 13, 177, 239, 57, 150, 0, 108, 7485, 67, 0, 1647, 0, 850, 0, 1458, 0, 0, 0, 2328, 164, 0, 0, 63, 0, 64, 793, 22, 514, 61, 0, 0, 446, 6617, 140, 222, 2165, 0, 2943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 7395, 1998, 0, 0, 61, 684, 3672, 0, 5, 0, 0, 0, 703, 2721, 151, 0, 744, 124, 252, 69, 0, 129, 682, 607, 340, 0, 165, 216, 67, 216, 0, 139, 7526, 90, 0, 3264, 0, 662, 0, 1320, 0, 0, 0, 2310, 252, 0, 0, 90, 0, 92, 771, 3, 417, 86, 0, 0, 376, 6669, 211, 282, 1689, 0, 1902] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7170, 1924, 1, 2, 53, 720, 4389, 0, 5, 0, 0, 0, 700, 2464, 131, 1, 484, 98, 244, 70, 0, 180, 781, 698, 322, 0, 155, 244, 53, 179, 4, 115, 7459, 83, 0, 1799, 0, 747, 0, 1409, 0, 0, 0, 2429, 243, 0, 0, 78, 0, 84, 817, 25, 491, 78, 0, 0, 417, 6627, 147, 277, 1988, 0, 2767] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[21, 6894, 3353, 70, 5, 55, 817, 2685, 7, 239, 120, 30, 9, 662, 2584, 149, 160, 639, 126, 199, 89, 101, 209, 649, 655, 346, 117, 146, 262, 97, 204, 6, 375, 6907, 89, 49, 1650, 125, 527, 2, 861, 2, 198, 18, 1930, 292, 18, 155, 90, 135, 80, 893, 31, 554, 151, 207, 19, 328, 6198, 244, 284, 2041, 136, 2858] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 6632, 1853, 1, 1, 65, 684, 4820, 0, 0, 0, 0, 0, 515, 2113, 160, 362, 433, 104, 257, 72, 0, 202, 864, 673, 345, 64, 140, 256, 71, 199, 0, 58, 7381, 123, 0, 894, 0, 1064, 0, 957, 0, 0, 0, 2308, 170, 0, 0, 15, 0, 90, 1175, 79, 506, 89, 0, 0, 325, 6427, 82, 124, 2398, 73, 3928] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2187, 11, 1, 56, 662, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 1, 37, 12, 3, 1, 649, 2628, 143, 2626, 191, 0, 0, 0, 0, 0, 0, 67, 606, 139, 246, 67, 3, 154, 769, 4012, 0, 0, 0, 0, 0, 0, 0, 421, 343, 33, 157, 251, 67, 230, 1, 4531, 0, 0, 0, 0, 0, 0, 0, 106, 3202, 92, 6, 1560, 16, 1063, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 5, 29, 7, 2263, 252, 3, 18, 1505, 870, 0, 0, 0, 0, 0, 0, 81, 20, 165, 1105, 34, 519, 105, 19, 2613, 1766, 0, 0, 0, 0, 0, 0, 18, 401, 406, 100, 249, 2183, 24, 2444, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 6291, 6063, 6034, 6046, 6202, 6427, 5825] avg=6144.0000 max=6427 max_rank=6 min=5825 min_rank=7 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24500 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1803, 4, 0, 64, 705, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 640, 2532, 162, 2699, 196, 0, 0, 0, 0, 0, 0, 0, 693, 110, 231, 82, 0, 169, 710, 4118, 0, 0, 0, 0, 0, 0, 0, 758, 328, 0, 141, 223, 80, 209, 0, 4645, 0, 0, 0, 0, 0, 0, 0, 142, 3291, 101, 0, 2583, 0, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1560, 0, 0, 0, 2246, 202, 0, 0, 1547, 795, 0, 0, 0, 0, 0, 0, 98, 0, 100, 692, 8, 464, 97, 0, 2677, 1812, 0, 0, 0, 0, 0, 0, 0, 381, 418, 199, 287, 1938, 0, 2424, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5635, 6233, 6113, 6384, 6842, 6350, 5948, 5647] avg=6144.0000 max=6842 max_rank=4 min=5635 min_rank=0 imbalance_ratio=1.113607 max_load_reduction=42310 max_load_reduction_pct=0.860799 moved_to_echo=24787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1997, 17, 1, 60, 639, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 573, 2276, 140, 2633, 192, 0, 0, 0, 0, 0, 0, 297, 475, 108, 241, 62, 0, 183, 852, 4039, 0, 0, 0, 0, 0, 0, 0, 644, 322, 5, 144, 242, 53, 189, 1, 4414, 0, 0, 0, 0, 0, 0, 0, 104, 3211, 102, 0, 1329, 0, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 0, 2384, 136, 0, 0, 1509, 926, 0, 0, 0, 0, 0, 0, 67, 0, 80, 1081, 54, 529, 80, 0, 2546, 1778, 0, 0, 0, 0, 0, 0, 0, 409, 410, 101, 179, 2228, 13, 3461, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6273, 5818, 6257, 6014, 5690, 6084, 6215, 6801] avg=6144.0000 max=6801 max_rank=7 min=5690 min_rank=4 imbalance_ratio=1.106934 max_load_reduction=42351 max_load_reduction_pct=0.861633 moved_to_echo=24790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1909, 12, 1, 54, 710, 3577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 686, 2389, 124, 2680, 198, 0, 0, 0, 0, 0, 0, 93, 471, 89, 242, 62, 0, 153, 806, 4164, 0, 0, 0, 0, 0, 0, 0, 558, 303, 13, 177, 239, 57, 150, 0, 4559, 0, 0, 0, 0, 0, 0, 0, 108, 3269, 67, 0, 1647, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 0, 0, 2328, 164, 0, 0, 1536, 931, 0, 0, 0, 0, 0, 0, 63, 0, 64, 793, 22, 514, 61, 0, 2629, 1833, 0, 0, 0, 0, 0, 0, 0, 446, 422, 140, 222, 2165, 0, 2943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 6078, 6080, 6056, 5941, 6417, 5979, 6338] avg=6144.0000 max=6417 max_rank=5 min=5941 min_rank=4 imbalance_ratio=1.044434 max_load_reduction=42735 max_load_reduction_pct=0.869446 moved_to_echo=24675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1998, 0, 0, 61, 684, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 703, 2721, 151, 2698, 204, 0, 0, 0, 0, 0, 0, 0, 744, 124, 252, 69, 0, 129, 682, 4201, 0, 0, 0, 0, 0, 0, 0, 607, 340, 0, 165, 216, 67, 216, 0, 4691, 0, 0, 0, 0, 0, 0, 0, 139, 3281, 90, 0, 3264, 0, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 0, 0, 0, 2310, 252, 0, 0, 1547, 762, 0, 0, 0, 0, 0, 0, 90, 0, 92, 771, 3, 417, 86, 0, 2704, 1850, 0, 0, 0, 0, 0, 0, 0, 376, 414, 211, 282, 1689, 0, 1902, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5653, 6482, 6201, 6302, 7436, 6191, 6013, 4874] avg=6144.0000 max=7436 max_rank=4 min=4874 min_rank=7 imbalance_ratio=1.210286 max_load_reduction=41716 max_load_reduction_pct=0.848714 moved_to_echo=24514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1853, 1, 1, 65, 684, 3825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 2113, 160, 2643, 192, 0, 0, 0, 0, 0, 0, 362, 433, 104, 257, 72, 0, 202, 864, 4044, 0, 0, 0, 0, 0, 0, 0, 673, 345, 64, 140, 256, 71, 199, 0, 4206, 0, 0, 0, 0, 0, 0, 0, 58, 3223, 123, 0, 894, 0, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 957, 0, 0, 0, 2308, 170, 0, 0, 1515, 995, 0, 0, 0, 0, 0, 0, 15, 0, 90, 1175, 79, 506, 89, 0, 2426, 1780, 0, 0, 0, 0, 0, 0, 0, 325, 411, 82, 124, 2398, 73, 3928, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6429, 5623, 6338, 5954, 5362, 5945, 6160, 7341] avg=6144.0000 max=7341 max_rank=7 min=5362 min_rank=4 imbalance_ratio=1.194824 max_load_reduction=41811 max_load_reduction_pct=0.850647 moved_to_echo=24808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1803, 4, 0, 64, 705, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 640, 2532, 162, 2699, 196, 0, 0, 0, 0, 0, 0, 0, 693, 110, 231, 82, 0, 169, 710, 4118, 0, 0, 0, 0, 0, 0, 0, 758, 328, 0, 141, 223, 80, 209, 0, 4645, 0, 0, 0, 0, 0, 0, 0, 142, 3291, 101, 0, 2583, 0, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1560, 0, 0, 0, 2246, 202, 0, 0, 1547, 795, 0, 0, 0, 0, 0, 0, 98, 0, 100, 692, 8, 464, 97, 0, 2677, 1812, 0, 0, 0, 0, 0, 0, 0, 381, 418, 199, 287, 1938, 0, 2424, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5635, 6233, 6113, 6384, 6842, 6350, 5948, 5647] avg=6144.0000 max=6842 max_rank=4 min=5635 min_rank=0 imbalance_ratio=1.113607 max_load_reduction=42310 max_load_reduction_pct=0.860799 moved_to_echo=24787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1924, 1, 2, 53, 720, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 700, 2464, 131, 2671, 198, 0, 0, 0, 0, 0, 0, 1, 484, 98, 244, 70, 0, 180, 781, 4170, 0, 0, 0, 0, 0, 0, 0, 698, 322, 0, 155, 244, 53, 179, 4, 4547, 0, 0, 0, 0, 0, 0, 0, 115, 3257, 83, 0, 1799, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 0, 0, 0, 2429, 243, 0, 0, 1531, 906, 0, 0, 0, 0, 0, 0, 78, 0, 84, 817, 25, 491, 78, 0, 2623, 1835, 0, 0, 0, 0, 0, 0, 0, 417, 424, 147, 277, 1988, 0, 2767, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6169, 6028, 6202, 6001, 6518, 6031, 6020] avg=6144.0000 max=6518 max_rank=5 min=6001 min_rank=4 imbalance_ratio=1.060872 max_load_reduction=42634 max_load_reduction_pct=0.867391 moved_to_echo=24570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 3353, 70, 5, 55, 817, 2131, 0, 0, 0, 0, 0, 0, 0, 0, 7, 239, 120, 30, 9, 662, 2584, 149, 2473, 185, 0, 0, 0, 0, 0, 0, 160, 639, 126, 199, 89, 101, 209, 649, 3900, 0, 0, 0, 0, 0, 0, 0, 655, 346, 117, 146, 262, 97, 204, 6, 4373, 0, 0, 0, 0, 0, 0, 0, 375, 3017, 89, 49, 1650, 125, 527, 2, 0, 0, 0, 0, 0, 0, 0, 0, 861, 2, 198, 18, 1930, 292, 18, 155, 1417, 554, 0, 0, 0, 0, 0, 0, 90, 135, 80, 893, 31, 554, 151, 207, 2521, 1717, 0, 0, 0, 0, 0, 0, 19, 328, 396, 244, 284, 2041, 136, 2858, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6452, 6458, 6072, 6206, 5834, 5445, 6379, 6306] avg=6144.0000 max=6458 max_rank=1 min=5445 min_rank=5 imbalance_ratio=1.051107 max_load_reduction=42694 max_load_reduction_pct=0.868612 moved_to_echo=23964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1909, 12, 1, 54, 710, 3577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 686, 2389, 124, 2680, 198, 0, 0, 0, 0, 0, 0, 93, 471, 89, 242, 62, 0, 153, 806, 4164, 0, 0, 0, 0, 0, 0, 0, 558, 303, 13, 177, 239, 57, 150, 0, 4559, 0, 0, 0, 0, 0, 0, 0, 108, 3269, 67, 0, 1647, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 0, 0, 2328, 164, 0, 0, 1536, 931, 0, 0, 0, 0, 0, 0, 63, 0, 64, 793, 22, 514, 61, 0, 2629, 1833, 0, 0, 0, 0, 0, 0, 0, 446, 422, 140, 222, 2165, 0, 2943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 6078, 6080, 6056, 5941, 6417, 5979, 6338] avg=6144.0000 max=6417 max_rank=5 min=5941 min_rank=4 imbalance_ratio=1.044434 max_load_reduction=42735 max_load_reduction_pct=0.869446 moved_to_echo=24675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1998, 0, 0, 61, 684, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 703, 2721, 151, 2698, 204, 0, 0, 0, 0, 0, 0, 0, 744, 124, 252, 69, 0, 129, 682, 4201, 0, 0, 0, 0, 0, 0, 0, 607, 340, 0, 165, 216, 67, 216, 0, 4691, 0, 0, 0, 0, 0, 0, 0, 139, 3281, 90, 0, 3264, 0, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 0, 0, 0, 2310, 252, 0, 0, 1547, 762, 0, 0, 0, 0, 0, 0, 90, 0, 92, 771, 3, 417, 86, 0, 2704, 1850, 0, 0, 0, 0, 0, 0, 0, 376, 414, 211, 282, 1689, 0, 1902, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5653, 6482, 6201, 6302, 7436, 6191, 6013, 4874] avg=6144.0000 max=7436 max_rank=4 min=4874 min_rank=7 imbalance_ratio=1.210286 max_load_reduction=41716 max_load_reduction_pct=0.848714 moved_to_echo=24514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 256, 768, 7424, 256, 4096, 768, 256, 20864, 14464, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[9984, 128, 256, 128, 18304, 1792, 128, 256, 12160, 6784, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 384, 256, 128, 128, 5248, 19712, 1280, 21248, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 25856, 768, 128, 14848, 256, 6656, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 3200, 3328, 1280, 1920, 16640, 256, 22784, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 17024, 128, 128, 512, 5632, 25984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[9984, 128, 256, 128, 18304, 1792, 128, 256, 12160, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 25856, 768, 128, 14848, 256, 6656, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 2688, 256, 1280, 2048, 640, 1664, 128, 35968, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 17024, 128, 128, 512, 5632, 25984, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1024, 4608, 1024, 1920, 640, 128, 1408, 6144, 32768, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[6, 8, 0, 325, 3, 1, 4, 17, 367, 224, 1362, 12, 913, 340, 7, 88, 57, 5, 1, 0, 24, 63, 0, 54, 5, 4117, 2, 281, 249, 1372, 26, 51, 898, 2163, 0, 6, 1, 1629, 8, 1050, 2296, 1272, 6038, 14, 589, 687, 842, 10, 1, 214, 21, 3186, 11, 19, 63, 4, 898, 19, 5132, 5677, 4361, 1892, 0, 167] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 303, 0, 0, 0, 0, 376, 556, 1379, 0, 841, 328, 0, 97, 60, 0, 0, 0, 0, 82, 0, 0, 2, 4675, 0, 321, 240, 1440, 0, 0, 680, 2315, 0, 0, 0, 1771, 0, 1030, 3154, 765, 5991, 0, 555, 1160, 1108, 0, 0, 186, 0, 2658, 0, 0, 60, 0, 816, 0, 5296, 6043, 3253, 1416, 0, 195] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 363, 0, 0, 1, 0, 356, 253, 1347, 0, 961, 322, 0, 82, 65, 0, 9, 0, 0, 62, 0, 2, 1, 4340, 0, 272, 256, 1120, 3, 0, 1234, 1491, 0, 0, 0, 1760, 0, 973, 1876, 1352, 6440, 0, 614, 598, 701, 3, 0, 232, 0, 3690, 2, 0, 62, 0, 863, 0, 4734, 5857, 4445, 2252, 0, 158] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 301, 0, 0, 0, 0, 358, 345, 1619, 0, 958, 302, 0, 61, 88, 0, 3, 0, 0, 62, 0, 0, 1, 4189, 0, 250, 231, 1308, 1, 0, 821, 1770, 0, 0, 0, 1783, 0, 1015, 1983, 848, 6349, 0, 629, 647, 681, 3, 0, 163, 0, 3613, 0, 0, 88, 0, 891, 0, 5026, 6061, 4664, 1917, 0, 123] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 351, 0, 0, 0, 0, 365, 459, 1805, 0, 827, 322, 0, 78, 84, 0, 6, 0, 0, 70, 0, 0, 3, 4329, 0, 261, 238, 1383, 1, 0, 918, 1474, 0, 0, 0, 1612, 0, 1073, 2410, 983, 5936, 0, 618, 772, 737, 4, 0, 200, 0, 3345, 0, 0, 84, 0, 862, 0, 5353, 5987, 4226, 1821, 0, 155] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 319, 0, 0, 0, 0, 391, 203, 1113, 0, 1059, 345, 0, 104, 58, 0, 2, 0, 0, 72, 0, 0, 0, 4111, 0, 280, 219, 1092, 0, 0, 1382, 1366, 0, 0, 0, 1811, 0, 974, 1350, 1638, 6380, 0, 484, 603, 736, 0, 0, 224, 0, 4555, 0, 0, 57, 0, 716, 0, 4070, 5875, 4729, 2658, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 201, 0, 0, 0, 0, 382, 475, 1942, 0, 880, 341, 0, 89, 95, 0, 0, 0, 0, 69, 0, 0, 4, 4464, 0, 286, 241, 1492, 6, 0, 434, 2614, 0, 0, 0, 1928, 0, 1004, 3474, 594, 5023, 0, 554, 1507, 1153, 7, 0, 132, 0, 2392, 0, 0, 95, 0, 940, 0, 5761, 5962, 3216, 1222, 0, 171] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[6, 8, 0, 325, 3, 1, 4, 17, 367, 224, 1362, 12, 913, 340, 7, 88, 57, 5, 1, 0, 24, 63, 0, 54, 5, 4117, 2, 281, 249, 1372, 26, 51, 898, 2163, 0, 6, 1, 1629, 8, 1050, 2296, 1272, 6038, 14, 589, 687, 842, 10, 1, 214, 21, 3186, 11, 19, 63, 4, 898, 19, 5132, 5677, 4361, 1892, 0, 167] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[33, 25, 2, 353, 141, 13, 41, 212, 412, 324, 2014, 113, 1064, 385, 77, 85, 42, 43, 0, 9, 121, 110, 0, 116, 70, 4288, 10, 372, 294, 1389, 84, 160, 594, 1898, 1, 63, 0, 1941, 60, 1116, 3480, 1492, 5137, 178, 521, 862, 1461, 56, 2, 190, 194, 2201, 32, 150, 63, 33, 1101, 91, 4180, 5033, 3052, 1415, 7, 146] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 303, 0, 0, 0, 0, 376, 556, 1379, 0, 841, 328, 0, 97, 60, 0, 0, 0, 0, 82, 0, 0, 2, 4675, 0, 321, 240, 1440, 0, 0, 680, 2315, 0, 0, 0, 1771, 0, 1030, 3154, 765, 5991, 0, 555, 1160, 1108, 0, 0, 186, 0, 2658, 0, 0, 60, 0, 816, 0, 5296, 6043, 3253, 1416, 0, 195] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6, 8, 0, 325, 3, 1, 4, 17, 5755, 135, 0, 0, 0, 0, 0, 0, 367, 224, 1362, 12, 913, 340, 7, 88, 2280, 282, 0, 0, 0, 0, 0, 0, 57, 5, 1, 0, 24, 63, 0, 54, 5677, 143, 0, 0, 0, 0, 0, 0, 5, 4037, 2, 281, 249, 1372, 26, 51, 0, 0, 0, 0, 0, 0, 0, 0, 898, 2163, 0, 6, 1, 1629, 8, 1050, 520, 80, 0, 0, 0, 0, 0, 0, 2296, 1272, 140, 14, 589, 687, 842, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 214, 21, 3186, 11, 19, 63, 4, 2717, 0, 0, 0, 0, 0, 0, 0, 898, 19, 0, 0, 3559, 1892, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6254, 5875, 6024, 6023, 6355, 5850, 6236, 6535] avg=6144.0000 max=6535 max_rank=7 min=5850 min_rank=5 imbalance_ratio=1.063639 max_load_reduction=42617 max_load_reduction_pct=0.867045 moved_to_echo=24976 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 301, 0, 0, 0, 0, 358, 345, 1619, 0, 958, 302, 0, 61, 88, 0, 3, 0, 0, 62, 0, 0, 1, 4189, 0, 250, 231, 1308, 1, 0, 821, 1770, 0, 0, 0, 1783, 0, 1015, 1983, 848, 6349, 0, 629, 647, 681, 3, 0, 163, 0, 3613, 0, 0, 88, 0, 891, 0, 5026, 6061, 4664, 1917, 0, 123] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 363, 0, 0, 1, 0, 6138, 125, 0, 0, 0, 0, 0, 0, 356, 253, 1347, 0, 961, 322, 0, 82, 2104, 287, 0, 0, 0, 0, 0, 0, 65, 0, 9, 0, 0, 62, 0, 2, 5857, 152, 0, 0, 0, 0, 0, 0, 1, 4255, 0, 272, 256, 1120, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 1491, 0, 0, 0, 1760, 0, 973, 530, 85, 0, 0, 0, 0, 0, 0, 1876, 1352, 150, 0, 614, 598, 701, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 3690, 2, 0, 62, 0, 2505, 0, 0, 0, 0, 0, 0, 0, 863, 0, 0, 0, 3628, 2252, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6627, 5712, 6147, 5907, 6073, 5294, 6491, 6901] avg=6144.0000 max=6901 max_rank=7 min=5294 min_rank=5 imbalance_ratio=1.123210 max_load_reduction=42251 max_load_reduction_pct=0.859599 moved_to_echo=24759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 301, 0, 0, 0, 0, 6051, 133, 0, 0, 0, 0, 0, 0, 358, 345, 1619, 0, 958, 302, 0, 61, 2234, 302, 0, 0, 0, 0, 0, 0, 88, 0, 3, 0, 0, 62, 0, 0, 6061, 150, 0, 0, 0, 0, 0, 0, 1, 4107, 0, 250, 231, 1308, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 1770, 0, 0, 0, 1783, 0, 1015, 556, 82, 0, 0, 0, 0, 0, 0, 1983, 848, 148, 0, 629, 647, 681, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 3613, 0, 0, 88, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 891, 0, 0, 0, 3806, 1917, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6485, 6179, 6364, 5898, 6027, 4939, 6523, 6737] avg=6144.0000 max=6737 max_rank=7 min=4939 min_rank=5 imbalance_ratio=1.096517 max_load_reduction=42415 max_load_reduction_pct=0.862935 moved_to_echo=24226 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 303, 0, 0, 0, 0, 5710, 141, 0, 0, 0, 0, 0, 0, 376, 556, 1379, 0, 841, 328, 0, 97, 2353, 210, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 82, 0, 0, 6043, 142, 0, 0, 0, 0, 0, 0, 2, 4584, 0, 321, 240, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 2315, 0, 0, 0, 1771, 0, 1030, 388, 91, 0, 0, 0, 0, 0, 0, 3154, 765, 139, 0, 555, 1160, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 2658, 0, 0, 60, 0, 2802, 0, 0, 0, 0, 0, 0, 0, 816, 0, 0, 0, 2655, 1416, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6154, 6140, 6327, 6587, 6275, 6881, 5706, 5082] avg=6144.0000 max=6881 max_rank=5 min=5082 min_rank=7 imbalance_ratio=1.119954 max_load_reduction=42271 max_load_reduction_pct=0.860006 moved_to_echo=23944 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 319, 0, 0, 0, 0, 6081, 107, 0, 0, 0, 0, 0, 0, 391, 203, 1113, 0, 1059, 345, 0, 104, 1809, 306, 0, 0, 0, 0, 0, 0, 58, 0, 2, 0, 0, 72, 0, 0, 5875, 151, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 280, 219, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 1366, 0, 0, 0, 1811, 0, 974, 564, 80, 0, 0, 0, 0, 0, 0, 1350, 1638, 148, 0, 484, 603, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 4555, 0, 0, 57, 0, 2154, 0, 0, 0, 0, 0, 0, 0, 716, 0, 0, 0, 3859, 2658, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6507, 5330, 6158, 5622, 6177, 4959, 6990, 7409] avg=6144.0000 max=7409 max_rank=7 min=4959 min_rank=5 imbalance_ratio=1.205892 max_load_reduction=41743 max_load_reduction_pct=0.849264 moved_to_echo=25535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 351, 0, 0, 0, 0, 5658, 141, 0, 0, 0, 0, 0, 0, 365, 459, 1805, 0, 827, 322, 0, 78, 2378, 273, 0, 0, 0, 0, 0, 0, 84, 0, 6, 0, 0, 70, 0, 0, 5987, 140, 0, 0, 0, 0, 0, 0, 3, 4244, 0, 261, 238, 1383, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 918, 1474, 0, 0, 0, 1612, 0, 1073, 504, 85, 0, 0, 0, 0, 0, 0, 2410, 983, 138, 0, 618, 772, 737, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 3345, 0, 0, 84, 0, 2834, 0, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 3449, 1821, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6507, 6287, 6130, 5666, 5662, 6463, 6287] avg=6144.0000 max=6507 max_rank=1 min=5662 min_rank=5 imbalance_ratio=1.059082 max_load_reduction=42645 max_load_reduction_pct=0.867615 moved_to_echo=24078 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 201, 0, 0, 0, 0, 4791, 154, 0, 0, 0, 0, 0, 0, 382, 475, 1942, 0, 880, 341, 0, 89, 2559, 212, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 69, 0, 0, 5962, 124, 0, 0, 0, 0, 0, 0, 4, 4373, 0, 286, 241, 1492, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 2614, 0, 0, 0, 1928, 0, 1004, 387, 91, 0, 0, 0, 0, 0, 0, 3474, 594, 108, 0, 554, 1507, 1153, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 2392, 0, 0, 95, 0, 3048, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 2617, 1222, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5148, 6880, 6250, 6402, 6458, 7397, 5667, 4950] avg=6144.0000 max=7397 max_rank=5 min=4950 min_rank=7 imbalance_ratio=1.203939 max_load_reduction=41755 max_load_reduction_pct=0.849508 moved_to_echo=24472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6, 8, 0, 325, 3, 1, 4, 17, 5755, 135, 0, 0, 0, 0, 0, 0, 367, 224, 1362, 12, 913, 340, 7, 88, 2280, 282, 0, 0, 0, 0, 0, 0, 57, 5, 1, 0, 24, 63, 0, 54, 5677, 143, 0, 0, 0, 0, 0, 0, 5, 4037, 2, 281, 249, 1372, 26, 51, 0, 0, 0, 0, 0, 0, 0, 0, 898, 2163, 0, 6, 1, 1629, 8, 1050, 520, 80, 0, 0, 0, 0, 0, 0, 2296, 1272, 140, 14, 589, 687, 842, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 214, 21, 3186, 11, 19, 63, 4, 2717, 0, 0, 0, 0, 0, 0, 0, 898, 19, 0, 0, 3559, 1892, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6254, 5875, 6024, 6023, 6355, 5850, 6236, 6535] avg=6144.0000 max=6535 max_rank=7 min=5850 min_rank=5 imbalance_ratio=1.063639 max_load_reduction=42617 max_load_reduction_pct=0.867045 moved_to_echo=24976 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[33, 25, 2, 353, 141, 13, 41, 212, 4896, 111, 0, 0, 0, 0, 0, 0, 412, 324, 2014, 113, 1064, 385, 77, 85, 1858, 197, 0, 0, 0, 0, 0, 0, 42, 43, 0, 9, 121, 110, 0, 116, 5033, 121, 0, 0, 0, 0, 0, 0, 70, 4204, 10, 372, 294, 1389, 84, 160, 0, 0, 0, 0, 0, 0, 0, 0, 594, 1898, 1, 63, 0, 1941, 60, 1116, 364, 84, 0, 0, 0, 0, 0, 0, 3480, 1492, 120, 178, 521, 862, 1461, 56, 0, 0, 0, 0, 0, 0, 0, 0, 2, 190, 194, 2201, 32, 150, 63, 33, 2211, 0, 0, 0, 0, 0, 0, 0, 1101, 91, 0, 0, 2491, 1415, 7, 146, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5827, 6529, 5595, 6583, 6121, 8170, 5076, 5251] avg=6144.0000 max=8170 max_rank=5 min=5076 min_rank=6 imbalance_ratio=1.329753 max_load_reduction=40982 max_load_reduction_pct=0.833781 moved_to_echo=24618 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 301, 0, 0, 0, 0, 6051, 133, 0, 0, 0, 0, 0, 0, 358, 345, 1619, 0, 958, 302, 0, 61, 2234, 302, 0, 0, 0, 0, 0, 0, 88, 0, 3, 0, 0, 62, 0, 0, 6061, 150, 0, 0, 0, 0, 0, 0, 1, 4107, 0, 250, 231, 1308, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 1770, 0, 0, 0, 1783, 0, 1015, 556, 82, 0, 0, 0, 0, 0, 0, 1983, 848, 148, 0, 629, 647, 681, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 3613, 0, 0, 88, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 891, 0, 0, 0, 3806, 1917, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6485, 6179, 6364, 5898, 6027, 4939, 6523, 6737] avg=6144.0000 max=6737 max_rank=7 min=4939 min_rank=5 imbalance_ratio=1.096517 max_load_reduction=42415 max_load_reduction_pct=0.862935 moved_to_echo=24226 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 201, 0, 0, 0, 0, 4791, 154, 0, 0, 0, 0, 0, 0, 382, 475, 1942, 0, 880, 341, 0, 89, 2559, 212, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 69, 0, 0, 5962, 124, 0, 0, 0, 0, 0, 0, 4, 4373, 0, 286, 241, 1492, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 2614, 0, 0, 0, 1928, 0, 1004, 387, 91, 0, 0, 0, 0, 0, 0, 3474, 594, 108, 0, 554, 1507, 1153, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 2392, 0, 0, 95, 0, 3048, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 2617, 1222, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5148, 6880, 6250, 6402, 6458, 7397, 5667, 4950] avg=6144.0000 max=7397 max_rank=5 min=4950 min_rank=7 imbalance_ratio=1.203939 max_load_reduction=41755 max_load_reduction_pct=0.849508 moved_to_echo=24472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 2944, 12672, 128, 7552, 2688, 128, 768, 17664, 2176, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[7040, 15104, 128, 128, 128, 14336, 128, 8320, 3840, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[20096, 8960, 1152, 256, 4608, 6912, 7424, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7168, 128, 0, 0, 26112, 14720, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1664, 256, 25728, 128, 256, 640, 128, 20992, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7168, 128, 0, 0, 26112, 14720, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[20096, 8960, 1152, 256, 4608, 6912, 7424, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 33920, 128, 2432, 2048, 10624, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 128, 128, 128, 256, 640, 0, 256, 46592, 1152, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 2560, 256, 128, 128, 256, 45184, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 128, 128, 128, 256, 640, 0, 256, 46592, 1152, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 52, 304, 304, 154, 4758, 0, 435, 0, 0, 17, 0, 85, 0, 0, 0, 1358, 0, 0, 0, 0, 2206, 0, 1347, 0, 105, 11, 499, 0, 0, 0, 54, 6259, 4, 82, 5289, 0, 0, 142, 1271, 253, 0, 0, 1410, 62, 0, 0, 7, 390, 0, 3553, 80, 4630, 52, 5989, 0, 44, 1585, 3382, 682, 1681, 429, 0, 187] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[2, 63, 305, 319, 135, 4576, 39, 442, 1, 12, 16, 2, 98, 26, 0, 12, 975, 0, 1, 0, 1, 1989, 5, 1581, 10, 103, 78, 634, 1, 5, 0, 60, 6219, 36, 91, 4996, 0, 20, 151, 958, 219, 24, 42, 1399, 63, 9, 28, 15, 360, 9, 3788, 81, 4591, 51, 5964, 7, 34, 1513, 3897, 735, 1431, 752, 5, 173] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 61, 347, 321, 47, 4370, 0, 433, 0, 0, 7, 0, 90, 0, 0, 0, 1390, 0, 0, 0, 0, 2738, 0, 1791, 0, 1, 57, 1106, 0, 0, 0, 61, 5978, 1, 90, 5227, 0, 0, 158, 678, 222, 0, 0, 2254, 69, 0, 0, 0, 402, 0, 3150, 88, 5215, 53, 5854, 0, 1, 972, 3266, 701, 1198, 541, 0, 214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 63, 291, 313, 49, 4364, 0, 431, 0, 0, 9, 0, 98, 0, 0, 0, 1512, 0, 0, 0, 0, 2506, 0, 1603, 0, 0, 47, 945, 0, 0, 0, 63, 6158, 0, 98, 5469, 0, 0, 179, 835, 248, 0, 0, 1940, 82, 0, 0, 0, 377, 0, 3335, 97, 5332, 61, 5880, 0, 16, 1427, 2897, 708, 1209, 302, 0, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 54, 330, 304, 151, 4716, 0, 417, 0, 0, 8, 0, 63, 0, 0, 0, 1393, 0, 0, 0, 0, 2423, 0, 1512, 0, 20, 21, 590, 0, 0, 0, 54, 6224, 1, 63, 5224, 0, 0, 123, 1114, 191, 0, 0, 1463, 62, 0, 0, 0, 398, 0, 3323, 61, 5221, 54, 5920, 0, 53, 1513, 3284, 631, 1452, 588, 0, 133] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 52, 304, 304, 154, 4758, 0, 435, 0, 0, 17, 0, 85, 0, 0, 0, 1358, 0, 0, 0, 0, 2206, 0, 1347, 0, 105, 11, 499, 0, 0, 0, 54, 6259, 4, 82, 5289, 0, 0, 142, 1271, 253, 0, 0, 1410, 62, 0, 0, 7, 390, 0, 3553, 80, 4630, 52, 5989, 0, 44, 1585, 3382, 682, 1681, 429, 0, 187] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 51, 325, 314, 147, 4584, 0, 450, 0, 0, 10, 0, 78, 0, 0, 0, 1392, 0, 0, 0, 0, 2411, 0, 1513, 0, 0, 22, 575, 0, 0, 0, 52, 6113, 2, 78, 5562, 0, 0, 148, 1055, 214, 0, 0, 1498, 70, 0, 0, 0, 399, 0, 2997, 78, 5329, 51, 5930, 0, 37, 1374, 3443, 696, 1425, 553, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[2, 63, 305, 319, 135, 4576, 39, 442, 1, 12, 16, 2, 98, 26, 0, 12, 975, 0, 1, 0, 1, 1989, 5, 1581, 10, 103, 78, 634, 1, 5, 0, 60, 6219, 36, 91, 4996, 0, 20, 151, 958, 219, 24, 42, 1399, 63, 9, 28, 15, 360, 9, 3788, 81, 4591, 51, 5964, 7, 34, 1513, 3897, 735, 1431, 752, 5, 173] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[39, 70, 330, 381, 199, 3374, 173, 437, 8, 40, 27, 53, 179, 233, 2, 39, 1102, 0, 4, 0, 29, 2018, 88, 1992, 40, 101, 363, 823, 33, 18, 13, 96, 5823, 99, 78, 4273, 1, 158, 159, 648, 229, 159, 326, 1411, 87, 22, 249, 86, 376, 4, 3534, 69, 4757, 49, 5721, 49, 32, 1587, 3534, 843, 1279, 982, 56, 168] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 61, 347, 321, 47, 4370, 0, 433, 0, 0, 7, 0, 90, 0, 0, 0, 1390, 0, 0, 0, 0, 2738, 0, 1791, 0, 1, 57, 1106, 0, 0, 0, 61, 5978, 1, 90, 5227, 0, 0, 158, 678, 222, 0, 0, 2254, 69, 0, 0, 0, 402, 0, 3150, 88, 5215, 53, 5854, 0, 1, 972, 3266, 701, 1198, 541, 0, 214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 62, 314, 329, 218, 5272, 0, 457, 0, 0, 6, 0, 150, 0, 0, 0, 1133, 0, 0, 0, 0, 2003, 0, 1565, 0, 100, 2, 401, 0, 0, 0, 62, 6141, 0, 89, 4837, 0, 0, 160, 1233, 309, 0, 0, 1224, 72, 0, 0, 58, 294, 0, 3869, 89, 4173, 62, 6002, 0, 26, 1955, 3448, 710, 1833, 397, 0, 97] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[39, 70, 330, 381, 199, 3374, 173, 437, 8, 40, 27, 53, 179, 233, 2, 39, 1102, 0, 4, 0, 29, 2018, 88, 1992, 40, 101, 363, 823, 33, 18, 13, 96, 5823, 99, 78, 4273, 1, 158, 159, 648, 229, 159, 326, 1411, 87, 22, 249, 86, 376, 4, 3534, 69, 4757, 49, 5721, 49, 32, 1587, 3534, 843, 1279, 982, 56, 168] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 52, 304, 304, 154, 4758, 0, 435, 223, 118, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 85, 0, 0, 0, 6114, 0, 0, 0, 0, 0, 0, 0, 1358, 0, 0, 0, 0, 2206, 0, 1347, 943, 0, 0, 0, 0, 0, 0, 0, 0, 105, 11, 499, 0, 0, 0, 54, 5169, 145, 0, 0, 0, 0, 0, 0, 0, 4, 82, 5066, 0, 0, 142, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 1410, 62, 0, 0, 7, 2570, 820, 600, 0, 0, 0, 0, 0, 390, 0, 3553, 80, 2060, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1585, 1721, 682, 1681, 429, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6348, 6216, 5854, 5983, 6565, 5722, 6135, 6329] avg=6144.0000 max=6565 max_rank=4 min=5722 min_rank=5 imbalance_ratio=1.068522 max_load_reduction=42587 max_load_reduction_pct=0.866435 moved_to_echo=24751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 63, 305, 319, 135, 4576, 39, 442, 211, 136, 0, 0, 0, 0, 0, 0, 1, 12, 16, 2, 98, 26, 0, 12, 6075, 0, 0, 0, 0, 0, 0, 0, 975, 0, 1, 0, 1, 1989, 5, 1581, 1086, 0, 0, 0, 0, 0, 0, 0, 10, 103, 78, 634, 1, 5, 0, 60, 5148, 144, 0, 0, 0, 0, 0, 0, 0, 36, 91, 4785, 0, 20, 151, 958, 0, 0, 0, 0, 0, 0, 0, 0, 219, 24, 42, 1399, 63, 9, 28, 15, 2548, 816, 692, 0, 0, 0, 0, 0, 360, 9, 3788, 81, 2043, 51, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1513, 1983, 735, 1431, 752, 5, 173, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6228, 6242, 5638, 6183, 6041, 5855, 6339, 6626] avg=6144.0000 max=6626 max_rank=7 min=5638 min_rank=2 imbalance_ratio=1.078451 max_load_reduction=42526 max_load_reduction_pct=0.865194 moved_to_echo=24861 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 63, 291, 313, 49, 4364, 0, 431, 231, 101, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 98, 0, 0, 0, 6016, 0, 0, 0, 0, 0, 0, 0, 1512, 0, 0, 0, 0, 2506, 0, 1603, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 945, 0, 0, 0, 63, 5076, 142, 0, 0, 0, 0, 0, 0, 0, 0, 98, 5238, 0, 0, 179, 835, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 1940, 82, 0, 0, 0, 2959, 804, 514, 0, 0, 0, 0, 0, 377, 0, 3335, 97, 2373, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1427, 1475, 708, 1209, 302, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5843, 6123, 6428, 6273, 6350, 6547, 6243, 5345] avg=6144.0000 max=6547 max_rank=5 min=5345 min_rank=7 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=24485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 54, 330, 304, 151, 4716, 0, 417, 220, 114, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 63, 0, 0, 0, 6080, 0, 0, 0, 0, 0, 0, 0, 1393, 0, 0, 0, 0, 2423, 0, 1512, 915, 0, 0, 0, 0, 0, 0, 0, 0, 20, 21, 590, 0, 0, 0, 54, 5110, 144, 0, 0, 0, 0, 0, 0, 0, 1, 63, 5004, 0, 0, 123, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 1463, 62, 0, 0, 0, 2898, 810, 583, 0, 0, 0, 0, 0, 398, 0, 3323, 61, 2323, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1513, 1672, 631, 1452, 588, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6306, 6151, 6243, 5939, 6305, 6007, 6159, 6042] avg=6144.0000 max=6306 max_rank=0 min=5939 min_rank=3 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 314, 329, 218, 5272, 0, 457, 204, 120, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 150, 0, 0, 0, 5998, 0, 0, 0, 0, 0, 0, 0, 1133, 0, 0, 0, 0, 2003, 0, 1565, 961, 0, 0, 0, 0, 0, 0, 0, 0, 100, 2, 401, 0, 0, 0, 62, 5180, 143, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4633, 0, 0, 160, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 1224, 72, 0, 0, 58, 2316, 822, 612, 0, 0, 0, 0, 0, 294, 0, 3869, 89, 1857, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1955, 1755, 710, 1833, 397, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6976, 6154, 5662, 5888, 6115, 5413, 6171, 6773] avg=6144.0000 max=6976 max_rank=0 min=5413 min_rank=5 imbalance_ratio=1.135417 max_load_reduction=42176 max_load_reduction_pct=0.858073 moved_to_echo=24472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 325, 314, 147, 4584, 0, 450, 235, 120, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 78, 0, 0, 0, 5971, 0, 0, 0, 0, 0, 0, 0, 1392, 0, 0, 0, 0, 2411, 0, 1513, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 575, 0, 0, 0, 52, 5118, 142, 0, 0, 0, 0, 0, 0, 0, 2, 78, 5327, 0, 0, 148, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 1498, 70, 0, 0, 0, 2958, 812, 611, 0, 0, 0, 0, 0, 399, 0, 2997, 78, 2371, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1374, 1752, 696, 1425, 553, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6226, 6059, 6276, 5909, 6610, 6163, 5896, 6013] avg=6144.0000 max=6610 max_rank=4 min=5896 min_rank=6 imbalance_ratio=1.075846 max_load_reduction=42542 max_load_reduction_pct=0.865519 moved_to_echo=24682 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 61, 347, 321, 47, 4370, 0, 433, 224, 116, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 90, 0, 0, 0, 5840, 0, 0, 0, 0, 0, 0, 0, 1390, 0, 0, 0, 0, 2738, 0, 1791, 914, 0, 0, 0, 0, 0, 0, 0, 0, 1, 57, 1106, 0, 0, 0, 61, 5054, 138, 0, 0, 0, 0, 0, 0, 0, 1, 90, 5003, 0, 0, 158, 678, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 2254, 69, 0, 0, 0, 2897, 800, 585, 0, 0, 0, 0, 0, 402, 0, 3150, 88, 2318, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 972, 1651, 701, 1198, 541, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5919, 5937, 6833, 6417, 5930, 6827, 6011, 5278] avg=6144.0000 max=6833 max_rank=2 min=5278 min_rank=7 imbalance_ratio=1.112142 max_load_reduction=42319 max_load_reduction_pct=0.860982 moved_to_echo=24046 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 54, 330, 304, 151, 4716, 0, 417, 220, 114, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 63, 0, 0, 0, 6080, 0, 0, 0, 0, 0, 0, 0, 1393, 0, 0, 0, 0, 2423, 0, 1512, 915, 0, 0, 0, 0, 0, 0, 0, 0, 20, 21, 590, 0, 0, 0, 54, 5110, 144, 0, 0, 0, 0, 0, 0, 0, 1, 63, 5004, 0, 0, 123, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 1463, 62, 0, 0, 0, 2898, 810, 583, 0, 0, 0, 0, 0, 398, 0, 3323, 61, 2323, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1513, 1672, 631, 1452, 588, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6306, 6151, 6243, 5939, 6305, 6007, 6159, 6042] avg=6144.0000 max=6306 max_rank=0 min=5939 min_rank=3 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[39, 70, 330, 381, 199, 3374, 173, 437, 180, 123, 0, 0, 0, 0, 0, 0, 8, 40, 27, 53, 179, 233, 2, 39, 5689, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 4, 0, 29, 2018, 88, 1992, 985, 0, 0, 0, 0, 0, 0, 0, 40, 101, 363, 823, 33, 18, 13, 96, 4939, 134, 0, 0, 0, 0, 0, 0, 0, 99, 78, 4093, 1, 158, 159, 648, 0, 0, 0, 0, 0, 0, 0, 0, 229, 159, 326, 1411, 87, 22, 249, 86, 2640, 782, 627, 0, 0, 0, 0, 0, 376, 4, 3534, 69, 2117, 49, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1587, 1799, 843, 1279, 982, 56, 168, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5306, 6270, 6218, 6560, 5236, 6618, 6198, 6746] avg=6144.0000 max=6746 max_rank=7 min=5236 min_rank=4 imbalance_ratio=1.097982 max_load_reduction=42406 max_load_reduction_pct=0.862752 moved_to_echo=24798 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 61, 347, 321, 47, 4370, 0, 433, 224, 116, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 90, 0, 0, 0, 5840, 0, 0, 0, 0, 0, 0, 0, 1390, 0, 0, 0, 0, 2738, 0, 1791, 914, 0, 0, 0, 0, 0, 0, 0, 0, 1, 57, 1106, 0, 0, 0, 61, 5054, 138, 0, 0, 0, 0, 0, 0, 0, 1, 90, 5003, 0, 0, 158, 678, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 2254, 69, 0, 0, 0, 2897, 800, 585, 0, 0, 0, 0, 0, 402, 0, 3150, 88, 2318, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 972, 1651, 701, 1198, 541, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5919, 5937, 6833, 6417, 5930, 6827, 6011, 5278] avg=6144.0000 max=6833 max_rank=2 min=5278 min_rank=7 imbalance_ratio=1.112142 max_load_reduction=42319 max_load_reduction_pct=0.860982 moved_to_echo=24046 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 12032, 13824, 5760, 11520, 4608, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 128, 27648, 768, 17536, 512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1920, 256, 384, 12672, 640, 128, 384, 256, 21888, 6528, 4864, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10368, 0, 128, 0, 128, 18304, 128, 12928, 7680, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1920, 256, 384, 12672, 640, 128, 384, 256, 21888, 6528, 4864, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 256, 768, 39168, 128, 256, 1280, 7808, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 512, 2560, 2688, 1152, 36096, 256, 3584, 1792, 1024, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 512, 640, 5632, 128, 128, 128, 512, 40832, 1152, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 896, 384, 128, 128, 47872, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[45, 0, 51, 16, 53, 5, 1, 0, 24, 2476, 0, 225, 96, 286, 403, 1501, 0, 7147, 19, 497, 1, 0, 74, 3, 7006, 18, 142, 250, 7, 588, 93, 6301, 6994, 4561, 23, 79, 14, 7, 0, 3, 137, 54, 1660, 31, 3, 1466, 257, 48, 399, 3, 1, 2704, 4, 0, 8, 12, 14, 3236, 0, 3, 7, 32, 0, 64] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 52, 0, 82, 0, 0, 0, 0, 1880, 0, 217, 80, 472, 409, 1871, 0, 7218, 0, 492, 0, 0, 62, 0, 7112, 0, 127, 242, 0, 592, 80, 6263, 7144, 4802, 0, 21, 0, 0, 0, 0, 83, 62, 2115, 0, 0, 1381, 242, 0, 402, 0, 0, 2271, 0, 0, 0, 0, 0, 3314, 0, 0, 0, 0, 0, 62] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[112, 0, 61, 1, 62, 0, 0, 0, 0, 1932, 0, 210, 88, 78, 377, 1600, 0, 7266, 0, 464, 0, 0, 95, 0, 7092, 0, 183, 252, 0, 557, 89, 6685, 6958, 5265, 0, 14, 0, 0, 0, 0, 90, 94, 976, 0, 0, 1673, 252, 0, 390, 0, 0, 3293, 0, 0, 0, 0, 0, 2848, 0, 0, 0, 0, 0, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 62, 0, 77, 0, 0, 0, 0, 1980, 0, 248, 88, 855, 325, 1532, 0, 7266, 0, 390, 0, 0, 57, 0, 7114, 0, 116, 257, 0, 583, 88, 5973, 7508, 4446, 0, 55, 0, 0, 0, 0, 88, 57, 2493, 0, 0, 1388, 257, 0, 415, 0, 0, 2125, 0, 0, 0, 0, 0, 3249, 0, 0, 0, 0, 0, 57] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 51, 0, 75, 0, 0, 0, 0, 2474, 0, 208, 78, 203, 419, 1631, 0, 7285, 0, 495, 0, 0, 84, 0, 7151, 0, 100, 244, 0, 553, 78, 6547, 7087, 4417, 0, 4, 0, 0, 0, 0, 78, 82, 1329, 0, 0, 1264, 244, 0, 416, 0, 0, 3015, 0, 0, 0, 0, 0, 3450, 0, 0, 0, 0, 0, 84] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[45, 0, 63, 0, 67, 0, 0, 0, 0, 1955, 0, 213, 98, 107, 380, 1545, 0, 7275, 0, 473, 0, 0, 60, 0, 7167, 0, 124, 231, 0, 704, 97, 6524, 7108, 4923, 0, 5, 0, 0, 0, 0, 98, 59, 1028, 0, 0, 1517, 231, 0, 396, 0, 0, 3418, 0, 0, 0, 0, 0, 3181, 0, 0, 0, 0, 0, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 54, 0, 97, 0, 0, 0, 0, 2279, 0, 161, 63, 313, 445, 1586, 0, 7250, 0, 508, 0, 0, 88, 0, 7206, 0, 126, 242, 0, 593, 61, 6453, 7025, 4975, 0, 1, 0, 0, 0, 0, 63, 88, 1601, 0, 0, 1199, 242, 0, 394, 0, 0, 2747, 0, 0, 0, 0, 0, 3198, 0, 0, 0, 0, 0, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 51, 0, 75, 0, 0, 0, 0, 2474, 0, 208, 78, 203, 419, 1631, 0, 7285, 0, 495, 0, 0, 84, 0, 7151, 0, 100, 244, 0, 553, 78, 6547, 7087, 4417, 0, 4, 0, 0, 0, 0, 78, 82, 1329, 0, 0, 1264, 244, 0, 416, 0, 0, 3015, 0, 0, 0, 0, 0, 3450, 0, 0, 0, 0, 0, 84] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 52, 0, 82, 0, 0, 0, 0, 1880, 0, 217, 80, 472, 409, 1871, 0, 7218, 0, 492, 0, 0, 62, 0, 7112, 0, 127, 242, 0, 592, 80, 6263, 7144, 4802, 0, 21, 0, 0, 0, 0, 83, 62, 2115, 0, 0, 1381, 242, 0, 402, 0, 0, 2271, 0, 0, 0, 0, 0, 3314, 0, 0, 0, 0, 0, 62] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[130, 12, 60, 34, 70, 49, 8, 0, 43, 2200, 0, 159, 154, 677, 319, 1002, 9, 6800, 133, 424, 5, 0, 160, 90, 6232, 191, 317, 269, 66, 1407, 86, 6357, 6561, 3418, 119, 378, 88, 81, 0, 61, 170, 59, 1320, 203, 16, 1830, 314, 279, 414, 26, 4, 3115, 20, 5, 29, 33, 124, 2497, 0, 25, 152, 231, 22, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 52, 0, 82, 0, 0, 0, 6041, 0, 0, 0, 0, 0, 0, 0, 0, 1880, 0, 217, 80, 472, 409, 1871, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 5582, 0, 492, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 242, 0, 592, 80, 4829, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 4802, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 62, 2115, 0, 0, 1381, 242, 0, 1636, 934, 63, 0, 0, 0, 0, 0, 402, 0, 0, 2271, 0, 0, 0, 0, 1856, 1071, 0, 0, 0, 0, 0, 0, 0, 3314, 0, 0, 0, 0, 0, 62, 2909, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6300, 6136, 5870, 6268, 6516, 5600, 6285] avg=6144.0000 max=6516 max_rank=5 min=5600 min_rank=6 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24669 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[45, 0, 51, 16, 53, 5, 1, 0, 5951, 0, 0, 0, 0, 0, 0, 0, 24, 2476, 0, 225, 96, 286, 403, 1501, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 5527, 19, 497, 1, 0, 74, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 142, 250, 7, 588, 93, 4859, 0, 0, 0, 0, 0, 0, 0, 0, 1415, 4561, 23, 79, 14, 7, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 137, 54, 1660, 31, 3, 1466, 257, 48, 1620, 914, 63, 0, 0, 0, 0, 0, 399, 3, 1, 2704, 4, 0, 8, 12, 1817, 1055, 0, 0, 0, 0, 0, 0, 14, 3236, 0, 3, 7, 32, 0, 64, 2848, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6390, 6121, 5957, 6102, 6253, 6003, 6204] avg=6144.0000 max=6390 max_rank=1 min=5957 min_rank=3 imbalance_ratio=1.040039 max_load_reduction=42762 max_load_reduction_pct=0.869995 moved_to_echo=24562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 54, 0, 97, 0, 0, 0, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 2279, 0, 161, 63, 313, 445, 1586, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 5607, 0, 508, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 242, 0, 593, 61, 4976, 0, 0, 0, 0, 0, 0, 0, 0, 1421, 4975, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 88, 1601, 0, 0, 1199, 242, 0, 1643, 918, 65, 0, 0, 0, 0, 0, 394, 0, 0, 2747, 0, 0, 0, 0, 1825, 1084, 0, 0, 0, 0, 0, 0, 0, 3198, 0, 0, 0, 0, 0, 88, 2861, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6279, 6259, 6203, 5998, 6397, 5819, 6050, 6147] avg=6144.0000 max=6397 max_rank=4 min=5819 min_rank=5 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 54, 0, 97, 0, 0, 0, 0, 2279, 0, 161, 63, 313, 445, 1586, 0, 7250, 0, 508, 0, 0, 88, 0, 7206, 0, 126, 242, 0, 593, 61, 6453, 7025, 4975, 0, 1, 0, 0, 0, 0, 63, 88, 1601, 0, 0, 1199, 242, 0, 394, 0, 0, 2747, 0, 0, 0, 0, 0, 3198, 0, 0, 0, 0, 0, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[45, 0, 51, 16, 53, 5, 1, 0, 24, 2476, 0, 225, 96, 286, 403, 1501, 0, 7147, 19, 497, 1, 0, 74, 3, 7006, 18, 142, 250, 7, 588, 93, 6301, 6994, 4561, 23, 79, 14, 7, 0, 3, 137, 54, 1660, 31, 3, 1466, 257, 48, 399, 3, 1, 2704, 4, 0, 8, 12, 14, 3236, 0, 3, 7, 32, 0, 64] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[112, 0, 61, 1, 62, 0, 0, 0, 0, 1932, 0, 210, 88, 78, 377, 1600, 0, 7266, 0, 464, 0, 0, 95, 0, 7092, 0, 183, 252, 0, 557, 89, 6685, 6958, 5265, 0, 14, 0, 0, 0, 0, 90, 94, 976, 0, 0, 1673, 252, 0, 390, 0, 0, 3293, 0, 0, 0, 0, 0, 2848, 0, 0, 0, 0, 0, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 62, 0, 77, 0, 0, 0, 6043, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 0, 248, 88, 855, 325, 1532, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 5619, 0, 390, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 257, 0, 583, 88, 4606, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 4446, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 57, 2493, 0, 0, 1388, 257, 0, 1647, 982, 60, 0, 0, 0, 0, 0, 415, 0, 0, 2125, 0, 0, 0, 0, 1951, 1071, 0, 0, 0, 0, 0, 0, 0, 3249, 0, 0, 0, 0, 0, 57, 3057, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6185, 6335, 6066, 5650, 6019, 6972, 5562, 6363] avg=6144.0000 max=6972 max_rank=5 min=5562 min_rank=6 imbalance_ratio=1.134766 max_load_reduction=42180 max_load_reduction_pct=0.858154 moved_to_echo=24916 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[45, 0, 63, 0, 67, 0, 0, 0, 6089, 0, 0, 0, 0, 0, 0, 0, 0, 1955, 0, 213, 98, 107, 380, 1545, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 5626, 0, 473, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 231, 0, 704, 97, 5030, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 4923, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 59, 1028, 0, 0, 1517, 231, 0, 1649, 929, 66, 0, 0, 0, 0, 0, 396, 0, 0, 3418, 0, 0, 0, 0, 1847, 1078, 0, 0, 0, 0, 0, 0, 0, 3181, 0, 0, 0, 0, 0, 60, 2894, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 5726, 6159, 6186, 6366, 5577, 6739, 6135] avg=6144.0000 max=6739 max_rank=6 min=5577 min_rank=5 imbalance_ratio=1.096842 max_load_reduction=42413 max_load_reduction_pct=0.862895 moved_to_echo=24817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[112, 0, 61, 1, 62, 0, 0, 0, 6025, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 210, 88, 78, 377, 1600, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 5615, 0, 464, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 252, 0, 557, 89, 5149, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 5265, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 94, 976, 0, 0, 1673, 252, 0, 1651, 914, 70, 0, 0, 0, 0, 0, 390, 0, 0, 3293, 0, 0, 0, 0, 1811, 1067, 0, 0, 0, 0, 0, 0, 0, 2848, 0, 0, 0, 0, 0, 95, 2837, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 5751, 6174, 6230, 6675, 5720, 6561, 5780] avg=6144.0000 max=6675 max_rank=4 min=5720 min_rank=5 imbalance_ratio=1.086426 max_load_reduction=42477 max_load_reduction_pct=0.864197 moved_to_echo=24736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 51, 0, 75, 0, 0, 0, 6074, 0, 0, 0, 0, 0, 0, 0, 0, 2474, 0, 208, 78, 203, 419, 1631, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 5634, 0, 495, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 244, 0, 553, 78, 5048, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 4417, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 82, 1329, 0, 0, 1264, 244, 0, 1651, 927, 66, 0, 0, 0, 0, 0, 416, 0, 0, 3015, 0, 0, 0, 0, 1841, 1077, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 0, 0, 0, 0, 84, 2886, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6206, 6446, 6213, 6023, 5854, 5641, 6349, 6420] avg=6144.0000 max=6446 max_rank=1 min=5641 min_rank=5 imbalance_ratio=1.049154 max_load_reduction=42706 max_load_reduction_pct=0.868856 moved_to_echo=24264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 52, 0, 82, 0, 0, 0, 6041, 0, 0, 0, 0, 0, 0, 0, 0, 1880, 0, 217, 80, 472, 409, 1871, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 5582, 0, 492, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 242, 0, 592, 80, 4829, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 4802, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 62, 2115, 0, 0, 1381, 242, 0, 1636, 934, 63, 0, 0, 0, 0, 0, 402, 0, 0, 2271, 0, 0, 0, 0, 1856, 1071, 0, 0, 0, 0, 0, 0, 0, 3314, 0, 0, 0, 0, 0, 62, 2909, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6300, 6136, 5870, 6268, 6516, 5600, 6285] avg=6144.0000 max=6516 max_rank=5 min=5600 min_rank=6 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24669 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[45, 0, 51, 16, 53, 5, 1, 0, 5951, 0, 0, 0, 0, 0, 0, 0, 24, 2476, 0, 225, 96, 286, 403, 1501, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 5527, 19, 497, 1, 0, 74, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 142, 250, 7, 588, 93, 4859, 0, 0, 0, 0, 0, 0, 0, 0, 1415, 4561, 23, 79, 14, 7, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 137, 54, 1660, 31, 3, 1466, 257, 48, 1620, 914, 63, 0, 0, 0, 0, 0, 399, 3, 1, 2704, 4, 0, 8, 12, 1817, 1055, 0, 0, 0, 0, 0, 0, 14, 3236, 0, 3, 7, 32, 0, 64, 2848, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6390, 6121, 5957, 6102, 6253, 6003, 6204] avg=6144.0000 max=6390 max_rank=1 min=5957 min_rank=3 imbalance_ratio=1.040039 max_load_reduction=42762 max_load_reduction_pct=0.869995 moved_to_echo=24562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[130, 12, 60, 34, 70, 49, 8, 0, 5295, 0, 0, 0, 0, 0, 0, 0, 43, 2200, 0, 159, 154, 677, 319, 1002, 1391, 0, 0, 0, 0, 0, 0, 0, 9, 5259, 133, 424, 5, 0, 160, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 317, 269, 66, 1407, 86, 4902, 0, 0, 0, 0, 0, 0, 0, 0, 1326, 3418, 119, 378, 88, 81, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 170, 59, 1320, 203, 16, 1830, 314, 279, 1541, 858, 64, 0, 0, 0, 0, 0, 414, 26, 4, 3115, 20, 5, 29, 33, 1705, 937, 0, 0, 0, 0, 0, 0, 124, 2497, 0, 25, 152, 231, 22, 95, 2672, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5658, 5945, 6080, 7238, 5471, 6654, 6288, 5818] avg=6144.0000 max=7238 max_rank=3 min=5471 min_rank=4 imbalance_ratio=1.178060 max_load_reduction=41914 max_load_reduction_pct=0.852743 moved_to_echo=24231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 54, 0, 97, 0, 0, 0, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 2279, 0, 161, 63, 313, 445, 1586, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 5607, 0, 508, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 242, 0, 593, 61, 4976, 0, 0, 0, 0, 0, 0, 0, 0, 1421, 4975, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 88, 1601, 0, 0, 1199, 242, 0, 1643, 918, 65, 0, 0, 0, 0, 0, 394, 0, 0, 2747, 0, 0, 0, 0, 1825, 1084, 0, 0, 0, 0, 0, 0, 0, 3198, 0, 0, 0, 0, 0, 88, 2861, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6279, 6259, 6203, 5998, 6397, 5819, 6050, 6147] avg=6144.0000 max=6397 max_rank=4 min=5819 min_rank=5 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[112, 0, 61, 1, 62, 0, 0, 0, 6025, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 210, 88, 78, 377, 1600, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 5615, 0, 464, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 252, 0, 557, 89, 5149, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 5265, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 94, 976, 0, 0, 1673, 252, 0, 1651, 914, 70, 0, 0, 0, 0, 0, 390, 0, 0, 3293, 0, 0, 0, 0, 1811, 1067, 0, 0, 0, 0, 0, 0, 0, 2848, 0, 0, 0, 0, 0, 95, 2837, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 5751, 6174, 6230, 6675, 5720, 6561, 5780] avg=6144.0000 max=6675 max_rank=4 min=5720 min_rank=5 imbalance_ratio=1.086426 max_load_reduction=42477 max_load_reduction_pct=0.864197 moved_to_echo=24736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 640, 12544, 256, 128, 11776, 2048, 384, 13056, 7424, 640, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 17280, 0, 1664, 768, 3072, 3200, 12288, 11264, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 25088, 0, 128, 256, 384, 128, 640, 23040, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11392, 36864, 256, 640, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[3328, 128, 128, 22784, 128, 128, 128, 128, 14720, 8448, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11392, 36864, 256, 640, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 256, 1280, 2048, 128, 5632, 768, 39424, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 44544, 256, 3840, 128, 0, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 128, 512, 128, 640, 128, 128, 0, 47744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 44544, 256, 3840, 128, 0, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 108, 2443, 0, 0, 1117, 0, 0, 3464, 1827, 0, 0, 0, 701, 0, 63, 304, 0, 1338, 4, 55, 0, 0, 6203, 39, 0, 83, 0, 0, 0, 0, 71, 324, 0, 0, 2800, 187, 0, 80, 1, 0, 5282, 0, 0, 0, 117, 0, 564, 0, 80, 0, 6713, 0, 23, 87, 6925, 0, 0, 0, 6654, 739, 756, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 104, 2515, 0, 0, 864, 0, 0, 4294, 1764, 0, 0, 0, 585, 0, 72, 329, 0, 1622, 0, 57, 0, 0, 6040, 71, 0, 88, 0, 0, 0, 0, 64, 345, 0, 0, 2656, 211, 0, 88, 0, 0, 5257, 0, 0, 0, 116, 0, 594, 0, 88, 0, 6132, 0, 0, 88, 7031, 0, 0, 0, 6691, 702, 684, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[2, 3, 134, 1673, 0, 9, 1218, 0, 0, 2773, 2533, 2, 28, 41, 666, 1, 63, 320, 25, 1418, 9, 52, 0, 6, 5857, 43, 0, 95, 7, 1, 0, 8, 156, 364, 27, 24, 2635, 233, 56, 88, 21, 21, 5613, 3, 5, 2, 215, 39, 523, 14, 88, 0, 6754, 31, 41, 93, 7085, 1, 9, 2, 6565, 702, 755, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 110, 1636, 0, 0, 1791, 0, 0, 2167, 2741, 0, 0, 0, 672, 0, 82, 313, 0, 1062, 0, 60, 0, 0, 5789, 0, 0, 97, 0, 0, 0, 0, 80, 330, 0, 0, 3057, 228, 0, 97, 0, 0, 5655, 0, 0, 0, 15, 0, 517, 0, 97, 0, 6998, 0, 82, 98, 7199, 0, 0, 0, 6640, 702, 837, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 104, 2515, 0, 0, 864, 0, 0, 4294, 1764, 0, 0, 0, 585, 0, 72, 329, 0, 1622, 0, 57, 0, 0, 6040, 71, 0, 88, 0, 0, 0, 0, 64, 345, 0, 0, 2656, 211, 0, 88, 0, 0, 5257, 0, 0, 0, 116, 0, 594, 0, 88, 0, 6132, 0, 0, 88, 7031, 0, 0, 0, 6691, 702, 684, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 89, 2170, 0, 0, 1192, 0, 0, 3211, 2025, 0, 0, 0, 715, 0, 62, 304, 0, 1223, 0, 88, 0, 0, 6130, 19, 0, 69, 0, 0, 0, 0, 96, 305, 0, 0, 2834, 189, 0, 61, 0, 0, 5584, 0, 0, 0, 105, 0, 529, 0, 61, 0, 6749, 0, 43, 62, 7144, 0, 0, 0, 6637, 713, 743, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[42, 19, 126, 1279, 44, 73, 1777, 0, 0, 1630, 3601, 12, 242, 335, 599, 19, 82, 375, 180, 1015, 61, 43, 4, 11, 5461, 39, 0, 128, 80, 12, 12, 35, 185, 338, 223, 103, 2038, 181, 282, 80, 55, 139, 5430, 0, 46, 57, 350, 323, 631, 183, 78, 1, 6893, 248, 103, 97, 6299, 3, 84, 15, 6013, 638, 700, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 110, 1636, 0, 0, 1791, 0, 0, 2167, 2741, 0, 0, 0, 672, 0, 82, 313, 0, 1062, 0, 60, 0, 0, 5789, 0, 0, 97, 0, 0, 0, 0, 80, 330, 0, 0, 3057, 228, 0, 97, 0, 0, 5655, 0, 0, 0, 15, 0, 517, 0, 97, 0, 6998, 0, 82, 98, 7199, 0, 0, 0, 6640, 702, 837, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 98, 2067, 0, 0, 1508, 0, 0, 2662, 1993, 0, 0, 0, 699, 0, 70, 316, 0, 1107, 1, 83, 0, 0, 5907, 9, 0, 92, 0, 0, 0, 0, 86, 322, 0, 0, 3139, 179, 0, 78, 2, 0, 5675, 0, 0, 1, 41, 0, 510, 0, 78, 0, 6862, 2, 61, 78, 7165, 0, 0, 0, 6660, 774, 827, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 108, 2443, 0, 0, 1117, 0, 0, 3464, 1827, 0, 0, 0, 701, 0, 63, 304, 0, 1338, 4, 55, 0, 0, 6203, 39, 0, 83, 0, 0, 0, 0, 71, 324, 0, 0, 2800, 187, 0, 80, 1, 0, 5282, 0, 0, 0, 117, 0, 564, 0, 80, 0, 6713, 0, 23, 87, 6925, 0, 0, 0, 6654, 739, 756, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 124, 1508, 0, 0, 2061, 0, 0, 1774, 3150, 0, 0, 0, 691, 0, 71, 321, 0, 882, 0, 95, 0, 0, 5586, 0, 0, 92, 0, 0, 0, 0, 148, 342, 0, 0, 2877, 255, 0, 89, 9, 0, 5801, 0, 0, 0, 36, 0, 467, 0, 88, 0, 6945, 21, 69, 87, 7320, 0, 0, 0, 6700, 720, 823, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 89, 2170, 0, 0, 1192, 0, 0, 3211, 2025, 0, 0, 0, 715, 0, 62, 304, 0, 1223, 0, 88, 0, 0, 6130, 19, 0, 69, 0, 0, 0, 0, 96, 305, 0, 0, 2834, 189, 0, 61, 0, 0, 5584, 0, 0, 0, 105, 0, 529, 0, 61, 0, 6749, 0, 43, 62, 7144, 0, 0, 0, 6637, 713, 743, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 108, 2443, 0, 0, 1117, 0, 2547, 73, 0, 0, 0, 0, 0, 0, 0, 3464, 1827, 0, 0, 0, 701, 0, 190, 0, 0, 0, 0, 0, 0, 0, 63, 304, 0, 1338, 4, 55, 0, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 6203, 39, 0, 83, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 71, 324, 0, 0, 2800, 187, 0, 80, 1867, 716, 0, 0, 0, 0, 0, 0, 1, 0, 5282, 0, 0, 0, 117, 0, 389, 0, 0, 0, 0, 0, 0, 0, 564, 0, 80, 0, 5287, 0, 23, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4714, 739, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6288, 6182, 6142, 6456, 6045, 5789, 6041, 6209] avg=6144.0000 max=6456 max_rank=3 min=5789 min_rank=5 imbalance_ratio=1.050781 max_load_reduction=42696 max_load_reduction_pct=0.868652 moved_to_echo=24084 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 134, 1673, 0, 9, 1218, 0, 2606, 72, 0, 0, 0, 0, 0, 0, 0, 2773, 2533, 2, 28, 41, 666, 1, 191, 0, 0, 0, 0, 0, 0, 0, 63, 320, 25, 1418, 9, 52, 0, 6, 4479, 0, 0, 0, 0, 0, 0, 0, 5857, 43, 0, 95, 7, 1, 0, 8, 132, 0, 0, 0, 0, 0, 0, 0, 156, 364, 27, 24, 2635, 233, 56, 88, 1842, 720, 0, 0, 0, 0, 0, 0, 21, 21, 5613, 3, 5, 2, 215, 39, 391, 0, 0, 0, 0, 0, 0, 0, 523, 14, 88, 0, 5320, 31, 41, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 2, 4651, 702, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5717, 6235, 6372, 6143, 6145, 6310, 6110, 6120] avg=6144.0000 max=6372 max_rank=2 min=5717 min_rank=0 imbalance_ratio=1.037109 max_load_reduction=42780 max_load_reduction_pct=0.870361 moved_to_echo=24685 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 104, 2515, 0, 0, 864, 0, 2586, 74, 0, 0, 0, 0, 0, 0, 0, 4294, 1764, 0, 0, 0, 585, 0, 174, 0, 0, 0, 0, 0, 0, 0, 72, 329, 0, 1622, 0, 57, 0, 0, 4445, 0, 0, 0, 0, 0, 0, 0, 6040, 71, 0, 88, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 64, 345, 0, 0, 2656, 211, 0, 88, 1878, 654, 0, 0, 0, 0, 0, 0, 0, 0, 5257, 0, 0, 0, 116, 0, 355, 0, 0, 0, 0, 0, 0, 0, 594, 0, 88, 0, 4829, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4739, 702, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6817, 6525, 6319, 5896, 5728, 5599, 6125] avg=6144.0000 max=6817 max_rank=1 min=5599 min_rank=6 imbalance_ratio=1.109538 max_load_reduction=42335 max_load_reduction_pct=0.861308 moved_to_echo=23348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 134, 1673, 0, 9, 1218, 0, 2606, 72, 0, 0, 0, 0, 0, 0, 0, 2773, 2533, 2, 28, 41, 666, 1, 191, 0, 0, 0, 0, 0, 0, 0, 63, 320, 25, 1418, 9, 52, 0, 6, 4479, 0, 0, 0, 0, 0, 0, 0, 5857, 43, 0, 95, 7, 1, 0, 8, 132, 0, 0, 0, 0, 0, 0, 0, 156, 364, 27, 24, 2635, 233, 56, 88, 1842, 720, 0, 0, 0, 0, 0, 0, 21, 21, 5613, 3, 5, 2, 215, 39, 391, 0, 0, 0, 0, 0, 0, 0, 523, 14, 88, 0, 5320, 31, 41, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 2, 4651, 702, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5717, 6235, 6372, 6143, 6145, 6310, 6110, 6120] avg=6144.0000 max=6372 max_rank=2 min=5717 min_rank=0 imbalance_ratio=1.037109 max_load_reduction=42780 max_load_reduction_pct=0.870361 moved_to_echo=24685 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 98, 2067, 0, 0, 1508, 0, 2635, 74, 0, 0, 0, 0, 0, 0, 0, 2662, 1993, 0, 0, 0, 699, 0, 194, 0, 0, 0, 0, 0, 0, 0, 70, 316, 0, 1107, 1, 83, 0, 0, 4530, 0, 0, 0, 0, 0, 0, 0, 5907, 9, 0, 92, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 86, 322, 0, 0, 3139, 179, 0, 78, 1869, 732, 0, 0, 0, 0, 0, 0, 2, 0, 5675, 0, 0, 1, 41, 0, 398, 0, 0, 0, 0, 0, 0, 0, 510, 0, 78, 0, 5404, 2, 61, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4717, 774, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6382, 5548, 6107, 6142, 6405, 6117, 6133, 6318] avg=6144.0000 max=6405 max_rank=4 min=5548 min_rank=1 imbalance_ratio=1.042480 max_load_reduction=42747 max_load_reduction_pct=0.869690 moved_to_echo=24973 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[42, 19, 126, 1279, 44, 73, 1777, 0, 2318, 66, 0, 0, 0, 0, 0, 0, 0, 1630, 3601, 12, 242, 335, 599, 19, 195, 0, 0, 0, 0, 0, 0, 0, 82, 375, 180, 1015, 61, 43, 4, 11, 3981, 0, 0, 0, 0, 0, 0, 0, 5461, 39, 0, 128, 80, 12, 12, 35, 135, 0, 0, 0, 0, 0, 0, 0, 185, 338, 223, 103, 2038, 181, 282, 80, 1687, 735, 0, 0, 0, 0, 0, 0, 55, 139, 5430, 0, 46, 57, 350, 323, 399, 0, 0, 0, 0, 0, 0, 0, 631, 183, 78, 1, 5429, 248, 103, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 84, 15, 4260, 638, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5744, 6633, 5752, 5902, 5852, 6799, 6770, 5700] avg=6144.0000 max=6799 max_rank=5 min=5700 min_rank=7 imbalance_ratio=1.106608 max_load_reduction=42353 max_load_reduction_pct=0.861674 moved_to_echo=25121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 110, 1636, 0, 0, 1791, 0, 2649, 73, 0, 0, 0, 0, 0, 0, 0, 2167, 2741, 0, 0, 0, 672, 0, 198, 0, 0, 0, 0, 0, 0, 0, 82, 313, 0, 1062, 0, 60, 0, 0, 4550, 0, 0, 0, 0, 0, 0, 0, 5789, 0, 0, 97, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 80, 330, 0, 0, 3057, 228, 0, 97, 1863, 746, 0, 0, 0, 0, 0, 0, 0, 0, 5655, 0, 0, 0, 15, 0, 406, 0, 0, 0, 0, 0, 0, 0, 517, 0, 97, 0, 5511, 0, 82, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4704, 702, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6259, 5778, 6067, 6023, 6401, 6076, 6305, 6243] avg=6144.0000 max=6401 max_rank=4 min=5778 min_rank=1 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=25025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 89, 2170, 0, 0, 1192, 0, 2629, 73, 0, 0, 0, 0, 0, 0, 0, 3211, 2025, 0, 0, 0, 715, 0, 191, 0, 0, 0, 0, 0, 0, 0, 62, 304, 0, 1223, 0, 88, 0, 0, 4515, 0, 0, 0, 0, 0, 0, 0, 6130, 19, 0, 69, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 96, 305, 0, 0, 2834, 189, 0, 61, 1862, 720, 0, 0, 0, 0, 0, 0, 0, 0, 5584, 0, 0, 0, 105, 0, 391, 0, 0, 0, 0, 0, 0, 0, 529, 0, 61, 0, 5315, 0, 43, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4702, 713, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6142, 6192, 6350, 6067, 6080, 6010, 6158] avg=6144.0000 max=6350 max_rank=3 min=6010 min_rank=6 imbalance_ratio=1.033529 max_load_reduction=42802 max_load_reduction_pct=0.870809 moved_to_echo=24315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 98, 2067, 0, 0, 1508, 0, 2635, 74, 0, 0, 0, 0, 0, 0, 0, 2662, 1993, 0, 0, 0, 699, 0, 194, 0, 0, 0, 0, 0, 0, 0, 70, 316, 0, 1107, 1, 83, 0, 0, 4530, 0, 0, 0, 0, 0, 0, 0, 5907, 9, 0, 92, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 86, 322, 0, 0, 3139, 179, 0, 78, 1869, 732, 0, 0, 0, 0, 0, 0, 2, 0, 5675, 0, 0, 1, 41, 0, 398, 0, 0, 0, 0, 0, 0, 0, 510, 0, 78, 0, 5404, 2, 61, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4717, 774, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6382, 5548, 6107, 6142, 6405, 6117, 6133, 6318] avg=6144.0000 max=6405 max_rank=4 min=5548 min_rank=1 imbalance_ratio=1.042480 max_load_reduction=42747 max_load_reduction_pct=0.869690 moved_to_echo=24973 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 124, 1508, 0, 0, 2061, 0, 2694, 79, 0, 0, 0, 0, 0, 0, 0, 1774, 3150, 0, 0, 0, 691, 0, 202, 0, 0, 0, 0, 0, 0, 0, 71, 321, 0, 882, 0, 95, 0, 0, 4626, 0, 0, 0, 0, 0, 0, 0, 5586, 0, 0, 92, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 148, 342, 0, 0, 2877, 255, 0, 89, 1885, 745, 0, 0, 0, 0, 0, 0, 9, 0, 5801, 0, 0, 0, 36, 0, 407, 0, 0, 0, 0, 0, 0, 0, 467, 0, 88, 0, 5452, 21, 69, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4736, 720, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6466, 5817, 5995, 5817, 6341, 6253, 6184, 6279] avg=6144.0000 max=6466 max_rank=0 min=5817 min_rank=1 imbalance_ratio=1.052409 max_load_reduction=42686 max_load_reduction_pct=0.868449 moved_to_echo=25057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[42, 19, 126, 1279, 44, 73, 1777, 0, 2318, 66, 0, 0, 0, 0, 0, 0, 0, 1630, 3601, 12, 242, 335, 599, 19, 195, 0, 0, 0, 0, 0, 0, 0, 82, 375, 180, 1015, 61, 43, 4, 11, 3981, 0, 0, 0, 0, 0, 0, 0, 5461, 39, 0, 128, 80, 12, 12, 35, 135, 0, 0, 0, 0, 0, 0, 0, 185, 338, 223, 103, 2038, 181, 282, 80, 1687, 735, 0, 0, 0, 0, 0, 0, 55, 139, 5430, 0, 46, 57, 350, 323, 399, 0, 0, 0, 0, 0, 0, 0, 631, 183, 78, 1, 5429, 248, 103, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 84, 15, 4260, 638, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5744, 6633, 5752, 5902, 5852, 6799, 6770, 5700] avg=6144.0000 max=6799 max_rank=5 min=5700 min_rank=7 imbalance_ratio=1.106608 max_load_reduction=42353 max_load_reduction_pct=0.861674 moved_to_echo=25121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 108, 2443, 0, 0, 1117, 0, 2547, 73, 0, 0, 0, 0, 0, 0, 0, 3464, 1827, 0, 0, 0, 701, 0, 190, 0, 0, 0, 0, 0, 0, 0, 63, 304, 0, 1338, 4, 55, 0, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 6203, 39, 0, 83, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 71, 324, 0, 0, 2800, 187, 0, 80, 1867, 716, 0, 0, 0, 0, 0, 0, 1, 0, 5282, 0, 0, 0, 117, 0, 389, 0, 0, 0, 0, 0, 0, 0, 564, 0, 80, 0, 5287, 0, 23, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4714, 739, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6288, 6182, 6142, 6456, 6045, 5789, 6041, 6209] avg=6144.0000 max=6456 max_rank=3 min=5789 min_rank=5 imbalance_ratio=1.050781 max_load_reduction=42696 max_load_reduction_pct=0.868652 moved_to_echo=24084 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[896, 2688, 256, 128, 22144, 1664, 384, 768, 14848, 5888, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 896, 15360, 128, 128, 11648, 0, 20736, 640, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 22016, 19712, 128, 384, 384, 5376, 128, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 896, 15360, 128, 128, 11648, 0, 20736, 640, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 2688, 256, 9728, 128, 640, 128, 128, 35584, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 256, 768, 128, 42624, 384, 512, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 256, 44416, 128, 128, 128, 1024, 384, 3200, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 37248, 5760, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 256, 44416, 128, 128, 128, 1024, 384, 3200, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 256, 768, 128, 42624, 384, 512, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[46976, 256, 0, 768, 128, 128, 128, 128, 1152, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2672, 1, 47, 84, 759, 4519, 13, 7, 1168, 64, 463, 0, 4, 5, 3, 0, 0, 87, 137, 39, 1, 88, 9, 5, 61, 2, 4, 12, 13, 89, 2, 2, 19, 0, 5883, 2, 0, 13, 3319, 904, 6, 7193, 1735, 7464, 56, 32, 4831, 88, 3, 23, 3, 87, 23, 7, 37, 4853, 671, 41, 7, 362, 598, 181, 270, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3133, 0, 0, 122, 701, 3267, 0, 0, 1229, 72, 873, 0, 0, 0, 0, 0, 0, 88, 98, 0, 0, 88, 0, 0, 58, 0, 0, 0, 34, 88, 0, 0, 0, 0, 6297, 0, 0, 0, 3701, 825, 0, 7313, 1092, 7592, 62, 0, 5195, 88, 0, 0, 0, 88, 45, 0, 0, 5140, 463, 0, 0, 368, 599, 161, 257, 15] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[2857, 0, 0, 80, 786, 3545, 0, 0, 1184, 61, 527, 0, 0, 0, 0, 0, 0, 80, 113, 0, 0, 80, 0, 0, 63, 0, 0, 0, 2, 80, 0, 1, 0, 0, 6401, 0, 0, 0, 4031, 890, 0, 7319, 1253, 7632, 47, 0, 4880, 80, 0, 0, 0, 80, 0, 0, 1, 5075, 602, 0, 0, 355, 622, 170, 244, 11] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[2933, 0, 0, 78, 747, 4468, 0, 0, 1025, 69, 428, 0, 0, 0, 0, 0, 0, 78, 121, 0, 0, 78, 0, 0, 91, 0, 0, 0, 1, 78, 0, 0, 0, 0, 6309, 1, 0, 0, 3350, 865, 0, 7375, 1449, 7908, 46, 0, 4726, 77, 0, 0, 0, 78, 0, 0, 0, 4881, 399, 0, 0, 372, 644, 182, 244, 51] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3133, 0, 0, 122, 701, 3267, 0, 0, 1229, 72, 873, 0, 0, 0, 0, 0, 0, 88, 98, 0, 0, 88, 0, 0, 58, 0, 0, 0, 34, 88, 0, 0, 0, 0, 6297, 0, 0, 0, 3701, 825, 0, 7313, 1092, 7592, 62, 0, 5195, 88, 0, 0, 0, 88, 45, 0, 0, 5140, 463, 0, 0, 368, 599, 161, 257, 15] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1819, 5, 394, 69, 643, 5626, 135, 72, 1263, 164, 393, 0, 22, 34, 34, 7, 1, 69, 321, 234, 78, 77, 133, 23, 68, 13, 18, 126, 82, 77, 25, 13, 135, 1, 4787, 3, 11, 88, 3711, 754, 1, 6662, 1742, 7009, 51, 229, 4530, 92, 53, 116, 35, 76, 0, 42, 188, 3787, 868, 370, 17, 512, 576, 107, 311, 250] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2914, 0, 0, 65, 792, 4265, 0, 0, 1166, 62, 480, 0, 0, 0, 0, 0, 0, 63, 114, 0, 0, 61, 0, 0, 95, 0, 0, 0, 0, 63, 0, 0, 0, 0, 6404, 0, 0, 0, 3255, 920, 0, 7293, 1341, 7680, 46, 0, 4945, 61, 0, 0, 0, 63, 0, 0, 0, 5013, 555, 0, 0, 356, 604, 177, 242, 57] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[2669, 0, 0, 98, 718, 4922, 0, 0, 1050, 80, 425, 0, 0, 0, 0, 0, 0, 98, 147, 0, 0, 97, 0, 0, 72, 0, 0, 0, 0, 98, 0, 2, 0, 0, 6187, 0, 0, 0, 3186, 834, 0, 7390, 1620, 7922, 52, 0, 4396, 87, 0, 0, 0, 98, 0, 0, 0, 5130, 360, 0, 0, 334, 593, 170, 231, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2672, 1, 47, 84, 759, 4519, 13, 7, 1168, 64, 463, 0, 4, 5, 3, 0, 0, 87, 137, 39, 1, 88, 9, 5, 61, 2, 4, 12, 13, 89, 2, 2, 19, 0, 5883, 2, 0, 13, 3319, 904, 6, 7193, 1735, 7464, 56, 32, 4831, 88, 3, 23, 3, 87, 23, 7, 37, 4853, 671, 41, 7, 362, 598, 181, 270, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2165, 0, 0, 90, 718, 5142, 0, 0, 1023, 69, 414, 0, 0, 0, 0, 0, 0, 90, 130, 0, 0, 89, 0, 0, 116, 0, 0, 0, 0, 90, 0, 0, 0, 0, 6145, 0, 0, 0, 3004, 841, 0, 7350, 2081, 7802, 49, 0, 4491, 72, 0, 0, 0, 90, 0, 0, 0, 5183, 418, 0, 0, 344, 593, 211, 252, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1819, 5, 394, 69, 643, 5626, 135, 72, 1263, 164, 393, 0, 22, 34, 34, 7, 1, 69, 321, 234, 78, 77, 133, 23, 68, 13, 18, 126, 82, 77, 25, 13, 135, 1, 4787, 3, 11, 88, 3711, 754, 1, 6662, 1742, 7009, 51, 229, 4530, 92, 53, 116, 35, 76, 0, 42, 188, 3787, 868, 370, 17, 512, 576, 107, 311, 250] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2672, 1, 47, 84, 759, 2626, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 64, 463, 0, 4, 5, 3, 0, 3137, 1219, 0, 0, 0, 0, 0, 0, 0, 87, 137, 39, 1, 88, 9, 5, 4056, 1651, 0, 0, 0, 0, 0, 0, 61, 2, 4, 12, 13, 89, 2, 2, 5813, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1762, 2, 0, 13, 3319, 904, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1735, 0, 56, 32, 4517, 88, 0, 0, 0, 0, 0, 0, 0, 0, 3, 23, 3, 87, 23, 7, 37, 4853, 805, 314, 0, 0, 0, 0, 0, 0, 671, 41, 7, 362, 598, 181, 270, 81, 2902, 1088, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6209, 6063, 6073, 5998, 6019, 6434, 6155, 6201] avg=6144.0000 max=6434 max_rank=5 min=5998 min_rank=3 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[2857, 0, 0, 80, 786, 3545, 0, 0, 1184, 61, 527, 0, 0, 0, 0, 0, 0, 80, 113, 0, 0, 80, 0, 0, 63, 0, 0, 0, 2, 80, 0, 1, 0, 0, 6401, 0, 0, 0, 4031, 890, 0, 7319, 1253, 7632, 47, 0, 4880, 80, 0, 0, 0, 80, 0, 0, 1, 5075, 602, 0, 0, 355, 622, 170, 244, 11] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[2933, 0, 0, 78, 747, 4468, 0, 0, 1025, 69, 428, 0, 0, 0, 0, 0, 0, 78, 121, 0, 0, 78, 0, 0, 91, 0, 0, 0, 1, 78, 0, 0, 0, 0, 6309, 1, 0, 0, 3350, 865, 0, 7375, 1449, 7908, 46, 0, 4726, 77, 0, 0, 0, 78, 0, 0, 0, 4881, 399, 0, 0, 372, 644, 182, 244, 51] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3133, 0, 0, 122, 701, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1229, 72, 873, 0, 0, 0, 0, 0, 3189, 1305, 0, 0, 0, 0, 0, 0, 0, 88, 98, 0, 0, 88, 0, 0, 4124, 1679, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 34, 88, 0, 0, 5913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1886, 0, 0, 0, 3701, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 62, 0, 4857, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 45, 0, 0, 5140, 582, 338, 0, 0, 0, 0, 0, 0, 463, 0, 0, 368, 599, 161, 257, 15, 3106, 786, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 6668, 6077, 6093, 6412, 6099, 6193, 5755] avg=6144.0000 max=6668 max_rank=1 min=5755 min_rank=7 imbalance_ratio=1.085286 max_load_reduction=42484 max_load_reduction_pct=0.864339 moved_to_echo=24459 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2857, 0, 0, 80, 786, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 61, 527, 0, 0, 0, 0, 0, 3192, 1327, 0, 0, 0, 0, 0, 0, 0, 80, 113, 0, 0, 80, 0, 0, 4127, 1688, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 2, 80, 0, 1, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1916, 0, 0, 0, 4031, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 47, 0, 4562, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 1, 5075, 632, 318, 0, 0, 0, 0, 0, 0, 602, 0, 0, 355, 622, 170, 244, 11, 3158, 853, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5783, 6291, 6088, 6090, 6837, 5942, 6106, 6015] avg=6144.0000 max=6837 max_rank=4 min=5783 min_rank=0 imbalance_ratio=1.112793 max_load_reduction=42315 max_load_reduction_pct=0.860901 moved_to_echo=24900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2669, 0, 0, 98, 718, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 80, 425, 0, 0, 0, 0, 0, 3224, 1282, 0, 0, 0, 0, 0, 0, 0, 98, 147, 0, 0, 97, 0, 0, 4166, 1753, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 98, 0, 2, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 0, 0, 3186, 834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 52, 0, 4110, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 5130, 877, 286, 0, 0, 0, 0, 0, 0, 360, 0, 0, 334, 593, 170, 231, 86, 3052, 1185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6345, 6061, 6261, 6341, 5873, 5869, 6391, 6011] avg=6144.0000 max=6391 max_rank=6 min=5869 min_rank=5 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24144 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2914, 0, 0, 65, 792, 2478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 62, 480, 0, 0, 0, 0, 0, 3182, 1327, 0, 0, 0, 0, 0, 0, 0, 63, 114, 0, 0, 61, 0, 0, 4111, 1699, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 63, 0, 0, 5981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 0, 0, 0, 3255, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1341, 0, 46, 0, 4623, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 5013, 760, 322, 0, 0, 0, 0, 0, 0, 555, 0, 0, 356, 604, 177, 242, 57, 3159, 1027, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6249, 6217, 6048, 6139, 6093, 6071, 6158, 6177] avg=6144.0000 max=6249 max_rank=0 min=6048 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=42903 max_load_reduction_pct=0.872864 moved_to_echo=24499 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1819, 5, 394, 69, 643, 3269, 135, 72, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 164, 393, 0, 22, 34, 34, 7, 2906, 992, 0, 0, 0, 0, 0, 0, 1, 69, 321, 234, 78, 77, 133, 23, 3756, 1551, 0, 0, 0, 0, 0, 0, 68, 13, 18, 126, 82, 77, 25, 13, 5458, 0, 0, 0, 0, 0, 0, 0, 135, 1, 1434, 3, 11, 88, 3711, 754, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1742, 0, 51, 229, 4235, 92, 0, 0, 0, 0, 0, 0, 0, 0, 53, 116, 35, 76, 0, 42, 188, 3787, 1003, 295, 0, 0, 0, 0, 0, 0, 868, 370, 17, 512, 576, 107, 311, 250, 2361, 1354, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6406, 5815, 6243, 5880, 6137, 6350, 5595, 6726] avg=6144.0000 max=6726 max_rank=7 min=5595 min_rank=6 imbalance_ratio=1.094727 max_load_reduction=42426 max_load_reduction_pct=0.863159 moved_to_echo=24808 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2933, 0, 0, 78, 747, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 69, 428, 0, 0, 0, 0, 0, 3217, 1307, 0, 0, 0, 0, 0, 0, 0, 78, 121, 0, 0, 78, 0, 0, 4158, 1749, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 1, 78, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1890, 1, 0, 0, 3350, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1449, 0, 46, 0, 4418, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 4881, 796, 308, 0, 0, 0, 0, 0, 0, 399, 0, 0, 372, 644, 182, 244, 51, 3112, 1076, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6354, 6046, 6184, 6329, 6106, 5990, 6063, 6080] avg=6144.0000 max=6354 max_rank=0 min=5990 min_rank=5 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24239 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2914, 0, 0, 65, 792, 2478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 62, 480, 0, 0, 0, 0, 0, 3182, 1327, 0, 0, 0, 0, 0, 0, 0, 63, 114, 0, 0, 61, 0, 0, 4111, 1699, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 63, 0, 0, 5981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 0, 0, 0, 3255, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1341, 0, 46, 0, 4623, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 5013, 760, 322, 0, 0, 0, 0, 0, 0, 555, 0, 0, 356, 604, 177, 242, 57, 3159, 1027, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6249, 6217, 6048, 6139, 6093, 6071, 6158, 6177] avg=6144.0000 max=6249 max_rank=0 min=6048 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=42903 max_load_reduction_pct=0.872864 moved_to_echo=24499 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2672, 1, 47, 84, 759, 2626, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 64, 463, 0, 4, 5, 3, 0, 3137, 1219, 0, 0, 0, 0, 0, 0, 0, 87, 137, 39, 1, 88, 9, 5, 4056, 1651, 0, 0, 0, 0, 0, 0, 61, 2, 4, 12, 13, 89, 2, 2, 5813, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1762, 2, 0, 13, 3319, 904, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1735, 0, 56, 32, 4517, 88, 0, 0, 0, 0, 0, 0, 0, 0, 3, 23, 3, 87, 23, 7, 37, 4853, 805, 314, 0, 0, 0, 0, 0, 0, 671, 41, 7, 362, 598, 181, 270, 81, 2902, 1088, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6209, 6063, 6073, 5998, 6019, 6434, 6155, 6201] avg=6144.0000 max=6434 max_rank=5 min=5998 min_rank=3 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2857, 0, 0, 80, 786, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 61, 527, 0, 0, 0, 0, 0, 3192, 1327, 0, 0, 0, 0, 0, 0, 0, 80, 113, 0, 0, 80, 0, 0, 4127, 1688, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 2, 80, 0, 1, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1916, 0, 0, 0, 4031, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 47, 0, 4562, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 1, 5075, 632, 318, 0, 0, 0, 0, 0, 0, 602, 0, 0, 355, 622, 170, 244, 11, 3158, 853, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5783, 6291, 6088, 6090, 6837, 5942, 6106, 6015] avg=6144.0000 max=6837 max_rank=4 min=5783 min_rank=0 imbalance_ratio=1.112793 max_load_reduction=42315 max_load_reduction_pct=0.860901 moved_to_echo=24900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2165, 0, 0, 90, 718, 2978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 69, 414, 0, 0, 0, 0, 0, 3207, 1278, 0, 0, 0, 0, 0, 0, 0, 90, 130, 0, 0, 89, 0, 0, 4143, 1726, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 90, 0, 0, 6076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1830, 0, 0, 0, 3004, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2081, 0, 49, 0, 4195, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 5183, 922, 296, 0, 0, 0, 0, 0, 0, 418, 0, 0, 344, 593, 211, 252, 90, 3037, 1242, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5951, 5991, 6178, 6282, 5675, 6397, 6491, 6187] avg=6144.0000 max=6491 max_rank=6 min=5675 min_rank=4 imbalance_ratio=1.056478 max_load_reduction=42661 max_load_reduction_pct=0.867940 moved_to_echo=24750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1819, 5, 394, 69, 643, 3269, 135, 72, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 164, 393, 0, 22, 34, 34, 7, 2906, 992, 0, 0, 0, 0, 0, 0, 1, 69, 321, 234, 78, 77, 133, 23, 3756, 1551, 0, 0, 0, 0, 0, 0, 68, 13, 18, 126, 82, 77, 25, 13, 5458, 0, 0, 0, 0, 0, 0, 0, 135, 1, 1434, 3, 11, 88, 3711, 754, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1742, 0, 51, 229, 4235, 92, 0, 0, 0, 0, 0, 0, 0, 0, 53, 116, 35, 76, 0, 42, 188, 3787, 1003, 295, 0, 0, 0, 0, 0, 0, 868, 370, 17, 512, 576, 107, 311, 250, 2361, 1354, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6406, 5815, 6243, 5880, 6137, 6350, 5595, 6726] avg=6144.0000 max=6726 max_rank=7 min=5595 min_rank=6 imbalance_ratio=1.094727 max_load_reduction=42426 max_load_reduction_pct=0.863159 moved_to_echo=24808 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[9216, 768, 4096, 0, 128, 128, 128, 128, 25344, 10112, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4352, 512, 128, 3072, 4864, 1408, 2176, 768, 23936, 8704, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 14592, 128, 128, 128, 27648, 6912, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[21248, 128, 512, 768, 5888, 20864, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 12416, 0, 512, 384, 35584, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 256, 128, 768, 128, 128, 256, 39168, 6400, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 768, 1280, 384, 128, 768, 256, 128, 32768, 13568, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 12416, 0, 512, 384, 35584, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[640, 128, 128, 256, 256, 768, 128, 128, 47616, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 768, 1280, 384, 128, 768, 256, 128, 32768, 13568, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[4521, 1183, 5, 1, 7, 35, 1593, 36, 30, 3918, 25, 980, 186, 6, 782, 0, 15, 175, 1754, 14, 11, 424, 10, 13, 0, 3, 4470, 650, 1636, 19, 593, 486, 47, 12, 9, 0, 38, 0, 1, 3912, 6, 38, 258, 98, 4867, 257, 5757, 14, 6456, 19, 2, 185, 0, 87, 8, 18, 63, 8, 0, 50, 2486, 91, 1, 783] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[4021, 620, 0, 0, 0, 0, 1611, 10, 0, 4861, 0, 901, 129, 0, 891, 0, 0, 160, 2198, 0, 0, 464, 32, 0, 0, 0, 3545, 651, 1759, 0, 535, 471, 0, 0, 0, 0, 0, 0, 0, 4671, 0, 0, 183, 104, 5137, 396, 6227, 0, 6513, 3, 0, 163, 0, 81, 1, 0, 62, 0, 0, 0, 1927, 80, 0, 745] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[4383, 705, 0, 0, 0, 0, 1422, 4, 0, 4168, 0, 1133, 132, 0, 937, 0, 0, 176, 2086, 0, 0, 421, 62, 0, 0, 0, 3331, 582, 1383, 0, 469, 360, 0, 0, 0, 0, 0, 0, 0, 5272, 0, 0, 251, 82, 4715, 361, 6491, 0, 6952, 52, 0, 171, 0, 88, 0, 0, 72, 0, 0, 0, 1985, 88, 0, 818] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[3721, 1608, 20, 1, 56, 80, 1514, 95, 104, 3220, 169, 809, 149, 56, 567, 1, 73, 154, 1395, 59, 71, 531, 75, 89, 33, 12, 3234, 589, 2365, 121, 484, 812, 405, 189, 63, 0, 317, 12, 11, 3469, 21, 296, 933, 86, 5151, 225, 5208, 170, 5925, 84, 16, 178, 0, 76, 1, 84, 82, 22, 1, 394, 2697, 92, 19, 658] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[4146, 722, 0, 0, 0, 0, 1896, 38, 0, 3793, 0, 884, 11, 0, 807, 0, 0, 156, 1764, 0, 0, 490, 0, 0, 0, 0, 4185, 661, 2151, 0, 593, 519, 0, 0, 0, 0, 0, 0, 0, 4319, 0, 0, 342, 120, 5571, 218, 6154, 0, 6468, 1, 0, 187, 0, 78, 0, 0, 70, 0, 0, 0, 1975, 78, 0, 755] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[4499, 659, 0, 0, 0, 0, 1823, 16, 0, 4339, 0, 983, 50, 0, 803, 0, 0, 124, 1575, 0, 0, 442, 0, 0, 0, 0, 4297, 688, 1878, 0, 515, 488, 0, 0, 0, 0, 0, 0, 0, 4466, 0, 0, 307, 116, 5210, 214, 6072, 0, 6490, 0, 0, 172, 0, 61, 0, 0, 62, 0, 0, 0, 2046, 63, 0, 694] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[4034, 1012, 0, 0, 0, 0, 1885, 70, 0, 3363, 0, 790, 0, 0, 683, 0, 0, 195, 1592, 0, 0, 489, 0, 0, 0, 0, 4584, 613, 2177, 0, 546, 618, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 512, 146, 5517, 178, 5812, 0, 6303, 0, 0, 167, 0, 97, 0, 1, 81, 0, 0, 0, 2866, 98, 0, 750] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[4521, 1183, 5, 1, 7, 35, 1593, 36, 30, 3918, 25, 980, 186, 6, 782, 0, 15, 175, 1754, 14, 11, 424, 10, 13, 0, 3, 4470, 650, 1636, 19, 593, 486, 47, 12, 9, 0, 38, 0, 1, 3912, 6, 38, 258, 98, 4867, 257, 5757, 14, 6456, 19, 2, 185, 0, 87, 8, 18, 63, 8, 0, 50, 2486, 91, 1, 783] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[3809, 1483, 0, 0, 0, 0, 2149, 82, 0, 3590, 0, 691, 0, 0, 656, 0, 0, 179, 1442, 0, 0, 459, 0, 0, 0, 0, 5134, 628, 1965, 0, 472, 646, 0, 0, 0, 0, 0, 0, 0, 3243, 0, 0, 669, 94, 5474, 123, 5449, 0, 6298, 0, 0, 229, 0, 89, 0, 1, 69, 0, 0, 0, 3107, 89, 0, 833] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[3721, 1608, 20, 1, 56, 80, 1514, 95, 104, 3220, 169, 809, 149, 56, 567, 1, 73, 154, 1395, 59, 71, 531, 75, 89, 33, 12, 3234, 589, 2365, 121, 484, 812, 405, 189, 63, 0, 317, 12, 11, 3469, 21, 296, 933, 86, 5151, 225, 5208, 170, 5925, 84, 16, 178, 0, 76, 1, 84, 82, 22, 1, 394, 2697, 92, 19, 658] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3645, 1183, 5, 1, 7, 35, 1593, 36, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3918, 25, 980, 186, 6, 782, 0, 446, 0, 0, 0, 0, 0, 0, 0, 15, 175, 1754, 14, 11, 424, 10, 13, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2725, 650, 1636, 19, 593, 486, 0, 0, 0, 0, 0, 0, 0, 0, 47, 12, 9, 0, 38, 0, 1, 3912, 982, 876, 137, 0, 0, 0, 0, 0, 6, 38, 258, 98, 4867, 257, 86, 14, 0, 0, 0, 0, 0, 0, 0, 0, 5873, 19, 2, 185, 0, 87, 8, 18, 0, 0, 0, 0, 0, 0, 0, 0, 63, 8, 0, 50, 2486, 91, 1, 783, 2037, 763, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6505, 6373, 6050, 6112, 6014, 5624, 6192, 6282] avg=6144.0000 max=6505 max_rank=0 min=5624 min_rank=5 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=24112 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3242, 620, 0, 0, 0, 0, 1611, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4861, 0, 901, 129, 0, 891, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 160, 2198, 0, 0, 464, 32, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2162, 651, 1759, 0, 535, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4671, 778, 779, 139, 0, 0, 0, 0, 0, 0, 0, 183, 104, 5137, 396, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5924, 3, 0, 163, 0, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 1927, 80, 0, 745, 2203, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5483, 7232, 6785, 5578, 6367, 5913, 6172, 5622] avg=6144.0000 max=7232 max_rank=1 min=5483 min_rank=0 imbalance_ratio=1.177083 max_load_reduction=41920 max_load_reduction_pct=0.852865 moved_to_echo=24074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[4021, 620, 0, 0, 0, 0, 1611, 10, 0, 4861, 0, 901, 129, 0, 891, 0, 0, 160, 2198, 0, 0, 464, 32, 0, 0, 0, 3545, 651, 1759, 0, 535, 471, 0, 0, 0, 0, 0, 0, 0, 4671, 0, 0, 183, 104, 5137, 396, 6227, 0, 6513, 3, 0, 163, 0, 81, 1, 0, 62, 0, 0, 0, 1927, 80, 0, 745] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[4499, 659, 0, 0, 0, 0, 1823, 16, 0, 4339, 0, 983, 50, 0, 803, 0, 0, 124, 1575, 0, 0, 442, 0, 0, 0, 0, 4297, 688, 1878, 0, 515, 488, 0, 0, 0, 0, 0, 0, 0, 4466, 0, 0, 307, 116, 5210, 214, 6072, 0, 6490, 0, 0, 172, 0, 61, 0, 0, 62, 0, 0, 0, 2046, 63, 0, 694] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3534, 705, 0, 0, 0, 0, 1422, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4168, 0, 1133, 132, 0, 937, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 176, 2086, 0, 0, 421, 62, 0, 4098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 582, 1383, 0, 469, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5272, 731, 849, 148, 0, 0, 0, 0, 0, 0, 0, 251, 82, 4715, 361, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6324, 52, 0, 171, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 1985, 88, 0, 818, 2297, 568, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5665, 6850, 6843, 4826, 7000, 5505, 6635, 5828] avg=6144.0000 max=7000 max_rank=4 min=4826 min_rank=3 imbalance_ratio=1.139323 max_load_reduction=42152 max_load_reduction_pct=0.857585 moved_to_echo=24968 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3343, 722, 0, 0, 0, 0, 1896, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 0, 884, 11, 0, 807, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 156, 1764, 0, 0, 490, 0, 0, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 661, 2151, 0, 593, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4319, 919, 803, 138, 0, 0, 0, 0, 0, 0, 0, 342, 120, 5571, 218, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5883, 1, 0, 187, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 1975, 78, 0, 755, 2178, 714, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 5942, 6295, 6476, 6179, 6342, 6149, 5770] avg=6144.0000 max=6476 max_rank=3 min=5770 min_rank=7 imbalance_ratio=1.054036 max_load_reduction=42676 max_load_reduction_pct=0.868245 moved_to_echo=24440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3000, 1608, 20, 1, 56, 80, 1514, 95, 0, 0, 0, 0, 0, 0, 0, 0, 104, 3220, 169, 809, 149, 56, 567, 1, 409, 0, 0, 0, 0, 0, 0, 0, 73, 154, 1395, 59, 71, 531, 75, 89, 3288, 0, 0, 0, 0, 0, 0, 0, 33, 12, 1972, 589, 2365, 121, 484, 812, 0, 0, 0, 0, 0, 0, 0, 0, 405, 189, 63, 0, 317, 12, 11, 3469, 710, 721, 126, 0, 0, 0, 0, 0, 21, 296, 933, 86, 5151, 225, 77, 170, 0, 0, 0, 0, 0, 0, 0, 0, 5390, 84, 16, 178, 0, 76, 1, 84, 0, 0, 0, 0, 0, 0, 0, 0, 82, 22, 1, 394, 2697, 92, 19, 658, 1843, 552, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6374, 5484, 5735, 6388, 6023, 6959, 5829, 6360] avg=6144.0000 max=6959 max_rank=5 min=5484 min_rank=1 imbalance_ratio=1.132650 max_load_reduction=42193 max_load_reduction_pct=0.858419 moved_to_echo=25171 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3645, 1183, 5, 1, 7, 35, 1593, 36, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3918, 25, 980, 186, 6, 782, 0, 446, 0, 0, 0, 0, 0, 0, 0, 15, 175, 1754, 14, 11, 424, 10, 13, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2725, 650, 1636, 19, 593, 486, 0, 0, 0, 0, 0, 0, 0, 0, 47, 12, 9, 0, 38, 0, 1, 3912, 982, 876, 137, 0, 0, 0, 0, 0, 6, 38, 258, 98, 4867, 257, 86, 14, 0, 0, 0, 0, 0, 0, 0, 0, 5873, 19, 2, 185, 0, 87, 8, 18, 0, 0, 0, 0, 0, 0, 0, 0, 63, 8, 0, 50, 2486, 91, 1, 783, 2037, 763, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6505, 6373, 6050, 6112, 6014, 5624, 6192, 6282] avg=6144.0000 max=6505 max_rank=0 min=5624 min_rank=5 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=24112 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3253, 1012, 0, 0, 0, 0, 1885, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3363, 0, 790, 0, 0, 683, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 195, 1592, 0, 0, 489, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 613, 2177, 0, 546, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 1007, 781, 134, 0, 0, 0, 0, 0, 0, 0, 512, 146, 5517, 178, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5734, 0, 0, 167, 0, 97, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 2866, 98, 0, 750, 2057, 782, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6220, 5271, 5945, 6749, 5895, 6439, 5999, 6634] avg=6144.0000 max=6749 max_rank=3 min=5271 min_rank=1 imbalance_ratio=1.098470 max_load_reduction=42403 max_load_reduction_pct=0.862691 moved_to_echo=24967 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3242, 620, 0, 0, 0, 0, 1611, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4861, 0, 901, 129, 0, 891, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 160, 2198, 0, 0, 464, 32, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2162, 651, 1759, 0, 535, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4671, 778, 779, 139, 0, 0, 0, 0, 0, 0, 0, 183, 104, 5137, 396, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5924, 3, 0, 163, 0, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 1927, 80, 0, 745, 2203, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5483, 7232, 6785, 5578, 6367, 5913, 6172, 5622] avg=6144.0000 max=7232 max_rank=1 min=5483 min_rank=0 imbalance_ratio=1.177083 max_load_reduction=41920 max_load_reduction_pct=0.852865 moved_to_echo=24074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3627, 659, 0, 0, 0, 0, 1823, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4339, 0, 983, 50, 0, 803, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 124, 1575, 0, 0, 442, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2620, 688, 1878, 0, 515, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4466, 944, 872, 138, 0, 0, 0, 0, 0, 0, 0, 307, 116, 5210, 214, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5904, 0, 0, 172, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 2046, 63, 0, 694, 2149, 733, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6125, 6623, 5974, 6189, 6420, 5937, 6137, 5747] avg=6144.0000 max=6623 max_rank=1 min=5747 min_rank=7 imbalance_ratio=1.077962 max_load_reduction=42529 max_load_reduction_pct=0.865255 moved_to_echo=24241 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3343, 722, 0, 0, 0, 0, 1896, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 0, 884, 11, 0, 807, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 156, 1764, 0, 0, 490, 0, 0, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 661, 2151, 0, 593, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4319, 919, 803, 138, 0, 0, 0, 0, 0, 0, 0, 342, 120, 5571, 218, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5883, 1, 0, 187, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 1975, 78, 0, 755, 2178, 714, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 5942, 6295, 6476, 6179, 6342, 6149, 5770] avg=6144.0000 max=6476 max_rank=3 min=5770 min_rank=7 imbalance_ratio=1.054036 max_load_reduction=42676 max_load_reduction_pct=0.868245 moved_to_echo=24440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3067, 1483, 0, 0, 0, 0, 2149, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 691, 0, 0, 656, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1442, 0, 0, 459, 0, 0, 3445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3123, 628, 1965, 0, 472, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 1131, 742, 138, 0, 0, 0, 0, 0, 0, 0, 669, 94, 5474, 123, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5720, 0, 0, 229, 0, 89, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 3107, 89, 0, 833, 1931, 880, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6781, 5377, 5525, 6834, 5254, 6433, 6039, 6909] avg=6144.0000 max=6909 max_rank=7 min=5254 min_rank=4 imbalance_ratio=1.124512 max_load_reduction=42243 max_load_reduction_pct=0.859436 moved_to_echo=24635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3000, 1608, 20, 1, 56, 80, 1514, 95, 0, 0, 0, 0, 0, 0, 0, 0, 104, 3220, 169, 809, 149, 56, 567, 1, 409, 0, 0, 0, 0, 0, 0, 0, 73, 154, 1395, 59, 71, 531, 75, 89, 3288, 0, 0, 0, 0, 0, 0, 0, 33, 12, 1972, 589, 2365, 121, 484, 812, 0, 0, 0, 0, 0, 0, 0, 0, 405, 189, 63, 0, 317, 12, 11, 3469, 710, 721, 126, 0, 0, 0, 0, 0, 21, 296, 933, 86, 5151, 225, 77, 170, 0, 0, 0, 0, 0, 0, 0, 0, 5390, 84, 16, 178, 0, 76, 1, 84, 0, 0, 0, 0, 0, 0, 0, 0, 82, 22, 1, 394, 2697, 92, 19, 658, 1843, 552, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6374, 5484, 5735, 6388, 6023, 6959, 5829, 6360] avg=6144.0000 max=6959 max_rank=5 min=5484 min_rank=1 imbalance_ratio=1.132650 max_load_reduction=42193 max_load_reduction_pct=0.858419 moved_to_echo=25171 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[26752, 8064, 128, 128, 128, 128, 13952, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[640, 128, 128, 512, 19200, 768, 128, 6144, 16768, 5632, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 20096, 5120, 15360, 256, 4224, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 13824, 128, 128, 3840, 256, 128, 29824, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[512, 256, 128, 0, 384, 128, 128, 33408, 7296, 6528, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 20096, 5120, 15360, 256, 4224, 4480, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 31360, 256, 7296, 768, 128, 6144, 128, 3584, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 13824, 128, 128, 3840, 256, 128, 29824, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 384, 3456, 896, 41728, 2048, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[46848, 256, 128, 1536, 0, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 92, 2612, 116, 0, 0, 2849, 0, 0, 16, 138, 510, 7245, 0, 52, 7465, 10, 0, 1, 3, 8, 2318, 5, 32, 23, 0, 9, 7825, 6, 55, 2385, 2, 11, 0, 125, 1, 16, 7138, 24, 7, 6, 0, 459, 0, 49, 57, 30, 20, 1, 37, 34, 11, 3, 354, 41, 87, 6, 5695, 0, 18, 453, 645, 38, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 80, 1477, 175, 0, 0, 4020, 0, 1, 0, 139, 322, 7369, 0, 0, 7578, 0, 0, 0, 0, 0, 3183, 0, 0, 0, 0, 0, 7885, 0, 0, 2178, 0, 1, 0, 132, 0, 0, 6753, 0, 0, 0, 0, 475, 0, 0, 0, 0, 20, 0, 0, 30, 0, 0, 384, 0, 80, 0, 6164, 0, 0, 635, 71, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 74, 2808, 70, 0, 0, 3494, 0, 0, 0, 160, 308, 7171, 0, 0, 7541, 0, 0, 0, 0, 0, 1502, 6, 0, 0, 0, 0, 7956, 0, 0, 2733, 0, 0, 0, 132, 0, 0, 7517, 0, 0, 0, 0, 457, 0, 0, 0, 0, 16, 0, 0, 3, 0, 0, 326, 0, 98, 0, 5831, 0, 0, 417, 532, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 85, 1213, 68, 0, 0, 3611, 0, 0, 0, 156, 439, 7376, 0, 0, 7546, 2, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 8060, 0, 0, 2063, 0, 0, 0, 105, 0, 0, 6069, 0, 0, 0, 0, 542, 0, 0, 0, 0, 33, 0, 0, 49, 0, 0, 447, 0, 88, 0, 6381, 0, 0, 704, 97, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 60, 1961, 10, 0, 0, 3610, 0, 0, 0, 118, 383, 7402, 0, 0, 7591, 0, 0, 0, 0, 0, 2515, 1, 0, 0, 0, 0, 8063, 0, 0, 2408, 0, 0, 0, 105, 0, 0, 7178, 0, 0, 0, 0, 452, 0, 0, 0, 0, 13, 0, 0, 34, 0, 0, 396, 0, 63, 0, 6147, 0, 0, 424, 218, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 92, 2612, 116, 0, 0, 2849, 0, 0, 16, 138, 510, 7245, 0, 52, 7465, 10, 0, 1, 3, 8, 2318, 5, 32, 23, 0, 9, 7825, 6, 55, 2385, 2, 11, 0, 125, 1, 16, 7138, 24, 7, 6, 0, 459, 0, 49, 57, 30, 20, 1, 37, 34, 11, 3, 354, 41, 87, 6, 5695, 0, 18, 453, 645, 38, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 88, 3203, 5, 0, 0, 3513, 0, 0, 0, 150, 298, 7150, 0, 0, 7497, 1, 0, 0, 0, 0, 888, 34, 0, 0, 0, 0, 8003, 0, 0, 3154, 0, 0, 0, 93, 0, 0, 7606, 0, 0, 0, 0, 465, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 290, 0, 90, 0, 5531, 0, 0, 318, 744, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 76, 2100, 38, 0, 0, 3466, 0, 0, 0, 132, 373, 7291, 0, 0, 7552, 0, 0, 0, 0, 0, 2439, 0, 0, 0, 0, 0, 8025, 0, 0, 2523, 0, 0, 0, 124, 0, 0, 7260, 0, 0, 0, 0, 469, 0, 0, 0, 0, 21, 0, 0, 19, 0, 0, 393, 0, 78, 0, 6035, 0, 0, 432, 306, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 60, 1961, 10, 0, 0, 3610, 0, 0, 0, 118, 383, 7402, 0, 0, 7591, 0, 0, 0, 0, 0, 2515, 1, 0, 0, 0, 0, 8063, 0, 0, 2408, 0, 0, 0, 105, 0, 0, 7178, 0, 0, 0, 0, 452, 0, 0, 0, 0, 13, 0, 0, 34, 0, 0, 396, 0, 63, 0, 6147, 0, 0, 424, 218, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[17, 113, 3488, 293, 16, 3, 2908, 0, 0, 6, 121, 236, 6606, 17, 375, 6905, 34, 4, 1, 46, 35, 1018, 42, 284, 25, 9, 18, 7059, 26, 456, 2203, 21, 60, 5, 81, 8, 40, 7012, 183, 102, 18, 0, 346, 0, 153, 490, 198, 101, 0, 91, 20, 90, 21, 285, 399, 82, 49, 4301, 0, 50, 383, 1938, 201, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 74, 2808, 70, 0, 0, 3494, 0, 0, 0, 160, 308, 7171, 0, 0, 7541, 0, 0, 0, 0, 0, 1502, 6, 0, 0, 0, 0, 7956, 0, 0, 2733, 0, 0, 0, 132, 0, 0, 7517, 0, 0, 0, 0, 457, 0, 0, 0, 0, 16, 0, 0, 3, 0, 0, 326, 0, 98, 0, 5831, 0, 0, 417, 532, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 80, 1477, 175, 0, 0, 4020, 0, 1, 0, 139, 322, 7369, 0, 0, 7578, 0, 0, 0, 0, 0, 3183, 0, 0, 0, 0, 0, 7885, 0, 0, 2178, 0, 1, 0, 132, 0, 0, 6753, 0, 0, 0, 0, 475, 0, 0, 0, 0, 20, 0, 0, 30, 0, 0, 384, 0, 80, 0, 6164, 0, 0, 635, 71, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 80, 1477, 175, 0, 0, 4020, 0, 179, 0, 0, 0, 0, 0, 0, 0, 1, 0, 139, 322, 5718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 0, 1651, 1043, 594, 580, 0, 0, 0, 0, 0, 0, 0, 3619, 0, 0, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 132, 0, 0, 5710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 0, 20, 3672, 1898, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 384, 0, 80, 5680, 0, 0, 0, 0, 0, 0, 0, 0, 5405, 0, 0, 635, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5931, 6180, 7051, 5797, 5843, 6065, 6174, 6111] avg=6144.0000 max=7051 max_rank=2 min=5797 min_rank=3 imbalance_ratio=1.147624 max_load_reduction=42101 max_load_reduction_pct=0.856547 moved_to_echo=24193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 74, 2808, 70, 0, 0, 3494, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 308, 5565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 6, 0, 1606, 1161, 599, 549, 0, 0, 0, 0, 0, 0, 0, 3652, 0, 0, 2733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 6356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 16, 3705, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 326, 0, 98, 5652, 0, 0, 0, 0, 0, 0, 0, 0, 5113, 0, 0, 417, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6615, 6033, 5423, 6385, 6488, 6067, 6079, 6062] avg=6144.0000 max=6615 max_rank=0 min=5423 min_rank=2 imbalance_ratio=1.076660 max_load_reduction=42537 max_load_reduction_pct=0.865417 moved_to_echo=24696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 76, 2100, 38, 0, 0, 3466, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 373, 5658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2439, 0, 0, 1633, 1121, 604, 568, 0, 0, 0, 0, 0, 0, 0, 3684, 0, 0, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 0, 0, 21, 3737, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 393, 0, 78, 5661, 0, 0, 0, 0, 0, 0, 0, 0, 5292, 0, 0, 432, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 6163, 6365, 6207, 6263, 6118, 6151, 6030] avg=6144.0000 max=6365 max_rank=2 min=5855 min_rank=0 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 92, 2612, 116, 0, 0, 2849, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 16, 138, 510, 5622, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 3, 8, 2318, 5, 32, 1623, 1103, 589, 536, 0, 0, 0, 0, 23, 0, 9, 3592, 6, 55, 2385, 2, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 125, 1, 16, 6035, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 459, 0, 49, 57, 30, 20, 3644, 1869, 0, 0, 0, 0, 0, 0, 1, 37, 34, 11, 3, 354, 41, 87, 5596, 0, 0, 0, 0, 0, 0, 0, 6, 4994, 0, 18, 453, 645, 38, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5834, 6338, 6228, 6072, 6219, 6134, 6164, 6163] avg=6144.0000 max=6338 max_rank=1 min=5834 min_rank=0 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24680 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 85, 1213, 68, 0, 0, 3611, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 439, 5724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4018, 0, 0, 1652, 937, 607, 601, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 5132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 0, 33, 3753, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 447, 0, 88, 5656, 0, 0, 0, 0, 0, 0, 0, 0, 5595, 0, 0, 704, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5162, 6319, 7817, 5763, 5237, 6218, 6240, 6396] avg=6144.0000 max=7817 max_rank=2 min=5162 min_rank=0 imbalance_ratio=1.272298 max_load_reduction=41335 max_load_reduction_pct=0.840963 moved_to_echo=24091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 60, 1961, 10, 0, 0, 3610, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 383, 5744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2515, 1, 0, 1658, 1109, 607, 579, 0, 0, 0, 0, 0, 0, 0, 3701, 0, 0, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 6069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 0, 13, 3755, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 396, 0, 63, 5689, 0, 0, 0, 0, 0, 0, 0, 0, 5390, 0, 0, 424, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5819, 6245, 6469, 6109, 6174, 6122, 6182, 6032] avg=6144.0000 max=6469 max_rank=2 min=5819 min_rank=0 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24510 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 74, 2808, 70, 0, 0, 3494, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 308, 5565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 6, 0, 1606, 1161, 599, 549, 0, 0, 0, 0, 0, 0, 0, 3652, 0, 0, 2733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 6356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 16, 3705, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 326, 0, 98, 5652, 0, 0, 0, 0, 0, 0, 0, 0, 5113, 0, 0, 417, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6615, 6033, 5423, 6385, 6488, 6067, 6079, 6062] avg=6144.0000 max=6615 max_rank=0 min=5423 min_rank=2 imbalance_ratio=1.076660 max_load_reduction=42537 max_load_reduction_pct=0.865417 moved_to_echo=24696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 80, 1477, 175, 0, 0, 4020, 0, 179, 0, 0, 0, 0, 0, 0, 0, 1, 0, 139, 322, 5718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 0, 1651, 1043, 594, 580, 0, 0, 0, 0, 0, 0, 0, 3619, 0, 0, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 132, 0, 0, 5710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 0, 20, 3672, 1898, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 384, 0, 80, 5680, 0, 0, 0, 0, 0, 0, 0, 0, 5405, 0, 0, 635, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5931, 6180, 7051, 5797, 5843, 6065, 6174, 6111] avg=6144.0000 max=7051 max_rank=2 min=5797 min_rank=3 imbalance_ratio=1.147624 max_load_reduction=42101 max_load_reduction_pct=0.856547 moved_to_echo=24193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 88, 3203, 5, 0, 0, 3513, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 298, 5545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 888, 34, 0, 1605, 1179, 606, 523, 0, 0, 0, 0, 0, 0, 0, 3666, 0, 0, 3154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 6427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 0, 0, 31, 3731, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 90, 5619, 0, 0, 0, 0, 0, 0, 0, 0, 4844, 0, 0, 318, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6973, 5993, 4836, 6820, 6520, 6105, 5999, 5906] avg=6144.0000 max=6973 max_rank=0 min=4836 min_rank=2 imbalance_ratio=1.134928 max_load_reduction=42179 max_load_reduction_pct=0.858134 moved_to_echo=24530 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 76, 2100, 38, 0, 0, 3466, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 373, 5658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2439, 0, 0, 1633, 1121, 604, 568, 0, 0, 0, 0, 0, 0, 0, 3684, 0, 0, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 0, 0, 21, 3737, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 393, 0, 78, 5661, 0, 0, 0, 0, 0, 0, 0, 0, 5292, 0, 0, 432, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 6163, 6365, 6207, 6263, 6118, 6151, 6030] avg=6144.0000 max=6365 max_rank=2 min=5855 min_rank=0 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[17, 113, 3488, 293, 16, 3, 2908, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 6, 121, 236, 5126, 17, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4, 1, 46, 35, 1018, 42, 284, 1480, 1083, 531, 405, 0, 0, 0, 0, 25, 9, 18, 3241, 26, 456, 2203, 21, 0, 0, 0, 0, 0, 0, 0, 0, 60, 5, 81, 8, 40, 5929, 183, 102, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 346, 0, 153, 490, 198, 101, 3287, 1730, 0, 0, 0, 0, 0, 0, 0, 91, 20, 90, 21, 285, 399, 82, 5175, 0, 0, 0, 0, 0, 0, 0, 49, 3771, 0, 50, 383, 1938, 201, 60, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6963, 5881, 4963, 5999, 6408, 6323, 6163, 6452] avg=6144.0000 max=6963 max_rank=0 min=4963 min_rank=2 imbalance_ratio=1.133301 max_load_reduction=42189 max_load_reduction_pct=0.858337 moved_to_echo=25346 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 88, 3203, 5, 0, 0, 3513, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 298, 5545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 888, 34, 0, 1605, 1179, 606, 523, 0, 0, 0, 0, 0, 0, 0, 3666, 0, 0, 3154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 6427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 0, 0, 31, 3731, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 90, 5619, 0, 0, 0, 0, 0, 0, 0, 0, 4844, 0, 0, 318, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6973, 5993, 4836, 6820, 6520, 6105, 5999, 5906] avg=6144.0000 max=6973 max_rank=0 min=4836 min_rank=2 imbalance_ratio=1.134928 max_load_reduction=42179 max_load_reduction_pct=0.858134 moved_to_echo=24530 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 17920, 128, 384, 12928, 8832, 4864, 4352, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 3712, 0, 256, 640, 256, 256, 29312, 14976, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 28928, 128, 512, 19712, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 768, 18944, 896, 128, 128, 27520, 0, 1408, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 1152, 2944, 44800, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 40448, 0, 128, 3840, 4608, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 256, 128, 128, 2944, 512, 768, 44800, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 40448, 0, 128, 3840, 4608, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 1024, 128, 128, 47872, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[68, 0, 0, 0, 0, 88, 0, 0, 0, 0, 176, 0, 0, 62, 1089, 0, 4376, 14, 88, 7567, 7523, 0, 0, 0, 0, 349, 702, 0, 7669, 0, 62, 0, 4123, 0, 0, 0, 0, 4503, 7578, 88, 0, 0, 0, 0, 0, 62, 0, 57, 0, 1458, 0, 226, 0, 0, 0, 0, 0, 0, 0, 112, 434, 0, 309, 369] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[79, 0, 0, 0, 0, 80, 3, 0, 0, 1, 161, 0, 0, 52, 1144, 0, 4087, 8, 81, 7600, 7567, 0, 0, 0, 0, 247, 818, 0, 7680, 0, 52, 0, 4931, 0, 0, 0, 1, 4126, 7526, 80, 0, 0, 0, 0, 0, 52, 0, 63, 0, 1116, 0, 197, 0, 0, 0, 0, 0, 0, 0, 221, 488, 0, 336, 355] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[35, 2, 333, 0, 477, 70, 244, 46, 25, 7, 205, 109, 4, 100, 964, 4, 4158, 373, 141, 6901, 6266, 2, 255, 113, 0, 178, 2902, 0, 6798, 175, 227, 13, 4265, 105, 173, 74, 148, 1595, 6380, 88, 2, 72, 13, 332, 11, 40, 2, 37, 25, 536, 51, 966, 0, 361, 0, 1, 146, 25, 297, 204, 450, 440, 788, 398] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[99, 0, 0, 0, 0, 80, 0, 0, 0, 0, 196, 0, 0, 64, 1269, 0, 4222, 222, 115, 7559, 7553, 0, 0, 0, 0, 0, 1421, 0, 7605, 0, 64, 0, 6030, 0, 10, 0, 0, 2440, 7396, 98, 0, 0, 0, 0, 0, 61, 0, 60, 0, 986, 0, 199, 0, 0, 0, 0, 0, 0, 1, 250, 479, 0, 339, 334] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[88, 0, 0, 0, 0, 44, 0, 0, 0, 0, 179, 0, 0, 61, 1347, 0, 3717, 203, 135, 7548, 7470, 0, 0, 0, 0, 0, 1967, 0, 7613, 0, 61, 0, 6295, 0, 49, 0, 0, 2133, 7409, 90, 0, 0, 0, 0, 0, 61, 0, 95, 0, 914, 0, 243, 0, 0, 0, 0, 0, 0, 0, 288, 467, 0, 331, 344] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[44, 0, 50, 0, 62, 87, 42, 11, 19, 1, 166, 13, 0, 67, 1221, 7, 3917, 261, 133, 7460, 7302, 2, 24, 15, 0, 42, 1478, 0, 7503, 20, 61, 0, 4982, 11, 29, 4, 24, 3518, 7319, 87, 1, 13, 0, 37, 0, 53, 0, 51, 24, 929, 22, 419, 0, 45, 0, 0, 22, 25, 32, 224, 493, 52, 379, 349] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[68, 0, 0, 0, 0, 88, 0, 0, 0, 0, 176, 0, 0, 62, 1089, 0, 4376, 14, 88, 7567, 7523, 0, 0, 0, 0, 349, 702, 0, 7669, 0, 62, 0, 4123, 0, 0, 0, 0, 4503, 7578, 88, 0, 0, 0, 0, 0, 62, 0, 57, 0, 1458, 0, 226, 0, 0, 0, 0, 0, 0, 0, 112, 434, 0, 309, 369] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[71, 0, 0, 0, 0, 74, 0, 0, 0, 0, 156, 0, 0, 51, 1214, 0, 4382, 104, 89, 7616, 7523, 0, 0, 0, 0, 144, 1052, 0, 7601, 0, 51, 0, 5486, 0, 0, 0, 0, 3196, 7492, 78, 0, 0, 0, 0, 0, 51, 0, 84, 0, 992, 0, 199, 0, 0, 0, 0, 0, 0, 0, 249, 495, 0, 330, 372] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[63, 0, 0, 0, 0, 61, 0, 0, 0, 0, 126, 0, 0, 54, 1163, 0, 4192, 64, 63, 7676, 7591, 0, 0, 0, 0, 182, 1026, 0, 7654, 0, 54, 0, 5258, 0, 3, 0, 0, 3528, 7470, 63, 0, 0, 0, 0, 0, 53, 0, 88, 0, 1139, 0, 169, 0, 0, 0, 0, 0, 0, 0, 246, 507, 0, 303, 356] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[99, 0, 0, 0, 0, 80, 0, 0, 0, 0, 196, 0, 0, 64, 1269, 0, 4222, 222, 115, 7559, 7553, 0, 0, 0, 0, 0, 1421, 0, 7605, 0, 64, 0, 6030, 0, 10, 0, 0, 2440, 7396, 98, 0, 0, 0, 0, 0, 61, 0, 60, 0, 986, 0, 199, 0, 0, 0, 0, 0, 0, 1, 250, 479, 0, 339, 334] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[35, 2, 333, 0, 477, 70, 244, 46, 25, 7, 205, 109, 4, 100, 964, 4, 4158, 373, 141, 6901, 6266, 2, 255, 113, 0, 178, 2902, 0, 6798, 175, 227, 13, 4265, 105, 173, 74, 148, 1595, 6380, 88, 2, 72, 13, 332, 11, 40, 2, 37, 25, 536, 51, 966, 0, 361, 0, 1, 146, 25, 297, 204, 450, 440, 788, 398] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[79, 0, 0, 0, 0, 80, 3, 0, 4436, 1551, 0, 0, 0, 0, 0, 0, 0, 1, 161, 0, 0, 52, 1144, 0, 4029, 813, 0, 0, 0, 0, 0, 0, 4087, 8, 81, 0, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 818, 0, 4574, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 0, 0, 0, 1, 4126, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 63, 6049, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 0, 197, 0, 0, 0, 0, 3090, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 488, 0, 336, 355, 2212, 2293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6200, 5927, 5691, 6926, 6164, 6190, 5905] avg=6144.0000 max=6926 max_rank=4 min=5691 min_rank=3 imbalance_ratio=1.127279 max_load_reduction=42226 max_load_reduction_pct=0.859090 moved_to_echo=25185 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 50, 0, 62, 87, 42, 11, 4314, 1521, 0, 0, 0, 0, 0, 0, 19, 1, 166, 13, 0, 67, 1221, 7, 3888, 794, 0, 0, 0, 0, 0, 0, 3917, 261, 133, 0, 1689, 2, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1478, 0, 4469, 20, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2747, 11, 29, 4, 24, 3518, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 37, 0, 53, 0, 51, 5939, 0, 0, 0, 0, 0, 0, 0, 24, 929, 22, 419, 0, 45, 0, 0, 3005, 1725, 0, 0, 0, 0, 0, 0, 22, 25, 32, 224, 493, 52, 379, 349, 2235, 2240, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6176, 6041, 6070, 6420, 6094, 6169, 6051] avg=6144.0000 max=6420 max_rank=4 min=6041 min_rank=2 imbalance_ratio=1.044922 max_load_reduction=42732 max_load_reduction_pct=0.869385 moved_to_echo=24734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[44, 0, 50, 0, 62, 87, 42, 11, 19, 1, 166, 13, 0, 67, 1221, 7, 3917, 261, 133, 7460, 7302, 2, 24, 15, 0, 42, 1478, 0, 7503, 20, 61, 0, 4982, 11, 29, 4, 24, 3518, 7319, 87, 1, 13, 0, 37, 0, 53, 0, 51, 24, 929, 22, 419, 0, 45, 0, 0, 22, 25, 32, 224, 493, 52, 379, 349] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[68, 0, 0, 0, 0, 88, 0, 0, 4466, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 62, 1089, 0, 4006, 812, 0, 0, 0, 0, 0, 0, 4376, 14, 88, 0, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 702, 0, 4568, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2274, 0, 0, 0, 0, 4503, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 57, 6024, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 226, 0, 0, 0, 0, 3112, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 434, 0, 309, 369, 1849, 2289, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6165, 6145, 6218, 5681, 6865, 6143, 6573, 5362] avg=6144.0000 max=6865 max_rank=4 min=5362 min_rank=7 imbalance_ratio=1.117350 max_load_reduction=42287 max_load_reduction_pct=0.860331 moved_to_echo=24943 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[79, 0, 0, 0, 0, 80, 3, 0, 4436, 1551, 0, 0, 0, 0, 0, 0, 0, 1, 161, 0, 0, 52, 1144, 0, 4029, 813, 0, 0, 0, 0, 0, 0, 4087, 8, 81, 0, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 818, 0, 4574, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 0, 0, 0, 1, 4126, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 63, 6049, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 0, 197, 0, 0, 0, 0, 3090, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 488, 0, 336, 355, 2212, 2293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6200, 5927, 5691, 6926, 6164, 6190, 5905] avg=6144.0000 max=6926 max_rank=4 min=5691 min_rank=3 imbalance_ratio=1.127279 max_load_reduction=42226 max_load_reduction_pct=0.859090 moved_to_echo=25185 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[99, 0, 0, 0, 0, 80, 0, 0, 4360, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 64, 1269, 0, 4022, 805, 0, 0, 0, 0, 0, 0, 4222, 222, 115, 0, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1421, 0, 4530, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 0, 10, 0, 0, 2440, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 60, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 986, 0, 199, 0, 0, 0, 0, 3036, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 1, 250, 479, 0, 339, 334, 2705, 2270, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6082, 6356, 6306, 6015, 5873, 6137, 6005, 6378] avg=6144.0000 max=6378 max_rank=7 min=5873 min_rank=4 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=24393 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[63, 0, 0, 0, 0, 61, 0, 0, 4404, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 54, 1163, 0, 4042, 810, 0, 0, 0, 0, 0, 0, 4192, 64, 63, 0, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 1026, 0, 4559, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 3, 0, 0, 3528, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 88, 6109, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 0, 169, 0, 0, 0, 0, 3066, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 507, 0, 303, 356, 2358, 2285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6195, 6075, 5821, 6494, 6250, 6167, 6055] avg=6144.0000 max=6494 max_rank=4 min=5821 min_rank=3 imbalance_ratio=1.056966 max_load_reduction=42658 max_load_reduction_pct=0.867879 moved_to_echo=24966 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[88, 0, 0, 0, 0, 44, 0, 0, 4368, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 61, 1347, 0, 3980, 809, 0, 0, 0, 0, 0, 0, 3717, 203, 135, 0, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1967, 0, 4526, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 49, 0, 0, 2133, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 95, 6008, 0, 0, 0, 0, 0, 0, 0, 0, 914, 0, 243, 0, 0, 0, 0, 3041, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 467, 0, 331, 344, 2827, 2278, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6040, 6376, 5778, 6554, 5740, 6164, 5965, 6535] avg=6144.0000 max=6554 max_rank=3 min=5740 min_rank=4 imbalance_ratio=1.066732 max_load_reduction=42598 max_load_reduction_pct=0.866659 moved_to_echo=24404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[71, 0, 0, 0, 0, 74, 0, 0, 4416, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 51, 1214, 0, 4006, 804, 0, 0, 0, 0, 0, 0, 4382, 104, 89, 0, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 1052, 0, 4528, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3025, 0, 0, 0, 0, 3196, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 84, 6063, 0, 0, 0, 0, 0, 0, 0, 0, 992, 0, 199, 0, 0, 0, 0, 3076, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 495, 0, 330, 372, 2461, 2269, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6231, 6315, 5775, 6299, 6198, 6044, 6176] avg=6144.0000 max=6315 max_rank=2 min=5775 min_rank=3 imbalance_ratio=1.027832 max_load_reduction=42837 max_load_reduction_pct=0.871521 moved_to_echo=24717 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[35, 2, 333, 0, 477, 70, 244, 46, 3761, 1408, 0, 0, 0, 0, 0, 0, 25, 7, 205, 109, 4, 100, 964, 4, 3336, 719, 0, 0, 0, 0, 0, 0, 4158, 373, 141, 0, 1450, 2, 255, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 2902, 0, 4050, 175, 227, 13, 0, 0, 0, 0, 0, 0, 0, 0, 2352, 105, 173, 74, 148, 1595, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 2, 72, 13, 332, 11, 40, 2, 37, 5493, 0, 0, 0, 0, 0, 0, 0, 25, 536, 51, 966, 0, 361, 0, 1, 2619, 1480, 0, 0, 0, 0, 0, 0, 146, 25, 297, 204, 450, 440, 788, 398, 1913, 2029, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6376, 5473, 6492, 7545, 4535, 6002, 6039, 6690] avg=6144.0000 max=7545 max_rank=3 min=4535 min_rank=4 imbalance_ratio=1.228027 max_load_reduction=41607 max_load_reduction_pct=0.846497 moved_to_echo=23266 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 50, 0, 62, 87, 42, 11, 4314, 1521, 0, 0, 0, 0, 0, 0, 19, 1, 166, 13, 0, 67, 1221, 7, 3888, 794, 0, 0, 0, 0, 0, 0, 3917, 261, 133, 0, 1689, 2, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1478, 0, 4469, 20, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2747, 11, 29, 4, 24, 3518, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 37, 0, 53, 0, 51, 5939, 0, 0, 0, 0, 0, 0, 0, 24, 929, 22, 419, 0, 45, 0, 0, 3005, 1725, 0, 0, 0, 0, 0, 0, 22, 25, 32, 224, 493, 52, 379, 349, 2235, 2240, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6176, 6041, 6070, 6420, 6094, 6169, 6051] avg=6144.0000 max=6420 max_rank=4 min=6041 min_rank=2 imbalance_ratio=1.044922 max_load_reduction=42732 max_load_reduction_pct=0.869385 moved_to_echo=24734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[63, 0, 0, 0, 0, 61, 0, 0, 4404, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 54, 1163, 0, 4042, 810, 0, 0, 0, 0, 0, 0, 4192, 64, 63, 0, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 1026, 0, 4559, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 3, 0, 0, 3528, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 88, 6109, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 0, 169, 0, 0, 0, 0, 3066, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 507, 0, 303, 356, 2358, 2285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6195, 6075, 5821, 6494, 6250, 6167, 6055] avg=6144.0000 max=6494 max_rank=4 min=5821 min_rank=3 imbalance_ratio=1.056966 max_load_reduction=42658 max_load_reduction_pct=0.867879 moved_to_echo=24966 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[88, 0, 0, 0, 0, 44, 0, 0, 4368, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 61, 1347, 0, 3980, 809, 0, 0, 0, 0, 0, 0, 3717, 203, 135, 0, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1967, 0, 4526, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 49, 0, 0, 2133, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 95, 6008, 0, 0, 0, 0, 0, 0, 0, 0, 914, 0, 243, 0, 0, 0, 0, 3041, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 467, 0, 331, 344, 2827, 2278, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6040, 6376, 5778, 6554, 5740, 6164, 5965, 6535] avg=6144.0000 max=6554 max_rank=3 min=5740 min_rank=4 imbalance_ratio=1.066732 max_load_reduction=42598 max_load_reduction_pct=0.866659 moved_to_echo=24404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 128, 384, 1920, 3840, 512, 3200, 2944, 18560, 18048, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 8192, 128, 2688, 0, 512, 0, 128, 24064, 13952, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 128, 384, 0, 640, 640, 384, 128, 34560, 12288, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[33152, 1280, 896, 0, 13696, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 1408, 128, 128, 512, 9472, 128, 31360, 6400, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[22912, 128, 384, 128, 256, 25088, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[33152, 1280, 896, 0, 13696, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 1152, 11392, 0, 35840, 256, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 384, 128, 512, 128, 640, 47744, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 8, 81, 0, 80, 2552, 0, 0, 0, 2807, 1715, 8107, 659, 0, 80, 7616, 0, 0, 7503, 461, 52, 0, 0, 0, 0, 109, 0, 52, 0, 0, 3205, 0, 683, 0, 0, 0, 0, 79, 0, 0, 0, 0, 1295, 3469, 0, 1475, 162, 1224, 80, 0, 0, 0, 2248, 152, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2535, 0, 0, 0, 0, 0, 0, 1, 78, 0, 78, 2454, 0, 0, 0, 2206, 1745, 8113, 664, 0, 78, 7570, 0, 0, 7369, 403, 51, 0, 0, 0, 0, 159, 0, 51, 0, 0, 3684, 0, 933, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1269, 3766, 0, 2340, 168, 1199, 78, 0, 0, 0, 1924, 158, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[1, 0, 41, 3445, 13, 22, 0, 0, 4, 1, 2, 86, 18, 79, 2475, 29, 7, 2, 2146, 1506, 7804, 655, 4, 88, 7441, 23, 1, 7220, 473, 54, 1, 6, 0, 46, 231, 50, 52, 3, 0, 3358, 35, 811, 8, 52, 1, 0, 85, 0, 9, 8, 64, 1098, 4273, 5, 2090, 211, 1113, 93, 10, 0, 57, 1539, 165, 38] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3218, 0, 0, 0, 0, 0, 0, 0, 90, 0, 88, 1759, 0, 0, 0, 2484, 1496, 8077, 612, 0, 89, 7588, 0, 0, 7326, 188, 61, 0, 0, 0, 0, 213, 0, 59, 0, 0, 3393, 0, 1235, 0, 0, 0, 0, 46, 0, 0, 0, 0, 792, 3754, 0, 3282, 257, 1207, 89, 0, 0, 0, 1476, 231, 42] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2911, 0, 0, 0, 0, 0, 0, 29, 88, 0, 88, 2631, 0, 0, 0, 3316, 1838, 8103, 720, 0, 88, 7586, 0, 0, 7451, 469, 62, 0, 0, 0, 0, 101, 0, 60, 0, 0, 2862, 0, 562, 0, 0, 0, 0, 88, 0, 0, 0, 0, 1474, 3448, 0, 1331, 168, 992, 88, 0, 0, 0, 2433, 165, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3086, 0, 0, 0, 0, 0, 0, 0, 98, 0, 98, 2031, 0, 0, 0, 2086, 1382, 8090, 669, 0, 98, 7488, 0, 0, 7291, 323, 64, 0, 0, 0, 0, 204, 0, 57, 0, 0, 3463, 0, 1114, 0, 0, 0, 0, 96, 0, 0, 0, 0, 882, 4070, 0, 2924, 253, 1137, 97, 0, 0, 0, 1843, 206, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[1, 0, 41, 3445, 13, 22, 0, 0, 4, 1, 2, 86, 18, 79, 2475, 29, 7, 2, 2146, 1506, 7804, 655, 4, 88, 7441, 23, 1, 7220, 473, 54, 1, 6, 0, 46, 231, 50, 52, 3, 0, 3358, 35, 811, 8, 52, 1, 0, 85, 0, 9, 8, 64, 1098, 4273, 5, 2090, 211, 1113, 93, 10, 0, 57, 1539, 165, 38] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2911, 0, 0, 0, 0, 0, 0, 29, 88, 0, 88, 2631, 0, 0, 0, 3316, 1838, 8103, 720, 0, 88, 7586, 0, 0, 7451, 469, 62, 0, 0, 0, 0, 101, 0, 60, 0, 0, 2862, 0, 562, 0, 0, 0, 0, 88, 0, 0, 0, 0, 1474, 3448, 0, 1331, 168, 992, 88, 0, 0, 0, 2433, 165, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2535, 0, 0, 0, 0, 0, 0, 1, 78, 0, 78, 2454, 0, 0, 0, 2206, 1745, 8113, 664, 0, 78, 7570, 0, 0, 7369, 403, 51, 0, 0, 0, 0, 159, 0, 51, 0, 0, 3684, 0, 933, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1269, 3766, 0, 2340, 168, 1199, 78, 0, 0, 0, 1924, 158, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 8, 81, 0, 80, 2552, 0, 0, 0, 2807, 1715, 8107, 659, 0, 80, 7616, 0, 0, 7503, 461, 52, 0, 0, 0, 0, 109, 0, 52, 0, 0, 3205, 0, 683, 0, 0, 0, 0, 79, 0, 0, 0, 0, 1295, 3469, 0, 1475, 162, 1224, 80, 0, 0, 0, 2248, 152, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 63, 0, 62, 2826, 0, 0, 0, 2434, 1835, 8124, 638, 0, 63, 7648, 0, 0, 7409, 440, 54, 0, 0, 0, 0, 134, 0, 39, 0, 0, 3443, 0, 795, 0, 0, 0, 0, 59, 0, 0, 0, 0, 1152, 3865, 0, 1840, 147, 1184, 61, 0, 0, 0, 1831, 131, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[3, 0, 335, 4246, 136, 214, 1, 0, 35, 22, 0, 79, 238, 65, 2151, 175, 10, 19, 834, 884, 6731, 596, 60, 79, 6536, 194, 17, 6099, 292, 101, 13, 1, 0, 322, 273, 459, 47, 3, 0, 2371, 235, 1057, 47, 429, 30, 0, 68, 3, 93, 69, 183, 1058, 5591, 37, 3780, 215, 980, 104, 71, 12, 424, 567, 370, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 2309, 525, 0, 0, 0, 0, 0, 0, 0, 0, 8, 81, 0, 80, 2552, 0, 2426, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 1715, 1601, 659, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5901, 461, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 52, 0, 0, 3205, 1602, 896, 0, 0, 0, 0, 0, 0, 0, 683, 0, 0, 0, 0, 79, 0, 5190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 2048, 0, 1475, 162, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 80, 0, 0, 0, 2248, 152, 2, 2982, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6030, 6362, 6862, 6414, 5864, 5952, 4980, 6688] avg=6144.0000 max=6862 max_rank=2 min=4980 min_rank=6 imbalance_ratio=1.116862 max_load_reduction=42290 max_load_reduction_pct=0.860392 moved_to_echo=23484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 41, 3445, 13, 22, 0, 0, 2223, 647, 0, 0, 0, 0, 0, 0, 4, 1, 2, 86, 18, 79, 2475, 29, 2370, 1169, 0, 0, 0, 0, 0, 0, 7, 2, 2146, 1506, 1541, 655, 4, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 5678, 473, 54, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 231, 50, 52, 3, 0, 3358, 1542, 1104, 0, 0, 0, 0, 0, 0, 35, 811, 8, 52, 1, 0, 85, 0, 5071, 0, 0, 0, 0, 0, 0, 0, 9, 8, 64, 1098, 2522, 5, 2090, 211, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 93, 10, 0, 57, 1539, 165, 38, 2871, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6392, 6233, 5949, 6236, 6386, 6063, 6007, 5886] avg=6144.0000 max=6392 max_rank=0 min=5886 min_rank=7 imbalance_ratio=1.040365 max_load_reduction=42760 max_load_reduction_pct=0.869954 moved_to_echo=24342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3086, 0, 0, 0, 0, 2304, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 98, 2031, 0, 2386, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 1382, 1598, 669, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5734, 323, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 57, 0, 0, 3463, 1557, 1051, 0, 0, 0, 0, 0, 0, 0, 1114, 0, 0, 0, 0, 96, 0, 5102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 882, 2403, 0, 2924, 253, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 97, 0, 0, 0, 1843, 206, 2, 2976, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6006, 5825, 5833, 6121, 6332, 6312, 6462, 6261] avg=6144.0000 max=6462 max_rank=6 min=5825 min_rank=1 imbalance_ratio=1.051758 max_load_reduction=42690 max_load_reduction_pct=0.868530 moved_to_echo=25367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2535, 0, 0, 0, 0, 2311, 570, 0, 0, 0, 0, 0, 0, 0, 0, 1, 78, 0, 78, 2454, 0, 2411, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 1745, 1602, 664, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5795, 403, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 51, 0, 0, 3684, 1574, 973, 0, 0, 0, 0, 0, 0, 0, 933, 0, 0, 0, 0, 73, 0, 5159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 2223, 0, 2340, 168, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 78, 0, 0, 0, 1924, 158, 5, 2984, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5416, 6238, 6295, 6249, 6441, 6165, 6000, 6348] avg=6144.0000 max=6441 max_rank=4 min=5416 min_rank=0 imbalance_ratio=1.048340 max_load_reduction=42711 max_load_reduction_pct=0.868958 moved_to_echo=24954 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2911, 0, 0, 0, 0, 2308, 522, 0, 0, 0, 0, 0, 0, 0, 0, 29, 88, 0, 88, 2631, 0, 2416, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 3316, 1838, 1600, 720, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5860, 469, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 60, 0, 0, 2862, 1591, 890, 0, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 88, 0, 5170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 2036, 0, 1331, 168, 0, 0, 0, 0, 0, 0, 0, 0, 992, 88, 0, 0, 0, 2433, 165, 0, 2981, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5741, 6466, 7562, 6391, 5504, 5820, 5009, 6659] avg=6144.0000 max=7562 max_rank=2 min=5009 min_rank=6 imbalance_ratio=1.230794 max_load_reduction=41590 max_load_reduction_pct=0.846151 moved_to_echo=22992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2873, 0, 0, 0, 0, 2314, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 62, 2826, 0, 2436, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 2434, 1835, 1604, 638, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5827, 440, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 39, 0, 0, 3443, 1582, 998, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 59, 0, 5212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 2282, 0, 1840, 147, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 61, 0, 0, 0, 1831, 131, 2, 2989, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5772, 6604, 6574, 6321, 6196, 6066, 5421, 6198] avg=6144.0000 max=6604 max_rank=1 min=5421 min_rank=6 imbalance_ratio=1.074870 max_load_reduction=42548 max_load_reduction_pct=0.865641 moved_to_echo=23881 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3086, 0, 0, 0, 0, 2304, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 98, 2031, 0, 2386, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 1382, 1598, 669, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5734, 323, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 57, 0, 0, 3463, 1557, 1051, 0, 0, 0, 0, 0, 0, 0, 1114, 0, 0, 0, 0, 96, 0, 5102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 882, 2403, 0, 2924, 253, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 97, 0, 0, 0, 1843, 206, 2, 2976, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6006, 5825, 5833, 6121, 6332, 6312, 6462, 6261] avg=6144.0000 max=6462 max_rank=6 min=5825 min_rank=1 imbalance_ratio=1.051758 max_load_reduction=42690 max_load_reduction_pct=0.868530 moved_to_echo=25367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 41, 3445, 13, 22, 0, 0, 2223, 647, 0, 0, 0, 0, 0, 0, 4, 1, 2, 86, 18, 79, 2475, 29, 2370, 1169, 0, 0, 0, 0, 0, 0, 7, 2, 2146, 1506, 1541, 655, 4, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 5678, 473, 54, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 231, 50, 52, 3, 0, 3358, 1542, 1104, 0, 0, 0, 0, 0, 0, 35, 811, 8, 52, 1, 0, 85, 0, 5071, 0, 0, 0, 0, 0, 0, 0, 9, 8, 64, 1098, 2522, 5, 2090, 211, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 93, 10, 0, 57, 1539, 165, 38, 2871, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6392, 6233, 5949, 6236, 6386, 6063, 6007, 5886] avg=6144.0000 max=6392 max_rank=0 min=5886 min_rank=7 imbalance_ratio=1.040365 max_load_reduction=42760 max_load_reduction_pct=0.869954 moved_to_echo=24342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 2309, 525, 0, 0, 0, 0, 0, 0, 0, 0, 8, 81, 0, 80, 2552, 0, 2426, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 1715, 1601, 659, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5901, 461, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 52, 0, 0, 3205, 1602, 896, 0, 0, 0, 0, 0, 0, 0, 683, 0, 0, 0, 0, 79, 0, 5190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 2048, 0, 1475, 162, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 80, 0, 0, 0, 2248, 152, 2, 2982, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6030, 6362, 6862, 6414, 5864, 5952, 4980, 6688] avg=6144.0000 max=6862 max_rank=2 min=4980 min_rank=6 imbalance_ratio=1.116862 max_load_reduction=42290 max_load_reduction_pct=0.860392 moved_to_echo=23484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3218, 0, 0, 0, 0, 2304, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 88, 1759, 0, 2418, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 2484, 1496, 1584, 612, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5756, 188, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 59, 0, 0, 3393, 1570, 973, 0, 0, 0, 0, 0, 0, 0, 1235, 0, 0, 0, 0, 46, 0, 5170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 2207, 0, 3282, 257, 0, 0, 0, 0, 0, 0, 0, 0, 1207, 89, 0, 0, 0, 1476, 231, 42, 2975, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6096, 5569, 6265, 6005, 6208, 6451, 6538, 6020] avg=6144.0000 max=6538 max_rank=6 min=5569 min_rank=1 imbalance_ratio=1.064128 max_load_reduction=42614 max_load_reduction_pct=0.866984 moved_to_echo=25217 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 335, 4246, 136, 214, 1, 0, 1917, 847, 0, 0, 0, 0, 0, 0, 35, 22, 0, 79, 238, 65, 2151, 175, 2082, 1008, 0, 0, 0, 0, 0, 0, 10, 19, 834, 884, 1330, 596, 60, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 17, 4797, 292, 101, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 273, 459, 47, 3, 0, 2371, 1302, 1444, 0, 0, 0, 0, 0, 0, 235, 1057, 47, 429, 30, 0, 68, 3, 4454, 0, 0, 0, 0, 0, 0, 0, 93, 69, 183, 1058, 3300, 37, 3780, 215, 0, 0, 0, 0, 0, 0, 0, 0, 980, 104, 71, 12, 424, 567, 370, 88, 2476, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7699, 5855, 3812, 5415, 6221, 6323, 8735, 5092] avg=6144.0000 max=8735 max_rank=6 min=3812 min_rank=2 imbalance_ratio=1.421712 max_load_reduction=40417 max_load_reduction_pct=0.822286 moved_to_echo=26371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2873, 0, 0, 0, 0, 2314, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 62, 2826, 0, 2436, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 2434, 1835, 1604, 638, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5827, 440, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 39, 0, 0, 3443, 1582, 998, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 59, 0, 5212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 2282, 0, 1840, 147, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 61, 0, 0, 0, 1831, 131, 2, 2989, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5772, 6604, 6574, 6321, 6196, 6066, 5421, 6198] avg=6144.0000 max=6604 max_rank=1 min=5421 min_rank=6 imbalance_ratio=1.074870 max_load_reduction=42548 max_load_reduction_pct=0.865641 moved_to_echo=23881 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 384, 25600, 256, 256, 128, 0, 18048, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9088, 768, 128, 128, 512, 13952, 1664, 256, 23296, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 256, 9088, 19072, 128, 19072, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 18432, 12416, 12544, 5248, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 768, 256, 640, 18944, 256, 19072, 9472, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 384, 1536, 512, 512, 128, 0, 25856, 12416, 8448, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9088, 768, 128, 128, 512, 13952, 1664, 256, 23296, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 18432, 12416, 12544, 5248, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 7296, 128, 512, 128, 0, 640, 128, 40576, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 256, 128, 45440, 3072, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 4946, 670, 1299, 7448, 6849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 667, 0, 0, 0, 0, 640, 1244, 97, 0, 0, 0, 0, 0, 178, 0, 0, 5906, 5843, 149, 0, 0, 813, 0, 0, 0, 0, 0, 0, 1283, 3, 0, 0, 0, 82, 4393, 97, 6361, 0, 0, 97, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[1, 4136, 637, 1120, 6450, 5983, 76, 0, 22, 34, 73, 110, 125, 37, 180, 155, 469, 169, 94, 13, 265, 1353, 1, 0, 8, 0, 878, 2535, 75, 0, 238, 21, 12, 23, 440, 22, 17, 5114, 3876, 126, 60, 10, 649, 42, 270, 106, 14, 423, 158, 1140, 322, 0, 59, 511, 75, 3673, 94, 5993, 39, 332, 72, 25, 19, 178] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 5482, 691, 1437, 7459, 7046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 834, 0, 0, 0, 0, 660, 684, 61, 0, 0, 0, 0, 0, 96, 0, 0, 6316, 4589, 116, 0, 0, 856, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 62, 5424, 129, 5833, 0, 0, 61, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 5086, 745, 1209, 6973, 6844, 13, 0, 5, 3, 7, 23, 6, 4, 29, 16, 59, 10, 83, 1, 35, 1002, 0, 0, 1, 0, 1012, 1071, 97, 0, 29, 8, 6, 6, 182, 1, 1, 6076, 4768, 113, 7, 0, 906, 1, 25, 19, 31, 36, 39, 1296, 20, 0, 7, 64, 63, 5350, 127, 5468, 2, 43, 87, 3, 6, 28] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 5923, 787, 1073, 7176, 7150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 768, 0, 0, 0, 0, 943, 538, 88, 0, 0, 0, 0, 0, 84, 0, 0, 6588, 3567, 139, 0, 8, 879, 0, 0, 0, 0, 0, 0, 1283, 0, 0, 0, 0, 72, 6415, 187, 5323, 0, 0, 88, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 5616, 763, 1238, 7205, 7106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 838, 0, 0, 0, 0, 840, 679, 80, 0, 0, 0, 0, 0, 65, 0, 0, 6341, 3828, 114, 0, 7, 915, 0, 0, 0, 0, 0, 0, 1275, 0, 0, 0, 0, 62, 6178, 141, 5718, 0, 0, 80, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 5106, 689, 1452, 7441, 6915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 858, 0, 0, 0, 0, 638, 760, 78, 0, 0, 0, 0, 0, 120, 0, 0, 6116, 5275, 121, 0, 0, 795, 0, 0, 0, 0, 0, 0, 1298, 0, 0, 0, 0, 70, 4944, 93, 6161, 0, 0, 78, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 5086, 745, 1209, 6973, 6844, 13, 0, 5, 3, 7, 23, 6, 4, 29, 16, 59, 10, 83, 1, 35, 1002, 0, 0, 1, 0, 1012, 1071, 97, 0, 29, 8, 6, 6, 182, 1, 1, 6076, 4768, 113, 7, 0, 906, 1, 25, 19, 31, 36, 39, 1296, 20, 0, 7, 64, 63, 5350, 127, 5468, 2, 43, 87, 3, 6, 28] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 4759, 664, 1350, 7416, 6852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 847, 0, 0, 0, 0, 672, 1430, 88, 0, 0, 0, 0, 0, 131, 0, 0, 5715, 6251, 130, 0, 0, 795, 0, 0, 0, 0, 0, 0, 1369, 10, 0, 0, 0, 69, 4056, 89, 6300, 0, 0, 89, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4005, 670, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 667, 0, 0, 2816, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 640, 1244, 97, 0, 0, 0, 2653, 941, 348, 38, 0, 0, 0, 0, 0, 0, 178, 0, 0, 1075, 5843, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 0, 0, 0, 0, 4033, 1196, 0, 0, 0, 0, 0, 0, 0, 1283, 3, 0, 0, 0, 82, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 97, 6323, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5974, 6252, 5748, 5961, 7245, 6042, 5413, 6517] avg=6144.0000 max=7245 max_rank=4 min=5413 min_rank=6 imbalance_ratio=1.179199 max_load_reduction=41907 max_load_reduction_pct=0.852600 moved_to_echo=25217 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4118, 745, 1209, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 7, 23, 6, 4, 29, 16, 5853, 0, 0, 0, 0, 0, 0, 0, 59, 10, 83, 1, 35, 1002, 0, 0, 2813, 2240, 0, 0, 0, 0, 0, 0, 1, 0, 1012, 1071, 97, 0, 29, 8, 2729, 968, 424, 33, 0, 0, 0, 0, 6, 6, 182, 1, 1, 1107, 4768, 113, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 906, 1, 25, 19, 31, 36, 4031, 1120, 0, 0, 0, 0, 0, 0, 39, 1296, 20, 0, 7, 64, 63, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 127, 5435, 2, 43, 87, 3, 6, 28, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 5946, 6243, 6372, 6184, 6176, 6415, 5731] avg=6144.0000 max=6415 max_rank=6 min=5731 min_rank=7 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=24506 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 5106, 689, 1452, 7441, 6915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 858, 0, 0, 0, 0, 638, 760, 78, 0, 0, 0, 0, 0, 120, 0, 0, 6116, 5275, 121, 0, 0, 795, 0, 0, 0, 0, 0, 0, 1298, 0, 0, 0, 0, 70, 4944, 93, 6161, 0, 0, 78, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[1, 4136, 637, 1120, 6450, 5983, 76, 0, 22, 34, 73, 110, 125, 37, 180, 155, 469, 169, 94, 13, 265, 1353, 1, 0, 8, 0, 878, 2535, 75, 0, 238, 21, 12, 23, 440, 22, 17, 5114, 3876, 126, 60, 10, 649, 42, 270, 106, 14, 423, 158, 1140, 322, 0, 59, 511, 75, 3673, 94, 5993, 39, 332, 72, 25, 19, 178] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 5482, 691, 1437, 7459, 7046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 834, 0, 0, 0, 0, 660, 684, 61, 0, 0, 0, 0, 0, 96, 0, 0, 6316, 4589, 116, 0, 0, 856, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 62, 5424, 129, 5833, 0, 0, 61, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4796, 787, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 768, 0, 0, 2939, 2429, 0, 0, 0, 0, 0, 0, 0, 0, 943, 538, 88, 0, 0, 0, 2959, 1127, 509, 32, 0, 0, 0, 0, 0, 0, 84, 0, 0, 1200, 3567, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 879, 0, 0, 0, 0, 0, 4211, 1153, 0, 0, 0, 0, 0, 0, 0, 1283, 0, 0, 0, 0, 72, 5906, 0, 0, 0, 0, 0, 0, 0, 0, 187, 5291, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6656, 6023, 6209, 6196, 4990, 6251, 7261, 5566] avg=6144.0000 max=7261 max_rank=6 min=4990 min_rank=4 imbalance_ratio=1.181803 max_load_reduction=41891 max_load_reduction_pct=0.852275 moved_to_echo=24068 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4005, 670, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 667, 0, 0, 2816, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 640, 1244, 97, 0, 0, 0, 2653, 941, 348, 38, 0, 0, 0, 0, 0, 0, 178, 0, 0, 1075, 5843, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 0, 0, 0, 0, 4033, 1196, 0, 0, 0, 0, 0, 0, 0, 1283, 3, 0, 0, 0, 82, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 97, 6323, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5974, 6252, 5748, 5961, 7245, 6042, 5413, 6517] avg=6144.0000 max=7245 max_rank=4 min=5413 min_rank=6 imbalance_ratio=1.179199 max_load_reduction=41907 max_load_reduction_pct=0.852600 moved_to_echo=25217 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4547, 763, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 838, 0, 0, 2921, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 840, 679, 80, 0, 0, 0, 2848, 1069, 490, 35, 0, 0, 0, 0, 0, 0, 65, 0, 0, 1155, 3828, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 915, 0, 0, 0, 0, 0, 4185, 1158, 0, 0, 0, 0, 0, 0, 0, 1275, 0, 0, 0, 0, 62, 5688, 0, 0, 0, 0, 0, 0, 0, 0, 141, 5683, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6548, 6047, 6160, 6041, 5162, 6265, 7025, 5904] avg=6144.0000 max=7025 max_rank=6 min=5162 min_rank=4 imbalance_ratio=1.143392 max_load_reduction=42127 max_load_reduction_pct=0.857076 moved_to_echo=24356 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4439, 691, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 834, 0, 0, 2897, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 660, 684, 61, 0, 0, 0, 2837, 1043, 430, 35, 0, 0, 0, 0, 0, 0, 96, 0, 0, 1150, 4589, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 856, 0, 0, 0, 0, 0, 4149, 1198, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 62, 4994, 0, 0, 0, 0, 0, 0, 0, 0, 129, 5798, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6567, 6261, 6138, 5750, 5951, 6203, 6294, 5988] avg=6144.0000 max=6567 max_rank=0 min=5750 min_rank=3 imbalance_ratio=1.068848 max_load_reduction=42585 max_load_reduction_pct=0.866394 moved_to_echo=24436 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4134, 689, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 858, 0, 0, 2842, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 638, 760, 78, 0, 0, 0, 2747, 972, 392, 37, 0, 0, 0, 0, 0, 0, 120, 0, 0, 1114, 5275, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 4073, 1195, 0, 0, 0, 0, 0, 0, 0, 1298, 0, 0, 0, 0, 70, 4552, 0, 0, 0, 0, 0, 0, 0, 0, 93, 6124, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6275, 6246, 6099, 5624, 6630, 6063, 5920, 6295] avg=6144.0000 max=6630 max_rank=4 min=5624 min_rank=3 imbalance_ratio=1.079102 max_load_reduction=42522 max_load_reduction_pct=0.865112 moved_to_echo=24908 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3850, 664, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 847, 0, 0, 2817, 2111, 0, 0, 0, 0, 0, 0, 0, 0, 672, 1430, 88, 0, 0, 0, 2573, 909, 327, 43, 0, 0, 0, 0, 0, 0, 131, 0, 0, 1031, 6251, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 4035, 1190, 0, 0, 0, 0, 0, 0, 0, 1369, 10, 0, 0, 0, 69, 3729, 0, 0, 0, 0, 0, 0, 0, 0, 89, 6257, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5864, 6226, 5845, 6042, 7543, 6020, 5177, 6435] avg=6144.0000 max=7543 max_rank=4 min=5177 min_rank=6 imbalance_ratio=1.227702 max_load_reduction=41609 max_load_reduction_pct=0.846537 moved_to_echo=25175 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4118, 745, 1209, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 7, 23, 6, 4, 29, 16, 5853, 0, 0, 0, 0, 0, 0, 0, 59, 10, 83, 1, 35, 1002, 0, 0, 2813, 2240, 0, 0, 0, 0, 0, 0, 1, 0, 1012, 1071, 97, 0, 29, 8, 2729, 968, 424, 33, 0, 0, 0, 0, 6, 6, 182, 1, 1, 1107, 4768, 113, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 906, 1, 25, 19, 31, 36, 4031, 1120, 0, 0, 0, 0, 0, 0, 39, 1296, 20, 0, 7, 64, 63, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 127, 5435, 2, 43, 87, 3, 6, 28, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 5946, 6243, 6372, 6184, 6176, 6415, 5731] avg=6144.0000 max=6415 max_rank=6 min=5731 min_rank=7 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=24506 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1, 3349, 637, 1120, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 34, 73, 110, 125, 37, 180, 155, 5415, 0, 0, 0, 0, 0, 0, 0, 469, 169, 94, 13, 265, 1353, 1, 0, 2460, 1886, 0, 0, 0, 0, 0, 0, 8, 0, 878, 2535, 75, 0, 238, 21, 2297, 787, 291, 36, 0, 0, 0, 0, 12, 23, 440, 22, 17, 931, 3876, 126, 0, 0, 0, 0, 0, 0, 0, 0, 60, 10, 649, 42, 270, 106, 14, 423, 3523, 1035, 0, 0, 0, 0, 0, 0, 158, 1140, 322, 0, 59, 511, 75, 3382, 0, 0, 0, 0, 0, 0, 0, 0, 94, 5957, 39, 332, 72, 25, 19, 178, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5183, 6151, 6710, 7166, 5447, 6132, 5647, 6716] avg=6144.0000 max=7166 max_rank=3 min=5183 min_rank=0 imbalance_ratio=1.166341 max_load_reduction=41986 max_load_reduction_pct=0.854207 moved_to_echo=23942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3850, 664, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 847, 0, 0, 2817, 2111, 0, 0, 0, 0, 0, 0, 0, 0, 672, 1430, 88, 0, 0, 0, 2573, 909, 327, 43, 0, 0, 0, 0, 0, 0, 131, 0, 0, 1031, 6251, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 4035, 1190, 0, 0, 0, 0, 0, 0, 0, 1369, 10, 0, 0, 0, 69, 3729, 0, 0, 0, 0, 0, 0, 0, 0, 89, 6257, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5864, 6226, 5845, 6042, 7543, 6020, 5177, 6435] avg=6144.0000 max=7543 max_rank=4 min=5177 min_rank=6 imbalance_ratio=1.227702 max_load_reduction=41609 max_load_reduction_pct=0.846537 moved_to_echo=25175 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4439, 691, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 834, 0, 0, 2897, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 660, 684, 61, 0, 0, 0, 2837, 1043, 430, 35, 0, 0, 0, 0, 0, 0, 96, 0, 0, 1150, 4589, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 856, 0, 0, 0, 0, 0, 4149, 1198, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 62, 4994, 0, 0, 0, 0, 0, 0, 0, 0, 129, 5798, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6567, 6261, 6138, 5750, 5951, 6203, 6294, 5988] avg=6144.0000 max=6567 max_rank=0 min=5750 min_rank=3 imbalance_ratio=1.068848 max_load_reduction=42585 max_load_reduction_pct=0.866394 moved_to_echo=24436 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 256, 768, 128, 384, 7168, 128, 0, 22528, 17792, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 6656, 128, 384, 128, 128, 512, 32256, 9344, 0, 0, 0, 0, 0, 0] @@ -11562,76 +11562,76 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 256, 256, 128, 256, 256, 48384, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[5850, 0, 0, 0, 0, 0, 0, 2672, 559, 0, 160, 0, 0, 0, 85, 5291, 0, 0, 0, 3012, 278, 409, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 1467, 0, 0, 0, 0, 0, 0, 4582, 0, 0, 0, 7429, 0, 0, 268, 0, 0, 0, 0, 3643, 0, 7829, 0, 0, 0, 0, 0, 4946, 148, 0, 0, 78] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[4037, 0, 0, 0, 0, 0, 0, 3101, 484, 0, 119, 0, 0, 0, 83, 5556, 0, 0, 0, 5345, 246, 579, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 4696, 0, 0, 0, 7523, 0, 0, 288, 0, 0, 0, 0, 2517, 0, 7837, 0, 0, 0, 0, 0, 4765, 159, 0, 0, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[5662, 0, 0, 0, 0, 0, 0, 2743, 556, 0, 163, 0, 0, 0, 64, 5368, 0, 0, 0, 3744, 248, 406, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 1296, 0, 0, 0, 0, 0, 0, 4356, 0, 0, 0, 7436, 0, 0, 275, 0, 0, 0, 0, 3193, 0, 7871, 0, 0, 0, 0, 0, 5166, 120, 0, 0, 63] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[6301, 0, 0, 0, 0, 0, 0, 2500, 525, 0, 223, 0, 0, 0, 130, 5079, 0, 0, 0, 2372, 270, 402, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 0, 0, 0, 5110, 0, 0, 0, 7365, 0, 0, 248, 0, 0, 0, 0, 3773, 0, 7656, 0, 0, 0, 0, 0, 5211, 177, 0, 0, 98] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[5018, 0, 0, 0, 0, 0, 0, 2785, 550, 0, 133, 0, 0, 0, 80, 5501, 0, 0, 0, 4734, 285, 488, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 0, 0, 0, 0, 0, 0, 4386, 0, 0, 0, 7465, 0, 0, 279, 0, 0, 0, 0, 2468, 0, 7860, 0, 0, 0, 0, 0, 4953, 142, 0, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[4037, 0, 0, 0, 0, 0, 0, 3101, 484, 0, 119, 0, 0, 0, 83, 5556, 0, 0, 0, 5345, 246, 579, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 4696, 0, 0, 0, 7523, 0, 0, 288, 0, 0, 0, 0, 2517, 0, 7837, 0, 0, 0, 0, 0, 4765, 159, 0, 0, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[5552, 65, 7, 6, 1, 1, 4, 2289, 539, 66, 311, 1, 8, 0, 107, 5126, 0, 15, 6, 3615, 266, 438, 0, 588, 0, 6, 0, 1, 2, 3, 0, 3, 1280, 56, 9, 0, 0, 0, 0, 4115, 0, 61, 7, 7163, 45, 56, 245, 2, 7, 24, 1, 3451, 16, 7571, 0, 36, 26, 15, 3, 5678, 149, 13, 10, 87] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[5850, 0, 0, 0, 0, 0, 0, 2672, 559, 0, 160, 0, 0, 0, 85, 5291, 0, 0, 0, 3012, 278, 409, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 1467, 0, 0, 0, 0, 0, 0, 4582, 0, 0, 0, 7429, 0, 0, 268, 0, 0, 0, 0, 3643, 0, 7829, 0, 0, 0, 0, 0, 4946, 148, 0, 0, 78] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[6301, 0, 0, 0, 0, 0, 0, 2500, 525, 0, 223, 0, 0, 0, 130, 5079, 0, 0, 0, 2372, 270, 402, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 0, 0, 0, 5110, 0, 0, 0, 7365, 0, 0, 248, 0, 0, 0, 0, 3773, 0, 7656, 0, 0, 0, 0, 0, 5211, 177, 0, 0, 98] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[6612, 0, 0, 0, 0, 0, 0, 2358, 500, 0, 231, 0, 0, 0, 92, 5157, 0, 0, 0, 1858, 259, 432, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 5194, 0, 0, 0, 7389, 0, 0, 292, 0, 0, 0, 0, 3558, 0, 7581, 0, 0, 0, 0, 0, 5892, 152, 0, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[6580, 440, 14, 8, 8, 19, 51, 1396, 436, 498, 1844, 38, 61, 2, 152, 4717, 4, 136, 46, 1369, 257, 320, 23, 685, 0, 23, 2, 9, 50, 56, 23, 49, 2272, 426, 71, 0, 5, 0, 1, 3662, 3, 500, 4, 5704, 235, 446, 286, 17, 11, 20, 4, 2624, 154, 6281, 0, 300, 5, 161, 7, 6346, 154, 62, 6, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[6612, 0, 0, 0, 0, 0, 0, 2358, 500, 0, 231, 0, 0, 0, 92, 5157, 0, 0, 0, 1858, 259, 432, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 5194, 0, 0, 0, 7389, 0, 0, 292, 0, 0, 0, 0, 3558, 0, 7581, 0, 0, 0, 0, 0, 5892, 152, 0, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2540, 0, 0, 0, 0, 0, 0, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 119, 0, 0, 0, 83, 5365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5345, 246, 579, 0, 75, 1113, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4811, 1043, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 4696, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2517, 0, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4765, 159, 0, 0, 88, 441, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5641, 6051, 7812, 5854, 6454, 6257, 5543, 5540] avg=6144.0000 max=7812 max_rank=2 min=5540 min_rank=7 imbalance_ratio=1.271484 max_load_reduction=41340 max_load_reduction_pct=0.841064 moved_to_echo=23794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3680, 0, 0, 0, 0, 0, 0, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 160, 0, 0, 0, 85, 5109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 278, 409, 0, 446, 1099, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4806, 1512, 0, 0, 0, 0, 0, 0, 1467, 0, 0, 0, 0, 0, 0, 4582, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5895, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3643, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4946, 148, 0, 0, 78, 435, 83, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 5913, 5902, 6318, 6148, 6163, 6666, 5690] avg=6144.0000 max=6666 max_rank=6 min=5690 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=42486 max_load_reduction_pct=0.864380 moved_to_echo=24667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3964, 0, 0, 0, 0, 0, 0, 2500, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 223, 0, 0, 0, 130, 4904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2372, 270, 402, 0, 622, 1090, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4700, 1628, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 0, 0, 0, 5110, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5843, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 0, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5211, 177, 0, 0, 98, 432, 80, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6464, 5782, 5465, 6328, 6295, 6091, 6729, 5998] avg=6144.0000 max=6729 max_rank=6 min=5465 min_rank=2 imbalance_ratio=1.095215 max_load_reduction=42423 max_load_reduction_pct=0.863098 moved_to_echo=25113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3156, 0, 0, 0, 0, 0, 0, 2785, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 133, 0, 0, 0, 80, 5312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4734, 285, 488, 0, 186, 1104, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4825, 1297, 0, 0, 0, 0, 0, 0, 1759, 0, 0, 0, 0, 0, 0, 4386, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2468, 0, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4953, 142, 0, 0, 80, 438, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5941, 6075, 7362, 6122, 6248, 6202, 5503, 5699] avg=6144.0000 max=7362 max_rank=2 min=5503 min_rank=6 imbalance_ratio=1.198242 max_load_reduction=41790 max_load_reduction_pct=0.850220 moved_to_echo=23652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3492, 65, 7, 6, 1, 1, 4, 2289, 0, 0, 0, 0, 0, 0, 0, 0, 539, 66, 311, 1, 8, 0, 107, 4950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 6, 3615, 266, 438, 0, 588, 1060, 625, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 2, 3, 0, 3, 4648, 1435, 0, 0, 0, 0, 0, 0, 1280, 56, 9, 0, 0, 0, 0, 4115, 96, 0, 0, 0, 0, 0, 0, 0, 0, 61, 7, 5683, 45, 56, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 24, 1, 3451, 16, 2923, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 26, 15, 3, 5678, 149, 13, 10, 87, 420, 80, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5865, 5982, 6613, 6098, 5556, 6099, 6458, 6481] avg=6144.0000 max=6613 max_rank=2 min=5556 min_rank=4 imbalance_ratio=1.076335 max_load_reduction=42539 max_load_reduction_pct=0.865458 moved_to_echo=24594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3562, 0, 0, 0, 0, 0, 0, 2743, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 163, 0, 0, 0, 64, 5184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 248, 406, 0, 422, 1100, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4832, 1463, 0, 0, 0, 0, 0, 0, 1296, 0, 0, 0, 0, 0, 0, 4356, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5900, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3193, 0, 3039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5166, 120, 0, 0, 63, 436, 84, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6305, 5967, 6557, 6295, 5752, 6175, 6232, 5869] avg=6144.0000 max=6557 max_rank=2 min=5752 min_rank=4 imbalance_ratio=1.067220 max_load_reduction=42595 max_load_reduction_pct=0.866597 moved_to_echo=24028 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2540, 0, 0, 0, 0, 0, 0, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 119, 0, 0, 0, 83, 5365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5345, 246, 579, 0, 75, 1113, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4811, 1043, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 4696, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2517, 0, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4765, 159, 0, 0, 88, 441, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5641, 6051, 7812, 5854, 6454, 6257, 5543, 5540] avg=6144.0000 max=7812 max_rank=2 min=5540 min_rank=7 imbalance_ratio=1.271484 max_load_reduction=41340 max_load_reduction_pct=0.841064 moved_to_echo=23794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3964, 0, 0, 0, 0, 0, 0, 2500, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 223, 0, 0, 0, 130, 4904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2372, 270, 402, 0, 622, 1090, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4700, 1628, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 0, 0, 0, 5110, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5843, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 0, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5211, 177, 0, 0, 98, 432, 80, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6464, 5782, 5465, 6328, 6295, 6091, 6729, 5998] avg=6144.0000 max=6729 max_rank=6 min=5465 min_rank=2 imbalance_ratio=1.095215 max_load_reduction=42423 max_load_reduction_pct=0.863098 moved_to_echo=25113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3680, 0, 0, 0, 0, 0, 0, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 160, 0, 0, 0, 85, 5109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 278, 409, 0, 446, 1099, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4806, 1512, 0, 0, 0, 0, 0, 0, 1467, 0, 0, 0, 0, 0, 0, 4582, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5895, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3643, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4946, 148, 0, 0, 78, 435, 83, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 5913, 5902, 6318, 6148, 6163, 6666, 5690] avg=6144.0000 max=6666 max_rank=6 min=5690 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=42486 max_load_reduction_pct=0.864380 moved_to_echo=24667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4151, 0, 0, 0, 0, 0, 0, 2358, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 231, 0, 0, 0, 92, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 259, 432, 0, 623, 1096, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4656, 1713, 0, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 5194, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5856, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5892, 152, 0, 0, 90, 437, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6509, 5796, 5016, 6369, 6174, 6148, 6483, 6657] avg=6144.0000 max=6657 max_rank=7 min=5016 min_rank=2 imbalance_ratio=1.083496 max_load_reduction=42495 max_load_reduction_pct=0.864563 moved_to_echo=25462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4139, 440, 14, 8, 8, 19, 51, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 436, 498, 1844, 38, 61, 2, 152, 4555, 0, 0, 0, 0, 0, 0, 0, 0, 4, 136, 46, 1369, 257, 320, 23, 685, 844, 741, 0, 0, 0, 0, 0, 0, 0, 23, 2, 9, 50, 56, 23, 49, 3856, 1700, 0, 0, 0, 0, 0, 0, 2272, 426, 71, 0, 5, 0, 1, 3662, 88, 0, 0, 0, 0, 0, 0, 0, 3, 500, 4, 4526, 235, 446, 286, 17, 0, 0, 0, 0, 0, 0, 0, 0, 11, 20, 4, 2624, 154, 2425, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 5, 161, 7, 6346, 154, 62, 6, 69, 334, 74, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 7586, 4425, 5768, 6525, 6017, 5538, 7218] avg=6144.0000 max=7586 max_rank=1 min=4425 min_rank=2 imbalance_ratio=1.234701 max_load_reduction=41566 max_load_reduction_pct=0.845662 moved_to_echo=25298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4151, 0, 0, 0, 0, 0, 0, 2358, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 231, 0, 0, 0, 92, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 259, 432, 0, 623, 1096, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4656, 1713, 0, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 5194, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5856, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5892, 152, 0, 0, 90, 437, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6509, 5796, 5016, 6369, 6174, 6148, 6483, 6657] avg=6144.0000 max=6657 max_rank=7 min=5016 min_rank=2 imbalance_ratio=1.083496 max_load_reduction=42495 max_load_reduction_pct=0.864563 moved_to_echo=25462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 37248, 11904, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 25344, 256, 23424, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[28800, 512, 128, 128, 128, 128, 128, 19968, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 256, 128, 26112, 2176, 3584, 128, 3712, 8576, 5248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 25344, 256, 23424, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 640, 3200, 128, 128, 128, 896, 40448, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11776, 512, 128, 0, 128, 0, 128, 36224, 896, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 256, 128, 43008, 1280, 128, 128, 768, 3456, 768, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 640, 128, 45696, 384, 512, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 256, 128, 43008, 1280, 128, 128, 768, 3456, 768, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 3771, 0, 0, 0, 0, 0, 0, 231, 0, 0, 7799, 0, 1768, 0, 75, 334, 64, 0, 0, 0, 362, 0, 954, 0, 0, 0, 0, 0, 0, 6963, 5010, 0, 265, 0, 0, 0, 201, 0, 1937, 2761, 0, 0, 0, 6695, 7895, 25, 0, 0, 0, 0, 231, 0, 98, 0, 0, 0, 1713, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 3110, 0, 0, 0, 0, 0, 0, 242, 0, 0, 7831, 0, 1761, 0, 12, 356, 54, 0, 0, 0, 328, 0, 581, 0, 0, 0, 0, 0, 0, 7114, 6156, 0, 217, 0, 0, 0, 177, 0, 1375, 3145, 0, 0, 0, 6893, 7884, 72, 0, 1, 0, 0, 242, 0, 63, 0, 0, 0, 1538, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 3207, 0, 0, 0, 0, 0, 0, 244, 0, 0, 7819, 0, 1673, 0, 25, 372, 51, 0, 0, 0, 389, 0, 784, 0, 0, 0, 0, 0, 0, 6986, 5956, 0, 296, 0, 0, 0, 200, 0, 1862, 2667, 0, 0, 0, 6719, 7882, 21, 0, 0, 0, 0, 244, 0, 78, 0, 0, 0, 1677, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[7, 2, 2518, 5, 7, 2, 4, 0, 59, 251, 8, 0, 7639, 2, 2438, 0, 95, 380, 67, 1, 0, 0, 322, 11, 798, 10, 9, 0, 8, 5, 0, 6878, 4779, 15, 229, 0, 59, 1, 197, 0, 2470, 3048, 29, 28, 71, 6568, 7553, 25, 72, 11, 0, 16, 331, 0, 89, 1, 0, 65, 1890, 57, 19, 2, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2186, 0, 0, 0, 0, 0, 0, 257, 0, 0, 7781, 0, 2101, 0, 0, 369, 62, 0, 0, 0, 304, 0, 536, 0, 0, 0, 0, 0, 0, 7099, 7140, 0, 312, 0, 0, 0, 184, 0, 1184, 3780, 0, 0, 0, 6546, 7329, 32, 0, 0, 0, 0, 257, 0, 88, 0, 0, 0, 1605, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2493, 0, 0, 1, 0, 0, 0, 242, 0, 0, 7812, 0, 2224, 0, 0, 355, 52, 0, 0, 0, 350, 0, 536, 0, 0, 0, 0, 1, 0, 7110, 6768, 0, 306, 0, 0, 0, 164, 0, 1248, 3285, 0, 0, 0, 6761, 7541, 40, 0, 0, 0, 0, 242, 0, 80, 0, 0, 0, 1541, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 3207, 0, 0, 0, 0, 0, 0, 244, 0, 0, 7819, 0, 1673, 0, 25, 372, 51, 0, 0, 0, 389, 0, 784, 0, 0, 0, 0, 0, 0, 6986, 5956, 0, 296, 0, 0, 0, 200, 0, 1862, 2667, 0, 0, 0, 6719, 7882, 21, 0, 0, 0, 0, 244, 0, 78, 0, 0, 0, 1677, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[27, 24, 3502, 107, 30, 10, 71, 24, 292, 263, 37, 1, 6242, 9, 1993, 6, 144, 426, 108, 14, 0, 16, 218, 17, 1678, 97, 88, 28, 39, 22, 13, 5555, 2017, 7, 135, 0, 430, 1, 384, 13, 3100, 3983, 22, 252, 525, 5992, 6299, 38, 530, 46, 2, 172, 808, 0, 92, 4, 10, 517, 2056, 481, 93, 21, 12, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 3110, 0, 0, 0, 0, 0, 0, 242, 0, 0, 7831, 0, 1761, 0, 12, 356, 54, 0, 0, 0, 328, 0, 581, 0, 0, 0, 0, 0, 0, 7114, 6156, 0, 217, 0, 0, 0, 177, 0, 1375, 3145, 0, 0, 0, 6893, 7884, 72, 0, 1, 0, 0, 242, 0, 63, 0, 0, 0, 1538, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 3444, 0, 0, 0, 0, 0, 0, 252, 0, 0, 7789, 0, 2382, 0, 78, 344, 61, 0, 0, 0, 354, 0, 1382, 0, 0, 0, 0, 0, 0, 6741, 4192, 0, 314, 0, 0, 0, 213, 0, 2282, 2329, 0, 0, 0, 6773, 7917, 6, 0, 0, 0, 0, 252, 0, 90, 0, 0, 0, 1957, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[27, 24, 3502, 107, 30, 10, 71, 24, 292, 263, 37, 1, 6242, 9, 1993, 6, 144, 426, 108, 14, 0, 16, 218, 17, 1678, 97, 88, 28, 39, 22, 13, 5555, 2017, 7, 135, 0, 430, 1, 384, 13, 3100, 3983, 22, 252, 525, 5992, 6299, 38, 530, 46, 2, 172, 808, 0, 92, 4, 10, 517, 2056, 481, 93, 21, 12, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3207, 0, 0, 0, 0, 0, 1873, 1289, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 3919, 0, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 372, 51, 0, 0, 0, 389, 0, 3463, 1998, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 0, 0, 5336, 0, 0, 0, 0, 0, 0, 0, 0, 5956, 0, 296, 0, 0, 0, 200, 0, 361, 0, 0, 0, 0, 0, 0, 0, 1862, 2667, 0, 0, 0, 954, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 78, 0, 5884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1677, 0, 0, 0, 0, 0, 2302, 2027, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6369, 5836, 6298, 6120, 6813, 5504, 6206, 6006] avg=6144.0000 max=6813 max_rank=4 min=5504 min_rank=5 imbalance_ratio=1.108887 max_load_reduction=42339 max_load_reduction_pct=0.861389 moved_to_echo=24529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3771, 0, 0, 0, 0, 0, 1869, 1285, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 3908, 0, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 334, 64, 0, 0, 0, 362, 0, 3451, 2002, 0, 0, 0, 0, 0, 0, 954, 0, 0, 0, 0, 0, 0, 5319, 0, 0, 0, 0, 0, 0, 0, 0, 5010, 0, 265, 0, 0, 0, 201, 0, 359, 0, 0, 0, 0, 0, 0, 0, 1937, 2761, 0, 0, 0, 950, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 98, 0, 5893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, 0, 0, 0, 0, 2294, 2022, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6925, 5907, 6288, 6273, 5835, 5673, 6222, 6029] avg=6144.0000 max=6925 max_rank=0 min=5673 min_rank=5 imbalance_ratio=1.127116 max_load_reduction=42227 max_load_reduction_pct=0.859111 moved_to_echo=23759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2186, 0, 0, 0, 0, 0, 1864, 1310, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 3900, 0, 2101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 62, 0, 0, 0, 304, 0, 3374, 1858, 0, 0, 0, 0, 0, 0, 536, 0, 0, 0, 0, 0, 0, 5423, 0, 0, 0, 0, 0, 0, 0, 0, 7140, 0, 312, 0, 0, 0, 184, 0, 366, 0, 0, 0, 0, 0, 0, 0, 1184, 3780, 0, 0, 0, 929, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 88, 0, 5471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1605, 0, 0, 0, 0, 0, 2243, 2017, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5360, 6258, 5967, 5959, 8002, 5925, 5816, 5865] avg=6144.0000 max=8002 max_rank=4 min=5360 min_rank=0 imbalance_ratio=1.302409 max_load_reduction=41150 max_load_reduction_pct=0.837199 moved_to_echo=25608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2493, 0, 0, 1, 0, 0, 1872, 1312, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 3915, 0, 2224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 52, 0, 0, 0, 350, 0, 3485, 1912, 0, 0, 0, 0, 0, 0, 536, 0, 0, 0, 0, 1, 0, 5431, 0, 0, 0, 0, 0, 0, 0, 0, 6768, 0, 306, 0, 0, 0, 164, 0, 367, 0, 0, 0, 0, 0, 0, 0, 1248, 3285, 0, 0, 0, 960, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 80, 0, 5629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 0, 0, 0, 0, 0, 2316, 2025, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5678, 6381, 6154, 5968, 7605, 5533, 5951, 5882] avg=6144.0000 max=7605 max_rank=4 min=5533 min_rank=5 imbalance_ratio=1.237793 max_load_reduction=41547 max_load_reduction_pct=0.845276 moved_to_echo=24971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[7, 2, 2518, 5, 7, 2, 4, 0, 1830, 1269, 0, 0, 0, 0, 0, 0, 59, 251, 8, 0, 3828, 2, 2438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 380, 67, 1, 0, 0, 322, 11, 3385, 1914, 0, 0, 0, 0, 0, 0, 798, 10, 9, 0, 8, 5, 0, 5254, 0, 0, 0, 0, 0, 0, 0, 0, 4779, 15, 229, 0, 59, 1, 197, 0, 355, 0, 0, 0, 0, 0, 0, 0, 2470, 3048, 29, 28, 71, 933, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 72, 11, 0, 16, 331, 0, 89, 1, 5639, 0, 0, 0, 0, 0, 0, 0, 0, 65, 1890, 57, 19, 2, 0, 1, 2250, 1981, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5644, 6586, 6175, 6084, 5635, 6604, 6159, 6265] avg=6144.0000 max=6604 max_rank=5 min=5635 min_rank=4 imbalance_ratio=1.074870 max_load_reduction=42548 max_load_reduction_pct=0.865641 moved_to_echo=24663 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3110, 0, 0, 0, 0, 0, 1876, 1313, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 3925, 0, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 356, 54, 0, 0, 0, 328, 0, 3553, 1999, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 5434, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 217, 0, 0, 0, 177, 0, 367, 0, 0, 0, 0, 0, 0, 0, 1375, 3145, 0, 0, 0, 979, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 242, 0, 63, 0, 5885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1538, 0, 0, 0, 0, 0, 2361, 2030, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6299, 5928, 6302, 6015, 6917, 5571, 6191, 5929] avg=6144.0000 max=6917 max_rank=4 min=5571 min_rank=5 imbalance_ratio=1.125814 max_load_reduction=42235 max_load_reduction_pct=0.859273 moved_to_echo=24608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2493, 0, 0, 1, 0, 0, 1872, 1312, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 3915, 0, 2224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 52, 0, 0, 0, 350, 0, 3485, 1912, 0, 0, 0, 0, 0, 0, 536, 0, 0, 0, 0, 1, 0, 5431, 0, 0, 0, 0, 0, 0, 0, 0, 6768, 0, 306, 0, 0, 0, 164, 0, 367, 0, 0, 0, 0, 0, 0, 0, 1248, 3285, 0, 0, 0, 960, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 80, 0, 5629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 0, 0, 0, 0, 0, 2316, 2025, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5678, 6381, 6154, 5968, 7605, 5533, 5951, 5882] avg=6144.0000 max=7605 max_rank=4 min=5533 min_rank=5 imbalance_ratio=1.237793 max_load_reduction=41547 max_load_reduction_pct=0.845276 moved_to_echo=24971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2186, 0, 0, 0, 0, 0, 1864, 1310, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 3900, 0, 2101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 62, 0, 0, 0, 304, 0, 3374, 1858, 0, 0, 0, 0, 0, 0, 536, 0, 0, 0, 0, 0, 0, 5423, 0, 0, 0, 0, 0, 0, 0, 0, 7140, 0, 312, 0, 0, 0, 184, 0, 366, 0, 0, 0, 0, 0, 0, 0, 1184, 3780, 0, 0, 0, 929, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 88, 0, 5471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1605, 0, 0, 0, 0, 0, 2243, 2017, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5360, 6258, 5967, 5959, 8002, 5925, 5816, 5865] avg=6144.0000 max=8002 max_rank=4 min=5360 min_rank=0 imbalance_ratio=1.302409 max_load_reduction=41150 max_load_reduction_pct=0.837199 moved_to_echo=25608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3207, 0, 0, 0, 0, 0, 1873, 1289, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 3919, 0, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 372, 51, 0, 0, 0, 389, 0, 3463, 1998, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 0, 0, 5336, 0, 0, 0, 0, 0, 0, 0, 0, 5956, 0, 296, 0, 0, 0, 200, 0, 361, 0, 0, 0, 0, 0, 0, 0, 1862, 2667, 0, 0, 0, 954, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 78, 0, 5884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1677, 0, 0, 0, 0, 0, 2302, 2027, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6369, 5836, 6298, 6120, 6813, 5504, 6206, 6006] avg=6144.0000 max=6813 max_rank=4 min=5504 min_rank=5 imbalance_ratio=1.108887 max_load_reduction=42339 max_load_reduction_pct=0.861389 moved_to_echo=24529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3444, 0, 0, 0, 0, 0, 1871, 1246, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 3893, 0, 2382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 344, 61, 0, 0, 0, 354, 0, 3495, 2008, 0, 0, 0, 0, 0, 0, 1382, 0, 0, 0, 0, 0, 0, 5143, 0, 0, 0, 0, 0, 0, 0, 0, 4192, 0, 314, 0, 0, 0, 213, 0, 352, 0, 0, 0, 0, 0, 0, 0, 2282, 2329, 0, 0, 0, 954, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 90, 0, 5909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1957, 0, 0, 0, 0, 0, 2324, 2025, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6561, 6527, 6340, 6525, 5071, 5571, 6251, 6306] avg=6144.0000 max=6561 max_rank=0 min=5071 min_rank=4 imbalance_ratio=1.067871 max_load_reduction=42591 max_load_reduction_pct=0.866516 moved_to_echo=23199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[27, 24, 3502, 107, 30, 10, 71, 24, 1496, 1025, 0, 0, 0, 0, 0, 0, 292, 263, 37, 1, 3128, 9, 1993, 6, 0, 0, 0, 0, 0, 0, 0, 0, 144, 426, 108, 14, 0, 16, 218, 17, 3088, 1597, 0, 0, 0, 0, 0, 0, 1678, 97, 88, 28, 39, 22, 13, 4243, 0, 0, 0, 0, 0, 0, 0, 0, 2017, 7, 135, 0, 430, 1, 384, 13, 287, 0, 0, 0, 0, 0, 0, 0, 3100, 3983, 22, 252, 525, 851, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 530, 46, 2, 172, 808, 0, 92, 4, 4702, 0, 0, 0, 0, 0, 0, 0, 10, 517, 2056, 481, 93, 21, 12, 9, 2053, 1618, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6316, 5729, 5628, 6208, 3274, 8771, 6356, 6870] avg=6144.0000 max=8771 max_rank=5 min=3274 min_rank=4 imbalance_ratio=1.427572 max_load_reduction=40381 max_load_reduction_pct=0.821554 moved_to_echo=25271 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3444, 0, 0, 0, 0, 0, 1871, 1246, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 3893, 0, 2382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 344, 61, 0, 0, 0, 354, 0, 3495, 2008, 0, 0, 0, 0, 0, 0, 1382, 0, 0, 0, 0, 0, 0, 5143, 0, 0, 0, 0, 0, 0, 0, 0, 4192, 0, 314, 0, 0, 0, 213, 0, 352, 0, 0, 0, 0, 0, 0, 0, 2282, 2329, 0, 0, 0, 954, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 90, 0, 5909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1957, 0, 0, 0, 0, 0, 2324, 2025, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6561, 6527, 6340, 6525, 5071, 5571, 6251, 6306] avg=6144.0000 max=6561 max_rank=0 min=5071 min_rank=4 imbalance_ratio=1.067871 max_load_reduction=42591 max_load_reduction_pct=0.866516 moved_to_echo=23199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 24320, 128, 128, 128, 128, 128, 14592, 10112, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[512, 2944, 640, 128, 0, 128, 2688, 128, 27392, 15360, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 2048, 128, 128, 30464, 128, 16384, 128, 0, 0, 0, 0, 0, 0, 0, 0] @@ -11648,631 +11648,631 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 71, 42, 64, 68, 49, 309, 36, 87, 7349, 504, 0, 0, 133, 7, 70, 11, 18, 0, 69, 58, 213, 43, 0, 0, 21, 0, 24, 945, 7371, 35, 0, 4, 69, 0, 2, 561, 6845, 5, 8025, 87, 1, 0, 3925, 409, 6, 188, 9, 16, 0, 4, 1, 0, 2928, 0, 0, 0, 0, 399, 7602, 7, 17, 0, 445] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 59, 0, 0, 0, 0, 286, 0, 97, 7497, 635, 0, 0, 97, 0, 0, 0, 0, 0, 63, 0, 219, 0, 0, 0, 0, 0, 0, 806, 7613, 0, 0, 0, 0, 0, 0, 463, 7115, 0, 8095, 98, 0, 0, 4642, 102, 0, 137, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 329, 7820, 0, 0, 0, 574] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 62, 0, 0, 0, 0, 281, 0, 88, 7536, 519, 0, 0, 88, 0, 56, 0, 0, 0, 62, 0, 169, 0, 0, 0, 0, 0, 0, 784, 7648, 0, 0, 0, 0, 0, 0, 335, 7205, 0, 8104, 88, 0, 0, 4274, 95, 0, 91, 0, 0, 0, 0, 0, 0, 3118, 0, 0, 0, 0, 345, 7855, 0, 0, 0, 349] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 56, 0, 0, 0, 0, 341, 1, 80, 7479, 488, 0, 0, 80, 0, 22, 0, 0, 0, 52, 0, 228, 0, 0, 0, 0, 0, 0, 838, 7641, 0, 0, 0, 0, 0, 0, 415, 7156, 0, 8112, 80, 0, 0, 4115, 81, 0, 80, 0, 0, 0, 0, 0, 0, 3353, 0, 0, 0, 0, 322, 7827, 0, 0, 0, 305] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 54, 0, 0, 0, 0, 292, 0, 61, 7442, 557, 0, 0, 61, 0, 12, 0, 0, 0, 54, 0, 200, 0, 0, 0, 0, 0, 0, 845, 7673, 0, 0, 0, 0, 0, 0, 505, 7129, 0, 8131, 63, 0, 0, 4583, 67, 0, 66, 0, 0, 0, 0, 0, 0, 2760, 0, 0, 0, 0, 305, 7882, 0, 0, 0, 410] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 59, 0, 0, 0, 0, 286, 0, 97, 7497, 635, 0, 0, 97, 0, 0, 0, 0, 0, 63, 0, 219, 0, 0, 0, 0, 0, 0, 806, 7613, 0, 0, 0, 0, 0, 0, 463, 7115, 0, 8095, 98, 0, 0, 4642, 102, 0, 137, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 329, 7820, 0, 0, 0, 574] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 57, 0, 0, 0, 0, 302, 0, 89, 7495, 573, 0, 0, 89, 0, 0, 0, 0, 0, 61, 0, 183, 0, 0, 0, 0, 0, 0, 826, 7591, 0, 0, 0, 0, 0, 0, 444, 7065, 0, 8103, 90, 0, 0, 4401, 124, 0, 150, 0, 0, 0, 0, 0, 0, 2755, 0, 0, 0, 0, 342, 7830, 0, 0, 0, 582] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 51, 0, 0, 0, 0, 309, 0, 78, 7461, 577, 0, 0, 78, 0, 0, 0, 0, 0, 51, 0, 210, 0, 0, 0, 0, 0, 0, 842, 7646, 0, 0, 0, 0, 0, 0, 482, 7111, 0, 8114, 78, 0, 0, 4629, 83, 0, 96, 0, 0, 0, 0, 0, 0, 2551, 0, 0, 0, 0, 322, 7833, 0, 0, 0, 550] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 65, 337, 499, 558, 304, 310, 250, 68, 6672, 486, 0, 1, 395, 31, 96, 38, 79, 0, 125, 448, 328, 292, 0, 2, 88, 11, 22, 1014, 5985, 413, 11, 69, 551, 0, 18, 947, 5274, 66, 7510, 69, 9, 22, 4232, 1336, 117, 818, 34, 130, 4, 10, 2, 1, 752, 1, 0, 12, 0, 577, 6096, 14, 34, 7, 1512] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 71, 42, 64, 68, 49, 309, 36, 2421, 2289, 793, 0, 0, 0, 0, 0, 87, 5344, 504, 0, 0, 133, 7, 70, 0, 0, 0, 0, 0, 0, 0, 0, 11, 18, 0, 69, 58, 213, 43, 0, 5736, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 24, 945, 5187, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 69, 0, 2, 561, 5513, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1, 0, 3925, 409, 6, 188, 9, 1194, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4, 1, 0, 2928, 0, 0, 2005, 1391, 138, 0, 0, 0, 0, 0, 0, 0, 399, 5181, 7, 17, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6145, 6148, 6212, 6154, 5819, 6483, 6049] avg=6144.0000 max=6483 max_rank=6 min=5819 min_rank=5 imbalance_ratio=1.055176 max_load_reduction=42669 max_load_reduction_pct=0.868103 moved_to_echo=24505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 59, 0, 0, 0, 0, 286, 0, 2491, 2310, 819, 0, 0, 0, 0, 0, 97, 5452, 635, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 219, 0, 0, 5785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 5357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 5731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 4642, 102, 0, 137, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 2045, 1437, 143, 0, 0, 0, 0, 0, 0, 0, 329, 5329, 0, 0, 0, 574, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 6281, 6067, 6163, 6194, 6220, 6030, 6232] avg=6144.0000 max=6281 max_rank=1 min=5965 min_rank=0 imbalance_ratio=1.022298 max_load_reduction=42871 max_load_reduction_pct=0.872213 moved_to_echo=24676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 71, 42, 64, 68, 49, 309, 36, 87, 7349, 504, 0, 0, 133, 7, 70, 11, 18, 0, 69, 58, 213, 43, 0, 0, 21, 0, 24, 945, 7371, 35, 0, 4, 69, 0, 2, 561, 6845, 5, 8025, 87, 1, 0, 3925, 409, 6, 188, 9, 16, 0, 4, 1, 0, 2928, 0, 0, 0, 0, 399, 7602, 7, 17, 0, 445] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 51, 0, 0, 0, 0, 309, 0, 78, 7461, 577, 0, 0, 78, 0, 0, 0, 0, 0, 51, 0, 210, 0, 0, 0, 0, 0, 0, 842, 7646, 0, 0, 0, 0, 0, 0, 482, 7111, 0, 8114, 78, 0, 0, 4629, 83, 0, 96, 0, 0, 0, 0, 0, 0, 2551, 0, 0, 0, 0, 322, 7833, 0, 0, 0, 550] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 54, 0, 0, 0, 0, 292, 0, 61, 7442, 557, 0, 0, 61, 0, 12, 0, 0, 0, 54, 0, 200, 0, 0, 0, 0, 0, 0, 845, 7673, 0, 0, 0, 0, 0, 0, 505, 7129, 0, 8131, 63, 0, 0, 4583, 67, 0, 66, 0, 0, 0, 0, 0, 0, 2760, 0, 0, 0, 0, 305, 7882, 0, 0, 0, 410] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 62, 0, 0, 0, 0, 281, 0, 88, 7536, 519, 0, 0, 88, 0, 56, 0, 0, 0, 62, 0, 169, 0, 0, 0, 0, 0, 0, 784, 7648, 0, 0, 0, 0, 0, 0, 335, 7205, 0, 8104, 88, 0, 0, 4274, 95, 0, 91, 0, 0, 0, 0, 0, 0, 3118, 0, 0, 0, 0, 345, 7855, 0, 0, 0, 349] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 56, 0, 0, 0, 0, 341, 1, 2493, 2315, 822, 0, 0, 0, 0, 0, 80, 5439, 488, 0, 0, 80, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 228, 0, 0, 5797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 5377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 5764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 4115, 81, 0, 80, 0, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3353, 0, 0, 2040, 1442, 144, 0, 0, 0, 0, 0, 0, 0, 322, 5334, 0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6028, 6109, 6077, 6215, 6179, 5604, 6979, 5961] avg=6144.0000 max=6979 max_rank=6 min=5604 min_rank=5 imbalance_ratio=1.135905 max_load_reduction=42173 max_load_reduction_pct=0.858012 moved_to_echo=24723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 0, 0, 0, 0, 281, 0, 2502, 2312, 823, 0, 0, 0, 0, 0, 88, 5480, 519, 0, 0, 88, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 169, 0, 0, 5792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 5381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 5804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 4274, 95, 0, 91, 0, 1256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3118, 0, 0, 2056, 1444, 145, 0, 0, 0, 0, 0, 0, 0, 345, 5353, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5980, 6231, 6023, 6165, 6139, 5804, 6763, 6047] avg=6144.0000 max=6763 max_rank=6 min=5804 min_rank=5 imbalance_ratio=1.100749 max_load_reduction=42389 max_load_reduction_pct=0.862406 moved_to_echo=24753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 54, 0, 0, 0, 0, 292, 0, 2511, 2320, 826, 0, 0, 0, 0, 0, 61, 5412, 557, 0, 0, 61, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 200, 0, 0, 5811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 5399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 5743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 4583, 67, 0, 66, 0, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2760, 0, 0, 2030, 1448, 143, 0, 0, 0, 0, 0, 0, 0, 305, 5371, 0, 0, 0, 410, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6003, 6103, 6065, 6244, 6248, 6022, 6381, 6086] avg=6144.0000 max=6381 max_rank=6 min=6003 min_rank=0 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 57, 0, 0, 0, 0, 302, 0, 2497, 2312, 821, 0, 0, 0, 0, 0, 89, 5445, 573, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 183, 0, 0, 5791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 5333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 5685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 4401, 124, 0, 150, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 0, 0, 2050, 1437, 145, 0, 0, 0, 0, 0, 0, 0, 342, 5333, 0, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6196, 6035, 6159, 6129, 6000, 6387, 6257] avg=6144.0000 max=6387 max_rank=6 min=5989 min_rank=0 imbalance_ratio=1.039551 max_load_reduction=42765 max_load_reduction_pct=0.870056 moved_to_echo=24773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 59, 0, 0, 0, 0, 286, 0, 2491, 2310, 819, 0, 0, 0, 0, 0, 97, 5452, 635, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 219, 0, 0, 5785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 5357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 5731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 4642, 102, 0, 137, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 2045, 1437, 143, 0, 0, 0, 0, 0, 0, 0, 329, 5329, 0, 0, 0, 574, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 6281, 6067, 6163, 6194, 6220, 6030, 6232] avg=6144.0000 max=6281 max_rank=1 min=5965 min_rank=0 imbalance_ratio=1.022298 max_load_reduction=42871 max_load_reduction_pct=0.872213 moved_to_echo=24676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 71, 42, 64, 68, 49, 309, 36, 2421, 2289, 793, 0, 0, 0, 0, 0, 87, 5344, 504, 0, 0, 133, 7, 70, 0, 0, 0, 0, 0, 0, 0, 0, 11, 18, 0, 69, 58, 213, 43, 0, 5736, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 24, 945, 5187, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 69, 0, 2, 561, 5513, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1, 0, 3925, 409, 6, 188, 9, 1194, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4, 1, 0, 2928, 0, 0, 2005, 1391, 138, 0, 0, 0, 0, 0, 0, 0, 399, 5181, 7, 17, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6145, 6148, 6212, 6154, 5819, 6483, 6049] avg=6144.0000 max=6483 max_rank=6 min=5819 min_rank=5 imbalance_ratio=1.055176 max_load_reduction=42669 max_load_reduction_pct=0.868103 moved_to_echo=24505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 0, 0, 0, 0, 309, 0, 2495, 2315, 823, 0, 0, 0, 0, 0, 78, 5425, 577, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 210, 0, 0, 5799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 5380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 5728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 4629, 83, 0, 96, 0, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2551, 0, 0, 2036, 1443, 143, 0, 0, 0, 0, 0, 0, 0, 322, 5338, 0, 0, 0, 550, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5993, 6158, 6060, 6222, 6210, 6126, 6173, 6210] avg=6144.0000 max=6222 max_rank=3 min=5993 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=42930 max_load_reduction_pct=0.873413 moved_to_echo=24719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 65, 337, 499, 558, 304, 310, 250, 1942, 2143, 644, 0, 0, 0, 0, 0, 68, 4852, 486, 0, 1, 395, 31, 96, 0, 0, 0, 0, 0, 0, 0, 0, 38, 79, 0, 125, 448, 328, 292, 0, 5367, 0, 0, 0, 0, 0, 0, 0, 2, 88, 11, 22, 1014, 4211, 413, 11, 0, 0, 0, 0, 0, 0, 0, 0, 69, 551, 0, 18, 947, 4248, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 9, 22, 4232, 1336, 117, 818, 34, 920, 0, 0, 0, 0, 0, 0, 0, 130, 4, 10, 2, 1, 752, 1, 0, 1820, 1130, 106, 0, 0, 0, 0, 0, 12, 0, 577, 4154, 14, 34, 7, 1512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7052, 5929, 6677, 5772, 5899, 7557, 3956, 6310] avg=6144.0000 max=7557 max_rank=5 min=3956 min_rank=6 imbalance_ratio=1.229980 max_load_reduction=41595 max_load_reduction_pct=0.846252 moved_to_echo=23722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 0, 0, 0, 0, 281, 0, 2502, 2312, 823, 0, 0, 0, 0, 0, 88, 5480, 519, 0, 0, 88, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 169, 0, 0, 5792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 5381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 5804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 4274, 95, 0, 91, 0, 1256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3118, 0, 0, 2056, 1444, 145, 0, 0, 0, 0, 0, 0, 0, 345, 5353, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5980, 6231, 6023, 6165, 6139, 5804, 6763, 6047] avg=6144.0000 max=6763 max_rank=6 min=5804 min_rank=5 imbalance_ratio=1.100749 max_load_reduction=42389 max_load_reduction_pct=0.862406 moved_to_echo=24753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 57, 0, 0, 0, 0, 302, 0, 2497, 2312, 821, 0, 0, 0, 0, 0, 89, 5445, 573, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 183, 0, 0, 5791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 5333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 5685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 4401, 124, 0, 150, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 0, 0, 2050, 1437, 145, 0, 0, 0, 0, 0, 0, 0, 342, 5333, 0, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6196, 6035, 6159, 6129, 6000, 6387, 6257] avg=6144.0000 max=6387 max_rank=6 min=5989 min_rank=0 imbalance_ratio=1.039551 max_load_reduction=42765 max_load_reduction_pct=0.870056 moved_to_echo=24773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 54, 0, 0, 0, 0, 292, 0, 2511, 2320, 826, 0, 0, 0, 0, 0, 61, 5412, 557, 0, 0, 61, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 200, 0, 0, 5811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 5399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 5743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 4583, 67, 0, 66, 0, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2760, 0, 0, 2030, 1448, 143, 0, 0, 0, 0, 0, 0, 0, 305, 5371, 0, 0, 0, 410, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6003, 6103, 6065, 6244, 6248, 6022, 6381, 6086] avg=6144.0000 max=6381 max_rank=6 min=6003 min_rank=0 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 512, 384, 640, 640, 384, 2432, 384, 19456, 18432, 6400, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 128, 128, 20736, 128, 0, 16128, 11264, 1152, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 128, 128, 34816, 2304, 128, 1664, 128, 9600, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[768, 42880, 4352, 0, 128, 1024, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 6912, 41728, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 2944, 41472, 128, 128, 128, 4736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[768, 42880, 4352, 0, 128, 1024, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 640, 512, 1792, 384, 0, 45952, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 640, 0, 128, 4224, 44288, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 640, 512, 1792, 384, 0, 45952, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True - [2026-08-26 15:39:24] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 803.0 | throughput per GPU (TFLOP/s/GPU): 158.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.563369E+00 | loss scale: 1.0 | grad norm: 14.007 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1423, 288, 765, 765, 227, 1222, 984, 626, 1220, 317, 786, 370, 251, 1704, 915, 950, 694, 976, 759, 673, 337, 525, 300, 1245, 411, 650, 871, 401, 840, 166, 766, 1415, 893, 766, 1114, 1243, 372, 215, 670, 988, 585, 515, 641, 626, 872, 944, 470, 448, 826, 711, 719, 708, 1002, 1265, 691, 865, 1487, 313, 1453, 733, 206, 792, 1300, 877] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1410, 339, 792, 859, 299, 981, 897, 778, 1358, 212, 692, 362, 278, 1515, 950, 1057, 719, 1044, 771, 669, 251, 553, 273, 1186, 447, 707, 683, 404, 722, 129, 764, 1271, 793, 760, 1045, 1167, 540, 191, 655, 972, 570, 578, 611, 713, 949, 1055, 515, 493, 771, 591, 800, 540, 998, 1502, 752, 865, 1457, 399, 1486, 727, 203, 771, 1344, 967] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1425, 397, 873, 885, 298, 863, 809, 796, 1387, 260, 733, 357, 311, 1504, 863, 1137, 682, 971, 806, 577, 304, 557, 298, 1206, 427, 724, 761, 394, 702, 126, 768, 1361, 814, 864, 1193, 1245, 465, 199, 597, 906, 572, 555, 588, 744, 874, 959, 528, 568, 693, 573, 762, 641, 974, 1452, 726, 870, 1509, 276, 1482, 747, 190, 837, 1244, 943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True + [2026-08-26 16:35:27] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 818.6 | throughput per GPU (TFLOP/s/GPU): 155.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.563369E+00 | loss scale: 1.0 | grad norm: 14.007 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1377, 360, 772, 936, 325, 1119, 846, 773, 1418, 186, 713, 325, 287, 1191, 892, 1099, 754, 943, 733, 623, 286, 534, 258, 1145, 588, 749, 721, 362, 691, 81, 851, 1444, 851, 808, 970, 1108, 537, 235, 578, 928, 673, 675, 569, 803, 907, 958, 584, 512, 774, 638, 767, 500, 1040, 1355, 790, 847, 1500, 299, 1454, 923, 165, 776, 1253, 993] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1360, 341, 804, 847, 270, 831, 786, 949, 1325, 197, 671, 292, 366, 1236, 955, 1165, 668, 955, 808, 545, 325, 501, 316, 1409, 558, 694, 824, 328, 704, 89, 926, 1264, 841, 1011, 844, 1139, 515, 179, 622, 1105, 612, 567, 535, 825, 877, 950, 682, 624, 708, 580, 797, 527, 983, 1236, 798, 814, 1617, 592, 1460, 769, 132, 741, 1128, 1033] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1453, 286, 702, 872, 279, 1103, 917, 790, 1341, 254, 706, 336, 312, 1449, 992, 1136, 792, 891, 840, 715, 284, 569, 276, 1259, 508, 634, 768, 397, 780, 103, 707, 1157, 791, 645, 1053, 1040, 692, 228, 585, 966, 617, 559, 512, 796, 918, 977, 600, 607, 816, 644, 745, 497, 974, 1444, 790, 907, 1458, 267, 1506, 765, 214, 787, 1274, 870] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1423, 288, 765, 765, 227, 1222, 984, 626, 1220, 317, 786, 370, 251, 1704, 915, 950, 694, 976, 759, 673, 337, 525, 300, 1245, 411, 650, 871, 401, 840, 166, 766, 1415, 893, 766, 1114, 1243, 372, 215, 670, 988, 585, 515, 641, 626, 872, 944, 470, 448, 826, 711, 719, 708, 1002, 1265, 691, 865, 1487, 313, 1453, 733, 206, 792, 1300, 877] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1415, 322, 708, 817, 228, 1324, 902, 625, 1192, 333, 938, 300, 213, 1738, 880, 883, 786, 1013, 962, 669, 257, 531, 317, 1110, 372, 658, 775, 426, 793, 124, 766, 1405, 929, 773, 1042, 1262, 486, 218, 649, 940, 502, 544, 676, 580, 911, 943, 498, 461, 734, 638, 677, 603, 984, 1551, 666, 744, 1480, 322, 1584, 769, 225, 786, 1357, 836] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1360, 341, 804, 847, 270, 831, 786, 949, 1325, 197, 671, 292, 366, 1236, 955, 1165, 668, 955, 808, 545, 325, 501, 316, 1409, 558, 694, 824, 328, 704, 89, 926, 1264, 841, 1011, 844, 1139, 515, 179, 622, 1105, 612, 567, 535, 825, 877, 950, 682, 624, 708, 580, 797, 527, 983, 1236, 798, 814, 1617, 592, 1460, 769, 132, 741, 1128, 1033] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1410, 339, 792, 859, 299, 981, 897, 778, 1358, 212, 692, 362, 278, 1515, 950, 1057, 719, 1044, 771, 669, 251, 553, 273, 1186, 447, 707, 683, 404, 722, 129, 764, 1271, 793, 760, 1045, 1167, 540, 191, 655, 972, 570, 578, 611, 713, 949, 1055, 515, 493, 771, 591, 800, 540, 998, 1502, 752, 865, 1457, 399, 1486, 727, 203, 771, 1344, 967] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1346, 304, 776, 832, 298, 891, 816, 959, 1373, 241, 680, 351, 326, 1440, 912, 996, 807, 966, 762, 478, 304, 511, 345, 1337, 542, 678, 826, 385, 753, 154, 879, 1406, 854, 858, 1246, 1126, 400, 260, 500, 998, 594, 585, 495, 807, 910, 919, 569, 675, 766, 596, 867, 562, 1026, 1213, 773, 813, 1480, 320, 1403, 754, 186, 687, 1178, 1058] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1453, 286, 702, 872, 279, 1103, 917, 790, 1341, 254, 706, 336, 312, 1449, 992, 1136, 792, 891, 840, 715, 284, 569, 276, 1259, 508, 634, 768, 397, 780, 103, 707, 1157, 791, 645, 1053, 1040, 692, 228, 585, 966, 617, 559, 512, 796, 918, 977, 600, 607, 816, 644, 745, 497, 974, 1444, 790, 907, 1458, 267, 1506, 765, 214, 787, 1274, 870] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1234, 397, 873, 885, 298, 863, 809, 796, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 260, 733, 357, 311, 1252, 863, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 682, 971, 806, 577, 304, 557, 298, 1206, 252, 191, 181, 43, 0, 0, 0, 0, 427, 724, 761, 394, 702, 126, 768, 1361, 802, 0, 0, 0, 0, 0, 0, 0, 814, 864, 1193, 1202, 465, 199, 597, 906, 0, 0, 0, 0, 0, 0, 0, 0, 572, 555, 588, 744, 874, 959, 528, 568, 388, 329, 0, 0, 0, 0, 0, 0, 693, 573, 762, 641, 974, 883, 726, 870, 0, 0, 0, 0, 0, 0, 0, 0, 378, 276, 1482, 747, 190, 837, 1244, 943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6300, 6068, 6065, 6240, 6105, 6122, 6097] avg=6144.0000 max=6300 max_rank=1 min=6065 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=42852 max_load_reduction_pct=0.871826 moved_to_echo=24564 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1425, 397, 873, 885, 298, 863, 809, 796, 1387, 260, 733, 357, 311, 1504, 863, 1137, 682, 971, 806, 577, 304, 557, 298, 1206, 427, 724, 761, 394, 702, 126, 768, 1361, 814, 864, 1193, 1245, 465, 199, 597, 906, 572, 555, 588, 744, 874, 959, 528, 568, 693, 573, 762, 641, 974, 1452, 726, 870, 1509, 276, 1482, 747, 190, 837, 1244, 943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1258, 286, 702, 872, 279, 1103, 917, 790, 0, 0, 0, 0, 0, 0, 0, 0, 1341, 254, 706, 336, 312, 1206, 992, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 792, 891, 840, 715, 284, 569, 276, 1259, 243, 195, 180, 36, 0, 0, 0, 0, 508, 634, 768, 397, 780, 103, 707, 1157, 774, 0, 0, 0, 0, 0, 0, 0, 791, 645, 1053, 1004, 692, 228, 585, 966, 0, 0, 0, 0, 0, 0, 0, 0, 617, 559, 512, 796, 918, 977, 600, 607, 386, 318, 0, 0, 0, 0, 0, 0, 816, 644, 745, 497, 974, 878, 790, 907, 0, 0, 0, 0, 0, 0, 0, 0, 366, 267, 1506, 765, 214, 787, 1274, 870, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6207, 6283, 6280, 5828, 5964, 6290, 6251, 6049] avg=6144.0000 max=6290 max_rank=5 min=5828 min_rank=3 imbalance_ratio=1.023763 max_load_reduction=42862 max_load_reduction_pct=0.872030 moved_to_echo=24554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1192, 360, 772, 936, 325, 1119, 846, 773, 0, 0, 0, 0, 0, 0, 0, 0, 1418, 186, 713, 325, 287, 992, 892, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 754, 943, 733, 623, 286, 534, 258, 1145, 199, 185, 169, 38, 0, 0, 0, 0, 588, 749, 721, 362, 691, 81, 851, 1444, 797, 0, 0, 0, 0, 0, 0, 0, 851, 808, 970, 1070, 537, 235, 578, 928, 0, 0, 0, 0, 0, 0, 0, 0, 673, 675, 569, 803, 907, 958, 584, 512, 362, 327, 0, 0, 0, 0, 0, 0, 774, 638, 767, 500, 1040, 824, 790, 847, 0, 0, 0, 0, 0, 0, 0, 0, 376, 299, 1454, 923, 165, 776, 1253, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6323, 5912, 5867, 6284, 5977, 6370, 6180, 6239] avg=6144.0000 max=6370 max_rank=5 min=5867 min_rank=2 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1232, 288, 765, 765, 227, 1222, 984, 626, 0, 0, 0, 0, 0, 0, 0, 0, 1220, 317, 786, 370, 251, 1418, 915, 950, 0, 0, 0, 0, 0, 0, 0, 0, 694, 976, 759, 673, 337, 525, 300, 1245, 286, 191, 158, 43, 0, 0, 0, 0, 411, 650, 871, 401, 840, 166, 766, 1415, 790, 0, 0, 0, 0, 0, 0, 0, 893, 766, 1114, 1200, 372, 215, 670, 988, 0, 0, 0, 0, 0, 0, 0, 0, 585, 515, 641, 626, 872, 944, 470, 448, 338, 324, 0, 0, 0, 0, 0, 0, 826, 711, 719, 708, 1002, 769, 691, 865, 0, 0, 0, 0, 0, 0, 0, 0, 373, 313, 1453, 733, 206, 792, 1300, 877, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6109, 6227, 6187, 6310, 6218, 5763, 6291, 6047] avg=6144.0000 max=6310 max_rank=3 min=5763 min_rank=5 imbalance_ratio=1.027018 max_load_reduction=42842 max_load_reduction_pct=0.871623 moved_to_echo=24319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1177, 341, 804, 847, 270, 831, 786, 949, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 197, 671, 292, 366, 1029, 955, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 668, 955, 808, 545, 325, 501, 316, 1409, 207, 183, 154, 39, 0, 0, 0, 0, 558, 694, 824, 328, 704, 89, 926, 1264, 859, 0, 0, 0, 0, 0, 0, 0, 841, 1011, 844, 1100, 515, 179, 622, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 612, 567, 535, 825, 877, 950, 682, 624, 331, 353, 0, 0, 0, 0, 0, 0, 708, 580, 797, 527, 983, 751, 798, 814, 0, 0, 0, 0, 0, 0, 0, 0, 405, 592, 1460, 769, 132, 741, 1128, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6005, 6000, 6110, 6246, 6217, 6356, 5958, 6260] avg=6144.0000 max=6356 max_rank=5 min=5958 min_rank=6 imbalance_ratio=1.034505 max_load_reduction=42796 max_load_reduction_pct=0.870687 moved_to_echo=24791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1221, 339, 792, 859, 299, 981, 897, 778, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 212, 692, 362, 278, 1261, 950, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 719, 1044, 771, 669, 251, 553, 273, 1186, 254, 189, 187, 40, 0, 0, 0, 0, 447, 707, 683, 404, 722, 129, 764, 1271, 774, 0, 0, 0, 0, 0, 0, 0, 793, 760, 1045, 1127, 540, 191, 655, 972, 0, 0, 0, 0, 0, 0, 0, 0, 570, 578, 611, 713, 949, 1055, 515, 493, 402, 318, 0, 0, 0, 0, 0, 0, 771, 591, 800, 540, 998, 913, 752, 865, 0, 0, 0, 0, 0, 0, 0, 0, 365, 399, 1486, 727, 203, 771, 1344, 967, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6170, 6136, 5901, 6083, 6204, 6230, 6262] avg=6144.0000 max=6262 max_rank=7 min=5901 min_rank=3 imbalance_ratio=1.019206 max_load_reduction=42890 max_load_reduction_pct=0.872599 moved_to_echo=24779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1192, 360, 772, 936, 325, 1119, 846, 773, 0, 0, 0, 0, 0, 0, 0, 0, 1418, 186, 713, 325, 287, 992, 892, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 754, 943, 733, 623, 286, 534, 258, 1145, 199, 185, 169, 38, 0, 0, 0, 0, 588, 749, 721, 362, 691, 81, 851, 1444, 797, 0, 0, 0, 0, 0, 0, 0, 851, 808, 970, 1070, 537, 235, 578, 928, 0, 0, 0, 0, 0, 0, 0, 0, 673, 675, 569, 803, 907, 958, 584, 512, 362, 327, 0, 0, 0, 0, 0, 0, 774, 638, 767, 500, 1040, 824, 790, 847, 0, 0, 0, 0, 0, 0, 0, 0, 376, 299, 1454, 923, 165, 776, 1253, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6323, 5912, 5867, 6284, 5977, 6370, 6180, 6239] avg=6144.0000 max=6370 max_rank=5 min=5867 min_rank=2 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1220, 322, 708, 817, 228, 1324, 902, 625, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 333, 938, 300, 213, 1443, 880, 883, 0, 0, 0, 0, 0, 0, 0, 0, 786, 1013, 962, 669, 257, 531, 317, 1110, 295, 195, 198, 49, 0, 0, 0, 0, 372, 658, 775, 426, 793, 124, 766, 1405, 790, 0, 0, 0, 0, 0, 0, 0, 929, 773, 1042, 1213, 486, 218, 649, 940, 0, 0, 0, 0, 0, 0, 0, 0, 502, 544, 676, 580, 911, 943, 498, 461, 420, 328, 0, 0, 0, 0, 0, 0, 734, 638, 677, 603, 984, 933, 666, 744, 0, 0, 0, 0, 0, 0, 0, 0, 362, 322, 1584, 769, 225, 786, 1357, 836, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6182, 6382, 6109, 6250, 5863, 5979, 6241] avg=6144.0000 max=6382 max_rank=2 min=5863 min_rank=5 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1177, 341, 804, 847, 270, 831, 786, 949, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 197, 671, 292, 366, 1029, 955, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 668, 955, 808, 545, 325, 501, 316, 1409, 207, 183, 154, 39, 0, 0, 0, 0, 558, 694, 824, 328, 704, 89, 926, 1264, 859, 0, 0, 0, 0, 0, 0, 0, 841, 1011, 844, 1100, 515, 179, 622, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 612, 567, 535, 825, 877, 950, 682, 624, 331, 353, 0, 0, 0, 0, 0, 0, 708, 580, 797, 527, 983, 751, 798, 814, 0, 0, 0, 0, 0, 0, 0, 0, 405, 592, 1460, 769, 132, 741, 1128, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6005, 6000, 6110, 6246, 6217, 6356, 5958, 6260] avg=6144.0000 max=6356 max_rank=5 min=5958 min_rank=6 imbalance_ratio=1.034505 max_load_reduction=42796 max_load_reduction_pct=0.870687 moved_to_echo=24791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1165, 304, 776, 832, 298, 891, 816, 959, 0, 0, 0, 0, 0, 0, 0, 0, 1373, 241, 680, 351, 326, 1199, 912, 996, 0, 0, 0, 0, 0, 0, 0, 0, 807, 966, 762, 478, 304, 511, 345, 1337, 241, 181, 151, 39, 0, 0, 0, 0, 542, 678, 826, 385, 753, 154, 879, 1406, 786, 0, 0, 0, 0, 0, 0, 0, 854, 858, 1246, 1087, 400, 260, 500, 998, 0, 0, 0, 0, 0, 0, 0, 0, 594, 585, 495, 807, 910, 919, 569, 675, 324, 323, 0, 0, 0, 0, 0, 0, 766, 596, 867, 562, 1026, 738, 773, 813, 0, 0, 0, 0, 0, 0, 0, 0, 371, 320, 1403, 754, 186, 687, 1178, 1058, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6041, 6078, 6122, 6409, 6203, 6201, 6141, 5957] avg=6144.0000 max=6409 max_rank=3 min=5957 min_rank=7 imbalance_ratio=1.043132 max_load_reduction=42743 max_load_reduction_pct=0.869609 moved_to_echo=24502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1258, 286, 702, 872, 279, 1103, 917, 790, 0, 0, 0, 0, 0, 0, 0, 0, 1341, 254, 706, 336, 312, 1206, 992, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 792, 891, 840, 715, 284, 569, 276, 1259, 243, 195, 180, 36, 0, 0, 0, 0, 508, 634, 768, 397, 780, 103, 707, 1157, 774, 0, 0, 0, 0, 0, 0, 0, 791, 645, 1053, 1004, 692, 228, 585, 966, 0, 0, 0, 0, 0, 0, 0, 0, 617, 559, 512, 796, 918, 977, 600, 607, 386, 318, 0, 0, 0, 0, 0, 0, 816, 644, 745, 497, 974, 878, 790, 907, 0, 0, 0, 0, 0, 0, 0, 0, 366, 267, 1506, 765, 214, 787, 1274, 870, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6207, 6283, 6280, 5828, 5964, 6290, 6251, 6049] avg=6144.0000 max=6290 max_rank=5 min=5828 min_rank=3 imbalance_ratio=1.023763 max_load_reduction=42862 max_load_reduction_pct=0.872030 moved_to_echo=24554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1234, 397, 873, 885, 298, 863, 809, 796, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 260, 733, 357, 311, 1252, 863, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 682, 971, 806, 577, 304, 557, 298, 1206, 252, 191, 181, 43, 0, 0, 0, 0, 427, 724, 761, 394, 702, 126, 768, 1361, 802, 0, 0, 0, 0, 0, 0, 0, 814, 864, 1193, 1202, 465, 199, 597, 906, 0, 0, 0, 0, 0, 0, 0, 0, 572, 555, 588, 744, 874, 959, 528, 568, 388, 329, 0, 0, 0, 0, 0, 0, 693, 573, 762, 641, 974, 883, 726, 870, 0, 0, 0, 0, 0, 0, 0, 0, 378, 276, 1482, 747, 190, 837, 1244, 943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6300, 6068, 6065, 6240, 6105, 6122, 6097] avg=6144.0000 max=6300 max_rank=1 min=6065 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=42852 max_load_reduction_pct=0.871826 moved_to_echo=24564 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9728, 2688, 6272, 6912, 2304, 8448, 7040, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6784, 6528, 8576, 9088, 4096, 1792, 4864, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6144, 4992, 6144, 4608, 8064, 6784, 6016, 6784, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10624, 2048, 6016, 2816, 2432, 9856, 7424, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6784, 6528, 8576, 9088, 4096, 1792, 4864, 7808, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6016, 7808, 6528, 4992, 2432, 4352, 2432, 9984, 2048, 1536, 1408, 384, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6144, 4992, 6144, 4608, 8064, 6784, 6016, 6784, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 2816, 11904, 6272, 1536, 6272, 10112, 7680, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4736, 4608, 4736, 6016, 7296, 7808, 4480, 4480, 3072, 2688, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3968, 5504, 6272, 3200, 6016, 1024, 6528, 10752, 6400, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 2816, 11904, 6272, 1536, 6272, 10112, 7680, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[684, 953, 730, 417, 1243, 227, 389, 1056, 78, 827, 1676, 376, 554, 844, 651, 1640, 862, 1323, 879, 478, 1120, 308, 1769, 1335, 946, 721, 806, 1123, 99, 425, 234, 151, 498, 289, 343, 82, 1069, 1166, 1129, 989, 2211, 1538, 705, 1098, 494, 975, 38, 278, 1142, 1021, 1399, 15, 168, 1426, 220, 1240, 919, 310, 709, 504, 995, 556, 324, 378] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[577, 755, 770, 553, 1004, 248, 371, 1156, 82, 1199, 1486, 399, 667, 1046, 855, 1525, 1091, 1245, 967, 449, 1481, 165, 1664, 974, 637, 683, 854, 864, 58, 341, 221, 112, 399, 171, 554, 182, 1141, 1414, 1007, 931, 2252, 1444, 1062, 1540, 406, 658, 31, 203, 842, 1163, 985, 15, 130, 1472, 249, 1186, 932, 386, 831, 563, 1174, 536, 290, 504] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[632, 854, 798, 497, 1052, 196, 336, 1245, 73, 969, 1478, 389, 562, 897, 866, 1544, 1163, 1194, 861, 590, 1351, 236, 1633, 1212, 760, 766, 815, 952, 123, 440, 175, 90, 423, 274, 450, 117, 1124, 1254, 950, 903, 2150, 1476, 901, 1212, 407, 926, 23, 224, 1047, 1195, 1101, 13, 188, 1504, 237, 1137, 856, 351, 1001, 568, 963, 599, 335, 494] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[626, 831, 658, 413, 1183, 217, 305, 1201, 77, 988, 1590, 369, 655, 834, 682, 1683, 990, 1294, 845, 386, 1365, 257, 1704, 1161, 795, 703, 827, 1000, 73, 288, 199, 121, 382, 231, 439, 139, 1299, 1159, 1033, 947, 2364, 1556, 886, 1184, 368, 932, 51, 311, 1079, 1148, 1043, 14, 128, 1680, 228, 1237, 919, 307, 794, 553, 1057, 571, 310, 483] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[669, 799, 806, 531, 973, 220, 283, 1302, 107, 938, 1358, 416, 629, 826, 800, 1502, 1149, 1279, 882, 483, 1408, 232, 1693, 1101, 709, 703, 811, 962, 116, 388, 203, 154, 472, 247, 466, 151, 1170, 1275, 978, 1042, 2138, 1427, 980, 1330, 361, 712, 36, 202, 969, 1280, 1134, 28, 171, 1625, 204, 1097, 931, 390, 865, 554, 1065, 569, 327, 524] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[622, 857, 668, 575, 900, 306, 410, 1018, 105, 903, 1376, 340, 699, 1029, 811, 1525, 1077, 1216, 942, 352, 1510, 204, 1772, 1004, 697, 751, 763, 846, 95, 287, 203, 120, 458, 227, 541, 184, 1132, 1497, 1219, 1031, 2202, 1486, 1159, 1385, 469, 724, 27, 241, 766, 1260, 827, 18, 196, 1521, 244, 1249, 884, 366, 824, 554, 1086, 554, 326, 512] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[632, 854, 798, 497, 1052, 196, 336, 1245, 73, 969, 1478, 389, 562, 897, 866, 1544, 1163, 1194, 861, 590, 1351, 236, 1633, 1212, 760, 766, 815, 952, 123, 440, 175, 90, 423, 274, 450, 117, 1124, 1254, 950, 903, 2150, 1476, 901, 1212, 407, 926, 23, 224, 1047, 1195, 1101, 13, 188, 1504, 237, 1137, 856, 351, 1001, 568, 963, 599, 335, 494] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[684, 953, 730, 417, 1243, 227, 389, 1056, 78, 827, 1676, 376, 554, 844, 651, 1640, 862, 1323, 879, 478, 1120, 308, 1769, 1335, 946, 721, 806, 1123, 99, 425, 234, 151, 498, 289, 343, 82, 1069, 1166, 1129, 989, 2211, 1538, 705, 1098, 494, 975, 38, 278, 1142, 1021, 1399, 15, 168, 1426, 220, 1240, 919, 310, 709, 504, 995, 556, 324, 378] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[716, 828, 554, 479, 1345, 221, 293, 1351, 102, 920, 1835, 399, 546, 849, 529, 1774, 869, 1343, 914, 403, 1274, 325, 1730, 1567, 754, 802, 795, 1057, 101, 300, 157, 108, 364, 183, 381, 81, 1213, 860, 885, 843, 2070, 1306, 688, 1089, 329, 990, 20, 260, 1604, 1127, 1582, 3, 90, 1277, 244, 1040, 1068, 346, 1010, 549, 1061, 572, 337, 440] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[650, 760, 871, 506, 884, 278, 387, 1061, 103, 967, 1294, 315, 672, 996, 913, 1494, 1174, 1259, 922, 365, 1535, 154, 1722, 959, 651, 691, 705, 822, 78, 322, 212, 115, 400, 200, 567, 206, 1256, 1835, 953, 995, 2344, 1358, 1286, 1444, 425, 675, 23, 189, 818, 1169, 801, 25, 137, 1526, 260, 1218, 839, 309, 832, 698, 1160, 513, 309, 545] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[626, 831, 658, 413, 1183, 217, 305, 1201, 512, 122, 0, 0, 0, 0, 0, 0, 77, 988, 1590, 369, 655, 834, 682, 957, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1294, 845, 386, 1078, 257, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 795, 703, 827, 1000, 73, 288, 199, 121, 1704, 477, 0, 0, 0, 0, 0, 0, 382, 231, 439, 139, 1299, 1159, 1033, 947, 280, 165, 0, 0, 0, 0, 0, 0, 1015, 1556, 886, 1184, 368, 932, 51, 311, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1148, 1043, 14, 128, 1400, 228, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 919, 307, 794, 553, 1057, 571, 310, 483, 872, 214, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6068, 6152, 6011, 6187, 6074, 6303, 6277, 6080] avg=6144.0000 max=6303 max_rank=5 min=6011 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[684, 953, 730, 417, 1243, 227, 389, 1056, 499, 100, 0, 0, 0, 0, 0, 0, 78, 827, 1676, 376, 554, 844, 651, 932, 0, 0, 0, 0, 0, 0, 0, 0, 862, 1323, 879, 478, 884, 308, 0, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 946, 721, 806, 1123, 99, 425, 234, 151, 1769, 447, 0, 0, 0, 0, 0, 0, 498, 289, 343, 82, 1069, 1166, 1129, 989, 238, 136, 0, 0, 0, 0, 0, 0, 949, 1538, 705, 1098, 494, 975, 38, 278, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 1021, 1399, 15, 168, 1188, 220, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 919, 310, 709, 504, 995, 556, 324, 378, 815, 209, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6298, 5938, 6069, 6721, 5939, 6075, 6393, 5719] avg=6144.0000 max=6721 max_rank=3 min=5719 min_rank=7 imbalance_ratio=1.093913 max_load_reduction=42431 max_load_reduction_pct=0.863261 moved_to_echo=24126 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[669, 799, 806, 531, 973, 220, 283, 1302, 107, 938, 1358, 416, 629, 826, 800, 1502, 1149, 1279, 882, 483, 1408, 232, 1693, 1101, 709, 703, 811, 962, 116, 388, 203, 154, 472, 247, 466, 151, 1170, 1275, 978, 1042, 2138, 1427, 980, 1330, 361, 712, 36, 202, 969, 1280, 1134, 28, 171, 1625, 204, 1097, 931, 390, 865, 554, 1065, 569, 327, 524] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[632, 854, 798, 497, 1052, 196, 336, 1245, 470, 121, 0, 0, 0, 0, 0, 0, 73, 969, 1478, 389, 562, 897, 866, 877, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 1194, 861, 590, 1066, 236, 0, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 760, 766, 815, 952, 123, 440, 175, 90, 1633, 434, 0, 0, 0, 0, 0, 0, 423, 274, 450, 117, 1124, 1254, 950, 903, 251, 164, 0, 0, 0, 0, 0, 0, 923, 1476, 901, 1212, 407, 926, 23, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1195, 1101, 13, 188, 1253, 237, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 856, 351, 1001, 568, 963, 599, 335, 494, 793, 197, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 6111, 6322, 6188, 5910, 6092, 6171, 6157] avg=6144.0000 max=6322 max_rank=2 min=5910 min_rank=4 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[622, 857, 668, 575, 900, 306, 410, 1018, 464, 135, 0, 0, 0, 0, 0, 0, 105, 903, 1376, 340, 699, 1029, 811, 867, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1216, 942, 352, 1192, 204, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 697, 751, 763, 846, 95, 287, 203, 120, 1772, 445, 0, 0, 0, 0, 0, 0, 458, 227, 541, 184, 1132, 1497, 1219, 1031, 254, 183, 0, 0, 0, 0, 0, 0, 945, 1486, 1159, 1385, 469, 724, 27, 241, 0, 0, 0, 0, 0, 0, 0, 0, 766, 1260, 827, 18, 196, 1267, 244, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 884, 366, 824, 554, 1086, 554, 326, 512, 812, 194, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5955, 6130, 5987, 5979, 6726, 6436, 5827, 6112] avg=6144.0000 max=6726 max_rank=4 min=5827 min_rank=6 imbalance_ratio=1.094727 max_load_reduction=42426 max_load_reduction_pct=0.863159 moved_to_echo=25101 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[577, 755, 770, 553, 1004, 248, 371, 1156, 464, 132, 0, 0, 0, 0, 0, 0, 82, 1199, 1486, 399, 667, 1046, 855, 867, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1245, 967, 449, 1170, 165, 0, 974, 0, 0, 0, 0, 0, 0, 0, 0, 637, 683, 854, 864, 58, 341, 221, 112, 1664, 455, 0, 0, 0, 0, 0, 0, 399, 171, 554, 182, 1141, 1414, 1007, 931, 245, 179, 0, 0, 0, 0, 0, 0, 967, 1444, 1062, 1540, 406, 658, 31, 203, 0, 0, 0, 0, 0, 0, 0, 0, 842, 1163, 985, 15, 130, 1227, 249, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 932, 386, 831, 563, 1174, 536, 290, 504, 830, 194, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6030, 6601, 6061, 5889, 6223, 6311, 5797, 6240] avg=6144.0000 max=6601 max_rank=1 min=5797 min_rank=6 imbalance_ratio=1.074382 max_load_reduction=42551 max_load_reduction_pct=0.865702 moved_to_echo=24571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[622, 857, 668, 575, 900, 306, 410, 1018, 464, 135, 0, 0, 0, 0, 0, 0, 105, 903, 1376, 340, 699, 1029, 811, 867, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1216, 942, 352, 1192, 204, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 697, 751, 763, 846, 95, 287, 203, 120, 1772, 445, 0, 0, 0, 0, 0, 0, 458, 227, 541, 184, 1132, 1497, 1219, 1031, 254, 183, 0, 0, 0, 0, 0, 0, 945, 1486, 1159, 1385, 469, 724, 27, 241, 0, 0, 0, 0, 0, 0, 0, 0, 766, 1260, 827, 18, 196, 1267, 244, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 884, 366, 824, 554, 1086, 554, 326, 512, 812, 194, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5955, 6130, 5987, 5979, 6726, 6436, 5827, 6112] avg=6144.0000 max=6726 max_rank=4 min=5827 min_rank=6 imbalance_ratio=1.094727 max_load_reduction=42426 max_load_reduction_pct=0.863159 moved_to_echo=25101 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[626, 831, 658, 413, 1183, 217, 305, 1201, 512, 122, 0, 0, 0, 0, 0, 0, 77, 988, 1590, 369, 655, 834, 682, 957, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1294, 845, 386, 1078, 257, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 795, 703, 827, 1000, 73, 288, 199, 121, 1704, 477, 0, 0, 0, 0, 0, 0, 382, 231, 439, 139, 1299, 1159, 1033, 947, 280, 165, 0, 0, 0, 0, 0, 0, 1015, 1556, 886, 1184, 368, 932, 51, 311, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1148, 1043, 14, 128, 1400, 228, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 919, 307, 794, 553, 1057, 571, 310, 483, 872, 214, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6068, 6152, 6011, 6187, 6074, 6303, 6277, 6080] avg=6144.0000 max=6303 max_rank=5 min=6011 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[684, 953, 730, 417, 1243, 227, 389, 1056, 499, 100, 0, 0, 0, 0, 0, 0, 78, 827, 1676, 376, 554, 844, 651, 932, 0, 0, 0, 0, 0, 0, 0, 0, 862, 1323, 879, 478, 884, 308, 0, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 946, 721, 806, 1123, 99, 425, 234, 151, 1769, 447, 0, 0, 0, 0, 0, 0, 498, 289, 343, 82, 1069, 1166, 1129, 989, 238, 136, 0, 0, 0, 0, 0, 0, 949, 1538, 705, 1098, 494, 975, 38, 278, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 1021, 1399, 15, 168, 1188, 220, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 919, 310, 709, 504, 995, 556, 324, 378, 815, 209, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6298, 5938, 6069, 6721, 5939, 6075, 6393, 5719] avg=6144.0000 max=6721 max_rank=3 min=5719 min_rank=7 imbalance_ratio=1.093913 max_load_reduction=42431 max_load_reduction_pct=0.863261 moved_to_echo=24126 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[650, 760, 871, 506, 884, 278, 387, 1061, 454, 137, 0, 0, 0, 0, 0, 0, 103, 967, 1294, 315, 672, 996, 913, 850, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 1259, 922, 365, 1212, 154, 0, 959, 0, 0, 0, 0, 0, 0, 0, 0, 651, 691, 705, 822, 78, 322, 212, 115, 1722, 473, 0, 0, 0, 0, 0, 0, 400, 200, 567, 206, 1256, 1835, 953, 995, 254, 186, 0, 0, 0, 0, 0, 0, 1007, 1358, 1286, 1444, 425, 675, 23, 189, 0, 0, 0, 0, 0, 0, 0, 0, 818, 1169, 801, 25, 137, 1272, 260, 1218, 0, 0, 0, 0, 0, 0, 0, 0, 839, 309, 832, 698, 1160, 513, 309, 545, 864, 190, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5988, 6110, 6045, 5791, 6852, 6407, 5700, 6259] avg=6144.0000 max=6852 max_rank=4 min=5700 min_rank=6 imbalance_ratio=1.115234 max_load_reduction=42300 max_load_reduction_pct=0.860596 moved_to_echo=25218 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[669, 799, 806, 531, 973, 220, 283, 1302, 457, 126, 0, 0, 0, 0, 0, 0, 107, 938, 1358, 416, 629, 826, 800, 854, 0, 0, 0, 0, 0, 0, 0, 0, 1149, 1279, 882, 483, 1111, 232, 0, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 709, 703, 811, 962, 116, 388, 203, 154, 1693, 432, 0, 0, 0, 0, 0, 0, 472, 247, 466, 151, 1170, 1275, 978, 1042, 271, 171, 0, 0, 0, 0, 0, 0, 918, 1427, 980, 1330, 361, 712, 36, 202, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1280, 1134, 28, 171, 1354, 204, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 931, 390, 865, 554, 1065, 569, 327, 524, 788, 191, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 5928, 6237, 6171, 6243, 5966, 6237, 6204] avg=6144.0000 max=6243 max_rank=4 min=5928 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=42909 max_load_reduction_pct=0.872986 moved_to_echo=24650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[716, 828, 554, 479, 1345, 221, 293, 1351, 542, 117, 0, 0, 0, 0, 0, 0, 102, 920, 1835, 399, 546, 849, 529, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 869, 1343, 914, 403, 999, 325, 0, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 754, 802, 795, 1057, 101, 300, 157, 108, 1730, 422, 0, 0, 0, 0, 0, 0, 364, 183, 381, 81, 1213, 860, 885, 843, 217, 158, 0, 0, 0, 0, 0, 0, 880, 1306, 688, 1089, 329, 990, 20, 260, 0, 0, 0, 0, 0, 0, 0, 0, 1604, 1127, 1582, 3, 90, 1060, 244, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 346, 1010, 549, 1061, 572, 337, 440, 768, 230, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6446, 6182, 6420, 6226, 5185, 5562, 6750, 6381] avg=6144.0000 max=6750 max_rank=6 min=5185 min_rank=4 imbalance_ratio=1.098633 max_load_reduction=42402 max_load_reduction_pct=0.862671 moved_to_echo=23878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[650, 760, 871, 506, 884, 278, 387, 1061, 454, 137, 0, 0, 0, 0, 0, 0, 103, 967, 1294, 315, 672, 996, 913, 850, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 1259, 922, 365, 1212, 154, 0, 959, 0, 0, 0, 0, 0, 0, 0, 0, 651, 691, 705, 822, 78, 322, 212, 115, 1722, 473, 0, 0, 0, 0, 0, 0, 400, 200, 567, 206, 1256, 1835, 953, 995, 254, 186, 0, 0, 0, 0, 0, 0, 1007, 1358, 1286, 1444, 425, 675, 23, 189, 0, 0, 0, 0, 0, 0, 0, 0, 818, 1169, 801, 25, 137, 1272, 260, 1218, 0, 0, 0, 0, 0, 0, 0, 0, 839, 309, 832, 698, 1160, 513, 309, 545, 864, 190, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5988, 6110, 6045, 5791, 6852, 6407, 5700, 6259] avg=6144.0000 max=6852 max_rank=4 min=5700 min_rank=6 imbalance_ratio=1.115234 max_load_reduction=42300 max_load_reduction_pct=0.860596 moved_to_echo=25218 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 6656, 5888, 4096, 8704, 1920, 2816, 9472, 3968, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7680, 11648, 7680, 10368, 3328, 6656, 256, 1920, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7424, 2816, 6912, 4608, 8576, 4480, 2560, 3968, 6656, 1664, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 7808, 12160, 3072, 4992, 7424, 6144, 7296, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7424, 2816, 6912, 4608, 8576, 4480, 2560, 3968, 6656, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 6656, 5888, 4096, 8704, 1920, 2816, 9472, 3968, 1024, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 5888, 6400, 7680, 768, 2816, 1664, 1024, 13696, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 1920, 3840, 1152, 9472, 10496, 8192, 7808, 2048, 1408, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8320, 9472, 8960, 256, 1280, 10112, 1920, 9472, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8448, 10240, 7296, 3584, 8832, 1920, 0, 9344, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 1920, 3840, 1152, 9472, 10496, 8192, 7808, 2048, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[163, 923, 433, 434, 125, 1050, 710, 318, 663, 388, 1297, 1099, 1680, 1373, 356, 1350, 1094, 1423, 310, 451, 103, 314, 1179, 866, 428, 782, 916, 701, 322, 510, 957, 408, 357, 976, 1212, 852, 1702, 1962, 1667, 524, 500, 284, 383, 263, 1655, 1103, 160, 1131, 1091, 245, 1037, 836, 168, 428, 262, 754, 1489, 420, 100, 469, 79, 1478, 335, 2104] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[151, 1006, 428, 476, 83, 968, 881, 315, 582, 368, 1362, 1452, 1650, 1497, 293, 1077, 1184, 1410, 328, 437, 126, 320, 1239, 907, 459, 777, 1008, 885, 427, 609, 885, 258, 397, 1109, 980, 540, 2051, 1686, 1447, 467, 701, 231, 361, 249, 1591, 1240, 167, 1249, 1208, 213, 861, 686, 151, 361, 361, 645, 1457, 348, 112, 357, 182, 1317, 439, 2140] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[9, 1133, 429, 487, 8, 1229, 839, 304, 534, 371, 1389, 1521, 1865, 840, 35, 614, 1262, 1719, 413, 336, 293, 160, 1111, 899, 768, 874, 1342, 1222, 281, 423, 970, 112, 395, 1178, 1115, 440, 2645, 1986, 1244, 426, 761, 174, 357, 302, 1499, 1306, 166, 1104, 1064, 110, 493, 889, 74, 116, 260, 492, 1512, 453, 114, 309, 123, 1734, 105, 2414] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[18, 1125, 377, 472, 12, 1273, 899, 211, 535, 356, 1507, 1562, 1462, 1125, 50, 886, 1204, 1573, 564, 421, 172, 175, 1350, 1069, 525, 855, 1553, 1080, 352, 503, 1121, 119, 297, 937, 904, 341, 2640, 1614, 1234, 442, 1047, 233, 355, 544, 1437, 1383, 201, 1214, 1180, 145, 355, 738, 21, 57, 318, 453, 1340, 258, 81, 593, 153, 1793, 154, 2184] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[50, 859, 488, 391, 26, 1308, 847, 311, 503, 430, 1425, 1255, 1787, 962, 145, 923, 1249, 1495, 426, 399, 81, 203, 1150, 1057, 581, 704, 1282, 1129, 336, 468, 891, 245, 377, 1202, 1393, 609, 2147, 1975, 1488, 423, 684, 219, 338, 231, 1499, 1366, 177, 1259, 1035, 155, 946, 775, 82, 226, 280, 559, 1541, 357, 80, 375, 162, 1505, 139, 2142] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[36, 1067, 487, 372, 30, 1185, 756, 263, 641, 412, 1336, 1349, 1770, 1068, 87, 1042, 1181, 1697, 431, 375, 198, 178, 1258, 1044, 647, 773, 1221, 951, 341, 496, 936, 210, 342, 1113, 1209, 489, 2333, 1898, 1326, 451, 723, 186, 347, 308, 1519, 1300, 173, 1205, 993, 132, 784, 836, 84, 118, 240, 539, 1574, 396, 103, 475, 87, 1626, 153, 2252] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[236, 742, 411, 405, 100, 1081, 753, 314, 644, 334, 1300, 1081, 1701, 1751, 553, 1527, 1148, 1536, 268, 489, 117, 351, 1153, 863, 404, 803, 885, 547, 386, 537, 919, 382, 361, 911, 1081, 761, 1460, 1875, 1650, 499, 464, 235, 372, 230, 1823, 1151, 157, 1330, 980, 270, 1057, 770, 229, 402, 239, 829, 1477, 382, 62, 514, 95, 1248, 547, 1970] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[9, 1133, 429, 487, 8, 1229, 839, 304, 534, 371, 1389, 1521, 1865, 840, 35, 614, 1262, 1719, 413, 336, 293, 160, 1111, 899, 768, 874, 1342, 1222, 281, 423, 970, 112, 395, 1178, 1115, 440, 2645, 1986, 1244, 426, 761, 174, 357, 302, 1499, 1306, 166, 1104, 1064, 110, 493, 889, 74, 116, 260, 492, 1512, 453, 114, 309, 123, 1734, 105, 2414] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[36, 1067, 487, 372, 30, 1185, 756, 263, 641, 412, 1336, 1349, 1770, 1068, 87, 1042, 1181, 1697, 431, 375, 198, 178, 1258, 1044, 647, 773, 1221, 951, 341, 496, 936, 210, 342, 1113, 1209, 489, 2333, 1898, 1326, 451, 723, 186, 347, 308, 1519, 1300, 173, 1205, 993, 132, 784, 836, 84, 118, 240, 539, 1574, 396, 103, 475, 87, 1626, 153, 2252] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[172, 861, 484, 370, 78, 1063, 800, 290, 595, 410, 1149, 1255, 1772, 1239, 175, 1211, 1128, 1389, 250, 401, 246, 307, 1190, 966, 535, 794, 985, 805, 317, 531, 943, 310, 400, 1112, 1195, 700, 1871, 1861, 1659, 522, 575, 250, 379, 233, 1792, 1135, 169, 1301, 919, 216, 969, 826, 128, 319, 303, 769, 1534, 455, 82, 482, 110, 1248, 414, 2203] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[163, 923, 433, 434, 125, 1050, 710, 318, 1680, 78, 100, 0, 0, 0, 0, 0, 663, 388, 1282, 1099, 0, 1373, 356, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 1423, 310, 451, 103, 314, 1179, 866, 46, 15, 0, 0, 0, 0, 0, 0, 428, 782, 916, 701, 322, 510, 957, 408, 716, 0, 0, 0, 0, 0, 0, 0, 357, 976, 1212, 852, 0, 1111, 1667, 524, 0, 0, 0, 0, 0, 0, 0, 0, 500, 284, 383, 263, 1655, 1103, 160, 1131, 290, 35, 0, 0, 0, 0, 0, 0, 1091, 245, 1037, 836, 168, 428, 262, 754, 1624, 0, 0, 0, 0, 0, 0, 0, 1489, 420, 100, 469, 79, 1478, 335, 1768, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 6511, 5801, 5740, 6699, 5804, 6445, 6138] avg=6144.0000 max=6699 max_rank=4 min=5740 min_rank=3 imbalance_ratio=1.090332 max_load_reduction=42453 max_load_reduction_pct=0.863708 moved_to_echo=25086 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[18, 1125, 377, 472, 12, 1273, 899, 211, 1462, 122, 82, 0, 0, 0, 0, 0, 535, 356, 1489, 1562, 0, 1125, 50, 886, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 1573, 564, 421, 172, 175, 1350, 1069, 48, 18, 0, 0, 0, 0, 0, 0, 525, 855, 1553, 1080, 352, 503, 1121, 119, 589, 0, 0, 0, 0, 0, 0, 0, 297, 937, 904, 341, 0, 914, 1234, 442, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 233, 355, 544, 1437, 1383, 201, 1214, 301, 29, 0, 0, 0, 0, 0, 0, 1180, 145, 355, 738, 21, 57, 318, 453, 2518, 0, 0, 0, 0, 0, 0, 0, 1340, 258, 81, 593, 153, 1793, 154, 1835, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6053, 6003, 6594, 6697, 5069, 6744, 5785, 6207] avg=6144.0000 max=6744 max_rank=5 min=5069 min_rank=4 imbalance_ratio=1.097656 max_load_reduction=42408 max_load_reduction_pct=0.862793 moved_to_echo=23805 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[151, 1006, 428, 476, 83, 968, 881, 315, 1650, 95, 86, 0, 0, 0, 0, 0, 582, 368, 1346, 1452, 0, 1497, 293, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 1410, 328, 437, 126, 320, 1239, 907, 47, 16, 0, 0, 0, 0, 0, 0, 459, 777, 1008, 885, 427, 609, 885, 258, 615, 0, 0, 0, 0, 0, 0, 0, 397, 1109, 980, 540, 0, 955, 1447, 467, 0, 0, 0, 0, 0, 0, 0, 0, 701, 231, 361, 249, 1591, 1240, 167, 1249, 295, 30, 0, 0, 0, 0, 0, 0, 1208, 213, 861, 686, 151, 361, 361, 645, 1956, 0, 0, 0, 0, 0, 0, 0, 1457, 348, 112, 357, 182, 1317, 439, 1798, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6615, 6014, 5923, 5895, 6114, 6442, 6010] avg=6144.0000 max=6615 max_rank=1 min=5895 min_rank=4 imbalance_ratio=1.076660 max_load_reduction=42537 max_load_reduction_pct=0.865417 moved_to_echo=24461 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[151, 1006, 428, 476, 83, 968, 881, 315, 582, 368, 1362, 1452, 1650, 1497, 293, 1077, 1184, 1410, 328, 437, 126, 320, 1239, 907, 459, 777, 1008, 885, 427, 609, 885, 258, 397, 1109, 980, 540, 2051, 1686, 1447, 467, 701, 231, 361, 249, 1591, 1240, 167, 1249, 1208, 213, 861, 686, 151, 361, 361, 645, 1457, 348, 112, 357, 182, 1317, 439, 2140] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[163, 923, 433, 434, 125, 1050, 710, 318, 663, 388, 1297, 1099, 1680, 1373, 356, 1350, 1094, 1423, 310, 451, 103, 314, 1179, 866, 428, 782, 916, 701, 322, 510, 957, 408, 357, 976, 1212, 852, 1702, 1962, 1667, 524, 500, 284, 383, 263, 1655, 1103, 160, 1131, 1091, 245, 1037, 836, 168, 428, 262, 754, 1489, 420, 100, 469, 79, 1478, 335, 2104] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[36, 1067, 487, 372, 30, 1185, 756, 263, 1770, 107, 97, 0, 0, 0, 0, 0, 641, 412, 1320, 1349, 0, 1068, 87, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 1697, 431, 375, 198, 178, 1258, 1044, 49, 16, 0, 0, 0, 0, 0, 0, 647, 773, 1221, 951, 341, 496, 936, 210, 693, 0, 0, 0, 0, 0, 0, 0, 342, 1113, 1209, 489, 0, 1074, 1326, 451, 0, 0, 0, 0, 0, 0, 0, 0, 723, 186, 347, 308, 1519, 1300, 173, 1205, 310, 34, 0, 0, 0, 0, 0, 0, 993, 132, 784, 836, 84, 118, 240, 539, 2226, 0, 0, 0, 0, 0, 0, 0, 1574, 396, 103, 475, 87, 1626, 153, 1893, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 5919, 6427, 6268, 6004, 6105, 5952, 6307] avg=6144.0000 max=6427 max_rank=2 min=5919 min_rank=1 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[50, 859, 488, 391, 26, 1308, 847, 311, 1787, 99, 101, 0, 0, 0, 0, 0, 503, 430, 1408, 1255, 0, 962, 145, 923, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 1495, 426, 399, 81, 203, 1150, 1057, 47, 17, 0, 0, 0, 0, 0, 0, 581, 704, 1282, 1129, 336, 468, 891, 245, 721, 0, 0, 0, 0, 0, 0, 0, 377, 1202, 1393, 609, 0, 1117, 1488, 423, 0, 0, 0, 0, 0, 0, 0, 0, 684, 219, 338, 231, 1499, 1366, 177, 1259, 295, 36, 0, 0, 0, 0, 0, 0, 1035, 155, 946, 775, 82, 226, 280, 559, 2048, 0, 0, 0, 0, 0, 0, 0, 1541, 357, 80, 375, 162, 1505, 139, 1800, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6267, 5626, 6124, 6357, 6609, 6104, 6106, 5959] avg=6144.0000 max=6609 max_rank=4 min=5626 min_rank=1 imbalance_ratio=1.075684 max_load_reduction=42543 max_load_reduction_pct=0.865540 moved_to_echo=24778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 1133, 429, 487, 8, 1229, 839, 304, 1865, 123, 105, 0, 0, 0, 0, 0, 534, 371, 1368, 1521, 0, 840, 35, 614, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 1719, 413, 336, 293, 160, 1111, 899, 56, 21, 0, 0, 0, 0, 0, 0, 768, 874, 1342, 1222, 281, 423, 970, 112, 729, 0, 0, 0, 0, 0, 0, 0, 395, 1178, 1115, 440, 0, 1112, 1244, 426, 0, 0, 0, 0, 0, 0, 0, 0, 761, 174, 357, 302, 1499, 1306, 166, 1104, 335, 40, 0, 0, 0, 0, 0, 0, 1064, 110, 493, 889, 74, 116, 260, 492, 2522, 0, 0, 0, 0, 0, 0, 0, 1512, 453, 114, 309, 123, 1734, 105, 2023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 5283, 6270, 6721, 5910, 6044, 6020, 6373] avg=6144.0000 max=6721 max_rank=3 min=5283 min_rank=1 imbalance_ratio=1.093913 max_load_reduction=42431 max_load_reduction_pct=0.863261 moved_to_echo=24347 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[172, 861, 484, 370, 78, 1063, 800, 290, 1772, 86, 95, 0, 0, 0, 0, 0, 595, 410, 1135, 1255, 0, 1239, 175, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 1389, 250, 401, 246, 307, 1190, 966, 48, 14, 0, 0, 0, 0, 0, 0, 535, 794, 985, 805, 317, 531, 943, 310, 679, 0, 0, 0, 0, 0, 0, 0, 400, 1112, 1195, 700, 0, 1054, 1659, 522, 0, 0, 0, 0, 0, 0, 0, 0, 575, 250, 379, 233, 1792, 1135, 169, 1301, 303, 33, 0, 0, 0, 0, 0, 0, 919, 216, 969, 826, 128, 319, 303, 769, 1785, 0, 0, 0, 0, 0, 0, 0, 1534, 455, 82, 482, 110, 1248, 414, 1852, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6071, 6020, 5939, 5899, 6642, 6170, 6234, 6177] avg=6144.0000 max=6642 max_rank=4 min=5899 min_rank=3 imbalance_ratio=1.081055 max_load_reduction=42510 max_load_reduction_pct=0.864868 moved_to_echo=25223 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[236, 742, 411, 405, 100, 1081, 753, 314, 1701, 68, 96, 0, 0, 0, 0, 0, 644, 334, 1285, 1081, 0, 1751, 553, 1527, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 1536, 268, 489, 117, 351, 1153, 863, 43, 15, 0, 0, 0, 0, 0, 0, 404, 803, 885, 547, 386, 537, 919, 382, 684, 0, 0, 0, 0, 0, 0, 0, 361, 911, 1081, 761, 0, 1061, 1650, 499, 0, 0, 0, 0, 0, 0, 0, 0, 464, 235, 372, 230, 1823, 1151, 157, 1330, 271, 34, 0, 0, 0, 0, 0, 0, 980, 270, 1057, 770, 229, 402, 239, 829, 1392, 0, 0, 0, 0, 0, 0, 0, 1477, 382, 62, 514, 95, 1248, 547, 1656, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5907, 7175, 5983, 5547, 6324, 6067, 6168, 5981] avg=6144.0000 max=7175 max_rank=1 min=5547 min_rank=3 imbalance_ratio=1.167806 max_load_reduction=41977 max_load_reduction_pct=0.854024 moved_to_echo=24540 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[36, 1067, 487, 372, 30, 1185, 756, 263, 1770, 107, 97, 0, 0, 0, 0, 0, 641, 412, 1320, 1349, 0, 1068, 87, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 1697, 431, 375, 198, 178, 1258, 1044, 49, 16, 0, 0, 0, 0, 0, 0, 647, 773, 1221, 951, 341, 496, 936, 210, 693, 0, 0, 0, 0, 0, 0, 0, 342, 1113, 1209, 489, 0, 1074, 1326, 451, 0, 0, 0, 0, 0, 0, 0, 0, 723, 186, 347, 308, 1519, 1300, 173, 1205, 310, 34, 0, 0, 0, 0, 0, 0, 993, 132, 784, 836, 84, 118, 240, 539, 2226, 0, 0, 0, 0, 0, 0, 0, 1574, 396, 103, 475, 87, 1626, 153, 1893, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 5919, 6427, 6268, 6004, 6105, 5952, 6307] avg=6144.0000 max=6427 max_rank=2 min=5919 min_rank=1 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[172, 861, 484, 370, 78, 1063, 800, 290, 1772, 86, 95, 0, 0, 0, 0, 0, 595, 410, 1135, 1255, 0, 1239, 175, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 1389, 250, 401, 246, 307, 1190, 966, 48, 14, 0, 0, 0, 0, 0, 0, 535, 794, 985, 805, 317, 531, 943, 310, 679, 0, 0, 0, 0, 0, 0, 0, 400, 1112, 1195, 700, 0, 1054, 1659, 522, 0, 0, 0, 0, 0, 0, 0, 0, 575, 250, 379, 233, 1792, 1135, 169, 1301, 303, 33, 0, 0, 0, 0, 0, 0, 919, 216, 969, 826, 128, 319, 303, 769, 1785, 0, 0, 0, 0, 0, 0, 0, 1534, 455, 82, 482, 110, 1248, 414, 1852, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6071, 6020, 5939, 5899, 6642, 6170, 6234, 6177] avg=6144.0000 max=6642 max_rank=4 min=5899 min_rank=3 imbalance_ratio=1.081055 max_load_reduction=42510 max_load_reduction_pct=0.864868 moved_to_echo=25223 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[151, 1006, 428, 476, 83, 968, 881, 315, 1650, 95, 86, 0, 0, 0, 0, 0, 582, 368, 1346, 1452, 0, 1497, 293, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 1410, 328, 437, 126, 320, 1239, 907, 47, 16, 0, 0, 0, 0, 0, 0, 459, 777, 1008, 885, 427, 609, 885, 258, 615, 0, 0, 0, 0, 0, 0, 0, 397, 1109, 980, 540, 0, 955, 1447, 467, 0, 0, 0, 0, 0, 0, 0, 0, 701, 231, 361, 249, 1591, 1240, 167, 1249, 295, 30, 0, 0, 0, 0, 0, 0, 1208, 213, 861, 686, 151, 361, 361, 645, 1956, 0, 0, 0, 0, 0, 0, 0, 1457, 348, 112, 357, 182, 1317, 439, 1798, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6615, 6014, 5923, 5895, 6114, 6442, 6010] avg=6144.0000 max=6615 max_rank=1 min=5895 min_rank=4 imbalance_ratio=1.076660 max_load_reduction=42537 max_load_reduction_pct=0.865417 moved_to_echo=24461 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[163, 923, 433, 434, 125, 1050, 710, 318, 1680, 78, 100, 0, 0, 0, 0, 0, 663, 388, 1282, 1099, 0, 1373, 356, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 1423, 310, 451, 103, 314, 1179, 866, 46, 15, 0, 0, 0, 0, 0, 0, 428, 782, 916, 701, 322, 510, 957, 408, 716, 0, 0, 0, 0, 0, 0, 0, 357, 976, 1212, 852, 0, 1111, 1667, 524, 0, 0, 0, 0, 0, 0, 0, 0, 500, 284, 383, 263, 1655, 1103, 160, 1131, 290, 35, 0, 0, 0, 0, 0, 0, 1091, 245, 1037, 836, 168, 428, 262, 754, 1624, 0, 0, 0, 0, 0, 0, 0, 1489, 420, 100, 469, 79, 1478, 335, 1768, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 6511, 5801, 5740, 6699, 5804, 6445, 6138] avg=6144.0000 max=6699 max_rank=4 min=5740 min_rank=3 imbalance_ratio=1.090332 max_load_reduction=42453 max_load_reduction_pct=0.863708 moved_to_echo=25086 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 1133, 429, 487, 8, 1229, 839, 304, 1865, 123, 105, 0, 0, 0, 0, 0, 534, 371, 1368, 1521, 0, 840, 35, 614, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 1719, 413, 336, 293, 160, 1111, 899, 56, 21, 0, 0, 0, 0, 0, 0, 768, 874, 1342, 1222, 281, 423, 970, 112, 729, 0, 0, 0, 0, 0, 0, 0, 395, 1178, 1115, 440, 0, 1112, 1244, 426, 0, 0, 0, 0, 0, 0, 0, 0, 761, 174, 357, 302, 1499, 1306, 166, 1104, 335, 40, 0, 0, 0, 0, 0, 0, 1064, 110, 493, 889, 74, 116, 260, 492, 2522, 0, 0, 0, 0, 0, 0, 0, 1512, 453, 114, 309, 123, 1734, 105, 2023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 5283, 6270, 6721, 5910, 6044, 6020, 6373] avg=6144.0000 max=6721 max_rank=3 min=5283 min_rank=1 imbalance_ratio=1.093913 max_load_reduction=42431 max_load_reduction_pct=0.863261 moved_to_echo=24347 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[18, 1125, 377, 472, 12, 1273, 899, 211, 1462, 122, 82, 0, 0, 0, 0, 0, 535, 356, 1489, 1562, 0, 1125, 50, 886, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 1573, 564, 421, 172, 175, 1350, 1069, 48, 18, 0, 0, 0, 0, 0, 0, 525, 855, 1553, 1080, 352, 503, 1121, 119, 589, 0, 0, 0, 0, 0, 0, 0, 297, 937, 904, 341, 0, 914, 1234, 442, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 233, 355, 544, 1437, 1383, 201, 1214, 301, 29, 0, 0, 0, 0, 0, 0, 1180, 145, 355, 738, 21, 57, 318, 453, 2518, 0, 0, 0, 0, 0, 0, 0, 1340, 258, 81, 593, 153, 1793, 154, 1835, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6053, 6003, 6594, 6697, 5069, 6744, 5785, 6207] avg=6144.0000 max=6744 max_rank=5 min=5069 min_rank=4 imbalance_ratio=1.097656 max_load_reduction=42408 max_load_reduction_pct=0.862793 moved_to_echo=23805 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5504, 1920, 2944, 2432, 12928, 9984, 1408, 9856, 2432, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8576, 1536, 6528, 6400, 1024, 2048, 2304, 5120, 16128, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2944, 8576, 9216, 4736, 0, 8448, 11776, 3840, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12032, 3072, 768, 3584, 1024, 12032, 2304, 14720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5504, 1920, 2944, 2432, 12928, 9984, 1408, 9856, 2432, 384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4736, 3072, 10752, 10624, 0, 9856, 1792, 8704, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 7808, 3584, 3456, 512, 9216, 6528, 2432, 13696, 896, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4352, 6400, 9216, 7424, 2816, 4096, 7680, 2048, 5504, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12032, 3072, 768, 3584, 1024, 12032, 2304, 14720, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9472, 12288, 3072, 3328, 1408, 2048, 9728, 7680, 384, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4352, 6400, 9216, 7424, 2816, 4096, 7680, 2048, 5504, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[392, 814, 1411, 391, 722, 559, 866, 798, 1384, 2206, 278, 456, 1633, 413, 937, 424, 1590, 97, 2102, 552, 605, 401, 16, 650, 1156, 1163, 663, 612, 597, 937, 1302, 819, 219, 691, 322, 335, 836, 764, 1898, 546, 226, 2075, 135, 546, 359, 619, 423, 3267, 49, 358, 204, 1013, 503, 1326, 219, 308, 409, 145, 1275, 627, 1064, 652, 494, 299] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[327, 715, 1175, 471, 657, 537, 678, 773, 1759, 2541, 194, 468, 1233, 530, 1153, 363, 1660, 117, 1922, 509, 473, 342, 13, 644, 1131, 1103, 676, 531, 517, 1164, 1506, 1130, 155, 652, 323, 335, 1000, 650, 1725, 492, 255, 2041, 192, 518, 342, 588, 433, 2623, 121, 357, 241, 972, 748, 1096, 359, 481, 414, 155, 1224, 577, 1001, 839, 532, 699] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[214, 419, 1495, 308, 636, 694, 746, 921, 1956, 3342, 367, 497, 1730, 412, 1268, 348, 1757, 73, 1919, 473, 658, 564, 1, 550, 1324, 1264, 600, 615, 537, 1015, 1242, 679, 264, 603, 179, 274, 890, 628, 1559, 638, 127, 2378, 101, 469, 267, 456, 341, 2894, 4, 253, 137, 1133, 493, 918, 166, 195, 369, 188, 1336, 531, 1088, 809, 431, 409] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[188, 357, 1591, 382, 803, 762, 761, 1003, 1790, 3183, 261, 455, 1139, 564, 1306, 370, 1926, 90, 1910, 402, 679, 502, 1, 537, 1245, 1291, 578, 424, 641, 1123, 1638, 801, 373, 421, 174, 313, 1128, 621, 1712, 647, 44, 2017, 131, 436, 301, 368, 342, 2700, 3, 259, 134, 1085, 562, 942, 194, 193, 523, 210, 1338, 510, 1103, 856, 395, 414] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[327, 715, 1175, 471, 657, 537, 678, 773, 1759, 2541, 194, 468, 1233, 530, 1153, 363, 1660, 117, 1922, 509, 473, 342, 13, 644, 1131, 1103, 676, 531, 517, 1164, 1506, 1130, 155, 652, 323, 335, 1000, 650, 1725, 492, 255, 2041, 192, 518, 342, 588, 433, 2623, 121, 357, 241, 972, 748, 1096, 359, 481, 414, 155, 1224, 577, 1001, 839, 532, 699] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[202, 381, 1667, 274, 754, 890, 732, 1011, 2245, 3306, 300, 475, 1339, 434, 1291, 359, 1707, 80, 2000, 490, 734, 460, 0, 591, 853, 1422, 587, 548, 482, 1192, 1371, 692, 294, 557, 235, 305, 975, 604, 1325, 613, 85, 2240, 142, 404, 235, 524, 350, 2554, 1, 283, 160, 1160, 424, 986, 184, 149, 463, 87, 1368, 509, 982, 922, 459, 704] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[240, 418, 1763, 286, 882, 756, 658, 1206, 1767, 3099, 475, 475, 1766, 436, 1251, 197, 1617, 56, 1707, 524, 754, 436, 4, 667, 1155, 1601, 608, 448, 552, 1009, 1239, 686, 430, 606, 225, 295, 832, 493, 1369, 665, 213, 2694, 145, 452, 300, 522, 318, 2866, 5, 265, 176, 988, 356, 966, 241, 234, 221, 142, 1218, 511, 961, 939, 334, 432] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[392, 814, 1411, 391, 722, 559, 866, 798, 1384, 2206, 278, 456, 1633, 413, 937, 424, 1590, 97, 2102, 552, 605, 401, 16, 650, 1156, 1163, 663, 612, 597, 937, 1302, 819, 219, 691, 322, 335, 836, 764, 1898, 546, 226, 2075, 135, 546, 359, 619, 423, 3267, 49, 358, 204, 1013, 503, 1326, 219, 308, 409, 145, 1275, 627, 1064, 652, 494, 299] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[470, 763, 1448, 347, 703, 534, 823, 751, 1226, 2217, 287, 423, 1636, 554, 835, 415, 1575, 84, 2340, 536, 591, 339, 25, 676, 1129, 1212, 660, 629, 466, 919, 1272, 813, 220, 758, 279, 322, 871, 727, 1863, 619, 414, 2326, 128, 533, 402, 685, 486, 2903, 70, 374, 220, 968, 496, 1068, 298, 377, 284, 196, 1453, 577, 995, 656, 544, 342] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[381, 711, 1346, 293, 675, 558, 706, 848, 1636, 2732, 343, 520, 1748, 434, 966, 414, 1654, 70, 2203, 532, 528, 388, 24, 676, 1309, 1289, 667, 602, 523, 897, 1315, 764, 399, 785, 277, 318, 928, 632, 1439, 628, 382, 2267, 152, 484, 350, 545, 395, 2983, 46, 237, 225, 1003, 442, 1078, 243, 274, 289, 155, 1378, 565, 1037, 738, 392, 334] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[202, 381, 1667, 274, 754, 890, 732, 1011, 2245, 3306, 300, 475, 1339, 434, 1291, 359, 1707, 80, 2000, 490, 734, 460, 0, 591, 853, 1422, 587, 548, 482, 1192, 1371, 692, 294, 557, 235, 305, 975, 604, 1325, 613, 85, 2240, 142, 404, 235, 524, 350, 2554, 1, 283, 160, 1160, 424, 986, 184, 149, 463, 87, 1368, 509, 982, 922, 459, 704] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[214, 419, 1495, 308, 636, 694, 746, 921, 394, 0, 0, 0, 0, 0, 0, 0, 1956, 180, 367, 497, 1730, 412, 1268, 348, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 73, 1919, 473, 658, 564, 1, 550, 171, 0, 0, 0, 0, 0, 0, 0, 1324, 1264, 600, 615, 537, 1015, 127, 679, 0, 0, 0, 0, 0, 0, 0, 0, 264, 603, 179, 274, 890, 628, 1559, 638, 493, 343, 0, 0, 0, 0, 0, 0, 127, 2378, 101, 469, 267, 456, 341, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 4, 253, 137, 1133, 493, 918, 166, 195, 2973, 0, 0, 0, 0, 0, 0, 0, 369, 188, 1336, 531, 1088, 809, 431, 409, 772, 189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5827, 6758, 6166, 6161, 5871, 5975, 6272, 6122] avg=6144.0000 max=6758 max_rank=1 min=5827 min_rank=0 imbalance_ratio=1.099935 max_load_reduction=42394 max_load_reduction_pct=0.862508 moved_to_echo=24240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[240, 418, 1763, 286, 882, 756, 658, 1206, 390, 0, 0, 0, 0, 0, 0, 0, 1767, 168, 475, 475, 1766, 436, 1251, 197, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 56, 1707, 524, 754, 436, 4, 667, 169, 0, 0, 0, 0, 0, 0, 0, 1155, 1601, 608, 448, 552, 1009, 126, 686, 0, 0, 0, 0, 0, 0, 0, 0, 430, 606, 225, 295, 832, 493, 1369, 665, 488, 343, 0, 0, 0, 0, 0, 0, 213, 2694, 145, 452, 300, 522, 318, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 5, 265, 176, 988, 356, 966, 241, 234, 2756, 0, 0, 0, 0, 0, 0, 0, 221, 142, 1218, 511, 961, 939, 334, 432, 770, 175, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6599, 6535, 5934, 6185, 5746, 6463, 5987, 5703] avg=6144.0000 max=6599 max_rank=0 min=5703 min_rank=7 imbalance_ratio=1.074056 max_load_reduction=42553 max_load_reduction_pct=0.865743 moved_to_echo=23899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[470, 763, 1448, 347, 703, 534, 823, 751, 1226, 2217, 287, 423, 1636, 554, 835, 415, 1575, 84, 2340, 536, 591, 339, 25, 676, 1129, 1212, 660, 629, 466, 919, 1272, 813, 220, 758, 279, 322, 871, 727, 1863, 619, 414, 2326, 128, 533, 402, 685, 486, 2903, 70, 374, 220, 968, 496, 1068, 298, 377, 284, 196, 1453, 577, 995, 656, 544, 342] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[188, 357, 1591, 382, 803, 762, 761, 1003, 368, 0, 0, 0, 0, 0, 0, 0, 1790, 172, 261, 455, 1139, 564, 1306, 370, 0, 0, 0, 0, 0, 0, 0, 0, 1926, 90, 1910, 402, 679, 502, 1, 537, 159, 0, 0, 0, 0, 0, 0, 0, 1245, 1291, 578, 424, 641, 1123, 167, 801, 0, 0, 0, 0, 0, 0, 0, 0, 373, 421, 174, 313, 1128, 621, 1712, 647, 460, 453, 0, 0, 0, 0, 0, 0, 44, 2017, 131, 436, 301, 368, 342, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 3, 259, 134, 1085, 562, 942, 194, 193, 2831, 0, 0, 0, 0, 0, 0, 0, 523, 210, 1338, 510, 1103, 856, 395, 414, 1018, 180, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6215, 6057, 6206, 6270, 6302, 5352, 6203, 6547] avg=6144.0000 max=6547 max_rank=7 min=5352 min_rank=5 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=24404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[327, 715, 1175, 471, 657, 537, 678, 773, 357, 0, 0, 0, 0, 0, 0, 0, 1759, 138, 194, 468, 1233, 530, 1153, 363, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 117, 1922, 509, 473, 342, 13, 644, 155, 0, 0, 0, 0, 0, 0, 0, 1131, 1103, 676, 531, 517, 1164, 153, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 155, 652, 323, 335, 1000, 650, 1725, 492, 446, 417, 0, 0, 0, 0, 0, 0, 255, 2041, 192, 518, 342, 588, 433, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 121, 357, 241, 972, 748, 1096, 359, 481, 2260, 0, 0, 0, 0, 0, 0, 0, 414, 155, 1224, 577, 1001, 839, 532, 699, 936, 143, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5690, 5838, 5835, 6405, 6195, 6034, 6635, 6520] avg=6144.0000 max=6635 max_rank=6 min=5690 min_rank=0 imbalance_ratio=1.079915 max_load_reduction=42517 max_load_reduction_pct=0.865011 moved_to_echo=25384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[392, 814, 1411, 391, 722, 559, 866, 798, 445, 0, 0, 0, 0, 0, 0, 0, 1384, 120, 278, 456, 1633, 413, 937, 424, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 97, 2102, 552, 605, 401, 16, 650, 193, 0, 0, 0, 0, 0, 0, 0, 1156, 1163, 663, 612, 597, 937, 133, 819, 0, 0, 0, 0, 0, 0, 0, 0, 219, 691, 322, 335, 836, 764, 1898, 546, 556, 360, 0, 0, 0, 0, 0, 0, 226, 2075, 135, 546, 359, 619, 423, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 49, 358, 204, 1013, 503, 1326, 219, 308, 1962, 0, 0, 0, 0, 0, 0, 0, 409, 145, 1275, 627, 1064, 652, 494, 299, 809, 124, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6398, 5645, 6206, 6080, 6527, 6456, 5942, 5898] avg=6144.0000 max=6527 max_rank=4 min=5645 min_rank=1 imbalance_ratio=1.062337 max_load_reduction=42625 max_load_reduction_pct=0.867208 moved_to_echo=24823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[214, 419, 1495, 308, 636, 694, 746, 921, 394, 0, 0, 0, 0, 0, 0, 0, 1956, 180, 367, 497, 1730, 412, 1268, 348, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 73, 1919, 473, 658, 564, 1, 550, 171, 0, 0, 0, 0, 0, 0, 0, 1324, 1264, 600, 615, 537, 1015, 127, 679, 0, 0, 0, 0, 0, 0, 0, 0, 264, 603, 179, 274, 890, 628, 1559, 638, 493, 343, 0, 0, 0, 0, 0, 0, 127, 2378, 101, 469, 267, 456, 341, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 4, 253, 137, 1133, 493, 918, 166, 195, 2973, 0, 0, 0, 0, 0, 0, 0, 369, 188, 1336, 531, 1088, 809, 431, 409, 772, 189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5827, 6758, 6166, 6161, 5871, 5975, 6272, 6122] avg=6144.0000 max=6758 max_rank=1 min=5827 min_rank=0 imbalance_ratio=1.099935 max_load_reduction=42394 max_load_reduction_pct=0.862508 moved_to_echo=24240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[470, 763, 1448, 347, 703, 534, 823, 751, 396, 0, 0, 0, 0, 0, 0, 0, 1226, 120, 287, 423, 1636, 554, 835, 415, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 84, 2340, 536, 591, 339, 25, 676, 171, 0, 0, 0, 0, 0, 0, 0, 1129, 1212, 660, 629, 466, 919, 129, 813, 0, 0, 0, 0, 0, 0, 0, 0, 220, 758, 279, 322, 871, 727, 1863, 619, 494, 352, 0, 0, 0, 0, 0, 0, 414, 2326, 128, 533, 402, 685, 486, 1842, 0, 0, 0, 0, 0, 0, 0, 0, 70, 374, 220, 968, 496, 1068, 298, 377, 1972, 0, 0, 0, 0, 0, 0, 0, 284, 196, 1453, 577, 995, 656, 544, 342, 791, 125, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6235, 5496, 6337, 5957, 6505, 6816, 5843, 5963] avg=6144.0000 max=6816 max_rank=5 min=5496 min_rank=1 imbalance_ratio=1.109375 max_load_reduction=42336 max_load_reduction_pct=0.861328 moved_to_echo=25127 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[381, 711, 1346, 293, 675, 558, 706, 848, 406, 0, 0, 0, 0, 0, 0, 0, 1636, 148, 343, 520, 1748, 434, 966, 414, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 70, 2203, 532, 528, 388, 24, 676, 176, 0, 0, 0, 0, 0, 0, 0, 1309, 1289, 667, 602, 523, 897, 134, 764, 0, 0, 0, 0, 0, 0, 0, 0, 399, 785, 277, 318, 928, 632, 1439, 628, 508, 364, 0, 0, 0, 0, 0, 0, 382, 2267, 152, 484, 350, 545, 395, 1893, 0, 0, 0, 0, 0, 0, 0, 0, 46, 237, 225, 1003, 442, 1078, 243, 274, 2430, 0, 0, 0, 0, 0, 0, 0, 289, 155, 1378, 565, 1037, 738, 392, 334, 817, 154, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5924, 6209, 6251, 6185, 6278, 6468, 5978, 5859] avg=6144.0000 max=6468 max_rank=5 min=5859 min_rank=7 imbalance_ratio=1.052734 max_load_reduction=42684 max_load_reduction_pct=0.868408 moved_to_echo=24583 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[392, 814, 1411, 391, 722, 559, 866, 798, 445, 0, 0, 0, 0, 0, 0, 0, 1384, 120, 278, 456, 1633, 413, 937, 424, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 97, 2102, 552, 605, 401, 16, 650, 193, 0, 0, 0, 0, 0, 0, 0, 1156, 1163, 663, 612, 597, 937, 133, 819, 0, 0, 0, 0, 0, 0, 0, 0, 219, 691, 322, 335, 836, 764, 1898, 546, 556, 360, 0, 0, 0, 0, 0, 0, 226, 2075, 135, 546, 359, 619, 423, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 49, 358, 204, 1013, 503, 1326, 219, 308, 1962, 0, 0, 0, 0, 0, 0, 0, 409, 145, 1275, 627, 1064, 652, 494, 299, 809, 124, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6398, 5645, 6206, 6080, 6527, 6456, 5942, 5898] avg=6144.0000 max=6527 max_rank=4 min=5645 min_rank=1 imbalance_ratio=1.062337 max_load_reduction=42625 max_load_reduction_pct=0.867208 moved_to_echo=24823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[202, 381, 1667, 274, 754, 890, 732, 1011, 353, 0, 0, 0, 0, 0, 0, 0, 2245, 171, 300, 475, 1339, 434, 1291, 359, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 80, 2000, 490, 734, 460, 0, 591, 155, 0, 0, 0, 0, 0, 0, 0, 853, 1422, 587, 548, 482, 1192, 133, 692, 0, 0, 0, 0, 0, 0, 0, 0, 294, 557, 235, 305, 975, 604, 1325, 613, 438, 382, 0, 0, 0, 0, 0, 0, 85, 2240, 142, 404, 235, 524, 350, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 1, 283, 160, 1160, 424, 986, 184, 149, 2945, 0, 0, 0, 0, 0, 0, 0, 463, 87, 1368, 509, 982, 922, 459, 704, 856, 190, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 6614, 6217, 5909, 5728, 5588, 6292, 6540] avg=6144.0000 max=6614 max_rank=1 min=5588 min_rank=5 imbalance_ratio=1.076497 max_load_reduction=42538 max_load_reduction_pct=0.865438 moved_to_echo=24148 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[470, 763, 1448, 347, 703, 534, 823, 751, 396, 0, 0, 0, 0, 0, 0, 0, 1226, 120, 287, 423, 1636, 554, 835, 415, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 84, 2340, 536, 591, 339, 25, 676, 171, 0, 0, 0, 0, 0, 0, 0, 1129, 1212, 660, 629, 466, 919, 129, 813, 0, 0, 0, 0, 0, 0, 0, 0, 220, 758, 279, 322, 871, 727, 1863, 619, 494, 352, 0, 0, 0, 0, 0, 0, 414, 2326, 128, 533, 402, 685, 486, 1842, 0, 0, 0, 0, 0, 0, 0, 0, 70, 374, 220, 968, 496, 1068, 298, 377, 1972, 0, 0, 0, 0, 0, 0, 0, 284, 196, 1453, 577, 995, 656, 544, 342, 791, 125, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6235, 5496, 6337, 5957, 6505, 6816, 5843, 5963] avg=6144.0000 max=6816 max_rank=5 min=5496 min_rank=1 imbalance_ratio=1.109375 max_load_reduction=42336 max_load_reduction_pct=0.861328 moved_to_echo=25127 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[240, 418, 1763, 286, 882, 756, 658, 1206, 390, 0, 0, 0, 0, 0, 0, 0, 1767, 168, 475, 475, 1766, 436, 1251, 197, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 56, 1707, 524, 754, 436, 4, 667, 169, 0, 0, 0, 0, 0, 0, 0, 1155, 1601, 608, 448, 552, 1009, 126, 686, 0, 0, 0, 0, 0, 0, 0, 0, 430, 606, 225, 295, 832, 493, 1369, 665, 488, 343, 0, 0, 0, 0, 0, 0, 213, 2694, 145, 452, 300, 522, 318, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 5, 265, 176, 988, 356, 966, 241, 234, 2756, 0, 0, 0, 0, 0, 0, 0, 221, 142, 1218, 511, 961, 939, 334, 432, 770, 175, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6599, 6535, 5934, 6185, 5746, 6463, 5987, 5703] avg=6144.0000 max=6599 max_rank=0 min=5703 min_rank=7 imbalance_ratio=1.074056 max_load_reduction=42553 max_load_reduction_pct=0.865743 moved_to_echo=23899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[327, 715, 1175, 471, 657, 537, 678, 773, 357, 0, 0, 0, 0, 0, 0, 0, 1759, 138, 194, 468, 1233, 530, 1153, 363, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 117, 1922, 509, 473, 342, 13, 644, 155, 0, 0, 0, 0, 0, 0, 0, 1131, 1103, 676, 531, 517, 1164, 153, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 155, 652, 323, 335, 1000, 650, 1725, 492, 446, 417, 0, 0, 0, 0, 0, 0, 255, 2041, 192, 518, 342, 588, 433, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 121, 357, 241, 972, 748, 1096, 359, 481, 2260, 0, 0, 0, 0, 0, 0, 0, 414, 155, 1224, 577, 1001, 839, 532, 699, 936, 143, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5690, 5838, 5835, 6405, 6195, 6034, 6635, 6520] avg=6144.0000 max=6635 max_rank=6 min=5690 min_rank=0 imbalance_ratio=1.079915 max_load_reduction=42517 max_load_reduction_pct=0.865011 moved_to_echo=25384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13568, 768, 16128, 4096, 5120, 3456, 128, 4992, 1408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 18048, 1152, 3968, 2560, 4352, 3200, 14464, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 4608, 11904, 2816, 5888, 5376, 6016, 7424, 3200, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 5120, 2048, 2560, 7552, 5120, 12928, 4864, 3968, 3072, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13824, 1280, 2560, 3840, 12288, 3840, 9088, 2944, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 2432, 1536, 8448, 4096, 8448, 1920, 2304, 20224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 18048, 1152, 3968, 2560, 4352, 3200, 14464, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 1280, 10624, 4480, 8320, 6528, 3584, 3712, 6784, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 2432, 1536, 8448, 4096, 8448, 1920, 2304, 20224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13824, 1280, 2560, 3840, 12288, 3840, 9088, 2944, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9344, 10368, 5120, 4480, 4352, 8320, 1152, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 4608, 11904, 2816, 5888, 5376, 6016, 7424, 3200, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[25, 2090, 1677, 515, 1063, 1562, 742, 977, 592, 1225, 2326, 193, 202, 625, 483, 883, 1384, 14, 248, 129, 145, 522, 325, 353, 797, 400, 1434, 600, 1838, 1212, 485, 112, 279, 86, 37, 1409, 967, 376, 187, 1454, 382, 719, 383, 989, 576, 15, 1692, 67, 330, 678, 320, 765, 465, 1281, 972, 881, 936, 826, 191, 938, 997, 2520, 957, 1299] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 1011, 1464, 898, 768, 1936, 600, 1464, 585, 1345, 2652, 109, 178, 616, 338, 546, 1736, 10, 121, 40, 176, 974, 335, 223, 670, 336, 1770, 185, 2956, 514, 329, 53, 231, 0, 0, 2635, 586, 48, 22, 1290, 174, 594, 11, 808, 348, 0, 2584, 1, 5, 1132, 167, 728, 406, 2227, 1343, 917, 423, 1054, 98, 1261, 888, 2011, 899, 1323] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 1106, 1466, 1111, 1018, 1578, 663, 1304, 496, 1100, 3055, 79, 146, 574, 323, 741, 1453, 7, 74, 52, 173, 629, 322, 237, 634, 370, 2360, 254, 2858, 714, 408, 49, 204, 0, 0, 2152, 685, 95, 56, 1158, 233, 232, 15, 847, 454, 0, 2378, 0, 36, 1081, 186, 1055, 402, 1639, 1274, 1073, 536, 852, 121, 1290, 1096, 2203, 1095, 1350] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 920, 1514, 1268, 1015, 1122, 630, 1258, 410, 1152, 2929, 77, 153, 831, 323, 621, 1175, 4, 146, 51, 202, 785, 375, 224, 792, 330, 2416, 264, 2750, 635, 349, 67, 254, 0, 2, 2442, 788, 149, 77, 1528, 267, 324, 5, 807, 535, 1, 2448, 0, 43, 1062, 204, 1048, 392, 1941, 1127, 986, 605, 714, 136, 1136, 1103, 2108, 931, 1201] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1142, 1402, 796, 1122, 2149, 599, 1486, 443, 795, 2442, 102, 134, 471, 295, 969, 1539, 3, 195, 60, 351, 610, 241, 228, 456, 346, 2577, 244, 2913, 643, 428, 71, 246, 1, 0, 2188, 720, 215, 55, 1401, 260, 345, 13, 938, 409, 0, 2227, 0, 19, 707, 232, 1140, 418, 1528, 1384, 1257, 441, 856, 145, 1385, 895, 2099, 1422, 954] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 1011, 1464, 898, 768, 1936, 600, 1464, 585, 1345, 2652, 109, 178, 616, 338, 546, 1736, 10, 121, 40, 176, 974, 335, 223, 670, 336, 1770, 185, 2956, 514, 329, 53, 231, 0, 0, 2635, 586, 48, 22, 1290, 174, 594, 11, 808, 348, 0, 2584, 1, 5, 1132, 167, 728, 406, 2227, 1343, 917, 423, 1054, 98, 1261, 888, 2011, 899, 1323] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2, 1909, 1690, 884, 1130, 1346, 733, 1213, 451, 1559, 2485, 129, 170, 535, 355, 1120, 1279, 35, 131, 54, 132, 552, 256, 413, 956, 417, 1665, 565, 1710, 945, 627, 83, 251, 19, 33, 1256, 1124, 242, 181, 1214, 375, 450, 62, 881, 586, 12, 1703, 3, 243, 867, 350, 794, 428, 1254, 882, 1146, 927, 775, 167, 1155, 1171, 2837, 997, 1236] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[25, 2090, 1677, 515, 1063, 1562, 742, 977, 592, 1225, 2326, 193, 202, 625, 483, 883, 1384, 14, 248, 129, 145, 522, 325, 353, 797, 400, 1434, 600, 1838, 1212, 485, 112, 279, 86, 37, 1409, 967, 376, 187, 1454, 382, 719, 383, 989, 576, 15, 1692, 67, 330, 678, 320, 765, 465, 1281, 972, 881, 936, 826, 191, 938, 997, 2520, 957, 1299] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 920, 1514, 1268, 1015, 1122, 630, 1258, 410, 1152, 2929, 77, 153, 831, 323, 621, 1175, 4, 146, 51, 202, 785, 375, 224, 792, 330, 2416, 264, 2750, 635, 349, 67, 254, 0, 2, 2442, 788, 149, 77, 1528, 267, 324, 5, 807, 535, 1, 2448, 0, 43, 1062, 204, 1048, 392, 1941, 1127, 986, 605, 714, 136, 1136, 1103, 2108, 931, 1201] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[12, 1860, 1668, 813, 1083, 1113, 677, 1214, 632, 1341, 2565, 99, 159, 642, 368, 750, 1487, 24, 165, 65, 154, 643, 284, 327, 987, 385, 1594, 458, 1859, 924, 542, 74, 250, 5, 9, 1589, 1095, 220, 126, 1155, 287, 700, 87, 907, 584, 5, 1981, 8, 123, 819, 253, 755, 434, 1336, 781, 1112, 934, 839, 175, 1218, 1224, 2623, 1064, 1486] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3, 1997, 1879, 815, 1109, 1156, 693, 942, 401, 1285, 2681, 87, 105, 615, 349, 956, 1289, 73, 119, 87, 143, 544, 221, 411, 1042, 393, 1493, 619, 1518, 1077, 597, 64, 292, 6, 15, 1145, 1235, 328, 146, 1199, 325, 546, 146, 822, 639, 0, 1752, 2, 335, 840, 339, 1040, 511, 1235, 827, 1251, 1018, 702, 166, 1126, 1223, 2630, 1174, 1374] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2, 1909, 1690, 884, 1130, 1346, 733, 1213, 451, 1559, 2485, 129, 170, 535, 355, 1120, 1279, 35, 131, 54, 132, 552, 256, 413, 956, 417, 1665, 565, 1710, 945, 627, 83, 251, 19, 33, 1256, 1124, 242, 181, 1214, 375, 450, 62, 881, 586, 12, 1703, 3, 243, 867, 350, 794, 428, 1254, 882, 1146, 927, 775, 167, 1155, 1171, 2837, 997, 1236] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 602, 0, 1111, 1018, 1578, 663, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 496, 1100, 2731, 79, 146, 574, 323, 741, 0, 0, 0, 0, 0, 0, 0, 0, 1453, 7, 74, 52, 173, 629, 322, 237, 2203, 616, 0, 0, 0, 0, 0, 0, 634, 370, 2360, 254, 1607, 714, 408, 49, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 2152, 685, 95, 56, 1158, 504, 324, 217, 178, 194, 0, 0, 0, 233, 232, 15, 847, 454, 0, 2378, 0, 850, 1057, 0, 0, 0, 0, 0, 0, 36, 1081, 186, 1055, 402, 1422, 1274, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 536, 852, 121, 1290, 1096, 0, 1095, 1172, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6276, 6190, 5766, 6396, 5767, 6066, 6529, 6162] avg=6144.0000 max=6529 max_rank=6 min=5766 min_rank=2 imbalance_ratio=1.062663 max_load_reduction=42623 max_load_reduction_pct=0.867167 moved_to_echo=24524 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 501, 0, 1268, 1015, 1122, 630, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 410, 1152, 2618, 77, 153, 831, 323, 621, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 4, 146, 51, 202, 785, 375, 224, 2108, 636, 0, 0, 0, 0, 0, 0, 792, 330, 2416, 264, 1547, 635, 349, 67, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 2, 2442, 788, 149, 77, 1528, 419, 311, 258, 158, 186, 0, 0, 0, 267, 324, 5, 807, 535, 1, 2448, 0, 878, 1017, 0, 0, 0, 0, 0, 0, 43, 1062, 204, 1048, 392, 1683, 1127, 986, 0, 0, 0, 0, 0, 0, 0, 0, 605, 714, 136, 1136, 1103, 0, 931, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5794, 6185, 5706, 6400, 6572, 6282, 6545, 5668] avg=6144.0000 max=6572 max_rank=4 min=5668 min_rank=7 imbalance_ratio=1.069661 max_load_reduction=42580 max_load_reduction_pct=0.866292 moved_to_echo=25067 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 622, 0, 796, 1122, 2149, 599, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 443, 795, 2183, 102, 134, 471, 295, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 3, 195, 60, 351, 610, 241, 228, 2099, 590, 0, 0, 0, 0, 0, 0, 456, 346, 2577, 244, 1637, 643, 428, 71, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1, 0, 2188, 720, 215, 55, 1401, 520, 259, 203, 126, 198, 0, 0, 0, 260, 345, 13, 938, 409, 0, 2227, 0, 812, 1078, 0, 0, 0, 0, 0, 0, 19, 707, 232, 1140, 418, 1325, 1384, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 441, 856, 145, 1385, 895, 0, 1422, 828, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6774, 5392, 5916, 6402, 6132, 6082, 6482, 5972] avg=6144.0000 max=6774 max_rank=0 min=5392 min_rank=1 imbalance_ratio=1.102539 max_load_reduction=42378 max_load_reduction_pct=0.862183 moved_to_echo=24668 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 1137, 0, 515, 1063, 1562, 742, 977, 0, 0, 0, 0, 0, 0, 0, 0, 592, 1225, 2079, 193, 202, 625, 483, 883, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 14, 248, 129, 145, 522, 325, 353, 2520, 704, 0, 0, 0, 0, 0, 0, 797, 400, 1434, 600, 1034, 1212, 485, 112, 0, 0, 0, 0, 0, 0, 0, 0, 279, 86, 37, 1409, 967, 376, 187, 1454, 953, 247, 170, 171, 124, 0, 0, 0, 382, 719, 383, 989, 576, 15, 1692, 67, 973, 680, 0, 0, 0, 0, 0, 0, 330, 678, 320, 765, 465, 1111, 972, 881, 0, 0, 0, 0, 0, 0, 0, 0, 936, 826, 191, 938, 997, 0, 957, 1128, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6021, 6282, 6344, 6074, 6460, 6476, 5522, 5973] avg=6144.0000 max=6476 max_rank=5 min=5522 min_rank=6 imbalance_ratio=1.054036 max_load_reduction=42676 max_load_reduction_pct=0.868245 moved_to_echo=24431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 545, 0, 898, 768, 1936, 600, 1464, 0, 0, 0, 0, 0, 0, 0, 0, 585, 1345, 2367, 109, 178, 616, 338, 546, 0, 0, 0, 0, 0, 0, 0, 0, 1736, 10, 121, 40, 176, 974, 335, 223, 2011, 616, 0, 0, 0, 0, 0, 0, 670, 336, 1770, 185, 1654, 514, 329, 53, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 2635, 586, 48, 22, 1290, 466, 285, 299, 180, 204, 0, 0, 0, 174, 594, 11, 808, 348, 0, 2584, 1, 848, 1098, 0, 0, 0, 0, 0, 0, 5, 1132, 167, 728, 406, 1928, 1343, 917, 0, 0, 0, 0, 0, 0, 0, 0, 423, 1054, 98, 1261, 888, 0, 899, 1143, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 6084, 6242, 5511, 6246, 6466, 6626, 5766] avg=6144.0000 max=6626 max_rank=6 min=5511 min_rank=3 imbalance_ratio=1.078451 max_load_reduction=42526 max_load_reduction_pct=0.865194 moved_to_echo=25104 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 1039, 0, 884, 1130, 1346, 733, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 451, 1559, 2221, 129, 170, 535, 355, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 35, 131, 54, 132, 552, 256, 413, 2837, 710, 0, 0, 0, 0, 0, 0, 956, 417, 1665, 565, 962, 945, 627, 83, 0, 0, 0, 0, 0, 0, 0, 0, 251, 19, 33, 1256, 1124, 242, 181, 1214, 870, 264, 166, 163, 116, 0, 0, 0, 375, 450, 62, 881, 586, 12, 1703, 3, 980, 632, 0, 0, 0, 0, 0, 0, 243, 867, 350, 794, 428, 1088, 882, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 927, 775, 167, 1155, 1171, 0, 997, 1073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6347, 6540, 6399, 6220, 5899, 5684, 5798, 6265] avg=6144.0000 max=6540 max_rank=1 min=5684 min_rank=5 imbalance_ratio=1.064453 max_load_reduction=42612 max_load_reduction_pct=0.866943 moved_to_echo=23646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 622, 0, 796, 1122, 2149, 599, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 443, 795, 2183, 102, 134, 471, 295, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 3, 195, 60, 351, 610, 241, 228, 2099, 590, 0, 0, 0, 0, 0, 0, 456, 346, 2577, 244, 1637, 643, 428, 71, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1, 0, 2188, 720, 215, 55, 1401, 520, 259, 203, 126, 198, 0, 0, 0, 260, 345, 13, 938, 409, 0, 2227, 0, 812, 1078, 0, 0, 0, 0, 0, 0, 19, 707, 232, 1140, 418, 1325, 1384, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 441, 856, 145, 1385, 895, 0, 1422, 828, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6774, 5392, 5916, 6402, 6132, 6082, 6482, 5972] avg=6144.0000 max=6774 max_rank=0 min=5392 min_rank=1 imbalance_ratio=1.102539 max_load_reduction=42378 max_load_reduction_pct=0.862183 moved_to_echo=24668 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 1012, 0, 813, 1083, 1113, 677, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 632, 1341, 2293, 99, 159, 642, 368, 750, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 24, 165, 65, 154, 643, 284, 327, 2623, 700, 0, 0, 0, 0, 0, 0, 987, 385, 1594, 458, 1046, 924, 542, 74, 0, 0, 0, 0, 0, 0, 0, 0, 250, 5, 9, 1589, 1095, 220, 126, 1155, 848, 272, 177, 196, 126, 0, 0, 0, 287, 700, 87, 907, 584, 5, 1981, 8, 968, 687, 0, 0, 0, 0, 0, 0, 123, 819, 253, 755, 434, 1159, 781, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 934, 839, 175, 1218, 1224, 0, 1064, 1290, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5924, 6284, 6472, 6010, 6068, 6214, 5436, 6744] avg=6144.0000 max=6744 max_rank=7 min=5436 min_rank=6 imbalance_ratio=1.097656 max_load_reduction=42408 max_load_reduction_pct=0.862793 moved_to_echo=24462 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 1137, 0, 515, 1063, 1562, 742, 977, 0, 0, 0, 0, 0, 0, 0, 0, 592, 1225, 2079, 193, 202, 625, 483, 883, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 14, 248, 129, 145, 522, 325, 353, 2520, 704, 0, 0, 0, 0, 0, 0, 797, 400, 1434, 600, 1034, 1212, 485, 112, 0, 0, 0, 0, 0, 0, 0, 0, 279, 86, 37, 1409, 967, 376, 187, 1454, 953, 247, 170, 171, 124, 0, 0, 0, 382, 719, 383, 989, 576, 15, 1692, 67, 973, 680, 0, 0, 0, 0, 0, 0, 330, 678, 320, 765, 465, 1111, 972, 881, 0, 0, 0, 0, 0, 0, 0, 0, 936, 826, 191, 938, 997, 0, 957, 1128, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6021, 6282, 6344, 6074, 6460, 6476, 5522, 5973] avg=6144.0000 max=6476 max_rank=5 min=5522 min_rank=6 imbalance_ratio=1.054036 max_load_reduction=42676 max_load_reduction_pct=0.868245 moved_to_echo=24431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 1087, 0, 815, 1109, 1156, 693, 942, 0, 0, 0, 0, 0, 0, 0, 0, 401, 1285, 2397, 87, 105, 615, 349, 956, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 73, 119, 87, 143, 544, 221, 411, 2630, 790, 0, 0, 0, 0, 0, 0, 1042, 393, 1493, 619, 854, 1077, 597, 64, 0, 0, 0, 0, 0, 0, 0, 0, 292, 6, 15, 1145, 1235, 328, 146, 1199, 910, 284, 164, 181, 103, 0, 0, 0, 325, 546, 146, 822, 639, 0, 1752, 2, 1089, 561, 0, 0, 0, 0, 0, 0, 335, 840, 339, 1040, 511, 1071, 827, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 702, 166, 1126, 1223, 0, 1174, 1193, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5805, 6195, 6307, 6139, 6008, 5882, 6214, 6602] avg=6144.0000 max=6602 max_rank=7 min=5805 min_rank=0 imbalance_ratio=1.074544 max_load_reduction=42550 max_load_reduction_pct=0.865682 moved_to_echo=24706 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 545, 0, 898, 768, 1936, 600, 1464, 0, 0, 0, 0, 0, 0, 0, 0, 585, 1345, 2367, 109, 178, 616, 338, 546, 0, 0, 0, 0, 0, 0, 0, 0, 1736, 10, 121, 40, 176, 974, 335, 223, 2011, 616, 0, 0, 0, 0, 0, 0, 670, 336, 1770, 185, 1654, 514, 329, 53, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 2635, 586, 48, 22, 1290, 466, 285, 299, 180, 204, 0, 0, 0, 174, 594, 11, 808, 348, 0, 2584, 1, 848, 1098, 0, 0, 0, 0, 0, 0, 5, 1132, 167, 728, 406, 1928, 1343, 917, 0, 0, 0, 0, 0, 0, 0, 0, 423, 1054, 98, 1261, 888, 0, 899, 1143, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 6084, 6242, 5511, 6246, 6466, 6626, 5766] avg=6144.0000 max=6626 max_rank=6 min=5511 min_rank=3 imbalance_ratio=1.078451 max_load_reduction=42526 max_load_reduction_pct=0.865194 moved_to_echo=25104 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5888, 6656, 1280, 9600, 8704, 0, 8576, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 7296, 2176, 7424, 3456, 10880, 8704, 8704, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 6656, 0, 7168, 8320, 12032, 5376, 9984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5888, 6656, 1280, 9600, 8704, 0, 8576, 8960, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4096, 9856, 18944, 896, 1280, 4992, 2944, 6656, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2304, 3968, 768, 7040, 4224, 128, 16768, 128, 7424, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11392, 256, 1280, 640, 1536, 5376, 2432, 2432, 19072, 5376, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 128, 128, 14848, 7296, 1792, 896, 10496, 5504, 2304, 1664, 1408, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11392, 256, 1280, 640, 1536, 5376, 2432, 2432, 19072, 5376, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 3072, 15360, 3200, 10368, 6784, 3840, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[627, 2068, 998, 285, 102, 303, 2243, 418, 38, 8, 679, 845, 1788, 5, 8, 1206, 657, 953, 3296, 39, 173, 1132, 291, 10, 174, 2034, 860, 132, 133, 182, 3994, 238, 1254, 447, 109, 1509, 875, 322, 1437, 876, 70, 19, 30, 1062, 97, 654, 1583, 637, 945, 1495, 1413, 872, 161, 608, 17, 1363, 1519, 475, 466, 554, 438, 383, 761, 782] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[555, 1907, 1003, 231, 125, 311, 2191, 249, 59, 2, 542, 965, 1702, 0, 8, 913, 772, 838, 3303, 55, 213, 882, 338, 6, 130, 1623, 728, 135, 140, 217, 3623, 219, 1514, 480, 159, 1731, 813, 332, 1790, 1125, 109, 2, 34, 1600, 97, 566, 1528, 626, 965, 1779, 1274, 939, 168, 580, 1, 1360, 1546, 505, 483, 631, 384, 407, 855, 784] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[734, 2098, 1106, 353, 226, 294, 2121, 381, 135, 23, 399, 956, 1848, 1, 27, 1307, 781, 929, 3005, 96, 211, 943, 417, 38, 242, 1339, 797, 122, 185, 228, 3000, 333, 1384, 618, 201, 2138, 689, 398, 1715, 789, 125, 55, 44, 1214, 323, 459, 1729, 810, 1051, 1439, 1187, 977, 237, 558, 20, 1600, 1242, 526, 499, 510, 322, 339, 676, 603] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[495, 2152, 795, 326, 64, 250, 1996, 743, 3, 0, 802, 740, 2612, 0, 0, 725, 507, 851, 3470, 20, 114, 1831, 296, 0, 75, 2124, 947, 111, 76, 139, 3705, 249, 1551, 432, 79, 1501, 765, 279, 1149, 850, 36, 0, 11, 1177, 92, 867, 1365, 656, 867, 1382, 1944, 1347, 142, 478, 0, 1288, 1639, 444, 409, 391, 245, 252, 656, 640] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[543, 1952, 746, 295, 83, 234, 2419, 657, 5, 0, 706, 669, 2002, 0, 0, 812, 502, 828, 3459, 28, 83, 1620, 232, 0, 182, 2082, 916, 125, 63, 162, 3894, 223, 1550, 374, 92, 1420, 906, 285, 1047, 1165, 30, 0, 17, 1040, 130, 992, 1476, 800, 864, 1308, 1582, 1677, 147, 467, 0, 1423, 1626, 450, 399, 356, 285, 245, 706, 771] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[555, 1907, 1003, 231, 125, 311, 2191, 249, 59, 2, 542, 965, 1702, 0, 8, 913, 772, 838, 3303, 55, 213, 882, 338, 6, 130, 1623, 728, 135, 140, 217, 3623, 219, 1514, 480, 159, 1731, 813, 332, 1790, 1125, 109, 2, 34, 1600, 97, 566, 1528, 626, 965, 1779, 1274, 939, 168, 580, 1, 1360, 1546, 505, 483, 631, 384, 407, 855, 784] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[687, 2464, 829, 242, 77, 261, 2422, 952, 4, 0, 728, 817, 2750, 0, 0, 967, 551, 835, 3628, 39, 51, 1665, 331, 0, 143, 2034, 946, 138, 68, 191, 3033, 401, 1762, 355, 68, 1647, 613, 297, 1063, 995, 45, 0, 6, 1177, 88, 707, 1118, 557, 824, 1308, 1891, 1265, 70, 480, 0, 1301, 1603, 353, 426, 305, 220, 210, 601, 543] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[627, 2068, 998, 285, 102, 303, 2243, 418, 38, 8, 679, 845, 1788, 5, 8, 1206, 657, 953, 3296, 39, 173, 1132, 291, 10, 174, 2034, 860, 132, 133, 182, 3994, 238, 1254, 447, 109, 1509, 875, 322, 1437, 876, 70, 19, 30, 1062, 97, 654, 1583, 637, 945, 1495, 1413, 872, 161, 608, 17, 1363, 1519, 475, 466, 554, 438, 383, 761, 782] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[488, 1919, 1032, 285, 127, 241, 2366, 269, 43, 0, 592, 883, 1736, 0, 1, 1199, 952, 885, 3291, 67, 304, 884, 207, 0, 122, 1792, 760, 115, 177, 178, 3981, 303, 1356, 435, 128, 1611, 876, 308, 1654, 1103, 210, 0, 77, 1356, 123, 588, 1482, 547, 928, 1510, 1183, 961, 163, 700, 0, 1250, 1587, 451, 463, 546, 422, 477, 730, 728] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[411, 1966, 770, 425, 60, 285, 2074, 748, 1, 1, 746, 633, 2714, 0, 2, 676, 516, 797, 3522, 24, 68, 1929, 333, 3, 127, 1614, 852, 131, 74, 155, 3556, 198, 2044, 368, 46, 1782, 799, 325, 1158, 975, 21, 0, 11, 808, 85, 659, 1464, 775, 990, 1170, 1771, 1660, 196, 531, 2, 1335, 1858, 473, 427, 350, 263, 214, 568, 613] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[687, 2464, 829, 242, 77, 261, 2422, 952, 4, 0, 728, 817, 2750, 0, 0, 967, 551, 835, 3628, 39, 51, 1665, 331, 0, 143, 2034, 946, 138, 68, 191, 3033, 401, 1762, 355, 68, 1647, 613, 297, 1063, 995, 45, 0, 6, 1177, 88, 707, 1118, 557, 824, 1308, 1891, 1265, 70, 480, 0, 1301, 1603, 353, 426, 305, 220, 210, 601, 543] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[734, 2098, 1106, 353, 226, 294, 1298, 381, 0, 0, 0, 0, 0, 0, 0, 0, 135, 23, 399, 956, 1848, 1, 27, 1307, 814, 280, 188, 0, 0, 0, 0, 0, 781, 929, 2454, 96, 211, 943, 417, 38, 0, 0, 0, 0, 0, 0, 0, 0, 242, 1339, 797, 122, 185, 228, 2186, 333, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 618, 201, 733, 689, 398, 1715, 789, 0, 0, 0, 0, 0, 0, 0, 0, 125, 55, 44, 1214, 323, 459, 1729, 810, 1405, 612, 0, 0, 0, 0, 0, 0, 1051, 1439, 387, 977, 237, 558, 20, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 1242, 526, 499, 510, 322, 339, 676, 603, 551, 543, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6490, 5978, 5869, 5432, 6527, 6776, 6269, 5811] avg=6144.0000 max=6776 max_rank=5 min=5432 min_rank=3 imbalance_ratio=1.102865 max_load_reduction=42376 max_load_reduction_pct=0.862142 moved_to_echo=25383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 2152, 795, 326, 64, 250, 1221, 743, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 802, 740, 2612, 0, 0, 725, 1006, 264, 308, 0, 0, 0, 0, 0, 507, 851, 2834, 20, 114, 1831, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 2124, 947, 111, 76, 139, 2699, 249, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 432, 79, 515, 765, 279, 1149, 850, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 11, 1177, 92, 867, 1365, 656, 986, 1003, 0, 0, 0, 0, 0, 0, 867, 1382, 633, 1347, 142, 478, 0, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 1639, 444, 409, 391, 245, 252, 656, 640, 636, 511, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 6460, 6453, 6420, 5620, 6193, 6137, 5823] avg=6144.0000 max=6460 max_rank=1 min=5620 min_rank=4 imbalance_ratio=1.051432 max_load_reduction=42692 max_load_reduction_pct=0.868571 moved_to_echo=23773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[495, 2152, 795, 326, 64, 250, 1996, 743, 3, 0, 802, 740, 2612, 0, 0, 725, 507, 851, 3470, 20, 114, 1831, 296, 0, 75, 2124, 947, 111, 76, 139, 3705, 249, 1551, 432, 79, 1501, 765, 279, 1149, 850, 36, 0, 11, 1177, 92, 867, 1365, 656, 867, 1382, 1944, 1347, 142, 478, 0, 1288, 1639, 444, 409, 391, 245, 252, 656, 640] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[543, 1952, 746, 295, 83, 234, 1480, 657, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 706, 669, 2002, 0, 0, 812, 1057, 320, 250, 0, 0, 0, 0, 0, 502, 828, 2825, 28, 83, 1620, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 2082, 916, 125, 63, 162, 2837, 223, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 374, 92, 487, 906, 285, 1047, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 17, 1040, 130, 992, 1476, 800, 933, 816, 0, 0, 0, 0, 0, 0, 864, 1308, 516, 1677, 147, 467, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 1626, 450, 399, 356, 285, 245, 706, 771, 634, 619, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5990, 5821, 6118, 6590, 5906, 6234, 6402, 6091] avg=6144.0000 max=6590 max_rank=3 min=5821 min_rank=1 imbalance_ratio=1.072591 max_load_reduction=42562 max_load_reduction_pct=0.865926 moved_to_echo=24633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[555, 1907, 1003, 231, 125, 311, 1341, 249, 0, 0, 0, 0, 0, 0, 0, 0, 59, 2, 542, 965, 1702, 0, 8, 913, 983, 289, 202, 0, 0, 0, 0, 0, 772, 838, 2697, 55, 213, 882, 338, 6, 0, 0, 0, 0, 0, 0, 0, 0, 130, 1623, 728, 135, 140, 217, 2640, 219, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 480, 159, 594, 813, 332, 1790, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 109, 2, 34, 1600, 97, 566, 1528, 626, 1137, 657, 0, 0, 0, 0, 0, 0, 965, 1779, 415, 939, 168, 580, 1, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 1546, 505, 483, 631, 384, 407, 855, 784, 606, 561, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5722, 5665, 5801, 5832, 6807, 6356, 6207, 6762] avg=6144.0000 max=6807 max_rank=4 min=5665 min_rank=1 imbalance_ratio=1.107910 max_load_reduction=42345 max_load_reduction_pct=0.861511 moved_to_echo=26132 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[627, 2068, 998, 285, 102, 303, 1373, 418, 0, 0, 0, 0, 0, 0, 0, 0, 38, 8, 679, 845, 1788, 5, 8, 1206, 1084, 296, 224, 0, 0, 0, 0, 0, 657, 953, 2692, 39, 173, 1132, 291, 10, 0, 0, 0, 0, 0, 0, 0, 0, 174, 2034, 860, 132, 133, 182, 2910, 238, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 447, 109, 518, 875, 322, 1437, 876, 0, 0, 0, 0, 0, 0, 0, 0, 70, 19, 30, 1062, 97, 654, 1583, 637, 991, 729, 0, 0, 0, 0, 0, 0, 945, 1495, 460, 872, 161, 608, 17, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 1519, 475, 466, 554, 438, 383, 761, 782, 604, 574, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6174, 6181, 5947, 6663, 5838, 5872, 5921, 6556] avg=6144.0000 max=6663 max_rank=3 min=5838 min_rank=4 imbalance_ratio=1.084473 max_load_reduction=42489 max_load_reduction_pct=0.864441 moved_to_echo=24187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[411, 1966, 770, 425, 60, 285, 1262, 748, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 746, 633, 2714, 0, 2, 676, 970, 278, 284, 0, 0, 0, 0, 0, 516, 797, 2872, 24, 68, 1929, 333, 3, 0, 0, 0, 0, 0, 0, 0, 0, 127, 1614, 852, 131, 74, 155, 2586, 198, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 368, 46, 608, 799, 325, 1158, 975, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 11, 808, 85, 659, 1464, 775, 1174, 918, 0, 0, 0, 0, 0, 0, 990, 1170, 569, 1660, 196, 531, 2, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 473, 427, 350, 263, 214, 568, 613, 650, 534, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5927, 6305, 6542, 5737, 6323, 5915, 6453, 5950] avg=6144.0000 max=6542 max_rank=2 min=5737 min_rank=3 imbalance_ratio=1.064779 max_load_reduction=42610 max_load_reduction_pct=0.866903 moved_to_echo=24641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[734, 2098, 1106, 353, 226, 294, 1298, 381, 0, 0, 0, 0, 0, 0, 0, 0, 135, 23, 399, 956, 1848, 1, 27, 1307, 814, 280, 188, 0, 0, 0, 0, 0, 781, 929, 2454, 96, 211, 943, 417, 38, 0, 0, 0, 0, 0, 0, 0, 0, 242, 1339, 797, 122, 185, 228, 2186, 333, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 618, 201, 733, 689, 398, 1715, 789, 0, 0, 0, 0, 0, 0, 0, 0, 125, 55, 44, 1214, 323, 459, 1729, 810, 1405, 612, 0, 0, 0, 0, 0, 0, 1051, 1439, 387, 977, 237, 558, 20, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 1242, 526, 499, 510, 322, 339, 676, 603, 551, 543, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6490, 5978, 5869, 5432, 6527, 6776, 6269, 5811] avg=6144.0000 max=6776 max_rank=5 min=5432 min_rank=3 imbalance_ratio=1.102865 max_load_reduction=42376 max_load_reduction_pct=0.862142 moved_to_echo=25383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[687, 2464, 829, 242, 77, 261, 1482, 952, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 728, 817, 2750, 0, 0, 967, 823, 320, 299, 0, 0, 0, 0, 0, 551, 835, 2963, 39, 51, 1665, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 2034, 946, 138, 68, 191, 2210, 401, 0, 0, 0, 0, 0, 0, 0, 0, 1762, 355, 68, 565, 613, 297, 1063, 995, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 6, 1177, 88, 707, 1118, 557, 1082, 975, 0, 0, 0, 0, 0, 0, 824, 1308, 617, 1265, 70, 480, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 353, 426, 305, 220, 210, 601, 543, 665, 620, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6994, 6708, 6435, 6131, 5718, 5755, 5865, 5546] avg=6144.0000 max=6994 max_rank=0 min=5546 min_rank=7 imbalance_ratio=1.138346 max_load_reduction=42158 max_load_reduction_pct=0.857707 moved_to_echo=22884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[488, 1919, 1032, 285, 127, 241, 1448, 269, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 592, 883, 1736, 0, 1, 1199, 1081, 312, 187, 0, 0, 0, 0, 0, 952, 885, 2688, 67, 304, 884, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 1792, 760, 115, 177, 178, 2900, 303, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 435, 128, 553, 876, 308, 1654, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 77, 1356, 123, 588, 1482, 547, 1058, 610, 0, 0, 0, 0, 0, 0, 928, 1510, 386, 961, 163, 700, 0, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 1587, 451, 463, 546, 422, 477, 730, 728, 603, 606, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5809, 6034, 5987, 6347, 6413, 6051, 5898, 6613] avg=6144.0000 max=6613 max_rank=7 min=5809 min_rank=0 imbalance_ratio=1.076335 max_load_reduction=42539 max_load_reduction_pct=0.865458 moved_to_echo=24975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 2152, 795, 326, 64, 250, 1221, 743, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 802, 740, 2612, 0, 0, 725, 1006, 264, 308, 0, 0, 0, 0, 0, 507, 851, 2834, 20, 114, 1831, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 2124, 947, 111, 76, 139, 2699, 249, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 432, 79, 515, 765, 279, 1149, 850, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 11, 1177, 92, 867, 1365, 656, 986, 1003, 0, 0, 0, 0, 0, 0, 867, 1382, 633, 1347, 142, 478, 0, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 1639, 444, 409, 391, 245, 252, 656, 640, 636, 511, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 6460, 6453, 6420, 5620, 6193, 6137, 5823] avg=6144.0000 max=6460 max_rank=1 min=5620 min_rank=4 imbalance_ratio=1.051432 max_load_reduction=42692 max_load_reduction_pct=0.868571 moved_to_echo=23773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[411, 1966, 770, 425, 60, 285, 1262, 748, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 746, 633, 2714, 0, 2, 676, 970, 278, 284, 0, 0, 0, 0, 0, 516, 797, 2872, 24, 68, 1929, 333, 3, 0, 0, 0, 0, 0, 0, 0, 0, 127, 1614, 852, 131, 74, 155, 2586, 198, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 368, 46, 608, 799, 325, 1158, 975, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 11, 808, 85, 659, 1464, 775, 1174, 918, 0, 0, 0, 0, 0, 0, 990, 1170, 569, 1660, 196, 531, 2, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 473, 427, 350, 263, 214, 568, 613, 650, 534, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5927, 6305, 6542, 5737, 6323, 5915, 6453, 5950] avg=6144.0000 max=6542 max_rank=2 min=5737 min_rank=3 imbalance_ratio=1.064779 max_load_reduction=42610 max_load_reduction_pct=0.866903 moved_to_echo=24641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 128, 256, 9472, 1152, 5504, 11776, 5504, 8832, 6400, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7552, 11392, 4096, 9728, 1408, 4480, 128, 11008, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 16640, 7296, 2560, 896, 2304, 11008, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12672, 3712, 3584, 3712, 2688, 2560, 5632, 5504, 4992, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7552, 11392, 4096, 9728, 1408, 4480, 128, 11008, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 128, 256, 9472, 1152, 5504, 11776, 5504, 8832, 6400, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 128, 5248, 6528, 17152, 128, 128, 7808, 7936, 2432, 2048, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12416, 3584, 896, 4608, 6400, 2560, 11136, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12672, 3712, 3584, 3712, 2688, 2560, 5632, 5504, 4992, 4608, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 7040, 22144, 384, 1280, 11008, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 14720, 6912, 1024, 1024, 1536, 20992, 2176, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[784, 130, 1, 3570, 69, 5, 276, 82, 0, 1006, 699, 699, 903, 956, 452, 763, 2, 3, 762, 18, 9, 417, 410, 2370, 135, 83, 1159, 5, 408, 110, 0, 3, 74, 1696, 257, 23, 1, 0, 0, 2078, 1230, 287, 220, 1916, 5369, 91, 1358, 3065, 417, 525, 0, 2127, 323, 649, 5892, 1, 4785, 1, 0, 19, 13, 2, 316, 128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[571, 91, 0, 3861, 92, 0, 192, 76, 0, 1487, 600, 801, 876, 320, 375, 603, 0, 0, 833, 1, 0, 485, 415, 1504, 133, 73, 514, 0, 151, 95, 0, 0, 79, 1462, 94, 0, 0, 0, 0, 3917, 1380, 266, 193, 1846, 5684, 93, 1067, 4358, 351, 485, 0, 1181, 362, 1094, 6067, 0, 4406, 0, 0, 3, 0, 0, 465, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[696, 115, 0, 4026, 92, 0, 185, 100, 0, 1208, 443, 587, 866, 590, 378, 570, 0, 0, 783, 0, 1, 518, 408, 1044, 130, 54, 625, 0, 126, 91, 0, 1, 101, 1586, 64, 0, 0, 2, 1, 3531, 1163, 239, 184, 1991, 5614, 94, 1247, 4236, 337, 387, 0, 1637, 368, 1438, 6166, 1, 4502, 0, 0, 8, 4, 1, 448, 165] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[653, 101, 0, 4151, 91, 0, 192, 83, 0, 1010, 473, 555, 835, 496, 386, 463, 0, 0, 719, 1, 1, 468, 342, 1367, 120, 68, 661, 0, 292, 226, 0, 0, 85, 1469, 136, 0, 0, 0, 0, 4007, 1280, 260, 191, 1859, 5524, 96, 834, 4363, 360, 163, 0, 2248, 383, 1066, 5964, 0, 4546, 0, 0, 2, 0, 0, 410, 152] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[631, 91, 0, 4220, 87, 0, 177, 78, 0, 1161, 473, 603, 882, 495, 404, 570, 0, 0, 712, 3, 0, 441, 375, 1292, 113, 61, 582, 0, 157, 233, 0, 0, 78, 1471, 101, 0, 0, 0, 0, 4222, 1229, 236, 177, 1842, 5724, 88, 985, 4306, 390, 307, 0, 1652, 317, 986, 6185, 0, 4540, 0, 0, 2, 0, 0, 322, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[762, 113, 4, 3778, 93, 11, 203, 90, 1, 973, 643, 639, 870, 921, 431, 566, 0, 1, 778, 10, 13, 471, 428, 1982, 119, 65, 1040, 0, 310, 81, 1, 1, 96, 1551, 213, 0, 3, 19, 20, 2801, 1301, 251, 187, 1879, 5370, 86, 1187, 3505, 389, 464, 7, 1952, 358, 748, 5921, 18, 4918, 2, 16, 16, 12, 1, 318, 145] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[694, 78, 0, 3583, 62, 0, 226, 52, 0, 850, 785, 781, 955, 940, 447, 665, 0, 0, 810, 2, 10, 424, 476, 2440, 117, 62, 1293, 0, 309, 264, 0, 0, 52, 1560, 214, 7, 0, 0, 0, 2434, 1202, 243, 183, 1947, 5439, 79, 1126, 3408, 404, 310, 0, 1898, 308, 573, 6142, 0, 4935, 0, 0, 7, 0, 0, 259, 97] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[631, 91, 0, 4220, 87, 0, 177, 78, 0, 1161, 473, 603, 882, 495, 404, 570, 0, 0, 712, 3, 0, 441, 375, 1292, 113, 61, 582, 0, 157, 233, 0, 0, 78, 1471, 101, 0, 0, 0, 0, 4222, 1229, 236, 177, 1842, 5724, 88, 985, 4306, 390, 307, 0, 1652, 317, 986, 6185, 0, 4540, 0, 0, 2, 0, 0, 322, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[653, 101, 0, 4151, 91, 0, 192, 83, 0, 1010, 473, 555, 835, 496, 386, 463, 0, 0, 719, 1, 1, 468, 342, 1367, 120, 68, 661, 0, 292, 226, 0, 0, 85, 1469, 136, 0, 0, 0, 0, 4007, 1280, 260, 191, 1859, 5524, 96, 834, 4363, 360, 163, 0, 2248, 383, 1066, 5964, 0, 4546, 0, 0, 2, 0, 0, 410, 152] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[696, 115, 0, 4026, 92, 0, 185, 100, 0, 1208, 443, 587, 866, 590, 378, 570, 0, 0, 783, 0, 1, 518, 408, 1044, 130, 54, 625, 0, 126, 91, 0, 1, 101, 1586, 64, 0, 0, 2, 1, 3531, 1163, 239, 184, 1991, 5614, 94, 1247, 4236, 337, 387, 0, 1637, 368, 1438, 6166, 1, 4502, 0, 0, 8, 4, 1, 448, 165] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[784, 130, 1, 3570, 69, 5, 276, 82, 0, 1006, 699, 699, 903, 956, 452, 763, 2, 3, 762, 18, 9, 417, 410, 2370, 135, 83, 1159, 5, 408, 110, 0, 3, 74, 1696, 257, 23, 1, 0, 0, 2078, 1230, 287, 220, 1916, 5369, 91, 1358, 3065, 417, 525, 0, 2127, 323, 649, 5892, 1, 4785, 1, 0, 19, 13, 2, 316, 128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[879, 171, 18, 3150, 149, 30, 260, 168, 40, 1208, 771, 798, 883, 788, 536, 610, 2, 23, 820, 68, 63, 604, 499, 1557, 106, 208, 1360, 6, 671, 210, 2, 53, 114, 1518, 288, 54, 11, 48, 22, 2045, 1422, 290, 231, 1995, 4658, 247, 1015, 3173, 394, 874, 75, 1853, 564, 842, 5422, 106, 4224, 38, 23, 108, 180, 0, 453, 154] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[694, 78, 0, 3583, 62, 0, 226, 52, 0, 850, 785, 781, 955, 940, 447, 665, 0, 0, 810, 2, 10, 424, 476, 2440, 117, 62, 1293, 0, 309, 264, 0, 0, 52, 1560, 214, 7, 0, 0, 0, 2434, 1202, 243, 183, 1947, 5439, 79, 1126, 3408, 404, 310, 0, 1898, 308, 573, 6142, 0, 4935, 0, 0, 7, 0, 0, 259, 97] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[784, 130, 1, 3570, 69, 5, 276, 82, 907, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 699, 699, 903, 956, 452, 763, 703, 338, 0, 0, 0, 0, 0, 0, 2, 3, 762, 18, 9, 417, 410, 2370, 1856, 856, 0, 0, 0, 0, 0, 0, 135, 83, 1159, 5, 408, 110, 0, 3, 4513, 0, 0, 0, 0, 0, 0, 0, 74, 1696, 257, 23, 1, 0, 0, 2078, 1177, 0, 0, 0, 0, 0, 0, 0, 1230, 287, 220, 1916, 0, 91, 1358, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 417, 525, 0, 2127, 323, 649, 2156, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4785, 1, 0, 19, 13, 2, 316, 128, 776, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5824, 6519, 6703, 6416, 5306, 6146, 6198, 6040] avg=6144.0000 max=6703 max_rank=2 min=5306 min_rank=4 imbalance_ratio=1.090983 max_load_reduction=42449 max_load_reduction_pct=0.863627 moved_to_echo=23690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[762, 113, 4, 3778, 93, 11, 203, 90, 1038, 0, 0, 0, 0, 0, 0, 0, 1, 973, 643, 639, 870, 921, 431, 566, 706, 387, 0, 0, 0, 0, 0, 0, 0, 1, 778, 10, 13, 471, 428, 1982, 1865, 856, 0, 0, 0, 0, 0, 0, 119, 65, 1040, 0, 310, 81, 1, 1, 4514, 0, 0, 0, 0, 0, 0, 0, 96, 1551, 213, 0, 3, 19, 20, 2801, 1183, 0, 0, 0, 0, 0, 0, 0, 1301, 251, 187, 1879, 0, 86, 1187, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 389, 464, 7, 1952, 358, 748, 2167, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4918, 2, 16, 16, 12, 1, 318, 145, 887, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6092, 6137, 6404, 6131, 5886, 6084, 6103, 6315] avg=6144.0000 max=6404 max_rank=2 min=5886 min_rank=4 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[653, 101, 0, 4151, 91, 0, 192, 83, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 473, 555, 835, 496, 386, 463, 712, 482, 0, 0, 0, 0, 0, 0, 0, 0, 719, 1, 1, 468, 342, 1367, 1879, 880, 0, 0, 0, 0, 0, 0, 120, 68, 661, 0, 292, 226, 0, 0, 4644, 0, 0, 0, 0, 0, 0, 0, 85, 1469, 136, 0, 0, 0, 0, 4007, 1192, 0, 0, 0, 0, 0, 0, 0, 1280, 260, 191, 1859, 0, 96, 834, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 360, 163, 0, 2248, 383, 1066, 2181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4546, 0, 0, 2, 0, 0, 410, 152, 1105, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6563, 5412, 5657, 6011, 6889, 6004, 6401, 6215] avg=6144.0000 max=6889 max_rank=4 min=5412 min_rank=1 imbalance_ratio=1.121257 max_load_reduction=42263 max_load_reduction_pct=0.859843 moved_to_echo=25509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[571, 91, 0, 3861, 92, 0, 192, 76, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 600, 801, 876, 320, 375, 603, 724, 481, 0, 0, 0, 0, 0, 0, 0, 0, 833, 1, 0, 485, 415, 1504, 1911, 907, 0, 0, 0, 0, 0, 0, 133, 73, 514, 0, 151, 95, 0, 0, 4777, 0, 0, 0, 0, 0, 0, 0, 79, 1462, 94, 0, 0, 0, 0, 3917, 1212, 0, 0, 0, 0, 0, 0, 0, 1380, 266, 193, 1846, 0, 93, 1067, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 351, 485, 0, 1181, 362, 1094, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4406, 0, 0, 3, 0, 0, 465, 150, 1104, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6173, 6267, 6056, 5743, 6764, 6328, 5693, 6128] avg=6144.0000 max=6764 max_rank=4 min=5693 min_rank=6 imbalance_ratio=1.100911 max_load_reduction=42388 max_load_reduction_pct=0.862386 moved_to_echo=24913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[784, 130, 1, 3570, 69, 5, 276, 82, 907, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 699, 699, 903, 956, 452, 763, 703, 338, 0, 0, 0, 0, 0, 0, 2, 3, 762, 18, 9, 417, 410, 2370, 1856, 856, 0, 0, 0, 0, 0, 0, 135, 83, 1159, 5, 408, 110, 0, 3, 4513, 0, 0, 0, 0, 0, 0, 0, 74, 1696, 257, 23, 1, 0, 0, 2078, 1177, 0, 0, 0, 0, 0, 0, 0, 1230, 287, 220, 1916, 0, 91, 1358, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 417, 525, 0, 2127, 323, 649, 2156, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4785, 1, 0, 19, 13, 2, 316, 128, 776, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5824, 6519, 6703, 6416, 5306, 6146, 6198, 6040] avg=6144.0000 max=6703 max_rank=2 min=5306 min_rank=4 imbalance_ratio=1.090983 max_load_reduction=42449 max_load_reduction_pct=0.863627 moved_to_echo=23690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[631, 91, 0, 4220, 87, 0, 177, 78, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 1161, 473, 603, 882, 495, 404, 570, 738, 475, 0, 0, 0, 0, 0, 0, 0, 0, 712, 3, 0, 441, 375, 1292, 1949, 912, 0, 0, 0, 0, 0, 0, 113, 61, 582, 0, 157, 233, 0, 0, 4812, 0, 0, 0, 0, 0, 0, 0, 78, 1471, 101, 0, 0, 0, 0, 4222, 1236, 0, 0, 0, 0, 0, 0, 0, 1229, 236, 177, 1842, 0, 88, 985, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 390, 307, 0, 1652, 317, 986, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4540, 0, 0, 2, 0, 0, 322, 151, 1090, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6559, 5801, 5684, 5958, 7108, 6023, 5914, 6105] avg=6144.0000 max=7108 max_rank=4 min=5684 min_rank=2 imbalance_ratio=1.156901 max_load_reduction=42044 max_load_reduction_pct=0.855387 moved_to_echo=25150 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[694, 78, 0, 3583, 62, 0, 226, 52, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 850, 785, 781, 955, 940, 447, 665, 733, 376, 0, 0, 0, 0, 0, 0, 0, 0, 810, 2, 10, 424, 476, 2440, 1935, 868, 0, 0, 0, 0, 0, 0, 117, 62, 1293, 0, 309, 264, 0, 0, 4571, 0, 0, 0, 0, 0, 0, 0, 52, 1560, 214, 7, 0, 0, 0, 2434, 1227, 0, 0, 0, 0, 0, 0, 0, 1202, 243, 183, 1947, 0, 79, 1126, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 404, 310, 0, 1898, 308, 573, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4935, 0, 0, 7, 0, 0, 259, 97, 863, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5704, 6532, 6965, 6616, 5494, 5940, 5740, 6161] avg=6144.0000 max=6965 max_rank=2 min=5494 min_rank=4 imbalance_ratio=1.133626 max_load_reduction=42187 max_load_reduction_pct=0.858297 moved_to_echo=23335 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[879, 171, 18, 3150, 149, 30, 260, 168, 939, 0, 0, 0, 0, 0, 0, 0, 40, 1208, 771, 798, 883, 788, 536, 610, 647, 350, 0, 0, 0, 0, 0, 0, 2, 23, 820, 68, 63, 604, 499, 1557, 1708, 742, 0, 0, 0, 0, 0, 0, 106, 208, 1360, 6, 671, 210, 2, 53, 3916, 0, 0, 0, 0, 0, 0, 0, 114, 1518, 288, 54, 11, 48, 22, 2045, 1083, 0, 0, 0, 0, 0, 0, 0, 1422, 290, 231, 1995, 0, 247, 1015, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 394, 874, 75, 1853, 564, 842, 1984, 106, 0, 0, 0, 0, 0, 0, 0, 0, 4224, 38, 23, 108, 180, 0, 453, 154, 803, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5764, 6631, 6086, 6532, 5183, 6281, 6692, 5983] avg=6144.0000 max=6692 max_rank=6 min=5183 min_rank=4 imbalance_ratio=1.089193 max_load_reduction=42460 max_load_reduction_pct=0.863851 moved_to_echo=24139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[762, 113, 4, 3778, 93, 11, 203, 90, 1038, 0, 0, 0, 0, 0, 0, 0, 1, 973, 643, 639, 870, 921, 431, 566, 706, 387, 0, 0, 0, 0, 0, 0, 0, 1, 778, 10, 13, 471, 428, 1982, 1865, 856, 0, 0, 0, 0, 0, 0, 119, 65, 1040, 0, 310, 81, 1, 1, 4514, 0, 0, 0, 0, 0, 0, 0, 96, 1551, 213, 0, 3, 19, 20, 2801, 1183, 0, 0, 0, 0, 0, 0, 0, 1301, 251, 187, 1879, 0, 86, 1187, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 389, 464, 7, 1952, 358, 748, 2167, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4918, 2, 16, 16, 12, 1, 318, 145, 887, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6092, 6137, 6404, 6131, 5886, 6084, 6103, 6315] avg=6144.0000 max=6404 max_rank=2 min=5886 min_rank=4 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[696, 115, 0, 4026, 92, 0, 185, 100, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 443, 587, 866, 590, 378, 570, 739, 472, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 1, 518, 408, 1044, 1947, 896, 0, 0, 0, 0, 0, 0, 130, 54, 625, 0, 126, 91, 0, 1, 4718, 0, 0, 0, 0, 0, 0, 0, 101, 1586, 64, 0, 0, 2, 1, 3531, 1237, 0, 0, 0, 0, 0, 0, 0, 1163, 239, 184, 1991, 0, 94, 1247, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 337, 387, 0, 1637, 368, 1438, 2243, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4502, 0, 0, 8, 4, 1, 448, 165, 1077, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6473, 5853, 5597, 5745, 6522, 6346, 6411, 6205] avg=6144.0000 max=6522 max_rank=4 min=5597 min_rank=2 imbalance_ratio=1.061523 max_load_reduction=42630 max_load_reduction_pct=0.867310 moved_to_echo=25484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[694, 78, 0, 3583, 62, 0, 226, 52, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 850, 785, 781, 955, 940, 447, 665, 733, 376, 0, 0, 0, 0, 0, 0, 0, 0, 810, 2, 10, 424, 476, 2440, 1935, 868, 0, 0, 0, 0, 0, 0, 117, 62, 1293, 0, 309, 264, 0, 0, 4571, 0, 0, 0, 0, 0, 0, 0, 52, 1560, 214, 7, 0, 0, 0, 2434, 1227, 0, 0, 0, 0, 0, 0, 0, 1202, 243, 183, 1947, 0, 79, 1126, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 404, 310, 0, 1898, 308, 573, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4935, 0, 0, 7, 0, 0, 259, 97, 863, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5704, 6532, 6965, 6616, 5494, 5940, 5740, 6161] avg=6144.0000 max=6965 max_rank=2 min=5494 min_rank=4 imbalance_ratio=1.133626 max_load_reduction=42187 max_load_reduction_pct=0.858297 moved_to_echo=23335 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10240, 2176, 1664, 15360, 0, 896, 8832, 10368, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 8960, 4992, 5504, 7168, 5632, 3456, 4864, 5760, 3456, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 6272, 128, 128, 3840, 3456, 13568, 15104, 7040, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3072, 3584, 128, 14592, 3072, 7424, 17536, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 12416, 1408, 128, 128, 128, 128, 25088, 9600, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 896, 128, 30464, 768, 128, 1792, 768, 9088, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 12416, 1408, 128, 128, 128, 128, 25088, 9600, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[36864, 128, 128, 256, 256, 128, 3072, 1152, 7808, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 768, 7296, 128, 2432, 1408, 128, 128, 36480, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[3647, 7088, 0, 83, 503, 1, 68, 1, 0, 379, 92, 190, 74, 328, 564, 465, 164, 1769, 0, 0, 145, 90, 523, 1128, 4800, 2478, 2558, 213, 1, 11, 114, 381, 0, 392, 432, 0, 87, 2842, 747, 112, 623, 1, 0, 59, 1098, 6100, 151, 0, 4224, 0, 588, 545, 469, 80, 8, 0, 11, 393, 0, 0, 0, 1686, 351, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3585, 7041, 0, 84, 495, 0, 58, 3, 0, 450, 77, 213, 143, 301, 637, 387, 166, 1561, 0, 0, 201, 85, 319, 630, 4854, 2405, 1975, 217, 2, 5, 116, 420, 0, 309, 403, 0, 92, 3735, 776, 138, 722, 1, 0, 73, 1515, 6021, 330, 0, 4158, 0, 468, 517, 423, 2, 0, 0, 3, 290, 0, 0, 0, 2095, 299, 352] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[3070, 5940, 3, 166, 626, 41, 63, 62, 0, 456, 179, 352, 60, 324, 464, 929, 341, 1043, 1, 1, 246, 119, 1353, 1664, 2574, 3282, 2095, 281, 8, 61, 155, 375, 0, 298, 879, 1, 67, 2472, 628, 133, 654, 3, 1, 83, 2035, 6335, 603, 1, 4399, 2, 660, 482, 482, 77, 342, 2, 73, 436, 1, 0, 0, 1212, 174, 283] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[3783, 7052, 4, 62, 552, 0, 77, 0, 0, 339, 148, 198, 120, 347, 691, 457, 190, 1614, 3, 0, 150, 107, 447, 1110, 4665, 2193, 2811, 214, 0, 0, 169, 291, 0, 571, 301, 0, 93, 2794, 825, 94, 687, 1, 0, 53, 889, 6134, 259, 0, 4080, 2, 534, 521, 427, 56, 12, 1, 40, 346, 2, 0, 0, 1804, 503, 329] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3607, 7084, 0, 96, 526, 6, 69, 3, 0, 322, 126, 199, 146, 335, 571, 507, 174, 1549, 0, 0, 173, 95, 582, 1112, 4056, 2862, 2259, 236, 0, 13, 132, 371, 0, 506, 387, 1, 92, 2630, 737, 131, 696, 3, 0, 67, 1273, 6291, 227, 0, 4191, 0, 590, 563, 451, 87, 125, 0, 9, 356, 0, 0, 0, 1755, 377, 396] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[3070, 5940, 3, 166, 626, 41, 63, 62, 0, 456, 179, 352, 60, 324, 464, 929, 341, 1043, 1, 1, 246, 119, 1353, 1664, 2574, 3282, 2095, 281, 8, 61, 155, 375, 0, 298, 879, 1, 67, 2472, 628, 133, 654, 3, 1, 83, 2035, 6335, 603, 1, 4399, 2, 660, 482, 482, 77, 342, 2, 73, 436, 1, 0, 0, 1212, 174, 283] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[3370, 6729, 33, 112, 587, 15, 75, 30, 1, 408, 193, 297, 88, 349, 552, 853, 199, 1275, 23, 3, 210, 101, 1200, 1459, 3069, 2788, 2276, 245, 5, 16, 193, 307, 5, 393, 624, 3, 80, 2278, 677, 114, 660, 16, 1, 65, 1622, 6171, 558, 23, 4149, 31, 756, 483, 474, 82, 344, 0, 121, 410, 13, 4, 8, 1288, 308, 330] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3021, 4953, 134, 199, 774, 84, 73, 158, 2, 509, 437, 697, 161, 395, 543, 811, 464, 1303, 68, 73, 407, 129, 698, 1434, 2532, 2650, 2081, 295, 92, 218, 319, 422, 81, 395, 792, 22, 103, 2727, 867, 192, 728, 9, 28, 101, 1871, 5266, 857, 16, 3502, 103, 765, 455, 526, 121, 347, 86, 309, 486, 78, 20, 21, 1365, 346, 431] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[3364, 6395, 0, 197, 568, 38, 37, 47, 0, 389, 149, 232, 60, 306, 489, 904, 336, 966, 0, 0, 207, 68, 1839, 1404, 2147, 3408, 1791, 266, 0, 1, 162, 316, 0, 266, 1051, 0, 62, 2076, 562, 121, 693, 7, 0, 59, 2220, 6513, 554, 0, 4929, 0, 751, 439, 462, 66, 420, 0, 47, 417, 0, 0, 0, 982, 116, 253] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3585, 7041, 0, 84, 495, 0, 58, 3, 0, 450, 77, 213, 143, 301, 637, 387, 166, 1561, 0, 0, 201, 85, 319, 630, 4854, 2405, 1975, 217, 2, 5, 116, 420, 0, 309, 403, 0, 92, 3735, 776, 138, 722, 1, 0, 73, 1515, 6021, 330, 0, 4158, 0, 468, 517, 423, 2, 0, 0, 3, 290, 0, 0, 0, 2095, 299, 352] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3647, 2020, 0, 83, 503, 1, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 92, 190, 74, 328, 564, 465, 3898, 0, 0, 0, 0, 0, 0, 0, 164, 1769, 0, 0, 145, 90, 523, 1128, 1202, 1138, 0, 0, 0, 0, 0, 0, 435, 2478, 2558, 213, 1, 11, 114, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 432, 0, 87, 2842, 747, 112, 1493, 0, 0, 0, 0, 0, 0, 0, 623, 1, 0, 59, 1098, 3369, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4224, 0, 588, 545, 469, 80, 8, 0, 36, 0, 0, 0, 0, 0, 0, 0, 11, 393, 0, 0, 0, 1686, 351, 295, 3227, 1170, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6323, 5990, 6159, 6191, 6105, 5301, 5950, 7133] avg=6144.0000 max=7133 max_rank=7 min=5301 min_rank=5 imbalance_ratio=1.160970 max_load_reduction=42019 max_load_reduction_pct=0.854879 moved_to_echo=24489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3607, 2019, 0, 96, 526, 6, 69, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 126, 199, 146, 335, 571, 507, 3896, 0, 0, 0, 0, 0, 0, 0, 174, 1549, 0, 0, 173, 95, 582, 1112, 1240, 961, 0, 0, 0, 0, 0, 0, 368, 2862, 2259, 236, 0, 13, 132, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 387, 1, 92, 2630, 737, 131, 1540, 0, 0, 0, 0, 0, 0, 0, 696, 3, 0, 67, 1273, 3474, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4191, 0, 590, 563, 451, 87, 125, 0, 37, 0, 0, 0, 0, 0, 0, 0, 9, 356, 0, 0, 0, 1755, 377, 396, 2727, 1169, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6326, 6102, 5886, 6241, 6024, 5740, 6044, 6789] avg=6144.0000 max=6789 max_rank=7 min=5740 min_rank=5 imbalance_ratio=1.104980 max_load_reduction=42363 max_load_reduction_pct=0.861877 moved_to_echo=24597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[3647, 7088, 0, 83, 503, 1, 68, 1, 0, 379, 92, 190, 74, 328, 564, 465, 164, 1769, 0, 0, 145, 90, 523, 1128, 4800, 2478, 2558, 213, 1, 11, 114, 381, 0, 392, 432, 0, 87, 2842, 747, 112, 623, 1, 0, 59, 1098, 6100, 151, 0, 4224, 0, 588, 545, 469, 80, 8, 0, 11, 393, 0, 0, 0, 1686, 351, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3021, 4953, 134, 199, 774, 84, 73, 158, 2, 509, 437, 697, 161, 395, 543, 811, 464, 1303, 68, 73, 407, 129, 698, 1434, 2532, 2650, 2081, 295, 92, 218, 319, 422, 81, 395, 792, 22, 103, 2727, 867, 192, 728, 9, 28, 101, 1871, 5266, 857, 16, 3502, 103, 765, 455, 526, 121, 347, 86, 309, 486, 78, 20, 21, 1365, 346, 431] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3070, 1693, 3, 166, 626, 41, 63, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 179, 352, 60, 324, 464, 929, 3267, 0, 0, 0, 0, 0, 0, 0, 341, 1043, 1, 1, 246, 119, 1353, 1664, 1248, 610, 0, 0, 0, 0, 0, 0, 234, 3282, 2095, 281, 8, 61, 155, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 879, 1, 67, 2472, 628, 133, 1551, 0, 0, 0, 0, 0, 0, 0, 654, 3, 1, 83, 2035, 3498, 603, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4399, 2, 660, 482, 482, 77, 342, 2, 38, 0, 0, 0, 0, 0, 0, 0, 73, 436, 1, 0, 0, 1212, 174, 283, 1730, 980, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5724, 6031, 6626, 6491, 6029, 6878, 6484, 4889] avg=6144.0000 max=6878 max_rank=5 min=4889 min_rank=7 imbalance_ratio=1.119466 max_load_reduction=42274 max_load_reduction_pct=0.860067 moved_to_echo=24280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3783, 2001, 4, 62, 552, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 148, 198, 120, 347, 691, 457, 3882, 0, 0, 0, 0, 0, 0, 0, 190, 1614, 3, 0, 150, 107, 447, 1110, 1213, 1109, 0, 0, 0, 0, 0, 0, 416, 2193, 2811, 214, 0, 0, 169, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 301, 0, 93, 2794, 825, 94, 1505, 0, 0, 0, 0, 0, 0, 0, 687, 1, 0, 53, 889, 3377, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2, 534, 521, 427, 56, 12, 1, 39, 0, 0, 0, 0, 0, 0, 0, 40, 346, 2, 0, 0, 1804, 503, 329, 3140, 1169, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6479, 6182, 5943, 6094, 6183, 5266, 5672, 7333] avg=6144.0000 max=7333 max_rank=7 min=5266 min_rank=5 imbalance_ratio=1.193522 max_load_reduction=41819 max_load_reduction_pct=0.850810 moved_to_echo=24454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3607, 2019, 0, 96, 526, 6, 69, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 126, 199, 146, 335, 571, 507, 3896, 0, 0, 0, 0, 0, 0, 0, 174, 1549, 0, 0, 173, 95, 582, 1112, 1240, 961, 0, 0, 0, 0, 0, 0, 368, 2862, 2259, 236, 0, 13, 132, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 387, 1, 92, 2630, 737, 131, 1540, 0, 0, 0, 0, 0, 0, 0, 696, 3, 0, 67, 1273, 3474, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4191, 0, 590, 563, 451, 87, 125, 0, 37, 0, 0, 0, 0, 0, 0, 0, 9, 356, 0, 0, 0, 1755, 377, 396, 2727, 1169, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6326, 6102, 5886, 6241, 6024, 5740, 6044, 6789] avg=6144.0000 max=6789 max_rank=7 min=5740 min_rank=5 imbalance_ratio=1.104980 max_load_reduction=42363 max_load_reduction_pct=0.861877 moved_to_echo=24597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3585, 2007, 0, 84, 495, 0, 58, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 77, 213, 143, 301, 637, 387, 3872, 0, 0, 0, 0, 0, 0, 0, 166, 1561, 0, 0, 201, 85, 319, 630, 1186, 1151, 0, 0, 0, 0, 0, 0, 440, 2405, 1975, 217, 2, 5, 116, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 403, 0, 92, 3735, 776, 138, 1474, 0, 0, 0, 0, 0, 0, 0, 722, 1, 0, 73, 1515, 3325, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4158, 0, 468, 517, 423, 2, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 3, 290, 0, 0, 0, 2095, 299, 352, 3263, 1162, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6232, 6080, 5299, 5580, 6927, 5966, 5604, 7464] avg=6144.0000 max=7464 max_rank=7 min=5299 min_rank=2 imbalance_ratio=1.214844 max_load_reduction=41688 max_load_reduction_pct=0.848145 moved_to_echo=25961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3021, 1412, 134, 199, 774, 84, 73, 158, 0, 0, 0, 0, 0, 0, 0, 0, 2, 509, 437, 697, 161, 395, 543, 811, 2724, 0, 0, 0, 0, 0, 0, 0, 464, 1303, 68, 73, 407, 129, 698, 1434, 1038, 600, 0, 0, 0, 0, 0, 0, 230, 2650, 2081, 295, 92, 218, 319, 422, 0, 0, 0, 0, 0, 0, 0, 0, 81, 395, 792, 22, 103, 2727, 867, 192, 1289, 0, 0, 0, 0, 0, 0, 0, 728, 9, 28, 101, 1871, 2908, 857, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 103, 765, 455, 526, 121, 347, 86, 31, 0, 0, 0, 0, 0, 0, 0, 309, 486, 78, 20, 21, 1365, 346, 431, 1702, 817, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 6279, 6214, 6307, 6468, 6518, 5936, 5575] avg=6144.0000 max=6518 max_rank=5 min=5575 min_rank=7 imbalance_ratio=1.060872 max_load_reduction=42634 max_load_reduction_pct=0.867391 moved_to_echo=24497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3370, 1917, 33, 112, 587, 15, 75, 30, 0, 0, 0, 0, 0, 0, 0, 0, 1, 408, 193, 297, 88, 349, 552, 853, 3701, 0, 0, 0, 0, 0, 0, 0, 199, 1275, 23, 3, 210, 101, 1200, 1459, 1216, 727, 0, 0, 0, 0, 0, 0, 279, 2788, 2276, 245, 5, 16, 193, 307, 0, 0, 0, 0, 0, 0, 0, 0, 5, 393, 624, 3, 80, 2278, 677, 114, 1510, 0, 0, 0, 0, 0, 0, 0, 660, 16, 1, 65, 1622, 3408, 558, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 31, 756, 483, 474, 82, 344, 0, 37, 0, 0, 0, 0, 0, 0, 0, 121, 410, 13, 4, 8, 1288, 308, 330, 2063, 1111, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6442, 6413, 6109, 5684, 6353, 6356, 5656] avg=6144.0000 max=6442 max_rank=1 min=5656 min_rank=7 imbalance_ratio=1.048503 max_load_reduction=42710 max_load_reduction_pct=0.868937 moved_to_echo=24049 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3647, 2020, 0, 83, 503, 1, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 92, 190, 74, 328, 564, 465, 3898, 0, 0, 0, 0, 0, 0, 0, 164, 1769, 0, 0, 145, 90, 523, 1128, 1202, 1138, 0, 0, 0, 0, 0, 0, 435, 2478, 2558, 213, 1, 11, 114, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 432, 0, 87, 2842, 747, 112, 1493, 0, 0, 0, 0, 0, 0, 0, 623, 1, 0, 59, 1098, 3369, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4224, 0, 588, 545, 469, 80, 8, 0, 36, 0, 0, 0, 0, 0, 0, 0, 11, 393, 0, 0, 0, 1686, 351, 295, 3227, 1170, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6323, 5990, 6159, 6191, 6105, 5301, 5950, 7133] avg=6144.0000 max=7133 max_rank=7 min=5301 min_rank=5 imbalance_ratio=1.160970 max_load_reduction=42019 max_load_reduction_pct=0.854879 moved_to_echo=24489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3364, 1823, 0, 197, 568, 38, 37, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 149, 232, 60, 306, 489, 904, 3517, 0, 0, 0, 0, 0, 0, 0, 336, 966, 0, 0, 207, 68, 1839, 1404, 1283, 509, 0, 0, 0, 0, 0, 0, 195, 3408, 1791, 266, 0, 1, 162, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 1051, 0, 62, 2076, 562, 121, 1594, 0, 0, 0, 0, 0, 0, 0, 693, 7, 0, 59, 2220, 3597, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4929, 0, 751, 439, 462, 66, 420, 0, 39, 0, 0, 0, 0, 0, 0, 0, 47, 417, 0, 0, 0, 982, 116, 253, 1443, 1055, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6074, 6046, 6612, 6139, 5732, 7130, 7106, 4313] avg=6144.0000 max=7130 max_rank=5 min=4313 min_rank=7 imbalance_ratio=1.160482 max_load_reduction=42022 max_load_reduction_pct=0.854940 moved_to_echo=24281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3783, 2001, 4, 62, 552, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 148, 198, 120, 347, 691, 457, 3882, 0, 0, 0, 0, 0, 0, 0, 190, 1614, 3, 0, 150, 107, 447, 1110, 1213, 1109, 0, 0, 0, 0, 0, 0, 416, 2193, 2811, 214, 0, 0, 169, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 301, 0, 93, 2794, 825, 94, 1505, 0, 0, 0, 0, 0, 0, 0, 687, 1, 0, 53, 889, 3377, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2, 534, 521, 427, 56, 12, 1, 39, 0, 0, 0, 0, 0, 0, 0, 40, 346, 2, 0, 0, 1804, 503, 329, 3140, 1169, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6479, 6182, 5943, 6094, 6183, 5266, 5672, 7333] avg=6144.0000 max=7333 max_rank=7 min=5266 min_rank=5 imbalance_ratio=1.193522 max_load_reduction=41819 max_load_reduction_pct=0.850810 moved_to_echo=24454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3021, 1412, 134, 199, 774, 84, 73, 158, 0, 0, 0, 0, 0, 0, 0, 0, 2, 509, 437, 697, 161, 395, 543, 811, 2724, 0, 0, 0, 0, 0, 0, 0, 464, 1303, 68, 73, 407, 129, 698, 1434, 1038, 600, 0, 0, 0, 0, 0, 0, 230, 2650, 2081, 295, 92, 218, 319, 422, 0, 0, 0, 0, 0, 0, 0, 0, 81, 395, 792, 22, 103, 2727, 867, 192, 1289, 0, 0, 0, 0, 0, 0, 0, 728, 9, 28, 101, 1871, 2908, 857, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 103, 765, 455, 526, 121, 347, 86, 31, 0, 0, 0, 0, 0, 0, 0, 309, 486, 78, 20, 21, 1365, 346, 431, 1702, 817, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 6279, 6214, 6307, 6468, 6518, 5936, 5575] avg=6144.0000 max=6518 max_rank=5 min=5575 min_rank=7 imbalance_ratio=1.060872 max_load_reduction=42634 max_load_reduction_pct=0.867391 moved_to_echo=24497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 11136, 128, 128, 1792, 896, 7040, 9984, 9728, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 3200, 128, 128, 128, 12288, 2560, 2688, 19328, 8704, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5504, 128, 128, 640, 12544, 27008, 3584, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[27520, 14976, 256, 1024, 4736, 256, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 3200, 4992, 128, 768, 21632, 5888, 1152, 12032, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 3200, 128, 128, 128, 12288, 2560, 2688, 19328, 8704, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 3328, 1408, 2432, 896, 2688, 4608, 5376, 28800, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 22144, 17920, 2048, 128, 384, 1408, 2944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[27520, 14976, 256, 1024, 4736, 256, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 3200, 4992, 128, 768, 21632, 5888, 1152, 12032, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[33664, 256, 5120, 4096, 3840, 640, 1664, 128, 384, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[579, 465, 4017, 4025, 1127, 0, 4810, 2596, 0, 184, 0, 1126, 0, 2139, 0, 229, 252, 0, 1444, 31, 471, 85, 365, 143, 69, 92, 1140, 0, 0, 163, 0, 177, 5684, 1085, 0, 92, 273, 1909, 538, 0, 426, 3, 7, 0, 0, 0, 300, 241, 1652, 0, 252, 286, 125, 2340, 1180, 0, 1386, 454, 130, 2661, 2298, 0, 101, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[591, 529, 4297, 4500, 1350, 6, 5189, 3307, 6, 158, 25, 928, 29, 1761, 2, 274, 261, 1, 1545, 43, 712, 55, 397, 142, 66, 89, 1179, 4, 2, 150, 32, 191, 5429, 1122, 3, 81, 208, 1591, 544, 11, 424, 7, 55, 15, 0, 29, 357, 166, 1346, 4, 246, 360, 166, 2037, 741, 22, 1642, 501, 154, 2117, 1763, 0, 178, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[565, 560, 4046, 4020, 582, 103, 4857, 2895, 35, 206, 24, 888, 147, 1938, 49, 431, 179, 48, 1201, 51, 633, 84, 614, 269, 68, 153, 1239, 25, 47, 194, 45, 196, 4830, 1264, 34, 123, 249, 1181, 549, 51, 454, 17, 254, 78, 111, 117, 431, 265, 891, 5, 386, 183, 930, 2337, 813, 80, 1280, 537, 127, 2694, 2009, 0, 433, 47] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[576, 394, 3910, 4268, 951, 0, 5266, 2529, 0, 173, 0, 1029, 0, 2149, 0, 203, 258, 0, 1323, 23, 408, 83, 387, 132, 68, 86, 1150, 0, 0, 155, 0, 171, 5740, 1159, 0, 87, 260, 2558, 534, 0, 415, 3, 11, 0, 0, 0, 326, 137, 1689, 0, 246, 205, 155, 2104, 1026, 0, 1083, 469, 112, 2724, 2341, 0, 76, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[589, 389, 3972, 4515, 537, 0, 5481, 2584, 0, 184, 0, 860, 0, 2063, 0, 141, 227, 0, 1285, 28, 506, 133, 353, 106, 58, 92, 1107, 0, 0, 150, 0, 178, 5642, 1377, 0, 98, 291, 2479, 572, 0, 394, 0, 16, 0, 0, 0, 339, 141, 1628, 0, 219, 187, 152, 1848, 747, 0, 766, 443, 293, 3234, 2620, 0, 128, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[572, 426, 3826, 4143, 886, 1, 5528, 2112, 2, 182, 1, 1074, 2, 2242, 0, 170, 248, 0, 1268, 9, 485, 54, 337, 142, 77, 91, 994, 0, 0, 170, 0, 186, 5529, 1197, 1, 92, 256, 2442, 590, 0, 437, 0, 13, 3, 1, 2, 351, 200, 1504, 0, 247, 112, 177, 2405, 1065, 0, 1009, 429, 86, 2814, 2848, 1, 113, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[576, 472, 4164, 4440, 1349, 0, 5462, 3724, 1, 134, 0, 812, 1, 1957, 0, 212, 251, 3, 1616, 70, 882, 73, 429, 143, 63, 67, 1290, 0, 4, 130, 1, 197, 5429, 1141, 0, 67, 196, 1095, 542, 1, 391, 0, 37, 0, 9, 1, 340, 129, 1191, 1, 247, 261, 183, 2590, 581, 0, 1745, 492, 91, 2355, 1360, 0, 154, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[591, 529, 4297, 4500, 1350, 6, 5189, 3307, 6, 158, 25, 928, 29, 1761, 2, 274, 261, 1, 1545, 43, 712, 55, 397, 142, 66, 89, 1179, 4, 2, 150, 32, 191, 5429, 1122, 3, 81, 208, 1591, 544, 11, 424, 7, 55, 15, 0, 29, 357, 166, 1346, 4, 246, 360, 166, 2037, 741, 22, 1642, 501, 154, 2117, 1763, 0, 178, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[548, 519, 4374, 4607, 1449, 0, 5318, 4020, 0, 124, 0, 801, 0, 1748, 0, 125, 255, 0, 1881, 36, 713, 77, 386, 102, 37, 62, 1393, 0, 0, 99, 0, 176, 5641, 1160, 0, 72, 200, 1077, 602, 0, 368, 0, 26, 0, 0, 0, 396, 119, 1582, 0, 260, 413, 133, 2099, 634, 0, 1630, 497, 101, 2064, 1133, 0, 95, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[565, 560, 4046, 4020, 582, 103, 4857, 2895, 35, 206, 24, 888, 147, 1938, 49, 431, 179, 48, 1201, 51, 633, 84, 614, 269, 68, 153, 1239, 25, 47, 194, 45, 196, 4830, 1264, 34, 123, 249, 1181, 549, 51, 454, 17, 254, 78, 111, 117, 431, 265, 891, 5, 386, 183, 930, 2337, 813, 80, 1280, 537, 127, 2694, 2009, 0, 433, 47] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[572, 426, 3826, 4143, 886, 1, 5528, 2112, 2, 182, 1, 1074, 2, 2242, 0, 170, 248, 0, 1268, 9, 485, 54, 337, 142, 77, 91, 994, 0, 0, 170, 0, 186, 5529, 1197, 1, 92, 256, 2442, 590, 0, 437, 0, 13, 3, 1, 2, 351, 200, 1504, 0, 247, 112, 177, 2405, 1065, 0, 1009, 429, 86, 2814, 2848, 1, 113, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[579, 465, 1072, 0, 1127, 0, 0, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 1126, 0, 2139, 0, 229, 2674, 61, 0, 0, 0, 0, 0, 0, 252, 0, 1444, 31, 471, 85, 365, 143, 3280, 7, 0, 0, 0, 0, 0, 0, 69, 92, 1140, 0, 0, 163, 0, 177, 4018, 129, 0, 0, 0, 0, 0, 0, 2343, 1085, 0, 92, 273, 1909, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 3, 7, 0, 0, 0, 300, 241, 4681, 0, 0, 0, 0, 0, 0, 0, 1652, 0, 252, 286, 125, 2340, 1180, 0, 602, 271, 0, 0, 0, 0, 0, 0, 1386, 454, 130, 2059, 2298, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5839, 6413, 6078, 5788, 6240, 5658, 6708, 6428] avg=6144.0000 max=6708 max_rank=6 min=5658 min_rank=5 imbalance_ratio=1.091797 max_load_reduction=42444 max_load_reduction_pct=0.863525 moved_to_echo=25034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[576, 394, 1044, 0, 951, 0, 0, 2529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 1029, 0, 2149, 0, 203, 2603, 61, 0, 0, 0, 0, 0, 0, 258, 0, 1323, 23, 408, 83, 387, 132, 3313, 7, 0, 0, 0, 0, 0, 0, 68, 86, 1150, 0, 0, 155, 0, 171, 4261, 140, 0, 0, 0, 0, 0, 0, 2366, 1159, 0, 87, 260, 2558, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 3, 11, 0, 0, 0, 326, 137, 5126, 0, 0, 0, 0, 0, 0, 0, 1689, 0, 246, 205, 155, 2104, 1026, 0, 616, 263, 0, 0, 0, 0, 0, 0, 1083, 469, 112, 2108, 2341, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5494, 6218, 5934, 6031, 6964, 6018, 6304, 6189] avg=6144.0000 max=6964 max_rank=4 min=5494 min_rank=0 imbalance_ratio=1.133464 max_load_reduction=42188 max_load_reduction_pct=0.858317 moved_to_echo=25475 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[565, 560, 1079, 0, 582, 103, 0, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 35, 206, 24, 888, 147, 1938, 49, 431, 2694, 51, 0, 0, 0, 0, 0, 0, 179, 48, 1201, 51, 633, 84, 614, 269, 2787, 7, 0, 0, 0, 0, 0, 0, 68, 153, 1239, 25, 47, 194, 45, 196, 4013, 130, 0, 0, 0, 0, 0, 0, 1992, 1264, 34, 123, 249, 1181, 549, 51, 0, 0, 0, 0, 0, 0, 0, 0, 454, 17, 254, 78, 111, 117, 431, 265, 4727, 0, 0, 0, 0, 0, 0, 0, 891, 5, 386, 183, 930, 2337, 813, 80, 609, 273, 0, 0, 0, 0, 0, 0, 1280, 537, 127, 2085, 2009, 0, 433, 47, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5784, 6463, 5873, 6110, 5443, 6454, 6507, 6518] avg=6144.0000 max=6518 max_rank=7 min=5443 min_rank=4 imbalance_ratio=1.060872 max_load_reduction=42634 max_load_reduction_pct=0.867391 moved_to_echo=24922 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[589, 389, 1060, 0, 537, 0, 0, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 860, 0, 2063, 0, 141, 2644, 60, 0, 0, 0, 0, 0, 0, 227, 0, 1285, 28, 506, 133, 353, 106, 3256, 8, 0, 0, 0, 0, 0, 0, 58, 92, 1107, 0, 0, 150, 0, 178, 4507, 147, 0, 0, 0, 0, 0, 0, 2326, 1377, 0, 98, 291, 2479, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 16, 0, 0, 0, 339, 141, 5334, 0, 0, 0, 0, 0, 0, 0, 1628, 0, 219, 187, 152, 1848, 747, 0, 732, 268, 0, 0, 0, 0, 0, 0, 766, 443, 293, 2502, 2620, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5159, 5952, 5902, 6239, 7143, 6224, 5781, 6752] avg=6144.0000 max=7143 max_rank=4 min=5159 min_rank=0 imbalance_ratio=1.162598 max_load_reduction=42009 max_load_reduction_pct=0.854675 moved_to_echo=25900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[591, 529, 1146, 0, 1350, 6, 0, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 6, 158, 25, 928, 29, 1761, 2, 274, 2861, 58, 0, 0, 0, 0, 0, 0, 261, 1, 1545, 43, 712, 55, 397, 142, 3133, 7, 0, 0, 0, 0, 0, 0, 66, 89, 1179, 4, 2, 150, 32, 191, 4493, 138, 0, 0, 0, 0, 0, 0, 2238, 1122, 3, 81, 208, 1591, 544, 11, 0, 0, 0, 0, 0, 0, 0, 0, 424, 7, 55, 15, 0, 29, 357, 166, 5051, 0, 0, 0, 0, 0, 0, 0, 1346, 4, 246, 360, 166, 2037, 741, 22, 479, 290, 0, 0, 0, 0, 0, 0, 1642, 501, 154, 1638, 1763, 0, 178, 12, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6929, 6102, 6296, 6344, 5798, 6104, 5691, 5888] avg=6144.0000 max=6929 max_rank=0 min=5691 min_rank=6 imbalance_ratio=1.127767 max_load_reduction=42223 max_load_reduction_pct=0.859029 moved_to_echo=23481 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[576, 394, 1044, 0, 951, 0, 0, 2529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 1029, 0, 2149, 0, 203, 2603, 61, 0, 0, 0, 0, 0, 0, 258, 0, 1323, 23, 408, 83, 387, 132, 3313, 7, 0, 0, 0, 0, 0, 0, 68, 86, 1150, 0, 0, 155, 0, 171, 4261, 140, 0, 0, 0, 0, 0, 0, 2366, 1159, 0, 87, 260, 2558, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 3, 11, 0, 0, 0, 326, 137, 5126, 0, 0, 0, 0, 0, 0, 0, 1689, 0, 246, 205, 155, 2104, 1026, 0, 616, 263, 0, 0, 0, 0, 0, 0, 1083, 469, 112, 2108, 2341, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5494, 6218, 5934, 6031, 6964, 6018, 6304, 6189] avg=6144.0000 max=6964 max_rank=4 min=5494 min_rank=0 imbalance_ratio=1.133464 max_load_reduction=42188 max_load_reduction_pct=0.858317 moved_to_echo=25475 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[576, 472, 1111, 0, 1349, 0, 0, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 1, 134, 0, 812, 1, 1957, 0, 212, 2772, 58, 0, 0, 0, 0, 0, 0, 251, 3, 1616, 70, 882, 73, 429, 143, 3133, 7, 0, 0, 0, 0, 0, 0, 63, 67, 1290, 0, 4, 130, 1, 197, 4433, 146, 0, 0, 0, 0, 0, 0, 2238, 1141, 0, 67, 196, 1095, 542, 1, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 37, 0, 9, 1, 340, 129, 5316, 0, 0, 0, 0, 0, 0, 0, 1191, 1, 247, 261, 183, 2590, 581, 0, 533, 281, 0, 0, 0, 0, 0, 0, 1745, 492, 91, 1822, 1360, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7232, 5947, 6607, 6331, 5280, 6223, 5868, 5664] avg=6144.0000 max=7232 max_rank=0 min=5280 min_rank=4 imbalance_ratio=1.177083 max_load_reduction=41920 max_load_reduction_pct=0.852865 moved_to_echo=23035 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[572, 426, 1015, 0, 886, 1, 0, 2112, 0, 0, 0, 0, 0, 0, 0, 0, 2, 182, 1, 1074, 2, 2242, 0, 170, 2551, 63, 0, 0, 0, 0, 0, 0, 248, 0, 1268, 9, 485, 54, 337, 142, 3196, 8, 0, 0, 0, 0, 0, 0, 77, 91, 994, 0, 0, 170, 0, 186, 4135, 148, 0, 0, 0, 0, 0, 0, 2270, 1197, 1, 92, 256, 2442, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 0, 13, 3, 1, 2, 351, 200, 5380, 0, 0, 0, 0, 0, 0, 0, 1504, 0, 247, 112, 177, 2405, 1065, 0, 640, 260, 0, 0, 0, 0, 0, 0, 1009, 429, 86, 2174, 2848, 1, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5012, 6287, 5747, 5801, 6848, 6387, 6410, 6660] avg=6144.0000 max=6848 max_rank=4 min=5012 min_rank=0 imbalance_ratio=1.114583 max_load_reduction=42304 max_load_reduction_pct=0.860677 moved_to_echo=26305 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[589, 389, 1060, 0, 537, 0, 0, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 860, 0, 2063, 0, 141, 2644, 60, 0, 0, 0, 0, 0, 0, 227, 0, 1285, 28, 506, 133, 353, 106, 3256, 8, 0, 0, 0, 0, 0, 0, 58, 92, 1107, 0, 0, 150, 0, 178, 4507, 147, 0, 0, 0, 0, 0, 0, 2326, 1377, 0, 98, 291, 2479, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 16, 0, 0, 0, 339, 141, 5334, 0, 0, 0, 0, 0, 0, 0, 1628, 0, 219, 187, 152, 1848, 747, 0, 732, 268, 0, 0, 0, 0, 0, 0, 766, 443, 293, 2502, 2620, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5159, 5952, 5902, 6239, 7143, 6224, 5781, 6752] avg=6144.0000 max=7143 max_rank=4 min=5159 min_rank=0 imbalance_ratio=1.162598 max_load_reduction=42009 max_load_reduction_pct=0.854675 moved_to_echo=25900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[548, 519, 1167, 0, 1449, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 801, 0, 1748, 0, 125, 2912, 60, 0, 0, 0, 0, 0, 0, 255, 0, 1881, 36, 713, 77, 386, 102, 3256, 9, 0, 0, 0, 0, 0, 0, 37, 62, 1393, 0, 0, 99, 0, 176, 4598, 143, 0, 0, 0, 0, 0, 0, 2325, 1160, 0, 72, 200, 1077, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 26, 0, 0, 0, 396, 119, 5175, 0, 0, 0, 0, 0, 0, 0, 1582, 0, 260, 413, 133, 2099, 634, 0, 467, 295, 0, 0, 0, 0, 0, 0, 1630, 497, 101, 1597, 1133, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7703, 5770, 6715, 6508, 5436, 6084, 5883, 5053] avg=6144.0000 max=7703 max_rank=0 min=5053 min_rank=7 imbalance_ratio=1.253743 max_load_reduction=41449 max_load_reduction_pct=0.843282 moved_to_echo=22456 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[565, 560, 1079, 0, 582, 103, 0, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 35, 206, 24, 888, 147, 1938, 49, 431, 2694, 51, 0, 0, 0, 0, 0, 0, 179, 48, 1201, 51, 633, 84, 614, 269, 2787, 7, 0, 0, 0, 0, 0, 0, 68, 153, 1239, 25, 47, 194, 45, 196, 4013, 130, 0, 0, 0, 0, 0, 0, 1992, 1264, 34, 123, 249, 1181, 549, 51, 0, 0, 0, 0, 0, 0, 0, 0, 454, 17, 254, 78, 111, 117, 431, 265, 4727, 0, 0, 0, 0, 0, 0, 0, 891, 5, 386, 183, 930, 2337, 813, 80, 609, 273, 0, 0, 0, 0, 0, 0, 1280, 537, 127, 2085, 2009, 0, 433, 47, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5784, 6463, 5873, 6110, 5443, 6454, 6507, 6518] avg=6144.0000 max=6518 max_rank=7 min=5443 min_rank=4 imbalance_ratio=1.060872 max_load_reduction=42634 max_load_reduction_pct=0.867391 moved_to_echo=24922 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[572, 426, 1015, 0, 886, 1, 0, 2112, 0, 0, 0, 0, 0, 0, 0, 0, 2, 182, 1, 1074, 2, 2242, 0, 170, 2551, 63, 0, 0, 0, 0, 0, 0, 248, 0, 1268, 9, 485, 54, 337, 142, 3196, 8, 0, 0, 0, 0, 0, 0, 77, 91, 994, 0, 0, 170, 0, 186, 4135, 148, 0, 0, 0, 0, 0, 0, 2270, 1197, 1, 92, 256, 2442, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 0, 13, 3, 1, 2, 351, 200, 5380, 0, 0, 0, 0, 0, 0, 0, 1504, 0, 247, 112, 177, 2405, 1065, 0, 640, 260, 0, 0, 0, 0, 0, 0, 1009, 429, 86, 2174, 2848, 1, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5012, 6287, 5747, 5801, 6848, 6387, 6410, 6660] avg=6144.0000 max=6848 max_rank=4 min=5012 min_rank=0 imbalance_ratio=1.114583 max_load_reduction=42304 max_load_reduction_pct=0.860677 moved_to_echo=26305 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18176, 9600, 128, 768, 2048, 14336, 4480, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11520, 128, 2176, 2048, 2048, 17792, 6912, 128, 4736, 2304, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4608, 3840, 8704, 0, 8320, 128, 0, 23808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10624, 3840, 1152, 16000, 16384, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 128, 7552, 256, 16000, 128, 1792, 21760, 512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11520, 128, 2176, 2048, 2048, 17792, 6912, 128, 4736, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10624, 3840, 1152, 16000, 16384, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2048, 128, 11648, 384, 4864, 768, 3328, 1280, 25472, 128, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 768, 9600, 128, 128, 1280, 128, 1536, 34560, 1152, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 128, 512, 128, 128, 256, 2944, 1408, 40832, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[33, 6234, 2333, 91, 4, 119, 1327, 2254, 6, 219, 52, 49, 48, 674, 2532, 185, 203, 686, 300, 185, 130, 66, 255, 785, 689, 421, 185, 239, 252, 107, 272, 73, 357, 6639, 153, 80, 2521, 159, 634, 9, 1023, 17, 133, 74, 1920, 273, 10, 100, 92, 109, 146, 906, 85, 584, 214, 207, 80, 400, 5788, 167, 297, 1915, 168, 2884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 7279, 1662, 7, 0, 59, 1022, 2500, 0, 44, 0, 0, 0, 688, 2409, 151, 0, 873, 143, 226, 76, 0, 201, 679, 1180, 319, 0, 136, 223, 85, 258, 5, 322, 7322, 93, 0, 3946, 0, 150, 0, 1402, 0, 0, 0, 2125, 227, 0, 0, 92, 0, 92, 966, 15, 433, 92, 0, 0, 350, 6406, 406, 263, 1303, 0, 2922] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 7272, 1423, 17, 0, 69, 968, 3447, 0, 20, 0, 0, 0, 670, 2574, 161, 0, 826, 146, 252, 83, 0, 209, 585, 594, 344, 0, 133, 232, 92, 301, 1, 286, 7368, 92, 0, 3675, 0, 312, 0, 1614, 0, 0, 0, 2105, 252, 0, 0, 92, 0, 110, 1055, 9, 453, 91, 0, 0, 358, 6263, 202, 250, 2028, 1, 2117] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 7294, 1379, 12, 0, 68, 935, 3055, 0, 28, 0, 0, 0, 711, 2672, 155, 0, 834, 74, 251, 77, 0, 195, 688, 769, 337, 0, 112, 227, 90, 220, 0, 276, 7353, 87, 0, 3891, 0, 259, 0, 1864, 0, 0, 0, 1772, 251, 0, 0, 87, 0, 88, 815, 12, 440, 85, 0, 0, 386, 6412, 288, 248, 1999, 0, 2356] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 7294, 1540, 21, 0, 77, 1112, 2786, 0, 41, 1, 0, 0, 667, 2790, 170, 0, 859, 118, 245, 100, 4, 201, 580, 676, 337, 0, 73, 246, 107, 224, 0, 307, 7287, 93, 2, 3549, 0, 302, 1, 1779, 0, 4, 1, 1613, 247, 0, 3, 92, 2, 104, 1190, 3, 447, 90, 4, 0, 337, 6455, 224, 263, 1753, 1, 2730] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 6922, 1314, 28, 0, 72, 964, 3844, 0, 4, 0, 0, 0, 646, 2413, 134, 39, 578, 60, 252, 71, 1, 168, 756, 586, 346, 56, 167, 253, 93, 135, 6, 84, 7273, 72, 1, 2705, 0, 717, 0, 1674, 0, 2, 0, 1946, 253, 0, 1, 66, 1, 71, 1028, 52, 525, 67, 1, 1, 414, 6341, 99, 258, 2483, 5, 3104] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 7294, 1379, 12, 0, 68, 935, 3055, 0, 28, 0, 0, 0, 711, 2672, 155, 0, 834, 74, 251, 77, 0, 195, 688, 769, 337, 0, 112, 227, 90, 220, 0, 276, 7353, 87, 0, 3891, 0, 259, 0, 1864, 0, 0, 0, 1772, 251, 0, 0, 87, 0, 88, 815, 12, 440, 85, 0, 0, 386, 6412, 288, 248, 1999, 0, 2356] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 7272, 1423, 17, 0, 69, 968, 3447, 0, 20, 0, 0, 0, 670, 2574, 161, 0, 826, 146, 252, 83, 0, 209, 585, 594, 344, 0, 133, 232, 92, 301, 1, 286, 7368, 92, 0, 3675, 0, 312, 0, 1614, 0, 0, 0, 2105, 252, 0, 0, 92, 0, 110, 1055, 9, 453, 91, 0, 0, 358, 6263, 202, 250, 2028, 1, 2117] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 7279, 1662, 7, 0, 59, 1022, 2500, 0, 44, 0, 0, 0, 688, 2409, 151, 0, 873, 143, 226, 76, 0, 201, 679, 1180, 319, 0, 136, 223, 85, 258, 5, 322, 7322, 93, 0, 3946, 0, 150, 0, 1402, 0, 0, 0, 2125, 227, 0, 0, 92, 0, 92, 966, 15, 433, 92, 0, 0, 350, 6406, 406, 263, 1303, 0, 2922] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 7022, 1186, 18, 0, 37, 1154, 4378, 0, 1, 0, 0, 0, 738, 2110, 100, 5, 458, 60, 253, 52, 0, 153, 814, 432, 317, 34, 159, 251, 58, 136, 3, 89, 7476, 66, 0, 1970, 0, 714, 0, 2006, 0, 0, 0, 2087, 240, 0, 0, 62, 0, 90, 998, 49, 494, 62, 0, 0, 400, 6358, 75, 215, 2930, 0, 2842] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7032, 1213, 27, 0, 67, 1060, 3716, 0, 57, 15, 0, 1, 692, 2251, 146, 14, 699, 84, 256, 85, 29, 180, 720, 549, 346, 21, 164, 262, 93, 240, 7, 148, 7299, 81, 25, 2829, 10, 579, 1, 1728, 1, 41, 8, 1929, 272, 6, 41, 81, 26, 81, 1040, 15, 548, 82, 34, 2, 380, 6221, 112, 260, 2444, 5, 2797] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[33, 6234, 2333, 91, 4, 119, 1327, 2254, 6, 219, 52, 49, 48, 674, 2532, 185, 203, 686, 300, 185, 130, 66, 255, 785, 689, 421, 185, 239, 252, 107, 272, 73, 357, 6639, 153, 80, 2521, 159, 634, 9, 1023, 17, 133, 74, 1920, 273, 10, 100, 92, 109, 146, 906, 85, 584, 214, 207, 80, 400, 5788, 167, 297, 1915, 168, 2884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 7022, 1186, 18, 0, 37, 1154, 4378, 0, 1, 0, 0, 0, 738, 2110, 100, 5, 458, 60, 253, 52, 0, 153, 814, 432, 317, 34, 159, 251, 58, 136, 3, 89, 7476, 66, 0, 1970, 0, 714, 0, 2006, 0, 0, 0, 2087, 240, 0, 0, 62, 0, 90, 998, 49, 494, 62, 0, 0, 400, 6358, 75, 215, 2930, 0, 2842] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 227, 1423, 17, 0, 69, 968, 3447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 670, 2574, 161, 2810, 0, 0, 0, 0, 0, 0, 0, 0, 826, 146, 252, 83, 0, 209, 585, 3982, 35, 0, 0, 0, 0, 0, 0, 594, 344, 0, 133, 232, 92, 301, 1, 4501, 0, 0, 0, 0, 0, 0, 0, 286, 2224, 92, 0, 3675, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1614, 0, 0, 0, 2105, 252, 0, 0, 2299, 0, 0, 0, 0, 0, 0, 0, 92, 0, 110, 1055, 9, 453, 91, 0, 2544, 1827, 0, 0, 0, 0, 0, 0, 0, 358, 454, 202, 250, 2028, 1, 2117, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 6235, 6118, 6198, 6589, 6270, 6181, 5410] avg=6144.0000 max=6589 max_rank=4 min=5410 min_rank=7 imbalance_ratio=1.072428 max_load_reduction=42563 max_load_reduction_pct=0.865946 moved_to_echo=24450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[33, 195, 2333, 91, 4, 119, 1327, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 6, 219, 52, 49, 48, 674, 2532, 185, 2532, 0, 0, 0, 0, 0, 0, 0, 203, 686, 300, 185, 130, 66, 255, 785, 3680, 31, 0, 0, 0, 0, 0, 0, 689, 421, 185, 239, 252, 107, 272, 73, 3858, 0, 0, 0, 0, 0, 0, 0, 357, 2004, 153, 80, 2521, 159, 634, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 17, 133, 74, 1920, 273, 10, 100, 2072, 0, 0, 0, 0, 0, 0, 0, 92, 109, 146, 906, 85, 584, 214, 207, 2181, 1688, 0, 0, 0, 0, 0, 0, 80, 400, 420, 167, 297, 1915, 168, 2884, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6356, 6297, 6321, 6096, 5917, 5622, 6212, 6331] avg=6144.0000 max=6356 max_rank=0 min=5622 min_rank=5 imbalance_ratio=1.034505 max_load_reduction=42796 max_load_reduction_pct=0.870687 moved_to_echo=24082 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 228, 1379, 12, 0, 68, 935, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 711, 2672, 155, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 834, 74, 251, 77, 0, 195, 688, 4077, 35, 0, 0, 0, 0, 0, 0, 769, 337, 0, 112, 227, 90, 220, 0, 4514, 0, 0, 0, 0, 0, 0, 0, 276, 2220, 87, 0, 3891, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 0, 0, 0, 1772, 251, 0, 0, 2294, 0, 0, 0, 0, 0, 0, 0, 87, 0, 88, 815, 12, 440, 85, 0, 2552, 1870, 0, 0, 0, 0, 0, 0, 0, 386, 465, 288, 248, 1999, 0, 2356, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5677, 6370, 6231, 6269, 6733, 6181, 5949, 5742] avg=6144.0000 max=6733 max_rank=4 min=5677 min_rank=0 imbalance_ratio=1.095866 max_load_reduction=42419 max_load_reduction_pct=0.863017 moved_to_echo=24605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 217, 1314, 28, 0, 72, 964, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 646, 2413, 134, 2773, 0, 0, 0, 0, 0, 0, 0, 39, 578, 60, 252, 71, 1, 168, 756, 4032, 34, 0, 0, 0, 0, 0, 0, 586, 346, 56, 167, 253, 93, 135, 6, 4284, 0, 0, 0, 0, 0, 0, 0, 84, 2196, 72, 1, 2705, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 0, 2, 0, 1946, 253, 0, 1, 2270, 0, 0, 0, 0, 0, 0, 0, 66, 1, 71, 1028, 52, 525, 67, 1, 2421, 1850, 0, 0, 0, 0, 0, 0, 1, 414, 459, 99, 258, 2483, 5, 3104, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6439, 5970, 5991, 5926, 5775, 6146, 6082, 6823] avg=6144.0000 max=6823 max_rank=7 min=5775 min_rank=4 imbalance_ratio=1.110514 max_load_reduction=42329 max_load_reduction_pct=0.861186 moved_to_echo=24826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 228, 1662, 7, 0, 59, 1022, 2500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 688, 2409, 151, 2792, 0, 0, 0, 0, 0, 0, 0, 0, 873, 143, 226, 76, 0, 201, 679, 4073, 34, 0, 0, 0, 0, 0, 0, 1180, 319, 0, 136, 223, 85, 258, 5, 4505, 0, 0, 0, 0, 0, 0, 0, 322, 2211, 93, 0, 3946, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 2125, 227, 0, 0, 2285, 0, 0, 0, 0, 0, 0, 0, 92, 0, 92, 966, 15, 433, 92, 0, 2546, 1869, 0, 0, 0, 0, 0, 0, 0, 350, 464, 406, 263, 1303, 0, 2922, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5478, 6084, 6305, 6711, 6722, 6039, 6105, 5708] avg=6144.0000 max=6722 max_rank=4 min=5478 min_rank=0 imbalance_ratio=1.094076 max_load_reduction=42430 max_load_reduction_pct=0.863241 moved_to_echo=24574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 219, 1540, 21, 0, 77, 1112, 2786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1, 0, 0, 667, 2790, 170, 2782, 0, 0, 0, 0, 0, 0, 0, 0, 859, 118, 245, 100, 4, 201, 580, 4110, 39, 0, 0, 0, 0, 0, 0, 676, 337, 0, 73, 246, 107, 224, 0, 4519, 0, 0, 0, 0, 0, 0, 0, 307, 2189, 93, 2, 3549, 0, 302, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1779, 0, 4, 1, 1613, 247, 0, 3, 2277, 0, 0, 0, 0, 0, 0, 0, 92, 2, 104, 1190, 3, 447, 90, 4, 2556, 1887, 0, 0, 0, 0, 0, 0, 0, 337, 458, 224, 263, 1753, 1, 2730, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5755, 6451, 6256, 6182, 6443, 5924, 6375, 5766] avg=6144.0000 max=6451 max_rank=1 min=5755 min_rank=0 imbalance_ratio=1.049967 max_load_reduction=42701 max_load_reduction_pct=0.868754 moved_to_echo=24508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 220, 1213, 27, 0, 67, 1060, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 15, 0, 1, 692, 2251, 146, 2783, 0, 0, 0, 0, 0, 0, 0, 14, 699, 84, 256, 85, 29, 180, 720, 3956, 34, 0, 0, 0, 0, 0, 0, 549, 346, 21, 164, 262, 93, 240, 7, 4352, 0, 0, 0, 0, 0, 0, 0, 148, 2204, 81, 25, 2829, 10, 579, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 1, 41, 8, 1929, 272, 6, 41, 2278, 0, 0, 0, 0, 0, 0, 0, 81, 26, 81, 1040, 15, 548, 82, 34, 2460, 1815, 0, 0, 0, 0, 0, 0, 2, 380, 450, 112, 260, 2444, 5, 2797, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6303, 5945, 6057, 6034, 5877, 6304, 6182, 6450] avg=6144.0000 max=6450 max_rank=7 min=5877 min_rank=4 imbalance_ratio=1.049805 max_load_reduction=42702 max_load_reduction_pct=0.868774 moved_to_echo=24813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 217, 1314, 28, 0, 72, 964, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 646, 2413, 134, 2773, 0, 0, 0, 0, 0, 0, 0, 39, 578, 60, 252, 71, 1, 168, 756, 4032, 34, 0, 0, 0, 0, 0, 0, 586, 346, 56, 167, 253, 93, 135, 6, 4284, 0, 0, 0, 0, 0, 0, 0, 84, 2196, 72, 1, 2705, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 0, 2, 0, 1946, 253, 0, 1, 2270, 0, 0, 0, 0, 0, 0, 0, 66, 1, 71, 1028, 52, 525, 67, 1, 2421, 1850, 0, 0, 0, 0, 0, 0, 1, 414, 459, 99, 258, 2483, 5, 3104, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6439, 5970, 5991, 5926, 5775, 6146, 6082, 6823] avg=6144.0000 max=6823 max_rank=7 min=5775 min_rank=4 imbalance_ratio=1.110514 max_load_reduction=42329 max_load_reduction_pct=0.861186 moved_to_echo=24826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 228, 1379, 12, 0, 68, 935, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 711, 2672, 155, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 834, 74, 251, 77, 0, 195, 688, 4077, 35, 0, 0, 0, 0, 0, 0, 769, 337, 0, 112, 227, 90, 220, 0, 4514, 0, 0, 0, 0, 0, 0, 0, 276, 2220, 87, 0, 3891, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 0, 0, 0, 1772, 251, 0, 0, 2294, 0, 0, 0, 0, 0, 0, 0, 87, 0, 88, 815, 12, 440, 85, 0, 2552, 1870, 0, 0, 0, 0, 0, 0, 0, 386, 465, 288, 248, 1999, 0, 2356, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5677, 6370, 6231, 6269, 6733, 6181, 5949, 5742] avg=6144.0000 max=6733 max_rank=4 min=5677 min_rank=0 imbalance_ratio=1.095866 max_load_reduction=42419 max_load_reduction_pct=0.863017 moved_to_echo=24605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 220, 1186, 18, 0, 37, 1154, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 738, 2110, 100, 2851, 0, 0, 0, 0, 0, 0, 0, 5, 458, 60, 253, 52, 0, 153, 814, 4043, 35, 0, 0, 0, 0, 0, 0, 432, 317, 34, 159, 251, 58, 136, 3, 4346, 0, 0, 0, 0, 0, 0, 0, 89, 2257, 66, 0, 1970, 0, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 0, 0, 2087, 240, 0, 0, 2333, 0, 0, 0, 0, 0, 0, 0, 62, 0, 90, 998, 49, 494, 62, 0, 2456, 1855, 0, 0, 0, 0, 0, 0, 0, 400, 460, 75, 215, 2930, 0, 2842, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6993, 5800, 5873, 5736, 5096, 6666, 6066, 6922] avg=6144.0000 max=6993 max_rank=0 min=5096 min_rank=4 imbalance_ratio=1.138184 max_load_reduction=42159 max_load_reduction_pct=0.857727 moved_to_echo=24750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[33, 195, 2333, 91, 4, 119, 1327, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 6, 219, 52, 49, 48, 674, 2532, 185, 2532, 0, 0, 0, 0, 0, 0, 0, 203, 686, 300, 185, 130, 66, 255, 785, 3680, 31, 0, 0, 0, 0, 0, 0, 689, 421, 185, 239, 252, 107, 272, 73, 3858, 0, 0, 0, 0, 0, 0, 0, 357, 2004, 153, 80, 2521, 159, 634, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 17, 133, 74, 1920, 273, 10, 100, 2072, 0, 0, 0, 0, 0, 0, 0, 92, 109, 146, 906, 85, 584, 214, 207, 2181, 1688, 0, 0, 0, 0, 0, 0, 80, 400, 420, 167, 297, 1915, 168, 2884, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6356, 6297, 6321, 6096, 5917, 5622, 6212, 6331] avg=6144.0000 max=6356 max_rank=0 min=5622 min_rank=5 imbalance_ratio=1.034505 max_load_reduction=42796 max_load_reduction_pct=0.870687 moved_to_echo=24082 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 219, 1540, 21, 0, 77, 1112, 2786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1, 0, 0, 667, 2790, 170, 2782, 0, 0, 0, 0, 0, 0, 0, 0, 859, 118, 245, 100, 4, 201, 580, 4110, 39, 0, 0, 0, 0, 0, 0, 676, 337, 0, 73, 246, 107, 224, 0, 4519, 0, 0, 0, 0, 0, 0, 0, 307, 2189, 93, 2, 3549, 0, 302, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1779, 0, 4, 1, 1613, 247, 0, 3, 2277, 0, 0, 0, 0, 0, 0, 0, 92, 2, 104, 1190, 3, 447, 90, 4, 2556, 1887, 0, 0, 0, 0, 0, 0, 0, 337, 458, 224, 263, 1753, 1, 2730, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5755, 6451, 6256, 6182, 6443, 5924, 6375, 5766] avg=6144.0000 max=6451 max_rank=1 min=5755 min_rank=0 imbalance_ratio=1.049967 max_load_reduction=42701 max_load_reduction_pct=0.868754 moved_to_echo=24508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 256, 896, 8064, 256, 3968, 896, 256, 19840, 14720, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 3072, 3712, 1664, 2176, 16896, 256, 21760, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1920, 17536, 768, 128, 25088, 256, 3712, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[13184, 128, 256, 128, 15616, 2048, 128, 256, 18176, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1792, 12160, 256, 128, 640, 8576, 25984, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 512, 128, 128, 128, 5504, 19840, 1280, 22144, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1920, 17536, 768, 128, 25088, 256, 3712, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5504, 2816, 384, 1280, 2048, 768, 1792, 128, 34944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 512, 128, 128, 128, 5504, 19840, 1280, 22144, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 5888, 1024, 1920, 768, 128, 1664, 5632, 32000, 384, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 207, 0, 0, 0, 0, 397, 703, 1525, 0, 802, 336, 0, 86, 59, 0, 0, 0, 0, 78, 0, 0, 1, 3481, 0, 291, 245, 1767, 1, 0, 521, 3174, 0, 0, 0, 1455, 0, 1045, 4203, 543, 6022, 0, 563, 1792, 1109, 0, 0, 130, 0, 2230, 0, 0, 59, 0, 1278, 0, 5047, 5814, 2107, 1909, 0, 172] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 346, 0, 0, 3, 1, 390, 413, 1327, 1, 788, 320, 1, 67, 68, 0, 13, 0, 0, 71, 0, 0, 5, 3058, 0, 238, 281, 1547, 1, 0, 745, 2290, 0, 0, 0, 1741, 1, 1222, 2651, 784, 6594, 0, 633, 731, 1052, 0, 0, 132, 1, 2703, 0, 0, 72, 0, 1460, 1, 4559, 5976, 3264, 3466, 0, 134] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 220, 0, 0, 0, 0, 404, 618, 1530, 0, 794, 344, 0, 92, 67, 0, 0, 0, 0, 83, 0, 0, 10, 3464, 0, 306, 261, 2181, 1, 0, 465, 3140, 0, 0, 0, 1482, 0, 1104, 3863, 627, 5860, 0, 551, 1457, 1061, 0, 0, 220, 0, 2029, 0, 0, 67, 0, 1202, 0, 5275, 5823, 2314, 2054, 0, 183] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[96, 134, 6, 306, 210, 28, 80, 121, 427, 525, 1965, 153, 941, 438, 138, 122, 81, 74, 10, 17, 188, 116, 0, 179, 66, 3111, 22, 381, 396, 1410, 122, 182, 816, 2485, 1, 75, 10, 1529, 95, 1276, 3436, 1220, 5480, 162, 615, 958, 1466, 106, 3, 291, 160, 1789, 59, 98, 139, 47, 1490, 93, 3519, 4501, 2468, 2544, 5, 171] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 223, 0, 0, 0, 0, 361, 553, 2171, 0, 825, 318, 0, 92, 73, 0, 0, 0, 0, 76, 0, 0, 1, 4126, 0, 292, 251, 1567, 2, 0, 585, 3495, 0, 0, 0, 1480, 0, 1113, 4561, 558, 5148, 0, 543, 1815, 1233, 0, 0, 246, 0, 2212, 0, 0, 73, 0, 1244, 0, 4284, 5777, 1832, 1839, 0, 182] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 211, 0, 0, 0, 4, 424, 653, 1886, 2, 806, 337, 0, 90, 53, 0, 0, 0, 4, 100, 0, 3, 2, 3741, 0, 345, 261, 1856, 1, 3, 345, 3526, 0, 0, 0, 1699, 0, 1118, 4139, 662, 5517, 0, 520, 1903, 1498, 6, 0, 142, 4, 2073, 0, 1, 53, 0, 1336, 3, 4365, 5776, 1903, 1598, 0, 182] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 9, 0, 231, 13, 0, 5, 38, 411, 559, 1430, 23, 879, 346, 20, 79, 61, 10, 1, 1, 24, 82, 0, 6, 3, 2992, 0, 297, 255, 1780, 29, 29, 732, 2607, 0, 6, 0, 1449, 0, 1188, 3232, 807, 6478, 24, 565, 1126, 1233, 5, 0, 164, 41, 2632, 3, 29, 66, 13, 1217, 38, 4613, 5721, 2911, 2481, 0, 158] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 328, 0, 0, 0, 0, 347, 493, 1544, 0, 918, 316, 0, 62, 59, 0, 0, 0, 0, 52, 0, 0, 2, 3038, 0, 152, 212, 1643, 0, 0, 877, 2081, 0, 0, 0, 1481, 0, 1177, 2147, 765, 6823, 0, 584, 722, 970, 0, 0, 149, 0, 3251, 0, 0, 59, 0, 1124, 0, 4638, 6022, 3956, 3036, 0, 124] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 346, 0, 0, 3, 1, 390, 413, 1327, 1, 788, 320, 1, 67, 68, 0, 13, 0, 0, 71, 0, 0, 5, 3058, 0, 238, 281, 1547, 1, 0, 745, 2290, 0, 0, 0, 1741, 1, 1222, 2651, 784, 6594, 0, 633, 731, 1052, 0, 0, 132, 1, 2703, 0, 0, 72, 0, 1460, 1, 4559, 5976, 3264, 3466, 0, 134] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 207, 0, 0, 0, 0, 5814, 60, 61, 0, 0, 0, 0, 0, 397, 703, 1525, 0, 802, 336, 0, 86, 1401, 1132, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 78, 0, 0, 5962, 0, 0, 0, 0, 0, 0, 0, 1, 3481, 0, 291, 245, 1767, 1, 0, 236, 94, 66, 0, 0, 0, 0, 0, 521, 3108, 0, 0, 0, 1455, 0, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 2708, 543, 0, 0, 563, 1792, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 2230, 0, 0, 59, 0, 3854, 0, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 1871, 1909, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6382, 6099, 6182, 6129, 6715, 6273, 5230] avg=6144.0000 max=6715 max_rank=5 min=5230 min_rank=7 imbalance_ratio=1.092936 max_load_reduction=42437 max_load_reduction_pct=0.863383 moved_to_echo=24347 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 223, 0, 0, 0, 0, 361, 553, 2171, 0, 825, 318, 0, 92, 73, 0, 0, 0, 0, 76, 0, 0, 1, 4126, 0, 292, 251, 1567, 2, 0, 585, 3495, 0, 0, 0, 1480, 0, 1113, 4561, 558, 5148, 0, 543, 1815, 1233, 0, 0, 246, 0, 2212, 0, 0, 73, 0, 1244, 0, 4284, 5777, 1832, 1839, 0, 182] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[96, 134, 6, 306, 210, 28, 80, 121, 427, 525, 1965, 153, 941, 438, 138, 122, 81, 74, 10, 17, 188, 116, 0, 179, 66, 3111, 22, 381, 396, 1410, 122, 182, 816, 2485, 1, 75, 10, 1529, 95, 1276, 3436, 1220, 5480, 162, 615, 958, 1466, 106, 3, 291, 160, 1789, 59, 98, 139, 47, 1490, 93, 3519, 4501, 2468, 2544, 5, 171] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 207, 0, 0, 0, 0, 397, 703, 1525, 0, 802, 336, 0, 86, 59, 0, 0, 0, 0, 78, 0, 0, 1, 3481, 0, 291, 245, 1767, 1, 0, 521, 3174, 0, 0, 0, 1455, 0, 1045, 4203, 543, 6022, 0, 563, 1792, 1109, 0, 0, 130, 0, 2230, 0, 0, 59, 0, 1278, 0, 5047, 5814, 2107, 1909, 0, 172] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 220, 0, 0, 0, 0, 5823, 58, 63, 0, 0, 0, 0, 0, 404, 618, 1530, 0, 794, 344, 0, 92, 1287, 1185, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 83, 0, 0, 5802, 0, 0, 0, 0, 0, 0, 0, 10, 3464, 0, 306, 261, 2181, 1, 0, 259, 87, 65, 0, 0, 0, 0, 0, 465, 3075, 0, 0, 0, 1482, 0, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 2489, 627, 0, 0, 551, 1457, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 2029, 0, 0, 67, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 1202, 0, 0, 0, 2055, 2054, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 6254, 5952, 6634, 6126, 6185, 6343, 5494] avg=6144.0000 max=6634 max_rank=3 min=5494 min_rank=7 imbalance_ratio=1.079753 max_load_reduction=42518 max_load_reduction_pct=0.865031 moved_to_echo=24148 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[96, 134, 6, 306, 210, 28, 80, 121, 4501, 55, 42, 0, 0, 0, 0, 0, 427, 525, 1965, 153, 941, 438, 138, 122, 1145, 791, 0, 0, 0, 0, 0, 0, 81, 74, 10, 17, 188, 116, 0, 179, 5425, 0, 0, 0, 0, 0, 0, 0, 66, 3111, 22, 381, 396, 1410, 122, 182, 276, 77, 52, 0, 0, 0, 0, 0, 816, 2433, 1, 75, 10, 1529, 95, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 1220, 0, 162, 615, 958, 1466, 106, 0, 0, 0, 0, 0, 0, 0, 0, 3, 291, 160, 1789, 59, 98, 139, 47, 2686, 0, 0, 0, 0, 0, 0, 0, 1490, 93, 0, 0, 2192, 2544, 5, 171, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5579, 6645, 6090, 6095, 6235, 6741, 5272, 6495] avg=6144.0000 max=6741 max_rank=5 min=5272 min_rank=6 imbalance_ratio=1.097168 max_load_reduction=42411 max_load_reduction_pct=0.862854 moved_to_echo=24743 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 223, 0, 0, 0, 0, 5777, 52, 53, 0, 0, 0, 0, 0, 361, 553, 2171, 0, 825, 318, 0, 92, 1520, 961, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 76, 0, 0, 5096, 0, 0, 0, 0, 0, 0, 0, 1, 4126, 0, 292, 251, 1567, 2, 0, 205, 102, 73, 0, 0, 0, 0, 0, 585, 3422, 0, 0, 0, 1480, 0, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 2939, 558, 0, 0, 543, 1815, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 2212, 0, 0, 73, 0, 3270, 0, 0, 0, 0, 0, 0, 0, 1244, 0, 0, 0, 1627, 1839, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 6801, 5245, 6619, 6600, 7088, 5801, 4892] avg=6144.0000 max=7088 max_rank=5 min=4892 min_rank=7 imbalance_ratio=1.153646 max_load_reduction=42064 max_load_reduction_pct=0.855794 moved_to_echo=24381 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 211, 0, 0, 0, 4, 5776, 56, 54, 0, 0, 0, 0, 0, 424, 653, 1886, 2, 806, 337, 0, 90, 1384, 979, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 4, 100, 0, 3, 5461, 0, 0, 0, 0, 0, 0, 0, 2, 3741, 0, 345, 261, 1856, 1, 3, 217, 98, 78, 0, 0, 0, 0, 0, 345, 3448, 0, 0, 0, 1699, 0, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 2657, 662, 0, 0, 520, 1903, 1498, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 4, 2073, 0, 1, 53, 0, 3332, 0, 0, 0, 0, 0, 0, 0, 1336, 3, 0, 0, 1686, 1598, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 6561, 5621, 6602, 6610, 7246, 5605, 4805] avg=6144.0000 max=7246 max_rank=5 min=4805 min_rank=7 imbalance_ratio=1.179362 max_load_reduction=41906 max_load_reduction_pct=0.852580 moved_to_echo=24266 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 231, 13, 0, 5, 38, 5721, 64, 55, 0, 0, 0, 0, 0, 411, 559, 1430, 23, 879, 346, 20, 79, 1077, 1035, 0, 0, 0, 0, 0, 0, 61, 10, 1, 1, 24, 82, 0, 6, 6414, 0, 0, 0, 0, 0, 0, 0, 3, 2992, 0, 297, 255, 1780, 29, 29, 326, 72, 54, 0, 0, 0, 0, 0, 732, 2553, 0, 6, 0, 1449, 0, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 2083, 807, 0, 24, 565, 1126, 1233, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 41, 2632, 3, 29, 66, 13, 3523, 0, 0, 0, 0, 0, 0, 0, 1217, 38, 0, 0, 2585, 2481, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 5859, 6599, 5837, 5928, 5843, 6471, 6479] avg=6144.0000 max=6599 max_rank=2 min=5837 min_rank=3 imbalance_ratio=1.074056 max_load_reduction=42553 max_load_reduction_pct=0.865743 moved_to_echo=24721 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 346, 0, 0, 3, 1, 5976, 65, 55, 0, 0, 0, 0, 0, 390, 413, 1327, 1, 788, 320, 1, 67, 883, 1024, 0, 0, 0, 0, 0, 0, 68, 0, 13, 0, 0, 71, 0, 0, 6529, 0, 0, 0, 0, 0, 0, 0, 5, 3058, 0, 238, 281, 1547, 1, 0, 365, 59, 48, 0, 0, 0, 0, 0, 745, 2242, 0, 0, 0, 1741, 1, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 784, 0, 0, 633, 731, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1, 2703, 0, 0, 72, 0, 3480, 0, 0, 0, 0, 0, 0, 0, 1460, 1, 0, 0, 2899, 3466, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6447, 5214, 6681, 5602, 5951, 4909, 6388, 7960] avg=6144.0000 max=7960 max_rank=7 min=4909 min_rank=5 imbalance_ratio=1.295573 max_load_reduction=41192 max_load_reduction_pct=0.838053 moved_to_echo=25208 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 231, 13, 0, 5, 38, 5721, 64, 55, 0, 0, 0, 0, 0, 411, 559, 1430, 23, 879, 346, 20, 79, 1077, 1035, 0, 0, 0, 0, 0, 0, 61, 10, 1, 1, 24, 82, 0, 6, 6414, 0, 0, 0, 0, 0, 0, 0, 3, 2992, 0, 297, 255, 1780, 29, 29, 326, 72, 54, 0, 0, 0, 0, 0, 732, 2553, 0, 6, 0, 1449, 0, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 2083, 807, 0, 24, 565, 1126, 1233, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 41, 2632, 3, 29, 66, 13, 3523, 0, 0, 0, 0, 0, 0, 0, 1217, 38, 0, 0, 2585, 2481, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 5859, 6599, 5837, 5928, 5843, 6471, 6479] avg=6144.0000 max=6599 max_rank=2 min=5837 min_rank=3 imbalance_ratio=1.074056 max_load_reduction=42553 max_load_reduction_pct=0.865743 moved_to_echo=24721 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 223, 0, 0, 0, 0, 5777, 52, 53, 0, 0, 0, 0, 0, 361, 553, 2171, 0, 825, 318, 0, 92, 1520, 961, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 76, 0, 0, 5096, 0, 0, 0, 0, 0, 0, 0, 1, 4126, 0, 292, 251, 1567, 2, 0, 205, 102, 73, 0, 0, 0, 0, 0, 585, 3422, 0, 0, 0, 1480, 0, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 2939, 558, 0, 0, 543, 1815, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 2212, 0, 0, 73, 0, 3270, 0, 0, 0, 0, 0, 0, 0, 1244, 0, 0, 0, 1627, 1839, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 6801, 5245, 6619, 6600, 7088, 5801, 4892] avg=6144.0000 max=7088 max_rank=5 min=4892 min_rank=7 imbalance_ratio=1.153646 max_load_reduction=42064 max_load_reduction_pct=0.855794 moved_to_echo=24381 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 328, 0, 0, 0, 0, 6022, 69, 57, 0, 0, 0, 0, 0, 347, 493, 1544, 0, 918, 316, 0, 62, 715, 1041, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 52, 0, 0, 6754, 0, 0, 0, 0, 0, 0, 0, 2, 3038, 0, 152, 212, 1643, 0, 0, 443, 48, 43, 0, 0, 0, 0, 0, 877, 2038, 0, 0, 0, 1481, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 765, 0, 0, 584, 722, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 3251, 0, 0, 59, 0, 3540, 0, 0, 0, 0, 0, 0, 0, 1124, 0, 0, 0, 3513, 3036, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6476, 5436, 6865, 5581, 5573, 4425, 6999, 7797] avg=6144.0000 max=7797 max_rank=7 min=4425 min_rank=5 imbalance_ratio=1.269043 max_load_reduction=41355 max_load_reduction_pct=0.841370 moved_to_echo=24794 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[96, 134, 6, 306, 210, 28, 80, 121, 4501, 55, 42, 0, 0, 0, 0, 0, 427, 525, 1965, 153, 941, 438, 138, 122, 1145, 791, 0, 0, 0, 0, 0, 0, 81, 74, 10, 17, 188, 116, 0, 179, 5425, 0, 0, 0, 0, 0, 0, 0, 66, 3111, 22, 381, 396, 1410, 122, 182, 276, 77, 52, 0, 0, 0, 0, 0, 816, 2433, 1, 75, 10, 1529, 95, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 1220, 0, 162, 615, 958, 1466, 106, 0, 0, 0, 0, 0, 0, 0, 0, 3, 291, 160, 1789, 59, 98, 139, 47, 2686, 0, 0, 0, 0, 0, 0, 0, 1490, 93, 0, 0, 2192, 2544, 5, 171, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5579, 6645, 6090, 6095, 6235, 6741, 5272, 6495] avg=6144.0000 max=6741 max_rank=5 min=5272 min_rank=6 imbalance_ratio=1.097168 max_load_reduction=42411 max_load_reduction_pct=0.862854 moved_to_echo=24743 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 211, 0, 0, 0, 4, 5776, 56, 54, 0, 0, 0, 0, 0, 424, 653, 1886, 2, 806, 337, 0, 90, 1384, 979, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 4, 100, 0, 3, 5461, 0, 0, 0, 0, 0, 0, 0, 2, 3741, 0, 345, 261, 1856, 1, 3, 217, 98, 78, 0, 0, 0, 0, 0, 345, 3448, 0, 0, 0, 1699, 0, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 2657, 662, 0, 0, 520, 1903, 1498, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 4, 2073, 0, 1, 53, 0, 3332, 0, 0, 0, 0, 0, 0, 0, 1336, 3, 0, 0, 1686, 1598, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 6561, 5621, 6602, 6610, 7246, 5605, 4805] avg=6144.0000 max=7246 max_rank=5 min=4805 min_rank=7 imbalance_ratio=1.179362 max_load_reduction=41906 max_load_reduction_pct=0.852580 moved_to_echo=24266 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 207, 0, 0, 0, 0, 5814, 60, 61, 0, 0, 0, 0, 0, 397, 703, 1525, 0, 802, 336, 0, 86, 1401, 1132, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 78, 0, 0, 5962, 0, 0, 0, 0, 0, 0, 0, 1, 3481, 0, 291, 245, 1767, 1, 0, 236, 94, 66, 0, 0, 0, 0, 0, 521, 3108, 0, 0, 0, 1455, 0, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 2708, 543, 0, 0, 563, 1792, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 2230, 0, 0, 59, 0, 3854, 0, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 1871, 1909, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6382, 6099, 6182, 6129, 6715, 6273, 5230] avg=6144.0000 max=6715 max_rank=5 min=5230 min_rank=7 imbalance_ratio=1.092936 max_load_reduction=42437 max_load_reduction_pct=0.863383 moved_to_echo=24347 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 4608, 13440, 256, 6784, 2816, 256, 768, 9472, 8192, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10368, 256, 0, 0, 18432, 18944, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 22400, 128, 128, 128, 12416, 128, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1536, 256, 18944, 128, 128, 640, 128, 27776, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18304, 6016, 0, 256, 4608, 10624, 9728, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 22400, 128, 128, 128, 12416, 128, 9344, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 27136, 128, 2304, 2176, 13824, 256, 256, 2432, 640, 512, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1536, 256, 18944, 128, 128, 640, 128, 27776, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 256, 128, 2176, 256, 128, 128, 256, 45440, 512, 512, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 128, 128, 128, 256, 768, 0, 256, 47488, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 68, 310, 329, 142, 2803, 0, 418, 0, 0, 25, 0, 87, 0, 0, 0, 1912, 0, 0, 0, 0, 2740, 0, 1357, 0, 0, 144, 705, 0, 0, 0, 68, 6473, 7, 87, 5864, 0, 0, 164, 770, 235, 0, 0, 2300, 78, 0, 0, 0, 320, 0, 2814, 86, 4540, 66, 6327, 0, 0, 1850, 2799, 904, 1379, 820, 0, 161] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 69, 319, 335, 123, 2617, 0, 410, 0, 0, 9, 0, 92, 0, 0, 0, 1370, 0, 0, 0, 0, 2682, 0, 1289, 0, 0, 84, 554, 0, 0, 0, 69, 6440, 0, 92, 6026, 0, 0, 175, 767, 250, 0, 0, 2301, 83, 0, 0, 0, 391, 0, 2820, 91, 4191, 68, 6354, 0, 0, 1843, 3416, 985, 1524, 1073, 0, 240] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 63, 321, 324, 320, 3167, 0, 408, 0, 1, 11, 0, 67, 0, 0, 0, 1988, 0, 0, 0, 0, 2192, 0, 1063, 0, 42, 24, 514, 0, 0, 0, 63, 6604, 4, 66, 5671, 0, 1, 138, 1355, 208, 1, 0, 1983, 71, 1, 1, 2, 399, 1, 3093, 67, 3511, 63, 6313, 1, 53, 1640, 3044, 905, 2075, 1185, 1, 127] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[9, 69, 327, 351, 383, 3019, 28, 427, 0, 7, 12, 9, 85, 32, 5, 5, 1889, 2, 0, 0, 3, 2037, 11, 1002, 4, 27, 116, 502, 1, 0, 0, 74, 6572, 33, 80, 5443, 0, 30, 161, 908, 237, 38, 60, 1726, 82, 2, 44, 15, 394, 1, 2852, 79, 3975, 66, 6219, 18, 11, 1724, 3623, 930, 2019, 1196, 16, 162] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 69, 319, 335, 123, 2617, 0, 410, 0, 0, 9, 0, 92, 0, 0, 0, 1370, 0, 0, 0, 0, 2682, 0, 1289, 0, 0, 84, 554, 0, 0, 0, 69, 6440, 0, 92, 6026, 0, 0, 175, 767, 250, 0, 0, 2301, 83, 0, 0, 0, 391, 0, 2820, 91, 4191, 68, 6354, 0, 0, 1843, 3416, 985, 1524, 1073, 0, 240] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 68, 310, 329, 142, 2803, 0, 418, 0, 0, 25, 0, 87, 0, 0, 0, 1912, 0, 0, 0, 0, 2740, 0, 1357, 0, 0, 144, 705, 0, 0, 0, 68, 6473, 7, 87, 5864, 0, 0, 164, 770, 235, 0, 0, 2300, 78, 0, 0, 0, 320, 0, 2814, 86, 4540, 66, 6327, 0, 0, 1850, 2799, 904, 1379, 820, 0, 161] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[114, 69, 455, 429, 361, 2070, 153, 570, 4, 100, 26, 85, 194, 215, 8, 49, 1336, 6, 20, 0, 57, 1836, 37, 1295, 42, 238, 320, 524, 69, 49, 24, 78, 6054, 89, 91, 4185, 2, 145, 192, 992, 306, 129, 342, 1856, 105, 60, 246, 96, 431, 38, 3206, 91, 3742, 66, 5836, 96, 125, 1921, 3272, 977, 1817, 1431, 116, 264] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 37, 315, 306, 472, 3144, 0, 488, 0, 0, 7, 0, 62, 0, 0, 0, 1716, 0, 0, 0, 0, 2061, 0, 983, 0, 59, 10, 488, 0, 0, 0, 37, 6670, 0, 62, 5624, 0, 0, 114, 1202, 156, 0, 0, 1402, 52, 0, 0, 0, 334, 0, 2870, 62, 3951, 37, 6493, 0, 44, 1797, 3581, 876, 2460, 1058, 0, 122] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 58, 300, 303, 326, 2523, 0, 432, 0, 0, 16, 0, 92, 0, 0, 0, 1522, 0, 0, 0, 0, 2609, 0, 1832, 0, 0, 176, 532, 0, 0, 0, 58, 6517, 2, 92, 5817, 0, 0, 168, 756, 237, 0, 0, 2022, 76, 0, 0, 0, 289, 0, 3041, 92, 4333, 58, 6320, 0, 0, 1845, 3269, 939, 1409, 856, 0, 235] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 77, 300, 348, 291, 2574, 3, 438, 0, 0, 15, 1, 92, 0, 0, 2, 1820, 0, 0, 0, 0, 2481, 0, 1662, 1, 0, 204, 675, 0, 0, 2, 79, 6387, 6, 92, 5466, 0, 2, 190, 622, 276, 4, 4, 2649, 100, 0, 4, 0, 388, 0, 3290, 90, 3909, 77, 6239, 2, 0, 1545, 3167, 839, 1536, 992, 1, 210] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 37, 315, 306, 472, 3144, 0, 488, 0, 0, 7, 0, 62, 0, 0, 0, 1716, 0, 0, 0, 0, 2061, 0, 983, 0, 59, 10, 488, 0, 0, 0, 37, 6670, 0, 62, 5624, 0, 0, 114, 1202, 156, 0, 0, 1402, 52, 0, 0, 0, 334, 0, 2870, 62, 3951, 37, 6493, 0, 44, 1797, 3581, 876, 2460, 1058, 0, 122] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[114, 69, 455, 429, 361, 2070, 153, 570, 4, 100, 26, 85, 194, 215, 8, 49, 1336, 6, 20, 0, 57, 1836, 37, 1295, 42, 238, 320, 524, 69, 49, 24, 78, 6054, 89, 91, 4185, 2, 145, 192, 992, 306, 129, 342, 1856, 105, 60, 246, 96, 431, 38, 3206, 91, 3742, 66, 5836, 96, 125, 1921, 3272, 977, 1817, 1431, 116, 264] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 310, 329, 142, 2803, 0, 418, 1342, 591, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 87, 0, 0, 0, 5974, 0, 0, 0, 0, 0, 0, 0, 1912, 0, 0, 0, 0, 2740, 0, 1357, 611, 245, 0, 0, 0, 0, 0, 0, 0, 0, 144, 705, 0, 0, 0, 68, 4863, 499, 0, 0, 0, 0, 0, 0, 0, 7, 87, 5253, 0, 0, 164, 770, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 2300, 78, 0, 0, 0, 1895, 1464, 0, 0, 0, 0, 0, 0, 320, 0, 2814, 86, 2953, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 313, 904, 1379, 820, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6003, 6086, 6865, 6279, 6281, 5972, 6239, 5427] avg=6144.0000 max=6865 max_rank=2 min=5427 min_rank=7 imbalance_ratio=1.117350 max_load_reduction=42287 max_load_reduction_pct=0.860331 moved_to_echo=23919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 69, 319, 335, 123, 2617, 0, 410, 1239, 721, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 92, 0, 0, 0, 5944, 0, 0, 0, 0, 0, 0, 0, 1370, 0, 0, 0, 0, 2682, 0, 1289, 628, 226, 0, 0, 0, 0, 0, 0, 0, 0, 84, 554, 0, 0, 0, 69, 4885, 496, 0, 0, 0, 0, 0, 0, 0, 0, 92, 5398, 0, 0, 175, 767, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 2301, 83, 0, 0, 0, 2313, 1469, 0, 0, 0, 0, 0, 0, 391, 0, 2820, 91, 2726, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1843, 382, 985, 1524, 1073, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5833, 6045, 6195, 6088, 6432, 6416, 6096, 6047] avg=6144.0000 max=6432 max_rank=4 min=5833 min_rank=0 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 58, 300, 303, 326, 2523, 0, 432, 1281, 690, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 92, 0, 0, 0, 6016, 0, 0, 0, 0, 0, 0, 0, 1522, 0, 0, 0, 0, 2609, 0, 1832, 606, 234, 0, 0, 0, 0, 0, 0, 0, 0, 176, 532, 0, 0, 0, 58, 4859, 501, 0, 0, 0, 0, 0, 0, 0, 2, 92, 5211, 0, 0, 168, 756, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 2022, 76, 0, 0, 0, 2214, 1461, 0, 0, 0, 0, 0, 0, 289, 0, 3041, 92, 2818, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1845, 365, 939, 1409, 856, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5913, 6124, 6803, 6126, 6229, 6010, 6298, 5649] avg=6144.0000 max=6803 max_rank=2 min=5649 min_rank=7 imbalance_ratio=1.107259 max_load_reduction=42349 max_load_reduction_pct=0.861593 moved_to_echo=24186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 63, 321, 324, 320, 3167, 0, 408, 1038, 643, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 67, 0, 0, 0, 6095, 0, 0, 0, 0, 0, 0, 0, 1988, 0, 0, 0, 0, 2192, 0, 1063, 591, 190, 0, 0, 0, 0, 0, 0, 0, 42, 24, 514, 0, 0, 0, 63, 4853, 509, 0, 0, 0, 0, 0, 0, 0, 4, 66, 5080, 0, 1, 138, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 208, 1, 0, 1983, 71, 1, 1, 2, 2061, 1460, 0, 0, 0, 0, 0, 0, 399, 1, 3093, 67, 2283, 63, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1640, 340, 905, 2075, 1185, 1, 127, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6174, 6024, 6005, 6644, 5788, 5907, 6326] avg=6144.0000 max=6644 max_rank=4 min=5788 min_rank=5 imbalance_ratio=1.081380 max_load_reduction=42508 max_load_reduction_pct=0.864827 moved_to_echo=24665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[114, 69, 455, 429, 361, 2070, 153, 570, 1106, 691, 0, 0, 0, 0, 0, 0, 4, 100, 26, 85, 194, 215, 8, 49, 5589, 0, 0, 0, 0, 0, 0, 0, 1336, 6, 20, 0, 57, 1836, 37, 1295, 436, 202, 0, 0, 0, 0, 0, 0, 42, 238, 320, 524, 69, 49, 24, 78, 4487, 465, 0, 0, 0, 0, 0, 0, 0, 89, 91, 3749, 2, 145, 192, 992, 0, 0, 0, 0, 0, 0, 0, 0, 306, 129, 342, 1856, 105, 60, 246, 96, 2216, 1349, 0, 0, 0, 0, 0, 0, 431, 38, 3206, 91, 2434, 66, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 125, 1921, 365, 977, 1817, 1431, 116, 264, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6018, 6270, 5225, 6296, 5260, 6705, 6362, 7016] avg=6144.0000 max=7016 max_rank=7 min=5225 min_rank=2 imbalance_ratio=1.141927 max_load_reduction=42136 max_load_reduction_pct=0.857259 moved_to_echo=25343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[9, 69, 327, 351, 383, 3019, 28, 427, 1175, 765, 0, 0, 0, 0, 0, 0, 0, 7, 12, 9, 85, 32, 5, 5, 6066, 0, 0, 0, 0, 0, 0, 0, 1889, 2, 0, 0, 3, 2037, 11, 1002, 567, 215, 0, 0, 0, 0, 0, 0, 4, 27, 116, 502, 1, 0, 0, 74, 4780, 506, 0, 0, 0, 0, 0, 0, 0, 33, 80, 4876, 0, 30, 161, 908, 0, 0, 0, 0, 0, 0, 0, 0, 237, 38, 60, 1726, 82, 2, 44, 15, 2454, 1439, 0, 0, 0, 0, 0, 0, 394, 1, 2852, 79, 2585, 66, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1724, 404, 930, 2019, 1196, 16, 162, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6553, 6221, 5726, 6010, 6088, 6097, 5995, 6462] avg=6144.0000 max=6553 max_rank=0 min=5726 min_rank=2 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=24642 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 300, 348, 291, 2574, 3, 438, 1158, 673, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 92, 0, 0, 2, 5896, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 2481, 0, 1662, 576, 216, 0, 0, 0, 0, 0, 0, 1, 0, 204, 675, 0, 0, 2, 79, 4797, 491, 0, 0, 0, 0, 0, 0, 0, 6, 92, 4890, 0, 2, 190, 622, 0, 0, 0, 0, 0, 0, 0, 0, 276, 4, 4, 2649, 100, 0, 4, 0, 2149, 1442, 0, 0, 0, 0, 0, 0, 388, 0, 3290, 90, 2535, 77, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 345, 839, 1536, 992, 1, 210, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5862, 6006, 6755, 6249, 5802, 6628, 6382, 5468] avg=6144.0000 max=6755 max_rank=2 min=5468 min_rank=7 imbalance_ratio=1.099447 max_load_reduction=42397 max_load_reduction_pct=0.862569 moved_to_echo=24280 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 69, 319, 335, 123, 2617, 0, 410, 1239, 721, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 92, 0, 0, 0, 5944, 0, 0, 0, 0, 0, 0, 0, 1370, 0, 0, 0, 0, 2682, 0, 1289, 628, 226, 0, 0, 0, 0, 0, 0, 0, 0, 84, 554, 0, 0, 0, 69, 4885, 496, 0, 0, 0, 0, 0, 0, 0, 0, 92, 5398, 0, 0, 175, 767, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 2301, 83, 0, 0, 0, 2313, 1469, 0, 0, 0, 0, 0, 0, 391, 0, 2820, 91, 2726, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1843, 382, 985, 1524, 1073, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5833, 6045, 6195, 6088, 6432, 6416, 6096, 6047] avg=6144.0000 max=6432 max_rank=4 min=5833 min_rank=0 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 37, 315, 306, 472, 3144, 0, 488, 1168, 756, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 62, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 1716, 0, 0, 0, 0, 2061, 0, 983, 586, 213, 0, 0, 0, 0, 0, 0, 0, 59, 10, 488, 0, 0, 0, 37, 4992, 513, 0, 0, 0, 0, 0, 0, 0, 0, 62, 5038, 0, 0, 114, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 1402, 52, 0, 0, 0, 2425, 1501, 0, 0, 0, 0, 0, 0, 334, 0, 2870, 62, 2570, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1797, 400, 876, 2460, 1058, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6686, 6226, 5559, 6099, 6416, 5536, 5873, 6757] avg=6144.0000 max=6757 max_rank=7 min=5536 min_rank=5 imbalance_ratio=1.099772 max_load_reduction=42395 max_load_reduction_pct=0.862528 moved_to_echo=24582 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[114, 69, 455, 429, 361, 2070, 153, 570, 1106, 691, 0, 0, 0, 0, 0, 0, 4, 100, 26, 85, 194, 215, 8, 49, 5589, 0, 0, 0, 0, 0, 0, 0, 1336, 6, 20, 0, 57, 1836, 37, 1295, 436, 202, 0, 0, 0, 0, 0, 0, 42, 238, 320, 524, 69, 49, 24, 78, 4487, 465, 0, 0, 0, 0, 0, 0, 0, 89, 91, 3749, 2, 145, 192, 992, 0, 0, 0, 0, 0, 0, 0, 0, 306, 129, 342, 1856, 105, 60, 246, 96, 2216, 1349, 0, 0, 0, 0, 0, 0, 431, 38, 3206, 91, 2434, 66, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 125, 1921, 365, 977, 1817, 1431, 116, 264, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6018, 6270, 5225, 6296, 5260, 6705, 6362, 7016] avg=6144.0000 max=7016 max_rank=7 min=5225 min_rank=2 imbalance_ratio=1.141927 max_load_reduction=42136 max_load_reduction_pct=0.857259 moved_to_echo=25343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 58, 300, 303, 326, 2523, 0, 432, 1281, 690, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 92, 0, 0, 0, 6016, 0, 0, 0, 0, 0, 0, 0, 1522, 0, 0, 0, 0, 2609, 0, 1832, 606, 234, 0, 0, 0, 0, 0, 0, 0, 0, 176, 532, 0, 0, 0, 58, 4859, 501, 0, 0, 0, 0, 0, 0, 0, 2, 92, 5211, 0, 0, 168, 756, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 2022, 76, 0, 0, 0, 2214, 1461, 0, 0, 0, 0, 0, 0, 289, 0, 3041, 92, 2818, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1845, 365, 939, 1409, 856, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5913, 6124, 6803, 6126, 6229, 6010, 6298, 5649] avg=6144.0000 max=6803 max_rank=2 min=5649 min_rank=7 imbalance_ratio=1.107259 max_load_reduction=42349 max_load_reduction_pct=0.861593 moved_to_echo=24186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 310, 329, 142, 2803, 0, 418, 1342, 591, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 87, 0, 0, 0, 5974, 0, 0, 0, 0, 0, 0, 0, 1912, 0, 0, 0, 0, 2740, 0, 1357, 611, 245, 0, 0, 0, 0, 0, 0, 0, 0, 144, 705, 0, 0, 0, 68, 4863, 499, 0, 0, 0, 0, 0, 0, 0, 7, 87, 5253, 0, 0, 164, 770, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 2300, 78, 0, 0, 0, 1895, 1464, 0, 0, 0, 0, 0, 0, 320, 0, 2814, 86, 2953, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 313, 904, 1379, 820, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6003, 6086, 6865, 6279, 6281, 5972, 6239, 5427] avg=6144.0000 max=6865 max_rank=2 min=5427 min_rank=7 imbalance_ratio=1.117350 max_load_reduction=42287 max_load_reduction_pct=0.860331 moved_to_echo=23919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 300, 348, 291, 2574, 3, 438, 1158, 673, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 92, 0, 0, 2, 5896, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 2481, 0, 1662, 576, 216, 0, 0, 0, 0, 0, 0, 1, 0, 204, 675, 0, 0, 2, 79, 4797, 491, 0, 0, 0, 0, 0, 0, 0, 6, 92, 4890, 0, 2, 190, 622, 0, 0, 0, 0, 0, 0, 0, 0, 276, 4, 4, 2649, 100, 0, 4, 0, 2149, 1442, 0, 0, 0, 0, 0, 0, 388, 0, 3290, 90, 2535, 77, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 345, 839, 1536, 992, 1, 210, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5862, 6006, 6755, 6249, 5802, 6628, 6382, 5468] avg=6144.0000 max=6755 max_rank=2 min=5468 min_rank=7 imbalance_ratio=1.099447 max_load_reduction=42397 max_load_reduction_pct=0.862569 moved_to_echo=24280 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 14208, 2944, 7424, 14336, 8704, 256, 1536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 512, 2688, 2816, 2432, 22016, 256, 3712, 9600, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 14208, 2944, 7424, 14336, 8704, 256, 1536, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 128, 24064, 768, 20992, 512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13568, 128, 128, 0, 128, 18688, 128, 10496, 4608, 1792, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 256, 512, 16256, 768, 128, 384, 128, 17792, 11648, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 256, 768, 39552, 128, 256, 1408, 7424, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 384, 1152, 4608, 128, 128, 128, 640, 38528, 4096, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 896, 256, 128, 128, 47744, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[55, 0, 68, 0, 71, 0, 0, 0, 0, 1484, 0, 162, 87, 150, 373, 1686, 2, 7133, 0, 475, 0, 0, 59, 0, 6813, 0, 357, 251, 0, 514, 86, 6078, 6940, 6473, 0, 714, 0, 0, 0, 0, 87, 57, 800, 0, 0, 1289, 251, 0, 96, 0, 0, 3229, 0, 0, 0, 0, 0, 3253, 0, 0, 0, 0, 0, 59] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[117, 0, 69, 0, 92, 0, 0, 0, 0, 1945, 0, 239, 91, 128, 359, 1622, 0, 7009, 0, 452, 0, 0, 67, 0, 6596, 0, 290, 252, 0, 558, 92, 6100, 6966, 6259, 0, 557, 0, 0, 0, 0, 92, 67, 837, 0, 0, 1353, 252, 0, 227, 0, 0, 3035, 0, 0, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 68] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[32, 0, 58, 0, 71, 0, 0, 0, 0, 1257, 0, 246, 92, 127, 349, 1562, 0, 7009, 0, 506, 0, 0, 73, 0, 6651, 0, 504, 227, 0, 584, 92, 5986, 6947, 6361, 0, 834, 0, 0, 0, 0, 92, 73, 775, 0, 0, 1523, 227, 0, 127, 0, 0, 3564, 0, 0, 0, 0, 0, 3130, 0, 0, 0, 0, 0, 73] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[175, 35, 100, 114, 98, 63, 50, 0, 75, 2189, 6, 269, 192, 1245, 520, 1620, 25, 6333, 159, 576, 11, 11, 241, 197, 5698, 186, 290, 249, 65, 804, 110, 5244, 6359, 3759, 223, 857, 69, 53, 6, 125, 249, 101, 1954, 236, 37, 1386, 287, 196, 427, 62, 21, 2723, 51, 30, 29, 24, 116, 2233, 17, 48, 161, 217, 25, 121] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[117, 0, 69, 0, 92, 0, 0, 0, 0, 1945, 0, 239, 91, 128, 359, 1622, 0, 7009, 0, 452, 0, 0, 67, 0, 6596, 0, 290, 252, 0, 558, 92, 6100, 6966, 6259, 0, 557, 0, 0, 0, 0, 92, 67, 837, 0, 0, 1353, 252, 0, 227, 0, 0, 3035, 0, 0, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 68] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 63, 10, 107, 0, 0, 0, 0, 2023, 0, 217, 67, 509, 415, 1470, 0, 7067, 1, 491, 0, 0, 68, 1, 6764, 1, 216, 255, 1, 550, 67, 5600, 6904, 5936, 1, 589, 1, 0, 0, 0, 71, 68, 1905, 0, 0, 1122, 253, 0, 357, 0, 0, 2389, 0, 0, 0, 0, 0, 3524, 0, 0, 0, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 37, 0, 61, 0, 0, 0, 0, 2531, 0, 177, 62, 532, 400, 1628, 0, 7187, 0, 433, 0, 0, 59, 0, 6815, 0, 156, 254, 0, 583, 62, 5834, 6948, 6003, 0, 346, 0, 0, 0, 0, 62, 59, 1255, 0, 0, 1079, 254, 0, 486, 0, 0, 2374, 0, 0, 0, 0, 0, 3410, 0, 0, 0, 0, 0, 59] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[58, 0, 78, 7, 89, 0, 0, 0, 0, 1497, 0, 213, 92, 174, 338, 1339, 2, 7087, 2, 430, 0, 0, 53, 2, 6613, 4, 399, 247, 2, 556, 90, 5991, 6909, 6226, 0, 786, 3, 0, 0, 0, 93, 52, 1066, 2, 0, 1404, 249, 1, 235, 0, 0, 3324, 0, 0, 1, 0, 1, 3379, 0, 0, 1, 3, 0, 54] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[28, 0, 66, 11, 97, 1, 2, 0, 20, 2525, 0, 172, 101, 527, 380, 1399, 0, 6956, 20, 458, 0, 0, 87, 19, 6688, 47, 198, 261, 27, 675, 79, 5681, 6848, 5700, 25, 503, 24, 5, 0, 4, 96, 62, 1345, 34, 0, 1224, 269, 47, 384, 1, 0, 2942, 6, 0, 4, 3, 24, 2959, 1, 2, 6, 37, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[55, 0, 68, 0, 71, 0, 0, 0, 5970, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 0, 162, 87, 150, 373, 1686, 993, 455, 175, 0, 0, 0, 0, 0, 2, 5650, 0, 475, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 251, 0, 514, 86, 5085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6018, 0, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 57, 800, 0, 0, 1289, 251, 0, 2226, 970, 0, 0, 0, 0, 0, 0, 96, 0, 0, 3229, 0, 0, 0, 0, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 3253, 0, 0, 0, 0, 0, 59, 1610, 1308, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 5565, 6186, 6293, 6732, 5680, 6302, 6230] avg=6144.0000 max=6732 max_rank=4 min=5565 min_rank=1 imbalance_ratio=1.095703 max_load_reduction=42420 max_load_reduction_pct=0.863037 moved_to_echo=24944 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 58, 0, 71, 0, 0, 0, 5977, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 0, 246, 92, 127, 349, 1562, 978, 447, 172, 0, 0, 0, 0, 0, 0, 5552, 0, 506, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 227, 0, 584, 92, 5008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5914, 0, 834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 73, 775, 0, 0, 1523, 227, 0, 2174, 970, 0, 0, 0, 0, 0, 0, 127, 0, 0, 3564, 0, 0, 0, 0, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 3130, 0, 0, 0, 0, 0, 73, 1571, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 5230, 6131, 6415, 6748, 5834, 6597, 6059] avg=6144.0000 max=6748 max_rank=4 min=5230 min_rank=1 imbalance_ratio=1.098307 max_load_reduction=42404 max_load_reduction_pct=0.862712 moved_to_echo=25238 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[175, 35, 100, 114, 98, 63, 50, 0, 75, 2189, 6, 269, 192, 1245, 520, 1620, 25, 6333, 159, 576, 11, 11, 241, 197, 5698, 186, 290, 249, 65, 804, 110, 5244, 6359, 3759, 223, 857, 69, 53, 6, 125, 249, 101, 1954, 236, 37, 1386, 287, 196, 427, 62, 21, 2723, 51, 30, 29, 24, 116, 2233, 17, 48, 161, 217, 25, 121] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[55, 0, 68, 0, 71, 0, 0, 0, 0, 1484, 0, 162, 87, 150, 373, 1686, 2, 7133, 0, 475, 0, 0, 59, 0, 6813, 0, 357, 251, 0, 514, 86, 6078, 6940, 6473, 0, 714, 0, 0, 0, 0, 87, 57, 800, 0, 0, 1289, 251, 0, 96, 0, 0, 3229, 0, 0, 0, 0, 0, 3253, 0, 0, 0, 0, 0, 59] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[58, 0, 78, 7, 89, 0, 0, 0, 0, 1497, 0, 213, 92, 174, 338, 1339, 2, 7087, 2, 430, 0, 0, 53, 2, 6613, 4, 399, 247, 2, 556, 90, 5991, 6909, 6226, 0, 786, 3, 0, 0, 0, 93, 52, 1066, 2, 0, 1404, 249, 1, 235, 0, 0, 3324, 0, 0, 1, 0, 1, 3379, 0, 0, 1, 3, 0, 54] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 37, 0, 61, 0, 0, 0, 0, 2531, 0, 177, 62, 532, 400, 1628, 0, 7187, 0, 433, 0, 0, 59, 0, 6815, 0, 156, 254, 0, 583, 62, 5834, 6948, 6003, 0, 346, 0, 0, 0, 0, 62, 59, 1255, 0, 0, 1079, 254, 0, 486, 0, 0, 2374, 0, 0, 0, 0, 0, 3410, 0, 0, 0, 0, 0, 59] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[117, 0, 69, 0, 92, 0, 0, 0, 5993, 0, 0, 0, 0, 0, 0, 0, 0, 1945, 0, 239, 91, 128, 359, 1622, 997, 440, 172, 0, 0, 0, 0, 0, 0, 5552, 0, 452, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 252, 0, 558, 92, 5103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5819, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 67, 837, 0, 0, 1353, 252, 0, 2155, 973, 0, 0, 0, 0, 0, 0, 227, 0, 0, 3035, 0, 0, 0, 0, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 68, 1558, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6271, 5993, 6071, 6295, 6376, 5729, 6145, 6272] avg=6144.0000 max=6376 max_rank=4 min=5729 min_rank=5 imbalance_ratio=1.037760 max_load_reduction=42776 max_load_reduction_pct=0.870280 moved_to_echo=24522 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[175, 35, 100, 114, 98, 63, 50, 0, 5471, 0, 0, 0, 0, 0, 0, 0, 75, 2189, 6, 269, 192, 1245, 520, 1620, 857, 264, 156, 0, 0, 0, 0, 0, 25, 5016, 159, 576, 11, 11, 241, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 290, 249, 65, 804, 110, 4387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3495, 223, 857, 69, 53, 6, 125, 0, 0, 0, 0, 0, 0, 0, 0, 249, 101, 1954, 236, 37, 1386, 287, 196, 1861, 888, 0, 0, 0, 0, 0, 0, 427, 62, 21, 2723, 51, 30, 29, 24, 2491, 0, 0, 0, 0, 0, 0, 0, 116, 2233, 17, 48, 161, 217, 25, 121, 1346, 1161, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 7393, 6236, 6091, 4828, 7195, 5858, 5445] avg=6144.0000 max=7393 max_rank=1 min=4828 min_rank=4 imbalance_ratio=1.203288 max_load_reduction=41759 max_load_reduction_pct=0.849589 moved_to_echo=23326 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 63, 10, 107, 0, 0, 0, 5939, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 0, 217, 67, 509, 415, 1470, 915, 418, 174, 0, 0, 0, 0, 0, 0, 5597, 1, 491, 0, 0, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 216, 255, 1, 550, 67, 4685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5518, 1, 589, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 68, 1905, 0, 0, 1122, 253, 0, 2210, 965, 0, 0, 0, 0, 0, 0, 357, 0, 0, 2389, 0, 0, 0, 0, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 0, 0, 0, 0, 0, 69, 1598, 1296, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6119, 6208, 6158, 5775, 6109, 6594, 5702, 6487] avg=6144.0000 max=6594 max_rank=5 min=5702 min_rank=6 imbalance_ratio=1.073242 max_load_reduction=42558 max_load_reduction_pct=0.865845 moved_to_echo=24892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[58, 0, 78, 7, 89, 0, 0, 0, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 1497, 0, 213, 92, 174, 338, 1339, 982, 443, 179, 0, 0, 0, 0, 0, 2, 5604, 2, 430, 0, 0, 53, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 399, 247, 2, 556, 90, 5009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5783, 0, 786, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 52, 1066, 2, 0, 1404, 249, 1, 2162, 965, 0, 0, 0, 0, 0, 0, 235, 0, 0, 3324, 0, 0, 1, 0, 2889, 0, 0, 0, 0, 0, 0, 0, 1, 3379, 0, 0, 1, 3, 0, 54, 1562, 1304, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 5257, 6093, 6307, 6572, 5994, 6449, 6304] avg=6144.0000 max=6572 max_rank=4 min=5257 min_rank=1 imbalance_ratio=1.069661 max_load_reduction=42580 max_load_reduction_pct=0.866292 moved_to_echo=25319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 58, 0, 71, 0, 0, 0, 5977, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 0, 246, 92, 127, 349, 1562, 978, 447, 172, 0, 0, 0, 0, 0, 0, 5552, 0, 506, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 227, 0, 584, 92, 5008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5914, 0, 834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 73, 775, 0, 0, 1523, 227, 0, 2174, 970, 0, 0, 0, 0, 0, 0, 127, 0, 0, 3564, 0, 0, 0, 0, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 3130, 0, 0, 0, 0, 0, 73, 1571, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 5230, 6131, 6415, 6748, 5834, 6597, 6059] avg=6144.0000 max=6748 max_rank=4 min=5230 min_rank=1 imbalance_ratio=1.098307 max_load_reduction=42404 max_load_reduction_pct=0.862712 moved_to_echo=25238 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 66, 11, 97, 1, 2, 0, 5891, 0, 0, 0, 0, 0, 0, 0, 20, 2525, 0, 172, 101, 527, 380, 1399, 928, 401, 171, 0, 0, 0, 0, 0, 0, 5509, 20, 458, 0, 0, 87, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 198, 261, 27, 675, 79, 4753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5299, 25, 503, 24, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 96, 62, 1345, 34, 0, 1224, 269, 47, 2185, 957, 0, 0, 0, 0, 0, 0, 384, 1, 0, 2942, 6, 0, 4, 3, 2922, 0, 0, 0, 0, 0, 0, 0, 24, 2959, 1, 2, 6, 37, 0, 72, 1581, 1276, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6096, 6624, 6093, 6040, 5860, 6219, 6262, 5958] avg=6144.0000 max=6624 max_rank=1 min=5860 min_rank=4 imbalance_ratio=1.078125 max_load_reduction=42528 max_load_reduction_pct=0.865234 moved_to_echo=24299 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 63, 10, 107, 0, 0, 0, 5939, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 0, 217, 67, 509, 415, 1470, 915, 418, 174, 0, 0, 0, 0, 0, 0, 5597, 1, 491, 0, 0, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 216, 255, 1, 550, 67, 4685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5518, 1, 589, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 68, 1905, 0, 0, 1122, 253, 0, 2210, 965, 0, 0, 0, 0, 0, 0, 357, 0, 0, 2389, 0, 0, 0, 0, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 0, 0, 0, 0, 0, 69, 1598, 1296, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6119, 6208, 6158, 5775, 6109, 6594, 5702, 6487] avg=6144.0000 max=6594 max_rank=5 min=5702 min_rank=6 imbalance_ratio=1.073242 max_load_reduction=42558 max_load_reduction_pct=0.865845 moved_to_echo=24892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[175, 35, 100, 114, 98, 63, 50, 0, 5471, 0, 0, 0, 0, 0, 0, 0, 75, 2189, 6, 269, 192, 1245, 520, 1620, 857, 264, 156, 0, 0, 0, 0, 0, 25, 5016, 159, 576, 11, 11, 241, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 290, 249, 65, 804, 110, 4387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3495, 223, 857, 69, 53, 6, 125, 0, 0, 0, 0, 0, 0, 0, 0, 249, 101, 1954, 236, 37, 1386, 287, 196, 1861, 888, 0, 0, 0, 0, 0, 0, 427, 62, 21, 2723, 51, 30, 29, 24, 2491, 0, 0, 0, 0, 0, 0, 0, 116, 2233, 17, 48, 161, 217, 25, 121, 1346, 1161, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 7393, 6236, 6091, 4828, 7195, 5858, 5445] avg=6144.0000 max=7393 max_rank=1 min=4828 min_rank=4 imbalance_ratio=1.203288 max_load_reduction=41759 max_load_reduction_pct=0.849589 moved_to_echo=23326 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 37, 0, 61, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 0, 2531, 0, 177, 62, 532, 400, 1628, 953, 422, 177, 0, 0, 0, 0, 0, 0, 5692, 0, 433, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 254, 0, 583, 62, 4881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5581, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 59, 1255, 0, 0, 1079, 254, 0, 2228, 970, 0, 0, 0, 0, 0, 0, 486, 0, 0, 2374, 0, 0, 0, 0, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 3410, 0, 0, 0, 0, 0, 59, 1610, 1318, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6082, 6882, 6184, 5936, 5927, 5907, 5837, 6397] avg=6144.0000 max=6882 max_rank=1 min=5837 min_rank=6 imbalance_ratio=1.120117 max_load_reduction=42270 max_load_reduction_pct=0.859985 moved_to_echo=24068 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[55, 0, 68, 0, 71, 0, 0, 0, 5970, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 0, 162, 87, 150, 373, 1686, 993, 455, 175, 0, 0, 0, 0, 0, 2, 5650, 0, 475, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 251, 0, 514, 86, 5085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6018, 0, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 57, 800, 0, 0, 1289, 251, 0, 2226, 970, 0, 0, 0, 0, 0, 0, 96, 0, 0, 3229, 0, 0, 0, 0, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 3253, 0, 0, 0, 0, 0, 59, 1610, 1308, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 5565, 6186, 6293, 6732, 5680, 6302, 6230] avg=6144.0000 max=6732 max_rank=4 min=5565 min_rank=1 imbalance_ratio=1.095703 max_load_reduction=42420 max_load_reduction_pct=0.863037 moved_to_echo=24944 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[58, 0, 78, 7, 89, 0, 0, 0, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 1497, 0, 213, 92, 174, 338, 1339, 982, 443, 179, 0, 0, 0, 0, 0, 2, 5604, 2, 430, 0, 0, 53, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 399, 247, 2, 556, 90, 5009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5783, 0, 786, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 52, 1066, 2, 0, 1404, 249, 1, 2162, 965, 0, 0, 0, 0, 0, 0, 235, 0, 0, 3324, 0, 0, 1, 0, 2889, 0, 0, 0, 0, 0, 0, 0, 1, 3379, 0, 0, 1, 3, 0, 54, 1562, 1304, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 5257, 6093, 6307, 6572, 5994, 6449, 6304] avg=6144.0000 max=6572 max_rank=4 min=5257 min_rank=1 imbalance_ratio=1.069661 max_load_reduction=42580 max_load_reduction_pct=0.866292 moved_to_echo=25319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 15488, 128, 1792, 896, 3456, 3200, 12416, 7680, 3328, 1408, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[256, 25344, 128, 128, 256, 384, 128, 640, 12544, 10240, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2432, 128, 128, 23680, 128, 128, 128, 128, 23040, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 640, 9984, 384, 128, 10496, 2048, 256, 17280, 7680, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[256, 25344, 128, 128, 256, 384, 128, 640, 12544, 10240, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 256, 2432, 2048, 128, 4864, 768, 38912, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 43520, 256, 5248, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 128, 640, 256, 768, 128, 128, 0, 47232, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 44288, 256, 3840, 128, 128, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[49, 22, 171, 1658, 94, 57, 1468, 8, 2, 1515, 3509, 12, 182, 313, 680, 30, 99, 377, 89, 1232, 49, 77, 2, 40, 5552, 108, 1, 305, 33, 28, 77, 60, 535, 318, 260, 200, 2115, 388, 258, 95, 183, 106, 5086, 0, 106, 47, 323, 326, 563, 149, 116, 23, 6389, 259, 117, 162, 5920, 25, 96, 102, 5432, 781, 764, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 143, 2122, 0, 0, 1925, 0, 0, 1003, 2955, 0, 0, 0, 636, 0, 76, 303, 0, 1078, 0, 73, 0, 0, 6057, 0, 0, 301, 0, 0, 0, 0, 239, 319, 0, 0, 2342, 618, 0, 92, 71, 0, 5875, 0, 0, 0, 113, 0, 486, 0, 92, 0, 6621, 92, 76, 92, 7099, 0, 0, 0, 6389, 710, 1154, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 118, 1413, 0, 2, 2178, 0, 0, 1052, 3775, 0, 2, 4, 661, 0, 100, 347, 0, 960, 0, 51, 0, 0, 5498, 0, 0, 306, 0, 0, 0, 1, 515, 361, 1, 3, 2048, 729, 3, 90, 44, 2, 5876, 0, 0, 4, 232, 4, 496, 3, 90, 0, 6501, 91, 100, 94, 7176, 0, 4, 0, 6469, 784, 963, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 60, 2960, 0, 0, 1062, 0, 0, 2145, 2354, 0, 0, 0, 684, 0, 52, 306, 0, 1391, 0, 58, 0, 0, 6331, 47, 0, 240, 0, 0, 0, 0, 113, 324, 0, 0, 2826, 479, 0, 62, 0, 0, 5815, 0, 0, 0, 288, 0, 414, 0, 62, 0, 5943, 0, 5, 62, 6991, 0, 0, 0, 6603, 762, 713, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 150, 1931, 0, 0, 2069, 0, 0, 1097, 2764, 0, 0, 0, 679, 0, 83, 335, 0, 1190, 0, 67, 0, 0, 5795, 0, 0, 399, 0, 0, 0, 0, 467, 344, 0, 0, 2229, 839, 2, 92, 33, 0, 5923, 0, 0, 0, 245, 0, 472, 0, 91, 0, 6433, 92, 83, 91, 7084, 0, 0, 0, 6463, 676, 934, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 60, 2738, 0, 0, 1164, 0, 0, 2144, 2361, 0, 0, 1, 731, 0, 71, 325, 0, 1450, 0, 67, 0, 1, 6402, 6, 0, 309, 0, 0, 0, 1, 89, 326, 0, 1, 2303, 746, 5, 67, 0, 1, 5641, 0, 0, 0, 251, 1, 532, 1, 67, 0, 5928, 45, 68, 68, 7029, 0, 0, 0, 6614, 752, 786, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 74, 1827, 0, 0, 2122, 0, 0, 998, 3031, 0, 0, 0, 706, 0, 78, 329, 0, 981, 1, 59, 0, 0, 5763, 0, 0, 263, 0, 0, 0, 0, 433, 338, 0, 0, 2437, 701, 0, 86, 0, 0, 5836, 0, 0, 0, 198, 0, 469, 0, 86, 0, 6639, 86, 78, 87, 7214, 0, 0, 0, 6541, 717, 974, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 143, 2122, 0, 0, 1925, 0, 0, 1003, 2955, 0, 0, 0, 636, 0, 76, 303, 0, 1078, 0, 73, 0, 0, 6057, 0, 0, 301, 0, 0, 0, 0, 239, 319, 0, 0, 2342, 618, 0, 92, 71, 0, 5875, 0, 0, 0, 113, 0, 486, 0, 92, 0, 6621, 92, 76, 92, 7099, 0, 0, 0, 6389, 710, 1154, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 6, 84, 1737, 3, 1, 1776, 1, 0, 1627, 2910, 3, 36, 62, 701, 2, 82, 358, 19, 1175, 12, 61, 0, 3, 5949, 3, 0, 355, 13, 3, 0, 33, 367, 341, 32, 5, 2543, 655, 46, 79, 20, 40, 5798, 0, 5, 3, 300, 58, 477, 42, 79, 1, 6223, 98, 86, 84, 6852, 1, 4, 0, 6393, 706, 799, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 118, 1413, 0, 2, 2178, 0, 0, 1052, 3775, 0, 2, 4, 661, 0, 100, 347, 0, 960, 0, 51, 0, 0, 5498, 0, 0, 306, 0, 0, 0, 1, 515, 361, 1, 3, 2048, 729, 3, 90, 44, 2, 5876, 0, 0, 4, 232, 4, 496, 3, 90, 0, 6501, 91, 100, 94, 7176, 0, 4, 0, 6469, 784, 963, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 60, 2960, 0, 0, 1062, 0, 0, 2145, 2354, 0, 0, 0, 684, 0, 52, 306, 0, 1391, 0, 58, 0, 0, 6331, 47, 0, 240, 0, 0, 0, 0, 113, 324, 0, 0, 2826, 479, 0, 62, 0, 0, 5815, 0, 0, 0, 288, 0, 414, 0, 62, 0, 5943, 0, 5, 62, 6991, 0, 0, 0, 6603, 762, 713, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[49, 22, 171, 1658, 94, 57, 1468, 8, 2, 1515, 3509, 12, 182, 313, 680, 30, 99, 377, 89, 1232, 49, 77, 2, 40, 5552, 108, 1, 305, 33, 28, 77, 60, 535, 318, 260, 200, 2115, 388, 258, 95, 183, 106, 5086, 0, 106, 47, 323, 326, 563, 149, 116, 23, 6389, 259, 117, 162, 5920, 25, 96, 102, 5432, 781, 764, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 150, 1931, 0, 0, 2069, 0, 1900, 188, 184, 0, 0, 0, 0, 0, 0, 1097, 2764, 0, 0, 0, 679, 0, 888, 96, 0, 0, 0, 0, 0, 0, 83, 335, 0, 1190, 0, 67, 0, 0, 4560, 0, 0, 0, 0, 0, 0, 0, 5663, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 344, 0, 0, 2229, 839, 2, 92, 2336, 0, 0, 0, 0, 0, 0, 0, 33, 0, 5923, 0, 0, 0, 245, 0, 36, 0, 0, 0, 0, 0, 0, 0, 472, 0, 91, 0, 5361, 92, 83, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4563, 676, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6422, 5524, 6235, 6062, 6309, 6237, 6190, 6173] avg=6144.0000 max=6422 max_rank=0 min=5524 min_rank=1 imbalance_ratio=1.045247 max_load_reduction=42730 max_load_reduction_pct=0.869344 moved_to_echo=24909 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[49, 22, 171, 1658, 94, 57, 1468, 8, 1597, 157, 183, 0, 0, 0, 0, 0, 2, 1515, 3509, 12, 182, 313, 680, 30, 882, 92, 0, 0, 0, 0, 0, 0, 99, 377, 89, 1232, 49, 77, 2, 40, 3813, 0, 0, 0, 0, 0, 0, 0, 5426, 108, 1, 305, 33, 28, 77, 60, 0, 0, 0, 0, 0, 0, 0, 0, 535, 318, 260, 200, 2115, 388, 258, 95, 1950, 0, 0, 0, 0, 0, 0, 0, 183, 106, 5086, 0, 106, 47, 323, 326, 34, 0, 0, 0, 0, 0, 0, 0, 563, 149, 116, 23, 5324, 259, 117, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 96, 102, 3835, 781, 764, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5464, 7217, 5778, 6038, 6119, 6211, 6713, 5612] avg=6144.0000 max=7217 max_rank=1 min=5464 min_rank=0 imbalance_ratio=1.174642 max_load_reduction=41935 max_load_reduction_pct=0.853170 moved_to_echo=24655 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 74, 1827, 0, 0, 2122, 0, 1923, 192, 190, 0, 0, 0, 0, 0, 0, 998, 3031, 0, 0, 0, 706, 0, 916, 96, 0, 0, 0, 0, 0, 0, 78, 329, 0, 981, 1, 59, 0, 0, 4644, 0, 0, 0, 0, 0, 0, 0, 5631, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 338, 0, 0, 2437, 701, 0, 86, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5836, 0, 0, 0, 198, 0, 36, 0, 0, 0, 0, 0, 0, 0, 469, 0, 86, 0, 5533, 86, 78, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4618, 717, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 5747, 6092, 5894, 6373, 6070, 6339, 6309] avg=6144.0000 max=6373 max_rank=4 min=5747 min_rank=1 imbalance_ratio=1.037272 max_load_reduction=42779 max_load_reduction_pct=0.870341 moved_to_echo=25091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 143, 2122, 0, 0, 1925, 0, 1878, 190, 190, 0, 0, 0, 0, 0, 0, 1003, 2955, 0, 0, 0, 636, 0, 914, 100, 0, 0, 0, 0, 0, 0, 76, 303, 0, 1078, 0, 73, 0, 0, 4570, 0, 0, 0, 0, 0, 0, 0, 5919, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 319, 0, 0, 2342, 618, 0, 92, 2339, 0, 0, 0, 0, 0, 0, 0, 71, 0, 5875, 0, 0, 0, 113, 0, 38, 0, 0, 0, 0, 0, 0, 0, 486, 0, 92, 0, 5517, 92, 76, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4511, 710, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6448, 5608, 6100, 6220, 5949, 6097, 6355, 6375] avg=6144.0000 max=6448 max_rank=0 min=5608 min_rank=1 imbalance_ratio=1.049479 max_load_reduction=42704 max_load_reduction_pct=0.868815 moved_to_echo=24776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 60, 2738, 0, 0, 1164, 0, 1944, 187, 170, 0, 0, 0, 0, 0, 0, 2144, 2361, 0, 0, 1, 731, 0, 818, 106, 0, 0, 0, 0, 0, 0, 71, 325, 0, 1450, 0, 67, 0, 1, 4525, 0, 0, 0, 0, 0, 0, 0, 6256, 6, 0, 309, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 89, 326, 0, 1, 2303, 746, 5, 67, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5641, 0, 0, 0, 251, 1, 40, 0, 0, 0, 0, 0, 0, 0, 532, 1, 67, 0, 4940, 45, 68, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4670, 752, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 6161, 6439, 6572, 5854, 5934, 5721, 6208] avg=6144.0000 max=6572 max_rank=3 min=5721 min_rank=6 imbalance_ratio=1.069661 max_load_reduction=42580 max_load_reduction_pct=0.866292 moved_to_echo=23717 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 60, 2960, 0, 0, 1062, 0, 1941, 186, 170, 0, 0, 0, 0, 0, 0, 2145, 2354, 0, 0, 0, 684, 0, 820, 105, 0, 0, 0, 0, 0, 0, 52, 306, 0, 1391, 0, 58, 0, 0, 4502, 0, 0, 0, 0, 0, 0, 0, 6187, 47, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 324, 0, 0, 2826, 479, 0, 62, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5815, 0, 0, 0, 288, 0, 39, 0, 0, 0, 0, 0, 0, 0, 414, 0, 62, 0, 4953, 0, 5, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4662, 762, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6379, 6108, 6309, 6474, 6107, 6142, 5496, 6137] avg=6144.0000 max=6474 max_rank=3 min=5496 min_rank=6 imbalance_ratio=1.053711 max_load_reduction=42678 max_load_reduction_pct=0.868286 moved_to_echo=23882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 84, 1737, 3, 1, 1776, 1, 1879, 182, 178, 0, 0, 0, 0, 0, 0, 1627, 2910, 3, 36, 62, 701, 2, 859, 99, 0, 0, 0, 0, 0, 0, 82, 358, 19, 1175, 12, 61, 0, 3, 4411, 0, 0, 0, 0, 0, 0, 0, 5813, 3, 0, 355, 13, 3, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 367, 341, 32, 5, 2543, 655, 46, 79, 2259, 0, 0, 0, 0, 0, 0, 0, 20, 40, 5798, 0, 5, 3, 300, 58, 37, 0, 0, 0, 0, 0, 0, 0, 477, 42, 79, 1, 5186, 98, 86, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4514, 706, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5847, 6299, 6121, 6220, 6327, 6261, 6053, 6024] avg=6144.0000 max=6327 max_rank=4 min=5847 min_rank=0 imbalance_ratio=1.029785 max_load_reduction=42825 max_load_reduction_pct=0.871277 moved_to_echo=24665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 60, 2738, 0, 0, 1164, 0, 1944, 187, 170, 0, 0, 0, 0, 0, 0, 2144, 2361, 0, 0, 1, 731, 0, 818, 106, 0, 0, 0, 0, 0, 0, 71, 325, 0, 1450, 0, 67, 0, 1, 4525, 0, 0, 0, 0, 0, 0, 0, 6256, 6, 0, 309, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 89, 326, 0, 1, 2303, 746, 5, 67, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5641, 0, 0, 0, 251, 1, 40, 0, 0, 0, 0, 0, 0, 0, 532, 1, 67, 0, 4940, 45, 68, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4670, 752, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 6161, 6439, 6572, 5854, 5934, 5721, 6208] avg=6144.0000 max=6572 max_rank=3 min=5721 min_rank=6 imbalance_ratio=1.069661 max_load_reduction=42580 max_load_reduction_pct=0.866292 moved_to_echo=23717 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 118, 1413, 0, 2, 2178, 0, 1906, 192, 191, 0, 0, 0, 0, 0, 0, 1052, 3775, 0, 2, 4, 661, 0, 899, 95, 0, 0, 0, 0, 0, 0, 100, 347, 0, 960, 0, 51, 0, 0, 4620, 0, 0, 0, 0, 0, 0, 0, 5365, 0, 0, 306, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 515, 361, 1, 3, 2048, 729, 3, 90, 2364, 0, 0, 0, 0, 0, 0, 0, 44, 2, 5876, 0, 0, 4, 232, 4, 38, 0, 0, 0, 0, 0, 0, 0, 496, 3, 90, 0, 5411, 91, 100, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4563, 784, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6001, 6488, 6078, 5672, 6114, 6200, 6285, 6314] avg=6144.0000 max=6488 max_rank=1 min=5672 min_rank=3 imbalance_ratio=1.055990 max_load_reduction=42664 max_load_reduction_pct=0.868001 moved_to_echo=24913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 60, 2960, 0, 0, 1062, 0, 1941, 186, 170, 0, 0, 0, 0, 0, 0, 2145, 2354, 0, 0, 0, 684, 0, 820, 105, 0, 0, 0, 0, 0, 0, 52, 306, 0, 1391, 0, 58, 0, 0, 4502, 0, 0, 0, 0, 0, 0, 0, 6187, 47, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 324, 0, 0, 2826, 479, 0, 62, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5815, 0, 0, 0, 288, 0, 39, 0, 0, 0, 0, 0, 0, 0, 414, 0, 62, 0, 4953, 0, 5, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4662, 762, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6379, 6108, 6309, 6474, 6107, 6142, 5496, 6137] avg=6144.0000 max=6474 max_rank=3 min=5496 min_rank=6 imbalance_ratio=1.053711 max_load_reduction=42678 max_load_reduction_pct=0.868286 moved_to_echo=23882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 74, 1827, 0, 0, 2122, 0, 1923, 192, 190, 0, 0, 0, 0, 0, 0, 998, 3031, 0, 0, 0, 706, 0, 916, 96, 0, 0, 0, 0, 0, 0, 78, 329, 0, 981, 1, 59, 0, 0, 4644, 0, 0, 0, 0, 0, 0, 0, 5631, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 338, 0, 0, 2437, 701, 0, 86, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5836, 0, 0, 0, 198, 0, 36, 0, 0, 0, 0, 0, 0, 0, 469, 0, 86, 0, 5533, 86, 78, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4618, 717, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 5747, 6092, 5894, 6373, 6070, 6339, 6309] avg=6144.0000 max=6373 max_rank=4 min=5747 min_rank=1 imbalance_ratio=1.037272 max_load_reduction=42779 max_load_reduction_pct=0.870341 moved_to_echo=25091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[49, 22, 171, 1658, 94, 57, 1468, 8, 1597, 157, 183, 0, 0, 0, 0, 0, 2, 1515, 3509, 12, 182, 313, 680, 30, 882, 92, 0, 0, 0, 0, 0, 0, 99, 377, 89, 1232, 49, 77, 2, 40, 3813, 0, 0, 0, 0, 0, 0, 0, 5426, 108, 1, 305, 33, 28, 77, 60, 0, 0, 0, 0, 0, 0, 0, 0, 535, 318, 260, 200, 2115, 388, 258, 95, 1950, 0, 0, 0, 0, 0, 0, 0, 183, 106, 5086, 0, 106, 47, 323, 326, 34, 0, 0, 0, 0, 0, 0, 0, 563, 149, 116, 23, 5324, 259, 117, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 96, 102, 3835, 781, 764, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5464, 7217, 5778, 6038, 6119, 6211, 6713, 5612] avg=6144.0000 max=7217 max_rank=1 min=5464 min_rank=0 imbalance_ratio=1.174642 max_load_reduction=41935 max_load_reduction_pct=0.853170 moved_to_echo=24655 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 896, 16512, 128, 128, 13824, 128, 14976, 1536, 1536, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 11648, 23680, 128, 256, 384, 5504, 128, 7040, 896, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 2688, 384, 256, 18944, 5248, 384, 768, 18304, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 35968, 5888, 7168, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 11648, 23680, 128, 256, 384, 5504, 128, 7040, 896, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 2688, 128, 9472, 128, 640, 128, 128, 35712, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3968, 256, 768, 128, 42240, 768, 640, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 35968, 5888, 7168, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 256, 45952, 0, 128, 128, 2048, 512, 384, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[46336, 256, 128, 2560, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3118, 0, 1, 72, 784, 5147, 0, 0, 1329, 71, 415, 0, 0, 0, 0, 0, 0, 67, 129, 1, 0, 67, 1, 0, 90, 0, 0, 1, 0, 67, 0, 50, 0, 0, 6789, 13, 0, 0, 2456, 858, 0, 7230, 1061, 7598, 63, 1, 4590, 65, 0, 0, 0, 67, 14, 0, 0, 4700, 291, 1, 0, 340, 629, 129, 254, 593] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[2169, 0, 0, 92, 716, 5824, 0, 0, 1026, 83, 428, 0, 0, 0, 0, 0, 0, 92, 152, 0, 0, 92, 0, 0, 110, 0, 0, 0, 0, 92, 0, 1, 0, 0, 6849, 0, 0, 0, 2341, 853, 0, 7611, 1626, 7566, 69, 0, 4475, 83, 0, 0, 0, 92, 0, 0, 0, 4233, 285, 0, 0, 318, 593, 213, 254, 814] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[3076, 0, 0, 63, 725, 5449, 0, 0, 1262, 51, 391, 0, 0, 0, 0, 0, 0, 62, 87, 0, 0, 62, 0, 0, 92, 0, 0, 0, 0, 62, 0, 22, 0, 0, 6634, 0, 0, 0, 2691, 873, 0, 7422, 954, 7572, 37, 0, 4771, 62, 0, 0, 0, 62, 18, 0, 0, 4337, 262, 0, 0, 422, 658, 119, 256, 598] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2292, 0, 0, 87, 728, 6016, 0, 0, 1136, 77, 430, 0, 0, 0, 0, 0, 0, 87, 146, 0, 0, 86, 0, 0, 95, 0, 0, 0, 1, 87, 0, 1, 0, 0, 6977, 0, 0, 0, 2628, 806, 0, 7736, 1323, 7645, 68, 0, 4248, 81, 0, 0, 0, 87, 0, 0, 0, 3950, 262, 0, 0, 331, 586, 133, 254, 768] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2267, 22, 384, 88, 732, 5710, 207, 60, 1929, 151, 410, 29, 25, 29, 43, 32, 0, 87, 251, 200, 91, 119, 166, 52, 128, 17, 24, 130, 67, 96, 13, 132, 121, 0, 5243, 21, 12, 182, 3050, 874, 23, 6306, 725, 6493, 88, 150, 4138, 126, 37, 164, 48, 96, 47, 51, 216, 3197, 750, 354, 24, 526, 684, 222, 345, 1148] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[2643, 0, 0, 92, 714, 5886, 0, 0, 1455, 76, 503, 0, 0, 0, 0, 0, 0, 92, 134, 0, 0, 92, 0, 0, 87, 0, 0, 0, 2, 92, 0, 0, 0, 0, 7010, 2, 0, 0, 3126, 877, 0, 7663, 1294, 7678, 58, 0, 3757, 87, 0, 0, 0, 92, 0, 0, 0, 3280, 280, 0, 0, 340, 609, 216, 229, 686] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[2599, 0, 70, 79, 719, 5708, 15, 7, 1267, 106, 446, 0, 5, 0, 2, 1, 0, 79, 166, 48, 28, 79, 25, 14, 87, 0, 0, 29, 1, 79, 0, 23, 15, 0, 6475, 0, 0, 8, 2430, 821, 1, 7335, 1217, 7435, 66, 46, 4523, 79, 11, 9, 7, 79, 0, 9, 15, 4352, 317, 71, 2, 344, 620, 143, 282, 758] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[3076, 0, 0, 63, 725, 5449, 0, 0, 1262, 51, 391, 0, 0, 0, 0, 0, 0, 62, 87, 0, 0, 62, 0, 0, 92, 0, 0, 0, 0, 62, 0, 22, 0, 0, 6634, 0, 0, 0, 2691, 873, 0, 7422, 954, 7572, 37, 0, 4771, 62, 0, 0, 0, 62, 18, 0, 0, 4337, 262, 0, 0, 422, 658, 119, 256, 598] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[2643, 0, 0, 92, 714, 5886, 0, 0, 1455, 76, 503, 0, 0, 0, 0, 0, 0, 92, 134, 0, 0, 92, 0, 0, 87, 0, 0, 0, 2, 92, 0, 0, 0, 0, 7010, 2, 0, 0, 3126, 877, 0, 7663, 1294, 7678, 58, 0, 3757, 87, 0, 0, 0, 92, 0, 0, 0, 3280, 280, 0, 0, 340, 609, 216, 229, 686] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2292, 0, 0, 87, 728, 6016, 0, 0, 1136, 77, 430, 0, 0, 0, 0, 0, 0, 87, 146, 0, 0, 86, 0, 0, 95, 0, 0, 0, 1, 87, 0, 1, 0, 0, 6977, 0, 0, 0, 2628, 806, 0, 7736, 1323, 7645, 68, 0, 4248, 81, 0, 0, 0, 87, 0, 0, 0, 3950, 262, 0, 0, 331, 586, 133, 254, 768] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2735, 0, 4, 92, 683, 5981, 1, 1, 1337, 100, 428, 0, 1, 1, 0, 0, 0, 92, 177, 4, 0, 90, 2, 0, 84, 0, 0, 3, 0, 92, 0, 3, 0, 0, 6896, 11, 0, 1, 2200, 860, 0, 7544, 1514, 7514, 77, 3, 3788, 79, 1, 0, 0, 92, 10, 0, 1, 4049, 324, 4, 0, 348, 632, 171, 248, 874] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3118, 0, 1, 72, 784, 5147, 0, 0, 1329, 71, 415, 0, 0, 0, 0, 0, 0, 67, 129, 1, 0, 67, 1, 0, 90, 0, 0, 1, 0, 67, 0, 50, 0, 0, 6789, 13, 0, 0, 2456, 858, 0, 7230, 1061, 7598, 63, 1, 4590, 65, 0, 0, 0, 67, 14, 0, 0, 4700, 291, 1, 0, 340, 629, 129, 254, 593] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2169, 0, 0, 92, 716, 2678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 83, 428, 0, 0, 0, 0, 0, 2779, 1630, 0, 0, 0, 0, 0, 0, 0, 92, 152, 0, 0, 92, 0, 0, 4348, 1565, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 92, 0, 1, 6001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2724, 0, 0, 0, 2341, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 1626, 0, 69, 0, 4475, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 4233, 2003, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 318, 593, 213, 254, 814, 2495, 1143, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5655, 5946, 6249, 6204, 5918, 6737, 6328, 6115] avg=6144.0000 max=6737 max_rank=5 min=5655 min_rank=0 imbalance_ratio=1.096517 max_load_reduction=42415 max_load_reduction_pct=0.862935 moved_to_echo=25098 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2292, 0, 0, 87, 728, 2766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1136, 77, 430, 0, 0, 0, 0, 0, 2824, 1661, 0, 0, 0, 0, 0, 0, 0, 87, 146, 0, 0, 86, 0, 0, 4420, 1581, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 1, 87, 0, 1, 6064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 0, 2628, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 1323, 0, 68, 0, 4248, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 3950, 2069, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 331, 586, 133, 254, 768, 2541, 1181, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5873, 6128, 6320, 6248, 6209, 6212, 6106, 6056] avg=6144.0000 max=6320 max_rank=2 min=5873 min_rank=0 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2267, 22, 384, 88, 732, 2625, 207, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 151, 410, 29, 25, 29, 43, 32, 2302, 1248, 0, 0, 0, 0, 0, 0, 0, 87, 251, 200, 91, 119, 166, 52, 3603, 1343, 0, 0, 0, 0, 0, 0, 128, 17, 24, 130, 67, 96, 13, 132, 5150, 0, 0, 0, 0, 0, 0, 0, 121, 0, 2086, 21, 12, 182, 3050, 874, 0, 0, 0, 0, 0, 0, 0, 0, 23, 401, 725, 0, 88, 150, 4138, 126, 0, 0, 0, 0, 0, 0, 0, 0, 37, 164, 48, 96, 47, 51, 216, 3197, 1964, 0, 0, 0, 0, 0, 0, 0, 750, 354, 24, 526, 684, 222, 345, 1148, 1909, 1121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6385, 6198, 5912, 5757, 6346, 5651, 5820, 7083] avg=6144.0000 max=7083 max_rank=7 min=5651 min_rank=5 imbalance_ratio=1.152832 max_load_reduction=42069 max_load_reduction_pct=0.855896 moved_to_echo=24900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2643, 0, 0, 92, 714, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1455, 76, 503, 0, 0, 0, 0, 0, 2798, 1669, 0, 0, 0, 0, 0, 0, 0, 92, 134, 0, 0, 92, 0, 0, 4378, 1589, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 2, 92, 0, 0, 6089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 2, 0, 0, 3126, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 1294, 0, 58, 0, 3757, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 3280, 2024, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 340, 609, 216, 229, 686, 2553, 1155, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 6501, 6285, 6270, 6793, 5683, 5396, 6068] avg=6144.0000 max=6793 max_rank=4 min=5396 min_rank=6 imbalance_ratio=1.105632 max_load_reduction=42359 max_load_reduction_pct=0.861796 moved_to_echo=23940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3118, 0, 1, 72, 784, 2367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 71, 415, 0, 0, 0, 0, 0, 2639, 1616, 0, 0, 0, 0, 0, 0, 0, 67, 129, 1, 0, 67, 1, 0, 4131, 1571, 0, 0, 0, 0, 0, 0, 90, 0, 0, 1, 0, 67, 0, 50, 6027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2700, 13, 0, 0, 2456, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 1061, 0, 63, 1, 4590, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 14, 0, 0, 4700, 1770, 0, 0, 0, 0, 0, 0, 0, 291, 1, 0, 340, 629, 129, 254, 593, 2473, 1010, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6342, 6070, 5967, 6235, 6027, 6240, 6551, 5720] avg=6144.0000 max=6551 max_rank=6 min=5720 min_rank=7 imbalance_ratio=1.066243 max_load_reduction=42601 max_load_reduction_pct=0.866720 moved_to_echo=24538 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2599, 0, 70, 79, 719, 2625, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1267, 106, 446, 0, 5, 0, 2, 1, 2678, 1541, 0, 0, 0, 0, 0, 0, 0, 79, 166, 48, 28, 79, 25, 14, 4191, 1538, 0, 0, 0, 0, 0, 0, 87, 0, 0, 29, 1, 79, 0, 23, 5897, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2576, 0, 0, 8, 2430, 821, 0, 0, 0, 0, 0, 0, 0, 0, 1, 466, 1217, 0, 66, 46, 4523, 79, 0, 0, 0, 0, 0, 0, 0, 0, 11, 9, 7, 79, 0, 9, 15, 4352, 1963, 0, 0, 0, 0, 0, 0, 0, 317, 71, 2, 344, 620, 143, 282, 758, 2358, 1120, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6046, 6168, 6116, 5850, 6398, 6445, 6015] avg=6144.0000 max=6445 max_rank=6 min=5850 min_rank=4 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=24708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2735, 0, 4, 92, 683, 2741, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 100, 428, 0, 1, 1, 0, 0, 2757, 1646, 0, 0, 0, 0, 0, 0, 0, 92, 177, 4, 0, 90, 2, 0, 4313, 1555, 0, 0, 0, 0, 0, 0, 84, 0, 0, 3, 0, 92, 0, 3, 5959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 11, 0, 1, 2200, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 1514, 0, 77, 3, 3788, 79, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 92, 10, 0, 1, 4049, 2062, 0, 0, 0, 0, 0, 0, 0, 324, 4, 0, 348, 632, 171, 248, 874, 2516, 1178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6257, 6270, 6233, 6141, 5806, 5935, 6215, 6295] avg=6144.0000 max=6295 max_rank=7 min=5806 min_rank=4 imbalance_ratio=1.024577 max_load_reduction=42857 max_load_reduction_pct=0.871928 moved_to_echo=24251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3076, 0, 0, 63, 725, 2506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 51, 391, 0, 0, 0, 0, 0, 2710, 1579, 0, 0, 0, 0, 0, 0, 0, 62, 87, 0, 0, 62, 0, 0, 4240, 1567, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 62, 0, 22, 6005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2639, 0, 0, 0, 2691, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 954, 0, 37, 0, 4771, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 18, 0, 0, 4337, 1874, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 422, 658, 119, 256, 598, 2416, 1069, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6370, 5993, 6018, 6181, 6203, 6296, 6291, 5800] avg=6144.0000 max=6370 max_rank=0 min=5800 min_rank=7 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24590 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2643, 0, 0, 92, 714, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1455, 76, 503, 0, 0, 0, 0, 0, 2798, 1669, 0, 0, 0, 0, 0, 0, 0, 92, 134, 0, 0, 92, 0, 0, 4378, 1589, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 2, 92, 0, 0, 6089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 2, 0, 0, 3126, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 1294, 0, 58, 0, 3757, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 3280, 2024, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 340, 609, 216, 229, 686, 2553, 1155, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 6501, 6285, 6270, 6793, 5683, 5396, 6068] avg=6144.0000 max=6793 max_rank=4 min=5396 min_rank=6 imbalance_ratio=1.105632 max_load_reduction=42359 max_load_reduction_pct=0.861796 moved_to_echo=23940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2267, 22, 384, 88, 732, 2625, 207, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 151, 410, 29, 25, 29, 43, 32, 2302, 1248, 0, 0, 0, 0, 0, 0, 0, 87, 251, 200, 91, 119, 166, 52, 3603, 1343, 0, 0, 0, 0, 0, 0, 128, 17, 24, 130, 67, 96, 13, 132, 5150, 0, 0, 0, 0, 0, 0, 0, 121, 0, 2086, 21, 12, 182, 3050, 874, 0, 0, 0, 0, 0, 0, 0, 0, 23, 401, 725, 0, 88, 150, 4138, 126, 0, 0, 0, 0, 0, 0, 0, 0, 37, 164, 48, 96, 47, 51, 216, 3197, 1964, 0, 0, 0, 0, 0, 0, 0, 750, 354, 24, 526, 684, 222, 345, 1148, 1909, 1121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6385, 6198, 5912, 5757, 6346, 5651, 5820, 7083] avg=6144.0000 max=7083 max_rank=7 min=5651 min_rank=5 imbalance_ratio=1.152832 max_load_reduction=42069 max_load_reduction_pct=0.855896 moved_to_echo=24900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2292, 0, 0, 87, 728, 2766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1136, 77, 430, 0, 0, 0, 0, 0, 2824, 1661, 0, 0, 0, 0, 0, 0, 0, 87, 146, 0, 0, 86, 0, 0, 4420, 1581, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 1, 87, 0, 1, 6064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 0, 2628, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 1323, 0, 68, 0, 4248, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 3950, 2069, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 331, 586, 133, 254, 768, 2541, 1181, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5873, 6128, 6320, 6248, 6209, 6212, 6106, 6056] avg=6144.0000 max=6320 max_rank=2 min=5873 min_rank=0 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2735, 0, 4, 92, 683, 2741, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 100, 428, 0, 1, 1, 0, 0, 2757, 1646, 0, 0, 0, 0, 0, 0, 0, 92, 177, 4, 0, 90, 2, 0, 4313, 1555, 0, 0, 0, 0, 0, 0, 84, 0, 0, 3, 0, 92, 0, 3, 5959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 11, 0, 1, 2200, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 1514, 0, 77, 3, 3788, 79, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 92, 10, 0, 1, 4049, 2062, 0, 0, 0, 0, 0, 0, 0, 324, 4, 0, 348, 632, 171, 248, 874, 2516, 1178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6257, 6270, 6233, 6141, 5806, 5935, 6215, 6295] avg=6144.0000 max=6295 max_rank=7 min=5806 min_rank=4 imbalance_ratio=1.024577 max_load_reduction=42857 max_load_reduction_pct=0.871928 moved_to_echo=24251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 512, 128, 3072, 5120, 1408, 2176, 6272, 19328, 9088, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10752, 768, 3456, 128, 128, 128, 128, 128, 21504, 12672, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 0, 21120, 128, 128, 256, 20992, 6912, 0, 0, 0, 0, 0, 0, 0, 0] @@ -12287,312 +12287,312 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[4619, 1832, 0, 0, 0, 0, 2069, 86, 0, 2986, 0, 1423, 0, 0, 703, 0, 0, 173, 3567, 0, 0, 424, 0, 0, 0, 0, 3244, 638, 1750, 0, 1007, 448, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 0, 297, 100, 4265, 153, 3898, 0, 6277, 27, 0, 168, 0, 86, 0, 4, 77, 0, 0, 0, 4168, 87, 0, 842] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[4761, 2268, 0, 0, 0, 0, 1990, 91, 0, 2943, 0, 1548, 0, 0, 697, 0, 0, 184, 3295, 0, 0, 433, 0, 0, 0, 0, 3590, 611, 1513, 0, 1031, 414, 0, 0, 0, 0, 0, 0, 0, 3677, 0, 0, 311, 73, 4384, 143, 3806, 0, 6417, 37, 0, 171, 0, 92, 0, 7, 83, 0, 0, 0, 3536, 91, 0, 955] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[4852, 1451, 0, 0, 0, 0, 1387, 67, 0, 3474, 0, 1553, 54, 0, 851, 0, 0, 134, 3599, 4, 0, 479, 50, 1, 0, 0, 2635, 836, 1198, 1, 1083, 383, 1, 0, 0, 0, 1, 0, 0, 4585, 0, 1, 184, 98, 4214, 212, 4276, 1, 6700, 19, 0, 160, 0, 67, 0, 20, 71, 0, 0, 0, 3554, 67, 0, 829] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[4145, 1677, 0, 0, 0, 0, 2303, 92, 0, 3007, 0, 1255, 0, 0, 704, 0, 0, 184, 3790, 0, 0, 442, 0, 0, 0, 0, 2509, 580, 1949, 0, 945, 539, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 386, 88, 4504, 164, 4488, 0, 6013, 44, 0, 174, 0, 92, 0, 0, 76, 0, 0, 0, 4011, 92, 0, 973] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[4247, 2035, 17, 32, 55, 90, 1195, 146, 82, 3136, 127, 1294, 243, 102, 691, 13, 89, 188, 3047, 231, 63, 493, 121, 96, 69, 50, 2518, 827, 1004, 75, 1053, 560, 382, 242, 31, 39, 260, 25, 50, 3276, 41, 329, 323, 84, 4644, 350, 3978, 216, 5946, 102, 19, 249, 7, 102, 41, 109, 95, 51, 18, 319, 2762, 124, 53, 896] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[5186, 1607, 1, 0, 8, 4, 1539, 85, 6, 2979, 24, 1839, 95, 10, 773, 0, 14, 158, 3045, 17, 7, 452, 74, 10, 1, 5, 3080, 683, 1376, 42, 1394, 430, 68, 44, 7, 0, 58, 8, 0, 3707, 5, 56, 279, 72, 4005, 168, 4157, 28, 6636, 33, 2, 159, 0, 79, 0, 35, 82, 1, 0, 72, 3485, 79, 3, 880] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[4095, 2655, 0, 0, 0, 0, 2094, 90, 1, 2488, 1, 1406, 1, 0, 737, 0, 1, 182, 3490, 0, 0, 454, 0, 0, 0, 4, 3554, 583, 1145, 2, 1005, 440, 4, 2, 1, 0, 4, 0, 0, 3868, 0, 3, 477, 83, 4396, 132, 3627, 3, 6275, 3, 0, 153, 0, 90, 0, 1, 100, 0, 0, 4, 4463, 92, 0, 943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[4761, 2268, 0, 0, 0, 0, 1990, 91, 0, 2943, 0, 1548, 0, 0, 697, 0, 0, 184, 3295, 0, 0, 433, 0, 0, 0, 0, 3590, 611, 1513, 0, 1031, 414, 0, 0, 0, 0, 0, 0, 0, 3677, 0, 0, 311, 73, 4384, 143, 3806, 0, 6417, 37, 0, 171, 0, 92, 0, 7, 83, 0, 0, 0, 3536, 91, 0, 955] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[4247, 2035, 17, 32, 55, 90, 1195, 146, 82, 3136, 127, 1294, 243, 102, 691, 13, 89, 188, 3047, 231, 63, 493, 121, 96, 69, 50, 2518, 827, 1004, 75, 1053, 560, 382, 242, 31, 39, 260, 25, 50, 3276, 41, 329, 323, 84, 4644, 350, 3978, 216, 5946, 102, 19, 249, 7, 102, 41, 109, 95, 51, 18, 319, 2762, 124, 53, 896] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[4619, 1832, 0, 0, 0, 0, 2069, 86, 0, 2986, 0, 1423, 0, 0, 703, 0, 0, 173, 3567, 0, 0, 424, 0, 0, 0, 0, 3244, 638, 1750, 0, 1007, 448, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 0, 297, 100, 4265, 153, 3898, 0, 6277, 27, 0, 168, 0, 86, 0, 4, 77, 0, 0, 0, 4168, 87, 0, 842] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[5510, 1168, 0, 0, 0, 0, 1347, 60, 0, 2998, 0, 1857, 0, 0, 955, 0, 0, 124, 3466, 0, 0, 391, 42, 0, 0, 0, 3229, 743, 1189, 0, 1290, 330, 0, 0, 0, 0, 0, 0, 0, 4487, 0, 0, 216, 61, 4502, 172, 4242, 0, 6762, 32, 0, 128, 0, 62, 0, 1, 52, 0, 0, 0, 2820, 62, 0, 854] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[4852, 1451, 0, 0, 0, 0, 1387, 67, 0, 3474, 0, 1553, 54, 0, 851, 0, 0, 134, 3599, 4, 0, 479, 50, 1, 0, 0, 2635, 836, 1198, 1, 1083, 383, 1, 0, 0, 0, 1, 0, 0, 4585, 0, 1, 184, 98, 4214, 212, 4276, 1, 6700, 19, 0, 160, 0, 67, 0, 20, 71, 0, 0, 0, 3554, 67, 0, 829] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2422, 1832, 0, 0, 0, 0, 2069, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2986, 0, 1423, 0, 0, 703, 0, 590, 204, 0, 0, 0, 0, 0, 0, 0, 173, 3567, 0, 0, 424, 0, 0, 1143, 840, 0, 0, 0, 0, 0, 0, 0, 0, 2654, 638, 1750, 0, 1007, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 2035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 100, 1390, 153, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5731, 27, 0, 168, 0, 86, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 4168, 87, 0, 842, 1054, 342, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6409, 5906, 6147, 6497, 5769, 5838, 6016, 6570] avg=6144.0000 max=6570 max_rank=7 min=5769 min_rank=4 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=24193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2174, 1677, 0, 0, 0, 0, 2303, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3007, 0, 1255, 0, 0, 704, 0, 456, 195, 0, 0, 0, 0, 0, 0, 0, 184, 3790, 0, 0, 442, 0, 0, 1025, 887, 0, 0, 0, 0, 0, 0, 0, 0, 2053, 580, 1949, 0, 945, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 88, 1468, 164, 4488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5490, 44, 0, 174, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 4011, 92, 0, 973, 946, 328, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6246, 5617, 6328, 6066, 6075, 6594, 5800, 6426] avg=6144.0000 max=6594 max_rank=5 min=5617 min_rank=1 imbalance_ratio=1.073242 max_load_reduction=42558 max_load_reduction_pct=0.865845 moved_to_echo=24895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2227, 2035, 17, 32, 55, 90, 1195, 146, 0, 0, 0, 0, 0, 0, 0, 0, 82, 3136, 127, 1294, 243, 102, 691, 13, 458, 193, 0, 0, 0, 0, 0, 0, 89, 188, 3047, 231, 63, 493, 121, 96, 1051, 914, 0, 0, 0, 0, 0, 0, 69, 50, 2060, 827, 1004, 75, 1053, 560, 0, 0, 0, 0, 0, 0, 0, 0, 382, 242, 31, 39, 260, 25, 50, 3276, 2216, 0, 0, 0, 0, 0, 0, 0, 41, 329, 323, 84, 1514, 350, 3978, 216, 0, 0, 0, 0, 0, 0, 0, 0, 5429, 102, 19, 249, 7, 102, 41, 109, 0, 0, 0, 0, 0, 0, 0, 0, 95, 51, 18, 319, 2762, 124, 53, 896, 969, 324, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5797, 6339, 6293, 5698, 6521, 6835, 6058, 5611] avg=6144.0000 max=6835 max_rank=5 min=5611 min_rank=7 imbalance_ratio=1.112467 max_load_reduction=42317 max_load_reduction_pct=0.860942 moved_to_echo=25025 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[4095, 2655, 0, 0, 0, 0, 2094, 90, 1, 2488, 1, 1406, 1, 0, 737, 0, 1, 182, 3490, 0, 0, 454, 0, 0, 0, 4, 3554, 583, 1145, 2, 1005, 440, 4, 2, 1, 0, 4, 0, 0, 3868, 0, 3, 477, 83, 4396, 132, 3627, 3, 6275, 3, 0, 153, 0, 90, 0, 1, 100, 0, 0, 4, 4463, 92, 0, 943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2496, 2268, 0, 0, 0, 0, 1990, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2943, 0, 1548, 0, 0, 697, 0, 653, 209, 0, 0, 0, 0, 0, 0, 0, 184, 3295, 0, 0, 433, 0, 0, 1178, 863, 0, 0, 0, 0, 0, 0, 0, 0, 2937, 611, 1513, 0, 1031, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 73, 1429, 143, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5858, 37, 0, 171, 0, 92, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 3536, 91, 0, 955, 1087, 350, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6845, 6050, 5953, 6506, 5769, 5762, 6165, 6102] avg=6144.0000 max=6845 max_rank=0 min=5762 min_rank=5 imbalance_ratio=1.114095 max_load_reduction=42307 max_load_reduction_pct=0.860738 moved_to_echo=23798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2719, 1607, 1, 0, 8, 4, 1539, 85, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2979, 24, 1839, 95, 10, 773, 0, 560, 216, 0, 0, 0, 0, 0, 0, 14, 158, 3045, 17, 7, 452, 74, 10, 1283, 788, 0, 0, 0, 0, 0, 0, 1, 5, 2520, 683, 1376, 42, 1394, 430, 0, 0, 0, 0, 0, 0, 0, 0, 68, 44, 7, 0, 58, 8, 0, 3707, 1911, 0, 0, 0, 0, 0, 0, 0, 5, 56, 279, 72, 1306, 168, 4157, 28, 0, 0, 0, 0, 0, 0, 0, 0, 6058, 33, 2, 159, 0, 79, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 82, 1, 0, 72, 3485, 79, 3, 880, 1184, 362, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5963, 6502, 5848, 6451, 5803, 6071, 6366, 6148] avg=6144.0000 max=6502 max_rank=1 min=5803 min_rank=4 imbalance_ratio=1.058268 max_load_reduction=42650 max_load_reduction_pct=0.867716 moved_to_echo=24388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2545, 1451, 0, 0, 0, 0, 1387, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3474, 0, 1553, 54, 0, 851, 0, 479, 218, 0, 0, 0, 0, 0, 0, 0, 134, 3599, 4, 0, 479, 50, 1, 1200, 830, 0, 0, 0, 0, 0, 0, 0, 0, 2156, 836, 1198, 1, 1083, 383, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 4585, 2011, 0, 0, 0, 0, 0, 0, 0, 0, 1, 184, 98, 1373, 212, 4276, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6117, 19, 0, 160, 0, 67, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 3554, 67, 0, 829, 1107, 365, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5450, 6629, 6297, 5657, 6598, 6145, 6383, 5993] avg=6144.0000 max=6629 max_rank=1 min=5450 min_rank=0 imbalance_ratio=1.078939 max_load_reduction=42523 max_load_reduction_pct=0.865133 moved_to_echo=25119 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2139, 2655, 0, 0, 0, 0, 2094, 90, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2488, 1, 1406, 1, 0, 737, 0, 650, 208, 0, 0, 0, 0, 0, 0, 1, 182, 3490, 0, 0, 454, 0, 0, 1017, 870, 0, 0, 0, 0, 0, 0, 0, 4, 2904, 583, 1145, 2, 1005, 440, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 1, 0, 4, 0, 0, 3868, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 3, 477, 83, 1423, 132, 3627, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5721, 3, 0, 153, 0, 90, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 4, 4463, 92, 0, 943, 939, 346, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6978, 5492, 6014, 6083, 5982, 5748, 5968, 6887] avg=6144.0000 max=6978 max_rank=0 min=5492 min_rank=1 imbalance_ratio=1.135742 max_load_reduction=42174 max_load_reduction_pct=0.858032 moved_to_echo=24585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2174, 1677, 0, 0, 0, 0, 2303, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3007, 0, 1255, 0, 0, 704, 0, 456, 195, 0, 0, 0, 0, 0, 0, 0, 184, 3790, 0, 0, 442, 0, 0, 1025, 887, 0, 0, 0, 0, 0, 0, 0, 0, 2053, 580, 1949, 0, 945, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 88, 1468, 164, 4488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5490, 44, 0, 174, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 4011, 92, 0, 973, 946, 328, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6246, 5617, 6328, 6066, 6075, 6594, 5800, 6426] avg=6144.0000 max=6594 max_rank=5 min=5617 min_rank=1 imbalance_ratio=1.073242 max_load_reduction=42558 max_load_reduction_pct=0.865845 moved_to_echo=24895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2227, 2035, 17, 32, 55, 90, 1195, 146, 0, 0, 0, 0, 0, 0, 0, 0, 82, 3136, 127, 1294, 243, 102, 691, 13, 458, 193, 0, 0, 0, 0, 0, 0, 89, 188, 3047, 231, 63, 493, 121, 96, 1051, 914, 0, 0, 0, 0, 0, 0, 69, 50, 2060, 827, 1004, 75, 1053, 560, 0, 0, 0, 0, 0, 0, 0, 0, 382, 242, 31, 39, 260, 25, 50, 3276, 2216, 0, 0, 0, 0, 0, 0, 0, 41, 329, 323, 84, 1514, 350, 3978, 216, 0, 0, 0, 0, 0, 0, 0, 0, 5429, 102, 19, 249, 7, 102, 41, 109, 0, 0, 0, 0, 0, 0, 0, 0, 95, 51, 18, 319, 2762, 124, 53, 896, 969, 324, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5797, 6339, 6293, 5698, 6521, 6835, 6058, 5611] avg=6144.0000 max=6835 max_rank=5 min=5611 min_rank=7 imbalance_ratio=1.112467 max_load_reduction=42317 max_load_reduction_pct=0.860942 moved_to_echo=25025 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2889, 1168, 0, 0, 0, 0, 1347, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 0, 1857, 0, 0, 955, 0, 587, 220, 0, 0, 0, 0, 0, 0, 0, 124, 3466, 0, 0, 391, 42, 0, 1363, 886, 0, 0, 0, 0, 0, 0, 0, 0, 2642, 743, 1189, 0, 1290, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4487, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 61, 1468, 172, 4242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6173, 32, 0, 128, 0, 62, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 2820, 62, 0, 854, 1258, 369, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5464, 6617, 6272, 6194, 6635, 6159, 6396, 5415] avg=6144.0000 max=6635 max_rank=4 min=5415 min_rank=7 imbalance_ratio=1.079915 max_load_reduction=42517 max_load_reduction_pct=0.865011 moved_to_echo=24605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2422, 1832, 0, 0, 0, 0, 2069, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2986, 0, 1423, 0, 0, 703, 0, 590, 204, 0, 0, 0, 0, 0, 0, 0, 173, 3567, 0, 0, 424, 0, 0, 1143, 840, 0, 0, 0, 0, 0, 0, 0, 0, 2654, 638, 1750, 0, 1007, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 2035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 100, 1390, 153, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5731, 27, 0, 168, 0, 86, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 4168, 87, 0, 842, 1054, 342, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6409, 5906, 6147, 6497, 5769, 5838, 6016, 6570] avg=6144.0000 max=6570 max_rank=7 min=5769 min_rank=4 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=24193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2139, 2655, 0, 0, 0, 0, 2094, 90, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2488, 1, 1406, 1, 0, 737, 0, 650, 208, 0, 0, 0, 0, 0, 0, 1, 182, 3490, 0, 0, 454, 0, 0, 1017, 870, 0, 0, 0, 0, 0, 0, 0, 4, 2904, 583, 1145, 2, 1005, 440, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 1, 0, 4, 0, 0, 3868, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 3, 477, 83, 1423, 132, 3627, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5721, 3, 0, 153, 0, 90, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 4, 4463, 92, 0, 943, 939, 346, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6978, 5492, 6014, 6083, 5982, 5748, 5968, 6887] avg=6144.0000 max=6978 max_rank=0 min=5492 min_rank=1 imbalance_ratio=1.135742 max_load_reduction=42174 max_load_reduction_pct=0.858032 moved_to_echo=24585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 24064, 256, 12288, 512, 128, 6144, 128, 4480, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19712, 14720, 128, 128, 128, 128, 13952, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 27392, 256, 128, 3584, 384, 128, 9344, 6912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 24064, 256, 12288, 512, 128, 6144, 128, 4480, 1664, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 128, 19968, 5504, 11136, 128, 8832, 3584, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 512, 2560, 768, 11392, 1536, 32512, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[512, 384, 128, 128, 384, 128, 128, 31360, 16768, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 128, 128, 512, 28800, 768, 128, 7296, 8576, 2816, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[46592, 384, 128, 1408, 128, 768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 91, 3926, 234, 0, 0, 4370, 0, 0, 0, 161, 1352, 7106, 0, 0, 7326, 0, 0, 0, 0, 0, 328, 18, 0, 0, 0, 0, 7779, 0, 0, 1762, 0, 0, 0, 159, 0, 0, 7120, 0, 4, 0, 0, 500, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 677, 0, 92, 0, 5211, 0, 0, 235, 370, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 87, 4008, 241, 0, 0, 4377, 0, 0, 0, 154, 1144, 7217, 0, 0, 7418, 1, 0, 0, 0, 0, 287, 10, 0, 0, 0, 0, 7782, 0, 0, 1529, 0, 0, 0, 141, 0, 0, 7386, 0, 0, 0, 0, 393, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 641, 0, 87, 0, 5396, 0, 0, 231, 341, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[1, 83, 3606, 269, 0, 2, 3750, 0, 0, 0, 145, 2078, 7159, 1, 78, 7289, 23, 0, 1, 6, 4, 426, 16, 49, 16, 0, 5, 7655, 4, 73, 1155, 6, 3, 3, 145, 0, 35, 6933, 8, 8, 0, 0, 428, 0, 5, 80, 54, 169, 0, 35, 25, 11, 6, 558, 56, 80, 14, 5771, 0, 17, 238, 534, 31, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 92, 3880, 226, 0, 0, 4772, 0, 0, 0, 150, 740, 7101, 0, 0, 7338, 0, 0, 0, 0, 0, 318, 1, 0, 0, 0, 0, 7811, 0, 0, 1696, 0, 0, 0, 141, 0, 0, 7115, 0, 0, 0, 0, 484, 0, 0, 0, 0, 582, 0, 0, 0, 0, 0, 805, 0, 92, 0, 5360, 0, 0, 216, 232, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[1, 92, 4245, 246, 0, 0, 4297, 0, 0, 0, 167, 1260, 7145, 1, 5, 7327, 5, 0, 0, 0, 0, 319, 3, 3, 2, 0, 0, 7763, 0, 4, 1696, 0, 0, 1, 109, 0, 8, 7225, 0, 1, 1, 0, 479, 0, 0, 3, 1, 132, 0, 2, 0, 0, 0, 626, 4, 92, 0, 5101, 0, 0, 243, 541, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[6, 132, 3953, 351, 61, 31, 3401, 8, 5, 22, 215, 1278, 6538, 32, 273, 6582, 66, 24, 5, 31, 31, 524, 107, 207, 32, 48, 47, 6921, 105, 334, 1301, 50, 117, 10, 110, 7, 118, 6241, 251, 62, 33, 3, 519, 1, 128, 371, 171, 218, 41, 110, 33, 126, 41, 469, 323, 124, 72, 4306, 0, 100, 289, 1763, 233, 41] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 67, 2989, 252, 0, 0, 4450, 0, 0, 0, 157, 1893, 7396, 0, 1, 7455, 2, 0, 0, 0, 0, 807, 1, 1, 1, 0, 0, 7808, 0, 1, 1244, 0, 9, 0, 130, 0, 0, 6633, 0, 0, 0, 0, 454, 0, 0, 1, 0, 170, 0, 0, 37, 0, 3, 628, 0, 67, 1, 6118, 0, 0, 274, 101, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 91, 3926, 234, 0, 0, 4370, 0, 0, 0, 161, 1352, 7106, 0, 0, 7326, 0, 0, 0, 0, 0, 328, 18, 0, 0, 0, 0, 7779, 0, 0, 1762, 0, 0, 0, 159, 0, 0, 7120, 0, 4, 0, 0, 500, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 677, 0, 92, 0, 5211, 0, 0, 235, 370, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[6, 132, 3953, 351, 61, 31, 3401, 8, 5, 22, 215, 1278, 6538, 32, 273, 6582, 66, 24, 5, 31, 31, 524, 107, 207, 32, 48, 47, 6921, 105, 334, 1301, 50, 117, 10, 110, 7, 118, 6241, 251, 62, 33, 3, 519, 1, 128, 371, 171, 218, 41, 110, 33, 126, 41, 469, 323, 124, 72, 4306, 0, 100, 289, 1763, 233, 41] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[1, 92, 4245, 246, 0, 0, 4297, 0, 0, 0, 167, 1260, 7145, 1, 5, 7327, 5, 0, 0, 0, 0, 319, 3, 3, 2, 0, 0, 7763, 0, 4, 1696, 0, 0, 1, 109, 0, 8, 7225, 0, 1, 1, 0, 479, 0, 0, 3, 1, 132, 0, 2, 0, 0, 0, 626, 4, 92, 0, 5101, 0, 0, 243, 541, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[1, 83, 3606, 269, 0, 2, 3750, 0, 0, 0, 145, 2078, 7159, 1, 78, 7289, 23, 0, 1, 6, 4, 426, 16, 49, 16, 0, 5, 7655, 4, 73, 1155, 6, 3, 3, 145, 0, 35, 6933, 8, 8, 0, 0, 428, 0, 5, 80, 54, 169, 0, 35, 25, 11, 6, 558, 56, 80, 14, 5771, 0, 17, 238, 534, 31, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 62, 2879, 254, 0, 0, 4043, 0, 0, 0, 100, 2275, 7415, 0, 0, 7530, 2, 0, 0, 0, 0, 1004, 0, 0, 0, 0, 0, 7839, 0, 0, 1162, 0, 0, 0, 99, 0, 0, 6604, 0, 0, 0, 0, 350, 0, 0, 0, 0, 231, 0, 0, 48, 0, 0, 523, 0, 62, 0, 6274, 0, 0, 300, 96, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 87, 4008, 241, 0, 0, 4377, 0, 0, 0, 154, 1144, 7217, 0, 0, 7418, 1, 0, 0, 0, 0, 287, 10, 0, 0, 0, 0, 7782, 0, 0, 1529, 0, 0, 0, 141, 0, 0, 7386, 0, 0, 0, 0, 393, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 641, 0, 87, 0, 5396, 0, 0, 231, 341, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 91, 3926, 234, 0, 0, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 1352, 4416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 18, 0, 5598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4676, 0, 0, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 6118, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 0, 0, 0, 331, 2142, 2178, 1002, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 0, 92, 3103, 1728, 548, 0, 0, 0, 0, 0, 0, 5103, 0, 0, 235, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6443, 5929, 5944, 6438, 6281, 6261, 6148, 5708] avg=6144.0000 max=6443 max_rank=0 min=5708 min_rank=7 imbalance_ratio=1.048665 max_load_reduction=42709 max_load_reduction_pct=0.868917 moved_to_echo=24398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 87, 4008, 241, 0, 0, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 1144, 4485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 287, 10, 0, 5668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4678, 0, 0, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 6346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 0, 281, 2175, 2182, 1040, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 87, 3104, 1750, 557, 0, 0, 0, 0, 0, 0, 5284, 0, 0, 231, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 5783, 5966, 6207, 6487, 6183, 6139, 5856] avg=6144.0000 max=6531 max_rank=0 min=5783 min_rank=1 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 92, 3880, 226, 0, 0, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 740, 4413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 1, 0, 5608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4695, 0, 0, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 0, 0, 0, 582, 2140, 2379, 1002, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 805, 0, 92, 3116, 1730, 548, 0, 0, 0, 0, 0, 0, 5249, 0, 0, 216, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6591, 5303, 5927, 6391, 6254, 6698, 6291, 5697] avg=6144.0000 max=6698 max_rank=5 min=5303 min_rank=1 imbalance_ratio=1.090169 max_load_reduction=42454 max_load_reduction_pct=0.863729 moved_to_echo=24940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 67, 2989, 252, 0, 0, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 1893, 4597, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 807, 1, 1, 5696, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4693, 0, 1, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 130, 0, 0, 5699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 1, 0, 170, 2229, 2218, 934, 127, 0, 0, 0, 0, 0, 0, 37, 0, 3, 628, 0, 67, 3115, 1759, 570, 0, 0, 0, 0, 0, 1, 5991, 0, 0, 274, 101, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5540, 6648, 6507, 5939, 5838, 6133, 6179, 6368] avg=6144.0000 max=6648 max_rank=1 min=5540 min_rank=0 imbalance_ratio=1.082031 max_load_reduction=42504 max_load_reduction_pct=0.864746 moved_to_echo=24518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 83, 3606, 269, 0, 2, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 2078, 4449, 1, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 1, 6, 4, 426, 16, 49, 5569, 0, 0, 0, 0, 0, 0, 0, 16, 0, 5, 4602, 4, 73, 1155, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 145, 0, 35, 5957, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 5, 80, 54, 169, 2158, 1869, 976, 120, 0, 0, 0, 0, 0, 35, 25, 11, 6, 558, 56, 80, 3053, 1720, 552, 0, 0, 0, 0, 0, 14, 5651, 0, 17, 238, 534, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5842, 6751, 6094, 5861, 6159, 5859, 6096, 6490] avg=6144.0000 max=6751 max_rank=1 min=5842 min_rank=0 imbalance_ratio=1.098796 max_load_reduction=42401 max_load_reduction_pct=0.862651 moved_to_echo=24604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[6, 132, 3953, 351, 61, 31, 1706, 8, 0, 0, 0, 0, 0, 0, 0, 0, 5, 22, 215, 1278, 4064, 32, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 24, 5, 31, 31, 524, 107, 207, 5030, 0, 0, 0, 0, 0, 0, 0, 32, 48, 47, 4160, 105, 334, 1301, 50, 0, 0, 0, 0, 0, 0, 0, 0, 117, 10, 110, 7, 118, 5362, 251, 62, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3, 519, 1, 128, 371, 171, 218, 1970, 1695, 879, 89, 0, 0, 0, 0, 41, 110, 33, 126, 41, 469, 323, 124, 2761, 1552, 504, 0, 0, 0, 0, 0, 72, 4217, 0, 100, 289, 1763, 233, 41, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6248, 5889, 6025, 6077, 6037, 6077, 6084, 6715] avg=6144.0000 max=6715 max_rank=7 min=5889 min_rank=1 imbalance_ratio=1.092936 max_load_reduction=42437 max_load_reduction_pct=0.863383 moved_to_echo=24913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 67, 2989, 252, 0, 0, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 1893, 4597, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 807, 1, 1, 5696, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4693, 0, 1, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 130, 0, 0, 5699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 1, 0, 170, 2229, 2218, 934, 127, 0, 0, 0, 0, 0, 0, 37, 0, 3, 628, 0, 67, 3115, 1759, 570, 0, 0, 0, 0, 0, 1, 5991, 0, 0, 274, 101, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5540, 6648, 6507, 5939, 5838, 6133, 6179, 6368] avg=6144.0000 max=6648 max_rank=1 min=5540 min_rank=0 imbalance_ratio=1.082031 max_load_reduction=42504 max_load_reduction_pct=0.864746 moved_to_echo=24518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 2879, 254, 0, 0, 2028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 2275, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1004, 0, 0, 5754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4712, 0, 0, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 5674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 231, 2235, 2015, 930, 130, 0, 0, 0, 0, 0, 0, 48, 0, 0, 523, 0, 62, 3127, 1776, 572, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 300, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5223, 6983, 6760, 5874, 5773, 5891, 6108, 6540] avg=6144.0000 max=6983 max_rank=1 min=5223 min_rank=0 imbalance_ratio=1.136556 max_load_reduction=42169 max_load_reduction_pct=0.857931 moved_to_echo=24312 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 87, 4008, 241, 0, 0, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 1144, 4485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 287, 10, 0, 5668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4678, 0, 0, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 6346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 0, 281, 2175, 2182, 1040, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 87, 3104, 1750, 557, 0, 0, 0, 0, 0, 0, 5284, 0, 0, 231, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 5783, 5966, 6207, 6487, 6183, 6139, 5856] avg=6144.0000 max=6531 max_rank=0 min=5783 min_rank=1 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 92, 4245, 246, 0, 0, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 1260, 4433, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 319, 3, 3, 5599, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4663, 0, 4, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 109, 0, 8, 6204, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 479, 0, 0, 3, 1, 132, 2157, 2147, 1021, 109, 0, 0, 0, 0, 0, 2, 0, 0, 0, 626, 4, 92, 3100, 1728, 555, 0, 0, 0, 0, 0, 0, 4992, 0, 0, 243, 541, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6734, 5866, 5929, 6365, 6323, 6050, 6107, 5778] avg=6144.0000 max=6734 max_rank=0 min=5778 min_rank=7 imbalance_ratio=1.096029 max_load_reduction=42418 max_load_reduction_pct=0.862996 moved_to_echo=24258 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 128, 3712, 128, 256, 512, 256, 2176, 17280, 16768, 7808, 1024, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 1280, 12032, 35584, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 768, 29568, 2176, 128, 128, 16896, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 256, 256, 256, 128, 4992, 384, 768, 24576, 13824, 4480, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 768, 29568, 2176, 128, 128, 16896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 1280, 12032, 35584, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 36992, 128, 512, 11648, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 4096, 256, 384, 44544, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 42752, 0, 128, 2048, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 1152, 128, 256, 47488, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[90, 0, 0, 0, 0, 82, 0, 0, 0, 0, 173, 0, 0, 68, 2596, 0, 3485, 250, 91, 7608, 7185, 0, 0, 0, 0, 117, 3285, 0, 7281, 0, 68, 0, 5756, 0, 29, 0, 0, 725, 7182, 87, 0, 0, 0, 0, 0, 62, 0, 59, 0, 662, 0, 570, 0, 0, 0, 0, 0, 0, 8, 217, 753, 0, 332, 331] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[89, 0, 0, 0, 0, 77, 0, 0, 0, 0, 184, 0, 0, 69, 2567, 0, 3534, 252, 94, 7512, 6959, 0, 0, 0, 0, 113, 3187, 0, 7123, 0, 69, 0, 6040, 0, 28, 0, 0, 815, 7200, 92, 0, 0, 0, 0, 0, 69, 0, 67, 0, 664, 0, 654, 0, 0, 0, 0, 0, 0, 2, 225, 796, 0, 353, 318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[85, 0, 3, 0, 5, 63, 1, 0, 0, 0, 183, 0, 0, 77, 2596, 0, 3225, 248, 90, 7550, 6910, 0, 4, 2, 0, 149, 4078, 0, 7171, 1, 77, 0, 5156, 2, 160, 0, 1, 505, 7166, 92, 0, 1, 0, 3, 0, 77, 0, 53, 0, 922, 1, 663, 0, 4, 0, 0, 0, 0, 34, 223, 837, 4, 379, 351] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 0, 62, 0, 0, 0, 0, 126, 0, 0, 40, 2360, 0, 3728, 254, 62, 7656, 7130, 0, 0, 0, 0, 129, 1825, 0, 7414, 0, 37, 0, 5786, 0, 3, 0, 0, 1709, 7160, 62, 0, 0, 0, 0, 0, 37, 0, 59, 0, 991, 0, 578, 0, 0, 0, 0, 0, 0, 0, 211, 1015, 0, 283, 422] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 1, 67, 34, 0, 0, 0, 135, 0, 0, 73, 2481, 0, 3800, 251, 67, 7661, 7048, 0, 1, 0, 0, 128, 1689, 0, 7472, 0, 63, 0, 5758, 1, 1, 0, 2, 1582, 7169, 67, 0, 0, 0, 1, 0, 63, 0, 70, 0, 1035, 0, 507, 0, 1, 0, 0, 0, 0, 2, 253, 992, 1, 331, 340] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[53, 0, 51, 0, 76, 79, 59, 7, 0, 1, 186, 17, 0, 89, 2258, 1, 3315, 269, 94, 7473, 6742, 0, 46, 10, 0, 138, 2880, 0, 7169, 30, 85, 0, 5305, 20, 59, 9, 12, 1606, 7015, 79, 0, 12, 0, 59, 6, 66, 0, 61, 0, 811, 7, 704, 0, 65, 0, 0, 33, 0, 87, 240, 935, 76, 396, 361] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[94, 0, 0, 0, 0, 90, 0, 0, 0, 0, 187, 0, 0, 58, 2713, 0, 3468, 226, 92, 7546, 7052, 0, 0, 0, 0, 125, 3386, 0, 7245, 0, 58, 0, 5604, 0, 12, 0, 0, 704, 7201, 92, 0, 0, 0, 0, 0, 56, 0, 73, 0, 894, 0, 651, 0, 0, 0, 0, 0, 0, 1, 238, 603, 0, 343, 340] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[90, 0, 0, 0, 0, 82, 0, 0, 0, 0, 173, 0, 0, 68, 2596, 0, 3485, 250, 91, 7608, 7185, 0, 0, 0, 0, 117, 3285, 0, 7281, 0, 68, 0, 5756, 0, 29, 0, 0, 725, 7182, 87, 0, 0, 0, 0, 0, 62, 0, 59, 0, 662, 0, 570, 0, 0, 0, 0, 0, 0, 8, 217, 753, 0, 332, 331] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[89, 0, 0, 0, 0, 77, 0, 0, 0, 0, 184, 0, 0, 69, 2567, 0, 3534, 252, 94, 7512, 6959, 0, 0, 0, 0, 113, 3187, 0, 7123, 0, 69, 0, 6040, 0, 28, 0, 0, 815, 7200, 92, 0, 0, 0, 0, 0, 69, 0, 67, 0, 664, 0, 654, 0, 0, 0, 0, 0, 0, 2, 225, 796, 0, 353, 318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[85, 0, 3, 0, 5, 63, 1, 0, 0, 0, 183, 0, 0, 77, 2596, 0, 3225, 248, 90, 7550, 6910, 0, 4, 2, 0, 149, 4078, 0, 7171, 1, 77, 0, 5156, 2, 160, 0, 1, 505, 7166, 92, 0, 1, 0, 3, 0, 77, 0, 53, 0, 922, 1, 663, 0, 4, 0, 0, 0, 0, 34, 223, 837, 4, 379, 351] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[81, 5, 238, 0, 351, 104, 559, 142, 46, 45, 144, 110, 3, 276, 1898, 8, 3443, 372, 162, 6660, 5459, 7, 155, 89, 6, 479, 3533, 0, 6180, 161, 333, 105, 4239, 98, 310, 43, 125, 669, 6206, 191, 9, 85, 9, 288, 18, 38, 1, 84, 37, 628, 38, 1231, 31, 266, 0, 6, 158, 17, 253, 206, 1231, 383, 736, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 0, 0, 0, 77, 0, 0, 4410, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 69, 2567, 0, 2858, 620, 0, 0, 0, 0, 0, 0, 3534, 252, 94, 0, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 3187, 0, 2850, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5420, 0, 28, 0, 0, 815, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 67, 6001, 0, 0, 0, 0, 0, 0, 0, 0, 664, 0, 654, 0, 0, 0, 0, 2790, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 2, 225, 796, 0, 353, 318, 2784, 1415, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6087, 6298, 6176, 6219, 6355, 6137, 5987, 5893] avg=6144.0000 max=6355 max_rank=4 min=5893 min_rank=7 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[90, 0, 0, 0, 0, 82, 0, 0, 4399, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 68, 2596, 0, 2922, 591, 0, 0, 0, 0, 0, 0, 3485, 250, 91, 0, 2371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 3285, 0, 2913, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5165, 0, 29, 0, 0, 725, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 59, 6078, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 570, 0, 0, 0, 0, 2783, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 8, 217, 753, 0, 332, 331, 2874, 1446, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6101, 6350, 6197, 6383, 6006, 6199, 5955, 5961] avg=6144.0000 max=6383 max_rank=3 min=5955 min_rank=6 imbalance_ratio=1.038900 max_load_reduction=42769 max_load_reduction_pct=0.870138 moved_to_echo=24121 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[53, 0, 51, 0, 76, 79, 59, 7, 4297, 1502, 0, 0, 0, 0, 0, 0, 0, 1, 186, 17, 0, 89, 2258, 1, 2877, 545, 0, 0, 0, 0, 0, 0, 3315, 269, 94, 0, 2224, 0, 46, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 2880, 0, 2868, 30, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4760, 20, 59, 9, 12, 1606, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 59, 6, 66, 0, 61, 5971, 0, 0, 0, 0, 0, 0, 0, 0, 811, 7, 704, 0, 65, 0, 0, 2718, 1821, 0, 0, 0, 0, 0, 0, 33, 0, 87, 240, 935, 76, 396, 361, 2697, 1424, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 5974, 5958, 6001, 6545, 6175, 6126, 6249] avg=6144.0000 max=6545 max_rank=4 min=5958 min_rank=2 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=25095 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 1, 67, 34, 0, 4391, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 73, 2481, 0, 2999, 591, 0, 0, 0, 0, 0, 0, 3800, 251, 67, 0, 2325, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 1689, 0, 2989, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5167, 1, 1, 0, 2, 1582, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 63, 0, 70, 6121, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 0, 507, 0, 1, 0, 0, 2778, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 2, 253, 992, 1, 331, 340, 2820, 1484, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6038, 6279, 6444, 4869, 6820, 6255, 6224, 6223] avg=6144.0000 max=6820 max_rank=4 min=4869 min_rank=3 imbalance_ratio=1.110026 max_load_reduction=42332 max_load_reduction_pct=0.861247 moved_to_echo=25522 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 0, 0, 0, 77, 0, 0, 4410, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 69, 2567, 0, 2858, 620, 0, 0, 0, 0, 0, 0, 3534, 252, 94, 0, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 3187, 0, 2850, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5420, 0, 28, 0, 0, 815, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 67, 6001, 0, 0, 0, 0, 0, 0, 0, 0, 664, 0, 654, 0, 0, 0, 0, 2790, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 2, 225, 796, 0, 353, 318, 2784, 1415, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6087, 6298, 6176, 6219, 6355, 6137, 5987, 5893] avg=6144.0000 max=6355 max_rank=4 min=5893 min_rank=7 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[94, 0, 0, 0, 0, 90, 0, 0, 4411, 1518, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 58, 2713, 0, 2907, 575, 0, 0, 0, 0, 0, 0, 3468, 226, 92, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 3386, 0, 2899, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5029, 0, 12, 0, 0, 704, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 73, 6028, 0, 0, 0, 0, 0, 0, 0, 0, 894, 0, 651, 0, 0, 0, 0, 2790, 1904, 0, 0, 0, 0, 0, 0, 0, 0, 1, 238, 603, 0, 343, 340, 2821, 1439, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6440, 6113, 6468, 5837, 6157, 6239, 5785] avg=6144.0000 max=6468 max_rank=3 min=5785 min_rank=7 imbalance_ratio=1.052734 max_load_reduction=42684 max_load_reduction_pct=0.868408 moved_to_echo=24018 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 0, 62, 0, 0, 4386, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 40, 2360, 0, 2975, 594, 0, 0, 0, 0, 0, 0, 3728, 254, 62, 0, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1825, 0, 2966, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5192, 0, 3, 0, 0, 1709, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 59, 6116, 0, 0, 0, 0, 0, 0, 0, 0, 991, 0, 578, 0, 0, 0, 0, 2774, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 1015, 0, 283, 422, 2852, 1473, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6001, 6095, 6396, 4957, 6966, 6212, 6269, 6256] avg=6144.0000 max=6966 max_rank=4 min=4957 min_rank=3 imbalance_ratio=1.133789 max_load_reduction=42186 max_load_reduction_pct=0.858276 moved_to_echo=25703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[90, 0, 0, 0, 0, 82, 0, 0, 4399, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 68, 2596, 0, 2922, 591, 0, 0, 0, 0, 0, 0, 3485, 250, 91, 0, 2371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 3285, 0, 2913, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5165, 0, 29, 0, 0, 725, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 59, 6078, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 570, 0, 0, 0, 0, 2783, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 8, 217, 753, 0, 332, 331, 2874, 1446, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6101, 6350, 6197, 6383, 6006, 6199, 5955, 5961] avg=6144.0000 max=6383 max_rank=3 min=5955 min_rank=6 imbalance_ratio=1.038900 max_load_reduction=42769 max_load_reduction_pct=0.870138 moved_to_echo=24121 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[81, 5, 238, 0, 351, 104, 559, 142, 3802, 1340, 0, 0, 0, 0, 0, 0, 46, 45, 144, 110, 3, 276, 1898, 8, 2480, 435, 0, 0, 0, 0, 0, 0, 3443, 372, 162, 0, 1801, 7, 155, 89, 0, 0, 0, 0, 0, 0, 0, 0, 6, 479, 3533, 0, 2473, 161, 333, 105, 0, 0, 0, 0, 0, 0, 0, 0, 3804, 98, 310, 43, 125, 669, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 9, 85, 9, 288, 18, 38, 1, 84, 5320, 0, 0, 0, 0, 0, 0, 0, 37, 628, 38, 1231, 31, 266, 0, 6, 2404, 1474, 0, 0, 0, 0, 0, 0, 158, 17, 253, 206, 1231, 383, 736, 364, 2184, 1227, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6622, 5445, 6029, 7090, 5240, 5852, 6115, 6759] avg=6144.0000 max=7090 max_rank=3 min=5240 min_rank=4 imbalance_ratio=1.153971 max_load_reduction=42062 max_load_reduction_pct=0.855754 moved_to_echo=23966 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[85, 0, 3, 0, 5, 63, 1, 0, 4390, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 77, 2596, 0, 2882, 533, 0, 0, 0, 0, 0, 0, 3225, 248, 90, 0, 2270, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 4078, 0, 2860, 1, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4623, 2, 160, 0, 1, 505, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 77, 0, 53, 6031, 0, 0, 0, 0, 0, 0, 0, 0, 922, 1, 663, 0, 4, 0, 0, 2776, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 34, 223, 837, 4, 379, 351, 2769, 1429, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 6271, 5839, 7165, 5383, 6165, 6237, 6026] avg=6144.0000 max=7165 max_rank=3 min=5383 min_rank=4 imbalance_ratio=1.166178 max_load_reduction=41987 max_load_reduction_pct=0.854228 moved_to_echo=23811 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[53, 0, 51, 0, 76, 79, 59, 7, 4297, 1502, 0, 0, 0, 0, 0, 0, 0, 1, 186, 17, 0, 89, 2258, 1, 2877, 545, 0, 0, 0, 0, 0, 0, 3315, 269, 94, 0, 2224, 0, 46, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 2880, 0, 2868, 30, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4760, 20, 59, 9, 12, 1606, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 59, 6, 66, 0, 61, 5971, 0, 0, 0, 0, 0, 0, 0, 0, 811, 7, 704, 0, 65, 0, 0, 2718, 1821, 0, 0, 0, 0, 0, 0, 33, 0, 87, 240, 935, 76, 396, 361, 2697, 1424, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 5974, 5958, 6001, 6545, 6175, 6126, 6249] avg=6144.0000 max=6545 max_rank=4 min=5958 min_rank=2 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=25095 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 0, 62, 0, 0, 4386, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 40, 2360, 0, 2975, 594, 0, 0, 0, 0, 0, 0, 3728, 254, 62, 0, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1825, 0, 2966, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5192, 0, 3, 0, 0, 1709, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 59, 6116, 0, 0, 0, 0, 0, 0, 0, 0, 991, 0, 578, 0, 0, 0, 0, 2774, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 1015, 0, 283, 422, 2852, 1473, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6001, 6095, 6396, 4957, 6966, 6212, 6269, 6256] avg=6144.0000 max=6966 max_rank=4 min=4957 min_rank=3 imbalance_ratio=1.133789 max_load_reduction=42186 max_load_reduction_pct=0.858276 moved_to_echo=25703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 512, 1920, 7168, 512, 3200, 2944, 21888, 11392, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 1408, 128, 128, 768, 19584, 128, 22912, 4608, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 128, 384, 0, 512, 640, 768, 256, 34560, 12032, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 23936, 0, 22912, 256, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[28032, 2176, 768, 0, 18048, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 23936, 0, 22912, 256, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[39168, 128, 640, 128, 256, 8320, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 6656, 128, 5632, 128, 384, 0, 128, 21888, 14720, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 384, 128, 512, 128, 640, 47744, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 4254, 0, 0, 0, 0, 0, 0, 0, 92, 0, 92, 1846, 0, 0, 0, 1855, 913, 7989, 639, 0, 92, 7219, 0, 0, 7226, 143, 58, 0, 0, 0, 0, 235, 0, 58, 0, 0, 3437, 0, 1069, 0, 0, 0, 0, 85, 0, 0, 0, 0, 392, 2504, 0, 4827, 293, 1013, 92, 0, 0, 0, 2271, 366, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[7, 0, 249, 4504, 340, 87, 7, 0, 30, 86, 5, 160, 187, 145, 2284, 32, 22, 52, 1332, 873, 6525, 647, 228, 95, 6354, 196, 10, 6154, 330, 86, 30, 14, 0, 150, 394, 334, 70, 38, 0, 3731, 154, 1019, 41, 272, 78, 5, 91, 38, 128, 76, 284, 489, 4566, 42, 3018, 348, 953, 128, 75, 22, 240, 696, 504, 97] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 92, 0, 91, 2004, 0, 0, 0, 2019, 1111, 7890, 611, 0, 91, 7245, 0, 0, 7341, 154, 69, 0, 0, 0, 0, 242, 0, 69, 0, 0, 4171, 0, 1129, 0, 0, 0, 0, 60, 0, 0, 0, 0, 393, 2604, 0, 4378, 310, 922, 92, 0, 0, 0, 2004, 217, 91] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 62, 0, 62, 2847, 0, 0, 0, 2025, 1390, 8004, 672, 0, 62, 7454, 0, 0, 7381, 453, 37, 0, 0, 0, 0, 120, 0, 36, 0, 0, 4544, 0, 684, 0, 0, 0, 0, 61, 0, 0, 0, 0, 561, 3141, 0, 2478, 159, 978, 62, 0, 0, 0, 2491, 149, 43] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 52, 3897, 61, 44, 0, 0, 2, 0, 0, 79, 18, 78, 2613, 21, 1, 0, 1521, 1225, 7634, 619, 14, 79, 7093, 45, 0, 7048, 375, 72, 0, 1, 0, 61, 186, 77, 66, 1, 0, 4407, 58, 1066, 6, 76, 1, 0, 78, 0, 13, 4, 75, 570, 4010, 2, 2654, 239, 905, 81, 11, 0, 77, 1598, 153, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 1, 3525, 0, 1, 0, 0, 0, 0, 8, 70, 0, 67, 2621, 0, 0, 0, 2181, 1347, 7919, 611, 1, 67, 7494, 0, 0, 7419, 399, 63, 0, 0, 0, 1, 126, 1, 63, 1, 0, 4385, 0, 710, 0, 1, 0, 0, 67, 0, 0, 0, 8, 539, 3152, 0, 2545, 190, 926, 67, 1, 0, 1, 2283, 224, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3773, 0, 0, 0, 0, 0, 0, 0, 87, 0, 87, 2005, 0, 0, 0, 1820, 1112, 7907, 625, 0, 87, 7206, 0, 0, 7341, 74, 68, 0, 0, 0, 0, 160, 0, 67, 0, 0, 4024, 0, 1046, 0, 0, 0, 0, 63, 0, 0, 0, 0, 385, 2686, 0, 4692, 228, 991, 86, 1, 0, 0, 2139, 306, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 52, 3897, 61, 44, 0, 0, 2, 0, 0, 79, 18, 78, 2613, 21, 1, 0, 1521, 1225, 7634, 619, 14, 79, 7093, 45, 0, 7048, 375, 72, 0, 1, 0, 61, 186, 77, 66, 1, 0, 4407, 58, 1066, 6, 76, 1, 0, 78, 0, 13, 4, 75, 570, 4010, 2, 2654, 239, 905, 81, 11, 0, 77, 1598, 153, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[7, 0, 249, 4504, 340, 87, 7, 0, 30, 86, 5, 160, 187, 145, 2284, 32, 22, 52, 1332, 873, 6525, 647, 228, 95, 6354, 196, 10, 6154, 330, 86, 30, 14, 0, 150, 394, 334, 70, 38, 0, 3731, 154, 1019, 41, 272, 78, 5, 91, 38, 128, 76, 284, 489, 4566, 42, 3018, 348, 953, 128, 75, 22, 240, 696, 504, 97] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 92, 0, 91, 2004, 0, 0, 0, 2019, 1111, 7890, 611, 0, 91, 7245, 0, 0, 7341, 154, 69, 0, 0, 0, 0, 242, 0, 69, 0, 0, 4171, 0, 1129, 0, 0, 0, 0, 60, 0, 0, 0, 0, 393, 2604, 0, 4378, 310, 922, 92, 0, 0, 0, 2004, 217, 91] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 4, 4075, 3, 5, 0, 0, 0, 0, 0, 92, 1, 92, 1748, 1, 0, 0, 1909, 1090, 7854, 650, 1, 92, 7216, 1, 0, 7251, 155, 77, 0, 0, 0, 4, 208, 3, 77, 1, 0, 4036, 2, 1071, 1, 5, 2, 0, 73, 0, 1, 0, 3, 386, 2975, 0, 4635, 285, 939, 90, 0, 0, 5, 1708, 235, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 62, 0, 62, 2847, 0, 0, 0, 2025, 1390, 8004, 672, 0, 62, 7454, 0, 0, 7381, 453, 37, 0, 0, 0, 0, 120, 0, 36, 0, 0, 4544, 0, 684, 0, 0, 0, 0, 61, 0, 0, 0, 0, 561, 3141, 0, 2478, 159, 978, 62, 0, 0, 0, 2491, 149, 43] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3773, 0, 0, 0, 0, 1519, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 87, 2005, 0, 2168, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 1112, 2483, 625, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5939, 74, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 67, 0, 0, 4024, 1402, 425, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 0, 0, 63, 0, 5038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 2686, 0, 2748, 228, 0, 0, 0, 0, 0, 0, 0, 0, 991, 86, 1, 0, 0, 2139, 306, 86, 2869, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 5897, 6127, 6081, 6078, 6147, 6047, 6478] avg=6144.0000 max=6478 max_rank=7 min=5897 min_rank=1 imbalance_ratio=1.054362 max_load_reduction=42674 max_load_reduction_pct=0.868205 moved_to_echo=24750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 52, 3897, 61, 44, 0, 0, 859, 970, 0, 0, 0, 0, 0, 0, 2, 0, 0, 79, 18, 78, 2613, 21, 2134, 1497, 0, 0, 0, 0, 0, 0, 1, 0, 1521, 1225, 2397, 619, 14, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 5702, 375, 72, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 186, 77, 66, 1, 0, 4407, 1346, 240, 0, 0, 0, 0, 0, 0, 58, 1066, 6, 76, 1, 0, 78, 0, 4959, 0, 0, 0, 0, 0, 0, 0, 13, 4, 75, 570, 4010, 2, 1555, 239, 0, 0, 0, 0, 0, 0, 0, 0, 905, 81, 11, 0, 77, 1598, 153, 85, 2770, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5883, 6442, 5856, 6195, 6384, 6244, 6468, 5680] avg=6144.0000 max=6468 max_rank=6 min=5680 min_rank=7 imbalance_ratio=1.052734 max_load_reduction=42684 max_load_reduction_pct=0.868408 moved_to_echo=24776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3752, 0, 0, 0, 0, 1418, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 91, 2004, 0, 2181, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 2019, 1111, 2477, 611, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5939, 154, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 69, 0, 0, 4171, 1402, 396, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 0, 0, 60, 0, 5064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 2604, 0, 2564, 310, 0, 0, 0, 0, 0, 0, 0, 0, 922, 92, 0, 0, 0, 2004, 217, 91, 2863, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6173, 5915, 6309, 6162, 6280, 6253, 5871, 6189] avg=6144.0000 max=6309 max_rank=2 min=5871 min_rank=6 imbalance_ratio=1.026855 max_load_reduction=42843 max_load_reduction_pct=0.871643 moved_to_echo=24593 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 3525, 0, 1, 0, 0, 824, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 8, 70, 0, 67, 2621, 0, 2256, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 1347, 2488, 611, 1, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6002, 399, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 126, 1, 63, 1, 0, 4385, 1417, 230, 0, 0, 0, 0, 0, 0, 0, 710, 0, 1, 0, 0, 67, 0, 5238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 539, 3152, 0, 1491, 190, 0, 0, 0, 0, 0, 0, 0, 0, 926, 67, 1, 0, 1, 2283, 224, 67, 2873, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5357, 6574, 6695, 6464, 6224, 6016, 5380, 6442] avg=6144.0000 max=6695 max_rank=2 min=5357 min_rank=0 imbalance_ratio=1.089681 max_load_reduction=42457 max_load_reduction_pct=0.863790 moved_to_echo=24062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4254, 0, 0, 0, 0, 1563, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 92, 1846, 0, 2173, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 913, 2509, 639, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5846, 143, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 58, 0, 0, 3437, 1380, 437, 0, 0, 0, 0, 0, 0, 0, 1069, 0, 0, 0, 0, 85, 0, 5046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 2504, 0, 2827, 293, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 92, 0, 0, 0, 2271, 366, 92, 2899, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6832, 5769, 6008, 6047, 5547, 6200, 6016, 6733] avg=6144.0000 max=6832 max_rank=0 min=5547 min_rank=4 imbalance_ratio=1.111979 max_load_reduction=42320 max_load_reduction_pct=0.861003 moved_to_echo=24496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 802, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 62, 2847, 0, 2244, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 1390, 2514, 672, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5972, 453, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 36, 0, 0, 4544, 1409, 224, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 61, 0, 5210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 3141, 0, 1452, 159, 0, 0, 0, 0, 0, 0, 0, 0, 978, 62, 0, 0, 0, 2491, 149, 43, 2904, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5015, 6784, 6663, 6462, 6333, 5955, 5313, 6627] avg=6144.0000 max=6784 max_rank=1 min=5015 min_rank=0 imbalance_ratio=1.104167 max_load_reduction=42368 max_load_reduction_pct=0.861979 moved_to_echo=24228 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 249, 4504, 340, 87, 7, 0, 977, 829, 0, 0, 0, 0, 0, 0, 30, 86, 5, 160, 187, 145, 2284, 32, 1913, 1279, 0, 0, 0, 0, 0, 0, 22, 52, 1332, 873, 2050, 647, 228, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 10, 4979, 330, 86, 30, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 394, 334, 70, 38, 0, 3731, 1175, 273, 0, 0, 0, 0, 0, 0, 154, 1019, 41, 272, 78, 5, 91, 38, 4441, 0, 0, 0, 0, 0, 0, 0, 128, 76, 284, 489, 4566, 42, 1768, 348, 0, 0, 0, 0, 0, 0, 0, 0, 953, 128, 75, 22, 240, 696, 504, 97, 2367, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7000, 6121, 5299, 5645, 6165, 6139, 7701, 5082] avg=6144.0000 max=7701 max_rank=6 min=5082 min_rank=7 imbalance_ratio=1.253418 max_load_reduction=41451 max_load_reduction_pct=0.843323 moved_to_echo=25087 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 3525, 0, 1, 0, 0, 824, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 8, 70, 0, 67, 2621, 0, 2256, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 1347, 2488, 611, 1, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6002, 399, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 126, 1, 63, 1, 0, 4385, 1417, 230, 0, 0, 0, 0, 0, 0, 0, 710, 0, 1, 0, 0, 67, 0, 5238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 539, 3152, 0, 1491, 190, 0, 0, 0, 0, 0, 0, 0, 0, 926, 67, 1, 0, 1, 2283, 224, 67, 2873, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5357, 6574, 6695, 6464, 6224, 6016, 5380, 6442] avg=6144.0000 max=6695 max_rank=2 min=5357 min_rank=0 imbalance_ratio=1.089681 max_load_reduction=42457 max_load_reduction_pct=0.863790 moved_to_echo=24062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3773, 0, 0, 0, 0, 1519, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 87, 2005, 0, 2168, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 1112, 2483, 625, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5939, 74, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 67, 0, 0, 4024, 1402, 425, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 0, 0, 63, 0, 5038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 2686, 0, 2748, 228, 0, 0, 0, 0, 0, 0, 0, 0, 991, 86, 1, 0, 0, 2139, 306, 86, 2869, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 5897, 6127, 6081, 6078, 6147, 6047, 6478] avg=6144.0000 max=6478 max_rank=7 min=5897 min_rank=1 imbalance_ratio=1.054362 max_load_reduction=42674 max_load_reduction_pct=0.868205 moved_to_echo=24750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 4075, 3, 5, 0, 0, 1505, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 1, 92, 1748, 1, 2172, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 1909, 1090, 2453, 650, 1, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5863, 155, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 208, 3, 77, 1, 0, 4036, 1388, 424, 0, 0, 0, 0, 0, 0, 2, 1071, 1, 5, 2, 0, 73, 0, 5044, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 386, 2975, 0, 2706, 285, 0, 0, 0, 0, 0, 0, 0, 0, 939, 90, 0, 0, 5, 1708, 235, 90, 2854, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6595, 5650, 6195, 6096, 6141, 6198, 6356, 5921] avg=6144.0000 max=6595 max_rank=0 min=5650 min_rank=1 imbalance_ratio=1.073405 max_load_reduction=42557 max_load_reduction_pct=0.865824 moved_to_echo=24616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 52, 3897, 61, 44, 0, 0, 859, 970, 0, 0, 0, 0, 0, 0, 2, 0, 0, 79, 18, 78, 2613, 21, 2134, 1497, 0, 0, 0, 0, 0, 0, 1, 0, 1521, 1225, 2397, 619, 14, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 5702, 375, 72, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 186, 77, 66, 1, 0, 4407, 1346, 240, 0, 0, 0, 0, 0, 0, 58, 1066, 6, 76, 1, 0, 78, 0, 4959, 0, 0, 0, 0, 0, 0, 0, 13, 4, 75, 570, 4010, 2, 1555, 239, 0, 0, 0, 0, 0, 0, 0, 0, 905, 81, 11, 0, 77, 1598, 153, 85, 2770, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5883, 6442, 5856, 6195, 6384, 6244, 6468, 5680] avg=6144.0000 max=6468 max_rank=6 min=5680 min_rank=7 imbalance_ratio=1.052734 max_load_reduction=42684 max_load_reduction_pct=0.868408 moved_to_echo=24776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 802, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 62, 2847, 0, 2244, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 1390, 2514, 672, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5972, 453, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 36, 0, 0, 4544, 1409, 224, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 61, 0, 5210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 3141, 0, 1452, 159, 0, 0, 0, 0, 0, 0, 0, 0, 978, 62, 0, 0, 0, 2491, 149, 43, 2904, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5015, 6784, 6663, 6462, 6333, 5955, 5313, 6627] avg=6144.0000 max=6784 max_rank=1 min=5015 min_rank=0 imbalance_ratio=1.104167 max_load_reduction=42368 max_load_reduction_pct=0.861979 moved_to_echo=24228 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 768, 256, 768, 18048, 128, 17280, 12160, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 768, 128, 128, 384, 15232, 2176, 768, 22400, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 384, 30976, 512, 256, 128, 0, 9472, 7936, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 768, 128, 128, 384, 15232, 2176, 768, 22400, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 14720, 9088, 19456, 5120, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 128, 384, 3840, 25728, 128, 17152, 2176, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 256, 1792, 512, 512, 128, 0, 32768, 11008, 2688, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 7808, 128, 384, 128, 128, 640, 128, 40064, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 256, 128, 46336, 2176, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 5474, 2218, 996, 7063, 7453, 0, 0, 0, 0, 0, 81, 0, 0, 1, 0, 1, 0, 132, 2, 0, 1489, 0, 0, 0, 0, 997, 187, 67, 0, 1, 0, 0, 0, 74, 0, 0, 6468, 3922, 178, 0, 57, 1225, 0, 0, 3, 0, 0, 0, 1403, 31, 0, 1, 1, 94, 3282, 82, 6101, 0, 1, 67, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 5059, 1569, 1079, 7040, 7471, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 128, 0, 0, 1818, 0, 0, 0, 0, 1086, 309, 86, 0, 0, 0, 0, 0, 86, 0, 0, 5893, 5842, 154, 0, 0, 1176, 0, 0, 6, 0, 0, 0, 1432, 226, 0, 0, 0, 84, 1730, 86, 6705, 0, 0, 86, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 5083, 1784, 1163, 6904, 7254, 9, 0, 1, 5, 15, 183, 10, 5, 34, 5, 83, 14, 150, 0, 41, 1947, 3, 0, 0, 0, 1032, 320, 79, 0, 60, 0, 1, 1, 158, 0, 3, 5984, 4119, 189, 4, 0, 1113, 5, 37, 81, 11, 86, 19, 1348, 201, 0, 1, 82, 119, 2656, 79, 6475, 7, 75, 79, 6, 2, 27] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 4965, 1888, 1078, 7019, 7361, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 144, 0, 0, 1878, 0, 0, 0, 0, 1080, 393, 92, 0, 0, 0, 0, 0, 101, 0, 0, 5881, 5385, 164, 0, 0, 1217, 0, 0, 0, 0, 0, 0, 1514, 271, 0, 0, 0, 91, 1783, 92, 6654, 0, 0, 92, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 5279, 1549, 1185, 6972, 7360, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 132, 0, 0, 1596, 0, 0, 0, 0, 1105, 591, 92, 0, 0, 0, 0, 0, 95, 0, 0, 5675, 5473, 152, 0, 0, 1240, 0, 0, 9, 0, 0, 0, 1527, 215, 0, 0, 0, 76, 1758, 92, 6866, 0, 0, 92, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 5474, 2218, 996, 7063, 7453, 0, 0, 0, 0, 0, 81, 0, 0, 1, 0, 1, 0, 132, 2, 0, 1489, 0, 0, 0, 0, 997, 187, 67, 0, 1, 0, 0, 0, 74, 0, 0, 6468, 3922, 178, 0, 57, 1225, 0, 0, 3, 0, 0, 0, 1403, 31, 0, 1, 1, 94, 3282, 82, 6101, 0, 1, 67, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 5331, 1642, 1058, 6983, 7393, 0, 0, 0, 1, 1, 20, 0, 1, 4, 1, 5, 0, 153, 0, 2, 2031, 0, 0, 0, 0, 1125, 442, 90, 0, 4, 0, 0, 0, 113, 0, 0, 5711, 5366, 194, 0, 0, 1151, 0, 3, 15, 1, 3, 0, 1651, 330, 0, 0, 5, 104, 1622, 90, 6411, 0, 3, 90, 0, 1, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 4965, 1888, 1078, 7019, 7361, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 144, 0, 0, 1878, 0, 0, 0, 0, 1080, 393, 92, 0, 0, 0, 0, 0, 101, 0, 0, 5881, 5385, 164, 0, 0, 1217, 0, 0, 0, 0, 0, 0, 1514, 271, 0, 0, 0, 91, 1783, 92, 6654, 0, 0, 92, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[44, 4245, 1997, 986, 6282, 6464, 195, 0, 29, 26, 31, 248, 215, 41, 71, 125, 309, 259, 147, 121, 140, 2172, 29, 0, 5, 0, 1455, 666, 99, 12, 122, 28, 12, 14, 443, 22, 18, 5263, 3125, 255, 63, 304, 1049, 76, 122, 140, 22, 295, 77, 1372, 381, 2, 53, 372, 187, 2468, 153, 5841, 29, 102, 109, 31, 46, 143] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 5484, 2000, 1100, 7084, 7507, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 116, 0, 0, 1555, 0, 0, 0, 0, 1048, 139, 62, 0, 0, 0, 0, 0, 88, 0, 0, 6448, 4049, 125, 0, 12, 1291, 0, 0, 0, 0, 0, 0, 1368, 15, 0, 0, 0, 62, 3041, 63, 6361, 0, 0, 62, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3167, 1569, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 1818, 0, 0, 3635, 415, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 309, 86, 0, 0, 0, 3482, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 1150, 5842, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 0, 0, 6, 0, 0, 3558, 1330, 0, 0, 0, 0, 0, 0, 0, 1432, 226, 0, 0, 0, 84, 1730, 1477, 541, 0, 0, 0, 0, 0, 0, 86, 6164, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5815, 6142, 5996, 6071, 7232, 6070, 5490, 6336] avg=6144.0000 max=7232 max_rank=4 min=5490 min_rank=6 imbalance_ratio=1.177083 max_load_reduction=41920 max_load_reduction_pct=0.852865 moved_to_echo=25128 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 5083, 1784, 1163, 6904, 7254, 9, 0, 1, 5, 15, 183, 10, 5, 34, 5, 83, 14, 150, 0, 41, 1947, 3, 0, 0, 0, 1032, 320, 79, 0, 60, 0, 1, 1, 158, 0, 3, 5984, 4119, 189, 4, 0, 1113, 5, 37, 81, 11, 86, 19, 1348, 201, 0, 1, 82, 119, 2656, 79, 6475, 7, 75, 79, 6, 2, 27] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[44, 4245, 1997, 986, 6282, 6464, 195, 0, 29, 26, 31, 248, 215, 41, 71, 125, 309, 259, 147, 121, 140, 2172, 29, 0, 5, 0, 1455, 666, 99, 12, 122, 28, 12, 14, 443, 22, 18, 5263, 3125, 255, 63, 304, 1049, 76, 122, 140, 22, 295, 77, 1372, 381, 2, 53, 372, 187, 2468, 153, 5841, 29, 102, 109, 31, 46, 143] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3109, 1888, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 6051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 1878, 0, 0, 3628, 407, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 393, 92, 0, 0, 0, 3472, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 1148, 5385, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1217, 0, 0, 0, 0, 0, 3547, 1310, 0, 0, 0, 0, 0, 0, 0, 1514, 271, 0, 0, 0, 91, 1783, 1449, 537, 0, 0, 0, 0, 0, 0, 92, 6117, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 6060, 6057, 6142, 6798, 6074, 5645, 6301] avg=6144.0000 max=6798 max_rank=4 min=5645 min_rank=6 imbalance_ratio=1.106445 max_load_reduction=42354 max_load_reduction_pct=0.861694 moved_to_echo=24818 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3182, 1784, 1163, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 15, 183, 10, 5, 34, 5, 5963, 0, 0, 0, 0, 0, 0, 0, 83, 14, 150, 0, 41, 1947, 3, 0, 3691, 417, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 320, 79, 0, 60, 0, 3415, 1125, 0, 0, 0, 0, 0, 0, 1, 1, 158, 0, 3, 1168, 4119, 189, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1113, 5, 37, 81, 11, 86, 3489, 1291, 0, 0, 0, 0, 0, 0, 19, 1348, 201, 0, 1, 82, 119, 2656, 1484, 522, 0, 0, 0, 0, 0, 0, 79, 5953, 7, 75, 79, 6, 2, 27, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6221, 6346, 6031, 5639, 6117, 6432, 6228] avg=6144.0000 max=6432 max_rank=6 min=5639 min_rank=4 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24416 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3427, 2218, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 1, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 1, 0, 132, 2, 0, 1489, 0, 0, 3990, 449, 0, 0, 0, 0, 0, 0, 0, 0, 997, 187, 67, 0, 1, 0, 3494, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 1262, 3922, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 1225, 0, 0, 3, 0, 0, 3569, 1327, 0, 0, 0, 0, 0, 0, 0, 1403, 31, 0, 1, 1, 94, 3282, 1598, 492, 0, 0, 0, 0, 0, 0, 82, 5609, 0, 1, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6641, 6208, 6063, 5962, 5436, 6181, 6902, 5759] avg=6144.0000 max=6902 max_rank=6 min=5436 min_rank=4 imbalance_ratio=1.123372 max_load_reduction=42250 max_load_reduction_pct=0.859578 moved_to_echo=24278 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3305, 1549, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 6051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 1596, 0, 0, 3500, 433, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 591, 92, 0, 0, 0, 3450, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 1108, 5473, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1240, 0, 0, 9, 0, 0, 3522, 1309, 0, 0, 0, 0, 0, 0, 0, 1527, 215, 0, 0, 0, 76, 1758, 1541, 554, 0, 0, 0, 0, 0, 0, 92, 6312, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6039, 6072, 5661, 6305, 6828, 6080, 5671, 6496] avg=6144.0000 max=6828 max_rank=4 min=5661 min_rank=2 imbalance_ratio=1.111328 max_load_reduction=42324 max_load_reduction_pct=0.861084 moved_to_echo=25075 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3109, 1888, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 6051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 1878, 0, 0, 3628, 407, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 393, 92, 0, 0, 0, 3472, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 1148, 5385, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1217, 0, 0, 0, 0, 0, 3547, 1310, 0, 0, 0, 0, 0, 0, 0, 1514, 271, 0, 0, 0, 91, 1783, 1449, 537, 0, 0, 0, 0, 0, 0, 92, 6117, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 6060, 6057, 6142, 6798, 6074, 5645, 6301] avg=6144.0000 max=6798 max_rank=4 min=5645 min_rank=6 imbalance_ratio=1.106445 max_load_reduction=42354 max_load_reduction_pct=0.861694 moved_to_echo=24818 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3167, 1569, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 1818, 0, 0, 3635, 415, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 309, 86, 0, 0, 0, 3482, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 1150, 5842, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 0, 0, 6, 0, 0, 3558, 1330, 0, 0, 0, 0, 0, 0, 0, 1432, 226, 0, 0, 0, 84, 1730, 1477, 541, 0, 0, 0, 0, 0, 0, 86, 6164, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5815, 6142, 5996, 6071, 7232, 6070, 5490, 6336] avg=6144.0000 max=7232 max_rank=4 min=5490 min_rank=6 imbalance_ratio=1.177083 max_load_reduction=41920 max_load_reduction_pct=0.852865 moved_to_echo=25128 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3433, 2000, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 6172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 1555, 0, 0, 3977, 450, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 139, 62, 0, 0, 0, 3505, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 1259, 4049, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1291, 0, 0, 0, 0, 0, 3579, 1335, 0, 0, 0, 0, 0, 0, 0, 1368, 15, 0, 0, 0, 62, 3041, 1601, 513, 0, 0, 0, 0, 0, 0, 63, 5848, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6533, 6244, 6098, 5966, 5521, 6217, 6600, 5973] avg=6144.0000 max=6600 max_rank=6 min=5521 min_rank=4 imbalance_ratio=1.074219 max_load_reduction=42552 max_load_reduction_pct=0.865723 moved_to_echo=24311 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[44, 2658, 1997, 986, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 26, 31, 248, 215, 41, 71, 125, 5313, 0, 0, 0, 0, 0, 0, 0, 309, 259, 147, 121, 140, 2172, 29, 0, 3246, 348, 0, 0, 0, 0, 0, 0, 5, 0, 1455, 666, 99, 12, 122, 28, 3107, 989, 0, 0, 0, 0, 0, 0, 12, 14, 443, 22, 18, 1028, 3125, 255, 0, 0, 0, 0, 0, 0, 0, 0, 63, 304, 1049, 76, 122, 140, 22, 295, 3175, 1151, 0, 0, 0, 0, 0, 0, 77, 1372, 381, 2, 53, 372, 187, 2468, 1239, 471, 0, 0, 0, 0, 0, 0, 153, 5370, 29, 102, 109, 31, 46, 143, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5880, 6099, 6771, 6483, 4917, 6397, 6622, 5983] avg=6144.0000 max=6771 max_rank=2 min=4917 min_rank=4 imbalance_ratio=1.102051 max_load_reduction=42381 max_load_reduction_pct=0.862244 moved_to_echo=23919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3182, 1784, 1163, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 15, 183, 10, 5, 34, 5, 5963, 0, 0, 0, 0, 0, 0, 0, 83, 14, 150, 0, 41, 1947, 3, 0, 3691, 417, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 320, 79, 0, 60, 0, 3415, 1125, 0, 0, 0, 0, 0, 0, 1, 1, 158, 0, 3, 1168, 4119, 189, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1113, 5, 37, 81, 11, 86, 3489, 1291, 0, 0, 0, 0, 0, 0, 19, 1348, 201, 0, 1, 82, 119, 2656, 1484, 522, 0, 0, 0, 0, 0, 0, 79, 5953, 7, 75, 79, 6, 2, 27, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6221, 6346, 6031, 5639, 6117, 6432, 6228] avg=6144.0000 max=6432 max_rank=6 min=5639 min_rank=4 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24416 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3331, 1642, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 20, 0, 1, 4, 1, 6078, 0, 0, 0, 0, 0, 0, 0, 5, 0, 153, 0, 2, 2031, 0, 0, 3527, 442, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 442, 90, 0, 4, 0, 3455, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 1108, 5366, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 0, 3, 15, 1, 3, 3528, 1315, 0, 0, 0, 0, 0, 0, 0, 1651, 330, 0, 0, 5, 104, 1622, 1558, 520, 0, 0, 0, 0, 0, 0, 90, 5891, 0, 3, 90, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6031, 6106, 6160, 6192, 6781, 6016, 5790, 6076] avg=6144.0000 max=6781 max_rank=4 min=5790 min_rank=6 imbalance_ratio=1.103678 max_load_reduction=42371 max_load_reduction_pct=0.862040 moved_to_echo=24663 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3433, 2000, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 6172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 1555, 0, 0, 3977, 450, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 139, 62, 0, 0, 0, 3505, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 1259, 4049, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1291, 0, 0, 0, 0, 0, 3579, 1335, 0, 0, 0, 0, 0, 0, 0, 1368, 15, 0, 0, 0, 62, 3041, 1601, 513, 0, 0, 0, 0, 0, 0, 63, 5848, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6533, 6244, 6098, 5966, 5521, 6217, 6600, 5973] avg=6144.0000 max=6600 max_rank=6 min=5521 min_rank=4 imbalance_ratio=1.074219 max_load_reduction=42552 max_load_reduction_pct=0.865723 moved_to_echo=24311 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 11648, 1792, 128, 128, 512, 896, 18432, 12032, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 384, 9472, 128, 256, 256, 128, 384, 28032, 10368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 11648, 1792, 128, 128, 512, 896, 18432, 12032, 4224, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[512, 384, 1152, 128, 256, 14592, 128, 0, 29312, 3456, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 1280, 128, 128, 9344, 37376, 1536, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 25728, 14720, 8704, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 8960, 3072, 768, 128, 256, 128, 27392, 8960, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 47360, 128, 256, 768, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 640, 256, 128, 128, 256, 48000, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[5764, 0, 0, 0, 0, 0, 0, 3231, 545, 0, 172, 0, 0, 0, 240, 6428, 0, 0, 0, 1247, 258, 393, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 0, 0, 0, 0, 0, 0, 2441, 0, 0, 0, 7272, 0, 0, 383, 0, 0, 0, 0, 4357, 0, 7478, 0, 0, 0, 0, 0, 7323, 113, 0, 0, 87] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[4006, 0, 0, 0, 0, 0, 0, 4182, 512, 0, 87, 0, 0, 0, 322, 6661, 0, 0, 0, 2530, 246, 366, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 1810, 0, 0, 0, 7470, 0, 0, 296, 0, 0, 0, 0, 4531, 0, 7786, 0, 0, 0, 0, 0, 6802, 66, 0, 0, 62] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[5423, 0, 0, 0, 0, 0, 0, 3166, 534, 0, 183, 0, 0, 0, 249, 6451, 0, 0, 0, 1507, 276, 472, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 952, 0, 0, 0, 0, 0, 0, 2424, 0, 0, 0, 7398, 0, 0, 409, 0, 0, 0, 0, 4511, 0, 7471, 0, 0, 0, 0, 0, 7206, 98, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[6027, 340, 45, 35, 25, 192, 49, 2508, 616, 206, 1455, 34, 69, 40, 396, 5944, 12, 75, 55, 875, 334, 415, 12, 674, 9, 10, 102, 18, 57, 70, 105, 76, 1876, 247, 125, 7, 23, 7, 3, 1717, 72, 126, 7, 6305, 130, 337, 319, 82, 14, 24, 2, 2626, 189, 6331, 0, 182, 9, 104, 23, 6779, 194, 94, 232, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[5580, 2, 0, 0, 0, 0, 0, 3221, 528, 5, 187, 0, 0, 0, 200, 6648, 0, 3, 0, 1191, 275, 441, 0, 553, 0, 0, 0, 0, 0, 1, 0, 1, 760, 2, 1, 0, 0, 0, 0, 2920, 0, 5, 0, 7272, 1, 4, 363, 0, 0, 0, 0, 3987, 2, 7378, 0, 4, 0, 2, 1, 7414, 108, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[5273, 53, 1, 2, 0, 0, 2, 3452, 533, 88, 496, 6, 11, 0, 421, 5989, 0, 42, 8, 1546, 259, 403, 15, 518, 4, 1, 0, 0, 1, 10, 0, 16, 1204, 65, 29, 0, 1, 0, 0, 2228, 0, 80, 0, 7130, 34, 73, 332, 2, 1, 0, 0, 4255, 32, 7225, 0, 55, 0, 10, 4, 6990, 155, 14, 4, 79] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[4238, 0, 0, 0, 0, 2, 0, 4166, 553, 1, 96, 0, 0, 0, 378, 6786, 0, 0, 0, 2534, 284, 413, 0, 155, 1, 0, 0, 0, 0, 1, 0, 0, 1151, 0, 0, 0, 0, 0, 0, 1965, 0, 1, 0, 7433, 0, 1, 293, 0, 0, 0, 0, 4028, 1, 7716, 0, 1, 0, 0, 0, 6816, 71, 0, 0, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[5580, 2, 0, 0, 0, 0, 0, 3221, 528, 5, 187, 0, 0, 0, 200, 6648, 0, 3, 0, 1191, 275, 441, 0, 553, 0, 0, 0, 0, 0, 1, 0, 1, 760, 2, 1, 0, 0, 0, 0, 2920, 0, 5, 0, 7272, 1, 4, 363, 0, 0, 0, 0, 3987, 2, 7378, 0, 4, 0, 2, 1, 7414, 108, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[5648, 0, 0, 0, 0, 0, 0, 3100, 519, 0, 309, 0, 0, 0, 197, 6805, 0, 0, 0, 1150, 285, 429, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 3169, 0, 0, 0, 7327, 0, 0, 378, 0, 0, 0, 0, 3598, 0, 7414, 0, 0, 0, 0, 0, 7193, 96, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2938, 0, 0, 0, 0, 0, 0, 3231, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 172, 0, 0, 0, 240, 4844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1247, 258, 393, 0, 412, 1917, 1584, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5347, 909, 0, 0, 0, 0, 0, 0, 1008, 0, 0, 0, 0, 0, 0, 2441, 1426, 1221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5735, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4357, 0, 2131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6102, 113, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6169, 5801, 5922, 6256, 6096, 6118, 6488, 6302] avg=6144.0000 max=6488 max_rank=6 min=5801 min_rank=1 imbalance_ratio=1.055990 max_load_reduction=42664 max_load_reduction_pct=0.868001 moved_to_echo=25004 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[4238, 0, 0, 0, 0, 2, 0, 4166, 553, 1, 96, 0, 0, 0, 378, 6786, 0, 0, 0, 2534, 284, 413, 0, 155, 1, 0, 0, 0, 0, 1, 0, 0, 1151, 0, 0, 0, 0, 0, 0, 1965, 0, 1, 0, 7433, 0, 1, 293, 0, 0, 0, 0, 4028, 1, 7716, 0, 1, 0, 0, 0, 6816, 71, 0, 0, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[5764, 0, 0, 0, 0, 0, 0, 3231, 545, 0, 172, 0, 0, 0, 240, 6428, 0, 0, 0, 1247, 258, 393, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 0, 0, 0, 0, 0, 0, 2441, 0, 0, 0, 7272, 0, 0, 383, 0, 0, 0, 0, 4357, 0, 7478, 0, 0, 0, 0, 0, 7323, 113, 0, 0, 87] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[6027, 340, 45, 35, 25, 192, 49, 2508, 616, 206, 1455, 34, 69, 40, 396, 5944, 12, 75, 55, 875, 334, 415, 12, 674, 9, 10, 102, 18, 57, 70, 105, 76, 1876, 247, 125, 7, 23, 7, 3, 1717, 72, 126, 7, 6305, 130, 337, 319, 82, 14, 24, 2, 2626, 189, 6331, 0, 182, 9, 104, 23, 6779, 194, 94, 232, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2764, 0, 0, 0, 0, 0, 0, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 183, 0, 0, 0, 249, 4861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1507, 276, 472, 0, 330, 1804, 1590, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5342, 855, 0, 0, 0, 0, 0, 0, 952, 0, 0, 0, 0, 0, 0, 2424, 1451, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5834, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4511, 0, 2129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6005, 98, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 5827, 6092, 6197, 6028, 6243, 6640, 6195] avg=6144.0000 max=6640 max_rank=6 min=5827 min_rank=1 imbalance_ratio=1.080729 max_load_reduction=42512 max_load_reduction_pct=0.864909 moved_to_echo=25106 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3071, 340, 45, 35, 25, 192, 49, 2508, 0, 0, 0, 0, 0, 0, 0, 0, 616, 206, 1455, 34, 69, 40, 396, 4479, 0, 0, 0, 0, 0, 0, 0, 0, 12, 75, 55, 875, 334, 415, 12, 674, 2005, 1465, 96, 0, 0, 0, 0, 0, 9, 10, 102, 18, 57, 70, 105, 76, 4527, 951, 0, 0, 0, 0, 0, 0, 1876, 247, 125, 7, 23, 7, 3, 1717, 1236, 1130, 0, 0, 0, 0, 0, 0, 72, 126, 7, 4973, 130, 337, 319, 82, 0, 0, 0, 0, 0, 0, 0, 0, 14, 24, 2, 2626, 189, 1804, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 9, 104, 23, 5649, 194, 94, 232, 86, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6265, 7295, 6018, 5925, 6371, 6046, 4841, 6391] avg=6144.0000 max=7295 max_rank=1 min=4841 min_rank=6 imbalance_ratio=1.187337 max_load_reduction=41857 max_load_reduction_pct=0.851583 moved_to_echo=23649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2161, 0, 0, 0, 0, 2, 0, 4166, 0, 0, 0, 0, 0, 0, 0, 0, 553, 1, 96, 0, 0, 0, 378, 5114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2534, 284, 413, 0, 155, 1409, 1672, 113, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 5517, 668, 0, 0, 0, 0, 0, 0, 1151, 0, 0, 0, 0, 0, 0, 1965, 1458, 1136, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5862, 0, 1, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 1, 2199, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5680, 71, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6329, 6142, 6580, 6187, 5710, 6157, 6229, 5818] avg=6144.0000 max=6580 max_rank=2 min=5710 min_rank=4 imbalance_ratio=1.070964 max_load_reduction=42572 max_load_reduction_pct=0.866130 moved_to_echo=23914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2687, 53, 1, 2, 0, 0, 2, 3452, 0, 0, 0, 0, 0, 0, 0, 0, 533, 88, 496, 6, 11, 0, 421, 4513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 8, 1546, 259, 403, 15, 518, 1754, 1476, 109, 0, 0, 0, 0, 0, 4, 1, 0, 0, 1, 10, 0, 16, 5166, 832, 0, 0, 0, 0, 0, 0, 1204, 65, 29, 0, 1, 0, 0, 2228, 1398, 1165, 0, 0, 0, 0, 0, 0, 0, 80, 0, 5623, 34, 73, 332, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4255, 32, 2059, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 5825, 155, 14, 4, 79, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6197, 6068, 6130, 6030, 6090, 6144, 6402, 6091] avg=6144.0000 max=6402 max_rank=6 min=6030 min_rank=3 imbalance_ratio=1.041992 max_load_reduction=42750 max_load_reduction_pct=0.869751 moved_to_echo=24727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2879, 0, 0, 0, 0, 0, 0, 3100, 0, 0, 0, 0, 0, 0, 0, 0, 519, 0, 309, 0, 0, 0, 197, 5128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 285, 429, 0, 390, 1878, 1677, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5301, 891, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 3169, 1437, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5778, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 0, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5994, 96, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5979, 6153, 5921, 6192, 6858, 6156, 5711, 6182] avg=6144.0000 max=6858 max_rank=4 min=5711 min_rank=6 imbalance_ratio=1.116211 max_load_reduction=42294 max_load_reduction_pct=0.860474 moved_to_echo=24907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2042, 0, 0, 0, 0, 0, 0, 4182, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 87, 0, 0, 0, 322, 5019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 246, 366, 0, 181, 1332, 1642, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5567, 632, 0, 0, 0, 0, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 1810, 1465, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5891, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4531, 0, 2219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5668, 66, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6224, 5940, 6411, 6199, 5645, 6187, 6750, 5796] avg=6144.0000 max=6750 max_rank=6 min=5645 min_rank=4 imbalance_ratio=1.098633 max_load_reduction=42402 max_load_reduction_pct=0.862671 moved_to_echo=24378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2161, 0, 0, 0, 0, 2, 0, 4166, 0, 0, 0, 0, 0, 0, 0, 0, 553, 1, 96, 0, 0, 0, 378, 5114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2534, 284, 413, 0, 155, 1409, 1672, 113, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 5517, 668, 0, 0, 0, 0, 0, 0, 1151, 0, 0, 0, 0, 0, 0, 1965, 1458, 1136, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5862, 0, 1, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 1, 2199, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5680, 71, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6329, 6142, 6580, 6187, 5710, 6157, 6229, 5818] avg=6144.0000 max=6580 max_rank=2 min=5710 min_rank=4 imbalance_ratio=1.070964 max_load_reduction=42572 max_load_reduction_pct=0.866130 moved_to_echo=23914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2938, 0, 0, 0, 0, 0, 0, 3231, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 172, 0, 0, 0, 240, 4844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1247, 258, 393, 0, 412, 1917, 1584, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5347, 909, 0, 0, 0, 0, 0, 0, 1008, 0, 0, 0, 0, 0, 0, 2441, 1426, 1221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5735, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4357, 0, 2131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6102, 113, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6169, 5801, 5922, 6256, 6096, 6118, 6488, 6302] avg=6144.0000 max=6488 max_rank=6 min=5801 min_rank=1 imbalance_ratio=1.055990 max_load_reduction=42664 max_load_reduction_pct=0.868001 moved_to_echo=25004 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2836, 2, 0, 0, 0, 0, 0, 3221, 0, 0, 0, 0, 0, 0, 0, 0, 528, 5, 187, 0, 0, 0, 200, 5006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1191, 275, 441, 0, 553, 1860, 1642, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 5280, 884, 0, 0, 0, 0, 0, 0, 760, 2, 1, 0, 0, 0, 0, 2920, 1431, 1240, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5728, 1, 4, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 2, 2098, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6174, 108, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6059, 5926, 6078, 6166, 6354, 6101, 6091, 6377] avg=6144.0000 max=6377 max_rank=7 min=5926 min_rank=1 imbalance_ratio=1.037923 max_load_reduction=42775 max_load_reduction_pct=0.870260 moved_to_echo=24923 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2879, 0, 0, 0, 0, 0, 0, 3100, 0, 0, 0, 0, 0, 0, 0, 0, 519, 0, 309, 0, 0, 0, 197, 5128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 285, 429, 0, 390, 1878, 1677, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5301, 891, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 3169, 1437, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5778, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 0, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5994, 96, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5979, 6153, 5921, 6192, 6858, 6156, 5711, 6182] avg=6144.0000 max=6858 max_rank=4 min=5711 min_rank=6 imbalance_ratio=1.116211 max_load_reduction=42294 max_load_reduction_pct=0.860474 moved_to_echo=24907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3071, 340, 45, 35, 25, 192, 49, 2508, 0, 0, 0, 0, 0, 0, 0, 0, 616, 206, 1455, 34, 69, 40, 396, 4479, 0, 0, 0, 0, 0, 0, 0, 0, 12, 75, 55, 875, 334, 415, 12, 674, 2005, 1465, 96, 0, 0, 0, 0, 0, 9, 10, 102, 18, 57, 70, 105, 76, 4527, 951, 0, 0, 0, 0, 0, 0, 1876, 247, 125, 7, 23, 7, 3, 1717, 1236, 1130, 0, 0, 0, 0, 0, 0, 72, 126, 7, 4973, 130, 337, 319, 82, 0, 0, 0, 0, 0, 0, 0, 0, 14, 24, 2, 2626, 189, 1804, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 9, 104, 23, 5649, 194, 94, 232, 86, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6265, 7295, 6018, 5925, 6371, 6046, 4841, 6391] avg=6144.0000 max=7295 max_rank=1 min=4841 min_rank=6 imbalance_ratio=1.187337 max_load_reduction=41857 max_load_reduction_pct=0.851583 moved_to_echo=23649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[9344, 384, 256, 128, 128, 128, 128, 18688, 11392, 9472, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 12672, 2304, 3456, 128, 3328, 14080, 12800, 896, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[21504, 512, 128, 128, 128, 256, 128, 27136, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32000, 256, 16768, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[21504, 512, 128, 128, 128, 256, 128, 27136, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 42112, 6656, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 384, 3072, 128, 128, 128, 2432, 39040, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 256, 128, 45440, 256, 512, 2816, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 47104, 1024, 128, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 632, 1, 0, 2, 0, 0, 1, 254, 0, 0, 7231, 0, 5452, 0, 56, 342, 107, 0, 0, 0, 1038, 0, 1632, 0, 1, 0, 0, 0, 0, 6669, 3186, 0, 230, 0, 1, 0, 805, 0, 2730, 6091, 0, 0, 1, 6509, 4484, 68, 1, 0, 0, 0, 254, 0, 67, 0, 0, 1, 1306, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 472, 0, 0, 0, 0, 0, 0, 251, 0, 0, 7168, 0, 5671, 0, 87, 331, 68, 0, 0, 0, 1110, 0, 2507, 0, 0, 0, 0, 0, 0, 6716, 1715, 0, 217, 0, 0, 0, 691, 0, 3441, 5247, 0, 0, 0, 6485, 5245, 59, 0, 0, 0, 0, 251, 0, 87, 0, 0, 0, 1333, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 548, 0, 0, 0, 0, 0, 0, 227, 0, 0, 7196, 0, 5408, 0, 92, 340, 58, 0, 0, 0, 845, 0, 3019, 0, 0, 0, 0, 0, 0, 6725, 1989, 0, 307, 0, 0, 0, 681, 0, 3183, 5526, 0, 0, 0, 6452, 4646, 73, 0, 0, 0, 0, 229, 0, 92, 0, 0, 0, 1516, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 497, 2, 0, 0, 1, 0, 2, 247, 0, 0, 7141, 0, 5956, 0, 92, 351, 84, 0, 0, 0, 864, 1, 2646, 0, 2, 0, 0, 0, 0, 6716, 1600, 0, 260, 0, 5, 0, 585, 0, 3608, 5325, 0, 3, 5, 6459, 4749, 53, 5, 0, 0, 0, 253, 0, 93, 0, 0, 4, 1539, 3, 0, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[55, 18, 698, 5, 17, 97, 147, 47, 64, 319, 46, 2, 5350, 13, 5512, 7, 179, 340, 131, 3, 4, 133, 680, 34, 3388, 109, 96, 109, 108, 25, 44, 5724, 1465, 28, 189, 0, 197, 4, 827, 147, 3825, 5240, 75, 99, 229, 5895, 2937, 85, 255, 56, 23, 245, 1053, 5, 123, 0, 22, 194, 1908, 242, 135, 46, 26, 73] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 433, 0, 0, 0, 0, 0, 0, 252, 0, 0, 7141, 0, 5562, 0, 92, 318, 80, 0, 0, 0, 1324, 0, 2710, 0, 0, 0, 0, 0, 0, 6711, 1795, 0, 306, 0, 0, 0, 687, 0, 3941, 4912, 0, 0, 0, 6429, 4669, 67, 0, 0, 0, 0, 252, 0, 92, 0, 0, 0, 1379, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 675, 0, 0, 0, 0, 0, 0, 254, 0, 0, 7303, 0, 5338, 0, 23, 422, 60, 0, 0, 0, 1049, 0, 1423, 0, 0, 0, 0, 0, 0, 6691, 3470, 0, 220, 0, 0, 0, 884, 0, 2877, 6157, 0, 0, 0, 6629, 3988, 59, 0, 0, 0, 0, 255, 0, 80, 0, 0, 0, 1295, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[8, 1, 537, 22, 8, 0, 8, 0, 66, 261, 1, 0, 6800, 1, 5440, 5, 82, 349, 101, 0, 0, 0, 819, 2, 2554, 13, 14, 3, 1, 0, 0, 6454, 2089, 1, 219, 0, 80, 0, 967, 0, 3734, 5569, 5, 32, 87, 6358, 4143, 61, 88, 9, 0, 14, 453, 0, 84, 0, 0, 82, 1427, 67, 26, 4, 0, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 472, 0, 0, 0, 0, 0, 4579, 1043, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 330, 0, 5671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 331, 68, 0, 0, 0, 1110, 0, 2620, 1906, 0, 0, 0, 0, 0, 0, 2507, 0, 0, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 0, 217, 0, 0, 0, 691, 0, 2122, 1178, 0, 0, 0, 0, 0, 0, 3441, 0, 0, 0, 0, 0, 3123, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 87, 0, 5795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 0, 0, 0, 0, 0, 2627, 1853, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 6252, 6122, 6192, 5923, 6623, 6133, 5813] avg=6144.0000 max=6623 max_rank=5 min=5813 min_rank=7 imbalance_ratio=1.077962 max_load_reduction=42529 max_load_reduction_pct=0.865255 moved_to_echo=24492 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 632, 1, 0, 2, 0, 0, 4596, 1052, 0, 0, 0, 0, 0, 0, 1, 254, 0, 0, 333, 0, 5452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 342, 107, 0, 0, 0, 1038, 0, 3042, 1913, 0, 0, 0, 0, 0, 0, 1632, 0, 1, 0, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 3186, 0, 230, 0, 1, 0, 805, 0, 1814, 1170, 0, 0, 0, 0, 0, 0, 2730, 0, 0, 0, 1, 0, 2670, 68, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 254, 0, 67, 0, 5846, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1306, 0, 0, 0, 0, 0, 3049, 1840, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 6040, 6498, 5292, 7206, 5469, 6168, 6196] avg=6144.0000 max=7206 max_rank=4 min=5292 min_rank=3 imbalance_ratio=1.172852 max_load_reduction=41946 max_load_reduction_pct=0.853394 moved_to_echo=25039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 548, 0, 0, 0, 0, 0, 4556, 1047, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 332, 0, 5408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 340, 58, 0, 0, 0, 845, 0, 2760, 1896, 0, 0, 0, 0, 0, 0, 3019, 0, 0, 0, 0, 0, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 0, 307, 0, 0, 0, 681, 0, 1880, 1180, 0, 0, 0, 0, 0, 0, 3183, 0, 0, 0, 0, 0, 2766, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 92, 0, 5817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1516, 0, 0, 0, 0, 0, 2766, 1855, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 5967, 5991, 6709, 6037, 6022, 6138, 6137] avg=6144.0000 max=6709 max_rank=3 min=5967 min_rank=1 imbalance_ratio=1.091960 max_load_reduction=42443 max_load_reduction_pct=0.863505 moved_to_echo=24334 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[55, 18, 698, 5, 17, 97, 147, 47, 4163, 779, 0, 0, 0, 0, 0, 0, 64, 319, 46, 2, 246, 13, 5512, 7, 0, 0, 0, 0, 0, 0, 0, 0, 179, 340, 131, 3, 4, 133, 680, 34, 2617, 1732, 0, 0, 0, 0, 0, 0, 3388, 109, 96, 109, 108, 25, 44, 3141, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 28, 189, 0, 197, 4, 827, 147, 1188, 1004, 0, 0, 0, 0, 0, 0, 3825, 0, 75, 99, 229, 0, 1749, 85, 0, 0, 0, 0, 0, 0, 0, 0, 255, 56, 23, 245, 1053, 5, 123, 0, 4325, 0, 0, 0, 0, 0, 0, 0, 22, 194, 1908, 242, 135, 46, 26, 73, 2623, 1579, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6026, 6209, 5853, 7020, 5049, 6062, 6085, 6848] avg=6144.0000 max=7020 max_rank=3 min=5049 min_rank=4 imbalance_ratio=1.142578 max_load_reduction=42132 max_load_reduction_pct=0.857178 moved_to_echo=24044 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 548, 0, 0, 0, 0, 0, 0, 227, 0, 0, 7196, 0, 5408, 0, 92, 340, 58, 0, 0, 0, 845, 0, 3019, 0, 0, 0, 0, 0, 0, 6725, 1989, 0, 307, 0, 0, 0, 681, 0, 3183, 5526, 0, 0, 0, 6452, 4646, 73, 0, 0, 0, 0, 229, 0, 92, 0, 0, 0, 1516, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 472, 0, 0, 0, 0, 0, 0, 251, 0, 0, 7168, 0, 5671, 0, 87, 331, 68, 0, 0, 0, 1110, 0, 2507, 0, 0, 0, 0, 0, 0, 6716, 1715, 0, 217, 0, 0, 0, 691, 0, 3441, 5247, 0, 0, 0, 6485, 5245, 59, 0, 0, 0, 0, 251, 0, 87, 0, 0, 0, 1333, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 675, 0, 0, 0, 0, 0, 0, 254, 0, 0, 7303, 0, 5338, 0, 23, 422, 60, 0, 0, 0, 1049, 0, 1423, 0, 0, 0, 0, 0, 0, 6691, 3470, 0, 220, 0, 0, 0, 884, 0, 2877, 6157, 0, 0, 0, 6629, 3988, 59, 0, 0, 0, 0, 255, 0, 80, 0, 0, 0, 1295, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 632, 1, 0, 2, 0, 0, 1, 254, 0, 0, 7231, 0, 5452, 0, 56, 342, 107, 0, 0, 0, 1038, 0, 1632, 0, 1, 0, 0, 0, 0, 6669, 3186, 0, 230, 0, 1, 0, 805, 0, 2730, 6091, 0, 0, 1, 6509, 4484, 68, 1, 0, 0, 0, 254, 0, 67, 0, 0, 1, 1306, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[55, 18, 698, 5, 17, 97, 147, 47, 64, 319, 46, 2, 5350, 13, 5512, 7, 179, 340, 131, 3, 4, 133, 680, 34, 3388, 109, 96, 109, 108, 25, 44, 5724, 1465, 28, 189, 0, 197, 4, 827, 147, 3825, 5240, 75, 99, 229, 5895, 2937, 85, 255, 56, 23, 245, 1053, 5, 123, 0, 22, 194, 1908, 242, 135, 46, 26, 73] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 497, 2, 0, 0, 1, 0, 2, 247, 0, 0, 7141, 0, 5956, 0, 92, 351, 84, 0, 0, 0, 864, 1, 2646, 0, 2, 0, 0, 0, 0, 6716, 1600, 0, 260, 0, 5, 0, 585, 0, 3608, 5325, 0, 3, 5, 6459, 4749, 53, 5, 0, 0, 0, 253, 0, 93, 0, 0, 4, 1539, 3, 0, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 433, 0, 0, 0, 0, 0, 4539, 1039, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 329, 0, 5562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 318, 80, 0, 0, 0, 1324, 0, 2453, 1890, 0, 0, 0, 0, 0, 0, 2710, 0, 0, 0, 0, 0, 0, 3683, 0, 0, 0, 0, 0, 0, 0, 0, 1795, 0, 306, 0, 0, 0, 687, 0, 1889, 1177, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 2780, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 92, 0, 5773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 0, 0, 0, 0, 0, 2459, 1851, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6011, 6143, 6157, 6393, 5854, 6788, 6117, 5689] avg=6144.0000 max=6788 max_rank=5 min=5689 min_rank=7 imbalance_ratio=1.104818 max_load_reduction=42364 max_load_reduction_pct=0.861898 moved_to_echo=24448 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 497, 2, 0, 0, 1, 0, 4561, 1044, 0, 0, 0, 0, 0, 0, 2, 247, 0, 0, 321, 0, 5956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 351, 84, 0, 0, 0, 864, 1, 2660, 1898, 0, 0, 0, 0, 0, 0, 2646, 0, 2, 0, 0, 0, 0, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 1600, 0, 260, 0, 5, 0, 585, 0, 1926, 1181, 0, 0, 0, 0, 0, 0, 3608, 0, 0, 3, 5, 0, 2823, 53, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 253, 0, 93, 0, 5776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1539, 3, 0, 0, 0, 1, 2665, 1857, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 6526, 5950, 6326, 5557, 6492, 6127, 6069] avg=6144.0000 max=6526 max_rank=1 min=5557 min_rank=4 imbalance_ratio=1.062174 max_load_reduction=42626 max_load_reduction_pct=0.867228 moved_to_echo=24245 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 537, 22, 8, 0, 8, 0, 4489, 990, 0, 0, 0, 0, 0, 0, 66, 261, 1, 0, 313, 1, 5440, 5, 0, 0, 0, 0, 0, 0, 0, 0, 82, 349, 101, 0, 0, 0, 819, 2, 2781, 1869, 0, 0, 0, 0, 0, 0, 2554, 13, 14, 3, 1, 0, 0, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 1, 219, 0, 80, 0, 967, 0, 1676, 1132, 0, 0, 0, 0, 0, 0, 3734, 0, 5, 32, 87, 0, 2467, 61, 0, 0, 0, 0, 0, 0, 0, 0, 88, 9, 0, 14, 453, 0, 84, 0, 5497, 0, 0, 0, 0, 0, 0, 0, 0, 82, 1427, 67, 26, 4, 0, 3, 2788, 1781, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6087, 6003, 6126, 6164, 6386, 6145, 6178] avg=6144.0000 max=6386 max_rank=5 min=6003 min_rank=2 imbalance_ratio=1.039388 max_load_reduction=42766 max_load_reduction_pct=0.870076 moved_to_echo=24873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 548, 0, 0, 0, 0, 0, 4556, 1047, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 332, 0, 5408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 340, 58, 0, 0, 0, 845, 0, 2760, 1896, 0, 0, 0, 0, 0, 0, 3019, 0, 0, 0, 0, 0, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 0, 307, 0, 0, 0, 681, 0, 1880, 1180, 0, 0, 0, 0, 0, 0, 3183, 0, 0, 0, 0, 0, 2766, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 92, 0, 5817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1516, 0, 0, 0, 0, 0, 2766, 1855, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 5967, 5991, 6709, 6037, 6022, 6138, 6137] avg=6144.0000 max=6709 max_rank=3 min=5967 min_rank=1 imbalance_ratio=1.091960 max_load_reduction=42443 max_load_reduction_pct=0.863505 moved_to_echo=24334 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 632, 1, 0, 2, 0, 0, 4596, 1052, 0, 0, 0, 0, 0, 0, 1, 254, 0, 0, 333, 0, 5452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 342, 107, 0, 0, 0, 1038, 0, 3042, 1913, 0, 0, 0, 0, 0, 0, 1632, 0, 1, 0, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 3186, 0, 230, 0, 1, 0, 805, 0, 1814, 1170, 0, 0, 0, 0, 0, 0, 2730, 0, 0, 0, 1, 0, 2670, 68, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 254, 0, 67, 0, 5846, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1306, 0, 0, 0, 0, 0, 3049, 1840, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 6040, 6498, 5292, 7206, 5469, 6168, 6196] avg=6144.0000 max=7206 max_rank=4 min=5292 min_rank=3 imbalance_ratio=1.172852 max_load_reduction=41946 max_load_reduction_pct=0.853394 moved_to_echo=25039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 675, 0, 0, 0, 0, 0, 4681, 1063, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 336, 0, 5338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 422, 60, 0, 0, 0, 1049, 0, 3076, 1948, 0, 0, 0, 0, 0, 0, 1423, 0, 0, 0, 0, 0, 0, 3671, 0, 0, 0, 0, 0, 0, 0, 0, 3470, 0, 220, 0, 0, 0, 884, 0, 1614, 1174, 0, 0, 0, 0, 0, 0, 2877, 0, 0, 0, 0, 0, 2374, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 80, 0, 5904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 0, 0, 0, 0, 0, 3081, 1846, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6419, 5928, 6578, 5094, 7362, 5310, 6239, 6222] avg=6144.0000 max=7362 max_rank=4 min=5094 min_rank=3 imbalance_ratio=1.198242 max_load_reduction=41790 max_load_reduction_pct=0.850220 moved_to_echo=25133 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 472, 0, 0, 0, 0, 0, 4579, 1043, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 330, 0, 5671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 331, 68, 0, 0, 0, 1110, 0, 2620, 1906, 0, 0, 0, 0, 0, 0, 2507, 0, 0, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 0, 217, 0, 0, 0, 691, 0, 2122, 1178, 0, 0, 0, 0, 0, 0, 3441, 0, 0, 0, 0, 0, 3123, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 87, 0, 5795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 0, 0, 0, 0, 0, 2627, 1853, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 6252, 6122, 6192, 5923, 6623, 6133, 5813] avg=6144.0000 max=6623 max_rank=5 min=5813 min_rank=7 imbalance_ratio=1.077962 max_load_reduction=42529 max_load_reduction_pct=0.865255 moved_to_echo=24492 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[55, 18, 698, 5, 17, 97, 147, 47, 4163, 779, 0, 0, 0, 0, 0, 0, 64, 319, 46, 2, 246, 13, 5512, 7, 0, 0, 0, 0, 0, 0, 0, 0, 179, 340, 131, 3, 4, 133, 680, 34, 2617, 1732, 0, 0, 0, 0, 0, 0, 3388, 109, 96, 109, 108, 25, 44, 3141, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 28, 189, 0, 197, 4, 827, 147, 1188, 1004, 0, 0, 0, 0, 0, 0, 3825, 0, 75, 99, 229, 0, 1749, 85, 0, 0, 0, 0, 0, 0, 0, 0, 255, 56, 23, 245, 1053, 5, 123, 0, 4325, 0, 0, 0, 0, 0, 0, 0, 22, 194, 1908, 242, 135, 46, 26, 73, 2623, 1579, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6026, 6209, 5853, 7020, 5049, 6062, 6085, 6848] avg=6144.0000 max=7020 max_rank=3 min=5049 min_rank=4 imbalance_ratio=1.142578 max_load_reduction=42132 max_load_reduction_pct=0.857178 moved_to_echo=24044 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 497, 2, 0, 0, 1, 0, 4561, 1044, 0, 0, 0, 0, 0, 0, 2, 247, 0, 0, 321, 0, 5956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 351, 84, 0, 0, 0, 864, 1, 2660, 1898, 0, 0, 0, 0, 0, 0, 2646, 0, 2, 0, 0, 0, 0, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 1600, 0, 260, 0, 5, 0, 585, 0, 1926, 1181, 0, 0, 0, 0, 0, 0, 3608, 0, 0, 3, 5, 0, 2823, 53, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 253, 0, 93, 0, 5776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1539, 3, 0, 0, 0, 1, 2665, 1857, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 6526, 5950, 6326, 5557, 6492, 6127, 6069] avg=6144.0000 max=6526 max_rank=1 min=5557 min_rank=4 imbalance_ratio=1.062174 max_load_reduction=42626 max_load_reduction_pct=0.867228 moved_to_echo=24245 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[17408, 128, 2048, 0, 384, 128, 6144, 256, 14208, 9216, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 4608, 128, 128, 128, 256, 128, 36224, 8064, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[19968, 128, 128, 128, 128, 128, 128, 28800, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[27392, 0, 128, 256, 384, 0, 20864, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 384, 11776, 384, 256, 128, 128, 128, 22144, 14464, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[19968, 128, 128, 128, 128, 128, 128, 28800, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 2816, 768, 128, 128, 256, 7808, 128, 22016, 15104, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 384, 11776, 384, 256, 128, 128, 128, 22144, 14464, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 2176, 128, 128, 2560, 128, 44416, 128, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 128, 128, 384, 3072, 128, 768, 0, 44800, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 68, 0, 0, 0, 0, 214, 125, 86, 7476, 184, 0, 0, 86, 0, 78, 2, 0, 0, 107, 0, 187, 0, 0, 0, 0, 0, 0, 859, 7362, 0, 0, 0, 0, 0, 0, 375, 6994, 0, 8106, 87, 0, 0, 3725, 97, 0, 271, 0, 0, 0, 0, 0, 0, 3571, 0, 0, 0, 0, 412, 7842, 0, 0, 0, 838] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 279, 200, 93, 282, 52, 447, 540, 87, 6299, 268, 13, 39, 160, 112, 225, 73, 21, 0, 470, 165, 452, 186, 0, 48, 115, 133, 69, 1131, 5481, 240, 3, 256, 197, 0, 41, 696, 5318, 51, 7451, 94, 14, 105, 3842, 1014, 23, 622, 37, 127, 23, 58, 5, 16, 1591, 5, 0, 179, 2, 718, 6437, 24, 32, 28, 2463] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 119, 51, 78, 92, 50, 301, 384, 79, 7323, 196, 0, 0, 105, 5, 93, 57, 16, 0, 246, 82, 256, 67, 0, 1, 14, 0, 0, 929, 6669, 68, 0, 1, 87, 0, 0, 593, 6247, 8, 8013, 79, 2, 2, 4391, 417, 8, 354, 1, 30, 0, 1, 0, 1, 2310, 0, 0, 0, 0, 488, 7548, 1, 0, 6, 1283] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 60, 0, 0, 0, 0, 274, 195, 92, 7499, 204, 0, 0, 92, 0, 76, 24, 0, 0, 271, 0, 277, 0, 0, 0, 0, 0, 0, 860, 7059, 0, 0, 0, 0, 0, 0, 185, 7010, 0, 8100, 92, 0, 0, 3674, 92, 0, 283, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 0, 0, 456, 7815, 0, 0, 0, 999] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 146, 0, 0, 0, 0, 265, 151, 92, 7487, 186, 0, 0, 92, 0, 83, 2, 0, 0, 165, 0, 192, 0, 0, 0, 0, 0, 0, 895, 7228, 0, 0, 0, 0, 0, 0, 312, 6918, 0, 8100, 92, 0, 0, 3097, 105, 0, 211, 0, 0, 0, 0, 0, 0, 4153, 0, 0, 0, 0, 483, 7816, 0, 0, 0, 881] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 268, 0, 1, 1, 1, 331, 7, 66, 7328, 331, 0, 0, 67, 0, 75, 31, 0, 0, 77, 1, 226, 1, 0, 0, 0, 0, 0, 768, 7303, 0, 0, 0, 1, 0, 0, 419, 7054, 0, 8123, 67, 0, 0, 4100, 82, 0, 78, 0, 0, 0, 0, 0, 0, 3420, 0, 0, 0, 0, 380, 7838, 0, 0, 0, 707] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 60, 0, 0, 0, 0, 274, 195, 92, 7499, 204, 0, 0, 92, 0, 76, 24, 0, 0, 271, 0, 277, 0, 0, 0, 0, 0, 0, 860, 7059, 0, 0, 0, 0, 0, 0, 185, 7010, 0, 8100, 92, 0, 0, 3674, 92, 0, 283, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 0, 0, 456, 7815, 0, 0, 0, 999] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 279, 200, 93, 282, 52, 447, 540, 87, 6299, 268, 13, 39, 160, 112, 225, 73, 21, 0, 470, 165, 452, 186, 0, 48, 115, 133, 69, 1131, 5481, 240, 3, 256, 197, 0, 41, 696, 5318, 51, 7451, 94, 14, 105, 3842, 1014, 23, 622, 37, 127, 23, 58, 5, 16, 1591, 5, 0, 179, 2, 718, 6437, 24, 32, 28, 2463] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 208, 0, 0, 0, 0, 278, 44, 62, 7408, 340, 0, 0, 62, 0, 52, 39, 0, 0, 39, 0, 184, 0, 0, 0, 0, 0, 0, 683, 7357, 0, 0, 0, 0, 0, 0, 405, 7093, 0, 8130, 62, 0, 0, 3761, 71, 0, 78, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 0, 0, 389, 7884, 0, 0, 0, 672] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 77, 3, 5, 5, 4, 275, 392, 90, 7493, 47, 0, 0, 90, 0, 100, 11, 0, 0, 148, 4, 194, 3, 0, 0, 1, 0, 0, 954, 6937, 4, 0, 0, 5, 0, 0, 335, 6967, 0, 8097, 92, 0, 0, 3606, 129, 0, 278, 0, 1, 0, 0, 0, 0, 3557, 0, 0, 0, 0, 465, 7813, 0, 0, 0, 970] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 119, 51, 78, 92, 50, 301, 384, 79, 7323, 196, 0, 0, 105, 5, 93, 57, 16, 0, 246, 82, 256, 67, 0, 1, 14, 0, 0, 929, 6669, 68, 0, 1, 87, 0, 0, 593, 6247, 8, 8013, 79, 2, 2, 4391, 417, 8, 354, 1, 30, 0, 1, 0, 1, 2310, 0, 0, 0, 0, 488, 7548, 1, 0, 6, 1283] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 0, 0, 0, 0, 214, 125, 3026, 2432, 0, 0, 0, 0, 0, 0, 86, 5775, 184, 0, 0, 86, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 107, 0, 187, 0, 0, 5674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 5499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 5896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 3725, 97, 0, 271, 0, 1098, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 0, 0, 1863, 996, 151, 0, 0, 0, 0, 0, 0, 0, 412, 4665, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5865, 6209, 5970, 6358, 6271, 5983, 6581, 5915] avg=6144.0000 max=6581 max_rank=6 min=5865 min_rank=0 imbalance_ratio=1.071126 max_load_reduction=42571 max_load_reduction_pct=0.866109 moved_to_echo=24750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 146, 0, 0, 0, 0, 265, 151, 3016, 2431, 0, 0, 0, 0, 0, 0, 92, 5783, 186, 0, 0, 92, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 165, 0, 192, 0, 0, 5669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 5399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 5832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 3097, 105, 0, 211, 0, 1086, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4153, 0, 0, 1829, 998, 150, 0, 0, 0, 0, 0, 0, 0, 483, 4650, 0, 0, 0, 881, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6009, 6236, 6028, 6294, 6144, 5297, 7130, 6014] avg=6144.0000 max=7130 max_rank=6 min=5297 min_rank=5 imbalance_ratio=1.160482 max_load_reduction=42022 max_load_reduction_pct=0.854940 moved_to_echo=24585 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 279, 200, 93, 282, 52, 447, 540, 2484, 2236, 0, 0, 0, 0, 0, 0, 87, 4866, 268, 13, 39, 160, 112, 225, 0, 0, 0, 0, 0, 0, 0, 0, 73, 21, 0, 470, 165, 452, 186, 0, 5215, 0, 0, 0, 0, 0, 0, 0, 48, 115, 133, 69, 1131, 4094, 240, 3, 0, 0, 0, 0, 0, 0, 0, 0, 256, 197, 0, 41, 696, 4484, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 14, 105, 3842, 1014, 23, 622, 37, 834, 594, 0, 0, 0, 0, 0, 0, 127, 23, 58, 5, 16, 1591, 5, 0, 1387, 839, 124, 0, 0, 0, 0, 0, 179, 2, 718, 3829, 24, 32, 28, 2463, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6613, 5770, 6582, 5833, 5725, 7179, 4175, 7275] avg=6144.0000 max=7275 max_rank=7 min=4175 min_rank=6 imbalance_ratio=1.184082 max_load_reduction=41877 max_load_reduction_pct=0.851990 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 119, 51, 78, 92, 50, 301, 384, 2913, 2404, 0, 0, 0, 0, 0, 0, 79, 5657, 196, 0, 0, 105, 5, 93, 0, 0, 0, 0, 0, 0, 0, 0, 57, 16, 0, 246, 82, 256, 67, 0, 5609, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 929, 4982, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 87, 0, 0, 593, 5267, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 2, 2, 4391, 417, 8, 354, 1, 980, 690, 0, 0, 0, 0, 0, 0, 30, 0, 1, 0, 1, 2310, 0, 0, 1687, 976, 145, 0, 0, 0, 0, 0, 0, 0, 488, 4490, 1, 0, 6, 1283, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6392, 6135, 6333, 5994, 5956, 6924, 5150, 6268] avg=6144.0000 max=6924 max_rank=5 min=5150 min_rank=6 imbalance_ratio=1.126953 max_load_reduction=42228 max_load_reduction_pct=0.859131 moved_to_echo=24298 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 60, 0, 0, 0, 0, 274, 195, 3016, 2432, 0, 0, 0, 0, 0, 0, 92, 5793, 204, 0, 0, 92, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 271, 0, 277, 0, 0, 5668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 5273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 5910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 3674, 92, 0, 283, 0, 1100, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 1786, 999, 150, 0, 0, 0, 0, 0, 0, 0, 456, 4649, 0, 0, 0, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6257, 6240, 6133, 6095, 5948, 6398, 6104] avg=6144.0000 max=6398 max_rank=6 min=5948 min_rank=5 imbalance_ratio=1.041341 max_load_reduction=42754 max_load_reduction_pct=0.869832 moved_to_echo=24545 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 268, 0, 1, 1, 1, 331, 7, 3025, 2437, 0, 0, 0, 0, 0, 0, 66, 5660, 331, 0, 0, 67, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 77, 1, 226, 1, 0, 5686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 5455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 419, 5947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 4100, 82, 0, 78, 0, 1107, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 0, 0, 1848, 977, 150, 0, 0, 0, 0, 0, 0, 0, 380, 4663, 0, 0, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6071, 6199, 6022, 6223, 6367, 6125, 6395, 5750] avg=6144.0000 max=6395 max_rank=6 min=5750 min_rank=7 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 3, 5, 5, 4, 275, 392, 3019, 2431, 0, 0, 0, 0, 0, 0, 90, 5780, 47, 0, 0, 90, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 148, 4, 194, 3, 0, 5666, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 954, 5179, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 335, 5869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 3606, 129, 0, 278, 0, 1098, 710, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3557, 0, 0, 1758, 1003, 154, 0, 0, 0, 0, 0, 0, 0, 465, 4640, 0, 0, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 6107, 6026, 6138, 6209, 5913, 6473, 6075] avg=6144.0000 max=6473 max_rank=6 min=5913 min_rank=5 imbalance_ratio=1.053548 max_load_reduction=42679 max_load_reduction_pct=0.868306 moved_to_echo=24670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 119, 51, 78, 92, 50, 301, 384, 2913, 2404, 0, 0, 0, 0, 0, 0, 79, 5657, 196, 0, 0, 105, 5, 93, 0, 0, 0, 0, 0, 0, 0, 0, 57, 16, 0, 246, 82, 256, 67, 0, 5609, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 929, 4982, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 87, 0, 0, 593, 5267, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 2, 2, 4391, 417, 8, 354, 1, 980, 690, 0, 0, 0, 0, 0, 0, 30, 0, 1, 0, 1, 2310, 0, 0, 1687, 976, 145, 0, 0, 0, 0, 0, 0, 0, 488, 4490, 1, 0, 6, 1283, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6392, 6135, 6333, 5994, 5956, 6924, 5150, 6268] avg=6144.0000 max=6924 max_rank=5 min=5150 min_rank=6 imbalance_ratio=1.126953 max_load_reduction=42228 max_load_reduction_pct=0.859131 moved_to_echo=24298 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 0, 0, 0, 0, 214, 125, 3026, 2432, 0, 0, 0, 0, 0, 0, 86, 5775, 184, 0, 0, 86, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 107, 0, 187, 0, 0, 5674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 5499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 5896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 3725, 97, 0, 271, 0, 1098, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 0, 0, 1863, 996, 151, 0, 0, 0, 0, 0, 0, 0, 412, 4665, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5865, 6209, 5970, 6358, 6271, 5983, 6581, 5915] avg=6144.0000 max=6581 max_rank=6 min=5865 min_rank=0 imbalance_ratio=1.071126 max_load_reduction=42571 max_load_reduction_pct=0.866109 moved_to_echo=24750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 279, 200, 93, 282, 52, 447, 540, 2484, 2236, 0, 0, 0, 0, 0, 0, 87, 4866, 268, 13, 39, 160, 112, 225, 0, 0, 0, 0, 0, 0, 0, 0, 73, 21, 0, 470, 165, 452, 186, 0, 5215, 0, 0, 0, 0, 0, 0, 0, 48, 115, 133, 69, 1131, 4094, 240, 3, 0, 0, 0, 0, 0, 0, 0, 0, 256, 197, 0, 41, 696, 4484, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 14, 105, 3842, 1014, 23, 622, 37, 834, 594, 0, 0, 0, 0, 0, 0, 127, 23, 58, 5, 16, 1591, 5, 0, 1387, 839, 124, 0, 0, 0, 0, 0, 179, 2, 718, 3829, 24, 32, 28, 2463, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6613, 5770, 6582, 5833, 5725, 7179, 4175, 7275] avg=6144.0000 max=7275 max_rank=7 min=4175 min_rank=6 imbalance_ratio=1.184082 max_load_reduction=41877 max_load_reduction_pct=0.851990 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 208, 0, 0, 0, 0, 278, 44, 3043, 2441, 0, 0, 0, 0, 0, 0, 62, 5723, 340, 0, 0, 62, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 39, 0, 184, 0, 0, 5689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 5496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 5980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 3761, 71, 0, 78, 0, 1113, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 1861, 987, 151, 0, 0, 0, 0, 0, 0, 0, 389, 4690, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 6239, 5951, 6179, 6385, 5783, 6850, 5751] avg=6144.0000 max=6850 max_rank=6 min=5751 min_rank=7 imbalance_ratio=1.114909 max_load_reduction=42302 max_load_reduction_pct=0.860636 moved_to_echo=24769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 3, 5, 5, 4, 275, 392, 3019, 2431, 0, 0, 0, 0, 0, 0, 90, 5780, 47, 0, 0, 90, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 148, 4, 194, 3, 0, 5666, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 954, 5179, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 335, 5869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 3606, 129, 0, 278, 0, 1098, 710, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3557, 0, 0, 1758, 1003, 154, 0, 0, 0, 0, 0, 0, 0, 465, 4640, 0, 0, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 6107, 6026, 6138, 6209, 5913, 6473, 6075] avg=6144.0000 max=6473 max_rank=6 min=5913 min_rank=5 imbalance_ratio=1.053548 max_load_reduction=42679 max_load_reduction_pct=0.868306 moved_to_echo=24670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 1280, 256, 256, 384, 128, 2432, 1920, 23552, 19328, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[256, 128, 128, 128, 128, 25984, 128, 0, 14080, 7808, 1280, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 128, 128, 30208, 2048, 128, 2176, 128, 8448, 5504, 0, 0, 0, 0, 0, 0] @@ -12609,11 +12609,11 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True - [2026-08-26 15:39:25] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 810.2 | throughput per GPU (TFLOP/s/GPU): 157.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.509861E+00 | loss scale: 1.0 | grad norm: 83.626 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 15:39:25 -[rank0]:[W826 15:39:26.665358424 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 15:39:31.192000 113834 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 15:39:31.192000 113834 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 113932 closing signal SIGTERM + [2026-08-26 16:35:27] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 811.6 | throughput per GPU (TFLOP/s/GPU): 157.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.509861E+00 | loss scale: 1.0 | grad norm: 83.626 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:35:27 +[rank0]:[W826 16:35:29.232097162 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 16:35:33.759000 233895 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 16:35:33.760000 233895 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 233992 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -12641,4 +12641,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 113834 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 233895 got signal: 15 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..c4027f8b8b12cddbc00fa5bc2a93c73c7b267b76 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,248 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 27 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 47803 +1, 47803 +2, 47803 +3, 47803 +4, 47515 +5, 47803 +6, 47803 +7, 47803 +0, 48483 +1, 48483 +2, 48483 +3, 48483 +4, 48373 +5, 48661 +6, 48661 +7, 48661 +0, 49539 +1, 49539 +2, 49539 +3, 49539 +4, 49251 +5, 49539 +6, 49539 +7, 49539 +0, 53697 +1, 53697 +2, 53697 +3, 53697 +4, 53409 +5, 53697 +6, 53697 +7, 53697 +0, 54105 +1, 54105 +2, 54105 +3, 54105 +4, 53817 +5, 54105 +6, 54105 +7, 54105 +0, 54421 +1, 54409 +2, 54421 +3, 54459 +4, 54185 +5, 54431 +6, 54413 +7, 54443 +0, 55543 +1, 55543 +2, 55543 +3, 55543 +4, 55255 +5, 55543 +6, 55543 +7, 55543 +0, 56183 +1, 56183 +2, 56183 +3, 56183 +4, 55895 +5, 56183 +6, 56183 +7, 56183 +0, 57321 +1, 57321 +2, 57321 +3, 57321 +4, 57033 +5, 57321 +6, 57321 +7, 57321 +0, 58095 +1, 58095 +2, 58095 +3, 58095 +4, 57807 +5, 58095 +6, 58095 +7, 58095 +0, 98719 +1, 98761 +2, 98761 +3, 98761 +4, 98731 +5, 98761 +6, 98761 +7, 98761 +0, 99365 +1, 99407 +2, 99407 +3, 99407 +4, 99377 +5, 99407 +6, 99407 +7, 99407 +0, 99367 +1, 99409 +2, 99409 +3, 99409 +4, 99379 +5, 99409 +6, 99409 +7, 99409 +0, 99367 +1, 99409 +2, 99409 +3, 99409 +4, 99379 +5, 99409 +6, 99409 +7, 99409 +0, 99367 +1, 99409 +2, 99409 +3, 99409 +4, 99379 +5, 99409 +6, 99409 +7, 99409 +0, 100731 +1, 101157 +2, 101157 +3, 101157 +4, 101127 +5, 101157 +6, 101157 +7, 101157 +0, 106371 +1, 105621 +2, 106653 +3, 106125 +4, 106335 +5, 106557 +6, 106749 +7, 107181 +0, 108169 +1, 107891 +2, 108169 +3, 107837 +4, 108269 +5, 108295 +6, 107829 +7, 108181 +0, 134563 +1, 134329 +2, 134713 +3, 134329 +4, 134627 +5, 134713 +6, 134329 +7, 134713 +0, 134563 +1, 134329 +2, 134713 +3, 134329 +4, 134627 +5, 134713 +6, 134329 +7, 134713 +0, 5886 +1, 134327 +2, 134711 +3, 134327 +4, 134625 +5, 134711 +6, 134327 +7, 134711 +0, 3963 +1, 4285 +2, 134711 +3, 5566 +4, 3966 +5, 3966 +6, 3966 +7, 3966 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/run.env b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..dfb091f4b12ab126cd6dc2566d05360350a6e7fe --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ac6a6317ac2bc1c926e03ca77d6c53026d9b13e7 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/train.log @@ -0,0 +1,7182 @@ +[16:27:23] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[16:27:23] WARN: token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0826 16:27:25.303000 216687 torch/distributed/run.py:766] +W0826 16:27:25.303000 216687 torch/distributed/run.py:766] ***************************************** +W0826 16:27:25.303000 216687 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:27:25.303000 216687 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 12 is connected to 4 is connected to 7 is connected to 7 peer ranks. 7 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : +7[Gloo] Rank 7 +[Gloo] Rank [Gloo] Rank +635 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 + + +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0[Gloo] Rank peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 000 is connected to + is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[rank5]:[W826 16:27:43.221633302 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 16:27:43.224130253 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 16:27:43.224154395 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:27:43.227412108 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:27:43.229811367 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 16:27:43.229847204 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 16:27:43.231374458 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 16:27:43.392486608 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.640 seconds +time to initialize megatron (seconds): 7.089 +[after megatron is initialized] datetime: 2026-08-26 16:27:48 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4898003968 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.k_layernorm.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:27:49 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 16:27:49 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (641.33, 664.39) + train/valid/test-data-iterators-setup ..........: (57.19, 64.84) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 16:27:49 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3582, 65, 17, 348, 173, 720, 6, 38, 93, 234, 1681, 554, 1835, 93, 233, 407, 142, 1767, 156, 15, 766, 17, 101, 1975, 1326, 75, 34, 1459, 303, 1912, 104, 216, 594, 1910, 110, 39, 109, 360, 58, 49, 1286, 52, 559, 2506, 31, 11, 243, 627, 184, 103, 0, 37, 1, 344, 78, 32, 81, 26, 23, 445, 1958, 51, 365, 198, 2233, 9, 3, 6, 1086, 21, 40, 1186, 763, 243, 1978, 168, 302, 7, 660, 925, 16, 10, 6, 46, 23, 2, 1051, 620, 14, 48, 1267, 4, 24, 145, 535, 142, 956, 143, 119, 2025, 93, 1, 53, 92, 209, 217, 753, 4732, 22, 151, 67, 75, 978, 43, 46, 88, 0, 405, 158, 486, 1208, 85, 1268, 431, 301, 219, 3, 3609] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3275, 77, 6, 177, 143, 360, 15, 19, 62, 166, 1593, 1553, 2007, 122, 228, 400, 246, 1894, 279, 10, 678, 29, 99, 1122, 518, 137, 56, 2124, 1089, 1664, 33, 110, 943, 2147, 54, 28, 94, 314, 68, 11, 745, 90, 518, 2679, 47, 21, 192, 487, 109, 347, 1, 61, 8, 693, 29, 8, 32, 24, 50, 523, 2127, 50, 559, 37, 1993, 45, 6, 5, 1362, 35, 12, 1426, 967, 121, 2164, 75, 535, 6, 627, 616, 46, 7, 0, 26, 191, 18, 568, 833, 16, 20, 925, 3, 12, 191, 265, 109, 1284, 218, 98, 3389, 66, 0, 41, 126, 759, 87, 349, 3809, 72, 111, 39, 68, 258, 23, 13, 40, 3, 359, 150, 626, 1393, 76, 1267, 386, 357, 121, 3, 3333] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3319, 91, 9, 325, 141, 771, 10, 0, 113, 131, 1583, 574, 1908, 98, 291, 400, 141, 1705, 44, 11, 526, 73, 84, 1479, 991, 46, 37, 1484, 600, 1555, 38, 361, 613, 2423, 39, 56, 153, 263, 95, 37, 1024, 35, 915, 2986, 44, 2, 246, 1124, 281, 240, 1, 18, 3, 576, 40, 30, 25, 37, 47, 586, 1487, 17, 275, 149, 2130, 74, 13, 2, 1225, 15, 54, 1289, 715, 56, 2134, 152, 477, 2, 779, 1073, 45, 21, 2, 34, 38, 3, 657, 745, 18, 56, 1257, 6, 64, 194, 280, 124, 954, 126, 99, 1887, 72, 13, 71, 122, 338, 133, 579, 5356, 46, 166, 60, 69, 339, 75, 5, 33, 11, 515, 109, 306, 1787, 70, 1401, 352, 186, 418, 3, 3295] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2556, 42, 35, 461, 182, 450, 13, 35, 64, 376, 2194, 396, 1708, 186, 108, 858, 303, 1598, 273, 11, 372, 22, 43, 1516, 794, 17, 33, 1125, 451, 2609, 106, 431, 707, 1920, 113, 21, 105, 709, 184, 163, 904, 31, 727, 2372, 48, 26, 191, 651, 282, 290, 7, 29, 5, 238, 118, 61, 192, 35, 34, 463, 1327, 32, 595, 411, 2335, 15, 6, 11, 1851, 5, 145, 1681, 1183, 218, 1290, 394, 290, 12, 543, 713, 27, 26, 3, 50, 82, 4, 722, 1085, 68, 60, 784, 10, 50, 298, 226, 124, 839, 179, 99, 1323, 158, 32, 190, 60, 360, 307, 584, 4653, 22, 198, 126, 25, 820, 36, 33, 57, 5, 1137, 216, 182, 809, 74, 1406, 416, 267, 185, 8, 4060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2665, 133, 14, 471, 269, 921, 14, 52, 65, 336, 1992, 615, 1704, 128, 116, 437, 47, 1632, 291, 3, 442, 11, 160, 2050, 728, 41, 144, 1375, 716, 2000, 107, 271, 480, 1912, 60, 37, 102, 501, 132, 41, 1399, 67, 766, 2873, 85, 57, 156, 1053, 444, 246, 2, 44, 2, 326, 97, 16, 203, 15, 27, 598, 1585, 41, 434, 342, 2007, 8, 1, 12, 1186, 8, 83, 1486, 1110, 213, 1137, 96, 316, 19, 485, 778, 46, 2, 2, 57, 88, 24, 486, 839, 32, 43, 674, 2, 17, 340, 254, 78, 1434, 115, 34, 1751, 122, 0, 87, 102, 323, 122, 746, 4321, 4, 124, 80, 83, 903, 82, 67, 46, 8, 689, 261, 399, 909, 69, 1362, 557, 120, 550, 17, 4027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2774, 81, 20, 440, 113, 825, 1, 7, 136, 273, 1522, 352, 1570, 94, 177, 387, 235, 1345, 284, 33, 919, 36, 29, 1509, 489, 70, 25, 1199, 452, 1511, 12, 376, 878, 2834, 125, 78, 175, 272, 175, 99, 1378, 40, 576, 2598, 65, 9, 257, 644, 184, 315, 3, 56, 9, 660, 54, 34, 188, 28, 21, 756, 1127, 28, 279, 236, 2031, 54, 1, 19, 1611, 7, 76, 1269, 1083, 94, 1524, 249, 482, 5, 368, 923, 23, 18, 2, 51, 52, 11, 519, 705, 39, 18, 862, 9, 112, 282, 241, 92, 2628, 39, 39, 1753, 126, 7, 88, 169, 431, 338, 490, 4414, 32, 231, 104, 65, 1090, 24, 28, 48, 8, 783, 114, 301, 1128, 63, 1940, 366, 244, 369, 12, 3755] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3419, 64, 28, 251, 137, 848, 33, 12, 76, 106, 1605, 391, 1987, 36, 318, 356, 166, 2384, 247, 50, 858, 11, 90, 1556, 781, 75, 13, 1523, 630, 1004, 48, 475, 733, 2475, 39, 61, 153, 188, 102, 42, 857, 33, 447, 3171, 69, 18, 216, 775, 179, 153, 0, 131, 1, 297, 102, 27, 105, 40, 56, 854, 1205, 22, 184, 346, 1707, 36, 2, 36, 2173, 1, 57, 967, 1056, 103, 1069, 219, 529, 22, 800, 1137, 39, 71, 10, 38, 84, 13, 1114, 765, 4, 45, 998, 5, 30, 137, 359, 119, 1929, 240, 33, 2444, 41, 1, 81, 127, 311, 147, 585, 4953, 244, 217, 66, 64, 797, 128, 64, 44, 3, 621, 94, 235, 1047, 107, 1257, 386, 289, 307, 13, 2257] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3717, 45, 34, 300, 95, 653, 52, 51, 104, 192, 1954, 454, 2109, 99, 125, 334, 421, 1669, 139, 48, 884, 11, 59, 1539, 1204, 46, 11, 1777, 700, 2391, 70, 239, 782, 1437, 128, 12, 147, 177, 94, 45, 626, 24, 911, 3248, 27, 0, 201, 749, 265, 152, 6, 29, 1, 164, 75, 47, 55, 33, 48, 504, 1107, 40, 260, 135, 2559, 19, 1, 15, 1338, 7, 43, 1005, 1350, 92, 1655, 231, 451, 10, 624, 918, 107, 20, 8, 54, 19, 11, 1470, 689, 17, 40, 1188, 53, 37, 117, 145, 28, 1156, 254, 48, 1829, 139, 4, 28, 65, 223, 134, 710, 4519, 63, 123, 56, 17, 829, 28, 23, 16, 27, 895, 161, 162, 1302, 83, 1006, 435, 211, 188, 22, 3404] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1762, 65, 17, 348, 173, 720, 6, 38, 93, 234, 1681, 554, 1835, 93, 233, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 1767, 156, 15, 766, 17, 101, 1975, 1326, 75, 34, 1459, 303, 155, 104, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 1910, 110, 39, 109, 360, 58, 49, 1286, 52, 559, 1581, 31, 11, 243, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 103, 0, 37, 1, 344, 78, 32, 81, 26, 23, 445, 1958, 51, 365, 198, 1820, 925, 988, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 9, 3, 6, 1086, 21, 40, 1186, 763, 243, 1978, 168, 302, 7, 660, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 10, 6, 46, 23, 2, 1051, 620, 14, 48, 1267, 4, 24, 145, 535, 142, 1955, 1832, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 143, 119, 2025, 93, 1, 53, 92, 209, 217, 753, 2777, 22, 151, 67, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 43, 46, 88, 0, 405, 158, 486, 1208, 85, 1268, 431, 301, 219, 3, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8611, 7619, 8345, 7798, 8811, 7753, 8340] avg=8192.0000 max=8811 max_rank=5 min=7619 min_rank=2 imbalance_ratio=1.075562 max_load_reduction=56725 max_load_reduction_pct=0.865555 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1823, 45, 34, 300, 95, 653, 52, 51, 104, 192, 1954, 454, 2109, 99, 125, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 1669, 139, 48, 884, 11, 59, 1539, 1204, 46, 11, 1777, 700, 189, 70, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 1437, 128, 12, 147, 177, 94, 45, 626, 24, 911, 2044, 27, 0, 201, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 152, 6, 29, 1, 164, 75, 47, 55, 33, 48, 504, 1107, 40, 260, 135, 1894, 1204, 935, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 19, 1, 15, 1338, 7, 43, 1005, 1350, 92, 1655, 231, 451, 10, 624, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 20, 8, 54, 19, 11, 1470, 689, 17, 40, 1188, 53, 37, 117, 145, 28, 1871, 2103, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 254, 48, 1829, 139, 4, 28, 65, 223, 134, 710, 2648, 63, 123, 56, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 28, 23, 16, 27, 895, 161, 162, 1302, 83, 1006, 435, 211, 188, 22, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8424, 9006, 7404, 7814, 8215, 9319, 7497, 7857] avg=8192.0000 max=9319 max_rank=5 min=7404 min_rank=2 imbalance_ratio=1.137573 max_load_reduction=56217 max_load_reduction_pct=0.857803 moved_to_echo=32888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1633, 91, 9, 325, 141, 771, 10, 0, 113, 131, 1583, 574, 1908, 98, 291, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 1705, 44, 11, 526, 73, 84, 1479, 991, 46, 37, 1484, 600, 127, 38, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 2423, 39, 56, 153, 263, 95, 37, 1024, 35, 915, 1884, 44, 2, 246, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 240, 1, 18, 3, 576, 40, 30, 25, 37, 47, 586, 1487, 17, 275, 149, 1686, 1102, 902, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 74, 13, 2, 1225, 15, 54, 1289, 715, 56, 2134, 152, 477, 2, 779, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 21, 2, 34, 38, 3, 657, 745, 18, 56, 1257, 6, 64, 194, 280, 124, 2212, 1747, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 126, 99, 1887, 72, 13, 71, 122, 338, 133, 579, 3144, 46, 166, 60, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 75, 5, 33, 11, 515, 109, 306, 1787, 70, 1401, 352, 186, 418, 3, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 7747, 8953, 8059, 8443, 8374, 7879, 8003] avg=8192.0000 max=8953 max_rank=2 min=7747 min_rank=1 imbalance_ratio=1.092896 max_load_reduction=56583 max_load_reduction_pct=0.863388 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1365, 81, 20, 440, 113, 825, 1, 7, 136, 273, 1522, 352, 1570, 94, 177, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 1345, 284, 33, 919, 36, 29, 1509, 489, 70, 25, 1199, 452, 123, 12, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 2834, 125, 78, 175, 272, 175, 99, 1378, 40, 576, 1639, 65, 9, 257, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 315, 3, 56, 9, 660, 54, 34, 188, 28, 21, 756, 1127, 28, 279, 236, 1409, 959, 1028, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 54, 1, 19, 1611, 7, 76, 1269, 1083, 94, 1524, 249, 482, 5, 368, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 18, 2, 51, 52, 11, 519, 705, 39, 18, 862, 9, 112, 282, 241, 92, 1823, 1666, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 39, 39, 1753, 126, 7, 88, 169, 431, 338, 490, 2591, 32, 231, 104, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 24, 28, 48, 8, 783, 114, 301, 1128, 63, 1940, 366, 244, 369, 12, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7363, 7136, 9244, 7916, 8130, 7371, 9131, 9245] avg=8192.0000 max=9245 max_rank=7 min=7136 min_rank=1 imbalance_ratio=1.128540 max_load_reduction=56291 max_load_reduction_pct=0.858932 moved_to_echo=33877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1611, 77, 6, 177, 143, 360, 15, 19, 62, 166, 1593, 1553, 2007, 122, 228, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1894, 279, 10, 678, 29, 99, 1122, 518, 137, 56, 2124, 1089, 135, 33, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 2147, 54, 28, 94, 314, 68, 11, 745, 90, 518, 1690, 47, 21, 192, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 347, 1, 61, 8, 693, 29, 8, 32, 24, 50, 523, 2127, 50, 559, 37, 1664, 989, 912, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 45, 6, 5, 1362, 35, 12, 1426, 967, 121, 2164, 75, 535, 6, 627, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 7, 0, 26, 191, 18, 568, 833, 16, 20, 925, 3, 12, 191, 265, 109, 1573, 1635, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 218, 98, 3389, 66, 0, 41, 126, 759, 87, 349, 2236, 72, 111, 39, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 23, 13, 40, 3, 359, 150, 626, 1393, 76, 1267, 386, 357, 121, 3, 2421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8539, 8559, 7449, 8820, 8360, 7370, 8943, 7496] avg=8192.0000 max=8943 max_rank=6 min=7370 min_rank=5 imbalance_ratio=1.091675 max_load_reduction=56593 max_load_reduction_pct=0.863541 moved_to_echo=32169 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1257, 42, 35, 461, 182, 450, 13, 35, 64, 376, 2194, 396, 1708, 186, 108, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 1598, 273, 11, 372, 22, 43, 1516, 794, 17, 33, 1125, 451, 212, 106, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 1920, 113, 21, 105, 709, 184, 163, 904, 31, 727, 1497, 48, 26, 191, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 290, 7, 29, 5, 238, 118, 61, 192, 35, 34, 463, 1327, 32, 595, 411, 1299, 875, 1111, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 15, 6, 11, 1851, 5, 145, 1681, 1183, 218, 1290, 394, 290, 12, 543, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 26, 3, 50, 82, 4, 722, 1085, 68, 60, 784, 10, 50, 298, 226, 124, 1922, 1915, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 179, 99, 1323, 158, 32, 190, 60, 360, 307, 584, 2731, 22, 198, 126, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 36, 33, 57, 5, 1137, 216, 182, 809, 74, 1406, 416, 267, 185, 8, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 7307, 7997, 8340, 8777, 8917, 7233, 8600] avg=8192.0000 max=8917 max_rank=5 min=7233 min_rank=6 imbalance_ratio=1.088501 max_load_reduction=56619 max_load_reduction_pct=0.863937 moved_to_echo=33527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1311, 133, 14, 471, 269, 921, 14, 52, 65, 336, 1992, 615, 1704, 128, 116, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 1632, 291, 3, 442, 11, 160, 2050, 728, 41, 144, 1375, 716, 163, 107, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 1912, 60, 37, 102, 501, 132, 41, 1399, 67, 766, 1813, 85, 57, 156, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 246, 2, 44, 2, 326, 97, 16, 203, 15, 27, 598, 1585, 41, 434, 342, 1354, 1060, 1102, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 8, 1, 12, 1186, 8, 83, 1486, 1110, 213, 1137, 96, 316, 19, 485, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 2, 2, 57, 88, 24, 486, 839, 32, 43, 674, 2, 17, 340, 254, 78, 1785, 1646, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1434, 115, 34, 1751, 122, 0, 87, 102, 323, 122, 746, 2536, 4, 124, 80, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 82, 67, 46, 8, 689, 261, 399, 909, 69, 1362, 557, 120, 550, 17, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8578, 8181, 8661, 8655, 7299, 7535, 7663, 8964] avg=8192.0000 max=8964 max_rank=7 min=7299 min_rank=4 imbalance_ratio=1.094238 max_load_reduction=56572 max_load_reduction_pct=0.863220 moved_to_echo=31461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1682, 64, 28, 251, 137, 848, 33, 12, 76, 106, 1605, 391, 1987, 36, 318, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 2384, 247, 50, 858, 11, 90, 1556, 781, 75, 13, 1523, 630, 82, 48, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 2475, 39, 61, 153, 188, 102, 42, 857, 33, 447, 2001, 69, 18, 216, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 153, 0, 131, 1, 297, 102, 27, 105, 40, 56, 854, 1205, 22, 184, 346, 1737, 1170, 618, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 36, 2, 36, 2173, 1, 57, 967, 1056, 103, 1069, 219, 529, 22, 800, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 71, 10, 38, 84, 13, 1114, 765, 4, 45, 998, 5, 30, 137, 359, 119, 2046, 1400, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 240, 33, 2444, 41, 1, 81, 127, 311, 147, 585, 2907, 244, 217, 66, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 128, 64, 44, 3, 621, 94, 235, 1047, 107, 1257, 386, 289, 307, 13, 1639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7930, 8989, 8209, 7587, 8514, 7839, 9437, 7031] avg=8192.0000 max=9437 max_rank=6 min=7031 min_rank=7 imbalance_ratio=1.151978 max_load_reduction=56099 max_load_reduction_pct=0.856003 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[384, 256, 128, 384, 640, 128, 6656, 6400, 256, 384, 8064, 128, 384, 1792, 2432, 896, 15232, 13952, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[5760, 17152, 768, 384, 1152, 2816, 1024, 512, 8320, 384, 5504, 14208, 512, 256, 1792, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[2048, 1920, 128, 512, 128, 3328, 640, 256, 896, 256, 384, 4736, 12032, 384, 3072, 1920, 12928, 8320, 7680, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1792, 14080, 1792, 256, 5504, 256, 768, 12800, 6912, 512, 384, 12160, 4992, 1280, 640, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[12544, 640, 256, 2816, 1280, 5632, 256, 256, 768, 1920, 14208, 4992, 14848, 896, 1664, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3072, 384, 128, 128, 11904, 128, 512, 10368, 8320, 1152, 13056, 1664, 3456, 128, 4992, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[11264, 1408, 640, 16512, 896, 128, 640, 896, 3072, 1536, 4864, 21632, 512, 1408, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[6016, 512, 384, 384, 128, 5504, 1280, 2816, 9600, 640, 11008, 3456, 2048, 2432, 128, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4, 814, 0, 2, 0, 0, 44, 62, 0, 0, 7372, 0, 0, 2024, 1, 1253, 0, 0, 29, 45, 0, 38, 0, 0, 0, 0, 2, 290, 0, 1, 1, 10, 1, 22, 0, 1, 3, 0, 0, 0, 0, 3706, 0, 4, 4, 0, 1, 0, 3, 0, 0, 4, 7678, 0, 0, 3976, 0, 2454, 1356, 1995, 0, 2, 93, 4201, 0, 0, 0, 109, 119, 7847, 0, 1, 71, 53, 0, 107, 0, 3, 201, 0, 4, 2491, 108, 0, 2, 3, 440, 0, 6177, 10, 2119, 2, 4, 0, 12, 365, 0, 0, 0, 0, 18, 1843, 0, 1, 0, 24, 38, 304, 30, 0, 0, 0, 0, 0, 0, 0, 33, 0, 513, 2, 0, 0, 20, 0, 4711, 0, 0, 255] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1331, 242, 0, 6709, 17, 0, 28, 2, 0, 480, 2779, 0, 0, 7, 0, 1, 0, 1, 0, 2008, 0, 0, 12, 0, 0, 0, 22, 119, 0, 0, 24, 101, 0, 0, 1, 0, 18, 243, 0, 0, 0, 2773, 4, 3, 0, 167, 0, 0, 0, 145, 0, 0, 681, 1, 202, 7580, 0, 7187, 550, 42, 0, 1664, 0, 2132, 224, 1, 0, 9, 1446, 8168, 10, 1, 0, 0, 0, 190, 0, 13, 20, 0, 0, 28, 0, 0, 0, 0, 0, 0, 5109, 0, 904, 0, 9, 0, 2149, 32, 0, 0, 9, 35, 0, 93, 0, 174, 28, 0, 0, 153, 0, 216, 165, 0, 0, 12, 0, 1, 667, 1, 138, 79, 0, 0, 196, 0, 7979, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[324, 775, 0, 3049, 0, 0, 2, 7, 0, 168, 8068, 0, 0, 48, 0, 1, 0, 25, 2, 42, 3, 0, 24, 0, 0, 0, 1, 443, 5, 0, 1, 7, 0, 0, 0, 0, 0, 99, 1, 1, 0, 31, 22, 23, 150, 0, 4, 1, 1, 11, 0, 0, 1265, 0, 3, 6109, 0, 242, 5403, 0, 0, 92, 0, 571, 19, 0, 0, 0, 565, 7899, 46, 0, 0, 0, 0, 68, 0, 0, 3, 0, 0, 43, 154, 573, 37, 0, 53, 0, 5181, 0, 3309, 0, 42, 0, 6072, 15, 0, 0, 0, 266, 0, 5079, 0, 0, 3, 26, 27, 0, 0, 612, 0, 0, 0, 26, 10, 0, 2, 0, 1458, 27, 0, 0, 3, 0, 6894, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[688, 184, 0, 2055, 0, 0, 5, 107, 0, 0, 3547, 0, 0, 269, 0, 61, 0, 109, 0, 1, 16, 0, 22, 0, 137, 4, 443, 700, 76, 1, 165, 119, 30, 0, 0, 1, 6, 0, 1, 0, 134, 0, 302, 1, 2, 0, 0, 4, 7, 969, 0, 1, 2935, 0, 6, 3686, 0, 166, 3327, 6612, 0, 36, 23, 5588, 0, 4, 0, 0, 1908, 8002, 95, 36, 3, 10, 0, 10, 3, 0, 3, 0, 0, 51, 126, 225, 0, 23, 29, 0, 7844, 0, 5135, 0, 9, 0, 7, 5, 27, 18, 2, 2, 4, 3366, 1, 0, 40, 0, 32, 422, 90, 0, 14, 0, 11, 4, 0, 0, 10, 2, 1437, 1, 2, 1, 0, 0, 3976, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1144, 22, 2, 944, 0, 0, 7, 29, 0, 12, 7774, 0, 20, 5, 0, 99, 0, 3, 151, 911, 0, 0, 0, 0, 38, 0, 23, 57, 3, 0, 24, 248, 0, 0, 3, 15, 98, 3, 0, 0, 4, 262, 42, 25, 5, 184, 0, 0, 14, 23, 0, 0, 4475, 1, 310, 6856, 0, 1970, 362, 155, 0, 2834, 69, 869, 18, 0, 0, 0, 1314, 8136, 0, 0, 0, 52, 0, 66, 0, 17, 1, 0, 0, 143, 5, 0, 0, 0, 29, 0, 7982, 1, 871, 0, 57, 7, 1674, 284, 1, 36, 0, 0, 15, 486, 0, 0, 0, 0, 180, 2, 3, 0, 0, 0, 1, 166, 0, 0, 58, 0, 5496, 158, 6, 0, 1, 0, 8108, 67, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[171, 212, 0, 647, 0, 0, 0, 0, 0, 0, 7648, 0, 2, 0, 0, 5, 5, 193, 97, 524, 0, 0, 1862, 0, 5, 0, 48, 5753, 2, 0, 0, 1, 194, 0, 0, 0, 32, 26, 0, 1, 148, 6, 24, 0, 239, 9, 3, 0, 0, 86, 49, 0, 780, 66, 212, 5369, 3, 287, 438, 15, 0, 516, 0, 1486, 0, 0, 0, 0, 1578, 6956, 9, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 283, 796, 4, 1, 0, 1876, 2, 8169, 0, 1026, 1, 373, 1, 6852, 16, 0, 0, 0, 0, 0, 1949, 0, 0, 98, 18, 180, 19, 1, 360, 64, 2, 0, 0, 1, 2, 0, 0, 40, 0, 0, 11, 6, 0, 7650, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1429, 392, 0, 7003, 12, 0, 3, 1, 34, 86, 6511, 0, 1, 63, 0, 0, 0, 0, 216, 355, 0, 1, 155, 0, 87, 1, 148, 2942, 6, 0, 1, 0, 12, 1, 0, 0, 1, 1555, 0, 6, 0, 42, 5, 2, 4, 15, 0, 0, 0, 283, 4, 9, 1365, 7, 27, 6376, 2, 271, 102, 1, 0, 15, 0, 597, 13, 0, 3, 0, 1450, 8182, 0, 0, 2, 0, 0, 743, 0, 0, 1, 0, 0, 1, 10, 2, 0, 0, 0, 11, 5948, 0, 915, 10, 17, 0, 2610, 5, 0, 0, 0, 0, 5, 2277, 12, 1, 3, 8, 0, 15, 1, 10, 10, 7, 0, 80, 0, 0, 0, 1, 7628, 0, 0, 1, 7, 0, 5390, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4190, 9, 13, 3708, 1, 0, 0, 2, 0, 2, 8052, 2, 0, 0, 1, 0, 0, 2, 0, 955, 0, 0, 0, 0, 1, 0, 50, 33, 10, 3, 34, 9, 0, 0, 1, 0, 0, 1, 6, 1, 3, 0, 0, 3, 1910, 0, 4, 0, 2, 1, 0, 0, 4502, 3, 156, 45, 0, 42, 1427, 4, 0, 143, 0, 2777, 15, 0, 0, 2, 75, 7645, 9, 1, 0, 0, 0, 0, 22, 0, 0, 57, 0, 13, 144, 5, 17, 183, 1012, 4, 7976, 0, 2294, 9, 41, 2, 2049, 743, 1, 0, 0, 0, 0, 5175, 0, 0, 0, 81, 1, 0, 0, 22, 0, 0, 0, 201, 20, 2, 1, 0, 1727, 0, 0, 0, 15, 25, 7821, 18, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[688, 184, 0, 2055, 0, 0, 5, 107, 0, 0, 1681, 0, 0, 269, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 1, 16, 0, 22, 0, 137, 4, 443, 700, 76, 1, 165, 119, 3599, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 1, 6, 0, 1, 0, 134, 0, 302, 1, 2, 0, 0, 4, 3686, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 969, 0, 1, 597, 0, 6, 0, 0, 166, 3327, 6612, 0, 36, 23, 5588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1908, 6968, 95, 36, 3, 10, 0, 10, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 126, 225, 0, 23, 29, 0, 2388, 0, 5135, 0, 9, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 18, 2, 2, 4, 3366, 1, 0, 40, 0, 32, 422, 90, 0, 14, 0, 2338, 1034, 579, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4, 0, 0, 10, 2, 1437, 1, 2, 1, 0, 0, 3397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5050, 6801, 6024, 17332, 9040, 7998, 8426, 4865] avg=8192.0000 max=17332 max_rank=3 min=4865 min_rank=7 imbalance_ratio=2.115723 max_load_reduction=48204 max_load_reduction_pct=0.735535 moved_to_echo=30329 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[324, 775, 0, 3049, 0, 0, 2, 7, 0, 168, 3820, 0, 0, 48, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2, 42, 3, 0, 24, 0, 0, 0, 1, 443, 5, 0, 1, 7, 2377, 3207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 1, 1, 0, 31, 22, 23, 150, 0, 4, 1, 6109, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 257, 0, 3, 0, 0, 242, 5403, 0, 0, 92, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 565, 6878, 46, 0, 0, 0, 0, 68, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 154, 573, 37, 0, 53, 0, 1578, 0, 3309, 0, 42, 0, 6072, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 5079, 0, 0, 3, 26, 27, 0, 0, 612, 0, 0, 1008, 1021, 1004, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 10, 0, 2, 0, 1458, 27, 0, 0, 3, 0, 5890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6137, 7667, 6580, 7579, 11876, 10087, 7416] avg=8192.0000 max=11876 max_rank=5 min=6137 min_rank=1 imbalance_ratio=1.449707 max_load_reduction=53660 max_load_reduction_pct=0.818787 moved_to_echo=36958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 814, 0, 2, 0, 0, 44, 62, 0, 0, 3491, 0, 0, 2024, 1, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 45, 0, 38, 0, 0, 0, 0, 2, 290, 0, 1, 1, 10, 2834, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 22, 0, 1, 3, 0, 0, 0, 0, 3706, 0, 4, 4, 0, 1, 0, 3976, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 1560, 0, 0, 0, 0, 2454, 1356, 1995, 0, 2, 93, 4201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 119, 6833, 0, 1, 71, 53, 0, 107, 0, 3, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2491, 108, 0, 2, 3, 440, 0, 1881, 10, 2119, 2, 4, 0, 12, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1843, 0, 1, 0, 24, 38, 304, 30, 0, 0, 0, 6118, 1014, 686, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 513, 2, 0, 0, 20, 0, 4025, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7695, 6180, 9180, 11668, 7497, 7441, 11027, 4848] avg=8192.0000 max=11668 max_rank=3 min=4848 min_rank=7 imbalance_ratio=1.424316 max_load_reduction=53868 max_load_reduction_pct=0.821960 moved_to_echo=30813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1429, 392, 0, 7003, 12, 0, 3, 1, 34, 86, 3083, 0, 1, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 355, 0, 1, 155, 0, 87, 1, 148, 2942, 6, 0, 1, 0, 2729, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 1, 1555, 0, 6, 0, 42, 5, 2, 4, 15, 0, 0, 6376, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 4, 9, 278, 7, 27, 0, 2, 271, 102, 1, 0, 15, 0, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 0, 1450, 7124, 0, 0, 2, 0, 0, 743, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 2, 0, 0, 0, 11, 1811, 0, 915, 10, 17, 0, 2610, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2277, 12, 1, 3, 8, 0, 15, 1, 10, 10, 7, 1087, 1058, 785, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 1, 7628, 0, 0, 1, 7, 0, 4605, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12107, 9229, 9427, 1596, 9336, 5392, 6119, 12330] avg=8192.0000 max=12330 max_rank=7 min=1596 min_rank=3 imbalance_ratio=1.505127 max_load_reduction=53206 max_load_reduction_pct=0.811859 moved_to_echo=33177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1331, 242, 0, 6709, 17, 0, 28, 2, 0, 480, 1317, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2008, 0, 0, 12, 0, 0, 0, 22, 119, 0, 0, 24, 101, 2344, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 243, 0, 0, 0, 2773, 4, 3, 0, 167, 0, 0, 7580, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 139, 1, 202, 0, 0, 7187, 550, 42, 0, 1664, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1, 0, 9, 1446, 7112, 10, 1, 0, 0, 0, 190, 0, 13, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 1556, 0, 904, 0, 9, 0, 2149, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 35, 0, 93, 0, 174, 28, 0, 0, 153, 0, 216, 165, 0, 542, 1056, 1162, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 667, 1, 138, 79, 0, 0, 196, 0, 6817, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10134, 5735, 11998, 12062, 9026, 4678, 3991, 7912] avg=8192.0000 max=12062 max_rank=3 min=3991 min_rank=6 imbalance_ratio=1.472412 max_load_reduction=53474 max_load_reduction_pct=0.815948 moved_to_echo=25607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4190, 9, 13, 3708, 1, 0, 0, 2, 0, 2, 3805, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 955, 0, 0, 0, 0, 1, 0, 50, 33, 10, 3, 34, 9, 3665, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 6, 1, 3, 0, 0, 3, 1910, 0, 4, 0, 45, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 910, 3, 156, 0, 0, 42, 1427, 4, 0, 143, 0, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 2, 75, 6653, 9, 1, 0, 0, 0, 0, 22, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 144, 5, 17, 183, 1012, 4, 2417, 0, 2294, 9, 41, 2, 2049, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5175, 0, 0, 0, 81, 1, 0, 0, 22, 0, 0, 3592, 992, 1141, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 20, 2, 1, 0, 1727, 0, 0, 0, 15, 25, 6680, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11733, 7965, 3868, 5465, 6834, 8933, 12049, 8689] avg=8192.0000 max=12049 max_rank=6 min=3868 min_rank=2 imbalance_ratio=1.470825 max_load_reduction=53487 max_load_reduction_pct=0.816147 moved_to_echo=36505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1144, 22, 2, 944, 0, 0, 7, 29, 0, 12, 3681, 0, 20, 5, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 151, 911, 0, 0, 0, 0, 38, 0, 23, 57, 3, 0, 24, 248, 3663, 3090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 15, 98, 3, 0, 0, 4, 262, 42, 25, 5, 184, 0, 0, 6856, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 23, 0, 0, 909, 1, 310, 0, 0, 1970, 362, 155, 0, 2834, 69, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1314, 7084, 0, 0, 0, 52, 0, 66, 0, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 5, 0, 0, 0, 29, 0, 2429, 1, 871, 0, 57, 7, 1674, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 0, 0, 15, 486, 0, 0, 0, 0, 180, 2, 3, 0, 0, 0, 3566, 1052, 1181, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 166, 0, 0, 58, 0, 5496, 158, 6, 0, 1, 0, 6927, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 8211, 9387, 7516, 8552, 5500, 7525, 12880] avg=8192.0000 max=12880 max_rank=7 min=5500 min_rank=5 imbalance_ratio=1.572266 max_load_reduction=52656 max_load_reduction_pct=0.803467 moved_to_echo=34457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[171, 212, 0, 647, 0, 0, 0, 0, 0, 0, 3621, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 193, 97, 524, 0, 0, 1862, 0, 5, 0, 48, 5753, 2, 0, 0, 1, 3748, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 32, 26, 0, 1, 148, 6, 24, 0, 239, 9, 3, 0, 5369, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 49, 0, 159, 66, 212, 0, 3, 287, 438, 15, 0, 516, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1578, 6057, 9, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 796, 4, 1, 0, 1876, 2, 2487, 0, 1026, 1, 373, 1, 6852, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 0, 0, 98, 18, 180, 19, 1, 360, 64, 2, 621, 899, 1114, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 40, 0, 0, 11, 6, 0, 6536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4658, 15278, 7985, 3317, 7672, 13718, 6312, 6596] avg=8192.0000 max=15278 max_rank=1 min=3317 min_rank=3 imbalance_ratio=1.864990 max_load_reduction=50258 max_load_reduction_pct=0.766876 moved_to_echo=34298 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 384, 512, 4864, 128, 128, 2176, 0, 384, 128, 768, 10368, 128, 128, 256, 512, 24960, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 384, 128, 20352, 128, 256, 256, 256, 512, 1024, 128, 1280, 256, 128, 18944, 8192, 7680, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 1536, 128, 128, 4864, 128, 1024, 0, 128, 12672, 13056, 8832, 0, 5376, 256, 18304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 3072, 1408, 896, 128, 256, 3456, 128, 16640, 128, 16640, 128, 640, 128, 21504, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[9344, 2688, 128, 24192, 128, 0, 128, 256, 128, 768, 24576, 128, 128, 2432, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 512, 128, 128, 896, 128, 18560, 384, 128, 128, 256, 128, 44928, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[384, 128, 128, 128, 8576, 54784, 256, 128, 128, 128, 0, 1280, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[256, 128, 128, 128, 256, 2048, 128, 128, 384, 6912, 512, 128, 2432, 384, 128, 128, 40064, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3, 17, 0, 0, 0, 8165, 24, 0, 1, 623, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 1, 7, 29, 3, 172, 7, 3, 0, 0, 0, 0, 0, 16, 0, 35, 17, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 3092, 0, 7906, 0, 0, 0, 12, 7106, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2049, 0, 2, 9, 8123, 2, 0, 0, 0, 1620, 0, 0, 0, 0, 0, 9, 2, 0, 39, 2954, 0, 0, 155, 0, 0, 176, 0, 0, 0, 0, 36, 2440, 7437, 0, 0, 0, 0, 3729, 0, 0, 13, 0, 0, 2, 0, 0, 0, 8166, 0, 0, 229, 15, 197, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7713, 1, 7, 0, 0, 3, 159, 0, 0, 0, 0, 0, 0, 812, 3, 0, 0, 43, 3, 0, 14, 0, 0, 0, 185, 0, 0, 0, 0, 27, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8, 171, 0, 0, 0, 0, 0, 550, 0, 7700, 0, 0, 0, 3, 8127, 0, 64, 1, 0, 272, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 7620, 101, 5, 2, 8, 8073, 0, 100, 0, 0, 0, 0, 0, 0, 2038, 0, 0, 3, 3652, 0, 1, 0, 0, 157, 0, 0, 4, 5, 1235, 2, 0, 0, 0, 7865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 119, 0, 471, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 5, 23, 0, 0, 26, 21, 0, 5092, 0, 14, 0, 0, 0, 0, 0, 152, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2, 1, 0, 0, 0, 0, 122, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3158, 0, 21, 0, 8, 7978, 0, 2249, 0, 0, 62, 0, 0, 0, 0, 0, 0, 12, 145, 0, 0, 5422, 1107, 0, 0, 0, 7606, 0, 1839, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 7, 1, 6, 0, 2, 0, 0, 0, 0, 7525, 5, 4688, 0, 85, 0, 0, 8181, 900, 0, 0, 0, 0, 17, 0, 0, 203, 5949, 0, 0, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 66, 0, 0, 7, 68, 0, 0, 0, 0, 31, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4949, 0, 0, 12, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7426, 0, 8145, 0, 0, 0, 1374, 6463, 0, 2564, 0, 0, 11, 0, 0, 0, 1, 0, 0, 6, 0, 0, 4, 8190, 205, 0, 0, 0, 4434, 0, 0, 0, 5, 0, 0, 0, 1, 529, 0, 0, 0, 3407, 0, 3, 19, 0, 0, 0, 0, 1, 0, 737, 0, 2, 0, 0, 6971, 3, 0, 697, 13, 0, 0, 0, 2, 2, 7831, 0, 0, 12, 1, 1263, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[10, 0, 3, 0, 2, 6231, 0, 0, 2, 6132, 0, 0, 0, 7, 65, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 27, 0, 1, 0, 7730, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 0, 1, 0, 0, 5, 0, 0, 1, 21, 0, 1069, 0, 1290, 2, 0, 0, 0, 8148, 0, 6, 348, 0, 0, 0, 3, 0, 44, 0, 0, 0, 0, 0, 43, 7140, 180, 0, 45, 16, 0, 69, 0, 0, 0, 0, 0, 0, 38, 0, 3361, 1, 8, 158, 0, 0, 0, 12, 32, 0, 0, 0, 5392, 4432, 0, 0, 3, 0, 4530, 0, 0, 50, 5, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 623, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[82, 1, 1, 0, 0, 7967, 0, 4, 0, 0, 0, 4, 1, 27, 0, 0, 0, 39, 4364, 30, 11, 0, 23, 0, 11, 0, 1, 0, 0, 66, 2, 26, 212, 0, 14, 0, 2, 0, 0, 0, 0, 17, 0, 3, 0, 0, 9, 0, 11, 313, 0, 0, 0, 0, 0, 1, 0, 7987, 0, 0, 4, 0, 8123, 0, 1974, 0, 0, 76, 5, 0, 0, 0, 0, 0, 4, 0, 2, 0, 7608, 0, 0, 5, 0, 0, 67, 90, 0, 0, 0, 459, 0, 0, 68, 0, 0, 0, 280, 0, 0, 445, 6, 1, 77, 0, 2560, 0, 5914, 0, 0, 7, 0, 7110, 54, 0, 12, 16, 0, 66, 475, 0, 57, 8124, 3, 331, 160, 117, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 8005, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 4, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 382, 10, 1, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 7914, 0, 1458, 0, 0, 69, 0, 5, 0, 2, 0, 0, 0, 0, 32, 0, 8103, 0, 0, 0, 77, 7449, 0, 0, 0, 2, 0, 5, 0, 0, 33, 88, 0, 0, 69, 0, 0, 127, 0, 0, 0, 1, 0, 22, 7126, 12, 0, 0, 7, 8188, 0, 10, 0, 0, 0, 2, 0, 0, 0, 7962, 11, 0, 4, 0, 60, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 6172, 0, 0, 0, 3, 0, 0, 0, 13, 0, 0, 1, 1, 144, 6, 0, 0, 0, 0, 0, 6, 0, 138, 11, 2, 0, 1, 0, 3, 0, 0, 1, 0, 0, 8, 16, 0, 0, 0, 0, 0, 12, 11, 7, 71, 2, 0, 0, 0, 0, 50, 0, 6162, 1, 22, 0, 957, 7581, 0, 2757, 0, 0, 0, 0, 1, 0, 113, 0, 0, 323, 70, 0, 0, 8029, 2, 0, 0, 6, 8128, 0, 0, 0, 0, 24, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 5, 0, 1, 0, 251, 0, 8007, 0, 0, 0, 0, 8018, 2, 0, 0, 0, 0, 0, 243, 0, 0, 8135, 0, 1, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 66, 0, 0, 7, 68, 0, 0, 0, 0, 31, 46, 1263, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 5708, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4949, 0, 0, 12, 0, 1, 0, 0, 0, 6304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7426, 0, 7905, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 0, 11, 0, 0, 0, 1, 0, 0, 6, 0, 0, 4, 6682, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4434, 0, 0, 0, 5, 0, 0, 0, 1, 529, 0, 0, 0, 987, 433, 240, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 3, 19, 0, 0, 0, 0, 1, 0, 715, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 697, 13, 0, 0, 0, 2, 2, 7398, 0, 0, 12, 1, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2014, 5883, 11271, 16706, 9473, 6651, 4147, 9391] avg=8192.0000 max=16706 max_rank=3 min=2014 min_rank=0 imbalance_ratio=2.039307 max_load_reduction=48830 max_load_reduction_pct=0.745087 moved_to_echo=29662 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7713, 1, 7, 0, 0, 3, 159, 0, 0, 0, 0, 1424, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 812, 3, 0, 0, 43, 3, 0, 14, 0, 0, 0, 185, 0, 0, 6441, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 171, 0, 0, 0, 0, 0, 550, 0, 7473, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 272, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 6218, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 8, 8073, 0, 100, 0, 0, 0, 0, 0, 0, 2038, 0, 0, 3, 918, 453, 227, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3652, 0, 1, 0, 0, 157, 0, 0, 4, 5, 1198, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7729, 0, 1, 119, 0, 471, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9791, 7700, 7965, 8205, 6660, 11864, 5019, 8332] avg=8192.0000 max=11864 max_rank=5 min=5019 min_rank=6 imbalance_ratio=1.448242 max_load_reduction=53672 max_load_reduction_pct=0.818970 moved_to_echo=31875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 17, 0, 0, 0, 8165, 24, 0, 1, 623, 0, 0, 0, 0, 0, 0, 675, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 1, 7, 29, 3, 172, 7, 3, 3054, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 35, 17, 0, 0, 0, 0, 0, 0, 0, 6932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 3092, 0, 7673, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2049, 0, 2, 9, 6628, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 0, 0, 0, 0, 9, 2, 0, 39, 2954, 0, 0, 979, 452, 233, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 176, 0, 0, 0, 0, 36, 2440, 7212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2, 0, 0, 0, 7714, 0, 0, 229, 15, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10024, 3459, 7000, 11651, 8700, 6513, 10019, 8170] avg=8192.0000 max=11651 max_rank=3 min=3459 min_rank=1 imbalance_ratio=1.422241 max_load_reduction=53885 max_load_reduction_pct=0.822220 moved_to_echo=33402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[82, 1, 1, 0, 0, 7967, 0, 4, 0, 0, 0, 4, 1, 27, 0, 0, 1288, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4364, 30, 11, 0, 23, 0, 11, 0, 1, 0, 0, 66, 2, 26, 5822, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 14, 0, 2, 0, 0, 0, 0, 17, 0, 3, 0, 0, 9, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 313, 0, 0, 0, 0, 0, 1, 0, 7752, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 0, 0, 76, 5, 0, 0, 0, 0, 0, 4, 0, 2, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 67, 90, 0, 0, 0, 459, 0, 0, 68, 0, 0, 0, 917, 449, 235, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 445, 6, 1, 77, 0, 2560, 0, 5735, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 12, 16, 0, 66, 475, 0, 57, 7675, 3, 331, 160, 117, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9859, 10595, 8180, 8081, 8268, 2469, 9111, 8973] avg=8192.0000 max=10595 max_rank=1 min=2469 min_rank=5 imbalance_ratio=1.293335 max_load_reduction=54941 max_load_reduction_pct=0.838333 moved_to_echo=28821 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 5, 23, 0, 0, 26, 1482, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 5092, 0, 14, 0, 0, 0, 0, 0, 152, 0, 0, 0, 2, 1, 6699, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2, 1, 0, 0, 0, 0, 122, 7781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3065, 0, 21, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, 62, 0, 0, 0, 0, 0, 0, 12, 145, 0, 0, 4425, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 0, 1839, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 7, 653, 329, 93, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 2, 0, 0, 0, 0, 7525, 5, 4546, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 17, 0, 0, 203, 5620, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1951, 12178, 7912, 3112, 8000, 13470, 12170, 6743] avg=8192.0000 max=13470 max_rank=5 min=1951 min_rank=0 imbalance_ratio=1.644287 max_load_reduction=52066 max_load_reduction_pct=0.794464 moved_to_echo=40383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 3, 0, 2, 6231, 0, 0, 2, 6132, 0, 0, 0, 7, 65, 0, 821, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 27, 0, 1, 0, 7730, 0, 0, 3709, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 0, 1, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 21, 0, 1069, 0, 1249, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 348, 0, 0, 0, 3, 0, 44, 0, 0, 0, 0, 0, 43, 5817, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 16, 0, 69, 0, 0, 0, 0, 0, 0, 38, 0, 3361, 1, 8, 865, 455, 41, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 12, 32, 0, 0, 0, 5392, 4295, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 5, 0, 0, 0, 0, 0, 7706, 0, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13731, 11669, 8036, 2347, 6441, 5036, 9892, 8384] avg=8192.0000 max=13731 max_rank=0 min=2347 min_rank=3 imbalance_ratio=1.676147 max_load_reduction=51805 max_load_reduction_pct=0.790482 moved_to_echo=29753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 6172, 0, 0, 0, 3, 0, 0, 0, 13, 0, 0, 1452, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 144, 6, 0, 0, 0, 0, 0, 6, 0, 138, 11, 2, 0, 1, 6566, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 8, 16, 0, 0, 0, 0, 0, 12, 11, 7395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 71, 2, 0, 0, 0, 0, 50, 0, 5981, 1, 22, 0, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 0, 0, 0, 1, 0, 113, 0, 0, 323, 70, 0, 0, 6551, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8128, 0, 0, 0, 0, 24, 0, 0, 0, 7, 0, 0, 0, 968, 450, 181, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 0, 1, 0, 251, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 243, 0, 0, 7685, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 7062, 7446, 7091, 9817, 10006, 8029, 7934] avg=8192.0000 max=10006 max_rank=5 min=7062 min_rank=1 imbalance_ratio=1.221436 max_load_reduction=55530 max_load_reduction_pct=0.847321 moved_to_echo=35786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 8005, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 1483, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 4, 0, 0, 0, 0, 0, 75, 6705, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 10, 1, 0, 0, 0, 1, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 0, 69, 0, 5, 0, 2, 0, 0, 0, 0, 32, 0, 6611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 7449, 0, 0, 0, 2, 0, 5, 0, 0, 33, 88, 0, 0, 977, 440, 241, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 127, 0, 0, 0, 1, 0, 22, 6911, 12, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 7522, 11, 0, 4, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10015, 6990, 7726, 8343, 8177, 9527, 7149, 7609] avg=8192.0000 max=10015 max_rank=0 min=6990 min_rank=1 imbalance_ratio=1.222534 max_load_reduction=55521 max_load_reduction_pct=0.847183 moved_to_echo=32462 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 37376, 256, 2048, 0, 128, 128, 512, 128, 128, 5632, 6528, 128, 128, 7296, 3584, 1536, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 44416, 128, 128, 128, 6912, 128, 256, 128, 128, 128, 128, 9984, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 10496, 128, 128, 0, 128, 128, 128, 128, 256, 256, 128, 8192, 128, 128, 44800, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 1536, 512, 128, 128, 128, 0, 12288, 128, 49152, 128, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[7808, 128, 128, 896, 128, 256, 128, 128, 10496, 7936, 38400, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11136, 384, 0, 512, 128, 128, 0, 256, 0, 0, 2432, 256, 128, 128, 49152, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[384, 128, 128, 0, 128, 128, 128, 5120, 128, 128, 128, 128, 128, 0, 128, 256, 60032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[1024, 128, 896, 128, 0, 128, 768, 128, 384, 59136, 128, 384, 640, 256, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 0, 0, 0, 0, 49, 0, 7496, 2, 0, 0, 0, 0, 1, 0, 0, 7292, 0, 0, 1017, 3, 0, 0, 0, 0, 0, 12, 0, 89, 0, 0, 22, 0, 51, 332, 5911, 0, 0, 0, 92, 0, 0, 0, 0, 0, 154, 0, 275, 527, 0, 0, 0, 0, 0, 0, 61, 0, 331, 42, 0, 36, 0, 7915, 0, 0, 0, 0, 0, 0, 7639, 146, 0, 0, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 93, 0, 1122, 0, 0, 0, 0, 2, 0, 6, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 22, 3, 8043, 0, 0, 11, 1174, 0, 0, 0, 26, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[31, 0, 0, 0, 0, 1352, 4, 829, 0, 0, 0, 8037, 0, 2, 0, 0, 0, 0, 1, 0, 129, 0, 1, 0, 71, 0, 13, 0, 0, 0, 0, 2568, 1, 114, 53, 0, 12, 0, 0, 1446, 0, 0, 0, 699, 0, 0, 0, 12, 0, 7680, 0, 0, 2238, 0, 24, 0, 0, 2, 0, 2, 0, 1, 6474, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 8177, 0, 1566, 0, 5, 0, 8154, 444, 0, 0, 0, 0, 0, 5901, 28, 0, 0, 0, 0, 0, 618, 1, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 3, 0, 0, 0, 0, 2, 26, 0, 50, 0, 0, 8137, 14, 0, 561, 0, 0, 44, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4686, 0, 0, 3, 886, 0, 0, 5, 0, 5025, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 7121, 0, 4131, 6, 992, 0, 0, 0, 1, 0, 2, 55, 0, 193, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6782, 0, 0, 0, 2, 2, 0, 0, 0, 441, 0, 7828, 193, 1152, 49, 0, 661, 8105, 0, 0, 0, 0, 0, 0, 87, 1, 0, 7, 0, 0, 0, 67, 0, 0, 0, 1, 2, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 8181, 0, 0, 7340, 964, 0, 2, 84, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[22, 0, 0, 69, 1, 556, 1, 7, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 1, 6511, 0, 0, 0, 0, 0, 0, 0, 4, 94, 0, 1, 0, 0, 3, 0, 0, 0, 0, 7388, 0, 0, 0, 120, 0, 0, 0, 0, 0, 73, 0, 1, 0, 7, 0, 9, 0, 0, 0, 0, 0, 1652, 12, 0, 7, 0, 42, 0, 0, 0, 33, 20, 0, 4212, 0, 0, 0, 0, 0, 8157, 1000, 0, 0, 5, 0, 8, 7680, 0, 0, 0, 0, 290, 57, 3666, 0, 0, 0, 6108, 0, 21, 0, 0, 0, 0, 0, 7, 0, 0, 166, 0, 0, 0, 0, 1099, 0, 63, 1, 0, 7983, 0, 0, 4, 243, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3106, 0, 8, 0, 8151, 0, 4414, 0, 0, 13, 162, 0, 0, 8, 0, 1896, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 966, 0, 1, 0, 0, 0, 4, 0, 8181, 0, 0, 0, 7149, 0, 0, 0, 0, 0, 2114, 0, 0, 1, 0, 0, 0, 0, 9, 1689, 0, 0, 0, 360, 0, 0, 0, 486, 864, 0, 0, 0, 0, 0, 6528, 31, 183, 0, 0, 0, 107, 0, 0, 1, 0, 0, 0, 2068, 0, 1, 0, 0, 0, 34, 7036, 0, 0, 0, 14, 0, 48, 0, 2, 24, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 7817, 0, 0, 2017, 6, 0, 0, 0, 13, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 93, 15, 0, 0, 125, 0, 309, 1175, 143, 306, 0, 2, 0, 2, 0, 0, 2, 0, 9, 370, 0, 0, 0, 0, 0, 0, 0, 0, 14, 217, 0, 0, 0, 0, 0, 0, 42, 17, 7737, 0, 765, 0, 8087, 0, 289, 0, 0, 0, 34, 0, 0, 783, 0, 73, 17, 8, 129, 26, 0, 0, 0, 7992, 0, 0, 2, 35, 143, 0, 0, 0, 2, 11, 5108, 0, 115, 33, 78, 35, 5591, 0, 0, 0, 0, 0, 0, 6763, 0, 24, 2, 45, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 1, 920, 0, 7954, 0, 9, 5429, 0, 0, 0, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2245, 0, 31, 402, 2, 887, 0, 0, 4, 6405, 0, 3, 0, 0, 0, 488, 1, 0, 56, 0, 1568, 0, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 5, 0, 2930, 6, 6, 0, 0, 0, 0, 1, 0, 51, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6329, 0, 2, 0, 6987, 0, 0, 578, 6, 0, 4, 9, 0, 0, 0, 30, 0, 73, 0, 0, 6, 0, 0, 0, 3410, 0, 1, 2, 0, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 983, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4525, 0, 23, 88, 0, 8046, 0, 0, 7812, 0, 0, 59, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 57, 0, 0, 6330, 0, 0, 39, 1937, 312, 599, 0, 0, 0, 0, 1, 7, 0, 15, 349, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 81, 42, 0, 0, 1, 13, 0, 0, 8022, 1, 0, 0, 3, 0, 0, 0, 0, 0, 290, 0, 0, 2, 6, 0, 0, 0, 0, 0, 15, 0, 1, 4, 5, 0, 0, 6708, 1770, 36, 0, 0, 0, 0, 5072, 0, 1883, 0, 0, 0, 0, 0, 0, 5, 571, 1, 5, 6741, 0, 4, 0, 1, 313, 0, 3998, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 8183, 0, 7551, 4487, 0, 0, 0, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 0, 0, 0, 0, 49, 0, 6507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7292, 0, 0, 1017, 3, 0, 0, 0, 0, 0, 12, 0, 3828, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 22, 0, 51, 332, 5081, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 275, 527, 0, 0, 0, 0, 0, 0, 61, 0, 331, 42, 0, 830, 903, 989, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 7915, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 6661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6582, 0, 0, 93, 0, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7639, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3, 2597, 0, 0, 11, 1174, 0, 0, 0, 26, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6561, 12834, 5667, 4799, 14758, 7797, 9283, 3837] avg=8192.0000 max=14758 max_rank=4 min=3837 min_rank=7 imbalance_ratio=1.801514 max_load_reduction=50778 max_load_reduction_pct=0.774811 moved_to_echo=35675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 0, 0, 0, 1352, 4, 829, 0, 0, 0, 6977, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 129, 0, 1, 0, 71, 0, 13, 0, 0, 0, 0, 2568, 3872, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 114, 53, 0, 12, 0, 0, 1243, 0, 0, 0, 699, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 0, 0, 2238, 0, 24, 0, 0, 2, 0, 2, 0, 1, 6474, 0, 203, 974, 1060, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 1566, 0, 5, 0, 7180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, 0, 0, 4886, 28, 0, 0, 0, 0, 0, 618, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 3, 0, 0, 0, 0, 8177, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 26, 0, 50, 0, 0, 2628, 14, 0, 561, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9195, 7160, 2134, 19168, 8755, 5977, 9822, 3325] avg=8192.0000 max=19168 max_rank=3 min=2134 min_rank=2 imbalance_ratio=2.339844 max_load_reduction=46368 max_load_reduction_pct=0.707520 moved_to_echo=27879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3106, 0, 8, 0, 8151, 0, 3832, 0, 0, 13, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1896, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 966, 3720, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7033, 0, 0, 0, 7149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2114, 0, 0, 1, 0, 0, 0, 0, 9, 1689, 0, 0, 0, 360, 0, 1148, 12, 582, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 864, 0, 0, 0, 0, 0, 0, 31, 183, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1713, 0, 1, 0, 0, 0, 34, 7036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 48, 0, 2, 24, 0, 0, 0, 0, 9, 0, 0, 1, 0, 6528, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 2525, 0, 0, 2017, 6, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15273, 6772, 14187, 6093, 1659, 8785, 8198, 4569] avg=8192.0000 max=15273 max_rank=0 min=1659 min_rank=4 imbalance_ratio=1.864380 max_load_reduction=50263 max_load_reduction_pct=0.766953 moved_to_echo=23211 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 93, 15, 0, 0, 125, 0, 309, 1175, 143, 306, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 370, 0, 0, 0, 0, 0, 0, 0, 0, 14, 217, 0, 3785, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 17, 6651, 0, 765, 0, 8087, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 783, 0, 73, 17, 8, 129, 26, 0, 0, 0, 7992, 0, 1086, 668, 0, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 143, 0, 0, 0, 2, 11, 0, 0, 115, 33, 78, 35, 4923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5599, 0, 24, 2, 45, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 5108, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 920, 0, 2569, 0, 9, 5429, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2170, 4976, 15851, 11401, 5377, 9949, 6876, 8936] avg=8192.0000 max=15851 max_rank=2 min=2170 min_rank=0 imbalance_ratio=1.934937 max_load_reduction=49685 max_load_reduction_pct=0.758133 moved_to_echo=31138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 57, 0, 0, 6330, 0, 0, 39, 1937, 312, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 15, 349, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 81, 3894, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 1, 13, 0, 0, 6896, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 2, 6, 0, 0, 0, 0, 0, 15, 0, 1, 4, 5, 1126, 0, 79, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6708, 1770, 36, 0, 0, 0, 0, 0, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 571, 1, 5, 5581, 0, 4, 0, 1, 313, 0, 3998, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2, 0, 0, 0, 0, 0, 5072, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 2643, 0, 7551, 4487, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9199, 4937, 6956, 2111, 10397, 10504, 6741, 14691] avg=8192.0000 max=14691 max_rank=7 min=2111 min_rank=3 imbalance_ratio=1.793335 max_load_reduction=50845 max_load_reduction_pct=0.775833 moved_to_echo=42333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2245, 0, 31, 402, 2, 887, 0, 0, 4, 6405, 0, 3, 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 56, 0, 1568, 0, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 2, 3829, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 2930, 6, 6, 0, 0, 0, 0, 1, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6329, 0, 0, 8, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6987, 0, 0, 578, 6, 0, 4, 0, 0, 0, 0, 30, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2824, 0, 1, 2, 0, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4525, 0, 23, 88, 0, 2599, 0, 0, 7812, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10467, 8843, 3001, 6823, 7672, 11001, 2623, 15106] avg=8192.0000 max=15106 max_rank=7 min=2623 min_rank=6 imbalance_ratio=1.843994 max_load_reduction=50430 max_load_reduction_pct=0.769501 moved_to_echo=36402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 69, 1, 556, 1, 7, 0, 0, 0, 7057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6511, 0, 0, 0, 0, 0, 0, 0, 4, 94, 0, 1, 3805, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 6347, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1, 0, 7, 0, 9, 0, 0, 0, 0, 0, 1652, 12, 0, 1041, 978, 1075, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 42, 0, 0, 0, 33, 20, 0, 0, 0, 0, 0, 0, 0, 7179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 0, 0, 5, 0, 8, 6349, 0, 0, 0, 0, 290, 57, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 21, 0, 0, 0, 0, 0, 7, 0, 0, 166, 0, 0, 0, 4212, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 63, 1, 0, 2569, 0, 0, 4, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7713, 11079, 6470, 5516, 7281, 11375, 12123, 3979] avg=8192.0000 max=12123 max_rank=6 min=3979 min_rank=7 imbalance_ratio=1.479858 max_load_reduction=53413 max_load_reduction_pct=0.815018 moved_to_echo=34758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4068, 0, 0, 3, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5025, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 3893, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 6122, 0, 4131, 6, 992, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 55, 0, 193, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6782, 0, 999, 968, 618, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 441, 0, 0, 193, 1152, 49, 0, 661, 7137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1, 0, 7, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7828, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 2643, 0, 0, 7340, 964, 0, 2, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4958, 8935, 11270, 9625, 9637, 148, 9870, 11093] avg=8192.0000 max=11270 max_rank=2 min=148 min_rank=5 imbalance_ratio=1.375732 max_load_reduction=54266 max_load_reduction_pct=0.828033 moved_to_echo=30748 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 10368, 128, 384, 3968, 128, 128, 128, 128, 256, 1792, 128, 0, 2048, 28032, 128, 6528, 4608, 4480, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 5760, 0, 256, 1152, 128, 20864, 128, 7680, 27776, 2432, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 256, 128, 128, 128, 128, 3328, 39424, 128, 4992, 128, 17152, 0, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1536, 0, 128, 640, 128, 128, 33664, 128, 128, 128, 128, 1792, 128, 27904, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[2304, 128, 128, 512, 128, 12416, 128, 1280, 1280, 16768, 640, 29056, 128, 128, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 23168, 0, 128, 4224, 128, 0, 128, 128, 128, 128, 256, 3712, 30720, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 6144, 128, 512, 1152, 128, 128, 128, 0, 128, 128, 128, 256, 0, 128, 128, 44672, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 22272, 2816, 128, 640, 128, 512, 128, 0, 384, 4992, 128, 128, 768, 33280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 10, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 1, 0, 325, 0, 0, 0, 0, 0, 3, 6910, 8, 5, 0, 0, 0, 1061, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 3, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 10, 6794, 39, 0, 0, 0, 2, 0, 0, 1, 0, 443, 6364, 0, 17, 28, 0, 57, 0, 14, 0, 6297, 321, 0, 0, 0, 0, 0, 1288, 1, 0, 6796, 0, 0, 0, 0, 8162, 0, 1, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5710, 0, 74, 0, 0, 11, 0, 4186, 0, 0, 0, 0, 0, 0, 8092, 33, 224, 0, 4, 0, 0, 0, 21, 2, 14, 0, 0, 0, 34, 0, 0, 3, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 309, 0, 6541, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 136, 0, 4, 0, 0, 0, 0, 0, 6, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4820, 0, 5, 3, 2, 0, 0, 0, 7, 0, 36, 7996, 0, 0, 0, 0, 0, 8186, 0, 0, 2071, 0, 0, 0, 1, 0, 0, 0, 28, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 3259, 0, 0, 31, 0, 0, 4, 2923, 0, 3, 0, 0, 0, 5, 8182, 18, 0, 0, 0, 125, 0, 1, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1243, 0, 0, 5244, 0, 0, 0, 58, 0, 1, 0, 0, 0, 0, 0, 29, 0, 0, 18, 2218, 5415, 0, 0, 0, 0, 0, 0, 47, 2, 10, 1, 0, 17, 0, 0, 2097, 4698, 0, 0, 0, 21, 7453, 5, 0, 0, 0, 21, 0, 0, 0, 0, 6318, 63, 0, 65, 0, 0, 0, 0, 0, 0, 1866, 7845, 0, 0, 0, 0, 78, 1233, 0, 0, 4854, 0, 0, 0, 0, 0, 0, 0, 4, 34] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 8175, 7564, 12, 0, 249, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 3, 385, 68, 0, 21, 0, 28, 1, 1, 0, 6, 102, 671, 7, 0, 0, 7876, 1, 0, 0, 0, 19, 0, 0, 32, 0, 0, 5, 1, 1, 0, 0, 395, 0, 0, 232, 3, 0, 2, 0, 0, 0, 8161, 0, 0, 1, 8141, 0, 0, 0, 0, 0, 0, 608, 0, 0, 304, 12, 0, 1, 98, 0, 0, 6848, 0, 5, 0, 0, 0, 7819, 0, 0, 6321, 0, 0, 0, 795, 296, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 3, 13, 5, 21, 0, 0, 0, 41, 9, 170, 0, 0, 0, 0, 0, 3, 7269, 7125, 0, 0, 7910, 0, 0, 1, 24, 1253, 1187, 2, 0, 0, 7365, 37, 24, 0, 10, 194, 58, 0, 0, 0, 498, 0, 0, 0, 0, 128, 2, 0, 0, 0, 240, 0, 1, 0, 14, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 41, 8, 0, 902, 4, 0, 0, 0, 8183, 0, 1, 33, 1796, 0, 0, 29, 0, 0, 1, 15, 40, 0, 242, 0, 28, 0, 0, 39, 0, 7125, 0, 0, 9, 0, 0, 5891, 0, 0, 6804, 0, 183, 0, 0, 66, 0, 0, 34, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 65, 0, 0, 0, 66, 4, 0, 0, 2, 0, 0, 0, 18, 0, 8, 0, 0, 6458, 0, 271, 0, 0, 0, 0, 0, 0, 5261, 0, 0, 5, 18, 0, 0, 0, 1401, 6, 8, 0, 0, 2, 0, 0, 0, 0, 0, 0, 18, 0, 2, 1888, 0, 0, 0, 0, 0, 2, 0, 5392, 0, 1673, 0, 0, 0, 9, 8189, 0, 0, 0, 2, 0, 2, 0, 17, 0, 0, 0, 0, 21, 7, 0, 267, 0, 0, 0, 0, 8030, 0, 2, 0, 0, 0, 0, 7773, 0, 0, 1171, 0, 0, 0, 126, 0, 0, 1, 920, 0, 0, 8143, 0, 16, 0, 0, 0, 9, 79, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 1014, 0, 0, 394, 0, 0, 128, 0, 0, 0, 28, 0, 0, 0, 70, 0, 313, 7701, 3062, 0, 0, 0, 0, 50, 0, 2436, 0, 0, 0, 6, 243, 17, 0, 0, 0, 140, 0, 0, 6287, 0, 0, 0, 1, 0, 8, 0, 0, 0, 2378, 0, 0, 11, 3, 0, 0, 7365, 0, 0, 1365, 304, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 51, 7882, 2, 0, 3, 0, 7933, 0, 0, 0, 7, 0, 0, 2, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 2656, 0, 3, 0, 0, 1, 0, 0, 0, 5363, 0, 6, 3, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 644, 0, 0, 0, 0, 0, 57, 0, 0, 0, 481, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 176, 0, 0, 0, 895, 454, 0, 6593, 8092, 33, 6, 0, 0, 2, 376, 0, 0, 0, 18, 10, 0, 231, 6, 0, 0, 4, 0, 0, 4741, 367, 0, 2, 0, 0, 0, 2, 4983, 137, 7866, 33, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 31, 0, 0, 0, 1, 0, 11, 8, 388, 100, 53, 0, 0, 7823, 1, 0, 2, 33, 0, 0, 117, 0, 0, 16, 0, 0, 0, 6745, 1, 0, 0, 0, 0, 0, 7499, 0, 0, 0, 2, 0, 5, 0, 0, 6371, 0, 100, 0, 0, 0, 0, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5710, 5077, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 11, 0, 4186, 0, 0, 0, 0, 0, 0, 3785, 33, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 21, 2, 14, 0, 0, 0, 34, 0, 0, 3, 811, 2992, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 309, 0, 6541, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 136, 0, 3820, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4820, 0, 5, 3, 2, 0, 0, 0, 7, 0, 36, 3432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1383, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19690, 8313, 4223, 7197, 5326, 2883, 8306, 9598] avg=8192.0000 max=19690 max_rank=0 min=2883 min_rank=5 imbalance_ratio=2.403564 max_load_reduction=45846 max_load_reduction_pct=0.699554 moved_to_echo=26113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 568, 4339, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 325, 0, 0, 0, 0, 0, 3, 3232, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 2555, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 153, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 10, 2455, 39, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 443, 6364, 0, 17, 28, 0, 57, 0, 14, 0, 6297, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 1, 0, 4535, 0, 0, 0, 0, 8162, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4955, 3574, 8468, 1166, 9317, 10698, 13360, 13998] avg=8192.0000 max=13998 max_rank=7 min=1166 min_rank=3 imbalance_ratio=1.708740 max_load_reduction=51538 max_load_reduction_pct=0.786407 moved_to_echo=47373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 3, 13, 5, 21, 5226, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 9, 170, 0, 0, 0, 0, 0, 3, 7269, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 1, 24, 1253, 1187, 2, 0, 0, 7365, 37, 24, 0, 10, 194, 2634, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 498, 0, 0, 0, 0, 128, 2, 0, 0, 0, 240, 0, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 3404, 1158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 8, 0, 902, 4, 0, 0, 0, 2957, 0, 1, 33, 1796, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 40, 0, 242, 0, 28, 0, 0, 39, 0, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 5891, 0, 0, 4541, 0, 183, 0, 0, 66, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6375, 10825, 21767, 2063, 4588, 5771, 3423, 10724] avg=8192.0000 max=21767 max_rank=2 min=2063 min_rank=3 imbalance_ratio=2.657104 max_load_reduction=43769 max_load_reduction_pct=0.667862 moved_to_echo=24506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 3259, 4760, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4, 2923, 0, 3, 0, 0, 0, 5, 3828, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 1, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1243, 3025, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5244, 0, 0, 0, 58, 0, 1, 0, 0, 0, 0, 0, 29, 0, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2218, 5415, 0, 0, 0, 0, 0, 0, 47, 2, 10, 1, 0, 17, 3748, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 4698, 0, 0, 0, 21, 2693, 5, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6318, 63, 0, 65, 0, 0, 0, 0, 0, 0, 1866, 3367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1233, 0, 0, 3240, 0, 0, 0, 0, 0, 0, 0, 4, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8767, 6812, 5206, 6143, 12805, 9535, 11679, 4589] avg=8192.0000 max=12805 max_rank=4 min=4589 min_rank=7 imbalance_ratio=1.563110 max_load_reduction=52731 max_load_reduction_pct=0.804611 moved_to_echo=38608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 169, 5212, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 8175, 3539, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 3, 385, 2796, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 21, 0, 28, 1, 1, 0, 6, 102, 671, 7, 0, 0, 7876, 1, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 32, 0, 0, 5, 1, 1, 0, 0, 395, 0, 3272, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 3, 0, 2, 0, 0, 0, 2949, 0, 0, 1, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 304, 12, 0, 1, 98, 0, 0, 2939, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7819, 0, 0, 4218, 0, 0, 0, 795, 296, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 11728, 4500, 9839, 4954, 11328, 3967, 13129] avg=8192.0000 max=13129 max_rank=7 min=3967 min_rank=6 imbalance_ratio=1.602661 max_load_reduction=52407 max_load_reduction_pct=0.799667 moved_to_echo=33378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 1014, 0, 0, 394, 5066, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 28, 0, 0, 0, 70, 0, 313, 3603, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 2436, 0, 0, 0, 6, 243, 17, 0, 0, 0, 140, 2847, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6287, 0, 0, 0, 1, 0, 8, 0, 0, 0, 2378, 0, 0, 11, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7365, 0, 0, 1365, 304, 0, 0, 0, 0, 0, 0, 328, 0, 1269, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 7882, 2, 0, 3, 0, 2867, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 1140, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3579, 0, 6, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6731, 7204, 6627, 9581, 11885, 10814, 9101, 3593] avg=8192.0000 max=11885 max_rank=4 min=3593 min_rank=7 imbalance_ratio=1.450806 max_load_reduction=53651 max_load_reduction_pct=0.818649 moved_to_echo=35393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 65, 0, 0, 0, 66, 4, 0, 0, 2, 0, 0, 0, 18, 5128, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 6458, 0, 271, 0, 0, 0, 0, 0, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 18, 0, 0, 0, 1401, 6, 8, 0, 0, 2, 0, 0, 0, 0, 0, 1945, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 2, 1888, 0, 0, 0, 0, 0, 2, 0, 5392, 0, 1673, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8189, 0, 0, 0, 2, 0, 2, 0, 17, 0, 0, 0, 0, 3891, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7, 0, 267, 0, 0, 0, 0, 2902, 0, 2, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 0, 126, 0, 0, 1, 920, 0, 0, 3494, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 79, 0, 5461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6041, 9198, 4740, 10343, 12965, 10972, 5728, 5549] avg=8192.0000 max=12965 max_rank=4 min=4740 min_rank=2 imbalance_ratio=1.582642 max_load_reduction=52571 max_load_reduction_pct=0.802170 moved_to_echo=35214 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 644, 0, 0, 0, 0, 0, 57, 0, 0, 0, 481, 0, 0, 5001, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 176, 0, 0, 0, 895, 207, 0, 6593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 33, 6, 0, 0, 2, 376, 0, 0, 0, 18, 10, 0, 231, 6, 0, 171, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4741, 367, 0, 2, 0, 0, 0, 2, 4983, 137, 7866, 33, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 31, 0, 0, 0, 1, 3586, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 8, 388, 100, 53, 0, 0, 2822, 1, 0, 2, 33, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6745, 1, 0, 0, 0, 0, 0, 3210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 4242, 0, 100, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6886, 7882, 10005, 19204, 3696, 3535, 9972, 4356] avg=8192.0000 max=19204 max_rank=3 min=3535 min_rank=5 imbalance_ratio=2.344238 max_load_reduction=46332 max_load_reduction_pct=0.706970 moved_to_echo=21559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[16256, 128, 256, 1152, 128, 5120, 1664, 128, 0, 128, 11392, 128, 128, 256, 128, 2816, 19072, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 9344, 0, 128, 0, 128, 128, 128, 0, 128, 1152, 512, 128, 10240, 39936, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 11648, 128, 7168, 384, 128, 128, 128, 256, 768, 128, 13184, 256, 24320, 128, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 9984, 20992, 0, 128, 1408, 384, 128, 128, 128, 512, 128, 128, 896, 128, 23168, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 20736, 7168, 128, 128, 7552, 128, 128, 128, 1152, 128, 8320, 20864, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 384, 2176, 14208, 128, 128, 128, 128, 22528, 128, 128, 128, 9984, 128, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 24448, 128, 0, 31232, 0, 384, 128, 896, 8576, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 256, 128, 6528, 256, 7808, 128, 256, 0, 128, 128, 16768, 24064, 3200, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 153, 1, 0, 8019, 0, 2552, 1715, 0, 0, 236, 0, 0, 4486, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2, 33, 0, 0, 282, 4, 14, 0, 0, 7845, 0, 0, 0, 145, 3, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 63, 0, 1, 0, 0, 7900, 0, 0, 209, 0, 43, 1, 0, 1785, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 31, 305, 4626, 1, 20, 0, 3492, 0, 0, 48, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 6367, 10, 0, 0, 0, 0, 0, 0, 7177, 0, 7837] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 9, 0, 0, 0, 8167, 1, 0, 0, 0, 2, 0, 0, 97, 0, 0, 7, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 133, 0, 0, 75, 0, 0, 0, 3126, 0, 1827, 0, 0, 0, 8128, 0, 2, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 41, 0, 0, 17, 0, 0, 8180, 0, 0, 5994, 0, 0, 0, 0, 47, 0, 2244, 0, 0, 0, 0, 0, 2, 0, 0, 4666, 0, 0, 0, 0, 0, 0, 0, 364, 1236, 0, 0, 1043, 7908, 0, 0, 0, 6495, 0, 0, 0, 0, 0, 1, 0, 5633, 0, 1, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 70, 7112, 0, 0, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 736, 2197, 0, 0, 0, 2434, 591, 0, 3487, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 2, 1, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7870, 0, 0, 0, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 4, 0, 515, 0, 0, 0, 1, 8093, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 7969, 0, 1333, 0, 2407, 5, 0, 0, 0, 0, 2082, 0, 0, 0, 3, 0, 0, 20, 0, 0, 283, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2778, 1, 0, 0, 0, 6, 57, 0, 0, 0, 2, 7941, 22, 0, 725, 0, 331, 0, 70, 6321, 0, 6869, 3, 2, 0, 7, 0, 0, 58, 2605, 3203, 0, 90, 0, 20, 0, 0, 0, 0, 659, 0, 1, 8176, 5317, 0, 0, 0, 34, 0, 1, 0, 0, 984, 0, 0, 630, 0, 0, 0, 5, 0, 46, 0, 0, 0, 4236, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 7640, 0, 0, 0, 0, 0, 497, 0, 7190, 0, 0, 10, 9, 0, 0, 2, 0, 65, 0, 1, 0, 6606, 0, 0, 0, 2, 0, 0, 4, 0, 0, 6773, 0, 0, 3, 0, 8, 0, 1, 0, 0, 21, 0, 0, 0, 7997, 154, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 415, 0, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 14, 9, 6, 1, 8, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4240, 0, 0, 0, 0, 46, 0, 0, 0, 6230, 0, 0, 307, 0, 0, 0, 614, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 131] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 275, 2136, 7747, 0, 1, 15, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 16, 33, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 7630, 2, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 12, 0, 8, 1162, 0, 0, 0, 0, 0, 1612, 0, 0, 215, 0, 8153, 521, 2370, 3, 0, 0, 0, 0, 6466, 0, 668, 0, 0, 0, 0, 2, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 7018, 0, 6, 0, 326, 6255, 0, 2, 0, 55, 0, 0, 0, 0, 192, 0, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 0, 0, 941, 0, 0, 5107, 0, 0, 0, 26, 0, 51, 1, 0, 8046, 2, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 357, 96, 0, 0, 0, 0, 0, 41, 197, 0, 0, 1103, 0, 0, 0, 6455, 0, 20, 0, 0, 0, 2391, 29, 0, 0, 0, 0, 12, 18, 237, 209, 0, 0, 0, 0, 0, 0, 92, 0, 0, 7955, 0, 149, 3, 0, 2, 0, 0, 8187, 0, 0, 0, 0, 1, 1080, 0, 0, 0, 2, 137, 0, 0, 0, 0, 59, 0, 0, 2701, 41, 37, 0, 952, 7614, 0, 83, 0, 59, 0, 0, 0, 7, 7, 0, 2351, 0, 0, 6980, 172, 0, 1, 0, 0, 0, 1498, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 2041, 0, 0, 1, 0, 0, 0, 3, 4570, 0, 0, 0, 9, 0, 0, 0, 0, 4996, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 6482, 0, 0, 18, 0, 708, 0, 129, 0, 0, 0, 0, 0, 0, 8172, 0, 372, 10, 1, 0, 1603, 46, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 23, 0, 0, 7575, 0, 0, 0, 222, 0, 0, 0, 6544, 0, 0, 0, 0, 0, 7, 0, 4, 0, 0, 57, 5, 9, 0, 0, 5129, 0, 0, 7784, 1266, 0, 0, 0, 7205, 0, 0, 0, 0, 12, 0, 0, 2, 0, 0, 1, 0, 0, 16, 55, 0, 15, 0, 379, 0, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 9, 0, 0, 0, 4399, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 7, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 133, 0, 0, 75, 0, 0, 0, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 8128, 0, 2, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4574, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 17, 0, 0, 1970, 0, 0, 5994, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2244, 0, 0, 0, 0, 0, 2, 0, 0, 4666, 0, 0, 0, 0, 0, 3120, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 1236, 0, 0, 1043, 1053, 0, 0, 0, 6495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5633, 0, 1, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 599, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4413, 5213, 2738, 15686, 8069, 12313, 10191, 6913] avg=8192.0000 max=15686 max_rank=3 min=2738 min_rank=2 imbalance_ratio=1.914795 max_load_reduction=49850 max_load_reduction_pct=0.760651 moved_to_echo=37486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 153, 1, 0, 4320, 0, 2552, 1715, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4486, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2, 33, 0, 0, 282, 4, 14, 0, 0, 6341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 3, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2019, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 1, 0, 0, 1903, 0, 0, 209, 0, 43, 1, 0, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3063, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 305, 4626, 1, 20, 0, 466, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 6367, 10, 0, 0, 0, 0, 0, 0, 7177, 0, 7837, 1504, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8981, 9413, 6716, 3252, 4005, 4075, 5497, 23597] avg=8192.0000 max=23597 max_rank=7 min=3252 min_rank=3 imbalance_ratio=2.880493 max_load_reduction=41939 max_load_reduction_pct=0.639938 moved_to_echo=37174 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 70, 7112, 0, 0, 0, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 736, 4907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 2434, 591, 0, 3487, 0, 0, 0, 0, 0, 0, 6578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 2, 0, 0, 4681, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 0, 0, 0, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 1235, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 515, 0, 0, 0, 1, 1077, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796, 1564, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8924, 5711, 15287, 13946, 9642, 3631, 1779, 6616] avg=8192.0000 max=15287 max_rank=2 min=1779 min_rank=6 imbalance_ratio=1.866089 max_load_reduction=50249 max_load_reduction_pct=0.766739 moved_to_echo=21668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 275, 2136, 7747, 0, 1, 9, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 33, 0, 0, 0, 0, 0, 0, 0, 0, 5083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 6167, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 12, 0, 8, 1162, 0, 0, 0, 3618, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1612, 0, 0, 215, 0, 1964, 521, 2370, 3, 0, 0, 0, 0, 6466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 2, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 5, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7018, 0, 6, 0, 326, 833, 0, 2, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10200, 5132, 6553, 8524, 13151, 8705, 8240, 5031] avg=8192.0000 max=13151 max_rank=4 min=5031 min_rank=7 imbalance_ratio=1.605347 max_load_reduction=52385 max_load_reduction_pct=0.799332 moved_to_echo=35127 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2041, 0, 0, 1, 0, 0, 0, 3, 2462, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4996, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 4723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6482, 0, 0, 18, 0, 708, 0, 129, 0, 0, 0, 0, 0, 0, 6606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 10, 1, 0, 1603, 46, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4167, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 23, 0, 0, 1824, 0, 0, 0, 222, 0, 0, 0, 6544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 0, 57, 5, 9, 0, 0, 5129, 1746, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 1266, 0, 0, 0, 960, 0, 0, 0, 0, 12, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 55, 0, 15, 0, 379, 0, 0, 0, 0, 3, 1566, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4516, 9728, 13943, 7264, 8629, 9035, 10024, 2397] avg=8192.0000 max=13943 max_rank=2 min=2397 min_rank=7 imbalance_ratio=1.702026 max_load_reduction=51593 max_load_reduction_pct=0.787247 moved_to_echo=30085 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7640, 0, 0, 0, 0, 0, 497, 0, 3873, 0, 0, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 0, 1, 0, 6606, 0, 0, 0, 2, 0, 0, 4, 0, 0, 5024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6773, 0, 0, 3, 0, 8, 0, 1, 0, 0, 21, 0, 0, 0, 6464, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 415, 0, 5, 0, 0, 2, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 14, 9, 6, 1, 8, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4240, 0, 0, 0, 0, 46, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6230, 0, 0, 307, 0, 0, 0, 614, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 131, 1533, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12029, 11704, 13424, 1519, 2252, 7033, 15339, 2236] avg=8192.0000 max=15339 max_rank=6 min=1519 min_rank=3 imbalance_ratio=1.872437 max_load_reduction=50197 max_load_reduction_pct=0.765945 moved_to_echo=26860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 0, 0, 941, 0, 0, 2751, 0, 0, 0, 26, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8046, 2, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 357, 4960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 41, 197, 0, 0, 1103, 0, 0, 0, 5218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 2391, 29, 0, 0, 0, 0, 12, 18, 237, 209, 0, 0, 4404, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 1916, 0, 149, 3, 0, 2, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1080, 0, 0, 0, 2, 137, 0, 0, 0, 0, 59, 1951, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2701, 41, 37, 0, 952, 1014, 0, 83, 0, 59, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2351, 0, 0, 6980, 172, 0, 1, 0, 0, 0, 1498, 0, 0, 0, 1237, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3791, 13371, 6655, 8399, 10349, 5426, 4894, 12651] avg=8192.0000 max=13371 max_rank=1 min=3791 min_rank=0 imbalance_ratio=1.632202 max_load_reduction=52165 max_load_reduction_pct=0.795975 moved_to_echo=33320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 7969, 0, 1333, 0, 1290, 5, 0, 0, 0, 0, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 20, 0, 0, 283, 0, 1, 0, 0, 3, 0, 4954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2778, 1, 0, 0, 3078, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 57, 0, 0, 0, 2, 1905, 22, 0, 725, 0, 331, 0, 70, 6321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6869, 3, 2, 0, 7, 0, 0, 58, 2605, 3203, 0, 90, 0, 20, 0, 923, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 1, 8176, 701, 0, 0, 0, 34, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 0, 0, 630, 0, 0, 0, 5, 0, 46, 0, 0, 0, 4236, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12682, 5264, 220, 6946, 9439, 15318, 9572, 6095] avg=8192.0000 max=15318 max_rank=5 min=220 min_rank=2 imbalance_ratio=1.869873 max_load_reduction=50218 max_load_reduction_pct=0.766266 moved_to_echo=40424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 128, 2432, 6272, 0, 16768, 256, 128, 128, 128, 384, 0, 1536, 11520, 0, 12800, 9472, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[2304, 128, 256, 128, 12160, 128, 2688, 8320, 0, 128, 512, 128, 4224, 384, 128, 128, 26624, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[16000, 0, 0, 128, 2560, 1408, 128, 4096, 0, 512, 1280, 128, 0, 0, 39936, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 9856, 128, 128, 128, 1152, 128, 128, 128, 2688, 18560, 128, 128, 0, 128, 5248, 14848, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 18688, 14080, 128, 128, 10880, 6144, 0, 128, 896, 14848, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 17664, 128, 128, 0, 6656, 128, 128, 384, 128, 128, 128, 128, 128, 1152, 39680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 9728, 128, 128, 384, 17280, 8960, 1920, 128, 20864, 128, 2560, 1792, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 1792, 0, 256, 256, 128, 15360, 640, 2560, 7040, 8064, 384, 128, 128, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1815, 0, 0, 0, 0, 3, 0, 0, 7854, 17, 1, 0, 501, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 125, 0, 0, 0, 0, 183, 0, 0, 0, 0, 16, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 16, 790, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4586, 0, 0, 0, 0, 3, 0, 807, 0, 0, 0, 0, 4733, 0, 4135, 0, 0, 0, 0, 0, 0, 3, 0, 8056, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 8159, 0, 0, 0, 7065, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 24, 0, 0, 8101, 0, 0, 0, 0, 0, 5, 0, 0, 0, 190, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 3, 0, 3, 8169, 0, 0, 4169, 0, 1, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 20, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 53, 0, 0, 0, 0, 394, 0, 23, 15, 0, 0, 52, 7726, 0, 1372, 0, 0, 0, 0, 72, 5992, 0, 0, 20, 0, 0, 0, 0, 0, 0, 7579, 0, 0, 43, 0, 715, 0, 0, 0, 4857, 5089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 1, 0, 0, 740, 20, 15, 0, 0, 0, 0, 0, 1, 0, 0, 10, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 8147, 7839, 0, 8191, 0, 1, 131, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8165, 12, 0, 1875, 0, 3, 157, 0, 1546, 8017, 0, 3, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 777, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4814, 0, 1, 26, 2, 6, 11, 0, 318, 0, 4, 0, 0, 0, 2133, 0, 0, 2, 0, 0, 1, 0, 0, 2, 0, 17, 0, 0, 0, 228, 0, 0, 2, 0, 1, 0, 0, 0, 7825, 81, 0, 0, 0, 189, 0, 0, 7553, 0, 0, 0, 360, 0, 112, 0, 0, 7980, 0, 0, 0, 5, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 2, 15, 269, 8133, 0, 8155, 0, 0, 66, 0, 0, 5017, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 2508, 0, 11, 0, 6964, 1273] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 3, 1019, 7, 0, 0, 134, 0, 0, 0, 6138, 0, 0, 0, 86, 6603, 3, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 3, 0, 1, 1, 0, 301, 0, 0, 0, 0, 0, 0, 0, 5, 40, 0, 0, 0, 0, 4865, 0, 133, 0, 8095, 38, 0, 49, 0, 7954, 19, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 3, 21, 473, 0, 0, 3892, 0, 25, 0, 6494, 0, 1, 29, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 2613, 0, 0, 105, 0, 0, 3653, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 32, 3, 0, 4032, 121, 2, 7, 7561, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[70, 1, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 13, 0, 9, 7961, 0, 1, 0, 5534, 0, 0, 0, 0, 0, 2, 0, 0, 0, 12, 2, 0, 0, 14, 0, 77, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1175, 4730, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 3509, 0, 1, 7548, 0, 0, 0, 1, 0, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8142, 0, 0, 0, 0, 40, 0, 0, 1, 137, 0, 1315, 10, 1, 77, 0, 0, 1193, 462, 0, 7681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 91, 0, 0, 0, 7771, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 1139, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 13, 0, 7141, 0, 0, 166, 0, 2296, 0, 0, 0, 0, 221, 0, 1, 0, 0, 14, 0, 694, 8, 36, 0, 465, 0, 0, 0, 5, 3, 0, 0, 106, 81, 0, 0, 0, 0, 0, 45, 178, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3592, 13, 0, 0, 0, 119, 130, 0, 0, 4268, 0, 0, 583, 0, 0, 0, 0, 0, 350, 0, 0, 10, 5920, 0, 7394, 0, 0, 0, 0, 13, 47, 2, 0, 0, 0, 0, 398, 0, 8043, 0, 0, 2643, 7031, 1, 17, 0, 0, 3583, 0, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 0, 7, 0, 0, 754, 7947] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 336, 0, 0, 8189, 0, 0, 0, 103, 7233, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4500, 0, 0, 1955, 0, 10, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 39, 136, 8189, 0, 29, 0, 0, 0, 0, 0, 0, 777, 0, 6732, 6426, 0, 8081, 0, 0, 33, 571, 0, 7595, 0, 144, 0, 435, 0, 1, 0, 0, 38, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1360, 1692] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 24, 0, 0, 8101, 0, 0, 0, 0, 0, 5, 0, 5317, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1130, 1494, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 3, 0, 3, 8169, 0, 0, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 20, 0, 0, 0, 0, 3235, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 53, 0, 0, 0, 0, 394, 0, 23, 15, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4737, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7579, 0, 0, 43, 0, 715, 0, 0, 0, 3363, 5089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13743, 2878, 2694, 9213, 11879, 3459, 4881, 16789] avg=8192.0000 max=16789 max_rank=7 min=2694 min_rank=2 imbalance_ratio=2.049438 max_load_reduction=48747 max_load_reduction_pct=0.743820 moved_to_echo=37008 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4814, 0, 1, 26, 2, 6, 11, 0, 4452, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 4, 0, 0, 0, 2133, 0, 0, 2, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 228, 0, 0, 2, 0, 1, 0, 0, 0, 7825, 81, 6714, 2142, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 7553, 0, 0, 0, 360, 0, 112, 0, 0, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 6192, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 3123, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 269, 4986, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 2508, 0, 11, 0, 4822, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10753, 2398, 17034, 10002, 7559, 3524, 5530, 8736] avg=8192.0000 max=17034 max_rank=2 min=2398 min_rank=1 imbalance_ratio=2.079346 max_load_reduction=48502 max_load_reduction_pct=0.740082 moved_to_echo=25349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1815, 0, 0, 0, 0, 3, 0, 0, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6315, 17, 1, 0, 501, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 3405, 2173, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 183, 0, 0, 0, 0, 16, 0, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 16, 790, 0, 0, 0, 0, 0, 0, 0, 6357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4586, 0, 0, 0, 0, 3, 0, 807, 0, 1817, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 8159, 0, 0, 0, 4892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2549, 7125, 13740, 2159, 7167, 8754, 10962, 13080] avg=8192.0000 max=13740 max_rank=2 min=2159 min_rank=3 imbalance_ratio=1.677246 max_load_reduction=51796 max_load_reduction_pct=0.790344 moved_to_echo=39963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 7116, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 1, 0, 0, 740, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 8147, 7839, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 131, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 6356, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 8, 0, 0, 0, 3, 157, 0, 1546, 0, 0, 3, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 777, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15522, 7024, 1881, 17962, 7391, 164, 14808, 784] avg=8192.0000 max=17962 max_rank=3 min=164 min_rank=5 imbalance_ratio=2.192627 max_load_reduction=47574 max_load_reduction_pct=0.725922 moved_to_echo=23147 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 3, 1019, 7, 0, 0, 134, 0, 0, 0, 3243, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4935, 0, 0, 0, 86, 6603, 3, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 3, 0, 1, 1, 0, 301, 0, 0, 0, 0, 0, 0, 0, 2151, 2325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 40, 0, 0, 0, 0, 4865, 0, 133, 0, 8095, 38, 0, 49, 0, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 3, 21, 6172, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 0, 0, 3892, 0, 25, 0, 6494, 0, 1, 29, 0, 0, 0, 0, 0, 1, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 32, 3, 0, 4032, 121, 2, 7, 5236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4874, 11796, 5539, 15007, 6634, 12118, 112, 9456] avg=8192.0000 max=15007 max_rank=3 min=112 min_rank=6 imbalance_ratio=1.831909 max_load_reduction=50529 max_load_reduction_pct=0.771011 moved_to_echo=28320 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[70, 1, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 13, 0, 9, 6816, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6401, 0, 1, 0, 5534, 0, 0, 0, 0, 0, 2, 0, 0, 0, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 77, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1175, 4730, 63, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 3509, 0, 1, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 5857, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8142, 0, 0, 0, 0, 40, 0, 0, 1, 3, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 1315, 7, 1, 0, 0, 0, 1193, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 91, 0, 0, 0, 5381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6952, 11952, 8453, 5211, 14617, 9754, 3115, 5482] avg=8192.0000 max=14617 max_rank=4 min=3115 min_rank=6 imbalance_ratio=1.784302 max_load_reduction=50919 max_load_reduction_pct=0.776962 moved_to_echo=32968 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 336, 0, 0, 6740, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6584, 0, 0, 0, 103, 7233, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 847, 0, 6654, 418, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4500, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1517, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 39, 136, 8189, 0, 29, 0, 0, 0, 0, 0, 0, 2467, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 0, 6732, 3941, 0, 0, 0, 0, 33, 571, 0, 0, 0, 144, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 38, 0, 13, 0, 0, 0, 0, 0, 0, 0, 942, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8544, 13926, 7938, 4955, 2386, 12468, 12633, 2686] avg=8192.0000 max=13926 max_rank=1 min=2386 min_rank=4 imbalance_ratio=1.699951 max_load_reduction=51610 max_load_reduction_pct=0.787506 moved_to_echo=30173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1139, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 13, 0, 16, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5739, 0, 0, 166, 0, 2296, 0, 0, 0, 0, 221, 0, 1, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 8, 36, 0, 465, 0, 0, 0, 5, 3, 0, 0, 106, 81, 0, 6621, 234, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 178, 0, 0, 0, 0, 0, 0, 0, 1, 0, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 119, 130, 0, 0, 4268, 0, 0, 583, 0, 0, 0, 0, 2789, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 10, 5920, 0, 7394, 0, 0, 0, 0, 13, 47, 2, 0, 157, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 2643, 7031, 1, 0, 0, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 0, 7, 0, 0, 520, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2599, 8437, 8257, 1027, 7903, 15295, 13495, 8523] avg=8192.0000 max=15295 max_rank=5 min=1027 min_rank=3 imbalance_ratio=1.867065 max_load_reduction=50241 max_load_reduction_pct=0.766617 moved_to_echo=45216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 256, 256, 128, 256, 12544, 256, 256, 256, 8576, 128, 24576, 7936, 128, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 0, 128, 7680, 128, 128, 128, 128, 16384, 128, 128, 128, 25216, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1024, 128, 16640, 16896, 128, 0, 0, 128, 4224, 8064, 1664, 0, 128, 256, 16896, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 1536, 128, 128, 128, 1024, 128, 0, 384, 128, 128, 0, 0, 128, 18048, 4864, 28288, 11264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[512, 384, 128, 3968, 384, 6144, 256, 34944, 0, 128, 128, 0, 512, 128, 896, 128, 10624, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 1152, 128, 0, 0, 128, 0, 128, 23936, 128, 128, 128, 256, 384, 128, 128, 33792, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[30336, 128, 256, 256, 6272, 25472, 2176, 256, 0, 768, 256, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 256, 128, 128, 256, 0, 896, 20992, 128, 0, 640, 128, 0, 128, 128, 38528, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 6289, 14, 0, 0, 0, 0, 8155, 0, 0, 1316, 5025, 0, 9, 0, 5688, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 35, 7989, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 676, 0, 0, 4, 4256, 0, 0, 0, 0, 2857, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 28, 0, 0, 0, 0, 2, 2710, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 450, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 4, 0, 0, 5922, 0, 0, 0, 0, 0, 0, 5402, 0, 0, 8, 3749, 0, 0, 0, 244, 0, 7824, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 1, 3, 0, 0, 0, 7, 0, 1, 2, 1, 0, 0, 0, 1, 0, 2, 4754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 6, 0, 8176, 56, 0, 1, 5, 0, 11, 35, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8158, 0, 0, 7, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 1150, 3, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 154, 0, 0, 0, 51, 0, 0, 0, 1, 0, 0, 0, 6945, 0, 0, 0, 0, 6896, 0, 0, 633, 65, 0, 0, 3, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4721, 2, 7, 17, 0, 0, 0, 6249, 1, 0, 0, 0, 0, 0, 0, 0, 1012, 7281, 0, 0, 0, 0, 2, 0, 0, 0, 39, 0, 0, 0, 152, 4589, 73, 1381, 0, 0, 0, 11, 0, 0, 0, 0, 0, 7651, 0, 0, 0, 0, 8147, 0, 0, 51, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4, 8125, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 3623, 0, 0, 0, 3918, 1, 0, 0, 0, 8180, 0, 36, 0, 0, 0, 404, 185, 3, 0, 12, 0, 0, 0, 0, 0, 0, 7929, 0, 241, 8110, 169, 0, 0, 0, 0, 0, 4346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 6248, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 10, 40, 0, 8137, 0, 0, 0, 0, 0, 1, 50, 0, 27, 0, 2286, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 2402, 1, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 941, 0, 0, 0, 122, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 443, 0, 4, 0, 0, 0, 0, 0, 7941, 0, 0, 17, 0, 6, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8186, 0, 0, 1233, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2, 7446, 0, 0, 0, 0, 17, 0, 0, 8182, 0, 0, 1, 3, 0, 0, 0, 297, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 0, 0, 0, 7, 0, 0, 0, 0, 6671, 5, 0, 0, 0, 0, 4150, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 0, 4393, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1, 3306, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8091, 0, 1, 14, 0, 4596, 2, 1123, 0, 0, 0, 0, 0, 18, 0, 46, 1, 1479, 68, 0, 0, 0, 0, 1, 0, 10, 7994, 68, 0, 0, 0, 0, 7564, 853, 0, 0, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5, 7023, 0, 0, 309, 0, 0, 0, 72, 452, 0, 0, 0, 3, 0, 6928, 0, 8152, 0, 0, 24, 0, 0, 0, 0, 0, 96, 0, 0, 0, 6, 81, 0, 0, 0, 1, 2033, 0, 0, 20, 0, 36, 0, 0, 0, 0, 0, 0, 15, 22, 0, 0, 0, 4901, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 34, 1302, 0, 17, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 28, 0, 0, 0, 0, 7809, 0, 0, 0, 7005, 2, 0, 0, 0, 0, 114, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 6806, 0, 0, 642, 0, 0, 0, 0, 43, 121, 0, 0, 662, 0, 0, 0, 0, 8132, 0, 0, 7534, 3639, 0, 0, 1236, 0, 0, 2, 0, 0, 0, 172, 0, 0, 0, 0, 0, 4325, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 3, 8191] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 2, 119, 0, 0, 0, 0, 0, 3, 0, 0, 4322, 0, 0, 3526, 1, 0, 0, 0, 1, 0, 147, 2, 141, 0, 0, 0, 1219, 1445, 0, 0, 0, 0, 7774, 0, 0, 193, 12, 0, 0, 1, 3166, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4157, 0, 76, 7919, 0, 0, 0, 0, 0, 0, 5, 27, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1372, 0, 0, 0, 0, 2, 0, 3730, 406, 8155, 0, 0, 0, 0, 0, 0, 1, 675, 0, 0, 0, 81, 0, 2891, 0, 0, 8143, 0, 458, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 450, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 1173, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 4, 0, 0, 5922, 0, 0, 0, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2514, 0, 0, 0, 244, 0, 7824, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 1, 3, 0, 0, 0, 7, 0, 1, 2, 1205, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 6, 0, 8176, 56, 0, 1, 5, 0, 11, 35, 0, 1337, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8066, 0, 0, 7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10141, 13049, 12668, 6304, 3407, 9837, 2007, 8123] avg=8192.0000 max=13049 max_rank=1 min=2007 min_rank=6 imbalance_ratio=1.592896 max_load_reduction=52487 max_load_reduction_pct=0.800888 moved_to_echo=23374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1150, 3, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 20, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 51, 0, 0, 0, 1, 0, 0, 0, 6945, 0, 0, 0, 2634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 44, 0, 0, 3, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4721, 2, 7, 17, 0, 0, 0, 6249, 1, 0, 1846, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 5211, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 152, 4589, 73, 1381, 0, 0, 0, 11, 0, 0, 0, 0, 1707, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7651, 0, 0, 0, 0, 8147, 0, 0, 51, 0, 0, 0, 0, 0, 0, 2555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4, 8034, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1352, 9785, 691, 12844, 6225, 8176, 18404, 8059] avg=8192.0000 max=18404 max_rank=6 min=691 min_rank=2 imbalance_ratio=2.246582 max_load_reduction=47132 max_load_reduction_pct=0.719177 moved_to_echo=40864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 443, 0, 4, 0, 0, 0, 0, 0, 7941, 0, 0, 17, 0, 2549, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1233, 5461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2, 7446, 0, 0, 0, 0, 17, 0, 0, 8182, 0, 2064, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 297, 5825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 0, 0, 0, 7, 0, 0, 0, 0, 6671, 5, 0, 0, 0, 2028, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4150, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 2, 2, 0, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 0, 4344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11003, 3163, 6710, 17863, 6126, 8972, 7308, 4391] avg=8192.0000 max=17863 max_rank=3 min=3163 min_rank=1 imbalance_ratio=2.180542 max_load_reduction=47673 max_load_reduction_pct=0.727432 moved_to_echo=26797 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 3623, 0, 0, 0, 3918, 1, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 36, 0, 0, 0, 404, 185, 3, 0, 12, 0, 0, 0, 0, 0, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 8110, 115, 0, 0, 0, 0, 0, 4346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 6248, 37, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8, 40, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 50, 0, 27, 0, 2286, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 2402, 1, 0, 123, 0, 0, 0, 0, 0, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 941, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7597, 11848, 12812, 6418, 8195, 4359, 5520, 8787] avg=8192.0000 max=12812 max_rank=2 min=4359 min_rank=5 imbalance_ratio=1.563965 max_load_reduction=52724 max_load_reduction_pct=0.804504 moved_to_echo=26861 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1572, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 6289, 14, 0, 0, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 3369, 0, 9, 0, 5688, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 35, 5719, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 4, 4256, 0, 0, 0, 0, 2857, 0, 0, 0, 0, 2019, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 28, 0, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2710, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9832, 11403, 10534, 3929, 5863, 10057, 3159, 10759] avg=8192.0000 max=11403 max_rank=1 min=3159 min_rank=6 imbalance_ratio=1.391968 max_load_reduction=54133 max_load_reduction_pct=0.826004 moved_to_echo=29838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 34, 1302, 0, 2192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 28, 0, 0, 0, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4696, 2, 0, 0, 0, 0, 114, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6806, 0, 0, 642, 0, 0, 0, 0, 43, 121, 2062, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 5820, 0, 0, 7534, 3639, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 172, 0, 0, 0, 0, 0, 4325, 0, 0, 0, 0, 1934, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 3, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3542, 3032, 4906, 9791, 18891, 6683, 10318, 8373] avg=8192.0000 max=18891 max_rank=4 min=3032 min_rank=1 imbalance_ratio=2.306030 max_load_reduction=46645 max_load_reduction_pct=0.711746 moved_to_echo=44265 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 3306, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8091, 0, 1, 14, 0, 2506, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4596, 2, 1123, 0, 0, 0, 0, 0, 18, 0, 46, 1, 1479, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 5350, 68, 0, 0, 0, 0, 7564, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 9, 0, 1784, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5019, 0, 0, 309, 0, 0, 0, 72, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6928, 0, 8152, 0, 0, 24, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 6, 81, 0, 0, 0, 1, 2033, 0, 0, 20, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 22, 0, 0, 0, 4843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14001, 7333, 13846, 2019, 5857, 15327, 2273, 4880] avg=8192.0000 max=15327 max_rank=5 min=2019 min_rank=3 imbalance_ratio=1.870972 max_load_reduction=50209 max_load_reduction_pct=0.766129 moved_to_echo=28337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 119, 0, 0, 0, 0, 0, 3, 0, 0, 4322, 0, 0, 3526, 1, 3, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 147, 2, 141, 0, 0, 0, 1219, 1445, 0, 0, 0, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 9, 0, 0, 1, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2976, 0, 76, 7919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 27, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1924, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 0, 0, 0, 0, 2, 0, 3730, 406, 8155, 0, 0, 0, 0, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 675, 0, 0, 0, 81, 0, 2891, 0, 0, 8051, 0, 458, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 5923, 3369, 6368, 10972, 2125, 16547, 12164] avg=8192.0000 max=16547 max_rank=6 min=2125 min_rank=5 imbalance_ratio=2.019897 max_load_reduction=48989 max_load_reduction_pct=0.747513 moved_to_echo=41808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[13056, 128, 3200, 128, 128, 256, 5504, 384, 128, 0, 6016, 7552, 9984, 128, 0, 0, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 13312, 0, 0, 128, 128, 18048, 128, 3840, 640, 10240, 0, 0, 128, 128, 128, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 768, 128, 256, 14080, 4352, 17792, 128, 0, 128, 13952, 128, 128, 128, 0, 12928, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 256, 11520, 21632, 128, 128, 128, 9216, 0, 19968, 3200, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 128, 256, 37376, 128, 128, 768, 0, 128, 128, 6272, 8320, 128, 12160, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 4992, 16256, 512, 128, 128, 0, 3712, 128, 0, 0, 24832, 128, 128, 4864, 128, 10112, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 768, 0, 128, 0, 1408, 34048, 128, 128, 23936, 3712, 128, 128, 1408, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 8448, 2816, 0, 128, 1152, 128, 2944, 128, 128, 41600, 0, 512, 128, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 5986, 0, 0, 0, 777, 0, 8, 76, 0, 7, 69, 0, 0, 0, 0, 0, 4166, 0, 0, 0, 8, 2, 0, 0, 4, 0, 0, 8, 0, 534, 1728, 2, 0, 0, 0, 0, 7579, 0, 0, 1, 3419, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5619, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4742, 0, 0, 3376, 0, 2165, 0, 0, 0, 7631, 0, 0, 0, 11, 0, 0, 3461, 1, 0, 0, 0, 0, 0, 2765, 0, 0, 155, 0, 0, 6, 0, 43, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8187, 0, 0, 254, 0, 5, 0, 0, 0, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4710, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 33, 6306, 0, 0, 0, 140, 8147, 18, 0, 0, 6827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7657, 40, 39, 0, 0, 0, 0, 0, 0, 13, 7861, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 870, 477, 0, 0, 0, 3, 0, 7156, 0, 0, 0, 0, 3273, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 144, 0, 0, 109, 8169, 0, 0, 976, 287, 0, 0, 0, 0, 0, 0, 5, 0, 67, 125, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 249, 1788, 0, 0, 0, 0, 2, 0, 0, 13, 96, 0, 0, 0, 636, 0, 6800, 0, 0, 345, 6718, 0, 0, 0, 0, 0, 14, 1033, 0, 212, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5812, 0, 0, 0, 8137, 0, 0, 0, 0, 10, 3929, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 1, 0, 0, 4, 0, 2, 0, 0, 0, 5160, 0, 0, 0, 0, 2561, 0, 0, 0, 0, 0, 0, 1261, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 7957, 0, 3, 833, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4, 2237, 0, 0, 1, 148, 0, 6, 8, 171, 0, 3966, 0, 0, 0, 1, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4, 3181, 0, 0, 45, 0, 0, 0, 0, 0, 0, 8051, 0, 12, 0, 4231, 0, 726, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 206, 255, 1, 5823, 13, 0, 11, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 10, 0, 0, 7245, 0, 0, 0, 2, 2, 7279, 0, 0, 0, 2, 0, 19, 0, 0, 0, 1, 8089, 0, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 580, 0, 0, 0, 30, 0, 0, 21, 0, 0, 0, 0, 0, 0, 2, 0, 4787, 48, 0, 0, 0, 0, 177, 4, 0, 0, 0, 0, 1, 0, 8186, 0, 2, 0, 0, 0, 240, 23, 0, 0, 6806, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 6, 321, 0, 37, 0, 688, 0, 8140, 0, 0, 0, 0, 118, 0, 22, 0, 7662, 3, 58, 0, 0, 7058, 0, 2, 3, 0, 3229, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 233, 0, 0, 0, 82, 3, 7296, 0, 0, 11, 0, 113, 0, 418, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 24, 5392, 0, 9, 0, 0, 0, 0, 0, 0, 161, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8055, 0, 0, 0, 0, 0, 7258, 0, 0, 5, 8180, 0, 0, 0, 6268, 71, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 253, 0, 0, 7929, 0, 0, 0, 0, 0, 5, 0, 124, 0, 0, 0, 0, 70, 0, 0, 2134, 0, 0, 0, 572, 0, 3, 0, 0, 3, 0, 36, 0, 3, 0, 0, 0, 0, 2475, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 8019, 0, 0, 32, 0, 0, 0, 0, 0, 7777, 7, 0, 0, 0, 0, 0, 450, 0, 120] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4005, 2, 0, 0, 0, 3, 0, 1, 0, 0, 250, 0, 0, 0, 0, 4, 6192, 0, 0, 0, 4186, 0, 0, 0, 0, 0, 65, 235, 0, 0, 0, 24, 0, 0, 0, 0, 2214, 0, 0, 0, 8090, 0, 0, 0, 4272, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 1541, 0, 0, 5097, 0, 142, 0, 0, 0, 0, 0, 7879, 0, 0, 7885, 0, 18, 0, 0, 0, 0, 0, 0, 296, 0, 0, 155, 0, 258, 0, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 7161, 0, 0, 1659, 0, 0, 0, 0, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 1435, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 476, 0, 0, 0, 0, 1, 1267, 0, 1, 131, 0, 3, 0, 0, 7686, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 47, 0, 8, 0, 7630, 3, 3, 0, 4, 1, 23, 0, 0, 0, 280, 0, 0, 0, 0, 0, 10, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 52, 4468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 6417, 3, 2, 92, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 3535, 4, 0, 0, 0, 0, 0, 8027, 0, 0, 0, 2, 8035, 0, 12, 0, 516, 39, 0, 0, 0, 0, 0, 0, 27, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5986, 0, 0, 0, 777, 0, 8, 76, 0, 7, 69, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 8, 2, 0, 0, 4, 0, 0, 8, 0, 534, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7579, 0, 0, 1, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5619, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4742, 0, 0, 1481, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 0, 2165, 0, 0, 0, 7631, 0, 0, 0, 11, 0, 0, 3461, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 0, 0, 155, 0, 0, 6, 0, 43, 145, 0, 0, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 5, 0, 0, 0, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1593, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7140, 2810, 9303, 12280, 15665, 6318, 6596, 5424] avg=8192.0000 max=15665 max_rank=4 min=2810 min_rank=1 imbalance_ratio=1.912231 max_load_reduction=49871 max_load_reduction_pct=0.760971 moved_to_echo=34003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4710, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 33, 6306, 0, 0, 0, 7109, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 1029, 18, 0, 0, 6827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3854, 40, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7861, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8, 0, 3317, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 870, 477, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3273, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 144, 0, 0, 109, 6579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 976, 287, 0, 0, 0, 0, 0, 0, 5, 0, 67, 125, 0, 0, 0, 0, 1590, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20516, 8014, 3934, 12054, 1351, 9735, 6835, 3097] avg=8192.0000 max=20516 max_rank=0 min=1351 min_rank=4 imbalance_ratio=2.504395 max_load_reduction=45020 max_load_reduction_pct=0.686951 moved_to_echo=21018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 580, 0, 0, 0, 30, 0, 0, 21, 0, 0, 0, 0, 0, 0, 2, 7613, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 48, 0, 0, 0, 0, 177, 4, 0, 0, 0, 0, 1, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 240, 23, 0, 0, 3418, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 6, 2951, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 37, 0, 688, 0, 8140, 0, 0, 0, 0, 118, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 58, 0, 0, 7058, 0, 2, 3, 0, 3229, 531, 0, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 233, 0, 0, 0, 82, 3, 5872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 113, 0, 418, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7999, 1424, 95, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8683, 9014, 3685, 3473, 9231, 14568, 6769, 10113] avg=8192.0000 max=14568 max_rank=5 min=3473 min_rank=3 imbalance_ratio=1.778320 max_load_reduction=50968 max_load_reduction_pct=0.777710 moved_to_echo=40681 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 249, 1788, 0, 0, 0, 0, 2, 0, 0, 13, 96, 0, 0, 0, 636, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 0, 345, 6718, 0, 0, 0, 0, 0, 14, 1033, 0, 212, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3929, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 1, 2518, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 5160, 0, 0, 0, 0, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 5230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 6408, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 1, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3153, 11367, 11062, 8214, 7727, 6648, 6779, 10586] avg=8192.0000 max=11367 max_rank=1 min=3153 min_rank=0 imbalance_ratio=1.387573 max_load_reduction=54169 max_load_reduction_pct=0.826553 moved_to_echo=31740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 24, 5392, 0, 9, 0, 0, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7258, 0, 0, 5, 4117, 0, 0, 0, 6268, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 253, 0, 0, 3544, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 0, 5, 0, 124, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2134, 0, 0, 0, 572, 0, 3, 0, 0, 3, 0, 36, 0, 3, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2475, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 6458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 7777, 7, 0, 0, 0, 0, 0, 450, 0, 120, 1561, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5944, 8145, 17719, 3817, 5827, 2874, 8962, 12248] avg=8192.0000 max=17719 max_rank=2 min=2874 min_rank=5 imbalance_ratio=2.162964 max_load_reduction=47817 max_load_reduction_pct=0.729630 moved_to_echo=29911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 2237, 0, 0, 1, 148, 0, 6, 8, 171, 0, 3966, 0, 0, 0, 1, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 4052, 0, 12, 0, 4231, 0, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 206, 255, 1, 5823, 13, 0, 3488, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 10, 0, 0, 7245, 0, 0, 0, 6252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 7279, 0, 0, 0, 2, 0, 19, 0, 0, 0, 1, 6515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7053, 4249, 9066, 10650, 2945, 13556, 13820, 4197] avg=8192.0000 max=13820 max_rank=6 min=2945 min_rank=4 imbalance_ratio=1.687012 max_load_reduction=51716 max_load_reduction_pct=0.789124 moved_to_echo=34518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4005, 2, 0, 0, 0, 3, 0, 1, 0, 0, 250, 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 782, 0, 0, 0, 4186, 0, 0, 0, 0, 0, 65, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 2214, 0, 0, 0, 4071, 0, 0, 0, 4272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 1541, 0, 0, 3505, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 0, 142, 0, 0, 0, 0, 0, 7879, 0, 0, 7885, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 155, 0, 258, 0, 10, 3, 0, 0, 0, 4762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 5767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1659, 0, 0, 0, 0, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 1435, 0, 1394, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4775, 5272, 10581, 5771, 19542, 5484, 5771, 8340] avg=8192.0000 max=19542 max_rank=4 min=4775 min_rank=0 imbalance_ratio=2.385498 max_load_reduction=45994 max_load_reduction_pct=0.701813 moved_to_echo=39137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 476, 0, 0, 0, 0, 1, 1267, 0, 1, 131, 0, 3, 0, 6376, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 47, 0, 8, 0, 7630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4, 1, 23, 0, 0, 0, 142, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 52, 121, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 92, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 4, 0, 0, 0, 0, 0, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8035, 0, 12, 0, 516, 39, 0, 0, 0, 0, 0, 0, 27, 0, 1562, 1297, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 16665, 186, 9277, 3248, 6353, 10004, 11531] avg=8192.0000 max=16665 max_rank=1 min=186 min_rank=2 imbalance_ratio=2.034302 max_load_reduction=48871 max_load_reduction_pct=0.745712 moved_to_echo=31136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3840, 384, 10880, 0, 512, 0, 13312, 128, 128, 128, 128, 8320, 0, 512, 1536, 8192, 12288, 6272, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[15104, 0, 5376, 0, 768, 0, 15872, 128, 14080, 512, 128, 8064, 0, 6144, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 17408, 128, 0, 128, 24320, 128, 128, 0, 22912, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 17792, 3712, 512, 128, 1024, 128, 128, 5504, 1536, 128, 4480, 6528, 0, 128, 640, 21120, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 128, 128, 0, 15360, 0, 256, 512, 0, 3584, 640, 128, 7296, 256, 384, 0, 35456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 5888, 128, 128, 384, 25856, 128, 256, 128, 128, 128, 128, 1280, 128, 896, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 17664, 128, 128, 0, 128, 128, 128, 256, 256, 128, 20608, 1024, 128, 20992, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 9856, 128, 0, 640, 3584, 640, 128, 256, 0, 128, 128, 50688, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 98, 0, 0, 0, 2889, 0, 5635, 0, 0, 0, 8052, 0, 0, 8073, 0, 7844, 0, 0, 8115, 0, 30, 0, 0, 0, 2, 0, 0, 89, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 0, 150, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1323, 0, 0, 0, 24, 0, 0, 5, 1130, 0, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 0, 0, 1177, 0, 2109] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 31, 42, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 80, 0, 0, 0, 536, 0, 7947, 0, 0, 0, 8174, 0, 0, 0, 65, 8084, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 8080, 0, 2, 5, 0, 1, 0, 0, 7211, 134, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 12, 0, 1, 0, 0, 1, 0, 1992, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8188, 0, 2, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 0, 4950] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[11, 8, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 20, 0, 4, 0, 0, 4262, 0, 166, 0, 0, 0, 8074, 0, 10, 0, 0, 7419, 0, 0, 1130, 0, 0, 2, 0, 650, 1, 12, 0, 0, 7776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8131, 28, 0, 0, 1, 6575, 0, 0, 0, 0, 77, 0, 67, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 7992, 0, 8, 0, 0, 1126, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 775, 0, 0, 0, 0, 0, 731, 3, 0, 0, 0, 0, 212, 1, 0, 0, 0, 0, 0, 254, 0, 1493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[6, 8082, 0, 0, 0, 58, 0, 0, 44, 13, 34, 75, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 1, 17, 0, 0, 0, 1, 0, 0, 7441, 0, 8135, 0, 0, 22, 0, 0, 0, 0, 64, 161, 0, 0, 0, 0, 3, 0, 0, 0, 0, 191, 6082, 0, 0, 7970, 0, 0, 0, 100, 8032, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 72, 33, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 667, 593, 0, 0, 0, 19, 2144, 0, 7090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 17, 0, 79, 6034, 0, 0, 0, 0, 3021, 0, 0, 37, 0, 8192, 2, 41, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 621, 0, 0, 0, 8154, 0, 0, 35, 3574, 8159, 0, 0, 0, 0, 651, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 4636, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7393, 0, 0, 0, 0, 688, 0, 0, 140, 0, 8081, 0, 0, 0, 0, 759, 0, 11, 0, 0, 0, 0, 0, 0, 6607, 0, 215, 0, 1393, 0, 0, 0, 7975, 0, 0, 100, 0, 0, 0, 83, 0, 0, 0, 0, 0, 3, 0, 8102, 0, 0, 12, 0, 0, 0, 721, 0, 0, 0, 3, 0, 0, 8, 0, 1, 1, 0, 101, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 7906, 4, 0, 0, 0, 6486, 1, 0, 0, 145, 0, 0, 0, 0, 15, 3, 0, 8136, 0, 0, 77, 0, 0, 14, 0, 0, 0, 0, 304, 0, 0, 0, 0, 14, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 7, 41, 0, 0, 0, 0, 0, 0, 3202, 0, 4705, 0, 0, 0, 1819, 0, 0, 0, 0, 5289, 0, 0, 0, 0, 7639, 0, 1195, 0, 0, 3, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 5205, 0, 6934, 0, 2, 8109, 0, 0, 0, 60, 37, 2139, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 2435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 4103, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 58, 0, 3956] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 216, 15, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 8192, 1016, 0, 0, 1, 0, 8191, 0, 389, 0, 8158, 0, 0, 4, 33, 22, 0, 0, 0, 0, 17, 0, 12, 0, 6443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 42, 7964, 8172, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 79, 7826] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 7, 41, 0, 0, 0, 0, 25, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3202, 0, 4465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5619, 0, 1195, 0, 0, 3, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5205, 0, 6934, 0, 2, 0, 0, 0, 0, 60, 7, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8109, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 2435, 2020, 454, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 4103, 0, 228, 0, 0, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 58, 0, 3956, 1919, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4577, 7667, 6848, 14347, 9756, 8745, 7543, 6053] avg=8192.0000 max=14347 max_rank=3 min=4577 min_rank=0 imbalance_ratio=1.751343 max_load_reduction=51189 max_load_reduction_pct=0.781082 moved_to_echo=32097 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 31, 42, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 80, 0, 4990, 3621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 7541, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 8080, 0, 2, 0, 0, 1, 0, 0, 1065, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 12, 0, 1, 0, 0, 1, 0, 1992, 0, 0, 0, 0, 8190, 0, 694, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8188, 0, 2, 0, 1, 0, 0, 4456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 0, 4950, 2934, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8788, 8151, 32, 9373, 4562, 12903, 12661, 9066] avg=8192.0000 max=12903 max_rank=5 min=32 min_rank=2 imbalance_ratio=1.575073 max_load_reduction=52633 max_load_reduction_pct=0.803116 moved_to_echo=39192 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 98, 0, 5470, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2889, 0, 5347, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 23, 0, 0, 0, 2, 0, 0, 89, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 4486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1323, 0, 0, 0, 24, 0, 0, 5, 1130, 0, 0, 0, 0, 896, 7, 673, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 4322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 0, 0, 1177, 0, 2109, 2849, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9140, 16309, 8239, 3099, 4636, 4347, 12347, 7419] avg=8192.0000 max=16309 max_rank=1 min=3099 min_rank=3 imbalance_ratio=1.990845 max_load_reduction=49227 max_load_reduction_pct=0.751144 moved_to_echo=28749 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 450, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 17, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 6023, 2, 41, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 8154, 0, 0, 35, 3574, 0, 0, 0, 0, 0, 97, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8159, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 2169, 261, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 0, 8190, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4636, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8036, 1096, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3491, 2577, 6104, 12484, 11521, 2581, 11568, 15210] avg=8192.0000 max=15210 max_rank=7 min=2577 min_rank=1 imbalance_ratio=1.856689 max_load_reduction=50326 max_load_reduction_pct=0.767914 moved_to_echo=40880 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 8082, 0, 0, 0, 58, 0, 0, 44, 13, 34, 75, 0, 0, 0, 0, 5559, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7441, 0, 5984, 0, 0, 22, 0, 0, 0, 0, 64, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 191, 6082, 0, 0, 0, 0, 0, 0, 100, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 72, 33, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 667, 593, 0, 0, 0, 19, 2144, 0, 7090, 0, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13872, 64, 13672, 7561, 8127, 2258, 8165, 11817] avg=8192.0000 max=13872 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.693359 max_load_reduction=51664 max_load_reduction_pct=0.788330 moved_to_echo=30367 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 8, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 20, 0, 4, 4550, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4262, 0, 158, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1130, 0, 0, 2, 0, 650, 1, 12, 0, 0, 7776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 28, 0, 0, 1, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 67, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 8131, 4498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 7992, 0, 8, 0, 0, 1126, 0, 637, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 775, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 3, 0, 0, 0, 0, 212, 1, 0, 0, 0, 0, 0, 254, 0, 1493, 2694, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 4430, 9571, 9190, 12830, 10025, 4874, 6442] avg=8192.0000 max=12830 max_rank=4 min=4430 min_rank=1 imbalance_ratio=1.566162 max_load_reduction=52706 max_load_reduction_pct=0.804230 moved_to_echo=34171 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 216, 15, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 34, 0, 5516, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1016, 0, 0, 1, 0, 0, 0, 389, 0, 8158, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 22, 0, 0, 0, 0, 17, 0, 12, 0, 6443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 42, 1168, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 4562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 79, 7826, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9412, 17760, 6527, 9384, 5167, 8088, 1, 9197] avg=8192.0000 max=17760 max_rank=1 min=1 min_rank=6 imbalance_ratio=2.167969 max_load_reduction=47776 max_load_reduction_pct=0.729004 moved_to_echo=22453 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7393, 0, 0, 0, 0, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 7668, 0, 0, 0, 0, 759, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4860, 0, 215, 0, 1393, 0, 0, 0, 7975, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 0, 0, 0, 3, 0, 0, 8, 0, 1, 1, 0, 101, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 7906, 4, 0, 0, 0, 6486, 1747, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 145, 0, 0, 0, 0, 15, 3, 0, 8136, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 304, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8578, 14543, 98, 8937, 16589, 8377, 332] avg=8192.0000 max=16589 max_rank=5 min=98 min_rank=3 imbalance_ratio=2.025024 max_load_reduction=48947 max_load_reduction_pct=0.746872 moved_to_echo=34235 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[768, 4736, 1408, 0, 0, 128, 640, 768, 768, 0, 0, 128, 128, 3712, 128, 35584, 11520, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 4352, 384, 0, 128, 128, 0, 128, 128, 21376, 128, 128, 128, 128, 27264, 8192, 2816, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 19328, 1024, 27776, 128, 128, 128, 0, 896, 512, 8192, 8320, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 12160, 128, 0, 128, 128, 128, 0, 128, 128, 7936, 128, 0, 128, 256, 768, 26624, 17920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[640, 128, 128, 0, 23552, 256, 21120, 128, 3584, 0, 128, 128, 128, 256, 5760, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[16768, 128, 22528, 128, 1536, 768, 1536, 128, 128, 128, 22400, 256, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 256, 128, 0, 128, 1792, 128, 32896, 896, 12288, 0, 256, 128, 0, 17536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 0, 0, 128, 1408, 128, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 40704, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 0, 2651, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 6682, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 7531, 5580, 7374, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2049, 1036, 0, 6404, 48, 7, 7, 6268, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 0, 3696, 75, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 165, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 3, 0, 0, 0, 0, 10, 0, 492, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 889, 0, 0, 0, 1, 94, 0, 0, 0, 7254, 35, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3591, 0, 2937, 305, 63, 0, 8090, 0, 21, 118, 0, 0, 0, 0, 0, 0, 0, 7845, 61, 34, 0, 0, 7027, 4494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 3003, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 0, 8142, 91, 0, 0, 0, 115, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 2494, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 4986, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 45, 0, 0, 0, 4539, 0, 0, 1248, 50, 0, 0, 7538, 0, 0, 0, 25, 0, 0, 0, 766, 0, 0, 366, 0, 0, 205, 7773, 264, 289, 0, 0, 0, 0, 0, 186, 0, 0, 0, 34, 0, 0, 3545, 0, 0, 469, 0, 0, 0, 0, 0, 7179, 0, 10, 1, 124, 0, 0, 0, 0, 0, 88, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 681, 469, 0, 0, 0, 0, 39, 0, 1, 0, 7756, 17, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 18, 0, 25, 66, 0, 8188, 0, 459, 0, 0, 158, 4, 3, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 1340, 8192, 0, 0, 0, 0, 0, 0, 0, 35, 4, 0, 0, 0, 0, 130, 79, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 4, 0, 0, 0, 207, 294, 0, 0, 0, 0, 2133, 8095, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 12, 7, 7847, 0, 0, 0, 1, 0, 0, 0, 4028, 6945, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 4, 7949, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2556, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 287, 0, 7909, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 1, 0, 0, 6893, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 27, 14, 3573, 0, 0, 7104, 0, 0, 0, 8, 0, 2, 0, 2, 4245, 0, 0, 0, 0, 871, 0, 0, 0, 0, 0, 0, 1, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 3200, 0, 7, 0, 0, 410, 2607, 13, 0, 0, 1442] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 0, 0, 3, 0, 0, 0, 1, 0, 0, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 12, 6745, 0, 8192, 406, 0, 0, 0, 0, 7167, 0, 0, 4144, 0, 7357, 0, 0, 381, 0, 8, 7509, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 485, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 65, 4951, 2, 0, 0, 0, 901, 0, 9, 0, 0, 496, 0, 0, 0, 2829, 7978, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 539, 37, 0, 0, 0, 2, 0, 0, 17, 37, 0, 0, 0, 0, 4, 0, 0, 0, 44, 0, 4771, 196, 0, 0, 0, 0, 772, 0, 0, 0, 1817, 0, 2, 0, 0, 0, 7849, 0, 0, 1360, 377, 0, 3998, 0, 0, 39, 0, 0, 0, 0, 514, 0, 5123, 7571, 2, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 47, 21, 1, 5, 0, 0, 1, 0, 0, 1208, 0, 136, 0, 0, 3, 101, 0, 0, 16, 0, 0, 6039, 0, 706, 5874, 6966, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 202, 0, 0, 2660, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 634, 0, 0, 0, 0, 0, 1381, 0, 199, 0, 1170, 0, 353, 0, 0, 0, 0, 5, 0, 0, 0, 0, 22, 3310, 0, 0, 0, 12, 346, 7525, 5, 43, 0, 0, 0, 0, 0, 32, 0, 0, 6620, 0, 0, 1076, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 35, 105, 0, 538, 0, 0, 0, 0, 18, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7914, 0, 11, 298, 0, 0, 17, 8148, 2655, 8188, 4251, 57, 0, 0, 6826, 0, 0, 524, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 0, 2651, 552, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 6682, 686, 0, 0, 0, 0, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 5580, 7374, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2049, 479, 0, 6404, 48, 7, 7, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 0, 3696, 75, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 165, 0, 0, 0, 63, 0, 0, 0, 3, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 18, 0, 0, 0, 0, 0, 0, 0, 5786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 10, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10841, 16516, 12976, 9019, 6156, 714, 7158, 2156] avg=8192.0000 max=16516 max_rank=1 min=714 min_rank=5 imbalance_ratio=2.016113 max_load_reduction=49020 max_load_reduction_pct=0.747986 moved_to_echo=16184 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 2494, 0, 0, 0, 0, 0, 0, 0, 476, 4539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 4986, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 1248, 50, 0, 0, 7538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 766, 0, 0, 366, 0, 0, 205, 7773, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 34, 0, 0, 3545, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7179, 0, 10, 1, 124, 0, 0, 0, 0, 0, 88, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 681, 469, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 1, 0, 7756, 17, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15672, 10149, 8881, 9424, 4234, 7423, 1938, 7815] avg=8192.0000 max=15672 max_rank=0 min=1938 min_rank=6 imbalance_ratio=1.913086 max_load_reduction=49864 max_load_reduction_pct=0.760864 moved_to_echo=21410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 889, 0, 3591, 758, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 94, 0, 0, 0, 7254, 35, 0, 0, 0, 0, 0, 0, 2, 878, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2937, 305, 63, 0, 5419, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 3661, 61, 34, 0, 0, 7027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 2, 9, 0, 0, 3426, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3003, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 7948, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 0, 0, 91, 0, 0, 0, 115, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6472, 9843, 8747, 10901, 12682, 11301, 4149, 1441] avg=8192.0000 max=12682 max_rank=4 min=1441 min_rank=7 imbalance_ratio=1.548096 max_load_reduction=52854 max_load_reduction_pct=0.806488 moved_to_echo=29573 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 202, 0, 0, 2660, 0, 0, 0, 342, 0, 0, 0, 0, 1381, 320, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 634, 0, 0, 0, 0, 0, 0, 0, 199, 0, 1170, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 22, 1545, 0, 0, 0, 12, 346, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 32, 0, 0, 6620, 0, 0, 1076, 0, 2, 0, 1445, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 35, 105, 0, 538, 0, 0, 0, 0, 7993, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7914, 0, 11, 298, 6947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4760, 2655, 0, 4251, 57, 0, 0, 6826, 0, 0, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5100, 3458, 2244, 1935, 9265, 9254, 15190, 19090] avg=8192.0000 max=19090 max_rank=7 min=1935 min_rank=3 imbalance_ratio=2.330322 max_load_reduction=46446 max_load_reduction_pct=0.708710 moved_to_echo=52799 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 0, 0, 3, 0, 0, 0, 1, 0, 0, 784, 3, 711, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 6745, 0, 5486, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7167, 0, 0, 4144, 0, 3433, 0, 0, 381, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 1, 0, 0, 3213, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7788, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 4951, 2, 0, 0, 0, 901, 0, 9, 0, 0, 6931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 2829, 0, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1715, 1599, 12649, 15133, 8497, 8855, 12861, 4227] avg=8192.0000 max=15133 max_rank=3 min=1599 min_rank=1 imbalance_ratio=1.847290 max_load_reduction=50403 max_load_reduction_pct=0.769089 moved_to_echo=34440 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 539, 37, 0, 0, 0, 2, 0, 0, 17, 37, 0, 0, 0, 7849, 732, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 44, 0, 4771, 196, 0, 0, 0, 0, 772, 0, 0, 293, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1817, 0, 2, 0, 0, 0, 0, 0, 0, 1360, 377, 0, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 514, 0, 5123, 3532, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 3307, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 47, 21, 1, 5, 6800, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1208, 0, 136, 0, 0, 3, 101, 0, 0, 16, 0, 7509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6039, 0, 413, 5874, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9379, 6860, 6235, 9210, 4873, 7503, 8974, 12502] avg=8192.0000 max=12502 max_rank=7 min=4873 min_rank=4 imbalance_ratio=1.526123 max_load_reduction=53034 max_load_reduction_pct=0.809235 moved_to_echo=33852 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 25, 66, 0, 8188, 0, 459, 0, 0, 158, 4, 3, 0, 0, 130, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 1340, 8192, 0, 0, 0, 0, 0, 0, 1, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 5481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 207, 138, 0, 0, 0, 0, 2133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 12, 7, 7847, 0, 0, 0, 128, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4028, 6945, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 3, 7949, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9079, 11134, 5599, 2482, 9105, 11605, 7473, 9059] avg=8192.0000 max=11605 max_rank=5 min=2482 min_rank=3 imbalance_ratio=1.416626 max_load_reduction=53931 max_load_reduction_pct=0.822922 moved_to_echo=37242 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2556, 0, 0, 0, 0, 0, 15, 0, 0, 3398, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 7909, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 1, 0, 0, 6893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 27, 14, 3573, 0, 0, 7104, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 2, 4245, 0, 0, 0, 0, 871, 0, 0, 0, 0, 0, 3123, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4767, 0, 0, 0, 7, 0, 0, 410, 2607, 13, 0, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7278, 5977, 8205, 7432, 10724, 8881, 7793, 9246] avg=8192.0000 max=10724 max_rank=4 min=5977 min_rank=1 imbalance_ratio=1.309082 max_load_reduction=54812 max_load_reduction_pct=0.836365 moved_to_echo=36644 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 7168, 128, 12928, 19456, 0, 4352, 128, 7808, 128, 8320, 2816, 512, 640, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 8064, 128, 3840, 128, 128, 0, 14592, 8320, 13952, 0, 0, 0, 896, 0, 1536, 9216, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 7296, 0, 640, 4992, 7424, 12800, 1024, 6528, 512, 256, 24192, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 640, 384, 128, 20096, 5760, 512, 0, 0, 640, 128, 128, 896, 1280, 32256, 2688, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 256, 128, 0, 1280, 4992, 1536, 128, 0, 128, 1024, 8448, 128, 768, 768, 46720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 4608, 21376, 128, 128, 256, 256, 896, 640, 128, 128, 128, 128, 33664, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 128, 256, 128, 128, 128, 24576, 128, 128, 23296, 1152, 128, 512, 12032, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 2560, 128, 128, 0, 128, 0, 0, 5760, 10880, 2944, 16384, 128, 19328, 8064, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 29, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7845, 0, 2607, 0, 0, 7471, 8, 0, 25, 0, 1, 8050, 0, 0, 0, 0, 0, 2, 2, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, 2, 6, 0, 2, 0, 8057, 1, 0, 1, 0, 0, 1, 0, 20, 486, 0, 0, 0, 0, 0, 0, 2, 6527, 0, 142, 0, 0, 0, 16, 6935, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7, 0, 0, 90, 306, 0, 2, 7954, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3313, 0, 1, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 4672, 0, 693, 0, 1223, 0, 92, 0, 0, 8043, 18, 353, 0, 485, 1049, 0, 14, 1778, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 3943, 2044, 0, 0, 0, 272, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 8188, 0, 0, 0, 0, 0, 249, 0, 0, 2760, 0, 0, 0, 0, 0, 2, 0, 0, 0, 154, 3, 0, 3676, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 25, 0, 6404, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3182, 0, 0, 95, 0, 0, 5, 0, 0, 728, 5, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7667, 0, 0, 0, 0, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 5837, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 0, 163, 6634, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 613, 0, 0, 6, 0, 0, 0, 0, 900, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 0, 0, 0, 1, 8, 8063, 0, 0, 43, 6272] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 313, 8139, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 3, 6423, 0, 0, 0, 5951, 0, 0, 0, 0, 0, 8120, 1798, 0, 2, 0, 0, 3, 0, 0, 0, 0, 25, 26, 73, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 1, 64, 0, 0, 0, 0, 0, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 7575, 1013, 0, 0, 641, 0, 0, 0, 3775, 91, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6905, 6075, 0, 0, 1, 1360, 0, 0, 0, 0, 0, 7244, 0, 0, 0, 0, 0, 161, 92, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 10, 5262, 322, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1561, 1, 145, 0, 0, 24, 3759, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 8191, 0, 4693, 0, 0, 0, 1096, 0, 5035, 0, 0, 0, 50, 0, 412, 0, 745, 492, 49, 0, 105, 713, 47, 0, 0, 0, 0, 8083, 0, 0, 0, 2799] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 587, 0, 0, 0, 0, 0, 0, 0, 2, 32, 349, 0, 0, 0, 0, 8034, 0, 0, 0, 216, 5449, 0, 0, 12, 1, 0, 8065, 85, 0, 5, 0, 473, 0, 11, 0, 0, 0, 0, 0, 0, 0, 34, 239, 37, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7621, 2, 0, 0, 7, 0, 7944, 0, 0, 0, 0, 8, 7330, 0, 5, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8, 0, 2931, 0, 0, 5, 477] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4850, 0, 3, 0, 0, 0, 33, 0, 4, 30, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7289, 2, 18, 0, 0, 610, 0, 0, 0, 0, 0, 8192, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 0, 0, 0, 0, 6179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 8147, 0, 0, 6, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 458, 1184, 0, 2082, 0, 0, 0, 0, 0, 2012, 8160, 0, 0, 22, 0, 0, 0, 0, 1711, 0, 0, 21, 0, 0, 1, 0, 0, 0, 8101, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 440, 0, 1804, 1283, 0, 0, 0, 0, 0, 2049, 0, 188, 672, 0, 0, 0, 0, 21, 0, 2, 0, 10, 2863, 8192, 5, 0, 28, 29, 0, 0, 0, 0, 0, 7641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6899, 0, 0, 0, 0, 87, 0, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 6546, 0, 9, 1, 0, 3351, 6892, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7320, 47, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3313, 0, 1, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 4672, 0, 693, 0, 1223, 0, 92, 0, 0, 3087, 18, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 1049, 0, 14, 1778, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 3943, 2044, 0, 0, 1560, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 3312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 2760, 0, 0, 0, 0, 0, 2, 0, 4876, 2653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 3, 0, 3676, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 25, 0, 5686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 17827, 7206, 8478, 3597, 10540, 3972, 5713] avg=8192.0000 max=17827 max_rank=1 min=3597 min_rank=4 imbalance_ratio=2.176147 max_load_reduction=47709 max_load_reduction_pct=0.727982 moved_to_echo=23822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 29, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 3312, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7471, 8, 0, 25, 0, 1, 3895, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6184, 0, 0, 0, 1521, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, 2, 6, 0, 2, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 20, 486, 0, 0, 0, 0, 0, 0, 2, 4798, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6527, 0, 142, 0, 0, 0, 16, 6935, 0, 0, 0, 0, 0, 0, 2, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 90, 306, 0, 2, 7063, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4955, 5619, 11404, 10489, 3839, 7973, 13787, 7470] avg=8192.0000 max=13787 max_rank=6 min=3839 min_rank=4 imbalance_ratio=1.682983 max_load_reduction=51749 max_load_reduction_pct=0.789627 moved_to_echo=33069 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 313, 8139, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 2, 6423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5951, 0, 0, 0, 0, 0, 3929, 1798, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 26, 73, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 1, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 0, 0, 641, 0, 0, 0, 3775, 91, 0, 0, 0, 0, 0, 0, 0, 4511, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7265, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9958, 6718, 11683, 5998, 11014, 12718, 181, 7266] avg=8192.0000 max=12718 max_rank=5 min=181 min_rank=6 imbalance_ratio=1.552490 max_load_reduction=52818 max_load_reduction_pct=0.805939 moved_to_echo=31179 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3182, 0, 0, 95, 0, 0, 5, 0, 0, 728, 5, 0, 0, 0, 0, 0, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7667, 0, 0, 0, 0, 0, 3960, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 5837, 0, 0, 0, 0, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 0, 163, 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 613, 0, 3950, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 900, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 0, 0, 0, 1, 8, 7159, 0, 0, 43, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5531, 8086, 11628, 6652, 2860, 7274, 9103, 14402] avg=8192.0000 max=14402 max_rank=7 min=2860 min_rank=4 imbalance_ratio=1.758057 max_load_reduction=51134 max_load_reduction_pct=0.780243 moved_to_echo=33639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 440, 0, 1804, 1283, 0, 0, 0, 0, 0, 1208, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2049, 0, 188, 672, 0, 0, 0, 0, 21, 0, 2, 0, 10, 1100, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 29, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6899, 0, 0, 0, 0, 87, 0, 7635, 0, 0, 0, 555, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 6546, 0, 9, 1, 0, 3351, 2788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4104, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6500, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 12239, 3755, 15844, 12724, 7108, 1169, 6547] avg=8192.0000 max=15844 max_rank=3 min=1169 min_rank=6 imbalance_ratio=1.934082 max_load_reduction=49692 max_load_reduction_pct=0.758240 moved_to_echo=27548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2915, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1360, 0, 0, 0, 0, 0, 3506, 0, 0, 0, 0, 0, 161, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 10, 5262, 322, 0, 0, 1339, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1561, 1, 145, 0, 0, 24, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 2238, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 4693, 0, 0, 0, 1096, 0, 5035, 0, 0, 0, 50, 0, 412, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 492, 49, 0, 105, 713, 47, 0, 0, 0, 0, 7177, 0, 0, 0, 2799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4259, 8733, 5120, 7675, 3253, 4724, 19645, 12127] avg=8192.0000 max=19645 max_rank=6 min=3253 min_rank=4 imbalance_ratio=2.398071 max_load_reduction=45891 max_load_reduction_pct=0.700241 moved_to_echo=39749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4850, 0, 3, 0, 0, 0, 33, 0, 4, 3077, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2799, 2, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 3964, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 0, 0, 0, 1413, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 3296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 458, 4851, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 2082, 0, 0, 0, 0, 0, 2012, 8160, 0, 0, 22, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 0, 21, 0, 0, 1, 0, 0, 0, 7192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9484, 2855, 4592, 8254, 9706, 8091, 13629, 8925] avg=8192.0000 max=13629 max_rank=6 min=2855 min_rank=1 imbalance_ratio=1.663696 max_load_reduction=51907 max_load_reduction_pct=0.792038 moved_to_echo=40351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 587, 0, 3396, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 349, 0, 0, 0, 0, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 5449, 0, 0, 12, 1, 0, 3896, 85, 0, 5, 0, 473, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 239, 37, 0, 0, 0, 0, 2, 0, 1562, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 2, 0, 0, 7, 0, 7944, 0, 0, 0, 0, 8, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4369, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8, 0, 2594, 0, 0, 5, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16996, 3459, 10148, 2146, 18543, 7108, 4050, 3086] avg=8192.0000 max=18543 max_rank=4 min=2146 min_rank=3 imbalance_ratio=2.263550 max_load_reduction=46993 max_load_reduction_pct=0.717056 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1024, 512, 128, 768, 0, 256, 128, 3840, 3456, 0, 0, 0, 128, 0, 640, 512, 33792, 21120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[16000, 128, 7168, 1024, 0, 3840, 2048, 19072, 7168, 8192, 0, 0, 128, 128, 512, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 6528, 128, 7680, 128, 128, 640, 128, 128, 24064, 128, 256, 128, 256, 3584, 22912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[6528, 0, 128, 7808, 128, 128, 384, 13440, 0, 2560, 1408, 128, 3968, 256, 640, 128, 17408, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[17920, 0, 256, 768, 4736, 0, 768, 128, 1664, 128, 128, 0, 384, 15744, 20736, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 768, 29696, 128, 128, 1920, 128, 128, 30848, 1920, 128, 128, 0, 512, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 2304, 128, 128, 128, 7040, 0, 128, 256, 128, 384, 128, 40192, 2432, 128, 128, 8064, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[768, 2304, 128, 0, 256, 1664, 128, 128, 384, 128, 128, 50688, 128, 128, 128, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 16:28:28] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38518.5 | throughput per GPU (TFLOP/s/GPU): 4.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115992E+01 | loss scale: 1.0 | grad norm: 44.778 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 81277.46484375 | max allocated: 88645.78466796875 | reserved: 98254.0 | max reserved: 98254.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4396, 162, 2, 510, 136, 840, 9, 2, 52, 290, 1562, 547, 1308, 107, 203, 866, 144, 2237, 274, 13, 1033, 18, 98, 1615, 424, 66, 42, 1913, 390, 988, 53, 195, 428, 1255, 57, 47, 80, 467, 48, 4, 1189, 16, 719, 3433, 25, 30, 152, 818, 210, 82, 0, 19, 0, 354, 118, 18, 118, 19, 31, 502, 1077, 20, 340, 100, 2324, 26, 14, 19, 2136, 7, 33, 1883, 1048, 130, 1811, 76, 444, 2, 752, 1031, 26, 20, 2, 27, 42, 5, 761, 1036, 35, 24, 613, 3, 15, 185, 262, 79, 926, 115, 50, 1860, 102, 3, 37, 98, 248, 67, 660, 4953, 33, 55, 47, 95, 681, 37, 10, 81, 3, 683, 120, 295, 1193, 37, 1885, 413, 267, 310, 41, 2989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3882, 82, 42, 464, 292, 453, 14, 19, 52, 288, 1082, 1084, 1511, 110, 589, 281, 175, 2286, 193, 34, 1090, 68, 177, 1369, 739, 153, 68, 2335, 776, 1391, 93, 286, 843, 1450, 81, 12, 357, 415, 160, 11, 977, 26, 501, 2988, 25, 33, 152, 624, 223, 223, 7, 48, 7, 243, 40, 41, 94, 77, 57, 725, 1249, 25, 628, 159, 2166, 26, 5, 18, 1431, 79, 125, 1494, 777, 164, 1409, 89, 388, 15, 625, 881, 43, 31, 3, 26, 34, 1, 649, 938, 11, 43, 617, 10, 24, 225, 239, 129, 801, 120, 76, 3116, 157, 3, 91, 254, 566, 106, 762, 4275, 30, 192, 77, 25, 781, 35, 21, 58, 7, 278, 146, 263, 1309, 44, 1269, 339, 147, 249, 8, 3237] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3495, 61, 42, 259, 178, 571, 6, 29, 70, 214, 1683, 441, 1991, 113, 353, 267, 114, 2618, 97, 44, 527, 74, 146, 1576, 766, 59, 36, 1579, 332, 1169, 91, 343, 461, 1856, 53, 87, 85, 415, 62, 71, 538, 71, 827, 3238, 39, 8, 326, 849, 260, 214, 4, 37, 6, 132, 79, 8, 33, 39, 51, 652, 2002, 32, 217, 93, 2814, 22, 3, 15, 1553, 16, 61, 1072, 765, 89, 2009, 122, 409, 18, 372, 803, 25, 18, 11, 66, 98, 17, 954, 848, 20, 75, 1080, 9, 12, 261, 275, 96, 1919, 179, 13, 2969, 56, 3, 11, 74, 424, 136, 697, 4726, 25, 86, 69, 75, 483, 25, 31, 22, 4, 555, 133, 408, 1145, 87, 1009, 365, 252, 263, 18, 3077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3707, 68, 25, 436, 236, 457, 12, 20, 63, 154, 2215, 412, 1272, 118, 112, 806, 148, 2480, 259, 9, 1318, 50, 172, 1656, 541, 60, 14, 1549, 506, 2068, 127, 270, 461, 1843, 175, 104, 250, 432, 98, 38, 1644, 51, 460, 3108, 25, 64, 94, 596, 204, 70, 4, 118, 8, 414, 53, 24, 170, 94, 41, 392, 1287, 11, 899, 195, 1338, 33, 8, 13, 1242, 13, 120, 1706, 748, 149, 1926, 142, 457, 19, 535, 772, 30, 13, 3, 24, 59, 7, 876, 548, 14, 23, 474, 12, 40, 166, 564, 75, 648, 123, 145, 1877, 177, 0, 103, 72, 549, 72, 509, 3597, 23, 196, 44, 39, 745, 109, 12, 131, 12, 751, 364, 191, 1297, 46, 2120, 408, 108, 237, 3, 3912] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3835, 120, 26, 419, 113, 348, 13, 12, 77, 263, 2564, 504, 1737, 90, 230, 435, 132, 2145, 484, 11, 1081, 28, 87, 1274, 508, 40, 76, 2035, 749, 2355, 82, 100, 510, 2885, 142, 30, 227, 427, 48, 69, 733, 63, 561, 2591, 40, 13, 141, 478, 388, 90, 1, 91, 2, 927, 33, 19, 240, 40, 50, 383, 924, 24, 519, 163, 1855, 20, 25, 1, 1356, 14, 73, 1108, 725, 45, 2149, 254, 659, 4, 444, 1094, 38, 9, 11, 16, 41, 5, 457, 562, 20, 31, 758, 15, 49, 100, 374, 112, 1227, 159, 107, 2249, 150, 0, 78, 300, 365, 114, 413, 3833, 25, 221, 41, 45, 674, 48, 18, 92, 11, 239, 190, 417, 1377, 54, 1997, 368, 155, 171, 10, 3139] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3698, 93, 55, 467, 125, 699, 35, 52, 31, 249, 1681, 253, 1339, 106, 193, 387, 306, 2012, 209, 20, 489, 10, 58, 1558, 678, 60, 34, 1831, 692, 2065, 121, 633, 782, 1658, 100, 68, 166, 189, 183, 67, 1089, 21, 619, 2126, 39, 9, 207, 893, 183, 210, 4, 72, 6, 299, 103, 24, 129, 24, 17, 517, 1055, 16, 562, 287, 1992, 25, 8, 14, 1742, 3, 66, 1572, 865, 103, 1247, 225, 481, 6, 550, 1364, 55, 13, 2, 42, 52, 10, 1289, 676, 59, 48, 934, 2, 29, 336, 272, 66, 1545, 258, 77, 1387, 194, 2, 69, 82, 280, 143, 343, 4992, 30, 134, 114, 56, 813, 145, 9, 153, 8, 211, 254, 443, 1441, 88, 1838, 438, 83, 486, 5, 3304] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[4024, 230, 10, 431, 54, 540, 53, 20, 50, 374, 1890, 410, 1445, 100, 201, 573, 81, 1542, 170, 15, 827, 27, 112, 1216, 691, 63, 50, 1491, 342, 1660, 30, 261, 641, 1675, 122, 36, 87, 357, 101, 35, 952, 19, 547, 2624, 22, 5, 136, 994, 272, 152, 1, 36, 2, 385, 150, 18, 113, 32, 18, 300, 1277, 68, 434, 128, 2704, 18, 15, 9, 1934, 2, 33, 1459, 980, 69, 1725, 261, 349, 10, 544, 771, 22, 16, 26, 28, 35, 36, 1197, 973, 27, 44, 1112, 16, 38, 103, 253, 67, 1349, 117, 17, 1989, 83, 6, 57, 82, 356, 93, 431, 5244, 27, 126, 53, 47, 923, 26, 14, 151, 1, 583, 147, 325, 1557, 39, 1758, 332, 377, 188, 5, 3435] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[4466, 59, 13, 293, 131, 445, 7, 34, 33, 278, 1154, 284, 2095, 194, 227, 516, 189, 2176, 305, 6, 695, 39, 73, 1400, 589, 25, 53, 2019, 1037, 2078, 27, 378, 682, 1571, 61, 32, 119, 433, 65, 32, 842, 11, 717, 2891, 31, 8, 109, 882, 225, 94, 8, 74, 2, 374, 43, 16, 177, 73, 31, 620, 939, 13, 449, 298, 2401, 27, 0, 6, 1954, 10, 37, 1160, 938, 179, 1729, 158, 516, 6, 494, 798, 62, 4, 0, 43, 105, 12, 770, 432, 18, 61, 1014, 3, 74, 138, 547, 79, 1442, 171, 18, 1677, 123, 5, 21, 115, 567, 81, 905, 4220, 18, 135, 62, 37, 744, 71, 13, 75, 2, 904, 237, 224, 1129, 17, 1387, 413, 255, 210, 6, 3237] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2099, 162, 2, 510, 136, 840, 9, 2, 52, 290, 1562, 547, 1308, 107, 203, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 274, 13, 1033, 18, 98, 1615, 424, 66, 42, 1896, 390, 988, 53, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 1255, 57, 47, 80, 467, 48, 4, 1189, 16, 719, 2787, 25, 30, 152, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 82, 0, 19, 0, 354, 118, 18, 118, 19, 31, 502, 1077, 20, 340, 100, 1769, 1654, 849, 646, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 26, 14, 19, 2136, 7, 33, 1883, 1048, 130, 1811, 76, 444, 2, 752, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 20, 2, 27, 42, 5, 761, 1036, 35, 24, 613, 3, 15, 185, 262, 79, 2237, 2198, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 115, 50, 1860, 102, 3, 37, 98, 248, 67, 660, 3184, 33, 55, 47, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, 37, 10, 81, 3, 683, 120, 295, 1193, 37, 1885, 413, 267, 310, 41, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8695, 7249, 8122, 7943, 9538, 8213, 7580, 8196] avg=8192.0000 max=9538 max_rank=4 min=7249 min_rank=1 imbalance_ratio=1.164307 max_load_reduction=55998 max_load_reduction_pct=0.854462 moved_to_echo=33527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1765, 93, 55, 467, 125, 699, 35, 52, 31, 249, 1681, 253, 1339, 106, 193, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 209, 20, 489, 10, 58, 1558, 678, 60, 34, 1815, 692, 2065, 121, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 1658, 100, 68, 166, 189, 183, 67, 1089, 21, 619, 1726, 39, 9, 207, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 210, 4, 72, 6, 299, 103, 24, 129, 24, 17, 517, 1055, 16, 562, 287, 1783, 1392, 939, 400, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 25, 8, 14, 1742, 3, 66, 1572, 865, 103, 1247, 225, 481, 6, 550, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 13, 2, 42, 52, 10, 1289, 676, 59, 48, 934, 2, 29, 336, 272, 66, 2012, 1884, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 258, 77, 1387, 194, 2, 69, 82, 280, 143, 343, 3209, 30, 134, 114, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 145, 9, 153, 8, 211, 254, 443, 1441, 88, 1838, 438, 83, 486, 5, 2365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7530, 8748, 7816, 8038, 8379, 8322, 7923, 8780] avg=8192.0000 max=8780 max_rank=7 min=7530 min_rank=0 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=33404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1669, 61, 42, 259, 178, 571, 6, 29, 70, 214, 1683, 441, 1991, 113, 353, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 97, 44, 527, 74, 146, 1576, 766, 59, 36, 1565, 332, 1169, 91, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 1856, 53, 87, 85, 415, 62, 71, 538, 71, 827, 2629, 39, 8, 326, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 214, 4, 37, 6, 132, 79, 8, 33, 39, 51, 652, 2002, 32, 217, 93, 1688, 1315, 874, 609, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 22, 3, 15, 1553, 16, 61, 1072, 765, 89, 2009, 122, 409, 18, 372, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 18, 11, 66, 98, 17, 954, 848, 20, 75, 1080, 9, 12, 261, 275, 96, 2618, 2661, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 179, 13, 2969, 56, 3, 11, 74, 424, 136, 697, 3038, 25, 86, 69, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 25, 31, 22, 4, 555, 133, 408, 1145, 87, 1009, 365, 252, 263, 18, 2203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7947, 6939, 8377, 8359, 7482, 9655, 9774, 7003] avg=8192.0000 max=9774 max_rank=6 min=6939 min_rank=1 imbalance_ratio=1.193115 max_load_reduction=55762 max_load_reduction_pct=0.850861 moved_to_echo=33914 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1831, 120, 26, 419, 113, 348, 13, 12, 77, 263, 2564, 504, 1737, 90, 230, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 484, 11, 1081, 28, 87, 1274, 508, 40, 76, 2017, 749, 2355, 82, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 2885, 142, 30, 227, 427, 48, 69, 733, 63, 561, 2104, 40, 13, 141, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 90, 1, 91, 2, 927, 33, 19, 240, 40, 50, 383, 924, 24, 519, 163, 1369, 1443, 892, 487, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 20, 25, 1, 1356, 14, 73, 1108, 725, 45, 2149, 254, 659, 4, 444, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 9, 11, 16, 41, 5, 457, 562, 20, 31, 758, 15, 49, 100, 374, 112, 2145, 1754, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 159, 107, 2249, 150, 0, 78, 300, 365, 114, 413, 2464, 25, 221, 41, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 674, 48, 18, 92, 11, 239, 190, 417, 1377, 54, 1997, 368, 155, 171, 10, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8782, 9024, 8471, 8103, 8072, 7058, 7958, 8068] avg=8192.0000 max=9024 max_rank=1 min=7058 min_rank=5 imbalance_ratio=1.101562 max_load_reduction=56512 max_load_reduction_pct=0.862305 moved_to_echo=31156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1853, 82, 42, 464, 292, 453, 14, 19, 52, 288, 1082, 1084, 1511, 110, 589, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 193, 34, 1090, 68, 177, 1369, 739, 153, 68, 2315, 776, 1391, 93, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 1450, 81, 12, 357, 415, 160, 11, 977, 26, 501, 2426, 25, 33, 152, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 223, 7, 48, 7, 243, 40, 41, 94, 77, 57, 725, 1249, 25, 628, 159, 1527, 1461, 920, 562, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 26, 5, 18, 1431, 79, 125, 1494, 777, 164, 1409, 89, 388, 15, 625, 881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 31, 3, 26, 34, 1, 649, 938, 11, 43, 617, 10, 24, 225, 239, 129, 2286, 2048, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 120, 76, 3116, 157, 3, 91, 254, 566, 106, 762, 2748, 30, 192, 77, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 35, 21, 58, 7, 278, 146, 263, 1309, 44, 1269, 339, 147, 249, 8, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8927, 8093, 8336, 7644, 7925, 9124, 7271] avg=8192.0000 max=9124 max_rank=6 min=7271 min_rank=7 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=31964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1922, 230, 10, 431, 54, 540, 53, 20, 50, 374, 1890, 410, 1445, 100, 201, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 170, 15, 827, 27, 112, 1216, 691, 63, 50, 1478, 342, 1660, 30, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 1675, 122, 36, 87, 357, 101, 35, 952, 19, 547, 2131, 22, 5, 136, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 152, 1, 36, 2, 385, 150, 18, 113, 32, 18, 300, 1277, 68, 434, 128, 1873, 1514, 976, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 18, 15, 9, 1934, 2, 33, 1459, 980, 69, 1725, 261, 349, 10, 544, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 16, 26, 28, 35, 36, 1197, 973, 27, 44, 1112, 16, 38, 103, 253, 67, 1542, 2557, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 117, 17, 1989, 83, 6, 57, 82, 356, 93, 431, 3371, 27, 126, 53, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 26, 14, 151, 1, 583, 147, 325, 1557, 39, 1758, 332, 377, 188, 5, 2459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 7023, 7860, 8255, 8326, 8680, 8204, 8885] avg=8192.0000 max=8885 max_rank=7 min=7023 min_rank=1 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=34095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1770, 68, 25, 436, 236, 457, 12, 20, 63, 154, 2215, 412, 1272, 118, 112, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 259, 9, 1318, 50, 172, 1656, 541, 60, 14, 1536, 506, 2068, 127, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 1843, 175, 104, 250, 432, 98, 38, 1644, 51, 460, 2523, 25, 64, 94, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 70, 4, 118, 8, 414, 53, 24, 170, 94, 41, 392, 1287, 11, 899, 195, 1285, 1395, 1112, 585, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 33, 8, 13, 1242, 13, 120, 1706, 748, 149, 1926, 142, 457, 19, 535, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 13, 3, 24, 59, 7, 876, 548, 14, 23, 474, 12, 40, 166, 564, 75, 2480, 1265, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 123, 145, 1877, 177, 0, 103, 72, 549, 72, 509, 2312, 23, 196, 44, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 109, 12, 131, 12, 751, 364, 191, 1297, 46, 2120, 408, 108, 237, 3, 2800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8734, 8858, 8374, 7956, 7215, 6889, 9334] avg=8192.0000 max=9334 max_rank=7 min=6889 min_rank=6 imbalance_ratio=1.139404 max_load_reduction=56202 max_load_reduction_pct=0.857574 moved_to_echo=31394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2124, 59, 13, 293, 131, 445, 7, 34, 33, 278, 1154, 284, 2095, 194, 227, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 305, 6, 695, 39, 73, 1400, 589, 25, 53, 1998, 1037, 2078, 27, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 1571, 61, 32, 119, 433, 65, 32, 842, 11, 717, 2344, 31, 8, 109, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 94, 8, 74, 2, 374, 43, 16, 177, 73, 31, 620, 939, 13, 449, 298, 1513, 1686, 925, 547, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 27, 0, 6, 1954, 10, 37, 1160, 938, 179, 1729, 158, 516, 6, 494, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4, 0, 43, 105, 12, 770, 432, 18, 61, 1014, 3, 74, 138, 547, 79, 2176, 2274, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1442, 171, 18, 1677, 123, 5, 21, 115, 567, 81, 905, 2707, 18, 135, 62, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 71, 13, 75, 2, 904, 237, 224, 1129, 17, 1387, 413, 255, 210, 6, 2312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7887, 8892, 7939, 8128, 8139, 8468, 8084, 7999] avg=8192.0000 max=8892 max_rank=1 min=7887 min_rank=0 imbalance_ratio=1.085449 max_load_reduction=56644 max_load_reduction_pct=0.864319 moved_to_echo=32690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[2048, 1152, 128, 512, 128, 3200, 640, 256, 1152, 512, 384, 4096, 9856, 256, 4096, 1536, 12928, 11904, 7552, 4352, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[384, 128, 128, 384, 512, 128, 7040, 6016, 256, 384, 6656, 128, 384, 1536, 2816, 768, 17536, 16768, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[5888, 512, 128, 768, 128, 4224, 1664, 2688, 10496, 512, 13312, 3200, 1664, 2176, 128, 18944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1408, 0, 2048, 256, 7168, 384, 1024, 11776, 4992, 640, 384, 14720, 4864, 13824, 640, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1024, 256, 128, 128, 13440, 256, 640, 11520, 6912, 1024, 14080, 1408, 3712, 128, 4352, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[9984, 1280, 512, 17152, 1152, 128, 512, 1152, 3456, 896, 4736, 23040, 256, 1152, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4864, 14208, 896, 512, 1408, 3200, 768, 384, 8064, 384, 4992, 18688, 256, 256, 1408, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[15104, 896, 256, 3328, 1280, 4480, 256, 256, 512, 2176, 13952, 3968, 12800, 1024, 2176, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[83, 4768, 1, 3233, 85, 0, 15, 3, 0, 102, 7615, 0, 0, 575, 3, 12, 1, 0, 2, 456, 0, 0, 3, 0, 0, 0, 360, 1588, 2, 3, 0, 0, 0, 0, 3, 20, 18, 6, 0, 1, 0, 652, 12, 0, 65, 3, 0, 0, 0, 4, 58, 0, 94, 6, 0, 8142, 0, 4635, 2841, 69, 0, 40, 63, 363, 330, 16, 0, 8, 1177, 8013, 0, 0, 0, 0, 0, 5, 2, 0, 67, 0, 0, 0, 0, 1, 0, 1, 64, 0, 1053, 2, 175, 2, 18, 0, 4143, 0, 0, 0, 445, 0, 13, 835, 43, 517, 12, 6, 24, 79, 0, 0, 0, 0, 1, 0, 17, 0, 2, 0, 4548, 1, 9, 9, 200, 0, 7685, 7, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 282, 0, 1579, 0, 0, 0, 3, 17, 71, 4450, 0, 0, 2442, 0, 0, 0, 27, 0, 1220, 1, 0, 0, 0, 8, 2, 9, 3, 0, 0, 1236, 9, 0, 115, 0, 0, 7, 24, 0, 528, 0, 785, 542, 0, 6, 8, 48, 20, 0, 0, 7, 0, 809, 0, 99, 5615, 0, 4148, 611, 2819, 0, 1068, 5, 1206, 322, 0, 0, 0, 1325, 8144, 0, 2, 0, 4, 80, 0, 0, 97, 1, 0, 1, 0, 0, 174, 1, 733, 0, 0, 6540, 0, 540, 0, 0, 33, 5121, 0, 25, 54, 0, 95, 8, 3278, 0, 59, 2, 487, 74, 75, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 7099, 59, 0, 2, 367, 1, 901, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[134, 3721, 0, 3384, 2, 0, 3, 9, 2, 21, 6991, 0, 0, 6, 0, 0, 0, 0, 3, 1434, 0, 0, 924, 0, 0, 0, 0, 1820, 47, 2, 126, 0, 0, 14, 1, 9, 0, 0, 0, 660, 0, 17, 613, 1, 1, 55, 3, 0, 22, 9, 1, 0, 2338, 0, 0, 5784, 4, 189, 2567, 0, 0, 8, 5, 185, 1, 0, 0, 0, 210, 8022, 0, 0, 1, 0, 0, 0, 3, 13, 1, 1, 0, 0, 4, 0, 0, 0, 95, 1, 7693, 1, 3546, 1, 13, 0, 1231, 168, 0, 3, 30, 0, 0, 2581, 22, 1, 0, 0, 474, 162, 1, 0, 0, 0, 0, 124, 0, 6, 0, 4, 2526, 0, 1, 0, 11, 13, 7456, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[768, 333, 1, 3185, 1, 5, 1, 0, 47, 2, 7785, 0, 0, 1271, 2, 117, 0, 351, 137, 248, 0, 1, 0, 0, 1, 2, 13, 108, 0, 6, 33, 329, 0, 0, 26, 0, 0, 1, 0, 0, 0, 148, 56, 51, 11, 12, 0, 2, 131, 27, 17, 44, 1548, 2, 8, 2204, 0, 585, 1112, 2055, 0, 1425, 106, 3314, 93, 10, 1, 0, 1175, 7756, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 201, 2096, 742, 106, 1403, 4, 7915, 1, 89, 0, 1885, 47, 2579, 78, 0, 0, 0, 0, 7, 2012, 0, 0, 0, 6, 14, 375, 2, 7, 0, 3, 2, 226, 0, 89, 102, 1, 1385, 10, 0, 0, 21, 7, 7356, 89, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1546, 1823, 5, 431, 32, 0, 0, 0, 1, 1460, 5628, 0, 12, 496, 0, 4, 0, 1, 106, 627, 0, 0, 0, 0, 0, 0, 2, 154, 2, 0, 4, 2, 1, 97, 0, 0, 0, 0, 5, 482, 4, 1949, 6, 0, 223, 5, 0, 0, 86, 1220, 81, 7, 4274, 10, 138, 6892, 0, 5024, 5, 4, 0, 1957, 0, 1017, 2, 0, 0, 35, 351, 8125, 0, 15, 0, 29, 0, 102, 0, 0, 2, 0, 1, 0, 0, 1, 0, 1414, 98, 19, 7473, 0, 137, 0, 6, 0, 5817, 3, 0, 0, 2, 57, 11, 578, 108, 3, 2, 166, 9, 0, 45, 5, 0, 0, 0, 252, 0, 20, 0, 0, 840, 0, 0, 0, 37, 0, 3948, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[171, 1250, 1, 34, 0, 0, 0, 0, 0, 216, 8019, 4, 22, 155, 0, 0, 40, 5, 1, 2873, 0, 0, 0, 0, 0, 0, 2, 865, 4, 0, 0, 84, 8, 0, 0, 19, 4, 11, 0, 2, 0, 103, 149, 0, 0, 683, 0, 0, 20, 79, 0, 0, 6417, 0, 12, 6010, 0, 928, 4586, 0, 0, 1021, 0, 644, 408, 0, 0, 0, 13, 8103, 22, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 519, 41, 1, 106, 16, 3517, 0, 6692, 0, 277, 6, 1, 0, 677, 0, 27, 1, 0, 1, 0, 2594, 0, 0, 0, 66, 0, 0, 36, 4, 0, 0, 0, 0, 0, 0, 0, 6, 248, 7, 0, 0, 5, 0, 7682, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[52, 28, 62, 1689, 1, 1, 2, 23, 0, 0, 4889, 0, 27, 33, 3, 122, 0, 2, 15, 581, 0, 0, 11, 0, 0, 10, 4, 816, 1, 0, 35, 737, 2, 2, 3, 0, 0, 0, 1, 3, 0, 2967, 1367, 0, 4, 0, 0, 0, 52, 0, 127, 0, 4617, 0, 3, 7351, 0, 681, 354, 2129, 0, 4203, 74, 1914, 262, 0, 6, 0, 954, 7870, 0, 51, 0, 0, 0, 19, 0, 0, 0, 0, 0, 1, 104, 98, 3, 13, 705, 7, 4136, 0, 9, 0, 4, 7, 5171, 0, 0, 0, 0, 383, 0, 1544, 0, 12, 0, 9, 239, 1296, 0, 28, 0, 0, 0, 22, 0, 0, 0, 0, 13, 0, 51, 0, 4, 0, 7517, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[391, 1862, 0, 307, 13, 0, 61, 273, 0, 0, 8065, 0, 9, 4612, 0, 35, 2, 0, 23, 2758, 0, 0, 266, 0, 0, 0, 2043, 27, 0, 0, 3, 0, 3, 2, 0, 0, 3, 591, 0, 0, 0, 39, 7, 0, 2, 51, 0, 0, 76, 1, 2, 1, 735, 68, 6, 6409, 0, 2, 1141, 63, 0, 49, 1, 665, 10, 0, 0, 0, 23, 8188, 20, 1, 0, 0, 0, 389, 0, 0, 0, 11, 0, 100, 0, 156, 0, 0, 0, 0, 6379, 1, 437, 0, 5, 0, 3538, 0, 1, 0, 0, 29, 0, 1449, 0, 5, 1, 0, 0, 0, 29, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 3, 82, 0, 7837, 16, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[768, 333, 1, 3185, 1, 5, 1, 0, 47, 2, 3203, 0, 0, 1271, 2, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 137, 248, 0, 1, 0, 0, 1, 2, 13, 108, 0, 6, 33, 329, 1010, 2211, 1394, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 1, 0, 0, 0, 148, 56, 51, 11, 12, 0, 2, 2204, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 27, 17, 44, 538, 2, 8, 0, 0, 585, 1112, 2055, 0, 1425, 106, 3314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 10, 1, 0, 1175, 6986, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 201, 2096, 742, 106, 1403, 4, 2847, 1, 89, 0, 1885, 47, 2579, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2012, 0, 0, 0, 6, 14, 375, 2, 7, 0, 3, 4031, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 226, 0, 89, 102, 1, 1385, 10, 0, 0, 21, 7, 5962, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8936, 6614, 3062, 9364, 8270, 12082, 9314, 7894] avg=8192.0000 max=12082 max_rank=5 min=3062 min_rank=2 imbalance_ratio=1.474854 max_load_reduction=53454 max_load_reduction_pct=0.815643 moved_to_echo=37560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[134, 3721, 0, 3384, 2, 0, 3, 9, 2, 21, 2876, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1434, 0, 0, 924, 0, 0, 0, 0, 1820, 47, 2, 126, 0, 1526, 2149, 1413, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 9, 0, 0, 0, 660, 0, 17, 613, 1, 1, 55, 3, 0, 5784, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 9, 1, 0, 812, 0, 0, 0, 4, 189, 2567, 0, 0, 8, 5, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 210, 7226, 0, 0, 1, 0, 0, 0, 3, 13, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 95, 1, 2767, 1, 3546, 1, 13, 0, 1231, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 30, 0, 0, 2581, 22, 1, 0, 0, 474, 162, 1, 0, 0, 0, 3620, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 6, 0, 4, 2526, 0, 1, 0, 11, 13, 6043, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10158, 10240, 7653, 3802, 7456, 7827, 9671, 8729] avg=8192.0000 max=10240 max_rank=1 min=3802 min_rank=3 imbalance_ratio=1.250000 max_load_reduction=55296 max_load_reduction_pct=0.843750 moved_to_echo=33683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 282, 0, 1579, 0, 0, 0, 3, 17, 71, 1831, 0, 0, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 1220, 1, 0, 0, 0, 8, 2, 9, 3, 0, 0, 1236, 9, 528, 1827, 170, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 7, 24, 0, 528, 0, 785, 542, 0, 6, 8, 48, 20, 5615, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 281, 0, 99, 0, 0, 4148, 611, 2819, 0, 1068, 5, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 1325, 7336, 0, 2, 0, 4, 80, 0, 0, 97, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 174, 1, 733, 0, 0, 2353, 0, 540, 0, 0, 33, 5121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 54, 0, 95, 8, 3278, 0, 59, 2, 487, 74, 75, 0, 0, 0, 1, 2304, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7099, 59, 0, 2, 367, 1, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6225, 5848, 8013, 10244, 9167, 8956, 8822, 8261] avg=8192.0000 max=10244 max_rank=3 min=5848 min_rank=1 imbalance_ratio=1.250488 max_load_reduction=55292 max_load_reduction_pct=0.843689 moved_to_echo=35206 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[52, 28, 62, 1689, 1, 1, 2, 23, 0, 0, 2012, 0, 27, 33, 3, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 581, 0, 0, 11, 0, 0, 10, 4, 816, 1, 0, 35, 737, 3015, 1155, 1425, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3, 0, 0, 0, 1, 3, 0, 2967, 1367, 0, 4, 0, 0, 0, 7351, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 127, 0, 1602, 0, 3, 0, 0, 681, 354, 2129, 0, 4203, 74, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 6, 0, 954, 7089, 0, 51, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 104, 98, 3, 13, 705, 7, 1488, 0, 9, 0, 4, 7, 5171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 1544, 0, 12, 0, 9, 239, 1296, 0, 28, 0, 0, 2531, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 13, 0, 51, 0, 4, 0, 6092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 8588, 12046, 11139, 8381, 7610, 7535, 6182] avg=8192.0000 max=12046 max_rank=2 min=4055 min_rank=0 imbalance_ratio=1.470459 max_load_reduction=53490 max_load_reduction_pct=0.816193 moved_to_echo=29708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1546, 1823, 5, 431, 32, 0, 0, 0, 1, 1460, 2316, 0, 12, 496, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 106, 627, 0, 0, 0, 0, 0, 0, 2, 154, 2, 0, 4, 2, 2791, 2088, 748, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 97, 0, 0, 0, 0, 5, 482, 4, 1949, 6, 0, 223, 5, 0, 0, 6892, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 1220, 81, 7, 1483, 10, 138, 0, 0, 5024, 5, 4, 0, 1957, 0, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 35, 351, 7319, 0, 15, 0, 29, 0, 102, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1414, 98, 19, 2688, 0, 137, 0, 6, 0, 5817, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 57, 11, 578, 108, 3, 2, 166, 9, 0, 45, 5, 0, 0, 2914, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 20, 0, 0, 840, 0, 0, 0, 37, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 7331, 10062, 11032, 7855, 10184, 6597, 4349] avg=8192.0000 max=11032 max_rank=3 min=4349 min_rank=7 imbalance_ratio=1.346680 max_load_reduction=54504 max_load_reduction_pct=0.831665 moved_to_echo=28985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[171, 1250, 1, 34, 0, 0, 0, 0, 0, 216, 3299, 4, 22, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 5, 1, 2873, 0, 0, 0, 0, 0, 0, 2, 865, 4, 0, 0, 84, 4190, 1870, 1456, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 19, 4, 11, 0, 2, 0, 103, 149, 0, 0, 683, 0, 0, 6010, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 79, 0, 0, 2227, 0, 12, 0, 0, 928, 4586, 0, 0, 1021, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 0, 13, 7299, 22, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 41, 1, 106, 16, 3517, 0, 2407, 0, 277, 6, 1, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1, 0, 1, 0, 2594, 0, 0, 0, 66, 0, 0, 36, 4, 0, 0, 4152, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 248, 7, 0, 0, 5, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5152, 12194, 7557, 9517, 7760, 7568, 9296, 6492] avg=8192.0000 max=12194 max_rank=1 min=5152 min_rank=0 imbalance_ratio=1.488525 max_load_reduction=53342 max_load_reduction_pct=0.813934 moved_to_echo=31116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[391, 1862, 0, 307, 13, 0, 61, 273, 0, 0, 3308, 0, 9, 4612, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 23, 2758, 0, 0, 266, 0, 0, 0, 2043, 27, 0, 0, 3, 0, 484, 1786, 1489, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 591, 0, 0, 0, 39, 7, 0, 2, 51, 0, 0, 6409, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 1, 2, 1, 251, 68, 6, 0, 0, 2, 1141, 63, 0, 49, 1, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 23, 7370, 20, 1, 0, 0, 0, 389, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 156, 0, 0, 0, 0, 2287, 1, 437, 0, 5, 0, 3538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 29, 0, 1449, 0, 5, 1, 0, 0, 0, 29, 0, 2, 0, 4182, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 3, 82, 0, 6348, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10871, 9699, 7682, 2326, 7824, 6524, 8004, 12606] avg=8192.0000 max=12606 max_rank=7 min=2326 min_rank=3 imbalance_ratio=1.538818 max_load_reduction=52930 max_load_reduction_pct=0.807648 moved_to_echo=34958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[83, 4768, 1, 3233, 85, 0, 15, 3, 0, 102, 3133, 0, 0, 575, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 456, 0, 0, 3, 0, 0, 0, 360, 1588, 2, 3, 0, 0, 61, 294, 1457, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 20, 18, 6, 0, 1, 0, 652, 12, 0, 65, 3, 0, 0, 8142, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 58, 0, 33, 6, 0, 0, 0, 4635, 2841, 69, 0, 40, 63, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 16, 0, 8, 1177, 7218, 0, 0, 0, 0, 0, 5, 2, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 64, 0, 379, 2, 175, 2, 18, 0, 4143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 13, 835, 43, 517, 12, 6, 24, 79, 0, 0, 0, 0, 3943, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 2, 0, 4548, 1, 9, 9, 200, 0, 6228, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12013, 5022, 9461, 8112, 8823, 4785, 6297, 11023] avg=8192.0000 max=12013 max_rank=0 min=4785 min_rank=5 imbalance_ratio=1.466431 max_load_reduction=53523 max_load_reduction_pct=0.816696 moved_to_echo=30928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 512, 384, 10240, 128, 128, 1280, 0, 128, 128, 2560, 5504, 128, 128, 1536, 1280, 13696, 13440, 9600, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 512, 640, 128, 14976, 256, 640, 128, 768, 896, 2048, 128, 128, 128, 128, 27776, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3200, 14080, 128, 13952, 256, 128, 128, 384, 128, 1920, 22016, 128, 128, 9600, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[512, 1408, 384, 128, 7296, 128, 384, 0, 128, 16256, 13312, 7168, 0, 9856, 256, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 640, 128, 128, 128, 128, 22912, 128, 128, 128, 768, 128, 40832, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 128, 128, 640, 128, 1792, 128, 6784, 2816, 128, 384, 896, 128, 128, 48512, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 640, 384, 2560, 896, 2304, 5888, 128, 17280, 128, 5248, 128, 2048, 128, 28288, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1536, 128, 128, 128, 5248, 57856, 128, 128, 128, 128, 128, 640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 9, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 0, 0, 0, 0, 8, 0, 0, 15, 5455, 0, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 34, 0, 2, 0, 24, 0, 3, 2, 0, 6036, 0, 3, 0, 0, 8156, 0, 4149, 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 25, 6949, 1067, 0, 8, 0, 7667, 1, 0, 0, 24, 0, 216, 1, 0, 0, 0, 32, 0, 0, 0, 0, 2, 0, 525, 0, 1, 88, 441, 8144, 46, 0, 0, 0, 7764, 0, 114, 5, 3, 27, 0, 0, 229, 16, 6842, 1, 0, 480, 451, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 12, 5, 0, 5744, 0, 22, 23, 57, 2, 259, 25, 4, 0, 41, 3, 0, 190, 0, 0, 249, 11, 0, 0, 1144, 0, 495, 1, 673, 278, 0, 0, 2, 0, 0, 0, 277, 0, 7, 0, 0, 2, 3, 0, 0, 0, 147, 0, 0, 2, 0, 0, 1, 0, 4951, 0, 4422, 0, 3, 0, 0, 7698, 0, 241, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 6800, 0, 0, 1, 0, 3116, 2, 743, 0, 0, 1, 3, 20, 0, 4416, 8, 0, 246, 0, 2, 0, 1, 0, 224, 715, 17, 2697, 0, 3997, 0, 0, 2, 3, 7524, 0, 2, 0, 146, 0, 0, 31, 0, 0, 7517, 0, 45, 18, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7894, 0, 0, 0, 170, 2, 0, 0, 0, 1, 11, 0, 0, 0, 0, 16, 0, 0, 0, 6, 1, 0, 0, 0, 1340, 0, 19, 2, 0, 0, 8, 0, 0, 10, 31, 8, 411, 0, 0, 0, 0, 250, 0, 214, 0, 0, 0, 0, 0, 0, 520, 0, 4890, 0, 0, 0, 5, 7328, 0, 2037, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 8191, 11, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 9, 0, 985, 1, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 196, 970, 6255, 0, 0, 0, 0, 7676, 0, 16, 72, 110, 0, 14, 1, 0, 15, 7868, 0, 0, 0, 0, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3532, 0, 5, 0, 0, 0, 15, 19, 9, 8, 0, 0, 0, 64, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 7848, 0, 0, 0, 0, 2, 151, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 410, 10, 1889, 1, 0, 0, 0, 8104, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 2, 177, 0, 0, 0, 7238, 76, 0, 0, 0, 7989, 93, 0, 0, 0, 0, 0, 0, 0, 549, 5, 0, 15, 0, 0, 0, 0, 3, 0, 2, 0, 0, 218, 8147, 55, 0, 0, 41, 6381, 169, 15, 4, 91, 0, 14, 0, 0, 570, 7894, 0, 3, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 14, 7680, 66, 0, 0, 33, 0, 2, 0, 38, 42, 0, 0, 0, 0, 0, 24, 33, 0, 0, 0, 0, 4, 0, 0, 976, 25, 891, 0, 0, 0, 0, 0, 0, 11, 13, 1, 113, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 0, 0, 1, 1, 0, 7997, 0, 0, 0, 2, 6866, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 275, 0, 6, 8103, 0, 0, 0, 2, 8149, 0, 0, 0, 0, 0, 0, 7, 0, 275, 0, 0, 0, 0, 0, 1, 20, 25, 1, 4, 0, 3, 0, 471, 1, 0, 0, 0, 8124, 0, 0, 69, 0, 0, 1, 1, 2, 0, 6932, 0, 0, 45, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 7195, 0, 1, 39, 35, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 180, 0, 8, 6, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1, 0, 0, 0, 0, 0, 36, 0, 8074, 0, 167, 1, 0, 4644, 0, 612, 0, 0, 0, 90, 0, 0, 0, 0, 0, 49, 0, 1, 0, 7444, 726, 0, 59, 51, 30, 0, 5, 0, 1, 0, 451, 0, 54, 46, 0, 5, 0, 140, 0, 0, 1958, 0, 36, 0, 0, 0, 6479, 5546, 1, 0, 1, 0, 7698, 393, 1, 3835, 0, 0, 0, 0, 0, 61, 8147, 1, 0, 258, 57, 21, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 7499, 8, 0, 0, 0, 0, 0, 0, 66, 159, 0, 0, 81, 0, 0, 0, 289, 0, 0, 0, 0, 0, 2, 0, 6428, 69, 0, 0, 0, 0, 0, 0, 23, 0, 2, 3, 0, 0, 0, 0, 2, 0, 57, 0, 244, 29, 17, 0, 0, 0, 338, 0, 7926, 0, 0, 0, 0, 7850, 10, 6153, 1, 0, 0, 77, 0, 0, 0, 6, 15, 0, 8, 0, 0, 7987, 0, 0, 0, 15, 26, 0, 1192, 4, 0, 0, 0, 0, 0, 1547, 0, 0, 532, 1, 10, 0, 0, 0, 0, 241, 0, 0, 11, 830, 0, 0, 0, 0, 7576, 5, 0, 13, 8, 6, 0, 1, 1, 1, 8001, 0, 72, 87, 0, 4, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 7860, 0, 4, 0, 0, 12, 10, 0, 0, 0, 4, 9, 0, 0, 2, 58, 0, 37, 0, 0, 0, 1, 20, 1257, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 204, 0, 0, 128, 0, 0, 4, 19, 0, 730, 34, 0, 0, 0, 0, 0, 6296, 0, 6372, 0, 0, 0, 0, 6986, 0, 7819, 0, 0, 10, 0, 0, 1, 0, 3, 2, 3, 0, 0, 0, 6768, 769, 8, 39, 0, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 1, 0, 2152, 2, 0, 0, 0, 16, 2, 1146, 0, 0, 0, 37, 7240, 0, 5, 0, 0, 7, 1, 5, 0, 273, 7826, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 12, 5, 0, 5744, 0, 22, 23, 57, 2, 259, 25, 4, 0, 41, 1489, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 190, 0, 0, 249, 11, 0, 0, 1144, 0, 495, 1, 673, 278, 0, 4648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 277, 0, 7, 0, 0, 2, 3, 0, 0, 0, 147, 7188, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4951, 0, 4422, 0, 3, 0, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3116, 2, 743, 0, 0, 1, 3, 20, 0, 4416, 8, 0, 246, 1965, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 224, 715, 17, 2697, 0, 3997, 0, 0, 2, 3, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 146, 0, 0, 31, 0, 0, 7097, 0, 45, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 7692, 8745, 9889, 3826, 10571, 9365, 7342] avg=8192.0000 max=10571 max_rank=5 min=3826 min_rank=4 imbalance_ratio=1.290405 max_load_reduction=54965 max_load_reduction_pct=0.838699 moved_to_echo=31104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 9, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1522, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 0, 0, 0, 0, 8, 0, 0, 15, 5455, 0, 5, 4796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 7616, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2, 0, 24, 0, 3, 2, 0, 6036, 0, 3, 0, 0, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 25, 3419, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 7667, 1, 0, 0, 24, 0, 216, 1, 0, 0, 0, 32, 0, 2008, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 525, 0, 1, 88, 441, 8144, 46, 0, 0, 0, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 5, 3, 27, 0, 0, 229, 16, 6460, 1, 0, 480, 451, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2343, 10292, 8779, 6644, 8665, 10009, 11009, 7795] avg=8192.0000 max=11009 max_rank=6 min=2343 min_rank=0 imbalance_ratio=1.343872 max_load_reduction=54527 max_load_reduction_pct=0.832016 moved_to_echo=37478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 14, 7680, 66, 0, 0, 33, 0, 2, 0, 38, 42, 0, 1775, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 33, 0, 0, 0, 0, 4, 0, 0, 976, 25, 891, 5018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 13, 1, 113, 0, 0, 0, 0, 0, 0, 6411, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 0, 0, 1, 1, 0, 7997, 0, 0, 0, 2, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 275, 0, 6, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8149, 0, 0, 0, 0, 0, 0, 7, 0, 275, 0, 0, 0, 2342, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 25, 1, 4, 0, 3, 0, 471, 1, 0, 0, 0, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 1, 1, 2, 0, 6545, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10046, 6971, 7757, 8471, 12429, 10829, 2370, 6663] avg=8192.0000 max=12429 max_rank=4 min=2370 min_rank=6 imbalance_ratio=1.517212 max_load_reduction=53107 max_load_reduction_pct=0.810349 moved_to_echo=32291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 7195, 0, 1, 39, 35, 0, 0, 0, 0, 0, 0, 1631, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 180, 0, 8, 6, 1, 0, 0, 6, 0, 0, 4755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 44, 4336, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 36, 0, 8074, 0, 167, 1, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 90, 0, 0, 0, 0, 0, 49, 0, 1, 0, 3662, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 51, 30, 0, 5, 0, 1, 0, 451, 0, 54, 46, 0, 5, 0, 2151, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 1958, 0, 36, 0, 0, 0, 6479, 5546, 1, 0, 1, 0, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 1, 3835, 0, 0, 0, 0, 0, 61, 7692, 1, 0, 258, 57, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9363, 4957, 6358, 8587, 5140, 2904, 15908, 12319] avg=8192.0000 max=15908 max_rank=6 min=2904 min_rank=5 imbalance_ratio=1.941895 max_load_reduction=49628 max_load_reduction_pct=0.757263 moved_to_echo=36271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 7499, 8, 0, 0, 0, 0, 0, 0, 66, 159, 0, 1750, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 289, 0, 0, 0, 0, 0, 2, 0, 6428, 69, 0, 4680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 2, 3, 0, 0, 0, 0, 2, 0, 57, 7330, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 29, 17, 0, 0, 0, 338, 0, 7926, 0, 0, 0, 0, 520, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 1, 0, 0, 77, 0, 0, 0, 6, 15, 0, 8, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 26, 0, 1192, 4, 0, 0, 0, 0, 0, 1547, 0, 0, 532, 2308, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 0, 0, 241, 0, 0, 11, 830, 0, 0, 0, 0, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 13, 8, 6, 0, 1, 1, 1, 7554, 0, 72, 87, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9931, 11549, 8543, 9084, 10189, 5674, 2813, 7753] avg=8192.0000 max=11549 max_rank=1 min=2813 min_rank=6 imbalance_ratio=1.409790 max_load_reduction=53987 max_load_reduction_pct=0.823776 moved_to_echo=26429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7894, 0, 0, 0, 170, 2, 0, 0, 0, 1, 11, 1799, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 1, 0, 0, 0, 1340, 0, 19, 4745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 0, 0, 10, 31, 8, 411, 0, 0, 0, 0, 250, 0, 6848, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 520, 0, 4890, 0, 0, 0, 5, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 4019, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 9, 0, 985, 1, 1, 0, 2373, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 196, 970, 6255, 0, 0, 0, 0, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 72, 110, 0, 14, 1, 0, 15, 7423, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10322, 6127, 8713, 6109, 6077, 11370, 9158, 7660] avg=8192.0000 max=11370 max_rank=5 min=6077 min_rank=4 imbalance_ratio=1.387939 max_load_reduction=54166 max_load_reduction_pct=0.826508 moved_to_echo=34265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3532, 0, 5, 0, 0, 0, 15, 19, 9, 8, 0, 1585, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 7848, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 151, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7568, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 410, 10, 1889, 1, 0, 0, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 2, 177, 0, 0, 0, 3561, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7989, 93, 0, 0, 0, 0, 0, 0, 0, 549, 5, 0, 15, 2092, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 218, 8147, 55, 0, 0, 41, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 15, 4, 91, 0, 14, 0, 0, 570, 7453, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5615, 12092, 8677, 2858, 7273, 10785, 9914, 8322] avg=8192.0000 max=12092 max_rank=1 min=2858 min_rank=3 imbalance_ratio=1.476074 max_load_reduction=53444 max_load_reduction_pct=0.815491 moved_to_echo=36294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 7860, 0, 4, 0, 0, 12, 10, 0, 0, 0, 4, 1482, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 58, 0, 37, 0, 0, 0, 1, 20, 1257, 0, 0, 0, 4472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 204, 0, 0, 128, 0, 0, 4, 19, 0, 6524, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 34, 0, 0, 0, 0, 0, 6296, 0, 6372, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7819, 0, 0, 10, 0, 0, 1, 0, 3, 2, 3, 0, 0, 0, 3330, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 39, 0, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 1956, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2152, 2, 0, 0, 0, 16, 2, 1146, 0, 0, 0, 37, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7, 1, 5, 0, 273, 7389, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9810, 5856, 7964, 13894, 11937, 3394, 4999, 7682] avg=8192.0000 max=13894 max_rank=3 min=3394 min_rank=5 imbalance_ratio=1.696045 max_load_reduction=51642 max_load_reduction_pct=0.787994 moved_to_echo=28012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 4224, 128, 896, 1024, 128, 3072, 8192, 34560, 128, 0, 128, 128, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 384, 128, 128, 640, 256, 256, 128, 1280, 128, 640, 1280, 22784, 384, 1024, 37120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[32640, 128, 0, 128, 256, 0, 128, 256, 128, 128, 256, 384, 128, 128, 29312, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 36224, 128, 2048, 128, 128, 128, 768, 128, 128, 7936, 128, 128, 896, 17280, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 47872, 128, 128, 128, 384, 128, 384, 128, 128, 256, 128, 13056, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 256, 128, 384, 128, 1152, 128, 640, 256, 128, 128, 128, 384, 256, 53888, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[1024, 384, 128, 128, 128, 128, 128, 12672, 128, 47616, 128, 256, 128, 128, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[640, 256, 4096, 384, 128, 128, 128, 256, 1024, 57728, 128, 128, 896, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[681, 20, 0, 10, 0, 1979, 0, 7, 0, 0, 206, 8168, 0, 11, 0, 0, 0, 123, 0, 6, 8040, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 7321, 0, 0, 0, 0, 0, 120, 0, 0, 1201, 0, 0, 0, 0, 71, 7, 3, 0, 0, 746, 0, 212, 0, 1233, 137, 0, 0, 4, 0, 3045, 6206, 1, 0, 7, 0, 58, 5444, 0, 76, 2, 0, 0, 0, 8172, 7, 0, 0, 3, 0, 0, 3943, 0, 0, 0, 5, 0, 2, 731, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 7482, 0, 0, 0, 0, 0, 19, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2278, 0, 5, 0, 19, 0, 2770, 0, 0, 0, 0, 0, 2936, 0, 0, 7656, 0, 0, 1, 0, 2, 0, 0, 21, 0, 7, 1, 0, 0, 0, 2, 0, 0, 0, 8138, 1, 0, 32, 24, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 46, 0, 0, 0, 6, 8, 227, 6106, 0, 0, 9, 0, 0, 63, 1, 0, 0, 426, 21, 0, 6876, 0, 0, 0, 6, 0, 0, 3292, 0, 1, 0, 0, 2158, 0, 8062, 0, 0, 5, 0, 0, 5356, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 4, 2, 0, 75, 55, 0, 6525, 0, 213, 213, 207, 2, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 1541, 0, 0, 7425, 5180, 0, 0, 0, 0, 0, 0, 864, 0, 5, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 2514, 0, 0, 0, 0, 0, 0, 7, 1463, 3, 299, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 7639, 5, 62, 0, 0, 1, 7493, 0, 0, 0, 0, 238, 6803, 1, 0, 180, 0, 0, 0, 8187, 8, 0, 3, 0, 0, 0, 258, 283, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6573, 2, 150, 65, 0, 0, 0, 32, 0, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 4, 0, 45, 0, 6011, 0, 35, 0, 0, 5, 0, 0, 44, 0, 1, 3080, 39, 0, 2, 0, 0, 0, 0, 37, 3, 71, 11, 0, 0, 0, 0, 0, 14, 9, 7494, 170, 0, 23, 2370, 3, 0, 10, 10, 0, 616, 0, 43, 0, 0, 0, 128, 3, 0, 0, 0, 0, 0, 3642, 0, 0, 0, 1625, 5603, 53, 670, 0, 129, 419, 5015, 0, 0, 35, 0, 0, 4302, 0, 0, 1, 2, 1, 597, 4368, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 35, 0, 0, 555, 0, 0, 0, 0, 0, 0, 17, 0, 0, 190, 0, 8, 0, 40, 0, 0, 385, 0, 548, 8, 0, 2793, 6235, 0, 1, 0, 0, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[155, 0, 0, 0, 0, 184, 0, 1372, 0, 27, 0, 8095, 0, 4, 0, 0, 0, 0, 53, 229, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 83, 0, 0, 4, 0, 0, 0, 767, 7463, 0, 0, 0, 48, 0, 0, 0, 0, 0, 7950, 0, 0, 123, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 13, 0, 0, 431, 6936, 2637, 0, 0, 5, 0, 8136, 0, 2, 922, 0, 0, 1918, 2, 0, 0, 0, 0, 0, 6240, 0, 1, 0, 0, 2567, 438, 11, 0, 0, 0, 0, 0, 190, 28, 0, 0, 0, 0, 4, 0, 33, 0, 0, 0, 0, 0, 2, 0, 0, 3739, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[119, 0, 101, 12, 0, 7291, 0, 0, 14, 0, 7, 3150, 0, 0, 1, 0, 0, 2, 1, 0, 1267, 1, 0, 0, 94, 18, 0, 0, 2, 0, 0, 10, 0, 347, 0, 0, 0, 4, 0, 8077, 2, 0, 2, 2, 0, 0, 0, 0, 0, 5215, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2132, 1, 0, 0, 5, 8153, 0, 0, 55, 0, 0, 732, 0, 462, 954, 0, 0, 32, 0, 0, 0, 306, 2, 0, 5099, 0, 1241, 0, 0, 0, 1, 7178, 0, 0, 0, 0, 28, 3447, 86, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 48, 2, 0, 0, 8155, 0, 0, 1643, 12, 0, 0, 2, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 5, 0, 0, 196, 0, 3, 0, 280, 0, 7861, 0, 0, 0, 787, 0, 0, 2, 0, 3348, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 634, 22, 0, 0, 416, 0, 0, 0, 0, 0, 4118, 1, 0, 1, 0, 169, 0, 0, 0, 72, 0, 0, 0, 8094, 0, 0, 0, 0, 50, 0, 0, 0, 2, 0, 2702, 7, 160, 868, 0, 0, 7147, 0, 0, 0, 0, 0, 0, 7966, 1, 145, 0, 1, 1, 26, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 22, 0, 0, 0, 0, 28, 4, 3730, 0, 8188, 0, 40, 4, 8184, 0, 0, 0, 7, 183] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 5, 0, 0, 0, 2, 0, 0, 464, 40, 7323, 0, 8, 0, 0, 0, 0, 0, 263, 757, 0, 0, 1578, 0, 0, 0, 0, 36, 13, 0, 1945, 0, 4, 9, 0, 0, 1, 7, 23, 0, 0, 0, 7962, 0, 0, 4, 0, 0, 0, 283, 31, 0, 53, 0, 0, 0, 10, 0, 0, 0, 0, 169, 144, 0, 0, 0, 5626, 0, 559, 0, 9, 0, 5873, 0, 0, 0, 0, 0, 7370, 0, 0, 0, 0, 0, 40, 6333, 0, 1, 0, 19, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 0, 4, 0, 613, 0, 0, 0, 0, 2, 0, 5, 7665, 0, 140, 283, 1799, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[681, 20, 0, 10, 0, 1979, 0, 7, 0, 0, 206, 6562, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 6, 8040, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2581, 2225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 7321, 0, 0, 0, 0, 1606, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 1201, 0, 0, 0, 0, 71, 7, 3, 0, 0, 746, 0, 1967, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 1233, 137, 0, 0, 4, 0, 3045, 1658, 1, 0, 7, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 2, 0, 0, 0, 3621, 7, 0, 0, 3, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 731, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5444, 2326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 5457, 0, 0, 0, 0, 0, 19, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9476, 12976, 9553, 5525, 6355, 7652, 8512, 5487] avg=8192.0000 max=12976 max_rank=1 min=5487 min_rank=7 imbalance_ratio=1.583984 max_load_reduction=52560 max_load_reduction_pct=0.802002 moved_to_echo=28006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2278, 0, 5, 0, 19, 0, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2936, 0, 0, 7656, 0, 0, 1, 0, 2, 0, 0, 21, 0, 7, 1, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8138, 1, 0, 32, 24, 0, 0, 0, 0, 544, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 46, 0, 0, 0, 6, 0, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 227, 6106, 0, 0, 9, 0, 0, 63, 1, 0, 0, 426, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1459, 0, 1, 0, 0, 2158, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5356, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 6876, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 75, 55, 0, 4759, 0, 213, 213, 207, 2, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4528, 11520, 9277, 2939, 6861, 11686, 13186, 5539] avg=8192.0000 max=13186 max_rank=6 min=2939 min_rank=3 imbalance_ratio=1.609619 max_load_reduction=52350 max_load_reduction_pct=0.798798 moved_to_echo=37272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[119, 0, 101, 12, 0, 7291, 0, 0, 14, 0, 7, 2531, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1267, 1, 0, 0, 94, 18, 0, 0, 2, 0, 0, 10, 304, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 4, 0, 8077, 2, 0, 2, 2, 0, 0, 0, 0, 619, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5215, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2132, 1, 232, 1537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8153, 0, 0, 55, 0, 0, 196, 0, 462, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 2, 0, 2260, 0, 1241, 0, 0, 0, 1, 7178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3447, 86, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 0, 32, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 2, 0, 0, 5948, 0, 0, 1643, 12, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10076, 3087, 9723, 9121, 9825, 10988, 5053, 7663] avg=8192.0000 max=10988 max_rank=5 min=3087 min_rank=1 imbalance_ratio=1.341309 max_load_reduction=54548 max_load_reduction_pct=0.832336 moved_to_echo=33529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 1541, 0, 0, 7425, 4162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 0, 5, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 2514, 3116, 2229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1463, 3, 299, 0, 0, 0, 0, 0, 0, 1018, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 2375, 1239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7639, 5, 62, 0, 0, 1, 2002, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 180, 0, 0, 0, 3628, 8, 0, 3, 0, 0, 0, 258, 283, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 6803, 2330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4794, 2, 150, 65, 0, 0, 0, 32, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13129, 8732, 3330, 11744, 9947, 4363, 9187, 5104] avg=8192.0000 max=13129 max_rank=0 min=3330 min_rank=2 imbalance_ratio=1.602661 max_load_reduction=52407 max_load_reduction_pct=0.799667 moved_to_echo=28601 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 4, 0, 45, 0, 6011, 0, 29, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1, 3080, 39, 0, 2, 0, 0, 0, 0, 37, 3, 71, 11, 2085, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 9, 7494, 170, 0, 23, 2370, 3, 0, 10, 10, 6, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 43, 0, 0, 0, 128, 3, 0, 0, 0, 0, 0, 3642, 0, 1589, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1625, 5603, 53, 670, 0, 129, 419, 1341, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 597, 1936, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 555, 0, 0, 0, 0, 0, 0, 17, 0, 0, 190, 0, 8, 4302, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 385, 0, 400, 8, 0, 2793, 6235, 0, 1, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6096, 6562, 10154, 6124, 9875, 10478, 6350, 9897] avg=8192.0000 max=10478 max_rank=5 min=6096 min_rank=0 imbalance_ratio=1.279053 max_load_reduction=55058 max_load_reduction_pct=0.840118 moved_to_echo=36600 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[155, 0, 0, 0, 0, 184, 0, 1372, 0, 27, 0, 6503, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 229, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 83, 3383, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 767, 7463, 0, 0, 0, 48, 0, 0, 0, 0, 1592, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 123, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 13, 2579, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 6936, 2637, 0, 0, 5, 0, 2174, 0, 2, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2765, 0, 1, 0, 0, 2567, 438, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 28, 0, 0, 0, 0, 4, 0, 33, 0, 0, 0, 0, 1918, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3739, 0, 2144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 7227, 10115, 11224, 13107, 5784, 3949, 5885] avg=8192.0000 max=13107 max_rank=4 min=3949 min_rank=6 imbalance_ratio=1.599976 max_load_reduction=52429 max_load_reduction_pct=0.800003 moved_to_echo=28725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 5, 0, 0, 0, 2, 0, 0, 464, 40, 5883, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 757, 0, 0, 1578, 0, 0, 0, 0, 36, 13, 0, 1945, 2442, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 9, 0, 0, 1, 7, 23, 0, 0, 0, 7962, 0, 0, 4, 0, 1440, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 31, 0, 53, 0, 0, 0, 10, 0, 0, 0, 0, 169, 144, 1861, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5626, 0, 559, 0, 9, 0, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 2807, 0, 1, 0, 19, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 0, 4, 0, 613, 0, 7370, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 5590, 0, 140, 283, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6403, 8758, 10080, 3996, 7764, 10859, 9857, 7819] avg=8192.0000 max=10859 max_rank=5 min=3996 min_rank=3 imbalance_ratio=1.325562 max_load_reduction=54677 max_load_reduction_pct=0.834305 moved_to_echo=36299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 5, 0, 0, 196, 0, 3, 0, 280, 0, 6310, 0, 0, 0, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3348, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 23, 1127, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 634, 22, 0, 0, 416, 0, 0, 0, 0, 1551, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4118, 1, 0, 1, 0, 169, 0, 0, 0, 72, 0, 0, 0, 8094, 0, 860, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 2, 0, 715, 7, 160, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 1, 145, 0, 1, 1, 26, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 22, 0, 0, 7147, 2269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4, 3730, 0, 5962, 0, 40, 4, 8184, 0, 0, 0, 7, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7583, 6674, 3304, 14863, 1802, 3726, 9442, 18142] avg=8192.0000 max=18142 max_rank=7 min=1802 min_rank=4 imbalance_ratio=2.214600 max_load_reduction=47394 max_load_reduction_pct=0.723175 moved_to_echo=33112 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 3200, 1024, 512, 25984, 128, 128, 1664, 128, 128, 128, 128, 128, 128, 128, 4608, 15104, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1664, 18048, 384, 128, 1408, 128, 256, 256, 128, 128, 128, 128, 0, 0, 22912, 256, 11520, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 7936, 128, 35072, 128, 640, 5120, 16512, 128, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 9600, 896, 0, 128, 128, 128, 128, 128, 128, 128, 256, 640, 128, 39936, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 256, 384, 128, 640, 22016, 128, 1408, 128, 128, 4736, 512, 35456, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 256, 9472, 34176, 2816, 1408, 128, 256, 3584, 9728, 128, 640, 3328, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1024, 128, 128, 128, 0, 12032, 128, 3072, 128, 6912, 7680, 34304, 0, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 384, 128, 128, 0, 128, 896, 33408, 256, 384, 128, 18176, 128, 0, 128, 128, 8448, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 1074, 0, 0, 2, 0, 0, 0, 0, 29, 0, 0, 0, 0, 2319, 6249, 12, 0, 8163, 0, 12, 7, 6760, 1, 0, 0, 0, 30, 8151, 26, 3, 0, 740, 0, 11, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 0, 0, 5427, 0, 3, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 2660, 0, 7877, 0, 0, 160, 698, 0, 1, 0, 2, 0, 0, 106, 7, 0, 0, 0, 0, 0, 0, 3, 0, 552, 1090, 232, 0, 9, 0, 206, 1, 0, 8040, 0, 231, 0, 0, 969, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 6, 2, 0, 24, 0, 0, 0, 0, 0, 7860, 0, 47, 0, 725, 78, 0, 5008, 0, 278, 0, 0, 1, 0, 77, 49, 0, 1780, 0, 0, 0, 149, 0, 0, 0, 0, 0, 11, 0, 20, 0, 0, 5, 17, 0, 102, 262, 972, 0, 0, 0, 0, 0, 0, 0, 240, 0, 919, 21, 0, 17, 2156, 178, 0, 0, 4, 8, 0, 0, 44, 0, 1668, 0, 0, 6, 0, 0, 7498, 0, 0, 0, 1842, 0, 7870, 1, 0, 0, 924, 0, 40, 0, 0, 0, 424, 88, 5, 29, 0, 0, 0, 0, 0, 6722, 820, 7490, 0, 2, 1, 13, 0, 0, 45, 0, 8172, 1, 1, 0, 736, 0, 0, 78, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1856, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 0, 971, 7222, 3801, 0, 104, 1084, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1, 0, 187, 0, 1983, 2, 6, 0, 0, 0, 1, 0, 0, 0, 6680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1265, 0, 0, 0, 7545, 8108, 0, 0, 0, 0, 8187, 0, 79, 0, 0, 5, 0, 0, 1, 1, 5622, 0, 0, 1, 0, 0, 13, 0, 0, 7, 0, 2825, 0, 0, 0, 0, 0, 0, 0, 12, 4517, 0, 0, 0, 38, 9, 0, 0, 0, 328] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 46, 1, 0, 2, 8, 1, 43, 2, 0, 7, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 18, 0, 8128, 0, 0, 5339, 132, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 598, 0, 8061, 0, 102, 191, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 2, 0, 0, 0, 85, 71, 0, 0, 7468, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 1625, 0, 0, 0, 0, 0, 996, 0, 0, 0, 0, 0, 0, 0, 10, 1754, 1564, 803, 0, 0, 2, 0, 0, 5937, 0, 0, 5740, 0, 0, 0, 0, 7957, 0, 0, 0, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 12, 1676, 0, 33, 24, 0, 4, 0, 56, 0, 1260, 0, 137, 0, 0, 0, 0, 37, 0, 0, 0, 0, 4326, 0, 0, 6478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6, 2994, 0, 0, 0, 0, 27, 0, 0, 0, 0, 437, 0, 0, 2, 57, 0, 8045, 0, 0, 0, 0, 717, 0, 0, 0, 1, 5989, 2456, 0, 0, 9, 235, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 27, 1, 0, 0, 0, 7546, 8186, 2, 2, 0, 7, 9, 110, 780, 421, 7650, 0, 0, 0, 1, 0, 0, 0, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3289, 0, 0, 7052, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 21, 7793, 0, 96, 2, 0, 0, 1, 0, 0, 1112, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1, 0, 0, 0, 73, 0, 0, 168, 0, 4, 0, 0, 5439, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7935, 0, 0, 15, 4, 0, 22, 16, 0, 0, 7172, 404, 0, 1, 38, 0, 0, 0, 0, 6795, 0, 3814, 0, 0, 0, 0, 0, 5787, 0, 540, 7645, 0, 29, 0, 0, 0, 19, 51, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 82, 0, 933, 0, 0, 8, 0, 0, 0, 0, 0, 4912, 8172, 235, 850, 0, 0, 0, 5086, 0, 3, 13, 0, 0, 0, 160, 0, 0, 0, 0, 6, 0, 0, 312, 0, 238, 0, 0, 0, 0, 0, 0, 0, 444, 0, 3313, 0, 0, 0, 11, 529, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 7327, 0, 0, 5317, 0, 0, 0, 0, 0, 29, 8087, 0, 0, 0, 0, 0, 1132, 0, 0, 7641, 0, 0, 2271, 0, 21, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 1, 74, 0, 1, 0, 0, 0, 0, 3098, 0, 0, 0, 10, 0, 0, 0, 501, 132, 0, 0, 0, 7916, 213, 0, 0, 8157, 0, 0, 0, 0, 190, 1321, 0, 325, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 565, 0, 0, 0, 323, 0, 0, 0, 12, 0, 7629, 0, 0, 5, 23, 6774, 0, 0, 0, 111, 2, 0, 0, 657, 22, 0, 2994, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 8168, 0, 234, 0, 268, 0, 13, 0, 1, 0, 13, 22, 1, 0, 0, 147, 2, 0, 0, 0, 64, 7069, 1, 0, 32, 0, 0, 311, 0, 5, 4523, 0, 166, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 1074, 4283, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 29, 0, 0, 0, 0, 2319, 6078, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 12, 7, 6760, 1, 0, 0, 0, 30, 8151, 26, 3, 0, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5427, 0, 3, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 2, 2488, 638, 171, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2660, 0, 3594, 0, 0, 160, 698, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 106, 7, 0, 0, 0, 0, 0, 0, 3, 0, 98, 1090, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 206, 1, 0, 5552, 0, 231, 0, 0, 969, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5441, 8440, 23255, 3985, 8786, 7114, 1538, 6977] avg=8192.0000 max=23255 max_rank=2 min=1538 min_rank=6 imbalance_ratio=2.838745 max_load_reduction=42281 max_load_reduction_pct=0.645157 moved_to_echo=24415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 12, 1676, 0, 33, 24, 0, 4, 0, 56, 3257, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 0, 137, 0, 0, 0, 0, 37, 0, 0, 0, 0, 4208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6, 2994, 0, 5354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 437, 0, 0, 2, 57, 0, 8045, 0, 2367, 508, 118, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 717, 0, 0, 0, 1, 2732, 2456, 0, 0, 9, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 27, 1, 0, 0, 0, 7546, 1441, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 110, 780, 421, 5283, 0, 0, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 5642, 7465, 8356, 11747, 6150, 13277, 6621] avg=8192.0000 max=13277 max_rank=6 min=5642 min_rank=1 imbalance_ratio=1.620728 max_load_reduction=52259 max_load_reduction_pct=0.797409 moved_to_echo=37795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 6, 2, 0, 24, 0, 0, 0, 0, 0, 7860, 4280, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 725, 78, 0, 5008, 0, 278, 0, 0, 1, 0, 75, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 0, 0, 0, 149, 0, 0, 0, 0, 0, 11, 0, 20, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 102, 262, 972, 0, 0, 0, 0, 0, 0, 0, 240, 0, 919, 21, 4899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2156, 178, 0, 0, 4, 8, 0, 0, 44, 0, 1668, 0, 0, 6, 2528, 139, 2, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7498, 0, 0, 0, 1842, 0, 3590, 1, 0, 0, 924, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 88, 5, 29, 0, 0, 0, 0, 0, 6722, 820, 1319, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 45, 0, 5644, 1, 1, 0, 736, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13274, 6261, 1826, 7432, 6920, 13895, 9409, 6519] avg=8192.0000 max=13895 max_rank=5 min=1826 min_rank=2 imbalance_ratio=1.696167 max_load_reduction=51641 max_load_reduction_pct=0.787979 moved_to_echo=36743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4452, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2948, 0, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6658, 3801, 0, 104, 1084, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 187, 0, 1983, 2, 6, 0, 0, 0, 1, 0, 0, 0, 6680, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1265, 0, 1397, 564, 83, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7545, 8108, 0, 0, 0, 0, 3735, 0, 79, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5622, 0, 0, 1, 0, 0, 13, 0, 0, 7, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3120, 0, 0, 0, 38, 9, 0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4875, 5787, 11661, 10707, 3383, 19472, 6144, 3507] avg=8192.0000 max=19472 max_rank=5 min=3383 min_rank=4 imbalance_ratio=2.376953 max_load_reduction=46064 max_load_reduction_pct=0.702881 moved_to_echo=32506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 1, 74, 0, 1, 0, 0, 0, 0, 3098, 4442, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 501, 132, 0, 0, 0, 7916, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 0, 0, 190, 1321, 0, 325, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 323, 0, 0, 0, 12, 0, 7629, 0, 4623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 23, 6774, 0, 0, 0, 111, 2, 0, 0, 657, 22, 0, 2994, 0, 1399, 637, 5, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 3726, 0, 234, 0, 268, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 22, 1, 0, 0, 147, 2, 0, 0, 0, 64, 1246, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 311, 0, 5, 3124, 0, 166, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8673, 8767, 9452, 13152, 12789, 7567, 1497, 3639] avg=8192.0000 max=13152 max_rank=3 min=1497 min_rank=6 imbalance_ratio=1.605469 max_load_reduction=52384 max_load_reduction_pct=0.799316 moved_to_echo=25492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3289, 0, 0, 7052, 4315, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 21, 7580, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1112, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1, 0, 0, 0, 73, 0, 0, 168, 0, 2494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5439, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2365, 0, 213, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3620, 0, 0, 15, 4, 0, 22, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7172, 404, 0, 1, 38, 0, 0, 0, 0, 6795, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5787, 0, 540, 5280, 0, 29, 0, 0, 0, 19, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15228, 7747, 1164, 2822, 8108, 3678, 15083, 11706] avg=8192.0000 max=15228 max_rank=0 min=1164 min_rank=2 imbalance_ratio=1.858887 max_load_reduction=50308 max_load_reduction_pct=0.767639 moved_to_echo=38575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 46, 1, 0, 2, 8, 1, 43, 2, 0, 7, 1, 0, 4373, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 18, 0, 7905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4922, 132, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 8061, 0, 102, 191, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 2, 0, 0, 0, 85, 71, 0, 0, 7468, 0, 1776, 417, 223, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3668, 0, 0, 0, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 0, 0, 0, 0, 0, 0, 10, 1754, 1564, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5937, 0, 0, 3964, 0, 0, 0, 0, 7957, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4603, 7928, 5445, 9481, 10447, 5295, 4466, 17871] avg=8192.0000 max=17871 max_rank=7 min=4466 min_rank=6 imbalance_ratio=2.181519 max_load_reduction=47665 max_load_reduction_pct=0.727310 moved_to_echo=38079 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 2825, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 933, 0, 0, 8, 0, 0, 0, 0, 0, 4912, 7944, 235, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5086, 0, 3, 13, 0, 0, 0, 160, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 238, 0, 0, 0, 0, 0, 0, 0, 444, 0, 3313, 0, 5294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 529, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2369, 0, 228, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 7327, 0, 0, 5317, 0, 0, 0, 0, 0, 29, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 0, 0, 5272, 0, 0, 2271, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7164, 14964, 5268, 9601, 3356, 2365, 14122, 8696] avg=8192.0000 max=14964 max_rank=1 min=2365 min_rank=5 imbalance_ratio=1.826660 max_load_reduction=50572 max_load_reduction_pct=0.771667 moved_to_echo=28539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 18560, 8064, 128, 128, 5376, 256, 128, 0, 128, 15360, 10112, 6912, 1152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[34304, 3968, 128, 5248, 8064, 256, 2560, 0, 384, 128, 8448, 128, 128, 1536, 896, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 3200, 128, 128, 128, 128, 1792, 128, 128, 128, 3328, 128, 128, 19200, 32256, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 15104, 12160, 0, 128, 4608, 128, 27136, 2560, 384, 256, 3584, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 13568, 896, 1024, 37248, 128, 512, 2304, 896, 8960, 128, 256, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 7680, 13440, 128, 0, 128, 256, 512, 128, 256, 768, 1792, 0, 19840, 128, 16768, 2944, 1152, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[640, 0, 8704, 384, 3968, 384, 128, 128, 384, 128, 128, 128, 768, 128, 25344, 128, 25472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 1408, 128, 3712, 128, 0, 5120, 640, 512, 0, 0, 128, 15232, 36992, 384, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 7122, 0, 0, 0, 1570, 0, 0, 0, 4651, 0, 0, 0, 0, 0, 0, 0, 0, 7100, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 130, 2, 0, 0, 0, 45, 0, 196, 0, 0, 0, 0, 4301, 0, 2, 8151, 1, 5, 0, 0, 0, 0, 0, 0, 162, 21, 0, 0, 0, 0, 75, 0, 0, 0, 0, 13, 0, 7, 0, 0, 8099, 0, 12, 4127, 0, 57, 0, 0, 7055, 1135, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 1, 3, 847, 2, 161, 46, 0, 0, 0, 0, 19, 517, 20, 1689, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 426, 0, 0, 3, 0, 3452, 10, 8, 5010, 0, 0, 0, 479, 0, 0, 0, 0, 420, 407, 1, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 142, 0, 0, 251, 0, 0, 0, 0, 2, 0, 2, 60, 4362, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 7359, 0, 0, 0, 0, 7230, 7588, 0, 0, 7, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 221, 0, 0, 0, 91, 8165, 0, 0, 0, 6192, 0, 2, 0, 1, 0, 20, 0, 2, 0, 0, 1771, 0, 0, 7999, 0, 0, 0, 0, 0, 0, 0, 709, 0, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 57, 0, 0, 1413, 0, 0, 0, 2721, 0, 10, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 1, 7, 0, 81, 0, 0, 0, 4, 0, 0, 131, 0, 0, 0, 0, 0, 0, 458, 5064, 26, 0, 0, 0, 0, 0, 0, 7996, 0, 5, 144, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 864, 7279, 0, 0, 8181, 0, 0, 0, 873, 6827, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 3, 0, 0, 0, 7267, 2870, 0, 2, 98, 8154, 0, 0, 8, 3443, 0, 0, 0, 6, 0, 0, 127, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 548, 0, 369] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[90, 8158, 0, 37, 0, 0, 5474, 0, 0, 10, 0, 0, 0, 0, 0, 2, 0, 1, 2299, 0, 0, 134, 347, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3016, 18, 0, 0, 0, 301, 509, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 6596, 0, 3138, 0, 0, 0, 27, 0, 1, 0, 0, 0, 0, 5312, 0, 0, 912, 0, 0, 1668, 6930, 0, 0, 0, 7175, 0, 1578, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 13, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 5436, 0, 0, 0, 0, 2, 0, 32, 5260, 0, 3, 0, 0, 0, 0, 0, 0, 8184, 30, 3, 0, 1, 0, 0, 0, 0, 0, 8, 0, 21, 32, 8177, 53, 0, 0, 147, 43, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 8175, 3791, 0, 0, 0, 0, 0, 0, 156, 0, 61, 0, 0, 0, 0, 1, 0, 0, 3, 7628, 0, 510, 0, 0, 1, 0, 0, 1775, 6087, 0, 0, 0, 4039, 0, 0, 0, 682, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 555] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 0, 0, 0, 556, 0, 7909, 0, 0, 0, 0, 0, 0, 0, 1, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 7023, 0, 5061, 0, 0, 0, 3115, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 32, 0, 2, 0, 48, 7925, 0, 1, 7241, 0, 0, 1, 0, 7200, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 90, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 0, 5356, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 374, 0, 122, 0, 0, 7249, 0, 0, 0, 0, 0, 960, 0, 133, 1, 0, 0, 1, 0, 13, 3, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 39, 2, 6005, 0, 13, 17, 539, 0, 0, 0, 4, 2046, 0, 0, 0, 1570, 146, 0, 0, 0, 2, 0, 0, 0, 3703, 0, 0, 8042, 6193, 0, 0, 0, 0, 293, 0, 8, 0, 5, 15, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8085, 2378, 0, 4, 0, 6270, 12, 1298, 0, 14, 0, 0, 0, 0, 3262, 36, 6606, 0, 0, 3, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 2259, 0, 0, 0, 0, 620, 0, 0, 7910, 0, 0, 0, 0, 0, 0, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 6298, 0, 1685, 0, 0, 1, 637, 43, 0, 264, 0, 0, 0, 0, 1, 12, 0, 0, 0, 67, 28, 1, 0, 0, 0, 5745, 0, 0, 8181, 0, 0, 0, 8, 8189, 0, 5696, 0, 12, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 0, 5, 0, 181, 1010, 0, 0, 1057, 7623, 0, 0, 1, 0, 1, 1, 0, 73, 0, 0, 30, 0, 0, 5715, 0, 0, 0, 0, 0, 0, 77, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 57, 0, 0, 1413, 0, 0, 0, 1793, 0, 10, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 1, 7, 0, 3919, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 4, 0, 0, 131, 0, 0, 0, 0, 0, 0, 458, 5064, 182, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 7996, 0, 5, 144, 0, 0, 0, 0, 0, 4235, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 864, 0, 0, 0, 8181, 0, 0, 0, 873, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 3, 0, 5095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7267, 2776, 0, 2, 98, 0, 0, 0, 8, 3443, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 548, 0, 369, 3442, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3296, 6248, 6014, 14759, 10971, 5411, 13600, 5237] avg=8192.0000 max=14759 max_rank=3 min=3296 min_rank=0 imbalance_ratio=1.801636 max_load_reduction=50777 max_load_reduction_pct=0.774796 moved_to_echo=35219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7122, 0, 0, 0, 1570, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7100, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 130, 408, 2431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 45, 0, 196, 0, 0, 0, 0, 4301, 0, 2, 8151, 1, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 162, 21, 0, 0, 0, 0, 75, 0, 0, 439, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 7, 0, 0, 0, 0, 12, 4127, 0, 57, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 5668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 1, 3, 0, 2, 161, 46, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 20, 1689, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3557, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11756, 10072, 13010, 3134, 5282, 6813, 8263, 7206] avg=8192.0000 max=13010 max_rank=2 min=3134 min_rank=3 imbalance_ratio=1.588135 max_load_reduction=52526 max_load_reduction_pct=0.801483 moved_to_echo=27564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 426, 0, 0, 3, 0, 3452, 10, 8, 3301, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 407, 1, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2977, 2170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 142, 0, 0, 251, 0, 0, 0, 0, 2, 0, 2, 60, 336, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4362, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 2, 0, 3215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7359, 0, 0, 0, 0, 0, 7588, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 221, 0, 5060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 7896, 0, 0, 0, 0, 0, 2, 0, 1, 0, 20, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 0, 0, 7999, 0, 0, 0, 0, 0, 0, 0, 709, 0, 32, 0, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7679, 6037, 4091, 7585, 14954, 5294, 8012, 11884] avg=8192.0000 max=14954 max_rank=4 min=4091 min_rank=2 imbalance_ratio=1.825439 max_load_reduction=50582 max_load_reduction_pct=0.771820 moved_to_echo=40144 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[90, 8158, 0, 37, 0, 0, 5474, 0, 0, 8, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2299, 0, 0, 134, 347, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3449, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3016, 18, 0, 0, 0, 301, 509, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 3726, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3138, 0, 0, 0, 27, 0, 1, 0, 0, 0, 0, 5312, 0, 0, 912, 2456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1668, 6702, 0, 0, 0, 0, 0, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 13, 0, 0, 0, 3325, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13769, 7301, 12207, 6029, 1009, 11846, 9948, 3427] avg=8192.0000 max=13769 max_rank=0 min=1009 min_rank=4 imbalance_ratio=1.680786 max_load_reduction=51767 max_load_reduction_pct=0.789902 moved_to_echo=26230 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2259, 0, 0, 0, 0, 620, 0, 0, 5210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 3665, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 6298, 0, 531, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 0, 1, 637, 43, 0, 264, 0, 0, 0, 0, 1, 12, 0, 0, 3958, 2823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 28, 1, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 8, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5696, 0, 12, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 181, 977, 0, 0, 1057, 0, 0, 0, 1, 0, 1, 1, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 5715, 0, 0, 0, 0, 0, 0, 77, 8, 0, 0, 4129, 2169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 7464, 6867, 9424, 9522, 9746, 2296, 12128] avg=8192.0000 max=12128 max_rank=7 min=2296 min_rank=6 imbalance_ratio=1.480469 max_load_reduction=53408 max_load_reduction_pct=0.814941 moved_to_echo=33692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5436, 0, 0, 0, 0, 2, 0, 32, 3465, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 30, 3, 0, 1, 0, 0, 0, 0, 0, 8, 0, 21, 32, 1941, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 53, 0, 0, 147, 43, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 353, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2098, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 0, 3791, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 1, 0, 0, 3, 7628, 0, 510, 0, 0, 1, 5723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 5887, 0, 0, 0, 0, 0, 0, 0, 682, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 555, 78, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8938, 12672, 8988, 5377, 5198, 13927, 8359, 2077] avg=8192.0000 max=13927 max_rank=5 min=2077 min_rank=7 imbalance_ratio=1.700073 max_load_reduction=51609 max_load_reduction_pct=0.787491 moved_to_echo=29561 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 374, 0, 122, 0, 0, 4775, 0, 0, 0, 0, 0, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 1, 0, 0, 1, 0, 13, 3, 0, 0, 0, 1, 0, 0, 0, 3014, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 39, 2, 6005, 0, 487, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 17, 539, 0, 0, 0, 4, 2046, 0, 0, 0, 1570, 146, 0, 0, 0, 3256, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3703, 0, 0, 0, 6193, 0, 0, 0, 0, 293, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 15, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 2300, 0, 4, 0, 0, 12, 1298, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3262, 36, 6606, 0, 0, 3, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6232, 5579, 6615, 7593, 10193, 5711, 11713, 11900] avg=8192.0000 max=11900 max_rank=7 min=5579 min_rank=1 imbalance_ratio=1.452637 max_load_reduction=53636 max_load_reduction_pct=0.818420 moved_to_echo=39517 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 0, 0, 0, 556, 0, 5199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 7023, 0, 537, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 0, 0, 0, 3115, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 2, 0, 48, 0, 0, 1, 7241, 0, 0, 1, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 0, 5547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 0, 5356, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 3633, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 10163, 7744, 11635, 8407, 6788, 3345, 11677] avg=8192.0000 max=11677 max_rank=7 min=3345 min_rank=6 imbalance_ratio=1.425415 max_load_reduction=53859 max_load_reduction_pct=0.821823 moved_to_echo=30217 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[14464, 128, 640, 128, 3840, 128, 640, 10496, 128, 128, 256, 1664, 256, 128, 128, 512, 20992, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3840, 128, 10240, 128, 0, 19200, 256, 128, 0, 0, 128, 128, 128, 1792, 128, 1024, 18176, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 256, 27904, 512, 128, 256, 640, 128, 128, 0, 0, 128, 128, 128, 128, 256, 19456, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 7552, 128, 3840, 1408, 1024, 0, 17664, 128, 27776, 128, 128, 384, 896, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 27264, 29824, 0, 128, 1280, 0, 128, 3072, 128, 4224, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[14336, 128, 0, 128, 384, 384, 768, 512, 0, 128, 0, 4352, 128, 128, 36096, 5248, 2816, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 23424, 128, 128, 384, 3072, 9728, 640, 128, 26880, 0, 128, 128, 512, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[1152, 9216, 128, 128, 128, 128, 128, 128, 128, 128, 8960, 128, 5888, 128, 256, 1024, 39296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 2144, 0, 5, 0, 0, 0, 16, 539, 0, 0, 2, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 34, 0, 0, 0, 100, 0, 0, 0, 2, 0, 12, 0, 0, 1, 8066, 0, 0, 0, 3865, 2, 8189, 2, 0, 5441, 0, 0, 0, 0, 0, 3, 0, 0, 29, 0, 0, 3, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 2, 5141, 3, 517, 18, 0, 0, 0, 0, 0, 0, 0, 6802, 0, 0, 0, 23, 0, 0, 9, 9, 6036, 0, 91, 0, 0, 0, 13, 60, 0, 0, 0, 7654, 0, 0, 1077, 0, 10, 0, 0, 7985, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4, 0, 0, 0, 1346, 0, 0, 0, 0, 20, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5, 470, 1, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 8165, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 89, 3, 0, 51, 83, 2, 0, 0, 0, 0, 0, 0, 1, 0, 5, 7447, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 7055, 1783, 51, 0, 0, 0, 0, 268, 0, 756, 124, 0, 3885, 7, 0, 0, 0, 0, 0, 8192, 176, 0, 0, 0, 7486, 0, 0, 0, 5094, 4862] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[136, 731, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 17, 0, 4, 0, 0, 0, 21, 0, 0, 249, 0, 0, 0, 4120, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6078, 0, 433, 0, 1, 0, 0, 0, 0, 1, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 5, 0, 0, 0, 0, 139, 0, 0, 7588, 0, 0, 0, 1, 0, 0, 511, 0, 0, 0, 2288, 0, 0, 5038, 0, 0, 2324, 8175, 0, 7446, 2, 0, 0, 1, 0, 1, 0, 4, 0, 6, 13, 0, 103, 0, 3374, 0, 0, 0, 7787, 8148] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5106, 0, 6576, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 3679, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 8180, 0, 0, 0, 0, 0, 0, 188, 0, 413, 0, 460, 1175, 0, 131, 0, 0, 4723, 48, 0, 1433, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 16, 0, 309, 66] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 538, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1127, 0, 4, 320, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 2, 0, 0, 7759, 0, 0, 0, 0, 0, 0, 0, 7221, 0, 0, 9, 0, 0, 8, 0, 8121, 0, 0, 0, 604, 124, 5, 0, 0, 6462, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 156, 0, 0, 20, 0, 0, 889, 0, 34, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 789, 7062, 0, 0, 5298, 0, 0, 0, 0, 0, 0, 6750, 8119, 0, 9, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 2139, 0, 0, 0, 0, 0, 9, 32, 0, 305, 0, 0, 649, 97, 0, 0, 0, 0, 0, 0, 981, 0, 0, 0, 2636, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 1, 0, 2, 0, 1, 23, 0, 7297, 0, 2, 0, 7200, 4433, 0, 6081, 0, 0, 0, 0, 2, 0, 403, 0, 0, 0, 0, 1, 0, 48, 0, 0, 0, 112, 0, 40, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 959, 80, 863, 8109, 0, 12, 0, 0, 0, 0, 0, 75, 0, 0, 1, 0, 0, 1095, 5744, 52, 1, 0, 4356, 8100] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[11, 0, 2, 0, 417, 0, 0, 29, 1584, 0, 0, 19, 29, 0, 0, 6, 0, 0, 0, 128, 0, 7204, 0, 38, 0, 7458, 0, 0, 0, 0, 0, 0, 0, 27, 20, 0, 0, 0, 282, 80, 0, 1, 0, 0, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 465, 0, 0, 0, 0, 5971, 0, 0, 0, 0, 10, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1888, 0, 0, 0, 42, 0, 129, 0, 30, 1, 811, 2072, 1, 0, 0, 7714, 30, 0, 356, 0, 1, 0, 0, 0, 7873, 0, 0, 7597, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 884, 4825] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 3, 0, 10, 0, 0, 1724, 0, 7240, 0, 6038, 1, 0, 0, 0, 2, 6466, 579, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 296, 0, 1, 0, 0, 0, 2734, 0, 14, 0, 0, 0, 0, 0, 43, 0, 1, 0, 0, 0, 0, 0, 18, 0, 422, 0, 0, 216, 0, 0, 0, 0, 0, 778, 0, 7604, 0, 0, 0, 0, 11, 34, 2, 5, 2790, 0, 0, 0, 11, 0, 3, 0, 0, 0, 13, 0, 0, 0, 0, 6903, 0, 1, 2, 0, 0, 2, 0, 0, 0, 0, 85, 0, 0, 7099, 0, 0, 1773, 3, 0, 5995, 1, 0, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 78, 0, 12, 4460] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2139, 0, 0, 0, 0, 0, 9, 32, 0, 305, 0, 0, 649, 97, 0, 5294, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 981, 0, 0, 0, 2636, 0, 8, 0, 0, 0, 0, 0, 1567, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 7446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 23, 0, 7297, 0, 2, 0, 2540, 4433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 403, 0, 0, 0, 0, 1, 0, 48, 0, 0, 8109, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 40, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 6, 0, 1997, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 959, 80, 863, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 1, 0, 0, 1095, 5744, 52, 1, 0, 2359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8946, 5425, 7456, 14299, 9350, 5548, 5185, 9327] avg=8192.0000 max=14299 max_rank=3 min=5185 min_rank=6 imbalance_ratio=1.745483 max_load_reduction=51237 max_load_reduction_pct=0.781815 moved_to_echo=29410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 2144, 0, 4738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 16, 539, 0, 0, 2, 0, 1042, 0, 0, 0, 0, 0, 4769, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 34, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 0, 1, 8066, 0, 0, 0, 3865, 2, 2887, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 29, 0, 0, 3, 0, 0, 0, 0, 6036, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 2, 5141, 3, 517, 18, 0, 0, 0, 0, 0, 3662, 3519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 0, 0, 0, 23, 0, 0, 9, 9, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 60, 0, 0, 0, 7654, 0, 0, 1077, 0, 10, 0, 0, 4323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7455, 8156, 135, 14837, 6774, 12877, 2165, 13137] avg=8192.0000 max=14837 max_rank=3 min=135 min_rank=2 imbalance_ratio=1.811157 max_load_reduction=50699 max_load_reduction_pct=0.773605 moved_to_echo=34953 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[136, 731, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 17, 0, 4, 0, 1, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 249, 0, 0, 0, 4120, 0, 0, 143, 0, 0, 0, 1604, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 368, 0, 0, 0, 0, 0, 0, 7489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6078, 0, 433, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 5, 7446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 7588, 0, 0, 0, 1, 0, 0, 511, 0, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 5038, 0, 0, 2324, 8175, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 6, 13, 0, 103, 0, 3374, 0, 0, 0, 4216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1385, 6373, 7914, 6513, 7600, 11810, 16224, 7717] avg=8192.0000 max=16224 max_rank=6 min=1385 min_rank=0 imbalance_ratio=1.980469 max_load_reduction=49312 max_load_reduction_pct=0.752441 moved_to_echo=43351 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4, 0, 0, 0, 1346, 0, 0, 0, 0, 20, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5, 470, 1, 0, 178, 0, 0, 4947, 1777, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 4471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 2880, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 89, 3, 0, 51, 83, 2, 0, 0, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 7447, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 2108, 1783, 51, 0, 0, 0, 0, 268, 0, 0, 124, 0, 3885, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 176, 0, 0, 0, 7486, 0, 0, 0, 2758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1625, 7519, 12342, 2915, 987, 13297, 8239, 18612] avg=8192.0000 max=18612 max_rank=7 min=987 min_rank=4 imbalance_ratio=2.271973 max_load_reduction=46924 max_load_reduction_pct=0.716003 moved_to_echo=41135 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 10, 0, 0, 1724, 0, 7240, 0, 6038, 1, 0, 0, 0, 2, 6621, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6466, 579, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 296, 0, 1, 0, 59, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 0, 14, 0, 0, 0, 0, 0, 43, 0, 1, 0, 0, 0, 4099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 422, 0, 0, 216, 0, 0, 0, 0, 0, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 34, 2, 5, 2790, 0, 0, 0, 11, 0, 3, 0, 5995, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 6903, 0, 1, 2, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 7099, 0, 0, 1773, 3, 0, 0, 1, 0, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 78, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21872, 7721, 6891, 1434, 9834, 6926, 10364, 494] avg=8192.0000 max=21872 max_rank=0 min=494 min_rank=7 imbalance_ratio=2.669922 max_load_reduction=43664 max_load_reduction_pct=0.666260 moved_to_echo=27618 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 538, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 4, 5626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 0, 4, 320, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 2, 0, 0, 7759, 0, 0, 0, 0, 0, 0, 0, 7221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8, 0, 8121, 0, 0, 0, 604, 124, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7062, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 20, 0, 0, 889, 0, 34, 0, 0, 0, 0, 67, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 789, 0, 0, 0, 5298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6750, 8119, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10028, 1452, 15056, 8868, 7996, 1168, 6089, 14879] avg=8192.0000 max=15056 max_rank=2 min=1168 min_rank=5 imbalance_ratio=1.837891 max_load_reduction=50480 max_load_reduction_pct=0.770264 moved_to_echo=30132 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6667, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 3, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5106, 0, 6576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 3679, 0, 0, 0, 0, 1433, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 8180, 0, 0, 0, 0, 0, 0, 188, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 135, 1175, 0, 131, 0, 0, 4723, 48, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 16, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6679, 8514, 8297, 11682, 14274, 8516, 6981, 593] avg=8192.0000 max=14274 max_rank=4 min=593 min_rank=7 imbalance_ratio=1.742432 max_load_reduction=51262 max_load_reduction_pct=0.782196 moved_to_echo=30364 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 2, 0, 417, 0, 0, 29, 1584, 0, 0, 19, 29, 0, 0, 6, 5198, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 7204, 0, 38, 0, 7458, 0, 0, 0, 0, 0, 0, 5409, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 20, 0, 0, 0, 282, 80, 0, 1, 0, 0, 0, 2600, 0, 0, 4435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 7873, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1888, 0, 0, 0, 42, 0, 129, 0, 30, 1, 811, 2072, 1, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 30, 0, 356, 0, 1, 0, 0, 0, 0, 0, 0, 7597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7546, 20376, 7445, 4988, 8721, 5394, 10289, 777] avg=8192.0000 max=20376 max_rank=1 min=777 min_rank=7 imbalance_ratio=2.487305 max_load_reduction=45160 max_load_reduction_pct=0.689087 moved_to_echo=25181 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 14976, 16000, 0, 128, 2176, 17792, 128, 128, 0, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[256, 2944, 128, 128, 1024, 128, 1792, 128, 8960, 5888, 6400, 128, 128, 768, 2304, 128, 34176, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[14720, 640, 128, 256, 128, 10112, 128, 128, 384, 14336, 1792, 128, 512, 256, 128, 0, 17152, 5248, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 512, 128, 22400, 256, 17408, 0, 11520, 128, 8320, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[512, 128, 7296, 3072, 128, 16000, 128, 128, 9856, 8704, 1664, 0, 128, 128, 18688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[0, 384, 128, 2048, 256, 128, 128, 36608, 128, 768, 128, 128, 128, 896, 2944, 128, 12160, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 2816, 128, 128, 0, 8192, 128, 128, 384, 128, 0, 128, 2688, 23296, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 512, 128, 11264, 128, 0, 3840, 128, 256, 128, 128, 44800, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 1294, 0, 1, 0, 3, 0, 0, 0, 0, 3, 7729, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3012, 62, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 6559, 0, 7188, 0, 0, 0, 0, 0, 0, 1, 33, 0, 0, 412, 0, 0, 0, 0, 0, 0, 677, 21, 0, 3, 0, 4, 0, 775, 0, 63, 3559, 0, 244, 0, 0, 186, 0, 0, 0, 0, 0, 1, 7280, 10, 0, 0, 8121, 0, 0, 7, 0, 46, 0, 8, 0, 0, 0, 0, 0, 0, 1, 502, 0, 140, 38, 11, 0, 5, 2, 0, 0, 0, 0, 247, 1, 0, 1596, 0, 1, 0, 0, 0, 7535, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 3632, 0, 11, 0, 34, 0, 0, 9, 0, 0, 50, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 31, 695, 1977, 39, 0, 0, 7202, 0, 91, 1633, 245, 0, 0, 0, 6135, 0, 8123, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 1968, 6217, 0, 14, 64, 0, 0, 259, 0, 0, 7541, 0, 0, 0, 4, 0, 21, 708, 0, 0, 0, 0, 0, 0, 0, 0, 122, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 7667, 5021, 28, 171, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 28, 0, 0, 95, 0, 0, 0, 0, 0, 0, 7619, 0, 7798, 0, 0, 220, 0, 0, 0, 0, 37, 0, 352, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4232, 591, 0, 0, 0, 0, 1, 0, 0, 0, 7381, 0, 0, 0, 10, 12, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 1436, 3, 1622, 0, 1679, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 752, 112, 0, 0, 0, 0, 34, 0, 6968, 0, 303, 0, 0, 0, 4571, 0, 0, 0, 0, 1317, 7, 0, 0, 0, 0, 0, 0, 0, 0, 722, 43, 8025, 2, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3062, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 3257, 597, 22, 0, 0, 0, 7262, 7352, 0, 0, 0, 0, 5, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 82, 0, 0, 3, 0, 0, 0, 0, 8133, 0, 55, 0, 0, 0, 90, 0, 7, 6857, 0, 0, 0, 4803, 0, 31, 0, 0, 0, 33, 5, 0, 0, 0, 0, 7665, 0, 0, 11, 7261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 20, 0, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 31] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 3, 3770, 0, 0, 23, 1, 0, 0, 0, 0, 5882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 7635, 0, 1, 0, 0, 0, 152, 0, 0, 2, 0, 206, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4160, 62, 0, 0, 1227, 0, 541, 0, 8189, 0, 58, 0, 0, 10, 0, 0, 0, 7887, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 6393, 0, 0, 44, 1430, 0, 4, 9, 0, 0, 3093, 0, 0, 1, 1, 0, 0, 0, 0, 1480, 0, 0, 1740, 168, 6, 0, 949, 1, 0, 0, 0, 1, 0, 231, 0, 0, 6747, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 11, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7884, 37, 0, 0, 2, 243, 257, 0, 0, 83, 8187, 0, 0, 0, 3, 0, 3499, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 1, 7988, 0, 0, 1, 0, 0, 0, 7567, 0, 240, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 0, 0, 0, 0, 1, 0, 939, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 4151, 0, 3, 1, 0, 0, 306, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1856, 1, 0, 2, 0, 0, 0, 0, 1141, 3984, 0, 0, 438, 0, 0, 4016, 7105, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 8023, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 1023, 0, 0, 8185, 1, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 7, 4, 0, 0, 1661, 8192, 0, 0, 0, 0, 3, 0, 0, 7972, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 34, 0, 102] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 2, 7370, 2, 0, 0, 0, 1, 386, 1, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 1, 9, 7941, 0, 0, 7932, 7405, 1, 3, 0, 0, 4, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 9, 0, 0, 7984, 0, 0, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 231, 0, 0, 50, 0, 123, 2, 170, 972, 1, 0, 5057, 0, 0, 0, 0, 0, 0, 0, 192, 0, 7, 0, 87, 4, 0, 0, 0, 0, 0, 0, 57, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1294, 0, 1, 0, 3, 0, 0, 0, 0, 3, 7729, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3012, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 6559, 0, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 33, 0, 0, 412, 0, 0, 0, 0, 0, 0, 677, 21, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 775, 0, 63, 2362, 0, 244, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7280, 10, 0, 0, 8121, 0, 0, 7, 0, 46, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 502, 0, 140, 38, 11, 0, 5, 2, 0, 0, 4153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 1, 0, 1596, 0, 1, 0, 0, 0, 7535, 0, 0, 0, 0, 2119, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9670, 3077, 15626, 1278, 3637, 15473, 4852, 11923] avg=8192.0000 max=15626 max_rank=2 min=1278 min_rank=3 imbalance_ratio=1.907471 max_load_reduction=49910 max_load_reduction_pct=0.761566 moved_to_echo=35885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 0, 0, 95, 0, 0, 0, 0, 0, 0, 7619, 0, 7798, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 37, 0, 352, 0, 0, 0, 0, 0, 0, 2, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 1, 0, 0, 0, 941, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 632, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5272, 0, 1436, 3, 1622, 0, 1679, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 752, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 6968, 0, 303, 0, 0, 0, 4571, 0, 0, 0, 4264, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 7, 0, 0, 0, 0, 0, 0, 0, 0, 722, 43, 8025, 2, 970, 2176, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16969, 4512, 1543, 952, 10013, 865, 16471, 14211] avg=8192.0000 max=16969 max_rank=0 min=865 min_rank=5 imbalance_ratio=2.071411 max_load_reduction=48567 max_load_reduction_pct=0.741074 moved_to_echo=41560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3632, 0, 11, 0, 34, 0, 0, 9, 0, 0, 50, 0, 0, 0, 0, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 31, 695, 1977, 39, 0, 6639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 1633, 245, 0, 0, 0, 6135, 0, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 1968, 6217, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 64, 0, 0, 259, 0, 0, 5001, 0, 0, 0, 4, 0, 21, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 4693, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 7667, 5021, 28, 171, 0, 0, 2395, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5091, 9474, 9139, 9303, 6071, 148, 10120, 16190] avg=8192.0000 max=16190 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.976318 max_load_reduction=49346 max_load_reduction_pct=0.752960 moved_to_echo=32529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 4151, 0, 3, 1, 0, 0, 306, 4, 0, 0, 0, 0, 0, 1, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 1, 0, 2, 0, 0, 0, 0, 1141, 3984, 0, 0, 438, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7105, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 8023, 7, 0, 0, 0, 0, 38, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 680, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 7, 4, 0, 0, 1661, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7972, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 34, 0, 102, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4649, 11124, 7130, 14401, 9957, 9712, 8301, 262] avg=8192.0000 max=14401 max_rank=3 min=262 min_rank=7 imbalance_ratio=1.757935 max_load_reduction=51135 max_load_reduction_pct=0.780258 moved_to_echo=28232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 2, 7370, 2, 0, 0, 0, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 386, 1, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 1, 7321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 7932, 7405, 1, 3, 0, 0, 4, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 5294, 0, 0, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 123, 2, 170, 972, 1, 0, 5057, 0, 0, 0, 0, 0, 255, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 7, 0, 87, 4, 0, 0, 0, 0, 0, 0, 57, 0, 130, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8965, 8080, 15411, 15962, 8172, 266, 7250, 1430] avg=8192.0000 max=15962 max_rank=3 min=266 min_rank=5 imbalance_ratio=1.948486 max_load_reduction=49574 max_load_reduction_pct=0.756439 moved_to_echo=17118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3062, 0, 0, 0, 0, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 3257, 597, 22, 0, 0, 6694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7352, 0, 0, 0, 0, 5, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 82, 0, 0, 3, 0, 0, 0, 0, 8133, 0, 259, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 90, 0, 7, 4548, 0, 0, 0, 4803, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 5, 0, 0, 0, 0, 7665, 0, 0, 11, 7122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 20, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 31, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4297, 10656, 7367, 8617, 9534, 14836, 8694, 1535] avg=8192.0000 max=14836 max_rank=5 min=1535 min_rank=7 imbalance_ratio=1.811035 max_load_reduction=50700 max_load_reduction_pct=0.773621 moved_to_echo=34599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7884, 37, 0, 0, 2, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 83, 8187, 0, 0, 0, 3, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 0, 0, 12, 308, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 5283, 0, 0, 1, 0, 0, 0, 7567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 0, 0, 0, 2023, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 939, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 1034, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4799, 8161, 8959, 511, 12853, 16378, 2761, 11114] avg=8192.0000 max=16378 max_rank=5 min=511 min_rank=3 imbalance_ratio=1.999268 max_load_reduction=49158 max_load_reduction_pct=0.750092 moved_to_echo=43106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3770, 0, 0, 23, 1, 0, 0, 0, 0, 5882, 0, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 2566, 0, 0, 0, 7037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 152, 0, 0, 2, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4160, 62, 0, 0, 1227, 0, 541, 0, 8189, 0, 298, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 10, 0, 0, 0, 5231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 6393, 0, 0, 44, 1403, 0, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 1, 1, 0, 0, 0, 0, 1480, 0, 0, 1740, 168, 6, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 1, 0, 0, 0, 1, 0, 231, 0, 0, 6747, 1, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11096, 10452, 361, 14512, 5299, 7858, 7087, 8871] avg=8192.0000 max=14512 max_rank=3 min=361 min_rank=2 imbalance_ratio=1.771484 max_load_reduction=51024 max_load_reduction_pct=0.778564 moved_to_echo=29115 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 7296, 128, 0, 8192, 22400, 128, 0, 1792, 24576, 128, 128, 896, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 3072, 512, 1024, 128, 1664, 128, 128, 256, 0, 7680, 28288, 128, 8320, 128, 1152, 7936, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 128, 18560, 896, 128, 128, 9344, 128, 640, 2688, 31232, 128, 2048, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 9216, 7168, 128, 128, 256, 128, 384, 128, 128, 128, 31744, 128, 7808, 128, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 0, 15104, 128, 17920, 15872, 128, 256, 0, 12800, 128, 3456, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 3840, 128, 8064, 256, 128, 7680, 1024, 5376, 128, 14720, 0, 4608, 1792, 256, 128, 15488, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 128, 2304, 0, 128, 33792, 0, 1792, 11136, 6528, 256, 1792, 8320, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[384, 512, 1920, 384, 128, 128, 128, 1280, 0, 128, 4352, 15232, 3968, 2048, 512, 128, 35584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4919, 19, 0, 0, 0, 0, 0, 0, 11, 21, 0, 1, 0, 0, 1, 0, 7608, 0, 144, 85, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 0, 450, 1, 18, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 1, 0, 385, 0, 0, 411, 0, 0, 1, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 31, 39, 0, 5870, 0, 0, 0, 0, 211, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 2, 0, 0, 1179, 0, 0, 0, 336, 0, 0, 38, 0, 10, 0, 0, 0, 0, 0, 7938, 3257, 4, 4100, 0, 42, 0, 0, 0, 16, 0, 200, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 7588, 2, 0, 43, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4173, 1, 0, 4336, 1396, 0, 1, 0, 7222, 0, 0, 0, 1, 0, 1851, 0, 0, 0, 2209, 0, 5998, 0, 0, 2, 8076, 0, 0, 0, 0, 1, 11, 137, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 938, 0, 0, 0, 0, 8030, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 409, 3249, 297, 275, 0, 64, 0, 2, 0, 0, 0, 7847, 0, 0, 0, 0, 2, 0, 0, 149, 9, 0, 0, 0, 0, 9, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 126, 7, 54, 0, 0, 0, 0, 8, 0, 0, 8186, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 19, 0, 0, 6, 0, 0, 25, 1, 1, 2, 0, 0, 0, 0, 5, 50, 0, 0, 0, 1, 0, 3, 0, 0, 1816, 0, 0, 1, 0, 701, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 39, 18, 0, 3, 0, 0, 0, 408, 22, 17, 0, 0, 0, 0, 0, 7666, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3228, 0, 0, 0, 0, 0, 0, 0, 7800, 8188, 11, 0, 0, 0, 0, 8169, 1, 4, 0, 0, 6491, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4182, 97] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 11, 512, 0, 0, 2, 0, 2, 28, 0, 0, 0, 1, 1, 0, 0, 0, 1886, 0, 2494, 0, 0, 0, 0, 0, 7468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 282, 0, 6548, 0, 0, 0, 0, 6839, 683, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 94, 6, 2114, 31, 194, 0, 0, 7, 0, 2944, 0, 8192, 0, 0, 1, 0, 0, 1380, 0, 2231, 19, 0, 0, 0, 1169, 0, 0, 69, 0, 0, 6730, 0, 0, 0, 0, 0, 4, 535, 354, 0, 0, 8119, 567, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 75, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 7078, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 6888, 960, 0, 0, 0, 3711, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6659, 0, 0, 0, 130, 0, 0, 0, 86, 4, 1, 0, 10, 0, 0, 0, 0, 0, 19, 0, 0, 2, 113, 15, 991, 0, 60, 53, 0, 0, 0, 14, 0, 0, 0, 1064, 2033, 11, 6194, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 1, 5354, 0, 0, 0, 2246, 291, 0, 0, 0, 0, 8070, 47, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 3176] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2, 5779, 0, 254, 11, 0, 0, 0, 42, 20, 4, 0, 0, 0, 0, 0, 0, 927, 0, 1505, 0, 3, 42, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 6364, 1, 0, 0, 0, 0, 0, 0, 0, 6495, 0, 7905, 53, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7129, 0, 0, 0, 0, 1452, 0, 0, 1, 0, 1, 130, 5, 0, 50, 0, 0, 0, 2173] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2452, 0, 0, 0, 0, 0, 0, 847, 12, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 2475, 0, 1, 0, 0, 0, 9, 0, 2, 125, 15, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 8, 5, 7982, 0, 0, 200, 21, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 533, 290, 2946, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 710, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 1, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 94, 0, 0, 0, 0, 6307, 0, 0, 0, 0, 0, 2, 8084, 0, 0, 209, 0, 0, 0, 6126, 0, 0, 5365, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 7784, 0, 0, 6866, 0, 6, 3266, 0, 0, 12, 0, 0, 0, 0, 0, 7280, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 26, 0, 0, 3, 0, 0, 0, 0, 58, 4, 0, 0, 0, 0, 257, 1423, 0, 0, 0, 0, 0, 4, 0, 0, 8130, 0, 0, 283, 2, 0, 3, 0, 0, 7514, 16, 11, 0, 0, 0, 0, 194, 0, 0, 15, 0, 0, 0, 0, 0, 30, 0, 73, 0, 0, 17, 1308, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 9, 0, 287, 0, 0, 0, 3, 0, 0, 0, 0, 114, 0, 0, 411, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 512, 0, 0, 2, 0, 2, 28, 0, 0, 0, 1, 1, 0, 0, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2494, 0, 0, 0, 0, 0, 7468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 282, 0, 3783, 0, 0, 0, 0, 6839, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1886, 237, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 94, 6, 2114, 31, 194, 0, 0, 7, 0, 2944, 0, 6591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1380, 0, 2231, 19, 0, 0, 0, 1169, 0, 0, 69, 4855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6730, 0, 0, 0, 0, 0, 4, 535, 354, 0, 0, 3176, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 75, 0, 1, 0, 0, 0, 0, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2158, 9963, 13801, 3861, 12057, 9724, 11366, 2606] avg=8192.0000 max=13801 max_rank=2 min=2158 min_rank=0 imbalance_ratio=1.684692 max_load_reduction=51735 max_load_reduction_pct=0.789413 moved_to_echo=35753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4919, 19, 0, 0, 0, 0, 0, 0, 11, 21, 0, 1, 0, 0, 1, 1594, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 85, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 1, 18, 0, 0, 0, 4688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 385, 0, 0, 411, 0, 0, 1, 0, 0, 0, 4029, 0, 0, 7608, 293, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 39, 0, 5870, 0, 0, 0, 0, 211, 0, 6560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 2, 0, 0, 1179, 0, 4747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 0, 38, 0, 10, 0, 0, 0, 0, 0, 3105, 3257, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4100, 0, 42, 0, 0, 0, 16, 0, 200, 0, 0, 0, 0, 0, 0, 0, 3133, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7319, 5614, 5157, 12814, 12711, 5934, 6750, 9237] avg=8192.0000 max=12814 max_rank=3 min=5157 min_rank=2 imbalance_ratio=1.564209 max_load_reduction=52722 max_load_reduction_pct=0.804474 moved_to_echo=34632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7588, 2, 0, 43, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4336, 1396, 0, 1, 0, 7222, 0, 0, 0, 1, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2209, 0, 5998, 0, 0, 2, 4658, 0, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 5, 4173, 295, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 409, 3249, 297, 275, 0, 64, 0, 2, 0, 0, 0, 3069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 149, 9, 0, 0, 0, 0, 9, 2, 0, 0, 0, 3123, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8719, 14217, 12879, 4709, 1109, 12759, 7437, 3707] avg=8192.0000 max=14217 max_rank=1 min=1109 min_rank=4 imbalance_ratio=1.735474 max_load_reduction=51319 max_load_reduction_pct=0.783066 moved_to_echo=25012 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 126, 7, 54, 0, 0, 0, 0, 8, 0, 0, 8186, 0, 0, 0, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 6, 0, 0, 25, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 50, 0, 0, 0, 1, 0, 3, 0, 0, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 701, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8169, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 18, 0, 3, 0, 0, 0, 408, 22, 17, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3228, 0, 4885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7800, 8188, 11, 0, 0, 0, 0, 3196, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6491, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4182, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9880, 54, 1875, 8964, 6674, 8115, 19200, 10774] avg=8192.0000 max=19200 max_rank=6 min=54 min_rank=1 imbalance_ratio=2.343750 max_load_reduction=46336 max_load_reduction_pct=0.707031 moved_to_echo=44763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 5779, 0, 254, 11, 0, 0, 0, 42, 20, 4, 0, 0, 0, 0, 0, 131, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1505, 0, 3, 42, 0, 0, 0, 0, 0, 0, 0, 0, 5506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 4588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 6364, 1, 0, 927, 287, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6495, 0, 7905, 53, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1452, 0, 0, 1, 0, 1, 130, 5, 0, 50, 0, 0, 0, 2173, 3066, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7021, 7056, 12514, 8209, 14992, 4265, 2796, 8683] avg=8192.0000 max=14992 max_rank=4 min=2796 min_rank=6 imbalance_ratio=1.830078 max_load_reduction=50544 max_load_reduction_pct=0.771240 moved_to_echo=30736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7078, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6888, 960, 0, 0, 0, 3711, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6659, 0, 0, 0, 76, 0, 0, 0, 86, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 19, 0, 0, 2, 113, 15, 991, 0, 60, 235, 4, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 14, 0, 0, 0, 1064, 2033, 11, 6194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5354, 0, 0, 0, 2246, 291, 0, 0, 0, 0, 3157, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 3176, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7482, 11560, 6826, 1536, 9369, 14340, 11097, 3326] avg=8192.0000 max=14340 max_rank=5 min=1536 min_rank=3 imbalance_ratio=1.750488 max_load_reduction=51196 max_load_reduction_pct=0.781189 moved_to_echo=38132 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7784, 0, 0, 6866, 0, 6, 3266, 0, 0, 12, 0, 0, 0, 0, 0, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3, 0, 0, 0, 0, 58, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 1423, 0, 0, 0, 0, 0, 4, 0, 0, 8130, 0, 0, 7280, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 2, 0, 3, 0, 0, 7514, 16, 11, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 30, 0, 73, 0, 0, 17, 1308, 0, 0, 0, 4843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 287, 0, 0, 0, 3, 0, 0, 0, 0, 114, 0, 0, 411, 7, 0, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18720, 8869, 91, 17186, 8023, 6286, 3711, 2650] avg=8192.0000 max=18720 max_rank=0 min=91 min_rank=2 imbalance_ratio=2.285156 max_load_reduction=46816 max_load_reduction_pct=0.714355 moved_to_echo=20670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2452, 0, 0, 0, 0, 0, 0, 847, 12, 0, 0, 0, 0, 0, 0, 138, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2475, 0, 1, 0, 0, 0, 9, 0, 2, 125, 15, 0, 5576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4398, 0, 8, 5, 7982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 21, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 533, 290, 2946, 0, 3905, 275, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 1, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 94, 0, 0, 0, 0, 2467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8084, 0, 0, 209, 0, 0, 0, 6126, 0, 0, 5365, 0, 2939, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4237, 8203, 12393, 8257, 601, 4113, 3179, 24553] avg=8192.0000 max=24553 max_rank=7 min=601 min_rank=4 imbalance_ratio=2.997192 max_load_reduction=40983 max_load_reduction_pct=0.625351 moved_to_echo=32446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 0, 8320, 128, 8192, 256, 128, 384, 384, 128, 128, 6400, 128, 0, 9984, 5504, 14848, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 15616, 3456, 128, 2304, 128, 0, 128, 384, 2560, 128, 4480, 128, 36992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 2688, 128, 22912, 0, 384, 128, 22272, 0, 128, 128, 8192, 8704, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 8064, 9344, 384, 384, 7808, 10624, 512, 640, 384, 0, 0, 24192, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[640, 128, 0, 3072, 1536, 0, 512, 384, 0, 128, 128, 128, 640, 19840, 3072, 128, 34304, 1408, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 128, 128, 128, 128, 128, 16256, 512, 15104, 2176, 128, 6272, 0, 3456, 384, 21248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 6656, 11392, 5760, 128, 128, 0, 18432, 0, 128, 256, 128, 128, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 27904, 8448, 384, 128, 8064, 0, 128, 4224, 128, 128, 128, 8192, 128, 0, 128, 5248, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 9, 0, 776, 0, 0, 0, 1, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4911, 0, 966, 0, 8171, 0, 0, 0, 0, 6477, 0, 0, 7, 8, 2783, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4451, 0, 3, 0, 0, 37, 0, 3, 0, 0, 7448, 451, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 3, 0, 0, 73, 0, 0, 0, 0, 2688, 29, 8140, 0, 0, 0, 652, 0, 7750, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 39, 0, 171, 0, 6, 133, 0, 0, 0, 0, 0, 0, 6602, 1, 88, 0, 50, 0, 0, 2031, 0, 5, 0, 84, 2, 0, 4, 12, 0, 48, 0, 1184, 0, 0, 7702, 1, 1, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 0, 781, 7224, 137, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 4519, 5684, 57, 0, 0, 0, 0, 41, 0, 6, 0, 0, 0, 0, 0, 0, 0, 96, 0, 229, 0, 0, 0, 3, 0, 0, 0, 303, 0, 0, 0, 16, 7, 0, 0, 0, 6694, 0, 0, 1, 13, 1674, 730, 0, 0, 205, 4, 8002, 0, 0, 0, 0, 347, 0, 147, 3, 36] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[2746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 1, 0, 0, 1793, 44, 0, 0, 19, 134, 0, 0, 0, 8191, 0, 0, 0, 23, 1055, 0, 0, 2, 0, 1282, 0, 30, 0, 0, 2523, 0, 0, 6832, 0, 0, 0, 3829, 1, 0, 0, 0, 0, 343, 0, 5841, 2319, 0, 3978, 0, 0, 0, 9, 8130, 4, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 36, 0, 0, 8131, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 258, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1378, 0, 0, 0, 3343, 0, 1, 0, 0, 0, 4, 0, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 8063, 0, 0, 0, 0, 0, 0, 1, 0, 0, 387, 780, 65, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 7823, 0, 127, 0, 0, 1582, 7, 0, 5439, 1, 0, 0, 2, 0, 0, 0, 0, 0, 55, 0, 0, 331, 0, 7382, 0, 0, 0, 0, 7550, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 74, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4211, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 33, 1, 6328, 0, 0, 0, 0, 0, 338, 0, 0, 42, 0, 8036, 0, 601, 0, 0, 0, 0, 0, 0, 6132] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 61, 0, 0, 243, 3799, 0, 577, 0, 0, 0, 0, 0, 0, 1, 6117, 0, 0, 0, 7512, 0, 0, 8052, 11, 0, 0, 0, 6773, 0, 0, 2, 0, 0, 8, 1, 3, 0, 7140, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2917, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 60, 0, 7524, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 23, 0, 0, 0, 0, 1941, 588, 0, 0, 247, 0, 0, 0, 0, 0, 3099, 0, 0, 0, 19, 4573] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[8161, 7670, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 46, 8037, 0, 0, 0, 394, 1, 7, 0, 692, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 6463, 0, 0, 0, 0, 88, 0, 0, 10, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2, 0, 161, 0, 0, 28, 796, 0, 0, 0, 0, 0, 0, 0, 1, 1, 10, 0, 0, 2433, 0, 0, 0, 0, 5, 85, 0, 0, 27, 0, 0, 81, 5047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 2664, 0, 0, 0, 0, 0, 5076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7241, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6526, 380, 0, 0, 0, 6050, 0, 0, 0, 55, 8056, 0, 0, 0, 0, 0, 86, 0, 8173, 0, 0, 5, 0, 0, 15, 0, 0, 0, 8135, 0, 118, 0, 0, 6992, 0, 0, 167, 0, 30, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 23, 74, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 1393, 0, 0, 946, 0, 0, 0, 63, 0, 0, 0, 0, 64, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 7903] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 806, 0, 0, 3, 0, 913, 0, 0, 0, 5846, 198, 0, 7782, 0, 5, 0, 0, 0, 0, 2123, 0, 0, 0, 0, 0, 0, 16, 7800, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 1660, 0, 0, 8012, 0, 0, 0, 0, 3666, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 129, 3, 0, 0, 0, 108, 0, 6459, 1250, 0, 0, 0, 5173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 198, 8047, 0, 0, 0, 517, 6, 13, 0, 0, 0, 0, 0, 0, 7, 3395, 0, 73, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 776, 0, 0, 0, 1, 0, 961, 0, 0, 0, 0, 0, 0, 4682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4911, 0, 966, 0, 946, 0, 0, 0, 0, 6477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 2783, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4451, 0, 3, 0, 0, 2, 0, 3, 0, 0, 7448, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 73, 0, 0, 0, 0, 2688, 29, 8140, 0, 0, 0, 652, 0, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7750, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6429, 13300, 2801, 12358, 279, 13706, 8255, 8408] avg=8192.0000 max=13706 max_rank=5 min=279 min_rank=4 imbalance_ratio=1.673096 max_load_reduction=51830 max_load_reduction_pct=0.790863 moved_to_echo=30648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 1, 0, 0, 1793, 4694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 19, 134, 0, 0, 0, 948, 0, 0, 0, 23, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1282, 0, 30, 0, 0, 2523, 0, 0, 1636, 0, 0, 0, 3829, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 5841, 2319, 0, 201, 0, 0, 0, 9, 8130, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 36, 0, 0, 3777, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 258, 5, 0, 0, 2126, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1378, 0, 0, 0, 3343, 0, 1, 0, 0, 3321, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10640, 2223, 9303, 16847, 5818, 10730, 8470, 1505] avg=8192.0000 max=16847 max_rank=3 min=1505 min_rank=7 imbalance_ratio=2.056519 max_load_reduction=48689 max_load_reduction_pct=0.742935 moved_to_echo=26523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 387, 780, 65, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 127, 0, 0, 1582, 7, 0, 1302, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 331, 0, 372, 0, 0, 0, 0, 7550, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 74, 0, 0, 2, 0, 2, 7010, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4211, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 33, 1, 6328, 0, 0, 0, 0, 2644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 42, 0, 8036, 0, 601, 0, 0, 0, 0, 0, 0, 4610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 1245, 10844, 8311, 8588, 5737, 9118, 13627] avg=8192.0000 max=13627 max_rank=7 min=1245 min_rank=1 imbalance_ratio=1.663452 max_load_reduction=51909 max_load_reduction_pct=0.792068 moved_to_echo=37070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 39, 0, 171, 0, 6, 133, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6602, 1, 88, 0, 8, 0, 0, 2031, 0, 5, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 12, 0, 48, 0, 1184, 0, 0, 1844, 1, 1, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 0, 781, 364, 137, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4519, 5684, 57, 0, 0, 0, 0, 41, 0, 6860, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 96, 0, 229, 0, 0, 0, 3, 0, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 16, 7, 0, 0, 0, 6694, 0, 0, 1, 13, 3744, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 730, 0, 0, 205, 4, 8002, 0, 0, 0, 0, 347, 0, 147, 3, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[379, 8819, 11269, 3520, 19275, 354, 10780, 11140] avg=8192.0000 max=19275 max_rank=4 min=354 min_rank=5 imbalance_ratio=2.352905 max_load_reduction=46261 max_load_reduction_pct=0.705887 moved_to_echo=41549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 61, 0, 0, 243, 3799, 0, 577, 0, 0, 0, 4305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6117, 0, 0, 0, 869, 0, 0, 8052, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6773, 0, 0, 2, 0, 0, 8, 1, 3, 0, 1709, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2917, 0, 0, 0, 0, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 60, 0, 7524, 0, 0, 0, 1, 3, 1950, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 23, 0, 0, 0, 0, 3471, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 588, 0, 0, 247, 0, 0, 0, 0, 0, 3099, 0, 0, 0, 19, 3436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8985, 15050, 8497, 4104, 4906, 10772, 3892, 9330] avg=8192.0000 max=15050 max_rank=1 min=3892 min_rank=6 imbalance_ratio=1.837158 max_load_reduction=50486 max_load_reduction_pct=0.770355 moved_to_echo=28900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8161, 7670, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 46, 8037, 0, 0, 0, 394, 1, 7, 0, 167, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 326, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2, 0, 161, 0, 0, 6137, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 796, 0, 0, 0, 0, 0, 0, 0, 1, 1, 10, 0, 0, 2433, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 85, 0, 0, 27, 0, 0, 81, 5047, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 2664, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15832, 8144, 8663, 5768, 6506, 4529, 5581, 10513] avg=8192.0000 max=15832 max_rank=0 min=4529 min_rank=5 imbalance_ratio=1.932617 max_load_reduction=49704 max_load_reduction_pct=0.758423 moved_to_echo=27129 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 806, 3353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 913, 0, 0, 0, 669, 198, 0, 7782, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2123, 0, 0, 0, 0, 0, 0, 16, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 1660, 0, 0, 399, 0, 0, 0, 0, 3666, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 7613, 2144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 129, 3, 0, 0, 0, 108, 0, 6459, 1250, 0, 0, 0, 5173, 1520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 198, 8047, 0, 0, 0, 517, 3795, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 13, 0, 0, 0, 0, 0, 0, 7, 3395, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 9570, 4000, 5811, 9760, 14644, 14055, 3494] avg=8192.0000 max=14644 max_rank=5 min=3494 min_rank=7 imbalance_ratio=1.787598 max_load_reduction=50892 max_load_reduction_pct=0.776550 moved_to_echo=41953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7241, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 380, 0, 0, 0, 6050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 8056, 0, 0, 0, 0, 0, 86, 0, 1957, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 8135, 0, 118, 0, 0, 352, 0, 0, 167, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 23, 74, 0, 0, 0, 0, 19, 3, 6640, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 1393, 0, 1694, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 0, 0, 63, 0, 0, 0, 0, 64, 0, 0, 0, 2, 0, 3973, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 5942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11003, 7185, 10159, 8817, 10404, 5064, 5385, 7519] avg=8192.0000 max=11003 max_rank=0 min=5064 min_rank=5 imbalance_ratio=1.343140 max_load_reduction=54533 max_load_reduction_pct=0.832108 moved_to_echo=28372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[8192, 896, 128, 256, 128, 0, 128, 4224, 3072, 128, 22400, 1280, 384, 128, 4608, 5248, 9472, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 1408, 128, 0, 4736, 5760, 256, 128, 128, 3200, 256, 128, 128, 38144, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[18176, 7680, 0, 896, 0, 0, 128, 128, 128, 1280, 13568, 128, 640, 256, 0, 2688, 20864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 27264, 896, 1152, 0, 4224, 640, 0, 17920, 128, 13696, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 13056, 128, 18816, 2688, 896, 2048, 256, 128, 256, 128, 27520, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[6912, 128, 30208, 128, 256, 128, 512, 5376, 128, 128, 10496, 128, 128, 8192, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[7808, 1152, 384, 128, 128, 128, 128, 1536, 0, 1280, 384, 29568, 0, 128, 128, 640, 21376, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3712, 1792, 0, 0, 896, 896, 24192, 4096, 640, 6144, 3200, 2048, 0, 256, 128, 18560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 967, 3, 0, 0, 4646, 0, 0, 0, 0, 0, 1, 0, 0, 22, 0, 427, 0, 0, 0, 0, 56, 67, 0, 0, 0, 0, 0, 0, 0, 1030, 1, 5035, 0, 0, 0, 0, 1, 7235, 0, 8182, 0, 0, 0, 0, 238, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7718, 3015, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6056, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 56, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4002, 1, 0, 0, 0, 0, 7, 282, 0, 4, 0, 1, 0, 0, 14, 4, 0, 8046, 0, 0, 0, 0, 0, 1, 392, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 5364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 283, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8118, 0, 0, 0, 709, 0, 1, 6, 8178, 0, 69, 0, 362, 0, 0, 0, 8, 101, 0, 1, 0, 1, 0, 0, 0, 4386, 18, 0, 0, 0, 356, 0, 2, 0, 0, 1929, 0, 0, 1, 0, 0, 0, 4907, 0, 0, 736, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 736, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 0, 0, 0, 0, 0, 8169, 0, 1762, 0, 0, 0, 0, 7283, 7, 3302, 0, 0, 979, 0, 0, 0, 0, 0, 0, 1245] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 128, 0, 166, 0, 0, 0, 7789, 0, 0, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 496, 7868, 0, 0, 0, 0, 2, 0, 0, 7394, 0, 307, 0, 0, 0, 8114, 0, 0, 9, 0, 28, 8173, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7829, 6832, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 172, 0, 5, 2, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 3, 4, 0, 0, 0, 0, 2, 15, 0, 0, 1, 3, 0, 0, 454, 0, 22, 1, 0, 0, 275, 33, 0, 0, 0, 0, 0, 7265, 0, 0, 364, 1047, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8039, 337, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 119, 8190, 0, 0, 0, 0, 2, 0, 0, 1, 368, 8094, 0, 1, 0, 1, 0, 0, 3, 2, 7731, 6844, 0, 1587, 0, 450, 0, 0, 0, 2704, 0, 0, 0, 30, 261, 0, 0, 0, 0, 0, 2392, 0, 0, 40, 0, 0, 0, 0, 13, 9, 26, 5409, 0, 0, 807, 286, 0, 0, 0, 0, 0, 0, 0, 1443, 0, 4405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8, 88, 262, 0, 0, 0, 0, 0, 0, 0, 0, 5319, 0, 0, 0, 0, 102, 15, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 0, 0, 2, 0, 132, 33, 0, 0, 0, 0, 0, 31, 0, 0, 1, 0, 0, 61, 0, 5489, 0, 8081, 0, 0, 0, 0, 0, 1102, 0, 8081, 6, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 15, 0, 1099, 0, 0, 0, 0, 5, 0, 43, 0, 0, 0, 0, 6, 0, 0, 0, 6600, 0, 0, 0, 2342, 4261, 3, 0, 0, 0, 0, 0, 0, 3, 318, 8, 0, 0, 2, 0, 0, 36, 0, 5178, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 8151, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 0, 1051, 237, 0, 0, 2006, 0, 519, 0, 0, 0, 69, 0, 0, 1238, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 2895, 0, 8160, 0, 7343, 0, 0, 0, 0, 28, 8192, 0, 0, 0, 2135, 5799, 9, 0, 0, 0, 7975, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 237, 1, 0, 3395, 239, 0, 0, 6533, 0, 0, 1699, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 167, 1174, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2392, 0, 7318, 8108, 0, 0, 1, 0, 1, 2, 0, 0, 6382, 0, 0, 7803, 7, 7594, 0, 0, 0, 231, 144, 7549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 398, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6363, 1, 1, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1425, 0, 0, 7649, 0, 0, 0, 0, 0, 0, 30, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6470, 0, 0, 0, 0, 0, 0, 0, 1406, 14, 0, 0, 0, 0, 0, 0, 1, 690, 0, 0, 0, 457, 0, 0, 78, 0, 0, 54, 5793, 1, 501, 348, 196, 0, 0, 0, 0, 0, 0, 0, 78, 38, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1588, 0, 0, 211, 5, 0, 0, 11, 8109, 0, 0, 6267, 0, 0, 7120, 0, 0, 25, 0, 0, 0, 0, 7033, 0, 0, 0, 2, 2100, 0, 0, 0, 0, 106, 0, 0, 0, 0, 2, 456, 0, 295, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 1, 54, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 1, 6, 0, 0, 34, 0, 362, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 1, 0, 1, 0, 0, 0, 4386, 18, 0, 0, 0, 292, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 0, 1, 0, 0, 0, 4907, 0, 0, 736, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 736, 0, 0, 34, 0, 0, 0, 0, 7544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 0, 0, 0, 0, 0, 8169, 0, 1762, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7283, 7, 3302, 0, 0, 979, 0, 0, 0, 0, 0, 0, 1245, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6186, 8464, 1120, 4801, 7593, 8327, 16166, 12879] avg=8192.0000 max=16166 max_rank=6 min=1120 min_rank=2 imbalance_ratio=1.973389 max_load_reduction=49370 max_load_reduction_pct=0.753326 moved_to_echo=44965 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 967, 3, 0, 0, 4646, 0, 0, 0, 0, 0, 1, 0, 0, 22, 4376, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 56, 67, 0, 0, 0, 0, 0, 0, 0, 1030, 1855, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 3872, 0, 0, 0, 0, 238, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6056, 0, 0, 7990, 0, 0, 0, 2362, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 6673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 1, 0, 0, 0, 0, 7, 282, 0, 4, 0, 1, 0, 0, 1160, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4, 0, 8046, 0, 0, 0, 0, 0, 1, 392, 4, 0, 0, 0, 0, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10577, 4094, 4117, 6321, 16440, 6750, 7405, 9832] avg=8192.0000 max=16440 max_rank=4 min=4094 min_rank=1 imbalance_ratio=2.006836 max_load_reduction=49096 max_load_reduction_pct=0.749146 moved_to_echo=40427 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 128, 0, 166, 0, 0, 0, 7789, 0, 0, 0, 0, 0, 586, 0, 0, 6426, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 496, 7868, 0, 0, 0, 0, 2, 0, 4202, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 0, 0, 0, 9, 0, 28, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 172, 0, 5, 2, 0, 0, 1, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 3, 7484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2, 15, 0, 0, 1, 3, 0, 0, 454, 0, 22, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 275, 33, 0, 0, 0, 0, 0, 7265, 0, 0, 364, 1047, 0, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15725, 13582, 8517, 6414, 220, 7571, 3131, 10376] avg=8192.0000 max=15725 max_rank=0 min=220 min_rank=4 imbalance_ratio=1.919556 max_load_reduction=49811 max_load_reduction_pct=0.760056 moved_to_echo=21298 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 0, 0, 2, 0, 132, 33, 0, 0, 0, 0, 0, 31, 7022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 61, 0, 5489, 0, 8081, 0, 0, 0, 0, 0, 1102, 4059, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 15, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 43, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2342, 4261, 3, 0, 0, 0, 0, 0, 0, 3, 318, 8, 0, 0, 2357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 36, 0, 5178, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 2541, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 237, 0, 0, 2006, 0, 519, 0, 0, 0, 69, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7312, 19852, 4984, 54, 9292, 5231, 13691, 5120] avg=8192.0000 max=19852 max_rank=1 min=54 min_rank=3 imbalance_ratio=2.423340 max_load_reduction=45684 max_load_reduction_pct=0.697083 moved_to_echo=33334 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8039, 337, 0, 0, 0, 145, 0, 0, 0, 0, 7033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 119, 8190, 0, 0, 0, 0, 2, 0, 0, 1, 1471, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 1, 0, 1, 0, 0, 3, 2, 7731, 6844, 0, 1587, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2704, 0, 0, 0, 30, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 13, 9, 26, 5409, 0, 0, 807, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1443, 0, 4405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8, 88, 262, 0, 0, 0, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5319, 0, 0, 0, 0, 102, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15554, 10845, 16987, 2995, 6590, 5848, 1281, 5436] avg=8192.0000 max=16987 max_rank=2 min=1281 min_rank=6 imbalance_ratio=2.073608 max_load_reduction=48549 max_load_reduction_pct=0.740799 moved_to_echo=19155 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 1012, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 0, 8160, 0, 7343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 8192, 0, 0, 0, 2135, 5799, 9, 0, 0, 0, 6529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 0, 11, 0, 0, 0, 835, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 237, 1, 0, 3395, 239, 0, 0, 6533, 0, 0, 1699, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1655, 409, 16874, 22692, 928, 8570, 699, 13709] avg=8192.0000 max=22692 max_rank=3 min=409 min_rank=1 imbalance_ratio=2.770020 max_load_reduction=42844 max_load_reduction_pct=0.653748 moved_to_echo=23906 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 167, 1174, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 552, 0, 0, 0, 4644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 8108, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6382, 0, 0, 7803, 7, 7594, 0, 0, 0, 231, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 398, 0, 0, 0, 0, 0, 2118, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6363, 1, 1, 0, 0, 0, 0, 61, 0, 0, 0, 0, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1425, 0, 0, 7649, 0, 0, 0, 2905, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1605, 5203, 11565, 22133, 2581, 8655, 13732, 62] avg=8192.0000 max=22133 max_rank=3 min=62 min_rank=7 imbalance_ratio=2.701782 max_load_reduction=43403 max_load_reduction_pct=0.662277 moved_to_echo=25030 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6470, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 14, 0, 0, 0, 0, 0, 0, 1, 690, 0, 0, 0, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 78, 0, 0, 54, 0, 1, 238, 348, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 38, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 5, 0, 0, 11, 8109, 0, 0, 6267, 0, 0, 7120, 0, 0, 25, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7033, 0, 0, 0, 2, 2100, 0, 0, 0, 0, 106, 0, 5343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 456, 0, 295, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 612, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6922, 3087, 1372, 126, 21892, 14584, 9431, 8122] avg=8192.0000 max=21892 max_rank=4 min=126 min_rank=3 imbalance_ratio=2.672363 max_load_reduction=43644 max_load_reduction_pct=0.665955 moved_to_echo=54029 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 17408, 128, 128, 7808, 2176, 18048, 128, 128, 128, 256, 19712, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 256, 4352, 4352, 128, 128, 8192, 128, 128, 23040, 512, 128, 17024, 384, 0, 128, 7936, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[896, 0, 128, 1152, 0, 128, 128, 0, 128, 12928, 16256, 15616, 128, 17152, 1408, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 12160, 128, 512, 128, 6656, 0, 128, 1792, 128, 8064, 16128, 1536, 1792, 128, 10880, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[384, 512, 128, 1408, 128, 512, 256, 13824, 512, 16000, 0, 128, 1280, 0, 128, 10368, 17280, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 1152, 5632, 0, 128, 12800, 8192, 256, 128, 256, 7808, 1024, 640, 256, 128, 24960, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 1152, 256, 15744, 384, 10752, 0, 3968, 1280, 128, 15872, 6656, 128, 512, 4096, 1280, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 13440, 7808, 128, 4480, 768, 2176, 0, 128, 128, 0, 128, 128, 36864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[22, 28, 0, 0, 1, 0, 0, 39, 0, 0, 0, 0, 0, 439, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 9, 637, 0, 7832, 0, 0, 38, 0, 707, 34, 0, 8188, 0, 76, 8108, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 7429, 0, 0, 0, 0, 819, 0, 0, 0, 0, 0, 0, 7417, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 156, 0, 0, 0, 0, 0, 0, 0, 0, 29, 323, 0, 0, 0, 0, 0, 0, 4098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 7053, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[6148, 0, 0, 0, 0, 0, 0, 64, 0, 1641, 0, 0, 0, 1, 0, 0, 57, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 262, 0, 0, 0, 1220, 0, 0, 4786, 0, 134, 0, 7825, 0, 0, 7984, 4, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6807, 0, 0, 0, 2, 0, 7885, 95, 0, 0, 1, 0, 0, 1260, 2944, 757, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1212, 0, 1, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 211, 0, 0, 0, 0, 195, 0, 8192, 0, 78, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[6848, 0, 0, 5, 0, 0, 215, 0, 239, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 38, 0, 0, 0, 6865, 7787, 0, 0, 0, 0, 23, 0, 43, 0, 5, 6118, 0, 0, 0, 0, 0, 904, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5945, 0, 0, 7, 0, 0, 8, 12, 1, 0, 2209, 34, 0, 0, 0, 0, 0, 0, 0, 7698, 0, 373, 0, 0, 0, 2838, 0, 0, 0, 0, 5134, 0, 0, 0, 0, 1, 161, 0, 0, 0, 0, 0, 0, 937, 470, 0, 0, 0, 0, 0, 0, 1687, 0, 0, 113, 0, 0, 38, 163, 0, 0, 558, 0, 3815, 0, 11, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[8168, 8142, 0, 0, 4, 0, 0, 0, 2, 0, 0, 2, 0, 0, 323, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 40, 0, 108, 0, 7252, 0, 0, 0, 0, 0, 3, 0, 228, 0, 0, 8067, 0, 1632, 0, 0, 9, 0, 0, 0, 0, 8, 1, 0, 0, 4, 0, 0, 0, 0, 4, 7290, 0, 0, 0, 153, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4948, 4, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 396, 0, 0, 3233, 0, 0, 1253, 0, 0, 0, 0, 1, 0, 2, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7854, 0, 6158, 0, 0, 0, 93] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 175, 0, 0, 0, 0, 47, 0, 0, 0, 0, 2, 0, 0, 0, 5613, 234, 0, 0, 7839, 0, 0, 0, 0, 1, 8040, 0, 3, 0, 2, 0, 0, 7, 12, 0, 23, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 1084, 2, 4813, 117, 0, 0, 0, 12, 0, 0, 8116, 0, 0, 0, 0, 0, 2227, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 0, 8009, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7016, 0, 0, 0, 56] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 140, 0, 0, 0, 0, 0, 0, 1, 0, 1058, 0, 0, 0, 0, 1833, 0, 43, 0, 6072, 7549, 0, 0, 0, 0, 0, 6454, 0, 0, 0, 933, 0, 2308, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 0, 0, 1, 0, 0, 0, 159, 0, 0, 7415, 0, 0, 0, 0, 0, 3744, 5, 0, 0, 5144, 0, 0, 0, 0, 6826, 0, 2982, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 1, 0, 935, 2, 0, 0, 222, 23, 0, 0, 0, 0, 0, 0, 0, 1, 0, 270, 0, 0, 0, 8083, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[22, 0, 183, 0, 1, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 0, 0, 0, 0, 7257, 5667, 0, 151, 45, 4605, 0, 0, 7648, 537, 22, 5811, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 25, 7948, 13, 5, 343, 0, 0, 0, 0, 0, 29, 0, 85, 0, 21, 0, 0, 0, 0, 0, 64, 1171, 0, 0, 0, 12, 0, 7816, 0, 0, 0, 0, 0, 16, 0, 0, 33, 0, 0, 0, 0, 0, 5, 520, 0, 0, 0, 0, 14, 3976, 56, 655, 0, 0, 0, 0, 12, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[7384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 108, 0, 0, 0, 3528, 0, 1, 0, 1871, 552, 0, 0, 0, 0, 4856, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 1, 0, 581, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 14, 3074, 0, 122, 0, 0, 107, 0, 0, 0, 0, 0, 7905, 0, 0, 0, 3972, 6738, 0, 2, 0, 186, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 0, 1, 5, 6, 645, 0, 1, 484, 0, 0, 0, 3902, 0, 4788, 0, 4118] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[22, 28, 0, 0, 1, 0, 0, 39, 0, 0, 0, 0, 0, 439, 3, 8, 2310, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 9, 637, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 707, 34, 0, 8188, 0, 76, 528, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3899, 0, 0, 0, 5270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7429, 0, 0, 0, 0, 819, 0, 0, 0, 0, 0, 0, 4312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 156, 0, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 323, 0, 0, 0, 0, 0, 0, 4098, 3780, 1229, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 5096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4726, 4712, 9572, 9189, 12560, 1817, 9770, 13190] avg=8192.0000 max=13190 max_rank=7 min=1817 min_rank=5 imbalance_ratio=1.610107 max_load_reduction=52346 max_load_reduction_pct=0.798737 moved_to_echo=37337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6848, 0, 0, 5, 0, 0, 215, 0, 239, 0, 368, 0, 0, 0, 0, 0, 1743, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 38, 0, 0, 0, 3552, 7787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 43, 0, 5, 399, 0, 0, 0, 0, 0, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5945, 0, 0, 7, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 0, 2209, 34, 0, 0, 0, 0, 0, 0, 0, 4476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 2838, 0, 0, 0, 0, 5134, 0, 0, 0, 0, 1, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 937, 470, 0, 0, 0, 0, 0, 0, 1687, 3313, 1275, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 38, 163, 0, 0, 558, 0, 2757, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11365, 15226, 1374, 9944, 6740, 9236, 8011, 3640] avg=8192.0000 max=15226 max_rank=1 min=1374 min_rank=2 imbalance_ratio=1.858643 max_load_reduction=50310 max_load_reduction_pct=0.767670 moved_to_echo=27627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[8168, 8142, 0, 0, 4, 0, 0, 0, 2, 0, 0, 2, 0, 0, 323, 0, 2299, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 40, 0, 108, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 228, 0, 0, 525, 0, 1632, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 4, 0, 0, 0, 0, 4, 7290, 0, 0, 0, 5243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 396, 0, 0, 3233, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 0, 0, 0, 1, 0, 2, 38, 0, 0, 0, 0, 0, 3504, 820, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7854, 0, 4450, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20191, 3946, 2397, 12550, 3036, 5129, 5889, 12398] avg=8192.0000 max=20191 max_rank=0 min=2397 min_rank=2 imbalance_ratio=2.464722 max_load_reduction=45345 max_load_reduction_pct=0.691910 moved_to_echo=26452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 140, 1839, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1058, 0, 0, 0, 0, 949, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6072, 7549, 0, 0, 0, 0, 0, 420, 0, 0, 0, 933, 0, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 4195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 159, 0, 0, 7415, 0, 0, 0, 0, 0, 2177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5144, 0, 0, 0, 0, 6826, 0, 2982, 0, 0, 0, 0, 0, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 1, 0, 935, 2, 0, 0, 222, 23, 0, 0, 884, 620, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 270, 0, 0, 0, 5841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2928, 2051, 17282, 7259, 9752, 16843, 3309, 6112] avg=8192.0000 max=17282 max_rank=2 min=2051 min_rank=1 imbalance_ratio=2.109619 max_load_reduction=48254 max_load_reduction_pct=0.736298 moved_to_echo=36016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[7384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 108, 0, 0, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1871, 552, 0, 0, 0, 0, 317, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 581, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 14, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 107, 0, 0, 0, 0, 0, 7905, 0, 0, 0, 3972, 6738, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 186, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 0, 509, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 6, 645, 0, 1, 484, 0, 0, 0, 2820, 0, 4788, 0, 4118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9544, 3642, 5229, 3807, 1804, 19754, 8888, 12868] avg=8192.0000 max=19754 max_rank=5 min=1804 min_rank=4 imbalance_ratio=2.411377 max_load_reduction=45782 max_load_reduction_pct=0.698578 moved_to_echo=43314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 175, 0, 0, 0, 0, 47, 0, 0, 0, 0, 6, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5613, 234, 0, 0, 7839, 0, 0, 0, 0, 1, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 7, 12, 0, 3, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 1084, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4813, 117, 0, 0, 0, 12, 0, 0, 8116, 0, 0, 0, 0, 0, 1295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 369, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5070, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[802, 21729, 1777, 1581, 14355, 3288, 16876, 5128] avg=8192.0000 max=21729 max_rank=1 min=802 min_rank=0 imbalance_ratio=2.652466 max_load_reduction=43807 max_load_reduction_pct=0.668442 moved_to_echo=39647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6148, 0, 0, 0, 0, 0, 0, 64, 0, 1641, 0, 0, 0, 1, 0, 0, 2279, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 262, 0, 0, 0, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4786, 0, 134, 0, 7825, 0, 0, 512, 4, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 0, 5193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6807, 0, 0, 0, 2, 0, 4577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 1, 0, 0, 1260, 2944, 757, 0, 50, 0, 0, 0, 0, 0, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1212, 0, 1, 0, 0, 0, 1, 0, 22, 0, 1310, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0, 195, 0, 5913, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12131, 2731, 13878, 9066, 11392, 7021, 2920, 6397] avg=8192.0000 max=13878 max_rank=2 min=2731 min_rank=1 imbalance_ratio=1.694092 max_load_reduction=51658 max_load_reduction_pct=0.788239 moved_to_echo=27730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 183, 0, 1, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1656, 1977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 0, 0, 0, 0, 3755, 5667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 45, 4605, 0, 0, 7648, 537, 22, 378, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 7948, 13, 5, 343, 0, 0, 0, 0, 0, 29, 0, 3777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 21, 0, 0, 0, 0, 0, 64, 1171, 0, 0, 0, 12, 0, 4544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 33, 0, 0, 0, 0, 0, 5, 520, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3976, 56, 655, 0, 0, 0, 0, 12, 0, 9, 0, 3502, 1295, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3849, 11499, 14027, 12140, 5897, 2448, 9873, 5803] avg=8192.0000 max=14027 max_rank=2 min=2448 min_rank=5 imbalance_ratio=1.712280 max_load_reduction=51509 max_load_reduction_pct=0.785965 moved_to_echo=24021 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 512, 0, 5248, 256, 2944, 1408, 3072, 7680, 0, 16128, 0, 512, 128, 4224, 12032, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 6912, 256, 128, 0, 14976, 128, 384, 768, 0, 16128, 26368, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[512, 128, 9344, 256, 128, 4096, 128, 11136, 1792, 128, 128, 0, 256, 128, 128, 13824, 15104, 7552, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[28672, 8192, 256, 128, 128, 128, 512, 128, 256, 1664, 384, 128, 128, 512, 384, 256, 13568, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 8576, 128, 128, 384, 512, 0, 128, 24064, 128, 128, 1152, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 11008, 14848, 768, 0, 24064, 640, 128, 3200, 128, 3456, 0, 3456, 640, 3328, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 4864, 256, 7552, 128, 0, 2432, 128, 896, 23552, 128, 0, 0, 128, 128, 26112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 896, 128, 256, 768, 0, 16768, 0, 37760, 0, 4992, 0, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True + [2026-08-26 16:28:28] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 637.6 | throughput per GPU (TFLOP/s/GPU): 283.6 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117604E+01 | loss scale: 1.0 | grad norm: 44.307 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4188, 72, 3, 125, 118, 478, 83, 0, 9, 332, 1230, 120, 1742, 281, 91, 426, 59, 1346, 33, 27, 829, 3, 14, 1770, 1429, 36, 3, 4115, 2327, 3519, 51, 490, 543, 1565, 173, 53, 27, 362, 10, 131, 1101, 2, 794, 2248, 94, 31, 17, 3463, 14, 90, 6, 11, 1, 311, 47, 7, 180, 17, 16, 455, 368, 13, 96, 282, 2973, 28, 1, 0, 812, 0, 9, 1098, 1504, 32, 1923, 222, 620, 10, 169, 219, 32, 1, 4, 21, 13, 0, 159, 1397, 3, 56, 188, 7, 51, 144, 379, 8, 501, 233, 13, 321, 101, 4, 50, 25, 410, 189, 412, 3557, 19, 149, 686, 97, 143, 602, 0, 93, 0, 122, 692, 297, 49, 19, 4628, 296, 59, 43, 31, 1735] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3737, 17, 1, 91, 110, 316, 120, 1, 17, 393, 1084, 175, 1545, 91, 335, 396, 66, 2327, 5, 22, 789, 9, 27, 805, 542, 39, 13, 3923, 2775, 3674, 22, 471, 1094, 2400, 144, 1, 150, 1040, 4, 76, 423, 23, 215, 2403, 2, 18, 16, 3429, 59, 82, 3, 4, 8, 372, 3, 8, 163, 16, 25, 183, 249, 12, 223, 58, 2938, 10, 1, 0, 1133, 2, 21, 703, 1029, 14, 1780, 840, 464, 7, 372, 571, 3, 3, 0, 103, 49, 1, 69, 1643, 6, 66, 159, 4, 14, 69, 308, 20, 240, 389, 53, 890, 28, 10, 47, 20, 197, 111, 369, 4050, 11, 177, 436, 44, 62, 974, 0, 52, 0, 125, 468, 407, 118, 4, 4656, 302, 27, 51, 2, 1970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3517, 70, 4, 230, 224, 621, 112, 2, 27, 504, 1279, 203, 1278, 109, 319, 215, 109, 1671, 1, 27, 872, 12, 54, 1718, 1402, 82, 10, 4594, 2361, 3668, 110, 623, 509, 1034, 183, 4, 84, 926, 11, 63, 1077, 9, 639, 2848, 4, 74, 20, 3590, 46, 147, 2, 21, 9, 142, 15, 6, 190, 12, 41, 345, 270, 130, 240, 229, 2794, 5, 3, 0, 1215, 0, 30, 475, 941, 23, 1281, 607, 374, 26, 504, 225, 28, 5, 9, 12, 43, 1, 75, 1226, 4, 115, 251, 3, 34, 160, 267, 13, 339, 191, 15, 946, 40, 4, 70, 104, 159, 325, 331, 3359, 2, 299, 489, 103, 381, 953, 1, 128, 0, 244, 542, 143, 32, 28, 4681, 413, 81, 44, 3, 1389] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4192, 30, 0, 257, 86, 689, 66, 0, 39, 418, 596, 186, 1491, 64, 209, 300, 47, 2021, 7, 85, 1105, 7, 31, 1434, 1145, 55, 5, 4768, 2374, 3597, 86, 621, 826, 1368, 245, 44, 139, 516, 7, 71, 1083, 12, 889, 3674, 0, 18, 39, 3528, 17, 49, 1, 6, 16, 259, 5, 11, 169, 45, 36, 205, 260, 111, 332, 190, 2539, 7, 0, 1, 1217, 10, 54, 398, 950, 15, 2704, 153, 522, 14, 359, 218, 25, 2, 0, 59, 33, 0, 111, 490, 8, 73, 388, 21, 138, 71, 97, 19, 161, 115, 22, 458, 34, 14, 140, 31, 124, 191, 417, 3261, 10, 162, 350, 76, 192, 917, 2, 85, 1, 198, 415, 91, 104, 60, 4850, 386, 83, 98, 15, 1595] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[4991, 34, 5, 431, 60, 705, 87, 0, 36, 223, 1112, 68, 1555, 60, 218, 435, 113, 2029, 7, 23, 1002, 13, 31, 829, 1067, 45, 0, 4035, 2102, 2762, 32, 412, 1019, 1295, 147, 20, 124, 400, 6, 14, 542, 7, 698, 3849, 1, 20, 31, 3546, 12, 57, 11, 19, 0, 246, 23, 4, 98, 10, 32, 544, 220, 43, 127, 129, 2813, 11, 2, 2, 1495, 0, 41, 612, 1275, 2, 2354, 318, 512, 3, 320, 233, 8, 20, 0, 14, 16, 10, 84, 620, 3, 96, 259, 18, 27, 32, 326, 14, 305, 390, 11, 819, 38, 3, 45, 29, 158, 84, 666, 4034, 20, 109, 309, 34, 191, 916, 1, 118, 0, 199, 347, 236, 107, 25, 4690, 372, 58, 84, 23, 1759] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3779, 4, 1, 197, 75, 459, 129, 0, 30, 377, 2170, 190, 1496, 101, 319, 382, 43, 2079, 11, 37, 756, 3, 51, 1377, 933, 50, 2, 4471, 2103, 2732, 34, 620, 964, 1355, 227, 9, 44, 887, 7, 95, 709, 11, 961, 3245, 11, 81, 75, 3331, 15, 78, 0, 20, 1, 286, 25, 11, 50, 9, 22, 200, 382, 7, 258, 398, 2825, 3, 2, 0, 894, 4, 32, 826, 1064, 25, 2345, 272, 439, 3, 398, 147, 12, 11, 0, 30, 47, 0, 104, 1376, 9, 66, 166, 6, 99, 190, 145, 34, 276, 197, 12, 734, 55, 12, 113, 39, 142, 167, 360, 3447, 10, 167, 379, 109, 134, 891, 4, 27, 1, 221, 511, 232, 71, 8, 4539, 491, 46, 57, 11, 1714] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3807, 66, 1, 130, 142, 360, 60, 0, 14, 405, 1525, 116, 1633, 147, 152, 460, 169, 1658, 28, 12, 992, 23, 18, 608, 631, 36, 1, 4001, 1766, 3412, 114, 264, 932, 1798, 162, 16, 92, 1105, 24, 128, 1210, 2, 539, 2688, 0, 44, 29, 4065, 8, 43, 4, 14, 1, 192, 15, 8, 235, 17, 11, 208, 293, 21, 153, 147, 2476, 2, 4, 1, 1573, 2, 23, 761, 1437, 89, 2118, 418, 416, 70, 591, 76, 23, 17, 0, 22, 33, 1, 148, 1608, 4, 80, 210, 39, 108, 67, 163, 2, 162, 268, 15, 408, 60, 9, 324, 86, 164, 450, 236, 4023, 2, 276, 388, 107, 214, 763, 0, 38, 0, 366, 381, 66, 68, 26, 4768, 328, 87, 48, 4, 1864] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[4033, 20, 1, 94, 174, 587, 203, 0, 1, 531, 1534, 252, 1297, 143, 208, 742, 117, 1910, 4, 7, 1277, 0, 40, 665, 1288, 26, 1, 4162, 2222, 3255, 176, 368, 749, 1054, 151, 2, 41, 1191, 24, 39, 533, 4, 389, 2277, 0, 26, 36, 4167, 7, 116, 4, 10, 4, 90, 5, 3, 200, 10, 13, 192, 262, 7, 258, 318, 2530, 5, 0, 0, 984, 2, 32, 988, 956, 8, 2411, 396, 593, 0, 506, 279, 8, 0, 0, 15, 25, 0, 67, 1515, 10, 49, 74, 1, 17, 214, 206, 7, 123, 366, 35, 421, 149, 2, 68, 5, 221, 147, 398, 4183, 3, 139, 371, 66, 91, 821, 0, 35, 1, 164, 908, 141, 47, 26, 5016, 399, 52, 58, 9, 1653] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3138, 72, 3, 125, 118, 478, 83, 0, 9, 332, 1230, 120, 1742, 281, 91, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 1346, 33, 27, 829, 3, 14, 1770, 1429, 36, 3, 0, 2327, 120, 51, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 1565, 173, 53, 27, 362, 10, 131, 1101, 2, 794, 2248, 94, 31, 17, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 90, 6, 11, 1, 311, 47, 7, 180, 17, 16, 455, 368, 13, 96, 282, 4115, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1527, 28, 1, 0, 812, 0, 9, 1098, 1504, 32, 1923, 222, 620, 10, 169, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1, 4, 21, 13, 0, 159, 1397, 3, 56, 188, 7, 51, 144, 379, 8, 3399, 1446, 1067, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 233, 13, 321, 101, 4, 50, 25, 410, 189, 412, 3557, 19, 149, 686, 97, 877, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 602, 0, 93, 0, 122, 692, 297, 49, 19, 3751, 296, 59, 43, 31, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8537, 7369, 8207, 8174, 8746, 8323, 7932] avg=8192.0000 max=8746 max_rank=5 min=7369 min_rank=2 imbalance_ratio=1.067627 max_load_reduction=56790 max_load_reduction_pct=0.866547 moved_to_echo=33175 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2635, 70, 4, 230, 224, 621, 112, 2, 27, 504, 1279, 203, 1278, 109, 319, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1671, 1, 27, 872, 12, 54, 1718, 1402, 82, 10, 0, 2361, 125, 110, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 1034, 183, 4, 84, 926, 11, 63, 1077, 9, 639, 2848, 4, 74, 20, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 147, 2, 21, 9, 142, 15, 6, 190, 12, 41, 345, 270, 130, 240, 229, 4594, 2258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1435, 5, 3, 0, 1215, 0, 30, 475, 941, 23, 1281, 607, 374, 26, 504, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 5, 9, 12, 43, 1, 75, 1226, 4, 115, 251, 3, 34, 160, 267, 13, 3543, 1359, 1106, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 191, 15, 946, 40, 4, 70, 104, 159, 325, 331, 3359, 2, 299, 489, 103, 887, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 953, 1, 128, 0, 244, 542, 143, 32, 28, 3794, 413, 81, 44, 3, 1389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7832, 9177, 7711, 8697, 7144, 8565, 8234, 8176] avg=8192.0000 max=9177 max_rank=1 min=7144 min_rank=4 imbalance_ratio=1.120239 max_load_reduction=56359 max_load_reduction_pct=0.859970 moved_to_echo=32119 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2831, 4, 1, 197, 75, 459, 129, 0, 30, 377, 2170, 190, 1496, 101, 319, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 2079, 11, 37, 756, 3, 51, 1377, 933, 50, 2, 0, 2103, 94, 34, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1355, 227, 9, 44, 887, 7, 95, 709, 11, 961, 3245, 11, 81, 75, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 78, 0, 20, 1, 286, 25, 11, 50, 9, 22, 200, 382, 7, 258, 398, 4471, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 3, 2, 0, 894, 4, 32, 826, 1064, 25, 2345, 272, 439, 3, 398, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 11, 0, 30, 47, 0, 104, 1376, 9, 66, 166, 6, 99, 190, 145, 34, 2638, 1374, 1026, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 197, 12, 734, 55, 12, 113, 39, 142, 167, 360, 3447, 10, 167, 379, 109, 860, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 891, 4, 27, 1, 221, 511, 232, 71, 8, 3679, 491, 46, 57, 11, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8761, 8193, 8891, 8328, 7905, 7668, 7692, 8098] avg=8192.0000 max=8891 max_rank=2 min=7668 min_rank=5 imbalance_ratio=1.085327 max_load_reduction=56645 max_load_reduction_pct=0.864334 moved_to_echo=31363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3141, 30, 0, 257, 86, 689, 66, 0, 39, 418, 596, 186, 1491, 64, 209, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2021, 7, 85, 1105, 7, 31, 1434, 1145, 55, 5, 0, 2374, 123, 86, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 1368, 245, 44, 139, 516, 7, 71, 1083, 12, 889, 3674, 0, 18, 39, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 49, 1, 6, 16, 259, 5, 11, 169, 45, 36, 205, 260, 111, 332, 190, 4768, 2219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1304, 7, 0, 1, 1217, 10, 54, 398, 950, 15, 2704, 153, 522, 14, 359, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2, 0, 59, 33, 0, 111, 490, 8, 73, 388, 21, 138, 71, 97, 19, 3474, 1235, 1087, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 115, 22, 458, 34, 14, 140, 31, 124, 191, 417, 3261, 10, 162, 350, 76, 919, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 917, 2, 85, 1, 198, 415, 91, 104, 60, 3931, 386, 83, 98, 15, 1595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7572, 9146, 9153, 8699, 7926, 7702, 7165, 8173] avg=8192.0000 max=9153 max_rank=2 min=7165 min_rank=6 imbalance_ratio=1.117310 max_load_reduction=56383 max_load_reduction_pct=0.860336 moved_to_echo=30966 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2800, 17, 1, 91, 110, 316, 120, 1, 17, 393, 1084, 175, 1545, 91, 335, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 2327, 5, 22, 789, 9, 27, 805, 542, 39, 13, 0, 2775, 126, 22, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 2400, 144, 1, 150, 1040, 4, 76, 423, 23, 215, 2403, 2, 18, 16, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 82, 3, 4, 8, 372, 3, 8, 163, 16, 25, 183, 249, 12, 223, 58, 3923, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1509, 10, 1, 0, 1133, 2, 21, 703, 1029, 14, 1780, 840, 464, 7, 372, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 103, 49, 1, 69, 1643, 6, 66, 159, 4, 14, 69, 308, 20, 3548, 1429, 1056, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 389, 53, 890, 28, 10, 47, 20, 197, 111, 369, 4050, 11, 177, 436, 44, 882, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 974, 0, 52, 0, 125, 468, 407, 118, 4, 3774, 302, 27, 51, 2, 1970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7492, 8038, 8225, 7548, 8456, 8881, 8560, 8336] avg=8192.0000 max=8881 max_rank=5 min=7492 min_rank=0 imbalance_ratio=1.084106 max_load_reduction=56655 max_load_reduction_pct=0.864487 moved_to_echo=34233 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3022, 20, 1, 94, 174, 587, 203, 0, 1, 531, 1534, 252, 1297, 143, 208, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 1910, 4, 7, 1277, 0, 40, 665, 1288, 26, 1, 0, 2222, 111, 176, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 749, 1054, 151, 2, 41, 1191, 24, 39, 533, 4, 389, 2277, 0, 26, 36, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 116, 4, 10, 4, 90, 5, 3, 200, 10, 13, 192, 262, 7, 258, 318, 4162, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1299, 5, 0, 0, 984, 2, 32, 988, 956, 8, 2411, 396, 593, 0, 506, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 15, 25, 0, 67, 1515, 10, 49, 74, 1, 17, 214, 206, 7, 3144, 1231, 1284, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 366, 35, 421, 149, 2, 68, 5, 221, 147, 398, 4183, 3, 139, 371, 66, 951, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 821, 0, 35, 1, 164, 908, 141, 47, 26, 4065, 399, 52, 58, 9, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8809, 8212, 6778, 8282, 8459, 8224, 8302, 8470] avg=8192.0000 max=8809 max_rank=0 min=6778 min_rank=2 imbalance_ratio=1.075317 max_load_reduction=56727 max_load_reduction_pct=0.865585 moved_to_echo=33455 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3730, 34, 5, 431, 60, 705, 87, 0, 36, 223, 1112, 68, 1555, 60, 218, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 2029, 7, 23, 1002, 13, 31, 829, 1067, 45, 0, 0, 2102, 91, 32, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1295, 147, 20, 124, 400, 6, 14, 542, 7, 698, 3849, 1, 20, 31, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 57, 11, 19, 0, 246, 23, 4, 98, 10, 32, 544, 220, 43, 127, 129, 4035, 2237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 11, 2, 2, 1495, 0, 41, 612, 1275, 2, 2354, 318, 512, 3, 320, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 20, 0, 14, 16, 10, 84, 620, 3, 96, 259, 18, 27, 32, 326, 14, 2671, 1374, 1097, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 390, 11, 819, 38, 3, 45, 29, 158, 84, 666, 4034, 20, 109, 309, 34, 893, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 916, 1, 118, 0, 199, 347, 236, 107, 25, 3797, 372, 58, 84, 23, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8759, 7796, 8385, 7847, 8619, 7136, 8761, 8233] avg=8192.0000 max=8761 max_rank=6 min=7136 min_rank=5 imbalance_ratio=1.069458 max_load_reduction=56775 max_load_reduction_pct=0.866318 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2852, 66, 1, 130, 142, 360, 60, 0, 14, 405, 1525, 116, 1633, 147, 152, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 1658, 28, 12, 992, 23, 18, 608, 631, 36, 1, 0, 1766, 117, 114, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 1798, 162, 16, 92, 1105, 24, 128, 1210, 2, 539, 2688, 0, 44, 29, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 43, 4, 14, 1, 192, 15, 8, 235, 17, 11, 208, 293, 21, 153, 147, 4001, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 2, 4, 1, 1573, 2, 23, 761, 1437, 89, 2118, 418, 416, 70, 591, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 17, 0, 22, 33, 1, 148, 1608, 4, 80, 210, 39, 108, 67, 163, 2, 3295, 1204, 1253, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 268, 15, 408, 60, 9, 324, 86, 164, 450, 236, 4023, 2, 276, 388, 107, 903, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 763, 0, 38, 0, 366, 381, 66, 68, 26, 3865, 328, 87, 48, 4, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8063, 6437, 9024, 7928, 8853, 8614, 8499, 8118] avg=8192.0000 max=9024 max_rank=2 min=6437 min_rank=1 imbalance_ratio=1.101562 max_load_reduction=56512 max_load_reduction_pct=0.862305 moved_to_echo=34084 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 15104, 128, 256, 7680, 128, 384, 9216, 8448, 384, 128, 0, 18048, 1024, 640, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[24192, 384, 128, 1664, 1024, 4224, 896, 128, 256, 3200, 10624, 1408, 12160, 1024, 1920, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[256, 128, 128, 384, 384, 128, 896, 9984, 128, 640, 1792, 128, 512, 1024, 1920, 128, 25728, 10752, 9088, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2176, 2176, 256, 5120, 512, 128, 896, 384, 1664, 1664, 3200, 29952, 128, 1536, 3456, 640, 7296, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[6656, 11904, 1536, 256, 768, 6528, 128, 640, 6784, 128, 5248, 23296, 128, 384, 384, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[11264, 128, 128, 128, 9344, 128, 256, 5888, 9216, 256, 17024, 3328, 3968, 256, 3328, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[1408, 6912, 128, 640, 128, 1664, 4352, 1664, 640, 256, 30720, 3072, 512, 512, 128, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[256, 768, 128, 128, 128, 1920, 256, 128, 1408, 256, 256, 2432, 2304, 384, 1792, 1792, 34176, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 0, 4, 1, 2, 0, 19, 0, 0, 0, 0, 1, 0, 7979, 0, 0, 0, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4, 0, 0, 0, 9, 6, 0, 0, 3, 0, 2, 1, 111, 0, 15, 0, 1, 0, 0, 33, 0, 1, 0, 7506, 1, 0, 0, 0, 11, 0, 15, 0, 0, 0, 0, 1, 0, 808, 0, 0, 21, 8152, 4, 0, 1, 0, 7763, 0, 8175, 1, 0, 483, 0, 0, 0, 535, 115, 8142, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4588, 0, 0, 0, 8174, 33, 59, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 25, 0, 221, 1, 2, 2, 0, 0, 0, 0, 0, 7871, 166, 0, 11, 0, 0, 7357, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 1, 69, 0, 0, 0, 90, 0, 0, 2, 0, 0, 0, 9, 0, 10, 0, 0, 0, 0, 4434, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 33, 0, 0, 596, 8034, 0, 0, 1, 0, 8151, 147, 8190, 0, 1, 63, 0, 11, 0, 605, 0, 3608, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 1, 0, 1, 7618, 0, 0, 0, 8185, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 21, 0, 15, 23, 7, 0, 0, 11, 0, 0, 0, 0, 0, 7626, 0, 0, 1, 0, 0, 7347, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 1, 16, 2, 0, 15, 27, 0, 0, 126, 0, 0, 1, 335, 0, 70, 0, 0, 0, 0, 7794, 0, 0, 0, 14, 0, 0, 0, 3, 0, 0, 21, 18, 0, 3, 0, 0, 74, 7998, 0, 1, 0, 0, 5334, 10, 8163, 0, 1166, 0, 0, 0, 0, 93, 0, 8043, 0, 0, 30, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 1, 3077, 0, 0, 0, 7675, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 21, 1, 0, 0, 3, 1, 0, 0, 10, 0, 0, 8145, 0, 0, 1, 6, 0, 7081, 0, 22, 16, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 2, 0, 104, 0, 0, 0, 0, 0, 0, 3, 0, 0, 235, 34, 4, 0, 0, 0, 0, 5, 0, 421, 0, 87, 359, 5392, 0, 0, 0, 0, 1, 0, 1, 3, 1, 0, 5, 0, 0, 393, 0, 0, 7, 7983, 0, 0, 0, 0, 7510, 13, 8191, 0, 3, 20, 0, 232, 0, 2517, 0, 7793, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 60, 4, 2, 605, 0, 0, 0, 8178, 2, 28, 11, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[11, 0, 0, 1, 0, 25, 2, 0, 6, 13, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 7, 3266, 2, 0, 0, 0, 0, 0, 7, 0, 14, 0, 7, 0, 0, 0, 10, 0, 1, 9, 0, 0, 0, 0, 62, 0, 1, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 6113, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7, 0, 806, 0, 0, 1406, 8084, 0, 0, 0, 0, 8156, 0, 8170, 0, 2, 109, 14, 0, 0, 1132, 2, 6802, 0, 0, 0, 0, 0, 10, 0, 0, 1, 2, 182, 26, 0, 15, 0, 2, 0, 234, 0, 203, 4080, 0, 0, 0, 8190, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[11, 1, 0, 0, 0, 1, 0, 0, 75, 190, 0, 0, 0, 275, 11, 0, 7850, 3, 0, 0, 0, 0, 7342, 0, 2, 0, 0, 0, 5, 0, 33, 0, 0, 0, 13, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 33, 108, 1, 0, 775, 0, 27, 246, 0, 1, 0, 166, 0, 1, 0, 1, 1, 2, 5554, 0, 1, 0, 0, 3, 0, 31, 0, 0, 0, 20, 0, 0, 0, 0, 2, 457, 5702, 1, 3, 2, 1, 8110, 0, 8116, 0, 6, 2, 1, 3, 0, 15, 1, 5119, 2, 0, 0, 20, 0, 0, 0, 2, 0, 0, 32, 0, 0, 0, 2, 0, 3, 36, 0, 0, 8137, 0, 1, 0, 6967, 2, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 6, 0, 5, 7931, 15, 0, 0, 0, 0, 8038, 0, 695, 18, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 1, 2, 6, 2, 1, 0, 0, 0, 0, 0, 2, 0, 0, 5, 58, 7, 0, 0, 0, 0, 0, 4, 0, 0, 32, 0, 3651, 9, 0, 0, 0, 0, 1, 30, 1, 0, 0, 0, 0, 1, 12, 1, 0, 9, 7901, 2, 22, 118, 0, 8168, 0, 8179, 0, 11, 28, 8, 47, 3, 4639, 10, 6674, 0, 1, 0, 2, 0, 5, 0, 0, 0, 1, 8, 1, 0, 0, 0, 0, 0, 636, 1, 0, 1421, 0, 0, 1, 7086, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 5, 7930, 0, 0, 0, 0, 0, 4688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 61, 1, 3, 339, 0, 2, 0, 70, 0, 0, 0, 0, 18, 35, 8, 1, 14, 0, 0, 0, 0, 6140, 6, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 3, 1307, 0, 0, 85, 8090, 0, 0, 2, 2, 8085, 0, 8160, 0, 0, 30, 0, 0, 0, 2106, 0, 8187, 1, 0, 9, 3, 0, 5, 0, 0, 0, 2, 3, 5, 0, 0, 0, 3, 0, 0, 2, 78, 1957, 9, 0, 0, 8053, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 25, 0, 221, 1, 2, 2, 0, 0, 0, 0, 0, 8034, 221, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2070, 166, 0, 11, 0, 0, 7357, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 1, 69, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 2, 0, 0, 0, 9, 0, 10, 0, 0, 0, 0, 4434, 1740, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 33, 0, 0, 5580, 2217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 0, 0, 0, 1, 0, 6475, 147, 0, 0, 1, 63, 0, 11, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7618, 0, 0, 0, 4228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8599, 9605, 8175, 6550, 7834, 7899, 5026, 11848] avg=8192.0000 max=11848 max_rank=7 min=5026 min_rank=6 imbalance_ratio=1.446289 max_load_reduction=53688 max_load_reduction_pct=0.819214 moved_to_echo=32607 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 0, 4, 1, 2, 0, 19, 0, 0, 0, 0, 1, 0, 8152, 224, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2099, 0, 0, 0, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4, 0, 0, 0, 9, 6, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 1, 111, 0, 15, 0, 1, 0, 0, 33, 0, 1, 0, 7506, 1738, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 15, 0, 0, 0, 0, 1, 0, 808, 0, 0, 5656, 2214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4, 0, 1, 0, 6167, 0, 0, 1, 0, 483, 0, 0, 0, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 8142, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7, 0, 0, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4588, 0, 0, 0, 4222, 33, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8505, 4786, 8118, 9663, 8706, 7212, 9644, 8902] avg=8192.0000 max=9663 max_rank=3 min=4786 min_rank=1 imbalance_ratio=1.179565 max_load_reduction=55873 max_load_reduction_pct=0.852554 moved_to_echo=34464 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 21, 0, 15, 23, 7, 0, 0, 11, 0, 0, 0, 0, 0, 7998, 214, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 0, 1, 0, 0, 7347, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 1, 16, 2, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 27, 0, 0, 126, 0, 0, 1, 335, 0, 70, 0, 0, 0, 0, 7794, 1632, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 3, 0, 0, 21, 18, 0, 3, 0, 0, 5406, 2079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 1, 0, 0, 4237, 10, 0, 0, 1166, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 0, 0, 30, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 1, 3077, 0, 0, 0, 3964, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8382, 9364, 8100, 10173, 7544, 5581, 8998, 7394] avg=8192.0000 max=10173 max_rank=3 min=5581 min_rank=5 imbalance_ratio=1.241821 max_load_reduction=55363 max_load_reduction_pct=0.844772 moved_to_echo=29517 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 1, 0, 25, 2, 0, 6, 13, 0, 0, 0, 0, 0, 0, 8084, 228, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 0, 0, 0, 0, 7, 3266, 2, 0, 0, 0, 0, 0, 7, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 10, 0, 1, 9, 0, 0, 0, 0, 62, 0, 1, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 6113, 1741, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7, 0, 806, 0, 0, 5775, 2219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 0, 0, 0, 0, 6479, 0, 0, 0, 2, 109, 14, 0, 0, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6802, 0, 0, 0, 0, 0, 10, 0, 0, 1, 2, 182, 26, 0, 15, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 234, 0, 203, 4080, 0, 0, 0, 4230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8462, 5440, 8168, 8310, 8813, 9142, 8452, 8749] avg=8192.0000 max=9142 max_rank=5 min=5440 min_rank=1 imbalance_ratio=1.115967 max_load_reduction=56394 max_load_reduction_pct=0.860504 moved_to_echo=35156 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[11, 1, 0, 0, 0, 1, 0, 0, 75, 190, 0, 0, 0, 275, 11, 0, 5702, 220, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 3, 0, 0, 0, 0, 7342, 0, 2, 0, 0, 0, 5, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 33, 108, 1, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 27, 246, 0, 1, 0, 166, 0, 1, 0, 1, 1, 2, 5554, 1481, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 31, 0, 0, 0, 20, 0, 0, 0, 0, 2, 5565, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 1, 3, 2, 1, 6442, 0, 0, 0, 6, 2, 1, 3, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5119, 2, 0, 0, 20, 0, 0, 0, 2, 0, 0, 32, 0, 0, 0, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 36, 0, 0, 8137, 0, 1, 0, 3599, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6579, 9450, 8185, 8519, 7509, 6933, 6580, 11781] avg=8192.0000 max=11781 max_rank=7 min=6579 min_rank=0 imbalance_ratio=1.438110 max_load_reduction=53755 max_load_reduction_pct=0.820236 moved_to_echo=32803 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 5, 8090, 222, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 0, 0, 0, 0, 0, 4688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 61, 1, 3, 339, 0, 2, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 18, 35, 8, 1, 14, 0, 0, 0, 0, 6140, 1712, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 3, 1307, 0, 0, 5622, 2181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 2, 2, 6422, 0, 0, 0, 0, 30, 0, 0, 0, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 9, 3, 0, 5, 0, 0, 0, 2, 3, 5, 0, 0, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 78, 1957, 9, 0, 0, 4160, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8418, 6774, 8474, 8261, 9132, 8647, 9615, 6215] avg=8192.0000 max=9615 max_rank=6 min=6215 min_rank=7 imbalance_ratio=1.173706 max_load_reduction=55921 max_load_reduction_pct=0.853287 moved_to_echo=33609 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 21, 1, 0, 0, 3, 1, 0, 0, 10, 0, 0, 7983, 228, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2143, 0, 0, 1, 6, 0, 7081, 0, 22, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 2, 0, 104, 0, 0, 0, 0, 0, 0, 3, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 34, 4, 0, 0, 0, 0, 5, 0, 421, 0, 87, 359, 5392, 1739, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3, 1, 0, 5, 0, 0, 393, 0, 0, 5774, 2215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 5966, 13, 0, 0, 3, 20, 0, 232, 0, 2517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7793, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 60, 4, 2, 605, 0, 0, 0, 4224, 2, 28, 11, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 9270, 8214, 8520, 8393, 8758, 9099, 4938] avg=8192.0000 max=9270 max_rank=1 min=4938 min_rank=7 imbalance_ratio=1.131592 max_load_reduction=56266 max_load_reduction_pct=0.858551 moved_to_echo=31188 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 6, 0, 5, 7901, 226, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2079, 15, 0, 0, 0, 0, 8038, 0, 695, 18, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 2, 6, 2, 1, 0, 0, 0, 0, 0, 2, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 58, 7, 0, 0, 0, 0, 0, 4, 0, 0, 32, 0, 3651, 1512, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 30, 1, 0, 0, 0, 0, 1, 12, 1, 0, 5626, 1924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 22, 118, 0, 6477, 0, 0, 0, 11, 28, 8, 47, 3, 4639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 6674, 0, 1, 0, 2, 0, 5, 0, 0, 0, 1, 8, 1, 0, 0, 1420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 636, 1, 0, 1421, 0, 0, 1, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 10847, 8102, 5540, 7605, 11364, 8122, 5709] avg=8192.0000 max=11364 max_rank=5 min=5540 min_rank=3 imbalance_ratio=1.387207 max_load_reduction=54172 max_load_reduction_pct=0.826599 moved_to_echo=32800 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 3456, 128, 128, 45056, 17024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 1408, 128, 384, 31488, 128, 128, 128, 32384, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 1024, 256, 128, 768, 0, 128, 128, 640, 128, 128, 512, 128, 128, 384, 46592, 13312, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[16768, 256, 0, 128, 128, 128, 47872, 128, 768, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[2688, 0, 128, 128, 128, 128, 48768, 256, 0, 128, 1280, 768, 128, 384, 128, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 54400, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 256, 128, 0, 128, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 256, 128, 256, 128, 0, 128, 384, 128, 128, 61952, 1792, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 128, 256, 128, 128, 128, 128, 640, 128, 128, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 4160, 8182, 0, 0, 0, 7984, 0, 0, 0, 0, 0, 0, 8, 0, 0, 643, 0, 0, 0, 0, 0, 27, 0, 0, 0, 5303, 0, 0, 0, 0, 0, 0, 1, 507, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 7500, 2, 0, 0, 0, 0, 2819, 0, 0, 2100, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 58, 0, 0, 0, 7, 0, 11, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8169, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 95, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2, 4, 0, 0, 2, 9, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 7960, 0, 0, 0, 5575, 0, 0, 0, 0, 0, 0, 0, 0, 9, 154, 0, 0, 3, 0, 0, 2, 0, 1, 0, 6388, 0, 0, 0, 0, 0, 1, 0, 444, 0, 0, 3, 0, 0, 0, 7842, 0, 0, 0, 0, 1, 0, 6754, 0, 0, 0, 1, 0, 8094, 0, 3, 3006, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 293, 1, 566, 0, 1311, 0, 0, 221, 27, 0, 0, 0, 278, 0, 0, 0, 0, 0, 74, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 0, 0, 309, 0, 19, 1, 0, 2, 0, 0, 149, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8130, 0, 1, 1492, 8067, 0, 0, 0, 7807, 10, 0, 0, 0, 0, 1, 8, 0, 0, 905, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 2289, 0, 1, 0, 0, 0, 0, 8105, 3, 0, 0, 5, 0, 0, 7157, 1, 0, 3, 0, 1, 7849, 0, 0, 390, 0, 0, 1, 0, 32, 0, 3, 1, 2, 20, 0, 42, 6, 0, 3, 35, 31, 119, 288, 0, 26, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 1, 0, 1990, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 32, 1, 58, 8182, 0, 0, 0, 7666, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2, 449, 0, 0, 0, 3, 0, 3, 20, 6513, 0, 0, 0, 0, 0, 0, 8130, 10, 0, 0, 0, 0, 2, 8143, 2, 0, 0, 0, 0, 7655, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 14, 130, 0, 18, 2, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 1, 25, 0, 1872, 0, 1, 0, 0, 0, 0, 0, 8, 4, 0, 0, 33, 0, 0, 0, 0, 0, 0, 7905, 0, 0, 2530, 8160, 0, 0, 0, 6395, 0, 0, 0, 65, 0, 0, 8, 4, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 4791, 0, 0, 16, 0, 0, 0, 0, 360, 0, 2, 0, 0, 0, 0, 8030, 0, 0, 10, 7, 0, 0, 7982, 354, 1, 0, 0, 0, 8161, 0, 0, 308, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 24, 0, 19, 0, 50, 0, 572, 133, 0, 4, 0, 0, 0, 0, 0, 24, 2, 0, 0, 0, 7469, 187, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 349, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 34, 7687, 0, 0, 6891, 8146, 0, 2, 0, 3951, 0, 0, 2, 0, 161, 0, 0, 0, 0, 473, 0, 0, 0, 40, 1, 108, 0, 0, 0, 2084, 0, 0, 2, 0, 0, 11, 0, 28, 4, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 7591, 195, 244, 0, 0, 0, 8054, 0, 0, 0, 0, 0, 3, 0, 0, 0, 86, 1, 1, 1, 0, 0, 0, 1, 0, 1, 11, 2922, 1, 0, 0, 4, 2, 4, 0, 1, 2, 0, 0, 0, 314, 8002, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 63, 0, 0, 2, 7971, 0, 1, 0, 0, 0, 1, 29, 0, 3, 0, 1, 0, 3, 0, 0, 0, 80, 0, 8130, 174, 0, 40, 8052, 0, 0, 21, 7385, 0, 0, 0, 15, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 48, 56, 0, 0, 0, 0, 0, 0, 0, 3824, 0, 4, 0, 0, 0, 12, 5609, 1, 4, 5, 0, 0, 22, 7639, 30, 0, 0, 0, 0, 4405, 0, 14, 2460, 0, 0, 51, 0, 0, 0, 0, 0, 0, 2, 170, 866, 0, 93, 0, 7, 15, 1, 2, 0, 17, 0, 0, 0, 0, 21, 0, 0, 0, 0, 1, 8178, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4323, 0, 0, 0, 0, 0, 2, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8130, 0, 0, 1594, 7848, 0, 0, 0, 7010, 0, 1, 0, 95, 35, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 2824, 1, 0, 1, 0, 0, 0, 0, 8164, 5, 5, 0, 1, 0, 0, 7544, 0, 1, 2, 0, 5, 0, 7196, 418, 14, 0, 0, 0, 770, 0, 0, 1018, 37, 0, 0, 0, 0, 0, 0, 0, 121, 14, 0, 0, 0, 0, 1, 15, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 2, 0, 7996, 0, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1657, 5083, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4160, 0, 0, 0, 0, 7133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 643, 0, 0, 0, 0, 0, 27, 0, 0, 0, 5303, 0, 2430, 1546, 851, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 507, 0, 0, 0, 0, 0, 1, 5761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 871, 2, 0, 0, 0, 0, 2819, 0, 0, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 58, 0, 0, 0, 7, 0, 11, 7319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8069, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7502, 8284, 11293, 10908, 6270, 5792, 7397, 8090] avg=8192.0000 max=11293 max_rank=2 min=5792 min_rank=5 imbalance_ratio=1.378540 max_load_reduction=54243 max_load_reduction_pct=0.827682 moved_to_echo=27549 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 0, 0, 309, 0, 19, 1, 0, 2, 0, 0, 149, 4850, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 8067, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1492, 0, 0, 0, 0, 6975, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 905, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 2405, 1475, 832, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2289, 0, 1, 0, 0, 0, 0, 5700, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 832, 1, 0, 3, 0, 1, 7849, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 32, 0, 3, 1, 2, 20, 0, 42, 6, 0, 3, 35, 31, 119, 7271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 26, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6099, 8174, 8479, 9735, 7993, 9081, 7566, 8409] avg=8192.0000 max=9735 max_rank=3 min=6099 min_rank=0 imbalance_ratio=1.188354 max_load_reduction=55801 max_load_reduction_pct=0.851456 moved_to_echo=33049 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 95, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4577, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2, 4, 0, 0, 2, 9, 0, 0, 0, 7960, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 154, 0, 0, 3, 0, 0, 2, 0, 1, 0, 6388, 0, 2327, 1392, 594, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 444, 0, 0, 3, 0, 0, 0, 5515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 785, 0, 0, 0, 1, 0, 8094, 0, 3, 3006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 293, 1, 566, 0, 1311, 0, 0, 7318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 27, 0, 0, 0, 278, 0, 0, 0, 0, 0, 74, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5447, 8091, 4982, 10970, 5963, 11890, 9505, 8688] avg=8192.0000 max=11890 max_rank=5 min=4982 min_rank=2 imbalance_ratio=1.451416 max_load_reduction=53646 max_load_reduction_pct=0.818573 moved_to_echo=36046 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 1, 25, 0, 1872, 5410, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 4, 0, 0, 33, 0, 0, 0, 0, 8160, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 0, 0, 0, 5713, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 4791, 0, 2383, 1646, 682, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 360, 0, 2, 0, 0, 0, 0, 5647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7, 0, 0, 926, 354, 1, 0, 0, 0, 8161, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 24, 0, 19, 0, 50, 0, 572, 7072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 4, 0, 0, 0, 0, 0, 24, 2, 0, 0, 0, 7378, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8051, 8304, 8308, 9612, 6025, 9767, 7741, 7728] avg=8192.0000 max=9767 max_rank=5 min=6025 min_rank=4 imbalance_ratio=1.192261 max_load_reduction=55769 max_load_reduction_pct=0.850967 moved_to_echo=31261 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 63, 0, 0, 2, 7971, 5177, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 29, 0, 3, 0, 1, 0, 3, 0, 0, 0, 8052, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 174, 0, 40, 0, 0, 0, 21, 6598, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 48, 56, 0, 1664, 1575, 787, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 0, 4, 0, 0, 0, 12, 3945, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 22, 887, 30, 0, 0, 0, 0, 4405, 0, 14, 2460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 2, 170, 866, 0, 93, 0, 7, 15, 1, 7271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 17, 0, 0, 0, 0, 21, 0, 0, 0, 0, 1, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13971, 8193, 6928, 4236, 7790, 7823, 8476, 8119] avg=8192.0000 max=13971 max_rank=0 min=4236 min_rank=3 imbalance_ratio=1.705444 max_load_reduction=51565 max_load_reduction_pct=0.786819 moved_to_echo=32208 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 1, 0, 1990, 5519, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1, 58, 0, 0, 0, 0, 6849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2, 449, 0, 2412, 1679, 817, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 20, 6513, 0, 0, 0, 0, 0, 0, 5718, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 945, 2, 0, 0, 0, 0, 7655, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 14, 7317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 18, 2, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8289, 6940, 5464, 12267, 8718, 7340, 8241] avg=8192.0000 max=12267 max_rank=4 min=5464 min_rank=3 imbalance_ratio=1.497437 max_load_reduction=53269 max_load_reduction_pct=0.812820 moved_to_echo=36566 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 349, 5145, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 34, 0, 0, 0, 6891, 0, 0, 2, 0, 3530, 0, 0, 2, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 0, 0, 0, 40, 1, 108, 0, 0, 0, 2084, 0, 2407, 1565, 421, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 11, 0, 28, 4, 0, 0, 0, 0, 0, 5705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 881, 195, 244, 0, 0, 0, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 86, 1, 1, 1, 0, 0, 0, 1, 0, 1, 11, 2922, 6875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 2, 4, 0, 1, 2, 0, 0, 0, 314, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 8245, 10622, 7197, 5750, 9374, 9902, 8232] avg=8192.0000 max=10622 max_rank=2 min=5750 min_rank=4 imbalance_ratio=1.296631 max_load_reduction=54914 max_load_reduction_pct=0.837921 moved_to_echo=33258 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4323, 4881, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7848, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1594, 0, 0, 0, 0, 6257, 0, 1, 0, 95, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 2824, 1, 2243, 1488, 753, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8164, 5, 5, 0, 1, 0, 0, 5301, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 827, 418, 14, 0, 0, 0, 770, 0, 0, 1018, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 14, 0, 0, 0, 0, 1, 15, 0, 187, 7271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 2, 0, 7895, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9975, 7956, 7984, 7414, 13478, 3091, 7609, 8029] avg=8192.0000 max=13478 max_rank=4 min=3091 min_rank=5 imbalance_ratio=1.645264 max_load_reduction=52058 max_load_reduction_pct=0.794342 moved_to_echo=32207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=67584 observed_loss=-2048 tokens_per_expert=[128, 128, 128, 128, 2304, 128, 128, 128, 128, 128, 256, 128, 128, 128, 25984, 128, 18304, 12416, 5760, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 384, 128, 128, 128, 128, 128, 128, 128, 18432, 40704, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 128, 128, 22144, 128, 128, 128, 128, 0, 128, 43392, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 7040, 1024, 384, 128, 128, 128, 47872, 0, 128, 9472, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 256, 128, 16768, 0, 0, 128, 128, 48128, 128, 128, 128, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 256, 1408, 128, 768, 128, 1536, 128, 3840, 57728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 0, 0, 128, 64640, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[896, 128, 128, 128, 128, 384, 128, 128, 256, 128, 0, 128, 384, 63616, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 0, 156, 0, 23, 0, 2371, 0, 1554, 0, 0, 0, 0, 466, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 8, 6110, 6096, 0, 3, 26, 0, 0, 5, 0, 8116, 0, 0, 0, 0, 0, 1, 7659, 6, 0, 0, 0, 0, 20, 0, 1, 148, 1, 0, 0, 2, 12, 0, 8182, 0, 0, 0, 0, 0, 5, 0, 3, 0, 1, 9, 6003, 0, 0, 8, 2, 0, 1, 0, 0, 0, 2, 3, 21, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 2707, 0, 0, 0, 205, 0, 0, 0, 0, 0, 5245, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 3, 0, 176, 1282, 0, 0, 0, 0, 434, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1, 0, 0, 8060, 0, 53, 0, 0, 0, 1, 74, 0, 2, 8, 0, 0, 0, 0, 1, 0, 7, 12, 0, 0, 0, 8029, 8108, 0, 0, 0, 0, 1, 77, 0, 8047, 3, 0, 0, 0, 0, 1, 7956, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 1, 0, 1, 3, 8007, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 8166, 14, 0, 0, 38, 0, 0, 0, 0, 2, 0, 16, 0, 0, 496, 0, 0, 0, 1, 0, 0, 0, 138, 0, 0, 0, 0, 0, 25, 0, 2, 0, 0, 31, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 15, 0, 0, 0, 7891, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 7270, 1288, 0, 0, 0, 0, 0, 976, 0, 7671, 20, 0, 0, 0, 4, 0, 7765, 0, 0, 21, 0, 380, 0, 0, 5, 0, 346, 0, 0, 7, 16, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8143, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7242, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 28, 0, 0, 0, 8077, 0, 0, 45, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 45, 0, 0, 0, 792, 0, 0, 0, 6116, 5784, 0, 2, 0, 0, 0, 176, 0, 4604, 0, 0, 0, 0, 0, 0, 7970, 0, 0, 0, 0, 58, 0, 0, 0, 72, 796, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 10, 0, 0, 0, 46, 0, 3, 56, 0, 0, 0, 30, 30, 0, 0, 0, 0, 8, 0, 6976, 1, 0, 0, 2752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 1, 3, 0, 1, 0, 1, 0, 0, 0, 0, 2, 7, 32, 4584, 0, 0, 0, 0, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 0, 0, 8180, 0, 0, 2, 9, 0, 1, 0, 31, 0, 5, 37, 3, 0, 0, 0, 0, 7, 0, 28, 0, 3, 6908, 7411, 16, 0, 0, 2, 0, 203, 2, 5458, 1, 0, 11, 0, 22, 0, 8096, 0, 0, 0, 0, 219, 0, 0, 5, 0, 0, 0, 1, 1, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8102, 0, 10, 0, 0, 0, 23, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 1, 56, 0, 7923, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 3, 0, 0, 0, 0, 1, 0, 574, 0, 0, 0, 0, 0, 3928, 0, 0, 3, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 7922, 4, 0, 2, 0, 0, 0, 0, 51, 0, 1, 0, 0, 0, 15, 7988, 0, 0, 58, 0, 0, 3, 0, 0, 0, 346, 0, 0, 0, 631, 0, 7586, 0, 0, 0, 25, 0, 0, 0, 0, 0, 162, 0, 8141, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7868, 1, 6, 5, 66, 0, 289, 0, 0, 0, 0, 1, 5, 12, 3207, 0, 1, 0, 0, 1, 0, 5, 0, 18, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 6342, 0, 191, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7452, 8167, 0, 0, 1, 2, 0, 10, 0, 3661, 7, 0, 1, 0, 1, 0, 2854, 0, 0, 1, 0, 16, 0, 0, 0, 2825, 0, 20, 0, 0, 100, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 57, 0, 0, 0, 5, 0, 0, 0, 0, 0, 931, 0, 0, 0, 0, 1, 0, 0, 427, 0, 0, 0, 0, 0, 7999, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7468, 0, 0, 0, 0, 0, 0, 2, 41, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 131, 8184, 0, 0, 0, 0, 0, 15, 0, 7409, 22, 0, 6, 0, 0, 0, 8189, 0, 0, 0, 0, 156, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 8116, 25, 0, 0, 0, 0, 0, 0, 22, 0, 2, 0, 8185, 0, 0, 0, 53, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8133, 0, 0, 0, 178, 0, 0, 0, 9, 0, 0, 0, 0, 0, 2569, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6560, 13, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 15, 0, 0, 0, 7891, 0, 8, 0, 0, 0, 0, 1, 0, 193, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 7270, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 976, 0, 7671, 20, 0, 0, 0, 4, 0, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 380, 0, 0, 5, 0, 346, 0, 0, 7, 16, 0, 7516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8143, 0, 0, 0, 0, 7, 1, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 5256, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7242, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 7639, 11180, 8291, 8416, 8391, 5493, 7822] avg=8192.0000 max=11180 max_rank=2 min=5493 min_rank=6 imbalance_ratio=1.364746 max_load_reduction=54356 max_load_reduction_pct=0.829407 moved_to_echo=30122 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 0, 0, 8180, 0, 0, 2, 9, 0, 1, 0, 31, 1113, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 37, 3, 0, 0, 0, 0, 7, 0, 28, 0, 3, 6908, 1997, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 203, 2, 5458, 1, 0, 11, 0, 22, 0, 2616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 5, 0, 0, 0, 1, 1, 0, 0, 7494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8102, 0, 10, 0, 0, 0, 23, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 1, 56, 0, 7923, 0, 9, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 3, 0, 0, 0, 5480, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 574, 0, 0, 0, 0, 0, 3928, 0, 0, 3, 2, 0, 0, 3331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9538, 9004, 8315, 7720, 8399, 8214, 6507, 7839] avg=8192.0000 max=9538 max_rank=0 min=6507 min_rank=6 imbalance_ratio=1.164307 max_load_reduction=55998 max_load_reduction_pct=0.854462 moved_to_echo=30959 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4911, 0, 0, 0, 0, 0, 0, 0, 0, 1190, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 2135, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 51, 0, 1, 0, 0, 0, 15, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 3, 0, 0, 0, 346, 0, 0, 0, 631, 0, 6969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 162, 0, 8141, 0, 0, 19, 1, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7868, 1, 6, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 66, 0, 289, 0, 0, 0, 0, 1, 5, 12, 3207, 0, 1, 0, 0, 5407, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 18, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 10305, 2650, 8007, 8585, 8073, 10029, 11604] avg=8192.0000 max=11604 max_rank=7 min=2650 min_rank=2 imbalance_ratio=1.416504 max_load_reduction=53932 max_load_reduction_pct=0.822937 moved_to_echo=38291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1, 0, 0, 8060, 0, 53, 0, 0, 0, 1, 74, 0, 1218, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 1, 0, 7, 12, 0, 0, 0, 8029, 2185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 77, 0, 8047, 3, 0, 0, 0, 0, 1, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 1, 0, 1, 3, 7356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 8166, 14, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 2, 0, 16, 0, 0, 496, 0, 0, 0, 1, 5385, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 25, 0, 2, 0, 0, 31, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9602, 10244, 10700, 7366, 8390, 8395, 6999, 3840] avg=8192.0000 max=10700 max_rank=2 min=3840 min_rank=7 imbalance_ratio=1.306152 max_load_reduction=54836 max_load_reduction_pct=0.836731 moved_to_echo=27624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 6342, 0, 191, 0, 1, 0, 0, 1, 1, 1227, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7452, 2201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 10, 0, 3661, 7, 0, 1, 0, 1, 0, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 2825, 0, 20, 0, 0, 100, 0, 7468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 5, 0, 0, 0, 0, 0, 931, 0, 0, 0, 0, 1931, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 427, 0, 0, 0, 0, 0, 7999, 0, 1, 1, 0, 0, 0, 3671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7965, 9654, 4606, 10430, 8436, 8353, 3992, 12100] avg=8192.0000 max=12100 max_rank=7 min=3992 min_rank=6 imbalance_ratio=1.477051 max_load_reduction=53436 max_load_reduction_pct=0.815369 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 156, 0, 23, 0, 2371, 0, 1554, 0, 0, 0, 0, 466, 0, 916, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 8, 6110, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 26, 0, 0, 5, 0, 8116, 0, 0, 0, 0, 0, 1, 2475, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1, 148, 1, 0, 0, 2, 12, 0, 7517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 1, 9, 6003, 0, 0, 8, 2, 0, 1, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 21, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2707, 0, 0, 0, 205, 0, 0, 0, 0, 0, 5245, 0, 0, 0, 0, 5184, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 3, 0, 176, 1282, 0, 0, 0, 0, 434, 0, 2740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5686, 7824, 10632, 7701, 6288, 8431, 14138, 4836] avg=8192.0000 max=14138 max_rank=6 min=4836 min_rank=7 imbalance_ratio=1.725830 max_load_reduction=51398 max_load_reduction_pct=0.784271 moved_to_echo=33693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7468, 0, 0, 0, 0, 0, 0, 2, 41, 1229, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 131, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 7409, 22, 0, 6, 0, 0, 0, 2646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 7456, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 2, 0, 8185, 0, 0, 0, 53, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8133, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 9, 0, 0, 0, 0, 0, 2569, 0, 1, 0, 0, 5543, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6560, 13, 0, 0, 0, 1, 0, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8934, 2362, 10098, 7642, 8516, 8360, 9371, 10253] avg=8192.0000 max=10253 max_rank=7 min=2362 min_rank=1 imbalance_ratio=1.251587 max_load_reduction=55283 max_load_reduction_pct=0.843552 moved_to_echo=36500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 28, 0, 0, 0, 8077, 0, 0, 45, 4, 0, 0, 0, 0, 872, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 45, 0, 0, 0, 792, 0, 0, 0, 6116, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 176, 0, 4604, 0, 0, 0, 0, 0, 0, 2573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 72, 796, 0, 0, 0, 0, 0, 7453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 10, 0, 0, 0, 46, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 56, 0, 0, 0, 30, 30, 0, 0, 0, 0, 8, 0, 6976, 1, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 1, 3, 0, 1, 5397, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 7, 32, 4584, 0, 0, 0, 0, 0, 12, 2604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9224, 8504, 7355, 8379, 8506, 7319, 9007, 7242] avg=8192.0000 max=9224 max_rank=0 min=7242 min_rank=7 imbalance_ratio=1.125977 max_load_reduction=56312 max_load_reduction_pct=0.859253 moved_to_echo=32074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 5888, 0, 384, 0, 256, 128, 0, 128, 128, 128, 12672, 128, 128, 0, 128, 39680, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[256, 128, 128, 1152, 128, 0, 128, 128, 256, 39680, 128, 128, 128, 128, 512, 128, 23808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 1536, 128, 45056, 128, 128, 128, 0, 128, 128, 18944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 0, 128, 896, 128, 0, 128, 50304, 14336, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 256, 128, 128, 0, 53376, 0, 1920, 128, 128, 0, 128, 640, 128, 8064, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 896, 128, 0, 128, 3072, 1536, 128, 128, 128, 768, 128, 59264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 0, 63616, 128, 128, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 256, 128, 63104, 128, 128, 128, 128, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 102, 0, 0, 0, 0, 5, 0, 298, 0, 0, 0, 0, 0, 0, 5, 0, 6432, 0, 0, 0, 2701, 0, 3, 2, 0, 0, 0, 0, 5031, 0, 0, 0, 8191, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1489, 7988, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8036, 0, 0, 0, 0, 30, 0, 0, 0, 7478, 0, 11, 0, 2, 0, 0, 0, 0, 3, 0, 1, 6, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 883, 0, 0, 0, 0, 0, 0, 7, 0, 8151, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6400, 0, 0, 0, 0, 0, 0, 4, 0, 5, 2, 3157, 8, 0, 120, 0, 0, 0, 0, 8075, 0, 0, 0, 8021, 8188, 0, 0, 3, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 7450, 1840, 0, 5, 1, 0, 1, 0, 0, 0, 0, 0, 51, 8159, 0, 0, 0, 0, 1, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 71, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 49, 10, 0, 0, 0, 32, 5, 0, 0, 0, 3279, 6, 0, 1, 0, 0, 0, 0, 0, 1899, 0, 0, 284, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 496, 0, 0, 0, 0, 0, 0, 758, 0, 1, 0, 27, 0, 0, 0, 0, 690, 0, 0, 0, 142, 0, 224, 0, 0, 0, 0, 0, 7132, 0, 0, 0, 7972, 8106, 0, 0, 0, 0, 2, 0, 0, 76, 0, 1, 0, 0, 0, 1890, 7739, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7938, 0, 0, 1, 0, 2, 4, 1, 1, 8150, 0, 15, 32, 0, 0, 2, 0, 1, 7, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 18, 6, 4209, 0, 20, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 590, 0, 0, 0, 0, 0, 7210, 0, 0, 0, 0, 1, 4, 1599, 1, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 476, 0, 0, 0, 8188, 8112, 0, 0, 0, 309, 0, 0, 0, 0, 0, 21, 0, 0, 0, 7906, 4199, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 2, 7625, 0, 0, 0, 0, 85, 0, 0, 0, 8111, 5, 0, 0, 14, 0, 6, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6905, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 106] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 163, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 6881, 0, 0, 0, 2320, 1, 0, 706, 284, 29, 0, 486, 452, 0, 0, 0, 8039, 8131, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 1, 1, 7718, 7263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7114, 0, 0, 0, 3, 321, 0, 0, 8, 3289, 0, 120, 0, 38, 0, 0, 0, 7, 1, 11, 459, 2, 0, 23, 7, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 41, 265, 0, 56, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 6, 0, 3, 7069, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 875, 0, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7362, 0, 245, 70, 0, 0, 0, 0, 7767, 0, 0, 0, 8145, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 7985, 7224, 0, 0, 0, 23, 31, 0, 0, 0, 0, 0, 0, 8008, 0, 0, 0, 0, 8, 0, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 31, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 155, 1, 0, 0, 0, 0, 0, 0, 0, 0, 180, 24, 0, 0, 0, 13, 0, 0, 25, 7396, 3, 648, 40, 0, 0, 0, 0, 24, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 334, 79, 39, 0, 18, 1, 0, 843, 0, 0, 0, 1648, 0, 0, 0, 26, 0, 0, 0, 5372, 13, 0, 0, 8174, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4881, 8115, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 440, 8110, 0, 0, 0, 0, 700, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 0, 0, 0, 0, 773, 0, 14, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 937, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 0, 0, 5, 1, 1479, 0, 0, 0, 0, 0, 3, 1135, 0, 0, 0, 3423, 7, 2, 2, 0, 1, 0, 404, 6162, 0, 0, 0, 8171, 8092, 0, 0, 0, 14, 0, 0, 0, 0, 0, 2, 6, 0, 3, 425, 7121, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 4, 7845, 0, 0, 0, 0, 40, 0, 0, 0, 7629, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 2363, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 201, 0, 0, 0, 0, 3, 13, 1, 0, 0, 3656, 0, 0, 0, 0, 0, 0, 3, 0, 7102, 0, 17, 15, 0, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 102, 0, 0, 0, 0, 5, 0, 298, 0, 0, 0, 0, 0, 0, 4087, 1759, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6432, 0, 0, 0, 2701, 0, 3, 2, 0, 0, 0, 0, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1489, 3901, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 7478, 0, 11, 0, 2, 0, 0, 0, 0, 1180, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 6, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 7573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 883, 0, 0, 0, 6277, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7735, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7059, 12186, 8202, 5399, 9117, 7856, 7721, 7996] avg=8192.0000 max=12186 max_rank=1 min=5399 min_rank=3 imbalance_ratio=1.487549 max_load_reduction=53350 max_load_reduction_pct=0.814056 moved_to_echo=32690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6400, 0, 0, 0, 0, 941, 1786, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 2, 3157, 8, 0, 120, 0, 0, 0, 0, 4883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8021, 10, 0, 0, 3, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7450, 899, 0, 5, 1, 0, 1, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 3, 0, 1, 1894, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 71, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 7614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 49, 10, 0, 0, 0, 32, 5, 0, 0, 0, 3279, 6, 0, 1, 6373, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1802, 0, 0, 284, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10425, 8179, 8156, 8407, 9943, 7734, 10605, 2087] avg=8192.0000 max=10605 max_rank=6 min=2087 min_rank=7 imbalance_ratio=1.294556 max_load_reduction=54931 max_load_reduction_pct=0.838181 moved_to_echo=30369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 163, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 3716, 1558, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6881, 0, 0, 0, 2320, 1, 0, 706, 284, 29, 0, 486, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 10, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7718, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 321, 0, 0, 8, 3289, 0, 120, 0, 38, 0, 0, 0, 7, 106, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 459, 2, 0, 23, 7, 48, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 41, 265, 0, 56, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5556, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 3, 6708, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5537, 10981, 8054, 11266, 4253, 8112, 6589, 10744] avg=8192.0000 max=11266 max_rank=3 min=4253 min_rank=4 imbalance_ratio=1.375244 max_load_reduction=54270 max_load_reduction_pct=0.828094 moved_to_echo=29698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 875, 0, 0, 3, 6, 0, 0, 3696, 1753, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7362, 0, 245, 70, 0, 0, 0, 0, 4698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 3528, 0, 0, 0, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 31, 0, 1821, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 1, 0, 0, 0, 0, 0, 0, 0, 0, 180, 24, 0, 0, 6255, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 25, 7019, 3, 648, 40, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7583, 12375, 8158, 11567, 3323, 7581, 7177, 7772] avg=8192.0000 max=12375 max_rank=1 min=3323 min_rank=4 imbalance_ratio=1.510620 max_load_reduction=53161 max_load_reduction_pct=0.811172 moved_to_echo=25853 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 334, 79, 39, 0, 18, 4151, 1775, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 843, 0, 0, 0, 1648, 0, 0, 0, 26, 0, 0, 0, 3249, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4881, 3964, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 0, 0, 0, 0, 773, 0, 14, 0, 0, 0, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 937, 0, 0, 0, 6335, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7629, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7262, 5780, 8184, 9286, 10348, 9158, 7885, 7633] avg=8192.0000 max=10348 max_rank=4 min=5780 min_rank=1 imbalance_ratio=1.263184 max_load_reduction=55188 max_load_reduction_pct=0.842102 moved_to_echo=35024 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 590, 0, 0, 0, 0, 0, 7210, 0, 0, 0, 0, 2148, 1669, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1599, 1, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 10, 0, 0, 0, 309, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7906, 2051, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 8111, 5, 0, 0, 14, 0, 6, 0, 0, 111, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 7, 0, 7543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5956, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6553, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11694, 1970, 8528, 9961, 8684, 7574, 6516, 10609] avg=8192.0000 max=11694 max_rank=0 min=1970 min_rank=1 imbalance_ratio=1.427490 max_load_reduction=53842 max_load_reduction_pct=0.821564 moved_to_echo=33383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 496, 0, 0, 0, 0, 0, 0, 758, 0, 1, 0, 27, 0, 0, 3962, 1738, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 690, 0, 0, 0, 142, 0, 224, 0, 0, 0, 0, 0, 4305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7972, 1, 0, 0, 0, 0, 2, 0, 0, 76, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1890, 3777, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4, 1, 1, 8150, 0, 15, 32, 0, 0, 2, 0, 1, 1675, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 7542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 1551, 0, 0, 0, 6200, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7662, 18, 6, 4209, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 5361, 8052, 5681, 10301, 7610, 8482, 11915] avg=8192.0000 max=11915 max_rank=7 min=5361 min_rank=1 imbalance_ratio=1.454468 max_load_reduction=53621 max_load_reduction_pct=0.818192 moved_to_echo=38308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 0, 0, 5, 1, 1479, 0, 0, 0, 0, 3643, 1717, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1135, 0, 0, 0, 3423, 7, 2, 2, 0, 1, 0, 404, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 9, 0, 0, 0, 14, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 425, 3478, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 7629, 0, 91, 0, 0, 0, 0, 0, 0, 1445, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 2363, 0, 0, 0, 1, 0, 0, 0, 0, 7525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 201, 0, 0, 0, 0, 3, 13, 1, 0, 0, 3656, 0, 0, 0, 6128, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6740, 0, 17, 15, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7842, 8704, 8202, 3969, 9567, 9911, 10561, 6780] avg=8192.0000 max=10561 max_rank=6 min=3969 min_rank=3 imbalance_ratio=1.289185 max_load_reduction=54975 max_load_reduction_pct=0.838852 moved_to_echo=36819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 640, 128, 128, 768, 128, 128, 0, 2048, 128, 15488, 128, 128, 0, 128, 26368, 13824, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 39808, 25216, 0, 128, 128, 128, 128, 0, 128, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 512, 512, 384, 0, 128, 0, 128, 128, 128, 256, 0, 10496, 128, 0, 128, 49152, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 17664, 128, 128, 128, 20864, 128, 512, 1024, 512, 128, 0, 896, 24576, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 51968, 128, 768, 12800, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 1280, 128, 128, 128, 51712, 128, 256, 128, 128, 0, 128, 128, 128, 9600, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 1408, 128, 0, 128, 128, 3584, 0, 128, 128, 128, 128, 0, 128, 0, 60544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 65024, 128, 0, 0, 128, 384, 128, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 7710, 0, 0, 8116, 1, 9, 0, 0, 4, 0, 164, 0, 0, 6960, 0, 0, 0, 0, 0, 69, 8147, 0, 24, 13, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 245, 0, 1715, 0, 0, 0, 13, 0, 6, 0, 0, 0, 1, 1, 5989, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 735, 0, 7751, 0, 0, 36, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 33, 47, 1, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4617, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 11, 0, 64, 8147, 2, 4, 1, 0, 1, 27, 2642, 0, 0, 0, 211, 0, 0, 46, 0, 0, 0, 0, 0, 7, 0, 72, 0, 0, 0, 0, 0, 1500, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 7497, 0, 6, 0, 0, 0, 0, 8083, 0, 0, 0, 20, 0, 3528, 0, 7653, 5, 1281, 0, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 1423, 0, 0, 0, 3, 521, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 5572, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 144, 0, 0, 0, 0, 4256, 0, 0, 8183, 0, 0, 2, 0, 0, 0, 0, 0, 0, 624, 0, 0, 0, 0, 59, 0, 2101, 0, 74, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 7962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 7907, 0, 0, 0, 0, 0, 0, 8190, 0, 3, 0, 0, 0, 8192, 0, 8102, 0, 123, 0, 0, 0, 0, 589, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 146, 179, 60, 0, 2, 0, 0, 8, 0, 0, 0, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8183, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 67, 0, 14, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 5, 7336, 0, 7948, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 3984, 7, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 16, 4064, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1, 0, 0, 12, 0, 1, 0, 3785, 39, 174, 102, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 0, 0, 0, 0, 0, 7935, 3, 0, 160, 0, 189, 3266, 0, 7992, 0, 1, 0, 0, 0, 0, 349, 0, 0, 0, 0, 48, 0, 5352, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7715, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 5, 0, 4685, 89, 3, 8157, 0, 0, 0, 7, 2, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 10, 7793, 0, 0, 2, 742, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 304, 0, 1, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4617, 0, 1, 2, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 1652, 0, 7836, 0, 3121, 0, 0, 2, 0, 3, 0, 0, 0, 6, 0, 0, 6369, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 22, 0, 0, 0, 1, 0, 0, 0, 2, 7782, 0, 6, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8172, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 535, 0, 0, 0, 0, 3, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 88, 0, 0, 7, 0, 2, 3, 0, 18, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 7601, 0, 0, 0, 0, 0, 0, 8056, 4, 0, 0, 0, 33, 8123, 0, 7889, 0, 114, 0, 0, 0, 0, 26, 0, 16, 0, 1, 0, 0, 8170, 0, 0, 8, 0, 62, 0, 0, 0, 0, 4, 1, 9, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 15, 0, 84, 3, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 7, 0, 0, 4001, 0, 0, 8070, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5758, 0, 0, 0, 3, 0, 0, 4144, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 8004, 0, 144, 0, 0, 2, 0, 8180, 0, 0, 0, 0, 0, 5909, 0, 8169, 0, 3161, 3, 16, 0, 0, 95, 0, 19, 6, 3, 0, 0, 3664, 0, 0, 0, 0, 0, 448, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 67, 0, 1872, 0, 0, 0, 0, 72, 0, 12, 0, 161, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4617, 0, 0, 2450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 64, 8147, 2, 4, 1, 0, 1, 27, 1585, 0, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 7, 0, 72, 0, 0, 0, 0, 0, 1500, 5699, 252, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 7497, 0, 6, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 3528, 0, 2215, 5, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 1423, 0, 0, 0, 3, 521, 513, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5572, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7092, 10053, 7637, 8437, 7049, 6636, 8107, 10525] avg=8192.0000 max=10525 max_rank=7 min=6636 min_rank=5 imbalance_ratio=1.284790 max_load_reduction=55011 max_load_reduction_pct=0.839401 moved_to_echo=32317 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8183, 0, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 67, 5724, 261, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8174, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7336, 0, 2301, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 9, 118, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10644, 8758, 6120, 8315, 9848, 8525, 8131, 5195] avg=8192.0000 max=10644 max_rank=0 min=5195 min_rank=7 imbalance_ratio=1.299316 max_load_reduction=54892 max_load_reduction_pct=0.837585 moved_to_echo=31699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8172, 0, 0, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 535, 0, 0, 0, 0, 3, 0, 4906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 88, 0, 0, 7, 0, 2, 3, 0, 18, 0, 5728, 259, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 7601, 0, 0, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 33, 8123, 0, 2284, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 16, 0, 1, 0, 0, 8170, 0, 0, 8, 0, 62, 1589, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 9, 0, 0, 0, 0, 0, 2, 0, 0, 1, 7995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 84, 3, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10635, 5445, 6168, 9389, 10558, 10149, 8012, 5180] avg=8192.0000 max=10635 max_rank=0 min=5180 min_rank=7 imbalance_ratio=1.298218 max_load_reduction=54901 max_load_reduction_pct=0.837723 moved_to_echo=33899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 7710, 0, 0, 2440, 1, 9, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 6960, 0, 0, 0, 0, 0, 69, 4887, 0, 24, 13, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 245, 5676, 255, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 0, 0, 0, 13, 0, 6, 0, 0, 0, 1, 1, 5989, 0, 0, 1677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 735, 0, 2243, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 1583, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 47, 1, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10166, 12550, 6241, 9402, 3014, 10046, 8124, 5993] avg=8192.0000 max=12550 max_rank=1 min=3014 min_rank=4 imbalance_ratio=1.531982 max_load_reduction=52986 max_load_reduction_pct=0.808502 moved_to_echo=27177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 3984, 7, 0, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 16, 4064, 0, 0, 0, 0, 0, 0, 4876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1, 0, 0, 12, 0, 1, 0, 3785, 5729, 263, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 174, 102, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 0, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 160, 0, 189, 3266, 0, 2313, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 48, 0, 5352, 0, 0, 0, 0, 45, 1579, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7715, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6460, 8959, 9923, 5873, 5932, 7654, 7876, 12859] avg=8192.0000 max=12859 max_rank=7 min=5873 min_rank=3 imbalance_ratio=1.569702 max_load_reduction=52677 max_load_reduction_pct=0.803787 moved_to_echo=34321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 144, 0, 0, 0, 0, 4256, 0, 0, 2460, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 624, 0, 0, 0, 0, 59, 0, 1261, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 7962, 5723, 266, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 7907, 0, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 2345, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 1, 408, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 179, 60, 0, 2, 0, 0, 8, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6862, 2018, 14028, 8408, 10663, 9452, 8523, 5582] avg=8192.0000 max=14028 max_rank=2 min=2018 min_rank=1 imbalance_ratio=1.712402 max_load_reduction=51508 max_load_reduction_pct=0.785950 moved_to_echo=34220 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 5, 0, 4685, 89, 3, 2452, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 10, 4675, 0, 0, 2, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5705, 258, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 1, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4617, 0, 1, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 0, 2267, 0, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 6, 0, 0, 6369, 0, 0, 0, 0, 0, 1514, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 22, 0, 0, 0, 1, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7782, 0, 6, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7244, 9514, 6032, 6573, 7042, 8170, 8127, 12834] avg=8192.0000 max=12834 max_rank=7 min=6032 min_rank=2 imbalance_ratio=1.566650 max_load_reduction=52702 max_load_reduction_pct=0.804169 moved_to_echo=36173 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 7, 0, 0, 4001, 0, 0, 2423, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5758, 0, 0, 0, 3, 0, 0, 2477, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3398, 5647, 272, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 8004, 0, 144, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5909, 0, 2355, 0, 3161, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 95, 0, 19, 6, 3, 0, 0, 3664, 0, 0, 0, 0, 0, 810, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 67, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1872, 0, 0, 0, 0, 72, 0, 12, 0, 161, 0, 0, 0, 0, 0, 5251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6433, 8239, 9387, 9139, 11430, 4904, 8636, 7368] avg=8192.0000 max=11430 max_rank=4 min=4904 min_rank=5 imbalance_ratio=1.395264 max_load_reduction=54106 max_load_reduction_pct=0.825592 moved_to_echo=32338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 256, 128, 128, 128, 0, 45696, 128, 128, 19712, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 23552, 128, 1024, 128, 0, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 40704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 256, 128, 256, 38784, 0, 18432, 128, 8064, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 128, 0, 256, 128, 128, 128, 0, 128, 128, 128, 128, 17024, 45696, 2176, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 5120, 0, 128, 128, 128, 128, 0, 49408, 0, 0, 128, 128, 640, 8192, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 0, 128, 38400, 128, 128, 128, 128, 128, 128, 25088, 0, 128, 128, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 2688, 128, 128, 128, 128, 0, 128, 128, 0, 384, 128, 128, 53760, 0, 256, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[512, 128, 128, 128, 128, 128, 256, 256, 128, 128, 128, 0, 128, 128, 128, 128, 64512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[5125, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 17, 21, 22, 0, 0, 0, 0, 0, 1, 2295, 0, 0, 0, 0, 0, 116, 0, 0, 0, 1, 232, 0, 0, 16, 554, 7539, 1, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 36, 17, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4, 0, 3, 0, 0, 0, 0, 3470, 0, 38, 0, 0, 0, 0, 5, 0, 5, 0, 725, 129, 0, 0, 0, 7789, 0, 7, 0, 8066, 5173, 0, 1, 0, 0, 0, 0, 0, 0, 3280, 3, 0, 0, 8137, 8154, 0, 87, 0, 1, 0, 0, 4438, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[368, 404, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6392, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 156, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7774, 0, 513, 3194, 5, 6959, 2, 14, 109, 8192, 49, 0, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1376, 7483, 0, 0, 0, 44, 0, 0, 3150, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[5479, 543, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8, 0, 0, 0, 33, 3, 0, 0, 0, 0, 0, 0, 36, 8127, 0, 0, 0, 0, 0, 49, 3, 1, 49, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4792, 100, 0, 0, 1, 6243, 0, 20, 0, 7237, 5271, 0, 114, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8058, 8044, 0, 0, 0, 0, 0, 3, 3002, 1688, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[6299, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 319, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7938, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7204, 0, 0, 275, 0, 0, 0, 0, 0, 8192, 143, 0, 624, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 8185, 8027, 0, 530, 0, 0, 0, 0, 7847, 1423, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 7874, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 195, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 0, 808, 16, 0, 5126, 0, 0, 0, 8192, 50, 0, 1469, 0, 0, 0, 0, 0, 0, 2434, 0, 0, 0, 8186, 7414, 0, 0, 9, 0, 0, 0, 8097, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 28, 12, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4554, 0, 67, 0, 2, 0, 95, 0, 91, 0, 0, 1, 0, 0, 0, 0, 432, 0, 0, 51, 1, 36, 0, 0, 0, 0, 0, 0, 1, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6806, 16, 0, 0, 3, 0, 0, 0, 0, 3, 0, 8135, 0, 0, 1411, 0, 507, 0, 0, 0, 8156, 0, 0, 292, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 7733, 1841, 0, 56, 551, 154, 0, 0, 8105, 1, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[158, 2485, 0, 0, 57, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 285, 0, 610, 0, 0, 0, 52, 0, 0, 0, 31, 0, 1, 0, 0, 7973, 0, 96, 17, 0, 0, 0, 0, 1, 1, 0, 0, 3, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3437, 0, 0, 11, 0, 11, 0, 0, 6, 0, 0, 8060, 0, 0, 0, 0, 2762, 0, 0, 0, 8127, 15, 46, 0, 0, 0, 117, 0, 0, 0, 10, 0, 0, 0, 8147, 8017, 0, 0, 1210, 0, 3, 0, 5517, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 411, 0, 0, 0, 0, 2, 0, 28, 45, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 534, 495, 0, 2, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 234, 4679, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 778, 0, 7634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 121, 1, 4, 0, 60, 0, 0, 6668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2118, 0, 47, 133, 0, 35, 0, 29, 0, 7608, 138, 0, 7246, 0, 1, 0, 1, 0, 0, 1317, 0, 0, 0, 8107, 7848, 0, 0, 617, 48, 0, 0, 8098, 2, 0, 0, 297, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2074, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7874, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 5841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7597, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 0, 808, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 0, 595, 50, 0, 1469, 0, 0, 0, 0, 0, 0, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 0, 9, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4310, 13809, 9696, 5311, 8467, 4786, 9674, 9483] avg=8192.0000 max=13809 max_rank=1 min=4310 min_rank=0 imbalance_ratio=1.685669 max_load_reduction=51727 max_load_reduction_pct=0.789291 moved_to_echo=32410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[368, 404, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 3050, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6392, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 156, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 3866, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 7597, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7774, 0, 513, 3194, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6959, 2, 14, 109, 595, 49, 0, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1391, 0, 0, 0, 44, 0, 0, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 7540, 10136, 4407, 7751, 14507, 10508, 4585] avg=8192.0000 max=14507 max_rank=5 min=4407 min_rank=3 imbalance_ratio=1.770874 max_load_reduction=51029 max_load_reduction_pct=0.778641 moved_to_echo=37351 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5125, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 17, 21, 22, 0, 0, 1316, 2426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2295, 0, 0, 0, 0, 0, 116, 0, 0, 0, 1, 232, 5807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 554, 7539, 1, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 17, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4213, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4, 0, 3, 0, 0, 0, 0, 7480, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1305, 0, 38, 0, 0, 0, 0, 5, 0, 5, 0, 725, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7789, 0, 7, 0, 586, 5173, 0, 1, 0, 0, 0, 0, 0, 0, 3280, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 0, 87, 0, 1, 0, 0, 4438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8928, 8452, 8971, 5738, 8360, 2207, 16839, 6041] avg=8192.0000 max=16839 max_rank=6 min=2207 min_rank=5 imbalance_ratio=2.055542 max_load_reduction=48697 max_load_reduction_pct=0.743057 moved_to_echo=33447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6299, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 319, 0, 0, 0, 4, 3012, 2388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 7, 0, 0, 0, 5840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4147, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7597, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2983, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7204, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 143, 0, 624, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 0, 530, 0, 0, 0, 0, 7847, 1423, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12023, 5864, 9248, 6212, 8470, 10464, 1961, 11294] avg=8192.0000 max=12023 max_rank=0 min=1961 min_rank=6 imbalance_ratio=1.467651 max_load_reduction=53513 max_load_reduction_pct=0.816544 moved_to_echo=32189 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 28, 12, 2, 8192, 0, 0, 0, 0, 0, 0, 2582, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4554, 0, 67, 0, 2, 0, 95, 0, 91, 0, 0, 1, 5519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 0, 51, 1, 36, 0, 0, 0, 0, 0, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 951, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7563, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2558, 16, 0, 0, 3, 0, 0, 0, 0, 3, 0, 8135, 0, 0, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 0, 0, 593, 0, 0, 292, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 56, 551, 154, 0, 0, 8105, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11363, 10329, 2186, 10405, 8392, 12126, 1522, 9213] avg=8192.0000 max=12126 max_rank=5 min=1522 min_rank=6 imbalance_ratio=1.480225 max_load_reduction=53410 max_load_reduction_pct=0.814972 moved_to_echo=31253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5479, 543, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2403, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8, 0, 0, 0, 33, 3, 0, 0, 0, 5749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 8127, 0, 0, 0, 0, 0, 49, 3, 1, 49, 0, 0, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 4156, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6711, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4792, 100, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6243, 0, 20, 0, 526, 5271, 0, 114, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 0, 0, 0, 3, 3002, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10825, 5796, 9815, 5885, 7580, 7272, 12175, 6188] avg=8192.0000 max=12175 max_rank=6 min=5796 min_rank=1 imbalance_ratio=1.486206 max_load_reduction=53361 max_load_reduction_pct=0.814224 moved_to_echo=33215 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[158, 2485, 0, 0, 57, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 234, 1304, 2385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 610, 0, 0, 0, 52, 0, 0, 0, 31, 5813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7973, 0, 96, 17, 0, 0, 0, 0, 1, 1, 0, 0, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4142, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7536, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 0, 0, 11, 0, 11, 0, 0, 6, 0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2762, 0, 0, 0, 591, 15, 46, 0, 0, 0, 117, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1490, 0, 0, 1210, 0, 3, 0, 5517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6624, 6791, 8930, 13646, 8404, 9380, 3541, 8220] avg=8192.0000 max=13646 max_rank=3 min=3541 min_rank=6 imbalance_ratio=1.665771 max_load_reduction=51890 max_load_reduction_pct=0.791779 moved_to_echo=29545 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 411, 0, 0, 0, 0, 2, 0, 28, 45, 1, 1, 0, 0, 0, 0, 2535, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 534, 495, 0, 2, 0, 0, 138, 0, 0, 0, 0, 0, 5785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 4679, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 7634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 1460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 121, 1, 4, 0, 60, 0, 0, 7059, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2118, 0, 47, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 29, 0, 549, 138, 0, 7246, 0, 1, 0, 1, 0, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1450, 0, 0, 617, 48, 0, 0, 8098, 2, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5361, 6955, 6554, 13932, 8112, 4794, 9316, 10512] avg=8192.0000 max=13932 max_rank=3 min=4794 min_rank=5 imbalance_ratio=1.700684 max_load_reduction=51604 max_load_reduction_pct=0.787415 moved_to_echo=32734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[17536, 3968, 0, 128, 128, 0, 128, 128, 128, 8320, 128, 384, 128, 128, 0, 256, 18304, 17024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[29440, 128, 128, 128, 4736, 10880, 128, 12544, 0, 128, 128, 128, 128, 0, 7808, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[896, 0, 24832, 128, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 29440, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[18176, 128, 128, 128, 128, 128, 0, 128, 128, 128, 0, 40832, 256, 1408, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 10624, 0, 768, 2432, 256, 128, 128, 48256, 3200, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 21760, 896, 128, 640, 128, 128, 512, 128, 128, 0, 128, 512, 41344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 1024, 52480, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 0, 128, 128, 128, 128, 0, 128, 0, 128, 59264, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 8189, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8168, 8141, 0, 0, 0, 2, 0, 0, 0, 0, 861, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2664, 41, 7, 0, 0, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7926, 4, 1, 0, 6482, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7091, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2, 0, 0, 7338] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8181, 0, 0, 8117, 74, 0, 0, 0, 0, 0, 8077, 8151, 0, 0, 0, 1966, 2, 0, 3, 0, 0, 0, 0, 1, 2, 0, 0, 14, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7879, 2, 0, 0, 0, 1, 10, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3, 16, 0, 6970, 585, 0, 3, 4512, 1, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 7633, 0, 1, 0, 0, 0, 198, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 22, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 459, 228, 2209] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8143, 8189, 0, 0, 0, 1375, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8192, 2, 0, 0, 4346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6461] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8160, 0, 0, 8175, 0, 0, 0, 242, 0, 6, 7962, 8179, 359, 0, 0, 3, 0, 4, 0, 1, 0, 16, 0, 0, 2, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 12, 0, 6, 0, 7647, 3, 0, 0, 0, 0, 0, 0, 20, 0, 11, 0, 59, 0, 0, 0, 0, 6, 0, 8184, 8, 0, 0, 11, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6336, 0, 1, 0, 0, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 7669] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8180, 0, 0, 8184, 0, 0, 0, 1, 0, 0, 8153, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 1528, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 21, 0, 0, 8181, 580, 0, 0, 7815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7855, 0, 0, 35, 0, 6, 5566, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 232, 0, 0, 3, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 13, 21, 0, 0, 0, 0, 0, 60, 835] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8164, 0, 0, 7381, 0, 0, 0, 21, 0, 0, 8192, 8179, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 412, 0, 0, 0, 6892, 0, 0, 0, 2, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 7787, 2268, 5, 0, 116, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 758, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6113] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8107, 0, 0, 7943, 0, 7, 0, 0, 0, 0, 5072, 8049, 3, 2, 122, 2859, 0, 0, 0, 0, 26, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 1, 0, 0, 0, 8001, 0, 0, 1, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 30, 0, 9, 0, 8111, 0, 0, 0, 7525, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 1, 0, 0, 0, 13, 8130, 0, 671, 61, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 23, 1, 0, 4, 0, 0, 0, 161, 181, 151, 0, 2, 0, 0, 0, 0, 0, 76, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 7882, 0, 0, 6268, 0, 0, 0, 0, 0, 0, 8037, 7980, 248, 0, 0, 0, 0, 0, 0, 0, 6865, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 213, 0, 42, 119, 0, 20, 0, 0, 0, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1826, 1, 8190, 733, 6, 0, 6395, 0, 5, 27, 922, 0, 0, 0, 0, 0, 26, 0, 0, 0, 6, 3, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 276] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 348, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 850, 0, 0, 0, 0, 6241, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3518, 3, 0, 0, 0, 1957, 659, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 7533, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6461, 2313, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8491, 8583, 6754, 7545, 7911, 7453, 8076, 10723] avg=8192.0000 max=10723 max_rank=7 min=6754 min_rank=2 imbalance_ratio=1.308960 max_load_reduction=54813 max_load_reduction_pct=0.836380 moved_to_echo=34163 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 348, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 861, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 6204, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2664, 41, 7, 0, 0, 1958, 638, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7288, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6482, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6559, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2, 0, 0, 7338, 2314, 1937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8517, 7211, 5846, 7626, 10066, 6582, 8084, 11604] avg=8192.0000 max=11604 max_rank=7 min=5846 min_rank=2 imbalance_ratio=1.416504 max_load_reduction=53932 max_load_reduction_pct=0.822937 moved_to_echo=36336 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 345, 74, 0, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1966, 2, 0, 3, 0, 0, 0, 0, 1, 2, 0, 0, 14, 34, 3, 0, 6213, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7879, 2, 0, 0, 0, 1942, 561, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3, 16, 0, 6409, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4512, 1, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 22, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7060, 0, 1, 0, 0, 0, 198, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 459, 228, 2209, 2294, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8496, 8355, 10957, 7030, 8162, 7271, 8065, 7200] avg=8192.0000 max=10957 max_rank=2 min=7030 min_rank=3 imbalance_ratio=1.337524 max_load_reduction=54579 max_load_reduction_pct=0.832809 moved_to_echo=30698 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 349, 0, 0, 0, 242, 0, 6, 7962, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 1, 0, 16, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6234, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 12, 0, 6, 0, 7647, 3, 0, 0, 0, 1955, 659, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 11, 0, 59, 0, 0, 0, 0, 6, 0, 7525, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5861, 0, 1, 0, 0, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 598, 0, 0, 0, 8043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 7669, 2310, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8918, 6377, 10889, 7629, 3574, 7505, 8642, 12002] avg=8192.0000 max=12002 max_rank=7 min=3574 min_rank=4 imbalance_ratio=1.465088 max_load_reduction=53534 max_load_reduction_pct=0.816864 moved_to_echo=31723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 315, 0, 0, 0, 21, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6234, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 412, 0, 0, 0, 6892, 0, 0, 0, 2, 1765, 627, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 7160, 2268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 116, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 1, 7576, 0, 0, 0, 0, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6113, 2086, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8528, 6439, 10325, 9984, 3337, 8551, 8226, 10146] avg=8192.0000 max=10325 max_rank=2 min=3337 min_rank=4 imbalance_ratio=1.260376 max_load_reduction=55211 max_load_reduction_pct=0.842453 moved_to_echo=30260 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 348, 0, 0, 0, 1, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 6205, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 1528, 0, 0, 0, 0, 1958, 658, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 21, 0, 0, 7523, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7266, 0, 0, 35, 0, 6, 5566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 232, 0, 0, 3, 0, 0, 0, 0, 0, 36, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 21, 0, 0, 0, 0, 0, 60, 835, 2313, 1937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8502, 6373, 4748, 8131, 11380, 12874, 8349, 5179] avg=8192.0000 max=12874 max_rank=5 min=4748 min_rank=2 imbalance_ratio=1.571533 max_load_reduction=52662 max_load_reduction_pct=0.803558 moved_to_echo=37782 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 8037, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6865, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6081, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 42, 119, 0, 20, 0, 0, 0, 0, 1614, 0, 0, 0, 0, 1499, 659, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1826, 1, 7531, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6395, 0, 5, 27, 922, 0, 0, 0, 0, 0, 26, 0, 0, 0, 2730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 7020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 7769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 276, 1771, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8553, 13061, 4736, 10091, 10111, 7029, 8006, 3949] avg=8192.0000 max=13061 max_rank=1 min=3949 min_rank=7 imbalance_ratio=1.594360 max_load_reduction=52475 max_load_reduction_pct=0.800705 moved_to_echo=29095 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 325, 0, 7, 0, 0, 0, 0, 5072, 0, 3, 2, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2859, 0, 0, 0, 0, 26, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6135, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 1, 0, 0, 0, 8001, 0, 0, 1, 0, 1905, 657, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 30, 0, 9, 0, 7454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 3465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 13, 7516, 0, 671, 61, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 23, 1, 0, 4, 7991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 181, 151, 0, 2, 0, 0, 0, 0, 0, 76, 0, 0, 2248, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5531, 9137, 11281, 7500, 10995, 8271, 8088, 4733] avg=8192.0000 max=11281 max_rank=2 min=4733 min_rank=7 imbalance_ratio=1.377075 max_load_reduction=54255 max_load_reduction_pct=0.827866 moved_to_echo=32087 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 256, 256, 256, 0, 256, 128, 0, 0, 0, 128, 640, 384, 30976, 17664, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[256, 256, 128, 128, 256, 0, 128, 512, 128, 128, 0, 39808, 128, 128, 128, 128, 14976, 5120, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 37248, 128, 128, 128, 1024, 0, 0, 128, 128, 0, 128, 0, 128, 128, 27264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[6400, 128, 128, 128, 128, 7808, 128, 0, 128, 128, 128, 896, 128, 128, 128, 0, 49664, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 0, 0, 128, 56320, 0, 768, 128, 0, 1792, 6656, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 2688, 128, 128, 0, 384, 0, 128, 61824, 0, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 768, 128, 128, 128, 0, 128, 128, 0, 128, 128, 2176, 128, 58496, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 384, 0, 256, 128, 0, 256, 640, 128, 0, 128, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 1, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8161, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 45, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7271, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 7952, 0, 0, 0, 8129, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[8170, 0, 0, 0, 2, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 8, 0, 0, 5, 0, 0, 0, 0, 0, 62, 0, 3, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 8132, 0, 0, 0, 44, 8032, 8176, 1, 5, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 37, 35, 0, 12, 0, 0, 0, 0, 0, 0, 59, 8153, 0, 20, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 78, 0, 8106, 0, 15, 0, 7990, 0, 0, 0, 0, 8086, 0, 0, 0, 9, 0, 0, 0, 0, 122, 0, 0, 0, 34, 0, 1, 0, 0, 0, 0, 17, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8130, 6246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1841, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 8163, 0, 0, 0, 0, 7994, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[8027, 0, 0, 161, 0, 0, 0, 1, 0, 3036, 0, 0, 4, 0, 85, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 5, 0, 2, 0, 0, 0, 47, 0, 37, 12, 0, 0, 8169, 0, 15, 2, 8, 8035, 8172, 0, 0, 0, 0, 0, 0, 54, 0, 20, 0, 0, 0, 0, 0, 0, 0, 526, 104, 0, 255, 0, 0, 0, 0, 1, 0, 0, 4, 7557, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7713, 0, 0, 2, 5176, 0, 0, 99, 0, 7923, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 200, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 5, 0, 1125, 0, 0, 6, 1, 5, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 8154, 7136, 2, 3, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 38, 120, 0, 0, 0, 0, 0, 241, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 4, 0, 0, 7658, 0, 0, 1, 0, 8150, 0, 0, 0, 0, 2, 0, 0, 58, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[7922, 0, 0, 0, 0, 0, 0, 7611, 0, 0, 101, 0, 0, 0, 0, 21, 0, 0, 13, 0, 100, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 66, 231, 0, 0, 0, 0, 0, 0, 7988, 0, 0, 0, 0, 5666, 7542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 166, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1526, 6, 8099, 0, 0, 0, 605, 0, 0, 0, 9, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 27, 0, 0, 2, 0, 6, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[8179, 0, 0, 0, 0, 0, 0, 4641, 22, 39, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5, 0, 0, 0, 0, 0, 6, 0, 570, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 8175, 7619, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 74, 1115, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 8055, 0, 0, 0, 1471, 15, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 2, 0, 623, 1, 114, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[8160, 0, 0, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6904, 0, 7, 0, 0, 321, 0, 2, 8156, 0, 0, 0, 39, 4624, 3565, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 1614, 0, 0, 0, 2, 0, 4, 0, 0, 0, 8150, 0, 7, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 7, 7188, 0, 0, 0, 8152, 0, 0, 92, 0, 8082, 5, 0, 3, 0, 0, 0, 0, 0, 13, 0, 0, 0, 15, 0, 0, 0, 0, 6, 4, 3, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6007, 0, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8129, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 7266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 6246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1841, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4063, 2185, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 2213, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 6762, 1718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6293, 8271, 7323, 7473, 9017, 8184, 10383, 8592] avg=8192.0000 max=10383 max_rank=6 min=6293 min_rank=0 imbalance_ratio=1.267456 max_load_reduction=55153 max_load_reduction_pct=0.841568 moved_to_echo=36176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6007, 0, 0, 0, 0, 0, 0, 1, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7216, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4163, 2185, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 3, 0, 0, 6788, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6434, 7358, 7266, 9185, 7554, 8036, 10396, 9307] avg=8192.0000 max=10396 max_rank=6 min=6434 min_rank=0 imbalance_ratio=1.269043 max_load_reduction=55140 max_load_reduction_pct=0.841370 moved_to_echo=35293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5991, 0, 0, 0, 2, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 5, 0, 0, 0, 0, 0, 62, 0, 3, 0, 0, 16, 0, 8091, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 7213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1211, 8176, 1, 5, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 37, 35, 0, 12, 0, 0, 0, 0, 0, 0, 59, 4110, 2179, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 20, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 78, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 7990, 0, 0, 0, 0, 2238, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 34, 0, 1, 0, 0, 0, 0, 17, 0, 0, 0, 6681, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6048, 8325, 7218, 9475, 7355, 8270, 10252, 8593] avg=8192.0000 max=10252 max_rank=6 min=6048 min_rank=0 imbalance_ratio=1.251465 max_load_reduction=55284 max_load_reduction_pct=0.843567 moved_to_echo=34470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6006, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 5, 0, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 5, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8067, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1230, 7136, 2, 3, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 120, 0, 0, 0, 0, 0, 241, 0, 0, 0, 4142, 2185, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 7658, 0, 0, 1, 0, 2256, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6782, 1752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7352, 8226, 7264, 8376, 7651, 8148, 9921, 8598] avg=8192.0000 max=9921 max_rank=6 min=7264 min_rank=2 imbalance_ratio=1.211060 max_load_reduction=55615 max_load_reduction_pct=0.848618 moved_to_echo=34318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5983, 0, 0, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6904, 0, 7, 0, 0, 321, 0, 2, 7234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 698, 3565, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 1614, 0, 0, 0, 2, 0, 4, 0, 0, 0, 4108, 2177, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 7, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 7, 7188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 92, 0, 2237, 5, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 15, 0, 0, 0, 0, 6, 4, 3, 0, 0, 3, 3846, 1737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5994, 8453, 14476, 4314, 8881, 7302, 10489, 5627] avg=8192.0000 max=14476 max_rank=2 min=4314 min_rank=3 imbalance_ratio=1.767090 max_load_reduction=51060 max_load_reduction_pct=0.779114 moved_to_echo=32299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5997, 0, 0, 0, 0, 0, 0, 4641, 22, 39, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5, 0, 0, 0, 0, 0, 6, 0, 570, 8127, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 7241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1233, 7619, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 74, 1115, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4152, 2182, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 15, 0, 0, 0, 2261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 2, 0, 623, 1, 114, 0, 0, 6800, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10705, 8854, 7247, 8858, 8587, 8137, 3747, 9401] avg=8192.0000 max=10705 max_rank=0 min=3747 min_rank=6 imbalance_ratio=1.306763 max_load_reduction=54831 max_load_reduction_pct=0.836655 moved_to_echo=29872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5881, 0, 0, 161, 0, 0, 0, 1, 0, 3036, 0, 0, 4, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 7505, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 47, 0, 37, 12, 0, 0, 7241, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 1202, 8172, 0, 0, 0, 0, 0, 0, 54, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 104, 0, 255, 0, 0, 0, 0, 1, 0, 0, 4, 4031, 2146, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5176, 0, 0, 99, 0, 2184, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 200, 0, 6688, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9168, 7699, 7359, 9458, 7995, 7766, 7470, 8621] avg=8192.0000 max=9458 max_rank=3 min=7359 min_rank=2 imbalance_ratio=1.154541 max_load_reduction=56078 max_load_reduction_pct=0.855682 moved_to_echo=31852 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5809, 0, 0, 0, 0, 0, 0, 7611, 0, 0, 101, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 100, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 9, 8128, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 66, 231, 0, 0, 0, 0, 0, 0, 7085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 855, 7542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 166, 4157, 2113, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1526, 6, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 0, 0, 0, 9, 2264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 27, 0, 0, 2, 0, 6, 0, 0, 0, 0, 1, 0, 4713, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13542, 8350, 7383, 8397, 8496, 9693, 2878, 6797] avg=8192.0000 max=13542 max_rank=0 min=2878 min_rank=6 imbalance_ratio=1.653076 max_load_reduction=51994 max_load_reduction_pct=0.793365 moved_to_echo=27864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 1408, 640, 640, 4864, 256, 128, 0, 128, 0, 256, 0, 0, 256, 33024, 17408, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 47360, 128, 0, 256, 128, 17920, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 256, 0, 384, 128, 128, 128, 128, 128, 128, 128, 1536, 128, 128, 256, 128, 49152, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[47744, 0, 0, 256, 128, 0, 128, 12288, 128, 4096, 128, 0, 128, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 8960, 56448, 128, 128, 0, 0, 128, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 7040, 256, 128, 0, 128, 384, 128, 128, 57856, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 1664, 128, 63360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 384, 128, 128, 128, 0, 128, 0, 128, 128, 128, 128, 128, 128, 640, 63360, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 937, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 7863, 0, 6, 0, 0, 0, 0, 6476, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 81, 101, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 8192, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 50, 0, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7043, 2, 0, 2, 0, 0, 0, 4935, 0, 0, 6599, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 8140, 0, 1, 0, 0, 0, 0, 12, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 8184, 11, 0, 5049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 0, 13, 0, 5, 0, 0, 0, 0, 0, 4, 0, 0, 20, 0, 0, 0, 0, 0, 106, 0, 374, 0, 0, 0, 0, 0, 4, 188, 0, 0, 7064, 0, 0, 26, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 75, 0, 0, 11, 0, 0, 0, 0, 0, 7, 0, 4629, 0, 10, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2338, 0, 0, 0, 0, 8014, 0, 0, 2351, 0, 12, 1, 7406, 1, 0, 0, 0, 0, 218, 330, 0, 0, 0, 0, 0, 0, 8192, 0, 8150, 1, 0, 7983, 0, 2, 0, 0, 0, 0, 11, 0, 0, 0, 4, 0, 105, 0, 0, 0, 7787, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 653, 0, 0, 0, 0, 167, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3, 25, 0, 0, 8040, 0, 12, 0, 0, 0, 0, 62, 0, 0, 7888, 0, 0, 13, 0, 0, 0, 1, 32, 0, 0, 6, 0, 344, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 358, 0, 2, 0, 0, 0, 1, 0, 2791, 0, 0, 0, 0, 0, 2, 1, 0, 126, 0, 0, 0, 4, 4, 65, 0, 0, 8058, 0, 0, 104, 0, 0, 13, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4830, 49, 8058, 0, 0, 8110, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1628, 0, 0, 0, 10, 0, 0, 0, 0, 13, 0, 14, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 7, 1608, 0, 142, 7680, 0, 0, 2, 0, 0, 0, 0, 0, 10, 29, 0, 0, 0, 0, 0, 0, 0, 74, 0, 14, 4, 0, 42, 0, 0, 1, 0, 0, 105, 0, 0, 184, 0, 540, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8175, 5, 0, 4763, 0, 1, 7, 484, 0, 0, 0, 0, 0, 1663, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 8185, 0, 0, 5596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3, 0, 0, 0, 0, 8033, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 21, 0, 0, 0, 55, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 0, 0, 3, 0, 0, 0, 0, 9, 0, 5139, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 192, 0, 106, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 9, 0, 2, 0, 0, 0, 7549, 0, 0, 6, 1, 0, 0, 2, 0, 129, 8102, 20, 0, 0, 0, 0, 117, 7463, 0, 55, 0, 0, 10, 0, 3, 0, 0, 7, 0, 0, 0, 8123, 0, 8174, 3, 0, 12, 19, 0, 0, 17, 0, 0, 0, 0, 0, 12, 296, 0, 0, 0, 0, 0, 8123, 30, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 3677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4826, 0, 22, 0, 0, 387, 0, 0, 0, 0, 0, 619, 0, 0, 3, 0, 117, 0, 90, 0, 0, 8192, 0, 0, 392, 0, 0, 106, 2948, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 8190, 0, 21, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1089, 0, 0, 197, 0, 0, 0, 6821, 2, 3, 7520, 0, 0, 0, 51, 2, 0, 0, 0, 0, 0, 0, 10, 3471, 0, 0, 0, 0, 0, 0, 26, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 14, 0, 7995, 0, 0, 0, 0, 0, 4987, 26, 0, 0, 25, 30, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8052, 0, 8102, 0, 0, 7658, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 937, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7863, 0, 6, 0, 0, 0, 0, 6476, 0, 0, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 5753, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 4205, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2997, 0, 0, 0, 0, 0, 81, 101, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1892, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 15161, 8283, 7023, 5178, 3220, 10063, 8359] avg=8192.0000 max=15161 max_rank=1 min=3220 min_rank=5 imbalance_ratio=1.850708 max_load_reduction=50375 max_load_reduction_pct=0.768661 moved_to_echo=26820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 50, 0, 5, 0, 0, 0, 0, 0, 0, 3, 7301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7043, 2, 0, 2, 0, 0, 0, 4935, 0, 0, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6599, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 345, 179, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 5785, 883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 8140, 0, 1, 0, 0, 0, 0, 12, 4205, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2997, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1903, 0, 0, 11, 0, 5049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7359, 12793, 7145, 6692, 12834, 3583, 6963, 8167] avg=8192.0000 max=12834 max_rank=4 min=3583 min_rank=5 imbalance_ratio=1.566650 max_load_reduction=52702 max_load_reduction_pct=0.804169 moved_to_echo=31547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 653, 0, 0, 0, 0, 167, 0, 0, 0, 0, 4, 0, 0, 0, 0, 7189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 25, 0, 0, 8040, 0, 12, 0, 0, 0, 0, 62, 0, 0, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 13, 0, 0, 0, 1, 32, 0, 0, 6, 0, 344, 0, 0, 176, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 358, 0, 2, 0, 0, 0, 1, 0, 3424, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 0, 2, 1, 0, 126, 0, 0, 0, 4, 4, 65, 4137, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2947, 0, 0, 104, 0, 0, 13, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1127, 49, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8013, 8940, 8477, 4663, 7409, 10696, 9292, 8046] avg=8192.0000 max=10696 max_rank=5 min=4663 min_rank=3 imbalance_ratio=1.305664 max_load_reduction=54840 max_load_reduction_pct=0.836792 moved_to_echo=35443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 7306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1077, 0, 0, 179, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4826, 0, 22, 5638, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 619, 0, 0, 3, 0, 117, 0, 90, 4205, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2997, 0, 0, 392, 0, 0, 106, 2948, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1854, 0, 0, 0, 21, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11028, 3158, 9451, 11370, 5880, 6445, 10032, 8172] avg=8192.0000 max=11370 max_rank=3 min=3158 min_rank=1 imbalance_ratio=1.387939 max_load_reduction=54166 max_load_reduction_pct=0.826508 moved_to_echo=30529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 0, 0, 0, 55, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 7292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 0, 0, 3, 0, 0, 0, 0, 9, 0, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5139, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 192, 0, 106, 177, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 5760, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 0, 0, 0, 7549, 0, 0, 6, 1, 0, 0, 2, 4159, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 2964, 20, 0, 0, 0, 0, 117, 7463, 0, 55, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 0, 0, 1894, 0, 0, 3, 0, 12, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 12, 296, 0, 0, 0, 0, 0, 8106, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7377, 4326, 5638, 14841, 12197, 10758, 1938, 8461] avg=8192.0000 max=14841 max_rank=3 min=1938 min_rank=6 imbalance_ratio=1.811646 max_load_reduction=50695 max_load_reduction_pct=0.773544 moved_to_echo=33354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1628, 0, 0, 0, 10, 0, 0, 0, 0, 13, 0, 7301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 7, 1608, 0, 142, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 0, 0, 2, 0, 0, 0, 0, 0, 10, 29, 0, 0, 0, 0, 0, 178, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 14, 4, 0, 42, 0, 0, 1, 0, 0, 105, 0, 0, 5755, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 540, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4197, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2991, 5, 0, 4763, 0, 1, 7, 484, 0, 0, 0, 0, 0, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1893, 0, 0, 0, 0, 5596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3, 0, 0, 0, 0, 8016, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8953, 10707, 7916, 6879, 5616, 9918, 7489, 8058] avg=8192.0000 max=10707 max_rank=1 min=5616 min_rank=4 imbalance_ratio=1.307007 max_load_reduction=54829 max_load_reduction_pct=0.836624 moved_to_echo=31081 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 13, 0, 5, 0, 0, 0, 0, 0, 4, 0, 0, 20, 0, 7270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 374, 0, 0, 0, 0, 0, 4, 188, 0, 0, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7064, 0, 0, 26, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 175, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 75, 0, 0, 11, 0, 0, 0, 0, 0, 7, 0, 5809, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4629, 0, 10, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2338, 0, 0, 4114, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2932, 0, 0, 2351, 0, 12, 1, 7406, 1, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 1910, 0, 0, 1, 0, 7983, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4, 0, 105, 0, 0, 0, 7770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7314, 1465, 7372, 6783, 11565, 12921, 10226, 7890] avg=8192.0000 max=12921 max_rank=5 min=1465 min_rank=1 imbalance_ratio=1.577271 max_load_reduction=52615 max_load_reduction_pct=0.802841 moved_to_echo=42602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 7228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1089, 0, 0, 197, 0, 0, 0, 6821, 2, 3, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 0, 51, 2, 0, 0, 0, 0, 0, 0, 10, 3471, 0, 0, 177, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5713, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 4109, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 2913, 0, 0, 0, 0, 0, 4987, 26, 0, 0, 25, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1871, 0, 0, 0, 0, 7658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7243, 8986, 11254, 7285, 4857, 7995, 9533, 8383] avg=8192.0000 max=11254 max_rank=2 min=4857 min_rank=4 imbalance_ratio=1.373779 max_load_reduction=54282 max_load_reduction_pct=0.828278 moved_to_echo=30768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 128, 38400, 128, 128, 256, 0, 0, 128, 20096, 128, 256, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 256, 23808, 128, 0, 7680, 0, 128, 5376, 26240, 128, 128, 128, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[7680, 0, 1280, 0, 128, 256, 128, 128, 16512, 128, 128, 128, 128, 2816, 128, 256, 33408, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[384, 0, 128, 128, 0, 0, 128, 14464, 128, 0, 128, 128, 50816, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 9344, 0, 128, 384, 0, 128, 0, 0, 5120, 128, 128, 43648, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 4480, 0, 128, 0, 2688, 256, 128, 128, 128, 128, 128, 0, 0, 128, 128, 58240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[58240, 0, 0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 5120, 0, 512, 1536, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 0, 256, 128, 512, 128, 128, 0, 0, 0, 64640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 7732, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 505, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 0, 0, 26, 8170, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7767, 0, 0, 0, 0, 0, 7757, 0, 3, 0, 0, 1725, 159, 0, 7, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 111, 5, 0, 0, 0, 0, 281, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 17, 8013, 0, 400, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7733, 0, 0, 0, 0, 0, 0, 7, 12, 1, 0, 8, 8192, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 6865, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7510, 0, 0, 0, 0, 0, 7189, 0, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8154, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5527, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 292, 3108, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 10, 0, 8165, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 4, 0, 5287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 9, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 8192, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 2, 140, 0, 0, 1, 0, 0, 0, 0, 6341, 3377, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 0, 7, 4, 0, 0, 0, 26, 0, 8186, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 7603, 0, 0, 35, 0, 0, 6557, 0, 6, 0, 0, 485, 0, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7446, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8176, 0, 0, 4, 0, 525, 6, 0, 0, 14, 0, 0, 8172, 0, 0, 0, 6, 93, 0, 0, 0, 0, 0, 0, 0, 0, 350, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 3, 6, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 43, 0, 0, 31, 6521, 0, 2, 0, 0, 0, 5811, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 1148, 0, 1, 0, 0, 0, 0, 0, 38, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 88, 0, 0, 315, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2, 0, 0, 8171, 0, 0, 0, 0, 13, 0, 5, 14, 0, 0, 0, 8174, 4, 18, 14, 0, 26, 0, 0, 0, 0, 0, 0, 0, 3173, 7020, 0, 0, 0, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 18, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 25, 0, 0, 0, 59, 0, 0, 0, 0, 8015, 316, 0, 0, 1, 0, 7407, 0, 0, 93, 0, 0, 0, 0, 4, 0, 0, 0, 5, 178, 0, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 580, 27, 0, 0, 0, 0, 312, 0, 0, 0, 7682, 0, 0, 3, 0, 2520, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 6766, 0, 0, 0, 13, 0, 0, 88, 0, 1, 0, 0, 8142, 0, 0, 0, 5, 1436, 0, 0, 0, 0, 74, 0, 0, 18, 5805, 0, 0, 0, 132] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 89, 8044, 116, 0, 0, 39, 0, 6019, 0, 2349, 26, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 72, 0, 0, 0, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5905, 1, 0, 3, 3, 22, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 11, 6, 0, 0, 0, 5, 0, 0, 8165, 0, 4, 11, 1, 260, 0, 0, 0, 19, 0, 0, 0, 8057, 1579, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 7685, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 213, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 0, 0, 26, 8170, 0, 0, 0, 0, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8225, 8590, 8041, 8493, 6076, 8575, 9339] avg=8192.0000 max=9339 max_rank=7 min=6076 min_rank=5 imbalance_ratio=1.140015 max_load_reduction=56197 max_load_reduction_pct=0.857498 moved_to_echo=32483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 0, 7189, 0, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 207, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5527, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 292, 3108, 0, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8136, 8339, 8037, 8489, 13586, 6036, 4662] avg=8192.0000 max=13586 max_rank=5 min=4662 min_rank=7 imbalance_ratio=1.658447 max_load_reduction=51950 max_load_reduction_pct=0.792694 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 10, 0, 8165, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7682, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 4, 0, 5287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 9, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 8188, 224, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 140, 0, 0, 1, 0, 0, 0, 0, 6341, 3377, 0, 0, 0, 3, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8172, 5694, 8077, 8505, 7078, 8726, 11043] avg=8192.0000 max=11043 max_rank=7 min=5694 min_rank=2 imbalance_ratio=1.348022 max_load_reduction=54493 max_load_reduction_pct=0.831497 moved_to_echo=35352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 0, 7, 4, 0, 0, 0, 26, 0, 8186, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 7666, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 0, 35, 0, 0, 6557, 0, 6, 0, 0, 485, 0, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7446, 210, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 4, 0, 525, 6, 0, 0, 14, 0, 0, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 93, 0, 0, 0, 0, 0, 0, 0, 0, 350, 1, 0, 0, 0, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8302, 8184, 9038, 7856, 7751, 13618, 9231, 1556] avg=8192.0000 max=13618 max_rank=5 min=1556 min_rank=7 imbalance_ratio=1.662354 max_load_reduction=51918 max_load_reduction_pct=0.792206 moved_to_echo=32156 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 18, 0, 0, 0, 0, 7656, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 25, 0, 0, 0, 59, 0, 0, 0, 0, 7638, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 316, 0, 0, 1, 0, 7407, 0, 0, 93, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 178, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 27, 0, 0, 0, 0, 312, 0, 0, 0, 7682, 221, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2520, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 5747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6766, 0, 0, 0, 13, 0, 0, 88, 0, 1, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1436, 0, 0, 0, 0, 74, 0, 0, 18, 5805, 0, 0, 0, 132, 1166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7740, 8237, 8219, 8120, 8910, 8302, 7372, 8636] avg=8192.0000 max=8910 max_rank=4 min=7372 min_rank=6 imbalance_ratio=1.087646 max_load_reduction=56626 max_load_reduction_pct=0.864044 moved_to_echo=33220 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 3, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 43, 0, 0, 31, 7668, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 2, 0, 0, 0, 5811, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 1148, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8167, 180, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 315, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2, 0, 4676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 13, 0, 5, 14, 0, 0, 0, 506, 4, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 26, 0, 0, 0, 0, 0, 0, 0, 3173, 7020, 0, 0, 0, 18, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8159, 6553, 9106, 8474, 5085, 8731, 11199] avg=8192.0000 max=11199 max_rank=7 min=5085 min_rank=5 imbalance_ratio=1.367065 max_load_reduction=54337 max_load_reduction_pct=0.829117 moved_to_echo=33489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8157, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7685, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 0, 0, 0, 0, 0, 7757, 0, 3, 0, 0, 1725, 159, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 111, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 17, 8013, 214, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7733, 0, 0, 0, 0, 0, 0, 7, 12, 1, 0, 8, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 6865, 0, 0, 0, 0, 0, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8178, 10037, 8228, 8628, 5973, 8268, 8000] avg=8192.0000 max=10037 max_rank=2 min=5973 min_rank=5 imbalance_ratio=1.225220 max_load_reduction=55499 max_load_reduction_pct=0.846848 moved_to_echo=30869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 120, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 89, 7666, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 116, 0, 0, 39, 0, 6019, 0, 2349, 26, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5905, 227, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 3, 22, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 11, 6, 0, 0, 0, 5, 0, 0, 499, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 260, 0, 0, 0, 19, 0, 0, 0, 8057, 1579, 0, 0, 0, 0, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8352, 8245, 9066, 8071, 6286, 5818, 8597, 11101] avg=8192.0000 max=11101 max_rank=7 min=5818 min_rank=5 imbalance_ratio=1.355103 max_load_reduction=54435 max_load_reduction_pct=0.830612 moved_to_echo=31802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 2176, 128, 0, 128, 128, 128, 0, 0, 24832, 29440, 128, 0, 0, 256, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 512, 128, 128, 20992, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 0, 44032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3072, 512, 128, 128, 128, 0, 54272, 0, 2432, 128, 0, 2688, 256, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 60800, 0, 0, 128, 128, 640, 128, 128, 128, 128, 0, 128, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 128, 128, 128, 128, 128, 0, 0, 128, 61440, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 384, 128, 640, 128, 128, 0, 128, 0, 384, 0, 128, 128, 61824, 1792, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 63872, 1152, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 64896, 128, 0, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 5569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 8192, 3479, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 3, 0, 0, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 7192, 0, 2, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 1, 0, 0, 903, 8088, 8, 0, 11, 269, 42, 0, 0, 0, 120, 0, 0, 0, 0, 7283, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 681, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 3200, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 2, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 13, 0, 0, 78, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 5, 0, 0, 0, 1, 8182, 0, 22, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 14, 0, 8068, 19, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7674, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 527, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 9, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 15, 4, 0, 0, 0, 0, 67, 0, 0, 7378, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 15, 2, 0, 0, 0, 0, 3, 6, 2, 0, 137, 4, 0, 0, 0, 0, 0, 0, 0, 7618, 8111, 1769, 0, 0, 0, 0, 0, 47, 0, 2681, 0, 279, 0, 0, 4326, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 27, 1, 1, 0, 5, 0, 4, 0, 4, 7614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8166, 866, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 0, 0, 21, 0, 0, 0, 0, 0, 0, 6181, 0, 602, 0, 0, 0, 0, 0, 22, 7978, 0, 0, 15, 7370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 73, 0, 0, 60, 0, 0, 0, 5859, 8171, 1, 0, 0, 0, 1, 0, 0, 0, 0, 12, 258, 0, 0, 8187, 0, 0, 5, 0, 0, 43, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2082, 1573, 0, 0, 0, 0, 0, 0, 1, 115, 0, 1, 6, 0, 1, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 19, 10, 0, 78] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1330, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8128, 2, 0, 0, 0, 5, 0, 0, 0, 8176, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 3, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 8000, 5226, 0, 0, 699, 54, 0, 0, 0, 862, 0, 0, 0, 0, 637, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 138, 0, 141, 0, 7860, 0, 108, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 9, 0, 0, 10, 8150, 0, 1, 0, 0, 0, 0, 0, 12, 0, 0, 7609, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 841, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 0, 0, 0, 0, 0, 0, 4, 0, 7090, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 69, 8172, 1199, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 242, 0, 0, 8137, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6641, 84, 0, 0, 0, 0, 7, 0, 1, 0, 0, 56, 0, 5104, 0, 0, 4, 8156, 0, 0, 0, 41, 0, 71, 0, 0, 1, 0, 8011, 4591, 0, 14, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 102, 0, 678, 0, 0, 0, 0, 823, 0, 0, 314, 0, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7789, 0, 0, 0, 7868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 7395, 0, 0, 0, 0, 75, 7989, 7995, 0, 0, 3376, 36, 0, 0, 0, 120, 0, 0, 0, 16, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4307, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 0, 0, 0, 0, 11, 0, 0, 28, 0, 0, 8159, 0, 0, 23, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 90, 0, 0, 0, 7536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 2930, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 3, 0, 0, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7884, 663, 259, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 5262, 1006, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7490, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7629, 7578, 8762, 6574, 10737, 9174, 7590, 7492] avg=8192.0000 max=10737 max_rank=4 min=6574 min_rank=3 imbalance_ratio=1.310669 max_load_reduction=54799 max_load_reduction_pct=0.836166 moved_to_echo=34993 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 0, 7501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 3155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 1, 0, 0, 903, 2893, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 269, 42, 0, 0, 0, 120, 0, 0, 0, 0, 6611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 681, 0, 8173, 664, 672, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 3200, 0, 0, 0, 5195, 890, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7505, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8562, 11355, 6579, 3889, 7053, 10604, 9986, 7508] avg=8192.0000 max=11355 max_rank=1 min=3889 min_rank=3 imbalance_ratio=1.386108 max_load_reduction=54181 max_load_reduction_pct=0.826736 moved_to_echo=35151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 13, 0, 0, 78, 0, 0, 0, 0, 0, 2929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 8182, 0, 22, 0, 0, 7392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 14, 0, 8068, 663, 752, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5258, 1005, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7494, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7065, 11722, 3240, 3025, 15602, 9885, 6896, 8101] avg=8192.0000 max=15602 max_rank=4 min=3025 min_rank=3 imbalance_ratio=1.904541 max_load_reduction=49934 max_load_reduction_pct=0.761932 moved_to_echo=40484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 0, 0, 21, 7478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 602, 0, 0, 0, 0, 0, 22, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 73, 0, 0, 60, 0, 0, 0, 5859, 2923, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 12, 258, 0, 0, 7431, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2082, 648, 756, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1573, 0, 0, 0, 0, 0, 0, 1, 115, 0, 1, 6, 0, 1, 0, 0, 5248, 987, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7316, 19, 10, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 10304, 10518, 8919, 7707, 3901, 8580, 7423] avg=8192.0000 max=10518 max_rank=2 min=3901 min_rank=5 imbalance_ratio=1.283936 max_load_reduction=55018 max_load_reduction_pct=0.839508 moved_to_echo=27611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1330, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 29, 0, 0, 0, 7500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8128, 2, 0, 0, 0, 5, 0, 0, 0, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 3, 0, 0, 0, 72, 0, 0, 3211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 2862, 5226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 699, 54, 0, 0, 0, 862, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 138, 0, 141, 0, 7860, 619, 58, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 9, 0, 0, 10, 5138, 1011, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 12, 0, 0, 6990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8909, 11733, 11451, 8132, 2194, 9185, 6929, 7003] avg=8192.0000 max=11733 max_rank=1 min=2194 min_rank=4 imbalance_ratio=1.432251 max_load_reduction=53803 max_load_reduction_pct=0.820969 moved_to_echo=25311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 9, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4, 7530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 4, 0, 0, 0, 0, 67, 0, 0, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 15, 2, 0, 0, 0, 0, 3, 2897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 137, 4, 0, 0, 0, 0, 0, 0, 0, 7618, 2902, 1769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 2681, 0, 279, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 27, 1, 1, 0, 5, 0, 4, 0, 4, 7614, 664, 399, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5209, 913, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 7502, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7553, 3323, 11081, 12438, 6934, 9052, 6777, 8378] avg=8192.0000 max=12438 max_rank=3 min=3323 min_rank=1 imbalance_ratio=1.518311 max_load_reduction=53098 max_load_reduction_pct=0.810211 moved_to_echo=31141 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 841, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 0, 0, 0, 0, 0, 0, 4, 0, 3099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 69, 2919, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 242, 0, 0, 7380, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6641, 656, 757, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 7, 0, 1, 0, 0, 56, 0, 5104, 0, 0, 4, 5253, 880, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 71, 0, 0, 1, 0, 7355, 4591, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 6090, 2978, 4189, 11437, 8380, 12040, 12073] avg=8192.0000 max=12073 max_rank=7 min=2978 min_rank=2 imbalance_ratio=1.473755 max_load_reduction=53463 max_load_reduction_pct=0.815781 moved_to_echo=43930 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 102, 0, 678, 0, 0, 0, 0, 823, 0, 0, 314, 0, 7364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 7395, 0, 0, 0, 0, 75, 2858, 7995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3376, 36, 0, 0, 0, 120, 0, 0, 0, 16, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4307, 663, 32, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5131, 964, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 28, 0, 0, 7496, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9285, 3431, 10927, 18370, 3872, 5355, 6738, 7558] avg=8192.0000 max=18370 max_rank=3 min=3431 min_rank=1 imbalance_ratio=2.242432 max_load_reduction=47166 max_load_reduction_pct=0.719696 moved_to_echo=23523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 256, 256, 128, 0, 7552, 128, 128, 0, 0, 14720, 23296, 19712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 0, 128, 37760, 128, 640, 0, 0, 128, 128, 128, 128, 27136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 40960, 0, 256, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 24320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1792, 128, 128, 0, 0, 128, 0, 128, 128, 0, 128, 128, 9088, 128, 0, 128, 41728, 7680, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 4352, 256, 0, 128, 128, 24064, 128, 896, 0, 128, 36224, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 256, 128, 896, 128, 52736, 5248, 3712, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 2176, 128, 0, 128, 128, 768, 128, 0, 128, 0, 896, 1920, 0, 384, 128, 59520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 128, 128, 0, 59264, 6016, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True + [2026-08-26 16:28:29] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 1016.7 | throughput per GPU (TFLOP/s/GPU): 177.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.693762E+00 | loss scale: 1.0 | grad norm: 66.371 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[267, 1, 0, 64, 120, 252, 491, 1, 2, 678, 347, 15, 206, 83, 164, 131, 11, 1277, 0, 7, 823, 1, 0, 1195, 1847, 12, 0, 7579, 5314, 6580, 2, 482, 243, 565, 203, 1, 26, 778, 0, 36, 115, 1, 419, 1065, 0, 164, 3, 7865, 0, 16, 17, 3, 0, 81, 4, 14, 42, 1, 19, 286, 16, 4, 5, 71, 2023, 1, 0, 0, 448, 0, 32, 197, 704, 0, 1200, 1747, 58, 18, 173, 10, 1, 6, 0, 22, 96, 1, 0, 778, 0, 86, 30, 5, 1, 3, 27, 5, 7, 103, 8, 19, 16, 9, 33, 13, 224, 495, 302, 611, 4, 76, 2598, 5, 0, 3576, 0, 21, 0, 31, 1234, 3, 1, 43, 8004, 343, 9, 6, 4, 51] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[317, 5, 0, 54, 33, 225, 666, 0, 0, 579, 657, 22, 430, 67, 185, 215, 30, 1098, 0, 14, 945, 8, 86, 1214, 1534, 13, 1, 7688, 4703, 5639, 8, 712, 90, 483, 202, 2, 11, 1498, 0, 107, 180, 3, 535, 1154, 0, 125, 21, 7532, 0, 19, 5, 6, 1, 78, 0, 0, 98, 4, 15, 184, 2, 4, 14, 40, 2018, 1, 14, 0, 617, 1, 7, 159, 639, 1, 1507, 1299, 65, 10, 211, 11, 0, 25, 2, 22, 124, 0, 1, 963, 0, 211, 49, 107, 35, 9, 66, 8, 11, 114, 3, 10, 105, 35, 43, 16, 216, 843, 282, 875, 2, 81, 1943, 30, 2, 3650, 0, 17, 1, 86, 855, 20, 6, 27, 8020, 382, 35, 11, 12, 60] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[99, 1, 0, 80, 103, 328, 549, 0, 1, 1039, 542, 44, 230, 53, 85, 169, 2, 1086, 0, 20, 1163, 0, 1, 1225, 2169, 7, 0, 7835, 5090, 6181, 13, 888, 111, 341, 199, 1, 22, 1154, 0, 7, 65, 0, 384, 1225, 0, 236, 4, 7968, 1, 12, 6, 2, 0, 77, 0, 6, 182, 0, 19, 350, 0, 0, 8, 35, 1593, 1, 0, 0, 809, 0, 13, 125, 520, 1, 1255, 984, 51, 3, 194, 7, 0, 2, 0, 0, 30, 0, 0, 587, 0, 136, 8, 34, 13, 15, 29, 6, 33, 137, 3, 9, 12, 5, 23, 5, 271, 419, 207, 521, 0, 54, 2113, 12, 1, 4104, 0, 6, 0, 43, 1124, 6, 0, 20, 8168, 312, 2, 12, 2, 78] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[249, 0, 0, 125, 109, 166, 583, 0, 0, 583, 283, 29, 159, 68, 270, 134, 13, 1026, 0, 14, 1033, 0, 0, 1036, 1380, 36, 0, 7474, 4630, 5898, 6, 1224, 294, 378, 320, 2, 52, 1505, 0, 0, 106, 0, 451, 1225, 0, 239, 37, 7779, 0, 47, 2, 0, 1, 53, 1, 6, 206, 1, 14, 133, 2, 1, 6, 26, 1935, 1, 1, 0, 597, 3, 15, 90, 710, 0, 2090, 1276, 98, 3, 209, 6, 0, 1, 0, 10, 33, 0, 1, 679, 3, 149, 18, 5, 20, 4, 56, 3, 3, 285, 0, 15, 25, 12, 69, 17, 478, 260, 311, 834, 0, 155, 2599, 19, 0, 2671, 0, 6, 0, 138, 1615, 5, 1, 2, 8131, 373, 9, 10, 1, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[169, 2, 0, 64, 91, 304, 441, 0, 5, 556, 274, 9, 150, 91, 93, 163, 8, 599, 0, 33, 815, 0, 13, 1457, 3237, 5, 0, 7442, 4410, 6061, 26, 522, 70, 364, 176, 0, 5, 946, 0, 49, 139, 1, 476, 887, 0, 143, 15, 7843, 0, 14, 5, 0, 0, 25, 1, 10, 104, 0, 37, 299, 1, 2, 3, 48, 1757, 0, 0, 0, 612, 0, 12, 162, 898, 0, 1825, 1873, 58, 20, 288, 1, 1, 0, 1, 4, 70, 0, 0, 677, 1, 107, 9, 6, 43, 9, 63, 0, 15, 132, 0, 3, 20, 2, 14, 7, 106, 700, 420, 720, 0, 142, 2235, 10, 2, 3207, 0, 4, 1, 99, 1707, 2, 1, 64, 8117, 486, 6, 12, 5, 97] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[321, 1, 0, 96, 133, 293, 601, 1, 3, 1193, 196, 42, 144, 33, 91, 168, 7, 864, 3, 7, 1091, 11, 1, 1700, 2070, 10, 0, 7410, 4850, 6550, 4, 817, 66, 407, 192, 1, 4, 881, 2, 11, 163, 8, 680, 1441, 0, 68, 4, 7886, 14, 9, 21, 3, 0, 97, 0, 13, 253, 7, 41, 237, 5, 1, 3, 41, 2104, 10, 0, 0, 482, 0, 18, 39, 860, 0, 1255, 996, 91, 1, 180, 8, 0, 2, 0, 1, 36, 0, 6, 547, 1, 109, 23, 20, 14, 7, 139, 6, 26, 116, 9, 45, 30, 3, 43, 3, 234, 305, 421, 690, 2, 80, 1692, 21, 4, 3204, 0, 8, 1, 122, 1301, 23, 0, 22, 8084, 750, 9, 13, 10, 40] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[288, 1, 0, 167, 61, 164, 475, 0, 0, 856, 254, 19, 144, 42, 105, 65, 6, 869, 1, 22, 805, 3, 1, 1134, 1740, 11, 0, 7507, 5108, 6570, 10, 1067, 137, 252, 349, 0, 15, 651, 0, 46, 182, 0, 600, 1303, 0, 76, 7, 7760, 1, 52, 10, 1, 0, 29, 2, 5, 147, 4, 26, 232, 1, 4, 37, 227, 2135, 1, 5, 0, 514, 0, 43, 19, 649, 0, 1307, 1136, 58, 8, 112, 4, 2, 8, 1, 3, 31, 0, 4, 632, 1, 76, 21, 33, 14, 14, 18, 0, 0, 114, 0, 7, 32, 12, 75, 12, 162, 1089, 217, 606, 1, 47, 3003, 7, 13, 3838, 0, 6, 0, 70, 1099, 3, 3, 36, 8028, 411, 9, 5, 7, 124] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[347, 0, 0, 83, 117, 361, 660, 0, 0, 897, 199, 31, 222, 77, 134, 267, 12, 1941, 0, 14, 1145, 0, 5, 1286, 1906, 4, 0, 7608, 5019, 5993, 33, 574, 106, 377, 310, 1, 21, 788, 0, 5, 182, 1, 310, 1145, 0, 162, 3, 7916, 0, 14, 17, 0, 0, 102, 0, 6, 214, 0, 19, 225, 1, 3, 0, 54, 2616, 0, 2, 1, 502, 0, 7, 61, 815, 0, 1099, 1054, 131, 10, 254, 20, 0, 3, 0, 2, 69, 1, 1, 723, 1, 337, 8, 21, 63, 6, 56, 2, 6, 281, 4, 14, 34, 7, 50, 1, 274, 340, 424, 637, 0, 102, 1989, 16, 8, 2922, 0, 15, 0, 93, 799, 5, 2, 12, 8070, 526, 6, 18, 0, 99] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[267, 1, 0, 64, 120, 252, 491, 1, 2, 678, 347, 15, 206, 83, 164, 131, 4210, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1277, 0, 7, 823, 1, 0, 1195, 1847, 12, 0, 0, 2172, 0, 2, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 565, 203, 1, 26, 778, 0, 36, 115, 1, 419, 1065, 0, 164, 3, 4392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 17, 3, 0, 81, 4, 14, 42, 1, 19, 286, 16, 4, 5, 71, 7579, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 1, 0, 0, 448, 0, 32, 197, 704, 0, 1200, 1747, 58, 18, 173, 10, 1855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 22, 96, 1, 0, 778, 0, 86, 30, 5, 1, 3, 27, 5, 6578, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 103, 8, 19, 16, 9, 33, 13, 224, 495, 302, 611, 4, 76, 2598, 5, 2541, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 21, 0, 31, 1234, 3, 1, 43, 2865, 343, 9, 6, 4, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7964, 7829, 8011, 8160, 8466, 8568, 8351, 8187] avg=8192.0000 max=8568 max_rank=5 min=7829 min_rank=1 imbalance_ratio=1.045898 max_load_reduction=56968 max_load_reduction_pct=0.869263 moved_to_echo=33572 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[169, 2, 0, 64, 91, 304, 441, 0, 5, 556, 274, 9, 150, 91, 93, 163, 4270, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 599, 0, 33, 815, 0, 13, 1457, 3237, 5, 0, 0, 1803, 0, 26, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 364, 176, 0, 5, 946, 0, 49, 139, 1, 476, 887, 0, 143, 15, 4379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 5, 0, 0, 25, 1, 10, 104, 0, 37, 299, 1, 2, 3, 48, 7442, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 0, 0, 0, 612, 0, 12, 162, 898, 0, 1825, 1873, 58, 20, 288, 1, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4, 70, 0, 0, 677, 1, 107, 9, 6, 43, 9, 63, 0, 6059, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 132, 0, 3, 20, 2, 14, 7, 106, 700, 420, 720, 0, 142, 2235, 10, 2534, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3207, 0, 4, 1, 99, 1707, 2, 1, 64, 2905, 486, 6, 12, 5, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7612, 8518, 7650, 7993, 9045, 7992, 8128, 8598] avg=8192.0000 max=9045 max_rank=4 min=7612 min_rank=0 imbalance_ratio=1.104126 max_load_reduction=56491 max_load_reduction_pct=0.861984 moved_to_echo=33763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[249, 0, 0, 125, 109, 166, 583, 0, 0, 583, 283, 29, 159, 68, 270, 134, 4277, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1026, 0, 14, 1033, 0, 0, 1036, 1380, 36, 0, 0, 1893, 0, 6, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 378, 320, 2, 52, 1505, 0, 0, 106, 0, 451, 1225, 0, 239, 37, 4344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2, 0, 1, 53, 1, 6, 206, 1, 14, 133, 2, 1, 6, 26, 7474, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1935, 1, 1, 0, 597, 3, 15, 90, 710, 0, 2090, 1276, 98, 3, 209, 6, 1616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 33, 0, 1, 679, 3, 149, 18, 5, 20, 4, 56, 3, 5895, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 285, 0, 15, 25, 12, 69, 17, 478, 260, 311, 834, 0, 155, 2599, 19, 2513, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2671, 0, 6, 0, 138, 1615, 5, 1, 2, 2910, 373, 9, 10, 1, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7957, 7661, 8953, 7976, 8650, 7821, 8716, 7802] avg=8192.0000 max=8953 max_rank=2 min=7661 min_rank=1 imbalance_ratio=1.092896 max_load_reduction=56583 max_load_reduction_pct=0.863388 moved_to_echo=32989 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[317, 5, 0, 54, 33, 225, 666, 0, 0, 579, 657, 22, 430, 67, 185, 215, 4219, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1098, 0, 14, 945, 8, 86, 1214, 1534, 13, 1, 0, 1922, 0, 8, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 483, 202, 2, 11, 1498, 0, 107, 180, 3, 535, 1154, 0, 125, 21, 4206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 5, 6, 1, 78, 0, 0, 98, 4, 15, 184, 2, 4, 14, 40, 7688, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2018, 1, 14, 0, 617, 1, 7, 159, 639, 1, 1507, 1299, 65, 10, 211, 11, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2, 22, 124, 0, 1, 963, 0, 211, 49, 107, 35, 9, 66, 8, 5637, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 114, 3, 10, 105, 35, 43, 16, 216, 843, 282, 875, 2, 81, 1943, 30, 2433, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3650, 0, 17, 1, 86, 855, 20, 6, 27, 2870, 382, 35, 11, 12, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8567, 7585, 8617, 8160, 8202, 8190, 8181, 8034] avg=8192.0000 max=8617 max_rank=2 min=7585 min_rank=1 imbalance_ratio=1.051880 max_load_reduction=56919 max_load_reduction_pct=0.868515 moved_to_echo=32607 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[99, 1, 0, 80, 103, 328, 549, 0, 1, 1039, 542, 44, 230, 53, 85, 169, 4297, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1086, 0, 20, 1163, 0, 1, 1225, 2169, 7, 0, 0, 2081, 0, 13, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 341, 199, 1, 22, 1154, 0, 7, 65, 0, 384, 1225, 0, 236, 4, 4450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 6, 2, 0, 77, 0, 6, 182, 0, 19, 350, 0, 0, 8, 35, 7835, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1593, 1, 0, 0, 809, 0, 13, 125, 520, 1, 1255, 984, 51, 3, 194, 7, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 30, 0, 0, 587, 0, 136, 8, 34, 13, 15, 29, 6, 6178, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 137, 3, 9, 12, 5, 23, 5, 271, 419, 207, 521, 0, 54, 2113, 12, 2574, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4104, 0, 6, 0, 43, 1124, 6, 0, 20, 2923, 312, 2, 12, 2, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8564, 8655, 8199, 8536, 7333, 7986, 7630, 8633] avg=8192.0000 max=8655 max_rank=1 min=7333 min_rank=4 imbalance_ratio=1.056519 max_load_reduction=56881 max_load_reduction_pct=0.867935 moved_to_echo=31582 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[288, 1, 0, 167, 61, 164, 475, 0, 0, 856, 254, 19, 144, 42, 105, 65, 4223, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 869, 1, 22, 805, 3, 1, 1134, 1740, 11, 0, 0, 2088, 0, 10, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 252, 349, 0, 15, 651, 0, 46, 182, 0, 600, 1303, 0, 76, 7, 4333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 52, 10, 1, 0, 29, 2, 5, 147, 4, 26, 232, 1, 4, 37, 227, 7507, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2135, 1, 5, 0, 514, 0, 43, 19, 649, 0, 1307, 1136, 58, 8, 112, 4, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 1, 3, 31, 0, 4, 632, 1, 76, 21, 33, 14, 14, 18, 0, 6567, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 7, 32, 12, 75, 12, 162, 1089, 217, 606, 1, 47, 3003, 7, 2507, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3838, 0, 6, 0, 70, 1099, 3, 3, 36, 2873, 411, 9, 5, 7, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7784, 7757, 7951, 8288, 7774, 8357, 9128, 8497] avg=8192.0000 max=9128 max_rank=6 min=7757 min_rank=1 imbalance_ratio=1.114258 max_load_reduction=56408 max_load_reduction_pct=0.860718 moved_to_echo=33756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[321, 1, 0, 96, 133, 293, 601, 1, 3, 1193, 196, 42, 144, 33, 91, 168, 4256, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 864, 3, 7, 1091, 11, 1, 1700, 2070, 10, 0, 0, 1974, 0, 4, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 407, 192, 1, 4, 881, 2, 11, 163, 8, 680, 1441, 0, 68, 4, 4395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 9, 21, 3, 0, 97, 0, 13, 253, 7, 41, 237, 5, 1, 3, 41, 7410, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2104, 10, 0, 0, 482, 0, 18, 39, 860, 0, 1255, 996, 91, 1, 180, 8, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 36, 0, 6, 547, 1, 109, 23, 20, 14, 7, 139, 6, 6547, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 116, 9, 45, 30, 3, 43, 3, 234, 305, 421, 690, 2, 80, 1692, 21, 2553, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3204, 0, 8, 1, 122, 1301, 23, 0, 22, 2885, 750, 9, 13, 10, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8510, 8559, 8323, 8158, 7742, 8401, 7451, 8392] avg=8192.0000 max=8559 max_rank=1 min=7451 min_rank=6 imbalance_ratio=1.044800 max_load_reduction=56977 max_load_reduction_pct=0.869400 moved_to_echo=31986 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[347, 0, 0, 83, 117, 361, 660, 0, 0, 897, 199, 31, 222, 77, 134, 267, 4245, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1941, 0, 14, 1145, 0, 5, 1286, 1906, 4, 0, 0, 2052, 0, 33, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 377, 310, 1, 21, 788, 0, 5, 182, 1, 310, 1145, 0, 162, 3, 4421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 17, 0, 0, 102, 0, 6, 214, 0, 19, 225, 1, 3, 0, 54, 7608, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2616, 0, 2, 1, 502, 0, 7, 61, 815, 0, 1099, 1054, 131, 10, 254, 20, 1752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 69, 1, 1, 723, 1, 337, 8, 21, 63, 6, 56, 2, 5991, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 281, 4, 14, 34, 7, 50, 1, 274, 340, 424, 637, 0, 102, 1989, 16, 2557, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2922, 0, 15, 0, 93, 799, 5, 2, 12, 2888, 526, 6, 18, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8578, 8972, 7832, 8265, 8324, 8221, 7951, 7393] avg=8192.0000 max=8972 max_rank=1 min=7393 min_rank=7 imbalance_ratio=1.095215 max_load_reduction=56564 max_load_reduction_pct=0.863098 moved_to_echo=31889 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 1408, 128, 128, 384, 128, 384, 128, 2048, 4480, 2688, 5504, 128, 768, 18176, 128, 20224, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[2176, 128, 0, 768, 768, 2176, 4480, 128, 128, 6400, 2816, 256, 1792, 640, 1152, 1408, 34048, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 8832, 128, 256, 7936, 128, 128, 10368, 16000, 128, 128, 0, 16000, 0, 128, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1152, 3200, 2048, 128, 256, 8320, 128, 384, 1152, 128, 3968, 9472, 0, 1280, 128, 34944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[16256, 128, 128, 128, 4608, 128, 256, 896, 5888, 128, 11648, 10368, 640, 128, 1664, 128, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 27264, 0, 128, 128, 768, 9856, 128, 128, 256, 23168, 3584, 128, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 512, 128, 128, 5632, 128, 1280, 256, 256, 256, 128, 512, 128, 49536, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 128, 128, 640, 128, 128, 1280, 128, 256, 2048, 128, 128, 128, 640, 60544, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 6, 0, 0, 333, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8178, 0, 0, 0, 0, 0, 20, 0, 11, 0, 7604, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 70, 0, 8187, 0, 0, 7, 0, 0, 0, 1, 0, 119, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 49, 0, 1, 0, 0, 1, 0, 0, 0, 7363, 8, 1, 0, 0, 0, 920, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8164, 0, 0, 17, 0, 0, 0, 0, 4, 0, 0, 8178, 0, 0, 0, 20, 0, 0, 4, 7127, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 2, 0, 4619, 28, 0, 0, 0, 1, 67, 2, 0, 0, 7777, 0, 0, 12, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8151, 0, 2, 0, 0, 48, 0, 8192, 0, 1, 20, 0, 3, 0, 33, 0, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 914, 0, 0, 4, 0, 0, 0, 0, 6, 8155, 0, 0, 0, 0, 1, 18, 0, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8174, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 6, 0, 0, 218, 7695, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7, 0, 0, 29, 0, 0, 0, 1, 7994, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 5, 0, 9, 0, 8192, 0, 0, 50, 0, 0, 0, 8, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 7486, 0, 0, 0, 0, 0, 835, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 8182, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 11, 5257, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8152, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 16, 0, 8190, 0, 1, 1, 0, 0, 0, 0, 0, 896, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 3, 0, 0, 0, 0, 490, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 10, 5236, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7749, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8179, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 7, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 0, 0, 2, 0, 0, 0, 0, 0, 7770, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8183, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 0, 38, 8083, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 1, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 41, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 12, 0, 0, 0, 0, 0, 8034, 1, 0, 0, 3, 0, 222, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 7988, 0, 0, 16, 16, 2, 0, 0, 0, 0, 0, 8188, 4, 0, 0, 5, 0, 0, 77, 7788, 0, 0, 0, 0, 0, 0, 7, 3, 1, 0, 0, 0, 15, 0, 11, 4, 6, 3, 2, 2, 7, 1, 0, 0, 21, 7977, 4, 2, 1, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8, 1, 0, 9, 8161, 0, 0, 2, 0, 132, 0, 8169, 2, 10, 222, 1, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 1, 0, 0, 13, 0, 1979, 0, 0, 3, 0, 0, 0, 0, 0, 6517, 0, 2, 4, 0, 3, 179, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 0, 7945, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 1, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 8050, 4, 0, 0, 0, 0, 2, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 7, 0, 8190, 0, 0, 0, 0, 0, 0, 3, 0, 157, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 332, 0, 0, 2, 0, 0, 0, 0, 0, 8081, 2, 0, 0, 1, 0, 35, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8174, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 6, 0, 0, 218, 7695, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7, 0, 0, 29, 0, 0, 7557, 716, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7994, 0, 0, 0, 0, 0, 0, 0, 1, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 5, 0, 9, 0, 0, 0, 0, 50, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 6401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7486, 0, 0, 0, 0, 0, 835, 0, 0, 0, 0, 133, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8863, 8329, 8560, 8193, 8190, 6699, 8491] avg=8192.0000 max=8863 max_rank=1 min=6699 min_rank=6 imbalance_ratio=1.081909 max_load_reduction=56673 max_load_reduction_pct=0.864761 moved_to_echo=31573 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 6, 0, 0, 333, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7076, 716, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8178, 0, 0, 0, 0, 0, 20, 0, 11, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 1, 0, 70, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 49, 0, 1, 0, 6401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7363, 8, 1, 0, 0, 0, 920, 0, 0, 0, 0, 133, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 9359, 7795, 8738, 8190, 8195, 6571, 8463] avg=8192.0000 max=9359 max_rank=1 min=6571 min_rank=6 imbalance_ratio=1.142456 max_load_reduction=56177 max_load_reduction_pct=0.857193 moved_to_echo=31419 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8164, 0, 0, 17, 0, 0, 0, 0, 4, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 939, 0, 0, 0, 20, 0, 0, 4, 7127, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 7237, 715, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4619, 28, 0, 0, 0, 1, 67, 2, 0, 0, 540, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8079, 0, 2, 0, 0, 48, 0, 0, 0, 1, 20, 0, 3, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 914, 0, 0, 4, 6391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8155, 0, 0, 0, 0, 1, 18, 0, 0, 1, 1, 133, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8110, 7968, 5271, 8208, 8187, 11219, 8352] avg=8192.0000 max=11219 max_rank=6 min=5271 min_rank=3 imbalance_ratio=1.369507 max_load_reduction=54317 max_load_reduction_pct=0.828812 moved_to_echo=35966 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 8182, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 0, 0, 0, 11, 5257, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7535, 716, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 1, 0, 0, 0, 1, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 16, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 896, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1779, 0, 0, 0, 6401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 3, 0, 0, 0, 0, 490, 1, 0, 0, 0, 133, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 6219, 8280, 8716, 8188, 8131, 9077, 8687] avg=8192.0000 max=9077 max_rank=6 min=6219 min_rank=1 imbalance_ratio=1.108032 max_load_reduction=56459 max_load_reduction_pct=0.861496 moved_to_echo=34083 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 10, 5236, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7611, 716, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7749, 0, 0, 0, 0, 0, 5, 0, 0, 0, 568, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 0, 0, 2, 6402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7770, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 133, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 6189, 8342, 8323, 8192, 8128, 10203, 7942] avg=8192.0000 max=10203 max_rank=6 min=6189 min_rank=1 imbalance_ratio=1.245483 max_load_reduction=55333 max_load_reduction_pct=0.844315 moved_to_echo=34465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8183, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 1, 0, 0, 38, 8083, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7544, 716, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 12, 6399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8034, 1, 0, 0, 3, 0, 222, 0, 0, 0, 0, 133, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 9087, 8264, 8751, 8192, 8122, 6469, 8430] avg=8192.0000 max=9087 max_rank=1 min=6469 min_rank=6 imbalance_ratio=1.109253 max_load_reduction=56449 max_load_reduction_pct=0.861343 moved_to_echo=31213 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 7988, 0, 0, 16, 16, 2, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 4, 0, 0, 5, 0, 0, 77, 7788, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 15, 0, 11, 4, 6, 3, 2, 2, 7, 1, 0, 7397, 719, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7977, 4, 2, 1, 0, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8078, 0, 0, 2, 0, 132, 0, 0, 2, 10, 222, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 1, 0, 0, 13, 0, 1979, 0, 0, 3, 6402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6517, 0, 2, 4, 0, 3, 179, 1, 0, 0, 0, 135, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8813, 8173, 8554, 8184, 8456, 8408, 6882] avg=8192.0000 max=8813 max_rank=1 min=6882 min_rank=7 imbalance_ratio=1.075806 max_load_reduction=56723 max_load_reduction_pct=0.865524 moved_to_echo=31930 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 0, 1, 0, 0, 7945, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 1, 0, 0, 2, 0, 1, 0, 0, 0, 7589, 716, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 4, 0, 0, 0, 0, 2, 0, 0, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8116, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 332, 0, 0, 2, 6398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 2, 0, 0, 1, 0, 35, 0, 0, 0, 0, 133, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8896, 8385, 8623, 8189, 8127, 6890, 8289] avg=8192.0000 max=8896 max_rank=1 min=6890 min_rank=6 imbalance_ratio=1.085938 max_load_reduction=56640 max_load_reduction_pct=0.864258 moved_to_echo=31495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 5376, 128, 128, 0, 0, 128, 128, 0, 0, 128, 0, 8960, 0, 128, 128, 51200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[7552, 128, 0, 0, 128, 128, 0, 768, 57216, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 0, 256, 0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 59648, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 60928, 128, 128, 128, 0, 128, 128, 128, 128, 0, 4480, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 0, 128, 61440, 128, 128, 128, 128, 128, 2816, 128, 0, 128, 128, 1152, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 128, 0, 128, 128, 128, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 128, 65280, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 64896, 0, 128, 128, 0, 384, 0, 0, 128, 128, 384, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[8186, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4781, 98, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 24, 0, 0, 8102, 0, 0, 0, 1, 8124, 0, 0, 0, 0, 0, 3006, 0, 8168, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8191, 0, 181] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[8178, 0, 0, 0, 75, 0, 0, 1207, 0, 0, 0, 5082, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 0, 0, 0, 8186, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 710, 0, 7030, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 208, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 87, 1, 0, 0, 0, 5, 0, 2172, 3, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 20, 49, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8189, 0, 0, 1, 2, 0, 0, 18, 0, 0, 0, 7787, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 4, 0, 0, 8007, 0, 0, 0, 3, 8094, 2, 0, 0, 0, 0, 252, 0, 8100, 2, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3, 13, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 416, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4529, 10, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 8191, 18, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 1739, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1788, 0, 0, 4, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 15, 0, 0, 20, 0, 0, 0, 7620, 5, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 8164, 1, 0, 0, 0, 7991, 0, 0, 0, 0, 0, 168, 0, 8101, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 51] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[8189, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5908, 37, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 46, 0, 0, 8122, 102, 0, 0, 0, 8184, 0, 0, 0, 7, 0, 1998, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3, 95, 4, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 116, 0, 0, 45, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 9, 0, 0, 113, 1, 1, 0, 3069, 16, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 8189, 0, 0, 0, 3, 8155, 0, 0, 0, 1, 0, 3682, 0, 8176, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 1, 0, 0, 6, 0, 0, 218, 0, 0, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1165, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[7952, 6, 0, 10, 236, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8154, 283, 0, 0, 8088, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 496, 0, 8057, 4, 3, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 0, 25, 3, 0, 3, 0, 1, 0, 0, 0, 206, 0, 7, 0, 18, 0, 0, 0, 0, 0, 139, 0, 0, 2, 0, 8162, 0, 0, 0, 0, 0, 0, 20, 48, 0, 1, 0, 0, 0, 3, 0, 7, 0, 8, 0, 0, 0, 0, 0, 970, 8, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 14, 8175, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2331, 0, 0, 0, 15, 0, 0, 20, 0, 0, 0, 7620, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 6763, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7965, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 5791, 1338, 168, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9991, 8203, 6933, 8190, 8158, 8437, 7430, 8194] avg=8192.0000 max=9991 max_rank=0 min=6933 min_rank=2 imbalance_ratio=1.219604 max_load_reduction=55545 max_load_reduction_pct=0.847549 moved_to_echo=32219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2330, 0, 0, 1, 2, 0, 0, 18, 0, 0, 0, 7787, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 4, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 252, 0, 6762, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8007, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8068, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 1, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 416, 0, 3, 5789, 1338, 168, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10141, 8203, 7021, 8036, 8155, 8058, 7764, 8158] avg=8192.0000 max=10141 max_rank=0 min=7021 min_rank=2 imbalance_ratio=1.237915 max_load_reduction=55395 max_load_reduction_pct=0.845261 moved_to_echo=32135 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2330, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4781, 98, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 24, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3006, 0, 6819, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8102, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8098, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 5786, 1349, 168, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8136, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7224, 8208, 9828, 8129, 8171, 8029, 7620, 8327] avg=8192.0000 max=9828 max_rank=2 min=7224 min_rank=0 imbalance_ratio=1.199707 max_load_reduction=55708 max_load_reduction_pct=0.850037 moved_to_echo=32147 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2327, 0, 0, 0, 75, 0, 0, 1207, 0, 0, 0, 5082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 0, 5869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 208, 4, 4, 0, 0, 0, 8186, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7916, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 1, 0, 0, 8022, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1, 0, 0, 0, 5, 0, 2172, 3, 0, 5781, 1161, 168, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 20, 49, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8693, 8075, 6579, 8434, 7986, 8136, 9424, 8209] avg=8192.0000 max=9424 max_rank=6 min=6579 min_rank=2 imbalance_ratio=1.150391 max_load_reduction=56112 max_load_reduction_pct=0.856201 moved_to_echo=33755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2331, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4529, 10, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1739, 0, 6795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1788, 0, 0, 5791, 1345, 168, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6872, 8083, 8552, 8218, 8210, 8312, 9138, 8151] avg=8192.0000 max=9138 max_rank=6 min=6872 min_rank=0 imbalance_ratio=1.115479 max_load_reduction=56398 max_load_reduction_pct=0.860565 moved_to_echo=33811 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2330, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5908, 37, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 46, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1998, 0, 6783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3, 8156, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 4, 0, 0, 1, 8019, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 116, 0, 0, 5789, 1342, 168, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8200, 8890, 8150, 8231, 8121, 7463, 8201] avg=8192.0000 max=8890 max_rank=2 min=7463 min_rank=6 imbalance_ratio=1.085205 max_load_reduction=56646 max_load_reduction_pct=0.864349 moved_to_echo=32016 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2331, 0, 0, 0, 9, 0, 0, 113, 1, 1, 0, 3069, 16, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8106, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 3682, 0, 6825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 1, 0, 0, 6, 0, 0, 8128, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 8017, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1165, 0, 0, 5790, 1351, 167, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5558, 8116, 10511, 8217, 8221, 8240, 8520, 8153] avg=8192.0000 max=10511 max_rank=2 min=5558 min_rank=0 imbalance_ratio=1.283081 max_load_reduction=55025 max_load_reduction_pct=0.839615 moved_to_echo=33134 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2253, 6, 0, 10, 236, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8154, 283, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 6722, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 0, 25, 3, 0, 8088, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 206, 0, 7, 0, 18, 0, 0, 0, 0, 0, 8094, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 2, 0, 7993, 0, 0, 0, 0, 0, 0, 20, 48, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 8, 0, 0, 0, 0, 0, 970, 8, 0, 5624, 1335, 169, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 14, 8112, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8777, 8448, 7222, 8162, 8404, 8203, 8177, 8143] avg=8192.0000 max=8777 max_rank=0 min=7222 min_rank=2 imbalance_ratio=1.071411 max_load_reduction=56759 max_load_reduction_pct=0.866074 moved_to_echo=32927 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 128, 128, 0, 7040, 128, 128, 46208, 10624, 1408, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[18688, 128, 0, 128, 384, 128, 0, 1408, 128, 128, 0, 45056, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 128, 0, 12160, 0, 53376, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1280, 128, 128, 128, 128, 64256, 0, 128, 0, 0, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 0, 128, 384, 0, 128, 128, 128, 0, 0, 128, 0, 128, 64640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 0, 128, 0, 128, 128, 128, 256, 128, 128, 128, 128, 0, 65152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 128, 0, 0, 128, 0, 128, 128, 128, 65152, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 0, 128, 128, 0, 0, 65152, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 8192, 7148, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 6, 0, 0, 0, 8192, 0, 0, 8192, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 6565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2534, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 8188, 7287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8187, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 852, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 217, 3, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 8186, 6207, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 1924, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8099, 8187, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 3, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 91, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 292, 8192, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 8192, 7461, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 761, 0, 0, 0, 0, 0, 0, 0, 0, 7605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 10, 0, 0, 0, 0, 207, 0, 289, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 6, 7, 0, 3, 0, 0, 0, 0, 0, 33, 8158, 4, 0, 15, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 0, 8171, 8147, 6824, 0, 0, 2, 6, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 8150, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 6878, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 56, 24, 0, 0, 0, 0, 16, 6, 6, 0, 0, 15, 0, 0, 42, 0, 0, 0, 0, 1212, 0, 1326, 14, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8187, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8189, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8190, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 763, 0, 7287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 294, 54, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 8173, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 852, 0, 0, 7134, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8158, 8052, 8184, 8544, 8216, 8188, 8002] avg=8192.0000 max=8544 max_rank=4 min=8002 min_rank=7 imbalance_ratio=1.042969 max_load_reduction=56992 max_load_reduction_pct=0.869629 moved_to_echo=32950 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 217, 3, 0, 0, 0, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8129, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 293, 54, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8171, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 1924, 0, 0, 7117, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8412, 8138, 6968, 8180, 8386, 8187, 8184, 9081] avg=8192.0000 max=9081 max_rank=7 min=6968 min_rank=2 imbalance_ratio=1.108521 max_load_reduction=56455 max_load_reduction_pct=0.861435 moved_to_echo=33838 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 294, 54, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7135, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8138, 8931, 8189, 8543, 8187, 8187, 7167] avg=8192.0000 max=8931 max_rank=2 min=7167 min_rank=7 imbalance_ratio=1.090210 max_load_reduction=56605 max_load_reduction_pct=0.863724 moved_to_echo=32084 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 754, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 291, 54, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8172, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 91, 1, 0, 7054, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8140, 8944, 8185, 8539, 8191, 8185, 7163] avg=8192.0000 max=8944 max_rank=2 min=7163 min_rank=7 imbalance_ratio=1.091797 max_load_reduction=56592 max_load_reduction_pct=0.863525 moved_to_echo=32078 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8133, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 762, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 294, 54, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8174, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 5, 0, 0, 7129, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8138, 8940, 8187, 8545, 8190, 8181, 7168] avg=8192.0000 max=8940 max_rank=2 min=7168 min_rank=7 imbalance_ratio=1.091309 max_load_reduction=56596 max_load_reduction_pct=0.863586 moved_to_echo=32084 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 3, 0, 0, 0, 0, 0, 33, 8099, 4, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 0, 755, 0, 6824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 6878, 3, 0, 0, 0, 0, 296, 59, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 56, 24, 0, 0, 0, 0, 16, 6, 8133, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 15, 0, 0, 42, 0, 0, 0, 0, 1212, 0, 1326, 14, 3, 7120, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8167, 7585, 8164, 7263, 8188, 8244, 9752] avg=8192.0000 max=9752 max_rank=7 min=7263 min_rank=4 imbalance_ratio=1.190430 max_load_reduction=55784 max_load_reduction_pct=0.851196 moved_to_echo=33447 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 756, 0, 7148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 6565, 0, 0, 0, 0, 0, 292, 54, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2534, 0, 0, 7077, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8211, 7904, 8255, 7001, 8187, 8182, 9631] avg=8192.0000 max=9631 max_rank=7 min=7001 min_rank=4 imbalance_ratio=1.175659 max_load_reduction=55905 max_load_reduction_pct=0.853043 moved_to_echo=33001 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 292, 8138, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 0, 7461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 0, 0, 0, 0, 0, 0, 0, 0, 7605, 0, 0, 0, 0, 0, 289, 54, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8177, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 10, 0, 0, 0, 0, 207, 0, 289, 0, 0, 7019, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8024, 8446, 8212, 8192, 8715, 8190, 8185, 7572] avg=8192.0000 max=8715 max_rank=4 min=7572 min_rank=7 imbalance_ratio=1.063843 max_load_reduction=56821 max_load_reduction_pct=0.867020 moved_to_echo=32662 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 1536, 128, 7040, 128, 128, 56832, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 128, 0, 0, 0, 6144, 0, 59520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 896, 128, 0, 0, 0, 0, 0, 0, 128, 61952, 128, 0, 0, 0, 0, 2432, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 0, 512, 65152, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 256, 128, 128, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 7687, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 13, 0, 0, 8150, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7887, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 16, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 7188, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4, 1504, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 7686, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 27, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 359, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8191, 0, 0, 0, 4, 0, 0, 0, 0, 8191, 0, 7262, 0, 3, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 1, 0, 0, 0, 0, 12, 0, 0, 0, 112, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3508, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 13, 8185, 0, 0, 0, 0, 1, 3, 0, 4702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 80, 0, 0, 98, 0, 0, 8018, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7701, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 54, 3, 0, 0, 8182, 0, 7853, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 95, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3281, 4427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 17, 886, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7986, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 7999, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 1, 20, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8176, 3, 0, 0, 0, 0, 1, 0, 8116, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 5, 1, 1, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 4, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 8162, 0, 133, 0, 0, 0, 0, 0, 1, 8163, 0, 8169, 3, 0, 0, 22, 0, 1, 18, 25, 8011, 0, 0, 0, 0, 3, 0, 0, 0, 83, 0, 4, 1, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 741, 1, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 8172, 2, 0, 14, 0, 1, 11, 3, 7933, 0, 19, 0, 1, 0, 0, 0, 0, 38, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7553, 0, 0, 1, 29, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7687, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 373, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 13, 0, 0, 8140, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7887, 0, 0, 0, 0, 0, 174, 20, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8045, 8160, 10702, 5919, 8186, 8257, 8091] avg=8192.0000 max=10702 max_rank=3 min=5919 min_rank=4 imbalance_ratio=1.306396 max_load_reduction=54834 max_load_reduction_pct=0.836700 moved_to_echo=30453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 2, 0, 0, 0, 0, 7825, 373, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 359, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 0, 0, 0, 0, 0, 0, 0, 7131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 4, 0, 0, 0, 0, 173, 20, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8482, 8215, 6659, 9283, 8186, 8176, 8366] avg=8192.0000 max=9283 max_rank=4 min=6659 min_rank=3 imbalance_ratio=1.133179 max_load_reduction=56253 max_load_reduction_pct=0.858353 moved_to_echo=34011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7827, 373, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 16, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 174, 20, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8500, 8209, 5709, 10185, 8186, 8182, 8392] avg=8192.0000 max=10185 max_rank=4 min=5709 min_rank=3 imbalance_ratio=1.243286 max_load_reduction=55351 max_load_reduction_pct=0.844589 moved_to_echo=34945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7262, 0, 3, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 112, 0, 5, 0, 0, 0, 0, 0, 0, 7617, 373, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3508, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2150, 0, 0, 0, 0, 1, 3, 0, 4702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 98, 0, 0, 8008, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7701, 0, 0, 0, 0, 0, 173, 20, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 7616, 8125, 10466, 6869, 8193, 8187, 7904] avg=8192.0000 max=10466 max_rank=3 min=6869 min_rank=4 imbalance_ratio=1.277588 max_load_reduction=55070 max_load_reduction_pct=0.840302 moved_to_echo=31153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 54, 3, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7853, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7755, 373, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3281, 4427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 0, 0, 0, 0, 0, 0, 17, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 7999, 0, 0, 0, 0, 0, 173, 20, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8210, 8231, 13201, 3055, 8189, 8209, 8219] avg=8192.0000 max=13201 max_rank=3 min=3055 min_rank=4 imbalance_ratio=1.611450 max_load_reduction=52335 max_load_reduction_pct=0.798569 moved_to_echo=27672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7188, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7829, 373, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1504, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2150, 0, 0, 0, 0, 0, 0, 0, 7686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 27, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 173, 20, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 7548, 8217, 7000, 9837, 8187, 8178, 8396] avg=8192.0000 max=9837 max_rank=4 min=7000 min_rank=3 imbalance_ratio=1.200806 max_load_reduction=55699 max_load_reduction_pct=0.849899 moved_to_echo=34598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 7830, 372, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2148, 3, 0, 0, 0, 0, 1, 0, 8116, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 5, 1, 1, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 4, 2, 173, 20, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8547, 8235, 5557, 10271, 8188, 8197, 8371] avg=8192.0000 max=10271 max_rank=4 min=5557 min_rank=3 imbalance_ratio=1.253784 max_load_reduction=55265 max_load_reduction_pct=0.843277 moved_to_echo=35027 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 1, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 3, 0, 0, 22, 0, 1, 18, 25, 347, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 4, 1, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 7664, 377, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 741, 1, 0, 0, 0, 0, 0, 0, 0, 15, 0, 5484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2133, 2, 0, 14, 0, 1, 11, 3, 7933, 0, 19, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 22, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7553, 0, 0, 1, 29, 0, 178, 23, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8588, 8144, 6242, 10117, 8221, 8150, 7797] avg=8192.0000 max=10117 max_rank=4 min=6242 min_rank=3 imbalance_ratio=1.234985 max_load_reduction=55419 max_load_reduction_pct=0.845627 moved_to_echo=34285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 10368, 11392, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 44032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 17280, 128, 0, 128, 0, 128, 128, 128, 48256, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[62720, 128, 128, 0, 128, 0, 128, 128, 128, 2944, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 63872, 128, 0, 128, 128, 128, 1408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 256, 128, 128, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 62208, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 256, 128, 128, 65024, 0, 128, 384, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 256, 0, 128, 128, 128, 0, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1232, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8185, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 475, 0, 0, 7, 0, 6322, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 3, 0, 0, 1, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 8192, 0, 45, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 97, 0, 0, 2, 0, 8015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 139, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8184, 0, 1, 1, 0, 8103, 0, 17, 0, 1, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 104, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 4751, 1, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3293, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 452, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 8184, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 218, 0, 0, 1, 0, 6546, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6587, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8187, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 1941, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 176, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 0, 0, 7962, 0, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 635, 15, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 937, 0, 0, 6, 0, 57, 0, 0, 18, 0, 0, 0, 0, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 8190, 5543, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 2, 0, 8111, 0, 0, 0, 0, 8180, 0, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 0, 0, 0, 25, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 3, 1, 0, 0, 792, 0, 1, 0, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8157, 39, 0, 0, 6, 21, 0, 0, 8166, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 6, 0, 0, 0, 8161, 0, 13, 16, 0, 8172, 0, 5048, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 17, 20, 0, 0, 0, 0, 0, 8155, 0, 0, 2, 4, 1, 0, 0, 0, 0, 0, 1, 0, 0, 3, 8153, 0, 0, 0, 13, 22, 1436, 0, 24, 0, 0, 266, 0, 0, 0, 8, 0, 0, 0, 280, 0, 0, 0, 0, 16, 0, 0, 12, 0, 0, 0, 0, 1, 0, 0, 0, 1095, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6206, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7057, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7684, 0, 0, 0, 0, 0, 218, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6546, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1985, 1127, 637, 506, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 0, 0, 0, 0, 0, 7498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6659, 8192, 8163, 7063, 8266, 7903, 11010, 8280] avg=8192.0000 max=11010 max_rank=6 min=6659 min_rank=0 imbalance_ratio=1.343994 max_load_reduction=54526 max_load_reduction_pct=0.832001 moved_to_echo=35459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6182, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 1, 0, 8184, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6987, 0, 17, 0, 1, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7684, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4751, 1, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1977, 1116, 637, 506, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3293, 0, 0, 0, 0, 0, 0, 7502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6321, 8194, 8238, 7011, 8109, 7689, 9179, 10795] avg=8192.0000 max=10795 max_rank=7 min=6321 min_rank=0 imbalance_ratio=1.317749 max_load_reduction=54741 max_load_reduction_pct=0.835281 moved_to_echo=35772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6207, 6587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7059, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7686, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1985, 1128, 637, 506, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 7504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12794, 8192, 8243, 7082, 8408, 7761, 5547, 7509] avg=8192.0000 max=12794 max_rank=0 min=5547 min_rank=6 imbalance_ratio=1.561768 max_load_reduction=52742 max_load_reduction_pct=0.804779 moved_to_echo=29225 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6207, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7058, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7686, 0, 0, 0, 0, 0, 475, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6322, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1985, 1127, 637, 506, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 7504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7439, 8192, 8241, 7065, 8069, 8168, 10772, 7590] avg=8192.0000 max=10772 max_rank=6 min=7065 min_rank=3 imbalance_ratio=1.314941 max_load_reduction=54764 max_load_reduction_pct=0.835632 moved_to_echo=34599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6182, 1941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 176, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6865, 0, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 0, 0, 0, 0, 635, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7664, 0, 0, 0, 0, 0, 937, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 18, 0, 0, 0, 0, 1395, 0, 0, 0, 0, 0, 0, 0, 1977, 1097, 637, 505, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 7503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8187, 8012, 10565, 8653, 8607, 5872, 7517] avg=8192.0000 max=10565 max_rank=3 min=5872 min_rank=6 imbalance_ratio=1.289673 max_load_reduction=54971 max_load_reduction_pct=0.838791 moved_to_echo=30649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 3, 0, 0, 1, 0, 6197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 50, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7064, 0, 45, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 97, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1982, 1128, 637, 506, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 7494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6203, 8192, 8237, 7136, 8007, 7784, 12454, 7523] avg=8192.0000 max=12454 max_rank=6 min=6203 min_rank=0 imbalance_ratio=1.520264 max_load_reduction=53082 max_load_reduction_pct=0.809967 moved_to_echo=35768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 6205, 5543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 2, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7053, 0, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8004, 0, 0, 0, 0, 25, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7685, 0, 3, 1, 0, 0, 792, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1985, 1127, 637, 506, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 7498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11758, 8192, 8163, 7503, 8032, 8482, 5676, 7730] avg=8192.0000 max=11758 max_rank=0 min=5676 min_rank=6 imbalance_ratio=1.435303 max_load_reduction=53778 max_load_reduction_pct=0.820587 moved_to_echo=29920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 6175, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 21, 0, 0, 8162, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 6, 0, 0, 0, 8161, 0, 13, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7042, 0, 5048, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 7965, 0, 0, 2, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 7643, 0, 0, 0, 13, 22, 1436, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 8, 0, 0, 0, 280, 0, 0, 0, 0, 16, 0, 0, 1982, 1130, 640, 510, 190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 0, 0, 1095, 0, 0, 0, 0, 0, 1, 7483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6239, 8195, 8239, 12111, 7992, 9142, 5026, 8592] avg=8192.0000 max=12111 max_rank=3 min=5026 min_rank=6 imbalance_ratio=1.478394 max_load_reduction=53425 max_load_reduction_pct=0.815201 moved_to_echo=30752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[28416, 128, 128, 128, 128, 0, 0, 0, 1792, 0, 0, 0, 0, 128, 0, 0, 15872, 9088, 5120, 4096, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 128, 128, 128, 128, 128, 0, 49664, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 56192, 0, 9344, 0, 128, 0, 0, 128, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 128, 0, 0, 0, 5632, 0, 0, 0, 0, 0, 128, 60032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 128, 61440, 0, 128, 128, 128, 128, 4096, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 0, 64000, 0, 0, 128, 128, 1536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 512, 0, 256, 0, 0, 128, 0, 128, 0, 65024, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8167, 4, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 10, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4438, 8190, 0, 8153, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 26, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 21, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8192, 0, 7233, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 10, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 8192, 0, 8173, 0, 1, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 0, 8180, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5245, 8192, 0, 8030, 0, 0, 0, 53, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 8183, 0, 0, 0, 0, 14, 0, 0, 0, 0, 3, 0, 0, 0, 9, 0, 0, 7, 0, 8190, 0, 7455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 110, 79, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 8191, 0, 7704, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 7528, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7958, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 32, 0, 8191, 0, 8178, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 147, 8190, 0, 8191, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8121, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 433, 740, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7578, 8191, 0, 7159, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7927, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 7670, 0, 0, 389, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 5835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 8192, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 43, 6364, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 41, 4, 8137, 0, 8160, 0, 14, 0, 1, 24, 0, 0, 0, 0, 11, 0, 3, 0, 0, 0, 0, 0, 865, 0, 0, 0, 0, 0, 0, 36, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 94, 8123, 0, 8171, 0, 0, 0, 0, 8139, 11, 0, 25, 0, 0, 0, 0, 0, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 299, 0, 7885, 0, 0, 0, 4, 17, 0, 0, 0, 0, 0, 0, 19, 8163, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 0, 8180, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 2936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 58, 0, 0, 0, 5457, 2120, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5245, 5665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 53, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 1, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8429, 5671, 7774, 10910, 8292, 7958, 8314, 8188] avg=8192.0000 max=10910 max_rank=3 min=5671 min_rank=1 imbalance_ratio=1.331787 max_load_reduction=54626 max_load_reduction_pct=0.833527 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2727, 4, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 10, 5, 0, 0, 0, 5440, 2120, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4438, 5663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 208, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 0, 7939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 34, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 26, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8423, 6449, 7620, 10102, 8372, 7958, 8348, 8264] avg=8192.0000 max=10102 max_rank=3 min=6449 min_rank=1 imbalance_ratio=1.233154 max_load_reduction=55434 max_load_reduction_pct=0.845856 moved_to_echo=32942 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 21, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2735, 0, 7233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 10, 43, 0, 0, 0, 5457, 2120, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 5665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 1, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 3, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8421, 9977, 7673, 6591, 8382, 7945, 8305, 8242] avg=8192.0000 max=9977 max_rank=1 min=6591 min_rank=3 imbalance_ratio=1.217896 max_load_reduction=55559 max_load_reduction_pct=0.847763 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 8183, 0, 0, 0, 0, 14, 0, 0, 0, 0, 3, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 7, 0, 2734, 0, 7455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 110, 79, 0, 0, 0, 5456, 2120, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 5664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7704, 0, 0, 0, 0, 208, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 7528, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7958, 1, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8440, 10205, 7807, 6946, 7913, 8562, 7656, 8007] avg=8192.0000 max=10205 max_rank=1 min=6946 min_rank=3 imbalance_ratio=1.245728 max_load_reduction=55331 max_load_reduction_pct=0.844284 moved_to_echo=32138 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2735, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 433, 740, 0, 0, 0, 5457, 2120, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7578, 5664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7159, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7927, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 7670, 0, 0, 389, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8430, 2738, 9095, 13250, 7367, 8493, 8049, 8114] avg=8192.0000 max=13250 max_rank=3 min=2738 min_rank=1 imbalance_ratio=1.617432 max_load_reduction=52286 max_load_reduction_pct=0.797821 moved_to_echo=32023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 5835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1855, 0, 0, 0, 5456, 2120, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 5665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8441, 8570, 9478, 6170, 8398, 7943, 8297, 8239] avg=8192.0000 max=9478 max_rank=2 min=6170 min_rank=3 imbalance_ratio=1.156982 max_load_reduction=56058 max_load_reduction_pct=0.855377 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 43, 6364, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 41, 4, 2711, 0, 8160, 0, 14, 0, 1, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3, 0, 0, 0, 0, 0, 865, 0, 0, 0, 5426, 2108, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 94, 5602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 204, 11, 0, 25, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 825, 0, 0, 0, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 299, 0, 7885, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 17, 0, 0, 0, 0, 0, 0, 19, 8163, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6648, 10979, 8454, 5738, 8413, 8733, 8318, 8253] avg=8192.0000 max=10979 max_rank=1 min=5738 min_rank=3 imbalance_ratio=1.340210 max_load_reduction=54557 max_load_reduction_pct=0.832474 moved_to_echo=33717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 32, 0, 2735, 0, 8178, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 5456, 2120, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 147, 5663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8121, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 10947, 7635, 5829, 8399, 7944, 8249, 8229] avg=8192.0000 max=10947 max_rank=1 min=5829 min_rank=3 imbalance_ratio=1.336304 max_load_reduction=54589 max_load_reduction_pct=0.832962 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 384, 128, 0, 128, 0, 0, 128, 128, 768, 3712, 0, 0, 0, 43648, 17024, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 21888, 128, 43520, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 20224, 45312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 63616, 0, 0, 128, 0, 128, 0, 128, 128, 0, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 2048, 128, 0, 0, 0, 0, 0, 63488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 63872, 0, 128, 0, 128, 1664, 128, 128, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 128, 384, 128, 64256, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 128, 0, 0, 128, 0, 128, 64768, 128, 0, 512, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 8152, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8164, 13, 0, 0, 0, 2, 0, 0, 39, 0, 618, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 26, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8191, 1, 0, 7, 0, 0, 8191, 1, 0, 1, 0, 0, 0, 0, 0, 1, 43, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6093, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2591, 0, 0, 7681, 8192, 0, 0, 15, 0, 0, 0, 0, 1, 0, 0, 0, 0, 520, 0, 7144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 10, 0, 0, 8031, 13, 0, 0, 3, 0, 0, 0, 0, 0, 575, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 7931, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1512, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 109, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 1, 0, 7981, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 8124, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2, 0, 0, 0, 8185, 0, 0, 41, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 0, 5, 0, 0, 0, 0, 2, 13, 0, 0, 38, 0, 0, 8022, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 0, 8178, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8184, 0, 1, 0, 8190, 0, 1, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 1, 0, 0, 3, 37, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 30, 10, 0, 0, 0, 0, 0, 339, 0, 0, 4, 88, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 34, 0, 6, 0, 0, 0, 0, 3, 34, 0, 0, 0, 0, 0, 0, 6604, 0, 0, 0, 1, 0, 0, 0, 22, 0, 0, 0, 0, 0, 1, 0, 7982, 7956, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 523, 0, 8097, 0, 0, 136, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 14, 8055, 0, 0, 2, 8146, 0, 38, 0, 0, 0, 8143, 27, 0, 0, 0, 0, 0, 0, 21, 37, 901, 116, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 0, 0, 0, 0, 0, 0, 0, 329, 270, 272, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8194, 8192, 8878, 8114, 7919, 8201, 7881] avg=8192.0000 max=8878 max_rank=3 min=7881 min_rank=7 imbalance_ratio=1.083740 max_load_reduction=56658 max_load_reduction_pct=0.864532 moved_to_echo=32115 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 26, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 3, 0, 0, 1, 0, 330, 275, 272, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7861, 1, 0, 1, 0, 0, 0, 0, 0, 1, 43, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8220, 8191, 9054, 7803, 7995, 8201, 7921] avg=8192.0000 max=9054 max_rank=3 min=7803 min_rank=4 imbalance_ratio=1.105225 max_load_reduction=56482 max_load_reduction_pct=0.861847 moved_to_echo=31920 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7185, 0, 0, 0, 0, 0, 0, 0, 329, 277, 272, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 7875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 7912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7835, 13, 0, 0, 0, 2, 0, 0, 39, 0, 618, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8195, 8193, 8089, 8218, 7969, 8202, 8511] avg=8192.0000 max=8511 max_rank=7 min=7969 min_rank=5 imbalance_ratio=1.038940 max_load_reduction=57025 max_load_reduction_pct=0.870132 moved_to_echo=32900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6568, 0, 0, 0, 0, 0, 0, 0, 330, 279, 272, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1512, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 3, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8194, 8192, 7474, 7913, 9430, 8201, 7974] avg=8192.0000 max=9430 max_rank=5 min=7474 min_rank=3 imbalance_ratio=1.151123 max_load_reduction=56106 max_load_reduction_pct=0.856110 moved_to_echo=33518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6093, 0, 0, 18, 0, 0, 0, 0, 324, 261, 237, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2591, 0, 0, 7420, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 520, 0, 6906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 7707, 13, 0, 0, 3, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8199, 8192, 6959, 10026, 7427, 8214, 8308] avg=8192.0000 max=10026 max_rank=4 min=6959 min_rank=3 imbalance_ratio=1.223877 max_load_reduction=55510 max_load_reduction_pct=0.847015 moved_to_echo=33975 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 330, 272, 270, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 1, 0, 7709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 7852, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8195, 8193, 8908, 7929, 7891, 8199, 8065] avg=8192.0000 max=8908 max_rank=3 min=7891 min_rank=5 imbalance_ratio=1.087402 max_load_reduction=56628 max_load_reduction_pct=0.864075 moved_to_echo=32084 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 10, 0, 0, 0, 0, 0, 339, 0, 0, 4, 88, 0, 0, 0, 7924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 34, 0, 6, 0, 0, 0, 0, 8141, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 34, 0, 0, 0, 0, 0, 0, 6604, 0, 0, 0, 1, 0, 0, 0, 332, 275, 273, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 1, 0, 7707, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 7821, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 14, 8055, 0, 0, 2, 5, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7811, 27, 0, 0, 0, 0, 0, 0, 21, 37, 901, 116, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8395, 8096, 8193, 7546, 7731, 8480, 8121, 8974] avg=8192.0000 max=8974 max_rank=7 min=7546 min_rank=3 imbalance_ratio=1.095459 max_load_reduction=56562 max_load_reduction_pct=0.863068 moved_to_echo=33306 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8185, 0, 0, 41, 13, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 0, 5, 0, 0, 0, 330, 273, 272, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 0, 38, 0, 0, 7749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 0, 7904, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8184, 0, 1, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7851, 0, 0, 0, 0, 0, 1, 0, 0, 3, 37, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8243, 8190, 8628, 7802, 8425, 8197, 7902] avg=8192.0000 max=8628 max_rank=3 min=7802 min_rank=4 imbalance_ratio=1.053223 max_load_reduction=56908 max_load_reduction_pct=0.868347 moved_to_echo=32326 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 0, 128, 0, 0, 0, 58368, 0, 0, 128, 128, 0, 128, 0, 2688, 2304, 2176, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 62720, 128, 0, 128, 128, 128, 128, 0, 128, 128, 2560, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 0, 3712, 128, 0, 61952, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 3328, 0, 62208, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 384, 0, 0, 128, 128, 0, 128, 0, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 0, 128, 0, 0, 128, 65408, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 65536, 128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 2, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8146, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8186, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 4, 0, 0, 8184, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 47, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8190, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 39, 0, 0, 0, 0, 0, 0, 15, 9, 0, 0, 0, 0, 8156, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 20, 0, 8188, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7887, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7855, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[23, 0, 0, 8140, 0, 0, 0, 0, 0, 4, 7440, 844, 10, 0, 0, 0, 0, 0, 0, 0, 7796, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 1, 8101, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 14, 8058, 0, 8152, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 25, 38, 0, 0, 0, 2, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 264, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 82, 0, 0, 16, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 28, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 31, 8188, 0, 0, 0, 0, 2, 0, 8191, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 7626, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8185, 0, 0, 0, 0, 0, 1, 8189, 541, 0, 0, 0, 2, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 8159, 0, 7822, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 16, 0, 8181, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 8109, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 7984, 8172, 8325, 8201, 8328, 8229, 8195] avg=8192.0000 max=8328 max_rank=5 min=7984 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32953 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 4, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 47, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8279, 8180, 8279, 8201, 8194, 8116, 8185] avg=8192.0000 max=8279 max_rank=1 min=8102 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32696 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 93, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 15, 9, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 91, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8074, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8322, 8201, 8327, 8201, 8081, 8112, 8188] avg=8192.0000 max=8327 max_rank=3 min=8081 min_rank=5 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32582 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7887, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 448, 0, 0, 0, 0, 0, 0, 8105, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8316, 8337, 8059, 8176, 7756, 8577, 8197] avg=8192.0000 max=8577 max_rank=6 min=7756 min_rank=5 imbalance_ratio=1.046997 max_load_reduction=56959 max_load_reduction_pct=0.869125 moved_to_echo=32706 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8317, 8174, 8341, 8201, 8080, 8141, 8185] avg=8192.0000 max=8341 max_rank=3 min=8080 min_rank=5 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32607 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7349, 844, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7796, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 1, 91, 25, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 14, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 25, 38, 0, 93, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 264, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 1, 0, 0, 0, 8018, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 28, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 11, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 7958, 8076, 8403, 8157, 8390, 8100, 8222] avg=8192.0000 max=8403 max_rank=3 min=7958 min_rank=1 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 31, 0, 0, 0, 0, 0, 2, 0, 8098, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7626, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 90, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8108, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8287, 8174, 7806, 8201, 8613, 8129, 8183] avg=8192.0000 max=8613 max_rank=5 min=7806 min_rank=3 imbalance_ratio=1.051392 max_load_reduction=56923 max_load_reduction_pct=0.868576 moved_to_echo=33126 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8096, 541, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 93, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7822, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 16, 0, 8067, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 1, 8091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8640, 8073, 8222, 7996, 8198, 8094, 8132, 8181] avg=8192.0000 max=8640 max_rank=0 min=7996 min_rank=3 imbalance_ratio=1.054688 max_load_reduction=56896 max_load_reduction_pct=0.868164 moved_to_echo=32605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 64512, 0, 128, 0, 128, 0, 0, 128, 128, 0, 128, 128, 768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 896, 384, 128, 0, 0, 128, 128, 0, 128, 0, 128, 128, 64256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 128, 128, 512, 0, 128, 128, 0, 0, 128, 64768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 0, 0, 128, 128, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[64128, 0, 0, 128, 0, 128, 128, 0, 0, 128, 128, 0, 128, 128, 128, 0, 768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 0, 0, 0, 0, 128, 128, 64128, 1408, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 256, 128, 0, 128, 128, 128, 0, 128, 0, 128, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7453, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7695, 0, 513, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7287, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 21, 0, 0, 0, 0, 0, 0, 7915, 0, 388, 1, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 1, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7249, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 2956, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6561, 0, 826, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2007, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6744, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6667, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 2624, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6894, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 5, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7296, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 2156, 0, 0, 43, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6873, 267, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 1, 0, 0, 0, 0, 7878, 0, 600, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 4, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6362, 0, 4, 207, 0, 0, 2, 0, 0, 0, 0, 2318, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 12, 0, 1, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7794, 2, 385, 0, 0, 0, 3, 0, 1, 0, 0, 1248, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8184] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[23, 0, 0, 2334, 27, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 9, 0, 0, 0, 0, 17, 9, 0, 0, 5, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 2, 0, 23, 7571, 0, 0, 50, 0, 0, 0, 0, 7825, 0, 0, 1, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 2072, 0, 25, 0, 8144, 36, 0, 0, 0, 0, 0, 13, 0, 2, 0, 0, 0, 0, 23, 7395, 24, 8, 0, 0, 59, 0, 0, 35, 11, 24, 5055, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 8107] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7453, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1364, 714, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7345, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7695, 0, 513, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6741, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9812, 7967, 8200, 8341, 8201, 7970, 6839, 8206] avg=8192.0000 max=9812 max_rank=0 min=6839 min_rank=6 imbalance_ratio=1.197754 max_load_reduction=55724 max_load_reduction_pct=0.850281 moved_to_echo=31216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7287, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1220, 714, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7338, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 21, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7915, 0, 388, 1, 0, 0, 0, 0, 132, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6029, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9713, 7414, 8234, 8439, 8205, 7894, 7435, 8202] avg=8192.0000 max=9713 max_rank=0 min=7414 min_rank=1 imbalance_ratio=1.185669 max_load_reduction=55823 max_load_reduction_pct=0.851791 moved_to_echo=31736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6744, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 714, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7345, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5545, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 2624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8802, 7742, 8113, 8303, 8197, 7970, 8200, 8209] avg=8192.0000 max=8802 max_rank=0 min=7742 min_rank=1 imbalance_ratio=1.074463 max_load_reduction=56734 max_load_reduction_pct=0.865692 moved_to_echo=32576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2956, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1108, 714, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7345, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6561, 0, 826, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 5, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4981, 13050, 8354, 7520, 8195, 7711, 7486, 8239] avg=8192.0000 max=13050 max_rank=1 min=4981 min_rank=0 imbalance_ratio=1.593018 max_load_reduction=52486 max_load_reduction_pct=0.800873 moved_to_echo=31631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6894, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 714, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7343, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 1, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 5, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6069, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 2156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9233, 7417, 8106, 8237, 8204, 7969, 8234, 8136] avg=8192.0000 max=9233 max_rank=0 min=7417 min_rank=1 imbalance_ratio=1.127075 max_load_reduction=56303 max_load_reduction_pct=0.859116 moved_to_echo=32543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6873, 267, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1070, 714, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7339, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 1, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7878, 0, 600, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 7970, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5292, 0, 4, 207, 0, 0, 2, 0, 0, 0, 0, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9134, 7471, 8112, 8610, 8202, 7976, 7823, 8208] avg=8192.0000 max=9134 max_rank=0 min=7471 min_rank=1 imbalance_ratio=1.114990 max_load_reduction=56402 max_load_reduction_pct=0.860626 moved_to_echo=32209 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 0, 2334, 27, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 1248, 687, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 17, 9, 0, 0, 5, 0, 0, 38, 0, 0, 7018, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 2, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 7571, 0, 0, 50, 0, 0, 0, 0, 120, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2072, 0, 25, 0, 7914, 36, 0, 0, 0, 0, 0, 13, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6147, 24, 8, 0, 0, 59, 0, 0, 35, 11, 24, 5055, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 8107, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4564, 7177, 8307, 7765, 8134, 10062, 11397, 8130] avg=8192.0000 max=11397 max_rank=6 min=4564 min_rank=0 imbalance_ratio=1.391235 max_load_reduction=54139 max_load_reduction_pct=0.826096 moved_to_echo=37723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1311, 702, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7218, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, 1, 7966, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6483, 2, 385, 0, 0, 0, 3, 0, 1, 0, 0, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8184, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9297, 7298, 8110, 8321, 8198, 7984, 8122, 8206] avg=8192.0000 max=9297 max_rank=0 min=7298 min_rank=1 imbalance_ratio=1.134888 max_load_reduction=56239 max_load_reduction_pct=0.858139 moved_to_echo=32510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 47744, 896, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 128, 9728, 5760, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 6656, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 0, 0, 58368, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 47872, 128, 512, 256, 0, 128, 128, 0, 128, 128, 128, 17024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[2176, 128, 128, 128, 63488, 128, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 62208, 0, 2432, 128, 0, 0, 0, 0, 1152, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 384, 0, 0, 128, 0, 128, 384, 0, 128, 0, 128, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 1, 0, 8, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 19, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8186, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8182, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 8191, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 0, 102, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8095, 0, 0, 0, 8189, 0, 0, 9, 8119, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 375, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 8184, 0, 0, 0, 8185, 0, 0, 0, 79, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 90, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 333, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7953, 0, 0, 41, 0, 10, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 49, 8142, 0, 2, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 1, 0, 8159, 2, 0, 0, 8098, 0, 0, 0, 8107, 0, 0, 3, 0, 21, 0, 47, 0, 0, 0, 0, 0, 8, 0, 0, 7853, 0, 26, 0, 0, 0, 0, 0, 21, 0, 2, 36, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 4, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 50, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8173, 0, 0, 0, 8190, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8242, 8082, 8212, 8168, 8229, 8196, 8228] avg=8192.0000 max=8242 max_rank=1 min=8082 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 1, 0, 8, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8035, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 19, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8246, 8061, 8221, 8211, 8217, 8195, 8197] avg=8192.0000 max=8246 max_rank=1 min=8061 min_rank=2 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8247, 8054, 8212, 8194, 8228, 8196, 8228] avg=8192.0000 max=8247 max_rank=1 min=8054 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8247, 8101, 8213, 8189, 8187, 8193, 8228] avg=8192.0000 max=8247 max_rank=1 min=8101 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 90, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7955, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7953, 0, 0, 41, 0, 10, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 49, 2, 0, 2, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8159, 2, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 3, 0, 21, 0, 47, 0, 0, 0, 0, 0, 8, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7853, 0, 26, 0, 0, 0, 0, 0, 21, 0, 2, 36, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8192, 8445, 8047, 8240, 8183, 8190, 7975] avg=8192.0000 max=8445 max_rank=2 min=7975 min_rank=7 imbalance_ratio=1.030884 max_load_reduction=57091 max_load_reduction_pct=0.871140 moved_to_echo=32588 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 8038, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8095, 0, 0, 0, 37, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8237, 8220, 8207, 8197, 8142, 8127, 8227] avg=8192.0000 max=8237 max_rank=1 min=8127 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8034, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 79, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 7906, 8454, 8223, 8139, 8139, 8269, 8228] avg=8192.0000 max=8454 max_rank=2 min=7906 min_rank=1 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 4, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 8039, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 5, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8173, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8219, 8119, 8201, 8198, 8211, 8170, 8225] avg=8192.0000 max=8225 max_rank=7 min=8119 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 64256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 65152, 0, 128, 0, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[65408, 0, 0, 128, 128, 128, 0, 128, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 65280, 128, 0, 128, 0, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 0, 65280, 128, 0, 0, 384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[65280, 0, 128, 128, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 65408, 128, 0, 128, 128, 128, 0, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 6231, 0, 8192, 0, 1201, 0, 0, 0, 8192, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 1905, 0, 0, 0, 0, 0, 0, 0, 7044, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 8187, 0, 29, 0, 0, 0, 8191, 0, 0, 36, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 8187, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 0, 0, 2, 0, 0, 0, 0, 0, 7547, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8113, 0, 8192, 0, 109, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 30, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 7759, 0, 8190, 0, 509, 0, 0, 0, 8161, 0, 0, 0, 193, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 7943, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 5703, 0, 8188, 0, 2095, 0, 0, 0, 8186, 1, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 4, 8190, 0, 20, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8112, 264, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 5, 0, 0, 8188, 0, 8178, 0, 0, 0, 8101, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7578, 1159, 0, 0, 0, 1, 0, 0, 1, 7035, 0, 7, 0, 0, 0, 0, 0, 0, 8192, 0, 228, 0, 0, 0, 87, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8185, 0, 0, 0, 17, 8132, 0, 8192, 0, 38, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7569, 465, 0, 0, 0, 3, 0, 0, 0, 8189, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 10, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 0, 0, 0, 24, 0, 0, 7722, 0, 25, 20, 0, 8179, 0, 7782, 0, 0, 0, 0, 0, 7856, 0, 0, 107, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 171, 0, 8136, 0, 335, 0, 0, 0, 0, 0, 0, 0, 15, 0, 16, 29, 0, 0, 7974, 0, 0, 0, 0, 0, 0, 0, 0, 6724, 15, 0, 0, 0, 25, 0, 3, 0, 8064, 0, 0, 0, 19, 201, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 275, 101, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8140, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5956, 0, 0, 0, 1201, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 1905, 0, 0, 0, 0, 0, 0, 0, 7044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8239, 8241, 7201, 8244, 8126, 9029, 8342] avg=8192.0000 max=9029 max_rank=6 min=7201 min_rank=3 imbalance_ratio=1.102173 max_load_reduction=56507 max_load_reduction_pct=0.862228 moved_to_echo=33741 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7765, 343, 101, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 1, 0, 0, 0, 8109, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 0, 0, 0, 509, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8190, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 80, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8215, 8246, 8118, 8246, 8117, 8275, 8095] avg=8192.0000 max=8275 max_rank=6 min=8095 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32733 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 359, 101, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7754, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 30, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8482, 8241, 8247, 7863, 8244, 8126, 8236, 8097] avg=8192.0000 max=8482 max_rank=0 min=7863 min_rank=3 imbalance_ratio=1.035400 max_load_reduction=57054 max_load_reduction_pct=0.870575 moved_to_echo=32703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 0, 7408, 0, 101, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 77, 1159, 0, 0, 0, 1, 0, 0, 1, 7035, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 228, 0, 0, 0, 87, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7924, 8142, 8228, 8185, 8240, 8123, 8285, 8409] avg=8192.0000 max=8409 max_rank=7 min=7924 min_rank=0 imbalance_ratio=1.026489 max_load_reduction=57127 max_load_reduction_pct=0.871689 moved_to_echo=33057 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7399, 360, 101, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 8135, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8185, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7772, 0, 0, 0, 38, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 465, 0, 0, 0, 3, 0, 0, 0, 8189, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7874, 8280, 8247, 7828, 8244, 8121, 8737, 8205] avg=8192.0000 max=8737 max_rank=6 min=7828 min_rank=3 imbalance_ratio=1.066528 max_load_reduction=56799 max_load_reduction_pct=0.866684 moved_to_echo=33307 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7930, 252, 101, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5451, 0, 0, 0, 2095, 0, 0, 0, 0, 1, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 8188, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8121, 0, 20, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 264, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 8234, 8239, 7743, 8246, 8146, 8531, 8093] avg=8192.0000 max=8531 max_rank=6 min=7743 min_rank=3 imbalance_ratio=1.041382 max_load_reduction=57005 max_load_reduction_pct=0.869827 moved_to_echo=33016 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7789, 362, 101, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8139, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7830, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 36, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 8118, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 2, 0, 0, 0, 0, 0, 7547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8241, 8245, 8675, 8239, 8145, 7630, 8088] avg=8192.0000 max=8675 max_rank=3 min=7630 min_rank=6 imbalance_ratio=1.058960 max_load_reduction=56861 max_load_reduction_pct=0.867630 moved_to_echo=32102 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 10, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7799, 365, 105, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7807, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 0, 0, 0, 24, 0, 0, 7722, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 20, 0, 7814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7782, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 8066, 0, 335, 0, 0, 0, 0, 0, 0, 0, 15, 0, 16, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 6724, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 3, 0, 7959, 0, 0, 0, 19, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8341, 7944, 7843, 9923, 7833, 8632, 6813, 8207] avg=8192.0000 max=9923 max_rank=3 min=6813 min_rank=6 imbalance_ratio=1.211304 max_load_reduction=55613 max_load_reduction_pct=0.848587 moved_to_echo=31485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 640, 3840, 0, 128, 0, 128, 0, 0, 128, 61056, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 50048, 0, 0, 0, 12160, 0, 0, 0, 0, 128, 0, 256, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 64640, 128, 512, 0, 128, 384, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 0, 0, 128, 0, 512, 0, 0, 0, 0, 61824, 2432, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 64768, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 65024, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 128, 0, 65152, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 65024, 0, 384, 128, 0, 0, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True + [2026-08-26 16:28:30] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 435.7 | throughput per GPU (TFLOP/s/GPU): 415.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338538E+01 | loss scale: 1.0 | grad norm: 19.947 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 44, 150, 216, 1626, 0, 0, 3247, 11, 0, 3, 5, 117, 23, 0, 206, 0, 0, 746, 0, 3, 1809, 4371, 0, 0, 8189, 1862, 7834, 0, 1144, 2, 20, 259, 0, 5, 312, 0, 18, 82, 0, 669, 471, 0, 649, 0, 8172, 0, 2, 42, 0, 0, 4, 0, 1, 318, 0, 34, 84, 0, 0, 0, 8, 601, 0, 0, 0, 679, 0, 2, 4, 756, 0, 695, 784, 6, 5, 265, 0, 0, 1, 0, 4, 61, 0, 0, 850, 0, 219, 0, 4, 4, 0, 7, 0, 0, 187, 0, 0, 19, 5, 0, 0, 221, 517, 646, 3, 1, 60, 3652, 1, 0, 4932, 0, 0, 0, 12, 675, 0, 0, 0, 6526, 373, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[5, 0, 0, 26, 155, 227, 1839, 0, 0, 3090, 3, 3, 19, 2, 97, 2, 0, 155, 0, 27, 721, 2, 0, 2525, 3477, 0, 0, 8172, 1802, 7344, 6, 1016, 5, 60, 229, 2, 2, 281, 0, 25, 65, 0, 477, 147, 0, 879, 9, 8172, 0, 1, 21, 0, 0, 5, 0, 0, 185, 1, 28, 291, 0, 0, 0, 27, 1183, 3, 0, 0, 723, 0, 3, 2, 794, 0, 805, 1023, 30, 6, 125, 0, 0, 0, 0, 1, 24, 0, 0, 921, 0, 261, 1, 4, 16, 2, 0, 7, 0, 86, 0, 0, 19, 4, 9, 2, 199, 967, 369, 1, 4, 97, 4371, 2, 1, 4208, 0, 3, 0, 13, 1022, 0, 1, 8, 5918, 667, 0, 3, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 194, 227, 1834, 0, 8, 3737, 1, 0, 26, 2, 32, 4, 0, 135, 0, 7, 940, 0, 0, 2099, 4325, 1, 0, 8176, 1276, 7486, 1, 226, 23, 49, 142, 0, 11, 433, 0, 6, 142, 2, 1017, 147, 0, 697, 39, 8179, 0, 2, 48, 1, 0, 5, 0, 3, 92, 0, 33, 259, 0, 2, 0, 10, 760, 0, 0, 0, 569, 0, 44, 2, 949, 0, 908, 857, 3, 9, 325, 0, 1, 1, 0, 3, 36, 0, 0, 562, 1, 316, 0, 5, 14, 2, 0, 3, 0, 197, 3, 0, 14, 8, 9, 5, 37, 934, 425, 20, 0, 97, 4090, 0, 0, 4189, 2, 0, 1, 36, 1653, 0, 2, 1, 5764, 584, 0, 2, 3, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 185, 96, 1582, 0, 0, 2997, 3, 2, 21, 5, 124, 32, 1, 294, 0, 18, 794, 0, 3, 1566, 4082, 2, 1, 8171, 1280, 7390, 0, 795, 34, 32, 150, 0, 5, 306, 0, 12, 80, 1, 1401, 185, 1, 680, 32, 8171, 0, 1, 65, 0, 0, 0, 0, 1, 144, 1, 19, 374, 0, 1, 0, 17, 997, 0, 0, 0, 584, 0, 9, 1, 774, 0, 1227, 1243, 21, 4, 150, 3, 17, 8, 2, 0, 56, 0, 2, 776, 6, 343, 0, 3, 35, 4, 12, 0, 2, 301, 5, 1, 22, 6, 1, 0, 208, 888, 691, 16, 1, 174, 3727, 0, 1, 4578, 0, 0, 0, 23, 854, 0, 0, 13, 6182, 373, 2, 18, 2, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 229, 195, 1393, 4, 6, 2008, 1, 2, 13, 0, 115, 13, 1, 129, 0, 13, 1125, 0, 0, 1946, 4040, 3, 0, 8165, 1417, 7540, 5, 809, 10, 35, 224, 0, 3, 214, 1, 16, 53, 3, 572, 132, 2, 1533, 3, 8167, 0, 1, 23, 2, 0, 6, 0, 6, 220, 1, 13, 241, 0, 2, 0, 28, 652, 0, 0, 0, 496, 0, 6, 12, 1032, 0, 763, 1296, 36, 21, 448, 0, 0, 0, 0, 0, 86, 0, 0, 1327, 0, 195, 1, 28, 1, 5, 0, 6, 1, 569, 0, 4, 14, 2, 0, 5, 130, 1117, 434, 12, 4, 171, 3560, 0, 0, 4796, 3, 0, 0, 8, 1199, 0, 2, 2, 6075, 319, 1, 0, 5, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4, 1, 0, 7, 94, 192, 1794, 0, 0, 2108, 1, 0, 4, 2, 63, 5, 0, 188, 0, 10, 922, 0, 3, 2014, 3745, 1, 0, 8176, 1481, 7494, 1, 1207, 4, 36, 220, 0, 1, 91, 0, 4, 65, 0, 676, 184, 0, 686, 4, 8143, 0, 0, 29, 1, 0, 1, 0, 0, 204, 0, 39, 178, 1, 0, 0, 27, 598, 0, 0, 0, 576, 0, 11, 1, 594, 0, 1406, 742, 13, 16, 212, 0, 0, 0, 2, 1, 78, 0, 1, 760, 1, 281, 0, 1, 10, 0, 3, 0, 0, 301, 1, 0, 12, 1, 2, 1, 224, 819, 621, 2, 0, 125, 4380, 0, 1, 5361, 0, 2, 0, 9, 1373, 0, 0, 0, 6363, 518, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 0, 12, 159, 341, 1605, 0, 0, 2595, 2, 3, 5, 3, 142, 14, 0, 136, 0, 20, 633, 0, 5, 1794, 4313, 0, 3, 8163, 1629, 7820, 11, 1128, 16, 61, 97, 0, 0, 187, 0, 9, 29, 2, 1092, 342, 0, 644, 2, 8157, 2, 2, 62, 0, 5, 1, 0, 2, 100, 3, 56, 251, 0, 1, 0, 13, 1054, 0, 0, 0, 398, 1, 18, 1, 904, 1, 1150, 1233, 15, 8, 145, 0, 2, 3, 1, 0, 104, 0, 0, 884, 7, 168, 0, 5, 2, 0, 0, 5, 0, 159, 8, 3, 11, 6, 0, 0, 162, 1103, 543, 9, 1, 74, 3407, 2, 3, 4145, 0, 2, 0, 9, 1202, 0, 1, 3, 6358, 537, 0, 1, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 180, 211, 1917, 0, 0, 2569, 2, 0, 7, 2, 78, 16, 0, 61, 0, 3, 1239, 0, 10, 2410, 3153, 0, 0, 8159, 1854, 7306, 2, 1441, 11, 41, 316, 0, 9, 179, 0, 1, 211, 0, 639, 105, 0, 642, 5, 8171, 0, 6, 22, 0, 0, 2, 0, 0, 239, 0, 19, 126, 0, 0, 4, 12, 820, 0, 0, 0, 563, 0, 4, 3, 954, 0, 859, 494, 22, 2, 246, 0, 0, 1, 0, 1, 43, 0, 2, 529, 0, 112, 0, 16, 4, 1, 2, 0, 0, 252, 1, 0, 16, 2, 2, 0, 120, 288, 456, 20, 0, 132, 4761, 1, 2, 5277, 0, 0, 0, 17, 744, 0, 0, 3, 7007, 362, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 26, 155, 227, 1839, 0, 0, 3090, 3, 3, 19, 2, 97, 2, 1524, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 27, 721, 2, 0, 2525, 2254, 0, 0, 0, 1802, 0, 6, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 60, 229, 2, 2, 281, 0, 25, 65, 0, 477, 147, 0, 879, 9, 5734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 21, 0, 0, 5, 0, 0, 185, 1, 28, 291, 0, 0, 0, 27, 7682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1183, 3, 0, 0, 723, 0, 3, 2, 794, 0, 805, 1023, 30, 6, 125, 0, 2583, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 24, 0, 0, 921, 0, 261, 1, 4, 16, 2, 0, 7, 6385, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 19, 4, 9, 2, 199, 967, 369, 1, 4, 97, 4371, 2, 1223, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4208, 0, 3, 0, 13, 1022, 0, 1, 8, 1811, 667, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8546, 8508, 7915, 8241, 8239, 8112, 8237, 7738] avg=8192.0000 max=8546 max_rank=0 min=7738 min_rank=7 imbalance_ratio=1.043213 max_load_reduction=56990 max_load_reduction_pct=0.869598 moved_to_echo=32326 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 44, 150, 216, 1626, 0, 0, 3247, 11, 0, 3, 5, 117, 23, 1680, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 746, 0, 3, 1809, 2833, 0, 0, 0, 1862, 0, 0, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 259, 0, 5, 312, 0, 18, 82, 0, 669, 471, 0, 649, 0, 5734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 42, 0, 0, 4, 0, 1, 318, 0, 34, 84, 0, 0, 0, 8, 7698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 0, 0, 679, 0, 2, 4, 756, 0, 695, 784, 6, 5, 265, 0, 2848, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 61, 0, 0, 850, 0, 219, 0, 4, 4, 0, 7, 0, 6811, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 19, 5, 0, 0, 221, 517, 646, 3, 1, 60, 3652, 1, 1538, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4932, 0, 0, 0, 12, 675, 0, 0, 0, 1998, 373, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8676, 8603, 8221, 8191, 7668, 8452, 7734, 7991] avg=8192.0000 max=8676 max_rank=0 min=7668 min_rank=4 imbalance_ratio=1.059082 max_load_reduction=56860 max_load_reduction_pct=0.867615 moved_to_echo=31845 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 185, 96, 1582, 0, 0, 2997, 3, 2, 21, 5, 124, 32, 1592, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 294, 0, 18, 794, 0, 3, 1566, 2646, 2, 1, 0, 1280, 0, 0, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 32, 150, 0, 5, 306, 0, 12, 80, 1, 1401, 185, 1, 680, 32, 5733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 65, 0, 0, 0, 0, 1, 144, 1, 19, 374, 0, 1, 0, 17, 7682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 0, 0, 584, 0, 9, 1, 774, 0, 1227, 1243, 21, 4, 150, 3, 2698, 966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8, 2, 0, 56, 0, 2, 776, 6, 343, 0, 3, 35, 4, 12, 0, 6424, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 301, 5, 1, 22, 6, 1, 0, 208, 888, 691, 16, 1, 174, 3727, 0, 1436, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4578, 0, 0, 0, 23, 854, 0, 0, 13, 1892, 373, 2, 18, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 7400, 8652, 8305, 8677, 8177, 8363, 7760] avg=8192.0000 max=8677 max_rank=4 min=7400 min_rank=1 imbalance_ratio=1.059204 max_load_reduction=56859 max_load_reduction_pct=0.867599 moved_to_echo=32977 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 194, 227, 1834, 0, 8, 3737, 1, 0, 26, 2, 32, 4, 1484, 1556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 7, 940, 0, 0, 2099, 2804, 1, 0, 0, 1276, 0, 1, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 49, 142, 0, 11, 433, 0, 6, 142, 2, 1017, 147, 0, 697, 39, 5738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 48, 1, 0, 5, 0, 3, 92, 0, 33, 259, 0, 2, 0, 10, 7687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 0, 0, 0, 569, 0, 44, 2, 949, 0, 908, 857, 3, 9, 325, 0, 2515, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3, 36, 0, 0, 562, 1, 316, 0, 5, 14, 2, 0, 3, 6507, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 3, 0, 14, 8, 9, 5, 37, 934, 425, 20, 0, 97, 4090, 0, 1521, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4189, 2, 0, 1, 36, 1653, 0, 2, 1, 1765, 584, 0, 2, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9114, 7489, 8446, 8142, 7920, 7940, 8245, 8240] avg=8192.0000 max=9114 max_rank=0 min=7489 min_rank=1 imbalance_ratio=1.112549 max_load_reduction=56422 max_load_reduction_pct=0.860931 moved_to_echo=32345 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 12, 159, 341, 1605, 0, 0, 2595, 2, 3, 5, 3, 142, 14, 1637, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 20, 633, 0, 5, 1794, 2796, 0, 3, 0, 1629, 0, 11, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 61, 97, 0, 0, 187, 0, 9, 29, 2, 1092, 342, 0, 644, 2, 5723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 62, 0, 5, 1, 0, 2, 100, 3, 56, 251, 0, 1, 0, 13, 7674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 0, 398, 1, 18, 1, 904, 1, 1150, 1233, 15, 8, 145, 0, 2775, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 1, 0, 104, 0, 0, 884, 7, 168, 0, 5, 2, 0, 0, 5, 6799, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 8, 3, 11, 6, 0, 0, 162, 1103, 543, 9, 1, 74, 3407, 2, 1517, 883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4145, 0, 2, 0, 9, 1202, 0, 1, 3, 1946, 537, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8072, 8155, 8204, 8172, 8724, 8469, 7888, 7852] avg=8192.0000 max=8724 max_rank=4 min=7852 min_rank=7 imbalance_ratio=1.064941 max_load_reduction=56812 max_load_reduction_pct=0.866882 moved_to_echo=32933 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 1, 0, 7, 94, 192, 1794, 0, 0, 2108, 1, 0, 4, 2, 63, 5, 1638, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 10, 922, 0, 3, 2014, 2428, 1, 0, 0, 1481, 0, 1, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 36, 220, 0, 1, 91, 0, 4, 65, 0, 676, 184, 0, 686, 4, 5713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1, 0, 1, 0, 0, 204, 0, 39, 178, 1, 0, 0, 27, 7686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 576, 0, 11, 1, 594, 0, 1406, 742, 13, 16, 212, 0, 2777, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 78, 0, 1, 760, 1, 281, 0, 1, 10, 0, 3, 0, 6514, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 1, 0, 12, 1, 2, 1, 224, 819, 621, 2, 0, 125, 4380, 0, 1317, 881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5361, 0, 2, 0, 9, 1373, 0, 0, 0, 1948, 518, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7462, 8255, 7684, 8166, 7926, 8142, 8687, 9214] avg=8192.0000 max=9214 max_rank=7 min=7462 min_rank=0 imbalance_ratio=1.124756 max_load_reduction=56322 max_load_reduction_pct=0.859406 moved_to_echo=33969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 229, 195, 1393, 4, 6, 2008, 1, 2, 13, 0, 115, 13, 1564, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 129, 0, 13, 1125, 0, 0, 1946, 2619, 3, 0, 0, 1417, 0, 5, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 35, 224, 0, 3, 214, 1, 16, 53, 3, 572, 132, 2, 1533, 3, 5730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 23, 2, 0, 6, 0, 6, 220, 1, 13, 241, 0, 2, 0, 28, 7675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 496, 0, 6, 12, 1032, 0, 763, 1296, 36, 21, 448, 0, 2651, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 1327, 0, 195, 1, 28, 1, 5, 0, 6, 6555, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 569, 0, 4, 14, 2, 0, 5, 130, 1117, 434, 12, 4, 171, 3560, 0, 1421, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796, 3, 0, 0, 8, 1199, 0, 2, 2, 1860, 319, 1, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7104, 8067, 8531, 8218, 8398, 8694, 8328, 8196] avg=8192.0000 max=8694 max_rank=5 min=7104 min_rank=0 imbalance_ratio=1.061279 max_load_reduction=56842 max_load_reduction_pct=0.867340 moved_to_echo=33616 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 180, 211, 1917, 0, 0, 2569, 2, 0, 7, 2, 78, 16, 1809, 1558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 3, 1239, 0, 10, 2410, 2039, 0, 0, 0, 1854, 0, 2, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 41, 316, 0, 9, 179, 0, 1, 211, 0, 639, 105, 0, 642, 5, 5724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 22, 0, 0, 2, 0, 0, 239, 0, 19, 126, 0, 0, 4, 12, 7671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 0, 0, 0, 563, 0, 4, 3, 954, 0, 859, 494, 22, 2, 246, 0, 3062, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 43, 0, 2, 529, 0, 112, 0, 16, 4, 1, 2, 0, 6351, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 1, 0, 16, 2, 2, 0, 120, 288, 456, 20, 0, 132, 4761, 1, 1114, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5277, 0, 0, 0, 17, 744, 0, 0, 3, 2136, 362, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 9059, 7883, 8101, 7984, 7550, 8054, 8545] avg=8192.0000 max=9059 max_rank=1 min=7550 min_rank=5 imbalance_ratio=1.105835 max_load_reduction=56477 max_load_reduction_pct=0.861771 moved_to_echo=32133 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 1408, 1792, 13696, 128, 128, 22400, 128, 128, 128, 128, 768, 128, 12928, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 1408, 0, 128, 7168, 128, 128, 16256, 20480, 128, 128, 0, 12672, 0, 128, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 2176, 128, 128, 128, 128, 128, 128, 1408, 6656, 4224, 128, 128, 1024, 32000, 128, 11136, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[6784, 128, 0, 0, 4608, 128, 128, 128, 6784, 128, 7936, 7680, 256, 128, 1920, 128, 22016, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 37504, 128, 128, 128, 128, 8832, 0, 128, 128, 15360, 3840, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 384, 1664, 128, 128, 2048, 128, 128, 768, 128, 6656, 1792, 128, 6528, 128, 45952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 512, 0, 128, 6656, 128, 1920, 128, 128, 128, 128, 128, 128, 52352, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 384, 128, 128, 128, 0, 128, 1536, 128, 256, 1920, 128, 128, 128, 256, 61568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8180, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8189, 40, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8188, 0, 3706, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 3210, 0, 0, 8154, 0, 2, 0, 2, 0, 17, 1, 0, 0, 0, 0, 1256, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8099, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 3, 0, 16, 0, 0, 0, 8191, 0, 0, 8180, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 38, 0, 13, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 8190, 0, 1293, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1849, 0, 0, 8150, 0, 4, 0, 0, 0, 57, 0, 2, 0, 0, 0, 5029, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8142, 0, 14, 0, 1, 0, 0, 1, 1, 5, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 14, 0, 8171, 0, 0, 2, 3, 0, 0, 0, 2, 1, 0, 1, 0, 0, 32, 0, 0, 7, 0, 0, 0, 3, 0, 6, 1, 4, 2, 101, 0, 0, 0, 0, 0, 9, 2, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 1, 4, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 8175, 0, 1, 2, 0, 0, 1, 8179, 0, 4370, 0, 0, 4, 0, 0, 0, 60, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2109, 0, 3, 7961, 2, 0, 3, 0, 0, 79, 1, 0, 3, 0, 0, 1688, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 10, 0, 2, 0, 8192, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2813, 0, 0, 8087, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 4908, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 142, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 2729, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3496, 0, 0, 8192, 0, 2, 0, 0, 0, 156, 0, 0, 0, 0, 0, 1646, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8165, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4, 70, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 9, 0, 1, 0, 8185, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 569, 0, 0, 8144, 0, 24, 0, 0, 0, 370, 0, 0, 0, 0, 0, 6737, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8183, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8183, 19, 9, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 5, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8147, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 1, 2, 0, 0, 4, 8192, 0, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1682, 0, 0, 8091, 0, 1, 0, 0, 0, 409, 0, 0, 0, 1, 0, 3392, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8142, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 8190, 0, 0, 8182, 7, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 8, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 13, 0, 0, 0, 8186, 0, 286, 19, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1277, 0, 0, 8186, 0, 14, 0, 0, 0, 51, 1, 2, 0, 0, 0, 6472, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8099, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 3, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 8151, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 38, 0, 13, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 1293, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1849, 0, 0, 6079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 57, 0, 2, 0, 0, 0, 5029, 0, 0, 0, 0, 2071, 2003, 32, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8199, 8233, 8162, 8193, 7473, 7928, 9224] avg=8192.0000 max=9224 max_rank=7 min=7473 min_rank=5 imbalance_ratio=1.125977 max_load_reduction=56312 max_load_reduction_pct=0.859253 moved_to_echo=32818 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 10, 0, 2, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2813, 0, 0, 6032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 4908, 0, 0, 0, 0, 2055, 2005, 32, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8162, 8177, 8214, 8192, 6538, 8845, 9195] avg=8192.0000 max=9195 max_rank=7 min=6538 min_rank=5 imbalance_ratio=1.122437 max_load_reduction=56341 max_load_reduction_pct=0.859695 moved_to_echo=32770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8180, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 40, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6182, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 3210, 0, 0, 6082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 17, 1, 0, 0, 0, 0, 1256, 0, 0, 0, 0, 2072, 2005, 32, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8200, 8181, 8153, 8193, 9888, 9303, 5416] avg=8192.0000 max=9888 max_rank=5 min=5416 min_rank=7 imbalance_ratio=1.207031 max_load_reduction=55648 max_load_reduction_pct=0.849121 moved_to_echo=32800 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 142, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6185, 0, 0, 0, 0, 0, 0, 0, 0, 2729, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3496, 0, 0, 6110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 156, 0, 0, 0, 0, 0, 1646, 0, 0, 0, 0, 2082, 2006, 32, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8299, 8195, 8171, 8194, 8924, 9606, 5951] avg=8192.0000 max=9606 max_rank=6 min=5951 min_rank=7 imbalance_ratio=1.172607 max_load_reduction=55930 max_load_reduction_pct=0.853424 moved_to_echo=32675 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8142, 0, 14, 0, 1, 0, 0, 1, 1, 5, 0, 11, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 8142, 0, 0, 2, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 32, 0, 0, 7, 0, 0, 0, 3, 0, 6, 1, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 101, 0, 0, 0, 0, 0, 9, 2, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6173, 0, 1, 2, 0, 0, 1, 0, 0, 4370, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2109, 0, 3, 5938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 79, 1, 0, 3, 0, 0, 1688, 0, 0, 0, 4, 2023, 2002, 32, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8164, 8213, 8244, 8193, 10552, 8113, 5864] avg=8192.0000 max=10552 max_rank=5 min=5864 min_rank=7 imbalance_ratio=1.288086 max_load_reduction=54984 max_load_reduction_pct=0.838989 moved_to_echo=32722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8165, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 70, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6174, 0, 0, 9, 0, 1, 0, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 569, 0, 0, 6074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 370, 0, 0, 0, 0, 0, 6737, 0, 0, 0, 0, 2070, 2002, 32, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8163, 8172, 8260, 8191, 6657, 6646, 11262] avg=8192.0000 max=11262 max_rank=7 min=6646 min_rank=6 imbalance_ratio=1.374756 max_load_reduction=54274 max_load_reduction_pct=0.828156 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8183, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8115, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6184, 0, 1, 2, 0, 0, 4, 0, 0, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1682, 0, 0, 6035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 409, 0, 0, 0, 1, 0, 3392, 0, 0, 0, 0, 2056, 2005, 32, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8182, 8181, 8132, 8189, 9005, 7717, 7922] avg=8192.0000 max=9005 max_rank=5 min=7717 min_rank=6 imbalance_ratio=1.099243 max_load_reduction=56531 max_load_reduction_pct=0.862595 moved_to_echo=32833 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8142, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 2, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 8142, 7, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6179, 0, 0, 13, 0, 0, 0, 0, 0, 286, 19, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1277, 0, 0, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 51, 1, 2, 0, 0, 0, 6472, 1, 0, 0, 0, 2086, 2008, 37, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8167, 8184, 8200, 8191, 6499, 7378, 10702] avg=8192.0000 max=10702 max_rank=7 min=6499 min_rank=5 imbalance_ratio=1.306396 max_load_reduction=54834 max_load_reduction_pct=0.836700 moved_to_echo=32770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 0, 1408, 128, 128, 128, 128, 0, 31232, 128, 0, 0, 128, 16640, 16128, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 128, 0, 17024, 0, 128, 48512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 49536, 0, 128, 128, 0, 128, 128, 0, 0, 16128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 256, 256, 128, 0, 128, 0, 0, 128, 128, 0, 0, 65280, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[65280, 128, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 0, 0, 128, 0, 65280, 256, 128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8192, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[8072, 3, 0, 0, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 8133, 8172, 0, 1, 1, 0, 0, 4, 8163, 0, 0, 0, 4, 0, 0, 0, 0, 0, 38, 10, 0, 0, 0, 12, 0, 0, 0, 0, 1, 17, 0, 8172, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 8166, 6, 1, 8158, 0, 0, 0, 0, 12, 0, 0, 0, 0, 5, 0, 0, 0, 0, 110, 0, 11, 0, 7, 0, 1, 0, 0, 0, 0, 20, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 1, 0, 0, 0, 0, 0, 0, 0, 8180, 8191, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 3, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8188, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[8163, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 139, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 8182, 0, 0, 1, 1, 0, 0, 8142, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 0, 4, 0, 8184, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8140, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8188, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 1, 0, 1, 1, 0, 0, 0, 8181, 8191, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[8158, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 2, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[8137, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 187, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 8185, 0, 0, 0, 0, 0, 3, 8179, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8190, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 1, 8033, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7, 0, 4, 0, 1, 0, 1, 0, 1, 4, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8163, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 139, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8142, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 0, 4, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 8140, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 8045, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8140, 8172, 8195, 8189, 8303, 8156, 8191] avg=8192.0000 max=8303 max_rank=5 min=8140 min_rank=1 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32839 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 8045, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8187, 8196, 8186, 8193, 7939, 8432, 8185] avg=8192.0000 max=8432 max_rank=6 min=7939 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8192, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 8178, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 146, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8045, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8188, 8145, 8199, 8194, 8230, 8158, 8196] avg=8192.0000 max=8230 max_rank=5 min=8145 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8072, 3, 0, 0, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 3, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4, 8163, 0, 0, 0, 4, 0, 0, 0, 0, 0, 38, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 12, 0, 0, 0, 0, 1, 17, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8172, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 146, 6, 1, 8158, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 110, 0, 11, 0, 7, 0, 1, 0, 0, 8019, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8046, 8222, 8201, 8182, 8326, 8235, 8210] avg=8192.0000 max=8326 max_rank=5 min=8046 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8188, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 1, 0, 1, 1, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8044, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8202, 8203, 8189, 8192, 8222, 8128, 8184] avg=8192.0000 max=8222 max_rank=5 min=8128 min_rank=6 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8158, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8043, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8180, 8203, 8204, 8194, 8244, 8135, 8184] avg=8192.0000 max=8244 max_rank=5 min=8135 min_rank=6 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 1, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8170, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 8041, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8324, 8193, 8174, 8199, 8095, 8158, 8188] avg=8192.0000 max=8324 max_rank=1 min=8095 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8137, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 187, 0, 1, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8179, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8171, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 1, 8033, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7, 0, 0, 8038, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 1, 0, 1, 0, 1, 4, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8269, 8202, 8188, 8193, 8177, 8134, 8198] avg=8192.0000 max=8269 max_rank=1 min=8134 min_rank=6 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 128, 0, 128, 0, 128, 128, 512, 0, 0, 64384, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 0, 128, 128, 128, 65408, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 1280, 128, 128, 64384, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 65408, 128, 0, 0, 128, 0, 128, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 1024, 128, 128, 128, 128, 128, 0, 0, 0, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[65408, 128, 128, 128, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7914, 0, 0, 0, 7955, 0, 0, 0, 0, 8182, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 193, 0, 0, 74, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 8190, 293, 0, 0, 0, 0, 8, 0, 8182, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 6, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8181, 0, 0, 0, 8175, 11, 0, 0, 0, 7966, 0, 8136, 0, 0, 1, 10, 3, 0, 0, 0, 0, 0, 13, 0, 4, 0, 0, 11, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 8123, 0, 0, 0, 0, 18, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 9, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 1, 0, 0, 1, 0, 75, 8169, 0, 0, 0, 1, 2, 0, 57, 8161, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7838, 0, 0, 0, 7692, 0, 0, 0, 0, 7861, 0, 8192, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 17, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8120, 0, 0, 0, 8141, 0, 0, 0, 0, 8182, 0, 8182, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 71, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 8191, 7, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 1, 8025, 0, 0, 0, 0, 8110, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 18, 0, 0, 302, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 0, 8191, 184, 1, 0, 0, 0, 0, 3, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7917, 0, 0, 0, 8143, 0, 0, 0, 0, 7745, 0, 8187, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 0, 0, 5, 0, 2, 280, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 1, 8189, 54, 0, 0, 1, 0, 0, 3, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7740, 0, 0, 0, 8165, 0, 0, 0, 0, 8182, 0, 8185, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 104, 0, 0, 521, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 7930, 0, 0, 0, 0, 0, 0, 8191, 93, 0, 0, 0, 0, 0, 0, 8181, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 8188, 0, 0, 0, 8148, 0, 0, 0, 0, 8061, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8056, 0, 0, 0, 0, 69, 0, 0, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 0, 3, 8191, 2, 0, 0, 0, 0, 0, 2, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7914, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 8182, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 0, 0, 0, 7991, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 74, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7595, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 8053, 0, 60, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 8, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8301, 8013, 8169, 8057, 8207, 8276, 8390] avg=8192.0000 max=8390 max_rank=7 min=8013 min_rank=2 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=32930 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7838, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 7861, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 3, 0, 0, 0, 7999, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7344, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 60, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8033, 8214, 8467, 8222, 8118, 8189, 8186, 8107] avg=8192.0000 max=8467 max_rank=2 min=8033 min_rank=0 imbalance_ratio=1.033569 max_load_reduction=57069 max_load_reduction_pct=0.870804 moved_to_echo=32600 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8120, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 8182, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1, 0, 0, 0, 7989, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7773, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8149, 0, 60, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8328, 8018, 8205, 8105, 8191, 8255, 8106] avg=8192.0000 max=8328 max_rank=0 min=8018 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32657 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8181, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 11, 0, 0, 0, 7966, 0, 0, 0, 0, 1, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4, 0, 0, 11, 0, 314, 0, 0, 0, 0, 7943, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 8123, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 7805, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 9, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 1, 60, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 75, 0, 0, 0, 0, 1, 2, 0, 57, 8068, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8426, 8110, 8286, 8168, 8125, 8191, 8017, 8213] avg=8192.0000 max=8426 max_rank=0 min=8017 min_rank=6 imbalance_ratio=1.028564 max_load_reduction=57110 max_load_reduction_pct=0.871429 moved_to_echo=32546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 117, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 7995, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 302, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 7662, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 8075, 0, 60, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 1, 0, 0, 0, 0, 3, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8229, 8074, 8229, 8190, 8190, 8187, 8285] avg=8192.0000 max=8285 max_rank=7 min=8074 min_rank=2 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7740, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 8182, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 7992, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 521, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7795, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 7930, 0, 60, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7947, 8319, 8018, 8226, 8614, 8190, 8041, 8181] avg=8192.0000 max=8614 max_rank=4 min=7947 min_rank=0 imbalance_ratio=1.051514 max_load_reduction=56922 max_load_reduction_pct=0.868561 moved_to_echo=33026 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 8188, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 5, 0, 0, 0, 7993, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8056, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7779, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 60, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 2, 0, 0, 0, 0, 0, 2, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8398, 8180, 8223, 8094, 8065, 8191, 8279, 8106] avg=8192.0000 max=8398 max_rank=0 min=8065 min_rank=4 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32641 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7917, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 7745, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 440, 0, 0, 0, 0, 7994, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 280, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7778, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 8135, 0, 63, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 54, 0, 0, 1, 0, 0, 3, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 7855, 8437, 8223, 8262, 8187, 8295, 8148] avg=8192.0000 max=8437 max_rank=2 min=7855 min_rank=1 imbalance_ratio=1.029907 max_load_reduction=57099 max_load_reduction_pct=0.871262 moved_to_echo=32892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 256, 128, 0, 0, 0, 64512, 128, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 1024, 128, 0, 0, 0, 64384, 0, 0, 384, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 128, 0, 128, 0, 1664, 128, 0, 0, 0, 64000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 63872, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 512, 0, 128, 2048, 128, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 61568, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 128, 128, 65280, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 0, 0, 128, 0, 0, 128, 128, 128, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 0, 128, 0, 768, 128, 0, 128, 128, 128, 128, 64768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 8111, 8192, 0, 8191, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 34, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4837, 8192, 0, 8192, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7991, 8191, 0, 8190, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 26, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 41, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 1, 7639, 8187, 0, 8192, 8166, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 6, 1, 5, 0, 8148, 0, 16, 0, 22, 0, 0, 0, 2844, 8154, 0, 8165, 8008, 0, 10, 0, 0, 5, 0, 6, 19, 1, 0, 0, 0, 0, 0, 0, 0, 95, 17, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 14, 0, 0, 8170, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 22, 8129, 0, 0, 0, 0, 0, 0, 1919, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 3571, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1, 8182, 0, 0, 0, 0, 0, 0, 1, 7403, 8191, 0, 8191, 8125, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 90, 0, 239, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8190, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 2463, 8177, 0, 8190, 8191, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 29, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 52, 0, 424, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5230, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 13, 2, 0, 0, 8190, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 2, 0, 8016, 8155, 0, 8190, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 40, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 0, 1, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7683, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 508, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10138, 8204, 8178, 8190, 7735, 8205, 6698, 8188] avg=8192.0000 max=10138 max_rank=0 min=6698 min_rank=6 imbalance_ratio=1.237549 max_load_reduction=55398 max_load_reduction_pct=0.845306 moved_to_echo=30826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2019, 0, 0, 0, 0, 0, 2, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7682, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 508, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 40, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10037, 8201, 8176, 8210, 7845, 8165, 6665, 8237] avg=8192.0000 max=10037 max_rank=0 min=6665 min_rank=6 imbalance_ratio=1.225220 max_load_reduction=55499 max_load_reduction_pct=0.846848 moved_to_echo=30912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 0, 0, 0, 4837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7683, 0, 0, 0, 0, 1, 0, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 6149, 508, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6863, 8210, 8178, 8190, 8689, 8202, 9014, 8190] avg=8192.0000 max=9014 max_rank=6 min=6863 min_rank=0 imbalance_ratio=1.100342 max_load_reduction=56522 max_load_reduction_pct=0.862457 moved_to_echo=34095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2024, 0, 0, 0, 0, 0, 0, 1, 7639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8166, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7684, 0, 0, 0, 0, 1, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 3, 0, 0, 0, 0, 0, 6142, 508, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9664, 8250, 8180, 8191, 7828, 8197, 7039, 8187] avg=8192.0000 max=9664 max_rank=0 min=7039 min_rank=6 imbalance_ratio=1.179688 max_load_reduction=55872 max_load_reduction_pct=0.852539 moved_to_echo=31251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 1, 5, 0, 2015, 0, 16, 0, 22, 0, 0, 0, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8008, 0, 10, 0, 0, 5, 0, 6, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 17, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 8134, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8169, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 7624, 0, 0, 0, 0, 0, 0, 1919, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8154, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 3571, 0, 0, 0, 4, 0, 0, 0, 0, 6116, 505, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4909, 8070, 8272, 8187, 9572, 8168, 10214, 8144] avg=8192.0000 max=10214 max_rank=6 min=4909 min_rank=0 imbalance_ratio=1.246826 max_load_reduction=55322 max_load_reduction_pct=0.844147 moved_to_echo=36098 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 2026, 0, 0, 0, 0, 0, 0, 0, 7991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7677, 0, 0, 0, 0, 26, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 6149, 508, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10018, 8212, 8176, 8192, 7716, 8203, 6832, 8187] avg=8192.0000 max=10018 max_rank=0 min=6832 min_rank=6 imbalance_ratio=1.222900 max_load_reduction=55518 max_load_reduction_pct=0.847137 moved_to_echo=30938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1, 2023, 0, 0, 0, 0, 0, 0, 1, 7403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8125, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 8160, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7665, 0, 0, 0, 0, 90, 0, 239, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 507, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9430, 8151, 8190, 8190, 7996, 8205, 7178, 8196] avg=8192.0000 max=9430 max_rank=0 min=7178 min_rank=6 imbalance_ratio=1.151123 max_load_reduction=56106 max_load_reduction_pct=0.856110 moved_to_echo=31575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2014, 0, 0, 0, 0, 0, 0, 0, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8191, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 8164, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7677, 0, 0, 0, 0, 52, 0, 424, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8177, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5230, 2, 2, 0, 0, 0, 0, 0, 0, 6135, 514, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 13, 2, 0, 0, 8180, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4477, 8238, 8186, 8186, 8155, 8191, 11896, 8207] avg=8192.0000 max=11896 max_rank=6 min=4477 min_rank=0 imbalance_ratio=1.452148 max_load_reduction=53640 max_load_reduction_pct=0.818481 moved_to_echo=36449 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 128, 16256, 0, 128, 0, 128, 0, 128, 128, 49408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 256, 0, 128, 12160, 128, 128, 128, 128, 0, 0, 0, 0, 49152, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 61440, 0, 0, 0, 0, 512, 0, 3840, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 65280, 0, 128, 0, 128, 128, 0, 128, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 128, 0, 0, 65536, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 65536, 128, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 8170, 1, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 38, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8164, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 5, 3, 0, 8177, 0, 8, 7819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 1, 0, 0, 78, 440, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 63, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 59, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8164, 0, 3, 0, 15, 0, 0, 0, 8165, 0, 0, 11, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 6, 1, 8139, 1, 0, 0, 0, 0, 0, 8161, 0, 0, 7913, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2, 0, 39, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 37, 37, 0, 2, 14, 0, 0, 8037, 0, 3, 0, 16, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 15, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 8112, 83, 37, 8031, 0, 0, 0, 0, 0, 3, 0, 171, 0, 0, 0, 0, 0, 143, 0, 68, 0, 0, 5, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7641, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8136, 0, 12, 8005, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 1, 0, 5, 2, 0, 0, 94, 0, 0, 42, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 8129, 0, 0, 1, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8189, 0, 43, 8027, 0, 0, 0, 1, 0, 2, 0, 86, 0, 0, 0, 4, 0, 83, 0, 2, 0, 0, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 14, 0, 0, 0, 1, 0, 8172, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 1, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 143, 83, 37, 8031, 0, 0, 0, 0, 0, 3, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 68, 0, 0, 5, 0, 0, 0, 58, 0, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8057, 8471, 8140, 8194, 8126, 8196, 8188] avg=8192.0000 max=8471 max_rank=2 min=8057 min_rank=1 imbalance_ratio=1.034058 max_load_reduction=57065 max_load_reduction_pct=0.870743 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 1, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 63, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8203, 8323, 8027, 8193, 8204, 8196, 8188] avg=8192.0000 max=8323 max_rank=2 min=8027 min_rank=3 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 144, 0, 0, 8170, 1, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8164, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 8095, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8224, 8328, 7981, 8193, 8178, 8198, 8188] avg=8192.0000 max=8328 max_rank=2 min=7981 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 144, 0, 8, 7819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 1, 0, 0, 78, 440, 0, 0, 1, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8092, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8185, 7979, 8494, 8194, 8208, 8194, 8192] avg=8192.0000 max=8494 max_rank=3 min=7979 min_rank=2 imbalance_ratio=1.036865 max_load_reduction=57042 max_load_reduction_pct=0.870392 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 7641, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8224, 7792, 8492, 8193, 8225, 8196, 8188] avg=8192.0000 max=8492 max_rank=3 min=7792 min_rank=2 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 3, 0, 15, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 6, 1, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 7913, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 39, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 21, 7913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 37, 37, 0, 2, 14, 0, 0, 8037, 0, 3, 0, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8107, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 15, 0, 0, 0, 0, 0, 0, 8069, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8227, 8067, 8316, 8176, 8165, 8160, 8183] avg=8192.0000 max=8316 max_rank=3 min=8067 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 12, 8005, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 2, 0, 0, 94, 0, 0, 42, 0, 0, 7889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 8095, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8209, 8288, 8033, 8197, 8222, 8189, 8213] avg=8192.0000 max=8288 max_rank=2 min=8033 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 50, 0, 0, 2, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 43, 8027, 0, 0, 0, 1, 0, 2, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 83, 0, 2, 0, 0, 16, 3, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 14, 0, 0, 0, 1, 0, 8172, 0, 0, 1, 1, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8164, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8098, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8207, 8288, 8053, 8196, 8208, 8207, 8196] avg=8192.0000 max=8288 max_rank=2 min=8053 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 384, 0, 128, 128, 0, 128, 1536, 0, 0, 128, 0, 128, 63488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 1152, 256, 128, 63744, 128, 0, 0, 128, 0, 128, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 64768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 512, 0, 128, 128, 128, 0, 0, 0, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 0, 128, 128, 0, 128, 128, 128, 0, 65280, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 65280, 0, 0, 128, 0, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 86, 5, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 6860, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1137, 0, 126, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 6769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1422, 0, 26, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 75, 7685, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 12, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 34, 7152, 0, 0, 0, 0, 3, 0, 0, 2, 0, 1, 0, 0, 942, 0, 21, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 147] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 1, 0, 0, 29, 0, 0, 0, 0, 42, 0, 0, 6667, 0, 35, 0, 292, 0, 0, 8, 0, 0, 15, 0, 0, 1120, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7, 22, 8, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 15, 2, 0, 0, 0, 8103, 0, 0, 0, 8157, 0, 5, 0, 0, 0, 0, 0, 0, 81, 8170, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 7212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 20, 2, 0, 0, 1, 8190, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 153, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 0, 31, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8074, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 7777, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 338, 0, 24, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 27] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 86, 5, 0, 5, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 6860, 16, 0, 0, 0, 0, 0, 0, 0, 0, 990, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1137, 0, 126, 0, 0, 0, 0, 0, 7151, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 51, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8112, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8098, 8418, 8188, 8104, 8177, 8155, 8128] avg=8192.0000 max=8418 max_rank=2 min=8098 min_rank=1 imbalance_ratio=1.027588 max_load_reduction=57118 max_load_reduction_pct=0.871552 moved_to_echo=32564 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 6769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1422, 0, 26, 0, 0, 0, 0, 0, 7150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 51, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 8112, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 7823, 8598, 8185, 8157, 8185, 8191, 8165] avg=8192.0000 max=8598 max_rank=2 min=7823 min_rank=1 imbalance_ratio=1.049561 max_load_reduction=56938 max_load_reduction_pct=0.868805 moved_to_echo=32698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 75, 7685, 0, 0, 0, 1, 0, 0, 0, 0, 0, 990, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 12, 0, 0, 0, 0, 0, 7151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 2, 0, 0, 0, 51, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 8111, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8776, 7621, 8187, 8230, 8188, 8234, 8129] avg=8192.0000 max=8776 max_rank=1 min=7621 min_rank=2 imbalance_ratio=1.071289 max_load_reduction=56760 max_load_reduction_pct=0.866089 moved_to_echo=32781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 7212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 1, 0, 0, 0, 0, 0, 7151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 51, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 8112, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8408, 7812, 8190, 8202, 8185, 8143, 8419] avg=8192.0000 max=8419 max_rank=7 min=7812 min_rank=2 imbalance_ratio=1.027710 max_load_reduction=57117 max_load_reduction_pct=0.871536 moved_to_echo=32949 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 1, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 6667, 0, 35, 0, 292, 0, 0, 8, 0, 0, 985, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1120, 0, 0, 0, 0, 0, 0, 0, 7117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7, 22, 8, 0, 6, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 15, 2, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 81, 8163, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 26, 51, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 8024, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8052, 8252, 8212, 8241, 8251, 8243, 8130] avg=8192.0000 max=8252 max_rank=2 min=8052 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 7777, 0, 0, 0, 0, 0, 0, 0, 1, 0, 990, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 24, 0, 0, 0, 0, 0, 7150, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 51, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 27, 8112, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8847, 7513, 8193, 8224, 8184, 8260, 8144] avg=8192.0000 max=8847 max_rank=1 min=7513 min_rank=2 imbalance_ratio=1.079956 max_load_reduction=56689 max_load_reduction_pct=0.865005 moved_to_echo=32812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 34, 7152, 0, 0, 0, 0, 3, 0, 0, 2, 0, 994, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 942, 0, 21, 0, 0, 0, 0, 0, 7142, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8178, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 54, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 147, 8111, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8216, 8107, 8189, 8242, 8182, 8166, 8266] avg=8192.0000 max=8266 max_rank=7 min=8107 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 20, 2, 0, 0, 1, 8169, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 0, 31, 0, 0, 0, 0, 0, 7149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 5, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 51, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 35, 8112, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7316, 9215, 8192, 8136, 8184, 8144, 8155] avg=8192.0000 max=9215 max_rank=2 min=7316 min_rank=1 imbalance_ratio=1.124878 max_load_reduction=56321 max_load_reduction_pct=0.859390 moved_to_echo=32619 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 8192, 0, 256, 0, 0, 0, 0, 0, 57216, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 0, 768, 56320, 128, 128, 0, 384, 128, 0, 128, 128, 0, 7936, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65152, 0, 0, 128, 0, 128, 128, 128, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 768, 64896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 128, 256, 128, 0, 128, 128, 65408, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 64896, 0, 0, 0, 0, 128, 0, 0, 128, 512, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 65536, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 8192, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 3138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 591, 8185, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 4463, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 26, 8190, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 1691, 8192, 0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8191, 1, 0, 0, 6347, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 8189, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 7887, 5, 0, 0, 1469, 8192, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 5490, 0, 0, 8182, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 138, 8192, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8192, 0, 0, 0, 6769, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 8192, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 905, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6782, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 8192, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 177, 8191, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6898, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 8190, 0, 62, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 1, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 1, 39, 8189, 0, 0, 8189, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7524, 0, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 11, 4, 0, 40, 0, 12, 0, 4, 0, 10, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7779, 0, 0, 15, 0, 8168, 0, 0, 8097, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 7326, 21, 0, 0, 0, 821, 0, 0, 0, 0, 0, 426, 1, 0, 0, 8142, 0, 9, 0, 8178, 0, 0, 8163, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 7987, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 3138, 0, 0, 0, 0, 0, 0, 6596, 476, 205, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7471, 0, 0, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1581, 0, 0, 0, 4463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8241, 10437, 8185, 8186, 8062, 8191, 6045] avg=8192.0000 max=10437 max_rank=2 min=6045 min_rank=7 imbalance_ratio=1.274048 max_load_reduction=55099 max_load_reduction_pct=0.840744 moved_to_echo=30484 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 26, 7985, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 6596, 488, 205, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7667, 0, 0, 0, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1580, 1, 0, 0, 6347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8036, 7433, 8202, 8194, 9359, 8191, 7932] avg=8192.0000 max=9359 max_rank=5 min=7433 min_rank=2 imbalance_ratio=1.142456 max_load_reduction=56177 max_load_reduction_pct=0.857193 moved_to_echo=33676 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 7987, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 0, 0, 0, 0, 6596, 485, 205, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7613, 0, 0, 0, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1581, 0, 0, 0, 6782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8037, 7836, 8207, 8194, 8518, 8191, 8363] avg=8192.0000 max=8518 max_rank=5 min=7836 min_rank=2 imbalance_ratio=1.039795 max_load_reduction=57018 max_load_reduction_pct=0.870026 moved_to_echo=33266 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 7987, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 1043, 0, 0, 0, 0, 0, 0, 6596, 488, 205, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7669, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1581, 0, 0, 0, 6898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8112, 8337, 8189, 8192, 7846, 8191, 8479] avg=8192.0000 max=8479 max_rank=7 min=7846 min_rank=5 imbalance_ratio=1.035034 max_load_reduction=57057 max_load_reduction_pct=0.870621 moved_to_echo=32708 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 7987, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 0, 0, 0, 0, 0, 0, 6596, 490, 205, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7697, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1581, 0, 0, 0, 6769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8211, 8359, 8194, 8193, 7835, 8190, 8364] avg=8192.0000 max=8364 max_rank=7 min=7835 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32582 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 6556, 466, 204, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4, 0, 40, 0, 12, 0, 4, 0, 10, 0, 3, 1, 0, 0, 0, 8097, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7313, 0, 0, 15, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 7325, 21, 0, 0, 0, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 1, 0, 0, 1571, 0, 9, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 7958, 7243, 8199, 8177, 7355, 8199, 10185] avg=8192.0000 max=10185 max_rank=7 min=7243 min_rank=2 imbalance_ratio=1.243286 max_load_reduction=55351 max_load_reduction_pct=0.844589 moved_to_echo=33916 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 7984, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 0, 0, 0, 6596, 472, 205, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7415, 5, 0, 0, 1469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1580, 0, 0, 0, 5490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8625, 8225, 8149, 8207, 8889, 8191, 7070] avg=8192.0000 max=8889 max_rank=5 min=7070 min_rank=7 imbalance_ratio=1.085083 max_load_reduction=56647 max_load_reduction_pct=0.864365 moved_to_echo=32357 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 7980, 0, 62, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 1, 1, 0, 0, 0, 6600, 490, 210, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 8189, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7624, 0, 0, 1, 39, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 0, 0, 0, 7524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8316, 7666, 8211, 8193, 7672, 8192, 9098] avg=8192.0000 max=9098 max_rank=7 min=7666 min_rank=2 imbalance_ratio=1.110596 max_load_reduction=56438 max_load_reduction_pct=0.861176 moved_to_echo=33155 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 0, 7296, 0, 128, 128, 0, 0, 128, 52736, 3968, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 512, 128, 0, 0, 12672, 128, 128, 0, 52480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 60544, 128, 0, 128, 5120, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 1280, 63872, 128, 512, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 128, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 64768, 128, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[39, 0, 8052, 0, 0, 4358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 1, 0, 0, 7649, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 2236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 548, 0, 250, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[5, 0, 8191, 0, 0, 5002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1, 0, 0, 0, 0, 0, 0, 8009, 8, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 4496, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 304, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 1, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7767, 2, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4190, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 621, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5302, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[33, 0, 8133, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7602, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 18, 8192, 0, 0, 0, 0, 0, 0, 0, 5388, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 185, 0, 447, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5486, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[7, 0, 8190, 0, 0, 3862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 604, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 11, 8172, 0, 0, 1524, 0, 0, 0, 0, 0, 60, 0, 0, 0, 8166, 0, 0, 0, 0, 66, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 390, 0, 0, 0, 0, 0, 0, 11, 0, 7762, 118, 0, 7225, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8107, 0, 6, 0, 16, 0, 0, 0, 5481, 0, 0, 0, 0, 0, 3, 0, 0, 8134, 0, 0, 0, 18, 0, 0, 0, 5, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 26, 0, 3, 8136, 0, 0, 0, 0, 1320, 0, 0, 21, 12, 0, 0, 7, 0, 0, 659, 0, 0, 0, 18, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 8166, 0, 0, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7188, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 37, 0, 0, 3, 0, 0, 0, 0, 7884, 7, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 1, 0, 0, 1, 7138, 0, 0, 0, 0, 0, 0, 0, 0, 7495, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 536, 0, 191, 0, 0, 0, 0, 0, 0, 0, 7, 1, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[45, 1, 8175, 0, 0, 6076, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8130, 0, 1, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 6050, 0, 0, 0, 1, 0, 0, 0, 0, 6581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 86, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[39, 0, 0, 0, 0, 4358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 1, 0, 0, 7649, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1481, 541, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 0, 548, 0, 250, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10159, 8075, 7972, 8335, 9788, 4523, 8329, 8355] avg=8192.0000 max=10159 max_rank=0 min=4523 min_rank=5 imbalance_ratio=1.240112 max_load_reduction=55377 max_load_reduction_pct=0.844986 moved_to_echo=30995 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 5002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1, 0, 0, 0, 0, 0, 0, 8009, 8, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1677, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4496, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1347, 541, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7525, 0, 0, 304, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10248, 8214, 8296, 8327, 9781, 6651, 7924, 6095] avg=8192.0000 max=10248 max_rank=0 min=6095 min_rank=7 imbalance_ratio=1.250977 max_load_reduction=55288 max_load_reduction_pct=0.843628 moved_to_echo=30451 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7767, 2, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 4190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 541, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 0, 621, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5297, 8215, 8029, 8331, 5868, 10283, 8261, 11252] avg=8192.0000 max=11252 max_rank=7 min=5297 min_rank=0 imbalance_ratio=1.373535 max_load_reduction=54284 max_load_reduction_pct=0.828308 moved_to_echo=35664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 3862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 541, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 0, 604, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10056, 8213, 8251, 8250, 9335, 5362, 8135, 7934] avg=8192.0000 max=10056 max_rank=0 min=5362 min_rank=5 imbalance_ratio=1.227539 max_load_reduction=55480 max_load_reduction_pct=0.846558 moved_to_echo=30766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7602, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 18, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 5388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 541, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 0, 185, 0, 447, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4468, 8156, 7861, 8352, 7066, 10038, 8159, 11436] avg=8192.0000 max=11436 max_rank=7 min=4468 min_rank=0 imbalance_ratio=1.395996 max_load_reduction=54100 max_load_reduction_pct=0.825500 moved_to_echo=36699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 1524, 0, 0, 0, 0, 0, 60, 0, 0, 0, 6496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 390, 0, 0, 0, 0, 0, 0, 11, 0, 7762, 118, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7225, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 0, 6, 0, 16, 0, 0, 0, 5481, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 18, 0, 0, 0, 5, 0, 0, 0, 0, 9, 0, 1670, 536, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3, 7474, 0, 0, 0, 0, 1320, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 7, 0, 0, 659, 0, 0, 0, 18, 0, 0, 0, 0, 0, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8270, 8539, 7407, 7166, 10635, 8844, 6584] avg=8192.0000 max=10635 max_rank=5 min=6584 min_rank=7 imbalance_ratio=1.298218 max_load_reduction=54901 max_load_reduction_pct=0.837723 moved_to_echo=33229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[45, 1, 0, 0, 0, 6076, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8130, 0, 1, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1677, 0, 0, 0, 0, 0, 0, 0, 6050, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 541, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 0, 86, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9518, 8198, 8395, 8301, 7728, 8259, 7631, 7506] avg=8192.0000 max=9518 max_rank=0 min=7506 min_rank=7 imbalance_ratio=1.161865 max_load_reduction=56018 max_load_reduction_pct=0.854767 moved_to_echo=31124 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8166, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 37, 0, 0, 3, 0, 0, 0, 0, 7884, 7, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1663, 0, 1, 0, 1, 0, 0, 1, 7138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7495, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 546, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 0, 0, 0, 7516, 0, 0, 536, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7699, 8195, 8193, 8233, 8804, 9785, 8253, 6374] avg=8192.0000 max=9785 max_rank=5 min=6374 min_rank=7 imbalance_ratio=1.194458 max_load_reduction=55751 max_load_reduction_pct=0.850693 moved_to_echo=33216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 0, 0, 0, 23296, 0, 0, 128, 0, 0, 128, 0, 0, 0, 42112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 128, 128, 18048, 0, 0, 128, 128, 0, 0, 128, 0, 0, 47616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 48256, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 10880, 4352, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13440, 0, 128, 0, 128, 0, 0, 128, 52224, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 60160, 0, 0, 2944, 0, 2560, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[64384, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 128, 128, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 512, 128, 128, 0, 128, 0, 128, 128, 0, 62848, 256, 128, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7690, 0, 0, 6717, 0, 0, 0, 8113, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 1844, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7651, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 541, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8144, 0, 0, 0, 8190, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 7290, 2, 0, 8139, 193, 0, 0, 8164, 0, 0, 48, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 8159, 0, 10, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 8083, 2, 0, 0, 0, 15, 8017, 6, 0, 0, 0, 691, 4, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 22, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 131, 72, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7411, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 6790, 0, 0, 0, 8171, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 6607, 0, 0, 0, 8161, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 7391, 2, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8165, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 792, 0, 0, 6717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1844, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6898, 87, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7510, 8119, 8181, 8236, 8208, 8113, 8323, 8846] avg=8192.0000 max=8846 max_rank=7 min=7510 min_rank=0 imbalance_ratio=1.079834 max_load_reduction=56690 max_load_reduction_pct=0.865021 moved_to_echo=33490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 0, 0, 7651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7349, 87, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8494, 8163, 8202, 8189, 8208, 8105, 8186, 7989] avg=8192.0000 max=8494 max_rank=0 min=7989 min_rank=7 imbalance_ratio=1.036865 max_load_reduction=57042 max_load_reduction_pct=0.870392 moved_to_echo=32488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 843, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7348, 87, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8988, 8182, 8207, 8186, 8210, 8105, 8168, 7490] avg=8192.0000 max=8988 max_rank=0 min=7490 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=56548 max_load_reduction_pct=0.862854 moved_to_echo=31973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 0, 0, 7411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 7349, 87, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8179, 8203, 8189, 8204, 8105, 8169, 8233] avg=8192.0000 max=8254 max_rank=0 min=8105 min_rank=5 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 751, 2, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 8151, 0, 0, 48, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 24, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 10, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 18, 2, 0, 0, 0, 15, 8017, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 4, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 22, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 72, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 17, 6539, 86, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8989, 8404, 8149, 8172, 8115, 8734, 8113, 6860] avg=8192.0000 max=8989 max_rank=0 min=6860 min_rank=7 imbalance_ratio=1.097290 max_load_reduction=56547 max_load_reduction_pct=0.862839 moved_to_echo=31822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 0, 0, 6607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7339, 87, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7449, 8155, 8203, 8192, 8204, 8105, 8217, 9011] avg=8192.0000 max=9011 max_rank=7 min=7449 min_rank=0 imbalance_ratio=1.099976 max_load_reduction=56525 max_load_reduction_pct=0.862503 moved_to_echo=33537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 0, 6790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7228, 87, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7621, 8163, 8202, 8189, 8195, 8142, 8182, 8842] avg=8192.0000 max=8842 max_rank=7 min=7621 min_rank=0 imbalance_ratio=1.079346 max_load_reduction=56694 max_load_reduction_pct=0.865082 moved_to_echo=33361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 838, 0, 0, 7391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8168, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8165, 0, 0, 0, 0, 1, 0, 0, 0, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 5, 0, 0, 1, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 7334, 88, 12, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8171, 8189, 8183, 8192, 8127, 8178, 8265] avg=8192.0000 max=8265 max_rank=7 min=8127 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 7296, 128, 0, 0, 128, 128, 0, 0, 128, 0, 128, 128, 0, 128, 57472, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 6656, 128, 0, 58880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 256, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 65408, 0, 0, 0, 128, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[768, 128, 128, 0, 0, 0, 0, 0, 0, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 65280, 0, 128, 128, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 65536, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 0, 256, 128, 0, 128, 0, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7774, 0, 0, 0, 0, 0, 0, 0, 1, 0, 110, 0, 0, 29, 8192, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 370, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 26, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 18, 0, 0, 0, 8005, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 48, 0, 8127, 3, 1130, 13, 7041, 0, 39, 0, 108, 16, 18, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 34, 1, 8151, 0, 1, 0, 0, 133, 0, 12, 8160, 0, 0, 0, 0, 0, 0, 5, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8023, 0, 0, 0, 0, 0, 0, 12, 19, 0, 7, 17, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 5, 8192, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 60, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 61, 8190, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 7949, 0, 0, 0, 0, 1, 0, 0, 8187, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 8192, 0, 0, 0, 7619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 0, 0, 96, 0, 0, 0, 8048, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 18, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 22, 0, 0, 0, 8163, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8190, 4, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 1, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 8187, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 8190, 0, 0, 0, 2, 0, 117, 0, 0, 0, 8, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7774, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 110, 0, 0, 29, 69, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 370, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 8120, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 26, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7859, 8294, 8204, 8360, 8238, 8204, 8185, 8192] avg=8192.0000 max=8360 max_rank=3 min=7859 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 18, 0, 0, 0, 7857, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8241, 8204, 8064, 8179, 8205, 8173, 8195] avg=8192.0000 max=8275 max_rank=0 min=8064 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 5, 69, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 60, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8120, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8202, 8204, 8137, 8181, 8205, 8192, 8195] avg=8192.0000 max=8220 max_rank=0 min=8137 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 69, 0, 0, 0, 7619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 0, 0, 96, 0, 0, 0, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 18, 8120, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 7712, 8198, 8661, 8202, 8204, 8199, 8194] avg=8192.0000 max=8661 max_rank=3 min=7712 min_rank=1 imbalance_ratio=1.057251 max_load_reduction=56875 max_load_reduction_pct=0.867844 moved_to_echo=32799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 8, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 22, 0, 0, 0, 8013, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8120, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8260, 8205, 8037, 8185, 8204, 8179, 8195] avg=8192.0000 max=8271 max_rank=0 min=8037 min_rank=3 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 48, 0, 68, 3, 1130, 13, 7041, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 16, 18, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 34, 1, 8001, 0, 1, 0, 0, 133, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 21, 0, 0, 0, 0, 0, 0, 0, 8056, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8023, 0, 0, 0, 0, 0, 0, 12, 19, 0, 7, 17, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8403, 8130, 8197, 8142, 8097, 8149, 8170] avg=8192.0000 max=8403 max_rank=1 min=8097 min_rank=5 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 61, 69, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 7803, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8118, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 8187, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8287, 8187, 8045, 8178, 8202, 8168, 8195] avg=8192.0000 max=8287 max_rank=1 min=8045 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 57, 4, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 3, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 1, 1, 0, 8023, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 8123, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 10, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 2, 0, 117, 0, 0, 0, 8, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8137, 8204, 8035, 8231, 8215, 8291, 8200] avg=8192.0000 max=8291 max_rank=6 min=8035 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32937 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 0, 128, 128, 640, 128, 1152, 128, 63488, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 128, 128, 65024, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1280, 0, 128, 640, 128, 128, 128, 63616, 0, 128, 0, 0, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 64896, 0, 0, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65408, 0, 0, 0, 128, 0, 256, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 65408, 0, 0, 0, 128, 0, 0, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 0, 65408, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 763, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 83, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 200, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7480, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 265, 0, 0, 0, 0, 0, 0, 7890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 167, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 9, 0, 0, 0, 8045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8054, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 119, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 7, 0, 0, 1, 0, 0, 25, 0, 4, 0, 0, 0, 0, 0, 0, 35, 0, 0, 7985, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3, 38, 0, 0, 19, 0, 0, 3, 0, 0, 3, 0, 0, 0, 7, 24, 8114, 8121, 0, 0, 15, 0, 15, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 17, 0, 10, 0, 0, 0, 0, 32, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 317, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7885, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 129, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 89, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 105, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 235, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8093, 1, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 0, 1, 7869, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 211, 0, 0, 0, 0, 0, 0, 7995, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 430, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 32, 0, 0, 0, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7766, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 98, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 468, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 15, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 8191, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 94, 1, 0, 1, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 763, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 83, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 200, 0, 0, 7760, 299, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7480, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8648, 8181, 8136, 8288, 7787, 8315, 7981, 8200] avg=8192.0000 max=8648 max_rank=0 min=7787 min_rank=4 imbalance_ratio=1.055664 max_load_reduction=56888 max_load_reduction_pct=0.868042 moved_to_echo=32283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 167, 0, 0, 7892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 9, 0, 0, 0, 8045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 7760, 300, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8054, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8179, 8190, 8114, 8361, 8196, 8243, 8194] avg=8192.0000 max=8361 max_rank=4 min=8059 min_rank=0 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32994 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 317, 0, 0, 7892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 7760, 300, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7885, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8182, 8194, 8094, 8191, 8206, 8267, 8193] avg=8192.0000 max=8267 max_rank=6 min=8094 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 89, 0, 0, 7892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 13, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 7760, 300, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7981, 8182, 8247, 8126, 8434, 8168, 8205, 8193] avg=8192.0000 max=8434 max_rank=4 min=7981 min_rank=0 imbalance_ratio=1.029541 max_load_reduction=57102 max_load_reduction_pct=0.871307 moved_to_echo=33000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 430, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 32, 0, 0, 0, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 238, 0, 0, 7760, 299, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7766, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8318, 8182, 8117, 8325, 8072, 8144, 8186, 8192] avg=8192.0000 max=8325 max_rank=3 min=8072 min_rank=4 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 7, 0, 0, 1, 0, 0, 25, 0, 4, 0, 0, 0, 0, 7852, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3, 38, 7693, 298, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3, 0, 0, 3, 0, 0, 0, 7, 24, 8114, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 15, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 17, 0, 10, 0, 0, 0, 0, 32, 0, 0, 0, 10, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7840, 8253, 8307, 8087, 8473, 8079, 8321, 8176] avg=8192.0000 max=8473 max_rank=4 min=7840 min_rank=0 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=33049 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 235, 0, 0, 7887, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8055, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 1, 0, 8093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 142, 0, 0, 7763, 303, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 0, 1, 7869, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8196, 8230, 8256, 8175, 8271, 8091, 8193] avg=8192.0000 max=8271 max_rank=5 min=8091 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 468, 0, 0, 7889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8054, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 15, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 7759, 299, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8057, 0, 94, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8357, 8181, 8115, 8246, 8043, 8157, 8242, 8195] avg=8192.0000 max=8357 max_rank=0 min=8043 min_rank=4 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 0, 128, 128, 0, 0, 128, 128, 63104, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 128, 0, 896, 128, 128, 62080, 2432, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 2560, 0, 0, 63104, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 128, 0, 128, 0, 128, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 64256, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 0, 0, 64512, 0, 1024, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 64768, 128, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 1152, 0, 256, 0, 0, 128, 0, 128, 0, 64384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True + [2026-08-26 16:28:30] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 441.6 | throughput per GPU (TFLOP/s/GPU): 409.5 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.009528E+01 | loss scale: 1.0 | grad norm: 83.558 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 1054, 256, 5692, 0, 0, 6050, 0, 0, 1, 1, 480, 0, 0, 7, 0, 1, 211, 0, 3, 2532, 240, 0, 0, 8188, 1149, 8180, 2, 3364, 0, 1, 978, 0, 0, 0, 0, 1, 38, 0, 2355, 1, 0, 5830, 0, 105, 0, 7, 1030, 0, 0, 0, 0, 0, 171, 0, 50, 1708, 0, 1, 0, 6, 5, 0, 0, 0, 815, 2, 1, 1, 781, 0, 687, 2, 1, 15, 213, 0, 1, 3, 0, 0, 132, 0, 0, 1898, 1, 130, 0, 25, 11, 0, 0, 1, 0, 2518, 0, 0, 0, 3, 1, 1, 10, 2435, 354, 0, 0, 18, 406, 1, 0, 4255, 0, 0, 0, 6, 81, 2, 0, 2, 11, 1011, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1591, 298, 4116, 0, 0, 6311, 0, 1, 1, 0, 240, 0, 1, 10, 0, 5, 322, 0, 1, 2850, 214, 1, 0, 8189, 757, 8175, 1, 3583, 0, 0, 905, 0, 2, 2, 0, 1, 24, 0, 3087, 0, 0, 5347, 1, 52, 0, 1, 712, 1, 9, 0, 0, 2, 325, 1, 65, 2006, 0, 1, 0, 6, 0, 0, 0, 0, 619, 0, 3, 0, 1037, 0, 814, 0, 1, 16, 278, 0, 0, 0, 0, 0, 88, 0, 0, 1629, 2, 74, 0, 42, 73, 1, 0, 0, 0, 2831, 1, 0, 3, 21, 2, 0, 15, 4048, 501, 0, 0, 77, 765, 3, 0, 2364, 0, 2, 0, 3, 60, 0, 0, 0, 4, 938, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 824, 362, 5128, 0, 0, 5768, 0, 5, 3, 0, 472, 2, 0, 5, 0, 28, 79, 0, 3, 2264, 401, 1, 0, 8192, 927, 8162, 2, 4158, 3, 4, 556, 0, 1, 5, 0, 3, 17, 1, 2442, 1, 0, 4971, 3, 57, 0, 0, 693, 0, 4, 1, 0, 0, 267, 0, 63, 1780, 0, 0, 0, 35, 0, 1, 0, 0, 823, 0, 0, 0, 870, 0, 1016, 12, 18, 25, 186, 0, 0, 0, 0, 7, 167, 0, 0, 2187, 0, 136, 0, 39, 16, 0, 3, 0, 0, 2501, 3, 0, 4, 7, 0, 0, 26, 3576, 800, 0, 2, 15, 374, 0, 0, 3931, 0, 0, 1, 0, 45, 0, 0, 0, 14, 1021, 4, 0, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 9, 940, 331, 5084, 2, 0, 6983, 0, 0, 0, 1, 131, 0, 1, 3, 0, 1, 234, 0, 1, 2102, 182, 0, 0, 8190, 618, 8188, 0, 4003, 0, 0, 1098, 0, 0, 1, 0, 0, 2, 0, 3383, 15, 1, 5727, 0, 61, 0, 0, 662, 1, 0, 1, 0, 2, 439, 0, 52, 1516, 0, 0, 0, 20, 0, 2, 0, 0, 644, 0, 3, 0, 861, 0, 646, 7, 12, 14, 369, 0, 0, 0, 0, 1, 35, 0, 0, 2556, 2, 111, 0, 15, 69, 0, 0, 3, 0, 2073, 2, 0, 4, 40, 3, 2, 5, 2477, 435, 0, 0, 33, 647, 1, 0, 2998, 0, 0, 0, 0, 42, 0, 0, 2, 6, 1430, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 544, 497, 5476, 0, 0, 6641, 8, 0, 0, 0, 366, 3, 0, 28, 0, 11, 390, 0, 2, 2593, 305, 0, 0, 8188, 858, 8172, 0, 2927, 11, 17, 541, 0, 2, 5, 1, 0, 9, 1, 3055, 4, 4, 5177, 8, 90, 0, 0, 733, 0, 0, 3, 0, 0, 404, 2, 55, 1516, 2, 0, 0, 12, 0, 2, 0, 2, 1056, 0, 0, 0, 709, 0, 902, 23, 1, 12, 316, 0, 0, 0, 0, 1, 171, 0, 0, 1569, 0, 176, 4, 9, 103, 0, 0, 0, 0, 2226, 0, 1, 2, 39, 0, 0, 43, 4648, 938, 0, 0, 63, 352, 0, 0, 2220, 0, 5, 0, 16, 78, 0, 1, 2, 21, 1153, 1, 1, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 721, 315, 4737, 0, 1, 5665, 5, 0, 0, 0, 635, 0, 2, 6, 0, 9, 348, 0, 4, 2026, 287, 0, 0, 8192, 677, 8174, 3, 3508, 0, 0, 349, 0, 1, 0, 0, 2, 13, 0, 2641, 1, 0, 5251, 2, 148, 0, 8, 886, 0, 0, 0, 0, 0, 374, 4, 52, 1389, 0, 0, 0, 60, 0, 0, 0, 0, 930, 0, 0, 1, 833, 0, 1227, 9, 31, 23, 240, 0, 0, 2, 2, 0, 337, 0, 0, 2739, 1, 140, 0, 46, 81, 1, 5, 0, 0, 1699, 2, 0, 1, 13, 0, 0, 25, 4460, 980, 0, 0, 46, 428, 0, 0, 3404, 0, 0, 0, 0, 55, 0, 0, 3, 49, 1224, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 768, 233, 4896, 0, 0, 6522, 0, 0, 0, 0, 436, 2, 0, 17, 0, 0, 326, 0, 0, 1962, 187, 0, 0, 8191, 654, 8187, 0, 3433, 2, 0, 880, 0, 4, 24, 0, 0, 18, 0, 2619, 27, 0, 5752, 1, 52, 0, 0, 1015, 0, 0, 4, 0, 1, 347, 0, 101, 1317, 0, 0, 0, 15, 4, 1, 0, 0, 1202, 0, 1, 0, 885, 0, 702, 23, 0, 5, 441, 0, 0, 0, 0, 0, 236, 0, 0, 2718, 0, 224, 0, 5, 21, 2, 0, 0, 0, 3699, 0, 0, 5, 12, 0, 0, 19, 3458, 583, 0, 0, 125, 461, 0, 0, 1937, 0, 0, 0, 3, 61, 0, 0, 0, 8, 692, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1303, 180, 4412, 0, 0, 7140, 0, 0, 0, 0, 345, 0, 0, 21, 0, 24, 320, 0, 11, 2671, 267, 0, 0, 8192, 898, 8184, 0, 3011, 1, 28, 1008, 1, 7, 14, 0, 4, 48, 0, 2625, 3, 0, 4055, 0, 86, 0, 0, 1374, 0, 0, 0, 0, 0, 380, 0, 81, 1292, 0, 0, 0, 16, 0, 1, 0, 0, 1016, 0, 1, 0, 956, 0, 851, 13, 6, 52, 174, 0, 0, 2, 1, 1, 235, 0, 0, 2092, 3, 200, 0, 14, 102, 2, 0, 0, 0, 2594, 0, 0, 4, 84, 1, 0, 6, 4723, 681, 0, 0, 44, 498, 0, 0, 1874, 0, 1, 0, 2, 145, 0, 0, 0, 16, 1128, 0, 1, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1591, 298, 4116, 0, 0, 1558, 0, 1, 1, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 5, 322, 0, 1, 2850, 214, 1, 0, 0, 757, 936, 1, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 905, 0, 2, 2, 0, 1, 24, 0, 3087, 0, 0, 4568, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 712, 1, 9, 0, 0, 2, 325, 1, 65, 2006, 0, 1, 0, 6, 2719, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 3, 0, 1037, 0, 814, 0, 1, 16, 278, 0, 4520, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 1629, 2, 74, 0, 42, 73, 1, 0, 0, 5604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 1, 0, 3, 21, 2, 0, 15, 4048, 501, 0, 0, 77, 765, 3, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2364, 0, 2, 0, 3, 60, 0, 0, 0, 4, 938, 0, 0, 0, 2, 4023, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7807, 8681, 8642, 8433, 8018, 7513, 8969, 7473] avg=8192.0000 max=8969 max_rank=6 min=7473 min_rank=7 imbalance_ratio=1.094849 max_load_reduction=56567 max_load_reduction_pct=0.863144 moved_to_echo=31973 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 1054, 256, 5692, 0, 0, 1493, 0, 0, 1, 1, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 211, 0, 3, 2532, 240, 0, 0, 0, 1149, 937, 2, 3364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 978, 0, 0, 0, 0, 1, 38, 0, 2355, 1, 0, 4981, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1030, 0, 0, 0, 0, 0, 171, 0, 50, 1708, 0, 1, 0, 6, 2721, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 815, 2, 1, 1, 781, 0, 687, 2, 1, 15, 213, 0, 4522, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 132, 0, 0, 1898, 1, 130, 0, 25, 11, 0, 0, 1, 5603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2518, 0, 0, 0, 3, 1, 1, 10, 2435, 354, 0, 0, 18, 406, 1, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4255, 0, 0, 0, 6, 81, 2, 0, 2, 11, 1011, 0, 0, 0, 0, 3857, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8980, 8446, 8460, 8279, 7745, 7805, 6512, 9309] avg=8192.0000 max=9309 max_rank=7 min=6512 min_rank=6 imbalance_ratio=1.136353 max_load_reduction=56227 max_load_reduction_pct=0.857956 moved_to_echo=31371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 824, 362, 5128, 0, 0, 1424, 0, 5, 3, 0, 472, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 28, 79, 0, 3, 2264, 401, 1, 0, 0, 927, 935, 2, 4158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 556, 0, 1, 5, 0, 3, 17, 1, 2442, 1, 0, 4248, 3, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 693, 0, 4, 1, 0, 0, 267, 0, 63, 1780, 0, 0, 0, 35, 2715, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 823, 0, 0, 0, 870, 0, 1016, 12, 18, 25, 186, 0, 4512, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 167, 0, 0, 2187, 0, 136, 0, 39, 16, 0, 3, 0, 5605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2501, 3, 0, 4, 7, 0, 0, 26, 3576, 800, 0, 2, 15, 374, 0, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 1, 0, 45, 0, 0, 0, 14, 1021, 4, 0, 10, 0, 3677, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8803, 7341, 8145, 8130, 8160, 7960, 8774] avg=8192.0000 max=8803 max_rank=1 min=7341 min_rank=2 imbalance_ratio=1.074585 max_load_reduction=56733 max_load_reduction_pct=0.865677 moved_to_echo=33024 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 9, 940, 331, 5084, 2, 0, 1723, 0, 0, 0, 1, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 1, 234, 0, 1, 2102, 182, 0, 0, 0, 618, 937, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 0, 0, 1, 0, 0, 2, 0, 3383, 15, 1, 4894, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 1, 0, 1, 0, 2, 439, 0, 52, 1516, 0, 0, 0, 20, 2724, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 644, 0, 3, 0, 861, 0, 646, 7, 12, 14, 369, 0, 4527, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 0, 0, 2556, 2, 111, 0, 15, 69, 0, 0, 3, 5603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2073, 2, 0, 4, 40, 3, 2, 5, 2477, 435, 0, 0, 33, 647, 1, 751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 0, 0, 0, 0, 42, 0, 0, 2, 6, 1430, 0, 0, 0, 0, 4452, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8082, 9455, 8004, 7893, 8395, 6473, 9012] avg=8192.0000 max=9455 max_rank=2 min=6473 min_rank=6 imbalance_ratio=1.154175 max_load_reduction=56081 max_load_reduction_pct=0.855728 moved_to_echo=31773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 544, 497, 5476, 0, 0, 1639, 8, 0, 0, 0, 366, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 11, 390, 0, 2, 2593, 305, 0, 0, 0, 858, 936, 0, 2927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 17, 541, 0, 2, 5, 1, 0, 9, 1, 3055, 4, 4, 4424, 8, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 0, 0, 3, 0, 0, 404, 2, 55, 1516, 2, 0, 0, 12, 2718, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 1056, 0, 0, 0, 709, 0, 902, 23, 1, 12, 316, 0, 4518, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 171, 0, 0, 1569, 0, 176, 4, 9, 103, 0, 0, 0, 5603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2226, 0, 1, 2, 39, 0, 0, 43, 4648, 938, 0, 0, 63, 352, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2220, 0, 5, 0, 16, 78, 0, 1, 2, 21, 1153, 1, 1, 5, 0, 4234, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8537, 8050, 8172, 8030, 8309, 7636, 8991, 7811] avg=8192.0000 max=8991 max_rank=6 min=7636 min_rank=5 imbalance_ratio=1.097534 max_load_reduction=56545 max_load_reduction_pct=0.862808 moved_to_echo=32747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 768, 233, 4896, 0, 0, 1610, 0, 0, 0, 0, 436, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 326, 0, 0, 1962, 187, 0, 0, 0, 654, 938, 0, 3433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 880, 0, 4, 24, 0, 0, 18, 0, 2619, 27, 0, 4914, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 0, 0, 4, 0, 1, 347, 0, 101, 1317, 0, 0, 0, 15, 2723, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 1202, 0, 1, 0, 885, 0, 702, 23, 0, 5, 441, 0, 4526, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 2718, 0, 224, 0, 5, 21, 2, 0, 0, 5604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 0, 0, 5, 12, 0, 0, 19, 3458, 583, 0, 0, 125, 461, 0, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1937, 0, 0, 0, 3, 61, 0, 0, 0, 8, 692, 0, 0, 2, 0, 4158, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7953, 7517, 8541, 8110, 8544, 8810, 9117, 6944] avg=8192.0000 max=9117 max_rank=6 min=6944 min_rank=7 imbalance_ratio=1.112915 max_load_reduction=56419 max_load_reduction_pct=0.860886 moved_to_echo=33415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 721, 315, 4737, 0, 1, 1399, 5, 0, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 9, 348, 0, 4, 2026, 287, 0, 0, 0, 677, 936, 3, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 1, 0, 0, 2, 13, 0, 2641, 1, 0, 4487, 2, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 886, 0, 0, 0, 0, 0, 374, 4, 52, 1389, 0, 0, 0, 60, 2719, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 1, 833, 0, 1227, 9, 31, 23, 240, 0, 4519, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 337, 0, 0, 2739, 1, 140, 0, 46, 81, 1, 5, 0, 5606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1699, 2, 0, 1, 13, 0, 0, 25, 4460, 980, 0, 0, 46, 428, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 0, 0, 0, 55, 0, 0, 3, 49, 1224, 0, 0, 2, 0, 3611, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7814, 7806, 7644, 8078, 8468, 8960, 8343, 8423] avg=8192.0000 max=8960 max_rank=5 min=7644 min_rank=2 imbalance_ratio=1.093750 max_load_reduction=56576 max_load_reduction_pct=0.863281 moved_to_echo=34194 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1303, 180, 4412, 0, 0, 1754, 0, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 24, 320, 0, 11, 2671, 267, 0, 0, 0, 898, 928, 0, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 1008, 1, 7, 14, 0, 4, 48, 0, 2625, 3, 0, 3456, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 380, 0, 81, 1292, 0, 0, 0, 16, 2727, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1016, 0, 1, 0, 956, 0, 851, 13, 6, 52, 174, 0, 4529, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 235, 0, 0, 2092, 3, 200, 0, 14, 102, 2, 0, 0, 5605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2594, 0, 0, 4, 84, 1, 0, 6, 4723, 681, 0, 0, 44, 498, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1874, 0, 1, 0, 2, 145, 0, 0, 0, 16, 1128, 0, 1, 4, 0, 4556, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8000, 8151, 7281, 8457, 8429, 8257, 9171, 7790] avg=8192.0000 max=9171 max_rank=6 min=7281 min_rank=2 imbalance_ratio=1.119507 max_load_reduction=56365 max_load_reduction_pct=0.860062 moved_to_echo=33647 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 7168, 128, 128, 128, 0, 128, 2816, 128, 640, 12544, 128, 128, 0, 256, 21888, 20736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 2304, 0, 128, 19072, 2176, 128, 0, 0, 6656, 7552, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 23040, 0, 128, 128, 128, 640, 128, 128, 128, 256, 8704, 128, 128, 128, 128, 32640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 7808, 2560, 39552, 128, 128, 12672, 128, 128, 128, 128, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 20224, 128, 128, 128, 256, 128, 128, 256, 29952, 5376, 0, 128, 512, 3968, 128, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 6400, 128, 128, 128, 128, 128, 256, 128, 22272, 128, 128, 36096, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 7168, 128, 128, 128, 7040, 0, 6912, 128, 128, 256, 2304, 0, 36224, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 1408, 0, 0, 17408, 128, 1280, 128, 256, 512, 128, 128, 128, 44928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8189, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 8161, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 8182, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 12, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8189, 0, 7763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 7673, 0, 8188, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8189, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 1, 0, 8191, 0, 665, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8191, 8113, 0, 8185, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1, 8184, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7956, 0, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8171, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 8191, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8184, 0, 7944, 5, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 1, 9, 0, 2, 0, 2, 8186, 8163, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 114, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 3, 0, 1, 8180, 1, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 8188, 0, 141, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7881, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8192, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8185, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[294, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 7945, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8105, 84, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7811, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8461, 8246, 7998, 8197, 8190, 8191, 8190, 8063] avg=8192.0000 max=8461 max_rank=0 min=7998 min_rank=2 imbalance_ratio=1.032837 max_load_reduction=57075 max_load_reduction_pct=0.870895 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[294, 0, 7763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7673, 0, 85, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 7952, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8189, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 8103, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 1, 0, 7811, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8057, 7760, 8227, 8193, 8191, 8192, 8196, 8720] avg=8192.0000 max=8720 max_rank=7 min=7760 min_rank=1 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=33299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[294, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8113, 0, 85, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1, 7947, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8172, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8100, 84, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7814, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8395, 8204, 7998, 8176, 8192, 8189, 8188, 8194] avg=8192.0000 max=8395 max_rank=0 min=7998 min_rank=2 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[294, 0, 7956, 0, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7814, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8271, 8191, 8185, 8192, 8191, 8190, 8057] avg=8192.0000 max=8271 max_rank=1 min=8057 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[294, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 7952, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8104, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7814, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8469, 8256, 7969, 8186, 8192, 8196, 8197, 8071] avg=8192.0000 max=8469 max_rank=0 min=7969 min_rank=2 imbalance_ratio=1.033813 max_load_reduction=57067 max_load_reduction_pct=0.870773 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[294, 0, 7944, 5, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 8163, 0, 85, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 3, 0, 1, 8180, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8100, 84, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8255, 8062, 8196, 8195, 8188, 8192, 8190] avg=8192.0000 max=8258 max_rank=0 min=8062 min_rank=2 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[294, 0, 7881, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8107, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7814, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8275, 8194, 8204, 8192, 8192, 8198, 8103] avg=8192.0000 max=8275 max_rank=1 min=8103 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[284, 0, 7175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7819, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7459, 8269, 8897, 8199, 8192, 8197, 8185, 8138] avg=8192.0000 max=8897 max_rank=2 min=7459 min_rank=0 imbalance_ratio=1.086060 max_load_reduction=56639 max_load_reduction_pct=0.864243 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 1152, 128, 0, 128, 0, 0, 128, 128, 0, 0, 128, 128, 0, 128, 0, 62592, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2432, 0, 63232, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 1920, 0, 128, 63616, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 64896, 0, 768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 64896, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 128, 128, 0, 128, 65536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 0, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 0, 0, 0, 0, 0, 476, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7966, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7415, 0, 0, 0, 0, 0, 1128, 0, 8037, 0, 0, 0, 0, 0, 0, 30, 0, 8190, 0, 0, 9, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7199, 0, 0, 0, 0, 0, 0, 0, 0, 762, 0, 8183, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 183, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 17, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 14, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6420, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 0, 8167, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 6, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 6411, 0, 3, 1, 195, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 20, 2, 0, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4649, 0, 0, 0, 3, 1, 0, 2, 0, 5725, 0, 8164, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 350, 0, 8152, 0, 0, 0, 0, 0, 0, 35, 0, 8115, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6427, 0, 0, 0, 0, 0, 0, 0, 0, 1725, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 8, 0, 8097, 0, 0, 0, 0, 0, 0, 2, 0, 8064, 0, 0, 17, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6507, 0, 0, 0, 0, 0, 0, 0, 0, 1861, 0, 8164, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 0, 680, 0, 7923, 0, 0, 0, 0, 0, 0, 19, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6297, 0, 1, 0, 0, 0, 0, 0, 0, 1742, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 13, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 8178, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 7, 10, 0, 8183, 0, 0, 0, 0, 0, 6, 0, 0, 7922, 0, 0, 1, 45, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8182, 0, 44, 0, 0, 4, 9, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6475, 0, 0, 4, 8, 0, 0, 2, 0, 1925, 0, 8155, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 96, 85, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7979, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6285, 0, 0, 0, 0, 0, 0, 0, 0, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8271, 8374, 8006, 8193, 8203, 8178, 8146] avg=8192.0000 max=8374 max_rank=2 min=8006 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=57162 max_load_reduction_pct=0.872223 moved_to_echo=32720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 0, 0, 0, 0, 117, 86, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7695, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8279, 8198, 8529, 8178, 8176, 8206, 7710] avg=8192.0000 max=8529 max_rank=3 min=7710 min_rank=7 imbalance_ratio=1.041138 max_load_reduction=57007 max_load_reduction_pct=0.869858 moved_to_echo=32270 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 94, 86, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6202, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8249, 8384, 8291, 8187, 8191, 8182, 7796] avg=8192.0000 max=8384 max_rank=2 min=7796 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32356 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 0, 94, 85, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 8030, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6209, 0, 0, 0, 0, 0, 0, 0, 0, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8280, 8139, 8428, 8147, 8183, 8206, 7934] avg=8192.0000 max=8428 max_rank=3 min=7934 min_rank=7 imbalance_ratio=1.028809 max_load_reduction=57108 max_load_reduction_pct=0.871399 moved_to_echo=32470 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 68, 67, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6344, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 8020, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4491, 0, 0, 0, 3, 1, 0, 2, 0, 5725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 7702, 8302, 6354, 8310, 8197, 8183, 10222] avg=8192.0000 max=10222 max_rank=7 min=6354 min_rank=3 imbalance_ratio=1.247803 max_load_reduction=55314 max_load_reduction_pct=0.844025 moved_to_echo=34912 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 13, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 2, 0, 8169, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 95, 83, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7839, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 8009, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8169, 0, 44, 0, 0, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8155, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6255, 0, 0, 4, 8, 0, 0, 2, 0, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8207, 8396, 7863, 8194, 8233, 8178, 8194] avg=8192.0000 max=8396 max_rank=2 min=7863 min_rank=3 imbalance_ratio=1.024902 max_load_reduction=57140 max_load_reduction_pct=0.871887 moved_to_echo=32799 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7415, 0, 0, 0, 0, 111, 91, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 8099, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6946, 0, 0, 0, 0, 0, 0, 0, 0, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8268, 7633, 9266, 8182, 8176, 8196, 7708] avg=8192.0000 max=9266 max_rank=3 min=7633 min_rank=2 imbalance_ratio=1.131104 max_load_reduction=56270 max_load_reduction_pct=0.858612 moved_to_echo=32262 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7909, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 92, 86, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6083, 0, 1, 0, 0, 0, 0, 0, 0, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7992, 8280, 8110, 8799, 8145, 8177, 8207, 7826] avg=8192.0000 max=8799 max_rank=3 min=7826 min_rank=7 imbalance_ratio=1.074097 max_load_reduction=56737 max_load_reduction_pct=0.865738 moved_to_echo=32355 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 50176, 0, 128, 128, 128, 128, 0, 128, 0, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 2944, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 62592, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 64000, 0, 0, 0, 0, 768, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 128, 0, 64256, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 64896, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 65408, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 1, 0, 0, 0, 0, 2, 17, 0, 0, 8185, 0, 7967, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 18, 3958, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 8178, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 4514, 0, 0, 0, 0, 8161, 33, 0, 0, 0, 1, 8048, 0, 0, 8189, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 6102, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 17, 0, 2082, 0, 0, 0, 0, 8191, 100, 0, 0, 0, 0, 8100, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 29, 0, 0, 0, 0, 0, 13, 0, 10, 8179, 0, 8178, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6271, 0, 0, 0, 0, 6, 4, 3, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 8072, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 746, 0, 1181, 0, 0, 0, 0, 8169, 3, 0, 0, 7, 0, 8151, 0, 15, 8180, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 6486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 954, 0, 0, 0, 0, 8192, 136, 0, 0, 0, 0, 8190, 1, 326, 8186, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 177, 0, 1317, 0, 0, 0, 0, 8183, 187, 0, 0, 0, 0, 8156, 0, 2281, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 5755, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 0, 1533, 0, 0, 0, 0, 8192, 296, 0, 0, 0, 0, 8116, 0, 642, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 7526, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6, 0, 612, 0, 0, 0, 0, 8169, 189, 0, 0, 0, 0, 8045, 0, 40, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 5863, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 400, 0, 1875, 0, 0, 0, 0, 8191, 133, 0, 0, 0, 0, 8105, 0, 6, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 29, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 10, 53, 0, 8178, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 67, 0, 0, 0, 8156, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 2, 0, 0, 8126, 31, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6271, 0, 0, 0, 2210, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 3, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 8072, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 5910, 0, 15, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8263, 8236, 8179, 8646, 8107, 8765, 7126] avg=8192.0000 max=8765 max_rank=6 min=7126 min_rank=7 imbalance_ratio=1.069946 max_load_reduction=56771 max_load_reduction_pct=0.866257 moved_to_echo=32644 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8170, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8122, 31, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4273, 0, 0, 0, 2212, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 5913, 0, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8236, 8187, 8169, 6663, 8194, 8201, 9698] avg=8192.0000 max=9698 max_rank=7 min=6663 min_rank=4 imbalance_ratio=1.183838 max_load_reduction=55838 max_load_reduction_pct=0.852020 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 1, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 17, 0, 0, 53, 0, 7967, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 26, 0, 0, 0, 8149, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8132, 30, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 18, 3958, 0, 0, 0, 2182, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 8178, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4514, 0, 0, 0, 0, 0, 33, 0, 0, 0, 1, 5836, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8041, 8191, 8179, 6326, 8195, 8025, 10388] avg=8192.0000 max=10388 max_rank=7 min=6326 min_rank=4 imbalance_ratio=1.268066 max_load_reduction=55148 max_load_reduction_pct=0.841492 moved_to_echo=32934 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 8139, 30, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 5863, 0, 0, 0, 2198, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 5877, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8216, 8184, 8212, 8241, 8181, 8424, 7891] avg=8192.0000 max=8424 max_rank=6 min=7891 min_rank=7 imbalance_ratio=1.028320 max_load_reduction=57112 max_load_reduction_pct=0.871460 moved_to_echo=32737 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8143, 34, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 5755, 0, 0, 0, 2205, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8018, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1533, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 5877, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8236, 8186, 8191, 8162, 8202, 8025, 8348] avg=8192.0000 max=8348 max_rank=7 min=8025 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32737 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 8156, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 30, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 7526, 0, 0, 0, 2182, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 5833, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8132, 8181, 8170, 9900, 8255, 8030, 6674] avg=8192.0000 max=9900 max_rank=4 min=6674 min_rank=7 imbalance_ratio=1.208496 max_load_reduction=55636 max_load_reduction_pct=0.848938 moved_to_echo=32859 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 30, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 6102, 0, 0, 0, 2196, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2082, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 5874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8167, 8184, 8164, 8473, 8255, 8041, 8056] avg=8192.0000 max=8473 max_rank=4 min=8041 min_rank=6 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=32825 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8179, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 8125, 31, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 6486, 0, 0, 0, 2221, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 5938, 1, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8245, 8187, 8272, 9125, 8147, 8025, 7355] avg=8192.0000 max=9125 max_rank=4 min=7355 min_rank=7 imbalance_ratio=1.113892 max_load_reduction=56411 max_load_reduction_pct=0.860764 moved_to_echo=32652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 384, 0, 0, 0, 128, 0, 0, 128, 46336, 0, 0, 0, 17664, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14080, 0, 0, 0, 0, 0, 1152, 0, 0, 128, 128, 47104, 128, 3328, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 64256, 0, 0, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 256, 128, 0, 0, 65152, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 512, 0, 65152, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7501, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8185, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1795, 0, 18, 2532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 8192, 0, 0, 0, 8179, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 789, 0, 0, 322, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 7037, 0, 0, 0, 22, 0, 0, 5, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8192, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8120, 0, 8192, 0, 0, 1, 8161, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 572, 4592, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 28, 3, 0, 0, 2914, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 4, 1, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 5, 8185, 4, 8182, 0, 2, 2, 8186, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 2787, 0, 310, 2504, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 11, 8, 0, 0, 0, 5, 0, 0, 0, 0, 0, 17, 0, 0, 2, 0, 0, 2774, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8151, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 6, 0, 6, 0, 0, 8169, 12, 0, 0, 0, 0, 0, 0, 0, 13, 7971, 2, 8167, 0, 0, 0, 8180, 2, 9, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7782, 0, 8192, 0, 0, 0, 8184, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2433, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6434, 0, 8192, 0, 0, 0, 8189, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 0, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6910, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4859, 0, 8192, 0, 0, 0, 8160, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 5098, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6627, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7501, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 2366, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6908, 6942, 8187, 9923, 8194, 8192, 8277, 8913] avg=8192.0000 max=9923 max_rank=3 min=6908 min_rank=0 imbalance_ratio=1.211304 max_load_reduction=55613 max_load_reduction_pct=0.848587 moved_to_echo=33576 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1795, 0, 5094, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6629, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2364, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8350, 9189, 8187, 6221, 8191, 8187, 8301, 8910] avg=8192.0000 max=9189 max_rank=1 min=6221 min_rank=3 imbalance_ratio=1.121704 max_load_reduction=56347 max_load_reduction_pct=0.859787 moved_to_echo=33589 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 5099, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 4592, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 6611, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 8171, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3, 0, 0, 2914, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 2366, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 4, 1, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8185, 4, 0, 0, 2, 2, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6579, 11793, 8193, 5366, 8189, 8196, 8301, 8919] avg=8192.0000 max=11793 max_rank=1 min=5366 min_rank=3 imbalance_ratio=1.439575 max_load_reduction=53743 max_load_reduction_pct=0.820053 moved_to_echo=33605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 789, 0, 5083, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 6615, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 7037, 0, 0, 0, 22, 0, 0, 5, 0, 0, 0, 0, 2359, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 1, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7412, 6955, 8187, 9455, 8215, 8217, 8255, 8840] avg=8192.0000 max=9455 max_rank=3 min=6955 min_rank=1 imbalance_ratio=1.154175 max_load_reduction=56081 max_load_reduction_pct=0.855728 moved_to_echo=33527 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2433, 0, 5100, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6629, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2367, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6434, 0, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9006, 6821, 8186, 10103, 8191, 8192, 7881, 7156] avg=8192.0000 max=10103 max_rank=3 min=6821 min_rank=1 imbalance_ratio=1.233276 max_load_reduction=55433 max_load_reduction_pct=0.845840 moved_to_echo=31420 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 2787, 0, 5095, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 2504, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6615, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 11, 8, 0, 0, 0, 5, 0, 0, 0, 0, 0, 17, 0, 8157, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2774, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2365, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 6, 0, 6, 0, 0, 8169, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7971, 2, 0, 0, 0, 0, 720, 2, 9, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9345, 9441, 8222, 5144, 8176, 8180, 8302, 8726] avg=8192.0000 max=9441 max_rank=1 min=5144 min_rank=3 imbalance_ratio=1.152466 max_load_reduction=56095 max_load_reduction_pct=0.855942 moved_to_echo=33384 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 0, 5097, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6629, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2366, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7782, 0, 0, 0, 0, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7610, 6728, 8187, 10047, 8191, 8186, 8084, 8503] avg=8192.0000 max=10047 max_rank=3 min=6728 min_rank=1 imbalance_ratio=1.226440 max_load_reduction=55489 max_load_reduction_pct=0.846695 moved_to_echo=32964 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 5087, 1457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6633, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6910, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2363, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4859, 0, 0, 0, 0, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10326, 7667, 8187, 9277, 8189, 8186, 8135, 5569] avg=8192.0000 max=10326 max_rank=0 min=5569 min_rank=7 imbalance_ratio=1.260498 max_load_reduction=55210 max_load_reduction_pct=0.842438 moved_to_echo=30079 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 13056, 0, 40832, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 46464, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 18944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 11648, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 52992, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 59776, 128, 0, 0, 128, 128, 5760, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 896, 128, 128, 128, 0, 0, 64640, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 16, 8183, 0, 1, 0, 0, 0, 0, 0, 7986, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8192, 0, 206, 0, 0, 8180, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8157, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 33, 0, 0, 8192, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 8026, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 166, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8057, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 135, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 6, 8180, 0, 0, 0, 0, 0, 0, 0, 7112, 8183, 22, 5, 0, 0, 0, 3, 0, 0, 5, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 2, 0, 0, 0, 8183, 0, 1100, 0, 4, 8171, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8180, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 12, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 8180, 8176, 1, 11, 0, 0, 0, 6, 43, 0, 16, 0, 0, 11, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 16, 15, 0, 0, 0, 0, 2, 0, 8169, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 7, 91, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8148, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 44, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 166, 0, 0, 7986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7984, 187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8183, 8203, 8159, 8190, 8173, 8196, 8200] avg=8192.0000 max=8232 max_rank=0 min=8159 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 16, 0, 0, 1, 0, 0, 0, 0, 0, 7986, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 206, 0, 0, 7977, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7972, 186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8179, 8202, 8195, 8168, 8160, 8195, 8221] avg=8192.0000 max=8221 max_rank=7 min=8160 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8057, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 135, 0, 0, 7988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7991, 187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8183, 8203, 8125, 8203, 8180, 8198, 8186] avg=8192.0000 max=8258 max_rank=0 min=8125 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 7988, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7989, 187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8359, 8183, 8203, 8032, 8195, 8178, 8197, 8189] avg=8192.0000 max=8359 max_rank=0 min=8032 min_rank=3 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7112, 201, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 5, 0, 0, 2, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1100, 0, 4, 7968, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 5, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 7982, 186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7346, 8192, 8181, 9088, 8170, 8182, 8186, 8191] avg=8192.0000 max=9088 max_rank=3 min=7346 min_rank=0 imbalance_ratio=1.109375 max_load_reduction=56448 max_load_reduction_pct=0.861328 moved_to_echo=32729 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 201, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 6, 43, 0, 16, 0, 0, 11, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8169, 0, 0, 0, 0, 0, 3, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 7, 91, 12, 7975, 185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8399, 8253, 8130, 7924, 8189, 8294, 8168, 8179] avg=8192.0000 max=8399 max_rank=0 min=7924 min_rank=3 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32830 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7988, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7991, 187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8381, 8183, 8207, 7992, 8207, 8180, 8201, 8185] avg=8192.0000 max=8381 max_rank=0 min=7992 min_rank=3 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 7977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8345, 8180, 8207, 8021, 8214, 8189, 8195, 8185] avg=8192.0000 max=8345 max_rank=0 min=8021 min_rank=3 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32783 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 64000, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 63872, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 128, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 0, 0, 0, 1792, 0, 128, 63872, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 65408, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 8192, 0, 0, 358, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 1463, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 1518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8184, 0, 6, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 8192, 0, 0, 24, 0, 0, 8173, 0, 0, 70, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 43, 0, 11, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 493, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 8180, 0, 0, 0, 8179, 3, 4, 0, 0, 0, 0, 10, 0, 7828, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 0, 0, 0, 0, 8183, 0, 0, 44, 0, 4, 8149, 0, 0, 579, 8182, 0, 0, 0, 0, 5, 5, 4, 3, 0, 0, 0, 0, 0, 0, 956, 1, 0, 0, 0, 0, 0, 56, 0, 11, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 5, 0, 8165, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6598, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 94, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 913, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 1413, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 7582, 0, 0, 0, 8174, 12, 0, 0, 25, 0, 0, 0, 0, 8083, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 0, 0, 3, 8168, 0, 0, 0, 0, 7, 8130, 13, 0, 103, 8173, 0, 0, 0, 0, 11, 0, 15, 13, 0, 0, 0, 0, 0, 0, 176, 2, 0, 3, 0, 0, 0, 1526, 8, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 8178, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 15, 0, 0, 0, 0, 0, 6744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 7935, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 6982, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8245, 7963, 8155, 7320, 8113, 8205, 9272] avg=8192.0000 max=9272 max_rank=7 min=7320 min_rank=4 imbalance_ratio=1.131836 max_load_reduction=56264 max_load_reduction_pct=0.858521 moved_to_echo=32910 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 71, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7869, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 7936, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 6982, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8257, 8229, 8048, 8828, 8254, 8187, 7469] avg=8192.0000 max=8828 max_rank=4 min=7469 min_rank=7 imbalance_ratio=1.077637 max_load_reduction=56708 max_load_reduction_pct=0.865295 moved_to_echo=32738 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 71, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 7851, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 7935, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 11, 0, 0, 0, 6, 0, 0, 0, 0, 6975, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1138, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8202, 7945, 8294, 7292, 7940, 8235, 9360] avg=8192.0000 max=9360 max_rank=7 min=7292 min_rank=4 imbalance_ratio=1.142578 max_load_reduction=56176 max_load_reduction_pct=0.857178 moved_to_echo=32827 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 0, 0, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 7936, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1413, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 6982, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8219, 8782, 8070, 8654, 8190, 8183, 7175] avg=8192.0000 max=8782 max_rank=2 min=7175 min_rank=7 imbalance_ratio=1.072021 max_load_reduction=56754 max_load_reduction_pct=0.865997 moved_to_echo=32202 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 8180, 0, 0, 0, 72, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 7828, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4, 7829, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4, 3, 0, 0, 0, 0, 0, 0, 956, 1, 7927, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 11, 0, 0, 0, 322, 0, 0, 0, 0, 6970, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 5, 0, 8165, 0, 0, 8, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6598, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1137, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 7924, 8456, 8912, 7615, 8273, 8174, 7917] avg=8192.0000 max=8912 max_rank=3 min=7615 min_rank=4 imbalance_ratio=1.087891 max_load_reduction=56624 max_load_reduction_pct=0.864014 moved_to_echo=31979 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 7582, 0, 0, 0, 71, 12, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7, 7810, 13, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 15, 13, 0, 0, 0, 0, 0, 0, 176, 2, 7912, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1526, 8, 0, 0, 0, 0, 316, 0, 0, 0, 0, 6966, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 8178, 0, 0, 10, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 15, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7693, 8164, 7938, 8139, 9075, 8288, 8190, 8049] avg=8192.0000 max=9075 max_rank=4 min=7693 min_rank=0 imbalance_ratio=1.107788 max_load_reduction=56461 max_load_reduction_pct=0.861526 moved_to_echo=33602 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 71, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 0, 0, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7935, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6982, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8263, 8362, 7949, 7994, 8241, 8181, 8281] avg=8192.0000 max=8362 max_rank=2 min=7949 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32697 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 7936, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6984, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8262, 7861, 7969, 8758, 8237, 8181, 8013] avg=8192.0000 max=8758 max_rank=4 min=7861 min_rank=2 imbalance_ratio=1.069092 max_load_reduction=56778 max_load_reduction_pct=0.866364 moved_to_echo=33189 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 55168, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 9216, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 6912, 128, 128, 0, 0, 0, 896, 0, 0, 0, 0, 55936, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 62848, 128, 0, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 2048, 128, 63488, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 0, 64896, 0, 0, 128, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 65024, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 0, 65024, 0, 0, 0, 640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 8187, 0, 8173, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 80, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8108, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 37, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6526, 8192, 0, 8166, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1990, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 95, 8183, 0, 8177, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 5, 0, 0, 0, 8170, 0, 0, 11, 33, 0, 0, 4, 0, 11, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8030, 0, 6, 8177, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 12, 0, 0, 5, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 6060, 8173, 0, 8131, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 16, 0, 0, 34, 7983, 0, 29, 1, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 1, 8167, 0, 0, 0, 10, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 2098, 4, 0, 0, 0, 19, 0, 14, 0, 0, 0, 3, 0, 0, 0, 0, 0, 12, 0, 164, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5824, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7457, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 11, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 0, 0, 4313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5872, 0, 0, 0, 0, 0, 0, 0, 3872, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6633, 8200, 8196, 8185, 8198, 9800, 8132] avg=8192.0000 max=9800 max_rank=6 min=6633 min_rank=1 imbalance_ratio=1.196289 max_load_reduction=55736 max_load_reduction_pct=0.850464 moved_to_echo=34315 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 0, 0, 4303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 10, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 46, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7347, 0, 0, 0, 0, 0, 0, 0, 3863, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8080, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 5145, 8253, 8201, 8196, 8160, 11266, 8125] avg=8192.0000 max=11266 max_rank=6 min=5145 min_rank=1 imbalance_ratio=1.375244 max_load_reduction=54270 max_load_reduction_pct=0.828094 moved_to_echo=35747 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1990, 0, 0, 4313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 3872, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6303, 8156, 8196, 8194, 8230, 10135, 8131] avg=8192.0000 max=10135 max_rank=6 min=6303 min_rank=1 imbalance_ratio=1.237183 max_load_reduction=55401 max_load_reduction_pct=0.845352 moved_to_echo=34690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 4305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8170, 0, 0, 11, 33, 0, 0, 4, 0, 11, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 6, 46, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 5, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 3865, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8075, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 4400, 8272, 8238, 8195, 8089, 12027, 8123] avg=8192.0000 max=12027 max_rank=6 min=4400 min_rank=1 imbalance_ratio=1.468140 max_load_reduction=53509 max_load_reduction_pct=0.816483 moved_to_echo=36434 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7457, 0, 0, 4313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 0, 0, 0, 0, 0, 0, 0, 3872, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 8090, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 11770, 8166, 8191, 8193, 8210, 4605, 8210] avg=8192.0000 max=11770 max_rank=1 min=4605 min_rank=6 imbalance_ratio=1.436768 max_load_reduction=53766 max_load_reduction_pct=0.820404 moved_to_echo=29218 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6060, 0, 0, 4281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 8131, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 34, 7983, 0, 29, 1, 57, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 1, 46, 0, 0, 0, 10, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 2098, 4, 0, 0, 0, 19, 0, 14, 3843, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 12, 0, 164, 0, 0, 0, 0, 8065, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 10341, 8180, 8124, 8177, 8180, 6050, 8286] avg=8192.0000 max=10341 max_rank=1 min=6050 min_rank=6 imbalance_ratio=1.262329 max_load_reduction=55195 max_load_reduction_pct=0.842209 moved_to_echo=30693 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6526, 0, 0, 4299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 46, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1697, 0, 0, 0, 0, 0, 0, 0, 3860, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8090, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 10825, 8151, 8187, 8195, 8240, 5613, 8134] avg=8192.0000 max=10825 max_rank=1 min=5613 min_rank=6 imbalance_ratio=1.321411 max_load_reduction=54711 max_load_reduction_pct=0.834824 moved_to_echo=30182 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5824, 0, 0, 4295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2105, 0, 0, 0, 0, 0, 0, 0, 3877, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 0, 8095, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 10119, 8158, 8203, 8201, 8229, 6040, 8395] avg=8192.0000 max=10119 max_rank=1 min=6040 min_rank=6 imbalance_ratio=1.235229 max_load_reduction=55417 max_load_reduction_pct=0.845596 moved_to_echo=30865 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 31232, 0, 0, 34432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 34176, 128, 0, 0, 0, 128, 0, 128, 30976, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 65152, 0, 128, 384, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 512, 0, 0, 0, 0, 64768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 384, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 65408, 0, 128, 128, 128, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2620, 8192, 0, 0, 125, 0, 0, 0, 0, 0, 53, 0, 0, 9, 0, 0, 0, 0, 5626, 0, 0, 8161, 0, 0, 0, 8191, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 8192, 0, 0, 27, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 7182, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 8175, 0, 0, 97, 12, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 8146, 0, 0, 0, 8186, 0, 0, 2, 1, 0, 0, 0, 24, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8173, 0, 5, 1, 3, 0, 0, 0, 7890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 0, 0, 0, 0, 8184, 5, 0, 32, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6864, 8192, 0, 0, 12, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1327, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 28, 0, 0, 8020, 0, 0, 0, 0, 0, 19, 0, 0, 0, 5094, 8144, 0, 0, 0, 41, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3302, 0, 0, 8167, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8161, 0, 0, 5, 7, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 21, 0, 12, 0, 2, 0, 0, 0, 0, 0, 0, 0, 121, 3, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 16, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7687, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7625, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 27, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7182, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 159, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 4098, 3988, 43, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5194, 8190, 11115, 8189, 8319, 8187, 8157, 8185] avg=8192.0000 max=11115 max_rank=2 min=5194 min_rank=0 imbalance_ratio=1.356812 max_load_reduction=54421 max_load_reduction_pct=0.830399 moved_to_echo=32848 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 125, 0, 0, 0, 0, 0, 53, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5626, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 158, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4084, 3975, 43, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6787, 8336, 9572, 8178, 8154, 8186, 8147, 8176] avg=8192.0000 max=9572 max_rank=2 min=6787 min_rank=0 imbalance_ratio=1.168457 max_load_reduction=55964 max_load_reduction_pct=0.853943 moved_to_echo=32663 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 97, 12, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8006, 158, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 5, 1, 3, 0, 0, 0, 7890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 3924, 43, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 0, 0, 0, 0, 8179, 5, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4486, 8249, 12038, 8213, 8066, 8193, 8057, 8234] avg=8192.0000 max=12038 max_rank=2 min=4486 min_rank=0 imbalance_ratio=1.469482 max_load_reduction=53498 max_load_reduction_pct=0.816315 moved_to_echo=32550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7687, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 159, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 4099, 3998, 43, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4687, 8152, 11676, 8189, 8214, 8186, 8245, 8187] avg=8192.0000 max=11676 max_rank=2 min=4687 min_rank=0 imbalance_ratio=1.425293 max_load_reduction=53860 max_load_reduction_pct=0.821838 moved_to_echo=32832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 12, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 159, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4099, 3995, 43, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11053, 8169, 5274, 8188, 8322, 8186, 8159, 8185] avg=8192.0000 max=11053 max_rank=0 min=5274 min_rank=2 imbalance_ratio=1.349243 max_load_reduction=54483 max_load_reduction_pct=0.831345 moved_to_echo=32852 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 28, 0, 0, 4105, 0, 0, 0, 0, 0, 19, 0, 0, 0, 5094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 41, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3302, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7994, 158, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 5, 7, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 21, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2, 0, 0, 0, 0, 0, 0, 0, 121, 3, 0, 0, 0, 0, 4087, 3915, 43, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9247, 8147, 7224, 8193, 8113, 8226, 8188, 8198] avg=8192.0000 max=9247 max_rank=0 min=7224 min_rank=2 imbalance_ratio=1.128784 max_load_reduction=56289 max_load_reduction_pct=0.858902 moved_to_echo=32725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 159, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 8020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 4099, 3999, 43, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11818, 8149, 4509, 8189, 8188, 8186, 8310, 8187] avg=8192.0000 max=11818 max_rank=0 min=4509 min_rank=2 imbalance_ratio=1.442627 max_load_reduction=53718 max_load_reduction_pct=0.819672 moved_to_echo=32871 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 161, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 4104, 4005, 48, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12264, 8144, 4128, 8197, 8160, 8186, 8273, 8184] avg=8192.0000 max=12264 max_rank=0 min=4128 min_rank=2 imbalance_ratio=1.497070 max_load_reduction=53272 max_load_reduction_pct=0.812866 moved_to_echo=32803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[384, 0, 128, 128, 0, 0, 0, 0, 0, 256, 128, 128, 128, 0, 0, 0, 32768, 31872, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 34048, 0, 0, 31488, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 33408, 0, 0, 0, 0, 0, 128, 0, 0, 0, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1408, 0, 128, 128, 128, 0, 0, 0, 64256, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 64256, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65152, 0, 0, 384, 128, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 45, 8183, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 93, 0, 0, 0, 0, 0, 1426, 0, 0, 0, 0, 0, 0, 6100, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 2, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 37, 0, 0, 0, 0, 0, 1771, 0, 0, 0, 0, 0, 0, 3663, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 8192, 0, 0, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 0, 0, 0, 1605, 0, 0, 0, 0, 0, 0, 2984, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 0, 0, 8177, 8, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 30, 8178, 0, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8156, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 48, 0, 0, 0, 7708, 10, 0, 0, 0, 0, 0, 15, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8106, 0, 0, 0, 4, 0, 2, 2, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 2763, 5, 0, 0, 0, 0, 1, 3063, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 31, 0, 0, 0, 0, 0, 4, 14, 0, 0, 0, 0, 5, 0, 0, 0, 14, 0, 6, 8155, 0, 8168, 0, 0, 7260, 0, 0, 0, 0, 0, 0, 0, 13, 3, 8137, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 8033, 0, 0, 0, 0, 10, 0, 0, 1, 0, 0, 18, 0, 0, 10, 4, 0, 0, 7, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 15, 0, 0, 12, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 225, 14, 0, 0, 0, 0, 0, 680, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 0, 4065, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 5400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2792, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 6, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 0, 7390, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4712, 0, 0, 0, 0, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 1771, 0, 0, 0, 0, 0, 0, 3663, 0, 0, 3277, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8209, 7638, 8193, 8261, 8195, 8114, 8758] avg=8192.0000 max=8758 max_rank=7 min=7638 min_rank=2 imbalance_ratio=1.069092 max_load_reduction=56778 max_load_reduction_pct=0.866364 moved_to_echo=33328 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4712, 45, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 1426, 0, 0, 0, 0, 0, 0, 6100, 0, 0, 3277, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8185, 5868, 8181, 7793, 8192, 8250, 10907] avg=8192.0000 max=10907 max_rank=7 min=5868 min_rank=2 imbalance_ratio=1.331421 max_load_reduction=54629 max_load_reduction_pct=0.833572 moved_to_echo=35142 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4712, 1, 0, 0, 0, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1605, 0, 0, 0, 0, 0, 0, 2984, 0, 0, 3277, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8193, 8436, 8201, 8297, 8193, 8154, 7877] avg=8192.0000 max=8436 max_rank=2 min=7877 min_rank=7 imbalance_ratio=1.029785 max_load_reduction=57100 max_load_reduction_pct=0.871277 moved_to_echo=32521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4698, 30, 0, 0, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8156, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 48, 0, 0, 0, 7708, 10, 0, 0, 0, 0, 0, 15, 0, 2, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8106, 0, 0, 0, 4, 0, 2, 2, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 2763, 5, 0, 0, 0, 0, 1, 3063, 0, 0, 3268, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8187, 7507, 8181, 7933, 8180, 8173, 9184] avg=8192.0000 max=9184 max_rank=7 min=7507 min_rank=2 imbalance_ratio=1.121094 max_load_reduction=56352 max_load_reduction_pct=0.859863 moved_to_echo=33470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 4, 14, 0, 0, 0, 0, 5, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 6, 4691, 0, 0, 0, 0, 7260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3, 8137, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 8033, 0, 0, 0, 0, 10, 0, 0, 1, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 10, 4, 0, 0, 7, 0, 8152, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 15, 0, 0, 12, 0, 0, 2, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 225, 14, 0, 0, 0, 0, 0, 680, 0, 2, 3262, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8211, 11971, 8164, 8263, 8192, 8242, 4206] avg=8192.0000 max=11971 max_rank=2 min=4206 min_rank=7 imbalance_ratio=1.461304 max_load_reduction=53565 max_load_reduction_pct=0.817337 moved_to_echo=28903 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4712, 0, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 0, 7390, 0, 0, 3277, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8185, 5335, 8199, 8312, 8193, 8183, 10947] avg=8192.0000 max=10947 max_rank=7 min=5335 min_rank=2 imbalance_ratio=1.336304 max_load_reduction=54589 max_load_reduction_pct=0.832962 moved_to_echo=35635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4712, 0, 0, 0, 0, 5400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2792, 0, 0, 3277, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8184, 10112, 8202, 8339, 8193, 8238, 6081] avg=8192.0000 max=10112 max_rank=2 min=6081 min_rank=7 imbalance_ratio=1.234375 max_load_reduction=55424 max_load_reduction_pct=0.845703 moved_to_echo=30851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4691, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 3282, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8182, 8669, 8215, 8338, 8198, 8182, 7576] avg=8192.0000 max=8669 max_rank=2 min=7576 min_rank=7 imbalance_ratio=1.058228 max_load_reduction=56867 max_load_reduction_pct=0.867722 moved_to_echo=32294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 0, 0, 128, 0, 0, 8320, 128, 0, 0, 0, 0, 128, 30848, 0, 128, 26240, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 37760, 128, 0, 0, 0, 27904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 0, 64256, 128, 0, 0, 0, 128, 0, 128, 128, 128, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 65536, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 64896, 128, 0, 0, 128, 0, 128, 128, 0, 256, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 0, 65536, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 37, 0, 4, 0, 29, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 8124, 0, 0, 0, 9, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 59, 9, 10, 0, 37, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 10, 0, 0, 0, 0, 7, 0, 0, 0, 8154, 0, 0, 0, 4, 0, 0, 0, 0, 19, 0, 3, 2, 0, 0, 0, 0, 8171, 0, 8155, 0, 31, 0, 0, 0, 3, 0, 0, 0, 0, 4, 0, 2, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2910, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 6217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 8143, 0, 0, 12, 21, 9, 1, 92, 0, 25, 0, 0, 0, 0, 0, 8153, 0, 0, 10, 0, 0, 36, 0, 0, 0, 0, 0, 8163, 0, 0, 16, 0, 11, 34, 19, 0, 10, 0, 0, 5, 681, 0, 0, 0, 0, 18, 0, 12, 9, 0, 0, 0, 0, 0, 0, 0, 6824, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 12, 0, 0, 0, 3, 8168, 0, 8133, 0, 0, 0, 0, 0, 6, 0, 761, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 2426, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8201, 8199, 8086, 10633, 8198, 5802, 8217] avg=8192.0000 max=10633 max_rank=4 min=5802 min_rank=6 imbalance_ratio=1.297974 max_load_reduction=54903 max_load_reduction_pct=0.837753 moved_to_echo=32850 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 37, 0, 4, 0, 29, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6873, 2419, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5713, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8196, 8167, 8142, 9374, 8197, 7044, 8216] avg=8192.0000 max=9374 max_rank=4 min=7044 min_rank=6 imbalance_ratio=1.144287 max_load_reduction=56162 max_load_reduction_pct=0.856964 moved_to_echo=32831 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 7, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 59, 9, 10, 0, 37, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 10, 0, 0, 0, 0, 7, 0, 0, 0, 8154, 2424, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 19, 0, 3, 2, 0, 0, 0, 0, 8109, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5724, 0, 0, 0, 31, 0, 0, 0, 3, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8194, 8170, 8178, 10679, 8160, 5764, 8220] avg=8192.0000 max=10679 max_rank=4 min=5764 min_rank=6 imbalance_ratio=1.303589 max_load_reduction=54857 max_load_reduction_pct=0.837051 moved_to_echo=32823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2910, 2429, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8201, 8198, 8080, 5421, 8199, 11022, 8215] avg=8192.0000 max=11022 max_rank=6 min=5421 min_rank=4 imbalance_ratio=1.345459 max_load_reduction=54514 max_load_reduction_pct=0.831818 moved_to_echo=32857 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 8133, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 8, 0, 0, 12, 21, 9, 1, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8153, 0, 0, 10, 0, 0, 36, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 16, 0, 11, 34, 19, 0, 10, 0, 0, 5, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 12, 9, 0, 0, 0, 0, 0, 0, 0, 6824, 2423, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 12, 0, 0, 0, 3, 8127, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 0, 0, 0, 0, 0, 0, 0, 6, 0, 761, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8252, 8232, 8827, 9368, 8175, 6497, 8024] avg=8192.0000 max=9368 max_rank=4 min=6497 min_rank=6 imbalance_ratio=1.143555 max_load_reduction=56168 max_load_reduction_pct=0.857056 moved_to_echo=32064 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 9, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 2430, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8166, 8177, 8080, 4544, 8203, 11956, 8210] avg=8192.0000 max=11956 max_rank=6 min=4544 min_rank=4 imbalance_ratio=1.459473 max_load_reduction=53580 max_load_reduction_pct=0.817566 moved_to_echo=32913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 2434, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8126, 8203, 8063, 4814, 8206, 11700, 8220] avg=8192.0000 max=11700 max_rank=6 min=4814 min_rank=4 imbalance_ratio=1.428223 max_load_reduction=53836 max_load_reduction_pct=0.821472 moved_to_echo=32940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2430, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8200, 8190, 8080, 10703, 8198, 5751, 8214] avg=8192.0000 max=10703 max_rank=4 min=5751 min_rank=6 imbalance_ratio=1.306519 max_load_reduction=54833 max_load_reduction_pct=0.836685 moved_to_echo=32866 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[45952, 0, 0, 0, 128, 0, 0, 0, 128, 0, 19712, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 45440, 19456, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 65408, 0, 0, 0, 0, 128, 0, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 128, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 64640, 0, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 182, 0, 0, 78, 0, 0, 0, 0, 0, 7996, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8128, 5, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 18, 0, 0, 89, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8175, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 8178, 0, 0, 0, 0, 0, 0, 0, 241, 253, 8172, 0, 17, 0, 0, 0, 4, 0, 2, 0, 0, 7780, 0, 1, 0, 0, 8182, 0, 8157, 0, 0, 8102, 0, 7, 0, 0, 8113, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 8179, 0, 0, 10, 0, 9, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 6, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 8158, 0, 0, 0, 0, 30, 0, 0, 902, 0, 7753, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8076, 0, 122, 0, 0, 8161, 4, 7200, 0, 0, 8135, 0, 0, 0, 16, 7944, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 6, 19, 0, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 12, 8170, 0, 454, 21, 0, 11, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 497, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 7679, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 11, 0, 0, 10, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 37, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 427, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8192, 0, 7739, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7897, 0, 18, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7815, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8004, 8423, 8222, 8190, 8109, 8198, 8196] avg=8192.0000 max=8423 max_rank=2 min=8004 min_rank=1 imbalance_ratio=1.028198 max_load_reduction=57113 max_load_reduction_pct=0.871475 moved_to_echo=32693 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7897, 0, 182, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7712, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8125, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8157, 8425, 8214, 8192, 8006, 8154, 8196] avg=8192.0000 max=8425 max_rank=2 min=8006 min_rank=5 imbalance_ratio=1.028442 max_load_reduction=57111 max_load_reduction_pct=0.871445 moved_to_echo=32548 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 8178, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 253, 7878, 0, 17, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 1, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8087, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 9, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 7503, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 6, 0, 17, 0, 0, 8100, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8182, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8395, 8388, 8167, 8193, 7849, 8158, 8194] avg=8192.0000 max=8395 max_rank=1 min=7849 min_rank=5 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32394 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7897, 0, 11, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7885, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7918, 8426, 8223, 8190, 8180, 8214, 8195] avg=8192.0000 max=8426 max_rank=2 min=7918 min_rank=1 imbalance_ratio=1.028564 max_load_reduction=57110 max_load_reduction_pct=0.871429 moved_to_echo=32779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 8158, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 902, 0, 7474, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 122, 0, 0, 0, 4, 7200, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 6, 19, 7918, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 12, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 21, 0, 11, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 7789, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8161, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8414, 7572, 8037, 8206, 8793, 8166, 8170] avg=8192.0000 max=8793 max_rank=5 min=7572 min_rank=2 imbalance_ratio=1.073364 max_load_reduction=56743 max_load_reduction_pct=0.865829 moved_to_echo=33335 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 7896, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 7679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8395, 7913, 8222, 8190, 8210, 8216, 8195] avg=8192.0000 max=8395 max_rank=1 min=7913 min_rank=2 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32811 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7897, 0, 2, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7865, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7936, 8425, 8223, 8190, 8159, 8215, 8195] avg=8192.0000 max=8425 max_rank=2 min=7936 min_rank=1 imbalance_ratio=1.028442 max_load_reduction=57111 max_load_reduction_pct=0.871445 moved_to_echo=32759 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 7889, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 7739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7898, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8317, 7964, 8228, 8185, 8230, 8215, 8195] avg=8192.0000 max=8317 max_rank=1 min=7964 min_rank=2 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32825 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 512, 128, 0, 128, 0, 0, 0, 0, 384, 0, 128, 0, 0, 0, 0, 62464, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 2176, 256, 62848, 0, 256, 0, 0, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1920, 0, 128, 0, 0, 0, 128, 63616, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 128, 65024, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True + [2026-08-26 16:28:31] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 438.3 | throughput per GPU (TFLOP/s/GPU): 412.6 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.361202E+01 | loss scale: 1.0 | grad norm: 121.632 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 0, 2, 30, 613, 14, 0, 0, 8049, 0, 0, 0, 1, 0, 5, 0, 0, 0, 1, 484, 1, 3, 2757, 11, 0, 0, 8192, 2, 8191, 0, 17, 0, 0, 9, 1, 1, 0, 0, 5, 1, 0, 21, 3, 0, 0, 1, 1, 0, 1, 3, 0, 0, 3, 3, 0, 0, 0, 0, 7894, 0, 0, 0, 3, 2, 0, 0, 0, 3400, 0, 2, 0, 3665, 1, 6354, 3, 1, 0, 2851, 0, 0, 2, 0, 0, 1, 0, 0, 6468, 0, 47, 0, 1, 2, 2, 3, 0, 0, 12, 0, 0, 0, 3, 0, 0, 0, 39, 5953, 0, 0, 2, 1, 0, 0, 27, 0, 0, 0, 1, 1, 0, 0, 0, 2, 364, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 32, 514, 33, 0, 0, 7996, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 447, 0, 0, 3788, 7, 1, 0, 8185, 4, 8184, 0, 0, 1, 3, 3, 0, 1, 0, 0, 0, 0, 1, 3, 0, 0, 0, 1, 0, 0, 5, 23, 0, 0, 0, 4, 0, 4, 4, 0, 8070, 0, 2, 1, 0, 2, 0, 0, 1, 3220, 0, 0, 0, 3466, 2, 6782, 8, 0, 3, 1687, 0, 0, 2, 0, 2, 13, 0, 1, 5817, 1, 3, 0, 0, 4, 0, 8, 0, 1, 41, 3, 0, 0, 0, 0, 0, 1, 3, 6603, 1, 0, 8, 3, 2, 0, 122, 0, 2, 0, 6, 1, 0, 0, 0, 0, 396, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 55, 743, 34, 0, 0, 8056, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 550, 0, 0, 2669, 31, 0, 0, 8192, 1, 8192, 0, 13, 0, 1, 5, 1, 1, 1, 0, 0, 3, 0, 44, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 7871, 0, 0, 0, 1, 0, 0, 0, 0, 3465, 0, 0, 0, 4032, 0, 6783, 1, 0, 0, 2240, 0, 0, 0, 0, 0, 4, 0, 0, 6101, 0, 11, 0, 1, 4, 0, 9, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 19, 5808, 0, 0, 3, 4, 0, 0, 106, 0, 0, 0, 3, 7, 0, 0, 1, 0, 432, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 23, 871, 17, 0, 0, 8055, 0, 0, 1, 0, 0, 0, 2, 1, 1, 0, 546, 0, 0, 2337, 8, 0, 0, 8190, 4, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1, 0, 0, 0, 7953, 1, 0, 0, 0, 0, 1, 0, 0, 2673, 0, 0, 0, 3990, 0, 6732, 2, 0, 0, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 6377, 0, 17, 0, 0, 8, 0, 3, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4, 6888, 0, 0, 1, 0, 0, 1, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 82, 581, 6, 0, 0, 8088, 0, 0, 0, 2, 1, 4, 0, 1, 0, 0, 578, 0, 0, 1960, 14, 0, 0, 8191, 4, 8190, 0, 4, 1, 0, 5, 0, 2, 2, 0, 1, 1, 0, 6, 1, 0, 1, 0, 0, 0, 3, 4, 1, 0, 0, 1, 0, 2, 0, 0, 7871, 0, 0, 0, 1, 1, 0, 0, 0, 3597, 0, 0, 0, 4214, 0, 6850, 9, 0, 0, 2472, 0, 1, 0, 0, 0, 1, 0, 0, 5803, 0, 20, 0, 0, 0, 0, 7, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 19, 6462, 0, 0, 3, 0, 0, 0, 21, 0, 2, 0, 0, 0, 0, 0, 1, 0, 422, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 47, 725, 22, 0, 0, 8155, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 555, 0, 0, 3126, 2, 0, 0, 8191, 0, 8191, 3, 1, 0, 0, 11, 0, 0, 0, 0, 0, 10, 0, 13, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 3, 0, 7967, 0, 0, 0, 2, 1, 0, 0, 0, 3545, 0, 0, 0, 3652, 0, 6998, 0, 0, 0, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 6027, 0, 25, 0, 0, 5, 0, 38, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 5697, 0, 0, 7, 0, 0, 0, 57, 0, 1, 0, 1, 2, 0, 0, 0, 1, 425, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 90, 575, 32, 0, 1, 8092, 0, 0, 0, 0, 3, 6, 0, 1, 0, 0, 563, 0, 0, 3526, 3, 2, 0, 8191, 13, 8190, 0, 1, 0, 0, 9, 0, 4, 1, 0, 3, 1, 0, 64, 3, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 3, 0, 3134, 0, 7212, 9, 0, 0, 1900, 0, 0, 1, 0, 0, 5, 0, 0, 5883, 0, 77, 0, 0, 17, 0, 68, 0, 0, 54, 0, 0, 0, 1, 0, 0, 0, 21, 5799, 0, 0, 12, 0, 0, 0, 25, 0, 0, 0, 9, 0, 1, 0, 0, 0, 394, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 40, 1059, 24, 0, 0, 8077, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 496, 0, 0, 3371, 6, 2, 0, 8192, 3, 8190, 0, 1, 0, 0, 4, 0, 0, 4, 0, 1, 1, 0, 11, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 3290, 0, 0, 2, 3507, 0, 6603, 2, 2, 0, 2012, 0, 0, 0, 0, 0, 1, 0, 0, 6417, 0, 1, 0, 0, 0, 2, 20, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 11, 5607, 0, 0, 4, 7, 0, 0, 194, 0, 0, 0, 0, 1, 0, 0, 1, 0, 397, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 32, 514, 33, 0, 0, 7333, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 3788, 7, 1, 0, 0, 4, 4700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3, 0, 1, 0, 0, 0, 0, 1, 3, 0, 0, 0, 1, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 23, 0, 0, 0, 4, 0, 4, 4, 0, 8070, 0, 2, 1, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3220, 0, 0, 0, 2511, 2, 0, 8, 0, 3, 1687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 13, 0, 1, 5817, 1, 3, 0, 0, 4, 0, 8, 0, 955, 557, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 41, 3, 0, 0, 0, 0, 0, 1, 3, 6603, 1, 0, 8, 3, 2, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 2, 0, 6, 1, 0, 0, 0, 0, 396, 0, 0, 0, 0, 6782, 883, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7916, 8947, 8160, 8336, 7434, 7803, 8710, 8230] avg=8192.0000 max=8947 max_rank=1 min=7434 min_rank=4 imbalance_ratio=1.092163 max_load_reduction=56589 max_load_reduction_pct=0.863480 moved_to_echo=32177 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 2, 30, 613, 14, 0, 0, 7381, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 484, 1, 3, 2757, 11, 0, 0, 0, 2, 4704, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 1, 0, 0, 5, 1, 0, 21, 3, 0, 0, 1, 1, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 3, 3, 0, 0, 0, 0, 7894, 0, 0, 0, 3, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3400, 0, 2, 0, 2655, 1, 0, 3, 1, 0, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 6468, 0, 47, 0, 1, 2, 2, 3, 0, 1010, 558, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3, 0, 0, 0, 39, 5953, 0, 0, 2, 1, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 1, 1, 0, 0, 0, 2, 364, 0, 0, 0, 0, 6354, 883, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 7980, 8198, 8132, 8915, 8537, 8056, 7669] avg=8192.0000 max=8915 max_rank=4 min=7669 min_rank=7 imbalance_ratio=1.088257 max_load_reduction=56621 max_load_reduction_pct=0.863968 moved_to_echo=33177 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 23, 871, 17, 0, 0, 7387, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 546, 0, 0, 2337, 8, 0, 0, 0, 4, 4703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1, 0, 0, 0, 7953, 1, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2673, 0, 0, 0, 2890, 0, 0, 2, 0, 0, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6377, 0, 17, 0, 0, 8, 0, 3, 0, 1100, 558, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4, 6888, 0, 0, 1, 0, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 0, 0, 0, 6732, 883, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8299, 7602, 8173, 8183, 7746, 8506, 8951, 8076] avg=8192.0000 max=8951 max_rank=6 min=7602 min_rank=1 imbalance_ratio=1.092651 max_load_reduction=56585 max_load_reduction_pct=0.863419 moved_to_echo=33279 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 90, 575, 32, 0, 1, 7421, 0, 0, 0, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 563, 0, 0, 3526, 3, 2, 0, 0, 13, 4703, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4, 1, 0, 3, 1, 0, 64, 3, 0, 0, 9, 1, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 8052, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 3, 0, 2270, 0, 0, 9, 0, 0, 1900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 5883, 0, 77, 0, 0, 17, 0, 68, 0, 864, 558, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 1, 0, 0, 0, 21, 5799, 0, 0, 12, 0, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 9, 0, 1, 0, 0, 0, 394, 3, 0, 0, 0, 7212, 883, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8812, 8249, 8282, 7649, 7918, 7933, 8564] avg=8192.0000 max=8812 max_rank=1 min=7649 min_rank=4 imbalance_ratio=1.075684 max_load_reduction=56724 max_load_reduction_pct=0.865540 moved_to_echo=32064 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 55, 743, 34, 0, 0, 7388, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 550, 0, 0, 2669, 31, 0, 0, 0, 1, 4705, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 1, 1, 1, 0, 0, 3, 0, 44, 2, 0, 0, 1, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 7871, 0, 0, 0, 1, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 0, 0, 0, 2921, 0, 0, 1, 0, 0, 2240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6101, 0, 11, 0, 1, 4, 0, 9, 0, 1111, 558, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 19, 5808, 0, 0, 3, 4, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 3, 7, 0, 0, 1, 0, 432, 0, 0, 0, 0, 6783, 883, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 7970, 8213, 8104, 8627, 8242, 7898, 8253] avg=8192.0000 max=8627 max_rank=4 min=7898 min_rank=6 imbalance_ratio=1.053101 max_load_reduction=56909 max_load_reduction_pct=0.868362 moved_to_echo=33020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 47, 725, 22, 0, 0, 7478, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 555, 0, 0, 3126, 2, 0, 0, 0, 0, 4704, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 10, 0, 13, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 3, 0, 7967, 0, 0, 0, 2, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3545, 0, 0, 0, 2645, 0, 0, 0, 0, 0, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6027, 0, 25, 0, 0, 5, 0, 38, 1, 1007, 558, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 5697, 0, 0, 7, 0, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 1, 0, 1, 2, 0, 0, 0, 1, 425, 0, 0, 0, 0, 6998, 883, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8392, 8188, 8213, 8190, 8110, 7762, 8405] avg=8192.0000 max=8405 max_rank=7 min=7762 min_rank=6 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=32467 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 82, 581, 6, 0, 0, 7417, 0, 0, 0, 2, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 578, 0, 0, 1960, 14, 0, 0, 0, 4, 4703, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 2, 2, 0, 1, 1, 0, 6, 1, 0, 1, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 1, 0, 0, 1, 0, 2, 0, 0, 7871, 0, 0, 0, 1, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3597, 0, 0, 0, 3052, 0, 0, 9, 0, 0, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5803, 0, 20, 0, 0, 0, 0, 7, 0, 1162, 558, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 19, 6462, 0, 0, 3, 0, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 2, 0, 0, 0, 0, 0, 1, 0, 422, 0, 0, 0, 0, 6850, 883, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 7264, 8173, 8109, 9131, 7997, 8538, 8217] avg=8192.0000 max=9131 max_rank=4 min=7264 min_rank=1 imbalance_ratio=1.114624 max_load_reduction=56405 max_load_reduction_pct=0.860672 moved_to_echo=33883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 40, 1059, 24, 0, 0, 7399, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 3371, 6, 2, 0, 0, 3, 4690, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 1, 1, 0, 11, 0, 0, 0, 2, 4, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 7942, 0, 0, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3290, 0, 0, 2, 2536, 0, 0, 2, 2, 0, 2012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6417, 0, 1, 0, 0, 0, 2, 20, 0, 971, 561, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 11, 5607, 0, 0, 4, 7, 0, 2050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 1, 0, 0, 1, 0, 397, 0, 0, 0, 0, 6603, 889, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8531, 8569, 8182, 8177, 7844, 8423, 7688, 8122] avg=8192.0000 max=8569 max_rank=1 min=7688 min_rank=6 imbalance_ratio=1.046021 max_load_reduction=56967 max_load_reduction_pct=0.869247 moved_to_echo=32077 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 4224, 128, 128, 23552, 128, 128, 0, 0, 128, 37632, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 26752, 0, 128, 128, 21504, 128, 0, 128, 128, 128, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 256, 128, 0, 0, 128, 0, 0, 128, 128, 48896, 128, 0, 128, 128, 128, 16384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 48896, 128, 256, 0, 128, 128, 128, 256, 128, 8192, 4480, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 640, 0, 128, 0, 128, 128, 128, 0, 128, 128, 3200, 128, 0, 0, 0, 54400, 7168, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 512, 5760, 256, 0, 128, 59264, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 63744, 128, 128, 128, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 256, 128, 0, 128, 128, 128, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8189, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8185, 1, 0, 1, 0, 0, 2, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 6, 2, 8187, 8184, 2, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 8186, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8181, 1, 0, 1, 0, 0, 2, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 8184, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8195, 8192, 8193, 8191, 8193, 8189, 8192] avg=8192.0000 max=8195 max_rank=1 min=8189 min_rank=6 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8187, 8193, 8196, 8192, 8192, 8193] avg=8192.0000 max=8196 max_rank=4 min=8187 min_rank=2 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8192, 8188, 8193, 8191, 8191, 8192, 8193] avg=8192.0000 max=8196 max_rank=0 min=8188 min_rank=2 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 8193, 8193, 8193, 8192, 8192, 8191] avg=8192.0000 max=8193 max_rank=0 min=8189 min_rank=1 imbalance_ratio=1.000122 max_load_reduction=57343 max_load_reduction_pct=0.874985 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8176, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8192, 8196, 8185, 8181, 8192, 8192, 8193] avg=8192.0000 max=8205 max_rank=0 min=8181 min_rank=4 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8196, 8193, 8191, 8192, 8192, 8191] avg=8192.0000 max=8196 max_rank=2 min=8189 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8187, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8195, 8192, 8193, 8192, 8192, 8192] avg=8192.0000 max=8195 max_rank=2 min=8188 min_rank=0 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8180, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8192, 8189, 8194, 8200, 8192, 8195, 8191] avg=8192.0000 max=8200 max_rank=4 min=8183 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 65536, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 128, 128, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[65536, 128, 128, 128, 0, 0, 128, 128, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 65536, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 8179, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8124, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 4, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8182, 0, 0, 8, 0, 0, 0, 0, 0, 8177, 0, 8180, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 6, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 6, 0, 0, 0, 0, 0, 0, 8183, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 18, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 8181, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 8177, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 8179, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 5, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8177, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 6, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 6, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8139, 8234, 8193, 8201, 8210, 8170, 8197] avg=8192.0000 max=8234 max_rank=2 min=8139 min_rank=1 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 18, 0, 0, 0, 8158, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 8174, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8202, 8191, 8196, 8191, 8169, 8203, 8190] avg=8192.0000 max=8203 max_rank=6 min=8169 min_rank=5 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 8158, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8176, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8202, 8191, 8192, 8191, 8188, 8187, 8196] avg=8192.0000 max=8202 max_rank=1 min=8187 min_rank=6 imbalance_ratio=1.001221 max_load_reduction=57334 max_load_reduction_pct=0.874847 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8158, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 8174, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8202, 8176, 8196, 8190, 8194, 8195, 8192] avg=8192.0000 max=8202 max_rank=1 min=8176 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=57334 max_load_reduction_pct=0.874847 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8158, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 8175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8202, 8174, 8194, 8191, 8162, 8229, 8190] avg=8192.0000 max=8229 max_rank=6 min=8162 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 8158, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 9, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8168, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8202, 8188, 8187, 8191, 8208, 8181, 8189] avg=8192.0000 max=8208 max_rank=5 min=8181 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 1, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8157, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8174, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8188, 8183, 8186, 8190, 8216, 8176, 8200] avg=8192.0000 max=8216 max_rank=5 min=8176 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 8160, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 25, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 8176, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8199, 8199, 8192, 8191, 8189, 8195, 8182] avg=8192.0000 max=8199 max_rank=1 min=8182 min_rank=7 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 256, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 65408, 0, 128, 0, 0, 128, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 65536, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 5, 0, 0, 0, 44, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8182, 0, 0, 0, 0, 1, 6245, 0, 0, 0, 0, 0, 0, 0, 8165, 8177, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4742, 0, 0, 0, 0, 0, 19, 0, 0, 0, 3, 5328, 8178, 0, 0, 0, 8, 1, 8182, 8173, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 0, 0, 0, 0, 8192, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 8192, 0, 0, 0, 0, 0, 8192, 8180, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 1, 7877, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7092, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1384, 8191, 0, 0, 0, 1, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2785, 8192, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 5, 0, 8175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7841, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 341, 0, 0, 0, 0, 1, 6245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8177, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7, 0, 8158, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4742, 0, 0, 0, 0, 0, 19, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5328, 0, 0, 0, 0, 8, 1, 0, 6743, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8225, 8196, 6596, 8190, 8183, 5866, 12090] avg=8192.0000 max=12090 max_rank=7 min=5866 min_rank=6 imbalance_ratio=1.475830 max_load_reduction=53446 max_load_reduction_pct=0.815521 moved_to_echo=34329 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7850, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 0, 0, 0, 0, 0, 0, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 0, 0, 0, 6758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8182, 8192, 8524, 8197, 8188, 8663, 7399] avg=8192.0000 max=8663 max_rank=6 min=7399 min_rank=7 imbalance_ratio=1.057495 max_load_reduction=56873 max_load_reduction_pct=0.867813 moved_to_echo=32447 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7850, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7537, 0, 0, 0, 0, 0, 0, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 0, 0, 0, 0, 0, 0, 0, 6758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8183, 8191, 8485, 8199, 8187, 8639, 7461] avg=8192.0000 max=8639 max_rank=6 min=7461 min_rank=7 imbalance_ratio=1.054565 max_load_reduction=56897 max_load_reduction_pct=0.868179 moved_to_echo=32486 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7850, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5508, 0, 0, 0, 0, 0, 0, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2785, 0, 0, 0, 0, 0, 0, 0, 6757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8182, 8191, 8433, 8197, 8189, 6609, 9542] avg=8192.0000 max=9542 max_rank=7 min=6609 min_rank=6 imbalance_ratio=1.164795 max_load_reduction=55994 max_load_reduction_pct=0.854401 moved_to_echo=32537 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7850, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 6749, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8182, 8191, 8316, 8160, 8226, 9291, 6970] avg=8192.0000 max=9291 max_rank=6 min=6970 min_rank=7 imbalance_ratio=1.134155 max_load_reduction=56245 max_load_reduction_pct=0.858231 moved_to_echo=32647 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7850, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8182, 8192, 8529, 8198, 8189, 9294, 6763] avg=8192.0000 max=9294 max_rank=6 min=6763 min_rank=7 imbalance_ratio=1.134521 max_load_reduction=56242 max_load_reduction_pct=0.858185 moved_to_echo=32444 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 1, 0, 0, 7849, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 1, 1, 7877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7092, 0, 0, 0, 0, 0, 2, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 0, 0, 0, 0, 1, 0, 0, 6758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8209, 8192, 8221, 8198, 8186, 8196, 8143] avg=8192.0000 max=8221 max_rank=3 min=8143 min_rank=7 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7856, 334, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7873, 0, 0, 0, 0, 0, 0, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 6753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8191, 8432, 8197, 8188, 8978, 7168] avg=8192.0000 max=8978 max_rank=6 min=7168 min_rank=7 imbalance_ratio=1.095947 max_load_reduction=56558 max_load_reduction_pct=0.863007 moved_to_echo=32531 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 11520, 0, 0, 0, 0, 128, 128, 0, 54144, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 56704, 0, 0, 0, 0, 0, 128, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 62848, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 2816, 0, 0, 0, 128, 128, 62848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 65536, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6793, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1591, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8179, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6774, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 65, 0, 0, 8110, 0, 0, 0, 86, 0, 0, 0, 0, 8179, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 13, 10, 0, 0, 6773, 1, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8125, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 7, 0, 4, 0, 26, 0, 9, 0, 0, 0, 0, 1140, 0, 62, 8, 0, 2, 8161, 0, 5, 0, 0, 0, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 8172, 0, 0, 0, 430, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 1, 0, 0, 5926, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8178, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1831, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7275, 0, 4, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 65, 0, 0, 8051, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 13, 10, 0, 0, 6773, 1, 0, 0, 0, 0, 0, 0, 0, 4006, 59, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8119, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 7, 0, 4, 0, 26, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 0, 62, 8, 0, 2, 4155, 0, 5, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8183, 10936, 8196, 8245, 8198, 8195, 5381] avg=8192.0000 max=10936 max_rank=2 min=5381 min_rank=7 imbalance_ratio=1.334961 max_load_reduction=54600 max_load_reduction_pct=0.833130 moved_to_echo=30019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8132, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 60, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6793, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8195, 5487, 8189, 8185, 8190, 8192, 10963] avg=8192.0000 max=10963 max_rank=7 min=5487 min_rank=2 imbalance_ratio=1.338257 max_load_reduction=54573 max_load_reduction_pct=0.832718 moved_to_echo=35530 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8119, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 60, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6774, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8186, 5508, 8202, 8185, 8195, 8194, 10944] avg=8192.0000 max=10944 max_rank=7 min=5508 min_rank=2 imbalance_ratio=1.335938 max_load_reduction=54592 max_load_reduction_pct=0.833008 moved_to_echo=35518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8132, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5573, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 60, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8195, 9664, 8189, 8185, 8190, 8192, 6788] avg=8192.0000 max=9664 max_rank=2 min=6788 min_rank=7 imbalance_ratio=1.179688 max_load_reduction=55872 max_load_reduction_pct=0.852539 moved_to_echo=31355 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8132, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6597, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 60, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1591, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8192, 10688, 8189, 8185, 8193, 8192, 5761] avg=8192.0000 max=10688 max_rank=2 min=5761 min_rank=7 imbalance_ratio=1.304688 max_load_reduction=54848 max_load_reduction_pct=0.836914 moved_to_echo=30331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 8112, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 1, 0, 0, 5926, 0, 0, 0, 1, 0, 0, 0, 0, 4020, 60, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8172, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1831, 0, 0, 0, 0, 0, 4168, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8546, 8195, 10035, 8193, 8183, 8191, 8192, 6001] avg=8192.0000 max=10035 max_rank=2 min=6001 min_rank=7 imbalance_ratio=1.224976 max_load_reduction=55501 max_load_reduction_pct=0.846878 moved_to_echo=30567 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 63, 9, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7275, 0, 4, 0, 0, 0, 4168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8197, 5017, 8187, 8183, 8189, 8187, 11447] avg=8192.0000 max=11447 max_rank=7 min=5017 min_rank=2 imbalance_ratio=1.397339 max_load_reduction=54089 max_load_reduction_pct=0.825333 moved_to_echo=36006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8132, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4110, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 60, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8193, 8201, 8191, 8185, 8190, 8192, 8251] avg=8192.0000 max=8251 max_rank=7 min=8133 min_rank=0 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32128, 0, 128, 128, 0, 128, 33408, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 32768, 128, 0, 0, 128, 0, 0, 0, 0, 32256, 512, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 65408, 0, 256, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 65536, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 65024, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8172, 2, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 195, 8092, 0, 4, 0, 0, 0, 8, 10, 0, 9, 0, 0, 8, 0, 8157, 0, 0, 0, 0, 0, 0, 1472, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 13, 6514, 0, 0, 0, 6, 0, 8179, 0, 0, 0, 1, 0, 4, 0, 0, 0, 8179, 0, 0, 0, 5, 0, 0, 0, 11, 0, 5, 0, 0, 10, 0, 11, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8176, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 317, 0, 0, 0, 0, 0, 0, 7879, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 285, 0, 0, 0, 0, 0, 0, 7906, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8179, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 7985, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8183, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 38, 8185, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 204, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8180, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8, 10, 0, 9, 0, 0, 8, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1472, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 7792, 141, 15, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 13, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6514, 0, 0, 0, 6, 0, 387, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 5, 0, 0, 0, 11, 0, 5, 0, 0, 10, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8196, 9442, 8064, 6912, 8218, 8250, 8180] avg=8192.0000 max=9442 max_rank=2 min=6912 min_rank=4 imbalance_ratio=1.152588 max_load_reduction=56094 max_load_reduction_pct=0.855927 moved_to_echo=31560 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7803, 143, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 317, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7879, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8192, 7974, 8361, 8267, 8189, 8184, 8193] avg=8192.0000 max=8361 max_rank=3 min=7974 min_rank=2 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32833 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 142, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 285, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7906, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8192, 7974, 8334, 8294, 8189, 8184, 8193] avg=8192.0000 max=8334 max_rank=3 min=7974 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32860 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7802, 141, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 8042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8192, 7975, 8168, 8457, 8189, 8184, 8195] avg=8192.0000 max=8457 max_rank=4 min=7975 min_rank=2 imbalance_ratio=1.032349 max_load_reduction=57079 max_load_reduction_pct=0.870956 moved_to_echo=33025 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 142, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8191, 7974, 8145, 8483, 8189, 8184, 8194] avg=8192.0000 max=8483 max_rank=4 min=7974 min_rank=2 imbalance_ratio=1.035522 max_load_reduction=57053 max_load_reduction_pct=0.870560 moved_to_echo=33050 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 142, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 2, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8192, 8000, 8227, 8375, 8189, 8184, 8193] avg=8192.0000 max=8375 max_rank=4 min=8000 min_rank=2 imbalance_ratio=1.022339 max_load_reduction=57161 max_load_reduction_pct=0.872208 moved_to_echo=32941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 7803, 141, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8196, 8177, 8044, 8343, 8190, 8184, 8195] avg=8192.0000 max=8343 max_rank=4 min=8044 min_rank=3 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32912 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7810, 143, 17, 10, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8185, 8020, 8193, 8405, 8183, 8182, 8193] avg=8192.0000 max=8405 max_rank=4 min=8020 min_rank=2 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=32963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 1792, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 62464, 1152, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[62464, 128, 0, 0, 128, 0, 3200, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 0, 128, 0, 128, 1280, 128, 0, 0, 0, 0, 128, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8174, 11, 0, 7590, 3574, 0, 0, 0, 0, 8172, 8, 0, 120, 0, 8062, 0, 9, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 10, 8037, 0, 0, 300, 0, 6, 0, 0, 0, 12, 3, 13, 11, 0, 638, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 8171, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4255, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8164] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 1, 0, 8136, 603, 0, 0, 0, 0, 8191, 1, 0, 2, 0, 8178, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 254, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 76, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 11, 0, 7177, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 0, 120, 0, 8062, 0, 9, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 10, 7916, 0, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 12, 3, 13, 11, 0, 638, 0, 0, 0, 0, 0, 0, 7493, 413, 121, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 8155, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4255, 0, 0, 0, 0, 0, 11, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10765, 8207, 8233, 8726, 8183, 8309, 8178, 4935] avg=8192.0000 max=10765 max_rank=0 min=4935 min_rank=7 imbalance_ratio=1.314087 max_load_reduction=54771 max_load_reduction_pct=0.835739 moved_to_echo=29605 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 8, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7518, 446, 123, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7746, 8172, 8091, 8103, 8204, 8169, 8195, 8856] avg=8192.0000 max=8856 max_rank=7 min=7746 min_rank=0 imbalance_ratio=1.081055 max_load_reduction=56680 max_load_reduction_pct=0.864868 moved_to_echo=33424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7518, 446, 123, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7746, 8194, 8111, 8103, 8189, 8176, 8194, 8823] avg=8192.0000 max=8823 max_rank=7 min=7746 min_rank=0 imbalance_ratio=1.077026 max_load_reduction=56713 max_load_reduction_pct=0.865372 moved_to_echo=33382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7518, 446, 123, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7746, 8196, 8239, 8103, 8189, 8176, 8194, 8693] avg=8192.0000 max=8693 max_rank=7 min=7746 min_rank=0 imbalance_ratio=1.061157 max_load_reduction=56843 max_load_reduction_pct=0.867355 moved_to_echo=33252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7518, 446, 123, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7746, 8197, 8344, 8103, 8192, 8176, 8194, 8584] avg=8192.0000 max=8584 max_rank=7 min=7746 min_rank=0 imbalance_ratio=1.047852 max_load_reduction=56952 max_load_reduction_pct=0.869019 moved_to_echo=33146 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7518, 446, 123, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7746, 8193, 8108, 8103, 8190, 8174, 8194, 8828] avg=8192.0000 max=8828 max_rank=7 min=7746 min_rank=0 imbalance_ratio=1.077637 max_load_reduction=56708 max_load_reduction_pct=0.865295 moved_to_echo=33386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7693, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 2, 0, 8178, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 76, 0, 0, 1, 0, 0, 0, 7514, 443, 123, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8187, 8322, 8176, 8186, 8181, 8193, 7993] avg=8192.0000 max=8322 max_rank=2 min=7993 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32553 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7523, 449, 130, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7743, 8190, 8088, 8119, 8203, 8175, 8194, 8824] avg=8192.0000 max=8824 max_rank=7 min=7743 min_rank=0 imbalance_ratio=1.077148 max_load_reduction=56712 max_load_reduction_pct=0.865356 moved_to_echo=33396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 60160, 0, 0, 0, 0, 0, 128, 0, 0, 0, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 768, 0, 0, 128, 0, 0, 0, 60160, 3584, 1024, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 61440, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 256, 65408, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 0, 65408, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 128, 64512, 0, 0, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8192, 0, 21, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8189, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 97, 3, 0, 0, 0, 0, 4, 7077, 0, 0, 0, 0, 1, 24, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 19, 0, 0, 1586, 0, 0, 27, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 21, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 8169, 0, 0, 0, 8, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 2, 0, 14, 0, 0, 0, 16, 0, 8170, 12, 0, 8173, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 90, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8316, 8196, 8191, 7973, 8286, 8192, 8191, 8191] avg=8192.0000 max=8316 max_rank=0 min=7973 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32860 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7972, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 89, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8318, 8196, 8192, 7975, 8285, 8192, 8189, 8189] avg=8192.0000 max=8318 max_rank=0 min=7975 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32855 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 89, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8302, 8215, 8192, 7974, 8283, 8192, 8189, 8189] avg=8192.0000 max=8302 max_rank=0 min=7974 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32853 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 90, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8187, 8192, 7973, 8293, 8192, 8190, 8189] avg=8192.0000 max=8320 max_rank=0 min=7973 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32864 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 89, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8212, 8192, 7974, 8264, 8192, 8189, 8189] avg=8192.0000 max=8324 max_rank=0 min=7974 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32834 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 97, 3, 0, 0, 0, 0, 4, 7077, 0, 0, 0, 0, 1, 24, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1586, 0, 0, 27, 14, 0, 0, 0, 0, 0, 0, 0, 0, 7943, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 21, 0, 7518, 0, 0, 0, 0, 0, 88, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 8, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 14, 0, 0, 0, 16, 0, 8167, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7336, 8146, 8193, 9592, 7652, 8198, 8208, 8211] avg=8192.0000 max=9592 max_rank=3 min=7336 min_rank=0 imbalance_ratio=1.170898 max_load_reduction=55944 max_load_reduction_pct=0.853638 moved_to_echo=32269 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 1, 3, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 89, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8181, 8192, 8101, 8194, 8191, 8191, 8190] avg=8192.0000 max=8296 max_rank=0 min=8101 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32766 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 89, 11, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8203, 8192, 7974, 8279, 8187, 8189, 8188] avg=8192.0000 max=8324 max_rank=0 min=7974 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32843 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 1792, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 63744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 64384, 0, 0, 0, 0, 128, 128, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 64768, 0, 0, 0, 0, 0, 768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 7711, 2, 0, 0, 8152, 0, 0, 3, 0, 0, 40, 0, 0, 0, 8170, 0, 0, 0, 0, 50, 0, 17, 73, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 8, 0, 4163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 60, 0, 0, 12, 0, 0, 0, 0, 3065, 0, 21, 0, 1370, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 13, 0, 0, 14, 0, 0, 5, 0, 0, 8163, 14, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 7744, 1, 1, 0, 419, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 0, 0, 0, 0, 0, 7197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7711, 2, 0, 0, 55, 0, 0, 3, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 17, 73, 10, 0, 0, 0, 0, 0, 0, 0, 8096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 0, 8, 0, 4163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 60, 0, 0, 12, 0, 0, 0, 0, 3065, 0, 21, 0, 505, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1370, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 14, 0, 0, 5, 0, 0, 8162, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7811, 8287, 11792, 8185, 8062, 3929, 9262, 8208] avg=8192.0000 max=11792 max_rank=2 min=3929 min_rank=5 imbalance_ratio=1.439453 max_load_reduction=53744 max_load_reduction_pct=0.820068 moved_to_echo=29461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 506, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8177, 7645, 8193, 8211, 8859, 8014, 8190] avg=8192.0000 max=8859 max_rank=5 min=7645 min_rank=2 imbalance_ratio=1.081421 max_load_reduction=56677 max_load_reduction_pct=0.864822 moved_to_echo=33274 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 506, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8177, 7645, 8193, 8211, 8909, 7964, 8190] avg=8192.0000 max=8909 max_rank=5 min=7645 min_rank=2 imbalance_ratio=1.087524 max_load_reduction=56627 max_load_reduction_pct=0.864059 moved_to_echo=33274 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 506, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8177, 7645, 8193, 8211, 8805, 8068, 8190] avg=8192.0000 max=8805 max_rank=5 min=7645 min_rank=2 imbalance_ratio=1.074829 max_load_reduction=56731 max_load_reduction_pct=0.865646 moved_to_echo=33274 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 506, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8177, 7645, 8193, 8211, 8947, 7926, 8190] avg=8192.0000 max=8947 max_rank=5 min=7645 min_rank=2 imbalance_ratio=1.092163 max_load_reduction=56589 max_load_reduction_pct=0.863480 moved_to_echo=33274 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7645, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 0, 0, 0, 0, 0, 7197, 0, 0, 0, 506, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8177, 7865, 8193, 8211, 8730, 7923, 8190] avg=8192.0000 max=8730 max_rank=5 min=7865 min_rank=2 imbalance_ratio=1.065674 max_load_reduction=56806 max_load_reduction_pct=0.866791 moved_to_echo=33054 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 55, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7644, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 7744, 1, 1, 0, 506, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8178, 7665, 8192, 8206, 8516, 8342, 8189] avg=8192.0000 max=8516 max_rank=5 min=7665 min_rank=2 imbalance_ratio=1.039551 max_load_reduction=57020 max_load_reduction_pct=0.870056 moved_to_echo=33253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 512, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8186, 7634, 8194, 8213, 8841, 8037, 8189] avg=8192.0000 max=8841 max_rank=5 min=7634 min_rank=2 imbalance_ratio=1.079224 max_load_reduction=56695 max_load_reduction_pct=0.865097 moved_to_echo=33280 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 896, 0, 0, 128, 0, 0, 0, 0, 58624, 128, 128, 0, 4096, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 61184, 0, 128, 0, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[2304, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 63360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 65152, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65408, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65152, 128, 0, 0, 512, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 276, 8149, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8166, 32, 1, 0, 81, 0, 0, 6, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 10, 0, 0, 8, 0, 17, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 0, 0, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6922, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 11, 8169, 13, 0, 0, 19, 0, 1, 10, 0, 0, 0, 1, 0, 0, 0, 8176, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 81, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 10, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8191, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 276, 8109, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1, 0, 81, 0, 0, 6, 13, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 10, 0, 0, 8, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 0, 0, 0, 7781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6922, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 35, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 11, 8165, 13, 0, 0, 19, 0, 1, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8397, 8277, 8117, 8179, 9042, 7105, 8226, 8193] avg=8192.0000 max=9042 max_rank=4 min=7105 min_rank=5 imbalance_ratio=1.103760 max_load_reduction=56494 max_load_reduction_pct=0.862030 moved_to_echo=32566 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 35, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8173, 8199, 8195, 8072, 8364, 8188, 8193] avg=8192.0000 max=8364 max_rank=5 min=8072 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 35, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8190, 8201, 8195, 8072, 8344, 8188, 8194] avg=8192.0000 max=8344 max_rank=5 min=8072 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 35, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8171, 8199, 8195, 8072, 8365, 8188, 8193] avg=8192.0000 max=8365 max_rank=5 min=8072 min_rank=4 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 120, 35, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8189, 8217, 8193, 8072, 8346, 8188, 8179] avg=8192.0000 max=8346 max_rank=5 min=8072 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 35, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8182, 8198, 8195, 8072, 8366, 8188, 8183] avg=8192.0000 max=8366 max_rank=5 min=8072 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 37, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8171, 8207, 8197, 8069, 8363, 8175, 8208] avg=8192.0000 max=8363 max_rank=5 min=8069 min_rank=4 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 81, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 35, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8187, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8183, 8198, 8187, 8065, 8283, 8195, 8193] avg=8192.0000 max=8283 max_rank=5 min=8065 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 1280, 0, 0, 0, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[64128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 1024, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 65024, 0, 0, 0, 128, 0, 0, 0, 0, 0, 640, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 384, 65280, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 65536, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 9, 18, 0, 8006, 8166, 0, 0, 62, 0, 1840, 0, 0, 4, 0, 0, 0, 0, 0, 0, 27, 1, 0, 0, 7165, 0, 8052, 0, 0, 17, 0, 12, 0, 13, 0, 0, 8165, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 18, 6, 0, 0, 8, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 168, 0, 0, 18, 18, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7300, 0, 0, 11, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 7934, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4, 0, 8181, 8191, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8188, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8124, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8178, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 9, 18, 0, 8006, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 1840, 0, 0, 4, 0, 0, 0, 0, 0, 0, 27, 1, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7165, 0, 155, 0, 0, 17, 0, 12, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 6, 0, 0, 8, 0, 0, 2, 0, 8152, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 168, 0, 0, 18, 18, 0, 0, 0, 0, 8032, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7300, 0, 0, 11, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8106, 9727, 7362, 8198, 8343, 7448, 8177] avg=8192.0000 max=9727 max_rank=2 min=7362 min_rank=3 imbalance_ratio=1.187378 max_load_reduction=55809 max_load_reduction_pct=0.851578 moved_to_echo=32166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8202, 7931, 8349, 8190, 8169, 8309, 8193] avg=8192.0000 max=8349 max_rank=3 min=7931 min_rank=2 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32861 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8185, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8202, 8186, 8091, 8191, 8169, 8309, 8194] avg=8192.0000 max=8309 max_rank=6 min=8091 min_rank=3 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32863 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8202, 7929, 8349, 8190, 8169, 8309, 8194] avg=8192.0000 max=8349 max_rank=3 min=7929 min_rank=2 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32862 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8202, 7943, 8335, 8190, 8169, 8309, 8194] avg=8192.0000 max=8335 max_rank=3 min=7943 min_rank=2 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32862 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8202, 7931, 8347, 8190, 8169, 8309, 8195] avg=8192.0000 max=8347 max_rank=3 min=7931 min_rank=2 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32863 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8192, 7930, 8346, 8197, 8178, 8300, 8194] avg=8192.0000 max=8346 max_rank=3 min=7930 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4, 0, 8181, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 157, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8062, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8124, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8228, 7959, 8357, 8190, 8170, 8243, 8195] avg=8192.0000 max=8357 max_rank=3 min=7959 min_rank=2 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32798 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 1920, 0, 0, 128, 0, 0, 0, 384, 0, 0, 128, 128, 0, 0, 63360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[64256, 0, 1280, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 256, 0, 0, 128, 128, 0, 0, 0, 0, 64512, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 0, 0, 0, 0, 0, 0, 0, 128, 0, 64640, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 9, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8112, 5, 0, 0, 0, 0, 0, 0, 0, 0, 98, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 11, 8049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 8165, 95, 0, 0, 0, 0, 0, 0, 0, 0, 58, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 22, 0, 0, 0, 8174, 0, 0, 8170, 0, 0, 0, 0, 0, 15, 0, 9, 5, 14, 0, 0, 8135, 0, 8173, 0, 2866, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1500, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 7246, 0, 0, 0, 4251, 0, 0, 0, 8, 0, 7, 4, 0, 9, 0, 0, 0, 0, 7, 8, 0, 7712, 0, 8168, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 384, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 15, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 7586, 4, 0, 0, 0, 0, 0, 0, 0, 0, 649, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 8191, 0, 5781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 1, 0, 8170, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 98, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 77, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 7630, 9, 0, 0, 0, 0, 0, 0, 0, 0, 556, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 43, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 44, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 7985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 22, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8170, 0, 0, 0, 0, 0, 15, 0, 9, 5, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, 2866, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 6884, 0, 0, 0, 4251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 7, 4, 0, 9, 0, 0, 0, 0, 7, 8, 0, 7712, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 47, 0, 8149, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7672, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8238, 4818, 11159, 7806, 8189, 8232, 8889] avg=8192.0000 max=11159 max_rank=3 min=4818 min_rank=2 imbalance_ratio=1.362183 max_load_reduction=54377 max_load_reduction_pct=0.829727 moved_to_echo=33116 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 11, 8049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 8165, 95, 0, 0, 0, 0, 0, 0, 0, 0, 58, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7726, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8085, 8782, 7782, 8250, 8192, 8178, 8078] avg=8192.0000 max=8782 max_rank=2 min=7782 min_rank=3 imbalance_ratio=1.072021 max_load_reduction=56754 max_load_reduction_pct=0.865997 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 15, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 7586, 4, 0, 0, 0, 0, 0, 0, 0, 0, 649, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7726, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8170, 8690, 7782, 8250, 8192, 8186, 8078] avg=8192.0000 max=8690 max_rank=2 min=7782 min_rank=3 imbalance_ratio=1.060791 max_load_reduction=56846 max_load_reduction_pct=0.867401 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 77, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 7630, 9, 0, 0, 0, 0, 0, 0, 0, 0, 556, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7726, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8213, 8645, 7782, 8250, 8192, 8185, 8078] avg=8192.0000 max=8645 max_rank=2 min=7782 min_rank=3 imbalance_ratio=1.055298 max_load_reduction=56891 max_load_reduction_pct=0.868088 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 9, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 8112, 5, 0, 0, 0, 0, 0, 0, 0, 0, 98, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7726, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8187, 8670, 7782, 8250, 8192, 8187, 8078] avg=8192.0000 max=8670 max_rank=2 min=7782 min_rank=3 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 43, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7726, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8246, 8623, 7772, 8246, 8192, 8187, 8079] avg=8192.0000 max=8623 max_rank=2 min=7772 min_rank=3 imbalance_ratio=1.052612 max_load_reduction=56913 max_load_reduction_pct=0.868423 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 5781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7687, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 1, 0, 8170, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8166, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8217, 8634, 7707, 8232, 8195, 8189, 8171] avg=8192.0000 max=8634 max_rank=2 min=7707 min_rank=3 imbalance_ratio=1.053955 max_load_reduction=56902 max_load_reduction_pct=0.868256 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 44, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 7985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7727, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8180, 8674, 7770, 8252, 8192, 8192, 8085] avg=8192.0000 max=8674 max_rank=2 min=7770 min_rank=3 imbalance_ratio=1.058838 max_load_reduction=56862 max_load_reduction_pct=0.867645 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3584, 0, 0, 0, 55936, 128, 128, 0, 0, 0, 0, 0, 0, 0, 6016, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 512, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 61824, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 61312, 0, 0, 0, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 256, 65152, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 128, 0, 128, 128, 0, 65152, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65536, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True + [2026-08-26 16:28:31] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 445.9 | throughput per GPU (TFLOP/s/GPU): 405.5 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.031948E+01 | loss scale: 1.0 | grad norm: 6.322 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 5, 394, 0, 0, 0, 165, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 713, 0, 1, 2872, 4, 0, 0, 8191, 1, 8191, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 4467, 0, 0, 0, 6823, 0, 7867, 9, 1, 0, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 7763, 0, 0, 0, 0, 2, 0, 19, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 604, 0, 0, 0, 177, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 972, 0, 0, 3028, 0, 0, 0, 8192, 1, 8192, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 8, 0, 0, 0, 4450, 0, 0, 0, 6455, 0, 7974, 7, 0, 0, 1375, 0, 0, 0, 0, 0, 14, 0, 0, 7908, 0, 2, 0, 0, 1, 0, 22, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1057, 1, 0, 0, 293, 0, 2, 0, 2, 0, 12, 0, 1, 1, 0, 1112, 0, 0, 2605, 0, 0, 0, 8189, 0, 8188, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 1, 0, 0, 0, 0, 3915, 0, 0, 0, 6425, 0, 7563, 9, 0, 0, 2315, 0, 0, 2, 0, 0, 0, 0, 2, 7380, 0, 1, 0, 2, 7, 3, 40, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4, 4, 8016, 0, 0, 5, 1, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 176, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 478, 0, 0, 0, 103, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 974, 0, 0, 2858, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 4439, 0, 0, 0, 6668, 0, 7967, 6, 0, 0, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 7757, 0, 1, 0, 0, 7, 0, 69, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 7935, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 23, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 2, 1, 0, 9, 660, 1, 0, 0, 325, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1155, 0, 0, 2232, 1, 0, 0, 8191, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4, 0, 2, 4, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 3503, 0, 0, 0, 6836, 0, 7773, 34, 1, 0, 2484, 0, 0, 0, 0, 0, 0, 0, 0, 7960, 0, 1, 1, 2, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7914, 0, 0, 4, 0, 0, 0, 1, 0, 2, 1, 1, 3, 2, 0, 0, 0, 34, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 20, 496, 0, 0, 0, 349, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 951, 0, 1, 2824, 0, 0, 0, 8192, 0, 8192, 3, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 3989, 0, 0, 0, 6748, 0, 7508, 3, 0, 0, 2204, 0, 0, 0, 0, 0, 0, 0, 0, 7773, 0, 3, 0, 0, 3, 0, 17, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7836, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 469, 0, 0, 0, 56, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 764, 0, 2, 2032, 1, 0, 0, 8192, 0, 8190, 1, 0, 0, 0, 0, 0, 1, 0, 0, 2, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8182, 0, 1, 0, 0, 0, 0, 0, 0, 5017, 0, 1, 0, 6661, 0, 7828, 0, 4, 0, 2159, 0, 0, 1, 1, 4, 5, 0, 0, 7945, 0, 0, 0, 0, 5, 0, 28, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7946, 0, 0, 4, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 786, 0, 0, 0, 133, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 0, 0, 2425, 3, 0, 0, 8192, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 4615, 0, 0, 0, 6883, 0, 7909, 2, 0, 0, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 7540, 0, 3, 0, 0, 1, 0, 10, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7765, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 20, 496, 0, 0, 0, 349, 0, 0, 0, 0, 3, 2, 4523, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 1, 2824, 0, 0, 0, 0, 0, 4643, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8189, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 1903, 0, 0, 3, 0, 0, 2204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7773, 0, 3, 0, 0, 3, 0, 17, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7836, 0, 0, 2, 2, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 7508, 322, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8422, 8189, 8203, 8099, 8197, 8102, 8053] avg=8192.0000 max=8422 max_rank=1 min=8053 min_rank=7 imbalance_ratio=1.028076 max_load_reduction=57114 max_load_reduction_pct=0.871490 moved_to_echo=32451 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 5, 394, 0, 0, 0, 165, 0, 0, 0, 0, 0, 6, 4573, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 713, 0, 1, 2872, 4, 0, 0, 0, 1, 4642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4467, 0, 0, 0, 1924, 0, 0, 9, 1, 0, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7763, 0, 0, 0, 0, 2, 0, 19, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 1, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7867, 326, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8234, 8187, 8196, 8197, 8185, 8299, 8215] avg=8192.0000 max=8299 max_rank=6 min=8023 min_rank=0 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32896 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1057, 1, 0, 0, 293, 0, 2, 0, 2, 0, 12, 4306, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1112, 0, 0, 2605, 0, 0, 0, 0, 0, 4642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 20, 2, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 0, 1812, 0, 0, 9, 0, 0, 2315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 7380, 0, 1, 0, 2, 7, 3, 40, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4, 4, 8016, 0, 0, 5, 1, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 176, 0, 1, 0, 0, 7563, 307, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8550, 8361, 8208, 8174, 8051, 7837, 8295, 8060] avg=8192.0000 max=8550 max_rank=0 min=7837 min_rank=5 imbalance_ratio=1.043701 max_load_reduction=56986 max_load_reduction_pct=0.869537 moved_to_echo=32243 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 478, 0, 0, 0, 103, 0, 0, 0, 0, 0, 22, 4469, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 974, 0, 0, 2858, 0, 0, 0, 0, 0, 4643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4439, 0, 0, 0, 1881, 0, 0, 6, 0, 0, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7757, 0, 1, 0, 0, 7, 0, 69, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 7935, 0, 0, 2, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 23, 1, 0, 0, 0, 7967, 318, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7950, 8475, 8189, 8190, 7972, 8235, 8203, 8322] avg=8192.0000 max=8475 max_rank=1 min=7950 min_rank=0 imbalance_ratio=1.034546 max_load_reduction=57061 max_load_reduction_pct=0.870682 moved_to_echo=32732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 469, 0, 0, 0, 56, 0, 0, 0, 0, 0, 1, 4465, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 764, 0, 2, 2032, 1, 0, 0, 0, 0, 4643, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 3, 0, 0, 4, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8182, 0, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5017, 0, 1, 0, 1878, 0, 0, 0, 4, 0, 2159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 5, 0, 0, 7945, 0, 0, 0, 0, 5, 0, 28, 1, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7946, 0, 0, 4, 1, 2, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 0, 0, 0, 7828, 318, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7870, 7444, 8194, 8194, 9059, 8390, 8214, 8171] avg=8192.0000 max=9059 max_rank=4 min=7444 min_rank=1 imbalance_ratio=1.105835 max_load_reduction=56477 max_load_reduction_pct=0.861771 moved_to_echo=33834 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 604, 0, 0, 0, 177, 0, 0, 1, 0, 0, 0, 4327, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 972, 0, 0, 3028, 0, 0, 0, 0, 1, 4643, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4450, 0, 0, 0, 1820, 0, 0, 7, 0, 0, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 7908, 0, 2, 0, 0, 1, 0, 22, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 2, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 7974, 308, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7987, 8648, 8188, 8195, 7660, 8348, 8211, 8299] avg=8192.0000 max=8648 max_rank=1 min=7660 min_rank=4 imbalance_ratio=1.055664 max_load_reduction=56888 max_load_reduction_pct=0.868042 moved_to_echo=32518 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1, 0, 9, 660, 1, 0, 0, 325, 0, 0, 0, 0, 0, 3, 4586, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 2232, 1, 0, 0, 0, 0, 4628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4, 0, 2, 4, 0, 2, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3503, 0, 0, 0, 1920, 0, 0, 34, 1, 0, 2484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7960, 0, 1, 1, 2, 0, 0, 22, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7914, 0, 0, 4, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 1, 3, 2, 0, 0, 0, 34, 0, 0, 0, 0, 7773, 330, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8468, 8016, 8197, 8183, 7942, 8389, 8186, 8155] avg=8192.0000 max=8468 max_rank=0 min=7942 min_rank=4 imbalance_ratio=1.033691 max_load_reduction=57068 max_load_reduction_pct=0.870789 moved_to_echo=32672 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 786, 0, 0, 0, 133, 1, 0, 0, 0, 0, 0, 4614, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 0, 0, 2425, 3, 0, 0, 0, 0, 4643, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4615, 0, 0, 0, 1940, 0, 0, 2, 0, 0, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7540, 0, 3, 0, 0, 1, 0, 10, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7765, 0, 0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7909, 329, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8417, 7936, 8184, 8201, 8556, 7955, 8026, 8261] avg=8192.0000 max=8556 max_rank=4 min=7936 min_rank=1 imbalance_ratio=1.044434 max_load_reduction=56980 max_load_reduction_pct=0.869446 moved_to_echo=32798 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 4992, 128, 0, 0, 1664, 128, 128, 128, 128, 128, 128, 35968, 23040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 7552, 0, 128, 20992, 128, 0, 0, 0, 128, 37248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 34432, 0, 128, 0, 15104, 0, 0, 128, 128, 0, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 62080, 0, 128, 128, 128, 128, 128, 256, 128, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 512, 128, 128, 0, 0, 62464, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 128, 128, 63488, 0, 0, 128, 128, 128, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 128, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 8189, 0, 8186, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7321, 8188, 0, 0, 0, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 865, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6127, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8186, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3094, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5870, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7691, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5513, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5935, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 8191, 1, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7029, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8186, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 8186, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7321, 2119, 0, 0, 0, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 1, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 865, 6066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8193, 8189, 9448, 8183, 8192, 6932, 8193] avg=8192.0000 max=9448 max_rank=3 min=6932 min_rank=6 imbalance_ratio=1.153320 max_load_reduction=56088 max_load_reduction_pct=0.855835 moved_to_echo=31500 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7691, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 6069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8194, 8193, 9811, 8181, 8192, 6571, 8193] avg=8192.0000 max=9811 max_rank=3 min=6571 min_rank=6 imbalance_ratio=1.197632 max_load_reduction=55725 max_load_reduction_pct=0.850296 moved_to_echo=31137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6127, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2087, 6069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8194, 8187, 8247, 8182, 8192, 8156, 8190] avg=8192.0000 max=8247 max_rank=3 min=8156 min_rank=6 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8186, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3094, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5101, 6069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 8193, 5214, 8195, 8192, 11170, 8190] avg=8192.0000 max=11170 max_rank=6 min=5214 min_rank=3 imbalance_ratio=1.363525 max_load_reduction=54366 max_load_reduction_pct=0.829559 moved_to_echo=35747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5870, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 6069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8191, 8193, 7990, 8197, 8192, 8392, 8193] avg=8192.0000 max=8392 max_rank=6 min=7990 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=57144 max_load_reduction_pct=0.871948 moved_to_echo=32974 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5935, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2258, 6069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8192, 8055, 8197, 8191, 8327, 8191] avg=8192.0000 max=8327 max_rank=6 min=8055 min_rank=3 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5513, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2681, 6069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8188, 8193, 7633, 8197, 8194, 8750, 8190] avg=8192.0000 max=8750 max_rank=6 min=7633 min_rank=3 imbalance_ratio=1.068115 max_load_reduction=56786 max_load_reduction_pct=0.866486 moved_to_echo=33331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8178, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7029, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8193, 8196, 9138, 8204, 8191, 7238, 8196] avg=8192.0000 max=9138 max_rank=3 min=7238 min_rank=6 imbalance_ratio=1.115479 max_load_reduction=56398 max_load_reduction_pct=0.860565 moved_to_echo=31829 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 17024, 48640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[48640, 17024, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[65536, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 128, 128, 65536, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6457, 0, 0, 1734, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5562, 0, 0, 2630, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5050, 0, 0, 3142, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 59, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 3, 8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 1, 4223, 0, 8144, 0, 8188, 0, 3, 0, 0, 1, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6764, 0, 0, 1428, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 29, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5566, 0, 0, 2596, 0, 8185, 0, 8191, 0, 1, 0, 0, 1, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6164, 0, 0, 2029, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2920, 0, 0, 5271, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6457, 0, 0, 2877, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1734, 0, 5315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8195, 8183, 8192, 8191, 8189, 9345, 7049] avg=8192.0000 max=9345 max_rank=6 min=7049 min_rank=7 imbalance_ratio=1.140747 max_load_reduction=56191 max_load_reduction_pct=0.857407 moved_to_echo=32774 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 1, 2860, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4223, 0, 5284, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 8231, 8194, 8195, 8203, 6820, 9511] avg=8192.0000 max=9511 max_rank=7 min=6820 min_rank=6 imbalance_ratio=1.161011 max_load_reduction=56025 max_load_reduction_pct=0.854874 moved_to_echo=32729 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5050, 0, 0, 2877, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3142, 0, 5315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8183, 8192, 8191, 8191, 7938, 8457] avg=8192.0000 max=8457 max_rank=7 min=7938 min_rank=6 imbalance_ratio=1.032349 max_load_reduction=57079 max_load_reduction_pct=0.870956 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5562, 0, 0, 2877, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2630, 0, 5315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8183, 8192, 8192, 8191, 8451, 7945] avg=8192.0000 max=8451 max_rank=6 min=7945 min_rank=7 imbalance_ratio=1.031616 max_load_reduction=57085 max_load_reduction_pct=0.871048 moved_to_echo=32779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2920, 0, 0, 2877, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5271, 0, 5315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8195, 8183, 8192, 8192, 8189, 5809, 10586] avg=8192.0000 max=10586 max_rank=7 min=5809 min_rank=6 imbalance_ratio=1.292236 max_load_reduction=54950 max_load_reduction_pct=0.838470 moved_to_echo=32776 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6164, 0, 0, 2877, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 0, 5314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8193, 8183, 8192, 8191, 8191, 9052, 7343] avg=8192.0000 max=9052 max_rank=6 min=7343 min_rank=7 imbalance_ratio=1.104980 max_load_reduction=56484 max_load_reduction_pct=0.861877 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6764, 0, 0, 2877, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1428, 0, 5315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8183, 8192, 8191, 8191, 9653, 6743] avg=8192.0000 max=9653 max_rank=6 min=6743 min_rank=7 imbalance_ratio=1.178345 max_load_reduction=55883 max_load_reduction_pct=0.852707 moved_to_echo=32778 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5566, 0, 0, 2881, 10, 2, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 0, 5304, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8188, 8207, 8190, 8193, 8191, 8468, 7902] avg=8192.0000 max=8468 max_rank=6 min=7902 min_rank=7 imbalance_ratio=1.033691 max_load_reduction=57068 max_load_reduction_pct=0.870789 moved_to_echo=32754 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[23168, 0, 42496, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 42496, 0, 128, 23040, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8190, 0, 0, 0, 0, 0, 0, 286, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7789, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 21, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7468, 0, 8079, 0, 4, 0, 0, 0, 0, 3582, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 2, 3518, 8186, 8165, 0, 0, 0, 3, 0, 8186, 8158, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8186, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7895, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7800, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7720, 4, 8018, 0, 1, 0, 0, 0, 0, 1597, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 0, 0, 0, 6010, 8191, 8190, 0, 0, 0, 1, 0, 8190, 8186, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7797, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7486, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7113, 705, 122, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7789, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8186, 8192, 8190, 7772, 8194, 7942, 8868] avg=8192.0000 max=8868 max_rank=7 min=7772 min_rank=4 imbalance_ratio=1.082520 max_load_reduction=56668 max_load_reduction_pct=0.864685 moved_to_echo=32776 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 1, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6825, 0, 0, 0, 4, 0, 0, 0, 0, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4, 0, 8165, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 7083, 643, 121, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3518, 0, 0, 0, 0, 0, 3, 0, 0, 1075, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8676, 8194, 8207, 10411, 8180, 9075, 4602] avg=8192.0000 max=10411 max_rank=4 min=4602 min_rank=7 imbalance_ratio=1.270874 max_load_reduction=55125 max_load_reduction_pct=0.841141 moved_to_echo=32268 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7113, 705, 120, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8074, 8192, 8190, 7669, 8193, 7941, 9085] avg=8192.0000 max=9085 max_rank=7 min=7669 min_rank=4 imbalance_ratio=1.109009 max_load_reduction=56451 max_load_reduction_pct=0.861374 moved_to_echo=32888 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7113, 705, 120, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8125, 8192, 8193, 7533, 8195, 7943, 9163] avg=8192.0000 max=9163 max_rank=7 min=7533 min_rank=4 imbalance_ratio=1.118530 max_load_reduction=56373 max_load_reduction_pct=0.860184 moved_to_echo=32834 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7486, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7113, 705, 120, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7800, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8079, 8192, 8190, 7870, 8194, 7940, 8879] avg=8192.0000 max=8879 max_rank=7 min=7870 min_rank=4 imbalance_ratio=1.083862 max_load_reduction=56657 max_load_reduction_pct=0.864517 moved_to_echo=32883 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7486, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7113, 705, 122, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8076, 8191, 8190, 7777, 8193, 7943, 8974] avg=8192.0000 max=8974 max_rank=7 min=7777 min_rank=4 imbalance_ratio=1.095459 max_load_reduction=56562 max_load_reduction_pct=0.863068 moved_to_echo=32887 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7049, 4, 0, 0, 1, 0, 0, 0, 0, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 7109, 671, 118, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6010, 0, 0, 0, 0, 0, 1, 0, 0, 1077, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8221, 8191, 8186, 8651, 8193, 8812, 7089] avg=8192.0000 max=8812 max_rank=6 min=7089 min_rank=7 imbalance_ratio=1.075684 max_load_reduction=56724 max_load_reduction_pct=0.865540 moved_to_echo=32745 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 0, 0, 0, 0, 0, 0, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7113, 705, 120, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7797, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8099, 8192, 8190, 7853, 8194, 7940, 8876] avg=8192.0000 max=8876 max_rank=7 min=7853 min_rank=4 imbalance_ratio=1.083496 max_load_reduction=56660 max_load_reduction_pct=0.864563 moved_to_echo=32863 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 56960, 0, 0, 0, 0, 0, 128, 0, 0, 8704, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 2176, 56960, 5632, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 58880, 128, 0, 0, 128, 0, 0, 0, 0, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 1152, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 128, 0, 0, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6632, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3751, 0, 0, 0, 0, 0, 0, 6501, 0, 0, 0, 157, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 4, 3, 0, 0, 0, 0, 8094, 0, 0, 0, 7, 0, 0, 0, 28, 0, 8160, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6877, 4685, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2522, 0, 4, 0, 0, 0, 0, 0, 0, 297, 4, 0, 3, 7955, 0, 4, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 239, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2413, 0, 0, 0, 0, 0, 0, 6573, 2, 0, 0, 64, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 1, 1, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5690, 5371, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4512, 0, 1, 0, 0, 0, 0, 0, 0, 21, 1, 0, 0, 8119, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7994, 0, 0, 0, 0, 0, 0, 5, 0, 80, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7722, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7591, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 239, 6994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 1184, 386, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7801, 8192, 8192, 8193, 7242, 8191, 9665, 8060] avg=8192.0000 max=9665 max_rank=6 min=7242 min_rank=4 imbalance_ratio=1.179810 max_load_reduction=55871 max_load_reduction_pct=0.852524 moved_to_echo=33158 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3751, 0, 0, 0, 0, 0, 0, 6194, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 4, 3, 0, 0, 0, 0, 8094, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 8160, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6877, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2522, 0, 4, 0, 0, 0, 0, 678, 307, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 4, 0, 3, 7818, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10102, 8192, 8199, 8189, 10884, 8191, 3652, 8127] avg=8192.0000 max=10884 max_rank=4 min=3652 min_rank=6 imbalance_ratio=1.328613 max_load_reduction=54652 max_load_reduction_pct=0.833923 moved_to_echo=30854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 6979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7994, 0, 0, 0, 0, 0, 0, 1182, 386, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 80, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7788, 8192, 8192, 8192, 7141, 8192, 9703, 8136] avg=8192.0000 max=9703 max_rank=6 min=7141 min_rank=4 imbalance_ratio=1.184448 max_load_reduction=55833 max_load_reduction_pct=0.851944 moved_to_echo=33172 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 0, 0, 0, 0, 0, 7775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 6954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6632, 0, 0, 0, 0, 0, 0, 1178, 385, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7784, 8192, 8192, 8193, 8255, 8191, 8336, 8393] avg=8192.0000 max=8393 max_rank=7 min=7784 min_rank=0 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=33175 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 7005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7722, 0, 0, 0, 0, 0, 0, 1186, 386, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7784, 8192, 8192, 8192, 7329, 8192, 9435, 8220] avg=8192.0000 max=9435 max_rank=6 min=7329 min_rank=4 imbalance_ratio=1.151733 max_load_reduction=56101 max_load_reduction_pct=0.856033 moved_to_echo=33176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 7800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 6995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7591, 0, 0, 0, 0, 0, 0, 1185, 386, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7803, 8192, 8192, 8192, 7136, 8192, 9303, 8526] avg=8192.0000 max=9303 max_rank=6 min=7136 min_rank=4 imbalance_ratio=1.135620 max_load_reduction=56233 max_load_reduction_pct=0.858047 moved_to_echo=33157 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2413, 0, 0, 0, 0, 0, 0, 6259, 2, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 1, 1, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5690, 4586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4512, 0, 1, 0, 0, 0, 0, 782, 314, 147, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 0, 7972, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8738, 8192, 8185, 8193, 10276, 8195, 5762, 7995] avg=8192.0000 max=10276 max_rank=4 min=5762 min_rank=6 imbalance_ratio=1.254395 max_load_reduction=55260 max_load_reduction_pct=0.843201 moved_to_echo=32228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 6966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 1180, 383, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7736, 8192, 8192, 8192, 7273, 8192, 9680, 8079] avg=8192.0000 max=9680 max_rank=6 min=7273 min_rank=4 imbalance_ratio=1.181641 max_load_reduction=55856 max_load_reduction_pct=0.852295 moved_to_echo=33224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 0, 52992, 0, 128, 0, 0, 0, 0, 8576, 2944, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 15104, 50560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 6272, 0, 0, 0, 0, 0, 0, 59264, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 1536, 128, 0, 128, 64128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 256, 128, 128, 0, 0, 0, 0, 65408, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8145, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 80, 0, 7997, 0, 3, 0, 0, 0, 0, 0, 258, 0, 0, 4, 0, 0, 0, 1, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4622, 0, 8106, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1214, 0, 6, 0, 8169, 0, 0, 3, 0, 8187, 0, 0, 0, 4, 8182, 0, 0, 0, 0, 7937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2352, 0, 0, 0, 178, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 8188, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 8177, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3062, 0, 8123, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 1, 0, 8188, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8191, 0, 0, 0, 1, 7991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 7232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8145, 0, 0, 960, 471, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8191, 7278, 9605, 8195, 8247, 7705, 8162] avg=8192.0000 max=9605 max_rank=3 min=7278 min_rank=2 imbalance_ratio=1.172485 max_load_reduction=55931 max_load_reduction_pct=0.853439 moved_to_echo=32309 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 80, 0, 7959, 0, 3, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4622, 0, 7157, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1214, 0, 6, 949, 471, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 3, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 38, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8198, 11787, 2685, 8179, 7999, 10052, 8336] avg=8192.0000 max=11787 max_rank=2 min=2685 min_rank=3 imbalance_ratio=1.438843 max_load_reduction=53749 max_load_reduction_pct=0.820145 moved_to_echo=34566 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 8188, 0, 0, 960, 471, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8191, 7233, 9653, 8192, 8246, 7705, 8163] avg=8192.0000 max=9653 max_rank=3 min=7233 min_rank=2 imbalance_ratio=1.178345 max_load_reduction=55883 max_load_reduction_pct=0.852707 moved_to_echo=32306 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 7232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 960, 471, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8191, 7292, 9593, 8195, 8247, 7705, 8162] avg=8192.0000 max=9593 max_rank=3 min=7292 min_rank=2 imbalance_ratio=1.171021 max_load_reduction=55943 max_load_reduction_pct=0.853622 moved_to_echo=32309 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 8177, 0, 0, 960, 471, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8191, 7238, 9646, 8194, 8247, 7705, 8163] avg=8192.0000 max=9646 max_rank=3 min=7238 min_rank=2 imbalance_ratio=1.177490 max_load_reduction=55890 max_load_reduction_pct=0.852814 moved_to_echo=32309 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 960, 471, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8191, 7236, 9649, 8195, 8247, 7705, 8162] avg=8192.0000 max=9649 max_rank=3 min=7236 min_rank=2 imbalance_ratio=1.177856 max_load_reduction=55887 max_load_reduction_pct=0.852768 moved_to_echo=32309 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3062, 0, 7170, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 1, 953, 478, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 42, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8192, 10239, 5055, 8191, 8056, 9254, 8226] avg=8192.0000 max=10239 max_rank=2 min=5055 min_rank=3 imbalance_ratio=1.249878 max_load_reduction=55297 max_load_reduction_pct=0.843765 moved_to_echo=33727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 960, 471, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8191, 7233, 9650, 8195, 8247, 7705, 8162] avg=8192.0000 max=9650 max_rank=3 min=7233 min_rank=2 imbalance_ratio=1.177979 max_load_reduction=55886 max_load_reduction_pct=0.852753 moved_to_echo=32309 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 0, 53888, 0, 128, 7680, 3840, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 7808, 0, 57728, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 61696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 65024, 0, 128, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 256, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 2, 0, 0, 8187, 0, 0, 0, 0, 8186, 4, 0, 8187, 0, 8185, 0, 2, 0, 4, 0, 7832, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 3, 6502, 0, 1, 0, 0, 0, 5, 5, 5, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 0, 0, 4, 0, 0, 1, 0, 3, 2, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 309] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8183, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 1, 8192, 0, 0, 0, 0, 8192, 0, 0, 8171, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 1, 0, 8191, 0, 8189, 0, 0, 0, 1, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7318, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 879, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 115] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 2, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 51, 0, 0, 0, 2, 0, 4, 0, 7832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 3, 6502, 0, 302, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 5, 5, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 0, 0, 4, 0, 0, 1, 7830, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 309, 8101, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8020, 7893, 6813, 8201, 8188, 9555, 8415, 8451] avg=8192.0000 max=9555 max_rank=5 min=6813 min_rank=2 imbalance_ratio=1.166382 max_load_reduction=55981 max_load_reduction_pct=0.854202 moved_to_echo=34609 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 8192, 0, 302, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7835, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8243, 8519, 8191, 8192, 7870, 8156, 8133] avg=8192.0000 max=8519 max_rank=2 min=7870 min_rank=5 imbalance_ratio=1.039917 max_load_reduction=57017 max_load_reduction_pct=0.870010 moved_to_echo=32351 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 8192, 0, 302, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7835, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8106, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8243, 8533, 8191, 8193, 7870, 8155, 8143] avg=8192.0000 max=8533 max_rank=2 min=7870 min_rank=5 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=32361 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8191, 0, 302, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7835, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8104, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8243, 8503, 8191, 8194, 7870, 8154, 8141] avg=8192.0000 max=8503 max_rank=2 min=7870 min_rank=5 imbalance_ratio=1.037964 max_load_reduction=57033 max_load_reduction_pct=0.870255 moved_to_echo=32359 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8192, 0, 302, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7835, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8243, 8500, 8191, 8192, 7870, 8155, 8143] avg=8192.0000 max=8500 max_rank=2 min=7870 min_rank=5 imbalance_ratio=1.037598 max_load_reduction=57036 max_load_reduction_pct=0.870300 moved_to_echo=32360 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 8192, 0, 302, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7835, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8242, 8542, 8191, 8194, 7870, 8154, 8121] avg=8192.0000 max=8542 max_rank=2 min=7870 min_rank=5 imbalance_ratio=1.042725 max_load_reduction=56994 max_load_reduction_pct=0.869659 moved_to_echo=32339 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 302, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7835, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8105, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8243, 8495, 8191, 8192, 7870, 8156, 8142] avg=8192.0000 max=8495 max_rank=2 min=7870 min_rank=5 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32360 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 40, 0, 0, 0, 0, 0, 1, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7318, 0, 306, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 879, 0, 0, 0, 1, 0, 0, 0, 7839, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 115, 8109, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8186, 7631, 8189, 8191, 8761, 8191, 8262] avg=8192.0000 max=8761 max_rank=5 min=7631 min_rank=2 imbalance_ratio=1.069458 max_load_reduction=56775 max_load_reduction_pct=0.866318 moved_to_echo=33405 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 128, 62976, 0, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2688, 0, 0, 0, 128, 0, 0, 128, 62720, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 512, 64896, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 512, 0, 0, 0, 128, 0, 128, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65152, 128, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7764, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8186, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 49, 132, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 5, 5, 0, 0, 8187, 0, 0, 3912, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 8181, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 6, 0, 4147, 64, 0, 8185, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 27, 38, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 8191, 0, 0, 1843, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 6304, 6, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8038, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7306, 855, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7336, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8203, 8190, 7491, 8200, 8192, 8193, 8906] avg=8192.0000 max=8906 max_rank=7 min=7491 min_rank=3 imbalance_ratio=1.087158 max_load_reduction=56630 max_load_reduction_pct=0.864105 moved_to_echo=33491 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8157, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 49, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 5, 7300, 854, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7333, 0, 0, 3912, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 6, 0, 4147, 64, 0, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8340, 8135, 8194, 11259, 8145, 8191, 8183, 5089] avg=8192.0000 max=11259 max_rank=3 min=5089 min_rank=7 imbalance_ratio=1.374390 max_load_reduction=54277 max_load_reduction_pct=0.828201 moved_to_echo=29608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7306, 855, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7335, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8203, 8190, 7361, 8200, 8193, 8193, 9034] avg=8192.0000 max=9034 max_rank=7 min=7361 min_rank=3 imbalance_ratio=1.102783 max_load_reduction=56502 max_load_reduction_pct=0.862152 moved_to_echo=33620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7306, 855, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7337, 0, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7764, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8203, 8190, 7765, 8200, 8192, 8193, 8630] avg=8192.0000 max=8630 max_rank=7 min=7765 min_rank=3 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=33215 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7306, 855, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7337, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8203, 8190, 7529, 8200, 8192, 8193, 8866] avg=8192.0000 max=8866 max_rank=7 min=7529 min_rank=3 imbalance_ratio=1.082275 max_load_reduction=56670 max_load_reduction_pct=0.864716 moved_to_echo=33451 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7306, 852, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7315, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8038, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8203, 8187, 7495, 8200, 8194, 8193, 8904] avg=8192.0000 max=8904 max_rank=7 min=7495 min_rank=3 imbalance_ratio=1.086914 max_load_reduction=56632 max_load_reduction_pct=0.864136 moved_to_echo=33491 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7306, 855, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7335, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8203, 8190, 7456, 8200, 8192, 8193, 8939] avg=8192.0000 max=8939 max_rank=7 min=7456 min_rank=3 imbalance_ratio=1.091187 max_load_reduction=56597 max_load_reduction_pct=0.863602 moved_to_echo=33524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 27, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7312, 857, 33, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7334, 0, 0, 1843, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 6304, 6, 0, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8183, 8205, 9180, 8191, 8190, 8195, 7168] avg=8192.0000 max=9180 max_rank=3 min=7168 min_rank=7 imbalance_ratio=1.120605 max_load_reduction=56356 max_load_reduction_pct=0.859924 moved_to_echo=31744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 58496, 6912, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 58624, 128, 0, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 58752, 0, 0, 6912, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65408, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 7643, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 249, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 6336, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 543, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 3205, 0, 0, 0, 0, 1, 0, 8185, 0, 0, 26, 0, 0, 0, 8169, 0, 0, 0, 0, 40, 0, 2262, 2719, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 0, 5, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8162, 6, 0, 7943, 0, 1, 5, 0, 0, 0, 0, 0, 0, 7, 0, 8183, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 6, 0, 0, 275, 0, 0, 0, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 7648, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 75, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 7694, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 54, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 7392, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 734, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 6119, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 7, 0, 0, 0, 8177, 0, 0, 0, 0, 7, 0, 636, 1423, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 8184, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 1, 0, 1, 1, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 7556, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 255, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7643, 0, 0, 0, 0, 0, 0, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 318, 0, 0, 0, 0, 0, 0, 0, 0, 6681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9130, 7248, 8192, 8195, 8191, 8222, 8219, 8139] avg=8192.0000 max=9130 max_rank=0 min=7248 min_rank=1 imbalance_ratio=1.114502 max_load_reduction=56406 max_load_reduction_pct=0.860687 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3205, 0, 0, 0, 0, 1, 0, 1489, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2262, 2719, 4, 0, 0, 0, 0, 0, 0, 0, 6690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 0, 5, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8153, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 6, 0, 7943, 0, 1, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 275, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8103, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4721, 11715, 8192, 8182, 8194, 7992, 8145, 8395] avg=8192.0000 max=11715 max_rank=1 min=4721 min_rank=0 imbalance_ratio=1.430054 max_load_reduction=53821 max_load_reduction_pct=0.821243 moved_to_echo=32726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7648, 0, 0, 0, 0, 0, 0, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 476, 0, 0, 0, 0, 0, 0, 0, 0, 6686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9137, 7237, 8192, 8195, 8191, 8222, 8218, 8144] avg=8192.0000 max=9137 max_rank=0 min=7237 min_rank=1 imbalance_ratio=1.115356 max_load_reduction=56399 max_load_reduction_pct=0.860580 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7694, 0, 0, 0, 0, 0, 0, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 445, 0, 0, 0, 0, 0, 0, 0, 0, 6694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9185, 7193, 8192, 8195, 8191, 8222, 8219, 8139] avg=8192.0000 max=9185 max_rank=0 min=7193 min_rank=1 imbalance_ratio=1.121216 max_load_reduction=56351 max_load_reduction_pct=0.859848 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6336, 0, 0, 0, 0, 0, 0, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 6695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7827, 8551, 8192, 8195, 8191, 8222, 8219, 8139] avg=8192.0000 max=8551 max_rank=1 min=7827 min_rank=0 imbalance_ratio=1.043823 max_load_reduction=56985 max_load_reduction_pct=0.869522 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7392, 0, 0, 0, 0, 0, 0, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 66, 0, 0, 0, 0, 0, 0, 0, 0, 6695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8883, 7495, 8192, 8195, 8191, 8222, 8219, 8139] avg=8192.0000 max=8883 max_rank=0 min=7495 min_rank=1 imbalance_ratio=1.084351 max_load_reduction=56653 max_load_reduction_pct=0.864456 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7556, 0, 0, 0, 0, 0, 0, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 384, 0, 0, 0, 0, 0, 0, 0, 0, 6693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9046, 7332, 8192, 8195, 8191, 8222, 8219, 8139] avg=8192.0000 max=9046 max_rank=0 min=7332 min_rank=1 imbalance_ratio=1.104248 max_load_reduction=56490 max_load_reduction_pct=0.861969 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6119, 0, 0, 0, 0, 0, 0, 1481, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 636, 1423, 1, 0, 0, 0, 0, 0, 0, 0, 6698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 0, 8184, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7607, 8765, 8192, 8184, 8196, 8212, 8078, 8302] avg=8192.0000 max=8765 max_rank=1 min=7607 min_rank=0 imbalance_ratio=1.069946 max_load_reduction=56771 max_load_reduction_pct=0.866257 moved_to_echo=32788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 4864, 7168, 128, 0, 0, 0, 0, 0, 0, 0, 53632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 53632, 0, 0, 0, 0, 128, 0, 12032, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 0, 512, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 256, 128, 0, 65280, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7948, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8184, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7829, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7438, 8166, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8178, 21, 4, 0, 8185, 0, 8168, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 8155, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 8183, 0, 0, 0, 7, 0, 3, 2, 0, 0, 0, 4, 0, 0, 0, 920, 0, 0, 0, 7, 3, 0, 0, 0, 0, 7987, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8018, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8020, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 1, 0, 8190, 0, 8184, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8190, 0, 0, 0, 2, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 233, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8157, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8061, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7948, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 116, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8192, 8196, 8702, 7807, 8193, 8189, 8076] avg=8192.0000 max=8702 max_rank=3 min=7807 min_rank=4 imbalance_ratio=1.062256 max_load_reduction=56834 max_load_reduction_pct=0.867218 moved_to_echo=32265 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8184, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 116, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8417, 8192, 8193, 7695, 8582, 8192, 8189, 8076] avg=8192.0000 max=8582 max_rank=4 min=7695 min_rank=3 imbalance_ratio=1.047607 max_load_reduction=56954 max_load_reduction_pct=0.869049 moved_to_echo=33039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7829, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 116, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8062, 8192, 8192, 7485, 9148, 8192, 8189, 8076] avg=8192.0000 max=9148 max_rank=4 min=7485 min_rank=3 imbalance_ratio=1.116699 max_load_reduction=56388 max_load_reduction_pct=0.860413 moved_to_echo=33605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8018, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 116, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8192, 8192, 7257, 9187, 8192, 8189, 8076] avg=8192.0000 max=9187 max_rank=4 min=7257 min_rank=3 imbalance_ratio=1.121460 max_load_reduction=56349 max_load_reduction_pct=0.859818 moved_to_echo=33644 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7438, 232, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4, 0, 0, 0, 8168, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8155, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 518, 113, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 8180, 0, 0, 0, 7, 0, 3, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 0, 0, 0, 7, 3, 0, 0, 0, 0, 7874, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7675, 8199, 8181, 8796, 7480, 8190, 8210, 8805] avg=8192.0000 max=8805 max_rank=7 min=7480 min_rank=4 imbalance_ratio=1.074829 max_load_reduction=56731 max_load_reduction_pct=0.865646 moved_to_echo=32685 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 116, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8419, 8192, 8191, 8772, 7504, 8193, 8189, 8076] avg=8192.0000 max=8772 max_rank=3 min=7504 min_rank=4 imbalance_ratio=1.070801 max_load_reduction=56764 max_load_reduction_pct=0.866150 moved_to_echo=31962 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8061, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 116, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 8192, 8191, 8158, 8243, 8193, 8189, 8076] avg=8192.0000 max=8294 max_rank=0 min=8076 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32701 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8020, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 8179, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8190, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 117, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8184, 0, 0, 0, 2, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8185, 8200, 8671, 7585, 8191, 8192, 8275] avg=8192.0000 max=8671 max_rank=3 min=7585 min_rank=4 imbalance_ratio=1.058472 max_load_reduction=56865 max_load_reduction_pct=0.867691 moved_to_echo=32243 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 0, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 60416, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 4224, 1024, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 63744, 1920, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 1280, 0, 0, 0, 128, 128, 0, 0, 0, 0, 64384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 65536, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 4465, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8183, 0, 0, 0, 8178, 0, 0, 37, 0, 8179, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 0, 7298, 0, 8150, 0, 0, 0, 2, 5, 0, 11, 0, 5, 0, 0, 8170, 0, 0, 2614, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 7747, 0, 0, 2210, 0, 0, 6, 4, 0, 0, 0, 0, 210, 0, 0, 0, 0, 7, 0, 0, 4, 0, 0, 0, 0, 5, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 5842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 8183, 0, 0, 4, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 8188, 0, 0, 0, 1, 1, 0, 8, 0, 0, 0, 0, 8190, 0, 0, 2075, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 0, 0, 519, 0, 0, 1, 1, 0, 0, 0, 0, 37, 0, 0, 0, 1, 5, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8951, 8194, 8331, 7608, 8191, 7904, 8163, 8194] avg=8192.0000 max=8951 max_rank=0 min=7608 min_rank=3 imbalance_ratio=1.092651 max_load_reduction=56585 max_load_reduction_pct=0.863419 moved_to_echo=32452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8953, 8192, 8328, 7608, 8191, 7904, 8163, 8197] avg=8192.0000 max=8953 max_rank=0 min=7608 min_rank=3 imbalance_ratio=1.092896 max_load_reduction=56583 max_load_reduction_pct=0.863388 moved_to_echo=32455 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 4465, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 471, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 135, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 0, 7298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7569, 0, 0, 0, 2, 5, 0, 11, 0, 5, 0, 0, 0, 0, 0, 2614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4, 0, 0, 0, 0, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7474, 0, 0, 2210, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 0, 7, 0, 0, 4, 0, 0, 0, 0, 5, 0, 5, 8040, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5219, 8187, 7478, 10206, 8193, 9698, 8381, 8174] avg=8192.0000 max=10206 max_rank=3 min=5219 min_rank=0 imbalance_ratio=1.245850 max_load_reduction=55330 max_load_reduction_pct=0.844269 moved_to_echo=34446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8952, 8192, 8330, 7608, 8192, 7903, 8163, 8196] avg=8192.0000 max=8952 max_rank=0 min=7608 min_rank=3 imbalance_ratio=1.092773 max_load_reduction=56584 max_load_reduction_pct=0.863403 moved_to_echo=32454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8949, 8195, 8330, 7609, 8191, 7905, 8163, 8194] avg=8192.0000 max=8949 max_rank=0 min=7609 min_rank=3 imbalance_ratio=1.092407 max_load_reduction=56587 max_load_reduction_pct=0.863449 moved_to_echo=32453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8950, 8195, 8330, 7608, 8191, 7904, 8164, 8194] avg=8192.0000 max=8950 max_rank=0 min=7608 min_rank=3 imbalance_ratio=1.092529 max_load_reduction=56586 max_load_reduction_pct=0.863434 moved_to_echo=32453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 5842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 477, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7596, 0, 0, 0, 1, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 2075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7892, 0, 0, 519, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 1, 5, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8015, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6615, 8186, 8075, 9681, 8196, 8414, 8176, 8193] avg=8192.0000 max=9681 max_rank=3 min=6615 min_rank=0 imbalance_ratio=1.181763 max_load_reduction=55855 max_load_reduction_pct=0.852280 moved_to_echo=32979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8947, 8195, 8334, 7608, 8191, 7904, 8163, 8194] avg=8192.0000 max=8947 max_rank=0 min=7608 min_rank=3 imbalance_ratio=1.092163 max_load_reduction=56589 max_load_reduction_pct=0.863480 moved_to_echo=32452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 59520, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 3840, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[60928, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 62848, 0, 0, 2816, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64384, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 1152, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 64512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7776, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6724, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7695, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8156, 1, 0, 0, 0, 0, 6, 14, 28, 0, 0, 0, 8184, 0, 8083, 3, 0, 0, 0, 0, 0, 2, 0, 4, 2, 6, 0, 0, 8170, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 6, 0, 0, 14, 0, 0, 0, 0, 0, 0, 2, 516, 0, 0, 0, 0, 0, 5, 5, 0, 2, 0, 0, 7613, 0, 5, 0, 6, 0, 0, 8107, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7655, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7761, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8189, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8185, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 114, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 8075, 0, 1, 0, 0, 0, 0, 8091, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1468, 0, 7631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6724, 0, 0, 0, 0, 0, 538, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8215, 8155, 9099, 7276, 8214, 8193, 8193] avg=8192.0000 max=9099 max_rank=3 min=7276 min_rank=4 imbalance_ratio=1.110718 max_load_reduction=56437 max_load_reduction_pct=0.861160 moved_to_echo=31876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 0, 7631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7776, 0, 0, 0, 0, 0, 538, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8215, 8155, 8047, 8328, 8214, 8193, 8193] avg=8192.0000 max=8328 max_rank=4 min=8047 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8155, 1, 0, 0, 0, 0, 6, 14, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8083, 3, 0, 0, 0, 0, 0, 2, 0, 4, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 14, 0, 0, 0, 0, 0, 0, 2, 516, 0, 7624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 2, 0, 0, 7613, 0, 5, 0, 6, 0, 538, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8122, 8361, 8162, 8188, 8135, 8183, 8181] avg=8192.0000 max=8361 max_rank=2 min=8122 min_rank=1 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 0, 7631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7761, 0, 0, 0, 0, 0, 538, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8215, 8155, 8062, 8313, 8214, 8193, 8193] avg=8192.0000 max=8313 max_rank=4 min=8062 min_rank=3 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 0, 7631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7655, 0, 0, 0, 0, 0, 538, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8215, 8155, 8168, 8207, 8214, 8193, 8193] avg=8192.0000 max=8215 max_rank=1 min=8155 min_rank=2 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 7631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 0, 0, 0, 0, 0, 538, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8215, 8155, 8042, 8333, 8214, 8193, 8193] avg=8192.0000 max=8333 max_rank=4 min=8042 min_rank=3 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 7631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7695, 0, 0, 0, 0, 0, 538, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8215, 8155, 8128, 8247, 8214, 8193, 8193] avg=8192.0000 max=8247 max_rank=4 min=8128 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 114, 0, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 8075, 0, 1, 0, 0, 0, 545, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8186, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8124, 8245, 7828, 8644, 8117, 8195, 8197] avg=8192.0000 max=8644 max_rank=4 min=7828 min_rank=3 imbalance_ratio=1.055176 max_load_reduction=56892 max_load_reduction_pct=0.868103 moved_to_echo=33153 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 4480, 0, 61056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 61184, 0, 128, 0, 128, 0, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 0, 65408, 128, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[65280, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65408, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True + [2026-08-26 16:28:32] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 454.9 | throughput per GPU (TFLOP/s/GPU): 397.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.890286E+00 | loss scale: 1.0 | grad norm: 5.630 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 4, 0, 2, 1, 749, 0, 0, 0, 23, 0, 0, 0, 0, 1, 0, 1, 3, 0, 0, 2086, 0, 0, 3193, 7, 0, 0, 8191, 0, 8189, 1, 4, 0, 3, 0, 0, 1, 0, 0, 1, 2, 0, 5, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 3797, 0, 0, 0, 5443, 0, 6942, 5, 0, 0, 2712, 0, 0, 1, 0, 0, 3, 0, 0, 8130, 0, 3, 0, 1, 0, 0, 125, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7646, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 63, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1577, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2112, 0, 0, 2518, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 0, 0, 0, 5433, 0, 6706, 0, 0, 0, 3103, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1029, 1, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2134, 0, 0, 2617, 0, 0, 0, 8192, 2, 8192, 2, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 0, 0, 0, 0, 0, 3381, 0, 0, 0, 5899, 0, 7136, 26, 1, 0, 2659, 0, 0, 0, 0, 0, 7, 0, 0, 7998, 0, 7, 0, 0, 3, 0, 289, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7666, 0, 0, 2, 1, 0, 0, 1, 0, 3, 0, 6, 0, 0, 0, 0, 0, 76, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 759, 0, 1, 0, 1, 0, 0, 0, 1, 2, 2, 0, 0, 0, 0, 2215, 0, 0, 2231, 0, 0, 0, 8191, 0, 8189, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 5, 0, 0, 0, 3730, 0, 0, 0, 5940, 0, 7004, 24, 0, 0, 3216, 0, 0, 0, 1, 4, 2, 0, 0, 7838, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7749, 0, 0, 16, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 42, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 669, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1901, 0, 0, 2941, 3, 0, 0, 8191, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 0, 0, 5423, 0, 6997, 4, 0, 0, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 7939, 0, 0, 0, 0, 0, 0, 50, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7807, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1475, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1796, 0, 0, 3011, 0, 0, 0, 8192, 3, 8192, 1, 0, 0, 0, 8, 0, 0, 1, 0, 0, 5, 0, 0, 2, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 0, 0, 0, 0, 3843, 0, 0, 0, 5384, 0, 6909, 4, 0, 1, 2726, 0, 0, 0, 0, 0, 1, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7601, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 3, 0, 0, 0, 0, 24, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 1121, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 2010, 0, 0, 2957, 0, 0, 0, 8191, 1, 8191, 1, 0, 1, 0, 6, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 3732, 0, 0, 0, 5155, 0, 7113, 1, 0, 1, 2987, 0, 0, 0, 0, 1, 0, 0, 0, 8002, 0, 1, 0, 0, 0, 2, 119, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7686, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 62, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1005, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2350, 1, 0, 2260, 0, 1, 1, 8190, 0, 8190, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 1, 0, 2, 3922, 0, 0, 2, 5739, 0, 6856, 2, 0, 0, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 2, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7657, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 2, 1, 749, 0, 0, 0, 23, 0, 0, 0, 0, 1, 0, 3638, 3426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 2086, 0, 0, 3193, 7, 0, 0, 0, 0, 3394, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 2, 0, 5, 0, 0, 1, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3797, 0, 0, 0, 1454, 0, 0, 5, 0, 0, 2712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 8130, 0, 3, 0, 1, 0, 0, 125, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7646, 0, 0, 1, 0, 2, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 63, 0, 0, 0, 0, 6942, 1157, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7844, 8689, 8196, 8195, 7969, 8321, 8147, 8175] avg=8192.0000 max=8689 max_rank=1 min=7844 min_rank=0 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=32612 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1029, 1, 0, 0, 4, 0, 0, 0, 0, 0, 1, 3639, 3713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2134, 0, 0, 2617, 0, 0, 0, 0, 2, 3396, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 2, 0, 0, 7, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3381, 0, 0, 0, 1576, 0, 0, 26, 1, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7998, 0, 7, 0, 0, 3, 0, 289, 2, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7666, 0, 0, 2, 1, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 6, 0, 0, 0, 0, 0, 76, 0, 1, 1, 0, 7136, 1157, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8388, 8151, 8197, 8190, 7643, 8369, 8208, 8390] avg=8192.0000 max=8390 max_rank=7 min=7643 min_rank=4 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=32610 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 759, 0, 1, 0, 1, 0, 0, 0, 1, 2, 2, 3638, 3738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2215, 0, 0, 2231, 0, 0, 0, 0, 0, 3394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 3, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3730, 0, 0, 0, 1588, 0, 0, 24, 0, 0, 3216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 2, 0, 0, 7838, 0, 0, 0, 0, 0, 0, 177, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7749, 0, 0, 16, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 42, 0, 0, 0, 0, 7004, 1157, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 7840, 8193, 8188, 8563, 8086, 8307, 8216] avg=8192.0000 max=8563 max_rank=4 min=7840 min_rank=1 imbalance_ratio=1.045288 max_load_reduction=56973 max_load_reduction_pct=0.869339 moved_to_echo=33172 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1577, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 3639, 3419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 0, 0, 2518, 0, 0, 0, 0, 0, 3396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 0, 0, 0, 1452, 0, 0, 0, 0, 0, 3103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 99, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 6706, 1157, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8642, 8026, 8184, 8200, 8233, 8206, 8123, 7922] avg=8192.0000 max=8642 max_rank=0 min=7922 min_rank=7 imbalance_ratio=1.054932 max_load_reduction=56894 max_load_reduction_pct=0.868134 moved_to_echo=32484 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 669, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 3638, 3413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1901, 0, 0, 2941, 3, 0, 0, 0, 0, 3396, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 2, 1, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 0, 0, 1449, 0, 0, 4, 0, 0, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7939, 0, 0, 0, 0, 0, 0, 50, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7807, 0, 0, 0, 0, 0, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 6997, 1157, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7731, 8243, 8189, 8196, 8563, 8048, 8304, 8262] avg=8192.0000 max=8563 max_rank=4 min=7731 min_rank=0 imbalance_ratio=1.045288 max_load_reduction=56973 max_load_reduction_pct=0.869339 moved_to_echo=33177 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 1121, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 3638, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2010, 0, 0, 2957, 0, 0, 0, 0, 1, 3396, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8171, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3732, 0, 0, 0, 1378, 0, 0, 1, 0, 1, 2987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8002, 0, 1, 0, 0, 0, 2, 119, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7686, 0, 0, 1, 0, 6, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 62, 0, 0, 0, 0, 7113, 1157, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8008, 8369, 8192, 8179, 8099, 8181, 8165, 8343] avg=8192.0000 max=8369 max_rank=1 min=8008 min_rank=0 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1475, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8, 3643, 3393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1796, 0, 0, 3011, 0, 0, 0, 0, 3, 3388, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 5, 0, 0, 2, 0, 0, 2, 1, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 0, 1421, 0, 0, 4, 0, 1, 2726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 159, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7601, 0, 0, 1, 0, 0, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 3, 0, 0, 0, 0, 24, 0, 0, 0, 0, 6909, 1161, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8524, 8199, 8203, 8188, 7995, 8223, 8097, 8107] avg=8192.0000 max=8524 max_rank=0 min=7995 min_rank=4 imbalance_ratio=1.040527 max_load_reduction=57012 max_load_reduction_pct=0.869934 moved_to_echo=32422 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1005, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3638, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2350, 1, 0, 2260, 0, 1, 1, 0, 0, 3395, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 3922, 0, 0, 2, 1533, 0, 0, 2, 0, 0, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 2, 0, 40, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7657, 0, 0, 4, 0, 0, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 1, 6856, 1157, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8019, 8182, 8200, 8471, 8102, 8185, 8121] avg=8192.0000 max=8471 max_rank=4 min=8019 min_rank=1 imbalance_ratio=1.034058 max_load_reduction=57065 max_load_reduction_pct=0.870743 moved_to_echo=32879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 8448, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 29184, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 16640, 128, 0, 21760, 128, 128, 128, 0, 128, 27264, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 29696, 0, 0, 128, 11904, 0, 0, 128, 128, 128, 24064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 128, 0, 128, 512, 0, 128, 128, 128, 55680, 9344, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 61440, 0, 0, 128, 128, 128, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 64000, 0, 128, 0, 128, 128, 128, 1152, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 187, 0, 0, 0, 8008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8188, 0, 170, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 8192, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8163, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 27, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 159, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 18, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 1, 0, 0, 0, 8177, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 5, 1, 0, 0, 8179, 0, 4, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 8191, 1, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 10, 1, 0, 1, 8181, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 8, 2, 0, 2, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 8177, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 10, 0, 0, 0, 0, 8182, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8193, 8187, 8196, 8199, 8191, 8191, 8198] avg=8192.0000 max=8199 max_rank=4 min=8181 min_rank=0 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32779 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 170, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8192, 8193, 8169, 8168, 8195, 8209, 8211] avg=8192.0000 max=8211 max_rank=7 min=8168 min_rank=4 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32783 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8198, 8192, 8196, 8182, 8194, 8190, 8189] avg=8192.0000 max=8198 max_rank=1 min=8182 min_rank=4 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 1, 0, 0, 8179, 0, 4, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8187, 0, 1, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8185, 8198, 8198, 8191, 8193, 8186] avg=8192.0000 max=8198 max_rank=3 min=8185 min_rank=2 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 187, 0, 0, 0, 8008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8192, 8198, 8194, 8191, 8187, 8187] avg=8192.0000 max=8198 max_rank=3 min=8187 min_rank=6 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 1, 0, 1, 8181, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 6, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8191, 8192, 8199, 8196, 8190, 8188, 8186] avg=8192.0000 max=8199 max_rank=3 min=8186 min_rank=7 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 2, 0, 2, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8181, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8190, 8191, 8197, 8198, 8189, 8189, 8187] avg=8192.0000 max=8198 max_rank=4 min=8187 min_rank=7 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 159, 0, 0, 0, 8026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8187, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8187, 8204, 8183, 8201, 8195, 8189, 8192] avg=8192.0000 max=8204 max_rank=2 min=8183 min_rank=3 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32777 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 65536, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 128, 0, 0, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 640, 128, 0, 128, 65024, 0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 98, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 8093, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8190, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 164, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8027, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8190, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8163, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 184, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 1, 0, 8187, 2, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 8006, 0, 8190, 0, 2, 0, 0, 1, 0, 0, 0, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8133, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8138, 8193, 8190, 8192, 8191, 8249] avg=8192.0000 max=8249 max_rank=7 min=8138 min_rank=2 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32822 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8027, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8298, 8192, 8193, 8191, 8190, 8086] avg=8192.0000 max=8298 max_rank=2 min=8086 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32660 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8105, 54, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8197, 8138, 8194, 8201, 8174, 8196, 8244] avg=8192.0000 max=8244 max_rank=7 min=8138 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32815 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8234, 8192, 8190, 8191, 8191, 8152] avg=8192.0000 max=8234 max_rank=2 min=8152 min_rank=7 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32724 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 54, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8138, 8192, 8189, 8193, 8192, 8250] avg=8192.0000 max=8250 max_rank=7 min=8138 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32824 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 54, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8138, 8192, 8190, 8190, 8197, 8244] avg=8192.0000 max=8244 max_rank=7 min=8138 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32821 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8133, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8119, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8006, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8321, 8190, 8192, 8195, 8189, 8067] avg=8192.0000 max=8321 max_rank=2 min=8067 min_rank=7 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32643 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 61, 3, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8187, 8131, 8191, 8191, 8210, 8190, 8244] avg=8192.0000 max=8244 max_rank=7 min=8131 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32835 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 0, 65152, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65152, 512, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8187, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 4, 8191, 8189, 0, 0, 0, 0, 0, 8192, 8191, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8156, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 1, 8192, 8188, 0, 0, 0, 0, 0, 8192, 8186, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3256, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 20, 8191, 8176, 0, 0, 0, 1, 0, 8189, 8189, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4140, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8, 8191, 8178, 0, 0, 0, 1, 0, 8190, 8174, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2270, 0, 2, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 16, 8190, 8164, 0, 0, 0, 2, 0, 8178, 8189, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 3, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 9, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8187, 0, 0, 0, 0, 1, 0, 1869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8177, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6302, 8194, 8191, 10067, 8196, 8200, 8194] avg=8192.0000 max=10067 max_rank=4 min=6302 min_rank=1 imbalance_ratio=1.228882 max_load_reduction=55469 max_load_reduction_pct=0.846390 moved_to_echo=34657 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 1869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6308, 8192, 8192, 10057, 8193, 8211, 8192] avg=8192.0000 max=10057 max_rank=4 min=6308 min_rank=1 imbalance_ratio=1.227661 max_load_reduction=55479 max_load_reduction_pct=0.846542 moved_to_echo=34653 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6301, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3256, 0, 1, 0, 0, 0, 0, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 8176, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 11239, 8190, 8192, 5125, 8190, 8211, 8198] avg=8192.0000 max=11239 max_rank=1 min=5125 min_rank=4 imbalance_ratio=1.371948 max_load_reduction=54297 max_load_reduction_pct=0.828506 moved_to_echo=29724 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 6295, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8156, 0, 0, 0, 0, 0, 0, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8125, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 6384, 8199, 8193, 10023, 8203, 8147, 8189] avg=8192.0000 max=10023 max_rank=4 min=6384 min_rank=1 imbalance_ratio=1.223511 max_load_reduction=55513 max_load_reduction_pct=0.847061 moved_to_echo=34562 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 6302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8144, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6346, 8195, 8192, 10058, 8193, 8165, 8195] avg=8192.0000 max=10058 max_rank=4 min=6346 min_rank=1 imbalance_ratio=1.227783 max_load_reduction=55478 max_load_reduction_pct=0.846527 moved_to_echo=34611 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4140, 0, 1, 0, 0, 0, 0, 1869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8174, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8178, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 10372, 8192, 8195, 6010, 8178, 8211, 8188] avg=8192.0000 max=10372 max_rank=1 min=6010 min_rank=4 imbalance_ratio=1.266113 max_load_reduction=55164 max_load_reduction_pct=0.841736 moved_to_echo=30587 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2270, 0, 2, 0, 0, 0, 0, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8164, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 12245, 8182, 8192, 4140, 8192, 8211, 8184] avg=8192.0000 max=12245 max_rank=1 min=4140 min_rank=4 imbalance_ratio=1.494751 max_load_reduction=53291 max_load_reduction_pct=0.813156 moved_to_echo=28727 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 6308, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 20, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6340, 8192, 8189, 10056, 8191, 8180, 8196] avg=8192.0000 max=10056 max_rank=4 min=6340 min_rank=1 imbalance_ratio=1.227539 max_load_reduction=55480 max_load_reduction_pct=0.846558 moved_to_echo=34623 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 15104, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 50432, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 50688, 0, 128, 0, 0, 128, 0, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65408, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 1, 1, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 405, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 51, 0, 1, 0, 0, 0, 0, 0, 0, 8047, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1965, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1743, 6438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3676, 4512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8164, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4327, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 1, 1, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 443, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 40, 0, 1, 0, 0, 0, 0, 0, 0, 8015, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 2, 2, 0, 0, 0, 0, 7764, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8184, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 422, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 107, 0, 2, 0, 0, 0, 0, 0, 0, 8076, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2468, 5723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4263, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8214, 8210, 8128, 8190, 8168, 8239] avg=8192.0000 max=8239 max_rank=7 min=8128 min_rank=4 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1936, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8191, 8201, 8210, 8163, 8191, 8169, 8213] avg=8192.0000 max=8213 max_rank=7 min=8163 min_rank=4 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 1, 1, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 18, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7815, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 51, 0, 1, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 77, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8191, 8240, 8150, 8220, 8192, 8219, 8127] avg=8192.0000 max=8240 max_rank=2 min=8127 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 1, 1, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7786, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 40, 0, 1, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 77, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8192, 8227, 8200, 8229, 8191, 8205, 8095] avg=8192.0000 max=8229 max_rank=4 min=8095 min_rank=7 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 8192, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3622, 4512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8164, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8192, 8203, 8214, 8160, 8186, 8162, 8253] avg=8192.0000 max=8253 max_rank=7 min=8160 min_rank=4 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1718, 6438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8201, 8210, 8157, 8187, 8168, 8229] avg=8192.0000 max=8229 max_rank=7 min=8157 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 2, 2, 0, 0, 0, 0, 7764, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 18, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8182, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 107, 0, 2, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8194, 8041, 8143, 8332, 8200, 8272, 8157] avg=8192.0000 max=8332 max_rank=4 min=8041 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32961 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2424, 5723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8209, 8199, 8147, 8199, 8173, 8223] avg=8192.0000 max=8223 max_rank=7 min=8147 min_rank=4 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 256, 65408, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 256, 0, 128, 0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 512, 128, 128, 0, 0, 0, 0, 64896, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 65152, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 37504, 28160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 1085, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 6793, 0, 0, 8, 0, 0, 0, 8192, 0, 0, 0, 1, 8184, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2397, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5482, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 8157, 0, 4141, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 3864, 0, 0, 55, 0, 0, 0, 8192, 0, 0, 0, 7, 8186, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 1574, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 6500, 0, 0, 26, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7539, 0, 2683, 0, 0, 2552, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 2897, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8112, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 656, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7070, 0, 3378, 0, 0, 3036, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1544, 0, 0, 0, 0, 1, 0, 8171, 0, 0, 2, 0, 8190, 0, 0, 0, 2, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1342, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7327, 0, 3485, 0, 0, 2009, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 2482, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 995, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 3286, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4752, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4498, 0, 1085, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 6793, 0, 0, 8, 0, 0, 3687, 327, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8185, 5929, 10836, 8229, 8192, 7820, 8200] avg=8192.0000 max=10836 max_rank=3 min=5929 min_rank=2 imbalance_ratio=1.322754 max_load_reduction=54700 max_load_reduction_pct=0.834656 moved_to_echo=32441 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4484, 0, 1574, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 6500, 0, 0, 26, 0, 0, 3676, 327, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8191, 6246, 10549, 8229, 8194, 7814, 8169] avg=8192.0000 max=10549 max_rank=3 min=6246 min_rank=2 imbalance_ratio=1.287720 max_load_reduction=54987 max_load_reduction_pct=0.839035 moved_to_echo=32406 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4502, 0, 2397, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5482, 0, 0, 0, 0, 0, 3690, 327, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8193, 7231, 9510, 8229, 8192, 7821, 8216] avg=8192.0000 max=9510 max_rank=3 min=7231 min_rank=2 imbalance_ratio=1.160889 max_load_reduction=56026 max_load_reduction_pct=0.854889 moved_to_echo=32458 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8186, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4483, 0, 4141, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 3864, 0, 0, 55, 0, 0, 3674, 327, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8189, 8847, 7965, 8232, 8191, 7813, 8152] avg=8192.0000 max=8847 max_rank=2 min=7813 min_rank=6 imbalance_ratio=1.079956 max_load_reduction=56689 max_load_reduction_pct=0.865005 moved_to_echo=32388 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 3485, 0, 0, 2009, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 2482, 0, 0, 0, 0, 0, 3300, 327, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8195, 9522, 6165, 8153, 8191, 8815, 8237] avg=8192.0000 max=9522 max_rank=2 min=6165 min_rank=3 imbalance_ratio=1.162354 max_load_reduction=56014 max_load_reduction_pct=0.854706 moved_to_echo=33396 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4143, 0, 2683, 0, 0, 2552, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 2897, 0, 0, 0, 0, 0, 3396, 327, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8112, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8193, 9379, 6641, 8152, 8193, 8476, 8237] avg=8192.0000 max=9379 max_rank=2 min=6641 min_rank=3 imbalance_ratio=1.144897 max_load_reduction=56157 max_load_reduction_pct=0.856888 moved_to_echo=33058 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 3378, 0, 0, 3036, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1544, 0, 0, 0, 0, 1, 3185, 327, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 2, 0, 0, 0, 0, 0, 2, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8195, 10301, 5070, 8092, 8192, 9161, 8238] avg=8192.0000 max=10301 max_rank=2 min=5070 min_rank=3 imbalance_ratio=1.257446 max_load_reduction=55235 max_load_reduction_pct=0.842819 moved_to_echo=33683 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4487, 0, 3286, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4752, 0, 0, 7, 0, 0, 3684, 330, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8195, 8081, 8800, 8220, 8191, 7816, 8087] avg=8192.0000 max=8800 max_rank=3 min=7816 min_rank=6 imbalance_ratio=1.074219 max_load_reduction=56736 max_load_reduction_pct=0.865723 moved_to_echo=32314 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 34432, 0, 0, 128, 0, 128, 28416, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 34560, 0, 22144, 0, 0, 9088, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65280, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 384, 0, 0, 128, 128, 0, 0, 0, 0, 128, 65280, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 384, 128, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 1, 0, 0, 0, 106, 0, 0, 8183, 0, 8191, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 344, 1, 0, 8191, 0, 8190, 0, 0, 0, 1, 0, 7909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8188, 2, 0, 0, 0, 38, 0, 0, 8172, 0, 8191, 0, 0, 0, 0, 0, 8188, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 22, 0, 0, 30, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 36, 0, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 5, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 249, 1, 5, 8191, 0, 8185, 0, 0, 0, 1, 0, 7994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7919, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 219] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 36, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 455, 2, 1, 8190, 0, 8186, 0, 0, 0, 2, 0, 7966, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7637, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 2, 2, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 320] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 6, 0, 0, 0, 1, 89, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 8, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 11, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8227, 8254, 8191, 8193, 8180, 8194, 8109] avg=8192.0000 max=8254 max_rank=2 min=8109 min_rank=7 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32676 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 5, 0, 0, 19, 0, 90, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 9, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8158, 8322, 8192, 8194, 8174, 8180, 8124] avg=8192.0000 max=8322 max_rank=2 min=8124 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32672 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 22, 0, 0, 30, 6, 90, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 21, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8156, 8385, 8187, 8193, 8162, 8171, 8089] avg=8192.0000 max=8385 max_rank=2 min=8089 min_rank=7 imbalance_ratio=1.023560 max_load_reduction=57151 max_load_reduction_pct=0.872055 moved_to_echo=32615 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 1, 0, 0, 0, 0, 0, 89, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8113, 8194, 8192, 8213, 8200, 8244] avg=8192.0000 max=8244 max_rank=7 min=8113 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32849 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 1, 5, 0, 0, 0, 0, 0, 0, 1, 0, 7927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7919, 0, 1, 0, 0, 0, 0, 0, 90, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 219, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8183, 8048, 8194, 8187, 8213, 8200, 8320] avg=8192.0000 max=8320 max_rank=7 min=8048 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32920 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 89, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8122, 8320, 8192, 8194, 8214, 8201, 8101] avg=8192.0000 max=8320 max_rank=2 min=8101 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32710 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 90, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 3, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 1, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8143, 8326, 8189, 8194, 8167, 8190, 8129] avg=8192.0000 max=8326 max_rank=2 min=8129 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32680 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 7899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7637, 0, 2, 0, 0, 0, 0, 0, 89, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 2, 2, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 320, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8359, 7768, 8197, 8189, 8213, 8200, 8420] avg=8192.0000 max=8420 max_rank=7 min=7768 min_rank=2 imbalance_ratio=1.027832 max_load_reduction=57116 max_load_reduction_pct=0.871521 moved_to_echo=33022 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 64512, 0, 128, 128, 0, 0, 128, 128, 768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 768, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 1280, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 128, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2878, 0, 0, 8192, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4289, 0, 0, 8192, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2649, 0, 0, 8192, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 8191, 0, 0, 8142, 5704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 2483, 0, 0, 8171, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 33, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5419, 0, 0, 8192, 8163, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2797, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 8185, 0, 0, 8144, 5556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 2624, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 29, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 0, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 8189, 0, 0, 8056, 4523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 3, 0, 0, 3659, 0, 0, 8154, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8189, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 42, 0, 8189, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5421, 1909, 1090, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5621, 11597, 8435, 7102, 8219, 8191, 8193, 8178] avg=8192.0000 max=11597 max_rank=1 min=5621 min_rank=0 imbalance_ratio=1.415649 max_load_reduction=53939 max_load_reduction_pct=0.823044 moved_to_echo=32781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6236, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5419, 1927, 1090, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 9034, 8452, 7101, 8220, 8192, 8193, 8183] avg=8192.0000 max=9034 max_rank=1 min=7101 min_rank=3 imbalance_ratio=1.102783 max_load_reduction=56502 max_load_reduction_pct=0.862152 moved_to_echo=32788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5419, 1926, 1090, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7031, 10168, 8450, 7102, 8220, 8195, 8192, 8178] avg=8192.0000 max=10168 max_rank=1 min=7031 min_rank=0 imbalance_ratio=1.241211 max_load_reduction=55368 max_load_reduction_pct=0.844849 moved_to_echo=32785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5416, 1904, 1090, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8192, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5390, 11826, 8425, 7102, 8224, 8198, 8192, 8179] avg=8192.0000 max=11826 max_rank=1 min=5390 min_rank=0 imbalance_ratio=1.443604 max_load_reduction=53710 max_load_reduction_pct=0.819550 moved_to_echo=32793 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 5420, 1311, 1090, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2624, 0, 0, 7099, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 29, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10951, 4256, 7838, 9726, 8173, 8190, 8193, 8209] avg=8192.0000 max=10951 max_rank=0 min=4256 min_rank=1 imbalance_ratio=1.336792 max_load_reduction=54585 max_load_reduction_pct=0.832901 moved_to_echo=32765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 5420, 1346, 1087, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2483, 0, 0, 7084, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 33, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10970, 4362, 7870, 9570, 8170, 8191, 8191, 8212] avg=8192.0000 max=10970 max_rank=0 min=4362 min_rank=1 imbalance_ratio=1.339111 max_load_reduction=54566 max_load_reduction_pct=0.832611 moved_to_echo=32764 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5424, 1938, 1094, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6353, 10828, 8476, 7098, 8225, 8187, 8192, 8177] avg=8192.0000 max=10828 max_rank=1 min=6353 min_rank=0 imbalance_ratio=1.321777 max_load_reduction=54708 max_load_reduction_pct=0.834778 moved_to_echo=32781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 5419, 1067, 1085, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3659, 0, 0, 7069, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 42, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11059, 3465, 7590, 10735, 8085, 8192, 8190, 8220] avg=8192.0000 max=11059 max_rank=0 min=3465 min_rank=1 imbalance_ratio=1.349976 max_load_reduction=54477 max_load_reduction_pct=0.831253 moved_to_echo=32687 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 43392, 13440, 8832, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 43136, 0, 0, 128, 0, 0, 0, 0, 0, 0, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 22016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 43520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 8832, 0, 0, 56832, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 317, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7931, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 486, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7731, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 190, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8177, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 39, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8159, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 1, 2, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 1, 1, 7850, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 223, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8188, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 2, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 1, 7730, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 563, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 8189, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7809, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 35, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8184, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 5, 0, 0, 0, 0, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 7621, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 585, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7639, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 190, 0, 0, 0, 0, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, 7912, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8077, 8156, 8192, 8218, 8186, 8193, 8205, 8309] avg=8192.0000 max=8309 max_rank=7 min=8077 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32893 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 317, 0, 0, 0, 0, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 7912, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 7931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8155, 8193, 8230, 8198, 8192, 8200, 8166] avg=8192.0000 max=8230 max_rank=3 min=8155 min_rank=1 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32756 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 486, 0, 0, 0, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7912, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 7731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8372, 8206, 8192, 8204, 8198, 8192, 8206, 7966] avg=8192.0000 max=8372 max_rank=0 min=7966 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32562 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 563, 0, 0, 0, 0, 0, 0, 7861, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 7912, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 7809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8445, 8110, 8204, 8245, 8195, 8197, 8096, 8044] avg=8192.0000 max=8445 max_rank=0 min=8044 min_rank=7 imbalance_ratio=1.030884 max_load_reduction=57091 max_load_reduction_pct=0.871140 moved_to_echo=32532 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 223, 0, 0, 0, 0, 0, 0, 7885, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 39, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 2, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 7466, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 221, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8234, 8190, 8009, 8194, 8190, 8208, 8399] avg=8192.0000 max=8399 max_rank=7 min=8009 min_rank=3 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32991 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 39, 0, 0, 0, 0, 0, 0, 7885, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 38, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 1, 2, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 7582, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 225, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7930, 8246, 8190, 8217, 8166, 8191, 8210, 8386] avg=8192.0000 max=8386 max_rank=7 min=7930 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32953 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 585, 0, 0, 0, 0, 0, 0, 7881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 7916, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 7639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8466, 8178, 8187, 8230, 8207, 8197, 8205, 7866] avg=8192.0000 max=8466 max_rank=0 min=7866 min_rank=7 imbalance_ratio=1.033447 max_load_reduction=57070 max_load_reduction_pct=0.870819 moved_to_echo=32475 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 0, 0, 0, 0, 0, 0, 7883, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 38, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 5, 0, 0, 0, 0, 0, 0, 286, 0, 0, 0, 0, 0, 7361, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 218, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7932, 8251, 8188, 8183, 8192, 8184, 8206, 8400] avg=8192.0000 max=8400 max_rank=7 min=7932 min_rank=0 imbalance_ratio=1.025391 max_load_reduction=57136 max_load_reduction_pct=0.871826 moved_to_echo=32982 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 768, 128, 128, 0, 0, 0, 0, 0, 128, 384, 0, 0, 0, 0, 128, 62080, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 1920, 0, 63744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 65536, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 2560, 0, 0, 0, 0, 0, 0, 63104, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 384, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[65536, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3602, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 8192, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4580, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1606, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6552, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3661, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4527, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2584, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5608, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2985, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 4, 1, 0, 8191, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5198, 0, 1, 1, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8173, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3130, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5062, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1516, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 4, 1, 0, 8191, 0, 8187, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6685, 0, 0, 1, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8176, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2297, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8183, 8, 2, 0, 8189, 0, 8179, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 1, 0, 0, 2, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5889, 0, 2, 2, 8189, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8147, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5523, 2648, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4580, 0, 0, 0, 2667, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9124, 8200, 8209, 8189, 8175, 8192, 7248, 8199] avg=8192.0000 max=9124 max_rank=0 min=7248 min_rank=6 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=31814 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5520, 2648, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6552, 0, 0, 0, 2666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7128, 8196, 8209, 8190, 8204, 8192, 9218, 8199] avg=8192.0000 max=9218 max_rank=6 min=7128 min_rank=0 imbalance_ratio=1.125244 max_load_reduction=56318 max_load_reduction_pct=0.859344 moved_to_echo=33813 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5524, 2648, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4527, 0, 0, 0, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9183, 8192, 8209, 8189, 8177, 8192, 7195, 8199] avg=8192.0000 max=9183 max_rank=0 min=7195 min_rank=6 imbalance_ratio=1.120972 max_load_reduction=56353 max_load_reduction_pct=0.859879 moved_to_echo=31763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5524, 2648, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5608, 0, 0, 0, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8190, 8209, 8189, 8174, 8193, 8276, 8199] avg=8192.0000 max=8276 max_rank=6 min=8106 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1516, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 8185, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5523, 2647, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6685, 0, 0, 1, 2667, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8176, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7037, 8192, 8191, 8192, 8182, 8192, 9365, 8185] avg=8192.0000 max=9365 max_rank=6 min=7037 min_rank=0 imbalance_ratio=1.143188 max_load_reduction=56171 max_load_reduction_pct=0.857101 moved_to_echo=33924 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 8186, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 5523, 2645, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5198, 0, 1, 1, 2667, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8173, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8502, 8192, 8188, 8200, 8193, 8193, 7878, 8190] avg=8192.0000 max=8502 max_rank=0 min=7878 min_rank=6 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=32454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5530, 2652, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5062, 0, 0, 0, 2662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8642, 8183, 8214, 8190, 8184, 8192, 7724, 8207] avg=8192.0000 max=8642 max_rank=0 min=7724 min_rank=6 imbalance_ratio=1.054932 max_load_reduction=56894 max_load_reduction_pct=0.868134 moved_to_echo=32307 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 8177, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 1, 0, 0, 2, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 5522, 2645, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5889, 0, 2, 2, 2667, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8147, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7814, 8191, 8107, 8197, 8247, 8190, 8632, 8158] avg=8192.0000 max=8632 max_rank=6 min=7814 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=33227 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 44288, 21248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 21504, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 44160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[44160, 0, 128, 128, 21376, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 0, 0, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1809, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6383, 0, 8192, 0, 0, 1, 0, 0, 0, 8186, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1584, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6608, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4637, 0, 8192, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8, 0, 0, 8, 0, 5638, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 2435, 0, 8187, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 10, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6576, 0, 8192, 0, 0, 5, 0, 0, 0, 8150, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 4089, 0, 8190, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8191, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 656, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7538, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8189, 0, 0, 0, 17, 0, 0, 10, 0, 3210, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 0, 4664, 0, 8186, 0, 0, 0, 1, 1, 0, 8189, 0, 0, 0, 0, 8184, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8180, 0, 0, 8189, 0, 0, 3, 1, 0, 0, 0, 0, 116, 0, 0, 0, 0, 32, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1584, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 6608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 63, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8187, 8262, 8129, 8192, 8196, 8185, 8193] avg=8192.0000 max=8262 max_rank=2 min=8129 min_rank=3 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 4637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 63, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8190, 8268, 8121, 8193, 8195, 8185, 8193] avg=8192.0000 max=8268 max_rank=2 min=8121 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8011, 63, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8083, 8279, 8193, 8187, 8216, 8195] avg=8192.0000 max=8279 max_rank=3 min=8083 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 2, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 6576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 63, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8196, 8298, 8092, 8193, 8192, 8185, 8193] avg=8192.0000 max=8298 max_rank=2 min=8092 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1809, 1, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 6383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 63, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8263, 8124, 8192, 8196, 8185, 8192] avg=8192.0000 max=8263 max_rank=2 min=8124 min_rank=3 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 5638, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 2435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 10, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8018, 63, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8194, 8150, 8231, 8193, 8188, 8200, 8188] avg=8192.0000 max=8231 max_rank=3 min=8150 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 7538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 67, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8182, 8259, 8123, 8193, 8192, 8196, 8192] avg=8192.0000 max=8259 max_rank=2 min=8123 min_rank=3 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8184, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3210, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 4664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8180, 0, 0, 4, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 32, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 7962, 63, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8204, 7953, 8437, 8187, 8190, 8184, 8190] avg=8192.0000 max=8437 max_rank=3 min=7953 min_rank=2 imbalance_ratio=1.029907 max_load_reduction=57099 max_load_reduction_pct=0.871262 moved_to_echo=32751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 22144, 128, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 43008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 65024, 0, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64640, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8166, 0, 3, 26, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8175, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 25, 0, 0, 2, 8, 199, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 8157, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 8188, 7336, 0, 0, 675, 1, 0, 0, 2, 0, 2, 0, 0, 8188, 0, 2, 1, 0, 0, 0, 8106, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8189, 0, 0, 119, 0, 0, 1, 1, 73, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 8190, 7676, 0, 0, 330, 0, 0, 0, 1, 0, 1, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 8118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8158, 0, 14, 34, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8174, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8185, 0, 0, 63, 0, 0, 1, 0, 72, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8143, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 8190, 7909, 0, 1, 166, 0, 0, 0, 1, 0, 1, 0, 0, 8189, 0, 1, 0, 1, 0, 0, 8160, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 160, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8201, 8201, 8370, 8041, 8212, 8192, 8192] avg=8192.0000 max=8370 max_rank=3 min=8041 min_rank=4 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32637 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 160, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 26, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8186, 8201, 8361, 8061, 8215, 8194, 8192] avg=8192.0000 max=8361 max_rank=3 min=8061 min_rank=4 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32662 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 160, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8186, 8201, 8387, 8032, 8215, 8197, 8192] avg=8192.0000 max=8387 max_rank=3 min=8032 min_rank=4 imbalance_ratio=1.023804 max_load_reduction=57149 max_load_reduction_pct=0.872025 moved_to_echo=32636 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8123, 0, 0, 119, 0, 0, 1, 1, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 7995, 160, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 195, 7676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 1, 0, 1, 0, 0, 8031, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 8201, 8195, 7937, 8364, 8142, 8187, 8193] avg=8192.0000 max=8364 max_rank=4 min=7937 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32886 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 160, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 34, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8186, 8201, 8353, 8080, 8215, 8182, 8192] avg=8192.0000 max=8353 max_rank=3 min=8080 min_rank=4 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32669 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8119, 0, 0, 63, 0, 0, 1, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 7995, 160, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 195, 7909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 166, 0, 0, 0, 1, 0, 1, 0, 0, 8029, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8189, 8159, 8164, 8200, 8184, 8194, 8191] avg=8192.0000 max=8255 max_rank=0 min=8159 min_rank=2 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32769 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 163, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8197, 8208, 8365, 8047, 8222, 8201, 8192] avg=8192.0000 max=8365 max_rank=3 min=8047 min_rank=4 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32662 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8120, 0, 0, 25, 0, 0, 2, 8, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 7993, 160, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 195, 7336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 1, 0, 0, 2, 0, 2, 0, 0, 8028, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8182, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8354, 8190, 8170, 7599, 8711, 8131, 8189, 8192] avg=8192.0000 max=8711 max_rank=4 min=7599 min_rank=3 imbalance_ratio=1.063354 max_load_reduction=56825 max_load_reduction_pct=0.867081 moved_to_echo=33223 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 64000, 1408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 1664, 63872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 65024, 0, 0, 256, 0, 0, 128, 128, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 1280, 128, 0, 0, 128, 0, 128, 0, 0, 64256, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True + [2026-08-26 16:28:32] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 452.2 | throughput per GPU (TFLOP/s/GPU): 399.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.646755E+00 | loss scale: 1.0 | grad norm: 5.538 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3, 2246, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 1, 1, 3, 2067, 0, 0, 3604, 8, 0, 0, 8186, 0, 8185, 0, 0, 1, 0, 0, 2, 0, 0, 0, 2, 2, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8186, 0, 1, 0, 0, 0, 1, 0, 0, 3155, 0, 0, 0, 4693, 0, 6111, 10, 3, 3, 2979, 0, 1, 4, 0, 2, 0, 1, 0, 7910, 0, 2, 1, 0, 0, 0, 536, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7574, 1, 0, 3, 0, 3, 0, 1, 0, 1, 1, 8, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1707, 0, 0, 0, 3, 0, 0, 4, 0, 3, 1, 5, 0, 0, 0, 2846, 0, 0, 2725, 0, 0, 0, 8191, 0, 8188, 2, 0, 0, 1, 0, 2, 1, 0, 0, 0, 2, 0, 0, 1, 0, 1, 1, 1, 2, 1, 1, 2, 0, 0, 0, 0, 0, 2, 2, 8177, 0, 0, 0, 0, 1, 0, 0, 0, 2805, 0, 1, 0, 5331, 0, 6155, 9, 0, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 8054, 0, 0, 0, 1, 0, 0, 314, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1214, 1, 1, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 2733, 1, 0, 2593, 0, 0, 0, 8189, 0, 8188, 2, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 3, 0, 0, 0, 0, 2882, 0, 1, 0, 5151, 0, 6222, 6, 0, 0, 4119, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 3, 1, 0, 0, 0, 146, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7588, 0, 0, 12, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 4, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 1003, 0, 0, 0, 9, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 2563, 0, 0, 2618, 2, 0, 0, 8192, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 1, 0, 1, 18, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 3403, 0, 0, 0, 5464, 0, 6207, 2, 0, 0, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 12, 0, 0, 2, 0, 238, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7556, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 182, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 14, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 0, 5, 2809, 8, 1, 0, 8192, 0, 8190, 0, 0, 1, 0, 0, 0, 5, 2, 0, 0, 2, 1, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 3, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 0, 4989, 0, 5833, 8, 0, 0, 3732, 0, 0, 0, 0, 0, 0, 1, 0, 7737, 0, 0, 0, 1, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 39, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 2700, 0, 0, 2253, 0, 0, 0, 8192, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 4, 0, 0, 0, 0, 2655, 0, 0, 0, 5772, 0, 6568, 3, 1, 1, 3809, 0, 0, 0, 0, 1, 1, 0, 0, 8018, 0, 1, 0, 0, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7462, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 167, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1239, 0, 0, 0, 0, 0, 3, 0, 0, 5, 11, 1, 0, 0, 0, 3161, 0, 0, 3296, 3, 0, 0, 8190, 1, 8191, 4, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 2591, 1, 1, 0, 5077, 0, 6420, 0, 1, 0, 3333, 0, 0, 0, 0, 0, 1, 0, 0, 7913, 0, 4, 0, 0, 3, 0, 281, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7516, 0, 1, 11, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 45, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 1134, 0, 1, 1, 3, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 2946, 1, 0, 3087, 0, 0, 0, 8190, 0, 8190, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 3410, 0, 0, 0, 5149, 0, 6154, 14, 1, 0, 2931, 0, 0, 0, 0, 0, 0, 0, 0, 7927, 0, 4, 0, 0, 0, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7573, 0, 0, 1, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 80, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 14, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 0, 5, 2809, 8, 1, 0, 0, 0, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 2, 0, 0, 2, 1, 0, 3, 0, 0, 4, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 0, 1550, 0, 0, 8, 0, 0, 3732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7620, 0, 0, 0, 1, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7617, 0, 0, 0, 0, 0, 476, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 39, 0, 3, 0, 0, 5833, 1867, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7850, 8197, 8196, 8190, 9133, 8004, 8209, 7757] avg=8192.0000 max=9133 max_rank=4 min=7757 min_rank=7 imbalance_ratio=1.114868 max_load_reduction=56403 max_load_reduction_pct=0.860641 moved_to_echo=33103 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1707, 0, 0, 0, 3, 0, 0, 4, 0, 3, 1, 3736, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2846, 0, 0, 2725, 0, 0, 0, 0, 0, 2586, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 0, 0, 0, 2, 0, 0, 1, 0, 1, 1, 1, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 2, 0, 0, 0, 0, 0, 2, 2, 8177, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2805, 0, 1, 0, 1656, 0, 0, 9, 0, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7932, 0, 0, 0, 1, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7671, 0, 0, 0, 0, 0, 509, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 6155, 1866, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8623, 8164, 8187, 8189, 7731, 8247, 8301, 8094] avg=8192.0000 max=8623 max_rank=0 min=7731 min_rank=4 imbalance_ratio=1.052612 max_load_reduction=56913 max_load_reduction_pct=0.868423 moved_to_echo=32373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1214, 1, 1, 3, 0, 0, 0, 1, 0, 0, 0, 3736, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2733, 1, 0, 2593, 0, 0, 0, 0, 0, 2586, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 8, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2882, 0, 1, 0, 1600, 0, 0, 6, 0, 0, 4119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 3, 1, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7588, 0, 0, 12, 0, 0, 492, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 4, 0, 1, 6222, 1866, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8015, 7917, 8188, 8194, 8608, 8101, 8215, 8298] avg=8192.0000 max=8608 max_rank=4 min=7917 min_rank=1 imbalance_ratio=1.050781 max_load_reduction=56928 max_load_reduction_pct=0.868652 moved_to_echo=33222 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3, 2246, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 3735, 2787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 2067, 0, 0, 3604, 8, 0, 0, 0, 0, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 2, 2, 0, 0, 0, 1, 0, 1, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8186, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3155, 0, 0, 0, 1458, 0, 0, 10, 3, 3, 2979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 2, 0, 1, 0, 7790, 0, 2, 1, 0, 0, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7574, 1, 0, 3, 0, 3, 448, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 8, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 6111, 1865, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8778, 8269, 8181, 8192, 7609, 8337, 8148, 8022] avg=8192.0000 max=8778 max_rank=0 min=7609 min_rank=4 imbalance_ratio=1.071533 max_load_reduction=56758 max_load_reduction_pct=0.866058 moved_to_echo=32116 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3737, 3428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2700, 0, 0, 2253, 0, 0, 0, 0, 0, 2586, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2655, 0, 0, 0, 1793, 0, 0, 3, 1, 1, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 7897, 0, 1, 0, 0, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7462, 0, 0, 2, 1, 0, 551, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 167, 1, 0, 0, 0, 6568, 1867, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 7542, 8185, 8193, 8262, 8315, 8136, 8619] avg=8192.0000 max=8619 max_rank=7 min=7542 min_rank=1 imbalance_ratio=1.052124 max_load_reduction=56917 max_load_reduction_pct=0.868484 moved_to_echo=33332 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 1003, 0, 0, 0, 9, 0, 0, 0, 0, 2, 5, 3738, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2563, 0, 0, 2618, 2, 0, 0, 0, 0, 2587, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 18, 0, 0, 0, 0, 0, 10, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3403, 0, 0, 0, 1697, 0, 0, 2, 0, 0, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7877, 0, 12, 0, 0, 2, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7556, 0, 0, 3, 0, 0, 522, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 182, 0, 0, 0, 0, 6207, 1867, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8016, 7779, 8209, 8191, 8739, 8129, 8203, 8270] avg=8192.0000 max=8739 max_rank=4 min=7779 min_rank=1 imbalance_ratio=1.066772 max_load_reduction=56797 max_load_reduction_pct=0.866653 moved_to_echo=33341 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1239, 0, 0, 0, 0, 0, 3, 0, 0, 5, 11, 3738, 3015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3161, 0, 0, 3296, 3, 0, 0, 0, 1, 2586, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 27, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8184, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2591, 1, 1, 0, 1577, 0, 0, 0, 1, 0, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7793, 0, 4, 0, 0, 3, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7516, 0, 1, 11, 0, 1, 485, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 45, 3, 0, 0, 0, 6420, 1867, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8017, 9052, 8207, 8189, 7504, 8082, 8133, 8352] avg=8192.0000 max=9052 max_rank=1 min=7504 min_rank=4 imbalance_ratio=1.104980 max_load_reduction=56484 max_load_reduction_pct=0.861877 moved_to_echo=32071 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 1134, 0, 1, 1, 3, 0, 0, 0, 0, 0, 4, 3743, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2946, 1, 0, 3087, 0, 0, 0, 0, 0, 2577, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3410, 0, 0, 0, 1592, 0, 0, 14, 1, 0, 2931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7801, 0, 4, 0, 0, 0, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7573, 0, 0, 1, 0, 0, 495, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 80, 0, 0, 0, 2, 6154, 1870, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7953, 8616, 8183, 8198, 7950, 8321, 8191, 8124] avg=8192.0000 max=8616 max_rank=1 min=7950 min_rank=4 imbalance_ratio=1.051758 max_load_reduction=56920 max_load_reduction_pct=0.868530 moved_to_echo=32586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 128, 128, 10880, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 29952, 24832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 21888, 128, 128, 23040, 128, 128, 0, 0, 128, 20736, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 24832, 128, 128, 0, 12928, 0, 0, 128, 128, 128, 27904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 0, 0, 128, 128, 896, 128, 128, 0, 128, 49792, 14976, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 0, 128, 128, 60672, 128, 128, 128, 128, 128, 4096, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 128, 0, 62720, 0, 128, 128, 128, 128, 0, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 0, 0, 128, 128, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8185, 0, 0, 0, 0, 0, 8189, 0, 5, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8189, 8191, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8178, 0, 114, 0, 0, 0, 8080, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8169, 0, 1, 0, 0, 0, 8184, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8186, 0, 6, 1, 1, 0, 8188, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 8188, 8189, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8189, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 28, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8173, 0, 0, 0, 0, 0, 8187, 1, 141, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8185, 0, 7, 2, 0, 2, 8186, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8190, 8190, 2, 0, 0, 10, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8176, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8185, 4, 200, 2, 0, 0, 7987, 0, 0, 0, 0, 0, 0, 1, 0, 5, 1, 0, 2, 0, 8187, 8186, 0, 0, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 8184, 0, 0, 0, 0, 0, 5, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 114, 0, 0, 0, 8058, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 8175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8189, 8206, 8190, 8196, 8190, 8195, 8191] avg=8192.0000 max=8206 max_rank=2 min=8179 min_rank=0 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8166, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8186, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8192, 8202, 8197, 8198, 8191, 8192, 8192] avg=8192.0000 max=8202 max_rank=2 min=8172 min_rank=0 imbalance_ratio=1.001221 max_load_reduction=57334 max_load_reduction_pct=0.874847 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 1, 1, 0, 8165, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 6, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8190, 8208, 8197, 8192, 8189, 8191, 8190] avg=8192.0000 max=8208 max_rank=2 min=8179 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 28, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8191, 8213, 8192, 8181, 8193, 8190, 8188] avg=8192.0000 max=8213 max_rank=2 min=8181 min_rank=4 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32752 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 8161, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8186, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 8165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8192, 8213, 8198, 8190, 8195, 8190, 8191] avg=8192.0000 max=8213 max_rank=2 min=8167 min_rank=0 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 200, 2, 0, 0, 7965, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 8184, 0, 0, 0, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 6, 0, 0, 0, 0, 0, 5, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 4, 0, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8197, 8208, 8180, 8185, 8195, 8196, 8198] avg=8192.0000 max=8208 max_rank=2 min=8177 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8164, 1, 141, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 8170, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8188, 8072, 8199, 8196, 8189, 8192, 8193] avg=8192.0000 max=8307 max_rank=0 min=8072 min_rank=2 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 2, 0, 2, 8153, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8182, 0, 2, 0, 0, 10, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 5, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8185, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 8180, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8197, 8214, 8183, 8198, 8194, 8190, 8193] avg=8192.0000 max=8214 max_rank=2 min=8167 min_rank=0 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32775 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 65536, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 0, 128, 128, 128, 128, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 65408, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[65536, 128, 128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 384, 128, 128, 128, 65024, 128, 256, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 64, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7279, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8183, 0, 171, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8179, 1, 0, 0, 6, 8181, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2717, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8104, 0, 5378, 0, 8184, 1, 7, 0, 0, 1, 0, 0, 0, 8185, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8192, 0, 8093, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7727, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 114, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1211, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 6865, 0, 8190, 0, 1, 0, 1, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 0, 8087, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 483, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 8187, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3012, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 4737, 0, 8190, 0, 2, 0, 0, 1, 0, 0, 0, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 58, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 7903, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 7008, 108, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8084, 8192, 8190, 7208, 8202, 9277] avg=8192.0000 max=9277 max_rank=7 min=7208 min_rank=5 imbalance_ratio=1.132446 max_load_reduction=56259 max_load_reduction_pct=0.858444 moved_to_echo=32877 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 0, 7007, 108, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 7279, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8146, 8193, 8191, 7968, 8190, 8464] avg=8192.0000 max=8464 max_rank=7 min=7968 min_rank=5 imbalance_ratio=1.033203 max_load_reduction=57072 max_load_reduction_pct=0.870850 moved_to_echo=32813 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1211, 0, 7004, 108, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 6865, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8196, 8196, 8194, 8190, 8330, 8189, 8051] avg=8192.0000 max=8330 max_rank=5 min=8051 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32760 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 7008, 108, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 7727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8084, 8192, 8190, 7584, 8190, 8911] avg=8192.0000 max=8911 max_rank=7 min=7584 min_rank=5 imbalance_ratio=1.087769 max_load_reduction=56625 max_load_reduction_pct=0.864029 moved_to_echo=32875 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8181, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 8179, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2717, 0, 6932, 108, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 0, 5378, 0, 0, 1, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8185, 8243, 8197, 8195, 9767, 8194, 6559] avg=8192.0000 max=9767 max_rank=5 min=6559 min_rank=7 imbalance_ratio=1.192261 max_load_reduction=55769 max_load_reduction_pct=0.850967 moved_to_echo=32715 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 7006, 108, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8084, 8192, 8192, 7222, 8192, 9271] avg=8192.0000 max=9271 max_rank=7 min=7222 min_rank=5 imbalance_ratio=1.131714 max_load_reduction=56265 max_load_reduction_pct=0.858536 moved_to_echo=32877 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 227, 0, 7007, 108, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 7903, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8142, 8191, 8194, 7348, 8190, 9089] avg=8192.0000 max=9089 max_rank=7 min=7348 min_rank=5 imbalance_ratio=1.109497 max_load_reduction=56447 max_load_reduction_pct=0.861313 moved_to_echo=32821 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3012, 0, 6974, 115, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 0, 4737, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8196, 8557, 8185, 8194, 10109, 8189, 5914] avg=8192.0000 max=10109 max_rank=5 min=5914 min_rank=7 imbalance_ratio=1.234009 max_load_reduction=55427 max_load_reduction_pct=0.845749 moved_to_echo=32406 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[9472, 0, 56192, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 0, 128, 8704, 0, 56064, 896, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[896, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 64768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 128, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 128, 128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 128, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7847, 0, 0, 0, 27, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 40, 8191, 8152, 0, 0, 0, 1, 0, 8191, 8191, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7857, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4, 0, 7, 6, 0, 0, 0, 8155, 0, 0, 0, 694, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 4, 74, 8181, 7804, 0, 0, 0, 7, 0, 8182, 8086, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1859, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 3017, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 8177, 1, 0, 0, 24, 8189, 8082, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7919, 0, 0, 0, 22, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 2, 1, 0, 0, 0, 8187, 0, 0, 0, 647, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 20, 8190, 8140, 0, 0, 0, 2, 0, 8181, 8166, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5829, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1920, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7878, 0, 0, 0, 5, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1209, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 3506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 2614, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 0, 0, 0, 0, 0, 0, 6380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5532, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7847, 109, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 8192, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 5419, 8197, 8183, 9834, 9480, 7965, 8265] avg=8192.0000 max=9834 max_rank=4 min=5419 min_rank=1 imbalance_ratio=1.200439 max_load_reduction=55702 max_load_reduction_pct=0.849945 moved_to_echo=35544 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 2614, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 1, 0, 0, 0, 0, 6378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5532, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 109, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 8152, 0, 0, 0, 1, 0, 0, 45, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 10733, 8190, 8182, 6528, 7165, 8307, 8239] avg=8192.0000 max=10733 max_rank=1 min=6528 min_rank=4 imbalance_ratio=1.310181 max_load_reduction=54803 max_load_reduction_pct=0.836227 moved_to_echo=30239 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 2614, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 0, 0, 0, 0, 0, 0, 6380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7919, 109, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 4790, 8191, 8183, 8372, 11510, 8037, 8260] avg=8192.0000 max=11510 max_rank=5 min=4790 min_rank=1 imbalance_ratio=1.405029 max_load_reduction=54026 max_load_reduction_pct=0.824371 moved_to_echo=36179 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2614, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1859, 0, 0, 0, 0, 0, 0, 6373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3017, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 8177, 109, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 24, 0, 8082, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7728, 8193, 8182, 8234, 8553, 8300, 8152] avg=8192.0000 max=8553 max_rank=5 min=7728 min_rank=1 imbalance_ratio=1.044067 max_load_reduction=56983 max_load_reduction_pct=0.869492 moved_to_echo=33239 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2581, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 6, 0, 0, 0, 6351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 5461, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 109, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 74, 0, 7804, 0, 0, 0, 7, 0, 0, 44, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 12141, 8193, 8235, 6368, 6168, 8307, 7940] avg=8192.0000 max=12141 max_rank=1 min=6168 min_rank=5 imbalance_ratio=1.482056 max_load_reduction=53395 max_load_reduction_pct=0.814743 moved_to_echo=28783 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 2614, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5829, 0, 0, 0, 0, 0, 0, 6378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1920, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7878, 109, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 5062, 8198, 8183, 12207, 7453, 7997, 8243] avg=8192.0000 max=12207 max_rank=4 min=5062 min_rank=1 imbalance_ratio=1.490112 max_load_reduction=53329 max_load_reduction_pct=0.813736 moved_to_echo=35900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2614, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1209, 0, 0, 0, 0, 0, 0, 6379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5532, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 109, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7794, 8192, 8185, 7589, 9039, 8307, 8237] avg=8192.0000 max=9039 max_rank=5 min=7589 min_rank=4 imbalance_ratio=1.103394 max_load_reduction=56497 max_load_reduction_pct=0.862076 moved_to_echo=33172 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2611, 1705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 2, 1, 0, 0, 0, 6369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 5519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 113, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 8140, 0, 0, 0, 2, 0, 0, 36, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 11869, 8182, 8203, 6404, 6168, 8316, 8200] avg=8192.0000 max=11869 max_rank=1 min=6168 min_rank=5 imbalance_ratio=1.448853 max_load_reduction=53667 max_load_reduction_pct=0.818893 moved_to_echo=29088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 30976, 0, 128, 0, 0, 0, 0, 256, 0, 0, 128, 128, 128, 0, 20992, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 21376, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 44288, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 128, 14592, 0, 128, 128, 0, 0, 0, 51072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 64640, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 256, 0, 65024, 0, 0, 0, 128, 0, 0, 384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8184, 8183, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 9, 0, 0, 0, 431, 3, 0, 0, 0, 0, 0, 7499, 0, 0, 0, 5, 0, 0, 0, 46, 0, 7995, 5461, 0, 0, 0, 5, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7811, 3464, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 1, 0, 181, 0, 7, 0, 0, 0, 0, 0, 0, 7937, 7, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8187, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 1, 1, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 6469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5419, 4508, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 8136, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 251, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8035, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 114, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 129, 0, 0, 0, 1, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8191, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1634, 7312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 4, 0, 7416, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 284, 8120, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 19, 0, 0, 0, 477, 2, 1, 0, 0, 0, 0, 7440, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8166, 7073, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 140, 0, 2, 0, 0, 0, 0, 0, 0, 7295, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 99, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 1, 1, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 6469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5419, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6, 0, 1, 0, 0, 0, 0, 7506, 470, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8183, 8279, 7147, 9189, 8191, 7991, 8361] avg=8192.0000 max=9189 max_rank=4 min=7147 min_rank=3 imbalance_ratio=1.121704 max_load_reduction=56347 max_load_reduction_pct=0.859787 moved_to_echo=33732 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 251, 6731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7510, 840, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8035, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8187, 8273, 8870, 7035, 8188, 8357, 8441] avg=8192.0000 max=8870 max_rank=3 min=7035 min_rank=4 imbalance_ratio=1.082764 max_load_reduction=56666 max_load_reduction_pct=0.864655 moved_to_echo=32021 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 114, 6814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7513, 850, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8187, 8272, 8871, 6931, 8188, 8369, 8523] avg=8192.0000 max=8871 max_rank=3 min=6931 min_rank=4 imbalance_ratio=1.082886 max_load_reduction=56665 max_load_reduction_pct=0.864639 moved_to_echo=32011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 9, 0, 8175, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 3, 0, 0, 0, 0, 0, 7499, 0, 0, 0, 5, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 662, 5461, 0, 0, 0, 5, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7811, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 1, 0, 181, 0, 7, 0, 0, 0, 0, 7333, 361, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7937, 7, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8195, 7972, 6252, 10709, 8194, 7901, 8124] avg=8192.0000 max=10709 max_rank=4 min=6252 min_rank=3 imbalance_ratio=1.307251 max_load_reduction=54827 max_load_reduction_pct=0.836594 moved_to_echo=34928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8181, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 129, 0, 0, 0, 1, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 8191, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1634, 6115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 7510, 762, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7416, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8201, 8245, 8872, 7752, 8189, 8293, 7791] avg=8192.0000 max=8872 max_rank=3 min=7752 min_rank=4 imbalance_ratio=1.083008 max_load_reduction=56664 max_load_reduction_pct=0.864624 moved_to_echo=32025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 99, 6838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7513, 853, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8187, 8274, 8871, 6949, 8188, 8372, 8501] avg=8192.0000 max=8871 max_rank=3 min=6949 min_rank=4 imbalance_ratio=1.082886 max_load_reduction=56665 max_load_reduction_pct=0.864639 moved_to_echo=32010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 284, 6790, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7512, 847, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8186, 8274, 8873, 7078, 8187, 8365, 8378] avg=8192.0000 max=8873 max_rank=3 min=7078 min_rank=4 imbalance_ratio=1.083130 max_load_reduction=56663 max_load_reduction_pct=0.864609 moved_to_echo=32008 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 19, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 2, 1, 0, 0, 0, 0, 7440, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 672, 7073, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 1917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 140, 0, 2, 0, 0, 0, 0, 7494, 243, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7295, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8210, 7947, 7780, 9893, 8211, 7888, 7417] avg=8192.0000 max=9893 max_rank=4 min=7417 min_rank=7 imbalance_ratio=1.207642 max_load_reduction=55643 max_load_reduction_pct=0.849045 moved_to_echo=33409 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 384, 0, 128, 0, 0, 0, 0, 59904, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 5504, 60032, 0, 0, 0, 128, 0, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 23680, 41984, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 63104, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 1152, 128, 128, 0, 128, 0, 0, 64000, 0, 0, 0, 128, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 31, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 89, 0, 0, 19, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 7300, 0, 0, 0, 0, 1313, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 1, 0, 8176, 0, 0, 0, 1, 7618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8113, 0, 22, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 18, 44, 0, 0, 50, 0, 0, 0, 8192, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 209, 0, 0, 0, 6, 0, 0, 0, 0, 0, 178, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4416, 0, 0, 0, 0, 4383, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 8071, 0, 0, 7, 0, 8085, 0, 0, 0, 7, 6570, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 8, 8181, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8184, 2, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5446, 0, 0, 0, 0, 2685, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 2, 0, 8177, 0, 0, 0, 2, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8189, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 19, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 0, 0, 8192, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 180, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 7287, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 689, 0, 0, 0, 0, 0, 4, 3, 0, 0, 4, 0, 0, 0, 8191, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 45, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 76, 0, 0, 4, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 31, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 89, 0, 0, 19, 0, 0, 7871, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8207, 8137, 8069, 8502, 8192, 8190, 8168] avg=8192.0000 max=8502 max_rank=4 min=8069 min_rank=3 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=33052 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8176, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7300, 0, 0, 0, 0, 1313, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 7867, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 1, 0, 0, 0, 0, 0, 1, 7618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8088, 8193, 8617, 8100, 7931, 8193, 8212, 8202] avg=8192.0000 max=8617 max_rank=2 min=7931 min_rank=4 imbalance_ratio=1.051880 max_load_reduction=56919 max_load_reduction_pct=0.868515 moved_to_echo=32538 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8186, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 22, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 18, 44, 0, 0, 50, 0, 0, 7871, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8067, 8203, 8185, 8069, 8502, 8191, 8157, 8162] avg=8192.0000 max=8502 max_rank=4 min=8067 min_rank=0 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=33012 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 180, 2, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8186, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7287, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 689, 0, 0, 0, 0, 0, 4, 3, 0, 0, 4, 0, 0, 7870, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8203, 7321, 8658, 8503, 8192, 8202, 8199] avg=8192.0000 max=8658 max_rank=3 min=7321 min_rank=2 imbalance_ratio=1.056885 max_load_reduction=56878 max_load_reduction_pct=0.867889 moved_to_echo=33096 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 209, 0, 0, 0, 6, 0, 0, 0, 0, 0, 178, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 8085, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4416, 0, 0, 0, 0, 4383, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 7754, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 7, 0, 0, 0, 0, 0, 7, 6570, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 8, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8456, 8112, 8805, 8662, 6898, 8196, 8202, 8205] avg=8192.0000 max=8805 max_rank=2 min=6898 min_rank=4 imbalance_ratio=1.074829 max_load_reduction=56731 max_load_reduction_pct=0.865646 moved_to_echo=31501 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 45, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 76, 0, 0, 4, 0, 0, 7871, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8208, 8164, 8054, 8503, 8191, 8166, 8180] avg=8192.0000 max=8503 max_rank=4 min=8054 min_rank=3 imbalance_ratio=1.037964 max_load_reduction=57033 max_load_reduction_pct=0.870255 moved_to_echo=33040 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 19, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 0, 7871, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8080, 8208, 8180, 7971, 8494, 8191, 8210, 8202] avg=8192.0000 max=8494 max_rank=4 min=7971 min_rank=3 imbalance_ratio=1.036865 max_load_reduction=57042 max_load_reduction_pct=0.870392 moved_to_echo=33097 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8177, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5440, 0, 0, 0, 0, 2685, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7817, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 2, 0, 0, 0, 0, 0, 2, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8446, 8202, 8127, 7953, 8203, 8190, 8197, 8218] avg=8192.0000 max=8446 max_rank=0 min=7953 min_rank=3 imbalance_ratio=1.031006 max_load_reduction=57090 max_load_reduction_pct=0.871124 moved_to_echo=32808 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 1792, 0, 0, 0, 0, 128, 128, 256, 0, 0, 128, 0, 128, 62848, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 2560, 0, 0, 128, 128, 0, 0, 0, 0, 128, 63104, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 384, 0, 128, 0, 128, 0, 0, 128, 0, 0, 768, 128, 0, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 65536, 128, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 56960, 0, 128, 0, 0, 8576, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 13, 1, 1, 8191, 0, 8188, 0, 0, 0, 1, 0, 8052, 0, 0, 0, 0, 429, 0, 0, 0, 1, 6069, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1641, 3, 0, 0, 0, 174] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 204, 7, 0, 8184, 1, 8180, 0, 0, 0, 8, 0, 6169, 0, 0, 0, 0, 897, 0, 0, 0, 0, 6348, 0, 8, 0, 0, 4, 0, 0, 2, 0, 0, 0, 8, 8, 8, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8, 0, 0, 1, 0, 4, 4, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1773, 0, 0, 0, 0, 787] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8098, 4, 0, 0, 0, 56, 0, 0, 8187, 0, 8188, 0, 0, 0, 0, 0, 7884, 0, 0, 5, 0, 0, 0, 0, 0, 0, 410, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 166] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 1, 0, 0, 0, 9, 0, 0, 8146, 0, 8191, 0, 0, 0, 0, 0, 8143, 0, 0, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8130, 0, 0, 0, 0, 151] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 151, 2, 67, 8190, 0, 8186, 0, 0, 0, 2, 0, 7551, 0, 0, 0, 0, 40, 0, 0, 0, 1, 5662, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 876, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1448, 2, 0, 0, 0, 585] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 1, 0, 0, 0, 0, 5541, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 25, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 5, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8524, 5769, 8189, 8064, 8133, 8194, 10458] avg=8192.0000 max=10458 max_rank=7 min=5769 min_rank=2 imbalance_ratio=1.276611 max_load_reduction=55078 max_load_reduction_pct=0.840424 moved_to_echo=34849 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 1, 332, 0, 0, 0, 0, 0, 1, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 1, 6069, 0, 1, 0, 0, 0, 0, 0, 5558, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1641, 3, 0, 0, 0, 174, 2269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8400, 12187, 8191, 8063, 8211, 8191, 4088] avg=8192.0000 max=12187 max_rank=2 min=4088 min_rank=7 imbalance_ratio=1.487671 max_load_reduction=53349 max_load_reduction_pct=0.814041 moved_to_echo=28553 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8098, 4, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 7884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 410, 0, 0, 6, 0, 0, 0, 2, 5555, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 3, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 166, 2268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8272, 6107, 8190, 8059, 8197, 8188, 10405] avg=8192.0000 max=10405 max_rank=7 min=6107 min_rank=2 imbalance_ratio=1.270142 max_load_reduction=55131 max_load_reduction_pct=0.841232 moved_to_echo=34849 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 5, 0, 0, 2, 0, 5550, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 24, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8524, 5758, 8189, 8062, 8166, 8177, 10455] avg=8192.0000 max=10455 max_rank=7 min=5758 min_rank=2 imbalance_ratio=1.276245 max_load_reduction=55081 max_load_reduction_pct=0.840469 moved_to_echo=34860 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 7, 0, 332, 1, 0, 0, 0, 0, 8, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 0, 0, 0, 0, 6348, 0, 8, 0, 0, 4, 0, 0, 5553, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8, 8, 8, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 176, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8, 0, 0, 1, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1773, 0, 0, 0, 0, 787, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 6721, 12939, 8207, 8229, 8212, 8193, 4827] avg=8192.0000 max=12939 max_rank=2 min=4827 min_rank=7 imbalance_ratio=1.579468 max_load_reduction=52597 max_load_reduction_pct=0.802567 moved_to_echo=29461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 5550, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 33, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8524, 5702, 8189, 8063, 8201, 8195, 10457] avg=8192.0000 max=10457 max_rank=7 min=5702 min_rank=2 imbalance_ratio=1.276489 max_load_reduction=55079 max_load_reduction_pct=0.840439 moved_to_echo=34916 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2, 0, 0, 7, 0, 5527, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8130, 0, 0, 0, 0, 151, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8484, 5679, 8189, 8064, 8203, 8195, 10538] avg=8192.0000 max=10538 max_rank=7 min=5679 min_rank=2 imbalance_ratio=1.286377 max_load_reduction=54998 max_load_reduction_pct=0.839203 moved_to_echo=35000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 2, 67, 314, 0, 0, 0, 0, 0, 2, 0, 7551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 1, 5662, 0, 2, 0, 0, 0, 0, 0, 5560, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 876, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1448, 2, 0, 0, 0, 585, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8087, 11395, 8192, 8932, 8213, 8203, 4308] avg=8192.0000 max=11395 max_rank=2 min=4308 min_rank=7 imbalance_ratio=1.390991 max_load_reduction=54141 max_load_reduction_pct=0.826126 moved_to_echo=29656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 1408, 0, 0, 0, 128, 18816, 0, 128, 128, 0, 128, 128, 128, 44416, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 45568, 128, 0, 0, 0, 1920, 18176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 512, 128, 128, 2688, 128, 0, 0, 0, 0, 128, 0, 62464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 64512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 128, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5886, 0, 0, 8192, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 2089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8183, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 377, 8183, 0, 0, 7742, 1649, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2855, 0, 0, 0, 0, 0, 7, 8, 0, 0, 8, 5, 0, 0, 3714, 0, 0, 8036, 9, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 8180, 0, 0, 5, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 13, 0, 0, 0, 3, 0, 0, 177, 0, 8181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5979, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6901, 0, 0, 8192, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8188, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7540, 0, 0, 8192, 8003, 0, 0, 2, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 8188, 0, 0, 8031, 6523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1421, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8187, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 18, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 8190, 0, 0, 7956, 4818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 1, 0, 0, 2580, 0, 0, 8110, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 8189, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 92, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6650, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 1515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5821, 950, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7268, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6808, 9841, 8898, 7242, 8203, 8199, 8191, 8154] avg=8192.0000 max=9841 max_rank=1 min=6808 min_rank=0 imbalance_ratio=1.201294 max_load_reduction=55695 max_load_reduction_pct=0.849838 moved_to_echo=32747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1407, 6523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 5707, 949, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1421, 0, 0, 7235, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7267, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 18, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9168, 7930, 7032, 8659, 8185, 8195, 8193, 8174] avg=8192.0000 max=9168 max_rank=0 min=7032 min_rank=2 imbalance_ratio=1.119141 max_load_reduction=56368 max_load_reduction_pct=0.860107 moved_to_echo=32747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5821, 950, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7268, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7823, 9697, 8024, 7242, 8203, 8200, 8193, 8154] avg=8192.0000 max=9697 max_rank=1 min=7242 min_rank=3 imbalance_ratio=1.183716 max_load_reduction=55839 max_load_reduction_pct=0.852036 moved_to_echo=32750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 922, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 377, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 1649, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2855, 0, 0, 0, 0, 0, 7, 8, 0, 0, 8, 5501, 931, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3714, 0, 0, 7105, 9, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7260, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 5, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 13, 0, 0, 0, 3, 0, 0, 177, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9487, 3007, 9348, 10841, 8154, 8172, 8189, 8338] avg=8192.0000 max=10841 max_rank=3 min=3007 min_rank=1 imbalance_ratio=1.323364 max_load_reduction=54695 max_load_reduction_pct=0.834579 moved_to_echo=32853 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 923, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 8003, 0, 0, 2, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5821, 949, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 7242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7264, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8464, 9569, 7437, 7319, 8201, 8196, 8190, 8160] avg=8192.0000 max=9569 max_rank=1 min=7319 min_rank=3 imbalance_ratio=1.168091 max_load_reduction=55967 max_load_reduction_pct=0.853989 moved_to_echo=32747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5821, 949, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7267, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7572, 9658, 8324, 7239, 8202, 8193, 8193, 8155] avg=8192.0000 max=9658 max_rank=1 min=7239 min_rank=3 imbalance_ratio=1.178955 max_load_reduction=55878 max_load_reduction_pct=0.852631 moved_to_echo=32743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5820, 950, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7268, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6901, 9630, 9014, 7242, 8203, 8198, 8194, 8154] avg=8192.0000 max=9630 max_rank=1 min=6901 min_rank=0 imbalance_ratio=1.175537 max_load_reduction=55906 max_load_reduction_pct=0.853058 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1386, 4818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 5657, 943, 39, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2580, 0, 0, 7167, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7270, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 92, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9313, 6204, 7459, 9752, 8185, 8183, 8193, 8247] avg=8192.0000 max=9752 max_rank=3 min=6204 min_rank=1 imbalance_ratio=1.190430 max_load_reduction=55784 max_load_reduction_pct=0.851196 moved_to_echo=32808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 0, 0, 0, 11776, 128, 0, 0, 0, 128, 128, 128, 0, 0, 128, 46080, 7680, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 11392, 53760, 0, 0, 128, 0, 0, 128, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 58240, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 7424, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 768, 57600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 7808, 0, 0, 57728, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 128, 384, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 65536, 0, 0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8096, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4849, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6514, 5906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6486, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4048, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7613, 8088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 2, 8, 0, 0, 0, 0, 0, 5, 697, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 1, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 0, 1, 7372, 0, 4276, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8066, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 7715, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 6829, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 4153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5197, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 1163, 0, 0, 0, 0, 2, 0, 8182, 0, 0, 40, 0, 0, 0, 0, 0, 3, 0, 0, 89, 0, 5570, 8178, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 0, 56, 0, 0, 0, 0, 0, 0, 2497, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7997, 8, 0, 0, 0, 1, 4, 36, 0, 0, 0, 0, 1, 7, 0, 8177, 0, 0, 0, 0, 0, 4, 0, 0, 0, 69, 7, 0, 0, 0, 0, 5, 0, 5, 9, 6432, 0, 7054, 0, 0, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 2471, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 16, 0, 6363, 8171, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 980, 1, 3, 0, 0, 0, 0, 0, 0, 1619, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 2, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 7026, 0, 5846, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8096, 0, 0, 0, 0, 0, 0, 2782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7652, 0, 0, 0, 0, 0, 0, 2031, 1464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5410, 3103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10878, 4205, 8189, 11147, 8513, 8178, 8152, 6274] avg=8192.0000 max=11147 max_rank=3 min=4205 min_rank=1 imbalance_ratio=1.360718 max_load_reduction=54389 max_load_reduction_pct=0.829910 moved_to_echo=31117 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4849, 0, 0, 0, 0, 0, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2876, 5906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6486, 1, 0, 0, 0, 0, 4, 2031, 1166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5406, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7628, 8782, 8199, 9688, 7879, 8180, 8175, 7005] avg=8192.0000 max=9688 max_rank=3 min=7005 min_rank=7 imbalance_ratio=1.182617 max_load_reduction=55848 max_load_reduction_pct=0.852173 moved_to_echo=31239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4048, 0, 0, 0, 0, 0, 0, 2781, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3361, 8088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 2, 8, 0, 0, 0, 0, 0, 5, 697, 0, 0, 0, 0, 0, 1827, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5410, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 1, 0, 0, 0, 10, 0, 0, 0, 0, 0, 17, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 5528, 0, 4276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6832, 11451, 8187, 4549, 8301, 8173, 8237, 9806] avg=8192.0000 max=11451 max_rank=1 min=4549 min_rank=3 imbalance_ratio=1.397827 max_load_reduction=54085 max_load_reduction_pct=0.825272 moved_to_echo=34517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7715, 0, 0, 0, 0, 0, 0, 2782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6200, 0, 0, 0, 0, 0, 0, 2031, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5410, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10497, 6178, 8189, 9696, 8514, 8178, 8058, 6226] avg=8192.0000 max=10497 max_rank=0 min=6178 min_rank=1 imbalance_ratio=1.281372 max_load_reduction=55039 max_load_reduction_pct=0.839828 moved_to_echo=30976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8066, 0, 0, 0, 0, 0, 0, 2782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3611, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5935, 0, 0, 0, 0, 0, 0, 2031, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5410, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10848, 5505, 8200, 9431, 8514, 8178, 8174, 6686] avg=8192.0000 max=10848 max_rank=0 min=5505 min_rank=1 imbalance_ratio=1.324219 max_load_reduction=54688 max_load_reduction_pct=0.834473 moved_to_echo=31552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1163, 0, 0, 0, 0, 2, 0, 2778, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 89, 0, 2460, 8178, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 0, 56, 0, 0, 0, 0, 0, 0, 2497, 0, 0, 0, 0, 0, 1594, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5404, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7984, 8, 0, 0, 0, 1, 4, 36, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 4, 0, 0, 0, 69, 7, 0, 0, 0, 0, 15, 13, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 9, 4823, 0, 7054, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3983, 10737, 8183, 6872, 7534, 8041, 8288, 11898] avg=8192.0000 max=11898 max_rank=7 min=3983 min_rank=0 imbalance_ratio=1.452393 max_load_reduction=53638 max_load_reduction_pct=0.818451 moved_to_echo=35761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6829, 0, 0, 0, 0, 0, 0, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3533, 4153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5197, 0, 0, 0, 0, 0, 2, 2031, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5409, 3037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9610, 7686, 8208, 8663, 8447, 8179, 8217, 6526] avg=8192.0000 max=9610 max_rank=0 min=6526 min_rank=7 imbalance_ratio=1.173096 max_load_reduction=55926 max_load_reduction_pct=0.853363 moved_to_echo=31369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2471, 0, 0, 0, 0, 0, 0, 2777, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 0, 2802, 8171, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1, 3, 0, 0, 0, 0, 0, 0, 1619, 0, 0, 0, 0, 0, 1744, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5412, 2418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 2, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 5264, 0, 5846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5260, 10992, 8181, 5490, 7834, 8429, 8235, 11115] avg=8192.0000 max=11115 max_rank=7 min=5260 min_rank=0 imbalance_ratio=1.356812 max_load_reduction=54421 max_load_reduction_pct=0.830399 moved_to_echo=35613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 3456, 128, 128, 0, 0, 0, 0, 0, 31488, 4864, 0, 0, 0, 0, 128, 15360, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 43392, 22272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 43264, 0, 0, 0, 0, 128, 0, 22272, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 0, 25984, 39552, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 46336, 0, 19200, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[65280, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 65280, 128, 0, 128, 0, 128, 128, 384, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5540, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 2646, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1601, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 3, 1, 0, 8191, 0, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6559, 0, 1, 1, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8173, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6767, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1417, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7224, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 968, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1272, 8164, 0, 0, 1, 0, 0, 0, 0, 6, 0, 8148, 32, 7, 0, 8181, 0, 8154, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 0, 0, 7, 0, 4, 0, 16, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 1, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 5946, 0, 4, 10, 8179, 0, 0, 0, 11, 0, 2, 2, 21, 0, 0, 1171, 0, 0, 0, 46, 0, 0, 0, 10, 6, 0, 0, 0, 0, 8052, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1951, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8188, 0, 8192, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6212, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2236, 8184, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 9, 2, 0, 8190, 0, 8183, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 5513, 0, 0, 2, 8189, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 442, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8152, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5739, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 8191, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 2436, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4190, 3951, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9696, 8189, 8266, 8188, 8148, 8194, 6648, 8207] avg=8192.0000 max=9696 max_rank=0 min=6648 min_rank=6 imbalance_ratio=1.183594 max_load_reduction=55840 max_load_reduction_pct=0.852051 moved_to_echo=31197 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 8183, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 1, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 4190, 3949, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6559, 0, 1, 1, 4001, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8173, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5754, 8188, 8138, 8195, 8158, 8189, 10725, 8189] avg=8192.0000 max=10725 max_rank=6 min=5754 min_rank=0 imbalance_ratio=1.309204 max_load_reduction=54811 max_load_reduction_pct=0.836349 moved_to_echo=35261 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4190, 3951, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10923, 8192, 8266, 8191, 8147, 8191, 5419, 8207] avg=8192.0000 max=10923 max_rank=0 min=5419 min_rank=6 imbalance_ratio=1.333374 max_load_reduction=54613 max_load_reduction_pct=0.833328 moved_to_echo=29964 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4190, 3951, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 968, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11380, 8188, 8266, 8188, 8148, 8189, 4970, 8207] avg=8192.0000 max=11380 max_rank=0 min=4970 min_rank=6 imbalance_ratio=1.389160 max_load_reduction=54156 max_load_reduction_pct=0.826355 moved_to_echo=29514 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4187, 3951, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6212, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 8198, 8265, 8210, 8147, 8187, 10216, 8207] avg=8192.0000 max=10216 max_rank=6 min=6106 min_rank=0 imbalance_ratio=1.247070 max_load_reduction=55320 max_load_reduction_pct=0.844116 moved_to_echo=34757 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1272, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 4133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 7, 0, 0, 0, 8150, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 0, 0, 7, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 16, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8157, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 1, 0, 0, 103, 0, 0, 0, 0, 0, 0, 4183, 3930, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5946, 0, 4, 10, 3996, 0, 0, 0, 11, 0, 2, 2, 21, 0, 0, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 10, 6, 0, 0, 0, 0, 8052, 1, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5412, 8197, 7940, 8184, 8319, 8191, 11163, 8130] avg=8192.0000 max=11163 max_rank=6 min=5412 min_rank=0 imbalance_ratio=1.362671 max_load_reduction=54373 max_load_reduction_pct=0.829666 moved_to_echo=35803 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4190, 3951, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2436, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9894, 8195, 8264, 8191, 8148, 8199, 6438, 8207] avg=8192.0000 max=9894 max_rank=0 min=6438 min_rank=6 imbalance_ratio=1.207764 max_load_reduction=55642 max_load_reduction_pct=0.849030 moved_to_echo=30992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2236, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 8174, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 2, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 4192, 3952, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5513, 0, 0, 2, 3997, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8152, 0, 0, 17, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6371, 8189, 8131, 8189, 8321, 8196, 9957, 8182] avg=8192.0000 max=9957 max_rank=6 min=6371 min_rank=0 imbalance_ratio=1.215454 max_load_reduction=55579 max_load_reduction_pct=0.848068 moved_to_echo=34656 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 256, 0, 0, 0, 128, 128, 0, 0, 256, 0, 0, 128, 0, 0, 0, 33536, 31616, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 32384, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 33280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[31744, 0, 128, 128, 32000, 0, 0, 0, 128, 0, 128, 128, 128, 128, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65408, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65024, 0, 0, 0, 0, 0, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 7982, 0, 8190, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8189, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 2, 0, 8067, 0, 0, 0, 0, 7940, 0, 8184, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 8185, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 64, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8177, 0, 0, 8189, 0, 0, 2, 1, 0, 0, 0, 0, 195, 0, 0, 0, 0, 61, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8185, 0, 8192, 0, 0, 4, 0, 0, 0, 8182, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 19, 0, 0, 8179, 0, 0, 0, 41, 0, 0, 101, 0, 8, 0, 0, 1, 0, 0, 0, 7927, 0, 0, 0, 0, 7668, 0, 8134, 0, 0, 0, 5, 6, 0, 8178, 0, 5, 0, 0, 8131, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 40, 7, 0, 0, 0, 0, 0, 0, 0, 7, 8114, 0, 13, 8181, 0, 0, 5, 2, 0, 0, 0, 0, 432, 0, 0, 0, 4, 77, 0, 0, 8, 0, 0, 0, 0, 9, 0, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8187, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8185, 0, 8192, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8189, 0, 8192, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8187, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 7982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 8176, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 7984, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8188, 8094, 8346, 8184, 8199, 8137, 8192] avg=8192.0000 max=8346 max_rank=3 min=8094 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8187, 8300, 8141, 8178, 8200, 8139, 8195] avg=8192.0000 max=8300 max_rank=2 min=8139 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8189, 8297, 8144, 8174, 8206, 8139, 8185] avg=8192.0000 max=8297 max_rank=2 min=8139 min_rank=6 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8184, 8303, 8141, 8178, 8200, 8139, 8195] avg=8192.0000 max=8303 max_rank=2 min=8139 min_rank=6 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8131, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 8171, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 8, 0, 0, 1, 0, 0, 0, 107, 0, 0, 0, 0, 7668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 0, 0, 5, 0, 0, 8080, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 40, 7, 0, 0, 0, 0, 8166, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8114, 0, 13, 8, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 4, 77, 0, 0, 8, 0, 0, 0, 0, 9, 0, 9, 7819, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8178, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8231, 7885, 8291, 8220, 8149, 8417, 8189] avg=8192.0000 max=8417 max_rank=6 min=7885 min_rank=2 imbalance_ratio=1.027466 max_load_reduction=57119 max_load_reduction_pct=0.871567 moved_to_echo=32975 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8187, 8302, 8142, 8177, 8200, 8139, 8192] avg=8192.0000 max=8302 max_rank=2 min=8139 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32708 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 2, 0, 100, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 64, 2, 0, 0, 0, 0, 8175, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8177, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 61, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 7960, 55, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8181, 8055, 8188, 8249, 8182, 8288, 8194] avg=8192.0000 max=8288 max_rank=6 min=8055 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32913 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8189, 8300, 8143, 8176, 8200, 8138, 8194] avg=8192.0000 max=8300 max_rank=2 min=8138 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32708 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[768, 0, 0, 0, 128, 256, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64256, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 128, 0, 128, 0, 896, 0, 0, 0, 0, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 0, 0, 128, 128, 0, 65152, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 82, 0, 0, 1, 6, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 6580, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 876, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1158, 7648, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 8179, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 130, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 167, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 8137, 0, 0, 543, 0, 0, 9, 31, 508, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4, 0, 0, 3, 0, 7, 1, 9, 0, 0, 5150, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2894, 0, 0, 0, 0, 0, 0, 8176, 108, 0, 394, 7573, 0, 0, 5, 8, 0, 1, 0, 0, 8166, 0, 8, 0, 6, 0, 0, 8060, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 7452, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 59, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 114, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 210, 0, 0, 2, 4, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 7189, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 8189, 19, 0, 267, 7654, 1, 0, 0, 2, 0, 2, 0, 0, 8188, 0, 2, 0, 0, 0, 0, 8094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 92, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 144, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7956, 8190, 8888, 7590, 8236, 8222, 8260, 8194] avg=8192.0000 max=8888 max_rank=2 min=7590 min_rank=3 imbalance_ratio=1.084961 max_load_reduction=56648 max_load_reduction_pct=0.864380 moved_to_echo=32912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7955, 0, 0, 82, 0, 0, 1, 6, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6580, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 602, 92, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 876, 0, 0, 0, 0, 0, 0, 7588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 7597, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 144, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8192, 7278, 8464, 8757, 8214, 8254, 8194] avg=8192.0000 max=8757 max_rank=4 min=7278 min_rank=2 imbalance_ratio=1.068970 max_load_reduction=56779 max_load_reduction_pct=0.866379 moved_to_echo=33419 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 92, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 144, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7956, 8191, 8887, 7590, 8164, 8222, 8332, 8194] avg=8192.0000 max=8887 max_rank=2 min=7590 min_rank=3 imbalance_ratio=1.084839 max_load_reduction=56649 max_load_reduction_pct=0.864395 moved_to_echo=32912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 92, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 144, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7956, 8191, 8887, 7590, 8253, 8222, 8243, 8194] avg=8192.0000 max=8887 max_rank=2 min=7590 min_rank=3 imbalance_ratio=1.084839 max_load_reduction=56649 max_load_reduction_pct=0.864395 moved_to_echo=32912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7956, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 92, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 144, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7967, 8190, 8887, 7590, 8153, 8211, 8345, 8193] avg=8192.0000 max=8887 max_rank=2 min=7590 min_rank=3 imbalance_ratio=1.084839 max_load_reduction=56649 max_load_reduction_pct=0.864395 moved_to_echo=32902 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 7903, 0, 0, 543, 0, 0, 9, 31, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4, 0, 0, 3, 0, 7, 1, 9, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5150, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 601, 91, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2894, 0, 0, 0, 0, 0, 0, 7575, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 7523, 0, 0, 5, 8, 0, 1, 0, 0, 0, 0, 8, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 7452, 0, 143, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9001, 8198, 5908, 10582, 7945, 8097, 7629, 8176] avg=8192.0000 max=10582 max_rank=3 min=5908 min_rank=2 imbalance_ratio=1.291748 max_load_reduction=54954 max_load_reduction_pct=0.838531 moved_to_echo=31847 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 602, 92, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 144, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7957, 8191, 8889, 7590, 8162, 8221, 8333, 8193] avg=8192.0000 max=8889 max_rank=2 min=7590 min_rank=3 imbalance_ratio=1.085083 max_load_reduction=56647 max_load_reduction_pct=0.864365 moved_to_echo=32909 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7945, 0, 0, 210, 0, 0, 2, 4, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7189, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 609, 93, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 7580, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 7592, 1, 0, 0, 2, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7967, 0, 148, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8560, 8193, 7912, 8540, 7866, 8127, 8140, 8198] avg=8192.0000 max=8560 max_rank=0 min=7866 min_rank=4 imbalance_ratio=1.044922 max_load_reduction=56976 max_load_reduction_pct=0.869385 moved_to_echo=32331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[59904, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 4864, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 4736, 0, 0, 0, 0, 0, 0, 60800, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 63616, 0, 0, 896, 0, 0, 128, 128, 1152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 64256, 0, 1280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2432, 63232, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 0, 0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True + [2026-08-26 16:28:33] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 438.3 | throughput per GPU (TFLOP/s/GPU): 412.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.552851E+00 | loss scale: 1.0 | grad norm: 5.413 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2686, 0, 0, 2996, 0, 0, 0, 8192, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 0, 0, 0, 4189, 0, 4149, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7988, 0, 3, 0, 0, 0, 0, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7527, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 2899, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 2067, 0, 1, 2353, 0, 0, 0, 8191, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 3679, 0, 0, 0, 4102, 0, 4592, 19, 0, 0, 4569, 1, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 2, 0, 0, 0, 0, 844, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 7661, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 131, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 18, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 2393, 0, 1, 2569, 0, 0, 0, 8192, 0, 8190, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 3392, 0, 0, 0, 4242, 0, 4651, 6, 2, 0, 4895, 0, 0, 0, 0, 0, 1, 0, 0, 8072, 0, 1, 0, 1, 0, 0, 868, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 7523, 0, 0, 9, 0, 0, 0, 1, 0, 4, 0, 0, 4, 0, 0, 0, 0, 106, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 2, 1706, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 0, 1, 3714, 6, 0, 0, 8183, 0, 8184, 1, 0, 0, 2, 1, 0, 0, 2, 0, 3, 2, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 2411, 1, 1, 0, 4718, 0, 4581, 2, 0, 0, 3776, 2, 0, 0, 1, 0, 2, 1, 0, 8120, 1, 8, 0, 0, 0, 1, 956, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 7860, 0, 0, 1, 0, 0, 0, 7, 0, 3, 0, 0, 2, 0, 0, 0, 1, 41, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 0, 0, 2119, 2, 0, 0, 8192, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2960, 0, 0, 0, 4396, 0, 5505, 10, 0, 0, 4609, 0, 0, 0, 0, 0, 1, 0, 0, 8023, 0, 6, 0, 0, 2, 0, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7650, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 7, 7, 2697, 0, 0, 1, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2641, 0, 0, 2632, 1, 2, 0, 8191, 0, 8192, 2, 1, 0, 2, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 3265, 0, 0, 0, 4525, 0, 4651, 29, 0, 1, 3905, 0, 0, 0, 0, 0, 1, 0, 0, 7946, 0, 4, 0, 6, 1, 0, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7677, 0, 0, 4, 0, 1, 0, 16, 0, 3, 0, 10, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1562, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2768, 0, 0, 2792, 2, 0, 0, 8192, 0, 8192, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2721, 0, 0, 0, 4779, 0, 5396, 12, 1, 0, 4299, 0, 0, 0, 0, 0, 0, 0, 0, 8018, 0, 5, 1, 1, 0, 0, 865, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7589, 0, 0, 2, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2080, 0, 0, 0, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2639, 0, 0, 2956, 2, 0, 0, 8192, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2640, 0, 6, 0, 4351, 0, 5311, 4, 0, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 17, 0, 0, 2, 0, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7729, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 249, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 2899, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 3259, 2107, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 1, 2353, 0, 0, 0, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3679, 0, 0, 0, 843, 0, 0, 19, 0, 0, 4569, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7235, 0, 2, 0, 0, 0, 0, 844, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 7661, 0, 0, 4, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 131, 0, 0, 0, 0, 5384, 2485, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8526, 7227, 8187, 8191, 9111, 8083, 8200, 8011] avg=8192.0000 max=9111 max_rank=4 min=7227 min_rank=1 imbalance_ratio=1.112183 max_load_reduction=56425 max_load_reduction_pct=0.860977 moved_to_echo=33405 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 1904, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2686, 0, 0, 2996, 0, 0, 0, 0, 0, 2807, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 4, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 0, 0, 0, 861, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7202, 0, 3, 0, 0, 0, 0, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7527, 0, 0, 3, 0, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 5385, 2245, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8498, 8197, 8195, 8110, 8509, 8061, 7893] avg=8192.0000 max=8509 max_rank=5 min=7893 min_rank=7 imbalance_ratio=1.038696 max_load_reduction=57027 max_load_reduction_pct=0.870163 moved_to_echo=32573 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 2, 1706, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3748, 2103, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 0, 1, 3714, 6, 0, 0, 0, 0, 2804, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 2, 0, 3, 2, 0, 0, 0, 1, 1, 0, 1, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 8179, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2411, 1, 1, 0, 970, 0, 0, 2, 0, 0, 3776, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 0, 7322, 1, 8, 0, 0, 0, 1, 956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 7860, 0, 0, 1, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 2, 0, 0, 0, 1, 41, 0, 0, 0, 0, 5380, 2478, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7822, 9565, 8189, 8184, 7163, 8292, 8402, 7919] avg=8192.0000 max=9565 max_rank=1 min=7163 min_rank=4 imbalance_ratio=1.167603 max_load_reduction=55971 max_load_reduction_pct=0.854050 moved_to_echo=31776 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 18, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3370, 2135, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 2393, 0, 1, 2569, 0, 0, 0, 0, 0, 2806, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 0, 0, 0, 872, 0, 0, 6, 2, 0, 4895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7278, 0, 1, 0, 1, 0, 0, 868, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 7523, 0, 0, 9, 0, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 4, 0, 0, 0, 0, 106, 0, 0, 0, 0, 5384, 2516, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7971, 7774, 8192, 8190, 9167, 8150, 8070, 8022] avg=8192.0000 max=9167 max_rank=4 min=7774 min_rank=1 imbalance_ratio=1.119019 max_load_reduction=56369 max_load_reduction_pct=0.860123 moved_to_echo=33409 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3493, 2525, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 0, 0, 2119, 2, 0, 0, 0, 0, 2807, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2960, 0, 0, 0, 903, 0, 0, 10, 0, 0, 4609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7234, 0, 6, 0, 0, 2, 0, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7650, 0, 0, 4, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 5385, 2980, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 7607, 8189, 8191, 8482, 8131, 8187, 8641] avg=8192.0000 max=8641 max_rank=7 min=7607 min_rank=1 imbalance_ratio=1.054810 max_load_reduction=56895 max_load_reduction_pct=0.868149 moved_to_echo=33441 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 7, 7, 2697, 0, 0, 1, 2, 0, 0, 0, 0, 2, 0, 3595, 2134, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 0, 2632, 1, 2, 0, 0, 0, 2807, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3265, 0, 0, 0, 930, 0, 0, 29, 0, 1, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7164, 0, 4, 0, 6, 1, 0, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7677, 0, 0, 4, 0, 1, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3, 0, 10, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 5385, 2517, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8698, 8086, 8190, 8196, 8130, 7947, 8210, 8079] avg=8192.0000 max=8698 max_rank=0 min=7947 min_rank=5 imbalance_ratio=1.061768 max_load_reduction=56838 max_load_reduction_pct=0.867279 moved_to_echo=32366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1562, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3797, 2475, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2768, 0, 0, 2792, 2, 0, 0, 0, 0, 2807, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2721, 0, 0, 0, 982, 0, 0, 12, 1, 0, 4299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7229, 0, 5, 1, 1, 0, 0, 865, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7589, 0, 0, 2, 1, 1, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 5385, 2921, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8093, 8370, 8188, 8193, 8015, 8103, 8127, 8447] avg=8192.0000 max=8447 max_rank=7 min=8015 min_rank=4 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2080, 0, 0, 0, 2, 0, 0, 0, 1, 1, 0, 3461, 2437, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2639, 0, 0, 2956, 2, 0, 0, 0, 0, 2800, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 14, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2640, 0, 6, 0, 890, 0, 0, 4, 0, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7276, 0, 17, 0, 0, 2, 0, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7729, 0, 0, 2, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 249, 0, 0, 0, 0, 5392, 2874, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8409, 8204, 8196, 7358, 8321, 8279, 8524] avg=8192.0000 max=8524 max_rank=7 min=7358 min_rank=4 imbalance_ratio=1.040527 max_load_reduction=57012 max_load_reduction_pct=0.869934 moved_to_echo=32482 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 20992, 0, 128, 22144, 128, 128, 0, 0, 0, 22528, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 0, 128, 128, 17664, 128, 128, 128, 128, 0, 0, 128, 128, 128, 128, 28160, 17920, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 24320, 128, 128, 0, 7296, 0, 0, 128, 128, 128, 34048, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 0, 0, 128, 128, 1408, 0, 0, 0, 0, 43136, 21120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 57984, 128, 128, 128, 128, 128, 128, 7552, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 128, 0, 128, 61312, 0, 0, 128, 128, 128, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 0, 128, 128, 0, 128, 128, 128, 128, 65536, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 56, 0, 0, 0, 7774, 0, 6337, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 2, 1, 0, 0, 8017, 1, 6769, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 8190, 1, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 1, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 8121, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 0, 7635, 0, 5890, 0, 0, 18, 0, 0, 3, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8017, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 233, 0, 0, 0, 7811, 0, 6080, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7282, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[28, 0, 253, 2, 0, 0, 6755, 0, 2709, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 8190, 0, 0, 0, 88, 0, 2, 0, 0, 1, 0, 0, 0, 4627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 8188, 0, 0, 0, 0, 0, 2, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 2, 0, 0, 7546, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[12, 0, 31, 0, 0, 0, 7115, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7362, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8, 0, 8, 0, 0, 0, 8062, 0, 7927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7935, 0, 7189, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 8167, 8192, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7894, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 2, 1, 0, 0, 2070, 1, 6769, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8093, 0, 1, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5633, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 38, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 7286, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 8121, 0, 0, 1, 0, 314, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8847, 8239, 7749, 8184, 8196, 7602, 8189, 8530] avg=8192.0000 max=8847 max_rank=0 min=7602 min_rank=5 imbalance_ratio=1.079956 max_load_reduction=56689 max_load_reduction_pct=0.865005 moved_to_echo=32517 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 56, 0, 0, 0, 2008, 0, 6337, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5462, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 7287, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 304, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8404, 8279, 8232, 8181, 8195, 7729, 8190, 8326] avg=8192.0000 max=8404 max_rank=0 min=7729 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=57132 max_load_reduction_pct=0.871765 moved_to_echo=32440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 0, 1972, 0, 5890, 0, 0, 18, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5364, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 7287, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8017, 0, 0, 0, 0, 299, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7896, 8249, 8569, 8228, 8192, 7802, 8192, 8408] avg=8192.0000 max=8569 max_rank=2 min=7802 min_rank=5 imbalance_ratio=1.046021 max_load_reduction=56967 max_load_reduction_pct=0.869247 moved_to_echo=32594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 253, 2, 0, 0, 1745, 0, 2709, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8094, 0, 0, 0, 0, 88, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4746, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 38, 0, 0, 0, 0, 0, 2, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 2596, 0, 0, 0, 7283, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 2, 0, 0, 7546, 0, 0, 0, 0, 264, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4738, 8187, 10240, 8194, 8190, 9885, 8187, 7915] avg=8192.0000 max=10240 max_rank=2 min=4738 min_rank=0 imbalance_ratio=1.250000 max_load_reduction=55296 max_load_reduction_pct=0.843750 moved_to_echo=34177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 8, 0, 0, 0, 2083, 0, 7927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5664, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 7287, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 315, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10026, 8106, 6776, 8215, 8196, 7443, 8189, 8585] avg=8192.0000 max=10026 max_rank=0 min=6776 min_rank=2 imbalance_ratio=1.223877 max_load_reduction=55510 max_load_reduction_pct=0.847015 moved_to_echo=32413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 233, 0, 0, 0, 2017, 0, 6080, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5488, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 610, 0, 0, 0, 7284, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7282, 0, 0, 0, 0, 306, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8144, 8921, 8121, 8186, 7899, 8202, 7725] avg=8192.0000 max=8921 max_rank=2 min=7725 min_rank=7 imbalance_ratio=1.088989 max_load_reduction=56615 max_load_reduction_pct=0.863876 moved_to_echo=32012 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 31, 0, 0, 0, 1838, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4999, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1912, 0, 0, 0, 7283, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7362, 0, 0, 0, 0, 278, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 8244, 7726, 8194, 8183, 9201, 8198, 7742] avg=8192.0000 max=9201 max_rank=5 min=7726 min_rank=2 imbalance_ratio=1.123169 max_load_reduction=56335 max_load_reduction_pct=0.859604 moved_to_echo=33324 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2040, 0, 7189, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5580, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 0, 0, 0, 7292, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7894, 0, 0, 0, 0, 315, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9239, 8088, 7323, 8219, 8198, 7975, 8189, 8305] avg=8192.0000 max=9239 max_rank=0 min=7323 min_rank=2 imbalance_ratio=1.127808 max_load_reduction=56297 max_load_reduction_pct=0.859024 moved_to_echo=32667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43008, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 640, 128, 0, 0, 15872, 128, 49152, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 7296, 0, 0, 0, 58368, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 128, 0, 0, 62336, 0, 0, 128, 0, 2432, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 64768, 0, 128, 0, 0, 896, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[65280, 384, 128, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 238, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 1, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8191, 0, 1, 0, 0, 1, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8086, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 177, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 3, 8171, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 8189, 0, 2, 1, 0, 0, 0, 3, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8141, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8015, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 139, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 37, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8381, 8191, 8022, 8140, 8191, 8225] avg=8192.0000 max=8381 max_rank=2 min=8022 min_rank=4 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32578 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 139, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 3, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8142, 8193, 8168, 8261, 8193, 8197] avg=8192.0000 max=8261 max_rank=5 min=8142 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32819 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 139, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 1, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8142, 8193, 8047, 8356, 8192, 8224] avg=8192.0000 max=8356 max_rank=5 min=8047 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32819 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8171, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8013, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 139, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 37, 0, 2, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8190, 8312, 8184, 8021, 8313, 8190, 8136] avg=8192.0000 max=8313 max_rank=5 min=8021 min_rank=4 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7907, 0, 139, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8192, 8142, 8193, 8290, 8095, 8192, 8230] avg=8192.0000 max=8290 max_rank=4 min=8095 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32807 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8013, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7969, 0, 139, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8194, 8142, 8193, 8345, 8157, 8194, 8124] avg=8192.0000 max=8345 max_rank=4 min=8124 min_rank=7 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32820 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 0, 139, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8193, 8142, 8193, 8610, 7842, 8192, 8175] avg=8192.0000 max=8610 max_rank=4 min=7842 min_rank=5 imbalance_ratio=1.051025 max_load_reduction=56926 max_load_reduction_pct=0.868622 moved_to_echo=32819 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8017, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 142, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8191, 8133, 8196, 8033, 8372, 8192, 8225] avg=8192.0000 max=8372 max_rank=5 min=8033 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32822 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64128, 0, 1152, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 128, 0, 64128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65152, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[65280, 0, 128, 0, 384, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 65536, 0, 128, 128, 0, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 469, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 1, 0, 0, 0, 0, 8189, 0, 0, 0, 5876, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7138, 0, 0, 0, 33, 8191, 7427, 0, 0, 0, 1, 0, 8145, 8185, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 7, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 7708, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 8186, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 22, 8192, 8158, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 39, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 8172, 0, 0, 0, 3472, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 1, 94, 8188, 6786, 0, 0, 0, 3, 0, 8186, 8145, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 7, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 7631, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 8188, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 269, 0, 0, 0, 1, 8192, 8184, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 18, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 7522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 1, 8192, 8192, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 469, 0, 0, 1, 1, 0, 0, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 1, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5876, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 855, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7138, 6082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 7427, 0, 0, 0, 1, 0, 253, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 5075, 8189, 8198, 8190, 6756, 13221, 7715] avg=8192.0000 max=13221 max_rank=6 min=5075 min_rank=1 imbalance_ratio=1.613892 max_load_reduction=52315 max_load_reduction_pct=0.798264 moved_to_echo=35882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 3, 0, 0, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 0, 8145, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3472, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 2, 0, 859, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 94, 0, 6786, 0, 0, 0, 3, 0, 255, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 7110, 8154, 8224, 8161, 4370, 14186, 7142] avg=8192.0000 max=14186 max_rank=6 min=4370 min_rank=5 imbalance_ratio=1.731689 max_load_reduction=51350 max_load_reduction_pct=0.783539 moved_to_echo=33859 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 7708, 0, 0, 0, 0, 0, 0, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 6117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8158, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8825, 8200, 8186, 8172, 9080, 6444, 8435] avg=8192.0000 max=9080 max_rank=5 min=6444 min_rank=6 imbalance_ratio=1.108398 max_load_reduction=56456 max_load_reduction_pct=0.861450 moved_to_echo=32131 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 6117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 9075, 8207, 8186, 8181, 9050, 6191, 8454] avg=8192.0000 max=9075 max_rank=1 min=6191 min_rank=6 imbalance_ratio=1.107788 max_load_reduction=56461 max_load_reduction_pct=0.861526 moved_to_echo=31876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 7631, 0, 0, 0, 0, 0, 0, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 269, 6117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8184, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8864, 8200, 8186, 8191, 9073, 6389, 8440] avg=8192.0000 max=9073 max_rank=5 min=6389 min_rank=6 imbalance_ratio=1.107544 max_load_reduction=56463 max_load_reduction_pct=0.861557 moved_to_echo=32093 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 6121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8192, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 9020, 8198, 8184, 8201, 9078, 6215, 8448] avg=8192.0000 max=9078 max_rank=5 min=6215 min_rank=6 imbalance_ratio=1.108154 max_load_reduction=56458 max_load_reduction_pct=0.861481 moved_to_echo=31942 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 7522, 0, 0, 0, 0, 0, 0, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 6116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8533, 8194, 8186, 8249, 9064, 6670, 8448] avg=8192.0000 max=9064 max_rank=5 min=6670 min_rank=6 imbalance_ratio=1.106445 max_load_reduction=56472 max_load_reduction_pct=0.861694 moved_to_echo=32431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 6117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 9034, 8194, 8186, 8191, 9065, 6220, 8454] avg=8192.0000 max=9065 max_rank=5 min=6220 min_rank=6 imbalance_ratio=1.106567 max_load_reduction=56471 max_load_reduction_pct=0.861679 moved_to_echo=31930 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 16640, 49024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 10496, 0, 0, 0, 0, 0, 0, 47488, 0, 0, 128, 128, 0, 0, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 58496, 128, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 6912, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 256, 0, 63360, 0, 0, 0, 128, 0, 2048, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 256, 0, 128, 128, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 9, 0, 1, 0, 6632, 1, 0, 0, 0, 0, 0, 6511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 6681, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5276, 7580, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 30, 0, 1, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 0, 0, 0, 0, 7158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1435, 0, 0, 1, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7932, 0, 0, 0, 0, 0, 0, 7697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 798, 0, 0, 12, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8038, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 12, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7876, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7772, 0, 0, 0, 0, 0, 0, 7165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 0, 2, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 0, 0, 2, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 114, 0, 0, 0, 6244, 4, 0, 0, 0, 0, 0, 5492, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7923, 6217, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7823, 6762, 0, 0, 0, 0, 0, 0, 0, 3, 27, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 143, 0, 3, 0, 0, 0, 0, 0, 0, 114, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 9, 0, 8169, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 912, 1, 0, 0, 0, 0, 0, 6511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 6681, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5276, 5600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 30, 0, 1, 0, 0, 0, 0, 7747, 367, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5353, 1980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8185, 7425, 7137, 10876, 8201, 8167, 7352] avg=8192.0000 max=10876 max_rank=4 min=7137 min_rank=3 imbalance_ratio=1.327637 max_load_reduction=54660 max_load_reduction_pct=0.834045 moved_to_echo=34596 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 7697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 798, 0, 0, 12, 6009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7769, 439, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6402, 2125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8176, 8788, 8615, 6819, 8187, 8226, 8530] avg=8192.0000 max=8788 max_rank=2 min=6819 min_rank=4 imbalance_ratio=1.072754 max_load_reduction=56748 max_load_reduction_pct=0.865906 moved_to_echo=31762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 0, 0, 0, 0, 0, 0, 7158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1435, 0, 0, 1, 6031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7758, 433, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6315, 2133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8176, 8234, 8615, 7467, 8187, 8209, 8458] avg=8192.0000 max=8615 max_rank=3 min=7467 min_rank=4 imbalance_ratio=1.051636 max_load_reduction=56921 max_load_reduction_pct=0.868546 moved_to_echo=32321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 114, 0, 8164, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 4, 0, 0, 0, 0, 0, 5492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 409, 6217, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7823, 4996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 27, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 143, 0, 3, 0, 0, 0, 0, 7514, 345, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 5040, 1766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8286, 6355, 6712, 12820, 8214, 8029, 6928] avg=8192.0000 max=12820 max_rank=4 min=6355 min_rank=2 imbalance_ratio=1.564941 max_load_reduction=52716 max_load_reduction_pct=0.804382 moved_to_echo=35991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8172, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 12, 6051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7769, 445, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6488, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8178, 9275, 8615, 6227, 8187, 8234, 8628] avg=8192.0000 max=9275 max_rank=2 min=6227 min_rank=4 imbalance_ratio=1.132202 max_load_reduction=56261 max_load_reduction_pct=0.858475 moved_to_echo=31276 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 0, 0, 2, 6042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7769, 443, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6461, 2136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8179, 8588, 8615, 6922, 8187, 8230, 8623] avg=8192.0000 max=8623 max_rank=7 min=6922 min_rank=4 imbalance_ratio=1.052612 max_load_reduction=56913 max_load_reduction_pct=0.868423 moved_to_echo=31962 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 0, 0, 0, 0, 0, 0, 7165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 0, 2, 5997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7764, 430, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6273, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8176, 8234, 8615, 7519, 8187, 8212, 8403] avg=8192.0000 max=8615 max_rank=3 min=7519 min_rank=4 imbalance_ratio=1.051636 max_load_reduction=56921 max_load_reduction_pct=0.868546 moved_to_echo=32321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 0, 0, 6047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7772, 439, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6362, 2145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8180, 8637, 8612, 6886, 8186, 8229, 8614] avg=8192.0000 max=8637 max_rank=2 min=6886 min_rank=4 imbalance_ratio=1.054321 max_load_reduction=56899 max_load_reduction_pct=0.868210 moved_to_echo=31915 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 384, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 48768, 16640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 5760, 0, 0, 13184, 46848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 8320, 128, 0, 0, 0, 0, 0, 57344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 0, 128, 128, 128, 128, 256, 0, 128, 0, 0, 0, 0, 61952, 3456, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 3456, 62080, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 152, 0, 0, 0, 2, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 899, 0, 0, 0, 0, 4579, 0, 0, 0, 3, 0, 0, 2265, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 1, 0, 0, 0, 0, 41, 0, 4, 0, 8092, 0, 0, 2, 0, 8128, 0, 0, 0, 3, 304, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 8188, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 75, 0, 1, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1272, 0, 0, 0, 0, 937, 0, 1, 0, 1, 0, 0, 5849, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 8179, 0, 0, 1, 0, 8184, 0, 0, 0, 1, 36, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 8190, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6945, 0, 0, 0, 0, 0, 7812, 0, 0, 0, 0, 0, 0, 34, 0, 0, 1327, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6436, 0, 0, 0, 0, 0, 7265, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2370, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7984, 0, 0, 0, 0, 0, 7428, 0, 0, 0, 0, 0, 0, 3, 0, 0, 733, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7116, 0, 0, 0, 0, 0, 7458, 0, 0, 0, 0, 0, 17, 3, 0, 0, 1476, 0, 0, 0, 8192, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7684, 0, 0, 0, 0, 0, 7282, 0, 0, 0, 0, 0, 21, 40, 0, 0, 965, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5781, 0, 0, 0, 0, 0, 7498, 0, 0, 0, 0, 0, 108, 17, 0, 0, 2681, 0, 0, 0, 8192, 0, 0, 0, 13, 8191, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 75, 0, 1, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 0, 0, 0, 0, 937, 0, 1, 0, 1, 0, 0, 5849, 0, 0, 692, 220, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7347, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 1, 0, 0, 0, 0, 0, 1, 36, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8379, 8195, 9029, 7353, 7986, 8193, 8193, 8208] avg=8192.0000 max=9029 max_rank=2 min=7353 min_rank=3 imbalance_ratio=1.102173 max_load_reduction=56507 max_load_reduction_pct=0.862228 moved_to_echo=32580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7984, 0, 0, 639, 220, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6789, 0, 0, 0, 0, 0, 0, 3, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8190, 9028, 7525, 8070, 8192, 8195, 8203] avg=8192.0000 max=9028 max_rank=2 min=7525 min_rank=3 imbalance_ratio=1.102051 max_load_reduction=56508 max_load_reduction_pct=0.862244 moved_to_echo=32660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8190, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6945, 0, 0, 672, 220, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7140, 0, 0, 0, 0, 0, 0, 34, 0, 0, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8194, 7913, 8501, 8236, 8191, 8194, 8177] avg=8192.0000 max=8501 max_rank=3 min=7913 min_rank=2 imbalance_ratio=1.037720 max_load_reduction=57035 max_load_reduction_pct=0.870285 moved_to_echo=32798 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6436, 0, 0, 625, 220, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6640, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8191, 7371, 9015, 8236, 8192, 8195, 8205] avg=8192.0000 max=9015 max_rank=3 min=7371 min_rank=2 imbalance_ratio=1.100464 max_load_reduction=56521 max_load_reduction_pct=0.862442 moved_to_echo=32828 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 152, 0, 0, 0, 2, 0, 0, 0, 0, 0, 144, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, 0, 0, 0, 4579, 0, 0, 0, 3, 0, 0, 2265, 0, 0, 701, 217, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7448, 0, 0, 0, 0, 1, 0, 0, 0, 0, 41, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7860, 0, 0, 2, 0, 0, 0, 0, 0, 3, 304, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 8195, 8720, 7494, 8173, 8193, 8191, 8203] avg=8192.0000 max=8720 max_rank=2 min=7494 min_rank=3 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7116, 0, 0, 642, 220, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6816, 0, 0, 0, 0, 0, 17, 3, 0, 0, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 7, 0, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8187, 8039, 8312, 8299, 8192, 8187, 8188] avg=8192.0000 max=8312 max_rank=3 min=8039 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32866 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5781, 0, 0, 645, 220, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6853, 0, 0, 0, 0, 0, 108, 17, 0, 0, 2681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 13, 0, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8190, 6714, 9659, 8305, 8192, 8189, 8155] avg=8192.0000 max=9659 max_rank=3 min=6714 min_rank=2 imbalance_ratio=1.179077 max_load_reduction=55877 max_load_reduction_pct=0.852615 moved_to_echo=32841 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7684, 0, 0, 631, 224, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6651, 0, 0, 0, 0, 0, 21, 40, 0, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8194, 8722, 7677, 8231, 8191, 8192, 8197] avg=8192.0000 max=8722 max_rank=2 min=7677 min_rank=3 imbalance_ratio=1.064697 max_load_reduction=56814 max_load_reduction_pct=0.866913 moved_to_echo=32811 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 2560, 0, 128, 0, 0, 5632, 0, 128, 0, 128, 0, 0, 50176, 0, 0, 5248, 1792, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 55808, 0, 0, 0, 0, 128, 256, 128, 0, 0, 9600, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 63616, 0, 0, 128, 128, 0, 0, 0, 0, 128, 2048, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 256, 0, 128, 0, 128, 0, 0, 0, 0, 0, 384, 0, 0, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 65408, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 7, 1, 147, 8190, 0, 8185, 1, 0, 0, 2, 0, 3803, 0, 0, 0, 0, 3426, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 2, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 498, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 142] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 42, 2, 54, 8188, 0, 8183, 0, 3, 0, 3, 0, 1626, 0, 0, 0, 0, 4223, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 3, 4, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 1203, 3, 0, 0, 1, 0, 2, 1, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 973, 7, 0, 0, 0, 128] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 1851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6384, 0, 0, 0, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 8192, 0, 0, 0, 0, 0, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8017, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4510, 0, 0, 0, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8045, 0, 8192, 0, 0, 0, 0, 0, 3820, 0, 0, 4, 0, 0, 0, 0, 0, 0, 43, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 7840, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4680, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8192, 0, 0, 0, 0, 0, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7100, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 8192, 0, 0, 0, 0, 0, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7474, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 8192, 0, 0, 0, 0, 0, 2075, 0, 0, 3, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6198, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 147, 5820, 0, 0, 1, 0, 0, 2, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3426, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 2, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8023, 0, 0, 498, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 142, 2370, 966, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 9781, 10646, 8196, 8116, 8523, 8195, 3876] avg=8192.0000 max=10646 max_rank=2 min=3876 min_rank=7 imbalance_ratio=1.299561 max_load_reduction=54890 max_load_reduction_pct=0.837555 moved_to_echo=28710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 2, 54, 5819, 0, 0, 0, 3, 0, 3, 0, 1626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4223, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 3, 4, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8021, 0, 0, 1203, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 973, 7, 0, 0, 0, 128, 2369, 965, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7549, 11444, 8199, 8149, 9229, 8191, 4577] avg=8192.0000 max=11444 max_rank=2 min=4577 min_rank=7 imbalance_ratio=1.396973 max_load_reduction=54092 max_load_reduction_pct=0.825378 moved_to_echo=30146 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5817, 0, 0, 0, 0, 0, 0, 0, 1851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7974, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6384, 0, 0, 0, 0, 6, 2369, 966, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 7668, 7227, 8191, 8213, 8002, 8196, 9857] avg=8192.0000 max=9857 max_rank=7 min=7227 min_rank=2 imbalance_ratio=1.203247 max_load_reduction=55679 max_load_reduction_pct=0.849594 moved_to_echo=34268 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5782, 0, 0, 0, 0, 0, 0, 0, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 7226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7853, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4510, 0, 0, 0, 0, 6, 2354, 966, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 9616, 7231, 8191, 8213, 7938, 8196, 7966] avg=8192.0000 max=9616 max_rank=1 min=7231 min_rank=2 imbalance_ratio=1.173828 max_load_reduction=55920 max_load_reduction_pct=0.853271 moved_to_echo=32313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5813, 0, 0, 0, 0, 0, 0, 0, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7986, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7100, 0, 0, 0, 0, 0, 2367, 966, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 6926, 7230, 8191, 8213, 8015, 8196, 10565] avg=8192.0000 max=10565 max_rank=7 min=6926 min_rank=1 imbalance_ratio=1.289673 max_load_reduction=54971 max_load_reduction_pct=0.838791 moved_to_echo=34989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5717, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 43, 0, 0, 1, 0, 0, 0, 3, 7226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7681, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4680, 0, 0, 0, 0, 0, 2328, 966, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 9537, 7277, 8190, 8205, 7848, 8177, 8101] avg=8192.0000 max=9537 max_rank=1 min=7277 min_rank=2 imbalance_ratio=1.164185 max_load_reduction=55999 max_load_reduction_pct=0.854477 moved_to_echo=32331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5795, 0, 0, 0, 0, 0, 0, 0, 2075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 2, 7226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6198, 0, 0, 0, 0, 0, 2360, 966, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7870, 7254, 8190, 8210, 7983, 8182, 9655] avg=8192.0000 max=9655 max_rank=7 min=7254 min_rank=2 imbalance_ratio=1.178589 max_load_reduction=55881 max_load_reduction_pct=0.852676 moved_to_echo=34030 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5791, 0, 0, 0, 0, 0, 0, 0, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7902, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7474, 0, 0, 0, 0, 2, 2361, 965, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 6589, 7227, 8188, 8217, 7998, 8203, 10939] avg=8192.0000 max=10939 max_rank=7 min=6589 min_rank=1 imbalance_ratio=1.335327 max_load_reduction=54597 max_load_reduction_pct=0.833084 moved_to_echo=35357 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 37632, 128, 0, 0, 0, 384, 18944, 7808, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 256, 46464, 0, 0, 128, 128, 0, 128, 0, 18944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 7680, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 128, 128, 57856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 63488, 0, 0, 2176, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 8190, 0, 0, 1641, 2181, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1029, 0, 0, 8070, 2, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3339, 8187, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 125, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8189, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8187, 0, 0, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 259, 8183, 0, 0, 2147, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7958, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 3, 0, 0, 1325, 0, 0, 8071, 4, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 8052, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 4, 0, 0, 182, 0, 8187, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8192, 62, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 2181, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1029, 0, 0, 7808, 2, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 7263, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1218, 2093, 924, 256, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 125, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14396, 2605, 8123, 8842, 8192, 10604, 4496, 8278] avg=8192.0000 max=14396 max_rank=0 min=2605 min_rank=1 imbalance_ratio=1.757324 max_load_reduction=51140 max_load_reduction_pct=0.780334 moved_to_echo=31570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7268, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6081, 2094, 924, 260, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 10300, 8236, 7926, 8191, 7270, 9360, 8153] avg=8192.0000 max=10300 max_rank=1 min=6100 min_rank=0 imbalance_ratio=1.257324 max_load_reduction=55236 max_load_reduction_pct=0.842834 moved_to_echo=32974 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7268, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6080, 2094, 924, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6098, 10304, 8236, 7926, 8192, 7269, 9358, 8153] avg=8192.0000 max=10304 max_rank=1 min=6098 min_rank=0 imbalance_ratio=1.257812 max_load_reduction=55232 max_load_reduction_pct=0.842773 moved_to_echo=32972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6094, 0, 0, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 259, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7958, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1325, 0, 0, 7809, 4, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 7144, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1593, 2093, 908, 256, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 4, 0, 0, 182, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14554, 1210, 8011, 9146, 8191, 11207, 4873, 8344] avg=8192.0000 max=14554 max_rank=0 min=1210 min_rank=1 imbalance_ratio=1.776611 max_load_reduction=50982 max_load_reduction_pct=0.777924 moved_to_echo=32615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7268, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6081, 2094, 924, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6098, 10280, 8232, 7926, 8192, 7296, 9359, 8153] avg=8192.0000 max=10280 max_rank=1 min=6098 min_rank=0 imbalance_ratio=1.254883 max_load_reduction=55256 max_load_reduction_pct=0.843140 moved_to_echo=33000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7265, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6081, 2093, 924, 260, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6098, 10253, 8223, 7926, 8195, 7329, 9359, 8153] avg=8192.0000 max=10253 max_rank=1 min=6098 min_rank=0 imbalance_ratio=1.251587 max_load_reduction=55283 max_load_reduction_pct=0.843552 moved_to_echo=33036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7266, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6081, 2094, 924, 260, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 10303, 8227, 7926, 8191, 7279, 9360, 8153] avg=8192.0000 max=10303 max_rank=1 min=6097 min_rank=0 imbalance_ratio=1.257690 max_load_reduction=55233 max_load_reduction_pct=0.842789 moved_to_echo=32983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2105, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7263, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6084, 2097, 929, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 10281, 8248, 7918, 8192, 7282, 9371, 8149] avg=8192.0000 max=10281 max_rank=1 min=6095 min_rank=0 imbalance_ratio=1.255005 max_load_reduction=55255 max_load_reduction_pct=0.843124 moved_to_echo=32994 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 39424, 16768, 7424, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 48896, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 384, 16384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 13696, 51968, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 7424, 58112, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 2432, 0, 0, 63232, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 384, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 65280, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8102, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2970, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 5053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 3949, 1, 0, 0, 0, 1, 0, 8190, 0, 0, 9, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 837, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 4333, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 2, 5943, 0, 1716, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 3386, 1, 0, 0, 0, 1, 0, 8188, 0, 0, 15, 0, 0, 0, 0, 0, 2, 0, 0, 27, 0, 0, 1019, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 5558, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7488, 4, 0, 0, 0, 0, 2, 8171, 0, 0, 0, 0, 0, 3, 0, 8186, 0, 0, 0, 0, 0, 1, 0, 0, 0, 42, 0, 0, 0, 0, 0, 2, 0, 2, 4, 4101, 0, 2952, 0, 0, 0, 1, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8045, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5811, 0, 0, 0, 0, 7991, 0, 0, 0, 0, 0, 0, 0, 2632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 7854, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4776, 0, 0, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8149, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4432, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8112, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5979, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8132, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3357, 0, 0, 0, 0, 7979, 0, 0, 0, 0, 0, 0, 0, 5269, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 7997, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3949, 1, 0, 0, 0, 1, 0, 1223, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 837, 1, 0, 0, 0, 0, 0, 0, 0, 6788, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4267, 0, 0, 0, 0, 0, 0, 0, 0, 4333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 2, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3207, 2740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 5943, 0, 1716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5183, 8810, 4777, 8600, 8230, 7928, 14139, 7869] avg=8192.0000 max=14139 max_rank=6 min=4777 min_rank=2 imbalance_ratio=1.725952 max_load_reduction=51397 max_load_reduction_pct=0.784256 moved_to_echo=38166 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8102, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6790, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4777, 0, 0, 0, 0, 0, 0, 0, 2970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4249, 0, 0, 0, 0, 0, 0, 0, 5053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3208, 2729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9325, 7962, 7747, 9302, 8227, 8333, 6241, 8399] avg=8192.0000 max=9325 max_rank=0 min=6241 min_rank=6 imbalance_ratio=1.138306 max_load_reduction=56211 max_load_reduction_pct=0.857712 moved_to_echo=31200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3386, 1, 0, 0, 0, 1, 0, 1222, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 27, 0, 0, 1019, 3, 0, 0, 0, 0, 0, 0, 0, 6787, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4774, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4264, 1, 0, 0, 0, 0, 0, 0, 0, 5558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7488, 4, 0, 0, 0, 0, 2, 141, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 1, 0, 0, 0, 42, 0, 0, 0, 0, 0, 3205, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4, 4101, 0, 2952, 0, 0, 0, 1, 2, 0, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4625, 9014, 4779, 9823, 8215, 7638, 14172, 7270] avg=8192.0000 max=14172 max_rank=6 min=4625 min_rank=0 imbalance_ratio=1.729980 max_load_reduction=51364 max_load_reduction_pct=0.783752 moved_to_echo=37295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8045, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6790, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4777, 0, 0, 0, 0, 0, 0, 0, 5811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4166, 0, 0, 0, 0, 0, 0, 0, 2632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3208, 2676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9268, 7946, 10588, 6798, 8215, 8333, 5989, 8399] avg=8192.0000 max=10588 max_rank=2 min=5989 min_rank=6 imbalance_ratio=1.292480 max_load_reduction=54948 max_load_reduction_pct=0.838440 moved_to_echo=30936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8112, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6790, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4777, 0, 0, 0, 0, 0, 0, 0, 5979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4266, 0, 0, 0, 0, 0, 0, 0, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8038, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3208, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9335, 7967, 10756, 5856, 8217, 8333, 6673, 8399] avg=8192.0000 max=10756 max_rank=2 min=5856 min_rank=3 imbalance_ratio=1.312988 max_load_reduction=54780 max_load_reduction_pct=0.835876 moved_to_echo=31622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8132, 0, 0, 0, 0, 0, 0, 1222, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6787, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4777, 0, 0, 0, 0, 0, 0, 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4160, 0, 0, 0, 0, 0, 0, 0, 5269, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 7859, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3208, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9357, 7936, 8134, 9431, 8067, 8317, 5895, 8399] avg=8192.0000 max=9431 max_rank=3 min=5895 min_rank=6 imbalance_ratio=1.151245 max_load_reduction=56105 max_load_reduction_pct=0.856094 moved_to_echo=30678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8149, 0, 0, 0, 0, 0, 0, 1222, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6786, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4777, 0, 0, 0, 0, 0, 0, 0, 4432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4198, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 7942, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3208, 2696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9374, 7943, 9209, 8226, 8130, 8324, 5930, 8400] avg=8192.0000 max=9374 max_rank=0 min=5930 min_rank=6 imbalance_ratio=1.144287 max_load_reduction=56162 max_load_reduction_pct=0.856964 moved_to_echo=30784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7854, 0, 0, 0, 0, 0, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6793, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4770, 0, 0, 0, 0, 0, 0, 0, 4776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4205, 0, 0, 0, 0, 0, 0, 0, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3213, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9069, 7958, 9546, 7500, 8235, 8330, 6497, 8401] avg=8192.0000 max=9546 max_rank=2 min=6497 min_rank=6 imbalance_ratio=1.165283 max_load_reduction=55990 max_load_reduction_pct=0.854340 moved_to_echo=31463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18176, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 25728, 21760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 38272, 0, 128, 0, 128, 0, 0, 0, 27392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 54400, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 33792, 128, 0, 0, 0, 0, 0, 0, 21888, 9984, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 55808, 128, 0, 0, 0, 128, 0, 9856, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 59264, 0, 4736, 0, 0, 0, 128, 128, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 64128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 64512, 128, 0, 0, 0, 0, 128, 1152, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 63, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 369, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1068, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8191, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 76, 8182, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8187, 7, 2, 0, 8188, 0, 8171, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7670, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 333, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4, 8187, 0, 0, 0, 5, 0, 2, 0, 4, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 8111, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 74, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 5, 1, 0, 8190, 0, 8183, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7643, 0, 0, 0, 0, 0, 2, 0, 1, 0, 8, 1252, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8190, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 7078, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8147, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 69, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8191, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 155, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 69, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8189, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 8179, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7643, 0, 0, 0, 0, 0, 2, 0, 1454, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 1252, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1921, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 1, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 6266, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6237, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 7078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8147, 0, 0, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8015, 8186, 9330, 3682, 8395, 6445, 13320, 8163] avg=8192.0000 max=13320 max_rank=6 min=3682 min_rank=3 imbalance_ratio=1.625977 max_load_reduction=52216 max_load_reduction_pct=0.796753 moved_to_echo=36323 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5032, 0, 0, 0, 0, 0, 0, 0, 1454, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1921, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 0, 0, 0, 0, 6271, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8015, 8188, 6717, 10013, 8109, 10042, 6246, 8206] avg=8192.0000 max=10042 max_rank=5 min=6246 min_rank=6 imbalance_ratio=1.225830 max_load_reduction=55494 max_load_reduction_pct=0.846771 moved_to_echo=32603 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6828, 0, 0, 0, 0, 0, 0, 0, 1454, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1922, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 0, 6270, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8321, 8187, 8513, 10588, 8108, 6964, 6649, 8206] avg=8192.0000 max=10588 max_rank=3 min=6649 min_rank=6 imbalance_ratio=1.292480 max_load_reduction=54948 max_load_reduction_pct=0.838440 moved_to_echo=29927 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 8167, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7670, 0, 0, 0, 0, 0, 3, 0, 1453, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 333, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1920, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 1, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 6262, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4, 6235, 0, 0, 0, 5, 0, 2, 0, 4, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 8111, 0, 0, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8183, 9358, 2756, 8489, 6443, 14152, 8130] avg=8192.0000 max=14152 max_rank=6 min=2756 min_rank=3 imbalance_ratio=1.727539 max_load_reduction=51384 max_load_reduction_pct=0.784058 moved_to_echo=37214 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6309, 0, 0, 0, 0, 0, 0, 0, 1454, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1922, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2372, 0, 0, 0, 0, 0, 0, 6270, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8021, 8203, 7994, 9967, 8109, 8725, 6311, 8206] avg=8192.0000 max=9967 max_rank=3 min=6311 min_rank=6 imbalance_ratio=1.216675 max_load_reduction=55569 max_load_reduction_pct=0.847916 moved_to_echo=31351 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6284, 0, 0, 0, 0, 0, 0, 0, 1453, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1921, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 0, 0, 0, 0, 0, 6271, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8186, 7968, 8674, 8109, 10008, 6278, 8206] avg=8192.0000 max=10008 max_rank=5 min=6278 min_rank=6 imbalance_ratio=1.221680 max_load_reduction=55528 max_load_reduction_pct=0.847290 moved_to_echo=32601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6444, 0, 0, 0, 0, 0, 0, 0, 1453, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1922, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2853, 0, 0, 0, 0, 0, 0, 6270, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8021, 8188, 8128, 9429, 8108, 9207, 6249, 8206] avg=8192.0000 max=9429 max_rank=3 min=6249 min_rank=6 imbalance_ratio=1.151001 max_load_reduction=56107 max_load_reduction_pct=0.856125 moved_to_echo=31770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5835, 0, 0, 0, 0, 0, 0, 0, 1458, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1922, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 0, 0, 0, 0, 0, 0, 6270, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 14, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9011, 8215, 7528, 10427, 8109, 7702, 6331, 8213] avg=8192.0000 max=10427 max_rank=3 min=6331 min_rank=6 imbalance_ratio=1.272827 max_load_reduction=55109 max_load_reduction_pct=0.840897 moved_to_echo=30355 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 46208, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15488, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 49920, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 14720, 0, 0, 0, 0, 0, 0, 50176, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 52096, 0, 0, 0, 0, 0, 128, 0, 11648, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 63616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 768, 0, 0, 128, 0, 0, 0, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 0, 0, 0, 0, 6086, 0, 8186, 0, 0, 0, 1, 0, 0, 8190, 0, 1, 0, 0, 8167, 0, 0, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 5, 8190, 0, 0, 2, 0, 1, 0, 0, 0, 102, 0, 0, 0, 1, 157, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8187, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 7909, 0, 0, 0, 0, 3493, 0, 8172, 2, 0, 0, 2, 1, 0, 8146, 0, 2, 0, 0, 8150, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4635, 3, 0, 0, 0, 0, 0, 0, 0, 3, 8183, 0, 29, 8187, 0, 0, 5, 2, 0, 0, 0, 0, 253, 0, 0, 0, 2, 91, 0, 0, 3, 0, 0, 0, 0, 4, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8154, 0, 8192, 0, 0, 46, 0, 0, 0, 8169, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 8159, 0, 8192, 0, 0, 33, 0, 0, 0, 8184, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 8162, 0, 8192, 0, 0, 28, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8183, 0, 8192, 0, 0, 14, 0, 0, 0, 8180, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 0, 0, 6086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8179, 0, 1, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 1, 0, 0, 0, 0, 7149, 66, 11, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8184, 0, 5, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 1, 157, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8188, 6937, 8202, 9271, 8193, 8364, 8192] avg=8192.0000 max=9271 max_rank=4 min=6937 min_rank=2 imbalance_ratio=1.131714 max_load_reduction=56265 max_load_reduction_pct=0.858536 moved_to_echo=34020 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 869, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7323, 66, 11, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8188, 9063, 8179, 7408, 8188, 8123, 8193] avg=8192.0000 max=9063 max_rank=2 min=7408 min_rank=4 imbalance_ratio=1.106323 max_load_reduction=56473 max_load_reduction_pct=0.861710 moved_to_echo=31912 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8171, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8183, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 839, 0, 0, 0, 0, 3493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 1, 0, 8135, 0, 2, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4635, 3, 0, 0, 0, 0, 7070, 67, 11, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8179, 0, 29, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 2, 91, 0, 0, 3, 0, 0, 0, 0, 4, 0, 4, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8192, 4344, 8179, 11794, 8218, 8437, 8193] avg=8192.0000 max=11794 max_rank=4 min=4344 min_rank=2 imbalance_ratio=1.439697 max_load_reduction=53742 max_load_reduction_pct=0.820038 moved_to_echo=36642 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7323, 67, 11, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8188, 9061, 8181, 7409, 8188, 8122, 8192] avg=8192.0000 max=9061 max_rank=2 min=7409 min_rank=4 imbalance_ratio=1.106079 max_load_reduction=56475 max_load_reduction_pct=0.861740 moved_to_echo=31911 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 869, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7323, 67, 11, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8195, 9029, 8206, 7409, 8188, 8122, 8192] avg=8192.0000 max=9029 max_rank=2 min=7409 min_rank=4 imbalance_ratio=1.102173 max_load_reduction=56507 max_load_reduction_pct=0.862228 moved_to_echo=31911 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7323, 66, 11, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8204, 9024, 8204, 7408, 8188, 8123, 8191] avg=8192.0000 max=9024 max_rank=2 min=7408 min_rank=4 imbalance_ratio=1.101562 max_load_reduction=56512 max_load_reduction_pct=0.862305 moved_to_echo=31910 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7323, 66, 11, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8197, 9052, 8183, 7408, 8188, 8123, 8191] avg=8192.0000 max=9052 max_rank=2 min=7408 min_rank=4 imbalance_ratio=1.104980 max_load_reduction=56484 max_load_reduction_pct=0.861877 moved_to_echo=31910 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 862, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7330, 66, 16, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8184, 9026, 8202, 7429, 8185, 8122, 8192] avg=8192.0000 max=9026 max_rank=2 min=7429 min_rank=4 imbalance_ratio=1.101807 max_load_reduction=56510 max_load_reduction_pct=0.862274 moved_to_echo=31928 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 128, 6912, 0, 0, 0, 0, 58624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6784, 128, 0, 0, 0, 0, 58240, 640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 0, 0, 0, 128, 256, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 65408, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 6030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 101, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 1, 0, 0, 0, 0, 0, 0, 7170, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 39, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 7409, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8185, 0, 0, 6651, 0, 0, 2, 10, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6964, 0, 0, 0, 0, 0, 0, 8189, 6, 0, 7930, 34, 0, 0, 0, 2, 0, 1, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 8094, 2, 0, 0, 0, 1, 0, 0, 1, 0, 1768, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8184, 0, 0, 6817, 0, 0, 4, 0, 1504, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3, 1, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5021, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 7861, 19, 1, 1, 0, 3, 0, 2, 0, 0, 8186, 0, 3, 2, 0, 0, 0, 8026, 4, 0, 0, 0, 1, 0, 0, 5, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 681, 0, 0, 6651, 0, 0, 2, 10, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 8144, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6563, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6964, 0, 0, 0, 0, 0, 0, 906, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 34, 0, 0, 0, 2, 0, 1, 0, 0, 45, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6858, 2, 0, 0, 0, 1, 0, 0, 1, 0, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 7461, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8630, 8190, 7802, 7876, 8013, 8630, 8204, 8191] avg=8192.0000 max=8630 max_rank=0 min=7802 min_rank=2 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=33038 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 682, 0, 0, 7966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 6566, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 0, 0, 0, 0, 0, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7467, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8648, 8190, 8270, 8795, 8241, 7013, 8187, 8192] avg=8192.0000 max=8795 max_rank=3 min=7013 min_rank=5 imbalance_ratio=1.073608 max_load_reduction=56741 max_load_reduction_pct=0.865799 moved_to_echo=31633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 682, 0, 0, 7176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 1, 6566, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7170, 0, 0, 0, 0, 0, 0, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 39, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7467, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7858, 8190, 8421, 8076, 8276, 8336, 8187, 8192] avg=8192.0000 max=8421 max_rank=2 min=7858 min_rank=0 imbalance_ratio=1.027954 max_load_reduction=57115 max_load_reduction_pct=0.871506 moved_to_echo=32991 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 681, 0, 0, 6817, 0, 0, 4, 0, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3, 1, 4, 0, 0, 8141, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6561, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5021, 0, 0, 0, 0, 0, 0, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7861, 19, 1, 1, 0, 3, 0, 2, 0, 0, 45, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6800, 4, 0, 0, 0, 1, 0, 0, 5, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 12, 0, 7460, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9006, 8196, 7791, 5926, 7937, 10294, 8195, 8191] avg=8192.0000 max=10294 max_rank=5 min=5926 min_rank=3 imbalance_ratio=1.256592 max_load_reduction=55242 max_load_reduction_pct=0.842926 moved_to_echo=34617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 682, 0, 0, 7030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 6566, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7409, 0, 0, 0, 0, 0, 0, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7467, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7712, 8190, 8193, 8315, 8242, 8505, 8187, 8192] avg=8192.0000 max=8505 max_rank=5 min=7712 min_rank=0 imbalance_ratio=1.038208 max_load_reduction=57031 max_load_reduction_pct=0.870224 moved_to_echo=33126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 682, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 6566, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 0, 0, 0, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7467, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8575, 8190, 7876, 8729, 8248, 7539, 8187, 8192] avg=8192.0000 max=8729 max_rank=3 min=7539 min_rank=5 imbalance_ratio=1.065552 max_load_reduction=56807 max_load_reduction_pct=0.866806 moved_to_echo=32166 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 672, 0, 0, 7723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 6568, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 0, 0, 0, 0, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7474, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8395, 8200, 8290, 8958, 8241, 7056, 8202, 8194] avg=8192.0000 max=8958 max_rank=3 min=7056 min_rank=5 imbalance_ratio=1.093506 max_load_reduction=56578 max_load_reduction_pct=0.863312 moved_to_echo=31693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 682, 0, 0, 6030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 6566, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 101, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7467, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6712, 8190, 8893, 8861, 8338, 8163, 8187, 8192] avg=8192.0000 max=8893 max_rank=2 min=6712 min_rank=0 imbalance_ratio=1.085571 max_load_reduction=56643 max_load_reduction_pct=0.864304 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 2944, 0, 128, 52608, 9984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 55296, 128, 0, 0, 0, 128, 0, 0, 128, 0, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 58368, 0, 0, 0, 0, 0, 0, 7296, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 59776, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 5504, 0, 0, 57344, 0, 0, 128, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[65024, 256, 128, 128, 0, 128, 0, 128, 0, 0, 384, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True + [2026-08-26 16:28:33] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 437.0 | throughput per GPU (TFLOP/s/GPU): 413.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.427359E+00 | loss scale: 1.0 | grad norm: 5.155 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 11, 3515, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2594, 0, 0, 3297, 0, 0, 0, 8191, 1, 8182, 0, 1, 0, 0, 2, 1, 1, 5, 0, 0, 7, 0, 0, 0, 0, 2, 4, 1, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 2990, 0, 0, 0, 3339, 0, 2287, 17, 3, 0, 3954, 2, 0, 0, 3, 4, 0, 1, 0, 8077, 0, 9, 0, 0, 0, 0, 2360, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 0, 8109, 0, 1, 4, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 2, 0, 0, 347, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 3443, 0, 0, 0, 0, 0, 4, 0, 0, 1, 16, 0, 6, 0, 0, 2644, 0, 1, 2554, 0, 1, 2, 8191, 2, 8187, 1, 0, 0, 4, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2, 3, 8178, 2, 0, 0, 0, 0, 0, 0, 0, 2892, 2, 0, 2, 3370, 0, 3341, 46, 0, 0, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 7, 2, 0, 2, 0, 2234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7962, 0, 0, 22, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 2, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 3243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2781, 0, 0, 3049, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2583, 0, 0, 0, 3656, 0, 3218, 17, 0, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 21, 0, 0, 0, 0, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 10, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1950, 0, 0, 3148, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 1, 0, 0, 1, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 0, 0, 0, 3304, 0, 3700, 48, 0, 0, 4589, 0, 0, 0, 0, 0, 1, 0, 0, 8166, 0, 3, 0, 0, 0, 0, 2258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 2878, 0, 0, 3583, 0, 0, 0, 8191, 1, 8191, 12, 0, 0, 0, 3, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 3070, 0, 0, 0, 3248, 0, 2691, 42, 0, 0, 3939, 0, 0, 0, 0, 0, 1, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 1718, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 14, 0, 1, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 17, 2988, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 0, 2906, 16, 0, 0, 8192, 1, 8192, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 1, 0, 0, 0, 3039, 0, 2, 0, 3301, 0, 3109, 31, 0, 0, 4382, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 3, 0, 0, 1, 0, 1403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8046, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 349, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2853, 3, 1, 2591, 0, 0, 0, 8186, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 2, 0, 3413, 0, 2, 0, 3237, 1, 3278, 57, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 1, 0, 8004, 0, 1, 1, 0, 0, 0, 2195, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 8019, 1, 1, 0, 0, 0, 1, 0, 4, 0, 1, 3, 0, 0, 0, 0, 0, 350, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4, 3494, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2700, 0, 0, 2496, 5, 0, 0, 8192, 0, 8191, 13, 0, 0, 0, 0, 0, 4, 0, 0, 1, 3, 2, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 3002, 0, 0, 1, 4100, 0, 3147, 53, 0, 1, 4158, 0, 0, 3, 0, 0, 2, 0, 0, 8041, 0, 30, 0, 1, 0, 0, 1496, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7895, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 1, 0, 0, 0, 0, 24, 1825, 1854, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2878, 0, 0, 3583, 0, 0, 0, 0, 1, 2528, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3070, 0, 0, 0, 1990, 0, 2691, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6199, 0, 0, 0, 0, 0, 0, 1718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 14, 0, 1, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 5663, 2114, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 9003, 8178, 8192, 7793, 7918, 8250, 8054] avg=8192.0000 max=9003 max_rank=1 min=7793 min_rank=4 imbalance_ratio=1.098999 max_load_reduction=56533 max_load_reduction_pct=0.862625 moved_to_echo=32015 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 3243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 1875, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2781, 0, 0, 3049, 0, 0, 0, 0, 0, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 17, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2583, 0, 0, 0, 2240, 0, 3218, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6269, 0, 21, 0, 0, 0, 0, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 1, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 5664, 2078, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8358, 8193, 8194, 8058, 8216, 8232, 8111] avg=8192.0000 max=8358 max_rank=1 min=8058 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32617 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 3443, 0, 0, 0, 0, 0, 4, 0, 0, 1, 16, 1804, 1883, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2644, 0, 1, 2554, 0, 1, 2, 0, 2, 2527, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 1, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2, 3, 8178, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2892, 2, 0, 2, 2064, 0, 3341, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6296, 0, 7, 2, 0, 2, 0, 2234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7962, 0, 0, 22, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 2, 0, 1, 0, 5660, 2090, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 7738, 8187, 8190, 8347, 8541, 8129, 8091] avg=8192.0000 max=8541 max_rank=5 min=7738 min_rank=1 imbalance_ratio=1.042603 max_load_reduction=56995 max_load_reduction_pct=0.869675 moved_to_echo=33108 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 11, 3515, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 1832, 1859, 1148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2594, 0, 0, 3297, 0, 0, 0, 0, 1, 2525, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 5, 0, 0, 7, 0, 0, 0, 0, 2, 4, 1, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2990, 0, 0, 0, 2046, 0, 2287, 17, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 1, 0, 6218, 0, 9, 0, 0, 0, 0, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 0, 8109, 0, 1, 4, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 2, 0, 0, 347, 0, 0, 0, 1, 5657, 2122, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8372, 8418, 8195, 8196, 7345, 8595, 8261, 8154] avg=8192.0000 max=8595 max_rank=5 min=7345 min_rank=4 imbalance_ratio=1.049194 max_load_reduction=56941 max_load_reduction_pct=0.868851 moved_to_echo=32355 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 10, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2127, 1880, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1950, 0, 0, 3148, 0, 0, 0, 0, 0, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 1, 0, 0, 1, 0, 0, 30, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 0, 0, 0, 2024, 0, 3700, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6286, 0, 3, 0, 0, 0, 0, 2258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 28, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 5664, 2462, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8008, 7626, 8211, 8190, 8348, 8548, 8205, 8400] avg=8192.0000 max=8548 max_rank=5 min=7626 min_rank=1 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=33501 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1818, 1842, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2853, 3, 1, 2591, 0, 0, 0, 0, 0, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 2, 0, 2, 0, 2, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8186, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3413, 0, 2, 0, 1983, 1, 3278, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6162, 0, 1, 1, 0, 0, 0, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 8019, 1, 1, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 1, 3, 0, 0, 0, 0, 0, 350, 0, 0, 0, 3, 5661, 2104, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7788, 7975, 8176, 8192, 8736, 8360, 8164, 8145] avg=8192.0000 max=8736 max_rank=4 min=7788 min_rank=0 imbalance_ratio=1.066406 max_load_reduction=56800 max_load_reduction_pct=0.866699 moved_to_echo=33405 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 17, 2988, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2031, 1863, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 0, 2906, 16, 0, 0, 0, 1, 2528, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 11, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3039, 0, 2, 0, 2022, 0, 3109, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6232, 0, 3, 0, 0, 1, 0, 1403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8046, 0, 0, 8, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 349, 0, 0, 0, 0, 5664, 2351, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8037, 8678, 8197, 8191, 8204, 7639, 8206, 8384] avg=8192.0000 max=8678 max_rank=1 min=7639 min_rank=5 imbalance_ratio=1.059326 max_load_reduction=56858 max_load_reduction_pct=0.867584 moved_to_echo=32433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4, 3494, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1928, 1854, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2700, 0, 0, 2496, 5, 0, 0, 0, 0, 2526, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 3, 2, 0, 0, 0, 0, 15, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3002, 0, 0, 1, 2501, 0, 3147, 53, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 6187, 0, 30, 0, 1, 0, 0, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7895, 0, 0, 13, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 5665, 2230, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8696, 7740, 8199, 8191, 8705, 7719, 8089, 8197] avg=8192.0000 max=8705 max_rank=4 min=7719 min_rank=5 imbalance_ratio=1.062622 max_load_reduction=56831 max_load_reduction_pct=0.867172 moved_to_echo=32710 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 25984, 0, 0, 128, 128, 128, 128, 0, 0, 128, 128, 15232, 14976, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 21632, 128, 128, 23680, 128, 128, 128, 0, 128, 20224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 23680, 128, 128, 128, 16896, 128, 24832, 384, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 49920, 0, 128, 128, 128, 128, 0, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 0, 2560, 128, 0, 128, 128, 45312, 17664, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 128, 128, 128, 64256, 128, 128, 128, 0, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 65536, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 87, 0, 0, 0, 7430, 0, 4239, 0, 0, 8192, 0, 0, 260, 0, 0, 0, 0, 0, 2754, 8192, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4216, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7700, 0, 6942, 0, 0, 8159, 0, 0, 623, 0, 0, 0, 0, 0, 1012, 8192, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7270, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 14, 0, 0, 0, 7891, 0, 5434, 0, 0, 8184, 0, 0, 89, 0, 0, 0, 0, 0, 1498, 8192, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6825, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 24, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 11, 5, 0, 0, 7090, 6, 4337, 4, 0, 8186, 0, 0, 148, 0, 2, 0, 0, 0, 4437, 8186, 6, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2603, 8187, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 120, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 7783, 0, 7058, 0, 0, 8183, 0, 0, 1192, 0, 0, 0, 0, 0, 421, 8192, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 143, 0, 0, 0, 7641, 0, 6629, 0, 0, 8190, 0, 0, 253, 0, 0, 0, 0, 0, 1334, 8192, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 169, 0, 0, 0, 5873, 0, 4468, 0, 0, 8188, 0, 0, 122, 0, 0, 0, 0, 0, 4296, 8192, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4250, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 14, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 0, 0, 8005, 0, 7482, 0, 0, 8190, 0, 0, 582, 0, 0, 0, 0, 0, 364, 8192, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7860, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 87, 0, 0, 0, 1904, 0, 4239, 0, 0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2754, 0, 0, 0, 0, 6188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6105, 1993, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4216, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5502, 0, 0, 0, 3533, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8147, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6490, 8942, 8241, 6304, 8180, 11024, 8190, 8165] avg=8192.0000 max=11024 max_rank=5 min=6304 min_rank=3 imbalance_ratio=1.345703 max_load_reduction=54512 max_load_reduction_pct=0.831787 moved_to_echo=35559 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 14, 0, 0, 0, 2023, 0, 5434, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1498, 0, 0, 0, 0, 6183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 2116, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6825, 2086, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2783, 0, 0, 0, 3752, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 24, 0, 0, 0, 0, 8140, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7562, 7681, 8282, 8915, 8184, 8523, 8197, 8192] avg=8192.0000 max=8915 max_rank=3 min=7562 min_rank=0 imbalance_ratio=1.088257 max_load_reduction=56621 max_load_reduction_pct=0.863968 moved_to_echo=33096 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 143, 0, 0, 0, 1959, 0, 6629, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1334, 0, 0, 0, 0, 6191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6105, 2049, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 3633, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 8145, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8984, 7525, 8268, 9958, 8190, 6239, 8195, 8177] avg=8192.0000 max=9958 max_rank=3 min=6239 min_rank=5 imbalance_ratio=1.215576 max_load_reduction=55578 max_load_reduction_pct=0.848053 moved_to_echo=30801 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 11, 5, 0, 0, 1817, 6, 4337, 4, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4437, 0, 6, 0, 0, 6188, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 1902, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2603, 2086, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5779, 0, 0, 0, 3371, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 120, 0, 0, 0, 0, 8141, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6329, 10635, 8074, 4695, 8200, 11139, 8178, 8286] avg=8192.0000 max=11139 max_rank=5 min=4695 min_rank=3 imbalance_ratio=1.359741 max_load_reduction=54397 max_load_reduction_pct=0.830032 moved_to_echo=35803 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 1995, 0, 7058, 0, 0, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, 6178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6105, 2087, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 3701, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8138, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10251, 6599, 8252, 10023, 8196, 5873, 8182, 8160] avg=8192.0000 max=10251 max_rank=0 min=5873 min_rank=5 imbalance_ratio=1.251343 max_load_reduction=55285 max_load_reduction_pct=0.843582 moved_to_echo=30411 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 169, 0, 0, 0, 1505, 0, 4468, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4296, 0, 0, 0, 0, 6186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6103, 1575, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4250, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5222, 0, 0, 0, 2793, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 45, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 14, 0, 0, 4, 0, 8143, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6267, 10482, 7843, 6339, 8199, 10004, 8223, 8179] avg=8192.0000 max=10482 max_rank=1 min=6267 min_rank=0 imbalance_ratio=1.279541 max_load_reduction=55054 max_load_reduction_pct=0.840057 moved_to_echo=34605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1973, 0, 6942, 0, 0, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6105, 2065, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7270, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 0, 0, 0, 3662, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 8114, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9538, 7188, 8261, 9357, 8188, 6662, 8194, 8148] avg=8192.0000 max=9538 max_rank=0 min=6662 min_rank=5 imbalance_ratio=1.164307 max_load_reduction=55998 max_load_reduction_pct=0.854462 moved_to_echo=31192 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 0, 0, 2042, 0, 7482, 0, 0, 0, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6107, 2152, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7860, 2085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 3811, 1971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 8145, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10115, 6484, 8315, 9945, 8199, 6072, 8177, 8229] avg=8192.0000 max=10115 max_rank=0 min=6072 min_rank=5 imbalance_ratio=1.234741 max_load_reduction=55421 max_load_reduction_pct=0.845657 moved_to_echo=30677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 512, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48896, 16000, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[48896, 16768, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 21504, 0, 0, 0, 28288, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 512, 128, 0, 0, 15232, 128, 46592, 128, 0, 0, 0, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 16128, 0, 128, 0, 0, 49536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 128, 128, 0, 0, 384, 0, 0, 128, 0, 65152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8142, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 5838, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 1523, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 37, 0, 8185, 2, 0, 0, 0, 8070, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 5738, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 8186, 0, 6, 0, 0, 5, 0, 0, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5917, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2277, 0, 7416, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5917, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8192, 8000, 8206, 8502, 9701, 8191, 6557] avg=8192.0000 max=9701 max_rank=5 min=6557 min_rank=7 imbalance_ratio=1.184204 max_load_reduction=55835 max_load_reduction_pct=0.851974 moved_to_echo=32951 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 43, 0, 7416, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8195, 7995, 8209, 8497, 7469, 8198, 8786] avg=8192.0000 max=8786 max_rank=7 min=7469 min_rank=5 imbalance_ratio=1.072510 max_load_reduction=56750 max_load_reduction_pct=0.865936 moved_to_echo=32950 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 7414, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8191, 8001, 8206, 8503, 7760, 8190, 8499] avg=8192.0000 max=8503 max_rank=4 min=7760 min_rank=5 imbalance_ratio=1.037964 max_load_reduction=57033 max_load_reduction_pct=0.870255 moved_to_echo=32952 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2354, 0, 7417, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5838, 0, 0, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8199, 7991, 8205, 8493, 9779, 8202, 6479] avg=8192.0000 max=9779 max_rank=5 min=6479 min_rank=7 imbalance_ratio=1.193726 max_load_reduction=55757 max_load_reduction_pct=0.850784 moved_to_echo=32953 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 7417, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8191, 8001, 8206, 8503, 7445, 8190, 8814] avg=8192.0000 max=8814 max_rank=7 min=7445 min_rank=5 imbalance_ratio=1.075928 max_load_reduction=56722 max_load_reduction_pct=0.865509 moved_to_echo=32952 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 12, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 0, 7996, 0, 0, 0, 0, 37, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 5738, 0, 0, 0, 0, 0, 2, 0, 5, 0, 178, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 0, 7413, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 640, 0, 6, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8191, 9558, 8084, 6062, 8496, 8185, 8763] avg=8192.0000 max=9558 max_rank=2 min=6062 min_rank=4 imbalance_ratio=1.166748 max_load_reduction=55978 max_load_reduction_pct=0.854156 moved_to_echo=31506 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 7418, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8191, 8001, 8206, 8463, 7456, 8190, 8816] avg=8192.0000 max=8816 max_rank=7 min=7456 min_rank=5 imbalance_ratio=1.076172 max_load_reduction=56720 max_load_reduction_pct=0.865479 moved_to_echo=32925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7422, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8186, 7989, 8214, 8513, 7430, 8190, 8822] avg=8192.0000 max=8822 max_rank=7 min=7430 min_rank=5 imbalance_ratio=1.076904 max_load_reduction=56714 max_load_reduction_pct=0.865387 moved_to_echo=32955 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 6144, 0, 59392, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[60416, 0, 0, 0, 5120, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[1536, 0, 0, 0, 0, 64000, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 63104, 0, 0, 0, 0, 0, 128, 0, 128, 0, 1536, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 5567, 8192, 1410, 0, 0, 0, 0, 0, 7921, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 1041, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 6, 0, 0, 0, 0, 8184, 0, 0, 0, 7992, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 154, 8185, 916, 0, 0, 0, 6, 0, 6098, 8170, 0, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1490, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, 3638, 8192, 1909, 0, 0, 0, 0, 0, 8034, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1306, 0, 0, 0, 0, 0, 0, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8159, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 358, 0, 0, 0, 5961, 8192, 676, 0, 0, 0, 0, 0, 8033, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 7914, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2288, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 3660, 8192, 1959, 0, 0, 0, 0, 0, 8082, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 8180, 0, 12, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 436, 0, 0, 0, 4710, 8192, 2561, 0, 0, 0, 0, 0, 7985, 8186, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 6687, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1316, 0, 0, 0, 4596, 8192, 2051, 0, 0, 0, 0, 0, 8108, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 6621, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1207, 0, 0, 0, 6521, 8192, 1577, 0, 0, 0, 0, 0, 7914, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 1028, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 6, 0, 0, 0, 0, 7259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 4366, 925, 12, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 916, 0, 0, 0, 6, 0, 1732, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 9178, 8152, 8375, 7280, 8027, 13527, 2814] avg=8192.0000 max=13527 max_rank=6 min=2814 min_rank=7 imbalance_ratio=1.651245 max_load_reduction=52009 max_load_reduction_pct=0.793594 moved_to_echo=31648 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 7256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 12, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 26, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 436, 5717, 924, 97, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4710, 0, 2561, 0, 0, 0, 0, 0, 2268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8034, 8234, 8166, 7908, 8238, 7219, 9539] avg=8192.0000 max=9539 max_rank=7 min=7219 min_rank=6 imbalance_ratio=1.164429 max_load_reduction=55997 max_load_reduction_pct=0.854446 moved_to_echo=32904 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 0, 0, 0, 0, 0, 0, 7995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1490, 0, 0, 0, 0, 0, 0, 7266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 5752, 926, 98, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 0, 1909, 0, 0, 0, 0, 0, 2282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8722, 8192, 8166, 8756, 8227, 7454, 7829] avg=8192.0000 max=8756 max_rank=4 min=7454 min_rank=6 imbalance_ratio=1.068848 max_load_reduction=56780 max_load_reduction_pct=0.866394 moved_to_echo=32266 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 7254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 26, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 358, 5751, 924, 97, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5961, 0, 676, 0, 0, 0, 0, 0, 2282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 9251, 8200, 8166, 7418, 8206, 7175, 8919] avg=8192.0000 max=9251 max_rank=1 min=7175 min_rank=6 imbalance_ratio=1.129272 max_load_reduction=56285 max_load_reduction_pct=0.858841 moved_to_echo=31718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 6586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 0, 0, 0, 0, 0, 0, 7260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1316, 5808, 928, 85, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4596, 0, 2051, 0, 0, 0, 0, 0, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 6828, 8196, 8165, 8810, 8224, 8173, 8947] avg=8192.0000 max=8947 max_rank=7 min=6828 min_rank=1 imbalance_ratio=1.092163 max_load_reduction=56589 max_load_reduction_pct=0.863480 moved_to_echo=34154 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 6529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, 0, 0, 0, 0, 0, 0, 7266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1207, 5666, 926, 80, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6521, 0, 1577, 0, 0, 0, 0, 0, 2248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 6593, 8177, 8166, 7929, 8224, 7911, 10346] avg=8192.0000 max=10346 max_rank=7 min=6593 min_rank=1 imbalance_ratio=1.262939 max_load_reduction=55190 max_load_reduction_pct=0.842133 moved_to_echo=34410 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 8021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 0, 0, 0, 0, 0, 7265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 26, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 233, 5671, 926, 98, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5567, 0, 1410, 0, 0, 0, 0, 0, 2250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8689, 8188, 8166, 7881, 8232, 6962, 9227] avg=8192.0000 max=9227 max_rank=7 min=6962 min_rank=6 imbalance_ratio=1.126343 max_load_reduction=56309 max_load_reduction_pct=0.859207 moved_to_echo=32302 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 7804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2288, 0, 0, 0, 0, 0, 0, 7266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 5786, 926, 96, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 0, 1959, 0, 0, 0, 0, 0, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8241, 8197, 8166, 9554, 8158, 7115, 7915] avg=8192.0000 max=9554 max_rank=4 min=7115 min_rank=6 imbalance_ratio=1.166260 max_load_reduction=55982 max_load_reduction_pct=0.854218 moved_to_echo=32742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 12672, 44544, 7424, 768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 34816, 0, 13184, 0, 0, 0, 128, 0, 17664, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 7552, 0, 128, 0, 0, 0, 0, 58112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 11776, 0, 0, 0, 0, 0, 0, 53888, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 384, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 256, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65280, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 2503, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6393, 0, 0, 8, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8178, 8186, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 2709, 0, 0, 0, 7987, 7, 4, 0, 0, 110, 0, 4174, 0, 0, 0, 0, 0, 0, 0, 30, 0, 1759, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7734, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 58, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 1285, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7195, 8192, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 7846, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 960, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7404, 0, 0, 6, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7787, 0, 0, 0, 4309, 0, 0, 0, 0, 0, 0, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2492, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 1374, 0, 0, 0, 0, 1, 0, 1832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5218, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7993, 0, 0, 0, 1658, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6207, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 1196, 0, 0, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 8192, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 6937, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 2709, 0, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7987, 7, 4, 0, 0, 110, 0, 4174, 0, 0, 0, 0, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 1759, 995, 0, 0, 0, 0, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7734, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 6867, 1019, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 58, 0, 6, 0, 0, 0, 0, 8165, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 3499, 16561, 3248, 9605, 7996, 8245, 8190] avg=8192.0000 max=16561 max_rank=2 min=3248 min_rank=3 imbalance_ratio=2.021606 max_load_reduction=48975 max_load_reduction_pct=0.747299 moved_to_echo=34036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 4377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7195, 1037, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7846, 0, 0, 0, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7155, 1042, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8961, 5780, 8244, 9760, 8226, 8175, 8207] avg=8192.0000 max=9760 max_rank=4 min=5780 min_rank=2 imbalance_ratio=1.191406 max_load_reduction=55776 max_load_reduction_pct=0.851074 moved_to_echo=34368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7787, 0, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4309, 0, 0, 0, 0, 0, 0, 1935, 0, 0, 0, 0, 0, 0, 4405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2492, 0, 0, 0, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7155, 1049, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8592, 10649, 9095, 4419, 8213, 8188, 8184] avg=8192.0000 max=10649 max_rank=2 min=4419 min_rank=4 imbalance_ratio=1.299927 max_load_reduction=54887 max_load_reduction_pct=0.837509 moved_to_echo=29004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 960, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 4393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7404, 0, 0, 6, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7154, 1046, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8984, 5456, 8987, 9332, 8209, 8187, 8184] avg=8192.0000 max=9332 max_rank=4 min=5456 min_rank=2 imbalance_ratio=1.139160 max_load_reduction=56204 max_load_reduction_pct=0.857605 moved_to_echo=33912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7993, 0, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 4397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6207, 0, 0, 0, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7155, 1047, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8797, 6617, 9194, 8129, 8231, 8187, 8184] avg=8192.0000 max=9194 max_rank=3 min=6617 min_rank=2 imbalance_ratio=1.122314 max_load_reduction=56342 max_load_reduction_pct=0.859711 moved_to_echo=32731 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1196, 0, 0, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0, 4404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 1037, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6937, 0, 0, 0, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 7155, 1049, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8902, 5912, 9042, 8862, 8239, 8181, 8213] avg=8192.0000 max=9042 max_rank=3 min=5912 min_rank=2 imbalance_ratio=1.103760 max_load_reduction=56494 max_load_reduction_pct=0.862030 moved_to_echo=33495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 0, 0, 1, 0, 1832, 0, 0, 0, 0, 0, 0, 4412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 1035, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5218, 0, 0, 0, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7157, 1055, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8827, 7619, 9168, 7132, 8225, 8187, 8188] avg=8192.0000 max=9168 max_rank=3 min=7132 min_rank=4 imbalance_ratio=1.119141 max_load_reduction=56368 max_load_reduction_pct=0.860107 moved_to_echo=31732 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2503, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 4337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 1037, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6393, 0, 0, 8, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7155, 1033, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8974, 6942, 8558, 8297, 8197, 8186, 8186] avg=8192.0000 max=8974 max_rank=1 min=6942 min_rank=2 imbalance_ratio=1.095459 max_load_reduction=56562 max_load_reduction_pct=0.863068 moved_to_echo=32866 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 21376, 128, 128, 0, 0, 128, 0, 9216, 0, 0, 0, 0, 0, 0, 35072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 42624, 0, 0, 7808, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 56960, 8448, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 56832, 8320, 0, 0, 0, 0, 0, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 59136, 0, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 14, 1, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 5, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7881, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 51, 12, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 27, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 944, 0, 0, 0, 1, 0, 0, 0, 0, 0, 742, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2390, 0, 0, 0, 6, 0, 0, 4997, 0, 0, 0, 0, 0, 7358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 1, 0, 7987, 0, 0, 6, 0, 7831, 0, 0, 0, 6, 0, 0, 0, 0, 73, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 6, 8184, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8186, 1, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7205, 0, 0, 0, 0, 0, 968, 0, 0, 0, 0, 0, 21, 1, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 6, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 9, 12, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 24, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7147, 0, 6, 0, 0, 0, 970, 0, 0, 0, 0, 0, 42, 19, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 73, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7736, 0, 0, 0, 0, 0, 456, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7774, 0, 15, 0, 0, 0, 377, 0, 0, 0, 0, 0, 26, 36, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8170, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7881, 0, 0, 415, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 51, 12, 0, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 27, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8004, 8193, 8459, 8090, 8224, 8192, 8193, 8181] avg=8192.0000 max=8459 max_rank=2 min=8004 min_rank=0 imbalance_ratio=1.032593 max_load_reduction=57077 max_load_reduction_pct=0.870926 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 415, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 14, 1, 0, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 5, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8004, 8190, 8686, 7844, 8222, 8191, 8201, 8198] avg=8192.0000 max=8686 max_rank=2 min=7844 min_rank=3 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=32812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8179, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7205, 0, 0, 415, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 968, 0, 0, 0, 0, 0, 21, 1, 0, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 6, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8006, 8189, 7774, 8755, 8223, 8192, 8199, 8198] avg=8192.0000 max=8755 max_rank=3 min=7774 min_rank=2 imbalance_ratio=1.068726 max_load_reduction=56781 max_load_reduction_pct=0.866409 moved_to_echo=32812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 944, 0, 0, 0, 1, 0, 0, 0, 0, 0, 742, 0, 0, 7804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 3, 0, 0, 8172, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2390, 0, 0, 0, 6, 0, 0, 4997, 0, 0, 21, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 6, 0, 7831, 0, 0, 0, 6, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 6, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9491, 8191, 7563, 7760, 7941, 8194, 8198, 8198] avg=8192.0000 max=9491 max_rank=0 min=7563 min_rank=2 imbalance_ratio=1.158569 max_load_reduction=56045 max_load_reduction_pct=0.855179 moved_to_echo=32531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7736, 0, 0, 415, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8012, 8189, 8305, 8222, 8226, 8192, 8189, 8201] avg=8192.0000 max=8305 max_rank=2 min=8012 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 8166, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7147, 0, 6, 415, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 0, 0, 0, 0, 0, 42, 19, 0, 0, 7762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 73, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8005, 8197, 7733, 8793, 8269, 8192, 8180, 8167] avg=8192.0000 max=8793 max_rank=3 min=7733 min_rank=2 imbalance_ratio=1.073364 max_load_reduction=56743 max_load_reduction_pct=0.865829 moved_to_echo=32808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7774, 0, 15, 415, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 0, 0, 0, 0, 26, 36, 0, 0, 7762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8004, 8189, 8357, 8201, 8207, 8192, 8184, 8202] avg=8192.0000 max=8357 max_rank=2 min=8004 min_rank=0 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8183, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 418, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 9, 12, 0, 0, 7761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 24, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8010, 8198, 8659, 7871, 8224, 8191, 8192, 8191] avg=8192.0000 max=8659 max_rank=2 min=7871 min_rank=3 imbalance_ratio=1.057007 max_load_reduction=56877 max_load_reduction_pct=0.867874 moved_to_echo=32798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 2432, 0, 0, 0, 128, 0, 0, 59008, 0, 128, 2944, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 10624, 0, 0, 0, 0, 0, 256, 128, 0, 0, 54784, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 256, 0, 0, 128, 256, 65152, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 1024, 0, 0, 0, 128, 0, 0, 0, 0, 0, 768, 0, 0, 63872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 0, 0, 0, 0, 0, 0, 6439, 0, 8192, 0, 0, 0, 0, 0, 5103, 0, 0, 9, 0, 4642, 0, 0, 0, 0, 35, 0, 0, 11, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 149, 0, 0, 0, 0, 82] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 13, 0, 0, 0, 0, 0, 0, 5768, 0, 8192, 0, 0, 0, 0, 0, 8025, 0, 0, 10, 0, 1541, 0, 0, 0, 0, 32, 0, 0, 11, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1174, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 0, 0, 7391, 0, 8192, 0, 0, 0, 0, 0, 8049, 0, 0, 1, 0, 644, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 6975, 0, 8192, 0, 0, 0, 0, 0, 7611, 0, 0, 0, 0, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 0, 0, 105] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7889, 0, 0, 0, 0, 0, 6, 1416, 6169, 3, 8177, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 7204, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6, 2, 6, 5, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1617, 6, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 222] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 6900, 0, 8192, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 0, 0, 674] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 6529, 0, 8192, 0, 0, 0, 0, 0, 7414, 0, 0, 0, 0, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 0, 46] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 0, 0, 6023, 0, 8192, 0, 0, 0, 0, 0, 5889, 0, 0, 1, 0, 3025, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 0, 912] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 1, 6349, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2247, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 644, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 1, 5144, 903, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0, 0, 7399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7186, 10296, 6701, 8186, 8206, 9012, 8202, 7747] avg=8192.0000 max=10296 max_rank=1 min=6701 min_rank=2 imbalance_ratio=1.256836 max_load_reduction=55240 max_load_reduction_pct=0.842896 moved_to_echo=33167 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 6365, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 0, 0, 0, 0, 0, 0, 0, 5103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4642, 0, 0, 0, 0, 35, 0, 0, 11, 0, 0, 0, 8, 4481, 905, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 822, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 149, 0, 0, 0, 0, 82, 7399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7171, 7061, 10092, 8190, 8200, 9014, 8173, 7635] avg=8192.0000 max=10092 max_rank=2 min=7061 min_rank=1 imbalance_ratio=1.231934 max_load_reduction=55444 max_load_reduction_pct=0.846008 moved_to_echo=33022 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 6368, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 0, 0, 0, 0, 0, 0, 0, 7611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4854, 906, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 822, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 0, 0, 105, 7398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7196, 9732, 6894, 8186, 8207, 9014, 8196, 8111] avg=8192.0000 max=9732 max_rank=1 min=6894 min_rank=2 imbalance_ratio=1.187988 max_load_reduction=55804 max_load_reduction_pct=0.851501 moved_to_echo=33528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 13, 6234, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1754, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1541, 0, 0, 0, 0, 32, 0, 0, 11, 0, 0, 0, 9, 4014, 887, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1174, 0, 0, 0, 0, 0, 7399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7060, 9779, 6505, 8176, 8207, 8996, 8239, 8574] avg=8192.0000 max=9779 max_rank=1 min=6505 min_rank=2 imbalance_ratio=1.193726 max_load_reduction=55757 max_load_reduction_pct=0.850784 moved_to_echo=34016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 6418, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2098, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4802, 913, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 0, 0, 674, 7398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7219, 8246, 7315, 8186, 8207, 9020, 8189, 9154] avg=8192.0000 max=9154 max_rank=7 min=7219 min_rank=0 imbalance_ratio=1.117432 max_load_reduction=56382 max_load_reduction_pct=0.860321 moved_to_echo=34570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7889, 0, 6439, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1416, 1876, 3, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7204, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 4293, 916, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6, 2, 6, 5, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 831, 0, 0, 1617, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 222, 7384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15274, 3308, 12424, 8202, 8082, 2454, 8181, 7611] avg=8192.0000 max=15274 max_rank=0 min=2454 min_rank=5 imbalance_ratio=1.864502 max_load_reduction=50262 max_load_reduction_pct=0.766937 moved_to_echo=26328 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 6378, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1985, 0, 0, 0, 0, 0, 0, 0, 7414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4544, 907, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 0, 46, 7399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7214, 9399, 7449, 8227, 8207, 9015, 8187, 7838] avg=8192.0000 max=9399 max_rank=1 min=7214 min_rank=0 imbalance_ratio=1.147339 max_load_reduction=56137 max_load_reduction_pct=0.856583 moved_to_echo=33247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 6401, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1826, 0, 0, 0, 0, 0, 0, 0, 5889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3025, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 4197, 913, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 0, 912, 7398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7216, 7715, 8156, 8183, 8220, 9011, 8169, 8866] avg=8192.0000 max=9011 max_rank=5 min=7216 min_rank=0 imbalance_ratio=1.099976 max_load_reduction=56525 max_load_reduction_pct=0.862503 moved_to_echo=34266 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 128, 0, 0, 8064, 128, 51072, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 21888, 0, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 36352, 7296, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 1536, 15872, 128, 0, 0, 128, 0, 128, 0, 48256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 6656, 0, 0, 59008, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 4352, 128, 0, 0, 0, 2048, 59264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 8191, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8172, 8178, 0, 0, 8185, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 8176, 0, 0, 8182, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 8192, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8185, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 8185, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7396, 0, 0, 0, 0, 0, 6, 6, 0, 0, 6, 2, 815, 0, 1374, 0, 0, 6823, 7, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7908, 5456, 0, 0, 8184, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 5, 0, 0, 1519, 0, 8181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 0, 0, 8192, 5884, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8191, 0, 0, 8192, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8182, 8192, 0, 0, 8190, 7754, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6718, 1191, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8185, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1427, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7000, 8117, 8287, 8087, 8232, 9626, 8194, 7993] avg=8192.0000 max=9626 max_rank=5 min=7000 min_rank=0 imbalance_ratio=1.175049 max_load_reduction=55910 max_load_reduction_pct=0.853119 moved_to_echo=34045 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 6707, 1190, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8169, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1425, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6994, 8186, 8276, 8085, 8225, 9582, 8188, 8000] avg=8192.0000 max=9582 max_rank=5 min=6994 min_rank=0 imbalance_ratio=1.169678 max_load_reduction=55954 max_load_reduction_pct=0.853790 moved_to_echo=33995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6718, 1191, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1428, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7001, 8129, 8295, 8086, 8233, 9605, 8195, 7992] avg=8192.0000 max=9605 max_rank=5 min=7001 min_rank=0 imbalance_ratio=1.172485 max_load_reduction=55931 max_load_reduction_pct=0.853439 moved_to_echo=34025 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 6705, 1189, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1425, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6992, 8130, 8294, 8086, 8205, 9642, 8185, 8002] avg=8192.0000 max=9642 max_rank=5 min=6992 min_rank=0 imbalance_ratio=1.177002 max_load_reduction=55894 max_load_reduction_pct=0.852875 moved_to_echo=34034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6717, 1191, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1427, 0, 0, 0, 5884, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7001, 10421, 8292, 8086, 8234, 7316, 8194, 7992] avg=8192.0000 max=10421 max_rank=1 min=7001 min_rank=0 imbalance_ratio=1.272095 max_load_reduction=55115 max_load_reduction_pct=0.840988 moved_to_echo=31736 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6995, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4474, 1190, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7396, 0, 0, 0, 0, 0, 6, 6, 0, 0, 6, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 815, 0, 1374, 0, 0, 6736, 7, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7906, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 5, 0, 0, 1519, 0, 7981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16551, 5866, 7502, 8942, 7946, 962, 8193, 9574] avg=8192.0000 max=16551 max_rank=0 min=962 min_rank=5 imbalance_ratio=2.020386 max_load_reduction=48985 max_load_reduction_pct=0.747452 moved_to_echo=26675 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6718, 1191, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1427, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7001, 8116, 8297, 8086, 8230, 9617, 8195, 7994] avg=8192.0000 max=9617 max_rank=5 min=7001 min_rank=0 imbalance_ratio=1.173950 max_load_reduction=55919 max_load_reduction_pct=0.853256 moved_to_echo=34036 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6723, 1194, 204, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8180, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1419, 0, 0, 0, 7754, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6996, 8571, 8293, 8078, 8231, 9186, 8192, 7989] avg=8192.0000 max=9186 max_rank=5 min=6996 min_rank=0 imbalance_ratio=1.121338 max_load_reduction=56350 max_load_reduction_pct=0.859833 moved_to_echo=33598 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 2944, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 51584, 9600, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 11008, 0, 0, 0, 54528, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 56064, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 64768, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 896, 0, 1408, 0, 0, 63360, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 1536, 0, 64000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8178, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5538, 0, 0, 0, 0, 3339, 0, 0, 0, 0, 0, 0, 0, 40, 8148, 0, 0, 0, 0, 12, 3, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8104, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5378, 0, 0, 0, 0, 3265, 0, 0, 0, 0, 0, 0, 0, 17, 7982, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8047, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 6619, 0, 0, 0, 0, 2378, 0, 0, 0, 0, 0, 0, 0, 40, 7739, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8123, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5714, 0, 0, 0, 0, 2768, 0, 0, 0, 0, 0, 0, 0, 1, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 7427, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4197, 0, 0, 0, 0, 5476, 0, 0, 0, 0, 0, 0, 0, 13, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 850, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 69, 0, 0, 0, 0, 0, 2, 0, 0, 90, 0, 0, 200, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 2, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 8095, 0, 16, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5711, 7, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 3, 2, 0, 7353, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1203, 0, 0, 0, 0, 0, 5, 0, 6, 7, 5, 0, 1201, 0, 7989, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8184, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1918, 0, 0, 0, 0, 6496, 0, 0, 0, 0, 0, 0, 0, 11, 8190, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8146, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4507, 0, 0, 0, 0, 3766, 0, 0, 0, 0, 0, 0, 0, 8, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8104, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6799, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 5378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3265, 0, 0, 0, 0, 0, 0, 0, 17, 3686, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 7837, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4139, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9157, 8142, 9333, 6970, 8207, 8449, 7093, 8185] avg=8192.0000 max=9333 max_rank=2 min=6970 min_rank=3 imbalance_ratio=1.139282 max_load_reduction=56203 max_load_reduction_pct=0.857590 moved_to_echo=31934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8178, 0, 0, 0, 0, 0, 0, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6791, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 5538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 0, 0, 0, 0, 0, 0, 40, 3762, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 7819, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4139, 3015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9229, 8162, 9493, 7153, 8184, 8499, 7154, 7662] avg=8192.0000 max=9493 max_rank=2 min=7153 min_rank=3 imbalance_ratio=1.158813 max_load_reduction=56043 max_load_reduction_pct=0.855148 moved_to_echo=31499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8123, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6800, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 5714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2768, 0, 0, 0, 0, 0, 0, 0, 1, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 7843, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4139, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9176, 8147, 9669, 6464, 8201, 8523, 7102, 8254] avg=8192.0000 max=9669 max_rank=2 min=6464 min_rank=3 imbalance_ratio=1.180298 max_load_reduction=55867 max_load_reduction_pct=0.852463 moved_to_echo=32080 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7427, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6801, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 4197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5476, 0, 0, 0, 0, 0, 0, 0, 13, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7838, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4139, 3013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8480, 8172, 8152, 9248, 8183, 8530, 7153, 7618] avg=8192.0000 max=9248 max_rank=3 min=7153 min_rank=6 imbalance_ratio=1.128906 max_load_reduction=56288 max_load_reduction_pct=0.858887 moved_to_echo=31484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8047, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6786, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 6619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2378, 0, 0, 0, 0, 0, 0, 0, 40, 3573, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 7819, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4139, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9097, 8095, 10574, 6013, 8188, 8406, 7002, 8161] avg=8192.0000 max=10574 max_rank=2 min=6013 min_rank=3 imbalance_ratio=1.290771 max_load_reduction=54962 max_load_reduction_pct=0.838654 moved_to_echo=31757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 850, 0, 0, 0, 0, 0, 0, 1052, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 90, 0, 0, 200, 6, 0, 0, 0, 0, 0, 0, 0, 6794, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 2, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 16, 0, 0, 0, 0, 0, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7837, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5711, 7, 0, 0, 0, 0, 0, 347, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7353, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1203, 0, 0, 0, 0, 0, 4131, 3004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 7, 5, 0, 1201, 0, 7989, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1971, 8459, 3984, 11859, 8189, 6070, 15694, 9310] avg=8192.0000 max=15694 max_rank=6 min=1971 min_rank=0 imbalance_ratio=1.915771 max_load_reduction=49842 max_load_reduction_pct=0.760529 moved_to_echo=39263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8146, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6801, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 4507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 0, 0, 0, 0, 0, 0, 8, 3771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7840, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4139, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9199, 8176, 8462, 7545, 8184, 8540, 7161, 8269] avg=8192.0000 max=9199 max_rank=0 min=7161 min_rank=6 imbalance_ratio=1.122925 max_load_reduction=56337 max_load_reduction_pct=0.859634 moved_to_echo=32154 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8184, 0, 0, 0, 0, 0, 0, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6801, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 1918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6496, 0, 0, 0, 0, 0, 0, 0, 11, 3773, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 7838, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4142, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9227, 8183, 5869, 10284, 8200, 8519, 7177, 8077] avg=8192.0000 max=10284 max_rank=3 min=5869 min_rank=2 imbalance_ratio=1.255371 max_load_reduction=55252 max_load_reduction_pct=0.843079 moved_to_echo=31973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7424, 0, 0, 0, 0, 0, 128, 0, 0, 0, 1280, 0, 0, 0, 0, 0, 33152, 23936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 31744, 0, 128, 0, 0, 0, 0, 0, 33920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 35584, 0, 128, 0, 0, 0, 0, 0, 256, 29824, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 54400, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 57088, 0, 0, 0, 0, 0, 0, 8448, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 62848, 128, 0, 0, 0, 0, 0, 2816, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 62720, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 55552, 0, 1280, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8190, 0, 0, 7464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8190, 0, 0, 7321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8181, 0, 0, 7761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8175, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8187, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 8184, 0, 0, 7487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 58, 8168, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8094, 56, 5, 0, 8184, 0, 8079, 9, 6, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 2, 0, 30, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4765, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 7901, 2, 0, 0, 8, 0, 1, 0, 164, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 8006, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 992, 0, 0, 7464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6596, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4334, 0, 0, 0, 0, 0, 0, 1116, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8180, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7024, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8456, 8190, 10846, 6052, 8193, 7390, 8213] avg=8192.0000 max=10846 max_rank=3 min=6052 min_rank=4 imbalance_ratio=1.323975 max_load_reduction=54690 max_load_reduction_pct=0.834503 moved_to_echo=29848 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 0, 0, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6598, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 1118, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8182, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8819, 8190, 7023, 9678, 8190, 7230, 8213] avg=8192.0000 max=9678 max_rank=4 min=7023 min_rank=3 imbalance_ratio=1.181396 max_load_reduction=55858 max_load_reduction_pct=0.852325 moved_to_echo=33311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 992, 0, 0, 7321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6596, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7729, 0, 0, 0, 0, 0, 0, 1119, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8181, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8313, 8190, 7711, 9450, 8189, 7274, 8213] avg=8192.0000 max=9450 max_rank=4 min=7274 min_rank=6 imbalance_ratio=1.153564 max_load_reduction=56086 max_load_reduction_pct=0.855804 moved_to_echo=33126 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 0, 0, 7487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6591, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7143, 0, 0, 0, 0, 0, 0, 1116, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8180, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7025, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8478, 8195, 8303, 8863, 8185, 7103, 8213] avg=8192.0000 max=8863 max_rank=4 min=7103 min_rank=6 imbalance_ratio=1.081909 max_load_reduction=56673 max_load_reduction_pct=0.864761 moved_to_echo=32364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 7761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6589, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5100, 0, 0, 0, 0, 0, 0, 1116, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8180, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7026, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8751, 8200, 10104, 6818, 8185, 7060, 8213] avg=8192.0000 max=10104 max_rank=3 min=6818 min_rank=4 imbalance_ratio=1.233398 max_load_reduction=55432 max_load_reduction_pct=0.845825 moved_to_echo=30276 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 5, 0, 0, 0, 978, 9, 6, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 30, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6507, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4765, 0, 0, 1, 0, 0, 0, 1080, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 8160, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 6801, 2, 0, 0, 8, 0, 1, 0, 164, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 8006, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 4748, 8190, 6551, 6461, 8224, 15169, 8039] avg=8192.0000 max=15169 max_rank=6 min=4748 min_rank=1 imbalance_ratio=1.851685 max_load_reduction=50367 max_load_reduction_pct=0.768539 moved_to_echo=37893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6584, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7164, 0, 0, 0, 0, 0, 0, 1118, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8875, 8189, 7768, 8883, 8183, 7224, 8213] avg=8192.0000 max=8883 max_rank=4 min=7224 min_rank=6 imbalance_ratio=1.084351 max_load_reduction=56653 max_load_reduction_pct=0.864456 moved_to_echo=32503 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6597, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7601, 0, 0, 0, 0, 0, 0, 1123, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 9096, 8192, 7230, 9331, 8187, 7086, 8219] avg=8192.0000 max=9331 max_rank=4 min=7086 min_rank=6 imbalance_ratio=1.139038 max_load_reduction=56205 max_load_reduction_pct=0.857620 moved_to_echo=32823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 12800, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 52736, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 51840, 0, 0, 128, 0, 0, 0, 8960, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 56064, 128, 0, 0, 128, 0, 128, 0, 256, 128, 0, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 7936, 128, 128, 57600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65408, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 115, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 6, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 157, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 179, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 33, 0, 8182, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8191, 334, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 266, 0, 0, 0, 0, 7660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 1, 0, 0, 5038, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 6512, 5, 2, 0, 7419, 0, 1, 0, 0, 2432, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 9, 4, 0, 8098, 6, 0, 0, 0, 0, 0, 0, 0, 6, 8166, 0, 1971, 8183, 0, 0, 6, 3, 0, 0, 0, 0, 99, 0, 0, 0, 0, 1087, 0, 0, 6, 0, 0, 0, 0, 6, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 148, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 86, 0, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 8181, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 577, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 16, 0, 0, 0, 0, 7608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 478, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 23, 0, 0, 0, 0, 7729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 283, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 1369, 0, 0, 0, 0, 6545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 115, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 6, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 26, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8182, 8430, 8144, 8194, 7949, 8250, 8193] avg=8192.0000 max=8430 max_rank=2 min=7949 min_rank=5 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 33, 0, 8171, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8191, 334, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 266, 0, 0, 0, 0, 7660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 26, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8210, 8657, 7954, 8140, 7938, 8250, 8193] avg=8192.0000 max=8657 max_rank=2 min=7938 min_rank=5 imbalance_ratio=1.056763 max_load_reduction=56879 max_load_reduction_pct=0.867905 moved_to_echo=32521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 157, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 179, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 26, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8185, 8473, 8069, 8221, 7951, 8250, 8193] avg=8192.0000 max=8473 max_rank=2 min=7951 min_rank=5 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=32615 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8166, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 1, 0, 0, 5038, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 5, 2, 0, 7419, 0, 1, 0, 0, 2432, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 0, 8098, 6, 0, 0, 0, 0, 26, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1971, 7942, 0, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 1087, 0, 0, 6, 0, 0, 0, 0, 6, 0, 7, 6487, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 8163, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8183, 5220, 9914, 8154, 9928, 7781, 8179] avg=8192.0000 max=9928 max_rank=5 min=5220 min_rank=2 imbalance_ratio=1.211914 max_load_reduction=55608 max_load_reduction_pct=0.848511 moved_to_echo=34042 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 148, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 86, 0, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 26, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8181, 8460, 8092, 8215, 7951, 8250, 8193] avg=8192.0000 max=8460 max_rank=2 min=7951 min_rank=5 imbalance_ratio=1.032715 max_load_reduction=57076 max_load_reduction_pct=0.870911 moved_to_echo=32609 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 8170, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 577, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 16, 0, 0, 0, 0, 7608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 26, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8210, 8897, 7652, 8211, 7930, 8249, 8193] avg=8192.0000 max=8897 max_rank=2 min=7652 min_rank=3 imbalance_ratio=1.086060 max_load_reduction=56639 max_load_reduction_pct=0.864243 moved_to_echo=32583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 478, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 23, 0, 0, 0, 0, 7729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 26, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8200, 8796, 7780, 8172, 7951, 8250, 8193] avg=8192.0000 max=8796 max_rank=2 min=7780 min_rank=3 imbalance_ratio=1.073730 max_load_reduction=56740 max_load_reduction_pct=0.865784 moved_to_echo=32566 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 283, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1369, 0, 0, 0, 0, 6545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 31, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8185, 8603, 7931, 8229, 7938, 8256, 8199] avg=8192.0000 max=8603 max_rank=2 min=7931 min_rank=3 imbalance_ratio=1.050171 max_load_reduction=56933 max_load_reduction_pct=0.868729 moved_to_echo=32622 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 62464, 2176, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 256, 128, 128, 0, 9472, 0, 128, 0, 0, 55936, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 2048, 63616, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 1152, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 63744, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 65280, 128, 0, 0, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 75, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 45, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 8108, 0, 0, 161, 0, 0, 7, 48, 8186, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7378, 0, 0, 0, 0, 0, 1, 0, 0, 336, 0, 0, 0, 0, 0, 0, 8180, 10, 0, 411, 0, 2, 0, 2, 6, 0, 6, 0, 0, 8180, 0, 6, 1, 0, 0, 0, 8009, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 97, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5385, 0, 0, 0, 0, 0, 0, 0, 75, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 57, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5431, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5463, 8191, 8299, 8151, 10852, 8207, 8181, 8192] avg=8192.0000 max=10852 max_rank=4 min=5463 min_rank=0 imbalance_ratio=1.324707 max_load_reduction=54684 max_load_reduction_pct=0.834412 moved_to_echo=35432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5385, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 57, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5433, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5469, 8192, 8291, 8151, 10843, 8215, 8183, 8192] avg=8192.0000 max=10843 max_rank=4 min=5469 min_rank=0 imbalance_ratio=1.323608 max_load_reduction=54693 max_load_reduction_pct=0.834549 moved_to_echo=35433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5385, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 57, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5433, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5887, 8192, 8290, 8151, 10425, 8216, 8183, 8192] avg=8192.0000 max=10425 max_rank=4 min=5887 min_rank=0 imbalance_ratio=1.272583 max_load_reduction=55111 max_load_reduction_pct=0.840927 moved_to_echo=35016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 5329, 0, 0, 161, 0, 0, 7, 48, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4, 7, 0, 0, 8144, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7378, 0, 0, 57, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 336, 0, 0, 0, 0, 0, 0, 8139, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 2, 0, 2, 6, 0, 6, 0, 0, 2732, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 7, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5425, 2722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13732, 8189, 7483, 8486, 3166, 8040, 8249, 8191] avg=8192.0000 max=13732 max_rank=0 min=3166 min_rank=4 imbalance_ratio=1.676270 max_load_reduction=51804 max_load_reduction_pct=0.790466 moved_to_echo=27646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5385, 0, 0, 0, 0, 0, 0, 0, 45, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 57, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5433, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5433, 8192, 8290, 8151, 10882, 8213, 8183, 8192] avg=8192.0000 max=10882 max_rank=4 min=5433 min_rank=0 imbalance_ratio=1.328369 max_load_reduction=54654 max_load_reduction_pct=0.833954 moved_to_echo=35470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5385, 0, 0, 0, 0, 0, 0, 0, 7033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 57, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5433, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12418, 8192, 8290, 8151, 3894, 8216, 8183, 8192] avg=8192.0000 max=12418 max_rank=0 min=3894 min_rank=4 imbalance_ratio=1.515869 max_load_reduction=53118 max_load_reduction_pct=0.810516 moved_to_echo=28485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5376, 0, 0, 0, 0, 0, 0, 0, 5967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 63, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5438, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11343, 8196, 8303, 8144, 4953, 8213, 8191, 8193] avg=8192.0000 max=11343 max_rank=0 min=4953 min_rank=4 imbalance_ratio=1.384644 max_load_reduction=54193 max_load_reduction_pct=0.826920 moved_to_echo=29550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5385, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 57, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5433, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5791, 8192, 8290, 8151, 10521, 8216, 8183, 8192] avg=8192.0000 max=10521 max_rank=4 min=5791 min_rank=0 imbalance_ratio=1.284302 max_load_reduction=55015 max_load_reduction_pct=0.839462 moved_to_echo=35112 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 43520, 22016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 43136, 0, 0, 256, 0, 0, 128, 128, 22400, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[43648, 0, 128, 0, 128, 128, 0, 128, 0, 0, 21888, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 64768, 0, 128, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 384, 0, 0, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 device_initiated_grouped_gemm=True + [2026-08-26 16:28:33] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 437.9 | throughput per GPU (TFLOP/s/GPU): 412.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.303469E+00 | loss scale: 1.0 | grad norm: 4.843 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:28:33 +W0826 16:28:33.993000 216687 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 16:28:33.994000 216687 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216781 closing signal SIGTERM +W0826 16:28:33.996000 216687 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216782 closing signal SIGTERM +W0826 16:28:33.998000 216687 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216783 closing signal SIGTERM +W0826 16:28:34.001000 216687 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216784 closing signal SIGTERM +W0826 16:28:34.002000 216687 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216785 closing signal SIGTERM +W0826 16:28:34.003000 216687 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216786 closing signal SIGTERM +W0826 16:28:34.004000 216687 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216787 closing signal SIGTERM +W0826 16:28:34.006000 216687 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216788 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 216687 got signal: 15 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..5ed6e5fbaa18623ce7320fae433c2dba920ed3ae --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv @@ -0,0 +1,240 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 47 +2, 4 +3, 629 +4, 629 +5, 629 +6, 629 +7, 4 +0, 909 +1, 889 +2, 897 +3, 899 +4, 897 +5, 893 +6, 905 +7, 907 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 47803 +1, 47803 +2, 47803 +3, 47803 +4, 47515 +5, 47803 +6, 47803 +7, 47803 +0, 49055 +1, 49055 +2, 49055 +3, 49055 +4, 48767 +5, 49055 +6, 49055 +7, 49055 +0, 49927 +1, 49927 +2, 49927 +3, 49927 +4, 49639 +5, 49927 +6, 49927 +7, 49927 +0, 53697 +1, 53697 +2, 53697 +3, 53697 +4, 53409 +5, 53697 +6, 53697 +7, 53697 +0, 54105 +1, 54105 +2, 54105 +3, 54105 +4, 53817 +5, 54105 +6, 54105 +7, 54105 +0, 54451 +1, 54443 +2, 54489 +3, 54479 +4, 54179 +5, 54459 +6, 54527 +7, 54471 +0, 55543 +1, 55543 +2, 55543 +3, 55543 +4, 55255 +5, 55543 +6, 55543 +7, 55543 +0, 56183 +1, 56183 +2, 56183 +3, 56183 +4, 55895 +5, 56183 +6, 56183 +7, 56183 +0, 58095 +1, 58095 +2, 58095 +3, 58095 +4, 57807 +5, 58095 +6, 58095 +7, 58095 +0, 58095 +1, 58095 +2, 58095 +3, 58095 +4, 57807 +5, 58095 +6, 58095 +7, 58095 +0, 98817 +1, 98913 +2, 98913 +3, 98915 +4, 98871 +5, 98905 +6, 98911 +7, 98897 +0, 99355 +1, 99407 +2, 99407 +3, 99407 +4, 99377 +5, 99407 +6, 99407 +7, 99407 +0, 99357 +1, 99409 +2, 99409 +3, 99409 +4, 99379 +5, 99409 +6, 99409 +7, 99409 +0, 99357 +1, 99409 +2, 99409 +3, 99409 +4, 99379 +5, 99409 +6, 99409 +7, 99409 +0, 99357 +1, 99409 +2, 99409 +3, 99409 +4, 99379 +5, 99409 +6, 99409 +7, 99409 +0, 103409 +1, 103461 +2, 103461 +3, 103461 +4, 103431 +5, 103461 +6, 103461 +7, 103461 +0, 104057 +1, 104109 +2, 104109 +3, 104109 +4, 104079 +5, 104109 +6, 104109 +7, 104109 +0, 108145 +1, 108271 +2, 108171 +3, 108343 +4, 108331 +5, 108227 +6, 108239 +7, 108207 +0, 134577 +1, 134699 +2, 134699 +3, 134699 +4, 134627 +5, 134699 +6, 134699 +7, 134699 +0, 134577 +1, 134699 +2, 134699 +3, 134699 +4, 134627 +5, 134699 +6, 134699 +7, 134699 +0, 134577 +1, 134699 +2, 134699 +3, 134699 +4, 134627 +5, 134699 +6, 134699 +7, 134699 +0, 3963 +1, 3966 +2, 3966 +3, 3966 +4, 3966 +5, 4286 +6, 134697 +7, 5566 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..2ea6fda9c8bc3bed2d023671e2b75d3b36747351 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f2b994a6b1dde06cbbd08d3a373752c4376ab1a7 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log @@ -0,0 +1,7182 @@ +[16:28:40] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[16:28:40] WARN: token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0826 16:28:41.867000 220093 torch/distributed/run.py:766] +W0826 16:28:41.867000 220093 torch/distributed/run.py:766] ***************************************** +W0826 16:28:41.867000 220093 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:28:41.867000 220093 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7[Gloo] Rank peer ranks. 1Expected number of connected peer ranks is : is connected to 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 76 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 7 +2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 6457 is connected to is connected to is connected to is connected to 7777 peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 +77 + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 16:29:00.129027120 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 16:29:00.129056323 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:29:00.134392732 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 16:29:00.134485525 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 16:29:00.135786598 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:29:00.135931544 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 16:29:00.136896657 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 16:29:00.281895242 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.641 seconds +time to initialize megatron (seconds): 7.133 +[after megatron is initialized] datetime: 2026-08-26 16:29:05 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4898003968 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.router.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:29:06 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 16:29:06 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (690.21, 701.19) + train/valid/test-data-iterators-setup ..........: (58.07, 74.26) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 16:29:06 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[532, 472, 529, 506, 506, 502, 527, 532, 546, 504, 526, 493, 514, 534, 524, 489, 540, 495, 531, 473, 519, 462, 523, 505, 519, 489, 536, 529, 509, 526, 542, 477, 536, 503, 521, 484, 552, 494, 504, 514, 483, 497, 531, 504, 519, 531, 502, 520, 498, 539, 516, 497, 512, 501, 519, 524, 501, 483, 522, 501, 491, 486, 517, 495, 526, 517, 514, 522, 540, 532, 508, 532, 517, 554, 503, 526, 503, 478, 521, 541, 527, 504, 488, 545, 473, 515, 493, 535, 490, 490, 535, 523, 513, 491, 516, 491, 540, 515, 484, 522, 555, 529, 481, 498, 523, 488, 539, 528, 521, 513, 497, 479, 502, 511, 551, 536, 487, 496, 532, 517, 514, 467, 495, 500, 521, 480, 520, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[492, 513, 547, 511, 525, 519, 499, 511, 486, 527, 506, 511, 518, 481, 484, 548, 478, 518, 525, 538, 495, 506, 540, 488, 530, 503, 524, 528, 480, 519, 464, 531, 500, 503, 519, 531, 516, 511, 477, 476, 498, 523, 507, 508, 508, 512, 518, 539, 492, 487, 534, 511, 537, 521, 501, 539, 547, 489, 512, 477, 521, 505, 515, 539, 562, 478, 513, 519, 501, 481, 518, 467, 509, 529, 510, 504, 519, 544, 527, 469, 524, 499, 510, 541, 529, 502, 589, 488, 475, 493, 530, 528, 483, 508, 514, 478, 521, 516, 488, 489, 473, 512, 517, 524, 497, 540, 505, 514, 523, 512, 533, 522, 504, 537, 513, 488, 546, 519, 498, 500, 552, 509, 513, 526, 499, 507, 563, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[473, 498, 483, 518, 506, 509, 534, 532, 487, 539, 526, 520, 508, 538, 511, 497, 494, 446, 523, 508, 528, 473, 503, 480, 503, 529, 505, 555, 527, 503, 513, 539, 526, 522, 530, 480, 558, 526, 510, 518, 544, 551, 522, 539, 554, 515, 514, 536, 449, 521, 476, 517, 501, 512, 468, 533, 533, 467, 520, 498, 500, 515, 507, 491, 502, 511, 549, 516, 514, 486, 525, 518, 526, 530, 522, 513, 523, 499, 503, 512, 498, 549, 516, 453, 509, 510, 487, 514, 485, 497, 538, 518, 539, 496, 519, 495, 485, 489, 533, 503, 520, 512, 520, 538, 502, 523, 508, 522, 516, 501, 488, 514, 473, 497, 521, 514, 566, 512, 539, 504, 514, 506, 509, 517, 508, 513, 515, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[547, 561, 479, 501, 514, 493, 555, 514, 519, 489, 506, 510, 497, 510, 491, 540, 529, 486, 517, 494, 543, 463, 497, 544, 497, 491, 526, 522, 500, 505, 544, 502, 504, 538, 482, 481, 542, 518, 498, 510, 495, 520, 487, 534, 569, 520, 543, 501, 526, 519, 510, 500, 495, 507, 518, 492, 537, 458, 523, 522, 530, 512, 491, 530, 474, 496, 534, 508, 509, 562, 531, 512, 540, 508, 513, 498, 483, 493, 535, 553, 493, 505, 539, 540, 538, 533, 495, 508, 515, 505, 528, 490, 496, 517, 536, 499, 512, 503, 482, 483, 532, 499, 547, 567, 505, 462, 502, 542, 491, 501, 493, 497, 525, 526, 495, 494, 484, 515, 497, 501, 471, 519, 515, 539, 506, 530, 537, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[506, 484, 511, 472, 493, 545, 499, 527, 521, 500, 483, 556, 525, 501, 507, 536, 542, 525, 534, 525, 527, 498, 510, 502, 492, 516, 530, 510, 465, 483, 487, 498, 523, 479, 508, 518, 516, 529, 505, 509, 489, 500, 534, 540, 513, 483, 533, 496, 540, 499, 513, 552, 527, 504, 528, 490, 540, 543, 519, 528, 547, 494, 521, 490, 492, 531, 487, 528, 500, 499, 502, 488, 538, 541, 528, 491, 474, 477, 483, 540, 495, 510, 523, 514, 492, 500, 515, 492, 529, 535, 554, 565, 535, 489, 521, 522, 486, 521, 500, 481, 454, 490, 553, 543, 507, 554, 517, 465, 536, 478, 460, 517, 537, 485, 511, 497, 547, 490, 503, 542, 519, 505, 516, 551, 525, 525, 494, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[494, 464, 509, 533, 538, 500, 492, 503, 530, 529, 512, 568, 517, 484, 506, 503, 523, 518, 484, 493, 516, 519, 489, 517, 484, 541, 504, 534, 466, 525, 569, 491, 534, 555, 502, 525, 517, 518, 520, 537, 512, 527, 509, 537, 504, 483, 527, 511, 533, 538, 512, 472, 510, 513, 525, 491, 514, 479, 510, 537, 497, 516, 475, 508, 514, 500, 507, 516, 473, 513, 490, 516, 523, 532, 506, 517, 553, 482, 492, 528, 544, 509, 521, 516, 482, 504, 540, 564, 487, 526, 545, 527, 492, 530, 531, 518, 454, 511, 481, 472, 491, 501, 500, 468, 554, 482, 480, 501, 561, 513, 534, 495, 501, 503, 518, 533, 525, 497, 532, 569, 525, 503, 497, 488, 517, 537, 475, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[500, 467, 517, 489, 503, 534, 522, 497, 512, 493, 530, 489, 524, 465, 562, 535, 552, 517, 480, 492, 488, 509, 520, 535, 502, 546, 524, 529, 537, 550, 528, 503, 513, 522, 477, 501, 541, 555, 515, 455, 507, 481, 512, 541, 529, 486, 560, 550, 486, 525, 500, 503, 474, 533, 561, 518, 494, 502, 535, 501, 542, 518, 508, 512, 510, 487, 495, 530, 493, 500, 522, 534, 481, 543, 532, 542, 538, 496, 492, 544, 516, 482, 502, 497, 508, 517, 500, 490, 493, 518, 476, 518, 497, 482, 510, 526, 555, 485, 504, 502, 499, 509, 507, 479, 533, 490, 519, 512, 529, 483, 533, 531, 530, 528, 512, 504, 512, 507, 476, 502, 502, 513, 525, 522, 513, 529, 494, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[507, 535, 534, 494, 526, 480, 505, 507, 513, 490, 467, 529, 510, 495, 509, 548, 461, 490, 531, 509, 533, 535, 501, 518, 516, 477, 541, 516, 520, 534, 505, 519, 538, 509, 512, 473, 514, 494, 482, 466, 557, 523, 535, 464, 539, 536, 501, 486, 519, 530, 517, 460, 496, 544, 512, 529, 534, 483, 493, 480, 522, 511, 509, 540, 513, 529, 503, 536, 483, 509, 484, 516, 484, 503, 559, 539, 513, 519, 506, 504, 483, 469, 522, 472, 514, 543, 496, 497, 533, 508, 516, 487, 552, 506, 501, 507, 561, 489, 509, 534, 500, 533, 528, 521, 527, 522, 503, 540, 506, 483, 554, 498, 549, 518, 533, 488, 556, 474, 510, 493, 499, 521, 522, 490, 519, 528, 523, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[492, 513, 547, 511, 525, 519, 499, 511, 486, 527, 506, 511, 518, 481, 484, 548, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 518, 525, 538, 495, 506, 540, 488, 530, 503, 524, 528, 480, 519, 464, 531, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 519, 531, 467, 511, 477, 476, 498, 523, 507, 508, 508, 512, 518, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 487, 534, 511, 537, 521, 501, 539, 547, 489, 512, 477, 521, 505, 515, 539, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 478, 513, 519, 501, 481, 518, 467, 509, 503, 510, 504, 519, 544, 527, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 499, 510, 541, 529, 502, 589, 488, 475, 493, 530, 528, 483, 508, 514, 478, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 488, 489, 473, 512, 517, 524, 497, 540, 505, 514, 523, 512, 533, 522, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 537, 513, 488, 507, 519, 498, 500, 552, 509, 513, 526, 499, 507, 563, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8180, 8097, 8255, 8124, 8206, 8226, 8252] avg=8192.0000 max=8255 max_rank=3 min=8097 min_rank=2 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[473, 498, 483, 518, 506, 509, 534, 532, 487, 539, 526, 520, 508, 538, 511, 497, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 446, 523, 508, 528, 473, 503, 480, 503, 529, 505, 555, 527, 503, 513, 539, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 522, 530, 480, 512, 526, 510, 518, 544, 551, 522, 539, 554, 515, 514, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 521, 476, 517, 501, 512, 468, 533, 533, 467, 520, 498, 500, 515, 507, 491, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 549, 516, 514, 486, 525, 518, 526, 513, 522, 513, 523, 499, 503, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 549, 516, 453, 509, 510, 487, 514, 485, 497, 538, 518, 539, 496, 519, 495, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 489, 533, 503, 520, 512, 520, 538, 502, 523, 508, 522, 516, 501, 488, 514, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 497, 521, 514, 538, 512, 539, 504, 514, 506, 509, 517, 508, 513, 515, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8137, 8399, 8030, 8232, 8135, 8214, 8201] avg=8192.0000 max=8399 max_rank=2 min=8030 min_rank=3 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[547, 561, 479, 501, 514, 493, 555, 514, 519, 489, 506, 510, 497, 510, 491, 540, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 486, 517, 494, 543, 463, 497, 544, 497, 491, 526, 522, 500, 505, 544, 502, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 538, 482, 481, 497, 518, 498, 510, 495, 520, 487, 534, 569, 520, 543, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 519, 510, 500, 495, 507, 518, 492, 537, 458, 523, 522, 530, 512, 491, 530, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 496, 534, 508, 509, 562, 531, 512, 540, 491, 513, 498, 483, 493, 535, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 539, 540, 538, 533, 495, 508, 515, 505, 528, 490, 496, 517, 536, 499, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 503, 482, 483, 532, 499, 547, 567, 505, 462, 502, 542, 491, 501, 493, 497, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 495, 494, 461, 515, 497, 501, 471, 519, 515, 539, 506, 530, 537, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8168, 8197, 8189, 8232, 8247, 8157, 8111] avg=8192.0000 max=8247 max_rank=5 min=8111 min_rank=7 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[500, 467, 517, 489, 503, 534, 522, 497, 512, 493, 530, 489, 524, 465, 562, 535, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 517, 480, 492, 488, 509, 520, 535, 502, 546, 524, 529, 537, 550, 528, 503, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 477, 501, 496, 555, 515, 455, 507, 481, 512, 541, 529, 486, 560, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 525, 500, 503, 474, 533, 561, 518, 494, 502, 535, 501, 542, 518, 508, 512, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 495, 530, 493, 500, 522, 534, 481, 526, 532, 542, 538, 496, 492, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 482, 502, 497, 508, 517, 500, 490, 493, 518, 476, 518, 497, 482, 510, 526, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 485, 504, 502, 499, 509, 507, 479, 533, 490, 519, 512, 529, 483, 533, 531, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 512, 504, 487, 507, 476, 502, 502, 513, 525, 522, 513, 529, 494, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8320, 8200, 8232, 8222, 8043, 8209, 8162] avg=8192.0000 max=8320 max_rank=1 min=8043 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[532, 472, 529, 506, 506, 502, 527, 532, 546, 504, 526, 493, 514, 534, 524, 489, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 495, 531, 473, 519, 462, 523, 505, 519, 489, 536, 529, 509, 526, 542, 477, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 503, 521, 484, 507, 494, 504, 514, 483, 497, 531, 504, 519, 531, 502, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 539, 516, 497, 512, 501, 519, 524, 501, 483, 522, 501, 491, 486, 517, 495, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 514, 522, 540, 532, 508, 532, 517, 536, 503, 526, 503, 478, 521, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 504, 488, 545, 473, 515, 493, 535, 490, 490, 535, 523, 513, 491, 516, 491, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 515, 484, 522, 555, 529, 481, 498, 523, 488, 539, 528, 521, 513, 497, 479, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 551, 536, 463, 496, 532, 517, 514, 467, 495, 500, 521, 480, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8184, 8150, 8122, 8316, 8139, 8251, 8129] avg=8192.0000 max=8316 max_rank=4 min=8122 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 484, 511, 472, 493, 545, 499, 527, 521, 500, 483, 556, 525, 501, 507, 536, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 525, 534, 525, 527, 498, 510, 502, 492, 516, 530, 510, 465, 483, 487, 498, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 479, 508, 518, 473, 529, 505, 509, 489, 500, 534, 540, 513, 483, 533, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 499, 513, 552, 527, 504, 528, 490, 540, 543, 519, 528, 547, 494, 521, 490, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 531, 487, 528, 500, 499, 502, 488, 538, 524, 528, 491, 474, 477, 483, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 510, 523, 514, 492, 500, 515, 492, 529, 535, 554, 565, 535, 489, 521, 522, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 521, 500, 481, 454, 490, 553, 543, 507, 554, 517, 465, 536, 478, 460, 517, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 485, 511, 497, 521, 490, 503, 542, 519, 505, 516, 551, 525, 525, 494, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8152, 8132, 8356, 8082, 8302, 8099, 8238] avg=8192.0000 max=8356 max_rank=3 min=8082 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 464, 509, 533, 538, 500, 492, 503, 530, 529, 512, 568, 517, 484, 506, 503, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 518, 484, 493, 516, 519, 489, 517, 484, 541, 504, 534, 466, 525, 569, 491, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 555, 502, 525, 474, 518, 520, 537, 512, 527, 509, 537, 504, 483, 527, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 538, 512, 472, 510, 513, 525, 491, 514, 479, 510, 537, 497, 516, 475, 508, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 507, 516, 473, 513, 490, 516, 523, 515, 506, 517, 553, 482, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 509, 521, 516, 482, 504, 540, 564, 487, 526, 545, 527, 492, 530, 531, 518, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 511, 481, 472, 491, 501, 500, 468, 554, 482, 480, 501, 561, 513, 534, 495, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 503, 518, 533, 499, 497, 532, 569, 525, 503, 497, 488, 517, 537, 475, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8181, 8275, 8151, 8145, 8347, 8035, 8211] avg=8192.0000 max=8347 max_rank=5 min=8035 min_rank=6 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 535, 534, 494, 526, 480, 505, 507, 513, 490, 467, 529, 510, 495, 509, 548, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 490, 531, 509, 533, 535, 501, 518, 516, 477, 541, 516, 520, 534, 505, 519, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 509, 512, 473, 471, 494, 482, 466, 557, 523, 535, 464, 539, 536, 501, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 517, 460, 496, 544, 512, 529, 534, 483, 493, 480, 522, 511, 509, 540, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 503, 536, 483, 509, 484, 516, 484, 486, 559, 539, 513, 519, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 469, 522, 472, 514, 543, 496, 497, 533, 508, 516, 487, 552, 506, 501, 507, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 489, 509, 534, 500, 533, 528, 521, 527, 522, 503, 540, 506, 483, 554, 498, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 518, 533, 488, 529, 474, 510, 493, 499, 521, 522, 490, 519, 528, 523, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8214, 8086, 8201, 8183, 8117, 8345, 8232] avg=8192.0000 max=8345 max_rank=6 min=8086 min_rank=2 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4352, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[540, 532, 508, 492, 543, 482, 546, 512, 520, 492, 533, 500, 505, 524, 534, 525, 541, 490, 500, 475, 506, 526, 512, 467, 509, 480, 499, 516, 530, 527, 512, 528, 522, 509, 509, 500, 521, 504, 540, 518, 511, 536, 528, 517, 496, 517, 459, 482, 536, 559, 521, 536, 495, 518, 498, 553, 489, 462, 519, 538, 513, 529, 481, 537, 508, 531, 527, 533, 522, 461, 509, 492, 535, 566, 489, 473, 520, 493, 497, 517, 531, 548, 513, 502, 515, 503, 507, 527, 522, 544, 510, 474, 527, 529, 506, 500, 518, 515, 493, 497, 489, 540, 514, 517, 474, 507, 503, 497, 530, 522, 509, 526, 533, 493, 487, 492, 494, 545, 536, 501, 498, 489, 507, 512, 485, 496, 502, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[521, 509, 523, 501, 487, 485, 538, 523, 530, 486, 477, 506, 484, 575, 509, 508, 510, 528, 523, 498, 540, 510, 495, 518, 519, 508, 503, 473, 505, 524, 529, 532, 556, 528, 522, 538, 531, 517, 509, 535, 513, 528, 536, 515, 514, 523, 461, 507, 475, 517, 517, 490, 498, 527, 506, 516, 522, 529, 520, 519, 541, 504, 506, 498, 472, 508, 500, 541, 530, 515, 532, 537, 528, 513, 482, 517, 548, 522, 536, 558, 503, 491, 497, 512, 502, 479, 524, 489, 510, 491, 520, 484, 494, 521, 518, 522, 558, 526, 539, 503, 536, 503, 517, 478, 479, 480, 517, 490, 485, 493, 535, 521, 520, 506, 504, 517, 544, 509, 469, 500, 505, 482, 524, 485, 487, 529, 518, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[507, 517, 525, 523, 531, 511, 533, 495, 531, 509, 536, 493, 557, 522, 519, 547, 509, 527, 492, 507, 505, 508, 465, 512, 501, 502, 543, 477, 507, 518, 496, 515, 482, 483, 504, 477, 526, 515, 490, 499, 547, 499, 488, 505, 562, 520, 523, 499, 517, 511, 499, 526, 535, 526, 515, 507, 539, 547, 500, 476, 559, 492, 520, 489, 516, 530, 555, 505, 504, 470, 558, 486, 508, 515, 555, 482, 520, 507, 512, 508, 529, 510, 516, 478, 504, 496, 503, 533, 505, 534, 503, 511, 506, 518, 507, 507, 505, 530, 521, 512, 512, 496, 518, 547, 500, 503, 459, 523, 524, 513, 562, 470, 518, 497, 507, 526, 530, 504, 522, 499, 497, 512, 480, 460, 498, 554, 552, 477] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[483, 491, 551, 490, 497, 521, 523, 534, 501, 524, 500, 480, 538, 478, 518, 537, 538, 534, 520, 539, 524, 498, 504, 549, 525, 492, 531, 531, 526, 527, 455, 532, 505, 511, 530, 481, 489, 564, 531, 473, 515, 508, 505, 558, 506, 514, 534, 465, 554, 508, 500, 541, 471, 524, 511, 493, 508, 518, 498, 526, 521, 515, 512, 527, 506, 526, 520, 521, 514, 479, 520, 517, 520, 505, 536, 499, 494, 514, 509, 502, 535, 508, 493, 487, 521, 501, 519, 502, 477, 540, 488, 476, 566, 495, 505, 527, 524, 508, 493, 514, 483, 510, 493, 512, 515, 518, 526, 529, 531, 493, 516, 525, 476, 531, 514, 511, 566, 506, 534, 513, 470, 501, 522, 506, 493, 457, 509, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[514, 509, 507, 506, 503, 502, 491, 518, 502, 501, 506, 500, 505, 549, 499, 549, 498, 530, 505, 493, 513, 543, 477, 500, 491, 527, 520, 544, 523, 509, 484, 519, 461, 498, 554, 502, 515, 497, 490, 536, 520, 501, 502, 507, 494, 504, 541, 506, 544, 498, 488, 506, 512, 503, 504, 500, 515, 533, 509, 537, 537, 546, 482, 523, 506, 514, 543, 501, 485, 529, 495, 498, 510, 507, 517, 532, 533, 545, 500, 500, 508, 498, 462, 534, 558, 515, 466, 546, 544, 479, 519, 555, 492, 506, 493, 507, 517, 532, 547, 522, 521, 523, 483, 516, 489, 486, 535, 512, 534, 530, 491, 501, 521, 483, 545, 515, 511, 518, 513, 505, 469, 543, 519, 514, 510, 510, 505, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[571, 517, 539, 519, 506, 474, 503, 518, 539, 527, 535, 516, 517, 525, 500, 463, 469, 488, 502, 494, 571, 495, 477, 502, 516, 521, 475, 509, 560, 501, 518, 533, 529, 525, 512, 524, 501, 530, 491, 534, 487, 524, 503, 527, 513, 529, 492, 547, 505, 520, 489, 529, 505, 500, 511, 532, 523, 510, 550, 511, 520, 514, 518, 477, 524, 503, 520, 547, 511, 493, 518, 497, 542, 510, 490, 498, 491, 500, 483, 503, 483, 525, 523, 491, 475, 512, 528, 529, 481, 531, 511, 526, 467, 546, 550, 505, 535, 501, 523, 510, 480, 522, 517, 557, 519, 508, 491, 516, 529, 515, 478, 517, 517, 519, 529, 512, 478, 523, 516, 461, 486, 488, 511, 524, 565, 490, 493, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[494, 480, 506, 487, 504, 542, 511, 536, 512, 523, 487, 497, 543, 471, 487, 515, 505, 560, 505, 530, 517, 508, 539, 523, 489, 485, 492, 482, 571, 534, 532, 509, 522, 523, 479, 526, 493, 492, 535, 472, 510, 522, 501, 545, 520, 469, 525, 543, 538, 522, 507, 486, 517, 508, 510, 503, 482, 509, 498, 547, 580, 513, 495, 525, 509, 525, 482, 511, 518, 530, 521, 479, 497, 519, 536, 480, 497, 475, 535, 475, 528, 508, 510, 501, 506, 499, 497, 509, 533, 504, 532, 522, 517, 566, 498, 537, 483, 518, 495, 552, 532, 481, 520, 529, 509, 506, 539, 529, 526, 516, 490, 514, 500, 502, 525, 534, 529, 507, 508, 491, 501, 492, 524, 549, 495, 510, 451, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[536, 543, 547, 541, 479, 550, 512, 528, 511, 468, 504, 526, 524, 521, 515, 510, 469, 550, 511, 561, 528, 499, 529, 562, 484, 491, 525, 512, 515, 512, 472, 530, 492, 540, 484, 489, 481, 515, 502, 511, 527, 538, 501, 533, 513, 509, 489, 518, 543, 508, 514, 512, 510, 489, 511, 451, 531, 514, 514, 542, 495, 532, 497, 520, 491, 521, 513, 505, 524, 506, 503, 512, 515, 567, 521, 471, 490, 520, 489, 538, 495, 505, 520, 496, 522, 499, 487, 523, 537, 526, 520, 495, 542, 506, 520, 551, 490, 521, 514, 485, 519, 505, 488, 520, 493, 517, 513, 480, 509, 465, 489, 516, 490, 505, 510, 501, 538, 492, 527, 545, 510, 543, 514, 517, 511, 451, 525, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 509, 490, 501, 487, 485, 538, 523, 530, 486, 477, 506, 484, 575, 509, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 528, 523, 498, 540, 510, 495, 518, 519, 508, 503, 473, 500, 524, 529, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 528, 522, 538, 531, 517, 509, 535, 513, 528, 536, 515, 514, 523, 461, 507, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 517, 517, 490, 498, 527, 506, 516, 522, 529, 520, 519, 504, 504, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 508, 500, 541, 530, 515, 532, 537, 528, 512, 482, 517, 548, 522, 536, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 497, 512, 502, 479, 524, 489, 510, 491, 520, 484, 494, 521, 518, 522, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 526, 539, 503, 536, 503, 517, 478, 479, 480, 517, 490, 485, 493, 535, 521, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 506, 504, 517, 544, 509, 469, 500, 505, 482, 524, 485, 487, 529, 518, 486, 37, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8210, 8334, 8148, 8338, 8062, 8174, 8141] avg=8192.0000 max=8338 max_rank=4 min=8062 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[536, 543, 513, 541, 479, 550, 512, 528, 511, 468, 504, 526, 524, 521, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 550, 511, 561, 528, 499, 529, 562, 484, 491, 525, 512, 510, 512, 472, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 540, 484, 489, 481, 515, 502, 511, 527, 538, 501, 533, 513, 509, 489, 518, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 508, 514, 512, 510, 489, 511, 451, 531, 514, 514, 542, 462, 532, 497, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 521, 513, 505, 524, 506, 503, 512, 515, 566, 521, 471, 490, 520, 489, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 505, 520, 496, 522, 499, 487, 523, 537, 526, 520, 495, 542, 506, 520, 551, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 521, 514, 485, 519, 505, 488, 520, 493, 517, 513, 480, 509, 465, 489, 516, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 510, 501, 538, 492, 527, 545, 510, 543, 514, 517, 511, 451, 525, 513, 33, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8245, 8143, 8150, 8185, 8249, 8039, 8244] avg=8192.0000 max=8281 max_rank=0 min=8039 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[571, 517, 505, 519, 506, 474, 503, 518, 539, 527, 535, 516, 517, 525, 500, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 488, 502, 494, 571, 495, 477, 502, 516, 521, 475, 509, 555, 501, 518, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 525, 512, 524, 501, 530, 491, 534, 487, 524, 503, 527, 513, 529, 492, 547, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 520, 489, 529, 505, 500, 511, 532, 523, 510, 550, 511, 485, 514, 518, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 503, 520, 547, 511, 493, 518, 497, 542, 509, 490, 498, 491, 500, 483, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 525, 523, 491, 475, 512, 528, 529, 481, 531, 511, 526, 467, 546, 550, 505, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 501, 523, 510, 480, 522, 517, 557, 519, 508, 491, 516, 529, 515, 478, 517, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 529, 512, 478, 523, 516, 461, 486, 488, 511, 524, 565, 490, 493, 511, 35, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8126, 8269, 8179, 8129, 8188, 8233, 8177] avg=8192.0000 max=8269 max_rank=2 min=8126 min_rank=1 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[540, 532, 465, 492, 543, 482, 546, 512, 520, 492, 533, 500, 505, 524, 534, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 490, 500, 475, 506, 526, 512, 467, 509, 480, 499, 516, 521, 527, 512, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 509, 509, 500, 521, 504, 540, 518, 511, 536, 528, 517, 496, 517, 459, 482, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 559, 521, 536, 495, 518, 498, 553, 489, 462, 519, 538, 474, 529, 481, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 531, 527, 533, 522, 461, 509, 492, 535, 564, 489, 473, 520, 493, 497, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 548, 513, 502, 515, 503, 507, 527, 522, 544, 510, 474, 527, 529, 506, 500, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 515, 493, 497, 489, 540, 514, 517, 474, 507, 503, 497, 530, 522, 509, 526, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 487, 492, 494, 545, 536, 501, 498, 489, 507, 512, 485, 496, 502, 525, 39, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8109, 8173, 8245, 8171, 8268, 8170, 8155] avg=8192.0000 max=8268 max_rank=5 min=8109 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[483, 491, 516, 490, 497, 521, 523, 534, 501, 524, 500, 480, 538, 478, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 534, 520, 539, 524, 498, 504, 549, 525, 492, 531, 531, 521, 527, 455, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 530, 481, 489, 564, 531, 473, 515, 508, 505, 558, 506, 514, 534, 465, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 508, 500, 541, 471, 524, 511, 493, 508, 518, 498, 526, 486, 515, 512, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 526, 520, 521, 514, 479, 520, 517, 520, 504, 536, 499, 494, 514, 509, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 508, 493, 487, 521, 501, 519, 502, 477, 540, 488, 476, 566, 495, 505, 527, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 508, 493, 514, 483, 510, 493, 512, 515, 518, 526, 529, 531, 493, 516, 525, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 531, 514, 511, 566, 506, 534, 513, 470, 501, 522, 506, 493, 457, 509, 508, 35, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8320, 8190, 8192, 8181, 8145, 8205, 8172] avg=8192.0000 max=8320 max_rank=1 min=8131 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 509, 475, 506, 503, 502, 491, 518, 502, 501, 506, 500, 505, 549, 499, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 530, 505, 493, 513, 543, 477, 500, 491, 527, 520, 544, 518, 509, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 498, 554, 502, 515, 497, 490, 536, 520, 501, 502, 507, 494, 504, 541, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 498, 488, 506, 512, 503, 504, 500, 515, 533, 509, 537, 501, 546, 482, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 543, 501, 485, 529, 495, 498, 510, 506, 517, 532, 533, 545, 500, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 498, 462, 534, 558, 515, 466, 546, 544, 479, 519, 555, 492, 506, 493, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 532, 547, 522, 521, 523, 483, 516, 489, 486, 535, 512, 534, 530, 491, 501, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 483, 545, 515, 511, 518, 513, 505, 469, 543, 519, 514, 510, 510, 505, 517, 36, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8171, 8129, 8201, 8214, 8187, 8253, 8252] avg=8192.0000 max=8253 max_rank=6 min=8129 min_rank=0 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[507, 517, 492, 523, 531, 511, 533, 495, 531, 509, 536, 493, 557, 522, 519, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 492, 507, 505, 508, 465, 512, 501, 502, 543, 477, 502, 518, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 483, 504, 477, 526, 515, 490, 499, 547, 499, 488, 505, 562, 520, 523, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 499, 526, 535, 526, 515, 507, 539, 547, 500, 476, 521, 492, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 530, 555, 505, 504, 470, 558, 486, 508, 514, 555, 482, 520, 507, 512, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 510, 516, 478, 504, 496, 503, 533, 505, 534, 503, 511, 506, 518, 507, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 530, 521, 512, 512, 496, 518, 547, 500, 503, 459, 523, 524, 513, 562, 470, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 497, 507, 526, 530, 504, 522, 499, 497, 512, 480, 460, 498, 554, 552, 477, 38, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8079, 8120, 8220, 8230, 8165, 8209, 8190] avg=8192.0000 max=8323 max_rank=0 min=8079 min_rank=1 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 480, 474, 487, 504, 542, 511, 536, 512, 523, 487, 497, 543, 471, 487, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 560, 505, 530, 517, 508, 539, 523, 489, 485, 492, 482, 566, 534, 532, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 523, 479, 526, 493, 492, 535, 472, 510, 522, 501, 545, 520, 469, 525, 543, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 522, 507, 486, 517, 508, 510, 503, 482, 509, 498, 547, 541, 513, 495, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 482, 511, 518, 530, 521, 479, 497, 518, 536, 480, 497, 475, 535, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 508, 510, 501, 506, 499, 497, 509, 533, 504, 532, 522, 517, 566, 498, 537, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 495, 552, 532, 481, 520, 529, 509, 506, 539, 529, 526, 516, 490, 514, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 525, 534, 529, 507, 508, 491, 501, 492, 524, 549, 495, 510, 451, 530, 39, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8063, 8276, 8178, 8201, 8088, 8272, 8253, 8205] avg=8192.0000 max=8276 max_rank=1 min=8063 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[513, 489, 515, 483, 561, 514, 534, 468, 531, 502, 480, 513, 489, 500, 520, 514, 526, 527, 557, 472, 497, 479, 480, 527, 523, 501, 498, 496, 501, 486, 508, 507, 547, 517, 494, 525, 480, 523, 482, 494, 500, 538, 505, 504, 498, 495, 530, 519, 513, 500, 488, 523, 506, 514, 488, 523, 497, 549, 531, 471, 510, 553, 532, 533, 538, 508, 539, 505, 524, 516, 526, 533, 519, 528, 538, 507, 489, 516, 508, 518, 510, 479, 492, 513, 510, 483, 500, 490, 515, 548, 495, 518, 497, 525, 525, 521, 499, 532, 491, 517, 496, 550, 544, 532, 533, 541, 534, 515, 515, 509, 510, 547, 520, 504, 537, 484, 522, 536, 505, 504, 518, 530, 523, 496, 486, 508, 496, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[494, 533, 502, 542, 532, 553, 501, 511, 495, 487, 498, 501, 547, 522, 527, 525, 478, 499, 532, 535, 575, 495, 493, 488, 500, 551, 494, 512, 484, 489, 522, 521, 530, 521, 512, 504, 505, 501, 554, 485, 496, 504, 503, 562, 516, 519, 539, 516, 488, 519, 462, 507, 499, 509, 524, 568, 561, 528, 543, 537, 518, 490, 507, 499, 510, 488, 503, 482, 545, 507, 534, 517, 516, 516, 560, 494, 495, 484, 520, 516, 509, 497, 522, 505, 488, 518, 487, 519, 518, 507, 499, 469, 491, 494, 482, 524, 511, 525, 507, 477, 502, 503, 508, 520, 546, 521, 516, 519, 523, 503, 503, 526, 516, 515, 490, 463, 530, 520, 540, 496, 510, 510, 495, 501, 527, 499, 515, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[501, 502, 500, 506, 487, 506, 508, 486, 508, 503, 489, 517, 494, 544, 513, 523, 488, 514, 531, 551, 504, 498, 523, 518, 523, 529, 536, 521, 511, 512, 570, 527, 524, 533, 535, 509, 527, 509, 498, 504, 538, 502, 464, 512, 512, 496, 516, 482, 488, 482, 543, 502, 496, 477, 490, 501, 489, 489, 550, 533, 522, 538, 526, 523, 519, 560, 553, 504, 528, 511, 528, 493, 490, 542, 511, 506, 541, 491, 525, 504, 476, 533, 484, 474, 562, 508, 504, 485, 508, 530, 510, 513, 483, 526, 513, 491, 550, 524, 519, 502, 541, 510, 507, 520, 527, 498, 512, 487, 512, 534, 483, 491, 535, 538, 552, 540, 489, 507, 483, 500, 512, 499, 502, 503, 488, 525, 515, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[506, 511, 504, 525, 523, 482, 522, 515, 533, 536, 518, 530, 489, 507, 527, 532, 515, 523, 533, 542, 533, 498, 542, 535, 477, 486, 515, 509, 512, 472, 529, 529, 502, 511, 537, 524, 499, 503, 511, 499, 522, 511, 497, 498, 506, 535, 516, 539, 524, 548, 520, 536, 501, 476, 550, 477, 496, 507, 551, 474, 512, 492, 499, 530, 539, 527, 502, 528, 521, 508, 534, 519, 488, 482, 531, 509, 521, 492, 513, 492, 529, 529, 561, 515, 483, 489, 504, 487, 479, 489, 535, 498, 512, 521, 538, 503, 522, 495, 504, 507, 497, 520, 492, 494, 505, 514, 480, 524, 512, 501, 528, 485, 509, 509, 498, 558, 485, 475, 481, 525, 532, 499, 496, 499, 514, 505, 538, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[505, 522, 513, 526, 509, 518, 508, 513, 547, 495, 486, 526, 526, 535, 547, 502, 455, 481, 558, 494, 516, 548, 517, 490, 511, 475, 515, 531, 525, 508, 491, 497, 495, 522, 515, 468, 480, 499, 532, 521, 514, 546, 502, 494, 513, 523, 511, 525, 524, 527, 488, 519, 489, 498, 517, 519, 495, 492, 510, 519, 496, 488, 521, 549, 566, 512, 490, 529, 496, 494, 494, 524, 529, 490, 549, 523, 525, 479, 517, 512, 542, 518, 514, 486, 527, 528, 525, 529, 492, 534, 501, 521, 517, 529, 515, 519, 506, 565, 539, 471, 535, 518, 484, 525, 501, 482, 500, 520, 533, 524, 524, 505, 521, 455, 477, 531, 521, 527, 511, 473, 502, 504, 528, 494, 516, 495, 515, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[515, 487, 509, 499, 489, 476, 482, 507, 524, 502, 511, 497, 531, 511, 490, 492, 530, 479, 535, 499, 463, 499, 514, 524, 521, 516, 488, 493, 542, 517, 480, 507, 505, 495, 542, 506, 537, 540, 517, 522, 518, 513, 494, 550, 496, 516, 553, 498, 549, 516, 479, 520, 525, 525, 505, 512, 502, 496, 513, 519, 554, 533, 544, 464, 521, 531, 579, 532, 485, 492, 537, 535, 512, 494, 541, 498, 457, 528, 536, 531, 475, 509, 497, 494, 506, 485, 488, 539, 513, 479, 526, 499, 483, 532, 524, 548, 529, 505, 506, 521, 514, 515, 506, 514, 455, 517, 495, 525, 551, 517, 510, 518, 491, 521, 490, 546, 497, 520, 484, 523, 544, 550, 529, 510, 519, 525, 492, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[532, 508, 513, 471, 540, 523, 543, 523, 535, 487, 516, 495, 521, 514, 496, 536, 537, 525, 497, 534, 496, 524, 499, 534, 479, 541, 513, 525, 491, 497, 534, 509, 471, 529, 447, 489, 522, 505, 474, 523, 513, 519, 556, 495, 577, 514, 494, 523, 523, 498, 504, 545, 499, 514, 522, 502, 512, 572, 491, 519, 534, 515, 465, 502, 511, 545, 519, 519, 549, 495, 462, 517, 515, 491, 489, 504, 534, 489, 523, 520, 547, 536, 534, 504, 512, 515, 470, 533, 515, 490, 505, 507, 546, 543, 497, 517, 506, 540, 478, 516, 479, 477, 529, 476, 545, 484, 538, 503, 469, 470, 498, 527, 527, 493, 500, 492, 513, 545, 520, 514, 496, 530, 469, 535, 547, 511, 474, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[546, 513, 525, 506, 501, 518, 510, 545, 501, 524, 463, 556, 506, 532, 521, 499, 522, 486, 541, 487, 512, 524, 560, 514, 515, 493, 493, 463, 521, 469, 490, 500, 517, 501, 476, 497, 544, 550, 489, 544, 528, 549, 512, 533, 512, 500, 541, 487, 486, 462, 532, 545, 470, 484, 474, 508, 514, 514, 494, 524, 504, 551, 523, 535, 504, 501, 566, 516, 503, 492, 506, 540, 549, 530, 496, 501, 487, 518, 534, 496, 473, 542, 543, 530, 490, 543, 477, 536, 460, 536, 505, 470, 500, 527, 487, 543, 513, 501, 509, 527, 532, 479, 497, 497, 507, 514, 488, 518, 552, 494, 514, 510, 527, 529, 478, 516, 492, 477, 491, 523, 539, 535, 534, 512, 530, 502, 514, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[515, 487, 509, 499, 489, 476, 482, 507, 521, 502, 511, 497, 531, 511, 490, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 479, 535, 499, 463, 499, 514, 524, 521, 516, 488, 493, 542, 517, 480, 507, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 542, 506, 537, 540, 517, 522, 518, 513, 494, 550, 496, 516, 534, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 516, 479, 516, 525, 525, 505, 512, 502, 496, 513, 519, 554, 533, 544, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 531, 520, 532, 485, 492, 537, 535, 512, 494, 541, 498, 457, 528, 536, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 509, 497, 494, 506, 485, 488, 539, 513, 479, 526, 499, 483, 532, 524, 548, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 505, 506, 521, 514, 515, 506, 514, 455, 517, 495, 525, 551, 517, 510, 518, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 521, 490, 546, 497, 520, 484, 523, 544, 550, 529, 510, 519, 525, 492, 504, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 8124, 8283, 8252, 8250, 8136, 8203, 8269] avg=8192.0000 max=8283 max_rank=2 min=8019 min_rank=0 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 533, 502, 542, 532, 553, 501, 511, 492, 487, 498, 501, 547, 522, 527, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 499, 532, 535, 575, 495, 493, 488, 500, 551, 494, 512, 484, 489, 522, 521, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 521, 512, 504, 505, 501, 554, 485, 496, 504, 503, 562, 516, 519, 521, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 519, 462, 503, 499, 509, 524, 568, 561, 528, 543, 537, 518, 490, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 453, 482, 545, 507, 534, 517, 516, 516, 560, 494, 495, 484, 520, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 497, 522, 505, 488, 518, 487, 519, 518, 507, 499, 469, 491, 494, 482, 524, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 525, 507, 477, 502, 503, 508, 520, 546, 521, 516, 519, 523, 503, 503, 526, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 490, 463, 530, 520, 540, 496, 510, 510, 495, 501, 527, 499, 515, 519, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8184, 8249, 8255, 8137, 8062, 8215, 8167] avg=8192.0000 max=8267 max_rank=0 min=8062 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[513, 489, 515, 483, 561, 514, 534, 468, 528, 502, 480, 513, 489, 500, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 557, 472, 497, 479, 480, 527, 523, 501, 498, 496, 501, 486, 508, 507, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 517, 494, 525, 480, 523, 482, 494, 500, 538, 505, 504, 498, 495, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 488, 519, 506, 514, 488, 523, 497, 549, 531, 471, 510, 553, 532, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 508, 485, 505, 524, 516, 526, 533, 519, 528, 538, 507, 489, 516, 508, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 479, 492, 513, 510, 483, 500, 490, 515, 548, 495, 518, 497, 525, 525, 521, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 491, 517, 496, 550, 544, 532, 533, 541, 534, 515, 515, 509, 510, 547, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 504, 537, 484, 522, 536, 505, 504, 518, 530, 523, 496, 486, 508, 496, 476, 18, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8101, 8133, 8227, 8258, 8157, 8370, 8167] avg=8192.0000 max=8370 max_rank=6 min=8101 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32952 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[532, 508, 513, 471, 540, 523, 543, 523, 532, 487, 516, 495, 521, 514, 496, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 497, 534, 496, 524, 499, 534, 479, 541, 513, 525, 491, 497, 534, 509, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 529, 447, 489, 522, 505, 474, 523, 513, 519, 556, 495, 577, 514, 478, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 498, 504, 541, 499, 514, 522, 502, 512, 572, 491, 519, 534, 515, 465, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 545, 467, 519, 549, 495, 462, 517, 515, 491, 489, 504, 534, 489, 523, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 536, 534, 504, 512, 515, 470, 533, 515, 490, 505, 507, 546, 543, 497, 517, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 540, 478, 516, 479, 477, 529, 476, 545, 484, 538, 503, 469, 470, 498, 527, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 493, 500, 492, 513, 545, 520, 514, 496, 530, 469, 535, 547, 511, 474, 526, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8250, 8135, 8213, 8130, 8306, 8040, 8212] avg=8192.0000 max=8306 max_rank=5 min=8040 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[505, 522, 513, 526, 509, 518, 508, 513, 544, 495, 486, 526, 526, 535, 547, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 481, 558, 494, 516, 548, 517, 490, 511, 475, 515, 531, 525, 508, 491, 497, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 515, 468, 480, 499, 532, 521, 514, 546, 502, 494, 513, 523, 495, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 488, 515, 489, 498, 517, 519, 495, 492, 510, 519, 496, 488, 521, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 512, 441, 529, 496, 494, 494, 524, 529, 490, 549, 523, 525, 479, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 514, 486, 527, 528, 525, 529, 492, 534, 501, 521, 517, 529, 515, 519, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 565, 539, 471, 535, 518, 484, 525, 501, 482, 500, 520, 533, 524, 524, 505, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 455, 477, 531, 521, 527, 511, 473, 502, 504, 528, 494, 516, 495, 515, 507, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8127, 8144, 8147, 8180, 8330, 8237, 8096] avg=8192.0000 max=8330 max_rank=5 min=8096 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[501, 502, 500, 506, 487, 506, 508, 486, 503, 503, 489, 517, 494, 544, 513, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 514, 531, 551, 504, 498, 523, 518, 523, 529, 536, 521, 511, 512, 570, 527, 17, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 533, 535, 509, 527, 509, 498, 504, 538, 502, 464, 512, 512, 496, 492, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 482, 543, 490, 496, 477, 490, 501, 489, 489, 550, 533, 522, 538, 526, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 560, 490, 504, 528, 511, 528, 493, 490, 542, 511, 506, 541, 491, 525, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 533, 484, 474, 562, 508, 504, 485, 508, 530, 510, 513, 483, 526, 513, 491, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 524, 519, 502, 541, 510, 507, 520, 527, 498, 512, 487, 512, 534, 483, 491, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 538, 552, 540, 489, 507, 483, 500, 512, 499, 502, 503, 488, 525, 515, 472, 23, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8378, 8137, 8137, 8243, 8140, 8229, 8190] avg=8192.0000 max=8378 max_rank=1 min=8082 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 511, 504, 525, 523, 482, 522, 515, 530, 536, 518, 530, 489, 507, 527, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 523, 533, 542, 533, 498, 542, 535, 477, 486, 515, 509, 512, 472, 529, 529, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 537, 524, 499, 503, 511, 499, 522, 511, 497, 498, 506, 535, 499, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 548, 520, 532, 501, 476, 550, 477, 496, 507, 551, 474, 512, 492, 499, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 527, 452, 528, 521, 508, 534, 519, 488, 482, 531, 509, 521, 492, 513, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 529, 561, 515, 483, 489, 504, 487, 479, 489, 535, 498, 512, 521, 538, 503, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 504, 507, 497, 520, 492, 494, 505, 514, 480, 524, 512, 501, 528, 485, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 498, 558, 485, 475, 481, 525, 532, 499, 496, 499, 514, 505, 538, 542, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8266, 8193, 8189, 8156, 8205, 8085, 8185] avg=8192.0000 max=8266 max_rank=1 min=8085 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[546, 513, 525, 506, 501, 518, 510, 545, 498, 524, 463, 556, 506, 532, 521, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 486, 541, 487, 512, 524, 560, 514, 515, 493, 493, 463, 521, 469, 490, 500, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 501, 476, 497, 544, 550, 489, 544, 528, 549, 512, 533, 512, 500, 523, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 462, 532, 541, 470, 484, 474, 508, 514, 514, 494, 524, 504, 551, 523, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 509, 516, 503, 492, 506, 540, 549, 530, 496, 501, 487, 518, 534, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 542, 543, 530, 490, 543, 477, 536, 460, 536, 505, 470, 500, 527, 487, 543, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 509, 527, 532, 479, 497, 497, 507, 514, 488, 518, 552, 494, 514, 510, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 529, 478, 516, 492, 477, 491, 523, 539, 535, 534, 512, 530, 502, 514, 528, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8106, 8262, 8116, 8182, 8200, 8157, 8250] avg=8192.0000 max=8263 max_rank=0 min=8106 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 3840, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4352, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 3968, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[515, 448, 549, 537, 508, 503, 531, 504, 497, 492, 525, 523, 506, 523, 532, 517, 502, 466, 510, 518, 468, 498, 533, 503, 484, 549, 554, 511, 504, 494, 523, 482, 505, 513, 533, 530, 528, 535, 481, 498, 498, 540, 505, 529, 502, 511, 512, 501, 510, 511, 515, 504, 521, 510, 511, 522, 532, 534, 487, 526, 516, 481, 510, 488, 496, 506, 501, 497, 517, 533, 548, 551, 522, 532, 546, 501, 501, 510, 499, 500, 477, 548, 527, 542, 519, 509, 531, 528, 513, 526, 501, 496, 498, 512, 498, 515, 496, 526, 478, 516, 486, 491, 533, 495, 501, 533, 550, 481, 524, 556, 523, 527, 521, 521, 505, 472, 508, 524, 515, 511, 486, 498, 511, 490, 517, 503, 535, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[496, 521, 520, 485, 495, 503, 473, 538, 491, 452, 497, 504, 514, 456, 520, 540, 501, 504, 485, 511, 498, 531, 524, 506, 530, 510, 497, 487, 515, 516, 502, 530, 504, 497, 497, 550, 491, 490, 517, 494, 511, 528, 533, 499, 552, 500, 493, 509, 508, 485, 544, 543, 518, 498, 520, 488, 477, 525, 556, 469, 530, 536, 528, 524, 504, 517, 483, 487, 509, 538, 539, 510, 511, 551, 500, 538, 493, 528, 510, 513, 494, 557, 512, 533, 514, 515, 531, 536, 521, 522, 515, 502, 527, 497, 508, 570, 501, 528, 508, 523, 496, 464, 526, 508, 519, 510, 507, 519, 549, 491, 506, 540, 510, 489, 516, 514, 517, 488, 532, 513, 491, 502, 483, 546, 532, 529, 524, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[509, 542, 543, 496, 476, 540, 498, 500, 526, 530, 515, 494, 515, 545, 525, 511, 531, 459, 524, 513, 511, 511, 495, 508, 509, 505, 516, 490, 538, 519, 507, 519, 487, 485, 526, 492, 530, 576, 518, 498, 492, 514, 536, 480, 519, 495, 532, 517, 510, 512, 517, 505, 527, 527, 508, 487, 452, 527, 483, 513, 503, 508, 514, 492, 511, 553, 473, 494, 521, 501, 510, 512, 503, 531, 508, 542, 517, 591, 527, 515, 520, 518, 532, 480, 493, 510, 483, 525, 542, 516, 521, 504, 526, 508, 566, 519, 527, 464, 521, 552, 525, 483, 492, 485, 506, 499, 529, 478, 519, 496, 511, 480, 528, 541, 532, 486, 505, 465, 497, 487, 513, 492, 543, 549, 481, 530, 525, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[536, 511, 487, 525, 483, 516, 503, 490, 483, 531, 508, 521, 533, 519, 479, 503, 538, 504, 496, 516, 470, 489, 484, 487, 530, 499, 548, 471, 525, 511, 486, 572, 544, 564, 559, 478, 546, 508, 502, 514, 527, 532, 540, 499, 505, 506, 515, 518, 511, 519, 529, 494, 503, 507, 556, 532, 494, 524, 509, 487, 504, 501, 493, 501, 547, 518, 542, 492, 518, 511, 536, 507, 533, 509, 503, 526, 514, 527, 533, 521, 512, 517, 500, 534, 506, 511, 531, 506, 506, 508, 502, 512, 502, 534, 568, 526, 540, 498, 512, 496, 479, 499, 508, 495, 535, 499, 496, 536, 495, 502, 506, 472, 456, 519, 536, 501, 529, 488, 504, 529, 490, 493, 505, 496, 551, 499, 473, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[514, 551, 503, 511, 497, 504, 506, 471, 504, 527, 512, 531, 530, 489, 489, 527, 533, 529, 499, 527, 484, 498, 541, 511, 526, 525, 509, 507, 511, 508, 535, 531, 537, 498, 491, 501, 492, 508, 505, 487, 525, 513, 509, 506, 500, 506, 508, 520, 537, 496, 517, 529, 503, 517, 510, 522, 519, 507, 520, 518, 491, 546, 470, 476, 490, 532, 505, 540, 512, 526, 516, 510, 505, 510, 525, 483, 506, 513, 535, 484, 521, 565, 500, 495, 506, 489, 534, 524, 515, 511, 509, 474, 525, 548, 477, 512, 492, 543, 530, 536, 516, 538, 501, 472, 486, 529, 482, 517, 519, 506, 519, 508, 510, 499, 528, 519, 522, 512, 521, 509, 532, 505, 480, 539, 517, 497, 539, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[515, 516, 478, 498, 504, 490, 529, 513, 553, 522, 530, 496, 497, 495, 496, 539, 547, 542, 509, 517, 498, 485, 506, 509, 488, 503, 506, 490, 530, 570, 526, 489, 493, 521, 515, 498, 460, 495, 493, 501, 506, 522, 519, 520, 479, 504, 530, 509, 506, 527, 518, 526, 524, 502, 514, 472, 530, 494, 510, 524, 481, 551, 477, 521, 543, 550, 496, 512, 524, 544, 553, 533, 547, 515, 516, 546, 464, 512, 515, 519, 496, 523, 498, 514, 505, 518, 527, 557, 490, 535, 489, 502, 484, 521, 492, 508, 534, 508, 513, 509, 511, 505, 467, 518, 523, 481, 511, 495, 529, 515, 536, 499, 536, 525, 536, 501, 499, 505, 464, 543, 519, 537, 537, 524, 492, 522, 477, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[500, 486, 505, 515, 534, 551, 484, 503, 531, 517, 527, 490, 503, 506, 503, 497, 520, 495, 508, 499, 526, 520, 534, 521, 531, 547, 510, 498, 491, 554, 538, 515, 505, 511, 557, 487, 522, 488, 515, 526, 505, 477, 542, 538, 529, 499, 553, 514, 551, 552, 550, 524, 513, 511, 473, 473, 492, 541, 497, 511, 517, 518, 523, 493, 536, 492, 521, 487, 494, 512, 507, 538, 524, 477, 495, 509, 502, 506, 521, 496, 499, 538, 530, 530, 512, 532, 525, 514, 476, 528, 486, 526, 501, 517, 515, 528, 530, 503, 471, 504, 491, 460, 536, 534, 511, 497, 489, 490, 507, 508, 502, 468, 534, 490, 509, 500, 518, 519, 508, 523, 543, 509, 500, 495, 514, 527, 514, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[530, 553, 442, 535, 523, 522, 534, 538, 462, 537, 504, 529, 517, 531, 522, 507, 515, 542, 540, 522, 537, 512, 485, 512, 524, 565, 512, 509, 502, 487, 538, 515, 527, 543, 514, 490, 486, 484, 496, 494, 526, 490, 453, 533, 505, 547, 544, 498, 492, 499, 512, 535, 502, 548, 516, 491, 532, 512, 506, 515, 501, 511, 495, 479, 493, 557, 534, 550, 512, 464, 536, 498, 491, 516, 544, 485, 504, 539, 542, 482, 528, 500, 509, 509, 555, 494, 491, 490, 512, 502, 543, 493, 539, 490, 504, 517, 514, 498, 514, 508, 493, 496, 518, 542, 503, 511, 523, 501, 518, 514, 486, 495, 485, 489, 504, 530, 507, 525, 508, 511, 488, 501, 471, 518, 502, 512, 511, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 448, 549, 537, 508, 503, 531, 504, 497, 492, 525, 523, 506, 523, 532, 517, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 466, 510, 518, 468, 498, 533, 503, 484, 536, 554, 511, 504, 494, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 533, 530, 528, 535, 481, 498, 498, 540, 505, 529, 502, 511, 512, 501, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 515, 504, 521, 510, 511, 522, 532, 534, 487, 526, 516, 481, 510, 488, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 506, 501, 497, 517, 533, 478, 551, 522, 532, 546, 501, 501, 510, 499, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 499, 527, 542, 519, 509, 531, 528, 513, 526, 501, 496, 498, 512, 498, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 526, 478, 516, 486, 491, 533, 495, 501, 533, 550, 481, 524, 556, 523, 527, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 521, 505, 472, 508, 524, 515, 511, 486, 498, 511, 490, 517, 503, 535, 495, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8086, 8224, 8193, 8190, 8191, 8281, 8142] avg=8192.0000 max=8281 max_rank=6 min=8086 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 542, 543, 496, 476, 540, 498, 500, 526, 530, 515, 494, 515, 545, 525, 511, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 459, 524, 513, 511, 511, 495, 508, 509, 493, 516, 490, 538, 519, 507, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 485, 526, 492, 530, 576, 518, 498, 492, 514, 536, 480, 519, 495, 532, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 512, 517, 505, 527, 527, 508, 487, 452, 527, 483, 513, 503, 508, 514, 492, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 553, 473, 494, 521, 501, 446, 512, 503, 531, 508, 542, 517, 591, 527, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 472, 532, 480, 493, 510, 483, 525, 542, 516, 521, 504, 526, 508, 566, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 464, 521, 552, 525, 483, 492, 485, 506, 499, 529, 478, 519, 496, 511, 480, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 541, 532, 486, 505, 465, 497, 487, 513, 492, 543, 549, 481, 530, 525, 521, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8143, 8200, 8099, 8245, 8217, 8127, 8222] avg=8192.0000 max=8283 max_rank=0 min=8099 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 486, 505, 515, 534, 551, 484, 503, 531, 517, 527, 490, 503, 506, 503, 497, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 495, 508, 499, 526, 520, 534, 521, 531, 534, 510, 498, 491, 554, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 557, 487, 522, 488, 515, 526, 505, 477, 542, 538, 529, 499, 553, 514, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 552, 550, 524, 513, 511, 473, 473, 492, 541, 497, 511, 517, 518, 523, 493, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 492, 521, 487, 494, 512, 443, 538, 524, 477, 495, 509, 502, 506, 521, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 490, 530, 530, 512, 532, 525, 514, 476, 528, 486, 526, 501, 517, 515, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 503, 471, 504, 491, 460, 536, 534, 511, 497, 489, 490, 507, 508, 502, 468, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 509, 500, 518, 519, 508, 523, 543, 509, 500, 495, 514, 527, 514, 492, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8294, 8271, 8254, 8053, 8209, 8061, 8223] avg=8192.0000 max=8294 max_rank=1 min=8053 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 511, 487, 525, 483, 516, 503, 490, 483, 531, 508, 521, 533, 519, 479, 503, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 504, 496, 516, 470, 489, 484, 487, 530, 482, 548, 471, 525, 511, 486, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 564, 559, 478, 546, 508, 502, 514, 527, 532, 540, 499, 505, 506, 515, 518, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 529, 494, 503, 507, 556, 532, 494, 524, 509, 487, 504, 501, 493, 501, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 518, 542, 492, 518, 511, 460, 507, 533, 509, 503, 526, 514, 527, 533, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 459, 500, 534, 506, 511, 531, 506, 506, 508, 502, 512, 502, 534, 568, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 498, 512, 496, 479, 499, 508, 495, 535, 499, 496, 536, 495, 502, 506, 472, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 519, 536, 501, 529, 488, 504, 529, 490, 493, 505, 496, 551, 499, 473, 512, 27, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8109, 8363, 8183, 8261, 8217, 8135, 8117] avg=8192.0000 max=8363 max_rank=2 min=8109 min_rank=1 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[496, 521, 520, 485, 495, 503, 473, 538, 491, 452, 497, 504, 514, 456, 520, 540, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 504, 485, 511, 498, 531, 524, 506, 530, 498, 497, 487, 515, 516, 502, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 497, 550, 491, 490, 517, 494, 511, 528, 533, 499, 552, 500, 493, 509, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 485, 544, 543, 518, 498, 520, 488, 477, 525, 556, 469, 530, 536, 528, 524, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 483, 487, 509, 538, 471, 510, 511, 551, 500, 538, 493, 528, 510, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 512, 533, 514, 515, 531, 536, 521, 522, 515, 502, 527, 497, 508, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 508, 523, 496, 464, 526, 508, 519, 510, 507, 519, 549, 491, 506, 540, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 489, 516, 514, 517, 488, 532, 513, 491, 502, 483, 546, 532, 529, 524, 504, 25, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8024, 8135, 8168, 8263, 8163, 8305, 8259, 8219] avg=8192.0000 max=8305 max_rank=5 min=8024 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32946 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 551, 503, 511, 497, 504, 506, 471, 504, 527, 512, 531, 530, 489, 489, 527, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 529, 499, 527, 484, 498, 541, 511, 526, 513, 509, 507, 511, 508, 535, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 498, 491, 501, 492, 508, 505, 487, 525, 513, 509, 506, 500, 506, 508, 520, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 496, 517, 529, 503, 517, 510, 522, 519, 507, 520, 518, 491, 546, 470, 476, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 532, 505, 540, 512, 526, 451, 510, 505, 510, 525, 483, 506, 513, 535, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 514, 500, 495, 506, 489, 534, 524, 515, 511, 509, 474, 525, 548, 477, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 543, 530, 536, 516, 538, 501, 472, 486, 529, 482, 517, 519, 506, 519, 508, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 499, 528, 519, 522, 512, 521, 509, 532, 505, 480, 539, 517, 497, 539, 492, 26, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8262, 8109, 8192, 8127, 8154, 8255, 8251] avg=8192.0000 max=8262 max_rank=1 min=8109 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[530, 553, 442, 535, 523, 522, 534, 538, 462, 537, 504, 529, 517, 531, 522, 507, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 542, 540, 522, 537, 512, 485, 512, 524, 552, 512, 509, 502, 487, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 543, 514, 490, 486, 484, 496, 494, 526, 490, 453, 533, 505, 547, 544, 498, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 499, 512, 535, 502, 548, 516, 491, 532, 512, 506, 515, 501, 511, 495, 479, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 557, 534, 550, 512, 464, 468, 498, 491, 516, 544, 485, 504, 539, 542, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 455, 509, 509, 555, 494, 491, 490, 512, 502, 543, 493, 539, 490, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 498, 514, 508, 493, 496, 518, 542, 503, 511, 523, 501, 518, 514, 486, 495, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 489, 504, 530, 507, 525, 508, 511, 488, 501, 471, 518, 502, 512, 511, 538, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8304, 8133, 8161, 8179, 8131, 8198, 8127] avg=8192.0000 max=8304 max_rank=1 min=8127 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 516, 478, 498, 504, 490, 529, 513, 553, 522, 530, 496, 497, 495, 496, 539, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 542, 509, 517, 498, 485, 506, 509, 488, 491, 506, 490, 530, 570, 526, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 521, 515, 498, 460, 495, 493, 501, 506, 522, 519, 520, 479, 504, 530, 509, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 527, 518, 526, 524, 502, 514, 472, 530, 494, 510, 524, 481, 551, 477, 521, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 550, 496, 512, 524, 544, 482, 533, 547, 515, 516, 546, 464, 512, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 476, 498, 514, 505, 518, 527, 557, 490, 535, 489, 502, 484, 521, 492, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 513, 509, 511, 505, 467, 518, 523, 481, 511, 495, 529, 515, 536, 499, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 525, 536, 501, 499, 505, 464, 543, 519, 537, 537, 524, 492, 522, 477, 489, 24, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8203, 8068, 8191, 8318, 8112, 8220, 8235] avg=8192.0000 max=8318 max_rank=4 min=8068 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4352, 4096, 4096, 4224, 4224, 3712, 4224, 4224, 4224, 4224, 4224, 4096, 4352, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[478, 531, 508, 491, 495, 486, 532, 506, 499, 484, 519, 519, 531, 540, 558, 515, 513, 514, 472, 505, 505, 496, 518, 518, 493, 521, 521, 514, 478, 518, 501, 530, 526, 537, 515, 519, 497, 491, 532, 496, 497, 534, 512, 517, 543, 501, 538, 521, 523, 545, 532, 549, 534, 524, 518, 505, 523, 494, 545, 494, 529, 513, 481, 484, 510, 524, 513, 500, 500, 526, 505, 515, 523, 508, 530, 537, 507, 514, 482, 482, 489, 493, 493, 500, 531, 519, 509, 525, 560, 520, 493, 491, 511, 479, 526, 519, 502, 505, 521, 526, 505, 508, 557, 516, 488, 534, 498, 493, 516, 515, 512, 510, 488, 541, 493, 503, 505, 522, 494, 494, 538, 490, 505, 510, 490, 520, 490, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[493, 499, 504, 521, 459, 507, 530, 511, 495, 563, 513, 552, 490, 506, 539, 545, 518, 516, 476, 496, 507, 515, 498, 481, 494, 547, 492, 525, 485, 476, 492, 528, 536, 500, 506, 510, 554, 516, 485, 526, 532, 495, 544, 495, 515, 478, 498, 500, 541, 495, 491, 513, 552, 491, 507, 509, 487, 503, 523, 489, 513, 525, 498, 477, 486, 484, 514, 484, 535, 512, 508, 499, 529, 519, 507, 504, 491, 516, 480, 504, 525, 505, 529, 536, 522, 507, 546, 488, 535, 500, 499, 494, 520, 528, 549, 529, 497, 490, 532, 537, 555, 525, 496, 502, 529, 534, 526, 542, 501, 557, 476, 500, 542, 491, 513, 483, 497, 538, 541, 545, 496, 545, 497, 493, 526, 547, 514, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[518, 520, 530, 553, 510, 506, 506, 498, 502, 512, 474, 497, 521, 512, 485, 483, 535, 566, 510, 500, 515, 534, 539, 490, 533, 483, 478, 520, 549, 501, 498, 501, 500, 496, 516, 548, 506, 512, 526, 475, 509, 485, 557, 462, 504, 518, 501, 514, 526, 543, 545, 469, 520, 525, 505, 531, 540, 517, 504, 519, 529, 503, 516, 530, 489, 489, 494, 507, 517, 510, 513, 496, 514, 492, 549, 547, 498, 529, 506, 502, 470, 472, 509, 484, 492, 510, 517, 513, 529, 502, 518, 472, 516, 519, 532, 500, 485, 502, 493, 508, 505, 530, 531, 522, 533, 545, 516, 503, 588, 516, 484, 512, 490, 545, 518, 521, 510, 487, 515, 509, 545, 542, 479, 515, 482, 524, 492, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[514, 498, 532, 483, 498, 511, 552, 546, 491, 510, 535, 475, 526, 525, 482, 510, 521, 508, 464, 533, 533, 524, 530, 518, 495, 517, 541, 573, 505, 518, 494, 519, 478, 520, 516, 490, 512, 497, 508, 505, 521, 511, 458, 487, 526, 521, 519, 477, 486, 501, 490, 467, 499, 509, 519, 511, 527, 524, 516, 514, 512, 521, 538, 501, 512, 546, 498, 554, 519, 573, 503, 515, 516, 509, 485, 525, 537, 505, 470, 484, 478, 499, 510, 531, 530, 475, 562, 530, 493, 487, 498, 489, 508, 527, 523, 518, 536, 516, 482, 524, 512, 514, 506, 509, 518, 499, 546, 524, 492, 503, 503, 474, 515, 500, 516, 536, 506, 555, 511, 464, 521, 524, 542, 529, 539, 511, 509, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[497, 516, 498, 534, 523, 486, 498, 531, 496, 512, 481, 469, 491, 475, 529, 537, 534, 488, 501, 520, 506, 529, 539, 498, 540, 536, 523, 556, 505, 498, 501, 499, 487, 527, 470, 504, 499, 473, 513, 526, 512, 491, 491, 493, 521, 521, 569, 489, 515, 530, 470, 480, 509, 513, 497, 532, 502, 535, 527, 524, 512, 497, 503, 523, 527, 503, 526, 506, 481, 493, 509, 558, 519, 542, 490, 498, 554, 516, 529, 536, 511, 520, 562, 499, 473, 497, 504, 500, 528, 551, 491, 516, 557, 510, 513, 534, 514, 500, 541, 504, 516, 513, 536, 506, 509, 492, 518, 527, 484, 488, 506, 509, 561, 526, 511, 460, 508, 493, 518, 520, 551, 524, 522, 476, 504, 478, 514, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[484, 485, 479, 518, 496, 477, 496, 516, 474, 501, 495, 529, 489, 508, 526, 516, 525, 524, 507, 523, 488, 592, 498, 553, 507, 502, 540, 525, 547, 501, 499, 450, 555, 497, 486, 509, 500, 498, 493, 502, 526, 492, 510, 496, 516, 510, 537, 564, 545, 510, 501, 537, 527, 541, 492, 508, 510, 468, 514, 507, 510, 461, 540, 534, 471, 500, 521, 517, 480, 506, 491, 496, 499, 515, 564, 557, 532, 496, 529, 484, 528, 527, 512, 508, 504, 514, 496, 570, 502, 508, 540, 501, 550, 511, 501, 526, 530, 528, 475, 540, 525, 501, 513, 520, 518, 511, 511, 520, 522, 476, 547, 484, 517, 518, 517, 534, 519, 525, 475, 514, 495, 505, 480, 526, 517, 525, 486, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[533, 508, 568, 520, 553, 516, 511, 472, 514, 488, 507, 516, 523, 513, 483, 546, 508, 517, 485, 498, 500, 509, 502, 526, 493, 532, 547, 477, 459, 493, 524, 512, 490, 501, 481, 521, 516, 487, 495, 515, 509, 539, 500, 569, 519, 533, 538, 519, 529, 502, 535, 515, 468, 508, 552, 499, 523, 516, 511, 472, 505, 499, 481, 486, 539, 537, 501, 516, 528, 514, 502, 515, 509, 510, 508, 539, 524, 480, 514, 526, 476, 520, 525, 512, 494, 488, 512, 497, 512, 517, 549, 454, 539, 502, 527, 546, 497, 510, 527, 520, 495, 529, 467, 515, 513, 475, 508, 537, 501, 532, 507, 523, 535, 528, 483, 522, 514, 503, 535, 490, 506, 513, 512, 546, 526, 541, 490, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[525, 489, 506, 501, 491, 511, 502, 496, 546, 535, 490, 492, 521, 517, 533, 527, 499, 513, 518, 502, 534, 511, 521, 504, 502, 508, 516, 538, 503, 541, 499, 528, 535, 492, 479, 499, 497, 486, 474, 501, 508, 496, 524, 502, 503, 525, 501, 472, 465, 483, 505, 527, 480, 510, 500, 501, 513, 525, 503, 487, 559, 508, 525, 508, 508, 527, 504, 541, 570, 496, 500, 525, 524, 514, 466, 542, 532, 521, 502, 473, 508, 511, 488, 512, 496, 504, 538, 534, 532, 508, 528, 507, 506, 566, 469, 542, 505, 531, 550, 499, 529, 507, 487, 497, 527, 506, 527, 505, 464, 490, 503, 525, 534, 548, 507, 491, 554, 541, 512, 515, 542, 516, 499, 500, 532, 541, 558, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 520, 530, 553, 510, 506, 506, 498, 502, 512, 474, 497, 521, 512, 485, 483, 15, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 566, 510, 500, 515, 534, 539, 490, 533, 483, 478, 515, 549, 501, 498, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 496, 516, 548, 506, 512, 526, 475, 509, 485, 557, 462, 504, 518, 501, 514, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 543, 545, 469, 520, 525, 505, 531, 540, 517, 504, 519, 529, 503, 516, 530, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 489, 494, 507, 517, 510, 513, 496, 514, 492, 549, 540, 498, 529, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 472, 509, 484, 492, 510, 517, 513, 529, 502, 518, 472, 516, 519, 532, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 502, 493, 497, 505, 530, 531, 522, 533, 545, 516, 503, 588, 516, 484, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 545, 518, 521, 510, 487, 515, 509, 545, 542, 479, 515, 482, 524, 492, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8247, 8180, 8333, 8145, 8040, 8262, 8169] avg=8192.0000 max=8333 max_rank=3 min=8040 min_rank=5 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32616 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 498, 532, 483, 498, 511, 552, 546, 491, 510, 535, 475, 526, 525, 482, 510, 15, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 508, 464, 533, 533, 524, 530, 518, 495, 517, 541, 567, 505, 518, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 520, 516, 490, 512, 497, 508, 505, 521, 511, 458, 487, 526, 521, 519, 477, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 501, 490, 467, 499, 509, 519, 511, 527, 524, 516, 514, 512, 521, 538, 501, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 546, 498, 554, 519, 573, 503, 515, 516, 509, 485, 518, 537, 505, 470, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 499, 510, 531, 530, 475, 562, 530, 493, 487, 498, 489, 508, 527, 523, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 516, 482, 513, 512, 514, 506, 509, 518, 499, 546, 524, 492, 503, 503, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 500, 516, 536, 506, 555, 511, 464, 521, 524, 542, 529, 539, 511, 509, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8287, 8095, 8147, 8244, 8143, 8147, 8252] avg=8192.0000 max=8287 max_rank=1 min=8095 min_rank=2 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[493, 499, 504, 521, 459, 507, 530, 511, 495, 563, 513, 552, 490, 506, 539, 545, 16, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 476, 496, 507, 515, 498, 481, 494, 547, 492, 520, 485, 476, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 500, 506, 510, 554, 516, 485, 526, 532, 495, 544, 495, 515, 478, 498, 500, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 495, 491, 513, 552, 491, 507, 509, 487, 503, 523, 489, 513, 525, 498, 477, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 484, 514, 484, 535, 512, 508, 499, 529, 519, 507, 497, 491, 516, 480, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 529, 536, 522, 507, 546, 488, 535, 500, 499, 494, 520, 528, 549, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 490, 532, 526, 555, 525, 496, 502, 529, 534, 526, 542, 501, 557, 476, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 491, 513, 483, 497, 538, 541, 545, 496, 545, 497, 493, 526, 547, 514, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8041, 8237, 8125, 8065, 8296, 8288, 8223] avg=8192.0000 max=8296 max_rank=5 min=8041 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[478, 531, 508, 491, 495, 486, 532, 506, 499, 484, 519, 519, 531, 540, 558, 515, 16, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 514, 472, 505, 505, 496, 518, 518, 493, 521, 521, 509, 478, 518, 501, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 537, 515, 519, 497, 491, 532, 496, 497, 534, 512, 517, 543, 501, 538, 521, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 545, 532, 549, 534, 524, 518, 505, 523, 494, 545, 494, 529, 513, 481, 484, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 513, 500, 500, 526, 505, 515, 523, 508, 530, 530, 507, 514, 482, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 493, 493, 500, 531, 519, 509, 525, 560, 520, 493, 491, 511, 479, 526, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 505, 521, 515, 505, 508, 557, 516, 488, 534, 498, 493, 516, 515, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 541, 493, 503, 505, 522, 494, 494, 538, 490, 505, 510, 490, 520, 490, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8112, 8325, 8304, 8169, 8142, 8195, 8063] avg=8192.0000 max=8325 max_rank=2 min=8063 min_rank=7 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[497, 516, 498, 534, 523, 486, 498, 531, 496, 512, 481, 469, 491, 475, 529, 537, 16, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 488, 501, 520, 506, 529, 539, 498, 540, 536, 523, 551, 505, 498, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 527, 470, 504, 499, 473, 513, 526, 512, 491, 491, 493, 521, 521, 569, 489, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 530, 470, 480, 509, 513, 497, 532, 502, 535, 527, 524, 512, 497, 503, 523, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 503, 526, 506, 481, 493, 509, 558, 519, 542, 490, 491, 554, 516, 529, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 520, 562, 499, 473, 497, 504, 500, 528, 551, 491, 516, 557, 510, 513, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 541, 493, 516, 513, 536, 506, 509, 492, 518, 527, 484, 488, 506, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 526, 511, 460, 508, 493, 518, 520, 551, 524, 522, 476, 504, 478, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8268, 8137, 8180, 8280, 8250, 8152, 8162] avg=8192.0000 max=8280 max_rank=4 min=8107 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[484, 485, 479, 518, 496, 477, 496, 516, 474, 501, 495, 529, 489, 508, 526, 516, 20, 15, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 524, 507, 523, 488, 592, 498, 553, 507, 502, 540, 516, 547, 501, 499, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 497, 486, 509, 500, 498, 493, 502, 526, 492, 510, 496, 516, 510, 537, 564, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 510, 501, 537, 527, 541, 492, 508, 510, 468, 514, 507, 510, 461, 540, 534, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 500, 521, 517, 480, 506, 491, 496, 499, 515, 564, 539, 532, 496, 529, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 527, 512, 508, 504, 514, 496, 570, 502, 508, 540, 501, 550, 511, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 475, 525, 525, 501, 513, 520, 518, 511, 511, 520, 522, 476, 547, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 518, 517, 534, 519, 525, 475, 514, 495, 505, 480, 526, 517, 525, 486, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8041, 8272, 8243, 8225, 8140, 8278, 8206, 8131] avg=8192.0000 max=8278 max_rank=5 min=8041 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[525, 489, 506, 501, 491, 511, 502, 496, 546, 535, 490, 492, 521, 517, 533, 527, 16, 11, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 513, 518, 502, 534, 511, 521, 504, 502, 508, 516, 533, 503, 541, 499, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 492, 479, 499, 497, 486, 474, 501, 508, 496, 524, 502, 503, 525, 501, 472, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 483, 505, 527, 480, 510, 500, 501, 513, 525, 503, 487, 559, 508, 525, 508, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 504, 541, 570, 496, 500, 525, 524, 514, 466, 535, 532, 521, 502, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 511, 488, 512, 496, 504, 538, 534, 532, 508, 528, 507, 506, 566, 469, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 550, 488, 529, 507, 487, 497, 527, 506, 527, 505, 464, 490, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 548, 507, 491, 554, 541, 512, 515, 542, 516, 499, 500, 532, 541, 558, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8232, 8039, 8110, 8238, 8233, 8141, 8328] avg=8192.0000 max=8328 max_rank=7 min=8039 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32940 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[533, 508, 568, 520, 553, 516, 511, 472, 514, 488, 507, 516, 523, 513, 483, 546, 16, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 485, 498, 500, 509, 502, 526, 493, 532, 547, 472, 459, 493, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 501, 481, 521, 516, 487, 495, 515, 509, 539, 500, 569, 519, 533, 538, 519, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 535, 515, 468, 508, 552, 499, 523, 516, 511, 472, 505, 499, 481, 486, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 537, 501, 516, 528, 514, 502, 515, 509, 510, 508, 532, 524, 480, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 520, 525, 512, 494, 488, 512, 497, 512, 517, 549, 454, 539, 502, 527, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 527, 509, 495, 529, 467, 515, 513, 475, 508, 537, 501, 532, 507, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 528, 483, 522, 514, 503, 535, 490, 506, 513, 512, 546, 526, 541, 490, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8077, 8280, 8112, 8255, 8154, 8145, 8208] avg=8192.0000 max=8305 max_rank=0 min=8077 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[540, 491, 509, 547, 487, 467, 498, 527, 479, 496, 515, 444, 481, 529, 539, 541, 537, 530, 476, 513, 518, 498, 496, 508, 471, 537, 502, 554, 546, 505, 489, 503, 525, 518, 514, 512, 527, 516, 536, 522, 481, 515, 549, 480, 534, 492, 500, 486, 476, 529, 518, 514, 541, 535, 520, 541, 485, 513, 506, 554, 511, 529, 526, 508, 560, 499, 510, 521, 489, 496, 513, 509, 469, 532, 517, 502, 463, 512, 522, 511, 486, 518, 494, 518, 477, 500, 490, 555, 496, 487, 524, 518, 530, 521, 485, 513, 493, 507, 543, 506, 540, 561, 511, 534, 521, 507, 510, 481, 524, 494, 474, 493, 508, 491, 541, 499, 499, 529, 524, 528, 467, 546, 511, 536, 553, 533, 525, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[507, 482, 523, 571, 526, 539, 527, 512, 518, 520, 494, 546, 539, 532, 518, 480, 516, 523, 504, 540, 529, 512, 500, 546, 524, 509, 507, 538, 515, 478, 489, 484, 520, 523, 523, 532, 476, 548, 499, 476, 513, 522, 540, 543, 556, 504, 505, 521, 550, 515, 524, 547, 524, 505, 543, 528, 508, 505, 507, 482, 499, 542, 491, 464, 503, 524, 487, 522, 492, 530, 497, 482, 533, 477, 552, 518, 480, 528, 502, 512, 486, 501, 543, 509, 500, 530, 517, 505, 518, 500, 531, 466, 490, 489, 487, 522, 493, 498, 540, 532, 470, 522, 502, 501, 490, 508, 489, 492, 513, 543, 512, 519, 457, 526, 500, 530, 509, 550, 490, 479, 523, 491, 505, 529, 524, 478, 496, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[557, 516, 544, 531, 496, 521, 526, 526, 490, 527, 545, 490, 501, 492, 525, 477, 543, 508, 475, 524, 517, 508, 504, 526, 471, 491, 471, 496, 561, 493, 523, 533, 522, 524, 527, 484, 516, 480, 483, 515, 528, 479, 522, 507, 492, 534, 483, 477, 547, 486, 498, 487, 545, 538, 529, 550, 498, 510, 531, 501, 521, 524, 564, 513, 502, 517, 485, 546, 519, 511, 488, 520, 517, 503, 509, 532, 496, 499, 526, 483, 468, 511, 488, 537, 528, 525, 504, 524, 535, 478, 526, 504, 512, 500, 513, 522, 483, 509, 571, 497, 505, 562, 528, 533, 498, 486, 587, 485, 505, 569, 496, 533, 510, 509, 542, 525, 521, 481, 427, 568, 502, 499, 486, 502, 495, 511, 468, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[517, 541, 524, 523, 527, 496, 561, 528, 543, 514, 529, 489, 493, 521, 517, 490, 522, 538, 509, 512, 523, 519, 504, 519, 493, 498, 479, 521, 519, 509, 516, 493, 508, 481, 468, 509, 522, 503, 523, 520, 528, 513, 503, 541, 521, 464, 468, 521, 507, 514, 510, 502, 534, 509, 511, 476, 511, 486, 512, 504, 487, 519, 537, 547, 527, 523, 506, 531, 487, 514, 520, 487, 539, 527, 533, 502, 533, 512, 535, 477, 499, 500, 526, 472, 465, 490, 508, 526, 464, 483, 519, 506, 518, 531, 496, 487, 518, 518, 550, 536, 503, 547, 510, 531, 535, 532, 519, 529, 479, 483, 499, 497, 476, 520, 504, 516, 530, 560, 508, 497, 551, 557, 511, 544, 507, 497, 467, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[524, 497, 550, 517, 531, 558, 503, 536, 519, 568, 512, 498, 514, 503, 487, 475, 490, 505, 494, 503, 560, 502, 537, 547, 497, 553, 549, 521, 554, 538, 528, 547, 519, 502, 482, 519, 513, 535, 517, 542, 478, 507, 537, 540, 497, 517, 510, 524, 537, 476, 502, 512, 494, 476, 480, 534, 541, 477, 504, 528, 489, 515, 533, 527, 487, 539, 483, 504, 488, 537, 498, 500, 486, 497, 485, 536, 539, 507, 492, 506, 516, 497, 556, 485, 512, 528, 507, 502, 523, 505, 509, 507, 480, 523, 516, 485, 532, 467, 466, 533, 502, 465, 496, 498, 474, 507, 482, 519, 502, 512, 498, 522, 462, 503, 534, 542, 502, 533, 524, 511, 542, 509, 474, 494, 541, 497, 539, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[540, 500, 523, 500, 520, 521, 532, 513, 473, 498, 484, 503, 463, 510, 515, 547, 528, 553, 475, 486, 468, 556, 496, 523, 502, 507, 496, 501, 529, 502, 470, 527, 557, 524, 489, 518, 497, 576, 509, 493, 520, 504, 485, 490, 534, 550, 527, 503, 530, 534, 505, 524, 517, 488, 532, 503, 515, 512, 492, 504, 529, 517, 453, 525, 502, 495, 496, 521, 487, 486, 565, 501, 500, 539, 498, 542, 508, 464, 505, 501, 529, 538, 510, 493, 484, 506, 496, 529, 510, 529, 531, 481, 504, 535, 500, 530, 487, 519, 509, 547, 512, 535, 534, 466, 520, 505, 509, 497, 489, 532, 528, 517, 559, 553, 511, 544, 515, 510, 556, 469, 494, 507, 527, 529, 507, 481, 517, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[480, 499, 493, 522, 499, 503, 565, 507, 504, 497, 536, 499, 526, 487, 487, 503, 480, 517, 545, 492, 524, 505, 537, 509, 521, 516, 461, 509, 495, 478, 528, 498, 503, 504, 490, 527, 510, 501, 490, 500, 529, 482, 484, 498, 526, 504, 548, 563, 497, 518, 520, 501, 517, 517, 539, 488, 497, 519, 541, 536, 528, 510, 505, 511, 534, 517, 505, 507, 494, 535, 539, 552, 490, 531, 527, 573, 509, 495, 472, 535, 524, 448, 490, 548, 522, 490, 499, 482, 459, 521, 506, 562, 486, 523, 541, 508, 483, 517, 533, 504, 535, 488, 486, 536, 540, 522, 487, 507, 523, 523, 538, 488, 515, 500, 527, 489, 515, 528, 509, 530, 559, 542, 518, 527, 473, 508, 507, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[506, 516, 491, 488, 484, 538, 539, 513, 537, 495, 529, 529, 547, 527, 524, 501, 499, 544, 519, 572, 501, 514, 511, 483, 501, 483, 513, 493, 539, 497, 544, 529, 515, 495, 506, 513, 536, 569, 499, 519, 514, 487, 509, 512, 462, 523, 477, 458, 492, 527, 499, 540, 511, 486, 500, 488, 500, 482, 513, 538, 501, 485, 495, 529, 538, 495, 513, 532, 527, 517, 465, 528, 538, 533, 540, 501, 503, 508, 504, 537, 506, 520, 504, 545, 525, 514, 530, 533, 484, 517, 495, 510, 524, 513, 497, 498, 515, 538, 502, 516, 510, 522, 518, 524, 534, 525, 501, 525, 501, 510, 547, 482, 498, 499, 542, 506, 513, 527, 523, 469, 509, 497, 473, 518, 489, 489, 514, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[517, 541, 524, 523, 527, 496, 526, 528, 543, 514, 529, 489, 493, 521, 517, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 538, 509, 512, 523, 519, 504, 519, 493, 498, 479, 521, 510, 509, 516, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 481, 468, 509, 522, 503, 523, 520, 528, 513, 503, 541, 521, 464, 468, 521, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 514, 510, 502, 516, 509, 511, 476, 511, 486, 512, 504, 487, 519, 537, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 523, 506, 531, 487, 514, 520, 487, 539, 527, 533, 502, 533, 512, 535, 477, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 500, 526, 472, 465, 490, 508, 526, 464, 483, 519, 506, 518, 531, 496, 487, 35, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 541, 536, 503, 547, 510, 531, 535, 532, 519, 529, 479, 483, 499, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 520, 504, 516, 530, 550, 508, 497, 551, 557, 511, 544, 507, 497, 467, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8165, 8103, 8148, 8263, 8051, 8277, 8251] avg=8192.0000 max=8278 max_rank=0 min=8051 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[507, 482, 523, 571, 526, 539, 494, 512, 518, 520, 494, 546, 539, 532, 518, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 523, 504, 540, 529, 512, 500, 546, 524, 509, 507, 538, 506, 478, 489, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 523, 532, 476, 548, 499, 476, 513, 522, 540, 543, 556, 504, 505, 521, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 515, 524, 547, 506, 505, 543, 528, 508, 505, 507, 482, 499, 542, 491, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 487, 522, 492, 530, 497, 482, 533, 477, 552, 518, 480, 528, 502, 512, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 501, 543, 509, 500, 530, 517, 505, 518, 500, 531, 466, 490, 489, 487, 522, 33, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 498, 531, 532, 470, 522, 502, 501, 490, 508, 489, 492, 513, 543, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 526, 500, 530, 509, 540, 490, 479, 523, 491, 505, 529, 524, 478, 496, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8205, 8311, 8216, 8149, 8153, 8115, 8086] avg=8192.0000 max=8311 max_rank=2 min=8086 min_rank=7 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[540, 491, 509, 547, 487, 467, 467, 527, 479, 496, 515, 444, 481, 529, 539, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 476, 513, 518, 498, 496, 508, 471, 537, 502, 554, 536, 505, 489, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 514, 512, 527, 516, 536, 522, 481, 515, 549, 480, 534, 492, 500, 486, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 529, 518, 514, 522, 535, 520, 541, 485, 513, 506, 554, 511, 529, 526, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 499, 510, 521, 489, 496, 513, 509, 469, 532, 517, 502, 463, 512, 522, 511, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 518, 494, 518, 477, 500, 490, 555, 496, 487, 524, 518, 530, 521, 485, 513, 31, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 507, 534, 506, 540, 561, 511, 534, 521, 507, 510, 481, 524, 494, 474, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 491, 541, 499, 499, 520, 524, 528, 467, 546, 511, 536, 553, 533, 525, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8173, 8218, 8287, 8134, 8170, 8190, 8305] avg=8192.0000 max=8305 max_rank=7 min=8059 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[557, 516, 544, 531, 496, 521, 493, 526, 490, 527, 545, 490, 501, 492, 525, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 508, 475, 524, 517, 508, 504, 526, 471, 491, 471, 496, 551, 493, 523, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 524, 527, 484, 516, 480, 483, 515, 528, 479, 522, 507, 492, 534, 483, 477, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 486, 498, 487, 526, 538, 529, 550, 498, 510, 531, 501, 521, 524, 564, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 517, 485, 546, 519, 511, 488, 520, 517, 503, 509, 532, 496, 499, 526, 483, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 511, 488, 537, 528, 525, 504, 524, 535, 478, 526, 504, 512, 500, 513, 522, 33, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 509, 561, 497, 505, 562, 528, 533, 498, 486, 587, 485, 505, 569, 496, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 542, 525, 521, 473, 427, 568, 502, 499, 486, 502, 495, 511, 468, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8134, 8084, 8323, 8161, 8236, 8337, 8030] avg=8192.0000 max=8337 max_rank=6 min=8030 min_rank=7 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 497, 550, 517, 531, 558, 472, 536, 519, 568, 512, 498, 514, 503, 487, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 494, 503, 560, 502, 537, 547, 497, 553, 549, 521, 544, 538, 528, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 502, 482, 519, 513, 535, 517, 542, 478, 507, 537, 540, 497, 517, 510, 524, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 476, 502, 512, 477, 476, 480, 534, 541, 477, 504, 528, 489, 515, 533, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 539, 483, 504, 488, 537, 498, 500, 486, 497, 485, 536, 539, 507, 492, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 497, 556, 485, 512, 528, 507, 502, 523, 505, 509, 507, 480, 523, 516, 485, 31, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 467, 458, 533, 502, 465, 496, 498, 474, 507, 482, 519, 502, 512, 498, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 503, 534, 542, 502, 524, 524, 511, 542, 509, 474, 494, 541, 497, 539, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8415, 8250, 8108, 8093, 8206, 7967, 8236] avg=8192.0000 max=8415 max_rank=1 min=7967 min_rank=6 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=32502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[480, 499, 493, 522, 499, 503, 525, 507, 504, 497, 536, 499, 526, 487, 487, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 517, 545, 492, 524, 505, 537, 509, 521, 516, 461, 509, 482, 478, 528, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 490, 527, 510, 501, 490, 500, 529, 482, 484, 498, 526, 504, 548, 563, 13, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 518, 520, 501, 496, 517, 539, 488, 497, 519, 541, 536, 528, 510, 505, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 505, 507, 494, 535, 539, 552, 490, 531, 527, 573, 509, 495, 472, 535, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 448, 490, 548, 522, 490, 499, 482, 459, 521, 506, 562, 486, 523, 541, 508, 40, 21, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 517, 515, 504, 535, 488, 486, 536, 540, 522, 487, 507, 523, 523, 538, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 500, 527, 489, 515, 513, 509, 530, 559, 542, 518, 527, 473, 508, 507, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8067, 8102, 8179, 8223, 8329, 8182, 8192, 8262] avg=8192.0000 max=8329 max_rank=4 min=8067 min_rank=0 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32965 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[506, 516, 491, 488, 484, 538, 505, 513, 537, 495, 529, 529, 547, 527, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 544, 519, 572, 501, 514, 511, 483, 501, 483, 513, 493, 529, 497, 544, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 495, 506, 513, 536, 569, 499, 519, 514, 487, 509, 512, 462, 523, 477, 458, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 527, 499, 540, 494, 486, 500, 488, 500, 482, 513, 538, 501, 485, 495, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 495, 513, 532, 527, 517, 465, 528, 538, 533, 540, 501, 503, 508, 504, 537, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 504, 545, 525, 514, 530, 533, 484, 517, 495, 510, 524, 513, 497, 498, 34, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 538, 493, 516, 510, 522, 518, 524, 534, 525, 501, 525, 501, 510, 547, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 542, 506, 513, 518, 523, 469, 509, 497, 473, 518, 489, 489, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8232, 8105, 8069, 8288, 8274, 8261, 8077] avg=8192.0000 max=8288 max_rank=4 min=8069 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[540, 500, 523, 500, 520, 521, 499, 513, 473, 498, 484, 503, 463, 510, 515, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 553, 475, 486, 468, 556, 496, 523, 502, 507, 496, 501, 520, 502, 470, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 524, 489, 518, 497, 576, 509, 493, 520, 504, 485, 490, 534, 550, 527, 503, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 534, 505, 524, 499, 488, 532, 503, 515, 512, 492, 504, 529, 517, 453, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 496, 521, 487, 486, 565, 501, 500, 539, 498, 542, 508, 464, 505, 501, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 538, 510, 493, 484, 506, 496, 529, 510, 529, 531, 481, 504, 535, 500, 530, 33, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 519, 500, 547, 512, 535, 534, 466, 520, 505, 509, 497, 489, 532, 528, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 553, 511, 544, 515, 501, 556, 469, 494, 507, 527, 529, 507, 481, 517, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8110, 8286, 8162, 8119, 8264, 8197, 8289] avg=8192.0000 max=8289 max_rank=7 min=8109 min_rank=0 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[480, 495, 480, 485, 523, 516, 487, 524, 524, 521, 513, 511, 535, 511, 519, 559, 498, 510, 529, 529, 496, 478, 478, 511, 553, 512, 511, 507, 461, 483, 528, 555, 505, 505, 522, 499, 473, 530, 509, 495, 543, 523, 524, 538, 514, 528, 529, 529, 516, 526, 491, 536, 499, 513, 557, 539, 526, 487, 500, 495, 535, 507, 474, 486, 497, 499, 494, 498, 528, 531, 497, 541, 543, 544, 526, 499, 520, 517, 496, 531, 515, 494, 514, 531, 536, 497, 516, 540, 486, 492, 522, 485, 540, 515, 523, 501, 541, 475, 510, 477, 536, 518, 548, 467, 502, 506, 503, 490, 510, 506, 517, 544, 505, 504, 481, 520, 518, 472, 519, 508, 493, 509, 519, 538, 520, 523, 459, 555] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[497, 491, 545, 503, 511, 515, 467, 506, 538, 533, 543, 503, 509, 512, 507, 522, 496, 513, 539, 560, 475, 518, 511, 498, 496, 567, 534, 465, 490, 563, 523, 518, 501, 512, 488, 545, 509, 479, 516, 494, 507, 518, 506, 473, 510, 484, 456, 516, 539, 520, 519, 509, 555, 496, 527, 519, 507, 541, 504, 512, 496, 560, 529, 534, 505, 487, 520, 548, 501, 490, 493, 538, 511, 498, 487, 523, 485, 510, 526, 511, 501, 517, 509, 501, 485, 501, 510, 545, 474, 495, 523, 485, 525, 493, 492, 498, 515, 535, 536, 518, 509, 506, 482, 543, 558, 494, 540, 510, 508, 479, 510, 521, 537, 507, 487, 497, 560, 536, 522, 518, 489, 550, 468, 532, 504, 506, 475, 548] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[484, 513, 491, 507, 515, 525, 528, 532, 507, 512, 515, 491, 513, 506, 501, 524, 563, 523, 499, 513, 528, 497, 507, 524, 542, 545, 476, 527, 523, 502, 495, 512, 468, 488, 555, 510, 516, 553, 500, 531, 464, 473, 523, 440, 502, 525, 534, 520, 515, 499, 538, 521, 508, 535, 475, 502, 549, 520, 481, 494, 508, 473, 534, 508, 521, 515, 480, 514, 502, 529, 521, 493, 505, 540, 501, 526, 525, 529, 472, 511, 545, 522, 456, 498, 530, 510, 533, 520, 481, 506, 528, 523, 503, 499, 542, 461, 524, 518, 510, 525, 528, 493, 523, 513, 506, 539, 511, 503, 510, 536, 506, 501, 482, 500, 564, 540, 501, 525, 514, 521, 449, 544, 504, 483, 518, 529, 524, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[495, 520, 522, 507, 475, 495, 478, 511, 505, 487, 515, 515, 479, 504, 519, 512, 518, 508, 530, 511, 502, 501, 554, 474, 482, 528, 519, 530, 488, 512, 525, 511, 470, 531, 502, 522, 513, 514, 502, 521, 485, 518, 506, 540, 529, 479, 494, 505, 517, 515, 522, 499, 526, 513, 505, 533, 472, 515, 514, 522, 526, 509, 522, 510, 547, 512, 566, 494, 543, 523, 502, 481, 516, 494, 500, 495, 514, 479, 508, 546, 512, 517, 515, 539, 525, 530, 528, 524, 473, 545, 540, 480, 504, 514, 517, 486, 526, 501, 534, 478, 527, 530, 521, 508, 485, 498, 491, 500, 527, 511, 510, 571, 539, 533, 499, 496, 502, 526, 478, 554, 512, 521, 558, 497, 490, 510, 512, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[529, 500, 478, 535, 509, 537, 489, 507, 503, 489, 527, 572, 501, 546, 516, 506, 547, 530, 513, 533, 462, 528, 477, 495, 506, 500, 538, 499, 513, 479, 554, 477, 485, 495, 539, 522, 517, 510, 494, 497, 492, 492, 513, 518, 510, 521, 528, 517, 486, 513, 503, 510, 526, 507, 522, 507, 469, 519, 529, 499, 508, 480, 558, 546, 493, 497, 500, 503, 510, 548, 551, 543, 523, 479, 506, 507, 554, 510, 535, 546, 515, 499, 528, 478, 531, 503, 530, 524, 507, 489, 509, 504, 507, 529, 482, 516, 536, 509, 500, 509, 489, 534, 532, 499, 487, 515, 526, 537, 477, 537, 498, 530, 483, 487, 523, 525, 530, 478, 517, 520, 528, 546, 505, 495, 508, 506, 494, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[499, 542, 521, 507, 530, 473, 516, 493, 554, 485, 579, 476, 494, 493, 485, 496, 521, 508, 482, 501, 551, 535, 499, 508, 494, 497, 487, 488, 535, 458, 512, 525, 492, 499, 534, 508, 545, 500, 526, 550, 546, 517, 490, 493, 520, 492, 525, 499, 493, 517, 497, 474, 521, 499, 538, 534, 544, 544, 517, 542, 457, 525, 496, 495, 489, 538, 496, 520, 534, 507, 520, 475, 533, 524, 534, 517, 504, 481, 497, 520, 542, 528, 503, 483, 523, 557, 518, 510, 478, 526, 507, 537, 503, 460, 492, 531, 534, 537, 510, 509, 500, 523, 562, 520, 476, 531, 534, 482, 525, 471, 528, 570, 546, 492, 511, 502, 503, 511, 542, 494, 517, 516, 468, 535, 499, 524, 510, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[522, 533, 495, 528, 517, 514, 502, 477, 477, 486, 506, 531, 500, 541, 505, 511, 563, 519, 509, 481, 501, 514, 541, 504, 507, 504, 531, 494, 483, 504, 492, 494, 521, 524, 559, 547, 501, 539, 510, 522, 517, 533, 479, 519, 508, 482, 501, 534, 523, 508, 528, 519, 523, 511, 515, 538, 480, 510, 529, 540, 546, 517, 469, 519, 504, 531, 496, 504, 522, 530, 574, 528, 532, 513, 490, 467, 507, 561, 477, 541, 510, 511, 504, 486, 533, 514, 490, 511, 529, 506, 466, 511, 517, 516, 502, 524, 484, 575, 499, 490, 522, 483, 513, 457, 467, 531, 477, 539, 547, 490, 521, 502, 538, 494, 514, 507, 518, 498, 537, 499, 512, 519, 496, 476, 557, 504, 477, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[513, 519, 465, 534, 549, 504, 506, 522, 492, 497, 502, 510, 508, 531, 512, 527, 504, 504, 542, 483, 514, 498, 486, 520, 490, 538, 459, 491, 468, 524, 537, 490, 504, 501, 523, 521, 494, 530, 529, 520, 550, 534, 522, 490, 530, 501, 495, 500, 521, 541, 526, 506, 509, 542, 518, 523, 487, 577, 512, 539, 509, 515, 524, 452, 526, 488, 531, 510, 497, 514, 504, 524, 524, 505, 491, 478, 497, 543, 512, 555, 536, 539, 512, 500, 526, 547, 476, 525, 507, 514, 465, 475, 520, 535, 516, 500, 508, 509, 499, 520, 497, 509, 533, 532, 530, 484, 530, 507, 539, 500, 498, 509, 527, 535, 493, 471, 495, 502, 496, 519, 519, 524, 547, 499, 491, 542, 472, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[497, 491, 545, 503, 511, 515, 467, 506, 538, 533, 543, 503, 509, 512, 507, 522, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 539, 560, 475, 518, 511, 498, 496, 567, 534, 465, 490, 563, 523, 518, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 488, 545, 509, 479, 516, 494, 507, 518, 506, 473, 510, 484, 456, 516, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 520, 519, 509, 555, 496, 527, 519, 507, 498, 504, 512, 496, 560, 529, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 520, 548, 501, 490, 493, 538, 511, 498, 487, 523, 485, 510, 526, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 517, 509, 501, 485, 501, 510, 545, 474, 495, 523, 485, 525, 493, 492, 498, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 535, 536, 518, 509, 506, 482, 543, 558, 494, 540, 510, 508, 479, 510, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 487, 497, 560, 536, 522, 518, 489, 550, 468, 532, 504, 506, 475, 548, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8292, 8025, 8324, 8106, 8077, 8245, 8237] avg=8192.0000 max=8324 max_rank=3 min=8025 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[480, 495, 480, 485, 523, 516, 487, 524, 524, 521, 513, 511, 535, 511, 519, 559, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 510, 529, 529, 496, 478, 478, 511, 553, 512, 511, 507, 461, 483, 528, 555, 14, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 505, 522, 499, 473, 530, 509, 495, 543, 523, 524, 538, 514, 528, 529, 529, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 526, 491, 536, 499, 513, 557, 539, 526, 447, 500, 495, 535, 507, 474, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 494, 498, 528, 531, 497, 541, 543, 544, 526, 499, 520, 517, 496, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 494, 514, 531, 536, 497, 516, 540, 486, 492, 522, 485, 540, 515, 523, 501, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 475, 510, 477, 536, 518, 548, 467, 502, 506, 503, 490, 510, 506, 517, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 481, 520, 518, 472, 519, 508, 493, 509, 519, 538, 520, 523, 459, 555, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8165, 8278, 8147, 8233, 8230, 8130, 8144] avg=8192.0000 max=8278 max_rank=2 min=8130 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 533, 495, 528, 517, 514, 502, 477, 477, 486, 506, 531, 500, 541, 505, 511, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 519, 509, 481, 501, 514, 541, 504, 507, 504, 531, 494, 483, 504, 492, 494, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 524, 559, 547, 501, 539, 510, 522, 517, 533, 479, 519, 508, 482, 501, 534, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 508, 528, 519, 523, 511, 515, 538, 480, 468, 529, 540, 546, 517, 469, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 531, 496, 504, 522, 530, 574, 528, 532, 513, 490, 467, 507, 561, 477, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 504, 486, 533, 514, 490, 511, 529, 506, 466, 511, 517, 516, 502, 524, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 575, 499, 490, 522, 483, 513, 457, 467, 531, 477, 539, 547, 490, 521, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 494, 514, 507, 518, 498, 537, 499, 512, 519, 496, 476, 557, 504, 477, 529, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8168, 8307, 8233, 8248, 8154, 8078, 8176] avg=8192.0000 max=8307 max_rank=2 min=8078 min_rank=6 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 542, 521, 507, 530, 473, 516, 493, 554, 485, 579, 476, 494, 493, 485, 496, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 508, 482, 501, 551, 535, 499, 508, 494, 497, 487, 488, 535, 458, 512, 525, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 499, 534, 508, 545, 500, 526, 550, 546, 517, 490, 493, 520, 492, 525, 499, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 497, 474, 521, 499, 538, 534, 544, 499, 517, 542, 457, 525, 496, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 538, 496, 520, 534, 507, 520, 475, 533, 524, 534, 517, 504, 481, 497, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 528, 503, 483, 523, 557, 518, 510, 478, 526, 507, 537, 503, 460, 492, 531, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 537, 510, 509, 500, 523, 562, 520, 476, 531, 534, 482, 525, 471, 528, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 492, 511, 502, 503, 511, 542, 494, 517, 516, 468, 535, 499, 524, 510, 494, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8128, 8249, 8148, 8162, 8222, 8290, 8165] avg=8192.0000 max=8290 max_rank=6 min=8128 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[484, 513, 491, 507, 515, 525, 528, 532, 507, 512, 515, 491, 513, 506, 501, 524, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 523, 499, 513, 528, 497, 507, 524, 542, 545, 476, 527, 523, 502, 495, 512, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 488, 555, 510, 516, 553, 500, 531, 464, 473, 523, 440, 502, 525, 534, 520, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 538, 521, 508, 535, 475, 502, 549, 478, 481, 494, 508, 473, 534, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 515, 480, 514, 502, 529, 521, 493, 505, 540, 501, 526, 525, 529, 472, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 522, 456, 498, 530, 510, 533, 520, 481, 506, 528, 523, 503, 499, 542, 461, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 510, 525, 528, 493, 523, 513, 506, 539, 511, 503, 510, 536, 506, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 500, 564, 540, 501, 525, 514, 521, 449, 544, 504, 483, 518, 529, 524, 549, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8302, 8113, 8118, 8157, 8180, 8227, 8248] avg=8192.0000 max=8302 max_rank=1 min=8113 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 520, 522, 507, 475, 495, 478, 511, 505, 487, 515, 515, 479, 504, 519, 512, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 530, 511, 502, 501, 554, 474, 482, 528, 519, 530, 488, 512, 525, 511, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 531, 502, 522, 513, 514, 502, 521, 485, 518, 506, 540, 529, 479, 494, 505, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 522, 499, 526, 513, 505, 533, 472, 473, 514, 522, 526, 509, 522, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 512, 566, 494, 543, 523, 502, 481, 516, 494, 500, 495, 514, 479, 508, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 515, 539, 525, 530, 528, 524, 473, 545, 540, 480, 504, 514, 517, 486, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 501, 534, 478, 527, 530, 521, 508, 485, 498, 491, 500, 527, 511, 510, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 533, 499, 496, 502, 526, 478, 554, 512, 521, 558, 497, 490, 510, 512, 539, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8220, 8144, 8178, 8191, 8274, 8196, 8267] avg=8192.0000 max=8274 max_rank=5 min=8066 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[529, 500, 478, 535, 509, 537, 489, 507, 503, 489, 527, 572, 501, 546, 516, 506, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 530, 513, 533, 462, 528, 477, 495, 506, 500, 538, 499, 513, 479, 554, 477, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 495, 539, 522, 517, 510, 494, 497, 492, 492, 513, 518, 510, 521, 528, 517, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 513, 503, 510, 526, 507, 522, 507, 469, 477, 529, 499, 508, 480, 558, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 497, 500, 503, 510, 548, 551, 543, 523, 479, 506, 507, 554, 510, 535, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 528, 478, 531, 503, 530, 524, 507, 489, 509, 504, 507, 529, 482, 516, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 509, 500, 509, 489, 534, 532, 499, 487, 515, 526, 537, 477, 537, 498, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 487, 523, 525, 530, 478, 517, 520, 528, 546, 505, 495, 508, 506, 494, 493, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8178, 8162, 8140, 8276, 8175, 8195, 8139] avg=8192.0000 max=8276 max_rank=4 min=8139 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 519, 465, 534, 549, 504, 506, 522, 492, 497, 502, 510, 508, 531, 512, 527, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 504, 542, 483, 514, 498, 486, 520, 490, 538, 459, 491, 468, 524, 537, 490, 19, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 523, 521, 494, 530, 529, 520, 550, 534, 522, 490, 530, 501, 495, 500, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 541, 526, 506, 509, 542, 518, 523, 487, 524, 512, 539, 509, 515, 524, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 488, 531, 510, 497, 514, 504, 524, 524, 505, 491, 478, 497, 543, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 539, 512, 500, 526, 547, 476, 525, 507, 514, 465, 475, 520, 535, 516, 500, 20, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 509, 499, 520, 497, 509, 533, 532, 530, 484, 530, 507, 539, 500, 498, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 535, 493, 471, 495, 502, 496, 519, 519, 524, 547, 499, 491, 542, 472, 524, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8083, 8258, 8248, 8163, 8224, 8175, 8160] avg=8192.0000 max=8258 max_rank=2 min=8083 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4352, 4096, 4096, 4096, 4224, 3968, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[512, 543, 487, 515, 498, 518, 560, 496, 489, 547, 518, 543, 521, 502, 509, 506, 518, 535, 540, 525, 512, 509, 532, 521, 485, 486, 511, 542, 551, 517, 491, 501, 479, 513, 535, 503, 500, 519, 497, 502, 523, 555, 525, 507, 513, 507, 530, 501, 483, 501, 521, 500, 536, 527, 547, 512, 528, 505, 517, 514, 511, 511, 531, 511, 498, 510, 496, 545, 513, 512, 516, 519, 524, 514, 494, 553, 496, 505, 512, 508, 531, 477, 515, 484, 504, 530, 511, 565, 509, 495, 526, 463, 515, 516, 485, 467, 506, 552, 453, 502, 533, 493, 501, 517, 519, 522, 520, 486, 522, 508, 559, 541, 498, 477, 479, 460, 494, 483, 558, 524, 494, 449, 513, 506, 552, 504, 505, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[490, 499, 518, 531, 513, 531, 542, 460, 491, 489, 499, 472, 523, 499, 506, 498, 515, 478, 514, 500, 501, 529, 518, 514, 516, 512, 519, 455, 514, 509, 478, 512, 488, 485, 487, 512, 535, 549, 509, 521, 475, 502, 521, 501, 524, 514, 538, 527, 486, 548, 474, 495, 513, 495, 505, 512, 508, 517, 481, 538, 526, 498, 536, 494, 539, 494, 455, 532, 510, 496, 504, 523, 513, 507, 532, 539, 539, 531, 550, 548, 521, 494, 513, 555, 534, 506, 539, 489, 521, 537, 515, 518, 485, 511, 526, 493, 514, 513, 486, 532, 512, 521, 506, 526, 482, 568, 523, 512, 539, 531, 509, 506, 516, 531, 522, 510, 515, 514, 484, 499, 524, 504, 566, 506, 538, 505, 485, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[499, 536, 535, 522, 499, 501, 539, 504, 511, 496, 477, 500, 521, 531, 533, 555, 534, 515, 511, 528, 527, 514, 542, 490, 556, 523, 453, 523, 511, 542, 511, 500, 503, 508, 510, 476, 500, 540, 499, 504, 490, 492, 509, 513, 492, 484, 534, 512, 535, 478, 518, 524, 511, 512, 532, 533, 535, 506, 469, 499, 516, 559, 499, 517, 505, 468, 512, 545, 496, 530, 499, 493, 517, 546, 504, 514, 491, 508, 524, 529, 518, 508, 541, 519, 531, 498, 488, 520, 553, 541, 506, 533, 487, 479, 494, 510, 497, 508, 535, 483, 506, 466, 506, 542, 494, 485, 481, 513, 523, 533, 491, 509, 537, 501, 464, 499, 517, 537, 557, 494, 504, 511, 527, 528, 530, 495, 505, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[476, 499, 494, 499, 523, 488, 500, 527, 520, 543, 528, 485, 526, 540, 530, 533, 533, 537, 545, 513, 496, 543, 526, 479, 519, 515, 517, 526, 527, 511, 514, 517, 504, 537, 581, 499, 499, 487, 522, 521, 536, 504, 501, 511, 513, 498, 517, 524, 520, 514, 528, 518, 499, 548, 517, 535, 544, 514, 496, 520, 502, 532, 529, 489, 498, 504, 528, 539, 504, 543, 526, 531, 527, 466, 516, 537, 490, 500, 524, 528, 530, 549, 495, 516, 475, 498, 506, 499, 511, 491, 482, 490, 510, 463, 537, 484, 515, 493, 519, 508, 548, 504, 492, 518, 513, 525, 493, 489, 513, 514, 526, 504, 504, 498, 492, 469, 490, 554, 496, 510, 478, 499, 494, 511, 485, 526, 509, 462] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[490, 544, 548, 474, 484, 519, 502, 521, 508, 492, 518, 481, 514, 513, 525, 493, 555, 503, 519, 494, 534, 498, 553, 497, 525, 536, 477, 510, 481, 488, 521, 494, 518, 548, 508, 534, 479, 490, 524, 552, 519, 489, 528, 488, 514, 506, 454, 482, 522, 525, 510, 532, 514, 457, 495, 537, 515, 527, 516, 482, 467, 525, 509, 482, 507, 549, 520, 527, 497, 496, 544, 540, 534, 525, 523, 558, 498, 509, 550, 508, 497, 494, 518, 526, 558, 442, 482, 513, 509, 513, 489, 518, 540, 488, 519, 514, 481, 546, 536, 505, 486, 495, 530, 528, 530, 504, 487, 539, 507, 477, 523, 501, 503, 554, 503, 549, 570, 517, 519, 524, 502, 531, 508, 490, 496, 507, 489, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[499, 499, 532, 530, 518, 531, 516, 501, 479, 475, 509, 519, 547, 518, 495, 543, 544, 475, 481, 527, 481, 500, 505, 548, 502, 483, 488, 542, 500, 505, 529, 530, 523, 542, 549, 511, 512, 533, 501, 518, 516, 503, 483, 495, 513, 495, 498, 530, 496, 526, 512, 539, 540, 521, 536, 493, 451, 530, 557, 514, 514, 545, 526, 510, 477, 491, 469, 488, 528, 520, 529, 495, 517, 518, 510, 539, 538, 490, 524, 504, 511, 484, 510, 492, 508, 508, 515, 550, 511, 524, 473, 538, 521, 493, 509, 511, 510, 562, 531, 499, 491, 507, 550, 509, 498, 485, 529, 488, 494, 525, 497, 524, 506, 525, 530, 521, 505, 529, 478, 493, 522, 505, 487, 504, 515, 491, 495, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[522, 546, 514, 523, 490, 481, 555, 505, 500, 497, 489, 516, 480, 499, 513, 531, 516, 514, 487, 522, 459, 490, 532, 539, 513, 558, 524, 475, 473, 510, 517, 537, 495, 518, 526, 502, 551, 546, 513, 479, 480, 531, 508, 532, 504, 524, 518, 520, 491, 488, 529, 513, 512, 494, 502, 508, 563, 508, 544, 505, 525, 509, 474, 505, 494, 515, 474, 489, 504, 524, 551, 467, 533, 499, 522, 494, 540, 515, 467, 439, 501, 494, 499, 517, 499, 520, 525, 521, 517, 500, 512, 521, 514, 515, 546, 511, 520, 516, 477, 520, 498, 551, 513, 526, 494, 524, 478, 493, 542, 512, 552, 516, 555, 502, 490, 556, 521, 513, 564, 502, 550, 535, 556, 452, 476, 523, 503, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[496, 506, 524, 506, 495, 506, 508, 479, 499, 524, 529, 538, 508, 509, 514, 524, 547, 543, 497, 554, 490, 539, 548, 482, 498, 503, 509, 498, 482, 505, 520, 517, 536, 511, 524, 485, 498, 516, 484, 523, 523, 493, 481, 523, 531, 525, 502, 501, 527, 511, 528, 500, 500, 514, 502, 527, 524, 521, 504, 513, 541, 541, 462, 520, 544, 522, 517, 544, 506, 484, 533, 536, 490, 546, 498, 506, 508, 495, 551, 506, 483, 490, 526, 502, 492, 562, 520, 522, 490, 488, 516, 516, 519, 547, 515, 552, 509, 513, 519, 508, 515, 519, 531, 488, 481, 508, 509, 518, 469, 499, 519, 525, 496, 529, 478, 515, 487, 534, 488, 530, 480, 541, 510, 519, 466, 506, 502, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 543, 487, 515, 498, 518, 560, 496, 489, 547, 518, 543, 521, 502, 509, 506, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 535, 540, 525, 512, 509, 532, 521, 485, 486, 511, 542, 551, 517, 491, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 513, 535, 503, 500, 519, 497, 502, 523, 555, 525, 507, 513, 507, 530, 501, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 501, 521, 500, 536, 527, 547, 512, 528, 505, 517, 514, 511, 485, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 510, 496, 545, 513, 512, 516, 519, 524, 514, 494, 519, 496, 505, 512, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 477, 515, 484, 504, 530, 511, 565, 509, 495, 526, 463, 515, 516, 485, 467, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 552, 453, 502, 533, 493, 501, 517, 519, 522, 520, 486, 522, 508, 559, 541, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 477, 479, 460, 494, 483, 558, 524, 494, 449, 513, 506, 552, 504, 505, 494, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8260, 8216, 8229, 8181, 8115, 8238, 8024] avg=8192.0000 max=8273 max_rank=0 min=8024 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32558 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[499, 536, 535, 522, 499, 501, 539, 504, 511, 496, 477, 500, 521, 531, 533, 555, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 511, 528, 527, 514, 542, 490, 556, 523, 453, 523, 511, 542, 511, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 508, 510, 476, 500, 540, 499, 504, 490, 492, 509, 513, 492, 484, 534, 512, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 478, 518, 524, 511, 512, 532, 533, 535, 506, 469, 499, 516, 530, 499, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 468, 512, 545, 496, 530, 499, 493, 517, 546, 504, 483, 491, 508, 524, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 541, 519, 531, 498, 488, 520, 553, 541, 506, 533, 487, 479, 494, 510, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 508, 535, 483, 506, 466, 506, 542, 494, 485, 481, 513, 523, 533, 491, 509, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 501, 464, 499, 517, 537, 557, 494, 504, 511, 527, 528, 530, 495, 505, 503, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8263, 8073, 8214, 8150, 8251, 8077, 8240] avg=8192.0000 max=8268 max_rank=0 min=8073 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[490, 499, 518, 531, 513, 531, 542, 460, 491, 489, 499, 472, 523, 499, 506, 498, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 478, 514, 500, 501, 529, 518, 514, 516, 512, 519, 455, 514, 509, 478, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 485, 487, 512, 535, 549, 509, 521, 475, 502, 521, 501, 524, 514, 538, 527, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 548, 474, 495, 513, 495, 505, 512, 508, 517, 481, 538, 526, 472, 536, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 494, 455, 532, 510, 496, 504, 523, 513, 507, 532, 506, 539, 531, 550, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 494, 513, 555, 534, 506, 539, 489, 521, 537, 515, 518, 485, 511, 526, 493, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 513, 486, 532, 512, 521, 506, 526, 482, 568, 523, 512, 539, 531, 509, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 531, 522, 510, 515, 514, 484, 499, 524, 504, 566, 506, 538, 505, 485, 509, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8068, 8195, 8100, 8279, 8279, 8284, 8261] avg=8192.0000 max=8284 max_rank=6 min=8068 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=33103 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[499, 499, 532, 530, 518, 531, 516, 501, 479, 475, 509, 519, 547, 518, 495, 543, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 475, 481, 527, 481, 500, 505, 548, 502, 483, 488, 542, 500, 505, 529, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 542, 549, 511, 512, 533, 501, 518, 516, 503, 483, 495, 513, 495, 498, 530, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 526, 512, 539, 540, 521, 536, 493, 451, 530, 557, 514, 514, 517, 526, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 491, 469, 488, 528, 520, 529, 495, 517, 518, 510, 506, 538, 490, 524, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 484, 510, 492, 508, 508, 515, 550, 511, 524, 473, 538, 521, 493, 509, 511, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 562, 531, 499, 491, 507, 550, 509, 498, 485, 529, 488, 494, 525, 497, 524, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 525, 530, 521, 505, 529, 478, 493, 522, 505, 487, 504, 515, 491, 495, 553, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8123, 8229, 8282, 8104, 8182, 8204, 8192] avg=8192.0000 max=8282 max_rank=3 min=8104 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[476, 499, 494, 499, 523, 488, 500, 527, 520, 543, 528, 485, 526, 540, 530, 533, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 537, 545, 513, 496, 543, 526, 479, 519, 515, 517, 526, 527, 511, 514, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 537, 581, 499, 499, 487, 522, 521, 536, 504, 501, 511, 513, 498, 517, 524, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 528, 518, 499, 548, 517, 535, 544, 514, 496, 520, 502, 505, 529, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 528, 539, 504, 543, 526, 531, 527, 466, 516, 504, 490, 500, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 549, 495, 516, 475, 498, 506, 499, 511, 491, 482, 490, 510, 463, 537, 484, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 493, 519, 508, 548, 504, 492, 518, 513, 525, 493, 489, 513, 514, 526, 504, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 492, 469, 490, 554, 496, 510, 478, 499, 494, 511, 485, 526, 509, 462, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8301, 8261, 8278, 8228, 8059, 8179, 8010] avg=8192.0000 max=8301 max_rank=1 min=8010 min_rank=7 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 546, 514, 523, 490, 481, 555, 505, 500, 497, 489, 516, 480, 499, 513, 531, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 487, 522, 459, 490, 532, 539, 513, 558, 524, 475, 473, 510, 517, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 518, 526, 502, 551, 546, 513, 479, 480, 531, 508, 532, 504, 524, 518, 520, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 488, 529, 513, 512, 494, 502, 508, 563, 508, 544, 505, 525, 483, 474, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 474, 489, 504, 524, 551, 467, 533, 499, 522, 464, 540, 515, 467, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 494, 499, 517, 499, 520, 525, 521, 517, 500, 512, 521, 514, 515, 546, 511, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 516, 477, 520, 498, 551, 513, 526, 494, 524, 478, 493, 542, 512, 552, 516, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 502, 490, 556, 521, 513, 564, 502, 550, 535, 556, 452, 476, 523, 503, 523, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8150, 8254, 8144, 7997, 8234, 8236, 8351] avg=8192.0000 max=8351 max_rank=7 min=7997 min_rank=4 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[490, 544, 548, 474, 484, 519, 502, 521, 508, 492, 518, 481, 514, 513, 525, 493, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 503, 519, 494, 534, 498, 553, 497, 525, 536, 477, 510, 481, 488, 521, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 548, 508, 534, 479, 490, 524, 552, 519, 489, 528, 488, 514, 506, 454, 482, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 510, 532, 514, 457, 495, 537, 515, 527, 516, 482, 467, 492, 509, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 549, 520, 527, 497, 496, 544, 540, 534, 525, 523, 515, 498, 509, 550, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 518, 526, 558, 442, 482, 513, 509, 513, 489, 518, 540, 488, 519, 514, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 546, 536, 505, 486, 495, 530, 528, 530, 504, 487, 539, 507, 477, 523, 501, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 554, 503, 549, 570, 517, 519, 524, 502, 531, 508, 490, 496, 507, 489, 535, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8156, 8145, 8082, 8342, 8152, 8183, 8335] avg=8192.0000 max=8342 max_rank=4 min=8082 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=33012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[496, 506, 524, 506, 495, 506, 508, 479, 499, 524, 529, 538, 508, 509, 514, 524, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 543, 497, 554, 490, 539, 548, 482, 498, 503, 509, 498, 482, 505, 520, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 511, 524, 485, 498, 516, 484, 523, 523, 493, 481, 523, 531, 525, 502, 501, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 511, 528, 500, 500, 514, 502, 527, 524, 521, 504, 513, 541, 513, 462, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 522, 517, 544, 506, 484, 533, 536, 490, 546, 498, 475, 508, 495, 551, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 490, 526, 502, 492, 562, 520, 522, 490, 488, 516, 516, 519, 547, 515, 552, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 519, 508, 515, 519, 531, 488, 481, 508, 509, 518, 469, 499, 519, 525, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 529, 478, 515, 487, 534, 488, 530, 480, 541, 510, 519, 466, 506, 502, 511, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8215, 8163, 8207, 8255, 8264, 8135, 8123] avg=8192.0000 max=8264 max_rank=5 min=8123 min_rank=7 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[536, 535, 548, 501, 501, 506, 534, 521, 514, 477, 482, 523, 468, 479, 526, 491, 492, 539, 502, 512, 528, 538, 553, 516, 516, 529, 485, 517, 523, 548, 463, 499, 479, 534, 530, 488, 545, 515, 477, 544, 504, 484, 531, 521, 526, 501, 513, 513, 541, 507, 517, 532, 495, 491, 524, 519, 514, 490, 494, 519, 506, 509, 515, 492, 518, 486, 504, 484, 543, 517, 561, 508, 528, 508, 493, 558, 551, 483, 526, 501, 499, 506, 505, 497, 514, 531, 487, 527, 515, 513, 493, 499, 480, 502, 552, 527, 528, 483, 468, 496, 543, 517, 491, 530, 470, 517, 480, 543, 484, 527, 513, 505, 487, 492, 541, 484, 495, 528, 529, 523, 518, 491, 553, 563, 515, 497, 501, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[534, 538, 540, 499, 553, 491, 527, 507, 521, 535, 539, 505, 502, 555, 480, 485, 544, 523, 511, 531, 528, 535, 519, 483, 545, 517, 525, 508, 536, 508, 509, 510, 500, 517, 497, 508, 467, 546, 543, 539, 535, 505, 507, 561, 467, 518, 546, 537, 502, 515, 519, 524, 518, 495, 510, 505, 528, 550, 502, 546, 447, 512, 529, 532, 505, 519, 525, 494, 525, 493, 462, 492, 506, 509, 515, 524, 468, 503, 494, 505, 520, 501, 503, 517, 523, 509, 472, 484, 535, 529, 499, 469, 506, 500, 504, 521, 505, 498, 526, 504, 505, 530, 509, 491, 538, 492, 518, 534, 487, 510, 542, 494, 493, 485, 495, 499, 482, 541, 508, 512, 526, 489, 510, 487, 512, 492, 483, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[517, 501, 483, 555, 495, 540, 527, 525, 485, 487, 506, 510, 502, 519, 553, 471, 452, 495, 530, 498, 536, 468, 511, 512, 537, 530, 474, 544, 470, 512, 500, 491, 501, 495, 474, 496, 518, 490, 539, 533, 513, 529, 496, 485, 534, 524, 507, 497, 511, 514, 487, 536, 511, 529, 528, 493, 511, 511, 508, 494, 507, 520, 504, 552, 522, 481, 527, 541, 492, 528, 503, 526, 548, 538, 494, 502, 486, 496, 491, 522, 515, 501, 503, 550, 505, 523, 523, 511, 465, 501, 509, 477, 555, 481, 541, 526, 601, 521, 508, 508, 509, 522, 505, 504, 509, 515, 518, 498, 519, 511, 515, 480, 497, 524, 569, 541, 510, 542, 531, 523, 560, 525, 527, 504, 464, 512, 515, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[525, 501, 497, 536, 455, 502, 530, 527, 508, 517, 513, 474, 521, 501, 498, 532, 488, 520, 504, 475, 509, 537, 513, 528, 501, 541, 508, 500, 492, 524, 504, 494, 487, 564, 504, 485, 517, 540, 507, 514, 522, 560, 484, 534, 539, 492, 514, 501, 523, 531, 542, 554, 516, 506, 548, 514, 516, 506, 496, 520, 494, 484, 523, 482, 518, 550, 484, 531, 539, 503, 500, 492, 529, 516, 530, 545, 498, 491, 536, 517, 543, 519, 504, 496, 489, 502, 517, 507, 560, 505, 491, 508, 490, 497, 482, 467, 504, 499, 535, 508, 503, 512, 497, 480, 549, 537, 530, 560, 524, 515, 516, 500, 493, 491, 543, 488, 535, 537, 512, 545, 541, 513, 481, 480, 497, 460, 507, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[470, 487, 526, 500, 500, 525, 522, 550, 510, 538, 480, 546, 527, 487, 517, 470, 485, 499, 515, 539, 500, 505, 534, 493, 572, 542, 500, 482, 528, 524, 513, 486, 498, 504, 483, 514, 517, 506, 539, 515, 552, 500, 554, 516, 554, 503, 533, 523, 516, 523, 569, 506, 520, 521, 528, 480, 518, 488, 521, 500, 526, 511, 509, 521, 452, 512, 501, 519, 551, 473, 526, 528, 532, 539, 503, 468, 549, 490, 481, 543, 488, 514, 516, 530, 556, 556, 531, 519, 535, 495, 477, 483, 523, 488, 521, 507, 537, 527, 494, 510, 506, 481, 541, 470, 525, 523, 487, 505, 484, 513, 498, 508, 474, 471, 529, 516, 485, 480, 528, 484, 505, 533, 519, 517, 532, 485, 527, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[501, 527, 524, 521, 516, 499, 493, 480, 511, 517, 487, 494, 522, 495, 546, 487, 514, 524, 523, 484, 517, 502, 519, 500, 538, 495, 549, 484, 506, 571, 506, 496, 479, 546, 482, 505, 524, 496, 480, 556, 475, 505, 537, 494, 542, 531, 517, 530, 510, 554, 507, 478, 510, 514, 538, 521, 462, 519, 498, 522, 530, 579, 548, 490, 545, 512, 516, 529, 489, 505, 484, 544, 522, 522, 542, 511, 492, 529, 501, 494, 501, 497, 513, 494, 490, 528, 509, 511, 545, 495, 571, 504, 493, 525, 534, 517, 479, 509, 526, 509, 521, 484, 476, 500, 505, 498, 499, 496, 483, 533, 520, 550, 503, 504, 534, 506, 508, 505, 521, 515, 502, 574, 497, 494, 490, 492, 486, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[537, 494, 512, 457, 553, 527, 511, 502, 525, 522, 490, 534, 501, 494, 533, 473, 528, 501, 546, 489, 503, 530, 499, 530, 481, 464, 484, 545, 494, 518, 525, 502, 507, 518, 529, 499, 519, 487, 498, 515, 511, 515, 537, 521, 494, 511, 497, 534, 511, 535, 508, 471, 492, 495, 502, 515, 528, 554, 476, 469, 513, 491, 494, 558, 508, 501, 515, 490, 524, 463, 465, 490, 503, 527, 532, 532, 539, 529, 521, 533, 546, 495, 525, 519, 525, 527, 494, 511, 549, 514, 544, 511, 483, 537, 514, 498, 534, 507, 562, 525, 508, 494, 529, 454, 498, 496, 518, 521, 552, 505, 516, 521, 528, 482, 544, 508, 495, 504, 504, 510, 516, 520, 541, 529, 500, 503, 501, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[533, 510, 500, 542, 508, 531, 540, 551, 523, 500, 538, 541, 531, 513, 490, 488, 501, 537, 515, 526, 484, 496, 493, 513, 502, 545, 513, 493, 489, 538, 532, 490, 526, 510, 521, 514, 483, 470, 518, 541, 527, 492, 518, 453, 520, 516, 517, 485, 496, 493, 500, 503, 509, 529, 466, 498, 468, 501, 504, 487, 508, 516, 520, 542, 512, 554, 495, 488, 501, 474, 505, 501, 528, 531, 523, 564, 507, 510, 517, 487, 539, 503, 554, 510, 565, 502, 527, 463, 502, 508, 520, 492, 497, 521, 475, 515, 552, 517, 514, 504, 500, 505, 462, 546, 491, 515, 518, 509, 488, 497, 506, 482, 542, 480, 499, 566, 529, 538, 516, 554, 519, 507, 570, 528, 503, 505, 518, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[536, 535, 548, 501, 501, 506, 533, 521, 514, 477, 482, 523, 468, 479, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 539, 502, 512, 528, 538, 553, 516, 516, 529, 485, 517, 523, 548, 463, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 534, 530, 488, 545, 515, 477, 523, 504, 484, 531, 521, 526, 501, 513, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 505, 517, 532, 495, 491, 524, 519, 514, 490, 494, 519, 506, 509, 515, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 504, 484, 543, 517, 561, 508, 528, 508, 493, 556, 551, 483, 526, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 505, 497, 514, 531, 487, 527, 515, 513, 493, 499, 480, 502, 552, 527, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 483, 468, 496, 543, 517, 491, 530, 470, 517, 480, 543, 484, 527, 513, 505, 21, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 492, 534, 484, 495, 528, 529, 523, 518, 491, 553, 563, 515, 497, 501, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8260, 8184, 8163, 8267, 8155, 8120, 8246] avg=8192.0000 max=8267 max_rank=4 min=8120 min_rank=6 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[501, 527, 524, 521, 516, 499, 492, 480, 511, 517, 487, 494, 522, 495, 546, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 524, 523, 484, 517, 502, 519, 500, 538, 495, 549, 484, 506, 571, 506, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 546, 482, 505, 524, 496, 480, 534, 475, 505, 537, 494, 542, 531, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 551, 507, 478, 510, 514, 538, 521, 462, 519, 498, 522, 530, 579, 548, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 512, 516, 529, 489, 505, 484, 544, 522, 522, 542, 509, 492, 529, 501, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 497, 513, 494, 490, 528, 509, 511, 545, 495, 571, 504, 493, 525, 534, 517, 3, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 509, 526, 509, 521, 484, 476, 500, 505, 498, 499, 496, 483, 533, 520, 550, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 527, 506, 508, 505, 521, 515, 502, 574, 497, 494, 490, 492, 486, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8228, 8177, 8277, 8235, 8236, 8114, 8150] avg=8192.0000 max=8277 max_rank=3 min=8114 min_rank=6 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 501, 483, 555, 495, 540, 526, 525, 485, 487, 506, 510, 502, 519, 553, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 495, 530, 498, 536, 468, 511, 512, 537, 530, 474, 544, 470, 512, 500, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 495, 474, 496, 518, 490, 539, 512, 513, 529, 496, 485, 534, 524, 507, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 512, 487, 536, 511, 529, 528, 493, 511, 511, 508, 494, 507, 520, 504, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 527, 541, 492, 528, 503, 526, 548, 538, 494, 500, 486, 496, 491, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 503, 550, 505, 523, 523, 511, 465, 501, 509, 477, 555, 481, 541, 526, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 521, 508, 508, 509, 522, 505, 504, 509, 515, 518, 498, 519, 511, 515, 480, 21, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 524, 562, 541, 510, 542, 531, 523, 560, 525, 527, 504, 464, 512, 515, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8060, 8110, 8214, 8195, 8194, 8268, 8320] avg=8192.0000 max=8320 max_rank=7 min=8060 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32977 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[534, 538, 540, 499, 553, 491, 526, 507, 521, 535, 539, 505, 502, 555, 480, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 523, 511, 531, 528, 535, 519, 483, 545, 517, 525, 508, 536, 508, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 517, 497, 508, 467, 546, 543, 518, 535, 505, 507, 561, 467, 518, 546, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 519, 524, 518, 495, 510, 505, 528, 550, 502, 546, 447, 512, 529, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 519, 525, 494, 525, 493, 462, 492, 506, 509, 515, 522, 468, 503, 494, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 503, 517, 523, 509, 472, 484, 535, 529, 499, 469, 506, 500, 504, 521, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 498, 526, 504, 505, 530, 509, 491, 538, 492, 518, 534, 487, 510, 542, 494, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 485, 490, 499, 482, 541, 508, 512, 526, 489, 510, 487, 512, 492, 483, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8332, 8272, 8232, 8037, 8099, 8207, 8047] avg=8192.0000 max=8332 max_rank=1 min=8037 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[525, 501, 497, 536, 455, 502, 528, 527, 508, 517, 513, 474, 521, 501, 498, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 520, 504, 475, 509, 537, 513, 528, 501, 541, 508, 500, 492, 519, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 564, 504, 485, 517, 540, 507, 490, 522, 560, 484, 534, 539, 492, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 542, 554, 516, 506, 548, 514, 516, 506, 496, 520, 494, 484, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 550, 484, 531, 539, 503, 500, 492, 529, 516, 530, 538, 498, 491, 536, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 519, 504, 496, 489, 502, 517, 507, 560, 505, 491, 508, 490, 497, 482, 467, 6, 8, 7, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 499, 535, 508, 503, 512, 497, 480, 549, 537, 530, 560, 524, 515, 516, 500, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 491, 530, 488, 535, 537, 512, 545, 541, 513, 481, 480, 497, 460, 507, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8133, 8240, 8247, 8272, 8105, 8300, 8104] avg=8192.0000 max=8300 max_rank=6 min=8104 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 510, 500, 542, 508, 531, 539, 551, 523, 500, 538, 541, 531, 513, 490, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 537, 515, 526, 484, 496, 493, 513, 502, 545, 513, 493, 489, 538, 532, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 510, 521, 514, 483, 470, 518, 520, 527, 492, 518, 453, 520, 516, 517, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 491, 500, 503, 509, 529, 466, 498, 468, 501, 504, 487, 508, 516, 520, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 554, 495, 488, 501, 474, 505, 501, 528, 531, 523, 562, 507, 510, 517, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 503, 554, 510, 565, 502, 527, 463, 502, 508, 520, 492, 497, 521, 475, 515, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 517, 514, 504, 500, 505, 462, 546, 491, 515, 518, 509, 488, 497, 506, 482, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 480, 494, 566, 529, 538, 516, 554, 519, 507, 570, 528, 503, 505, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8167, 8090, 8038, 8195, 8200, 8130, 8378] avg=8192.0000 max=8378 max_rank=7 min=8038 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[537, 494, 512, 457, 553, 527, 510, 502, 525, 522, 490, 534, 501, 494, 533, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 501, 546, 489, 503, 530, 499, 530, 481, 464, 484, 545, 494, 518, 525, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 529, 499, 519, 487, 498, 495, 511, 515, 537, 521, 494, 511, 497, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 533, 508, 471, 492, 495, 502, 515, 528, 554, 476, 469, 513, 491, 494, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 515, 490, 524, 463, 465, 490, 503, 527, 532, 530, 539, 529, 521, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 495, 525, 519, 525, 527, 494, 511, 549, 514, 544, 511, 483, 537, 514, 498, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 507, 562, 525, 508, 494, 529, 454, 498, 496, 518, 521, 552, 505, 516, 521, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 482, 537, 508, 495, 504, 504, 510, 516, 520, 541, 529, 500, 503, 501, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8139, 8172, 8110, 8170, 8300, 8264, 8217] avg=8192.0000 max=8300 max_rank=5 min=8110 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32951 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[470, 487, 526, 500, 500, 525, 521, 550, 510, 538, 480, 546, 527, 487, 517, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 499, 515, 539, 500, 505, 534, 493, 572, 542, 500, 482, 528, 524, 513, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 483, 514, 517, 506, 539, 495, 552, 500, 554, 516, 554, 503, 533, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 521, 569, 506, 520, 521, 528, 480, 518, 488, 521, 500, 526, 511, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 512, 501, 519, 551, 473, 526, 528, 532, 539, 503, 466, 549, 490, 481, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 514, 516, 530, 556, 556, 531, 519, 535, 495, 477, 483, 523, 488, 521, 507, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 527, 494, 510, 506, 481, 541, 470, 525, 523, 487, 505, 484, 513, 498, 508, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 471, 522, 516, 485, 480, 528, 484, 505, 533, 519, 517, 532, 485, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8217, 8291, 8255, 8165, 8247, 8133, 8074] avg=8192.0000 max=8291 max_rank=2 min=8074 min_rank=7 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32619 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4352, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4352, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4096, 4096, 4224, 4096, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[529, 528, 535, 498, 515, 507, 516, 513, 506, 501, 496, 527, 530, 558, 499, 516, 491, 501, 494, 524, 486, 527, 562, 539, 514, 492, 488, 531, 492, 496, 508, 492, 508, 508, 512, 515, 477, 539, 520, 528, 524, 525, 538, 520, 545, 472, 502, 516, 535, 481, 522, 486, 498, 526, 509, 517, 498, 457, 502, 513, 515, 522, 509, 488, 524, 457, 512, 534, 536, 535, 507, 544, 513, 499, 530, 490, 484, 495, 489, 519, 495, 501, 456, 535, 541, 498, 508, 484, 522, 524, 505, 526, 518, 496, 523, 520, 502, 537, 556, 513, 518, 514, 529, 464, 489, 531, 479, 513, 524, 544, 483, 505, 511, 508, 483, 548, 476, 545, 504, 518, 481, 552, 516, 533, 536, 522, 494, 550] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[519, 544, 519, 495, 533, 532, 541, 504, 487, 462, 476, 502, 511, 533, 532, 496, 536, 513, 515, 543, 530, 503, 506, 518, 476, 524, 501, 525, 529, 495, 522, 526, 521, 473, 524, 490, 501, 525, 484, 457, 490, 535, 510, 530, 543, 506, 526, 497, 526, 515, 491, 497, 511, 503, 514, 504, 551, 519, 482, 502, 545, 503, 495, 546, 516, 542, 498, 524, 497, 521, 479, 486, 521, 501, 543, 518, 486, 550, 476, 501, 512, 483, 532, 519, 513, 491, 513, 503, 535, 530, 542, 519, 530, 451, 504, 512, 506, 520, 522, 508, 536, 522, 525, 527, 542, 499, 471, 486, 515, 515, 544, 535, 495, 481, 528, 533, 484, 522, 544, 546, 519, 520, 483, 529, 522, 492, 469, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[503, 533, 480, 490, 546, 501, 536, 508, 513, 487, 483, 498, 503, 519, 493, 482, 499, 488, 543, 515, 503, 543, 510, 496, 485, 518, 494, 523, 525, 486, 530, 513, 493, 528, 516, 529, 513, 551, 512, 528, 530, 509, 501, 480, 473, 528, 529, 510, 497, 559, 523, 476, 530, 498, 532, 496, 507, 496, 505, 507, 508, 508, 537, 526, 518, 462, 513, 507, 519, 494, 533, 493, 511, 506, 512, 544, 511, 513, 509, 514, 517, 513, 522, 517, 506, 539, 516, 496, 486, 485, 498, 521, 534, 503, 519, 499, 506, 475, 531, 493, 489, 500, 516, 526, 495, 523, 506, 471, 546, 502, 524, 510, 544, 532, 516, 502, 521, 526, 565, 561, 519, 515, 532, 487, 533, 508, 553, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[484, 496, 512, 513, 520, 529, 520, 512, 496, 545, 515, 520, 507, 515, 554, 495, 540, 521, 515, 525, 512, 513, 513, 506, 527, 523, 527, 507, 513, 519, 481, 506, 534, 532, 511, 497, 539, 498, 523, 496, 505, 526, 552, 471, 511, 522, 533, 530, 530, 506, 524, 521, 485, 513, 547, 534, 516, 511, 487, 520, 486, 522, 477, 523, 498, 495, 510, 528, 498, 512, 493, 492, 515, 522, 505, 498, 541, 484, 510, 493, 546, 504, 477, 529, 510, 548, 495, 492, 496, 538, 504, 494, 492, 510, 505, 530, 505, 490, 493, 524, 458, 539, 513, 516, 482, 511, 493, 507, 478, 507, 554, 482, 524, 502, 521, 508, 536, 542, 534, 510, 508, 517, 513, 515, 507, 492, 501, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[501, 493, 506, 492, 498, 554, 517, 549, 534, 526, 490, 508, 529, 494, 509, 474, 500, 526, 523, 494, 534, 475, 535, 519, 557, 502, 513, 496, 462, 527, 508, 504, 522, 512, 536, 542, 482, 497, 481, 526, 528, 497, 500, 507, 500, 506, 536, 503, 533, 505, 503, 502, 529, 498, 499, 487, 519, 506, 473, 501, 590, 494, 508, 497, 542, 541, 515, 504, 520, 531, 519, 513, 507, 514, 532, 524, 481, 529, 501, 521, 526, 515, 535, 493, 537, 521, 524, 520, 488, 496, 534, 518, 503, 540, 507, 517, 503, 433, 537, 522, 494, 532, 524, 491, 519, 500, 505, 504, 539, 480, 506, 518, 501, 541, 488, 555, 502, 537, 518, 523, 507, 501, 498, 521, 459, 560, 486, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[529, 508, 545, 535, 547, 518, 497, 526, 528, 530, 501, 547, 502, 487, 509, 505, 530, 539, 511, 477, 508, 492, 552, 523, 508, 499, 543, 512, 544, 511, 502, 547, 498, 522, 537, 484, 546, 492, 519, 491, 518, 510, 509, 485, 488, 532, 491, 502, 495, 514, 510, 523, 506, 545, 514, 524, 498, 487, 491, 523, 492, 522, 524, 516, 507, 497, 491, 520, 509, 545, 487, 489, 464, 487, 510, 497, 520, 541, 524, 462, 532, 533, 524, 529, 547, 504, 544, 536, 501, 505, 501, 501, 502, 563, 519, 516, 489, 494, 463, 506, 543, 486, 488, 463, 491, 502, 492, 506, 517, 504, 516, 470, 514, 493, 514, 508, 506, 528, 488, 526, 525, 520, 536, 498, 546, 527, 491, 559] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[515, 513, 569, 545, 501, 512, 508, 539, 538, 523, 498, 506, 486, 484, 523, 478, 506, 520, 541, 490, 524, 505, 486, 523, 488, 481, 517, 530, 508, 522, 501, 491, 556, 469, 536, 525, 511, 529, 537, 494, 509, 471, 501, 481, 504, 515, 552, 483, 489, 527, 538, 490, 544, 508, 546, 540, 519, 521, 489, 509, 487, 497, 512, 518, 541, 531, 484, 473, 516, 501, 521, 498, 549, 534, 538, 475, 497, 518, 515, 512, 528, 485, 506, 492, 493, 565, 535, 546, 472, 498, 480, 518, 486, 533, 545, 520, 493, 507, 509, 466, 483, 489, 514, 490, 504, 506, 498, 524, 518, 534, 487, 561, 528, 541, 522, 540, 528, 517, 517, 511, 516, 549, 511, 539, 502, 480, 481, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[505, 512, 483, 520, 532, 505, 475, 519, 526, 491, 514, 521, 543, 507, 485, 494, 513, 532, 515, 514, 540, 513, 524, 529, 527, 527, 470, 519, 529, 509, 497, 527, 512, 518, 506, 526, 531, 502, 533, 536, 545, 542, 521, 534, 524, 526, 492, 525, 464, 491, 481, 500, 517, 523, 507, 488, 543, 486, 518, 514, 496, 549, 497, 491, 493, 477, 495, 523, 474, 507, 490, 533, 517, 528, 502, 488, 518, 509, 486, 480, 472, 515, 518, 523, 529, 499, 485, 532, 507, 508, 500, 546, 506, 506, 525, 522, 477, 512, 498, 520, 525, 533, 528, 488, 508, 515, 554, 504, 499, 541, 496, 520, 525, 535, 489, 545, 506, 487, 521, 517, 495, 493, 531, 526, 500, 522, 540, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[529, 528, 535, 498, 504, 507, 516, 513, 506, 501, 496, 527, 530, 558, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 494, 524, 486, 527, 539, 539, 514, 492, 488, 531, 492, 496, 508, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 491, 515, 477, 539, 520, 528, 524, 525, 538, 520, 545, 472, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 481, 522, 486, 498, 526, 509, 517, 498, 457, 502, 513, 515, 522, 509, 488, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 457, 512, 534, 536, 535, 507, 544, 513, 499, 530, 490, 484, 495, 489, 519, 21, 23, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 501, 456, 535, 520, 498, 508, 484, 522, 524, 505, 526, 518, 496, 523, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 556, 513, 518, 514, 529, 464, 489, 531, 479, 513, 524, 544, 483, 505, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 483, 477, 476, 545, 504, 518, 481, 552, 516, 533, 536, 522, 494, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8114, 8228, 8105, 8217, 8131, 8272, 8206] avg=8192.0000 max=8272 max_rank=6 min=8105 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 533, 480, 490, 531, 501, 536, 508, 513, 487, 483, 498, 503, 519, 493, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 488, 543, 515, 503, 543, 486, 496, 485, 518, 494, 523, 525, 486, 530, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 528, 486, 529, 513, 551, 512, 528, 530, 509, 501, 480, 473, 528, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 559, 523, 476, 530, 498, 532, 496, 507, 496, 505, 507, 508, 508, 537, 526, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 462, 513, 507, 519, 494, 533, 493, 511, 506, 512, 544, 511, 513, 509, 514, 24, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 513, 522, 517, 478, 539, 516, 496, 486, 485, 498, 521, 534, 503, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 475, 531, 493, 489, 500, 516, 526, 495, 523, 506, 471, 546, 502, 524, 510, 68, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 532, 516, 434, 521, 526, 565, 561, 519, 515, 532, 487, 533, 508, 553, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8060, 8147, 8200, 8240, 8215, 8143, 8187, 8344] avg=8192.0000 max=8344 max_rank=7 min=8060 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32889 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[484, 496, 512, 513, 509, 529, 520, 512, 496, 545, 515, 520, 507, 515, 554, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 521, 515, 525, 512, 513, 492, 506, 527, 523, 527, 507, 513, 519, 481, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 490, 497, 539, 498, 523, 496, 505, 526, 552, 471, 511, 522, 533, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 506, 524, 521, 485, 513, 547, 534, 516, 511, 487, 520, 486, 522, 477, 523, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 510, 528, 498, 512, 493, 492, 515, 522, 505, 498, 541, 484, 510, 493, 21, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 504, 477, 529, 490, 548, 495, 492, 496, 538, 504, 494, 492, 510, 505, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 490, 493, 524, 458, 539, 513, 516, 482, 511, 493, 507, 478, 507, 554, 482, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 502, 521, 442, 536, 542, 534, 510, 508, 517, 513, 515, 507, 492, 501, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8227, 8259, 8228, 8141, 8150, 8118, 8191] avg=8192.0000 max=8259 max_rank=2 min=8118 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32600 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 512, 483, 520, 521, 505, 475, 519, 526, 491, 514, 521, 543, 507, 485, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 532, 515, 514, 540, 513, 503, 529, 527, 527, 470, 519, 529, 509, 497, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 485, 526, 531, 502, 533, 536, 545, 542, 521, 534, 524, 526, 492, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 491, 481, 500, 517, 523, 507, 488, 543, 486, 518, 514, 496, 549, 497, 491, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 477, 495, 523, 474, 507, 490, 533, 517, 528, 502, 488, 518, 509, 486, 480, 21, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 515, 518, 523, 508, 499, 485, 532, 507, 508, 500, 546, 506, 506, 525, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 512, 498, 520, 525, 533, 528, 488, 508, 515, 554, 504, 499, 541, 496, 520, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 535, 489, 474, 506, 487, 521, 517, 495, 493, 531, 526, 500, 522, 540, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8264, 8352, 8092, 8067, 8172, 8289, 8179] avg=8192.0000 max=8352 max_rank=2 min=8067 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 493, 506, 492, 488, 554, 517, 549, 534, 526, 490, 508, 529, 494, 509, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 526, 523, 494, 534, 475, 513, 519, 557, 502, 513, 496, 462, 527, 508, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 514, 542, 482, 497, 481, 526, 528, 497, 500, 507, 500, 506, 536, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 505, 503, 502, 529, 498, 499, 487, 519, 506, 473, 501, 590, 494, 508, 497, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 541, 515, 504, 520, 531, 519, 513, 507, 514, 532, 524, 481, 529, 501, 521, 22, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 535, 493, 516, 521, 524, 520, 488, 496, 534, 518, 503, 540, 507, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 433, 537, 522, 494, 532, 524, 491, 519, 500, 505, 504, 539, 480, 506, 518, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 541, 488, 483, 502, 537, 518, 523, 507, 501, 498, 521, 459, 560, 486, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8153, 8153, 8170, 8343, 8253, 8179, 8121] avg=8192.0000 max=8343 max_rank=4 min=8121 min_rank=7 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[519, 544, 519, 495, 522, 532, 541, 504, 487, 462, 476, 502, 511, 533, 532, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 513, 515, 543, 530, 503, 486, 518, 476, 524, 501, 525, 529, 495, 522, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 473, 503, 490, 501, 525, 484, 457, 490, 535, 510, 530, 543, 506, 526, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 491, 497, 511, 503, 514, 504, 551, 519, 482, 502, 545, 503, 495, 546, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 542, 498, 524, 497, 521, 479, 486, 521, 501, 543, 518, 486, 550, 476, 501, 21, 20, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 483, 532, 519, 493, 491, 513, 503, 535, 530, 542, 519, 530, 451, 504, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 522, 508, 536, 522, 525, 527, 542, 499, 471, 486, 515, 515, 544, 535, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 481, 528, 464, 484, 522, 544, 546, 519, 520, 483, 529, 522, 492, 469, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8242, 8091, 8230, 8205, 8169, 8342, 8082] avg=8192.0000 max=8342 max_rank=6 min=8082 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 513, 569, 545, 491, 512, 508, 539, 538, 523, 498, 506, 486, 484, 523, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 541, 490, 524, 505, 466, 523, 488, 481, 517, 530, 508, 522, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 469, 514, 525, 511, 529, 537, 494, 509, 471, 501, 481, 504, 515, 552, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 527, 538, 490, 544, 508, 546, 540, 519, 521, 489, 509, 487, 497, 512, 518, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 531, 484, 473, 516, 501, 521, 498, 549, 534, 538, 475, 497, 518, 515, 512, 22, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 485, 506, 492, 474, 565, 535, 546, 472, 498, 480, 518, 486, 533, 545, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 507, 509, 466, 483, 489, 514, 490, 504, 506, 498, 524, 518, 534, 487, 561, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 541, 522, 470, 528, 517, 517, 511, 516, 549, 511, 539, 502, 480, 481, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8113, 8151, 8259, 8249, 8183, 8153, 8200] avg=8192.0000 max=8259 max_rank=3 min=8113 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[529, 508, 545, 535, 536, 518, 497, 526, 528, 530, 501, 547, 502, 487, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 539, 511, 477, 508, 492, 530, 523, 508, 499, 543, 512, 544, 511, 502, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 522, 515, 484, 546, 492, 519, 491, 518, 510, 509, 485, 488, 532, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 514, 510, 523, 506, 545, 514, 524, 498, 487, 491, 523, 492, 522, 524, 516, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 497, 491, 520, 509, 545, 487, 489, 464, 487, 510, 497, 520, 541, 524, 462, 22, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 533, 524, 529, 525, 504, 544, 536, 501, 505, 501, 501, 502, 563, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 494, 463, 506, 543, 486, 488, 463, 491, 502, 492, 506, 517, 504, 516, 470, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 493, 514, 442, 506, 528, 488, 526, 525, 520, 536, 498, 546, 527, 491, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8276, 8102, 8212, 8099, 8335, 7996, 8213] avg=8192.0000 max=8335 max_rank=5 min=7996 min_rank=6 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 3712, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[491, 496, 493, 501, 550, 560, 511, 532, 472, 520, 515, 525, 484, 544, 510, 542, 524, 498, 475, 505, 510, 486, 520, 499, 515, 497, 521, 519, 529, 506, 495, 522, 516, 520, 489, 495, 511, 523, 496, 559, 512, 513, 491, 498, 537, 498, 505, 490, 503, 521, 525, 502, 499, 548, 534, 543, 486, 495, 487, 532, 500, 507, 521, 509, 520, 502, 524, 542, 540, 495, 502, 494, 499, 550, 509, 488, 488, 511, 497, 487, 512, 510, 526, 508, 524, 511, 508, 532, 471, 554, 520, 509, 517, 522, 536, 483, 547, 485, 532, 519, 522, 512, 517, 499, 551, 551, 481, 540, 499, 510, 516, 495, 509, 533, 491, 490, 501, 489, 504, 539, 499, 495, 522, 525, 510, 504, 492, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[505, 566, 542, 530, 533, 546, 512, 533, 497, 478, 527, 509, 480, 504, 497, 512, 511, 493, 516, 510, 512, 530, 503, 480, 502, 470, 492, 544, 512, 532, 504, 533, 530, 505, 512, 514, 501, 502, 494, 489, 484, 476, 486, 509, 536, 498, 522, 559, 518, 487, 504, 507, 520, 489, 550, 523, 513, 513, 494, 490, 500, 523, 536, 489, 479, 527, 507, 501, 527, 486, 501, 544, 497, 533, 532, 526, 510, 524, 516, 525, 515, 502, 520, 511, 543, 514, 541, 508, 480, 523, 500, 546, 528, 530, 490, 470, 517, 488, 501, 484, 511, 492, 521, 561, 539, 495, 471, 536, 523, 520, 489, 495, 507, 487, 542, 512, 534, 500, 497, 500, 545, 503, 483, 559, 497, 517, 561, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[474, 512, 489, 514, 500, 514, 477, 493, 518, 554, 495, 522, 507, 512, 486, 492, 478, 544, 532, 519, 498, 487, 527, 504, 512, 545, 488, 511, 510, 565, 524, 510, 505, 524, 525, 547, 548, 515, 543, 485, 495, 479, 505, 502, 496, 523, 503, 496, 504, 510, 522, 476, 495, 491, 480, 454, 522, 561, 497, 533, 504, 487, 536, 552, 519, 544, 470, 538, 539, 547, 511, 493, 499, 481, 509, 498, 507, 509, 498, 489, 520, 542, 502, 503, 466, 526, 530, 530, 456, 509, 535, 511, 503, 534, 508, 509, 454, 489, 491, 529, 528, 529, 510, 510, 512, 512, 515, 547, 480, 506, 513, 533, 521, 527, 509, 550, 505, 547, 514, 551, 505, 516, 526, 521, 528, 545, 525, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[487, 499, 479, 524, 502, 558, 527, 463, 481, 521, 507, 512, 527, 517, 557, 556, 522, 521, 495, 587, 453, 494, 491, 487, 514, 490, 493, 532, 481, 499, 503, 518, 527, 518, 497, 460, 536, 476, 511, 519, 479, 526, 508, 545, 502, 527, 524, 505, 529, 541, 489, 516, 482, 519, 505, 537, 553, 488, 568, 524, 512, 562, 506, 548, 499, 473, 493, 538, 539, 564, 502, 532, 497, 554, 527, 495, 480, 508, 480, 479, 491, 538, 524, 514, 532, 519, 483, 504, 515, 455, 532, 506, 520, 532, 504, 456, 513, 510, 520, 540, 489, 523, 481, 532, 528, 511, 483, 542, 497, 527, 518, 515, 522, 497, 517, 520, 512, 537, 513, 540, 490, 473, 529, 518, 481, 528, 518, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[571, 501, 521, 495, 508, 531, 506, 500, 550, 487, 536, 502, 528, 508, 552, 510, 505, 538, 507, 511, 499, 484, 529, 502, 496, 520, 488, 515, 504, 558, 508, 520, 505, 504, 582, 497, 501, 521, 531, 497, 542, 529, 491, 508, 523, 512, 507, 510, 505, 529, 555, 516, 526, 483, 507, 503, 501, 530, 469, 495, 515, 522, 489, 489, 492, 516, 503, 491, 504, 520, 500, 561, 531, 497, 470, 539, 529, 513, 517, 500, 519, 538, 496, 551, 470, 570, 497, 503, 508, 528, 525, 498, 472, 517, 511, 519, 495, 466, 499, 478, 536, 548, 508, 512, 502, 508, 483, 498, 496, 502, 510, 515, 521, 505, 471, 520, 533, 525, 487, 487, 537, 526, 489, 531, 509, 517, 526, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[549, 542, 512, 495, 519, 530, 494, 534, 547, 581, 542, 510, 492, 526, 538, 525, 470, 483, 461, 499, 535, 528, 508, 493, 499, 523, 539, 509, 488, 513, 515, 486, 516, 500, 497, 514, 500, 509, 506, 530, 489, 526, 540, 490, 493, 506, 486, 551, 550, 488, 486, 517, 488, 534, 500, 517, 529, 474, 517, 508, 458, 516, 531, 489, 512, 524, 494, 506, 528, 528, 495, 508, 519, 485, 488, 508, 496, 517, 536, 491, 490, 509, 508, 537, 501, 541, 538, 498, 527, 533, 537, 529, 529, 549, 516, 524, 492, 517, 472, 468, 515, 504, 516, 506, 506, 515, 509, 478, 493, 521, 557, 487, 523, 513, 525, 554, 506, 517, 516, 540, 490, 516, 540, 474, 532, 501, 494, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[500, 558, 534, 528, 544, 504, 546, 500, 506, 556, 511, 517, 525, 519, 485, 565, 513, 486, 530, 521, 549, 503, 484, 488, 525, 537, 524, 478, 490, 494, 504, 524, 540, 477, 514, 478, 482, 526, 498, 502, 536, 523, 509, 500, 505, 492, 476, 510, 520, 488, 546, 474, 483, 519, 488, 534, 540, 529, 532, 508, 551, 488, 511, 523, 490, 497, 536, 523, 501, 492, 506, 479, 480, 534, 518, 474, 499, 553, 532, 478, 518, 518, 526, 502, 550, 502, 508, 514, 518, 553, 544, 511, 498, 546, 500, 531, 528, 510, 517, 526, 525, 519, 537, 497, 491, 505, 487, 490, 505, 516, 490, 501, 493, 534, 507, 520, 510, 533, 487, 509, 496, 495, 514, 497, 549, 484, 507, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[498, 537, 491, 501, 532, 499, 526, 533, 513, 512, 494, 525, 515, 477, 529, 510, 529, 530, 498, 470, 528, 542, 468, 535, 491, 526, 478, 543, 493, 475, 497, 532, 491, 498, 522, 530, 486, 522, 539, 557, 489, 540, 495, 498, 532, 503, 503, 482, 503, 510, 535, 541, 515, 497, 485, 535, 519, 527, 500, 504, 499, 486, 491, 524, 520, 546, 541, 508, 492, 528, 510, 518, 520, 553, 517, 533, 510, 498, 520, 527, 485, 554, 488, 514, 492, 547, 504, 508, 530, 501, 511, 489, 510, 499, 556, 512, 518, 514, 491, 520, 506, 502, 493, 515, 498, 506, 492, 503, 503, 503, 511, 496, 506, 542, 486, 530, 511, 479, 507, 552, 500, 521, 502, 529, 547, 518, 537, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[491, 496, 493, 501, 550, 483, 511, 532, 472, 520, 515, 525, 484, 544, 510, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 498, 475, 505, 510, 486, 520, 499, 515, 497, 521, 519, 529, 506, 495, 522, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 520, 489, 495, 511, 523, 496, 559, 512, 513, 491, 498, 537, 498, 505, 490, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 521, 525, 502, 499, 548, 534, 543, 486, 495, 487, 532, 500, 507, 521, 509, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 502, 524, 542, 540, 495, 502, 494, 499, 550, 509, 488, 488, 511, 497, 487, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 510, 526, 508, 524, 468, 508, 532, 471, 554, 520, 509, 517, 522, 536, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 485, 532, 519, 522, 512, 517, 499, 551, 551, 481, 540, 499, 510, 516, 495, 28, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 491, 490, 501, 489, 504, 511, 499, 495, 522, 525, 510, 504, 492, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8176, 8182, 8215, 8159, 8200, 8326, 8109] avg=8192.0000 max=8326 max_rank=6 min=8109 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[474, 512, 489, 514, 500, 444, 477, 493, 518, 554, 495, 522, 507, 512, 486, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 544, 532, 519, 498, 487, 527, 504, 512, 545, 488, 511, 510, 565, 524, 510, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 524, 525, 547, 548, 515, 543, 485, 495, 479, 505, 502, 496, 523, 503, 496, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 510, 522, 476, 495, 491, 480, 454, 522, 561, 497, 533, 504, 487, 536, 552, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 544, 470, 538, 539, 547, 511, 493, 499, 481, 509, 498, 507, 509, 498, 489, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 542, 502, 503, 466, 482, 530, 530, 456, 509, 535, 511, 503, 534, 508, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 489, 491, 529, 528, 529, 510, 510, 512, 512, 515, 547, 480, 506, 513, 533, 28, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 509, 550, 505, 547, 514, 522, 505, 516, 526, 521, 528, 545, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7989, 8304, 8222, 8127, 8162, 8140, 8206, 8386] avg=8192.0000 max=8386 max_rank=7 min=7989 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[500, 558, 534, 528, 544, 435, 546, 500, 506, 556, 511, 517, 525, 519, 485, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 486, 530, 521, 549, 503, 484, 488, 525, 537, 524, 478, 490, 494, 504, 524, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 477, 514, 478, 482, 526, 498, 502, 536, 523, 509, 500, 505, 492, 476, 510, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 488, 546, 474, 483, 519, 488, 534, 540, 529, 532, 508, 551, 488, 511, 523, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 497, 536, 523, 501, 492, 506, 479, 480, 534, 518, 474, 499, 553, 532, 478, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 526, 502, 550, 460, 508, 514, 518, 553, 544, 511, 498, 546, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 510, 517, 526, 525, 519, 537, 497, 491, 505, 487, 490, 505, 516, 490, 501, 27, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 534, 507, 520, 510, 533, 487, 484, 496, 495, 514, 497, 549, 484, 507, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8329, 8199, 8096, 8236, 8102, 8297, 8191, 8086] avg=8192.0000 max=8329 max_rank=0 min=8086 min_rank=7 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[505, 566, 542, 530, 533, 472, 512, 533, 497, 478, 527, 509, 480, 504, 497, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 493, 516, 510, 512, 530, 503, 480, 502, 470, 492, 544, 512, 532, 504, 533, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 512, 514, 501, 502, 494, 489, 484, 476, 486, 509, 536, 498, 522, 559, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 487, 504, 507, 520, 489, 550, 523, 513, 513, 494, 490, 500, 523, 536, 489, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 527, 507, 501, 527, 486, 501, 544, 497, 533, 532, 526, 510, 524, 516, 525, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 502, 520, 511, 543, 471, 541, 508, 480, 523, 500, 546, 528, 530, 490, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 488, 501, 484, 511, 492, 521, 561, 539, 495, 471, 536, 523, 520, 489, 495, 28, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 542, 512, 534, 500, 497, 475, 545, 503, 483, 559, 497, 517, 561, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8197, 8145, 8158, 8245, 8178, 8192, 8224] avg=8192.0000 max=8245 max_rank=4 min=8145 min_rank=2 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[571, 501, 521, 495, 508, 450, 506, 500, 550, 487, 536, 502, 528, 508, 552, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 538, 507, 511, 499, 484, 529, 502, 496, 520, 488, 515, 504, 558, 508, 520, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 582, 497, 501, 521, 531, 497, 542, 529, 491, 508, 523, 512, 507, 510, 22, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 529, 555, 516, 526, 483, 507, 503, 501, 530, 469, 495, 515, 522, 489, 489, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 503, 491, 504, 520, 500, 561, 531, 497, 470, 539, 529, 513, 517, 500, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 538, 496, 551, 470, 513, 497, 503, 508, 528, 525, 498, 472, 517, 511, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 466, 499, 478, 536, 548, 508, 512, 502, 508, 483, 498, 496, 502, 510, 515, 35, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 471, 520, 533, 525, 487, 448, 537, 526, 489, 531, 509, 517, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8240, 8300, 8139, 8199, 8165, 8116, 8152] avg=8192.0000 max=8300 max_rank=2 min=8116 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[487, 499, 479, 524, 502, 481, 527, 463, 481, 521, 507, 512, 527, 517, 557, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 521, 495, 587, 453, 494, 491, 487, 514, 490, 493, 532, 481, 499, 503, 518, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 518, 497, 460, 536, 476, 511, 519, 479, 526, 508, 545, 502, 527, 524, 505, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 541, 489, 516, 482, 519, 505, 537, 553, 488, 568, 524, 512, 562, 506, 548, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 473, 493, 538, 539, 564, 502, 532, 497, 554, 527, 495, 480, 508, 480, 479, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 538, 524, 514, 532, 475, 483, 504, 515, 455, 532, 506, 520, 532, 504, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 520, 540, 489, 523, 481, 532, 528, 511, 483, 542, 497, 527, 518, 515, 28, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 497, 517, 520, 512, 537, 513, 512, 490, 473, 529, 518, 481, 528, 518, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8135, 8190, 8382, 8171, 8081, 8279, 8158] avg=8192.0000 max=8382 max_rank=3 min=8081 min_rank=5 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[498, 537, 491, 501, 532, 431, 526, 533, 513, 512, 494, 525, 515, 477, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 530, 498, 470, 528, 542, 468, 535, 491, 526, 478, 543, 493, 475, 497, 532, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 522, 530, 486, 522, 539, 557, 489, 540, 495, 498, 532, 503, 503, 482, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 535, 541, 515, 497, 485, 535, 519, 527, 500, 504, 499, 486, 491, 524, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 546, 541, 508, 492, 528, 510, 518, 520, 553, 517, 533, 510, 498, 520, 527, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 554, 488, 514, 492, 501, 504, 508, 530, 501, 511, 489, 510, 499, 556, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 514, 491, 520, 506, 502, 493, 515, 498, 506, 492, 503, 503, 503, 511, 496, 30, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 542, 486, 530, 511, 479, 507, 523, 500, 521, 502, 529, 547, 518, 537, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8184, 8218, 8174, 8352, 8154, 8120, 8210] avg=8192.0000 max=8352 max_rank=4 min=8120 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[549, 542, 512, 495, 519, 457, 494, 534, 547, 581, 542, 510, 492, 526, 538, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 483, 461, 499, 535, 528, 508, 493, 499, 523, 539, 509, 488, 513, 515, 486, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 500, 497, 514, 500, 509, 506, 530, 489, 526, 540, 490, 493, 506, 486, 551, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 488, 486, 517, 488, 534, 500, 517, 529, 474, 517, 508, 458, 516, 531, 489, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 494, 506, 528, 528, 495, 508, 519, 485, 488, 508, 496, 517, 536, 491, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 509, 508, 537, 501, 496, 538, 498, 527, 533, 537, 529, 529, 549, 516, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 517, 472, 468, 515, 504, 516, 506, 506, 515, 509, 478, 493, 521, 557, 487, 29, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 525, 554, 506, 517, 516, 512, 490, 516, 540, 474, 532, 501, 494, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8363, 8101, 8183, 8105, 8146, 8321, 8106, 8211] avg=8192.0000 max=8363 max_rank=0 min=8101 min_rank=1 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 3712, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4352, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[549, 560, 479, 482, 522, 496, 535, 537, 518, 548, 517, 541, 527, 494, 506, 511, 490, 518, 528, 509, 492, 515, 481, 511, 501, 486, 518, 530, 526, 512, 474, 497, 508, 533, 517, 530, 497, 478, 508, 492, 505, 499, 530, 541, 517, 466, 536, 532, 514, 528, 515, 520, 526, 479, 535, 501, 540, 515, 541, 527, 523, 485, 490, 505, 484, 548, 499, 530, 469, 538, 540, 516, 511, 489, 504, 481, 492, 508, 509, 495, 512, 536, 503, 530, 516, 505, 488, 471, 515, 524, 546, 528, 514, 479, 473, 541, 503, 560, 514, 485, 465, 531, 513, 532, 528, 553, 521, 509, 534, 527, 496, 503, 483, 518, 502, 488, 517, 473, 515, 478, 504, 555, 520, 474, 532, 555, 517, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[502, 523, 492, 516, 516, 502, 497, 530, 536, 482, 485, 513, 491, 543, 483, 517, 517, 535, 531, 473, 494, 557, 537, 544, 486, 535, 473, 496, 524, 490, 535, 565, 492, 516, 491, 546, 503, 535, 514, 557, 515, 511, 523, 526, 507, 492, 503, 495, 513, 508, 517, 525, 515, 535, 494, 509, 521, 516, 503, 568, 538, 488, 521, 532, 494, 506, 503, 507, 508, 526, 495, 496, 517, 504, 523, 551, 518, 470, 500, 534, 509, 479, 509, 526, 492, 490, 491, 522, 520, 530, 455, 542, 542, 500, 515, 493, 509, 502, 511, 480, 546, 479, 506, 532, 534, 532, 483, 488, 477, 558, 496, 501, 555, 483, 504, 511, 505, 513, 534, 520, 510, 517, 522, 505, 475, 483, 524, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[536, 513, 535, 487, 490, 496, 482, 548, 523, 535, 522, 501, 527, 475, 518, 563, 519, 551, 517, 514, 520, 488, 527, 511, 506, 559, 517, 516, 520, 505, 503, 548, 519, 463, 504, 510, 475, 506, 482, 513, 510, 536, 531, 497, 521, 510, 504, 520, 515, 530, 507, 498, 483, 531, 488, 535, 519, 538, 520, 499, 478, 529, 506, 531, 506, 462, 490, 525, 492, 539, 499, 516, 488, 503, 538, 516, 497, 497, 506, 474, 527, 530, 496, 493, 492, 513, 507, 519, 471, 528, 533, 563, 465, 496, 535, 543, 503, 524, 514, 513, 494, 498, 526, 498, 512, 506, 487, 572, 502, 492, 528, 537, 499, 497, 492, 519, 540, 516, 558, 491, 515, 509, 501, 521, 498, 553, 512, 470] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[529, 524, 558, 508, 548, 521, 501, 505, 509, 544, 482, 501, 519, 538, 517, 511, 497, 473, 513, 508, 543, 505, 510, 483, 490, 553, 480, 543, 513, 521, 509, 518, 518, 501, 529, 533, 488, 495, 510, 527, 544, 514, 557, 536, 507, 544, 558, 527, 527, 524, 544, 532, 492, 512, 521, 504, 532, 532, 502, 523, 497, 540, 531, 493, 516, 519, 500, 475, 497, 493, 532, 476, 473, 499, 504, 513, 476, 529, 471, 520, 509, 494, 494, 516, 517, 510, 498, 510, 459, 521, 497, 521, 498, 508, 521, 530, 491, 454, 515, 507, 490, 524, 479, 525, 515, 532, 538, 524, 537, 523, 507, 491, 523, 527, 518, 477, 497, 527, 502, 499, 524, 466, 469, 530, 486, 527, 526, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[525, 555, 511, 485, 486, 485, 485, 501, 549, 490, 519, 497, 554, 505, 497, 510, 491, 480, 521, 540, 509, 517, 509, 527, 535, 523, 485, 531, 545, 513, 538, 490, 527, 437, 506, 498, 502, 504, 526, 491, 478, 539, 491, 487, 517, 528, 492, 490, 496, 502, 459, 538, 512, 500, 540, 503, 485, 511, 532, 538, 536, 519, 514, 503, 523, 528, 523, 504, 529, 497, 558, 507, 552, 520, 527, 513, 501, 549, 526, 507, 535, 483, 521, 513, 536, 555, 518, 534, 453, 506, 512, 524, 466, 528, 498, 517, 503, 562, 514, 518, 537, 513, 513, 519, 503, 469, 519, 495, 476, 519, 492, 518, 484, 495, 550, 498, 523, 505, 506, 450, 574, 513, 526, 525, 530, 517, 497, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[510, 536, 515, 528, 545, 554, 517, 501, 492, 500, 526, 515, 530, 532, 485, 496, 489, 504, 503, 506, 536, 517, 526, 471, 509, 504, 506, 482, 536, 496, 496, 507, 512, 518, 516, 503, 525, 505, 502, 563, 459, 533, 504, 497, 520, 542, 519, 505, 524, 485, 491, 502, 505, 522, 523, 564, 548, 494, 487, 496, 539, 480, 530, 543, 538, 505, 534, 533, 536, 489, 523, 487, 511, 544, 539, 486, 510, 531, 527, 528, 501, 465, 539, 502, 496, 487, 490, 503, 524, 507, 498, 476, 509, 487, 482, 524, 539, 505, 550, 510, 490, 515, 532, 502, 497, 497, 498, 536, 533, 547, 550, 464, 520, 525, 511, 558, 525, 499, 469, 509, 493, 506, 483, 554, 519, 509, 485, 469] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[537, 505, 467, 478, 500, 525, 535, 471, 474, 540, 479, 484, 531, 511, 509, 501, 548, 520, 510, 488, 509, 520, 527, 509, 486, 525, 459, 495, 535, 546, 525, 556, 503, 503, 550, 480, 540, 496, 504, 502, 534, 511, 509, 532, 528, 491, 522, 522, 526, 498, 483, 511, 502, 497, 536, 501, 522, 486, 518, 476, 481, 449, 486, 525, 520, 531, 482, 505, 504, 541, 528, 504, 494, 534, 532, 523, 535, 504, 486, 500, 490, 492, 518, 568, 512, 527, 493, 481, 484, 514, 506, 507, 547, 536, 507, 502, 517, 505, 536, 515, 500, 499, 496, 499, 501, 507, 517, 479, 508, 502, 491, 513, 549, 575, 539, 521, 519, 524, 527, 511, 535, 520, 533, 499, 541, 537, 569, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[472, 534, 519, 546, 555, 501, 512, 538, 499, 528, 516, 523, 515, 506, 464, 549, 527, 517, 463, 534, 531, 522, 513, 528, 499, 511, 537, 513, 496, 547, 504, 494, 485, 523, 537, 511, 495, 510, 492, 547, 500, 523, 530, 548, 523, 522, 517, 526, 510, 499, 513, 510, 539, 504, 493, 502, 502, 508, 540, 522, 506, 491, 506, 461, 510, 457, 520, 494, 506, 492, 565, 524, 500, 500, 515, 469, 519, 511, 509, 527, 489, 499, 521, 535, 469, 514, 469, 509, 496, 505, 504, 518, 550, 523, 551, 504, 484, 527, 482, 497, 529, 505, 533, 486, 489, 513, 517, 525, 494, 513, 524, 480, 521, 531, 531, 501, 509, 525, 513, 538, 508, 484, 489, 511, 514, 495, 544, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[529, 495, 558, 508, 548, 521, 501, 505, 509, 544, 482, 501, 519, 538, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 473, 513, 508, 543, 505, 510, 483, 490, 532, 480, 543, 513, 521, 509, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 529, 533, 488, 495, 510, 512, 544, 514, 557, 536, 507, 544, 558, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 524, 544, 532, 492, 512, 521, 504, 526, 532, 502, 523, 497, 540, 531, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 500, 475, 497, 493, 532, 476, 473, 499, 504, 513, 476, 529, 471, 520, 21, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 494, 516, 517, 510, 498, 510, 459, 521, 497, 521, 498, 508, 521, 530, 29, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 454, 515, 507, 490, 524, 479, 525, 515, 532, 538, 524, 537, 523, 507, 491, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 527, 518, 477, 497, 527, 502, 499, 524, 466, 469, 530, 486, 505, 526, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8138, 8373, 8300, 8019, 8151, 8171, 8098] avg=8192.0000 max=8373 max_rank=2 min=8019 min_rank=4 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[502, 494, 492, 516, 516, 502, 497, 530, 536, 482, 485, 513, 491, 543, 483, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 535, 531, 473, 494, 557, 537, 544, 486, 515, 473, 496, 524, 490, 535, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 491, 546, 503, 535, 514, 542, 515, 511, 523, 526, 507, 492, 503, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 508, 517, 525, 515, 535, 494, 509, 516, 516, 503, 568, 538, 488, 521, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 506, 503, 507, 508, 526, 495, 496, 517, 504, 523, 551, 518, 470, 500, 534, 20, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 479, 509, 526, 492, 490, 491, 522, 520, 530, 455, 542, 542, 500, 515, 493, 29, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 502, 511, 480, 546, 479, 506, 532, 534, 532, 483, 488, 477, 558, 496, 501, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 483, 504, 511, 505, 513, 534, 520, 510, 517, 522, 505, 475, 462, 524, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8272, 8211, 8298, 8177, 8162, 8152, 8165] avg=8192.0000 max=8298 max_rank=3 min=8099 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[472, 504, 519, 546, 555, 501, 512, 538, 499, 528, 516, 523, 515, 506, 464, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 517, 463, 534, 531, 522, 513, 528, 499, 492, 537, 513, 496, 547, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 523, 537, 511, 495, 510, 492, 532, 500, 523, 530, 548, 523, 522, 517, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 499, 513, 510, 539, 504, 493, 502, 497, 508, 540, 522, 506, 491, 506, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 457, 520, 494, 506, 492, 565, 524, 500, 500, 515, 469, 519, 511, 509, 527, 19, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 499, 521, 535, 469, 514, 469, 509, 496, 505, 504, 518, 550, 523, 551, 504, 30, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 482, 497, 529, 505, 533, 486, 489, 513, 517, 525, 494, 513, 524, 480, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 531, 531, 501, 509, 525, 513, 538, 508, 484, 489, 511, 514, 474, 544, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8217, 8274, 8101, 8142, 8204, 8116, 8235] avg=8192.0000 max=8274 max_rank=2 min=8101 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[549, 529, 479, 482, 522, 496, 535, 537, 518, 548, 517, 541, 527, 494, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 518, 528, 509, 492, 515, 481, 511, 501, 468, 518, 530, 526, 512, 474, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 533, 517, 530, 497, 478, 508, 479, 505, 499, 530, 541, 517, 466, 536, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 528, 515, 520, 526, 479, 535, 501, 534, 515, 541, 527, 523, 485, 490, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 548, 499, 530, 469, 538, 540, 516, 511, 489, 504, 481, 492, 508, 509, 495, 18, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 536, 503, 530, 516, 505, 488, 471, 515, 524, 546, 528, 514, 479, 473, 541, 31, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 560, 514, 485, 465, 531, 513, 532, 528, 553, 521, 509, 534, 527, 496, 503, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 502, 488, 517, 473, 515, 478, 504, 555, 520, 474, 532, 532, 517, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8070, 8176, 8238, 8135, 8232, 8292, 8102] avg=8192.0000 max=8292 max_rank=6 min=8070 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[536, 485, 535, 487, 490, 496, 482, 548, 523, 535, 522, 501, 527, 475, 518, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 551, 517, 514, 520, 488, 527, 511, 506, 538, 517, 516, 520, 505, 503, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 463, 504, 510, 475, 506, 482, 500, 510, 536, 531, 497, 521, 510, 504, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 530, 507, 498, 483, 531, 488, 535, 514, 538, 520, 499, 478, 529, 506, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 462, 490, 525, 492, 539, 499, 516, 488, 503, 538, 516, 497, 497, 506, 474, 21, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 530, 496, 493, 492, 513, 507, 519, 471, 528, 533, 563, 465, 496, 535, 543, 28, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 514, 513, 494, 498, 526, 498, 512, 506, 487, 572, 502, 492, 528, 537, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 497, 492, 519, 540, 516, 558, 491, 515, 509, 501, 521, 498, 530, 512, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8300, 8088, 8202, 8073, 8259, 8223, 8168] avg=8192.0000 max=8300 max_rank=1 min=8073 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[537, 477, 467, 478, 500, 525, 535, 471, 474, 540, 479, 484, 531, 511, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 520, 510, 488, 509, 520, 527, 509, 486, 505, 459, 495, 535, 546, 525, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 550, 480, 540, 496, 504, 489, 534, 511, 509, 532, 528, 491, 522, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 498, 483, 511, 502, 497, 536, 501, 516, 486, 518, 476, 481, 449, 486, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 531, 482, 505, 504, 541, 528, 504, 494, 534, 532, 523, 535, 504, 486, 500, 20, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 492, 518, 568, 512, 527, 493, 481, 484, 514, 506, 507, 547, 536, 507, 502, 28, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 536, 515, 500, 499, 496, 499, 501, 507, 517, 479, 508, 502, 491, 513, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 575, 539, 521, 519, 524, 527, 511, 535, 520, 533, 499, 541, 514, 569, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 8238, 8214, 7991, 8247, 8232, 8103, 8492] avg=8192.0000 max=8492 max_rank=7 min=7991 min_rank=3 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=33074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[525, 520, 511, 485, 486, 485, 485, 501, 549, 490, 519, 497, 554, 505, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 480, 521, 540, 509, 517, 509, 527, 535, 501, 485, 531, 545, 513, 538, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 437, 506, 498, 502, 504, 526, 470, 478, 539, 491, 487, 517, 528, 492, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 502, 459, 538, 512, 500, 540, 503, 476, 511, 532, 538, 536, 519, 514, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 528, 523, 504, 529, 497, 558, 507, 552, 520, 527, 513, 501, 549, 526, 507, 22, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 483, 521, 513, 536, 555, 518, 534, 453, 506, 512, 524, 466, 528, 498, 517, 35, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 562, 514, 518, 537, 513, 513, 519, 503, 469, 519, 495, 476, 519, 492, 518, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 495, 550, 498, 523, 505, 506, 450, 574, 513, 526, 525, 530, 487, 497, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8232, 7992, 8179, 8398, 8257, 8195, 8164] avg=8192.0000 max=8398 max_rank=4 min=7992 min_rank=2 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=33014 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[510, 506, 515, 528, 545, 554, 517, 501, 492, 500, 526, 515, 530, 532, 485, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 504, 503, 506, 536, 517, 526, 471, 509, 485, 506, 482, 536, 496, 496, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 516, 503, 525, 505, 502, 548, 459, 533, 504, 497, 520, 542, 519, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 485, 491, 502, 505, 522, 523, 564, 542, 494, 487, 496, 539, 480, 530, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 505, 534, 533, 536, 489, 523, 487, 511, 544, 539, 486, 510, 531, 527, 528, 19, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 465, 539, 502, 496, 487, 490, 503, 524, 507, 498, 476, 509, 487, 482, 524, 30, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 505, 550, 510, 490, 515, 532, 502, 497, 497, 498, 536, 533, 547, 550, 464, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 525, 511, 558, 525, 499, 469, 509, 493, 506, 483, 554, 519, 487, 485, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8069, 8208, 8227, 8345, 8039, 8284, 8112] avg=8192.0000 max=8345 max_rank=4 min=8039 min_rank=5 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4352, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 16:29:44] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 37874.7 | throughput per GPU (TFLOP/s/GPU): 4.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115716E+01 | loss scale: 1.0 | grad norm: 44.805 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 81277.48046875 | max allocated: 88645.78466796875 | reserved: 98244.0 | max reserved: 98244.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[483, 490, 480, 478, 540, 483, 490, 516, 526, 529, 510, 514, 500, 560, 466, 541, 517, 486, 517, 537, 500, 524, 548, 506, 529, 506, 516, 534, 492, 467, 541, 521, 517, 480, 532, 541, 501, 480, 467, 542, 515, 476, 487, 504, 514, 500, 511, 506, 517, 484, 527, 503, 543, 520, 497, 488, 523, 519, 509, 497, 550, 489, 514, 560, 509, 530, 532, 502, 500, 507, 511, 497, 533, 510, 537, 579, 528, 531, 530, 526, 540, 477, 515, 551, 525, 516, 526, 519, 513, 519, 498, 532, 529, 494, 522, 494, 538, 500, 499, 514, 504, 506, 536, 526, 496, 537, 504, 486, 497, 520, 499, 510, 491, 482, 498, 507, 501, 509, 475, 492, 527, 500, 530, 519, 506, 532, 542, 461] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[530, 505, 525, 523, 538, 470, 511, 505, 513, 536, 573, 502, 480, 530, 499, 538, 534, 499, 508, 515, 512, 499, 512, 492, 532, 524, 537, 518, 527, 542, 558, 512, 487, 509, 524, 491, 465, 513, 456, 518, 497, 495, 482, 491, 508, 513, 512, 515, 480, 521, 470, 526, 542, 526, 487, 506, 528, 559, 531, 523, 523, 477, 564, 497, 514, 510, 485, 471, 467, 539, 545, 506, 517, 502, 519, 545, 504, 520, 477, 515, 526, 468, 500, 508, 485, 511, 503, 475, 482, 474, 548, 497, 493, 539, 500, 465, 497, 525, 530, 496, 554, 524, 528, 572, 514, 500, 492, 578, 502, 522, 519, 477, 518, 505, 475, 509, 524, 530, 518, 498, 497, 513, 501, 543, 511, 539, 528, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[487, 521, 510, 481, 488, 496, 506, 536, 536, 508, 516, 521, 530, 507, 512, 527, 478, 517, 537, 522, 534, 502, 523, 509, 510, 499, 504, 490, 525, 525, 519, 511, 513, 532, 511, 476, 517, 509, 532, 491, 525, 518, 523, 527, 501, 456, 500, 508, 527, 516, 523, 494, 533, 485, 507, 484, 517, 509, 520, 533, 530, 480, 514, 471, 497, 494, 526, 516, 534, 527, 511, 554, 503, 516, 534, 540, 479, 512, 519, 496, 482, 449, 509, 545, 532, 527, 496, 555, 480, 528, 484, 514, 477, 522, 513, 534, 496, 507, 481, 474, 538, 530, 506, 514, 471, 497, 563, 482, 516, 554, 503, 535, 522, 513, 533, 491, 535, 485, 514, 518, 530, 526, 505, 505, 512, 519, 545, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[543, 533, 488, 503, 524, 526, 513, 511, 497, 492, 516, 539, 490, 489, 528, 518, 529, 524, 529, 488, 534, 519, 499, 518, 470, 532, 505, 512, 527, 517, 498, 504, 504, 503, 465, 534, 531, 549, 532, 495, 526, 488, 506, 558, 533, 478, 538, 506, 532, 497, 537, 526, 506, 501, 512, 521, 446, 506, 520, 506, 498, 504, 465, 493, 519, 494, 490, 504, 519, 490, 551, 515, 477, 533, 537, 539, 512, 504, 530, 483, 508, 488, 460, 517, 544, 533, 513, 525, 522, 485, 511, 490, 543, 562, 531, 520, 502, 550, 527, 483, 539, 494, 505, 499, 501, 482, 553, 454, 566, 511, 484, 550, 459, 501, 488, 486, 545, 506, 557, 508, 522, 514, 518, 496, 512, 512, 535, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[488, 501, 516, 473, 525, 511, 492, 523, 504, 512, 509, 467, 549, 527, 494, 500, 530, 528, 514, 502, 454, 516, 508, 517, 535, 505, 526, 518, 496, 490, 491, 543, 462, 546, 504, 524, 509, 498, 532, 495, 550, 528, 530, 502, 577, 530, 491, 520, 525, 504, 487, 513, 512, 523, 537, 475, 520, 537, 523, 529, 550, 544, 523, 502, 496, 496, 518, 491, 509, 479, 539, 527, 473, 553, 509, 522, 530, 515, 516, 499, 498, 488, 511, 518, 553, 521, 480, 547, 522, 508, 480, 474, 510, 519, 527, 484, 501, 474, 481, 490, 521, 520, 527, 494, 509, 520, 510, 521, 500, 484, 515, 520, 511, 510, 528, 546, 511, 501, 480, 473, 529, 522, 545, 527, 533, 505, 525, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[518, 546, 500, 531, 510, 480, 528, 526, 486, 493, 500, 498, 483, 537, 491, 483, 512, 496, 549, 529, 543, 514, 545, 541, 502, 491, 488, 509, 516, 534, 496, 465, 488, 509, 478, 486, 540, 496, 522, 521, 508, 520, 503, 503, 546, 536, 513, 520, 499, 521, 535, 512, 514, 528, 522, 541, 523, 493, 461, 524, 510, 491, 508, 500, 493, 534, 514, 501, 488, 442, 489, 476, 523, 498, 541, 515, 497, 515, 538, 491, 543, 498, 510, 528, 501, 518, 525, 524, 565, 505, 477, 539, 556, 515, 492, 540, 511, 503, 535, 482, 536, 543, 515, 511, 503, 514, 547, 551, 524, 480, 531, 503, 511, 507, 508, 503, 511, 510, 471, 520, 500, 511, 521, 506, 572, 510, 489, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[491, 468, 507, 527, 515, 504, 523, 528, 519, 528, 490, 510, 514, 551, 490, 492, 506, 515, 550, 548, 539, 526, 521, 545, 525, 518, 525, 495, 523, 497, 488, 534, 550, 458, 543, 550, 524, 539, 485, 482, 517, 515, 513, 486, 517, 504, 522, 452, 521, 524, 515, 520, 507, 478, 497, 532, 478, 530, 537, 470, 512, 518, 556, 499, 494, 463, 525, 508, 527, 537, 524, 499, 545, 489, 514, 522, 533, 536, 455, 515, 502, 501, 514, 513, 498, 505, 560, 515, 476, 499, 496, 501, 516, 506, 510, 507, 552, 481, 513, 543, 492, 507, 485, 470, 500, 507, 509, 543, 498, 530, 537, 498, 532, 485, 494, 546, 512, 495, 526, 501, 532, 534, 519, 504, 513, 497, 517, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[549, 470, 557, 513, 481, 507, 532, 499, 508, 516, 491, 565, 505, 470, 502, 554, 510, 476, 532, 531, 521, 496, 533, 510, 494, 498, 524, 545, 488, 519, 514, 525, 486, 531, 494, 558, 490, 540, 489, 505, 532, 533, 547, 497, 496, 523, 492, 526, 486, 518, 516, 527, 524, 524, 492, 522, 488, 507, 536, 524, 496, 517, 476, 498, 508, 527, 494, 516, 522, 501, 525, 499, 488, 550, 514, 530, 518, 535, 519, 503, 510, 484, 485, 509, 482, 550, 533, 486, 519, 492, 532, 527, 479, 481, 505, 496, 487, 517, 522, 522, 517, 527, 553, 478, 483, 525, 530, 511, 467, 515, 525, 539, 490, 520, 536, 488, 481, 525, 481, 496, 531, 501, 536, 508, 503, 537, 519, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[483, 490, 480, 478, 540, 483, 490, 516, 526, 529, 510, 514, 500, 560, 466, 541, 10, 6, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 486, 468, 537, 500, 524, 548, 506, 529, 506, 516, 534, 492, 467, 541, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 480, 532, 541, 501, 480, 467, 542, 515, 476, 487, 504, 514, 500, 511, 506, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 484, 527, 503, 542, 520, 497, 488, 523, 519, 509, 497, 550, 489, 514, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 530, 532, 502, 500, 507, 511, 497, 533, 510, 537, 569, 528, 531, 530, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 477, 515, 551, 525, 516, 526, 519, 513, 519, 498, 532, 529, 494, 522, 494, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 500, 499, 514, 504, 506, 536, 526, 496, 537, 494, 486, 497, 520, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 482, 498, 507, 501, 509, 475, 492, 527, 500, 530, 519, 506, 532, 537, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8192, 8099, 8239, 8352, 8297, 8162, 8067] avg=8192.0000 max=8352 max_rank=4 min=8067 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[530, 505, 525, 523, 538, 470, 511, 505, 513, 536, 573, 502, 480, 530, 499, 538, 15, 10, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 499, 451, 515, 512, 499, 512, 492, 532, 524, 537, 518, 527, 542, 558, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 509, 524, 491, 465, 513, 456, 518, 497, 495, 482, 491, 508, 513, 512, 515, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 521, 470, 526, 537, 526, 487, 506, 528, 559, 531, 523, 523, 477, 564, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 485, 471, 467, 539, 545, 506, 517, 502, 519, 530, 504, 520, 477, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 468, 500, 508, 485, 511, 503, 475, 482, 474, 548, 497, 493, 539, 500, 465, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 525, 530, 496, 554, 524, 528, 572, 514, 500, 473, 578, 502, 522, 519, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 475, 509, 524, 530, 518, 498, 497, 513, 501, 543, 511, 539, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8264, 8010, 8255, 8121, 8006, 8311, 8254] avg=8192.0000 max=8315 max_rank=0 min=8006 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[488, 501, 516, 473, 525, 511, 492, 523, 504, 512, 509, 467, 549, 527, 494, 500, 9, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 465, 502, 454, 516, 508, 517, 535, 505, 526, 518, 496, 490, 491, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 546, 504, 524, 509, 498, 532, 495, 550, 528, 530, 502, 577, 530, 491, 520, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 504, 487, 513, 512, 523, 537, 475, 520, 537, 523, 529, 550, 544, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 496, 518, 491, 509, 479, 539, 527, 473, 553, 509, 513, 530, 515, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 488, 511, 518, 553, 521, 480, 547, 522, 508, 480, 474, 510, 519, 527, 484, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 474, 481, 490, 521, 520, 527, 494, 509, 520, 500, 521, 500, 484, 515, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 528, 546, 511, 501, 480, 473, 529, 522, 545, 527, 533, 505, 520, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8124, 8324, 8304, 8163, 8167, 8077, 8266] avg=8192.0000 max=8324 max_rank=2 min=8077 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[487, 521, 510, 481, 488, 496, 506, 536, 536, 508, 516, 521, 530, 507, 512, 527, 9, 7, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 517, 486, 522, 534, 502, 523, 509, 510, 499, 504, 490, 525, 525, 519, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 532, 511, 476, 517, 509, 532, 491, 525, 518, 523, 527, 501, 456, 500, 508, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 516, 523, 494, 532, 485, 507, 484, 517, 509, 520, 533, 530, 480, 514, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 526, 516, 534, 527, 511, 554, 503, 516, 534, 531, 479, 512, 519, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 449, 509, 545, 532, 527, 496, 555, 480, 528, 484, 514, 477, 522, 513, 534, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 507, 481, 474, 538, 530, 506, 514, 471, 497, 552, 482, 516, 554, 503, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 513, 533, 491, 535, 485, 514, 518, 530, 526, 505, 505, 512, 519, 540, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8154, 8166, 8142, 8249, 8175, 8156, 8290] avg=8192.0000 max=8290 max_rank=7 min=8142 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[543, 533, 488, 503, 524, 526, 513, 511, 497, 492, 516, 539, 490, 489, 528, 518, 9, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 524, 479, 488, 534, 519, 499, 518, 470, 532, 505, 512, 527, 517, 498, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 465, 534, 531, 549, 532, 495, 526, 488, 506, 558, 533, 478, 538, 506, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 537, 526, 506, 501, 512, 521, 446, 506, 520, 506, 498, 504, 465, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 494, 490, 504, 519, 490, 551, 515, 477, 533, 537, 530, 512, 504, 530, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 488, 460, 517, 544, 533, 513, 525, 522, 485, 511, 490, 543, 562, 531, 520, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 550, 527, 483, 539, 494, 505, 499, 501, 482, 542, 454, 566, 511, 484, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 501, 488, 486, 545, 506, 557, 508, 522, 514, 518, 496, 512, 512, 530, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8155, 8272, 8070, 8188, 8280, 8189, 8151] avg=8192.0000 max=8280 max_rank=5 min=8070 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 546, 500, 531, 510, 480, 528, 526, 486, 493, 500, 498, 483, 537, 491, 483, 9, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 496, 497, 529, 543, 514, 545, 541, 502, 491, 488, 509, 516, 534, 496, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 509, 478, 486, 540, 496, 522, 521, 508, 520, 503, 503, 546, 536, 513, 520, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 535, 512, 514, 528, 522, 541, 523, 493, 461, 524, 510, 491, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 534, 514, 501, 488, 442, 489, 476, 523, 498, 541, 506, 497, 515, 538, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 498, 510, 528, 501, 518, 525, 524, 565, 505, 477, 539, 556, 515, 492, 540, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 503, 535, 482, 536, 543, 515, 511, 503, 514, 536, 551, 524, 480, 531, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 507, 508, 503, 511, 510, 471, 520, 500, 511, 521, 506, 572, 510, 485, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8178, 8216, 8182, 8046, 8365, 8278, 8141] avg=8192.0000 max=8365 max_rank=5 min=8046 min_rank=4 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[491, 468, 507, 527, 515, 504, 523, 528, 519, 528, 490, 510, 514, 551, 490, 492, 9, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 498, 548, 539, 526, 521, 545, 525, 518, 525, 495, 523, 497, 488, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 458, 543, 550, 524, 539, 485, 482, 517, 515, 513, 486, 517, 504, 522, 452, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 524, 515, 520, 507, 478, 497, 532, 478, 530, 537, 470, 512, 518, 556, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 463, 525, 508, 527, 537, 524, 499, 545, 489, 514, 513, 533, 536, 455, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 514, 513, 498, 505, 560, 515, 476, 499, 496, 501, 516, 506, 510, 507, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 481, 513, 543, 492, 507, 485, 470, 500, 507, 499, 543, 498, 530, 537, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 485, 494, 546, 512, 495, 526, 501, 532, 534, 519, 504, 513, 497, 512, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8303, 8184, 8194, 8177, 8148, 8155, 8198] avg=8192.0000 max=8303 max_rank=1 min=8148 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[549, 470, 557, 513, 481, 507, 532, 499, 508, 516, 491, 565, 505, 470, 502, 554, 9, 6, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 476, 482, 531, 521, 496, 533, 510, 494, 498, 524, 545, 488, 519, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 531, 494, 558, 490, 540, 489, 505, 532, 533, 547, 497, 496, 523, 492, 526, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 518, 516, 527, 523, 524, 492, 522, 488, 507, 536, 524, 496, 517, 476, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 494, 516, 522, 501, 525, 499, 488, 550, 514, 521, 518, 535, 519, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 484, 485, 509, 482, 550, 533, 486, 519, 492, 532, 527, 479, 481, 505, 496, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 517, 522, 522, 517, 527, 553, 478, 483, 525, 520, 511, 467, 515, 525, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 520, 536, 488, 481, 525, 481, 496, 531, 501, 536, 508, 503, 537, 514, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8166, 8265, 8150, 8240, 8098, 8208, 8169] avg=8192.0000 max=8265 max_rank=2 min=8098 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 3840, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 3968, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[497, 558, 517, 486, 479, 498, 513, 522, 505, 500, 556, 479, 493, 514, 491, 466, 532, 500, 482, 505, 495, 480, 511, 524, 523, 497, 479, 510, 548, 502, 530, 512, 533, 518, 530, 509, 526, 517, 495, 536, 478, 539, 530, 481, 507, 494, 532, 524, 538, 524, 533, 522, 508, 532, 528, 496, 558, 516, 545, 527, 507, 472, 520, 508, 486, 532, 488, 472, 524, 484, 535, 535, 492, 546, 507, 527, 493, 546, 524, 507, 489, 482, 521, 500, 491, 474, 524, 525, 525, 520, 560, 529, 518, 547, 491, 538, 519, 506, 474, 511, 536, 513, 485, 509, 490, 555, 526, 530, 529, 539, 484, 561, 501, 516, 499, 499, 528, 527, 502, 491, 490, 516, 523, 468, 491, 537, 484, 478] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[502, 493, 528, 527, 476, 487, 528, 501, 490, 517, 517, 491, 531, 473, 539, 509, 551, 523, 509, 512, 495, 513, 537, 492, 499, 487, 493, 471, 544, 487, 497, 519, 528, 516, 511, 534, 509, 507, 507, 505, 530, 475, 530, 485, 512, 525, 530, 499, 521, 536, 564, 489, 508, 520, 518, 498, 525, 489, 489, 495, 543, 506, 497, 493, 493, 561, 521, 493, 525, 500, 535, 520, 531, 478, 502, 490, 511, 514, 539, 537, 533, 534, 499, 525, 508, 518, 534, 488, 500, 493, 535, 526, 489, 518, 481, 523, 536, 509, 540, 492, 518, 508, 508, 545, 517, 501, 528, 524, 501, 497, 538, 534, 493, 483, 489, 552, 517, 539, 490, 493, 507, 512, 535, 479, 520, 540, 507, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[547, 516, 506, 578, 525, 496, 516, 529, 483, 504, 507, 491, 470, 503, 482, 509, 548, 510, 511, 497, 504, 468, 508, 515, 488, 527, 527, 534, 528, 530, 532, 494, 543, 470, 520, 570, 542, 520, 549, 486, 502, 520, 521, 497, 519, 455, 523, 516, 516, 490, 493, 450, 518, 519, 497, 530, 459, 551, 531, 492, 473, 545, 508, 525, 509, 494, 516, 508, 492, 562, 538, 513, 512, 510, 531, 521, 537, 484, 510, 520, 492, 517, 497, 541, 473, 538, 489, 527, 516, 508, 486, 490, 485, 507, 516, 522, 514, 549, 517, 504, 549, 506, 502, 490, 525, 486, 495, 523, 510, 501, 532, 510, 508, 507, 530, 534, 499, 531, 504, 524, 511, 492, 500, 491, 532, 548, 494, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[496, 537, 520, 527, 515, 539, 498, 512, 475, 550, 550, 504, 516, 517, 529, 467, 548, 500, 522, 508, 521, 511, 502, 504, 532, 534, 502, 521, 526, 541, 499, 517, 493, 474, 517, 532, 528, 457, 515, 494, 535, 507, 495, 497, 553, 503, 553, 473, 544, 479, 496, 492, 501, 501, 530, 516, 540, 535, 502, 516, 505, 527, 513, 546, 520, 501, 513, 474, 490, 513, 539, 527, 486, 491, 554, 514, 530, 524, 476, 508, 504, 522, 562, 506, 501, 534, 522, 556, 523, 484, 519, 547, 470, 527, 518, 473, 529, 503, 511, 495, 520, 534, 525, 540, 476, 498, 550, 502, 509, 515, 513, 490, 506, 498, 468, 496, 492, 535, 477, 526, 504, 490, 495, 494, 515, 485, 493, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[575, 511, 528, 525, 498, 513, 486, 499, 496, 507, 499, 499, 489, 532, 495, 500, 574, 501, 545, 514, 496, 481, 512, 525, 539, 513, 531, 523, 509, 493, 517, 545, 508, 549, 498, 456, 529, 547, 507, 548, 535, 542, 525, 504, 493, 543, 509, 524, 490, 476, 496, 510, 518, 469, 500, 510, 495, 489, 493, 491, 517, 525, 519, 501, 506, 462, 560, 531, 515, 508, 533, 483, 467, 533, 507, 557, 521, 517, 511, 522, 525, 515, 521, 494, 527, 513, 506, 523, 533, 489, 491, 540, 552, 496, 478, 513, 542, 508, 499, 528, 524, 527, 533, 529, 526, 475, 496, 506, 492, 450, 483, 513, 542, 476, 512, 531, 520, 502, 508, 496, 491, 509, 481, 506, 533, 533, 544, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[526, 521, 519, 497, 533, 519, 503, 545, 549, 509, 456, 534, 495, 472, 545, 517, 529, 536, 530, 568, 484, 541, 522, 500, 515, 495, 522, 489, 524, 494, 483, 520, 524, 500, 493, 519, 498, 516, 539, 464, 497, 536, 490, 503, 531, 485, 501, 529, 521, 524, 550, 532, 511, 506, 527, 512, 517, 518, 549, 522, 502, 525, 541, 511, 530, 545, 524, 518, 509, 506, 514, 524, 518, 493, 469, 497, 491, 514, 551, 502, 463, 517, 480, 507, 483, 529, 494, 499, 510, 479, 526, 542, 529, 530, 518, 495, 528, 517, 484, 500, 488, 517, 505, 476, 521, 530, 524, 526, 510, 522, 497, 479, 502, 556, 512, 526, 509, 504, 476, 508, 447, 548, 508, 489, 515, 526, 488, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[493, 498, 522, 500, 515, 502, 526, 524, 543, 525, 515, 567, 549, 533, 516, 513, 512, 550, 521, 551, 568, 517, 506, 539, 509, 504, 465, 543, 515, 521, 453, 532, 507, 502, 474, 493, 500, 486, 487, 490, 463, 505, 506, 504, 530, 508, 524, 520, 539, 519, 507, 524, 477, 492, 511, 526, 521, 494, 510, 476, 508, 502, 500, 487, 508, 528, 509, 509, 522, 528, 503, 502, 519, 534, 492, 532, 538, 490, 540, 515, 547, 510, 509, 547, 538, 505, 506, 520, 506, 480, 498, 485, 501, 529, 470, 522, 515, 519, 470, 530, 477, 548, 500, 522, 518, 525, 485, 516, 510, 519, 492, 511, 505, 566, 510, 487, 490, 518, 456, 555, 504, 522, 507, 525, 497, 520, 494, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[523, 540, 516, 558, 550, 490, 500, 504, 493, 504, 524, 498, 519, 489, 538, 517, 501, 514, 523, 490, 477, 495, 500, 540, 512, 500, 482, 486, 483, 507, 529, 515, 504, 514, 491, 488, 542, 499, 516, 523, 515, 478, 515, 498, 509, 522, 512, 502, 543, 512, 499, 530, 566, 533, 516, 553, 537, 517, 476, 513, 498, 469, 501, 512, 557, 549, 468, 500, 524, 530, 484, 474, 524, 547, 508, 486, 525, 515, 491, 517, 488, 518, 516, 512, 521, 556, 524, 515, 526, 504, 527, 538, 494, 519, 471, 538, 495, 523, 490, 529, 516, 491, 523, 455, 505, 567, 537, 508, 501, 488, 505, 523, 514, 517, 523, 530, 468, 554, 472, 500, 536, 503, 503, 502, 531, 526, 485, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[502, 493, 528, 520, 476, 487, 528, 501, 490, 517, 517, 491, 531, 473, 539, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 523, 509, 512, 495, 513, 537, 492, 499, 487, 493, 471, 544, 487, 497, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 516, 511, 534, 509, 507, 507, 505, 530, 475, 530, 485, 512, 525, 530, 499, 7, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 536, 564, 489, 508, 520, 518, 498, 525, 489, 489, 495, 543, 506, 497, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 561, 521, 493, 525, 500, 505, 520, 531, 478, 502, 490, 511, 514, 539, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 534, 499, 525, 508, 518, 534, 488, 500, 493, 535, 523, 489, 518, 481, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 509, 540, 492, 518, 508, 508, 545, 517, 501, 528, 524, 501, 497, 538, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 483, 489, 552, 517, 539, 490, 493, 507, 512, 535, 479, 520, 540, 507, 498, 30, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8109, 8218, 8184, 8220, 8201, 8294, 8208] avg=8192.0000 max=8294 max_rank=6 min=8102 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32923 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[547, 516, 506, 571, 525, 496, 516, 529, 483, 504, 507, 491, 470, 503, 482, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 510, 511, 497, 504, 468, 508, 515, 488, 527, 527, 534, 528, 530, 532, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 470, 520, 570, 542, 520, 549, 486, 502, 520, 521, 497, 519, 455, 523, 516, 7, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 493, 450, 518, 519, 497, 530, 459, 551, 531, 492, 473, 545, 508, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 516, 508, 492, 562, 508, 513, 512, 510, 531, 521, 537, 484, 510, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 517, 497, 541, 473, 538, 489, 527, 516, 508, 486, 487, 485, 507, 516, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 549, 517, 504, 549, 506, 502, 490, 525, 486, 495, 523, 510, 501, 532, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 530, 534, 499, 531, 504, 524, 511, 492, 500, 491, 532, 548, 494, 524, 30, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8201, 8268, 8090, 8227, 8101, 8211, 8283] avg=8192.0000 max=8283 max_rank=7 min=8090 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[497, 558, 517, 476, 479, 498, 513, 522, 505, 500, 556, 479, 493, 514, 491, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 482, 505, 495, 480, 511, 524, 523, 497, 479, 510, 548, 502, 530, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 530, 509, 526, 517, 495, 536, 478, 539, 530, 481, 507, 494, 532, 524, 10, 8, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 524, 533, 522, 508, 532, 528, 496, 558, 516, 545, 527, 507, 472, 520, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 532, 488, 472, 524, 484, 502, 535, 492, 546, 507, 527, 493, 546, 524, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 482, 521, 500, 491, 474, 524, 525, 525, 520, 560, 521, 518, 547, 491, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 506, 474, 511, 536, 513, 485, 509, 490, 555, 526, 530, 529, 539, 484, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 499, 499, 528, 527, 502, 491, 490, 516, 523, 468, 491, 537, 484, 478, 33, 25, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8064, 8105, 8280, 8318, 8165, 8226, 8263, 8115] avg=8192.0000 max=8318 max_rank=3 min=8064 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 537, 520, 520, 515, 539, 498, 512, 475, 550, 550, 504, 516, 517, 529, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 522, 508, 521, 511, 502, 504, 532, 534, 502, 521, 526, 541, 499, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 474, 517, 532, 528, 457, 515, 494, 535, 507, 495, 497, 553, 503, 553, 473, 7, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 479, 496, 492, 501, 501, 530, 516, 540, 535, 502, 516, 505, 527, 513, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 513, 474, 490, 513, 509, 527, 486, 491, 554, 514, 530, 524, 476, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 562, 506, 501, 534, 522, 556, 523, 484, 519, 543, 470, 527, 518, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 503, 511, 495, 520, 534, 525, 540, 476, 498, 550, 502, 509, 515, 513, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 498, 468, 496, 492, 535, 477, 526, 504, 490, 495, 494, 515, 485, 493, 515, 30, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8268, 8142, 8236, 8130, 8264, 8208, 8043] avg=8192.0000 max=8268 max_rank=1 min=8043 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[493, 498, 522, 494, 515, 502, 526, 524, 543, 525, 515, 567, 549, 533, 516, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 550, 521, 551, 568, 517, 506, 539, 509, 504, 465, 543, 515, 521, 453, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 502, 474, 493, 500, 486, 487, 490, 463, 505, 506, 504, 530, 508, 524, 520, 6, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 519, 507, 524, 477, 492, 511, 526, 521, 494, 510, 476, 508, 502, 500, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 528, 509, 509, 522, 528, 475, 502, 519, 534, 492, 532, 538, 490, 540, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 510, 509, 547, 538, 505, 506, 520, 506, 480, 498, 482, 501, 529, 470, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 470, 530, 477, 548, 500, 522, 518, 525, 485, 516, 510, 519, 492, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 566, 510, 487, 490, 518, 456, 555, 504, 522, 507, 525, 497, 520, 494, 542, 28, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8335, 8287, 8013, 8086, 8241, 8170, 8155, 8249] avg=8192.0000 max=8335 max_rank=0 min=8013 min_rank=2 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 540, 516, 551, 550, 490, 500, 504, 493, 504, 524, 498, 519, 489, 538, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 514, 523, 490, 477, 495, 500, 540, 512, 500, 482, 486, 483, 507, 529, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 491, 488, 542, 499, 516, 523, 515, 478, 515, 498, 509, 522, 512, 502, 7, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 512, 499, 530, 566, 533, 516, 553, 537, 517, 476, 513, 498, 469, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 549, 468, 500, 524, 530, 457, 474, 524, 547, 508, 486, 525, 515, 491, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 516, 512, 521, 556, 524, 515, 526, 504, 527, 535, 494, 519, 471, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 523, 490, 529, 516, 491, 523, 455, 505, 567, 537, 508, 501, 488, 505, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 517, 523, 530, 468, 554, 472, 500, 536, 503, 503, 502, 531, 526, 485, 530, 27, 18, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8036, 8143, 8268, 8172, 8264, 8154, 8243] avg=8192.0000 max=8268 max_rank=3 min=8036 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[575, 511, 528, 518, 498, 513, 486, 499, 496, 507, 499, 499, 489, 532, 495, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 501, 545, 514, 496, 481, 512, 525, 539, 513, 531, 523, 509, 493, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 549, 498, 456, 529, 547, 507, 548, 535, 542, 525, 504, 493, 543, 509, 524, 7, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 476, 496, 510, 518, 469, 500, 510, 495, 489, 493, 491, 517, 525, 519, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 462, 560, 531, 515, 508, 504, 483, 467, 533, 507, 557, 521, 517, 511, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 515, 521, 494, 527, 513, 506, 523, 533, 489, 491, 537, 552, 496, 478, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 508, 499, 528, 524, 527, 533, 529, 526, 475, 496, 506, 492, 450, 483, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 476, 512, 531, 520, 502, 508, 496, 491, 509, 481, 506, 533, 533, 544, 486, 29, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8297, 8332, 7993, 8204, 8213, 8129, 8223] avg=8192.0000 max=8332 max_rank=2 min=7993 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 521, 519, 491, 533, 519, 503, 545, 549, 509, 456, 534, 495, 472, 545, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 530, 568, 484, 541, 522, 500, 515, 495, 522, 489, 524, 494, 483, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 500, 493, 519, 498, 516, 539, 464, 497, 536, 490, 503, 531, 485, 501, 529, 6, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 524, 550, 532, 511, 506, 527, 512, 517, 518, 549, 522, 502, 525, 541, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 545, 524, 518, 509, 506, 486, 524, 518, 493, 469, 497, 491, 514, 551, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 517, 480, 507, 483, 529, 494, 499, 510, 479, 526, 538, 529, 530, 518, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 484, 500, 488, 517, 505, 476, 521, 530, 524, 526, 510, 522, 497, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 556, 512, 526, 509, 504, 476, 508, 447, 548, 508, 489, 515, 526, 488, 507, 28, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8233, 8140, 8361, 8177, 8097, 8122, 8172] avg=8192.0000 max=8361 max_rank=3 min=8097 min_rank=5 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[570, 524, 550, 520, 529, 495, 493, 482, 520, 507, 498, 516, 494, 463, 549, 487, 533, 527, 498, 509, 508, 543, 498, 494, 473, 498, 469, 507, 502, 488, 495, 485, 501, 536, 477, 476, 510, 494, 481, 532, 507, 499, 538, 504, 495, 513, 513, 505, 528, 524, 522, 493, 544, 503, 535, 512, 491, 493, 516, 493, 503, 542, 524, 540, 516, 539, 520, 530, 523, 505, 473, 518, 528, 500, 526, 510, 520, 534, 469, 493, 516, 510, 495, 514, 483, 511, 562, 513, 528, 530, 486, 520, 536, 532, 517, 502, 498, 519, 533, 527, 491, 471, 540, 485, 523, 534, 518, 492, 515, 527, 519, 522, 512, 519, 503, 566, 518, 522, 521, 483, 542, 507, 517, 532, 546, 494, 521, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[539, 558, 523, 540, 524, 514, 476, 467, 479, 509, 524, 525, 470, 530, 520, 533, 508, 569, 495, 510, 514, 552, 473, 516, 511, 489, 535, 487, 491, 484, 509, 499, 518, 500, 535, 513, 526, 510, 513, 490, 489, 503, 539, 518, 523, 536, 503, 515, 520, 498, 517, 518, 529, 469, 536, 503, 504, 495, 519, 478, 495, 542, 538, 550, 500, 548, 495, 494, 486, 491, 522, 539, 544, 526, 494, 499, 494, 487, 528, 558, 490, 504, 520, 482, 488, 488, 489, 553, 520, 518, 532, 465, 558, 479, 512, 522, 482, 492, 519, 507, 509, 518, 490, 526, 520, 489, 498, 565, 545, 502, 495, 512, 518, 524, 507, 493, 511, 498, 501, 537, 525, 498, 526, 506, 517, 530, 525, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[527, 504, 486, 515, 517, 513, 505, 493, 544, 494, 502, 488, 490, 476, 503, 551, 491, 515, 507, 505, 511, 507, 520, 539, 541, 543, 553, 524, 510, 538, 507, 534, 518, 499, 496, 503, 519, 471, 516, 496, 498, 516, 518, 486, 493, 513, 518, 500, 457, 505, 526, 493, 552, 508, 518, 494, 514, 470, 548, 522, 494, 476, 531, 546, 519, 536, 494, 490, 470, 483, 514, 529, 518, 556, 479, 529, 474, 428, 571, 496, 505, 528, 472, 485, 508, 512, 527, 527, 518, 526, 519, 473, 530, 542, 503, 549, 499, 530, 512, 518, 475, 517, 512, 499, 549, 493, 514, 510, 542, 517, 528, 543, 556, 538, 505, 523, 523, 512, 523, 524, 555, 460, 474, 547, 525, 498, 520, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[488, 522, 518, 478, 508, 520, 515, 510, 524, 499, 595, 506, 510, 523, 507, 513, 542, 536, 552, 526, 545, 521, 502, 524, 519, 491, 501, 481, 494, 504, 484, 506, 507, 530, 533, 545, 526, 553, 509, 528, 516, 545, 493, 489, 496, 494, 531, 540, 498, 499, 493, 533, 494, 512, 511, 484, 497, 518, 524, 495, 513, 531, 497, 490, 486, 519, 514, 523, 441, 473, 540, 537, 533, 505, 521, 551, 490, 544, 521, 501, 469, 496, 506, 510, 524, 525, 508, 529, 488, 534, 498, 494, 551, 505, 525, 525, 551, 562, 492, 525, 495, 478, 492, 502, 493, 515, 517, 547, 489, 520, 482, 501, 545, 509, 487, 492, 527, 517, 538, 505, 539, 499, 497, 466, 482, 482, 510, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[474, 491, 478, 528, 500, 529, 524, 484, 510, 506, 503, 495, 505, 508, 507, 523, 503, 507, 549, 515, 516, 510, 510, 535, 513, 515, 525, 500, 513, 482, 513, 513, 512, 516, 468, 504, 564, 494, 528, 491, 499, 496, 492, 495, 491, 513, 494, 510, 510, 520, 528, 524, 532, 514, 536, 543, 533, 491, 515, 549, 517, 492, 554, 512, 490, 501, 513, 493, 544, 504, 506, 512, 468, 510, 498, 514, 509, 508, 505, 502, 536, 535, 555, 537, 514, 494, 456, 544, 517, 542, 566, 535, 511, 527, 508, 535, 480, 491, 520, 534, 529, 491, 491, 524, 524, 514, 525, 479, 523, 500, 518, 479, 521, 514, 519, 515, 502, 528, 511, 498, 521, 498, 508, 526, 528, 494, 505, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[514, 498, 550, 544, 524, 546, 524, 521, 520, 517, 501, 531, 472, 529, 514, 491, 521, 542, 469, 501, 510, 522, 460, 532, 536, 517, 491, 520, 521, 479, 513, 517, 486, 502, 534, 501, 536, 459, 521, 494, 521, 556, 501, 500, 469, 528, 511, 519, 494, 511, 474, 484, 530, 564, 459, 526, 523, 481, 524, 517, 548, 482, 535, 504, 557, 501, 495, 527, 521, 460, 495, 527, 496, 519, 531, 526, 494, 558, 493, 513, 549, 510, 524, 518, 482, 501, 564, 515, 538, 541, 509, 486, 477, 520, 506, 462, 479, 540, 513, 546, 537, 498, 478, 493, 487, 537, 533, 524, 514, 574, 495, 546, 513, 538, 526, 504, 494, 492, 522, 539, 491, 488, 489, 459, 523, 497, 483, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[527, 472, 513, 513, 558, 485, 540, 487, 526, 475, 480, 505, 515, 531, 495, 474, 510, 509, 525, 491, 503, 540, 498, 503, 502, 479, 536, 530, 509, 506, 512, 515, 478, 548, 500, 506, 487, 497, 477, 498, 533, 488, 544, 550, 535, 503, 534, 525, 503, 545, 531, 523, 515, 535, 485, 497, 529, 505, 529, 522, 555, 525, 542, 557, 511, 547, 566, 489, 552, 535, 485, 486, 513, 491, 533, 527, 516, 512, 482, 508, 526, 532, 518, 503, 470, 522, 515, 523, 462, 488, 522, 536, 485, 489, 487, 531, 487, 481, 519, 532, 526, 485, 516, 485, 512, 533, 513, 513, 505, 513, 530, 478, 522, 517, 517, 539, 500, 511, 506, 477, 498, 540, 484, 519, 514, 528, 532, 477] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[492, 490, 499, 535, 485, 491, 485, 574, 500, 520, 537, 488, 538, 525, 542, 533, 499, 508, 536, 508, 534, 560, 518, 488, 510, 501, 509, 531, 462, 483, 541, 536, 510, 531, 495, 489, 516, 514, 501, 520, 514, 510, 508, 523, 548, 527, 520, 497, 519, 543, 484, 484, 504, 496, 513, 537, 507, 526, 518, 540, 516, 508, 539, 497, 524, 513, 550, 527, 498, 525, 521, 493, 501, 515, 512, 500, 517, 499, 518, 509, 512, 516, 495, 525, 541, 557, 507, 505, 507, 506, 506, 525, 484, 499, 522, 509, 513, 533, 522, 524, 472, 474, 489, 487, 457, 528, 520, 474, 542, 503, 515, 502, 548, 496, 474, 509, 518, 537, 503, 495, 495, 490, 484, 497, 504, 521, 545, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[570, 524, 550, 520, 529, 495, 493, 482, 520, 507, 498, 516, 494, 463, 549, 487, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 527, 498, 509, 508, 543, 498, 494, 473, 498, 469, 507, 502, 488, 495, 485, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 477, 476, 510, 494, 481, 532, 507, 499, 538, 504, 495, 513, 513, 505, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 524, 522, 493, 544, 503, 535, 512, 491, 493, 516, 493, 503, 542, 484, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 539, 520, 530, 523, 505, 473, 518, 528, 500, 526, 510, 520, 534, 469, 493, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 495, 514, 483, 511, 562, 492, 528, 530, 486, 520, 536, 532, 517, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 519, 533, 527, 491, 471, 540, 485, 523, 534, 518, 492, 515, 527, 519, 522, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 503, 566, 518, 522, 521, 483, 542, 507, 517, 532, 546, 494, 521, 492, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8030, 8105, 8223, 8213, 8234, 8230, 8296] avg=8192.0000 max=8296 max_rank=7 min=8030 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[527, 472, 513, 513, 558, 485, 540, 487, 526, 475, 480, 505, 515, 531, 495, 474, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 525, 491, 503, 540, 498, 503, 502, 479, 536, 530, 509, 506, 512, 515, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 548, 500, 506, 487, 497, 477, 498, 533, 488, 544, 550, 535, 503, 534, 525, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 545, 531, 523, 515, 535, 485, 497, 529, 505, 529, 522, 555, 525, 500, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 547, 566, 489, 552, 535, 485, 486, 513, 491, 533, 527, 516, 512, 482, 508, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 532, 518, 503, 470, 522, 515, 501, 462, 488, 522, 536, 485, 489, 487, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 481, 519, 532, 526, 485, 516, 485, 512, 533, 513, 513, 505, 513, 530, 478, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 517, 539, 500, 511, 506, 477, 498, 540, 484, 519, 514, 528, 532, 477, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8171, 8228, 8356, 8262, 8087, 8145, 8182] avg=8192.0000 max=8356 max_rank=3 min=8087 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[527, 504, 486, 515, 517, 513, 505, 493, 544, 494, 502, 488, 490, 476, 503, 551, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 507, 505, 511, 507, 520, 539, 541, 543, 553, 524, 510, 538, 507, 534, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 499, 496, 503, 519, 471, 516, 496, 498, 516, 518, 486, 493, 513, 518, 500, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 505, 526, 493, 552, 508, 518, 494, 514, 470, 548, 522, 494, 476, 481, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 536, 494, 490, 470, 483, 514, 529, 518, 556, 479, 529, 474, 428, 571, 496, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 472, 485, 508, 512, 527, 491, 518, 526, 519, 473, 530, 542, 503, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 530, 512, 518, 475, 517, 512, 499, 549, 493, 514, 510, 542, 517, 528, 543, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 538, 505, 523, 523, 512, 523, 524, 555, 460, 474, 547, 525, 498, 520, 518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8353, 8087, 8104, 8101, 8188, 8278, 8304] avg=8192.0000 max=8353 max_rank=1 min=8087 min_rank=2 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[474, 491, 478, 528, 500, 529, 524, 484, 510, 506, 503, 495, 505, 508, 507, 523, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 507, 549, 515, 516, 510, 510, 535, 513, 515, 525, 500, 513, 482, 513, 513, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 516, 468, 504, 564, 494, 528, 491, 499, 496, 492, 495, 491, 513, 494, 510, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 528, 524, 532, 514, 536, 543, 533, 491, 515, 549, 517, 492, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 501, 513, 493, 544, 504, 506, 512, 468, 510, 498, 514, 509, 508, 505, 502, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 535, 555, 537, 514, 494, 456, 521, 517, 542, 566, 535, 511, 527, 508, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 491, 520, 534, 529, 491, 491, 524, 524, 514, 525, 479, 523, 500, 518, 479, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 514, 519, 515, 502, 528, 511, 498, 521, 498, 508, 526, 528, 494, 505, 516, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8074, 8223, 8092, 8328, 8086, 8389, 8139, 8205] avg=8192.0000 max=8389 max_rank=5 min=8074 min_rank=0 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[488, 522, 518, 478, 508, 520, 515, 510, 524, 499, 595, 506, 510, 523, 507, 513, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 536, 552, 526, 545, 521, 502, 524, 519, 491, 501, 481, 494, 504, 484, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 533, 545, 526, 553, 509, 528, 516, 545, 493, 489, 496, 494, 531, 540, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 493, 533, 494, 512, 511, 484, 497, 518, 524, 495, 513, 531, 459, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 514, 523, 441, 473, 540, 537, 533, 505, 521, 551, 490, 544, 521, 501, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 496, 506, 510, 524, 525, 508, 506, 488, 534, 498, 494, 551, 505, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 562, 492, 525, 495, 478, 492, 502, 493, 515, 517, 547, 489, 520, 482, 501, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 487, 492, 527, 517, 538, 505, 539, 499, 497, 466, 482, 482, 510, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8232, 8358, 8051, 8208, 8164, 8176, 8102] avg=8192.0000 max=8358 max_rank=2 min=8051 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[539, 558, 523, 540, 524, 514, 476, 467, 479, 509, 524, 525, 470, 530, 520, 533, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 569, 495, 510, 514, 552, 473, 516, 511, 489, 535, 487, 491, 484, 509, 499, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 535, 513, 526, 510, 513, 490, 489, 503, 539, 518, 523, 536, 503, 515, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 498, 517, 518, 529, 469, 536, 503, 504, 495, 519, 478, 495, 542, 496, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 548, 495, 494, 486, 491, 522, 539, 544, 526, 494, 499, 494, 487, 528, 558, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 504, 520, 482, 488, 488, 489, 530, 520, 518, 532, 465, 558, 479, 512, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 492, 519, 507, 509, 518, 490, 526, 520, 489, 498, 565, 545, 502, 495, 512, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 524, 507, 493, 511, 498, 501, 537, 525, 498, 526, 506, 517, 530, 525, 511, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8146, 8256, 8169, 8214, 8097, 8186, 8228] avg=8192.0000 max=8256 max_rank=2 min=8097 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 498, 550, 544, 524, 546, 524, 521, 520, 517, 501, 531, 472, 529, 514, 491, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 542, 469, 501, 510, 522, 460, 532, 536, 517, 491, 520, 521, 479, 513, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 534, 501, 536, 459, 521, 494, 521, 556, 501, 500, 469, 528, 511, 519, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 474, 484, 530, 564, 459, 526, 523, 481, 524, 517, 548, 482, 495, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 501, 495, 527, 521, 460, 495, 527, 496, 519, 531, 526, 494, 558, 493, 513, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 510, 524, 518, 482, 501, 564, 494, 538, 541, 509, 486, 477, 520, 506, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 540, 513, 546, 537, 498, 478, 493, 487, 537, 533, 524, 514, 574, 495, 546, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 538, 526, 504, 494, 492, 522, 539, 491, 488, 489, 459, 523, 497, 483, 528, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 8154, 8162, 8116, 8222, 8181, 8310, 8087] avg=8192.0000 max=8310 max_rank=6 min=8087 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 490, 499, 535, 485, 491, 485, 574, 500, 520, 537, 488, 538, 525, 542, 533, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 536, 508, 534, 560, 518, 488, 510, 501, 509, 531, 462, 483, 541, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 531, 495, 489, 516, 514, 501, 520, 514, 510, 508, 523, 548, 527, 520, 497, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 543, 484, 484, 504, 496, 513, 537, 507, 526, 518, 540, 516, 508, 497, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 550, 527, 498, 525, 521, 493, 501, 515, 512, 500, 517, 499, 518, 509, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 516, 495, 525, 541, 557, 507, 485, 507, 506, 506, 525, 484, 499, 522, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 533, 522, 524, 472, 474, 489, 487, 457, 528, 520, 474, 542, 503, 515, 502, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 496, 474, 509, 518, 537, 503, 495, 495, 490, 484, 497, 504, 521, 545, 515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8227, 8248, 8189, 8230, 8196, 8072, 8132] avg=8192.0000 max=8248 max_rank=2 min=8072 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4352, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[494, 507, 520, 502, 481, 536, 531, 463, 490, 511, 535, 513, 485, 484, 518, 484, 500, 518, 518, 489, 551, 547, 520, 511, 502, 530, 527, 532, 561, 543, 538, 529, 514, 486, 501, 527, 475, 505, 506, 504, 511, 479, 508, 526, 522, 471, 529, 489, 504, 466, 524, 522, 568, 501, 479, 503, 513, 515, 468, 544, 513, 510, 489, 558, 510, 501, 486, 518, 512, 486, 552, 508, 537, 545, 482, 494, 510, 525, 526, 482, 504, 565, 543, 530, 564, 533, 541, 496, 526, 449, 546, 485, 543, 551, 468, 482, 508, 502, 491, 548, 519, 535, 504, 509, 471, 548, 519, 490, 582, 515, 524, 522, 514, 513, 507, 527, 508, 469, 511, 502, 505, 471, 532, 479, 537, 486, 482, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[514, 537, 544, 476, 521, 500, 537, 497, 523, 526, 489, 551, 494, 527, 526, 483, 516, 474, 479, 530, 548, 516, 529, 509, 508, 528, 528, 507, 517, 561, 512, 549, 504, 528, 527, 474, 479, 526, 478, 545, 520, 479, 461, 502, 532, 517, 514, 498, 544, 530, 501, 539, 536, 538, 533, 525, 504, 508, 512, 501, 479, 495, 527, 503, 517, 505, 505, 502, 541, 528, 498, 523, 527, 506, 523, 520, 537, 512, 511, 526, 491, 461, 488, 492, 477, 523, 490, 475, 494, 514, 452, 503, 497, 541, 492, 490, 564, 521, 483, 517, 480, 517, 486, 517, 524, 540, 498, 526, 533, 488, 494, 491, 511, 523, 523, 533, 512, 477, 536, 550, 529, 512, 521, 524, 485, 544, 509, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[525, 518, 502, 499, 501, 472, 545, 531, 516, 466, 525, 514, 550, 480, 533, 504, 493, 509, 519, 519, 491, 497, 522, 467, 466, 504, 545, 493, 488, 515, 520, 513, 517, 473, 504, 505, 485, 538, 447, 541, 494, 506, 504, 530, 508, 508, 522, 513, 465, 516, 504, 489, 536, 521, 491, 511, 533, 507, 562, 536, 537, 481, 541, 515, 530, 523, 502, 514, 560, 521, 497, 524, 491, 528, 523, 555, 531, 515, 531, 545, 500, 517, 503, 552, 495, 485, 519, 521, 525, 500, 535, 514, 545, 480, 536, 494, 491, 501, 502, 511, 516, 539, 540, 503, 509, 511, 529, 528, 504, 502, 517, 507, 518, 504, 515, 494, 471, 490, 517, 495, 507, 468, 545, 519, 535, 524, 510, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[536, 515, 505, 526, 550, 494, 524, 535, 508, 532, 507, 511, 513, 529, 509, 475, 485, 556, 521, 474, 551, 517, 503, 521, 493, 516, 435, 554, 531, 567, 509, 529, 502, 499, 549, 528, 506, 539, 486, 505, 500, 497, 524, 542, 515, 497, 487, 489, 515, 504, 502, 516, 539, 503, 533, 508, 519, 546, 460, 485, 519, 535, 513, 535, 510, 475, 492, 475, 501, 529, 533, 499, 557, 528, 518, 489, 524, 500, 493, 504, 497, 503, 477, 483, 524, 524, 480, 514, 523, 560, 486, 532, 522, 534, 490, 542, 470, 521, 501, 519, 517, 497, 536, 488, 497, 496, 530, 487, 505, 497, 491, 507, 524, 519, 482, 495, 529, 539, 531, 520, 471, 520, 528, 537, 506, 519, 530, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[493, 509, 545, 493, 527, 515, 504, 519, 527, 492, 493, 546, 487, 508, 495, 540, 513, 526, 520, 495, 520, 480, 480, 520, 528, 532, 462, 517, 507, 564, 499, 476, 516, 534, 530, 505, 509, 512, 518, 500, 517, 495, 526, 506, 503, 519, 519, 501, 527, 488, 511, 531, 496, 489, 522, 509, 539, 552, 527, 536, 547, 490, 505, 538, 508, 521, 505, 507, 523, 543, 493, 548, 515, 495, 527, 532, 493, 482, 531, 498, 470, 530, 509, 534, 510, 493, 502, 552, 492, 523, 494, 514, 526, 537, 503, 539, 506, 484, 504, 513, 505, 525, 485, 531, 490, 526, 479, 516, 502, 527, 548, 496, 519, 521, 534, 560, 516, 492, 538, 459, 529, 487, 528, 469, 506, 504, 473, 466] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[494, 511, 491, 502, 471, 517, 501, 520, 531, 497, 504, 500, 530, 517, 548, 488, 514, 476, 496, 518, 520, 564, 534, 497, 498, 533, 492, 570, 477, 482, 510, 518, 498, 499, 522, 516, 499, 513, 517, 523, 540, 487, 524, 530, 532, 543, 547, 475, 553, 472, 485, 481, 555, 521, 495, 504, 544, 554, 553, 529, 515, 531, 495, 552, 499, 494, 521, 489, 486, 530, 525, 546, 491, 509, 490, 523, 498, 511, 523, 511, 520, 475, 495, 505, 505, 478, 498, 496, 496, 478, 502, 478, 508, 511, 517, 509, 492, 516, 555, 570, 516, 491, 529, 474, 500, 519, 506, 547, 514, 500, 532, 512, 541, 512, 520, 496, 521, 511, 496, 507, 522, 491, 552, 497, 543, 518, 491, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[509, 501, 497, 519, 478, 497, 531, 524, 499, 510, 519, 501, 526, 514, 488, 496, 489, 469, 536, 501, 523, 548, 519, 523, 503, 546, 513, 513, 534, 533, 524, 503, 493, 516, 518, 519, 467, 522, 501, 503, 516, 530, 493, 502, 503, 478, 544, 503, 521, 532, 512, 525, 502, 493, 509, 494, 489, 511, 477, 535, 518, 551, 519, 483, 534, 485, 507, 495, 486, 520, 522, 528, 509, 481, 540, 507, 520, 519, 493, 491, 484, 524, 528, 489, 506, 515, 511, 495, 540, 543, 513, 556, 540, 534, 478, 480, 518, 531, 504, 527, 507, 531, 520, 532, 566, 485, 535, 482, 491, 495, 493, 545, 528, 492, 501, 480, 531, 495, 501, 526, 522, 518, 505, 521, 568, 515, 504, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[545, 508, 506, 516, 527, 492, 465, 525, 512, 482, 542, 547, 517, 508, 529, 513, 481, 517, 524, 530, 481, 510, 516, 517, 529, 515, 507, 532, 531, 533, 512, 494, 565, 525, 462, 497, 511, 492, 527, 488, 522, 497, 513, 501, 514, 457, 509, 541, 544, 505, 515, 482, 521, 545, 512, 549, 511, 469, 467, 501, 512, 497, 533, 547, 555, 452, 542, 502, 497, 484, 519, 500, 502, 484, 492, 487, 537, 476, 544, 535, 478, 497, 536, 569, 526, 507, 482, 490, 475, 531, 489, 518, 453, 515, 506, 541, 551, 502, 507, 526, 496, 519, 542, 532, 487, 501, 517, 510, 530, 481, 502, 509, 517, 522, 509, 566, 539, 486, 496, 530, 556, 497, 505, 506, 541, 552, 506, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[494, 507, 520, 502, 481, 536, 531, 463, 490, 511, 535, 513, 485, 484, 518, 484, 7, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 518, 518, 489, 551, 547, 520, 511, 502, 530, 527, 532, 561, 499, 538, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 486, 501, 527, 475, 505, 506, 504, 511, 479, 508, 526, 522, 471, 529, 489, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 466, 524, 522, 512, 501, 479, 503, 513, 515, 468, 544, 513, 510, 489, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 501, 486, 518, 512, 486, 552, 503, 537, 545, 482, 494, 510, 525, 526, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 565, 543, 530, 564, 533, 541, 496, 526, 449, 546, 485, 543, 551, 468, 482, 43, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 502, 491, 538, 519, 535, 504, 509, 471, 548, 519, 490, 582, 515, 524, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 513, 507, 527, 508, 469, 511, 502, 505, 471, 532, 479, 534, 486, 482, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8069, 8372, 8110, 8121, 8169, 8372, 8277, 8046] avg=8192.0000 max=8372 max_rank=1 min=8046 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 515, 505, 526, 550, 494, 524, 535, 508, 532, 507, 511, 513, 529, 509, 475, 6, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 556, 521, 474, 551, 517, 503, 521, 493, 516, 435, 554, 531, 521, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 499, 549, 528, 506, 539, 486, 505, 500, 497, 524, 542, 515, 497, 487, 489, 53, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 502, 516, 486, 503, 533, 508, 519, 546, 460, 485, 519, 535, 513, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 475, 492, 475, 501, 529, 533, 494, 557, 528, 518, 489, 524, 500, 493, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 503, 477, 483, 524, 524, 480, 514, 523, 560, 486, 532, 522, 534, 490, 542, 45, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 521, 501, 510, 517, 497, 536, 488, 497, 496, 530, 487, 505, 497, 491, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 519, 482, 495, 529, 539, 531, 520, 471, 520, 528, 537, 504, 519, 530, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8216, 8219, 8179, 8122, 8239, 8050, 8229] avg=8192.0000 max=8282 max_rank=0 min=8050 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 501, 497, 519, 478, 497, 531, 524, 499, 510, 519, 501, 526, 514, 488, 496, 6, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 469, 536, 501, 523, 548, 519, 523, 503, 546, 513, 513, 534, 490, 524, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 516, 518, 519, 467, 522, 501, 503, 516, 530, 493, 502, 503, 478, 544, 503, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 532, 512, 525, 453, 493, 509, 494, 489, 511, 477, 535, 518, 551, 519, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 485, 507, 495, 486, 520, 522, 522, 509, 481, 540, 507, 520, 519, 493, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 524, 528, 489, 506, 515, 511, 495, 540, 543, 513, 556, 540, 534, 478, 480, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 531, 504, 518, 507, 531, 520, 532, 566, 485, 535, 482, 491, 495, 493, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 492, 501, 480, 531, 495, 501, 526, 522, 518, 505, 521, 565, 515, 504, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8234, 8158, 8122, 8131, 8281, 8253, 8233] avg=8192.0000 max=8281 max_rank=5 min=8122 min_rank=3 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[493, 509, 545, 493, 527, 515, 504, 519, 527, 492, 493, 546, 487, 508, 495, 540, 6, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 520, 495, 520, 480, 480, 520, 528, 532, 462, 517, 507, 518, 499, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 534, 530, 505, 509, 512, 518, 500, 517, 495, 526, 506, 503, 519, 519, 501, 48, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 488, 511, 531, 448, 489, 522, 509, 539, 552, 527, 536, 547, 490, 505, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 505, 507, 523, 543, 493, 542, 515, 495, 527, 532, 493, 482, 531, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 530, 509, 534, 510, 493, 502, 552, 492, 523, 494, 514, 526, 537, 503, 539, 45, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 484, 504, 504, 505, 525, 485, 531, 490, 526, 479, 516, 502, 527, 548, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 521, 534, 560, 516, 492, 538, 459, 529, 487, 528, 469, 504, 504, 473, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8093, 8259, 8259, 8215, 8276, 8128, 8099] avg=8192.0000 max=8276 max_rank=5 min=8093 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[525, 518, 502, 499, 501, 472, 545, 531, 516, 466, 525, 514, 550, 480, 533, 504, 12, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 509, 519, 519, 491, 497, 522, 467, 466, 504, 545, 493, 488, 463, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 473, 504, 505, 485, 538, 447, 541, 494, 506, 504, 530, 508, 508, 522, 513, 57, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 516, 504, 489, 479, 521, 491, 511, 533, 507, 562, 536, 537, 481, 541, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 523, 502, 514, 560, 521, 497, 515, 491, 528, 523, 555, 531, 515, 531, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 517, 503, 552, 495, 485, 519, 521, 525, 500, 535, 514, 545, 480, 536, 494, 44, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 502, 493, 516, 539, 540, 503, 509, 511, 529, 528, 504, 502, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 504, 515, 494, 471, 490, 517, 495, 507, 468, 545, 519, 528, 524, 510, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8009, 8160, 8188, 8381, 8271, 8192, 8126] avg=8192.0000 max=8381 max_rank=4 min=8009 min_rank=1 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[494, 511, 491, 502, 471, 517, 501, 520, 531, 497, 504, 500, 530, 517, 548, 488, 7, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 476, 496, 518, 520, 564, 534, 497, 498, 533, 492, 570, 477, 443, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 522, 516, 499, 513, 517, 523, 540, 487, 524, 530, 532, 543, 547, 475, 54, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 472, 485, 481, 501, 521, 495, 504, 544, 554, 553, 529, 515, 531, 495, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 494, 521, 489, 486, 530, 525, 540, 491, 509, 490, 523, 498, 511, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 475, 495, 505, 505, 478, 498, 496, 496, 478, 502, 478, 508, 511, 517, 509, 38, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 555, 560, 516, 491, 529, 474, 500, 519, 506, 547, 514, 500, 532, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 512, 520, 496, 521, 511, 496, 507, 522, 491, 552, 497, 540, 518, 491, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8160, 8320, 8285, 8140, 8012, 8263, 8218] avg=8192.0000 max=8320 max_rank=2 min=8012 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 537, 544, 476, 521, 500, 537, 497, 523, 526, 489, 551, 494, 527, 526, 483, 6, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 474, 479, 530, 548, 516, 529, 509, 508, 528, 528, 507, 517, 516, 512, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 528, 527, 474, 479, 526, 478, 545, 520, 479, 461, 502, 532, 517, 514, 498, 52, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 530, 501, 539, 484, 538, 533, 525, 504, 508, 512, 501, 479, 495, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 505, 502, 541, 528, 498, 517, 527, 506, 523, 520, 537, 512, 511, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 461, 488, 492, 477, 523, 490, 475, 494, 514, 452, 503, 497, 541, 492, 490, 44, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 521, 483, 508, 480, 517, 486, 517, 524, 540, 498, 526, 533, 488, 494, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 523, 523, 533, 512, 477, 536, 550, 529, 512, 521, 524, 483, 544, 509, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8266, 8137, 8223, 8275, 7927, 8170, 8279] avg=8192.0000 max=8279 max_rank=7 min=7927 min_rank=5 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[545, 508, 506, 516, 527, 492, 465, 525, 512, 482, 542, 547, 517, 508, 529, 513, 6, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 517, 524, 530, 481, 510, 516, 517, 529, 515, 507, 532, 531, 490, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 525, 462, 497, 511, 492, 527, 488, 522, 497, 513, 501, 514, 457, 509, 541, 51, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 505, 515, 482, 470, 545, 512, 549, 511, 469, 467, 501, 512, 497, 533, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 452, 542, 502, 497, 484, 519, 495, 502, 484, 492, 487, 537, 476, 544, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 497, 536, 569, 526, 507, 482, 490, 475, 531, 489, 518, 453, 515, 506, 541, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 502, 507, 517, 496, 519, 542, 532, 487, 501, 517, 510, 530, 481, 502, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 509, 566, 539, 486, 496, 530, 556, 497, 505, 506, 538, 552, 506, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8186, 8173, 8159, 8103, 8158, 8203, 8306] avg=8192.0000 max=8306 max_rank=7 min=8103 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4224, 4096, 3968, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 3840, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[505, 510, 504, 545, 541, 518, 547, 470, 529, 506, 472, 534, 506, 530, 487, 481, 520, 493, 520, 467, 482, 551, 491, 540, 512, 515, 466, 507, 512, 542, 537, 510, 511, 494, 512, 527, 547, 506, 509, 498, 502, 515, 533, 497, 524, 509, 499, 469, 523, 516, 517, 531, 477, 493, 497, 540, 525, 486, 534, 518, 535, 528, 553, 503, 488, 559, 519, 537, 509, 480, 501, 544, 529, 524, 518, 531, 509, 491, 526, 518, 511, 517, 500, 528, 489, 477, 494, 526, 516, 491, 492, 540, 524, 523, 494, 503, 499, 490, 505, 523, 509, 517, 520, 516, 513, 512, 491, 532, 502, 534, 488, 535, 512, 509, 490, 491, 513, 532, 536, 475, 521, 487, 519, 545, 520, 496, 525, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[537, 492, 480, 515, 503, 509, 510, 498, 527, 509, 532, 475, 471, 509, 502, 518, 518, 479, 531, 511, 501, 525, 496, 519, 568, 536, 529, 492, 508, 513, 497, 529, 518, 482, 492, 522, 517, 509, 542, 555, 526, 541, 531, 512, 541, 524, 529, 528, 494, 472, 521, 482, 495, 499, 543, 532, 508, 490, 539, 554, 525, 525, 502, 522, 506, 512, 480, 490, 486, 508, 514, 531, 490, 505, 504, 534, 522, 512, 528, 535, 539, 490, 484, 551, 515, 531, 485, 488, 499, 518, 484, 545, 511, 492, 517, 524, 465, 514, 495, 511, 540, 510, 480, 490, 509, 479, 480, 516, 515, 523, 530, 500, 507, 534, 531, 546, 520, 494, 542, 523, 538, 496, 484, 513, 522, 471, 526, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[483, 547, 505, 497, 435, 476, 513, 505, 536, 494, 513, 523, 510, 552, 524, 486, 492, 542, 518, 466, 514, 537, 546, 544, 525, 537, 529, 511, 515, 534, 496, 537, 507, 518, 525, 536, 504, 498, 498, 514, 510, 519, 557, 555, 522, 525, 497, 463, 539, 493, 500, 514, 531, 491, 523, 490, 485, 508, 519, 517, 541, 507, 467, 485, 526, 484, 494, 531, 532, 501, 509, 528, 496, 506, 485, 529, 520, 451, 487, 536, 503, 512, 503, 513, 528, 480, 493, 505, 535, 514, 486, 511, 502, 534, 502, 516, 519, 515, 521, 524, 498, 479, 523, 515, 526, 491, 503, 483, 556, 536, 509, 544, 541, 505, 501, 507, 513, 509, 513, 552, 489, 518, 508, 502, 532, 536, 512, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[517, 502, 478, 484, 511, 539, 491, 503, 519, 521, 486, 474, 561, 502, 541, 492, 481, 560, 543, 508, 522, 478, 517, 516, 528, 529, 497, 470, 500, 500, 467, 516, 509, 535, 510, 537, 520, 502, 538, 519, 511, 513, 525, 535, 515, 507, 534, 493, 516, 526, 493, 527, 486, 509, 513, 490, 527, 509, 487, 532, 512, 512, 516, 490, 497, 517, 501, 477, 489, 522, 519, 478, 519, 512, 522, 505, 529, 481, 497, 520, 521, 491, 498, 522, 535, 505, 541, 533, 490, 505, 505, 542, 512, 535, 531, 515, 478, 487, 548, 547, 519, 537, 506, 556, 548, 542, 492, 527, 489, 538, 503, 500, 534, 518, 504, 485, 531, 480, 528, 537, 507, 531, 519, 536, 441, 492, 489, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[522, 503, 504, 514, 456, 538, 525, 499, 499, 487, 521, 487, 506, 504, 520, 512, 547, 521, 477, 490, 502, 528, 522, 494, 511, 527, 520, 551, 509, 539, 486, 508, 502, 537, 512, 515, 523, 514, 485, 510, 535, 490, 496, 516, 513, 494, 485, 493, 504, 510, 513, 503, 517, 512, 492, 478, 527, 502, 505, 487, 522, 524, 504, 526, 496, 478, 543, 521, 541, 504, 546, 526, 526, 512, 526, 542, 492, 530, 503, 522, 525, 508, 502, 498, 533, 513, 489, 524, 533, 488, 547, 503, 545, 504, 509, 483, 542, 538, 527, 501, 502, 480, 524, 500, 521, 532, 570, 525, 497, 494, 545, 502, 529, 514, 491, 496, 521, 536, 522, 509, 533, 497, 480, 538, 466, 510, 511, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[502, 527, 496, 487, 511, 487, 506, 503, 509, 521, 484, 502, 493, 554, 512, 548, 526, 477, 506, 464, 513, 550, 494, 497, 484, 551, 523, 471, 553, 513, 513, 543, 535, 496, 499, 528, 524, 541, 498, 546, 509, 532, 517, 497, 485, 519, 479, 568, 485, 491, 465, 507, 499, 517, 505, 504, 520, 476, 518, 484, 504, 476, 536, 553, 522, 506, 506, 490, 512, 537, 501, 527, 517, 499, 513, 492, 519, 530, 495, 496, 496, 523, 489, 480, 502, 486, 498, 507, 550, 535, 507, 486, 513, 516, 494, 545, 490, 501, 494, 556, 532, 490, 507, 523, 498, 529, 501, 546, 504, 548, 547, 477, 524, 518, 527, 529, 539, 502, 507, 498, 526, 542, 489, 512, 545, 532, 556, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[541, 516, 541, 496, 527, 538, 500, 501, 527, 484, 513, 484, 493, 502, 522, 533, 530, 527, 509, 527, 518, 521, 497, 509, 543, 526, 491, 485, 531, 525, 476, 542, 518, 484, 522, 503, 520, 495, 504, 538, 515, 552, 525, 521, 471, 505, 533, 541, 518, 523, 504, 518, 525, 534, 525, 487, 523, 498, 527, 471, 510, 487, 507, 496, 531, 529, 562, 482, 519, 537, 483, 507, 496, 499, 509, 507, 526, 520, 516, 551, 518, 503, 547, 499, 470, 539, 501, 478, 508, 510, 526, 499, 507, 482, 514, 513, 517, 510, 528, 492, 482, 494, 485, 484, 488, 528, 517, 523, 505, 508, 531, 539, 517, 570, 464, 535, 477, 529, 533, 509, 498, 510, 506, 517, 496, 468, 512, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[474, 484, 485, 550, 502, 504, 491, 514, 474, 534, 541, 501, 489, 498, 515, 546, 514, 506, 541, 545, 503, 498, 528, 484, 559, 558, 509, 504, 506, 486, 537, 544, 470, 507, 524, 498, 504, 497, 464, 532, 525, 509, 513, 504, 504, 484, 486, 560, 480, 513, 506, 495, 496, 535, 508, 561, 559, 505, 515, 509, 520, 488, 481, 481, 512, 488, 525, 528, 513, 545, 487, 502, 490, 488, 509, 485, 533, 487, 541, 492, 513, 517, 488, 515, 487, 505, 548, 553, 498, 542, 512, 524, 490, 533, 561, 482, 534, 472, 515, 547, 530, 518, 561, 517, 491, 524, 516, 547, 498, 513, 534, 536, 497, 509, 487, 521, 533, 493, 516, 483, 489, 501, 510, 525, 503, 515, 529, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[505, 510, 504, 545, 541, 518, 547, 470, 529, 506, 472, 534, 506, 530, 487, 481, 41, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 493, 520, 467, 482, 551, 491, 540, 512, 474, 466, 507, 512, 542, 537, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 494, 512, 527, 547, 506, 509, 468, 502, 515, 533, 497, 524, 509, 499, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 517, 531, 477, 493, 497, 540, 525, 486, 534, 518, 535, 528, 553, 503, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 559, 519, 537, 509, 480, 501, 544, 529, 524, 518, 531, 509, 491, 526, 518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 517, 500, 528, 489, 477, 494, 526, 516, 491, 492, 540, 524, 523, 494, 503, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 490, 505, 487, 509, 517, 520, 516, 513, 512, 491, 532, 502, 534, 488, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 490, 491, 513, 532, 530, 475, 521, 487, 519, 545, 520, 496, 525, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8124, 8122, 8316, 8286, 8137, 8150, 8158] avg=8192.0000 max=8316 max_rank=3 min=8122 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[483, 547, 505, 497, 435, 476, 513, 505, 536, 494, 513, 523, 510, 552, 524, 486, 43, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 542, 518, 466, 514, 537, 546, 544, 525, 494, 529, 511, 515, 534, 496, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 525, 536, 504, 498, 498, 484, 510, 519, 557, 555, 522, 525, 497, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 493, 500, 514, 531, 491, 523, 490, 485, 508, 519, 517, 541, 507, 467, 485, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 484, 494, 531, 532, 501, 509, 528, 496, 506, 485, 529, 520, 451, 487, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 512, 503, 513, 528, 480, 493, 505, 535, 514, 486, 511, 502, 534, 502, 516, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 515, 521, 488, 498, 479, 523, 515, 526, 491, 503, 483, 556, 536, 509, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 505, 501, 507, 513, 509, 507, 552, 489, 518, 508, 502, 532, 536, 512, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8300, 8218, 8150, 8118, 8149, 8206, 8236] avg=8192.0000 max=8300 max_rank=1 min=8118 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[517, 502, 478, 484, 511, 539, 491, 503, 519, 521, 486, 474, 561, 502, 541, 492, 42, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 560, 543, 508, 522, 478, 517, 516, 528, 487, 497, 470, 500, 500, 467, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 535, 510, 537, 520, 502, 538, 489, 511, 513, 525, 535, 515, 507, 534, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 526, 493, 527, 486, 509, 513, 490, 527, 509, 487, 532, 512, 512, 516, 490, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 501, 477, 489, 522, 519, 478, 519, 512, 522, 505, 529, 481, 497, 520, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 498, 522, 535, 505, 541, 533, 490, 505, 505, 542, 512, 535, 531, 515, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 487, 548, 510, 519, 537, 506, 556, 548, 542, 492, 527, 489, 538, 503, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 518, 504, 485, 531, 480, 522, 537, 507, 531, 519, 536, 441, 492, 489, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8090, 8273, 8185, 8088, 8293, 8280, 8146] avg=8192.0000 max=8293 max_rank=5 min=8088 min_rank=4 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[522, 503, 504, 514, 456, 538, 525, 499, 499, 487, 521, 487, 506, 504, 520, 512, 46, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 521, 477, 490, 502, 528, 522, 494, 511, 481, 520, 551, 509, 539, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 512, 515, 523, 514, 485, 472, 535, 490, 496, 516, 513, 494, 485, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 510, 513, 503, 517, 512, 492, 478, 527, 502, 505, 487, 522, 524, 504, 526, 25, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 478, 543, 521, 541, 504, 546, 526, 526, 512, 526, 542, 492, 530, 503, 522, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 508, 502, 498, 533, 513, 489, 524, 533, 488, 547, 503, 545, 504, 509, 483, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 538, 527, 460, 502, 480, 524, 500, 521, 532, 570, 525, 497, 494, 545, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 514, 491, 496, 521, 536, 509, 509, 533, 497, 480, 538, 466, 510, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8186, 8082, 8173, 8314, 8224, 8259, 8136] avg=8192.0000 max=8314 max_rank=4 min=8082 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[537, 492, 480, 515, 503, 509, 510, 498, 527, 509, 532, 475, 471, 509, 502, 518, 43, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 479, 531, 511, 501, 525, 496, 519, 568, 493, 529, 492, 508, 513, 497, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 482, 492, 522, 517, 509, 542, 522, 526, 541, 531, 512, 541, 524, 529, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 472, 521, 482, 495, 499, 543, 532, 508, 490, 539, 554, 525, 525, 502, 522, 23, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 480, 490, 486, 508, 514, 531, 490, 505, 504, 534, 522, 512, 528, 535, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 490, 484, 551, 515, 531, 485, 488, 499, 518, 484, 545, 511, 492, 517, 524, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 514, 495, 476, 540, 510, 480, 490, 509, 479, 480, 516, 515, 523, 530, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 534, 531, 546, 520, 494, 536, 523, 538, 496, 484, 513, 522, 471, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8209, 8336, 8244, 8160, 8186, 8022, 8232] avg=8192.0000 max=8336 max_rank=2 min=8022 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32600 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[541, 516, 541, 496, 527, 538, 500, 501, 527, 484, 513, 484, 493, 502, 522, 533, 42, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 527, 509, 527, 518, 521, 497, 509, 543, 484, 491, 485, 531, 525, 476, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 484, 522, 503, 520, 495, 504, 506, 515, 552, 525, 521, 471, 505, 533, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 523, 504, 518, 525, 534, 525, 487, 523, 498, 527, 471, 510, 487, 507, 496, 22, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 529, 562, 482, 519, 537, 483, 507, 496, 499, 509, 507, 526, 520, 516, 551, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 547, 499, 470, 539, 501, 478, 508, 510, 526, 499, 507, 482, 514, 513, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 528, 459, 482, 494, 485, 484, 488, 528, 517, 523, 505, 508, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 570, 464, 535, 477, 529, 527, 509, 498, 510, 506, 517, 496, 468, 512, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8215, 8215, 8192, 8277, 8127, 8098, 8136] avg=8192.0000 max=8277 max_rank=4 min=8098 min_rank=6 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 527, 496, 487, 511, 487, 506, 503, 509, 521, 484, 502, 493, 554, 512, 548, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 477, 506, 464, 513, 550, 494, 497, 484, 507, 523, 471, 553, 513, 513, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 496, 499, 528, 524, 541, 498, 513, 509, 532, 517, 497, 485, 519, 479, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 491, 465, 507, 499, 517, 505, 504, 520, 476, 518, 484, 504, 476, 536, 553, 23, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 506, 506, 490, 512, 537, 501, 527, 517, 499, 513, 492, 519, 530, 495, 496, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 523, 489, 480, 502, 486, 498, 507, 550, 535, 507, 486, 513, 516, 494, 545, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 501, 494, 518, 532, 490, 507, 523, 498, 529, 501, 546, 504, 548, 547, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 527, 529, 539, 502, 501, 498, 526, 542, 489, 512, 545, 532, 556, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8134, 8240, 8083, 8165, 8140, 8205, 8365] avg=8192.0000 max=8365 max_rank=7 min=8083 min_rank=3 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[474, 484, 485, 550, 502, 504, 491, 514, 474, 534, 541, 501, 489, 498, 515, 546, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 541, 545, 503, 498, 528, 484, 559, 514, 509, 504, 506, 486, 537, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 507, 524, 498, 504, 497, 464, 501, 525, 509, 513, 504, 504, 484, 486, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 513, 506, 495, 496, 535, 508, 561, 559, 505, 515, 509, 520, 488, 481, 481, 22, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 525, 528, 513, 545, 487, 502, 490, 488, 509, 485, 533, 487, 541, 492, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 517, 488, 515, 487, 505, 548, 553, 498, 542, 512, 524, 490, 533, 561, 482, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 472, 515, 510, 530, 518, 561, 517, 491, 524, 516, 547, 498, 513, 534, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 487, 521, 533, 493, 510, 483, 489, 501, 510, 525, 503, 515, 529, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8278, 8050, 8193, 8128, 8280, 8316, 8127] avg=8192.0000 max=8316 max_rank=6 min=8050 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4352, 3968, 4096, 4096, 4224, 4224, 4096, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[489, 482, 556, 516, 508, 513, 472, 502, 509, 499, 535, 547, 503, 488, 534, 524, 535, 514, 497, 528, 535, 506, 510, 511, 488, 477, 504, 525, 510, 546, 539, 542, 484, 533, 486, 514, 498, 474, 502, 521, 512, 487, 533, 533, 532, 528, 510, 503, 516, 540, 520, 489, 532, 562, 533, 506, 519, 489, 507, 503, 489, 500, 495, 518, 520, 497, 553, 552, 504, 527, 503, 562, 539, 529, 480, 493, 500, 537, 514, 466, 510, 506, 499, 488, 511, 535, 500, 482, 487, 498, 523, 506, 523, 498, 527, 506, 504, 503, 493, 490, 534, 510, 477, 486, 532, 481, 528, 498, 514, 463, 521, 520, 528, 545, 488, 495, 511, 505, 521, 536, 476, 550, 491, 529, 522, 539, 533, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[501, 560, 522, 482, 497, 543, 529, 488, 535, 516, 498, 522, 523, 497, 526, 507, 516, 502, 524, 524, 533, 489, 498, 480, 503, 486, 526, 527, 539, 521, 513, 506, 540, 495, 513, 476, 500, 480, 529, 505, 489, 528, 510, 512, 482, 488, 528, 525, 559, 520, 546, 530, 508, 533, 517, 568, 506, 459, 509, 512, 511, 504, 514, 540, 506, 553, 500, 513, 517, 533, 509, 482, 524, 513, 501, 488, 538, 507, 494, 525, 485, 498, 523, 518, 495, 534, 534, 531, 556, 481, 514, 519, 503, 517, 497, 507, 515, 502, 496, 516, 536, 511, 483, 484, 500, 481, 564, 503, 542, 482, 515, 509, 527, 492, 520, 502, 511, 494, 488, 512, 549, 508, 487, 508, 489, 516, 505, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[515, 510, 501, 477, 496, 524, 529, 514, 537, 543, 480, 496, 465, 512, 527, 490, 517, 567, 498, 534, 532, 519, 501, 509, 519, 507, 502, 513, 503, 514, 513, 489, 489, 508, 504, 478, 531, 505, 502, 562, 546, 510, 466, 512, 529, 529, 482, 494, 507, 513, 489, 449, 558, 515, 522, 502, 507, 502, 473, 494, 504, 525, 555, 524, 514, 520, 506, 555, 518, 522, 501, 533, 509, 542, 495, 530, 538, 516, 523, 526, 542, 503, 488, 497, 514, 481, 529, 512, 546, 528, 519, 505, 487, 564, 484, 532, 478, 503, 518, 515, 523, 508, 550, 507, 503, 512, 479, 528, 513, 503, 487, 514, 515, 507, 484, 536, 489, 529, 501, 513, 539, 509, 498, 519, 546, 517, 499, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[510, 478, 544, 529, 505, 494, 516, 529, 492, 508, 474, 498, 553, 498, 509, 517, 506, 503, 550, 515, 519, 489, 482, 511, 523, 510, 523, 515, 549, 512, 504, 526, 500, 540, 538, 501, 491, 474, 491, 488, 520, 522, 488, 538, 511, 545, 480, 519, 499, 522, 489, 495, 489, 509, 477, 510, 523, 544, 488, 477, 514, 554, 489, 506, 517, 532, 526, 554, 499, 498, 532, 489, 519, 485, 492, 449, 518, 560, 495, 491, 533, 477, 520, 519, 502, 524, 482, 491, 498, 544, 519, 532, 543, 523, 513, 520, 525, 529, 535, 510, 533, 536, 474, 534, 538, 550, 467, 524, 501, 497, 540, 543, 486, 493, 512, 546, 512, 507, 555, 519, 500, 486, 485, 550, 511, 497, 510, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[510, 482, 493, 490, 541, 508, 518, 563, 512, 516, 531, 518, 514, 505, 522, 516, 510, 486, 499, 523, 521, 520, 546, 504, 502, 482, 516, 535, 505, 522, 501, 515, 554, 487, 521, 547, 512, 518, 490, 537, 523, 519, 503, 503, 488, 450, 483, 556, 506, 526, 521, 523, 516, 509, 506, 517, 506, 492, 511, 491, 551, 495, 501, 494, 488, 520, 547, 508, 515, 541, 539, 543, 505, 553, 502, 510, 519, 495, 533, 499, 494, 505, 496, 540, 518, 474, 504, 504, 508, 484, 509, 519, 528, 538, 495, 477, 490, 535, 481, 488, 512, 497, 522, 504, 533, 510, 499, 521, 491, 550, 517, 481, 515, 535, 500, 461, 526, 499, 517, 541, 521, 516, 512, 526, 481, 485, 564, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[539, 483, 501, 538, 510, 534, 506, 454, 539, 536, 520, 529, 526, 532, 527, 508, 555, 507, 520, 525, 507, 534, 530, 536, 477, 497, 494, 534, 493, 508, 525, 505, 529, 507, 533, 525, 507, 525, 533, 554, 497, 506, 486, 537, 480, 512, 489, 469, 505, 470, 532, 531, 510, 500, 482, 536, 523, 490, 500, 532, 535, 509, 491, 531, 549, 541, 514, 500, 528, 502, 502, 535, 471, 537, 550, 496, 521, 528, 442, 512, 521, 489, 516, 528, 559, 503, 465, 494, 528, 509, 480, 561, 479, 516, 500, 522, 489, 524, 512, 509, 498, 489, 523, 522, 556, 506, 502, 544, 496, 484, 508, 511, 479, 509, 507, 534, 535, 516, 513, 483, 526, 508, 533, 520, 473, 450, 491, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[498, 536, 522, 552, 497, 547, 528, 473, 515, 511, 524, 495, 495, 540, 513, 530, 505, 538, 518, 485, 556, 494, 432, 529, 501, 564, 520, 537, 502, 525, 488, 481, 495, 497, 522, 513, 540, 514, 525, 500, 480, 522, 520, 540, 514, 552, 517, 529, 508, 517, 522, 505, 504, 481, 543, 510, 487, 538, 487, 481, 503, 538, 533, 528, 539, 548, 548, 491, 553, 506, 529, 498, 510, 485, 484, 554, 502, 503, 462, 528, 479, 550, 512, 485, 525, 500, 494, 505, 505, 517, 521, 511, 515, 493, 517, 493, 526, 543, 486, 511, 529, 526, 520, 493, 501, 501, 469, 549, 511, 495, 494, 480, 524, 549, 526, 484, 522, 509, 553, 458, 505, 488, 488, 513, 511, 501, 489, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[522, 526, 513, 517, 486, 511, 503, 536, 516, 543, 502, 492, 498, 557, 480, 519, 498, 486, 517, 553, 516, 517, 521, 503, 480, 488, 506, 555, 549, 487, 453, 492, 549, 553, 499, 513, 456, 513, 538, 483, 512, 523, 517, 518, 531, 506, 508, 505, 488, 465, 523, 532, 525, 492, 524, 514, 520, 499, 483, 519, 528, 516, 518, 495, 491, 532, 507, 510, 561, 533, 506, 504, 484, 534, 533, 511, 484, 512, 516, 555, 526, 540, 491, 523, 498, 546, 536, 492, 501, 511, 500, 518, 533, 504, 490, 534, 523, 512, 498, 501, 515, 504, 473, 527, 523, 496, 543, 510, 510, 512, 496, 489, 498, 530, 492, 525, 491, 478, 524, 523, 492, 540, 531, 504, 506, 515, 538, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[489, 482, 556, 516, 508, 492, 472, 502, 509, 499, 535, 547, 503, 488, 534, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 514, 497, 528, 535, 506, 510, 511, 488, 477, 504, 511, 510, 546, 539, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 533, 486, 514, 498, 474, 502, 521, 512, 487, 533, 533, 532, 528, 510, 503, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 540, 520, 489, 532, 562, 533, 506, 519, 489, 507, 503, 489, 500, 495, 518, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 439, 553, 552, 504, 527, 503, 562, 539, 529, 480, 493, 500, 537, 514, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 499, 488, 511, 535, 500, 482, 487, 498, 523, 506, 523, 498, 527, 506, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 493, 490, 534, 510, 477, 486, 532, 481, 528, 498, 514, 463, 521, 520, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 545, 488, 495, 511, 505, 521, 536, 476, 550, 491, 529, 522, 539, 533, 526, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8253, 8162, 8229, 8218, 8114, 8088, 8316] avg=8192.0000 max=8316 max_rank=7 min=8088 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 478, 544, 529, 505, 474, 516, 529, 492, 508, 474, 498, 553, 498, 509, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 503, 550, 515, 519, 489, 482, 511, 523, 510, 523, 502, 549, 512, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 540, 538, 501, 491, 474, 491, 488, 520, 522, 488, 538, 511, 545, 480, 519, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 522, 489, 495, 489, 509, 477, 510, 523, 544, 488, 477, 514, 554, 489, 506, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 470, 526, 554, 499, 498, 532, 489, 519, 485, 492, 449, 518, 560, 495, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 477, 520, 519, 502, 524, 482, 491, 498, 544, 519, 532, 543, 523, 513, 520, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 529, 535, 510, 533, 536, 474, 534, 538, 550, 467, 524, 501, 497, 540, 543, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 493, 512, 546, 512, 507, 555, 519, 500, 486, 485, 550, 511, 497, 510, 513, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8224, 8158, 8095, 8094, 8254, 8373, 8204] avg=8192.0000 max=8373 max_rank=6 min=8094 min_rank=4 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32925 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[539, 483, 501, 538, 510, 505, 506, 454, 539, 536, 520, 529, 526, 532, 527, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 507, 520, 525, 507, 534, 530, 536, 477, 497, 494, 511, 493, 508, 525, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 533, 525, 507, 525, 533, 554, 497, 506, 486, 537, 480, 512, 489, 469, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 470, 532, 531, 510, 500, 482, 536, 523, 490, 500, 532, 535, 509, 491, 531, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 468, 514, 500, 528, 502, 502, 535, 471, 537, 550, 496, 521, 528, 442, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 489, 516, 528, 559, 503, 465, 494, 528, 509, 480, 561, 479, 516, 500, 522, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 524, 512, 509, 498, 489, 523, 522, 556, 506, 502, 544, 496, 484, 508, 511, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 509, 507, 534, 535, 516, 513, 483, 526, 508, 533, 520, 473, 450, 491, 493, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8224, 8209, 8192, 8155, 8194, 8213, 8096] avg=8192.0000 max=8253 max_rank=0 min=8096 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 482, 493, 490, 541, 488, 518, 563, 512, 516, 531, 518, 514, 505, 522, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 486, 499, 523, 521, 520, 546, 504, 502, 482, 516, 520, 505, 522, 501, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 487, 521, 547, 512, 518, 490, 537, 523, 519, 503, 503, 488, 450, 483, 556, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 526, 521, 523, 516, 509, 506, 517, 506, 492, 511, 491, 551, 495, 501, 494, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 459, 547, 508, 515, 541, 539, 543, 505, 553, 502, 510, 519, 495, 533, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 505, 496, 540, 518, 474, 504, 504, 508, 484, 509, 519, 528, 538, 495, 477, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 535, 481, 488, 512, 497, 522, 504, 533, 510, 499, 521, 491, 550, 517, 481, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 535, 500, 461, 526, 499, 517, 541, 521, 516, 512, 526, 481, 485, 564, 514, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8172, 8204, 8176, 8256, 8107, 8167, 8235] avg=8192.0000 max=8256 max_rank=4 min=8107 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 510, 501, 477, 496, 503, 529, 514, 537, 543, 480, 496, 465, 512, 527, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 567, 498, 534, 532, 519, 501, 509, 519, 507, 502, 500, 503, 514, 513, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 508, 504, 478, 531, 505, 502, 562, 546, 510, 466, 512, 529, 529, 482, 494, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 513, 489, 449, 558, 515, 522, 502, 507, 502, 473, 494, 504, 525, 555, 524, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 459, 506, 555, 518, 522, 501, 533, 509, 542, 495, 530, 538, 516, 523, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 503, 488, 497, 514, 481, 529, 512, 546, 528, 519, 505, 487, 564, 484, 532, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 503, 518, 515, 523, 508, 550, 507, 503, 512, 479, 528, 513, 503, 487, 514, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 507, 484, 536, 489, 529, 501, 513, 539, 509, 498, 519, 546, 517, 499, 476, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8224, 8159, 8149, 8287, 8246, 8177, 8199] avg=8192.0000 max=8287 max_rank=4 min=8095 min_rank=0 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[498, 536, 522, 552, 497, 525, 528, 473, 515, 511, 524, 495, 495, 540, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 538, 518, 485, 556, 494, 432, 529, 501, 564, 520, 522, 502, 525, 488, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 522, 513, 540, 514, 525, 500, 480, 522, 520, 540, 514, 552, 517, 529, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 522, 505, 504, 481, 543, 510, 487, 538, 487, 481, 503, 538, 533, 528, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 484, 548, 491, 553, 506, 529, 498, 510, 485, 484, 554, 502, 503, 462, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 550, 512, 485, 525, 500, 494, 505, 505, 517, 521, 511, 515, 493, 517, 493, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 543, 486, 511, 529, 526, 520, 493, 501, 501, 469, 549, 511, 495, 494, 480, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 549, 526, 484, 522, 509, 553, 458, 505, 488, 488, 513, 511, 501, 489, 504, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8160, 8294, 8196, 8176, 8137, 8172, 8147] avg=8192.0000 max=8294 max_rank=2 min=8137 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[501, 560, 522, 482, 497, 521, 529, 488, 535, 516, 498, 522, 523, 497, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 524, 524, 533, 489, 498, 480, 503, 486, 526, 513, 539, 521, 513, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 495, 513, 476, 500, 480, 529, 505, 489, 528, 510, 512, 482, 488, 528, 525, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 520, 546, 530, 508, 533, 517, 568, 506, 459, 509, 512, 511, 504, 514, 540, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 489, 500, 513, 517, 533, 509, 482, 524, 513, 501, 488, 538, 507, 494, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 498, 523, 518, 495, 534, 534, 531, 556, 481, 514, 519, 503, 517, 497, 507, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 502, 496, 516, 536, 511, 483, 484, 500, 481, 564, 503, 542, 482, 515, 509, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 492, 520, 502, 511, 494, 488, 512, 549, 508, 487, 508, 489, 516, 505, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8173, 8113, 8347, 8139, 8227, 8177, 8136] avg=8192.0000 max=8347 max_rank=3 min=8113 min_rank=2 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 526, 513, 517, 486, 491, 503, 536, 516, 543, 502, 492, 498, 557, 480, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 486, 517, 553, 516, 517, 521, 503, 480, 488, 506, 540, 549, 487, 453, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 553, 499, 513, 456, 513, 538, 483, 512, 523, 517, 518, 531, 506, 508, 505, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 465, 523, 532, 525, 492, 524, 514, 520, 499, 483, 519, 528, 516, 518, 495, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 470, 507, 510, 561, 533, 506, 504, 484, 534, 533, 511, 484, 512, 516, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 540, 491, 523, 498, 546, 536, 492, 501, 511, 500, 518, 533, 504, 490, 534, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 512, 498, 501, 515, 504, 473, 527, 523, 496, 543, 510, 510, 512, 496, 489, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 530, 492, 525, 491, 478, 524, 523, 492, 540, 531, 504, 506, 515, 538, 494, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8106, 8237, 8152, 8211, 8257, 8169, 8203] avg=8192.0000 max=8257 max_rank=5 min=8106 min_rank=1 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3840, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[521, 490, 562, 497, 534, 477, 507, 476, 532, 505, 498, 520, 528, 518, 546, 504, 500, 504, 495, 527, 579, 475, 467, 502, 541, 494, 540, 530, 481, 532, 479, 526, 535, 544, 489, 524, 489, 535, 513, 491, 525, 540, 512, 489, 492, 492, 496, 519, 542, 497, 554, 519, 504, 555, 498, 525, 515, 502, 515, 516, 545, 493, 503, 531, 500, 507, 499, 471, 538, 544, 547, 506, 445, 495, 526, 482, 554, 476, 549, 522, 473, 505, 512, 496, 492, 523, 496, 528, 524, 511, 536, 508, 475, 507, 548, 489, 517, 468, 518, 507, 478, 472, 552, 494, 542, 497, 482, 495, 507, 548, 508, 497, 520, 507, 567, 520, 542, 512, 509, 526, 529, 502, 499, 510, 488, 501, 517, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[477, 513, 510, 525, 482, 487, 550, 528, 502, 487, 487, 534, 471, 567, 512, 496, 525, 495, 539, 488, 493, 516, 527, 469, 500, 496, 528, 520, 508, 496, 521, 493, 496, 494, 545, 509, 551, 528, 526, 516, 504, 454, 524, 485, 543, 496, 501, 504, 548, 520, 567, 542, 487, 502, 534, 508, 487, 530, 492, 530, 506, 516, 510, 474, 529, 491, 534, 512, 515, 521, 552, 549, 518, 487, 481, 534, 521, 517, 495, 502, 519, 555, 499, 506, 517, 527, 540, 518, 499, 515, 521, 489, 466, 534, 529, 481, 513, 546, 529, 494, 488, 529, 472, 502, 509, 503, 501, 491, 527, 522, 538, 505, 506, 518, 512, 533, 517, 475, 524, 504, 507, 547, 523, 490, 531, 509, 488, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[520, 563, 485, 485, 555, 511, 491, 483, 489, 522, 507, 523, 521, 516, 489, 498, 503, 509, 525, 545, 523, 492, 533, 431, 484, 494, 483, 548, 537, 492, 562, 481, 522, 473, 507, 515, 517, 525, 519, 487, 558, 493, 549, 526, 498, 564, 528, 473, 508, 496, 534, 518, 489, 557, 480, 517, 510, 524, 517, 514, 514, 501, 546, 491, 524, 494, 461, 525, 474, 529, 469, 514, 536, 532, 514, 505, 539, 523, 512, 514, 489, 505, 473, 525, 473, 504, 480, 524, 557, 542, 525, 510, 508, 506, 486, 513, 496, 510, 526, 490, 506, 532, 478, 502, 528, 491, 505, 506, 499, 530, 519, 537, 494, 544, 531, 535, 519, 496, 492, 539, 527, 546, 512, 503, 513, 532, 517, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[548, 508, 529, 480, 549, 520, 516, 516, 534, 485, 497, 493, 503, 519, 521, 520, 473, 482, 516, 510, 477, 516, 529, 516, 501, 492, 522, 541, 535, 550, 517, 503, 463, 516, 531, 529, 535, 494, 528, 476, 509, 520, 501, 528, 504, 526, 505, 519, 513, 512, 511, 536, 493, 497, 531, 549, 477, 474, 534, 512, 498, 515, 492, 484, 461, 505, 461, 508, 546, 480, 551, 499, 509, 505, 560, 478, 480, 477, 506, 534, 508, 517, 494, 537, 522, 512, 546, 514, 485, 517, 522, 503, 494, 469, 513, 475, 534, 519, 480, 509, 587, 481, 545, 488, 550, 512, 501, 516, 520, 511, 488, 523, 530, 570, 496, 494, 508, 503, 521, 503, 504, 489, 577, 501, 557, 549, 519, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[530, 466, 487, 474, 499, 529, 498, 479, 520, 530, 502, 498, 555, 514, 495, 522, 498, 494, 483, 492, 505, 498, 516, 509, 504, 511, 496, 543, 548, 540, 505, 541, 456, 517, 517, 527, 494, 520, 520, 514, 547, 530, 478, 521, 511, 503, 478, 504, 490, 519, 517, 515, 543, 510, 510, 512, 485, 501, 547, 516, 511, 542, 535, 495, 528, 513, 527, 498, 462, 531, 529, 514, 516, 486, 510, 535, 537, 539, 528, 526, 523, 526, 504, 499, 527, 557, 544, 550, 522, 500, 470, 508, 523, 458, 541, 529, 539, 509, 496, 576, 493, 523, 528, 503, 516, 515, 493, 517, 544, 530, 521, 507, 519, 543, 513, 482, 484, 501, 493, 472, 510, 507, 526, 486, 512, 445, 509, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[522, 513, 470, 479, 502, 502, 504, 494, 502, 513, 498, 523, 519, 524, 494, 476, 488, 503, 546, 516, 513, 494, 549, 531, 528, 513, 477, 557, 569, 499, 502, 481, 507, 557, 513, 484, 526, 459, 521, 522, 534, 505, 498, 482, 543, 490, 517, 536, 502, 516, 560, 502, 511, 556, 547, 515, 511, 528, 483, 499, 548, 483, 532, 518, 496, 495, 526, 542, 507, 513, 477, 499, 490, 563, 539, 502, 565, 514, 496, 551, 499, 473, 526, 485, 506, 504, 510, 472, 528, 524, 491, 505, 523, 524, 544, 513, 503, 487, 519, 475, 498, 503, 539, 503, 512, 513, 498, 533, 514, 503, 520, 529, 490, 561, 516, 528, 476, 513, 494, 526, 479, 522, 497, 530, 516, 530, 473, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[518, 493, 503, 519, 517, 539, 510, 496, 500, 515, 522, 495, 554, 520, 482, 488, 491, 505, 486, 553, 551, 466, 546, 493, 493, 530, 485, 512, 512, 546, 521, 495, 528, 513, 474, 525, 517, 537, 517, 525, 520, 550, 506, 525, 478, 497, 488, 554, 531, 469, 510, 485, 513, 523, 503, 529, 499, 536, 532, 517, 492, 524, 526, 507, 499, 537, 495, 531, 532, 533, 524, 541, 510, 517, 469, 498, 530, 482, 516, 497, 546, 498, 485, 516, 490, 496, 536, 474, 538, 535, 495, 454, 496, 518, 475, 527, 568, 509, 506, 537, 510, 515, 502, 513, 494, 504, 514, 501, 480, 504, 553, 516, 501, 520, 510, 521, 500, 507, 521, 515, 520, 538, 566, 502, 475, 504, 493, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[516, 503, 496, 498, 533, 484, 528, 547, 519, 489, 516, 537, 500, 528, 528, 497, 515, 493, 524, 524, 519, 560, 516, 485, 497, 500, 515, 525, 510, 498, 479, 493, 560, 514, 514, 517, 517, 469, 528, 468, 515, 505, 539, 532, 512, 541, 536, 454, 512, 520, 505, 532, 510, 475, 519, 477, 472, 520, 512, 546, 516, 549, 507, 527, 462, 495, 555, 558, 493, 512, 512, 475, 502, 501, 560, 486, 533, 529, 515, 507, 511, 513, 525, 524, 483, 502, 509, 503, 544, 528, 502, 508, 491, 494, 521, 538, 509, 527, 498, 525, 485, 489, 532, 507, 545, 515, 478, 541, 521, 476, 538, 514, 507, 485, 541, 520, 496, 509, 497, 496, 501, 527, 511, 515, 498, 534, 516, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[521, 490, 562, 497, 534, 477, 507, 476, 532, 505, 498, 520, 528, 518, 546, 504, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 504, 495, 527, 579, 475, 467, 502, 541, 494, 540, 530, 481, 532, 479, 526, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 544, 489, 524, 489, 535, 513, 491, 518, 540, 512, 489, 492, 492, 496, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 497, 512, 519, 504, 555, 498, 525, 515, 502, 515, 516, 545, 493, 503, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 507, 499, 471, 538, 544, 547, 506, 445, 495, 526, 482, 546, 476, 549, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 505, 512, 496, 492, 523, 496, 528, 524, 511, 536, 508, 475, 507, 548, 489, 23, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 468, 518, 507, 478, 472, 552, 494, 540, 497, 482, 495, 507, 548, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 484, 567, 520, 542, 512, 509, 526, 529, 502, 499, 510, 488, 501, 517, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8188, 8178, 8272, 8153, 8154, 8080, 8261] avg=8192.0000 max=8272 max_rank=3 min=8080 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[477, 513, 510, 525, 482, 487, 550, 528, 502, 487, 487, 534, 471, 567, 512, 496, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 495, 539, 488, 493, 516, 527, 469, 500, 496, 528, 520, 508, 496, 521, 493, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 545, 509, 551, 528, 526, 516, 497, 454, 524, 485, 543, 496, 501, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 520, 524, 542, 487, 502, 534, 508, 487, 530, 492, 530, 506, 516, 510, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 491, 534, 512, 515, 521, 552, 549, 518, 487, 481, 534, 513, 517, 495, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 555, 499, 506, 517, 527, 540, 518, 499, 515, 521, 489, 466, 534, 529, 481, 24, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 546, 529, 494, 488, 529, 472, 502, 507, 503, 501, 491, 527, 522, 538, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 494, 512, 533, 517, 475, 524, 504, 507, 547, 523, 490, 531, 509, 488, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8130, 8169, 8210, 8250, 8247, 8167, 8199] avg=8192.0000 max=8250 max_rank=4 min=8130 min_rank=1 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32863 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[530, 466, 487, 474, 499, 529, 498, 479, 520, 530, 502, 498, 555, 514, 495, 522, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 483, 492, 505, 498, 516, 509, 504, 511, 496, 543, 548, 540, 505, 541, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 517, 517, 527, 494, 520, 520, 514, 539, 530, 478, 521, 511, 503, 478, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, 478, 515, 543, 510, 510, 512, 485, 501, 547, 516, 511, 542, 535, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 527, 498, 462, 531, 529, 514, 516, 486, 510, 535, 529, 539, 528, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 504, 499, 527, 557, 544, 550, 522, 500, 470, 508, 523, 458, 541, 529, 25, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 509, 496, 576, 493, 523, 528, 503, 514, 515, 493, 517, 544, 530, 521, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 513, 482, 484, 501, 493, 472, 510, 507, 526, 486, 512, 445, 509, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8200, 8129, 8209, 8271, 8313, 8308, 7975] avg=8192.0000 max=8313 max_rank=5 min=7975 min_rank=7 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 513, 470, 479, 502, 502, 504, 494, 502, 513, 498, 523, 519, 524, 494, 476, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 503, 546, 516, 513, 494, 549, 531, 528, 513, 477, 557, 569, 499, 502, 481, 13, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 557, 513, 484, 526, 459, 521, 522, 516, 505, 498, 482, 543, 490, 517, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 516, 509, 502, 511, 556, 547, 515, 511, 528, 483, 499, 548, 483, 532, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 526, 542, 507, 513, 477, 499, 490, 563, 539, 502, 552, 514, 496, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 473, 526, 485, 506, 504, 510, 472, 528, 524, 491, 505, 523, 524, 544, 513, 28, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 487, 519, 475, 498, 503, 539, 503, 507, 513, 498, 533, 514, 503, 520, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 533, 516, 528, 476, 513, 494, 526, 479, 522, 497, 530, 516, 530, 473, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8294, 8176, 8260, 8262, 8174, 8144, 8151] avg=8192.0000 max=8294 max_rank=1 min=8075 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 563, 485, 485, 555, 511, 491, 483, 489, 522, 507, 523, 521, 516, 489, 498, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 509, 525, 545, 523, 492, 533, 431, 484, 494, 483, 548, 537, 492, 562, 481, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 473, 507, 515, 517, 525, 519, 487, 550, 493, 549, 526, 498, 564, 528, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 494, 518, 489, 557, 480, 517, 510, 524, 517, 514, 514, 501, 546, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 494, 461, 525, 474, 529, 469, 514, 536, 532, 514, 505, 531, 523, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 505, 473, 525, 473, 504, 480, 524, 557, 542, 525, 510, 508, 506, 486, 513, 25, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 526, 490, 506, 532, 478, 502, 526, 491, 505, 506, 499, 530, 519, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 519, 531, 535, 519, 496, 492, 539, 527, 546, 512, 503, 513, 532, 517, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8159, 8246, 8176, 8157, 8152, 8153, 8301] avg=8192.0000 max=8301 max_rank=7 min=8152 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 503, 496, 498, 533, 484, 528, 547, 519, 489, 516, 537, 500, 528, 528, 497, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 493, 524, 524, 519, 560, 516, 485, 497, 500, 515, 525, 510, 498, 479, 493, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 514, 514, 517, 517, 469, 528, 468, 508, 505, 539, 532, 512, 541, 536, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 520, 467, 532, 510, 475, 519, 477, 472, 520, 512, 546, 516, 549, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 495, 555, 558, 493, 512, 512, 475, 502, 501, 560, 486, 525, 529, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 525, 524, 483, 502, 509, 503, 544, 528, 502, 508, 491, 494, 521, 538, 22, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 498, 525, 485, 489, 532, 507, 543, 515, 478, 541, 521, 476, 538, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 463, 541, 520, 496, 509, 497, 496, 501, 527, 511, 515, 498, 534, 516, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8169, 8214, 8161, 8187, 8225, 8198, 8131] avg=8192.0000 max=8251 max_rank=0 min=8131 min_rank=7 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 493, 503, 519, 517, 539, 510, 496, 500, 515, 522, 495, 554, 520, 482, 488, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 505, 486, 553, 551, 466, 546, 493, 493, 530, 485, 512, 512, 546, 521, 495, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 474, 525, 517, 537, 517, 525, 513, 550, 506, 525, 478, 497, 488, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 469, 472, 485, 513, 523, 503, 529, 499, 536, 532, 517, 492, 524, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 537, 495, 531, 532, 533, 524, 541, 510, 517, 469, 498, 522, 482, 516, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 498, 485, 516, 490, 496, 536, 474, 538, 535, 495, 454, 496, 518, 475, 527, 24, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 509, 506, 537, 510, 515, 502, 513, 492, 504, 514, 501, 480, 504, 553, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 496, 510, 521, 500, 507, 521, 515, 520, 538, 566, 502, 475, 504, 493, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8201, 8247, 8158, 8203, 8110, 8224, 8190] avg=8192.0000 max=8247 max_rank=2 min=8110 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[548, 508, 529, 480, 549, 520, 516, 516, 534, 485, 497, 493, 503, 519, 521, 520, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 482, 516, 510, 477, 516, 529, 516, 501, 492, 522, 541, 535, 550, 517, 503, 7, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 516, 531, 529, 535, 494, 528, 476, 502, 520, 501, 528, 504, 526, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 512, 473, 536, 493, 497, 531, 549, 477, 474, 534, 512, 498, 515, 492, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 505, 461, 508, 546, 480, 551, 499, 509, 505, 560, 478, 473, 477, 506, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 494, 537, 522, 512, 546, 514, 485, 517, 522, 503, 494, 469, 513, 475, 26, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 519, 480, 509, 587, 481, 545, 488, 548, 512, 501, 516, 520, 511, 488, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 544, 496, 494, 508, 503, 521, 503, 504, 489, 577, 501, 557, 549, 519, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8195, 8177, 8090, 8053, 8161, 8262, 8328] avg=8192.0000 max=8328 max_rank=7 min=8053 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4352, 4224, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[521, 495, 511, 461, 525, 535, 486, 526, 525, 518, 474, 489, 481, 533, 522, 552, 539, 523, 518, 496, 508, 513, 510, 481, 545, 522, 512, 495, 513, 537, 524, 513, 517, 506, 500, 486, 479, 500, 521, 523, 496, 471, 509, 538, 488, 495, 521, 477, 522, 520, 506, 532, 488, 531, 492, 493, 502, 535, 473, 577, 474, 510, 526, 545, 508, 495, 520, 528, 475, 520, 500, 493, 532, 530, 560, 513, 528, 519, 525, 511, 511, 558, 523, 505, 526, 543, 499, 495, 511, 493, 544, 550, 504, 515, 521, 534, 510, 471, 555, 490, 489, 500, 510, 499, 490, 521, 521, 523, 468, 500, 479, 539, 506, 499, 542, 542, 514, 525, 481, 542, 504, 544, 502, 507, 501, 533, 476, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[474, 532, 539, 552, 544, 503, 506, 532, 498, 554, 533, 479, 498, 536, 556, 490, 497, 540, 468, 456, 520, 505, 489, 518, 520, 468, 524, 501, 495, 511, 492, 519, 523, 534, 508, 479, 486, 495, 506, 491, 485, 471, 490, 523, 515, 508, 527, 519, 516, 493, 531, 515, 546, 502, 509, 502, 492, 531, 571, 512, 519, 489, 514, 485, 536, 489, 490, 493, 542, 534, 523, 502, 501, 513, 535, 511, 535, 488, 473, 496, 514, 492, 482, 517, 514, 512, 550, 520, 539, 541, 511, 548, 500, 480, 533, 526, 528, 521, 481, 521, 522, 493, 512, 517, 523, 504, 540, 515, 483, 524, 524, 498, 526, 501, 531, 527, 521, 484, 504, 537, 529, 511, 503, 523, 507, 526, 520, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[538, 513, 515, 502, 533, 530, 526, 489, 509, 496, 538, 495, 498, 485, 505, 513, 538, 521, 500, 556, 470, 482, 517, 461, 497, 491, 541, 540, 532, 534, 526, 525, 502, 511, 522, 530, 502, 488, 473, 533, 528, 486, 516, 494, 510, 475, 505, 484, 531, 495, 505, 508, 499, 518, 517, 509, 517, 498, 567, 480, 532, 496, 523, 513, 548, 565, 502, 512, 491, 540, 514, 548, 491, 538, 493, 472, 498, 497, 554, 512, 526, 493, 461, 535, 520, 541, 546, 509, 547, 519, 530, 479, 507, 511, 523, 486, 495, 510, 504, 522, 535, 503, 521, 558, 509, 514, 546, 501, 480, 516, 487, 469, 495, 532, 488, 542, 545, 505, 511, 494, 512, 493, 503, 499, 525, 512, 495, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[533, 549, 535, 516, 499, 517, 508, 513, 542, 530, 521, 513, 516, 483, 555, 498, 504, 495, 500, 505, 524, 517, 540, 525, 533, 538, 516, 519, 532, 514, 496, 494, 534, 510, 515, 513, 511, 529, 545, 541, 510, 477, 494, 526, 504, 510, 531, 490, 547, 527, 538, 487, 490, 495, 513, 514, 562, 498, 494, 509, 538, 486, 499, 490, 581, 523, 464, 500, 471, 509, 502, 522, 523, 533, 491, 468, 492, 532, 508, 476, 504, 532, 505, 478, 496, 541, 482, 456, 449, 512, 483, 490, 537, 513, 484, 503, 479, 513, 461, 531, 522, 543, 521, 543, 549, 488, 485, 506, 539, 545, 531, 516, 532, 519, 529, 508, 490, 520, 517, 493, 551, 517, 526, 499, 497, 501, 476, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[523, 496, 473, 481, 483, 485, 528, 473, 507, 510, 485, 511, 512, 488, 518, 446, 515, 511, 524, 493, 521, 497, 509, 466, 485, 540, 498, 521, 508, 551, 531, 488, 507, 534, 469, 522, 519, 497, 493, 536, 537, 528, 512, 511, 515, 511, 527, 489, 532, 525, 523, 511, 546, 509, 518, 497, 503, 522, 495, 517, 503, 535, 525, 507, 491, 513, 518, 490, 526, 517, 503, 498, 501, 520, 530, 528, 538, 539, 557, 463, 535, 498, 505, 523, 506, 517, 507, 521, 458, 479, 561, 510, 532, 522, 477, 506, 522, 496, 526, 519, 509, 532, 509, 521, 504, 517, 522, 542, 514, 512, 494, 529, 502, 532, 520, 525, 530, 515, 532, 530, 505, 554, 568, 505, 460, 488, 554, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[540, 510, 474, 457, 463, 532, 522, 456, 516, 515, 516, 518, 551, 487, 525, 515, 483, 543, 505, 573, 523, 511, 517, 508, 527, 533, 527, 515, 484, 518, 462, 534, 484, 495, 524, 481, 517, 540, 523, 502, 518, 502, 522, 509, 520, 533, 508, 506, 521, 544, 505, 519, 555, 543, 553, 519, 514, 499, 503, 547, 518, 493, 547, 481, 511, 515, 533, 543, 501, 540, 488, 445, 499, 534, 465, 492, 500, 489, 474, 494, 491, 552, 515, 530, 541, 497, 511, 484, 489, 505, 519, 501, 509, 486, 520, 534, 511, 526, 497, 495, 519, 544, 507, 529, 482, 504, 522, 495, 535, 496, 531, 502, 545, 503, 520, 503, 480, 500, 523, 539, 521, 523, 532, 477, 566, 496, 487, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[517, 513, 516, 509, 556, 507, 489, 488, 515, 463, 488, 529, 512, 493, 515, 517, 511, 505, 477, 515, 504, 531, 514, 528, 468, 525, 538, 499, 508, 499, 513, 502, 511, 524, 491, 515, 509, 480, 492, 529, 497, 500, 478, 493, 505, 507, 490, 516, 527, 467, 512, 467, 503, 517, 569, 496, 494, 478, 519, 512, 496, 518, 520, 554, 490, 483, 533, 538, 502, 544, 511, 520, 548, 537, 518, 520, 533, 506, 546, 479, 507, 476, 498, 522, 534, 561, 497, 517, 522, 455, 473, 514, 498, 554, 507, 507, 505, 513, 534, 521, 512, 537, 535, 534, 522, 528, 518, 470, 503, 532, 552, 529, 548, 532, 491, 531, 496, 554, 505, 519, 515, 514, 499, 495, 519, 489, 534, 550] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[522, 502, 498, 546, 531, 490, 487, 484, 524, 530, 482, 518, 510, 471, 494, 487, 462, 508, 548, 507, 533, 521, 528, 505, 482, 494, 502, 530, 515, 539, 491, 511, 511, 528, 484, 522, 541, 488, 533, 513, 503, 536, 513, 517, 509, 472, 524, 526, 511, 523, 550, 514, 511, 523, 526, 514, 512, 533, 500, 553, 478, 521, 474, 490, 480, 523, 514, 492, 524, 489, 473, 567, 534, 500, 493, 539, 529, 523, 487, 516, 496, 513, 527, 533, 508, 513, 528, 527, 475, 496, 528, 515, 500, 539, 496, 507, 520, 519, 480, 509, 567, 494, 481, 494, 521, 542, 524, 490, 482, 514, 507, 508, 502, 579, 508, 504, 539, 526, 551, 519, 482, 513, 541, 539, 519, 455, 533, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[474, 532, 539, 552, 544, 503, 506, 532, 498, 554, 533, 479, 498, 536, 556, 490, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 540, 468, 456, 520, 505, 489, 518, 520, 468, 524, 501, 495, 511, 492, 519, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 534, 508, 479, 486, 495, 506, 491, 485, 471, 490, 523, 515, 508, 527, 519, 54, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 493, 531, 515, 546, 502, 509, 502, 492, 531, 571, 512, 519, 489, 514, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 489, 490, 493, 542, 534, 523, 502, 501, 513, 535, 511, 535, 488, 473, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 492, 482, 517, 514, 512, 550, 520, 539, 541, 511, 548, 500, 480, 533, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 481, 521, 522, 493, 512, 502, 523, 504, 540, 515, 483, 524, 524, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 447, 531, 527, 521, 484, 504, 537, 529, 511, 503, 523, 507, 526, 520, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 8028, 8119, 8188, 8161, 8284, 8191, 8200] avg=8192.0000 max=8365 max_rank=0 min=8028 min_rank=1 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 495, 511, 461, 525, 535, 486, 526, 525, 518, 474, 489, 481, 533, 522, 552, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 523, 518, 496, 508, 513, 510, 481, 545, 522, 512, 495, 513, 537, 524, 513, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 506, 500, 486, 479, 500, 521, 523, 496, 471, 509, 538, 488, 495, 521, 477, 54, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 520, 506, 532, 488, 531, 492, 493, 502, 535, 473, 577, 474, 510, 526, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 520, 528, 475, 520, 500, 493, 532, 530, 560, 513, 528, 519, 525, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 558, 523, 505, 526, 543, 499, 495, 511, 493, 544, 550, 504, 515, 521, 534, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 471, 555, 490, 489, 500, 510, 484, 490, 521, 521, 523, 468, 500, 479, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 445, 542, 542, 514, 525, 481, 542, 504, 544, 502, 507, 501, 533, 476, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8254, 8086, 8187, 8257, 8337, 8050, 8172] avg=8192.0000 max=8337 max_rank=5 min=8050 min_rank=6 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[540, 510, 474, 457, 463, 532, 522, 456, 516, 515, 516, 518, 551, 487, 525, 515, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 543, 505, 573, 523, 511, 517, 508, 527, 533, 527, 515, 484, 518, 462, 534, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 495, 524, 481, 517, 540, 523, 502, 518, 502, 522, 509, 520, 533, 508, 506, 54, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 544, 505, 519, 555, 543, 553, 519, 514, 499, 503, 547, 518, 493, 547, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 515, 533, 543, 501, 540, 488, 445, 499, 534, 465, 492, 500, 489, 474, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 552, 515, 530, 541, 497, 511, 484, 489, 505, 519, 501, 509, 486, 520, 534, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 526, 497, 495, 519, 544, 507, 514, 482, 504, 522, 495, 535, 496, 531, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 449, 520, 503, 480, 500, 523, 539, 521, 523, 532, 477, 566, 496, 487, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8268, 8243, 8322, 8023, 8189, 8180, 8175] avg=8192.0000 max=8322 max_rank=3 min=8023 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32567 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 496, 473, 481, 483, 485, 528, 473, 507, 510, 485, 511, 512, 488, 518, 446, 39, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 524, 493, 521, 497, 509, 466, 485, 540, 498, 521, 508, 551, 531, 488, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 534, 469, 522, 519, 497, 493, 536, 537, 528, 512, 511, 515, 511, 527, 489, 61, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 525, 523, 511, 546, 509, 518, 497, 503, 522, 495, 517, 503, 535, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 513, 518, 490, 526, 517, 503, 498, 501, 520, 530, 528, 538, 539, 557, 463, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 498, 505, 523, 506, 517, 507, 521, 458, 479, 561, 510, 532, 522, 477, 506, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 496, 526, 519, 509, 532, 509, 491, 504, 517, 522, 542, 514, 512, 494, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 471, 520, 525, 530, 515, 532, 530, 505, 554, 568, 505, 460, 488, 554, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7968, 8168, 8277, 8220, 8234, 8165, 8238, 8266] avg=8192.0000 max=8277 max_rank=2 min=7968 min_rank=0 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[538, 513, 515, 502, 533, 530, 526, 489, 509, 496, 538, 495, 498, 485, 505, 513, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 521, 500, 556, 470, 482, 517, 461, 497, 491, 541, 540, 532, 534, 526, 525, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 522, 530, 502, 488, 473, 533, 528, 486, 516, 494, 510, 475, 505, 484, 58, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 495, 505, 508, 499, 518, 517, 509, 517, 498, 567, 480, 532, 496, 523, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 565, 502, 512, 491, 540, 514, 548, 491, 538, 493, 472, 498, 497, 554, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 493, 461, 535, 520, 541, 546, 509, 547, 519, 530, 479, 507, 511, 523, 486, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 510, 504, 522, 535, 503, 521, 542, 509, 514, 546, 501, 480, 516, 487, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 474, 488, 542, 545, 505, 511, 494, 512, 493, 503, 499, 525, 512, 495, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8237, 8122, 8168, 8275, 8238, 8154, 8117] avg=8192.0000 max=8275 max_rank=4 min=8117 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[533, 549, 535, 516, 499, 517, 508, 513, 542, 530, 521, 513, 516, 483, 555, 498, 36, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 495, 500, 505, 524, 517, 540, 525, 533, 538, 516, 519, 532, 514, 496, 494, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 515, 513, 511, 529, 545, 541, 510, 477, 494, 526, 504, 510, 531, 490, 56, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 527, 538, 487, 490, 495, 513, 514, 562, 498, 494, 509, 538, 486, 499, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 523, 464, 500, 471, 509, 502, 522, 523, 533, 491, 468, 492, 532, 508, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 532, 505, 478, 496, 541, 482, 456, 449, 512, 483, 490, 537, 513, 484, 503, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 513, 461, 531, 522, 543, 521, 527, 549, 488, 485, 506, 539, 545, 531, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 463, 529, 508, 490, 520, 517, 493, 551, 517, 526, 499, 497, 501, 476, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8369, 8258, 8301, 8146, 8095, 7970, 8256, 8141] avg=8192.0000 max=8369 max_rank=0 min=7970 min_rank=5 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 513, 516, 509, 556, 507, 489, 488, 515, 463, 488, 529, 512, 493, 515, 517, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 505, 477, 515, 504, 531, 514, 528, 468, 525, 538, 499, 508, 499, 513, 502, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 524, 491, 515, 509, 480, 492, 529, 497, 500, 478, 493, 505, 507, 490, 516, 58, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 467, 512, 467, 503, 517, 569, 496, 494, 478, 519, 512, 496, 518, 520, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 483, 533, 538, 502, 544, 511, 520, 548, 537, 518, 520, 533, 506, 546, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 476, 498, 522, 534, 561, 497, 517, 522, 455, 473, 514, 498, 554, 507, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 534, 521, 512, 537, 535, 519, 522, 528, 518, 470, 503, 532, 552, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 474, 491, 531, 496, 554, 505, 519, 515, 514, 499, 495, 519, 489, 534, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8142, 8100, 8110, 8308, 8147, 8330, 8233] avg=8192.0000 max=8330 max_rank=6 min=8100 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=33018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 502, 498, 546, 531, 490, 487, 484, 524, 530, 482, 518, 510, 471, 494, 487, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 508, 548, 507, 533, 521, 528, 505, 482, 494, 502, 530, 515, 539, 491, 511, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 528, 484, 522, 541, 488, 533, 513, 503, 536, 513, 517, 509, 472, 524, 526, 63, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 523, 550, 514, 511, 523, 526, 514, 512, 533, 500, 553, 478, 521, 474, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 514, 492, 524, 489, 473, 567, 534, 500, 493, 539, 529, 523, 487, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 527, 533, 508, 513, 528, 527, 475, 496, 528, 515, 500, 539, 496, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 519, 480, 509, 567, 494, 481, 479, 521, 542, 524, 490, 482, 514, 507, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 516, 508, 504, 539, 526, 551, 519, 482, 513, 541, 539, 519, 455, 533, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8181, 8288, 8195, 8183, 8206, 8137, 8232] avg=8192.0000 max=8288 max_rank=2 min=8114 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4352, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3968, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3840, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[512, 539, 508, 505, 535, 464, 510, 524, 542, 511, 498, 511, 500, 488, 510, 529, 506, 493, 486, 548, 531, 485, 494, 520, 531, 501, 518, 494, 550, 502, 467, 517, 499, 535, 544, 528, 473, 505, 528, 517, 520, 546, 517, 515, 529, 511, 517, 519, 477, 553, 509, 522, 505, 513, 485, 524, 512, 493, 522, 533, 504, 524, 527, 503, 517, 503, 508, 573, 486, 549, 543, 550, 513, 517, 505, 491, 492, 505, 505, 544, 516, 524, 507, 488, 533, 511, 497, 526, 506, 509, 494, 492, 494, 510, 507, 480, 497, 494, 537, 512, 523, 492, 520, 499, 494, 535, 560, 478, 476, 506, 545, 503, 524, 507, 509, 511, 513, 479, 516, 477, 523, 533, 526, 526, 483, 482, 501, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[484, 520, 534, 520, 537, 492, 555, 546, 545, 503, 480, 507, 544, 511, 492, 539, 491, 450, 489, 547, 516, 569, 516, 525, 513, 498, 511, 551, 505, 499, 500, 493, 519, 546, 531, 482, 522, 518, 508, 536, 537, 538, 491, 525, 497, 494, 522, 547, 485, 496, 532, 530, 492, 481, 532, 461, 509, 528, 507, 534, 472, 480, 541, 497, 555, 514, 493, 520, 537, 531, 512, 520, 504, 491, 477, 530, 502, 501, 494, 495, 457, 522, 494, 494, 501, 480, 529, 487, 497, 493, 491, 520, 505, 561, 519, 501, 503, 560, 484, 495, 485, 514, 495, 534, 493, 521, 547, 503, 503, 509, 510, 490, 496, 524, 517, 498, 489, 516, 517, 542, 573, 522, 545, 491, 538, 524, 505, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[525, 475, 518, 515, 499, 549, 505, 516, 487, 480, 499, 501, 507, 508, 504, 514, 495, 509, 503, 495, 514, 501, 551, 523, 477, 511, 520, 482, 511, 533, 501, 472, 533, 504, 527, 548, 509, 493, 531, 484, 545, 530, 506, 509, 505, 543, 540, 510, 516, 461, 475, 500, 494, 524, 526, 508, 536, 538, 526, 525, 512, 485, 533, 546, 526, 505, 513, 549, 501, 552, 491, 512, 544, 520, 533, 487, 526, 492, 516, 476, 510, 520, 473, 510, 531, 488, 515, 501, 457, 527, 555, 523, 512, 488, 518, 533, 542, 483, 518, 513, 526, 501, 484, 522, 506, 494, 506, 492, 528, 504, 476, 518, 525, 522, 526, 522, 497, 523, 503, 534, 522, 556, 548, 487, 490, 527, 509, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[506, 562, 530, 532, 497, 533, 489, 531, 493, 504, 497, 511, 506, 490, 519, 515, 499, 491, 507, 524, 508, 506, 534, 539, 486, 487, 548, 517, 525, 492, 505, 491, 503, 496, 494, 502, 526, 494, 519, 519, 509, 532, 487, 485, 505, 507, 541, 498, 485, 500, 518, 514, 488, 516, 490, 531, 511, 507, 551, 521, 509, 535, 528, 521, 520, 506, 491, 541, 496, 479, 501, 553, 499, 531, 522, 480, 507, 525, 538, 483, 486, 484, 516, 538, 484, 487, 541, 521, 517, 481, 511, 505, 537, 535, 535, 553, 526, 539, 547, 496, 506, 472, 525, 493, 495, 530, 499, 510, 498, 504, 557, 514, 515, 527, 518, 498, 515, 523, 499, 500, 505, 562, 498, 493, 556, 507, 481, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[501, 496, 526, 508, 515, 488, 523, 499, 491, 517, 497, 537, 517, 505, 506, 502, 567, 504, 496, 492, 468, 495, 545, 538, 540, 525, 499, 546, 531, 496, 529, 525, 515, 508, 531, 493, 553, 492, 528, 478, 471, 522, 503, 520, 521, 542, 529, 470, 518, 512, 531, 513, 514, 530, 514, 496, 500, 531, 489, 530, 531, 496, 517, 498, 496, 515, 527, 483, 509, 541, 488, 506, 477, 509, 471, 542, 503, 506, 537, 452, 532, 530, 482, 535, 499, 498, 496, 539, 445, 492, 568, 529, 482, 504, 511, 536, 469, 504, 506, 511, 555, 517, 504, 503, 553, 481, 505, 483, 526, 519, 488, 503, 495, 483, 475, 518, 544, 493, 543, 494, 511, 551, 536, 543, 543, 543, 532, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[522, 519, 487, 507, 492, 490, 508, 478, 517, 492, 540, 497, 521, 560, 522, 559, 532, 501, 513, 494, 484, 465, 522, 478, 501, 518, 506, 507, 525, 540, 483, 557, 539, 495, 516, 508, 506, 524, 516, 527, 518, 498, 511, 514, 532, 520, 501, 516, 479, 521, 545, 484, 529, 528, 503, 549, 530, 529, 480, 530, 542, 496, 541, 515, 501, 515, 506, 522, 510, 512, 505, 490, 533, 487, 482, 486, 530, 503, 493, 488, 500, 511, 488, 497, 507, 524, 525, 498, 523, 534, 504, 536, 529, 515, 535, 533, 497, 499, 496, 489, 468, 518, 513, 525, 529, 527, 496, 545, 495, 492, 555, 485, 545, 526, 521, 517, 551, 490, 493, 522, 496, 515, 515, 490, 496, 468, 543, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[493, 520, 506, 516, 462, 513, 500, 536, 491, 502, 503, 570, 498, 472, 525, 498, 477, 509, 517, 481, 513, 506, 493, 492, 543, 519, 492, 529, 474, 498, 540, 523, 573, 500, 493, 497, 520, 522, 485, 549, 518, 561, 522, 550, 506, 533, 488, 540, 456, 486, 520, 528, 532, 540, 527, 486, 522, 517, 528, 499, 503, 508, 509, 538, 524, 568, 478, 541, 525, 488, 506, 493, 535, 528, 494, 526, 514, 522, 491, 506, 522, 506, 508, 529, 521, 485, 532, 490, 484, 503, 492, 492, 528, 516, 516, 558, 508, 464, 525, 520, 522, 506, 514, 533, 546, 477, 516, 530, 544, 475, 527, 480, 504, 522, 463, 502, 494, 507, 538, 529, 523, 524, 516, 508, 486, 523, 517, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[518, 468, 505, 507, 502, 512, 507, 493, 503, 543, 533, 524, 522, 502, 508, 511, 474, 507, 506, 505, 510, 528, 540, 534, 516, 527, 492, 512, 470, 529, 531, 498, 534, 500, 526, 480, 473, 555, 521, 487, 513, 523, 486, 529, 537, 506, 520, 489, 509, 498, 508, 520, 517, 496, 513, 492, 501, 502, 556, 522, 500, 524, 594, 491, 524, 533, 496, 512, 497, 502, 524, 511, 485, 518, 473, 485, 495, 513, 515, 522, 518, 521, 538, 506, 514, 490, 493, 489, 523, 516, 520, 537, 490, 511, 528, 535, 531, 475, 541, 504, 510, 494, 523, 522, 498, 479, 519, 528, 524, 480, 534, 509, 531, 538, 476, 529, 545, 506, 505, 538, 489, 507, 472, 537, 529, 521, 567, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 539, 508, 505, 535, 464, 510, 524, 542, 511, 498, 511, 500, 488, 510, 529, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 493, 486, 548, 531, 485, 494, 520, 531, 501, 518, 494, 550, 502, 467, 517, 23, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 535, 544, 528, 473, 505, 528, 517, 520, 487, 517, 515, 529, 511, 517, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 553, 509, 522, 505, 513, 485, 524, 512, 493, 522, 533, 504, 524, 511, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 503, 508, 573, 486, 549, 543, 550, 513, 517, 505, 491, 492, 505, 505, 544, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 524, 507, 488, 533, 511, 497, 526, 506, 509, 494, 492, 494, 510, 507, 480, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 537, 512, 523, 492, 520, 499, 494, 535, 560, 478, 476, 506, 545, 503, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 509, 511, 513, 479, 516, 477, 523, 488, 526, 526, 483, 482, 501, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8174, 8244, 8190, 8322, 8112, 8207, 8087] avg=8192.0000 max=8322 max_rank=4 min=8087 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[506, 562, 530, 532, 497, 533, 489, 531, 493, 504, 497, 511, 506, 490, 519, 515, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 491, 507, 524, 508, 506, 534, 539, 486, 487, 548, 517, 525, 492, 505, 491, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 496, 494, 502, 526, 494, 519, 519, 509, 474, 487, 485, 505, 507, 541, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 500, 518, 514, 488, 516, 490, 531, 511, 507, 551, 521, 509, 535, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 506, 491, 541, 496, 479, 501, 553, 499, 531, 522, 480, 507, 525, 538, 483, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 484, 516, 538, 484, 487, 541, 521, 517, 481, 511, 505, 537, 535, 535, 553, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 539, 547, 496, 506, 472, 525, 493, 495, 530, 499, 510, 498, 504, 557, 514, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 527, 518, 498, 515, 523, 499, 500, 505, 513, 498, 493, 556, 507, 481, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8191, 8059, 8208, 8195, 8250, 8246, 8157] avg=8192.0000 max=8250 max_rank=5 min=8059 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[525, 475, 518, 515, 499, 549, 505, 516, 487, 480, 499, 501, 507, 508, 504, 514, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 503, 495, 514, 501, 551, 523, 477, 511, 520, 482, 511, 533, 501, 472, 28, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 504, 527, 548, 509, 493, 531, 484, 545, 464, 506, 509, 505, 543, 540, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 461, 475, 500, 494, 524, 526, 508, 536, 538, 526, 525, 512, 485, 510, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 513, 549, 501, 552, 491, 512, 544, 520, 533, 487, 526, 492, 516, 476, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 473, 510, 531, 488, 515, 501, 457, 527, 555, 523, 512, 488, 518, 533, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 483, 518, 513, 526, 501, 484, 522, 506, 494, 506, 492, 528, 504, 476, 518, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 526, 522, 497, 523, 503, 534, 522, 498, 548, 487, 490, 527, 509, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8138, 8251, 8182, 8269, 8185, 8151, 8239] avg=8192.0000 max=8269 max_rank=4 min=8121 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[522, 519, 487, 507, 492, 490, 508, 478, 517, 492, 540, 497, 521, 560, 522, 559, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 501, 513, 494, 484, 465, 522, 478, 501, 518, 506, 507, 525, 540, 483, 557, 21, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 495, 516, 508, 506, 524, 516, 527, 518, 445, 511, 514, 532, 520, 501, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 521, 545, 484, 529, 528, 503, 549, 530, 529, 480, 530, 542, 496, 524, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 506, 522, 510, 512, 505, 490, 533, 487, 482, 486, 530, 503, 493, 488, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 488, 497, 507, 524, 525, 498, 523, 534, 504, 536, 529, 515, 535, 533, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 496, 489, 468, 518, 513, 525, 529, 527, 496, 545, 495, 492, 555, 485, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 526, 521, 517, 551, 490, 493, 522, 496, 470, 515, 490, 496, 468, 543, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8155, 8188, 8284, 8084, 8277, 8161, 8161] avg=8192.0000 max=8284 max_rank=3 min=8084 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[484, 520, 534, 520, 537, 492, 555, 546, 545, 503, 480, 507, 544, 511, 492, 539, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 450, 489, 547, 516, 569, 516, 525, 513, 498, 511, 551, 505, 499, 500, 493, 23, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 546, 531, 482, 522, 518, 508, 536, 537, 480, 491, 525, 497, 494, 522, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 496, 532, 530, 492, 481, 532, 461, 509, 528, 507, 534, 472, 480, 524, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 514, 493, 520, 537, 531, 512, 520, 504, 491, 477, 530, 502, 501, 494, 495, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 522, 494, 494, 501, 480, 529, 487, 497, 493, 491, 520, 505, 561, 519, 501, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 560, 484, 495, 485, 514, 495, 534, 493, 521, 547, 503, 503, 509, 510, 490, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 517, 498, 489, 516, 517, 542, 573, 477, 545, 491, 538, 524, 505, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8204, 8255, 8060, 8197, 8069, 8181, 8246] avg=8192.0000 max=8324 max_rank=0 min=8060 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[501, 496, 526, 508, 515, 488, 523, 499, 491, 517, 497, 537, 517, 505, 506, 502, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 504, 496, 492, 468, 495, 545, 538, 540, 525, 499, 546, 531, 496, 529, 525, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 531, 493, 553, 492, 528, 478, 471, 466, 503, 520, 521, 542, 529, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 531, 513, 514, 530, 514, 496, 500, 531, 489, 530, 531, 496, 501, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 515, 527, 483, 509, 541, 488, 506, 477, 509, 471, 542, 503, 506, 537, 452, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 530, 482, 535, 499, 498, 496, 539, 445, 492, 568, 529, 482, 504, 511, 536, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 504, 506, 511, 555, 517, 504, 503, 553, 481, 505, 483, 526, 519, 488, 503, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 483, 475, 518, 544, 493, 543, 494, 511, 503, 536, 543, 543, 543, 532, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8327, 8120, 8204, 8084, 8197, 8161, 8301] avg=8192.0000 max=8327 max_rank=1 min=8084 min_rank=4 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[493, 520, 506, 516, 462, 513, 500, 536, 491, 502, 503, 570, 498, 472, 525, 498, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 509, 517, 481, 513, 506, 493, 492, 543, 519, 492, 529, 474, 498, 540, 523, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 500, 493, 497, 520, 522, 485, 549, 518, 500, 522, 550, 506, 533, 488, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 486, 520, 528, 532, 540, 527, 486, 522, 517, 528, 499, 503, 508, 493, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 568, 478, 541, 525, 488, 506, 493, 535, 528, 494, 526, 514, 522, 491, 506, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 506, 508, 529, 521, 485, 532, 490, 484, 503, 492, 492, 528, 516, 516, 558, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 464, 525, 520, 522, 506, 514, 533, 546, 477, 516, 530, 544, 475, 527, 480, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 463, 502, 494, 507, 538, 529, 523, 479, 516, 508, 486, 523, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8138, 8296, 8183, 8260, 8200, 8224, 8116] avg=8192.0000 max=8296 max_rank=2 min=8116 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[518, 468, 505, 507, 502, 512, 507, 493, 503, 543, 533, 524, 522, 502, 508, 511, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 507, 506, 505, 510, 528, 540, 534, 516, 527, 492, 512, 470, 529, 531, 498, 22, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 500, 526, 480, 473, 555, 521, 487, 513, 467, 486, 529, 537, 506, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 508, 520, 517, 496, 513, 492, 501, 502, 556, 522, 500, 524, 576, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 533, 496, 512, 497, 502, 524, 511, 485, 518, 473, 485, 495, 513, 515, 522, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 538, 506, 514, 490, 493, 489, 523, 516, 520, 537, 490, 511, 528, 535, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 475, 541, 504, 510, 494, 523, 522, 498, 479, 519, 528, 524, 480, 534, 509, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 538, 476, 529, 545, 506, 505, 538, 489, 464, 472, 537, 529, 521, 567, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8209, 8123, 8225, 8125, 8246, 8205, 8229] avg=8192.0000 max=8246 max_rank=5 min=8123 min_rank=2 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 3968, 4224, 4096, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 4352, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3840, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4352, 4096, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4096, 3968, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[525, 527, 536, 502, 496, 479, 467, 473, 522, 472, 520, 516, 525, 521, 538, 487, 519, 510, 532, 505, 512, 495, 536, 503, 496, 526, 519, 490, 546, 522, 530, 553, 497, 530, 486, 488, 511, 559, 541, 531, 516, 516, 531, 515, 509, 515, 495, 508, 522, 484, 504, 526, 485, 532, 510, 479, 526, 485, 491, 515, 515, 532, 481, 502, 508, 546, 533, 525, 514, 509, 541, 476, 544, 514, 526, 510, 501, 505, 548, 524, 490, 526, 519, 486, 517, 498, 487, 496, 519, 525, 545, 470, 515, 512, 520, 501, 512, 489, 516, 504, 499, 481, 528, 519, 503, 491, 531, 522, 487, 516, 496, 498, 519, 548, 518, 531, 561, 493, 477, 487, 485, 531, 516, 531, 505, 516, 510, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[519, 512, 523, 543, 511, 488, 486, 534, 509, 515, 548, 519, 481, 516, 528, 513, 494, 511, 514, 501, 504, 502, 493, 517, 530, 509, 500, 525, 498, 518, 540, 516, 467, 556, 489, 533, 487, 492, 490, 511, 519, 492, 544, 512, 528, 500, 516, 517, 523, 492, 513, 482, 501, 466, 519, 564, 532, 507, 517, 481, 524, 501, 518, 518, 517, 548, 488, 512, 558, 490, 566, 520, 523, 528, 504, 485, 523, 522, 522, 502, 487, 512, 535, 523, 503, 496, 527, 553, 494, 508, 530, 509, 530, 521, 519, 554, 496, 480, 561, 489, 487, 510, 487, 534, 490, 529, 479, 497, 520, 496, 514, 501, 533, 520, 513, 529, 484, 538, 499, 511, 494, 475, 512, 516, 501, 495, 506, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[506, 529, 488, 512, 524, 521, 538, 515, 521, 540, 556, 513, 561, 524, 479, 478, 497, 512, 494, 518, 488, 501, 541, 498, 499, 507, 503, 465, 492, 475, 512, 478, 482, 485, 553, 494, 514, 522, 506, 558, 555, 503, 518, 517, 495, 490, 521, 552, 450, 535, 543, 524, 510, 510, 555, 534, 545, 504, 548, 488, 502, 526, 529, 524, 497, 517, 526, 522, 522, 551, 515, 499, 496, 511, 504, 512, 490, 512, 505, 525, 530, 525, 503, 503, 526, 475, 510, 533, 522, 513, 493, 508, 495, 517, 523, 535, 496, 494, 481, 495, 528, 511, 512, 501, 485, 556, 507, 487, 504, 481, 491, 529, 496, 498, 496, 492, 489, 528, 527, 514, 567, 504, 529, 509, 472, 503, 529, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[524, 502, 473, 523, 537, 518, 508, 511, 493, 547, 495, 498, 501, 512, 502, 526, 518, 508, 509, 534, 501, 456, 523, 512, 517, 486, 547, 539, 520, 500, 530, 520, 532, 456, 484, 512, 528, 507, 471, 515, 529, 531, 518, 511, 510, 466, 509, 505, 519, 513, 499, 521, 507, 515, 523, 506, 565, 509, 514, 491, 527, 551, 558, 538, 543, 503, 477, 489, 533, 512, 514, 481, 510, 463, 500, 487, 493, 512, 510, 528, 538, 503, 505, 498, 478, 534, 521, 533, 524, 508, 550, 548, 504, 506, 536, 539, 479, 530, 528, 511, 487, 509, 521, 516, 500, 529, 495, 520, 522, 513, 537, 476, 492, 522, 486, 511, 553, 492, 495, 504, 498, 507, 545, 522, 494, 499, 532, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[446, 540, 474, 518, 513, 497, 506, 529, 507, 495, 515, 528, 463, 502, 501, 521, 529, 523, 509, 469, 532, 478, 521, 492, 516, 528, 503, 528, 518, 501, 516, 491, 501, 506, 533, 499, 517, 526, 501, 522, 565, 454, 534, 497, 498, 549, 473, 485, 549, 508, 578, 507, 482, 549, 540, 527, 547, 538, 497, 512, 483, 526, 540, 528, 504, 477, 537, 535, 505, 501, 523, 530, 517, 521, 517, 494, 480, 551, 536, 472, 530, 483, 540, 518, 532, 523, 477, 507, 492, 516, 512, 490, 494, 493, 522, 509, 518, 496, 496, 503, 505, 494, 496, 510, 538, 509, 536, 508, 496, 513, 504, 492, 545, 475, 507, 511, 520, 541, 537, 531, 475, 493, 556, 530, 513, 539, 500, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[547, 511, 536, 491, 540, 519, 541, 504, 521, 519, 499, 498, 500, 529, 504, 541, 484, 488, 539, 502, 509, 511, 488, 538, 516, 516, 507, 520, 513, 516, 491, 505, 494, 535, 523, 528, 545, 515, 532, 541, 504, 525, 529, 506, 500, 487, 493, 534, 470, 515, 543, 504, 519, 525, 492, 514, 479, 492, 512, 548, 553, 486, 498, 467, 525, 516, 497, 478, 496, 523, 474, 473, 530, 496, 547, 516, 513, 512, 504, 526, 498, 471, 513, 521, 538, 510, 500, 523, 530, 514, 516, 485, 468, 547, 530, 480, 518, 522, 492, 516, 506, 495, 525, 499, 515, 525, 501, 508, 510, 511, 525, 541, 489, 545, 500, 541, 543, 479, 521, 515, 517, 503, 517, 483, 499, 540, 516, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[509, 508, 524, 488, 505, 531, 543, 499, 505, 518, 534, 523, 505, 509, 488, 561, 531, 556, 496, 531, 544, 556, 537, 499, 522, 549, 525, 510, 521, 486, 484, 517, 514, 500, 541, 517, 465, 507, 538, 465, 532, 498, 538, 500, 487, 511, 535, 484, 512, 474, 534, 482, 486, 480, 539, 506, 517, 502, 503, 468, 502, 506, 498, 519, 539, 510, 497, 545, 516, 508, 504, 487, 488, 498, 491, 512, 532, 554, 500, 490, 506, 495, 478, 472, 513, 532, 537, 529, 521, 538, 513, 487, 493, 551, 513, 517, 556, 497, 510, 497, 533, 506, 520, 532, 562, 534, 501, 480, 516, 498, 542, 521, 491, 503, 490, 456, 516, 520, 490, 507, 536, 536, 518, 514, 499, 489, 522, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[474, 533, 502, 534, 553, 480, 505, 507, 492, 531, 465, 540, 522, 464, 503, 518, 540, 485, 545, 492, 480, 478, 529, 530, 553, 519, 535, 524, 523, 478, 546, 517, 524, 494, 493, 510, 536, 497, 493, 524, 508, 553, 530, 520, 529, 510, 485, 504, 503, 489, 532, 511, 544, 467, 498, 502, 477, 504, 520, 514, 533, 536, 545, 495, 499, 506, 530, 512, 521, 496, 495, 529, 530, 477, 548, 523, 479, 463, 492, 504, 505, 517, 525, 543, 519, 505, 519, 503, 524, 560, 496, 519, 529, 505, 526, 507, 529, 500, 553, 462, 506, 496, 510, 522, 485, 539, 509, 489, 495, 487, 504, 510, 490, 501, 546, 519, 530, 503, 488, 522, 511, 490, 520, 502, 552, 513, 518, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[525, 527, 536, 502, 494, 479, 467, 473, 522, 472, 520, 516, 525, 521, 538, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 532, 505, 512, 495, 528, 503, 496, 526, 519, 490, 546, 522, 530, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 530, 486, 488, 511, 559, 541, 531, 516, 516, 531, 515, 509, 515, 495, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 484, 490, 526, 485, 532, 510, 479, 526, 485, 491, 515, 515, 532, 481, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 546, 533, 525, 514, 509, 541, 476, 544, 514, 526, 510, 501, 505, 548, 524, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 526, 519, 486, 517, 498, 487, 496, 519, 525, 545, 470, 515, 512, 495, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 489, 516, 504, 499, 481, 528, 519, 503, 491, 531, 522, 487, 516, 496, 498, 25, 14, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 548, 518, 531, 561, 493, 477, 487, 485, 531, 511, 531, 505, 516, 510, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8286, 8248, 8075, 8328, 8101, 8142, 8252] avg=8192.0000 max=8328 max_rank=4 min=8075 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[519, 512, 523, 543, 509, 488, 486, 534, 509, 515, 548, 519, 481, 516, 528, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 514, 501, 504, 502, 486, 517, 530, 509, 500, 525, 498, 518, 540, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 556, 489, 533, 487, 492, 490, 511, 519, 492, 544, 512, 528, 500, 516, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 492, 499, 482, 501, 466, 519, 564, 532, 507, 517, 481, 524, 501, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 548, 488, 512, 558, 490, 566, 520, 523, 528, 504, 485, 523, 522, 522, 502, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 512, 535, 523, 503, 496, 527, 553, 494, 508, 530, 509, 530, 521, 494, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 480, 561, 489, 487, 510, 487, 534, 490, 529, 479, 497, 520, 496, 514, 501, 25, 14, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 520, 513, 529, 484, 538, 499, 511, 494, 475, 507, 516, 501, 495, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8165, 8153, 8144, 8312, 8276, 8119, 8124] avg=8192.0000 max=8312 max_rank=4 min=8119 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[506, 529, 488, 512, 522, 521, 538, 515, 521, 540, 556, 513, 561, 524, 479, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 494, 518, 488, 501, 533, 498, 499, 507, 503, 465, 492, 475, 512, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 485, 553, 494, 514, 522, 506, 558, 555, 503, 518, 517, 495, 490, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 535, 528, 524, 510, 510, 555, 534, 545, 504, 548, 488, 502, 526, 529, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 526, 522, 522, 551, 515, 499, 496, 511, 504, 512, 490, 512, 505, 525, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 525, 503, 503, 526, 475, 510, 533, 522, 513, 493, 508, 495, 517, 498, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 481, 495, 528, 511, 512, 501, 485, 556, 507, 487, 504, 481, 491, 529, 25, 15, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 498, 496, 492, 489, 528, 527, 514, 567, 504, 524, 509, 472, 503, 529, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 7972, 8265, 8312, 8208, 8186, 8109, 8181] avg=8192.0000 max=8312 max_rank=3 min=7972 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[446, 540, 474, 518, 509, 497, 506, 529, 507, 495, 515, 528, 463, 502, 501, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 523, 509, 469, 532, 478, 510, 492, 516, 528, 503, 528, 518, 501, 516, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 533, 499, 517, 526, 501, 522, 565, 454, 527, 497, 498, 549, 473, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 508, 560, 507, 482, 549, 540, 527, 547, 538, 497, 512, 483, 526, 540, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 477, 537, 535, 505, 501, 523, 530, 517, 521, 517, 494, 480, 551, 536, 472, 5, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 483, 540, 518, 532, 523, 477, 507, 492, 516, 512, 490, 494, 493, 493, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 496, 496, 503, 505, 494, 496, 510, 538, 509, 536, 508, 496, 513, 504, 492, 29, 18, 11, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 475, 507, 511, 520, 541, 537, 531, 475, 493, 544, 530, 513, 539, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8051, 8143, 8153, 8393, 8216, 8109, 8179, 8292] avg=8192.0000 max=8393 max_rank=3 min=8051 min_rank=0 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 502, 473, 523, 535, 518, 508, 511, 493, 547, 495, 498, 501, 512, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 509, 534, 501, 456, 515, 512, 517, 486, 547, 539, 520, 500, 530, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 456, 484, 512, 528, 507, 471, 515, 529, 531, 518, 511, 510, 466, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 513, 485, 521, 507, 515, 523, 506, 565, 509, 514, 491, 527, 551, 558, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 477, 489, 533, 512, 514, 481, 510, 463, 500, 487, 493, 512, 510, 528, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 503, 505, 498, 478, 534, 521, 533, 524, 508, 550, 548, 504, 506, 510, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 530, 528, 511, 487, 509, 521, 516, 500, 529, 495, 520, 522, 513, 537, 476, 26, 14, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 486, 511, 553, 492, 495, 504, 498, 507, 540, 522, 494, 499, 532, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8212, 8084, 8342, 8059, 8299, 8224, 8148] avg=8192.0000 max=8342 max_rank=3 min=8059 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[474, 533, 502, 534, 551, 480, 505, 507, 492, 531, 465, 540, 522, 464, 503, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 485, 545, 492, 480, 478, 521, 530, 553, 519, 535, 524, 523, 478, 546, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 494, 493, 510, 536, 497, 493, 524, 508, 553, 530, 520, 529, 510, 485, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 489, 517, 511, 544, 467, 498, 502, 477, 504, 520, 514, 533, 536, 545, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 530, 512, 521, 496, 495, 529, 530, 477, 548, 523, 479, 463, 492, 504, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 517, 525, 543, 519, 505, 519, 503, 524, 560, 496, 519, 529, 505, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 500, 553, 462, 506, 496, 510, 522, 485, 539, 509, 489, 495, 487, 504, 510, 25, 15, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 501, 546, 519, 530, 503, 488, 522, 511, 490, 515, 502, 552, 513, 518, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8266, 8210, 8155, 8108, 8277, 8147, 8252] avg=8192.0000 max=8277 max_rank=5 min=8108 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 508, 524, 488, 503, 531, 543, 499, 505, 518, 534, 523, 505, 509, 488, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 556, 496, 531, 544, 556, 529, 499, 522, 549, 525, 510, 521, 486, 484, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 541, 517, 465, 507, 538, 465, 532, 498, 538, 500, 487, 511, 535, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 474, 519, 482, 486, 480, 539, 506, 517, 502, 503, 468, 502, 506, 498, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 510, 497, 545, 516, 508, 504, 487, 488, 498, 491, 512, 532, 554, 500, 490, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 478, 472, 513, 532, 537, 529, 521, 538, 513, 487, 493, 551, 488, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 497, 510, 497, 533, 506, 520, 532, 562, 534, 501, 480, 516, 498, 542, 521, 25, 15, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 503, 490, 456, 516, 520, 490, 507, 536, 536, 513, 514, 499, 489, 522, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8356, 8132, 8013, 8175, 8170, 8356, 8086] avg=8192.0000 max=8356 max_rank=1 min=8013 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[547, 511, 536, 491, 538, 519, 541, 504, 521, 519, 499, 498, 500, 529, 504, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 488, 539, 502, 509, 511, 481, 538, 516, 516, 507, 520, 513, 516, 491, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 535, 523, 528, 545, 515, 532, 541, 504, 525, 529, 506, 500, 487, 493, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 515, 528, 504, 519, 525, 492, 514, 479, 492, 512, 548, 553, 486, 498, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 516, 497, 478, 496, 523, 474, 473, 530, 496, 547, 516, 513, 512, 504, 526, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 471, 513, 521, 538, 510, 500, 523, 530, 514, 516, 485, 468, 547, 504, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 522, 492, 516, 506, 495, 525, 499, 515, 525, 501, 508, 510, 511, 525, 541, 26, 15, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 545, 500, 541, 543, 479, 521, 515, 517, 503, 512, 483, 499, 540, 516, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8136, 8291, 8102, 8130, 8118, 8260, 8201] avg=8192.0000 max=8298 max_rank=0 min=8102 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4352, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[542, 506, 511, 487, 575, 524, 498, 523, 485, 514, 530, 507, 501, 520, 504, 510, 513, 524, 473, 502, 508, 501, 530, 508, 494, 497, 462, 523, 510, 536, 538, 509, 533, 513, 494, 517, 494, 552, 484, 493, 481, 512, 514, 538, 560, 525, 526, 515, 508, 486, 514, 496, 483, 533, 489, 542, 517, 520, 522, 449, 532, 466, 494, 509, 500, 525, 539, 521, 522, 492, 513, 496, 495, 470, 516, 580, 545, 510, 542, 514, 528, 526, 501, 547, 507, 494, 525, 507, 479, 503, 521, 512, 540, 550, 526, 536, 506, 510, 523, 514, 504, 487, 500, 521, 509, 560, 502, 527, 493, 519, 497, 478, 524, 503, 504, 499, 484, 510, 523, 522, 502, 525, 499, 551, 515, 488, 492, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[507, 541, 520, 503, 531, 501, 567, 476, 514, 504, 519, 537, 512, 535, 466, 515, 484, 505, 540, 492, 480, 516, 500, 514, 547, 503, 519, 523, 460, 477, 488, 516, 517, 544, 515, 543, 479, 506, 507, 506, 549, 514, 529, 553, 489, 505, 448, 514, 509, 506, 521, 542, 491, 483, 521, 532, 520, 499, 507, 545, 528, 536, 527, 513, 489, 525, 542, 507, 473, 489, 489, 498, 530, 534, 552, 535, 520, 489, 523, 544, 516, 503, 497, 512, 531, 471, 522, 535, 493, 530, 509, 511, 529, 489, 462, 468, 523, 511, 507, 470, 486, 479, 519, 547, 484, 522, 524, 522, 536, 524, 525, 552, 521, 517, 465, 539, 473, 509, 521, 518, 512, 494, 557, 514, 518, 471, 519, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[518, 493, 512, 527, 534, 495, 484, 543, 513, 498, 548, 521, 512, 481, 483, 513, 503, 500, 489, 509, 548, 548, 526, 528, 479, 489, 515, 503, 494, 541, 511, 522, 464, 512, 532, 525, 587, 501, 522, 511, 487, 557, 541, 475, 476, 487, 498, 527, 527, 535, 525, 544, 510, 476, 508, 501, 535, 496, 507, 509, 479, 477, 484, 519, 520, 512, 490, 506, 492, 483, 534, 533, 491, 520, 502, 535, 504, 522, 517, 530, 523, 535, 502, 550, 503, 521, 531, 488, 505, 506, 501, 515, 509, 530, 525, 524, 507, 518, 508, 507, 522, 470, 486, 513, 506, 497, 505, 502, 496, 534, 554, 550, 504, 550, 493, 480, 493, 510, 517, 505, 508, 492, 511, 500, 536, 519, 529, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[480, 511, 520, 507, 524, 485, 510, 499, 525, 485, 497, 506, 532, 502, 454, 535, 512, 532, 521, 547, 469, 522, 526, 483, 542, 480, 539, 466, 513, 509, 468, 516, 520, 520, 498, 536, 514, 513, 511, 530, 530, 474, 508, 458, 493, 555, 525, 507, 506, 541, 525, 513, 551, 513, 521, 557, 453, 504, 529, 504, 496, 498, 509, 550, 517, 507, 543, 495, 538, 496, 495, 501, 469, 532, 514, 549, 545, 478, 506, 488, 484, 522, 510, 520, 517, 497, 521, 525, 486, 543, 505, 505, 524, 542, 505, 536, 505, 494, 522, 531, 546, 487, 543, 518, 482, 516, 528, 498, 474, 511, 522, 500, 493, 492, 567, 526, 487, 503, 577, 511, 508, 512, 515, 517, 521, 515, 523, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[513, 480, 501, 497, 485, 527, 497, 512, 571, 499, 501, 530, 498, 518, 480, 462, 477, 505, 510, 533, 514, 522, 489, 550, 534, 544, 517, 491, 524, 483, 586, 512, 512, 512, 483, 538, 531, 497, 525, 470, 524, 491, 486, 545, 541, 547, 472, 499, 532, 514, 533, 501, 526, 494, 567, 520, 511, 524, 506, 536, 535, 520, 518, 504, 507, 529, 503, 528, 529, 520, 527, 534, 530, 507, 517, 486, 502, 523, 506, 516, 554, 504, 472, 509, 516, 515, 531, 480, 487, 475, 489, 499, 497, 473, 508, 502, 509, 513, 537, 488, 520, 525, 504, 500, 510, 500, 502, 528, 521, 506, 475, 537, 484, 497, 572, 531, 524, 505, 525, 526, 490, 531, 522, 526, 478, 525, 506, 468] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[482, 548, 500, 521, 545, 482, 501, 493, 543, 541, 502, 474, 502, 512, 468, 494, 529, 517, 488, 489, 512, 508, 494, 493, 486, 492, 505, 564, 523, 523, 468, 502, 525, 505, 510, 532, 519, 538, 521, 508, 522, 493, 488, 495, 550, 509, 498, 501, 518, 522, 545, 537, 462, 508, 531, 498, 505, 536, 517, 500, 493, 526, 559, 519, 545, 523, 469, 549, 508, 533, 493, 523, 517, 522, 504, 497, 518, 523, 498, 538, 524, 520, 483, 513, 493, 512, 532, 536, 510, 497, 492, 481, 528, 511, 498, 511, 509, 513, 511, 507, 507, 530, 488, 526, 524, 497, 525, 503, 537, 524, 517, 521, 517, 495, 496, 496, 485, 521, 513, 540, 507, 509, 532, 527, 516, 559, 500, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[507, 528, 548, 489, 465, 520, 487, 542, 537, 495, 480, 518, 480, 497, 499, 506, 502, 524, 519, 490, 495, 515, 526, 533, 469, 518, 507, 519, 538, 506, 497, 521, 525, 533, 502, 503, 581, 462, 538, 520, 477, 510, 509, 502, 529, 531, 511, 500, 498, 535, 538, 540, 527, 484, 513, 516, 546, 491, 500, 534, 534, 525, 491, 524, 502, 519, 484, 518, 526, 508, 486, 512, 499, 523, 534, 501, 555, 502, 514, 520, 503, 509, 483, 509, 508, 539, 527, 498, 550, 525, 520, 522, 517, 533, 470, 485, 532, 525, 492, 537, 517, 474, 476, 499, 533, 513, 530, 521, 498, 534, 487, 512, 528, 537, 517, 508, 503, 489, 502, 540, 484, 476, 504, 507, 506, 515, 506, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[506, 542, 534, 521, 498, 526, 501, 486, 494, 552, 539, 542, 504, 498, 545, 491, 479, 518, 482, 514, 492, 507, 476, 513, 509, 505, 512, 484, 497, 520, 518, 484, 468, 497, 507, 530, 476, 520, 506, 522, 563, 498, 501, 500, 517, 504, 541, 475, 550, 511, 525, 529, 480, 521, 515, 541, 517, 499, 510, 543, 480, 554, 541, 494, 501, 544, 528, 521, 480, 531, 491, 505, 527, 533, 488, 476, 462, 510, 524, 514, 528, 523, 493, 515, 498, 483, 559, 506, 497, 527, 492, 533, 517, 515, 511, 523, 538, 494, 519, 480, 517, 498, 527, 521, 539, 521, 501, 494, 481, 504, 488, 484, 490, 478, 546, 477, 554, 558, 493, 544, 485, 520, 519, 540, 545, 487, 564, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[518, 493, 512, 527, 534, 495, 484, 543, 513, 498, 548, 521, 512, 481, 483, 513, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 500, 489, 509, 548, 548, 526, 528, 479, 489, 515, 503, 494, 541, 511, 522, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 512, 532, 517, 587, 501, 522, 511, 487, 557, 541, 475, 476, 487, 498, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 535, 473, 544, 510, 476, 508, 501, 535, 496, 507, 509, 479, 477, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 490, 506, 492, 483, 534, 533, 491, 520, 502, 535, 504, 522, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 535, 502, 550, 503, 521, 531, 488, 505, 506, 501, 515, 509, 530, 525, 524, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 508, 507, 522, 470, 486, 513, 506, 497, 505, 502, 496, 534, 554, 550, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 550, 493, 480, 493, 510, 517, 490, 508, 492, 511, 500, 536, 519, 529, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8257, 8194, 8080, 8166, 8276, 8187, 8173] avg=8192.0000 max=8276 max_rank=5 min=8080 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[482, 548, 500, 521, 545, 482, 501, 493, 543, 541, 502, 474, 502, 512, 468, 494, 26, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 517, 488, 489, 512, 508, 494, 493, 486, 492, 505, 564, 523, 523, 468, 502, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 510, 523, 519, 538, 521, 508, 522, 493, 488, 495, 550, 509, 498, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 522, 490, 537, 462, 508, 531, 498, 505, 536, 517, 500, 493, 526, 559, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 497, 469, 549, 508, 533, 493, 523, 517, 522, 504, 497, 518, 523, 498, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 483, 513, 493, 512, 532, 536, 510, 497, 492, 481, 528, 511, 498, 511, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 511, 507, 507, 530, 488, 526, 524, 497, 525, 503, 537, 524, 517, 521, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 496, 496, 485, 521, 513, 524, 507, 509, 532, 527, 516, 559, 500, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8147, 8205, 8221, 8234, 8150, 8252, 8189] avg=8192.0000 max=8252 max_rank=6 min=8138 min_rank=0 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[542, 506, 511, 487, 575, 524, 498, 523, 485, 514, 530, 507, 501, 520, 504, 510, 26, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 473, 502, 508, 501, 530, 508, 494, 497, 462, 523, 510, 536, 538, 509, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 513, 494, 509, 494, 552, 484, 493, 481, 512, 514, 538, 560, 525, 526, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 486, 463, 496, 483, 533, 489, 542, 517, 520, 522, 449, 532, 466, 494, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 499, 539, 521, 522, 492, 513, 496, 495, 470, 516, 580, 545, 510, 542, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 526, 501, 547, 507, 494, 525, 507, 479, 503, 521, 512, 540, 550, 526, 536, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 523, 514, 504, 487, 500, 521, 509, 560, 502, 527, 493, 519, 497, 478, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 503, 504, 499, 484, 510, 523, 507, 502, 525, 499, 551, 515, 488, 492, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8179, 8243, 8009, 8254, 8310, 8162, 8113] avg=8192.0000 max=8310 max_rank=5 min=8009 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[513, 480, 501, 497, 485, 527, 497, 512, 571, 499, 501, 530, 498, 518, 480, 462, 29, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 505, 510, 533, 514, 522, 489, 550, 534, 544, 517, 491, 524, 483, 586, 512, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 512, 483, 526, 531, 497, 525, 470, 524, 491, 486, 545, 541, 547, 472, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 514, 471, 501, 526, 494, 567, 520, 511, 524, 506, 536, 535, 520, 518, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 503, 528, 529, 520, 527, 534, 530, 507, 517, 486, 502, 523, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 504, 472, 509, 516, 515, 531, 480, 487, 475, 489, 499, 497, 473, 508, 502, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 537, 488, 520, 525, 504, 500, 510, 500, 502, 528, 521, 506, 475, 537, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 497, 572, 531, 524, 505, 525, 497, 490, 531, 522, 526, 478, 525, 506, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8347, 8161, 8279, 8235, 8027, 8191, 8181] avg=8192.0000 max=8347 max_rank=1 min=8027 min_rank=5 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 541, 520, 503, 531, 501, 567, 476, 514, 504, 519, 537, 512, 535, 466, 515, 26, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 505, 540, 492, 480, 516, 500, 514, 547, 503, 519, 523, 460, 477, 488, 516, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 544, 515, 534, 479, 506, 507, 506, 549, 514, 529, 553, 489, 505, 448, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 506, 470, 542, 491, 483, 521, 532, 520, 499, 507, 545, 528, 536, 527, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 499, 542, 507, 473, 489, 489, 498, 530, 534, 552, 535, 520, 489, 523, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 503, 497, 512, 531, 471, 522, 535, 493, 530, 509, 511, 529, 489, 462, 468, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 507, 470, 486, 479, 519, 547, 484, 522, 524, 522, 536, 524, 525, 552, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 465, 539, 473, 509, 521, 503, 512, 494, 557, 514, 518, 471, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8115, 8209, 8229, 8213, 8087, 8243, 8163] avg=8192.0000 max=8277 max_rank=0 min=8087 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 528, 548, 489, 465, 520, 487, 542, 537, 495, 480, 518, 480, 497, 499, 506, 25, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 519, 490, 495, 515, 526, 533, 469, 518, 507, 519, 538, 506, 497, 521, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 533, 502, 495, 581, 462, 538, 520, 477, 510, 509, 502, 529, 531, 511, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 535, 484, 540, 527, 484, 513, 516, 546, 491, 500, 534, 534, 525, 491, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 494, 484, 518, 526, 508, 486, 512, 499, 523, 534, 501, 555, 502, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 509, 483, 509, 508, 539, 527, 498, 550, 525, 520, 522, 517, 533, 470, 485, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 492, 537, 517, 474, 476, 499, 533, 513, 530, 521, 498, 534, 487, 512, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 517, 508, 503, 489, 502, 524, 484, 476, 504, 507, 506, 515, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8232, 8225, 8242, 8178, 8206, 8193, 8133] avg=8192.0000 max=8242 max_rank=3 min=8127 min_rank=0 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[480, 511, 520, 507, 524, 485, 510, 499, 525, 485, 497, 506, 532, 502, 454, 535, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 532, 521, 547, 469, 522, 526, 483, 542, 480, 539, 466, 513, 509, 468, 516, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 520, 498, 527, 514, 513, 511, 530, 530, 474, 508, 458, 493, 555, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 541, 473, 513, 551, 513, 521, 557, 453, 504, 529, 504, 496, 498, 509, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 482, 543, 495, 538, 496, 495, 501, 469, 532, 514, 549, 545, 478, 506, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 522, 510, 520, 517, 497, 521, 525, 486, 543, 505, 505, 524, 542, 505, 536, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 494, 522, 531, 546, 487, 543, 518, 482, 516, 528, 498, 474, 511, 522, 500, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 567, 526, 487, 503, 577, 496, 508, 512, 515, 517, 521, 515, 523, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8197, 8183, 8218, 8148, 8251, 8189, 8250] avg=8192.0000 max=8251 max_rank=5 min=8100 min_rank=0 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[506, 542, 534, 521, 498, 526, 501, 486, 494, 552, 539, 542, 504, 498, 545, 491, 27, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 518, 482, 514, 492, 507, 476, 513, 509, 505, 512, 484, 497, 520, 518, 484, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 497, 507, 521, 476, 520, 506, 522, 563, 498, 501, 500, 517, 504, 541, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 511, 473, 529, 480, 521, 515, 541, 517, 499, 510, 543, 480, 554, 541, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 517, 528, 521, 480, 531, 491, 505, 527, 533, 488, 476, 462, 510, 524, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 523, 493, 515, 498, 483, 559, 506, 497, 527, 492, 533, 517, 515, 511, 523, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 494, 519, 480, 517, 498, 527, 521, 539, 521, 501, 494, 481, 504, 488, 484, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 478, 546, 477, 554, 558, 493, 527, 485, 520, 519, 540, 545, 487, 564, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8062, 8116, 8258, 8108, 8229, 8119, 8334] avg=8192.0000 max=8334 max_rank=7 min=8062 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3840, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3968, 4224, 4096, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 3968, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[512, 534, 515, 527, 566, 493, 521, 522, 491, 524, 486, 512, 506, 529, 519, 488, 543, 527, 518, 506, 507, 506, 482, 569, 472, 529, 504, 498, 518, 501, 528, 541, 515, 537, 516, 507, 525, 545, 496, 527, 495, 498, 531, 520, 470, 498, 495, 558, 484, 485, 507, 531, 526, 515, 526, 506, 508, 517, 499, 554, 479, 528, 524, 506, 502, 524, 515, 483, 496, 463, 504, 492, 511, 513, 534, 499, 512, 504, 499, 517, 476, 504, 532, 502, 519, 516, 497, 518, 530, 514, 528, 520, 506, 486, 558, 509, 494, 498, 483, 532, 510, 533, 500, 494, 547, 499, 540, 489, 543, 489, 515, 523, 483, 539, 549, 501, 513, 494, 542, 490, 485, 515, 508, 487, 535, 499, 475, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[517, 485, 535, 509, 522, 471, 525, 537, 527, 499, 491, 522, 513, 482, 511, 505, 525, 485, 517, 512, 561, 481, 510, 529, 548, 502, 523, 515, 450, 500, 506, 515, 511, 498, 541, 511, 545, 510, 501, 542, 494, 512, 525, 484, 531, 506, 476, 539, 517, 484, 472, 500, 516, 496, 512, 476, 513, 533, 514, 504, 484, 535, 522, 478, 521, 521, 522, 472, 529, 489, 513, 508, 537, 515, 509, 541, 522, 488, 518, 522, 546, 539, 513, 496, 533, 472, 529, 540, 519, 514, 535, 522, 476, 504, 535, 507, 479, 524, 539, 498, 511, 489, 523, 506, 515, 483, 526, 517, 512, 553, 499, 541, 524, 507, 552, 512, 524, 527, 520, 480, 472, 486, 505, 509, 526, 496, 535, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[474, 497, 524, 488, 533, 506, 485, 479, 514, 507, 509, 492, 512, 549, 518, 465, 504, 497, 532, 485, 562, 517, 498, 529, 490, 503, 506, 516, 540, 486, 516, 489, 520, 545, 531, 533, 519, 489, 504, 493, 546, 493, 526, 521, 520, 520, 523, 546, 475, 509, 520, 482, 507, 504, 483, 514, 504, 479, 552, 509, 508, 527, 497, 502, 529, 507, 526, 525, 512, 522, 526, 497, 515, 562, 502, 495, 539, 478, 468, 541, 504, 514, 517, 525, 525, 485, 529, 518, 510, 506, 505, 492, 520, 512, 500, 483, 495, 532, 514, 499, 519, 504, 530, 531, 499, 539, 556, 564, 518, 509, 528, 520, 516, 528, 490, 529, 502, 507, 525, 470, 509, 525, 504, 468, 513, 573, 479, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[506, 533, 511, 539, 516, 543, 501, 513, 501, 493, 498, 532, 520, 523, 504, 530, 506, 526, 543, 514, 554, 476, 529, 474, 520, 539, 538, 517, 503, 543, 472, 560, 522, 490, 515, 489, 545, 496, 507, 509, 500, 510, 510, 490, 493, 497, 493, 507, 534, 482, 549, 520, 536, 510, 503, 521, 523, 506, 500, 517, 495, 537, 517, 516, 532, 504, 486, 528, 566, 501, 514, 512, 474, 535, 489, 517, 498, 536, 513, 486, 559, 515, 519, 493, 471, 536, 510, 514, 485, 502, 499, 489, 533, 507, 497, 489, 463, 500, 502, 525, 508, 524, 517, 514, 523, 485, 505, 519, 518, 504, 545, 516, 488, 465, 522, 514, 509, 493, 530, 477, 516, 553, 509, 524, 503, 545, 464, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[516, 569, 534, 447, 521, 513, 552, 516, 477, 518, 479, 505, 498, 462, 542, 545, 497, 480, 513, 550, 507, 536, 530, 496, 530, 498, 534, 522, 531, 516, 500, 528, 526, 504, 545, 528, 508, 544, 496, 538, 515, 479, 512, 516, 498, 492, 493, 513, 512, 503, 513, 502, 479, 514, 515, 528, 529, 518, 496, 500, 511, 482, 539, 520, 530, 494, 501, 494, 506, 501, 505, 498, 523, 531, 488, 494, 534, 510, 511, 540, 492, 498, 534, 494, 448, 497, 519, 488, 527, 530, 533, 534, 508, 528, 522, 479, 524, 496, 528, 457, 530, 493, 524, 524, 537, 502, 507, 546, 525, 504, 505, 502, 459, 536, 508, 514, 534, 533, 504, 466, 498, 511, 545, 534, 547, 470, 545, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[486, 519, 518, 500, 472, 508, 528, 507, 502, 511, 478, 510, 535, 487, 512, 524, 513, 527, 517, 494, 511, 473, 503, 529, 535, 493, 550, 497, 537, 493, 529, 500, 553, 531, 480, 517, 513, 539, 516, 531, 536, 521, 499, 508, 499, 495, 502, 499, 501, 514, 486, 541, 530, 475, 533, 508, 523, 556, 479, 526, 504, 511, 503, 524, 500, 551, 508, 529, 487, 500, 508, 505, 503, 491, 504, 491, 537, 544, 512, 491, 490, 521, 499, 542, 540, 515, 466, 495, 508, 521, 524, 477, 568, 489, 511, 526, 482, 527, 523, 492, 517, 534, 494, 531, 496, 518, 487, 521, 546, 531, 515, 533, 512, 527, 492, 486, 535, 520, 554, 497, 493, 546, 471, 533, 531, 476, 491, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[479, 519, 536, 487, 501, 501, 484, 521, 542, 531, 543, 536, 524, 519, 513, 558, 513, 471, 497, 530, 534, 513, 571, 516, 525, 483, 535, 528, 509, 516, 526, 498, 508, 484, 514, 506, 560, 533, 477, 495, 502, 499, 533, 546, 517, 524, 509, 526, 521, 525, 532, 535, 495, 493, 509, 524, 510, 502, 478, 508, 512, 520, 524, 504, 506, 488, 515, 499, 492, 513, 495, 504, 502, 478, 553, 524, 520, 518, 535, 544, 515, 511, 524, 476, 520, 517, 508, 502, 513, 491, 484, 524, 471, 510, 469, 516, 542, 474, 516, 501, 485, 481, 502, 537, 528, 528, 502, 523, 511, 533, 514, 497, 519, 522, 467, 511, 541, 529, 554, 479, 438, 501, 507, 546, 539, 471, 523, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[506, 540, 519, 533, 499, 531, 490, 531, 503, 499, 490, 510, 534, 501, 529, 485, 533, 476, 500, 516, 525, 534, 550, 492, 492, 511, 515, 475, 512, 480, 493, 475, 513, 537, 525, 518, 507, 524, 476, 512, 489, 500, 499, 529, 505, 534, 489, 515, 534, 529, 510, 500, 508, 511, 519, 520, 504, 510, 503, 544, 498, 515, 521, 524, 482, 518, 486, 521, 529, 512, 499, 483, 497, 503, 491, 497, 478, 514, 561, 555, 498, 521, 541, 534, 490, 481, 525, 526, 525, 518, 515, 515, 514, 510, 539, 478, 519, 481, 501, 498, 504, 511, 517, 519, 518, 519, 508, 490, 517, 471, 538, 530, 520, 499, 499, 518, 524, 565, 526, 498, 537, 518, 485, 520, 543, 538, 538, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[512, 534, 515, 527, 566, 493, 521, 522, 491, 524, 486, 512, 506, 529, 519, 488, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 527, 518, 506, 486, 506, 482, 569, 472, 529, 504, 498, 518, 501, 528, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 537, 516, 507, 504, 545, 496, 527, 495, 498, 531, 520, 470, 498, 495, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 485, 507, 531, 526, 515, 526, 506, 508, 517, 499, 554, 479, 528, 524, 506, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 515, 483, 496, 463, 504, 492, 511, 513, 534, 499, 512, 504, 499, 517, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 504, 532, 502, 519, 516, 497, 518, 530, 514, 528, 520, 506, 486, 558, 509, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 498, 483, 532, 510, 533, 500, 494, 547, 499, 540, 489, 524, 489, 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 539, 549, 501, 513, 494, 542, 490, 485, 515, 508, 487, 535, 499, 475, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8228, 8212, 8211, 8088, 8235, 8170, 8142] avg=8192.0000 max=8250 max_rank=0 min=8088 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[516, 569, 534, 447, 521, 513, 552, 516, 477, 518, 479, 505, 498, 462, 542, 545, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 480, 513, 550, 486, 536, 530, 496, 530, 498, 534, 522, 531, 516, 500, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 504, 545, 528, 487, 544, 496, 538, 515, 479, 512, 516, 498, 492, 493, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 503, 513, 502, 479, 514, 515, 528, 529, 518, 496, 500, 511, 482, 539, 520, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 494, 501, 494, 506, 501, 505, 498, 523, 531, 488, 494, 534, 510, 511, 540, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 498, 534, 494, 448, 497, 519, 488, 527, 530, 533, 534, 508, 528, 522, 479, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 528, 457, 530, 493, 524, 524, 537, 502, 507, 546, 506, 504, 505, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 536, 508, 514, 534, 533, 504, 466, 498, 511, 545, 534, 547, 470, 545, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8247, 8186, 8177, 8180, 8151, 8185, 8211] avg=8192.0000 max=8247 max_rank=1 min=8151 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[517, 485, 535, 509, 522, 471, 525, 537, 527, 499, 491, 522, 513, 482, 511, 505, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 485, 517, 512, 538, 481, 510, 529, 548, 502, 523, 515, 450, 500, 506, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 498, 541, 511, 523, 510, 501, 542, 494, 512, 525, 484, 531, 506, 476, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 484, 472, 500, 516, 496, 512, 476, 513, 533, 514, 504, 484, 535, 522, 478, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 521, 522, 472, 529, 489, 513, 508, 537, 515, 509, 541, 522, 488, 518, 522, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 539, 513, 496, 533, 472, 529, 540, 519, 514, 535, 522, 476, 504, 535, 507, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 524, 539, 498, 511, 489, 523, 506, 515, 483, 526, 517, 495, 553, 499, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 552, 512, 524, 527, 520, 480, 472, 486, 505, 509, 526, 496, 535, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8156, 8204, 8071, 8248, 8302, 8198, 8202] avg=8192.0000 max=8302 max_rank=5 min=8071 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[479, 519, 536, 487, 501, 501, 484, 521, 542, 531, 543, 536, 524, 519, 513, 558, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 471, 497, 530, 499, 513, 571, 516, 525, 483, 535, 528, 509, 516, 526, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 484, 514, 506, 532, 533, 477, 495, 502, 499, 533, 546, 517, 524, 509, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 525, 532, 535, 495, 493, 509, 524, 510, 502, 478, 508, 512, 520, 524, 504, 18, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 488, 515, 499, 492, 513, 495, 504, 502, 478, 553, 524, 520, 518, 535, 544, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 524, 476, 520, 517, 508, 502, 513, 491, 484, 524, 471, 510, 469, 516, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 474, 516, 501, 485, 481, 502, 537, 528, 528, 502, 523, 477, 533, 514, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 522, 467, 511, 541, 529, 554, 479, 438, 501, 507, 546, 539, 471, 523, 494, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8230, 8205, 8214, 8218, 8078, 8140, 8148] avg=8192.0000 max=8303 max_rank=0 min=8078 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[474, 497, 524, 488, 533, 506, 485, 479, 514, 507, 509, 492, 512, 549, 518, 465, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 532, 485, 539, 517, 498, 529, 490, 503, 506, 516, 540, 486, 516, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 545, 531, 533, 498, 489, 504, 493, 546, 493, 526, 521, 520, 520, 523, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 509, 520, 482, 507, 504, 483, 514, 504, 479, 552, 509, 508, 527, 497, 502, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 526, 525, 512, 522, 526, 497, 515, 562, 502, 495, 539, 478, 468, 541, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 517, 525, 525, 485, 529, 518, 510, 506, 505, 492, 520, 512, 500, 483, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 532, 514, 499, 519, 504, 530, 531, 499, 539, 556, 564, 501, 509, 528, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 528, 490, 529, 502, 507, 525, 470, 509, 525, 504, 468, 513, 573, 479, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8056, 8147, 8308, 8087, 8264, 8167, 8340, 8167] avg=8192.0000 max=8340 max_rank=6 min=8056 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[506, 533, 511, 539, 516, 543, 501, 513, 501, 493, 498, 532, 520, 523, 504, 530, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 526, 543, 514, 531, 476, 529, 474, 520, 539, 538, 517, 503, 543, 472, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 515, 489, 523, 496, 507, 509, 500, 510, 510, 490, 493, 497, 493, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 482, 549, 520, 536, 510, 503, 521, 523, 506, 500, 517, 495, 537, 517, 516, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 486, 528, 566, 501, 514, 512, 474, 535, 489, 517, 498, 536, 513, 486, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 515, 519, 493, 471, 536, 510, 514, 485, 502, 499, 489, 533, 507, 497, 489, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 500, 502, 525, 508, 524, 517, 514, 523, 485, 505, 519, 501, 504, 545, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 465, 522, 514, 509, 493, 530, 477, 516, 553, 509, 524, 503, 545, 464, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8291, 8051, 8281, 8212, 8140, 8151, 8143] avg=8192.0000 max=8291 max_rank=1 min=8051 min_rank=2 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[486, 519, 518, 500, 472, 508, 528, 507, 502, 511, 478, 510, 535, 487, 512, 524, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 527, 517, 494, 490, 473, 503, 529, 535, 493, 550, 497, 537, 493, 529, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 531, 480, 517, 492, 539, 516, 531, 536, 521, 499, 508, 499, 495, 502, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 514, 486, 541, 530, 475, 533, 508, 523, 556, 479, 526, 504, 511, 503, 524, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 551, 508, 529, 487, 500, 508, 505, 503, 491, 504, 491, 537, 544, 512, 491, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 521, 499, 542, 540, 515, 466, 495, 508, 521, 524, 477, 568, 489, 511, 526, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 527, 523, 492, 517, 534, 494, 531, 496, 518, 487, 521, 527, 531, 515, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 492, 486, 535, 520, 554, 497, 493, 546, 471, 533, 531, 476, 491, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8180, 8218, 8230, 8181, 8212, 8228, 8185] avg=8192.0000 max=8230 max_rank=3 min=8102 min_rank=0 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[506, 540, 519, 533, 499, 531, 490, 531, 503, 499, 490, 510, 534, 501, 529, 485, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 476, 500, 516, 503, 534, 550, 492, 492, 511, 515, 475, 512, 480, 493, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 525, 518, 487, 524, 476, 512, 489, 500, 499, 529, 505, 534, 489, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 529, 510, 500, 508, 511, 519, 520, 504, 510, 503, 544, 498, 515, 521, 524, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 518, 486, 521, 529, 512, 499, 483, 497, 503, 491, 497, 478, 514, 561, 555, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 521, 541, 534, 490, 481, 525, 526, 525, 518, 515, 515, 514, 510, 539, 478, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 481, 501, 498, 504, 511, 517, 519, 518, 519, 508, 490, 500, 471, 538, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 499, 518, 524, 565, 526, 498, 537, 518, 485, 520, 543, 538, 538, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8057, 8152, 8265, 8145, 8251, 8124, 8338] avg=8192.0000 max=8338 max_rank=7 min=8057 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4352, 3968, 3968, 4224, 4096, 4224, 4352, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True + [2026-08-26 16:29:45] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 645.8 | throughput per GPU (TFLOP/s/GPU): 280.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117269E+01 | loss scale: 1.0 | grad norm: 44.201 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[518, 489, 504, 511, 495, 507, 533, 542, 522, 533, 505, 523, 493, 516, 524, 503, 506, 531, 493, 488, 511, 535, 526, 510, 465, 505, 528, 554, 517, 506, 490, 541, 468, 503, 519, 497, 552, 525, 491, 524, 515, 526, 513, 513, 511, 499, 483, 505, 495, 524, 490, 530, 498, 515, 502, 490, 538, 512, 498, 527, 541, 507, 507, 498, 473, 466, 528, 539, 544, 510, 546, 487, 531, 549, 534, 502, 510, 498, 522, 508, 520, 499, 526, 519, 520, 529, 506, 542, 500, 473, 508, 558, 553, 460, 543, 564, 484, 496, 501, 525, 494, 509, 529, 489, 475, 506, 495, 525, 519, 506, 488, 475, 511, 516, 513, 491, 543, 540, 497, 509, 474, 502, 502, 525, 507, 517, 559, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[501, 500, 526, 483, 534, 511, 500, 528, 515, 514, 515, 552, 513, 517, 480, 521, 494, 514, 509, 513, 501, 474, 519, 520, 490, 526, 556, 474, 523, 511, 549, 495, 518, 525, 508, 524, 524, 493, 508, 497, 509, 528, 511, 493, 505, 498, 468, 492, 506, 523, 526, 512, 516, 549, 489, 497, 514, 512, 488, 509, 515, 489, 478, 531, 485, 507, 487, 501, 514, 512, 517, 486, 484, 525, 532, 490, 500, 509, 501, 510, 548, 519, 491, 539, 525, 525, 554, 493, 522, 503, 519, 532, 528, 506, 554, 540, 536, 547, 534, 523, 535, 527, 502, 467, 507, 513, 486, 483, 508, 500, 532, 542, 508, 490, 548, 512, 513, 508, 514, 514, 503, 513, 482, 540, 544, 535, 494, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[529, 547, 520, 516, 495, 505, 543, 495, 553, 524, 513, 485, 471, 520, 515, 507, 531, 516, 541, 502, 529, 522, 530, 497, 500, 505, 523, 558, 494, 533, 527, 460, 493, 528, 497, 473, 546, 523, 557, 535, 509, 501, 504, 511, 556, 567, 487, 540, 501, 495, 522, 483, 502, 509, 530, 487, 511, 472, 539, 465, 516, 547, 491, 486, 507, 522, 530, 530, 558, 504, 504, 524, 498, 516, 540, 501, 511, 499, 502, 478, 506, 512, 510, 484, 475, 512, 532, 473, 514, 510, 535, 565, 488, 488, 541, 534, 480, 531, 509, 527, 465, 521, 484, 484, 502, 508, 509, 538, 519, 511, 490, 521, 514, 495, 501, 514, 476, 505, 541, 510, 510, 508, 551, 494, 495, 489, 537, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[519, 524, 499, 544, 511, 504, 509, 498, 519, 495, 515, 518, 522, 489, 502, 499, 522, 493, 538, 508, 544, 517, 528, 473, 494, 523, 487, 515, 512, 531, 473, 506, 512, 483, 511, 518, 517, 539, 500, 508, 542, 519, 527, 548, 495, 495, 496, 522, 519, 507, 523, 483, 488, 500, 484, 506, 535, 541, 513, 501, 465, 523, 527, 497, 513, 516, 494, 497, 530, 496, 484, 501, 504, 512, 515, 532, 484, 510, 475, 509, 537, 507, 516, 495, 478, 503, 474, 514, 540, 577, 458, 537, 542, 480, 533, 506, 543, 547, 544, 489, 487, 477, 525, 543, 520, 492, 522, 527, 549, 569, 482, 530, 552, 498, 534, 512, 545, 506, 492, 493, 496, 517, 509, 518, 508, 527, 536, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[504, 564, 487, 453, 516, 524, 529, 499, 497, 498, 508, 518, 506, 508, 485, 501, 519, 466, 498, 529, 466, 478, 477, 542, 545, 502, 519, 516, 518, 528, 488, 528, 464, 500, 520, 549, 526, 533, 522, 548, 500, 505, 501, 511, 539, 481, 516, 515, 537, 496, 491, 502, 504, 547, 519, 511, 488, 508, 519, 490, 529, 485, 531, 527, 497, 498, 488, 497, 524, 496, 510, 504, 553, 506, 475, 521, 512, 485, 512, 504, 557, 491, 473, 514, 487, 494, 504, 566, 524, 530, 522, 512, 500, 505, 534, 509, 498, 524, 505, 524, 521, 554, 523, 523, 503, 531, 543, 532, 510, 472, 515, 509, 497, 557, 532, 516, 530, 517, 486, 525, 533, 525, 527, 497, 526, 521, 494, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[526, 549, 535, 507, 521, 472, 554, 472, 516, 501, 515, 518, 539, 516, 527, 512, 495, 521, 484, 529, 501, 487, 525, 529, 515, 515, 523, 520, 506, 502, 486, 508, 486, 545, 510, 480, 502, 497, 534, 486, 498, 525, 493, 532, 479, 477, 523, 509, 531, 472, 537, 506, 497, 493, 521, 511, 531, 471, 561, 554, 481, 485, 478, 542, 499, 509, 501, 537, 502, 488, 544, 493, 533, 539, 476, 495, 522, 522, 509, 493, 539, 538, 506, 515, 530, 492, 528, 475, 493, 491, 517, 516, 490, 516, 515, 517, 528, 551, 498, 501, 498, 498, 479, 529, 503, 504, 517, 519, 521, 533, 513, 506, 517, 506, 510, 524, 548, 482, 551, 541, 536, 520, 512, 499, 496, 498, 538, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[479, 489, 535, 482, 514, 497, 522, 507, 512, 520, 532, 485, 500, 515, 486, 501, 526, 496, 492, 505, 495, 507, 565, 525, 491, 528, 476, 526, 513, 559, 526, 509, 512, 515, 511, 540, 531, 538, 506, 489, 493, 503, 519, 538, 516, 537, 510, 528, 512, 518, 514, 492, 507, 515, 479, 534, 504, 538, 453, 478, 514, 528, 547, 499, 512, 581, 505, 532, 475, 487, 490, 555, 501, 520, 528, 455, 500, 568, 457, 527, 495, 497, 479, 548, 517, 507, 545, 499, 540, 529, 496, 520, 522, 492, 514, 476, 510, 473, 507, 552, 475, 487, 525, 557, 514, 513, 530, 532, 536, 518, 501, 505, 515, 543, 487, 512, 517, 518, 533, 485, 516, 506, 483, 477, 559, 510, 514, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[483, 477, 491, 507, 521, 473, 536, 467, 493, 497, 467, 475, 513, 537, 469, 482, 520, 509, 524, 515, 474, 515, 532, 506, 462, 533, 524, 572, 497, 528, 544, 473, 505, 528, 509, 481, 532, 474, 521, 507, 511, 525, 546, 511, 514, 546, 458, 495, 501, 471, 491, 487, 456, 497, 516, 514, 569, 501, 510, 515, 512, 521, 524, 516, 542, 544, 557, 526, 530, 498, 525, 494, 506, 532, 509, 470, 499, 519, 524, 505, 527, 498, 517, 566, 519, 505, 524, 559, 485, 521, 534, 531, 493, 484, 489, 517, 553, 493, 502, 490, 509, 538, 521, 555, 522, 522, 520, 548, 503, 515, 466, 502, 493, 513, 492, 528, 521, 509, 551, 586, 543, 517, 554, 523, 500, 524, 486, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[519, 524, 499, 544, 511, 504, 509, 498, 519, 495, 515, 518, 522, 489, 502, 499, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 493, 538, 508, 544, 517, 528, 473, 494, 523, 487, 515, 512, 531, 473, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 483, 511, 518, 515, 539, 500, 508, 542, 519, 527, 548, 495, 495, 496, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 523, 483, 488, 500, 484, 506, 535, 541, 513, 501, 465, 523, 527, 497, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 516, 494, 497, 530, 496, 484, 501, 504, 512, 515, 532, 484, 510, 475, 509, 18, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 516, 495, 478, 503, 474, 514, 540, 577, 458, 487, 542, 480, 533, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 547, 544, 489, 487, 477, 525, 543, 520, 492, 522, 509, 549, 569, 482, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 498, 534, 512, 486, 506, 492, 493, 496, 517, 509, 518, 508, 527, 536, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8164, 8230, 8171, 8097, 8147, 8328, 8187] avg=8192.0000 max=8328 max_rank=6 min=8097 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[529, 547, 520, 516, 495, 505, 543, 495, 553, 524, 513, 485, 471, 520, 515, 507, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 516, 541, 502, 529, 522, 530, 497, 500, 505, 523, 558, 494, 533, 527, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 528, 497, 473, 544, 523, 557, 535, 509, 501, 504, 511, 556, 567, 487, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 495, 522, 483, 502, 509, 530, 487, 511, 472, 539, 465, 516, 547, 491, 486, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 522, 530, 530, 558, 504, 504, 524, 498, 516, 540, 501, 511, 499, 502, 478, 18, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 510, 484, 475, 512, 532, 473, 514, 510, 535, 513, 488, 488, 541, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 531, 509, 527, 465, 521, 484, 484, 502, 508, 509, 520, 519, 511, 490, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 495, 501, 514, 425, 505, 541, 510, 510, 508, 551, 494, 495, 489, 537, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8268, 8325, 8107, 8249, 8127, 8081, 8094] avg=8192.0000 max=8325 max_rank=2 min=8081 min_rank=6 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[479, 489, 535, 482, 514, 497, 522, 507, 512, 520, 532, 485, 500, 515, 486, 501, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 492, 505, 495, 507, 565, 525, 491, 528, 476, 526, 513, 559, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 511, 540, 529, 538, 506, 489, 493, 503, 519, 538, 516, 537, 510, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 514, 492, 507, 515, 479, 534, 504, 538, 453, 478, 514, 528, 547, 499, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 581, 505, 532, 475, 487, 490, 555, 501, 520, 528, 455, 500, 568, 457, 527, 18, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 479, 548, 517, 507, 545, 499, 540, 529, 496, 472, 522, 492, 514, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 473, 507, 552, 475, 487, 525, 557, 514, 513, 530, 514, 536, 518, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 543, 487, 512, 462, 518, 533, 485, 516, 506, 483, 477, 559, 510, 514, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8239, 8284, 8187, 8218, 8128, 8217, 8144] avg=8192.0000 max=8284 max_rank=2 min=8119 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[504, 564, 487, 453, 516, 524, 529, 499, 497, 498, 508, 518, 506, 508, 485, 501, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 466, 498, 529, 466, 478, 477, 542, 545, 502, 519, 514, 518, 528, 488, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 500, 520, 549, 518, 533, 522, 548, 500, 505, 501, 511, 539, 481, 516, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 496, 491, 502, 504, 547, 519, 511, 488, 508, 519, 490, 529, 485, 531, 527, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 488, 497, 524, 496, 510, 504, 553, 506, 475, 521, 512, 485, 512, 504, 21, 10, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 491, 473, 514, 487, 494, 504, 566, 524, 530, 522, 456, 500, 505, 534, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 505, 524, 521, 554, 523, 523, 503, 531, 543, 511, 510, 472, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 557, 532, 516, 465, 517, 486, 525, 533, 525, 527, 497, 526, 521, 494, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8117, 8222, 8245, 8123, 8166, 8266, 8250] avg=8192.0000 max=8266 max_rank=6 min=8117 min_rank=1 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 489, 504, 511, 495, 507, 533, 542, 522, 533, 505, 523, 493, 516, 524, 503, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 531, 493, 488, 511, 535, 526, 510, 465, 505, 528, 554, 517, 506, 490, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 503, 519, 497, 549, 525, 491, 524, 515, 526, 513, 513, 511, 499, 483, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 490, 530, 498, 515, 502, 490, 538, 512, 498, 527, 541, 507, 507, 498, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 466, 528, 539, 544, 510, 546, 487, 531, 549, 534, 502, 510, 498, 522, 508, 18, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 526, 519, 520, 529, 506, 542, 500, 473, 508, 507, 553, 460, 543, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 496, 501, 525, 494, 509, 529, 489, 475, 506, 495, 507, 519, 506, 488, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 516, 513, 491, 485, 540, 497, 509, 474, 502, 502, 525, 507, 517, 559, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8206, 8141, 8230, 8273, 8269, 7998, 8155] avg=8192.0000 max=8273 max_rank=4 min=7998 min_rank=6 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[501, 500, 526, 483, 534, 511, 500, 528, 515, 514, 515, 552, 513, 517, 480, 521, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 514, 509, 513, 501, 474, 519, 520, 490, 526, 556, 474, 523, 511, 549, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 508, 524, 522, 493, 508, 497, 509, 528, 511, 493, 505, 498, 468, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 526, 512, 516, 549, 489, 497, 514, 512, 488, 509, 515, 489, 478, 531, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 507, 487, 501, 514, 512, 517, 486, 484, 525, 532, 490, 500, 509, 501, 510, 16, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 519, 491, 539, 525, 525, 554, 493, 522, 503, 519, 483, 528, 506, 554, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 547, 534, 523, 535, 527, 502, 467, 507, 513, 486, 467, 508, 500, 532, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 548, 512, 458, 508, 514, 514, 503, 513, 482, 540, 544, 535, 494, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8168, 8099, 8209, 8083, 8349, 8226, 8148] avg=8192.0000 max=8349 max_rank=5 min=8083 min_rank=4 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 549, 535, 507, 521, 472, 554, 472, 516, 501, 515, 518, 539, 516, 527, 512, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 521, 484, 529, 501, 487, 525, 529, 515, 515, 523, 520, 506, 502, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 545, 510, 480, 500, 497, 534, 486, 498, 525, 493, 532, 479, 477, 523, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 472, 537, 506, 497, 493, 521, 511, 531, 471, 561, 554, 481, 485, 478, 542, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 501, 537, 502, 488, 544, 493, 533, 539, 476, 495, 522, 522, 509, 493, 18, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 538, 506, 515, 530, 492, 528, 475, 493, 491, 517, 468, 490, 516, 515, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 551, 498, 501, 498, 498, 479, 529, 503, 504, 517, 501, 521, 533, 513, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 506, 510, 524, 489, 482, 551, 541, 536, 520, 512, 499, 496, 498, 538, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8146, 8074, 8230, 8187, 8130, 8180, 8266] avg=8192.0000 max=8323 max_rank=0 min=8074 min_rank=2 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[483, 477, 491, 507, 521, 473, 536, 467, 493, 497, 467, 475, 513, 537, 469, 482, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 524, 515, 474, 515, 532, 506, 462, 533, 524, 572, 497, 528, 544, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 509, 481, 530, 474, 521, 507, 511, 525, 546, 511, 514, 546, 458, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 471, 491, 487, 456, 497, 516, 514, 569, 501, 510, 515, 512, 521, 524, 516, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 544, 557, 526, 530, 498, 525, 494, 506, 532, 509, 470, 499, 519, 524, 505, 19, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 498, 517, 566, 519, 505, 524, 559, 485, 521, 534, 482, 493, 484, 489, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 493, 502, 490, 509, 538, 521, 555, 522, 522, 520, 529, 503, 515, 466, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 492, 528, 465, 509, 551, 586, 543, 517, 554, 523, 500, 524, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7932, 8228, 8161, 8157, 8306, 8220, 8240, 8292] avg=8192.0000 max=8306 max_rank=4 min=7932 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=33058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4352, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 3968, 4224, 4224, 4352, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4352, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 3840, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[499, 508, 510, 539, 486, 519, 511, 574, 507, 498, 536, 518, 466, 469, 464, 480, 537, 530, 507, 475, 558, 520, 479, 509, 498, 502, 549, 501, 515, 543, 560, 490, 496, 490, 514, 476, 516, 527, 503, 487, 506, 540, 487, 460, 564, 483, 523, 519, 507, 495, 502, 548, 546, 501, 534, 502, 510, 539, 506, 529, 479, 517, 510, 481, 527, 497, 546, 507, 541, 540, 500, 506, 496, 501, 497, 526, 545, 516, 530, 526, 516, 544, 495, 495, 505, 494, 508, 504, 474, 477, 520, 510, 519, 508, 501, 510, 522, 513, 532, 490, 509, 507, 485, 522, 531, 508, 463, 554, 528, 520, 519, 474, 517, 538, 516, 556, 490, 507, 525, 530, 532, 497, 505, 497, 491, 562, 543, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[514, 508, 514, 472, 490, 520, 543, 475, 521, 500, 510, 483, 511, 475, 503, 505, 522, 525, 558, 499, 559, 504, 565, 487, 516, 527, 492, 516, 514, 519, 516, 510, 525, 487, 503, 490, 511, 492, 521, 544, 527, 538, 513, 531, 499, 466, 516, 552, 491, 497, 521, 508, 543, 464, 481, 527, 488, 502, 499, 528, 505, 523, 514, 500, 511, 538, 470, 510, 475, 495, 535, 516, 549, 482, 522, 512, 506, 536, 535, 500, 506, 509, 510, 527, 524, 509, 484, 476, 510, 526, 507, 488, 562, 516, 487, 506, 494, 518, 500, 512, 555, 498, 528, 496, 531, 525, 527, 504, 490, 491, 532, 516, 534, 511, 509, 516, 505, 531, 527, 531, 467, 538, 519, 527, 552, 553, 477, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[490, 478, 523, 502, 527, 491, 518, 512, 521, 455, 510, 477, 526, 498, 479, 505, 527, 495, 504, 517, 473, 500, 514, 516, 528, 556, 486, 521, 504, 503, 532, 498, 512, 490, 484, 526, 544, 550, 515, 539, 494, 537, 500, 539, 509, 500, 518, 510, 534, 503, 476, 505, 525, 507, 503, 544, 493, 541, 515, 507, 526, 487, 502, 517, 516, 501, 517, 486, 527, 521, 524, 555, 542, 510, 503, 551, 516, 519, 518, 551, 503, 492, 528, 526, 523, 522, 505, 544, 528, 501, 530, 544, 505, 500, 481, 528, 491, 481, 521, 501, 485, 510, 495, 482, 518, 491, 518, 512, 504, 506, 501, 531, 526, 495, 522, 513, 491, 526, 520, 499, 518, 503, 518, 528, 481, 544, 499, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[530, 531, 518, 510, 526, 520, 496, 514, 542, 477, 527, 516, 474, 495, 505, 528, 540, 504, 487, 473, 506, 498, 548, 513, 500, 512, 553, 503, 526, 512, 528, 499, 515, 550, 541, 490, 535, 477, 487, 520, 524, 488, 504, 504, 510, 522, 536, 541, 471, 511, 511, 533, 538, 509, 492, 508, 554, 514, 536, 511, 520, 487, 497, 505, 511, 484, 523, 528, 529, 484, 513, 489, 501, 484, 532, 532, 521, 519, 488, 498, 520, 499, 545, 486, 520, 533, 482, 502, 515, 492, 491, 530, 508, 509, 511, 496, 485, 532, 484, 497, 551, 491, 508, 472, 542, 499, 521, 538, 512, 496, 519, 505, 502, 515, 522, 518, 492, 541, 543, 500, 488, 545, 504, 526, 507, 500, 553, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[495, 517, 514, 514, 511, 495, 498, 497, 526, 521, 499, 519, 523, 505, 482, 480, 532, 521, 527, 514, 522, 506, 492, 506, 513, 519, 507, 519, 555, 497, 550, 535, 527, 496, 508, 541, 519, 473, 543, 529, 516, 527, 484, 533, 502, 532, 535, 503, 497, 537, 506, 540, 549, 502, 515, 485, 527, 496, 495, 502, 541, 504, 488, 523, 521, 563, 479, 522, 484, 547, 466, 528, 503, 539, 527, 531, 511, 529, 519, 498, 496, 500, 525, 529, 521, 524, 540, 525, 507, 525, 506, 510, 487, 483, 504, 515, 537, 516, 514, 475, 494, 512, 506, 551, 516, 526, 505, 526, 532, 515, 509, 514, 479, 476, 526, 480, 530, 489, 492, 507, 450, 492, 531, 529, 498, 511, 464, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[472, 539, 549, 535, 519, 506, 500, 519, 506, 527, 478, 520, 504, 521, 474, 492, 497, 501, 481, 538, 518, 507, 546, 485, 515, 536, 486, 518, 506, 451, 506, 504, 520, 480, 547, 563, 486, 531, 502, 516, 474, 505, 533, 502, 519, 480, 520, 540, 485, 475, 552, 504, 487, 518, 499, 523, 518, 490, 505, 484, 500, 526, 519, 543, 547, 522, 488, 496, 532, 519, 503, 526, 546, 488, 520, 548, 506, 506, 457, 489, 512, 507, 503, 529, 494, 523, 499, 516, 520, 553, 549, 492, 543, 514, 507, 521, 505, 482, 535, 526, 540, 495, 500, 527, 523, 510, 507, 513, 523, 524, 508, 541, 530, 514, 500, 506, 532, 505, 497, 505, 525, 516, 507, 524, 511, 483, 501, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[489, 503, 516, 494, 505, 523, 530, 528, 565, 472, 492, 508, 566, 538, 529, 521, 491, 527, 506, 528, 500, 532, 537, 534, 492, 490, 522, 483, 535, 515, 492, 531, 474, 496, 516, 500, 523, 502, 554, 498, 544, 526, 510, 533, 535, 469, 514, 480, 491, 504, 462, 504, 495, 475, 508, 475, 512, 498, 506, 506, 516, 502, 499, 508, 502, 490, 524, 520, 516, 481, 472, 549, 494, 481, 501, 537, 519, 533, 552, 516, 519, 522, 499, 533, 493, 519, 537, 525, 549, 543, 516, 504, 515, 516, 525, 510, 520, 493, 512, 501, 485, 499, 523, 511, 495, 537, 546, 478, 508, 526, 472, 555, 471, 502, 564, 524, 522, 507, 514, 495, 519, 529, 518, 513, 517, 519, 503, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[469, 516, 544, 539, 500, 524, 514, 465, 501, 497, 495, 527, 531, 520, 534, 485, 522, 517, 518, 486, 509, 485, 503, 486, 503, 545, 512, 522, 500, 513, 542, 490, 558, 531, 542, 518, 537, 504, 529, 524, 518, 517, 511, 514, 530, 518, 507, 516, 504, 526, 540, 492, 532, 505, 516, 504, 541, 524, 487, 571, 546, 517, 502, 513, 505, 499, 494, 496, 521, 544, 482, 487, 488, 485, 467, 501, 512, 530, 490, 528, 492, 504, 496, 529, 520, 471, 550, 541, 489, 514, 524, 533, 524, 520, 483, 524, 483, 510, 513, 518, 514, 474, 493, 497, 527, 504, 498, 515, 535, 502, 511, 498, 504, 472, 494, 522, 537, 519, 567, 525, 478, 546, 461, 530, 500, 525, 509, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 508, 514, 472, 490, 520, 543, 475, 521, 500, 510, 483, 511, 475, 503, 505, 40, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 558, 499, 559, 504, 565, 487, 516, 527, 492, 516, 514, 519, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 487, 503, 490, 511, 492, 521, 544, 527, 498, 513, 531, 499, 466, 516, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 497, 521, 508, 543, 464, 481, 527, 488, 502, 499, 528, 505, 523, 514, 500, 19, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 538, 470, 510, 475, 495, 535, 516, 549, 482, 522, 497, 506, 536, 535, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 510, 527, 524, 509, 484, 476, 510, 526, 507, 488, 548, 516, 487, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 518, 500, 512, 555, 498, 528, 496, 531, 525, 527, 504, 490, 491, 532, 516, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 509, 516, 505, 531, 527, 531, 467, 538, 519, 527, 552, 534, 477, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8303, 8175, 8121, 8177, 8133, 8240, 8282] avg=8192.0000 max=8303 max_rank=1 min=8105 min_rank=0 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[530, 531, 518, 510, 526, 520, 496, 514, 542, 477, 527, 516, 474, 495, 505, 528, 36, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 504, 487, 473, 506, 498, 548, 513, 500, 512, 553, 503, 526, 512, 502, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 550, 541, 490, 535, 477, 487, 520, 524, 452, 504, 504, 510, 522, 536, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 511, 511, 533, 538, 509, 492, 508, 554, 514, 536, 511, 520, 487, 497, 505, 17, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 484, 523, 528, 529, 484, 513, 489, 501, 484, 532, 517, 521, 519, 488, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 545, 486, 520, 533, 482, 502, 515, 492, 491, 530, 495, 509, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 532, 484, 497, 551, 491, 508, 472, 542, 499, 521, 538, 512, 496, 519, 505, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 522, 518, 492, 541, 543, 500, 488, 545, 504, 526, 507, 483, 553, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8176, 8208, 8225, 8121, 8126, 8174, 8240] avg=8192.0000 max=8266 max_rank=0 min=8121 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[499, 508, 510, 539, 486, 519, 511, 574, 507, 498, 536, 518, 466, 469, 464, 480, 45, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 507, 475, 558, 520, 479, 509, 498, 502, 549, 501, 515, 543, 523, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 490, 514, 476, 516, 527, 503, 487, 506, 495, 487, 460, 564, 483, 523, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 502, 548, 546, 501, 534, 502, 510, 539, 506, 529, 479, 517, 510, 481, 23, 9, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 497, 546, 507, 541, 540, 500, 506, 496, 501, 497, 504, 545, 516, 530, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 544, 495, 495, 505, 494, 508, 504, 474, 477, 520, 510, 503, 508, 501, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 513, 532, 490, 509, 507, 485, 522, 531, 508, 463, 554, 528, 520, 519, 474, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 538, 516, 556, 490, 507, 525, 530, 532, 497, 505, 497, 491, 539, 543, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8236, 8046, 8248, 8279, 8064, 8206, 8301] avg=8192.0000 max=8301 max_rank=7 min=8046 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[489, 503, 516, 494, 505, 523, 530, 528, 565, 472, 492, 508, 566, 538, 529, 521, 39, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 527, 506, 528, 500, 532, 537, 534, 492, 490, 522, 483, 535, 515, 467, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 496, 516, 500, 523, 502, 554, 498, 544, 487, 510, 533, 535, 469, 514, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 504, 462, 504, 495, 475, 508, 475, 512, 498, 506, 506, 516, 502, 499, 508, 17, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 490, 524, 520, 516, 481, 472, 549, 494, 481, 501, 522, 519, 533, 552, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 522, 499, 533, 493, 519, 537, 525, 549, 543, 516, 504, 502, 516, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 493, 512, 501, 485, 499, 523, 511, 495, 537, 546, 478, 508, 526, 472, 555, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 502, 564, 524, 522, 507, 514, 495, 519, 529, 518, 513, 517, 502, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8190, 8135, 7989, 8172, 8312, 8183, 8217] avg=8192.0000 max=8338 max_rank=0 min=7989 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 478, 523, 502, 527, 491, 518, 512, 521, 455, 510, 477, 526, 498, 479, 505, 40, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 495, 504, 517, 473, 500, 514, 516, 528, 556, 486, 521, 504, 503, 505, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 484, 526, 544, 550, 515, 539, 494, 497, 500, 539, 509, 500, 518, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 503, 476, 505, 525, 507, 503, 544, 493, 541, 515, 507, 526, 487, 502, 517, 18, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 501, 517, 486, 527, 521, 524, 555, 542, 510, 503, 536, 516, 519, 518, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 528, 526, 523, 522, 505, 544, 528, 501, 530, 544, 492, 500, 481, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 481, 521, 501, 485, 510, 495, 482, 518, 491, 518, 512, 504, 506, 501, 531, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 495, 522, 513, 491, 526, 520, 499, 518, 503, 518, 528, 481, 526, 499, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8074, 8147, 8227, 8214, 8342, 8247, 8069, 8216] avg=8192.0000 max=8342 max_rank=4 min=8069 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[472, 539, 549, 535, 519, 506, 500, 519, 506, 527, 478, 520, 504, 521, 474, 492, 38, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 501, 481, 538, 518, 507, 546, 485, 515, 536, 486, 518, 506, 451, 481, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 480, 547, 563, 486, 531, 502, 516, 474, 467, 533, 502, 519, 480, 520, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 475, 552, 504, 487, 518, 499, 523, 518, 490, 505, 484, 500, 526, 519, 543, 16, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 522, 488, 496, 532, 519, 503, 526, 546, 488, 520, 533, 506, 506, 457, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 507, 503, 529, 494, 523, 499, 516, 520, 553, 549, 492, 529, 514, 507, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 482, 535, 526, 540, 495, 500, 527, 523, 510, 507, 513, 523, 524, 508, 541, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 500, 506, 532, 505, 497, 505, 525, 516, 507, 524, 511, 467, 501, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8070, 8180, 8155, 8178, 8268, 8282, 8184] avg=8192.0000 max=8282 max_rank=6 min=8070 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 517, 514, 514, 511, 495, 498, 497, 526, 521, 499, 519, 523, 505, 482, 480, 39, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 521, 527, 514, 522, 506, 492, 506, 513, 519, 507, 519, 555, 497, 523, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 496, 508, 541, 519, 473, 543, 529, 516, 488, 484, 533, 502, 532, 535, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 537, 506, 540, 549, 502, 515, 485, 527, 496, 495, 502, 541, 504, 488, 523, 17, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 563, 479, 522, 484, 547, 466, 528, 503, 539, 527, 516, 511, 529, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 500, 525, 529, 521, 524, 540, 525, 507, 525, 506, 510, 475, 483, 504, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 514, 475, 494, 512, 506, 551, 516, 526, 505, 526, 532, 515, 509, 514, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 476, 526, 480, 530, 489, 492, 507, 450, 492, 531, 529, 498, 494, 464, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8288, 8229, 8235, 8252, 8185, 8269, 7921] avg=8192.0000 max=8288 max_rank=1 min=7921 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[469, 516, 544, 539, 500, 524, 514, 465, 501, 497, 495, 527, 531, 520, 534, 485, 38, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 518, 486, 509, 485, 503, 486, 503, 545, 512, 522, 500, 513, 515, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 531, 542, 518, 537, 504, 529, 524, 518, 479, 511, 514, 530, 518, 507, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 526, 540, 492, 532, 505, 516, 504, 541, 524, 487, 571, 546, 517, 502, 513, 18, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 499, 494, 496, 521, 544, 482, 487, 488, 485, 467, 487, 512, 530, 490, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 496, 529, 520, 471, 550, 541, 489, 514, 524, 533, 511, 520, 483, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 510, 513, 518, 514, 474, 493, 497, 527, 504, 498, 515, 535, 502, 511, 498, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 472, 494, 522, 537, 519, 567, 525, 478, 546, 461, 530, 500, 507, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8126, 8336, 8349, 8015, 8201, 8113, 8175] avg=8192.0000 max=8349 max_rank=3 min=8015 min_rank=4 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 4096, 4224, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[536, 535, 534, 499, 555, 512, 553, 538, 504, 509, 520, 529, 506, 531, 490, 493, 504, 498, 526, 530, 517, 508, 513, 505, 525, 534, 505, 508, 522, 547, 585, 524, 495, 492, 519, 492, 523, 503, 486, 530, 500, 510, 491, 499, 540, 498, 522, 491, 507, 477, 535, 507, 521, 525, 499, 560, 542, 531, 503, 508, 509, 475, 518, 510, 533, 536, 489, 524, 500, 562, 492, 526, 531, 493, 549, 477, 511, 519, 506, 517, 515, 545, 474, 508, 480, 513, 475, 475, 547, 506, 486, 516, 512, 490, 538, 502, 467, 474, 515, 514, 527, 534, 533, 512, 510, 480, 486, 458, 485, 513, 482, 499, 509, 496, 529, 534, 460, 554, 503, 510, 480, 546, 495, 536, 504, 520, 526, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[504, 505, 541, 520, 538, 514, 502, 488, 508, 510, 500, 545, 544, 511, 518, 542, 526, 496, 507, 515, 510, 480, 514, 481, 521, 501, 519, 506, 503, 521, 505, 510, 520, 505, 465, 532, 478, 538, 536, 492, 525, 511, 502, 529, 546, 492, 577, 543, 495, 503, 532, 496, 486, 536, 545, 510, 510, 492, 486, 490, 473, 498, 538, 474, 492, 506, 489, 526, 514, 470, 554, 474, 487, 498, 465, 511, 498, 520, 514, 471, 522, 503, 470, 490, 532, 533, 552, 492, 502, 504, 528, 532, 549, 544, 496, 531, 543, 482, 506, 514, 515, 528, 500, 499, 522, 514, 540, 525, 539, 534, 511, 508, 509, 510, 514, 512, 507, 516, 526, 465, 491, 529, 533, 516, 493, 543, 531, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[495, 486, 516, 538, 474, 523, 474, 513, 537, 474, 494, 528, 516, 509, 525, 542, 531, 504, 516, 540, 484, 483, 511, 499, 507, 526, 484, 500, 516, 501, 513, 501, 482, 487, 490, 479, 479, 535, 485, 541, 472, 525, 502, 517, 510, 531, 487, 507, 485, 524, 512, 555, 489, 479, 506, 551, 495, 508, 501, 542, 492, 528, 538, 529, 490, 559, 517, 538, 505, 501, 529, 499, 494, 508, 533, 538, 550, 527, 534, 531, 485, 518, 536, 466, 487, 522, 498, 500, 494, 531, 531, 530, 526, 542, 516, 541, 517, 521, 522, 520, 493, 513, 547, 505, 509, 480, 540, 515, 491, 509, 504, 512, 535, 480, 549, 532, 532, 492, 499, 482, 499, 506, 549, 522, 500, 516, 534, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[500, 501, 538, 496, 512, 520, 505, 515, 540, 482, 498, 504, 509, 507, 545, 480, 499, 479, 486, 530, 554, 523, 477, 502, 499, 539, 518, 523, 504, 516, 484, 514, 523, 492, 491, 494, 501, 542, 558, 522, 567, 484, 491, 478, 525, 525, 492, 537, 531, 509, 504, 542, 514, 518, 479, 531, 511, 522, 520, 509, 518, 478, 501, 472, 494, 568, 533, 475, 485, 567, 489, 502, 514, 519, 560, 526, 540, 492, 510, 503, 553, 520, 512, 547, 517, 496, 521, 452, 502, 491, 506, 514, 499, 460, 523, 522, 520, 498, 473, 519, 539, 525, 485, 534, 524, 508, 503, 538, 508, 512, 524, 511, 518, 482, 507, 534, 552, 494, 547, 460, 519, 528, 536, 518, 506, 506, 521, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[511, 507, 506, 535, 521, 553, 509, 552, 549, 477, 482, 515, 493, 496, 498, 512, 494, 509, 475, 489, 517, 506, 540, 497, 518, 543, 522, 480, 534, 507, 527, 525, 536, 514, 543, 489, 519, 510, 529, 518, 507, 503, 491, 512, 501, 522, 500, 487, 512, 490, 502, 497, 526, 523, 502, 491, 499, 499, 458, 526, 533, 517, 508, 509, 494, 487, 513, 508, 511, 515, 482, 494, 525, 524, 489, 476, 501, 521, 494, 523, 489, 538, 511, 539, 493, 473, 546, 511, 479, 510, 518, 495, 508, 512, 549, 521, 536, 484, 504, 547, 502, 528, 502, 531, 482, 506, 526, 511, 511, 551, 543, 524, 528, 557, 514, 522, 522, 556, 507, 499, 483, 511, 495, 535, 512, 537, 508, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[523, 480, 487, 514, 510, 534, 518, 521, 531, 519, 499, 549, 496, 527, 508, 533, 529, 528, 553, 520, 507, 503, 467, 542, 521, 484, 521, 542, 513, 528, 518, 515, 534, 531, 498, 533, 537, 500, 503, 489, 523, 486, 510, 514, 519, 493, 498, 510, 553, 520, 531, 526, 486, 519, 511, 483, 502, 487, 505, 497, 500, 503, 475, 499, 526, 520, 516, 499, 522, 528, 496, 514, 509, 484, 525, 525, 523, 523, 540, 514, 509, 503, 518, 488, 486, 517, 507, 512, 503, 560, 516, 488, 474, 497, 540, 522, 497, 495, 538, 501, 482, 542, 517, 489, 490, 530, 529, 504, 520, 530, 500, 544, 488, 485, 504, 510, 474, 526, 473, 521, 517, 525, 525, 523, 536, 510, 500, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[512, 519, 521, 531, 561, 495, 514, 506, 498, 507, 532, 550, 501, 530, 538, 474, 496, 535, 502, 521, 459, 524, 521, 519, 494, 516, 482, 522, 494, 511, 499, 544, 538, 538, 501, 518, 483, 497, 529, 497, 513, 538, 535, 483, 535, 505, 462, 533, 498, 511, 476, 572, 529, 488, 458, 513, 517, 458, 516, 509, 505, 536, 491, 506, 537, 522, 495, 512, 515, 470, 507, 518, 506, 530, 526, 505, 536, 490, 510, 529, 544, 533, 480, 504, 519, 500, 513, 508, 515, 490, 504, 526, 491, 500, 562, 487, 529, 499, 535, 524, 517, 506, 487, 484, 511, 527, 521, 516, 522, 520, 516, 465, 527, 491, 508, 486, 515, 521, 520, 531, 513, 527, 515, 502, 552, 534, 506, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[540, 501, 535, 498, 504, 491, 479, 460, 452, 558, 524, 501, 472, 511, 557, 539, 513, 527, 518, 503, 501, 503, 548, 512, 529, 475, 508, 543, 487, 487, 486, 542, 499, 478, 496, 534, 549, 502, 512, 503, 542, 527, 529, 546, 491, 528, 485, 532, 512, 517, 493, 505, 517, 490, 513, 520, 483, 537, 509, 508, 489, 513, 530, 462, 514, 513, 505, 519, 472, 524, 522, 528, 521, 516, 524, 486, 490, 503, 541, 527, 520, 487, 530, 512, 513, 511, 494, 515, 492, 522, 526, 517, 481, 507, 506, 522, 535, 519, 551, 537, 531, 493, 531, 504, 509, 520, 532, 514, 556, 516, 533, 507, 524, 495, 507, 504, 514, 480, 487, 522, 538, 466, 514, 526, 548, 505, 505, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[504, 505, 541, 520, 538, 514, 502, 488, 508, 510, 500, 511, 544, 511, 518, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 507, 515, 510, 480, 514, 481, 521, 501, 519, 506, 503, 521, 505, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 465, 532, 478, 538, 536, 492, 525, 511, 502, 529, 546, 492, 577, 543, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 503, 532, 496, 486, 536, 545, 510, 510, 492, 486, 490, 473, 498, 538, 474, 34, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 497, 489, 526, 514, 470, 554, 474, 487, 498, 465, 511, 498, 520, 514, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 470, 490, 532, 533, 552, 492, 502, 504, 528, 532, 549, 544, 496, 531, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 482, 506, 514, 515, 528, 500, 499, 522, 514, 540, 525, 539, 507, 511, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 510, 514, 512, 507, 516, 526, 465, 491, 529, 533, 497, 493, 543, 531, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8116, 8300, 8125, 7980, 8298, 8253, 8208] avg=8192.0000 max=8300 max_rank=2 min=7980 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[536, 535, 534, 499, 555, 512, 553, 538, 504, 509, 520, 496, 506, 531, 490, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 526, 530, 517, 508, 513, 505, 525, 534, 505, 508, 522, 547, 585, 524, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 492, 519, 492, 523, 503, 486, 530, 500, 510, 491, 499, 540, 498, 522, 491, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 477, 535, 507, 521, 525, 499, 560, 542, 531, 503, 508, 509, 475, 518, 510, 33, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 527, 489, 524, 500, 562, 492, 526, 531, 493, 549, 477, 511, 519, 506, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 545, 474, 508, 480, 513, 475, 475, 547, 506, 486, 516, 512, 490, 538, 502, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 474, 515, 514, 527, 534, 533, 512, 510, 480, 486, 458, 485, 488, 482, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 496, 529, 534, 460, 554, 503, 510, 480, 546, 495, 515, 504, 520, 526, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8352, 8101, 8285, 8256, 8101, 7964, 8166] avg=8192.0000 max=8352 max_rank=1 min=7964 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[500, 501, 538, 496, 512, 520, 505, 515, 540, 482, 498, 472, 509, 507, 545, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 479, 486, 530, 554, 523, 477, 502, 499, 539, 518, 523, 504, 516, 484, 514, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 492, 491, 494, 501, 542, 558, 522, 567, 484, 491, 478, 525, 525, 492, 537, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 504, 542, 514, 518, 479, 531, 511, 522, 520, 509, 518, 478, 501, 472, 32, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 558, 533, 475, 485, 567, 489, 502, 514, 519, 560, 526, 540, 492, 510, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 520, 512, 547, 517, 496, 521, 452, 502, 491, 506, 514, 499, 460, 523, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 498, 473, 519, 539, 525, 485, 534, 524, 508, 503, 538, 508, 487, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 482, 507, 534, 552, 494, 547, 460, 519, 528, 536, 499, 506, 506, 521, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8148, 8232, 8216, 8267, 8153, 8196, 8204] avg=8192.0000 max=8267 max_rank=4 min=8120 min_rank=0 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 486, 516, 538, 474, 523, 474, 513, 537, 474, 494, 491, 516, 509, 525, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 516, 540, 484, 483, 511, 499, 507, 526, 484, 500, 516, 501, 513, 501, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 487, 490, 479, 479, 535, 485, 541, 472, 525, 502, 517, 510, 531, 487, 507, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 524, 512, 555, 489, 479, 506, 551, 495, 508, 501, 542, 492, 528, 538, 529, 37, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 541, 517, 538, 505, 501, 529, 499, 494, 508, 533, 538, 550, 527, 534, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 518, 536, 466, 487, 522, 498, 500, 494, 531, 531, 530, 526, 542, 516, 541, 22, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 521, 522, 520, 493, 513, 547, 505, 509, 480, 540, 515, 491, 476, 504, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 480, 549, 532, 532, 492, 499, 482, 499, 506, 549, 495, 500, 516, 534, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8121, 8047, 8301, 8335, 8248, 8165, 8212] avg=8192.0000 max=8335 max_rank=4 min=8047 min_rank=2 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32960 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[511, 507, 506, 535, 521, 553, 509, 552, 549, 477, 482, 483, 493, 496, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 509, 475, 489, 517, 506, 540, 497, 518, 543, 522, 480, 534, 507, 527, 525, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 543, 489, 519, 510, 529, 518, 507, 503, 491, 512, 501, 522, 500, 487, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 502, 497, 526, 523, 502, 491, 499, 499, 458, 526, 533, 517, 508, 509, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 478, 513, 508, 511, 515, 482, 494, 525, 524, 489, 476, 501, 521, 494, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 538, 511, 539, 493, 473, 546, 511, 479, 510, 518, 495, 508, 512, 549, 521, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 484, 504, 547, 502, 528, 502, 531, 482, 506, 526, 511, 511, 524, 543, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 557, 514, 522, 522, 556, 507, 499, 483, 511, 495, 514, 512, 537, 508, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8184, 8191, 8151, 8048, 8211, 8261, 8306] avg=8192.0000 max=8306 max_rank=7 min=8048 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 519, 521, 531, 561, 495, 514, 506, 498, 507, 532, 515, 501, 530, 538, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 535, 502, 521, 459, 524, 521, 519, 494, 516, 482, 522, 494, 511, 499, 544, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 538, 501, 518, 483, 497, 529, 497, 513, 538, 535, 483, 535, 505, 462, 533, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 511, 476, 572, 529, 488, 458, 513, 517, 458, 516, 509, 505, 536, 491, 506, 35, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 513, 495, 512, 515, 470, 507, 518, 506, 530, 526, 505, 536, 490, 510, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 533, 480, 504, 519, 500, 513, 508, 515, 490, 504, 526, 491, 500, 562, 487, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 499, 535, 524, 517, 506, 487, 484, 511, 527, 521, 516, 522, 494, 516, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 491, 508, 486, 515, 521, 520, 531, 513, 527, 515, 483, 552, 534, 506, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8140, 8214, 8144, 8199, 8194, 8153, 8238] avg=8192.0000 max=8254 max_rank=0 min=8140 min_rank=1 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 480, 487, 514, 510, 534, 518, 521, 531, 519, 499, 514, 496, 527, 508, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 553, 520, 507, 503, 467, 542, 521, 484, 521, 542, 513, 528, 518, 515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 531, 498, 533, 537, 500, 503, 489, 523, 486, 510, 514, 519, 493, 498, 510, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 520, 531, 526, 486, 519, 511, 483, 502, 487, 505, 497, 500, 503, 475, 499, 35, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 511, 516, 499, 522, 528, 496, 514, 509, 484, 525, 525, 523, 523, 540, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 503, 518, 488, 486, 517, 507, 512, 503, 560, 516, 488, 474, 497, 540, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 538, 501, 482, 542, 517, 489, 490, 530, 529, 504, 520, 504, 500, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 485, 504, 510, 474, 526, 473, 521, 517, 525, 525, 503, 536, 510, 500, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8292, 8188, 8158, 8255, 8158, 8182, 8089] avg=8192.0000 max=8292 max_rank=1 min=8089 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[540, 501, 535, 498, 504, 491, 479, 460, 452, 558, 524, 469, 472, 511, 557, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 527, 518, 503, 501, 503, 548, 512, 529, 475, 508, 543, 487, 487, 486, 542, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 478, 496, 534, 549, 502, 512, 503, 542, 527, 529, 546, 491, 528, 485, 532, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 493, 505, 517, 490, 513, 520, 483, 537, 509, 508, 489, 513, 530, 462, 32, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 505, 519, 472, 524, 522, 528, 521, 516, 524, 486, 490, 503, 541, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 530, 512, 513, 511, 494, 515, 492, 522, 526, 517, 481, 507, 506, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 551, 537, 531, 493, 531, 504, 509, 520, 532, 514, 556, 490, 533, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 495, 507, 504, 514, 480, 487, 522, 538, 466, 514, 506, 548, 505, 505, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8183, 8263, 8156, 8196, 8173, 8362, 8113] avg=8192.0000 max=8362 max_rank=6 min=8090 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 3968, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 4352, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[540, 511, 505, 523, 502, 501, 519, 514, 549, 496, 485, 491, 539, 500, 526, 487, 522, 520, 497, 549, 544, 508, 473, 468, 491, 504, 501, 504, 534, 526, 495, 511, 493, 505, 513, 521, 486, 533, 487, 500, 530, 530, 524, 493, 477, 505, 517, 538, 512, 518, 492, 509, 560, 509, 481, 485, 514, 491, 489, 547, 512, 513, 510, 557, 531, 503, 519, 513, 512, 565, 549, 552, 513, 495, 534, 514, 517, 509, 505, 521, 532, 506, 531, 499, 483, 546, 523, 481, 495, 537, 482, 535, 520, 516, 492, 496, 554, 494, 551, 483, 494, 524, 507, 514, 558, 493, 512, 482, 491, 508, 501, 571, 503, 501, 516, 531, 538, 509, 509, 500, 483, 503, 513, 464, 466, 545, 486, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[514, 514, 533, 561, 526, 540, 530, 482, 549, 515, 521, 504, 507, 466, 518, 492, 511, 494, 490, 533, 497, 513, 518, 499, 555, 515, 493, 509, 531, 523, 537, 495, 543, 547, 520, 509, 534, 526, 477, 551, 516, 539, 490, 486, 514, 494, 532, 528, 488, 544, 533, 503, 506, 492, 484, 502, 539, 531, 489, 536, 521, 508, 510, 488, 510, 507, 487, 537, 530, 519, 485, 485, 512, 495, 498, 504, 520, 549, 494, 511, 493, 557, 511, 532, 474, 533, 507, 495, 519, 548, 497, 531, 512, 499, 509, 506, 522, 469, 486, 533, 483, 503, 514, 530, 536, 491, 487, 513, 513, 531, 518, 503, 535, 478, 495, 528, 525, 489, 482, 519, 517, 517, 494, 490, 503, 525, 473, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[522, 509, 498, 517, 564, 548, 505, 513, 519, 507, 508, 491, 494, 498, 515, 529, 458, 544, 505, 492, 487, 488, 518, 537, 496, 494, 511, 497, 473, 476, 505, 499, 537, 532, 506, 527, 476, 485, 464, 546, 559, 511, 557, 522, 515, 506, 488, 546, 528, 518, 494, 515, 499, 515, 487, 524, 533, 521, 495, 500, 527, 484, 499, 513, 508, 516, 525, 462, 510, 486, 513, 489, 520, 518, 534, 521, 499, 538, 492, 512, 485, 491, 547, 514, 535, 468, 510, 525, 489, 500, 466, 530, 508, 528, 527, 508, 513, 522, 513, 506, 519, 545, 532, 508, 509, 508, 535, 544, 537, 514, 547, 571, 516, 508, 495, 526, 511, 539, 506, 473, 522, 491, 492, 528, 512, 533, 493, 548] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[518, 521, 485, 506, 512, 510, 494, 523, 528, 515, 505, 509, 514, 521, 488, 526, 540, 478, 517, 506, 497, 516, 494, 558, 501, 552, 512, 520, 507, 499, 544, 511, 554, 519, 509, 544, 502, 554, 557, 495, 505, 504, 507, 534, 517, 499, 535, 514, 525, 524, 510, 528, 537, 503, 516, 467, 502, 552, 517, 490, 556, 495, 506, 529, 510, 504, 535, 486, 514, 526, 474, 486, 533, 500, 517, 509, 499, 511, 485, 522, 485, 521, 510, 531, 536, 516, 509, 516, 507, 531, 498, 502, 495, 518, 489, 534, 539, 536, 561, 499, 500, 509, 487, 524, 508, 464, 498, 511, 482, 510, 519, 514, 495, 475, 502, 511, 524, 538, 475, 510, 528, 499, 501, 518, 516, 483, 510, 448] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[501, 518, 531, 516, 508, 553, 480, 487, 531, 505, 549, 500, 467, 559, 555, 536, 494, 536, 535, 512, 505, 503, 506, 483, 504, 469, 531, 480, 474, 502, 510, 508, 543, 526, 507, 525, 505, 570, 499, 491, 448, 500, 503, 530, 537, 517, 502, 522, 512, 488, 549, 488, 513, 482, 550, 514, 491, 501, 517, 518, 530, 553, 490, 518, 471, 562, 527, 516, 549, 503, 520, 516, 518, 499, 507, 522, 473, 519, 486, 480, 475, 511, 517, 556, 529, 531, 520, 499, 526, 508, 448, 545, 517, 487, 530, 496, 524, 512, 542, 558, 505, 535, 507, 506, 466, 527, 530, 543, 492, 546, 502, 513, 448, 509, 465, 494, 513, 500, 517, 480, 511, 532, 519, 492, 517, 523, 534, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[492, 520, 509, 535, 541, 526, 523, 516, 500, 514, 475, 537, 523, 533, 525, 495, 465, 482, 553, 509, 504, 503, 489, 546, 531, 561, 513, 528, 496, 495, 468, 529, 550, 529, 535, 464, 497, 548, 531, 509, 490, 547, 549, 529, 539, 513, 525, 512, 504, 499, 501, 558, 470, 520, 504, 464, 487, 503, 473, 582, 516, 559, 531, 490, 520, 560, 497, 499, 496, 516, 537, 554, 516, 501, 495, 534, 513, 491, 476, 521, 466, 468, 516, 461, 488, 513, 539, 506, 476, 523, 509, 528, 507, 557, 491, 478, 493, 504, 545, 522, 521, 523, 507, 483, 498, 512, 491, 531, 519, 485, 519, 491, 502, 507, 528, 561, 518, 493, 508, 498, 441, 528, 495, 526, 527, 506, 508, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[457, 505, 501, 484, 499, 496, 527, 484, 516, 532, 541, 517, 525, 531, 485, 553, 505, 496, 504, 508, 538, 483, 476, 500, 539, 514, 476, 508, 480, 553, 523, 513, 507, 491, 504, 508, 545, 510, 502, 517, 517, 523, 523, 490, 482, 543, 574, 511, 551, 504, 477, 542, 555, 492, 516, 501, 505, 514, 527, 536, 499, 556, 500, 503, 500, 510, 536, 527, 519, 533, 523, 514, 516, 488, 515, 527, 509, 508, 481, 493, 540, 503, 511, 518, 505, 528, 496, 500, 502, 483, 509, 498, 509, 479, 534, 502, 531, 512, 507, 468, 513, 521, 532, 498, 538, 483, 484, 535, 499, 509, 561, 513, 498, 532, 510, 501, 531, 518, 522, 554, 544, 471, 508, 484, 510, 537, 516, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[520, 529, 519, 494, 515, 525, 514, 497, 494, 553, 508, 569, 504, 526, 506, 505, 505, 485, 520, 462, 509, 489, 513, 516, 507, 542, 512, 508, 529, 507, 521, 498, 495, 550, 492, 493, 536, 506, 496, 506, 513, 496, 537, 500, 504, 544, 502, 470, 506, 503, 502, 526, 491, 520, 528, 513, 518, 507, 495, 532, 501, 511, 543, 530, 488, 502, 519, 529, 527, 542, 512, 493, 486, 502, 526, 523, 512, 513, 490, 522, 501, 503, 533, 534, 510, 494, 526, 496, 501, 508, 549, 522, 513, 514, 528, 488, 519, 496, 502, 525, 547, 515, 494, 482, 509, 498, 499, 533, 526, 490, 492, 494, 524, 512, 488, 463, 526, 527, 487, 544, 539, 548, 496, 561, 513, 483, 468, 563] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[540, 511, 505, 523, 502, 462, 519, 514, 549, 496, 485, 491, 539, 500, 526, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 497, 549, 544, 508, 473, 468, 491, 504, 501, 504, 534, 526, 495, 511, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 513, 521, 486, 468, 487, 500, 530, 530, 524, 493, 477, 505, 517, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 492, 509, 560, 509, 481, 485, 514, 491, 489, 531, 512, 513, 510, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 503, 519, 513, 512, 565, 549, 552, 513, 495, 534, 514, 517, 509, 505, 521, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 506, 531, 499, 483, 546, 523, 481, 495, 537, 482, 535, 520, 516, 492, 496, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 494, 527, 483, 494, 524, 507, 514, 558, 493, 512, 482, 491, 508, 501, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 501, 516, 531, 538, 509, 509, 500, 483, 503, 513, 464, 466, 545, 486, 520, 39, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8207, 8087, 8183, 8353, 8206, 8213, 8138] avg=8192.0000 max=8353 max_rank=4 min=8087 min_rank=2 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 509, 498, 517, 564, 506, 505, 513, 519, 507, 508, 491, 494, 498, 515, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 544, 505, 492, 487, 488, 518, 537, 496, 494, 511, 497, 473, 476, 505, 499, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 532, 506, 527, 476, 425, 464, 546, 559, 511, 557, 522, 515, 506, 488, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 518, 494, 515, 499, 515, 487, 524, 533, 521, 495, 487, 527, 484, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 516, 525, 462, 510, 486, 513, 489, 520, 518, 534, 521, 499, 538, 492, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 491, 547, 514, 535, 468, 510, 525, 489, 500, 466, 530, 508, 528, 527, 508, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 490, 506, 519, 545, 532, 508, 509, 508, 535, 544, 537, 514, 547, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 495, 526, 511, 539, 506, 473, 522, 491, 492, 528, 512, 533, 493, 548, 42, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8035, 8217, 8139, 8143, 8160, 8400, 8247] avg=8192.0000 max=8400 max_rank=6 min=8035 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=57136 max_load_reduction_pct=0.871826 moved_to_echo=32950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[492, 520, 509, 535, 541, 482, 523, 516, 500, 514, 475, 537, 523, 533, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 482, 553, 509, 504, 503, 489, 546, 531, 561, 513, 528, 496, 495, 468, 529, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 529, 535, 464, 497, 472, 531, 509, 490, 547, 549, 529, 539, 513, 525, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 499, 501, 558, 470, 520, 504, 464, 487, 503, 473, 558, 516, 559, 531, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 560, 497, 499, 496, 516, 537, 554, 516, 501, 495, 534, 513, 491, 476, 521, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 468, 516, 461, 488, 513, 539, 506, 476, 523, 509, 528, 507, 557, 491, 478, 20, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 504, 514, 522, 521, 523, 507, 483, 498, 512, 491, 531, 519, 485, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 528, 561, 518, 493, 508, 498, 441, 528, 495, 526, 527, 506, 508, 530, 44, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8239, 8291, 8137, 8230, 8066, 8113, 8240] avg=8192.0000 max=8291 max_rank=2 min=8066 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 518, 531, 516, 508, 510, 480, 487, 531, 505, 549, 500, 467, 559, 555, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 536, 535, 512, 505, 503, 506, 483, 504, 469, 531, 480, 474, 502, 510, 508, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 526, 507, 525, 505, 500, 499, 491, 448, 500, 503, 530, 537, 517, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 549, 488, 513, 482, 550, 514, 491, 501, 517, 504, 530, 553, 490, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 562, 527, 516, 549, 503, 520, 516, 518, 499, 507, 522, 473, 519, 486, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 511, 517, 556, 529, 531, 520, 499, 526, 508, 448, 545, 517, 487, 530, 496, 17, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 518, 558, 505, 535, 507, 506, 466, 527, 530, 543, 492, 546, 502, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 509, 465, 494, 513, 500, 517, 480, 511, 532, 519, 492, 517, 523, 534, 524, 43, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8116, 8155, 8200, 8168, 8226, 8284, 8134] avg=8192.0000 max=8284 max_rank=6 min=8116 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[457, 505, 501, 484, 499, 458, 527, 484, 516, 532, 541, 517, 525, 531, 485, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 496, 504, 508, 538, 483, 476, 500, 539, 514, 476, 508, 480, 553, 523, 513, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 491, 504, 508, 545, 448, 502, 517, 517, 523, 523, 490, 482, 543, 574, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 504, 477, 542, 555, 492, 516, 501, 505, 514, 527, 521, 499, 556, 500, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 536, 527, 519, 533, 523, 514, 516, 488, 515, 527, 509, 508, 481, 493, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 503, 511, 518, 505, 528, 496, 500, 502, 483, 509, 498, 509, 479, 534, 502, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 484, 468, 513, 521, 532, 498, 538, 483, 484, 535, 499, 509, 561, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 532, 510, 501, 531, 518, 522, 554, 544, 471, 508, 484, 510, 537, 516, 486, 38, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8173, 8185, 8263, 8200, 8147, 8181, 8272] avg=8192.0000 max=8272 max_rank=7 min=8115 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 514, 533, 561, 526, 498, 530, 482, 549, 515, 521, 504, 507, 466, 518, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 494, 490, 533, 497, 513, 518, 499, 555, 515, 493, 509, 531, 523, 537, 495, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 547, 520, 509, 534, 462, 477, 551, 516, 539, 490, 486, 514, 494, 532, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 544, 533, 503, 506, 492, 484, 502, 539, 531, 489, 521, 521, 508, 510, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 487, 537, 530, 519, 485, 485, 512, 495, 498, 504, 520, 549, 494, 511, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 557, 511, 532, 474, 533, 507, 495, 519, 548, 497, 531, 512, 499, 509, 506, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 469, 465, 533, 483, 503, 514, 530, 536, 491, 487, 513, 513, 531, 518, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 478, 495, 528, 525, 489, 482, 519, 517, 517, 494, 490, 503, 525, 473, 503, 42, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8272, 8242, 8159, 8144, 8252, 8111, 8126] avg=8192.0000 max=8272 max_rank=1 min=8111 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 521, 485, 506, 512, 471, 494, 523, 528, 515, 505, 509, 514, 521, 488, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 478, 517, 506, 497, 516, 494, 558, 501, 552, 512, 520, 507, 499, 544, 511, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 519, 509, 544, 502, 486, 557, 495, 505, 504, 507, 534, 517, 499, 535, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 524, 510, 528, 537, 503, 516, 467, 502, 552, 517, 477, 556, 495, 506, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 535, 486, 514, 526, 474, 486, 533, 500, 517, 509, 499, 511, 485, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 521, 510, 531, 536, 516, 509, 516, 507, 531, 498, 502, 495, 518, 489, 534, 18, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 536, 536, 499, 500, 509, 487, 524, 508, 464, 498, 511, 482, 510, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 475, 502, 511, 524, 538, 475, 510, 528, 499, 501, 518, 516, 483, 510, 448, 39, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8314, 8281, 8244, 8111, 8229, 8136, 8085] avg=8192.0000 max=8314 max_rank=1 min=8085 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 529, 519, 494, 515, 485, 514, 497, 494, 553, 508, 569, 504, 526, 506, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 485, 520, 462, 509, 489, 513, 516, 507, 542, 512, 508, 529, 507, 521, 498, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 550, 492, 493, 536, 444, 496, 506, 513, 496, 537, 500, 504, 544, 502, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 503, 502, 526, 491, 520, 528, 513, 518, 507, 495, 517, 501, 511, 543, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 502, 519, 529, 527, 542, 512, 493, 486, 502, 526, 523, 512, 513, 490, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 503, 533, 534, 510, 494, 526, 496, 501, 508, 549, 522, 513, 514, 528, 488, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 479, 525, 547, 515, 494, 482, 509, 498, 499, 533, 526, 490, 492, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 488, 463, 526, 527, 487, 544, 539, 548, 496, 561, 513, 483, 468, 563, 40, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8180, 8078, 8211, 8187, 8250, 8098, 8294] avg=8192.0000 max=8294 max_rank=7 min=8078 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 3712, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[508, 534, 505, 488, 523, 498, 515, 483, 504, 487, 575, 487, 498, 483, 538, 533, 512, 499, 528, 538, 493, 485, 523, 501, 512, 490, 506, 525, 526, 529, 503, 517, 517, 522, 523, 520, 504, 473, 556, 524, 526, 511, 528, 495, 523, 564, 513, 502, 506, 548, 543, 574, 526, 533, 532, 530, 501, 515, 500, 553, 516, 527, 494, 509, 508, 514, 483, 480, 537, 486, 511, 529, 521, 513, 486, 506, 499, 491, 539, 528, 501, 496, 508, 534, 501, 519, 477, 509, 549, 511, 499, 497, 529, 530, 496, 476, 492, 504, 497, 489, 507, 492, 542, 489, 518, 483, 538, 492, 522, 524, 514, 521, 514, 481, 509, 495, 481, 480, 513, 482, 488, 529, 513, 501, 519, 545, 529, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[508, 534, 508, 512, 505, 526, 528, 469, 501, 509, 551, 522, 500, 491, 526, 518, 484, 527, 484, 505, 520, 477, 503, 482, 532, 554, 512, 468, 519, 525, 492, 491, 523, 491, 537, 514, 502, 490, 525, 500, 505, 534, 526, 528, 489, 519, 533, 494, 493, 539, 518, 532, 503, 516, 491, 504, 545, 482, 543, 504, 522, 493, 532, 522, 496, 510, 504, 507, 544, 510, 543, 461, 520, 488, 518, 561, 497, 515, 500, 539, 523, 528, 517, 517, 476, 509, 539, 499, 517, 505, 472, 502, 474, 500, 545, 502, 508, 504, 519, 532, 510, 522, 490, 531, 488, 502, 498, 554, 525, 508, 477, 505, 542, 479, 513, 512, 495, 497, 519, 546, 538, 546, 524, 543, 466, 532, 531, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[502, 513, 531, 485, 490, 519, 535, 503, 550, 534, 506, 527, 525, 497, 506, 511, 497, 509, 511, 484, 534, 506, 464, 529, 539, 517, 547, 487, 522, 496, 523, 533, 519, 518, 587, 505, 505, 502, 529, 491, 497, 503, 523, 477, 504, 502, 482, 511, 516, 561, 506, 529, 494, 498, 495, 508, 494, 541, 526, 482, 550, 536, 516, 520, 513, 518, 532, 446, 525, 522, 517, 499, 484, 507, 521, 504, 520, 508, 524, 529, 481, 504, 533, 489, 525, 514, 506, 516, 494, 525, 544, 489, 461, 546, 499, 517, 542, 544, 542, 521, 528, 552, 476, 517, 534, 504, 503, 503, 533, 485, 487, 561, 517, 509, 479, 487, 505, 526, 492, 470, 535, 476, 523, 499, 490, 507, 538, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[499, 509, 485, 469, 547, 506, 512, 483, 507, 472, 461, 481, 516, 525, 518, 513, 527, 551, 533, 532, 542, 499, 518, 519, 552, 496, 480, 521, 545, 487, 529, 489, 520, 521, 482, 524, 535, 497, 555, 525, 472, 484, 484, 550, 523, 508, 494, 529, 551, 543, 503, 526, 519, 502, 482, 555, 509, 501, 471, 481, 520, 480, 517, 497, 496, 519, 514, 500, 524, 508, 510, 463, 529, 525, 504, 532, 477, 500, 532, 480, 501, 481, 507, 472, 518, 521, 502, 540, 488, 521, 504, 502, 524, 475, 533, 522, 520, 513, 526, 501, 584, 516, 545, 527, 533, 531, 497, 532, 483, 520, 552, 534, 496, 481, 541, 530, 529, 477, 512, 501, 522, 485, 535, 534, 510, 524, 542, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[534, 488, 518, 502, 495, 537, 509, 551, 529, 512, 518, 501, 479, 512, 476, 515, 535, 500, 530, 518, 499, 530, 532, 516, 491, 505, 526, 519, 527, 532, 494, 489, 512, 488, 530, 525, 512, 504, 527, 531, 498, 533, 473, 466, 528, 525, 503, 517, 515, 471, 467, 494, 517, 515, 533, 519, 544, 501, 510, 498, 525, 538, 504, 480, 496, 521, 509, 475, 486, 500, 500, 492, 532, 473, 511, 507, 533, 468, 541, 511, 499, 517, 506, 471, 505, 517, 506, 480, 525, 479, 535, 541, 547, 504, 510, 533, 551, 501, 501, 505, 508, 510, 519, 470, 526, 534, 502, 534, 516, 550, 524, 520, 535, 573, 486, 525, 492, 452, 590, 516, 532, 515, 534, 546, 523, 505, 471, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[559, 515, 574, 507, 494, 516, 537, 470, 524, 498, 496, 487, 507, 466, 550, 519, 512, 504, 502, 501, 542, 492, 493, 480, 505, 510, 526, 550, 506, 499, 507, 506, 542, 543, 512, 563, 481, 535, 517, 508, 512, 541, 481, 534, 536, 535, 490, 510, 521, 519, 475, 510, 539, 513, 557, 470, 528, 520, 504, 545, 530, 488, 476, 530, 481, 513, 523, 537, 527, 504, 494, 516, 532, 477, 505, 516, 507, 565, 476, 506, 502, 500, 496, 524, 478, 511, 532, 522, 501, 505, 533, 490, 497, 517, 535, 492, 507, 537, 511, 523, 495, 538, 506, 500, 544, 510, 502, 507, 510, 522, 500, 509, 524, 479, 487, 514, 467, 517, 515, 522, 522, 501, 502, 483, 493, 525, 532, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[523, 486, 558, 504, 500, 504, 476, 529, 553, 518, 518, 483, 479, 467, 483, 485, 515, 527, 480, 524, 530, 487, 457, 481, 520, 502, 505, 523, 508, 499, 493, 536, 526, 542, 524, 511, 511, 464, 514, 492, 527, 525, 526, 481, 535, 496, 495, 502, 515, 512, 517, 538, 542, 521, 514, 539, 556, 486, 533, 514, 486, 476, 522, 574, 503, 492, 536, 527, 521, 523, 532, 490, 527, 486, 472, 507, 507, 526, 511, 554, 502, 478, 515, 489, 504, 514, 576, 487, 486, 522, 499, 501, 509, 502, 535, 512, 498, 503, 476, 515, 524, 514, 537, 532, 500, 528, 512, 521, 504, 485, 529, 516, 498, 522, 513, 484, 494, 516, 527, 516, 565, 537, 522, 515, 534, 522, 557, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[514, 526, 501, 486, 509, 529, 540, 501, 478, 504, 521, 491, 507, 533, 540, 478, 542, 512, 530, 506, 545, 518, 461, 490, 505, 516, 491, 508, 510, 498, 540, 522, 537, 542, 525, 550, 556, 485, 491, 533, 504, 500, 513, 501, 525, 518, 559, 507, 474, 519, 496, 510, 519, 492, 529, 508, 528, 500, 508, 509, 533, 531, 531, 497, 521, 535, 539, 504, 476, 469, 492, 509, 525, 496, 494, 501, 521, 475, 544, 517, 529, 525, 509, 517, 545, 534, 518, 534, 520, 506, 518, 498, 507, 512, 489, 521, 521, 527, 468, 518, 502, 461, 516, 539, 508, 475, 537, 521, 522, 507, 466, 505, 544, 511, 509, 498, 494, 497, 484, 512, 515, 545, 518, 490, 500, 538, 512, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 534, 505, 488, 523, 498, 515, 483, 504, 487, 575, 487, 498, 483, 538, 533, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 499, 528, 538, 493, 485, 523, 501, 512, 490, 506, 525, 526, 529, 503, 517, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 480, 520, 504, 473, 556, 524, 526, 511, 528, 495, 523, 564, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 548, 543, 518, 526, 533, 532, 530, 501, 515, 500, 553, 516, 527, 494, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 483, 480, 537, 486, 511, 529, 521, 513, 486, 506, 499, 491, 539, 528, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 496, 508, 534, 501, 519, 477, 509, 549, 511, 499, 497, 529, 530, 496, 476, 21, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 497, 489, 507, 492, 542, 489, 518, 483, 538, 492, 522, 524, 514, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 481, 509, 495, 481, 480, 513, 482, 488, 529, 513, 501, 519, 545, 529, 516, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8199, 8258, 8351, 8177, 8168, 8090, 8096] avg=8192.0000 max=8351 max_rank=3 min=8090 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 513, 531, 485, 490, 519, 535, 503, 550, 534, 506, 527, 525, 497, 506, 511, 32, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 511, 484, 534, 506, 464, 529, 539, 517, 547, 487, 522, 496, 523, 533, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 538, 505, 505, 502, 529, 491, 497, 503, 523, 477, 504, 502, 482, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 561, 506, 476, 494, 498, 495, 508, 494, 541, 526, 482, 550, 536, 516, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 532, 446, 525, 522, 517, 499, 484, 507, 521, 504, 520, 508, 524, 529, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 533, 489, 525, 514, 506, 516, 494, 525, 544, 489, 461, 546, 499, 517, 23, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 544, 542, 521, 528, 552, 476, 517, 534, 504, 503, 503, 533, 485, 487, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 479, 487, 505, 526, 492, 470, 535, 476, 523, 499, 490, 507, 538, 480, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8211, 8106, 8219, 8212, 8183, 8294, 8035] avg=8192.0000 max=8294 max_rank=6 min=8035 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 534, 508, 512, 505, 526, 528, 469, 501, 509, 551, 522, 500, 491, 526, 518, 29, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 484, 505, 520, 477, 503, 482, 532, 554, 512, 468, 519, 525, 492, 491, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 491, 492, 514, 502, 490, 525, 500, 505, 534, 526, 528, 489, 519, 533, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 539, 518, 479, 503, 516, 491, 504, 545, 482, 543, 504, 522, 493, 532, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 504, 507, 544, 510, 543, 461, 520, 488, 518, 561, 497, 515, 500, 539, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 528, 517, 517, 476, 509, 539, 499, 517, 505, 472, 502, 474, 500, 545, 502, 20, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 504, 519, 532, 510, 522, 490, 531, 488, 502, 498, 554, 525, 508, 477, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 479, 513, 512, 495, 497, 519, 546, 538, 546, 524, 543, 466, 532, 531, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8086, 8165, 8186, 8256, 8161, 8141, 8294] avg=8192.0000 max=8294 max_rank=7 min=8086 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32852 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[499, 509, 485, 469, 547, 506, 512, 483, 507, 472, 461, 481, 516, 525, 518, 513, 26, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 551, 533, 532, 542, 499, 518, 519, 552, 496, 480, 521, 545, 487, 529, 489, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 521, 442, 524, 535, 497, 555, 525, 472, 484, 484, 550, 523, 508, 494, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 543, 503, 475, 519, 502, 482, 555, 509, 501, 471, 481, 520, 480, 517, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 514, 500, 524, 508, 510, 463, 529, 525, 504, 532, 477, 500, 532, 480, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 481, 507, 472, 518, 521, 502, 540, 488, 521, 504, 502, 524, 475, 533, 522, 22, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 513, 526, 501, 584, 516, 545, 527, 533, 531, 497, 532, 483, 520, 552, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 481, 541, 530, 529, 477, 512, 501, 522, 485, 535, 534, 510, 524, 542, 496, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8038, 8332, 8163, 8106, 8155, 8147, 8378, 8217] avg=8192.0000 max=8378 max_rank=6 min=8038 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 486, 558, 504, 500, 504, 476, 529, 553, 518, 518, 483, 479, 467, 483, 485, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 527, 480, 524, 530, 487, 457, 481, 520, 502, 505, 523, 508, 499, 493, 536, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 542, 480, 511, 511, 464, 514, 492, 527, 525, 526, 481, 535, 496, 495, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 517, 485, 542, 521, 514, 539, 556, 486, 533, 514, 486, 476, 522, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 536, 527, 521, 523, 532, 490, 527, 486, 472, 507, 507, 526, 511, 554, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 478, 515, 489, 504, 514, 576, 487, 486, 522, 499, 501, 509, 502, 535, 512, 21, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 476, 515, 524, 514, 537, 532, 500, 528, 512, 521, 504, 485, 529, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 522, 513, 484, 494, 516, 527, 516, 565, 537, 522, 515, 534, 522, 557, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8099, 8127, 8292, 8257, 8168, 8160, 8329] avg=8192.0000 max=8329 max_rank=7 min=8099 min_rank=1 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[534, 488, 518, 502, 495, 537, 509, 551, 529, 512, 518, 501, 479, 512, 476, 515, 34, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 530, 518, 499, 530, 532, 516, 491, 505, 526, 519, 527, 532, 494, 489, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 476, 525, 512, 504, 527, 531, 498, 533, 473, 466, 528, 525, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 471, 467, 436, 517, 515, 533, 519, 544, 501, 510, 498, 525, 538, 504, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 521, 509, 475, 486, 500, 500, 492, 532, 473, 511, 507, 533, 468, 541, 511, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 506, 471, 505, 517, 506, 480, 525, 479, 535, 541, 547, 504, 510, 533, 25, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 501, 501, 505, 508, 510, 519, 470, 526, 534, 502, 534, 516, 550, 524, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 573, 486, 525, 492, 452, 590, 516, 532, 515, 534, 546, 523, 505, 471, 518, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8258, 8118, 8073, 8100, 8220, 8224, 8320] avg=8192.0000 max=8320 max_rank=7 min=8073 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[559, 515, 574, 507, 494, 516, 537, 470, 524, 498, 496, 487, 507, 466, 550, 519, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 502, 501, 542, 492, 493, 480, 505, 510, 526, 550, 506, 499, 507, 506, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 543, 469, 563, 481, 535, 517, 508, 512, 541, 481, 534, 536, 535, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 475, 460, 539, 513, 557, 470, 528, 520, 504, 545, 530, 488, 476, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 513, 523, 537, 527, 504, 494, 516, 532, 477, 505, 516, 507, 565, 476, 506, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 496, 524, 478, 511, 532, 522, 501, 505, 533, 490, 497, 517, 535, 492, 21, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 537, 511, 523, 495, 538, 506, 500, 544, 510, 502, 507, 510, 522, 500, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 487, 514, 467, 517, 515, 522, 522, 501, 502, 483, 493, 525, 532, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8146, 8297, 8175, 8220, 8171, 8188, 8083] avg=8192.0000 max=8297 max_rank=2 min=8083 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 526, 501, 486, 509, 529, 540, 501, 478, 504, 521, 491, 507, 533, 540, 478, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 512, 530, 506, 545, 518, 461, 490, 505, 516, 491, 508, 510, 498, 540, 522, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 542, 481, 550, 556, 485, 491, 533, 504, 500, 513, 501, 525, 518, 559, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 519, 496, 460, 519, 492, 529, 508, 528, 500, 508, 509, 533, 531, 531, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 535, 539, 504, 476, 469, 492, 509, 525, 496, 494, 501, 521, 475, 544, 517, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 525, 509, 517, 545, 534, 518, 534, 520, 506, 518, 498, 507, 512, 489, 521, 20, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 468, 518, 502, 461, 516, 539, 508, 475, 537, 521, 522, 507, 466, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 511, 509, 498, 494, 497, 484, 512, 515, 545, 518, 490, 500, 538, 512, 494, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8205, 8302, 8134, 8159, 8318, 8061, 8162] avg=8192.0000 max=8318 max_rank=5 min=8061 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 3840, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[492, 503, 521, 531, 492, 486, 473, 509, 513, 499, 501, 501, 550, 515, 493, 548, 488, 501, 535, 504, 481, 513, 526, 488, 506, 483, 514, 557, 522, 493, 515, 523, 532, 502, 531, 524, 521, 515, 523, 517, 483, 540, 516, 519, 535, 520, 544, 511, 505, 508, 522, 486, 500, 513, 515, 552, 534, 505, 495, 514, 497, 513, 504, 517, 505, 509, 517, 510, 519, 509, 537, 500, 481, 498, 491, 512, 556, 496, 507, 498, 512, 505, 497, 527, 479, 509, 493, 521, 497, 484, 526, 512, 524, 496, 533, 538, 498, 526, 481, 513, 547, 543, 502, 516, 521, 519, 546, 533, 501, 485, 507, 487, 515, 531, 493, 544, 492, 524, 497, 469, 509, 542, 521, 545, 510, 512, 477, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[467, 497, 515, 509, 502, 529, 510, 510, 544, 525, 528, 481, 520, 533, 491, 505, 508, 521, 536, 497, 514, 505, 515, 513, 521, 526, 528, 553, 537, 552, 492, 503, 511, 518, 505, 527, 518, 557, 502, 501, 498, 517, 543, 486, 532, 485, 484, 520, 504, 542, 537, 508, 487, 511, 498, 536, 558, 498, 518, 512, 536, 516, 502, 504, 477, 511, 485, 518, 526, 511, 530, 481, 485, 501, 522, 522, 487, 513, 563, 497, 516, 482, 486, 494, 521, 524, 456, 517, 521, 532, 484, 522, 541, 511, 495, 513, 522, 528, 489, 526, 480, 520, 517, 484, 477, 485, 485, 505, 520, 513, 508, 551, 519, 493, 491, 511, 541, 458, 472, 534, 512, 520, 512, 554, 540, 538, 501, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[516, 505, 523, 542, 488, 509, 493, 474, 520, 517, 521, 504, 525, 513, 507, 492, 519, 494, 538, 486, 527, 518, 530, 506, 538, 521, 516, 496, 524, 521, 555, 497, 532, 463, 492, 529, 519, 555, 534, 469, 482, 465, 501, 499, 495, 468, 494, 522, 538, 478, 486, 527, 524, 540, 526, 532, 527, 489, 551, 504, 536, 513, 527, 495, 504, 505, 514, 543, 493, 537, 506, 514, 515, 520, 495, 524, 495, 505, 507, 518, 533, 490, 527, 523, 532, 489, 469, 505, 537, 510, 495, 486, 490, 534, 455, 508, 492, 504, 510, 519, 520, 524, 488, 478, 536, 532, 528, 507, 548, 541, 504, 539, 520, 505, 497, 499, 518, 485, 489, 487, 539, 498, 578, 502, 522, 532, 548, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[508, 520, 558, 543, 556, 551, 502, 518, 522, 530, 575, 527, 529, 517, 520, 523, 488, 494, 501, 526, 491, 500, 460, 517, 518, 506, 525, 541, 506, 518, 494, 506, 524, 473, 537, 512, 496, 497, 507, 528, 527, 490, 548, 497, 521, 478, 498, 518, 502, 527, 531, 499, 527, 529, 537, 517, 509, 509, 499, 475, 545, 540, 506, 520, 486, 531, 517, 490, 512, 500, 467, 486, 502, 526, 532, 491, 510, 501, 456, 496, 503, 503, 469, 532, 529, 461, 512, 513, 536, 524, 507, 519, 498, 513, 536, 521, 525, 518, 509, 522, 533, 537, 516, 476, 520, 494, 486, 526, 517, 520, 526, 538, 489, 495, 489, 516, 493, 528, 518, 541, 543, 484, 485, 521, 476, 515, 495, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[502, 485, 504, 546, 532, 537, 483, 482, 560, 497, 488, 531, 499, 544, 490, 507, 478, 493, 495, 524, 524, 520, 479, 509, 505, 486, 471, 493, 519, 517, 541, 529, 541, 469, 488, 572, 487, 538, 506, 589, 491, 495, 520, 533, 503, 506, 557, 541, 520, 475, 511, 519, 512, 508, 485, 568, 509, 518, 491, 488, 505, 545, 544, 487, 540, 537, 547, 500, 534, 500, 493, 508, 529, 515, 514, 530, 502, 503, 491, 497, 516, 514, 547, 505, 503, 503, 520, 502, 505, 523, 530, 546, 500, 472, 520, 525, 519, 527, 451, 567, 526, 497, 521, 526, 469, 500, 534, 520, 507, 509, 520, 503, 492, 497, 524, 507, 500, 496, 513, 471, 488, 490, 497, 495, 592, 510, 503, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[533, 497, 466, 473, 558, 565, 485, 523, 531, 558, 518, 494, 533, 527, 484, 525, 498, 525, 490, 535, 524, 548, 494, 521, 523, 517, 477, 515, 499, 501, 529, 510, 497, 511, 495, 519, 516, 539, 521, 535, 522, 513, 497, 491, 494, 534, 474, 532, 507, 512, 478, 560, 507, 504, 493, 528, 494, 523, 507, 523, 511, 493, 520, 511, 563, 519, 512, 494, 512, 494, 504, 542, 497, 477, 512, 496, 555, 511, 532, 539, 551, 511, 487, 511, 480, 518, 491, 522, 469, 543, 541, 484, 550, 492, 512, 483, 489, 507, 514, 497, 493, 515, 523, 534, 518, 506, 525, 507, 523, 520, 511, 504, 499, 518, 481, 475, 482, 527, 465, 523, 520, 506, 494, 497, 544, 508, 517, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[467, 495, 497, 524, 526, 525, 524, 486, 495, 486, 527, 504, 525, 515, 515, 478, 535, 503, 540, 506, 491, 492, 521, 483, 475, 524, 521, 521, 495, 510, 547, 516, 541, 525, 511, 481, 501, 495, 510, 511, 509, 524, 496, 504, 476, 513, 510, 513, 479, 501, 471, 545, 493, 532, 534, 514, 494, 481, 509, 530, 541, 529, 546, 515, 514, 500, 476, 505, 512, 535, 554, 483, 496, 504, 511, 500, 494, 472, 496, 550, 488, 541, 459, 505, 510, 546, 508, 538, 499, 508, 540, 504, 509, 513, 516, 525, 503, 472, 500, 520, 549, 524, 510, 510, 504, 516, 502, 519, 542, 537, 542, 557, 551, 524, 516, 505, 512, 510, 521, 514, 514, 522, 511, 554, 492, 512, 532, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[485, 512, 505, 543, 525, 512, 521, 523, 481, 529, 568, 506, 517, 519, 477, 493, 472, 504, 499, 500, 520, 530, 491, 512, 552, 483, 489, 493, 554, 542, 498, 562, 546, 542, 517, 492, 539, 514, 499, 553, 514, 505, 514, 481, 516, 504, 534, 535, 505, 534, 493, 496, 568, 490, 499, 530, 509, 496, 541, 509, 505, 511, 525, 494, 479, 501, 531, 505, 490, 498, 539, 544, 501, 499, 537, 496, 509, 507, 494, 494, 496, 517, 495, 509, 503, 506, 526, 463, 466, 485, 506, 503, 561, 533, 518, 525, 486, 503, 495, 507, 477, 514, 551, 504, 499, 514, 541, 491, 512, 501, 488, 541, 530, 503, 518, 519, 508, 517, 540, 504, 558, 517, 485, 534, 495, 514, 487, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[467, 497, 515, 509, 502, 529, 510, 510, 544, 525, 509, 481, 520, 533, 491, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 536, 497, 514, 505, 515, 513, 521, 526, 528, 553, 537, 552, 492, 503, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 518, 505, 527, 518, 557, 502, 501, 498, 517, 543, 486, 532, 485, 484, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 542, 537, 508, 487, 511, 498, 505, 558, 498, 518, 512, 536, 516, 502, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 511, 485, 518, 526, 511, 530, 481, 485, 501, 522, 522, 487, 513, 563, 497, 31, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 482, 486, 494, 521, 524, 456, 517, 521, 532, 484, 522, 541, 511, 495, 513, 18, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 489, 526, 480, 520, 517, 484, 477, 485, 485, 505, 520, 513, 508, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 493, 491, 511, 541, 458, 472, 534, 512, 520, 512, 554, 540, 538, 501, 528, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8323, 8191, 8236, 8170, 8152, 8092, 8225] avg=8192.0000 max=8323 max_rank=1 min=8092 min_rank=6 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[492, 503, 521, 531, 492, 486, 473, 509, 513, 499, 482, 501, 550, 515, 493, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 501, 535, 504, 481, 513, 526, 488, 506, 483, 514, 557, 522, 493, 515, 523, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 502, 531, 524, 521, 515, 523, 517, 483, 540, 516, 519, 535, 520, 544, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 522, 486, 500, 513, 515, 520, 534, 505, 495, 514, 497, 513, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 517, 510, 519, 509, 537, 500, 481, 498, 491, 512, 556, 496, 507, 498, 32, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 497, 527, 479, 509, 493, 521, 497, 484, 526, 512, 524, 496, 533, 538, 16, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 526, 481, 513, 547, 543, 502, 516, 521, 519, 546, 533, 501, 485, 507, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 531, 493, 544, 492, 524, 497, 469, 509, 542, 521, 545, 510, 512, 477, 543, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8151, 8319, 8148, 8187, 8189, 8209, 8225] avg=8192.0000 max=8319 max_rank=2 min=8108 min_rank=0 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[502, 485, 504, 546, 532, 537, 483, 482, 560, 497, 469, 531, 499, 544, 490, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 493, 495, 524, 524, 520, 479, 509, 505, 486, 471, 493, 519, 517, 541, 529, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 469, 488, 572, 487, 538, 506, 589, 491, 495, 520, 533, 503, 506, 557, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 475, 511, 519, 512, 508, 485, 535, 509, 518, 491, 488, 505, 545, 544, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 537, 547, 500, 534, 500, 493, 508, 529, 515, 514, 530, 502, 503, 491, 497, 33, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 547, 505, 503, 503, 520, 502, 505, 523, 530, 546, 500, 472, 520, 525, 17, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 527, 451, 567, 526, 497, 521, 526, 469, 500, 534, 520, 507, 509, 520, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 497, 524, 507, 500, 496, 513, 471, 488, 490, 497, 495, 592, 510, 503, 503, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8085, 8322, 8152, 8283, 8268, 8179, 8079] avg=8192.0000 max=8322 max_rank=2 min=8079 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[516, 505, 523, 542, 488, 509, 493, 474, 520, 517, 502, 504, 525, 513, 507, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 494, 538, 486, 527, 518, 530, 506, 538, 521, 516, 496, 524, 521, 555, 497, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 463, 492, 529, 519, 555, 534, 469, 482, 465, 501, 499, 495, 468, 494, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 478, 486, 527, 524, 540, 526, 501, 527, 489, 551, 504, 536, 513, 527, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 514, 543, 493, 537, 506, 514, 515, 520, 495, 524, 495, 505, 507, 518, 31, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 490, 527, 523, 532, 489, 469, 505, 537, 510, 495, 486, 490, 534, 455, 508, 18, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 510, 519, 520, 524, 488, 478, 536, 532, 528, 507, 548, 541, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 497, 499, 518, 485, 489, 487, 539, 498, 578, 502, 522, 532, 548, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8288, 8005, 8262, 8236, 8121, 8252, 8242] avg=8192.0000 max=8288 max_rank=1 min=8005 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[508, 520, 558, 543, 556, 551, 502, 518, 522, 530, 554, 527, 529, 517, 520, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 494, 501, 526, 491, 500, 460, 517, 518, 506, 525, 541, 506, 518, 494, 506, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 473, 537, 512, 496, 497, 507, 528, 527, 490, 548, 497, 521, 478, 498, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 527, 531, 499, 527, 529, 537, 487, 509, 509, 499, 475, 545, 540, 506, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 531, 517, 490, 512, 500, 467, 486, 502, 526, 532, 491, 510, 501, 456, 496, 30, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 469, 532, 529, 461, 512, 513, 536, 524, 507, 519, 498, 513, 536, 521, 18, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 509, 522, 533, 537, 516, 476, 520, 494, 486, 526, 517, 520, 526, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 495, 489, 516, 493, 528, 518, 541, 543, 484, 485, 521, 476, 515, 495, 493, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8478, 8093, 8138, 8242, 8044, 8214, 8245, 8082] avg=8192.0000 max=8478 max_rank=0 min=8044 min_rank=4 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[533, 497, 466, 473, 558, 565, 485, 523, 531, 558, 499, 494, 533, 527, 484, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 490, 535, 524, 548, 494, 521, 523, 517, 477, 515, 499, 501, 529, 510, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 511, 495, 519, 516, 539, 521, 535, 522, 513, 497, 491, 494, 534, 474, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 478, 560, 507, 504, 493, 498, 494, 523, 507, 523, 511, 493, 520, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 519, 512, 494, 512, 494, 504, 542, 497, 477, 512, 496, 555, 511, 532, 539, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 511, 487, 511, 480, 518, 491, 522, 469, 543, 541, 484, 550, 492, 512, 483, 17, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 507, 514, 497, 493, 515, 523, 534, 518, 506, 525, 507, 523, 520, 511, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 518, 481, 475, 482, 527, 465, 523, 520, 506, 494, 497, 544, 508, 517, 553, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8208, 8177, 8141, 8299, 8181, 8169, 8110] avg=8192.0000 max=8299 max_rank=4 min=8110 min_rank=7 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[467, 495, 497, 524, 526, 525, 524, 486, 495, 486, 508, 504, 525, 515, 515, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 503, 540, 506, 491, 492, 521, 483, 475, 524, 521, 521, 495, 510, 547, 516, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 511, 481, 501, 495, 510, 511, 509, 524, 496, 504, 476, 513, 510, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 501, 471, 545, 493, 532, 534, 484, 494, 481, 509, 530, 541, 529, 546, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 476, 505, 512, 535, 554, 483, 496, 504, 511, 500, 494, 472, 496, 550, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 541, 459, 505, 510, 546, 508, 538, 499, 508, 540, 504, 509, 513, 516, 525, 19, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 472, 500, 520, 549, 524, 510, 510, 504, 516, 502, 519, 542, 537, 542, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 524, 516, 505, 512, 510, 521, 514, 514, 522, 511, 554, 492, 512, 532, 525, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8182, 8106, 8184, 8142, 8248, 8288, 8316] avg=8192.0000 max=8316 max_rank=7 min=8070 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32994 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[485, 512, 505, 543, 525, 512, 521, 523, 481, 529, 536, 506, 517, 519, 477, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 504, 499, 500, 520, 530, 491, 512, 552, 483, 489, 493, 554, 542, 498, 562, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 542, 517, 492, 539, 514, 499, 553, 514, 505, 514, 481, 516, 504, 534, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 534, 493, 496, 568, 490, 499, 496, 509, 496, 541, 509, 505, 511, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 501, 531, 505, 490, 498, 539, 544, 501, 499, 537, 496, 509, 507, 494, 494, 34, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 517, 495, 509, 503, 506, 526, 463, 466, 485, 506, 503, 561, 533, 518, 525, 22, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 503, 495, 507, 477, 514, 551, 504, 499, 514, 541, 491, 512, 501, 488, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 503, 518, 519, 508, 517, 540, 504, 558, 517, 485, 534, 495, 514, 487, 520, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8206, 8278, 8171, 8175, 8163, 8102, 8257] avg=8192.0000 max=8278 max_rank=2 min=8102 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 3968, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[543, 473, 503, 470, 510, 497, 480, 485, 502, 509, 509, 570, 496, 515, 530, 514, 508, 521, 505, 478, 475, 495, 514, 522, 521, 515, 530, 511, 512, 493, 517, 531, 531, 513, 517, 514, 551, 517, 514, 514, 492, 549, 509, 499, 503, 515, 484, 526, 477, 521, 492, 522, 528, 518, 508, 563, 548, 504, 546, 525, 504, 538, 512, 510, 529, 493, 532, 500, 532, 553, 502, 575, 501, 504, 491, 503, 527, 533, 538, 513, 466, 543, 495, 509, 532, 476, 516, 527, 476, 512, 494, 536, 498, 522, 499, 507, 510, 505, 500, 524, 521, 500, 505, 512, 487, 514, 548, 521, 516, 496, 487, 528, 522, 507, 486, 487, 525, 507, 493, 507, 485, 501, 504, 516, 518, 511, 533, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[511, 498, 517, 507, 469, 510, 499, 547, 484, 539, 496, 518, 553, 521, 540, 527, 519, 543, 545, 523, 512, 521, 503, 519, 554, 508, 519, 530, 510, 500, 492, 471, 512, 521, 499, 533, 502, 514, 526, 520, 478, 538, 494, 511, 500, 511, 490, 528, 519, 529, 506, 453, 513, 520, 505, 492, 514, 551, 519, 524, 494, 542, 528, 506, 528, 500, 489, 482, 508, 518, 537, 522, 498, 515, 505, 517, 492, 534, 533, 561, 460, 497, 509, 508, 557, 538, 515, 525, 515, 542, 520, 512, 515, 504, 535, 494, 502, 473, 498, 496, 525, 488, 546, 518, 513, 486, 503, 495, 511, 472, 517, 511, 509, 508, 531, 513, 503, 504, 473, 475, 514, 519, 539, 500, 488, 526, 510, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[501, 540, 534, 527, 516, 509, 529, 475, 479, 496, 503, 503, 508, 524, 527, 545, 458, 521, 491, 533, 516, 528, 513, 555, 536, 515, 528, 501, 503, 524, 501, 532, 514, 494, 492, 545, 504, 527, 510, 508, 549, 514, 510, 504, 542, 473, 500, 520, 549, 495, 533, 488, 501, 517, 533, 517, 566, 497, 494, 496, 526, 539, 542, 531, 499, 492, 505, 465, 536, 502, 484, 490, 508, 528, 480, 511, 504, 525, 506, 502, 531, 483, 516, 488, 505, 518, 498, 523, 534, 497, 502, 540, 494, 476, 529, 554, 532, 482, 494, 522, 480, 484, 497, 512, 536, 540, 513, 513, 535, 528, 514, 514, 491, 501, 495, 498, 502, 510, 536, 501, 490, 479, 502, 545, 528, 524, 485, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[540, 516, 504, 501, 510, 526, 511, 451, 512, 503, 497, 513, 490, 488, 518, 504, 533, 518, 501, 537, 498, 517, 506, 490, 477, 523, 453, 519, 477, 509, 537, 478, 487, 504, 505, 530, 534, 542, 503, 527, 509, 529, 530, 524, 495, 494, 513, 495, 518, 509, 494, 519, 515, 513, 503, 508, 503, 477, 512, 508, 495, 553, 523, 493, 502, 508, 519, 508, 508, 523, 502, 534, 503, 519, 526, 541, 472, 474, 543, 515, 486, 508, 510, 510, 496, 527, 523, 525, 500, 507, 543, 504, 516, 514, 483, 518, 554, 491, 504, 528, 496, 552, 527, 531, 545, 509, 524, 494, 519, 514, 513, 498, 549, 530, 484, 528, 541, 526, 521, 527, 534, 558, 513, 519, 505, 485, 524, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[513, 528, 534, 495, 505, 517, 491, 475, 472, 521, 536, 486, 517, 532, 474, 507, 501, 488, 545, 520, 531, 454, 510, 535, 532, 483, 502, 536, 484, 508, 504, 515, 549, 487, 495, 500, 510, 504, 504, 522, 545, 535, 531, 517, 515, 519, 500, 518, 523, 475, 554, 502, 522, 485, 533, 533, 494, 471, 494, 500, 502, 501, 498, 493, 531, 498, 521, 565, 522, 549, 533, 488, 489, 549, 519, 516, 514, 544, 511, 482, 484, 508, 518, 541, 529, 512, 517, 509, 505, 492, 503, 532, 501, 548, 518, 539, 485, 508, 506, 475, 521, 499, 530, 523, 516, 512, 527, 510, 496, 527, 514, 490, 541, 506, 552, 489, 462, 507, 553, 523, 507, 509, 487, 520, 511, 491, 529, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[516, 492, 504, 495, 519, 496, 504, 500, 500, 478, 539, 529, 524, 498, 498, 533, 530, 510, 534, 532, 522, 507, 525, 527, 535, 504, 524, 543, 474, 478, 511, 497, 458, 507, 476, 526, 494, 503, 527, 534, 474, 501, 518, 556, 546, 501, 490, 484, 510, 515, 478, 534, 516, 502, 531, 496, 492, 525, 511, 492, 518, 529, 532, 496, 516, 519, 493, 527, 529, 514, 522, 537, 496, 479, 506, 548, 549, 505, 541, 516, 537, 489, 497, 535, 489, 522, 526, 516, 504, 510, 506, 519, 519, 491, 518, 514, 497, 597, 537, 497, 539, 493, 516, 500, 539, 526, 534, 515, 537, 518, 530, 528, 481, 499, 465, 515, 497, 525, 502, 510, 512, 490, 520, 492, 508, 499, 529, 450] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[542, 508, 495, 496, 501, 505, 552, 509, 484, 517, 511, 515, 531, 488, 505, 547, 506, 513, 514, 544, 484, 455, 473, 515, 464, 521, 507, 491, 495, 492, 507, 537, 493, 494, 529, 527, 521, 518, 517, 502, 495, 521, 504, 533, 455, 509, 514, 534, 523, 520, 508, 512, 482, 509, 538, 532, 451, 473, 511, 529, 532, 499, 536, 525, 541, 542, 544, 500, 502, 507, 503, 507, 491, 472, 500, 529, 521, 489, 519, 503, 527, 519, 559, 497, 545, 526, 488, 486, 545, 526, 522, 529, 525, 505, 505, 522, 539, 533, 498, 489, 471, 493, 546, 525, 488, 533, 517, 534, 507, 526, 518, 521, 495, 524, 521, 509, 527, 474, 506, 507, 541, 539, 500, 520, 509, 537, 500, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[510, 569, 542, 530, 516, 481, 525, 493, 489, 540, 530, 505, 478, 522, 543, 534, 516, 513, 533, 510, 510, 509, 519, 487, 516, 523, 519, 489, 521, 508, 524, 520, 536, 520, 513, 544, 527, 515, 490, 482, 511, 522, 499, 489, 498, 498, 483, 521, 483, 528, 477, 494, 530, 478, 557, 541, 500, 522, 481, 473, 514, 517, 503, 522, 501, 515, 523, 493, 505, 522, 510, 515, 520, 469, 483, 492, 514, 522, 515, 546, 478, 496, 564, 495, 537, 477, 497, 474, 565, 501, 519, 542, 513, 507, 533, 516, 509, 539, 510, 514, 503, 495, 530, 522, 525, 513, 493, 498, 529, 535, 501, 517, 488, 522, 511, 491, 531, 494, 499, 507, 546, 477, 536, 530, 490, 535, 500, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 498, 517, 507, 469, 510, 499, 547, 484, 539, 496, 518, 553, 521, 540, 527, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 543, 545, 523, 512, 521, 503, 519, 554, 508, 519, 530, 510, 500, 492, 471, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 521, 499, 533, 502, 514, 526, 520, 478, 538, 494, 511, 500, 511, 490, 528, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 529, 506, 453, 513, 520, 505, 492, 514, 551, 519, 524, 494, 540, 528, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 489, 482, 508, 518, 537, 522, 498, 515, 505, 517, 492, 534, 507, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 497, 509, 508, 557, 538, 515, 525, 515, 542, 520, 494, 515, 504, 535, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 473, 498, 496, 525, 488, 521, 518, 513, 486, 503, 495, 511, 472, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 531, 513, 503, 504, 473, 475, 514, 519, 539, 500, 488, 526, 510, 488, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8288, 8179, 8213, 8213, 8228, 8029, 8132] avg=8192.0000 max=8288 max_rank=1 min=8029 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 528, 534, 495, 505, 517, 491, 475, 472, 521, 536, 486, 517, 532, 474, 507, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 488, 545, 520, 531, 454, 510, 535, 532, 483, 502, 536, 484, 508, 504, 515, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 487, 495, 500, 510, 504, 504, 522, 545, 535, 531, 517, 515, 519, 500, 518, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 475, 554, 502, 522, 485, 533, 533, 494, 471, 494, 500, 502, 499, 498, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 498, 521, 565, 522, 549, 533, 488, 489, 549, 519, 516, 514, 544, 486, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 518, 541, 529, 512, 517, 509, 505, 492, 503, 513, 501, 548, 518, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 508, 506, 475, 521, 499, 506, 523, 516, 512, 527, 510, 496, 527, 514, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 506, 552, 489, 462, 507, 553, 523, 507, 509, 487, 520, 511, 491, 529, 541, 25, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8166, 8253, 8078, 8306, 8237, 8115, 8259] avg=8192.0000 max=8306 max_rank=4 min=8078 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[543, 473, 503, 470, 510, 497, 480, 485, 502, 509, 509, 570, 496, 515, 530, 514, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 505, 478, 475, 495, 514, 522, 521, 515, 530, 511, 512, 493, 517, 531, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 513, 517, 514, 551, 517, 514, 514, 492, 549, 509, 499, 503, 515, 484, 526, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 521, 492, 522, 528, 518, 508, 563, 548, 504, 546, 525, 504, 536, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 493, 532, 500, 532, 553, 502, 575, 501, 504, 491, 503, 527, 533, 512, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 543, 495, 509, 532, 476, 516, 527, 476, 512, 494, 517, 498, 522, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 505, 500, 524, 521, 500, 482, 512, 487, 514, 548, 521, 516, 496, 487, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 507, 486, 487, 525, 507, 493, 507, 485, 501, 504, 516, 518, 511, 533, 508, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8165, 8250, 8314, 8300, 8089, 8151, 8142] avg=8192.0000 max=8314 max_rank=3 min=8089 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[501, 540, 534, 527, 516, 509, 529, 475, 479, 496, 503, 503, 508, 524, 527, 545, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 521, 491, 533, 516, 528, 513, 555, 536, 515, 528, 501, 503, 524, 501, 532, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 494, 492, 545, 504, 527, 510, 508, 549, 514, 510, 504, 542, 473, 500, 520, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 495, 533, 488, 501, 517, 533, 517, 566, 497, 494, 496, 526, 537, 542, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 492, 505, 465, 536, 502, 484, 490, 508, 528, 480, 511, 504, 525, 482, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 483, 516, 488, 505, 518, 498, 523, 534, 497, 502, 521, 494, 476, 529, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 482, 494, 522, 480, 484, 474, 512, 536, 540, 513, 513, 535, 528, 514, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 495, 498, 502, 510, 536, 501, 490, 479, 502, 545, 528, 524, 485, 527, 24, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8272, 8208, 8322, 8013, 8169, 8173, 8144] avg=8192.0000 max=8322 max_rank=3 min=8013 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[540, 516, 504, 501, 510, 526, 511, 451, 512, 503, 497, 513, 490, 488, 518, 504, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 501, 537, 498, 517, 506, 490, 477, 523, 453, 519, 477, 509, 537, 478, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 504, 505, 530, 534, 542, 503, 527, 509, 529, 530, 524, 495, 494, 513, 495, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 509, 494, 519, 515, 513, 503, 508, 503, 477, 512, 508, 495, 551, 523, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 508, 519, 508, 508, 523, 502, 534, 503, 519, 526, 541, 472, 474, 517, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 508, 510, 510, 496, 527, 523, 525, 500, 507, 543, 486, 516, 514, 483, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 491, 504, 528, 496, 552, 503, 531, 545, 509, 524, 494, 519, 514, 513, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 530, 484, 528, 541, 526, 521, 527, 534, 558, 513, 519, 505, 485, 524, 505, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8091, 8223, 8141, 8171, 8152, 8275, 8381] avg=8192.0000 max=8381 max_rank=7 min=8091 min_rank=1 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32979 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 492, 504, 495, 519, 496, 504, 500, 500, 478, 539, 529, 524, 498, 498, 533, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 510, 534, 532, 522, 507, 525, 527, 535, 504, 524, 543, 474, 478, 511, 497, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 507, 476, 526, 494, 503, 527, 534, 474, 501, 518, 556, 546, 501, 490, 484, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 478, 534, 516, 502, 531, 496, 492, 525, 511, 492, 518, 527, 532, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 493, 527, 529, 514, 522, 537, 496, 479, 506, 548, 549, 505, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 489, 497, 535, 489, 522, 526, 516, 504, 510, 506, 501, 519, 491, 518, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 597, 537, 497, 539, 493, 492, 500, 539, 526, 534, 515, 537, 518, 530, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 499, 465, 515, 497, 525, 502, 510, 512, 490, 520, 492, 508, 499, 529, 450, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8271, 8097, 8175, 8271, 8174, 8379, 8026] avg=8192.0000 max=8379 max_rank=6 min=8026 min_rank=7 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 569, 542, 530, 516, 481, 525, 493, 489, 540, 530, 505, 478, 522, 543, 534, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 513, 533, 510, 510, 509, 519, 487, 516, 523, 519, 489, 521, 508, 524, 520, 23, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 520, 513, 544, 527, 515, 490, 482, 511, 522, 499, 489, 498, 498, 483, 521, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 528, 477, 494, 530, 478, 557, 541, 500, 522, 481, 473, 514, 512, 503, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 523, 493, 505, 522, 510, 515, 520, 469, 483, 492, 514, 522, 487, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 496, 564, 495, 537, 477, 497, 474, 565, 501, 519, 508, 513, 507, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 539, 510, 514, 503, 495, 498, 522, 525, 513, 493, 498, 529, 535, 501, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 522, 511, 491, 531, 494, 499, 507, 546, 477, 536, 530, 490, 535, 500, 495, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8330, 8246, 8158, 8115, 8117, 8180, 8201, 8189] avg=8192.0000 max=8330 max_rank=0 min=8115 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[542, 508, 495, 496, 501, 505, 552, 509, 484, 517, 511, 515, 531, 488, 505, 547, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 514, 544, 484, 455, 473, 515, 464, 521, 507, 491, 495, 492, 507, 537, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 494, 529, 527, 521, 518, 517, 502, 495, 521, 504, 533, 455, 509, 514, 534, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 520, 508, 512, 482, 509, 538, 532, 451, 473, 511, 529, 532, 497, 536, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 542, 544, 500, 502, 507, 503, 507, 491, 472, 500, 529, 521, 489, 494, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 559, 497, 545, 526, 488, 486, 545, 526, 522, 510, 525, 505, 505, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 533, 498, 489, 471, 493, 521, 525, 488, 533, 517, 534, 507, 526, 518, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 521, 509, 527, 474, 506, 507, 541, 539, 500, 520, 509, 537, 500, 523, 25, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8037, 8168, 8178, 8145, 8307, 8213, 8263] avg=8192.0000 max=8307 max_rank=5 min=8037 min_rank=1 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 3968, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[530, 502, 509, 529, 520, 536, 538, 505, 506, 500, 510, 508, 534, 549, 501, 488, 483, 529, 504, 521, 522, 507, 527, 498, 512, 508, 489, 491, 508, 530, 535, 513, 534, 510, 518, 521, 518, 471, 522, 497, 534, 480, 559, 489, 503, 519, 485, 500, 511, 545, 519, 511, 501, 504, 490, 527, 483, 525, 509, 510, 523, 474, 541, 507, 495, 487, 513, 468, 496, 510, 500, 571, 502, 513, 541, 501, 536, 500, 555, 529, 505, 506, 460, 502, 503, 535, 517, 517, 524, 497, 485, 489, 522, 514, 515, 502, 540, 508, 534, 501, 491, 508, 515, 512, 512, 527, 537, 518, 507, 546, 504, 476, 525, 510, 515, 541, 549, 519, 523, 447, 509, 514, 530, 515, 461, 546, 513, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[487, 489, 517, 481, 494, 472, 514, 481, 492, 515, 490, 523, 501, 538, 496, 508, 446, 535, 507, 481, 512, 543, 517, 516, 518, 515, 516, 511, 509, 516, 531, 489, 493, 489, 534, 550, 499, 481, 503, 505, 530, 534, 482, 537, 501, 483, 496, 512, 502, 506, 482, 513, 502, 523, 516, 477, 484, 468, 496, 516, 505, 552, 505, 516, 508, 523, 489, 552, 526, 517, 531, 542, 503, 572, 505, 534, 521, 500, 546, 528, 525, 516, 537, 542, 501, 559, 505, 494, 498, 508, 510, 502, 508, 556, 496, 506, 516, 517, 490, 547, 553, 498, 502, 515, 506, 494, 487, 556, 530, 531, 536, 494, 540, 522, 525, 534, 499, 521, 499, 514, 500, 505, 455, 524, 523, 510, 540, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[533, 515, 527, 533, 504, 491, 512, 487, 490, 456, 498, 523, 525, 494, 523, 536, 519, 455, 533, 524, 519, 497, 523, 500, 514, 558, 469, 498, 551, 514, 502, 512, 528, 475, 546, 520, 523, 505, 542, 534, 513, 532, 534, 513, 524, 474, 522, 499, 520, 474, 522, 516, 504, 515, 513, 552, 476, 549, 502, 489, 479, 490, 518, 532, 475, 550, 488, 517, 506, 468, 491, 475, 524, 527, 498, 533, 533, 533, 508, 504, 558, 508, 508, 551, 521, 504, 517, 517, 553, 554, 511, 515, 509, 515, 514, 506, 476, 525, 483, 531, 496, 457, 510, 503, 511, 494, 520, 510, 490, 541, 545, 522, 510, 518, 522, 481, 504, 507, 537, 496, 500, 511, 522, 471, 543, 518, 537, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[523, 470, 532, 480, 503, 530, 528, 500, 489, 512, 530, 496, 516, 503, 506, 524, 470, 497, 476, 500, 542, 504, 495, 503, 484, 534, 514, 529, 485, 522, 526, 525, 507, 508, 519, 499, 571, 517, 508, 518, 496, 488, 509, 524, 515, 511, 521, 544, 487, 493, 540, 515, 539, 548, 509, 505, 484, 501, 496, 520, 470, 482, 480, 509, 518, 516, 496, 548, 458, 541, 532, 492, 487, 552, 497, 531, 497, 536, 521, 506, 493, 538, 562, 511, 522, 530, 526, 497, 516, 524, 499, 520, 480, 537, 500, 549, 469, 484, 522, 492, 485, 549, 540, 476, 530, 497, 519, 499, 486, 529, 498, 559, 542, 521, 484, 513, 532, 502, 507, 518, 543, 523, 509, 540, 503, 484, 519, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[526, 519, 491, 512, 516, 490, 508, 524, 518, 518, 468, 541, 523, 530, 511, 514, 493, 520, 526, 509, 514, 496, 493, 523, 520, 465, 500, 502, 549, 549, 505, 536, 521, 491, 481, 514, 495, 450, 507, 469, 521, 530, 489, 524, 507, 495, 522, 526, 543, 490, 524, 491, 511, 495, 546, 537, 483, 514, 500, 493, 554, 557, 547, 561, 526, 541, 522, 513, 502, 501, 497, 516, 500, 518, 493, 534, 477, 501, 483, 505, 534, 477, 487, 520, 548, 487, 513, 487, 481, 506, 506, 575, 507, 553, 472, 529, 479, 500, 522, 466, 518, 522, 517, 531, 513, 522, 538, 559, 527, 494, 516, 513, 499, 460, 514, 523, 491, 525, 520, 495, 467, 548, 583, 524, 513, 535, 486, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[482, 522, 492, 527, 542, 528, 530, 533, 500, 501, 478, 548, 515, 494, 506, 553, 546, 526, 517, 525, 514, 497, 524, 472, 509, 530, 523, 569, 549, 506, 529, 502, 532, 525, 484, 493, 522, 540, 554, 519, 481, 574, 506, 471, 527, 484, 513, 523, 501, 532, 451, 498, 522, 536, 484, 495, 499, 523, 568, 502, 483, 514, 511, 510, 539, 504, 480, 485, 502, 474, 488, 528, 490, 480, 512, 525, 462, 507, 512, 501, 477, 529, 533, 484, 541, 515, 545, 503, 540, 481, 489, 506, 502, 504, 517, 510, 515, 528, 505, 531, 505, 535, 498, 503, 555, 533, 528, 498, 532, 476, 519, 510, 520, 525, 502, 518, 506, 558, 514, 491, 486, 478, 482, 489, 540, 513, 498, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[496, 521, 504, 516, 511, 519, 548, 486, 520, 541, 496, 548, 481, 533, 538, 500, 502, 463, 499, 503, 508, 534, 501, 539, 518, 484, 505, 503, 478, 549, 508, 494, 542, 550, 544, 544, 488, 518, 517, 515, 515, 481, 478, 508, 498, 502, 537, 495, 528, 507, 472, 480, 507, 498, 560, 474, 544, 473, 494, 519, 504, 536, 472, 503, 551, 459, 540, 495, 515, 471, 505, 475, 510, 527, 516, 504, 515, 470, 506, 522, 546, 522, 521, 536, 556, 509, 513, 519, 508, 498, 504, 489, 492, 493, 495, 519, 523, 513, 520, 554, 496, 521, 546, 498, 515, 483, 533, 525, 526, 495, 514, 527, 499, 517, 504, 506, 528, 543, 538, 513, 482, 543, 520, 489, 527, 533, 545, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[517, 555, 528, 516, 560, 563, 523, 507, 518, 504, 487, 518, 479, 492, 531, 526, 552, 474, 566, 524, 533, 523, 489, 507, 522, 519, 538, 536, 499, 514, 525, 497, 531, 510, 530, 491, 487, 511, 526, 497, 508, 510, 513, 551, 487, 501, 485, 484, 480, 458, 510, 491, 535, 506, 514, 503, 526, 471, 550, 555, 527, 482, 522, 489, 533, 529, 500, 510, 546, 511, 552, 510, 500, 516, 546, 515, 525, 471, 514, 521, 505, 527, 508, 489, 507, 495, 521, 539, 532, 515, 519, 490, 483, 496, 547, 509, 539, 488, 489, 545, 507, 507, 500, 505, 510, 516, 508, 475, 474, 486, 497, 545, 540, 492, 511, 511, 526, 490, 498, 497, 516, 509, 521, 521, 494, 465, 505, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[487, 489, 517, 481, 494, 472, 514, 481, 492, 515, 490, 517, 501, 538, 496, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 535, 507, 481, 512, 543, 517, 516, 518, 515, 516, 511, 509, 511, 531, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 489, 534, 550, 499, 481, 503, 505, 530, 534, 482, 537, 501, 483, 496, 512, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 506, 482, 513, 502, 523, 516, 477, 484, 468, 496, 516, 505, 552, 505, 516, 29, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 523, 489, 552, 526, 517, 531, 542, 503, 572, 505, 534, 521, 500, 546, 528, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 516, 537, 542, 472, 559, 505, 494, 498, 508, 510, 502, 508, 556, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 490, 547, 553, 498, 502, 515, 506, 494, 475, 556, 530, 531, 536, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 522, 525, 534, 499, 521, 499, 514, 500, 505, 455, 524, 523, 510, 540, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7992, 8157, 8136, 8109, 8405, 8234, 8260, 8243] avg=8192.0000 max=8405 max_rank=4 min=7992 min_rank=0 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=33142 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[533, 515, 527, 533, 504, 491, 512, 487, 490, 456, 498, 517, 525, 494, 523, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 455, 533, 524, 519, 497, 523, 500, 514, 558, 469, 498, 551, 509, 502, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 475, 546, 520, 523, 505, 542, 534, 513, 532, 534, 513, 524, 474, 522, 499, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 474, 522, 516, 504, 515, 513, 552, 476, 549, 502, 489, 479, 490, 518, 532, 30, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 550, 488, 517, 506, 468, 491, 475, 524, 527, 498, 533, 533, 533, 508, 504, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 508, 508, 551, 491, 504, 517, 517, 553, 554, 511, 515, 509, 515, 514, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 525, 483, 531, 496, 457, 510, 503, 511, 494, 507, 510, 490, 541, 545, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 518, 522, 481, 504, 507, 537, 496, 500, 511, 522, 471, 543, 518, 537, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8183, 8291, 8199, 8137, 8331, 8101, 8153] avg=8192.0000 max=8331 max_rank=5 min=8101 min_rank=6 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[526, 519, 491, 512, 516, 490, 508, 524, 518, 518, 468, 535, 523, 530, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 526, 509, 514, 496, 493, 523, 520, 465, 500, 502, 549, 544, 505, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 481, 514, 495, 450, 507, 469, 521, 530, 489, 524, 507, 495, 522, 526, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 490, 524, 491, 511, 495, 546, 537, 483, 514, 500, 493, 554, 557, 547, 561, 31, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 541, 522, 513, 502, 501, 497, 516, 500, 518, 493, 534, 477, 501, 483, 505, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 477, 487, 520, 517, 487, 513, 487, 481, 506, 506, 575, 507, 553, 472, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 500, 522, 466, 518, 522, 517, 531, 513, 522, 525, 559, 527, 494, 516, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 460, 514, 523, 491, 525, 520, 495, 467, 548, 583, 524, 513, 535, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8195, 8049, 8395, 8136, 8151, 8224, 8183] avg=8192.0000 max=8395 max_rank=3 min=8049 min_rank=2 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 502, 509, 529, 520, 536, 538, 505, 506, 500, 510, 502, 534, 549, 501, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 529, 504, 521, 522, 507, 527, 498, 512, 508, 489, 491, 508, 525, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 518, 521, 518, 471, 522, 497, 534, 480, 559, 489, 503, 519, 485, 500, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 545, 519, 511, 501, 504, 490, 527, 483, 525, 509, 510, 523, 474, 541, 507, 29, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 487, 513, 468, 496, 510, 500, 571, 502, 513, 541, 501, 536, 500, 555, 529, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 506, 460, 502, 474, 535, 517, 517, 524, 497, 485, 489, 522, 514, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 508, 534, 501, 491, 508, 515, 512, 512, 527, 524, 518, 507, 546, 504, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 515, 541, 549, 519, 523, 447, 509, 514, 530, 515, 461, 546, 513, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8172, 8167, 8227, 8224, 8064, 8223, 8200] avg=8192.0000 max=8259 max_rank=0 min=8064 min_rank=5 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 555, 528, 516, 560, 563, 523, 507, 518, 504, 487, 502, 479, 492, 531, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 474, 566, 524, 533, 523, 489, 507, 522, 519, 538, 536, 499, 507, 525, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 530, 491, 487, 511, 526, 497, 508, 510, 513, 551, 487, 501, 485, 484, 7, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 458, 510, 491, 535, 506, 514, 503, 526, 471, 550, 555, 527, 482, 522, 489, 33, 18, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 529, 500, 510, 546, 511, 552, 510, 500, 516, 546, 515, 525, 471, 514, 521, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 508, 489, 474, 495, 521, 539, 532, 515, 519, 490, 483, 496, 547, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 488, 489, 545, 507, 507, 500, 505, 510, 516, 490, 475, 474, 486, 497, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 492, 511, 511, 526, 490, 498, 497, 516, 509, 521, 521, 494, 465, 505, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8311, 8138, 8181, 8314, 8149, 8073, 8062] avg=8192.0000 max=8314 max_rank=4 min=8062 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 470, 532, 480, 503, 530, 528, 500, 489, 512, 530, 490, 516, 503, 506, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 497, 476, 500, 542, 504, 495, 503, 484, 534, 514, 529, 485, 517, 526, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 508, 519, 499, 571, 517, 508, 518, 496, 488, 509, 524, 515, 511, 521, 544, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 493, 540, 515, 539, 548, 509, 505, 484, 501, 496, 520, 470, 482, 480, 509, 30, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 496, 548, 458, 541, 532, 492, 487, 552, 497, 531, 497, 536, 521, 506, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 538, 562, 511, 492, 530, 526, 497, 516, 524, 499, 520, 480, 537, 500, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 484, 522, 492, 485, 549, 540, 476, 530, 497, 506, 499, 486, 529, 498, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 521, 484, 513, 532, 502, 507, 518, 543, 523, 509, 540, 503, 484, 519, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8101, 8262, 8126, 8236, 8274, 8121, 8280] avg=8192.0000 max=8280 max_rank=7 min=8101 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32911 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[482, 522, 492, 527, 542, 528, 530, 533, 500, 501, 478, 542, 515, 494, 506, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 526, 517, 525, 514, 497, 524, 472, 509, 530, 523, 569, 549, 502, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 484, 493, 522, 540, 554, 519, 481, 574, 506, 471, 527, 484, 513, 523, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 451, 498, 522, 536, 484, 495, 499, 523, 568, 502, 483, 514, 511, 510, 31, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 480, 485, 502, 474, 488, 528, 490, 480, 512, 525, 462, 507, 512, 501, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 529, 533, 484, 510, 515, 545, 503, 540, 481, 489, 506, 502, 504, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 528, 505, 531, 505, 535, 498, 503, 555, 533, 515, 498, 532, 476, 519, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 525, 502, 518, 506, 558, 514, 491, 486, 478, 482, 489, 540, 513, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8334, 8254, 8178, 7996, 8145, 8258, 8126] avg=8192.0000 max=8334 max_rank=1 min=7996 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32525 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[496, 521, 504, 516, 511, 519, 548, 486, 520, 541, 496, 542, 481, 533, 538, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 463, 499, 503, 508, 534, 501, 539, 518, 484, 505, 503, 478, 544, 508, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 550, 544, 544, 488, 518, 517, 515, 515, 481, 478, 508, 498, 502, 537, 495, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 507, 472, 480, 507, 498, 560, 474, 544, 473, 494, 519, 504, 536, 472, 503, 32, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 459, 540, 495, 515, 471, 505, 475, 510, 527, 516, 504, 515, 470, 506, 522, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 522, 521, 536, 524, 509, 513, 519, 508, 498, 504, 489, 492, 493, 495, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 520, 554, 496, 521, 546, 498, 515, 483, 520, 525, 526, 495, 514, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 517, 504, 506, 528, 543, 538, 513, 482, 543, 520, 489, 527, 533, 545, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8083, 8239, 8121, 8088, 8188, 8276, 8289] avg=8192.0000 max=8289 max_rank=7 min=8083 min_rank=1 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[540, 514, 491, 526, 497, 506, 520, 494, 483, 485, 544, 513, 535, 489, 512, 562, 502, 536, 468, 502, 519, 558, 488, 511, 498, 515, 527, 533, 497, 512, 527, 475, 538, 542, 512, 518, 495, 511, 512, 499, 534, 509, 508, 562, 555, 524, 538, 515, 501, 516, 520, 554, 495, 497, 508, 491, 490, 513, 450, 499, 481, 510, 490, 513, 489, 495, 467, 514, 515, 497, 527, 479, 497, 543, 503, 507, 503, 483, 489, 496, 530, 511, 501, 537, 533, 514, 498, 513, 504, 523, 490, 494, 553, 543, 514, 534, 527, 525, 518, 505, 528, 492, 537, 529, 526, 470, 543, 481, 487, 511, 470, 562, 509, 530, 525, 514, 503, 540, 508, 541, 496, 540, 532, 509, 497, 508, 504, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[556, 535, 501, 532, 554, 505, 490, 536, 517, 506, 528, 481, 527, 519, 501, 525, 479, 483, 519, 497, 497, 478, 542, 483, 526, 501, 483, 521, 457, 507, 497, 530, 516, 527, 574, 523, 502, 511, 515, 481, 488, 481, 484, 495, 528, 502, 494, 516, 544, 487, 524, 528, 539, 501, 454, 511, 552, 480, 525, 534, 493, 511, 493, 521, 522, 515, 491, 540, 502, 544, 501, 528, 507, 532, 511, 533, 520, 529, 483, 506, 493, 533, 529, 489, 528, 529, 519, 497, 545, 521, 496, 537, 516, 544, 503, 517, 509, 490, 498, 513, 562, 527, 491, 514, 507, 530, 510, 536, 492, 523, 481, 509, 502, 493, 545, 486, 519, 489, 502, 523, 520, 486, 513, 515, 500, 508, 525, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[515, 511, 548, 538, 495, 532, 516, 541, 520, 517, 502, 512, 516, 531, 533, 519, 473, 453, 518, 482, 519, 537, 538, 518, 529, 522, 525, 533, 502, 474, 528, 526, 504, 498, 494, 503, 516, 502, 524, 527, 501, 521, 480, 489, 503, 514, 533, 507, 537, 536, 483, 519, 509, 511, 513, 504, 513, 518, 516, 477, 499, 509, 498, 500, 484, 518, 490, 485, 509, 506, 475, 575, 505, 509, 508, 479, 519, 503, 540, 514, 530, 527, 485, 513, 550, 510, 531, 488, 540, 503, 503, 503, 509, 520, 531, 516, 515, 513, 544, 493, 520, 530, 534, 536, 468, 531, 526, 515, 499, 533, 431, 495, 507, 524, 495, 522, 512, 481, 537, 539, 510, 531, 489, 502, 521, 521, 483, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[551, 552, 519, 494, 488, 525, 503, 535, 479, 474, 527, 527, 514, 523, 518, 494, 529, 539, 512, 543, 531, 547, 547, 508, 532, 566, 507, 498, 501, 484, 518, 543, 458, 501, 554, 533, 495, 510, 526, 475, 545, 522, 560, 507, 488, 508, 537, 536, 529, 521, 483, 475, 524, 531, 539, 489, 501, 523, 517, 515, 489, 472, 503, 457, 521, 503, 514, 509, 509, 543, 480, 495, 509, 533, 507, 516, 543, 518, 504, 490, 544, 480, 513, 482, 505, 525, 522, 516, 521, 508, 496, 535, 512, 506, 555, 525, 530, 455, 469, 547, 499, 490, 537, 476, 473, 483, 495, 510, 552, 473, 500, 526, 540, 527, 547, 530, 468, 493, 513, 498, 524, 522, 481, 496, 500, 485, 486, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[506, 497, 506, 546, 510, 489, 503, 493, 497, 510, 528, 496, 484, 450, 501, 522, 496, 476, 515, 512, 509, 475, 510, 515, 511, 510, 535, 510, 492, 529, 503, 539, 554, 540, 494, 487, 508, 515, 500, 531, 487, 521, 498, 503, 549, 500, 522, 513, 518, 498, 501, 527, 481, 469, 525, 548, 536, 522, 545, 503, 514, 479, 528, 514, 524, 495, 553, 507, 510, 556, 520, 523, 537, 516, 497, 514, 509, 504, 518, 539, 532, 538, 519, 514, 515, 492, 475, 480, 499, 554, 539, 504, 517, 513, 516, 491, 505, 511, 487, 508, 510, 504, 503, 552, 509, 449, 509, 563, 483, 522, 519, 550, 510, 491, 520, 517, 515, 512, 510, 523, 522, 530, 511, 481, 512, 553, 506, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[543, 502, 547, 539, 498, 494, 499, 542, 525, 548, 496, 486, 544, 544, 532, 511, 481, 520, 506, 507, 539, 515, 465, 539, 516, 479, 540, 519, 528, 443, 514, 540, 527, 515, 515, 519, 566, 501, 500, 525, 528, 497, 503, 505, 515, 522, 510, 476, 510, 490, 477, 472, 485, 494, 519, 541, 535, 501, 504, 469, 487, 502, 504, 520, 492, 503, 522, 521, 522, 524, 522, 541, 513, 527, 526, 563, 540, 499, 542, 568, 481, 480, 525, 495, 513, 509, 508, 507, 497, 519, 527, 488, 518, 507, 541, 500, 467, 520, 471, 474, 511, 527, 511, 510, 456, 509, 486, 506, 559, 479, 537, 514, 513, 497, 522, 508, 527, 538, 495, 501, 520, 485, 570, 489, 520, 548, 500, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[526, 546, 513, 504, 500, 535, 526, 483, 506, 530, 492, 496, 543, 511, 487, 490, 509, 542, 514, 541, 510, 473, 519, 480, 501, 539, 493, 561, 519, 516, 481, 560, 525, 491, 508, 531, 455, 482, 485, 542, 518, 502, 533, 490, 473, 554, 519, 534, 548, 468, 512, 488, 509, 502, 482, 537, 496, 493, 533, 495, 495, 505, 535, 536, 516, 521, 512, 512, 544, 542, 524, 549, 516, 486, 543, 503, 488, 484, 517, 545, 515, 526, 533, 527, 460, 514, 507, 545, 536, 536, 528, 495, 528, 493, 549, 508, 489, 476, 498, 516, 490, 513, 473, 515, 497, 479, 484, 519, 522, 504, 526, 545, 475, 519, 520, 517, 502, 530, 545, 503, 472, 489, 514, 535, 489, 498, 531, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[482, 512, 513, 538, 489, 523, 496, 545, 499, 535, 532, 589, 508, 531, 516, 524, 537, 541, 528, 475, 526, 538, 498, 475, 487, 490, 514, 521, 519, 481, 544, 519, 497, 454, 559, 518, 512, 522, 478, 530, 489, 479, 478, 482, 550, 557, 542, 487, 504, 493, 523, 523, 513, 491, 538, 526, 508, 496, 529, 490, 520, 510, 473, 521, 529, 519, 503, 512, 505, 514, 500, 493, 493, 524, 538, 518, 515, 525, 514, 511, 526, 509, 529, 523, 517, 511, 521, 511, 553, 524, 521, 483, 514, 485, 510, 522, 496, 509, 511, 461, 529, 508, 558, 499, 510, 502, 506, 521, 479, 495, 521, 510, 491, 501, 478, 519, 501, 507, 507, 493, 538, 492, 503, 517, 545, 509, 512, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[481, 514, 491, 526, 497, 506, 520, 494, 483, 485, 544, 513, 535, 489, 512, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 536, 468, 502, 519, 558, 488, 511, 498, 515, 527, 533, 497, 512, 527, 475, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 542, 505, 518, 495, 511, 512, 499, 534, 509, 508, 562, 555, 524, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 520, 554, 495, 497, 508, 491, 490, 513, 450, 499, 481, 510, 490, 513, 59, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 495, 467, 514, 515, 462, 527, 479, 497, 543, 503, 507, 503, 483, 489, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 511, 501, 537, 533, 514, 498, 513, 504, 523, 490, 494, 553, 543, 463, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 518, 505, 528, 492, 537, 529, 526, 470, 543, 481, 487, 511, 470, 562, 36, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 530, 525, 514, 503, 540, 508, 541, 496, 540, 532, 509, 497, 508, 504, 494, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8173, 8365, 8102, 7969, 8241, 8276, 8258] avg=8192.0000 max=8365 max_rank=2 min=7969 min_rank=4 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[495, 535, 501, 532, 554, 505, 490, 536, 517, 506, 528, 481, 527, 519, 501, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 483, 519, 497, 497, 478, 542, 483, 526, 501, 483, 521, 457, 507, 497, 530, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 527, 566, 523, 502, 511, 515, 481, 488, 481, 484, 495, 528, 502, 494, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 487, 524, 528, 539, 501, 454, 511, 552, 480, 525, 534, 493, 511, 493, 521, 61, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 515, 491, 540, 502, 505, 501, 528, 507, 532, 511, 533, 520, 529, 483, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 533, 529, 489, 528, 529, 519, 497, 545, 521, 496, 537, 516, 544, 452, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 498, 513, 562, 527, 491, 514, 507, 530, 510, 536, 492, 523, 481, 509, 36, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 493, 545, 486, 519, 489, 502, 523, 520, 486, 513, 515, 500, 508, 525, 511, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8006, 8129, 8273, 8225, 8245, 8260, 8146] avg=8192.0000 max=8273 max_rank=3 min=8006 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[459, 511, 548, 538, 495, 532, 516, 541, 520, 517, 502, 512, 516, 531, 533, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 453, 518, 482, 519, 537, 538, 518, 529, 522, 525, 533, 502, 474, 528, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 487, 503, 516, 502, 524, 527, 501, 521, 480, 489, 503, 514, 533, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 536, 483, 519, 509, 511, 513, 504, 513, 518, 516, 477, 499, 509, 498, 500, 56, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 518, 490, 485, 509, 470, 475, 575, 505, 509, 508, 479, 519, 503, 540, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 527, 485, 513, 550, 510, 531, 488, 540, 503, 503, 503, 509, 520, 478, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 513, 544, 493, 520, 530, 534, 536, 468, 531, 526, 515, 499, 533, 431, 495, 38, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 524, 495, 522, 512, 481, 537, 539, 510, 531, 489, 502, 521, 521, 483, 520, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8182, 8109, 8213, 8083, 8206, 8251, 8202] avg=8192.0000 max=8290 max_rank=0 min=8083 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[491, 552, 519, 494, 488, 525, 503, 535, 479, 474, 527, 527, 514, 523, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 539, 512, 543, 531, 547, 547, 508, 532, 566, 507, 498, 501, 484, 518, 543, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 501, 546, 533, 495, 510, 526, 475, 545, 522, 560, 507, 488, 508, 537, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 521, 483, 475, 524, 531, 539, 489, 501, 523, 517, 515, 489, 472, 503, 457, 60, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 514, 509, 509, 504, 480, 495, 509, 533, 507, 516, 543, 518, 504, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 480, 513, 482, 505, 525, 522, 516, 521, 508, 496, 535, 512, 506, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 455, 469, 547, 499, 490, 537, 476, 473, 483, 495, 510, 552, 473, 500, 526, 39, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 527, 547, 530, 468, 493, 513, 498, 524, 522, 481, 496, 500, 485, 486, 521, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8411, 8247, 8144, 8155, 8190, 8086, 8140] avg=8192.0000 max=8411 max_rank=1 min=8086 min_rank=6 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32571 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[464, 546, 513, 504, 500, 535, 526, 483, 506, 530, 492, 496, 543, 511, 487, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 542, 514, 541, 510, 473, 519, 480, 501, 539, 493, 561, 519, 516, 481, 560, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 491, 494, 531, 455, 482, 485, 542, 518, 502, 533, 490, 473, 554, 519, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 468, 512, 488, 509, 502, 482, 537, 496, 493, 533, 495, 495, 505, 535, 536, 62, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 521, 512, 512, 544, 497, 524, 549, 516, 486, 543, 503, 488, 484, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 526, 533, 527, 460, 514, 507, 545, 536, 536, 528, 495, 528, 493, 487, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 476, 498, 516, 490, 513, 473, 515, 497, 479, 484, 519, 522, 504, 526, 545, 43, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 519, 520, 517, 502, 530, 545, 503, 472, 489, 514, 535, 489, 498, 531, 527, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8267, 8128, 8215, 8257, 8238, 8123, 8182] avg=8192.0000 max=8267 max_rank=1 min=8123 min_rank=6 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[484, 502, 547, 539, 498, 494, 499, 542, 525, 548, 496, 486, 544, 544, 532, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 520, 506, 507, 539, 515, 465, 539, 516, 479, 540, 519, 528, 443, 514, 540, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 515, 508, 519, 566, 501, 500, 525, 528, 497, 503, 505, 515, 522, 510, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 477, 472, 485, 494, 519, 541, 535, 501, 504, 469, 487, 502, 504, 520, 59, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 503, 522, 521, 522, 487, 522, 541, 513, 527, 526, 563, 540, 499, 542, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 480, 525, 495, 513, 509, 508, 507, 497, 519, 527, 488, 518, 507, 487, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 520, 471, 474, 511, 527, 511, 510, 456, 509, 486, 506, 559, 479, 537, 514, 38, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 497, 522, 508, 527, 538, 495, 501, 520, 485, 570, 489, 520, 548, 500, 491, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8156, 8217, 8085, 8388, 8061, 8106, 8232] avg=8192.0000 max=8388 max_rank=4 min=8061 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[451, 497, 506, 546, 510, 489, 503, 493, 497, 510, 528, 496, 484, 450, 501, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 476, 515, 512, 509, 475, 510, 515, 511, 510, 535, 510, 492, 529, 503, 539, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 540, 487, 487, 508, 515, 500, 531, 487, 521, 498, 503, 549, 500, 522, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 498, 501, 527, 481, 469, 525, 548, 536, 522, 545, 503, 514, 479, 528, 514, 55, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 495, 553, 507, 510, 516, 520, 523, 537, 516, 497, 514, 509, 504, 518, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 538, 519, 514, 515, 492, 475, 480, 499, 554, 539, 504, 517, 513, 465, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 487, 508, 510, 504, 503, 552, 509, 449, 509, 563, 483, 522, 519, 550, 36, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 520, 517, 515, 512, 510, 523, 522, 530, 511, 481, 512, 553, 506, 514, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7983, 8142, 8215, 8278, 8282, 8147, 8253, 8236] avg=8192.0000 max=8282 max_rank=4 min=7983 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[429, 512, 513, 538, 489, 523, 496, 545, 499, 535, 532, 589, 508, 531, 516, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 541, 528, 475, 526, 538, 498, 475, 487, 490, 514, 521, 519, 481, 544, 519, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 454, 551, 518, 512, 522, 478, 530, 489, 479, 478, 482, 550, 557, 542, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 493, 523, 523, 513, 491, 538, 526, 508, 496, 529, 490, 520, 510, 473, 521, 53, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 503, 512, 505, 478, 500, 493, 493, 524, 538, 518, 515, 525, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 509, 529, 523, 517, 511, 521, 511, 553, 524, 521, 483, 514, 485, 459, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 509, 511, 461, 529, 508, 558, 499, 510, 502, 506, 521, 479, 495, 521, 510, 36, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 478, 519, 501, 507, 507, 493, 538, 492, 503, 517, 545, 509, 512, 519, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8199, 8126, 8226, 8177, 8208, 8181, 8140] avg=8192.0000 max=8279 max_rank=0 min=8126 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3840, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 3968, 3968, 4096, 4224, 4096, 4096, 4096, 4224, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[524, 504, 537, 506, 539, 499, 568, 549, 521, 500, 524, 528, 490, 507, 515, 516, 481, 528, 548, 547, 577, 552, 502, 481, 502, 553, 542, 510, 504, 536, 532, 542, 507, 510, 552, 510, 521, 484, 482, 538, 476, 491, 484, 501, 486, 526, 476, 462, 492, 510, 511, 541, 506, 476, 496, 482, 521, 547, 505, 499, 508, 501, 512, 514, 520, 488, 499, 536, 502, 485, 519, 491, 516, 523, 527, 502, 486, 515, 501, 555, 506, 474, 542, 528, 517, 519, 502, 514, 484, 486, 493, 509, 498, 460, 514, 540, 537, 495, 534, 515, 527, 490, 490, 509, 544, 480, 500, 504, 505, 524, 561, 522, 486, 526, 521, 541, 529, 500, 499, 524, 507, 493, 525, 484, 482, 500, 545, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[524, 504, 541, 489, 514, 545, 492, 478, 477, 520, 549, 496, 542, 508, 554, 530, 493, 493, 483, 467, 514, 510, 491, 503, 531, 502, 523, 513, 527, 522, 523, 468, 492, 516, 504, 504, 531, 530, 513, 535, 515, 548, 493, 522, 481, 544, 523, 546, 493, 490, 520, 484, 531, 487, 492, 527, 530, 489, 479, 537, 526, 523, 530, 532, 524, 482, 529, 493, 531, 516, 558, 565, 479, 508, 507, 548, 527, 547, 497, 521, 499, 480, 504, 525, 517, 526, 494, 507, 498, 518, 498, 526, 531, 548, 558, 509, 508, 514, 487, 488, 513, 494, 502, 531, 516, 463, 493, 510, 475, 548, 470, 506, 491, 552, 497, 512, 501, 496, 527, 548, 529, 489, 498, 487, 510, 496, 526, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[522, 530, 553, 505, 513, 507, 523, 492, 505, 493, 532, 496, 518, 521, 515, 517, 525, 549, 481, 501, 505, 520, 507, 552, 489, 484, 499, 483, 513, 483, 561, 518, 502, 495, 559, 523, 514, 501, 575, 508, 559, 499, 528, 501, 511, 500, 476, 490, 497, 495, 527, 524, 513, 511, 525, 496, 480, 517, 498, 499, 496, 535, 513, 527, 507, 476, 500, 499, 490, 499, 526, 511, 518, 516, 555, 490, 516, 481, 531, 469, 489, 520, 531, 518, 561, 500, 506, 520, 568, 478, 522, 515, 465, 509, 547, 512, 488, 511, 510, 512, 493, 543, 485, 514, 529, 506, 567, 502, 494, 526, 542, 511, 530, 497, 527, 517, 523, 489, 527, 535, 489, 533, 455, 540, 515, 498, 505, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[530, 532, 483, 503, 530, 517, 517, 495, 497, 478, 559, 488, 485, 534, 514, 508, 523, 498, 481, 531, 518, 540, 519, 515, 533, 489, 485, 500, 502, 518, 516, 533, 504, 487, 506, 507, 525, 518, 519, 547, 529, 522, 505, 532, 516, 512, 478, 535, 526, 496, 524, 510, 503, 519, 514, 532, 589, 471, 510, 512, 507, 512, 510, 506, 492, 530, 517, 474, 492, 543, 545, 535, 505, 531, 474, 518, 543, 555, 495, 508, 493, 479, 494, 520, 506, 496, 547, 499, 481, 521, 498, 502, 502, 504, 520, 516, 486, 527, 505, 540, 503, 539, 502, 506, 500, 497, 465, 558, 506, 493, 495, 515, 487, 523, 493, 523, 509, 558, 532, 506, 507, 482, 508, 526, 520, 513, 527, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[544, 545, 542, 507, 503, 536, 530, 517, 476, 531, 471, 515, 490, 511, 500, 501, 530, 491, 491, 525, 501, 524, 534, 502, 480, 504, 547, 463, 482, 510, 502, 531, 535, 500, 523, 504, 514, 526, 521, 489, 487, 523, 499, 500, 512, 514, 499, 538, 532, 552, 485, 495, 518, 510, 548, 535, 524, 538, 544, 477, 535, 530, 483, 563, 561, 473, 493, 477, 503, 464, 542, 499, 512, 524, 507, 532, 501, 487, 541, 544, 540, 518, 539, 532, 493, 493, 487, 482, 517, 546, 495, 483, 501, 521, 512, 460, 499, 488, 519, 537, 496, 503, 465, 489, 538, 528, 497, 558, 507, 514, 505, 489, 528, 543, 509, 461, 508, 550, 501, 514, 512, 483, 496, 540, 545, 483, 534, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[545, 544, 527, 510, 524, 512, 473, 515, 522, 539, 509, 513, 461, 547, 538, 510, 537, 502, 550, 508, 499, 497, 491, 515, 546, 511, 483, 516, 481, 480, 486, 515, 471, 472, 515, 530, 501, 530, 549, 490, 477, 523, 499, 522, 530, 475, 514, 507, 559, 566, 521, 537, 510, 484, 537, 492, 536, 516, 495, 524, 518, 515, 541, 529, 530, 520, 517, 522, 521, 499, 491, 500, 535, 510, 547, 518, 504, 470, 505, 507, 497, 517, 523, 491, 531, 514, 531, 560, 508, 509, 533, 522, 532, 450, 516, 502, 462, 515, 519, 477, 513, 472, 530, 512, 522, 455, 546, 504, 520, 516, 521, 520, 476, 478, 525, 517, 510, 519, 486, 494, 483, 523, 551, 498, 533, 510, 505, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[499, 514, 489, 544, 469, 514, 545, 525, 483, 507, 522, 499, 509, 500, 521, 509, 498, 504, 499, 517, 485, 525, 479, 516, 510, 563, 543, 520, 484, 508, 507, 495, 548, 509, 557, 538, 509, 525, 515, 501, 540, 498, 505, 515, 503, 525, 497, 503, 529, 519, 502, 541, 498, 527, 523, 531, 494, 507, 539, 526, 504, 568, 492, 519, 508, 527, 488, 499, 527, 469, 480, 536, 481, 491, 505, 531, 505, 497, 546, 493, 483, 514, 535, 510, 566, 522, 533, 511, 539, 545, 487, 514, 548, 475, 492, 527, 475, 519, 497, 522, 500, 504, 505, 479, 551, 428, 540, 511, 503, 491, 546, 479, 540, 593, 469, 549, 511, 481, 519, 494, 481, 526, 536, 497, 489, 464, 516, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[540, 527, 511, 495, 507, 526, 542, 534, 507, 544, 487, 479, 502, 507, 496, 520, 502, 477, 454, 501, 489, 526, 503, 532, 512, 477, 520, 559, 536, 537, 513, 495, 512, 512, 518, 517, 493, 517, 506, 511, 520, 487, 547, 471, 554, 520, 459, 502, 528, 495, 515, 505, 523, 489, 515, 511, 508, 497, 530, 547, 507, 535, 506, 519, 508, 524, 501, 539, 490, 502, 534, 528, 523, 495, 530, 512, 517, 479, 562, 517, 459, 503, 509, 503, 500, 484, 533, 508, 490, 498, 560, 490, 505, 522, 481, 485, 523, 486, 502, 533, 516, 520, 525, 532, 520, 467, 511, 512, 485, 514, 501, 548, 475, 526, 521, 508, 538, 562, 525, 518, 517, 514, 508, 487, 566, 510, 527, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[483, 504, 537, 506, 539, 499, 568, 549, 521, 500, 524, 528, 490, 507, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 528, 548, 547, 577, 552, 502, 481, 502, 553, 542, 510, 504, 536, 532, 542, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 552, 510, 521, 484, 482, 538, 476, 491, 484, 501, 486, 526, 476, 462, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 510, 511, 541, 506, 476, 496, 482, 521, 547, 505, 499, 508, 448, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 488, 499, 536, 502, 485, 519, 491, 516, 523, 527, 502, 486, 515, 501, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 474, 542, 528, 517, 519, 502, 514, 484, 486, 493, 509, 498, 460, 514, 540, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 495, 534, 515, 527, 490, 490, 509, 544, 480, 500, 504, 505, 524, 561, 522, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 526, 521, 541, 529, 500, 499, 524, 507, 493, 525, 484, 482, 500, 545, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8454, 8009, 8068, 8165, 8109, 8288, 8157] avg=8192.0000 max=8454 max_rank=1 min=8009 min_rank=2 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[488, 532, 483, 503, 530, 517, 517, 495, 497, 478, 559, 488, 485, 534, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 498, 481, 531, 518, 540, 519, 515, 533, 489, 485, 500, 502, 518, 516, 533, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 487, 506, 507, 525, 518, 519, 547, 529, 522, 505, 532, 516, 512, 478, 535, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 524, 510, 503, 519, 514, 532, 589, 471, 510, 512, 507, 458, 510, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 530, 517, 474, 492, 543, 545, 535, 505, 531, 474, 518, 543, 555, 495, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 479, 494, 520, 506, 496, 547, 499, 481, 521, 498, 502, 502, 504, 520, 516, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 527, 505, 540, 503, 539, 502, 506, 500, 497, 465, 558, 506, 493, 495, 515, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 523, 493, 523, 509, 558, 532, 506, 507, 482, 508, 526, 520, 513, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8219, 8245, 8187, 8257, 8101, 8189, 8210] avg=8192.0000 max=8257 max_rank=4 min=8101 min_rank=5 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[465, 530, 553, 505, 513, 507, 523, 492, 505, 493, 532, 496, 518, 521, 515, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 549, 481, 501, 505, 520, 507, 552, 489, 484, 499, 483, 513, 483, 561, 518, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 559, 523, 514, 501, 575, 508, 559, 499, 528, 501, 511, 500, 476, 490, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 527, 524, 513, 511, 525, 496, 480, 517, 498, 499, 496, 475, 513, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 476, 500, 499, 490, 499, 524, 511, 518, 516, 555, 490, 516, 481, 531, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 520, 531, 518, 561, 500, 506, 520, 568, 478, 522, 515, 465, 509, 547, 512, 24, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 511, 510, 512, 493, 543, 485, 514, 529, 506, 567, 502, 494, 526, 542, 511, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 497, 527, 517, 523, 489, 527, 535, 489, 533, 455, 540, 515, 498, 505, 472, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8192, 8247, 8093, 8082, 8288, 8290, 8159] avg=8192.0000 max=8290 max_rank=6 min=8082 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[483, 504, 541, 489, 514, 545, 492, 478, 477, 520, 549, 496, 542, 508, 554, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 483, 467, 514, 510, 491, 503, 531, 502, 523, 513, 527, 522, 523, 468, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 504, 504, 531, 530, 513, 535, 515, 548, 493, 522, 481, 544, 523, 546, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 490, 520, 484, 531, 487, 492, 527, 530, 489, 479, 537, 526, 468, 530, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 482, 529, 493, 531, 516, 558, 565, 479, 508, 507, 548, 527, 547, 497, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 480, 504, 525, 517, 526, 494, 507, 498, 518, 498, 526, 531, 548, 558, 509, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 487, 488, 513, 494, 502, 531, 516, 463, 493, 510, 475, 548, 470, 506, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 552, 497, 512, 501, 496, 527, 548, 529, 489, 498, 487, 510, 496, 526, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8080, 8300, 8115, 8332, 8261, 8071, 8155] avg=8192.0000 max=8332 max_rank=4 min=8071 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 545, 542, 507, 503, 536, 530, 517, 476, 531, 471, 515, 490, 511, 500, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 491, 491, 525, 501, 524, 534, 502, 480, 504, 547, 463, 482, 510, 502, 531, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 523, 504, 514, 526, 521, 489, 487, 523, 499, 500, 512, 514, 499, 538, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 552, 485, 495, 518, 510, 548, 535, 524, 538, 544, 477, 535, 474, 483, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 473, 493, 477, 503, 464, 542, 499, 512, 524, 507, 532, 501, 487, 541, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 518, 539, 532, 493, 493, 487, 482, 517, 546, 495, 483, 501, 521, 512, 460, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 488, 519, 537, 496, 503, 465, 489, 538, 528, 497, 558, 507, 514, 505, 489, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 543, 509, 461, 508, 550, 501, 514, 512, 483, 496, 540, 545, 483, 534, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8135, 8187, 8313, 8160, 8142, 8186, 8236] avg=8192.0000 max=8313 max_rank=3 min=8135 min_rank=1 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 544, 527, 510, 524, 512, 473, 515, 522, 539, 509, 513, 461, 547, 538, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 502, 550, 508, 499, 497, 491, 515, 546, 511, 483, 516, 481, 480, 486, 515, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 472, 515, 530, 501, 530, 549, 490, 477, 523, 499, 522, 530, 475, 514, 507, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 566, 521, 537, 510, 484, 537, 492, 536, 516, 495, 524, 518, 461, 541, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 520, 517, 522, 521, 499, 491, 500, 535, 510, 547, 518, 504, 470, 505, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 523, 491, 531, 514, 531, 560, 508, 509, 533, 522, 532, 450, 516, 502, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 515, 519, 477, 513, 472, 530, 512, 522, 455, 546, 504, 520, 516, 521, 520, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 478, 525, 517, 510, 519, 486, 494, 483, 523, 551, 498, 533, 510, 505, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8135, 8108, 8326, 8196, 8259, 8156, 8109] avg=8192.0000 max=8326 max_rank=3 min=8108 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[459, 514, 489, 544, 469, 514, 545, 525, 483, 507, 522, 499, 509, 500, 521, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 499, 517, 485, 525, 479, 516, 510, 563, 543, 520, 484, 508, 507, 495, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 509, 557, 538, 509, 525, 515, 501, 540, 498, 505, 515, 503, 525, 497, 503, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 502, 541, 498, 527, 523, 531, 494, 507, 539, 526, 504, 509, 492, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 488, 499, 527, 469, 480, 536, 481, 491, 505, 531, 505, 497, 546, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 514, 535, 510, 566, 522, 533, 511, 539, 545, 487, 514, 548, 475, 492, 527, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 519, 497, 522, 500, 504, 505, 479, 551, 428, 540, 511, 503, 491, 546, 479, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 593, 469, 549, 511, 481, 519, 494, 481, 526, 536, 497, 489, 464, 516, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8170, 8291, 8260, 8083, 8323, 8107, 8193] avg=8192.0000 max=8323 max_rank=5 min=8083 min_rank=4 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 527, 511, 495, 507, 526, 542, 534, 507, 544, 487, 479, 502, 507, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 477, 454, 501, 489, 526, 503, 532, 512, 477, 520, 559, 536, 537, 513, 495, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 512, 518, 517, 493, 517, 506, 511, 520, 487, 547, 471, 554, 520, 459, 502, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 495, 515, 505, 523, 489, 515, 511, 508, 497, 530, 547, 507, 479, 506, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 524, 501, 539, 490, 502, 534, 528, 523, 495, 530, 512, 517, 479, 562, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 503, 509, 503, 500, 484, 533, 508, 490, 498, 560, 490, 505, 522, 481, 485, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 486, 502, 533, 516, 520, 525, 532, 520, 467, 511, 512, 485, 514, 501, 548, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 526, 521, 508, 538, 562, 525, 518, 517, 514, 508, 487, 566, 510, 527, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8151, 8149, 8174, 8261, 8053, 8249, 8317] avg=8192.0000 max=8317 max_rank=7 min=8053 min_rank=5 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 3840, 4224, 4224, 4096, 4224, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3840, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4352, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 3968, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4352, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[523, 515, 491, 521, 501, 536, 491, 496, 522, 493, 497, 492, 545, 493, 508, 504, 491, 506, 518, 540, 505, 529, 525, 525, 517, 521, 525, 491, 474, 505, 529, 490, 503, 523, 532, 563, 523, 559, 488, 531, 556, 499, 518, 509, 476, 470, 498, 483, 514, 506, 508, 540, 511, 531, 550, 483, 484, 541, 506, 467, 530, 496, 524, 474, 519, 501, 518, 518, 499, 513, 515, 522, 517, 528, 521, 557, 509, 500, 466, 527, 501, 508, 513, 548, 498, 513, 476, 513, 525, 513, 537, 510, 495, 516, 492, 530, 549, 529, 496, 475, 509, 502, 479, 568, 494, 498, 547, 509, 509, 487, 517, 477, 499, 509, 543, 550, 481, 528, 515, 516, 485, 519, 521, 540, 493, 529, 515, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[484, 496, 525, 514, 490, 510, 490, 495, 505, 491, 533, 546, 535, 518, 503, 509, 519, 500, 525, 523, 471, 507, 520, 532, 522, 521, 502, 521, 546, 495, 487, 541, 496, 498, 509, 473, 517, 531, 501, 494, 519, 513, 453, 501, 538, 506, 516, 520, 505, 510, 483, 503, 490, 543, 505, 529, 533, 508, 524, 476, 486, 500, 504, 541, 519, 532, 534, 471, 515, 492, 497, 490, 491, 504, 525, 495, 499, 524, 496, 502, 533, 546, 510, 488, 500, 560, 510, 501, 544, 563, 519, 509, 563, 530, 481, 526, 522, 529, 504, 501, 548, 540, 545, 554, 518, 512, 493, 496, 500, 505, 514, 487, 487, 530, 552, 502, 491, 514, 519, 496, 504, 527, 535, 502, 514, 530, 490, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[505, 499, 571, 515, 535, 505, 535, 532, 478, 465, 497, 519, 512, 499, 517, 521, 482, 505, 553, 503, 515, 516, 506, 492, 494, 520, 504, 481, 518, 503, 513, 532, 540, 519, 509, 503, 515, 491, 531, 525, 509, 487, 513, 528, 492, 510, 553, 554, 523, 501, 484, 486, 493, 530, 512, 473, 507, 547, 502, 499, 526, 499, 549, 480, 518, 489, 508, 487, 475, 490, 510, 541, 527, 502, 516, 526, 506, 495, 503, 460, 488, 518, 500, 513, 560, 516, 527, 506, 533, 492, 533, 500, 532, 504, 526, 523, 498, 494, 529, 494, 548, 554, 444, 518, 517, 520, 464, 528, 525, 525, 511, 509, 530, 545, 553, 507, 548, 485, 515, 497, 528, 512, 527, 525, 523, 504, 490, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[491, 521, 555, 501, 480, 492, 533, 477, 503, 527, 506, 532, 526, 534, 488, 536, 498, 472, 509, 487, 543, 530, 514, 534, 506, 483, 536, 489, 492, 543, 502, 512, 474, 507, 529, 534, 523, 501, 504, 473, 522, 512, 515, 508, 491, 528, 513, 504, 535, 494, 492, 524, 508, 482, 524, 498, 510, 537, 508, 544, 557, 555, 533, 533, 510, 526, 500, 549, 510, 510, 488, 510, 517, 503, 504, 525, 525, 531, 495, 532, 498, 506, 517, 491, 464, 525, 509, 498, 525, 516, 524, 509, 494, 494, 505, 501, 519, 454, 545, 561, 526, 517, 543, 480, 505, 533, 519, 480, 509, 536, 510, 536, 533, 520, 494, 537, 480, 504, 528, 501, 502, 517, 506, 501, 490, 517, 495, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[517, 521, 517, 524, 538, 521, 518, 453, 483, 540, 528, 527, 501, 544, 486, 492, 489, 513, 510, 473, 514, 538, 503, 500, 498, 496, 525, 520, 506, 541, 504, 500, 472, 469, 521, 544, 522, 474, 529, 489, 531, 474, 522, 489, 511, 573, 520, 541, 513, 539, 568, 545, 506, 533, 516, 510, 541, 522, 473, 514, 503, 518, 495, 501, 475, 500, 507, 552, 507, 509, 462, 526, 502, 506, 526, 475, 519, 502, 550, 493, 537, 504, 519, 509, 483, 489, 506, 510, 533, 510, 492, 489, 478, 510, 510, 502, 475, 522, 561, 531, 529, 576, 466, 512, 514, 529, 510, 529, 503, 495, 497, 530, 514, 514, 501, 495, 531, 515, 501, 517, 520, 540, 552, 520, 508, 506, 503, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[536, 509, 485, 486, 492, 537, 494, 475, 549, 540, 516, 504, 453, 525, 527, 511, 466, 497, 518, 497, 508, 542, 520, 515, 537, 481, 557, 511, 514, 494, 496, 552, 509, 531, 517, 524, 522, 504, 521, 506, 515, 493, 509, 516, 521, 506, 539, 498, 491, 514, 520, 504, 489, 491, 516, 530, 534, 514, 513, 489, 514, 511, 508, 543, 516, 527, 517, 505, 512, 522, 506, 508, 541, 515, 569, 548, 485, 517, 513, 519, 503, 493, 496, 516, 531, 526, 458, 515, 492, 572, 500, 517, 505, 530, 554, 514, 532, 495, 514, 494, 491, 487, 510, 517, 492, 499, 514, 527, 516, 530, 491, 504, 507, 503, 524, 503, 539, 467, 514, 513, 501, 509, 483, 493, 491, 519, 497, 562] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[475, 504, 496, 483, 520, 523, 490, 547, 537, 484, 512, 498, 530, 453, 482, 463, 511, 506, 488, 553, 528, 475, 539, 499, 477, 526, 528, 518, 531, 520, 548, 503, 502, 508, 493, 520, 502, 522, 517, 558, 542, 526, 466, 515, 470, 562, 540, 538, 506, 562, 530, 499, 558, 528, 524, 537, 519, 495, 523, 486, 498, 509, 504, 505, 503, 540, 496, 521, 539, 519, 502, 504, 518, 486, 541, 537, 503, 546, 510, 477, 463, 528, 496, 524, 536, 524, 501, 518, 504, 533, 529, 527, 530, 495, 513, 489, 531, 512, 546, 499, 511, 505, 484, 535, 513, 499, 484, 484, 522, 522, 495, 489, 533, 523, 528, 502, 480, 520, 526, 517, 527, 491, 480, 509, 496, 488, 530, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[530, 481, 515, 505, 500, 492, 515, 499, 495, 531, 501, 474, 526, 512, 516, 559, 476, 519, 514, 530, 523, 497, 510, 503, 524, 501, 511, 509, 498, 498, 531, 521, 530, 503, 553, 509, 520, 482, 534, 528, 526, 506, 504, 479, 522, 470, 499, 527, 533, 533, 544, 527, 495, 490, 514, 505, 539, 545, 514, 534, 550, 527, 509, 473, 485, 506, 537, 527, 497, 482, 506, 537, 500, 534, 535, 533, 465, 522, 491, 491, 502, 494, 532, 533, 489, 481, 499, 479, 514, 532, 538, 508, 468, 535, 486, 509, 531, 511, 474, 545, 551, 560, 535, 566, 511, 527, 491, 493, 505, 513, 504, 500, 466, 516, 501, 490, 515, 503, 501, 527, 505, 524, 516, 508, 504, 484, 525, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[484, 496, 525, 514, 490, 510, 490, 495, 505, 491, 533, 546, 535, 518, 503, 509, 37, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 525, 523, 471, 507, 520, 532, 522, 521, 502, 521, 546, 495, 487, 541, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 498, 509, 473, 517, 531, 501, 494, 511, 513, 453, 501, 538, 506, 516, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 483, 503, 490, 543, 505, 529, 533, 471, 524, 476, 486, 500, 504, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 532, 534, 471, 515, 492, 497, 490, 491, 504, 525, 495, 499, 524, 496, 502, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 546, 510, 488, 500, 560, 510, 501, 544, 563, 519, 509, 563, 530, 481, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 529, 504, 501, 548, 540, 545, 534, 518, 512, 493, 496, 500, 505, 514, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 530, 552, 502, 491, 514, 519, 496, 504, 527, 535, 502, 514, 530, 490, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8240, 8077, 8103, 8100, 8383, 8248, 8198] avg=8192.0000 max=8383 max_rank=5 min=8077 min_rank=2 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[523, 515, 491, 521, 501, 536, 491, 496, 522, 493, 497, 492, 545, 493, 508, 504, 39, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 506, 518, 540, 505, 529, 525, 525, 517, 521, 525, 491, 474, 505, 529, 490, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 523, 532, 563, 523, 559, 488, 531, 547, 499, 518, 509, 476, 470, 498, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 508, 540, 511, 531, 550, 483, 484, 502, 506, 467, 530, 496, 524, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 501, 518, 518, 499, 513, 515, 522, 517, 528, 521, 557, 509, 500, 466, 527, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 508, 513, 548, 498, 513, 476, 513, 525, 513, 537, 510, 495, 516, 492, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 529, 496, 475, 509, 502, 479, 547, 494, 498, 547, 509, 509, 487, 517, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 543, 550, 481, 528, 515, 516, 485, 519, 521, 540, 493, 529, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8200, 8222, 8126, 8244, 8188, 8124, 8258] avg=8192.0000 max=8258 max_rank=7 min=8124 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[517, 521, 517, 524, 538, 521, 518, 453, 483, 540, 528, 527, 501, 544, 486, 492, 38, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 513, 510, 473, 514, 538, 503, 500, 498, 496, 525, 520, 506, 541, 504, 500, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 469, 521, 544, 522, 474, 529, 489, 522, 474, 522, 489, 511, 573, 520, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 539, 568, 545, 506, 533, 516, 510, 541, 484, 473, 514, 503, 518, 495, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 500, 507, 552, 507, 509, 462, 526, 502, 506, 526, 475, 519, 502, 550, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 504, 519, 509, 483, 489, 506, 510, 533, 510, 492, 489, 478, 510, 510, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 522, 561, 531, 529, 576, 466, 493, 514, 529, 510, 529, 503, 495, 497, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 501, 495, 531, 515, 501, 517, 520, 540, 552, 520, 508, 506, 503, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8139, 8172, 8259, 8124, 8081, 8260, 8247] avg=8192.0000 max=8260 max_rank=6 min=8081 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[536, 509, 485, 486, 492, 537, 494, 475, 549, 540, 516, 504, 453, 525, 527, 511, 40, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 497, 518, 497, 508, 542, 520, 515, 537, 481, 557, 511, 514, 494, 496, 552, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 531, 517, 524, 522, 504, 521, 506, 492, 493, 509, 516, 521, 506, 539, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 514, 520, 504, 489, 491, 516, 530, 534, 474, 513, 489, 514, 511, 508, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 527, 517, 505, 512, 522, 506, 508, 541, 515, 569, 548, 485, 517, 513, 519, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 493, 496, 516, 531, 526, 458, 515, 492, 572, 500, 517, 505, 530, 554, 514, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 495, 514, 494, 491, 487, 510, 485, 492, 499, 514, 527, 516, 530, 491, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 524, 503, 539, 467, 514, 513, 501, 509, 483, 493, 491, 519, 497, 562, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8222, 8208, 8141, 8338, 8228, 8081, 8129] avg=8192.0000 max=8338 max_rank=4 min=8081 min_rank=6 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[505, 499, 571, 515, 535, 505, 535, 532, 478, 465, 497, 519, 512, 499, 517, 521, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 505, 553, 503, 515, 516, 506, 492, 494, 520, 504, 481, 518, 503, 513, 532, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 519, 509, 503, 515, 491, 531, 525, 501, 487, 513, 528, 492, 510, 553, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 501, 484, 486, 493, 530, 512, 473, 507, 508, 502, 499, 526, 499, 549, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 508, 487, 475, 490, 510, 541, 527, 502, 516, 526, 506, 495, 503, 460, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 500, 513, 560, 516, 527, 506, 533, 492, 533, 500, 532, 504, 526, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 529, 494, 548, 554, 444, 499, 517, 520, 464, 528, 525, 525, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 545, 553, 507, 548, 485, 515, 497, 528, 512, 527, 525, 523, 504, 490, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8145, 8271, 8072, 8066, 8271, 8159, 8302] avg=8192.0000 max=8302 max_rank=7 min=8066 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[491, 521, 555, 501, 480, 492, 533, 477, 503, 527, 506, 532, 526, 534, 488, 536, 39, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 472, 509, 487, 543, 530, 514, 534, 506, 483, 536, 489, 492, 543, 502, 512, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 507, 529, 534, 523, 501, 504, 473, 514, 512, 515, 508, 491, 528, 513, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 494, 492, 524, 508, 482, 524, 498, 510, 498, 508, 544, 557, 555, 533, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 526, 500, 549, 510, 510, 488, 510, 517, 503, 504, 525, 525, 531, 495, 532, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 506, 517, 491, 464, 525, 509, 498, 525, 516, 524, 509, 494, 494, 505, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 454, 545, 561, 526, 517, 543, 463, 505, 533, 519, 480, 509, 536, 510, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 520, 494, 537, 480, 504, 528, 501, 502, 517, 506, 501, 490, 517, 495, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8158, 8130, 8295, 8247, 8076, 8256, 8128] avg=8192.0000 max=8295 max_rank=3 min=8076 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[530, 481, 515, 505, 500, 492, 515, 499, 495, 531, 501, 474, 526, 512, 516, 559, 39, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 519, 514, 530, 523, 497, 510, 503, 524, 501, 511, 509, 498, 498, 531, 521, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 503, 553, 509, 520, 482, 534, 528, 518, 506, 504, 479, 522, 470, 499, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 533, 544, 527, 495, 490, 514, 505, 539, 506, 514, 534, 550, 527, 509, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 506, 537, 527, 497, 482, 506, 537, 500, 534, 535, 533, 465, 522, 491, 491, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 494, 532, 533, 489, 481, 499, 479, 514, 532, 538, 508, 468, 535, 486, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 511, 474, 545, 551, 560, 535, 545, 511, 527, 491, 493, 505, 513, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 516, 501, 490, 515, 503, 501, 527, 505, 524, 516, 508, 504, 484, 525, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8173, 8184, 8293, 8162, 8099, 8296, 8132] avg=8192.0000 max=8296 max_rank=6 min=8099 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[475, 504, 496, 483, 520, 523, 490, 547, 537, 484, 512, 498, 530, 453, 482, 463, 36, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 488, 553, 528, 475, 539, 499, 477, 526, 528, 518, 531, 520, 548, 503, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 508, 493, 520, 502, 522, 517, 558, 533, 526, 466, 515, 470, 562, 540, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 562, 530, 499, 558, 528, 524, 537, 519, 459, 523, 486, 498, 509, 504, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 540, 496, 521, 539, 519, 502, 504, 518, 486, 541, 537, 503, 546, 510, 477, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 528, 496, 524, 536, 524, 501, 518, 504, 533, 529, 527, 530, 495, 513, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 546, 499, 511, 505, 484, 516, 513, 499, 484, 484, 522, 522, 495, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 523, 528, 502, 480, 520, 526, 517, 527, 491, 480, 509, 496, 488, 530, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8039, 8259, 8272, 8247, 8255, 8210, 8112, 8142] avg=8192.0000 max=8272 max_rank=2 min=8039 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4352, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[3968, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4352, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4352, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[537, 516, 494, 477, 528, 540, 514, 532, 474, 487, 514, 517, 506, 502, 497, 495, 564, 486, 513, 534, 537, 504, 486, 500, 553, 535, 479, 514, 492, 475, 516, 510, 504, 498, 523, 515, 512, 519, 520, 508, 491, 502, 496, 492, 530, 536, 507, 533, 525, 494, 518, 509, 527, 510, 495, 503, 492, 502, 506, 526, 525, 534, 497, 523, 526, 509, 515, 513, 547, 493, 501, 523, 484, 524, 511, 507, 525, 522, 532, 493, 491, 535, 528, 501, 501, 535, 514, 516, 523, 497, 501, 483, 464, 529, 500, 484, 528, 482, 485, 513, 551, 494, 541, 525, 506, 472, 528, 519, 507, 518, 554, 501, 515, 518, 497, 490, 569, 519, 557, 527, 510, 502, 484, 528, 531, 501, 526, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[538, 586, 523, 485, 511, 480, 524, 534, 508, 540, 540, 474, 496, 473, 499, 502, 484, 501, 525, 528, 543, 532, 473, 523, 490, 529, 516, 506, 484, 532, 541, 521, 555, 499, 529, 534, 522, 502, 522, 541, 514, 516, 468, 513, 555, 543, 463, 481, 483, 526, 506, 523, 535, 500, 479, 511, 520, 514, 530, 508, 530, 518, 511, 518, 524, 522, 552, 497, 490, 507, 509, 503, 500, 491, 500, 554, 536, 507, 501, 524, 498, 482, 525, 541, 525, 515, 450, 488, 507, 540, 529, 520, 541, 515, 557, 486, 521, 498, 495, 510, 508, 500, 495, 527, 519, 510, 486, 485, 515, 481, 527, 448, 491, 514, 535, 544, 490, 530, 520, 515, 493, 504, 518, 510, 518, 482, 518, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[537, 497, 550, 504, 502, 540, 524, 510, 519, 543, 502, 476, 490, 542, 507, 555, 504, 514, 511, 480, 496, 532, 498, 518, 525, 528, 519, 535, 500, 534, 502, 545, 479, 497, 546, 530, 499, 554, 508, 491, 503, 537, 507, 514, 474, 480, 491, 518, 521, 519, 519, 515, 528, 518, 496, 529, 495, 555, 513, 511, 498, 504, 481, 507, 493, 486, 515, 502, 537, 512, 509, 506, 514, 498, 509, 545, 528, 527, 501, 501, 503, 504, 492, 495, 528, 490, 546, 483, 496, 516, 537, 518, 480, 514, 492, 557, 510, 489, 520, 518, 496, 498, 520, 549, 491, 514, 564, 548, 493, 541, 516, 511, 503, 528, 485, 533, 486, 505, 486, 496, 492, 517, 513, 520, 473, 516, 514, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[530, 509, 497, 500, 538, 494, 549, 489, 504, 522, 500, 519, 554, 493, 536, 497, 531, 516, 494, 508, 506, 535, 538, 532, 523, 468, 515, 511, 555, 560, 515, 509, 502, 499, 511, 547, 462, 523, 519, 497, 560, 506, 502, 530, 526, 538, 541, 502, 513, 499, 534, 492, 503, 526, 512, 521, 522, 563, 499, 480, 494, 488, 489, 555, 522, 500, 514, 514, 504, 497, 514, 539, 515, 505, 484, 510, 521, 507, 506, 524, 535, 498, 499, 507, 497, 495, 497, 459, 517, 520, 520, 505, 483, 505, 474, 502, 525, 499, 522, 470, 490, 531, 504, 543, 548, 484, 522, 499, 539, 522, 493, 482, 504, 493, 490, 493, 526, 490, 527, 546, 554, 535, 501, 471, 519, 512, 503, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[525, 532, 505, 484, 561, 538, 537, 501, 516, 540, 531, 477, 499, 530, 526, 514, 521, 490, 464, 524, 467, 510, 499, 499, 506, 493, 535, 554, 464, 505, 504, 489, 540, 508, 460, 536, 509, 478, 497, 494, 524, 515, 518, 521, 531, 556, 513, 517, 513, 557, 517, 488, 521, 537, 525, 494, 514, 558, 528, 538, 514, 519, 499, 504, 536, 498, 516, 485, 456, 508, 531, 527, 548, 522, 547, 520, 540, 490, 521, 500, 496, 510, 492, 487, 505, 543, 502, 545, 515, 493, 473, 504, 534, 506, 505, 526, 483, 529, 523, 534, 488, 527, 538, 524, 479, 491, 530, 499, 522, 505, 483, 516, 505, 530, 536, 512, 502, 494, 488, 497, 499, 500, 505, 514, 502, 486, 533, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[514, 489, 500, 492, 510, 519, 538, 504, 489, 504, 509, 517, 503, 513, 492, 552, 503, 506, 511, 496, 542, 525, 495, 537, 514, 510, 489, 534, 466, 511, 527, 509, 520, 517, 503, 501, 504, 545, 532, 506, 470, 492, 507, 496, 476, 470, 493, 541, 533, 481, 494, 520, 466, 488, 538, 511, 477, 522, 488, 489, 484, 497, 542, 551, 523, 534, 537, 536, 489, 499, 529, 502, 508, 507, 511, 509, 475, 528, 513, 522, 533, 536, 504, 501, 485, 545, 525, 511, 569, 538, 513, 523, 527, 549, 469, 512, 476, 487, 502, 509, 529, 487, 521, 515, 542, 548, 500, 483, 488, 539, 521, 526, 500, 512, 501, 549, 528, 516, 503, 547, 526, 536, 531, 504, 542, 503, 509, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[512, 495, 499, 502, 510, 500, 508, 499, 533, 563, 513, 519, 502, 520, 490, 508, 508, 517, 473, 506, 483, 544, 504, 518, 478, 558, 510, 491, 498, 511, 499, 552, 497, 522, 511, 495, 523, 509, 522, 507, 511, 452, 504, 512, 547, 488, 546, 509, 526, 507, 535, 515, 496, 544, 539, 523, 517, 523, 520, 544, 517, 508, 475, 524, 499, 504, 511, 473, 480, 552, 539, 501, 537, 490, 554, 521, 536, 471, 512, 498, 477, 519, 476, 513, 531, 499, 533, 496, 504, 545, 512, 517, 511, 510, 501, 537, 530, 499, 500, 515, 487, 522, 528, 522, 527, 482, 514, 533, 512, 506, 519, 497, 493, 512, 494, 516, 506, 510, 504, 519, 524, 539, 515, 536, 530, 498, 498, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[503, 507, 484, 497, 530, 496, 507, 497, 538, 483, 486, 499, 528, 468, 497, 526, 534, 490, 513, 479, 496, 517, 544, 512, 505, 521, 484, 526, 496, 496, 523, 543, 519, 530, 530, 502, 525, 491, 526, 524, 522, 505, 540, 526, 524, 494, 500, 544, 498, 533, 555, 545, 480, 519, 538, 488, 544, 482, 530, 492, 498, 493, 493, 557, 504, 530, 521, 512, 523, 500, 508, 511, 554, 547, 511, 521, 545, 530, 514, 479, 532, 464, 518, 493, 494, 475, 508, 501, 519, 493, 556, 508, 548, 535, 531, 515, 470, 506, 512, 507, 508, 502, 528, 509, 531, 524, 538, 489, 514, 550, 499, 514, 505, 495, 493, 496, 467, 493, 521, 484, 507, 490, 505, 526, 548, 505, 510, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[538, 586, 523, 485, 511, 480, 523, 534, 508, 540, 540, 474, 496, 473, 499, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 525, 528, 543, 532, 473, 523, 490, 529, 516, 506, 484, 532, 541, 521, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 499, 529, 534, 522, 502, 522, 541, 514, 516, 468, 513, 549, 543, 463, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 526, 506, 523, 535, 500, 479, 511, 520, 514, 530, 508, 530, 518, 511, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 552, 497, 490, 507, 509, 503, 500, 491, 500, 554, 509, 507, 501, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 482, 525, 541, 525, 515, 450, 488, 507, 540, 529, 520, 541, 515, 557, 486, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 498, 495, 510, 508, 500, 495, 527, 519, 510, 486, 485, 515, 481, 527, 448, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 514, 535, 544, 490, 530, 520, 515, 493, 504, 518, 510, 518, 482, 518, 483, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8230, 8251, 8186, 8190, 8243, 8037, 8187] avg=8192.0000 max=8251 max_rank=2 min=8037 min_rank=6 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[537, 516, 494, 477, 528, 540, 513, 532, 474, 487, 514, 517, 506, 502, 497, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 486, 513, 534, 537, 504, 486, 500, 553, 535, 479, 514, 492, 475, 516, 510, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 523, 515, 512, 519, 520, 508, 491, 502, 496, 492, 525, 536, 507, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 494, 518, 509, 527, 510, 495, 503, 492, 502, 506, 526, 525, 534, 497, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 509, 515, 513, 547, 493, 501, 523, 484, 524, 511, 507, 499, 522, 532, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 535, 528, 501, 501, 535, 514, 516, 523, 497, 501, 483, 464, 529, 500, 484, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 482, 485, 513, 551, 494, 541, 525, 506, 472, 528, 519, 507, 518, 554, 501, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 518, 497, 490, 569, 519, 557, 527, 510, 502, 484, 528, 531, 501, 526, 511, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8200, 8181, 8159, 8199, 8127, 8235, 8306] avg=8192.0000 max=8306 max_rank=7 min=8127 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[530, 509, 497, 500, 538, 494, 547, 489, 504, 522, 500, 519, 554, 493, 536, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 516, 494, 508, 506, 535, 538, 532, 523, 468, 515, 511, 555, 560, 515, 509, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 499, 511, 547, 462, 523, 519, 497, 560, 506, 502, 530, 521, 538, 541, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 499, 534, 492, 503, 526, 512, 521, 522, 563, 499, 480, 494, 488, 489, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 500, 514, 514, 504, 497, 514, 539, 515, 505, 484, 510, 495, 507, 506, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 498, 499, 507, 497, 495, 497, 459, 517, 520, 520, 505, 483, 505, 474, 502, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 499, 522, 470, 490, 531, 504, 543, 548, 484, 522, 499, 539, 522, 493, 482, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 493, 490, 493, 526, 490, 527, 546, 554, 535, 501, 471, 519, 512, 503, 508, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8319, 8260, 8162, 8150, 8039, 8184, 8193] avg=8192.0000 max=8319 max_rank=1 min=8039 min_rank=5 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[525, 532, 505, 484, 561, 538, 530, 501, 516, 540, 531, 477, 499, 530, 526, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 490, 464, 524, 467, 510, 499, 499, 506, 493, 535, 554, 464, 505, 504, 489, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 508, 460, 536, 509, 478, 497, 494, 524, 515, 518, 521, 520, 556, 513, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 557, 517, 488, 521, 537, 525, 494, 514, 558, 528, 538, 514, 519, 499, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 516, 485, 456, 508, 531, 527, 548, 522, 547, 520, 504, 490, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 492, 487, 505, 543, 502, 545, 515, 493, 473, 504, 534, 506, 505, 526, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 529, 523, 534, 488, 527, 538, 524, 479, 491, 530, 499, 522, 505, 483, 516, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 530, 536, 512, 502, 494, 488, 497, 499, 500, 505, 514, 502, 486, 533, 498, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8309, 8033, 8206, 8290, 8209, 8164, 8192, 8133] avg=8192.0000 max=8309 max_rank=0 min=8033 min_rank=1 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[537, 497, 550, 504, 502, 540, 523, 510, 519, 543, 502, 476, 490, 542, 507, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 511, 480, 496, 532, 498, 518, 525, 528, 519, 535, 500, 534, 502, 545, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 497, 546, 530, 499, 554, 508, 491, 503, 537, 507, 514, 469, 480, 491, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 519, 515, 528, 518, 496, 529, 495, 555, 513, 511, 498, 504, 481, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 486, 515, 502, 537, 512, 509, 506, 514, 498, 509, 545, 501, 527, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 492, 495, 528, 490, 546, 483, 496, 516, 537, 518, 480, 514, 492, 557, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 489, 520, 518, 496, 498, 520, 549, 491, 514, 564, 548, 493, 541, 516, 511, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 528, 485, 533, 486, 505, 486, 496, 492, 517, 513, 520, 473, 516, 514, 481, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8243, 8123, 8183, 8156, 8175, 8289, 8070] avg=8192.0000 max=8297 max_rank=0 min=8070 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 489, 500, 492, 510, 519, 536, 504, 489, 504, 509, 517, 503, 513, 492, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 506, 511, 496, 542, 525, 495, 537, 514, 510, 489, 534, 466, 511, 527, 509, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 503, 501, 504, 545, 532, 506, 470, 492, 507, 496, 471, 470, 493, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 481, 494, 520, 466, 488, 538, 511, 477, 522, 488, 489, 484, 497, 542, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 534, 537, 536, 489, 499, 529, 502, 508, 507, 511, 509, 451, 528, 513, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 536, 504, 501, 485, 545, 525, 511, 569, 538, 513, 523, 527, 549, 469, 512, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 487, 502, 509, 529, 487, 521, 515, 542, 548, 500, 483, 488, 539, 521, 526, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 501, 549, 528, 516, 503, 547, 526, 536, 531, 504, 542, 503, 509, 520, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8178, 8068, 8053, 8198, 8366, 8183, 8347] avg=8192.0000 max=8366 max_rank=5 min=8053 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=33094 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[512, 495, 499, 502, 510, 500, 507, 499, 533, 563, 513, 519, 502, 520, 490, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 473, 506, 483, 544, 504, 518, 478, 558, 510, 491, 498, 511, 499, 552, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 522, 511, 495, 523, 509, 522, 507, 511, 452, 504, 512, 542, 488, 546, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 535, 515, 496, 544, 539, 523, 517, 523, 520, 544, 517, 508, 475, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 504, 511, 473, 480, 552, 539, 501, 537, 490, 554, 521, 509, 471, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 519, 476, 513, 531, 499, 533, 496, 504, 545, 512, 517, 511, 510, 501, 537, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 500, 515, 487, 522, 528, 522, 527, 482, 514, 533, 512, 506, 519, 497, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 512, 494, 516, 506, 510, 504, 519, 524, 539, 515, 536, 530, 498, 498, 499, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8152, 8150, 8286, 8151, 8206, 8204, 8215] avg=8192.0000 max=8286 max_rank=3 min=8150 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[503, 507, 484, 497, 530, 496, 506, 497, 538, 483, 486, 499, 528, 468, 497, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 513, 479, 496, 517, 544, 512, 505, 521, 484, 526, 496, 496, 523, 543, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 530, 502, 525, 491, 526, 524, 522, 505, 540, 526, 519, 494, 500, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 533, 555, 545, 480, 519, 538, 488, 544, 482, 530, 492, 498, 493, 493, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 530, 521, 512, 523, 500, 508, 511, 554, 547, 511, 521, 518, 530, 514, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 464, 518, 493, 494, 475, 508, 501, 519, 493, 556, 508, 548, 535, 531, 515, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 506, 512, 507, 508, 502, 528, 509, 531, 524, 538, 489, 514, 550, 499, 514, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 493, 496, 467, 493, 521, 484, 507, 490, 505, 526, 548, 505, 510, 518, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8045, 8181, 8297, 8217, 8283, 8216, 8212, 8085] avg=8192.0000 max=8297 max_rank=2 min=8045 min_rank=0 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True + [2026-08-26 16:29:46] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 967.5 | throughput per GPU (TFLOP/s/GPU): 186.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.700315E+00 | loss scale: 1.0 | grad norm: 69.482 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[499, 509, 493, 501, 560, 508, 505, 478, 560, 542, 527, 522, 513, 532, 535, 517, 498, 491, 520, 530, 513, 551, 535, 516, 512, 513, 548, 523, 518, 485, 511, 509, 529, 517, 495, 501, 479, 513, 488, 529, 545, 470, 557, 493, 476, 504, 564, 499, 565, 553, 480, 457, 509, 543, 512, 509, 492, 509, 507, 524, 485, 552, 472, 484, 550, 497, 497, 514, 508, 505, 517, 486, 526, 518, 486, 481, 549, 502, 511, 498, 487, 550, 512, 545, 509, 485, 483, 499, 501, 524, 486, 493, 550, 490, 497, 533, 533, 498, 488, 531, 509, 487, 500, 489, 502, 513, 531, 531, 502, 510, 515, 549, 503, 497, 509, 542, 535, 496, 504, 486, 546, 528, 517, 523, 474, 501, 509, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[506, 498, 526, 520, 503, 478, 513, 507, 505, 530, 518, 460, 521, 562, 547, 461, 513, 487, 497, 470, 505, 561, 531, 538, 513, 488, 525, 508, 513, 535, 499, 516, 519, 509, 503, 525, 489, 486, 537, 487, 489, 501, 516, 498, 520, 490, 456, 517, 504, 534, 553, 480, 481, 536, 532, 498, 507, 500, 504, 521, 470, 506, 513, 509, 508, 540, 477, 486, 499, 548, 543, 518, 522, 497, 531, 503, 500, 528, 496, 527, 523, 495, 513, 509, 531, 531, 513, 484, 502, 537, 506, 481, 514, 557, 516, 515, 491, 502, 535, 552, 522, 539, 502, 537, 514, 492, 527, 511, 513, 501, 509, 481, 552, 496, 489, 519, 507, 505, 533, 571, 472, 516, 548, 452, 536, 569, 516, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[515, 522, 527, 507, 462, 541, 575, 533, 504, 523, 498, 513, 516, 511, 499, 566, 541, 515, 525, 527, 517, 499, 514, 532, 489, 491, 488, 509, 502, 521, 551, 490, 507, 499, 518, 474, 562, 535, 494, 484, 523, 496, 544, 511, 493, 471, 481, 489, 516, 575, 497, 515, 472, 512, 482, 532, 507, 500, 523, 510, 543, 488, 511, 524, 532, 508, 517, 551, 472, 541, 525, 539, 560, 520, 527, 541, 531, 528, 513, 522, 540, 487, 475, 508, 511, 484, 511, 467, 524, 502, 508, 535, 482, 466, 467, 526, 504, 529, 491, 496, 500, 470, 483, 524, 486, 481, 513, 477, 517, 557, 509, 527, 536, 552, 502, 502, 532, 525, 515, 531, 504, 490, 525, 498, 501, 468, 530, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[530, 488, 514, 463, 503, 528, 503, 486, 547, 526, 501, 495, 528, 532, 464, 511, 470, 530, 555, 506, 501, 537, 495, 542, 507, 521, 521, 514, 510, 530, 491, 502, 502, 513, 508, 511, 492, 505, 511, 504, 502, 525, 511, 483, 516, 498, 511, 497, 526, 507, 515, 541, 502, 515, 522, 539, 506, 532, 549, 571, 475, 530, 489, 488, 505, 521, 529, 514, 516, 496, 485, 545, 499, 521, 495, 541, 486, 566, 509, 510, 515, 510, 553, 518, 568, 524, 509, 531, 508, 480, 512, 493, 513, 505, 529, 500, 513, 527, 499, 513, 511, 531, 499, 524, 559, 495, 476, 514, 499, 489, 495, 531, 553, 494, 455, 471, 483, 517, 495, 498, 506, 491, 549, 523, 480, 477, 519, 597] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[532, 529, 504, 478, 514, 495, 495, 532, 493, 513, 525, 486, 532, 507, 536, 490, 502, 500, 517, 482, 525, 527, 539, 530, 484, 510, 547, 503, 500, 493, 532, 497, 526, 514, 519, 494, 458, 535, 502, 513, 490, 505, 506, 539, 514, 535, 540, 522, 531, 525, 499, 508, 525, 502, 486, 547, 483, 547, 487, 456, 524, 488, 528, 504, 519, 510, 460, 522, 494, 522, 479, 562, 502, 535, 486, 509, 531, 496, 545, 553, 526, 508, 510, 544, 497, 505, 526, 520, 494, 480, 487, 520, 505, 520, 553, 471, 495, 540, 521, 510, 529, 525, 516, 524, 521, 498, 479, 520, 495, 541, 523, 475, 533, 541, 526, 528, 521, 520, 508, 509, 470, 523, 508, 478, 524, 556, 486, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[568, 468, 501, 550, 512, 485, 489, 531, 496, 530, 519, 491, 507, 514, 491, 489, 464, 561, 523, 481, 503, 541, 488, 504, 501, 524, 511, 518, 534, 499, 522, 528, 501, 531, 477, 505, 511, 513, 532, 512, 501, 509, 507, 508, 502, 519, 497, 550, 500, 514, 507, 501, 553, 501, 542, 499, 531, 490, 503, 510, 502, 537, 561, 507, 506, 486, 492, 567, 523, 506, 533, 532, 512, 574, 490, 525, 508, 474, 500, 487, 499, 519, 510, 485, 507, 484, 506, 544, 505, 480, 512, 515, 494, 527, 502, 504, 504, 540, 555, 506, 543, 525, 505, 458, 523, 546, 514, 469, 543, 536, 503, 479, 504, 497, 533, 485, 551, 510, 508, 522, 492, 515, 481, 523, 509, 509, 553, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[504, 545, 512, 533, 507, 487, 485, 496, 534, 501, 519, 484, 523, 494, 520, 524, 492, 515, 518, 482, 540, 481, 527, 511, 504, 528, 519, 539, 480, 509, 508, 487, 508, 528, 511, 449, 499, 522, 478, 520, 536, 555, 529, 523, 492, 482, 541, 535, 484, 540, 523, 521, 530, 535, 492, 509, 483, 526, 539, 511, 480, 507, 496, 527, 532, 535, 485, 476, 504, 486, 497, 517, 501, 492, 534, 578, 527, 494, 533, 533, 503, 487, 497, 475, 506, 516, 476, 528, 491, 510, 494, 478, 529, 530, 486, 543, 538, 541, 527, 534, 508, 527, 525, 492, 494, 521, 518, 494, 484, 519, 497, 479, 488, 513, 529, 578, 484, 477, 542, 526, 504, 558, 513, 523, 521, 546, 511, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[496, 480, 537, 538, 515, 514, 546, 498, 490, 506, 481, 507, 505, 534, 512, 529, 503, 511, 493, 507, 512, 522, 482, 491, 471, 496, 528, 519, 528, 542, 511, 530, 496, 524, 535, 479, 507, 495, 492, 532, 520, 541, 538, 537, 544, 492, 523, 519, 517, 487, 497, 493, 503, 534, 529, 557, 528, 504, 524, 476, 534, 523, 487, 525, 531, 504, 521, 537, 495, 462, 506, 535, 493, 512, 506, 498, 491, 500, 469, 550, 557, 529, 512, 521, 491, 517, 499, 523, 544, 484, 511, 490, 526, 507, 509, 533, 528, 531, 507, 485, 514, 485, 496, 479, 526, 530, 505, 539, 477, 503, 535, 490, 497, 495, 502, 499, 519, 535, 484, 537, 542, 491, 506, 501, 532, 504, 535, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 498, 526, 520, 503, 478, 513, 507, 505, 530, 518, 460, 521, 561, 547, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 487, 497, 470, 505, 554, 531, 538, 513, 488, 525, 508, 513, 535, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 509, 503, 525, 489, 486, 537, 487, 489, 501, 516, 498, 520, 490, 456, 517, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 523, 553, 480, 481, 536, 532, 498, 507, 500, 504, 521, 470, 506, 513, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 540, 477, 486, 499, 548, 543, 486, 522, 497, 531, 503, 500, 528, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 495, 513, 509, 531, 531, 513, 484, 502, 537, 506, 481, 514, 557, 516, 515, 37, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 502, 535, 552, 522, 539, 502, 537, 514, 492, 527, 511, 513, 501, 509, 481, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 496, 489, 519, 507, 505, 533, 571, 472, 516, 548, 452, 536, 569, 516, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8192, 8077, 8137, 8191, 8269, 8239, 8277] avg=8192.0000 max=8277 max_rank=7 min=8077 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[499, 509, 493, 501, 560, 508, 505, 478, 560, 542, 527, 522, 513, 531, 535, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 491, 520, 530, 513, 544, 535, 516, 512, 513, 548, 523, 518, 485, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 517, 495, 501, 479, 513, 488, 529, 545, 470, 557, 493, 476, 504, 564, 499, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 541, 480, 457, 509, 543, 512, 509, 492, 509, 507, 524, 485, 552, 472, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 497, 497, 514, 508, 505, 517, 456, 526, 518, 486, 481, 549, 502, 511, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 550, 512, 545, 509, 485, 483, 499, 501, 524, 486, 493, 550, 490, 497, 533, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 498, 488, 531, 509, 487, 500, 489, 502, 513, 531, 531, 502, 510, 515, 549, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 509, 542, 535, 496, 504, 486, 546, 528, 517, 523, 474, 501, 509, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8266, 8193, 8141, 8115, 8184, 8199, 8138] avg=8192.0000 max=8300 max_rank=0 min=8115 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[568, 468, 501, 550, 512, 485, 489, 531, 496, 530, 519, 491, 507, 513, 491, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 561, 523, 481, 503, 534, 488, 504, 501, 524, 511, 518, 534, 499, 522, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 531, 477, 505, 511, 513, 532, 512, 501, 509, 507, 508, 502, 519, 497, 550, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 507, 501, 553, 501, 542, 499, 531, 490, 503, 510, 502, 537, 561, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 486, 492, 567, 523, 506, 533, 499, 512, 574, 490, 525, 508, 474, 500, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 519, 510, 485, 507, 484, 506, 544, 505, 480, 512, 515, 494, 527, 502, 504, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 540, 555, 506, 543, 525, 505, 458, 523, 546, 514, 469, 543, 536, 503, 479, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 533, 485, 551, 510, 508, 522, 492, 515, 481, 523, 509, 509, 553, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8195, 8211, 8247, 8182, 8133, 8260, 8168] avg=8192.0000 max=8260 max_rank=6 min=8133 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[515, 522, 527, 507, 462, 541, 575, 533, 504, 523, 498, 513, 516, 509, 499, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 515, 525, 527, 517, 490, 514, 532, 489, 491, 488, 509, 502, 521, 551, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 499, 518, 474, 562, 535, 494, 484, 523, 496, 544, 511, 493, 471, 481, 489, 33, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 555, 497, 515, 472, 512, 482, 532, 507, 500, 523, 510, 543, 488, 511, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 508, 517, 551, 472, 541, 525, 496, 560, 520, 527, 541, 531, 528, 513, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 487, 475, 508, 511, 484, 511, 467, 524, 502, 508, 535, 482, 466, 467, 526, 41, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 529, 491, 496, 500, 470, 483, 524, 486, 481, 513, 477, 517, 557, 509, 527, 9, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 552, 502, 502, 532, 525, 515, 531, 504, 490, 525, 498, 501, 468, 530, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8202, 8127, 8187, 8384, 8044, 8082, 8200] avg=8192.0000 max=8384 max_rank=4 min=8044 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[530, 488, 514, 463, 503, 528, 503, 486, 547, 526, 501, 495, 528, 531, 464, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 530, 555, 506, 501, 530, 495, 542, 507, 521, 521, 514, 510, 530, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 508, 511, 492, 505, 511, 504, 502, 525, 511, 483, 516, 498, 511, 497, 29, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 515, 541, 502, 515, 522, 539, 506, 532, 549, 571, 475, 530, 489, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 529, 514, 516, 496, 485, 511, 499, 521, 495, 541, 486, 566, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 553, 518, 568, 524, 509, 531, 508, 480, 512, 493, 513, 505, 529, 500, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 527, 499, 513, 511, 531, 499, 524, 559, 495, 476, 514, 499, 489, 495, 531, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 494, 455, 471, 483, 517, 495, 498, 506, 491, 549, 523, 480, 477, 519, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8225, 8126, 8296, 8204, 8314, 8186, 8067] avg=8192.0000 max=8314 max_rank=5 min=8067 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[532, 529, 504, 478, 514, 495, 495, 532, 493, 513, 525, 486, 532, 506, 536, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 517, 482, 525, 521, 539, 530, 484, 510, 547, 503, 500, 493, 532, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 519, 494, 458, 535, 502, 513, 490, 505, 506, 539, 514, 535, 540, 522, 29, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 514, 499, 508, 525, 502, 486, 547, 483, 547, 487, 456, 524, 488, 528, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 460, 522, 494, 522, 479, 528, 502, 535, 486, 509, 531, 496, 545, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 508, 510, 544, 497, 505, 526, 520, 494, 480, 487, 520, 505, 520, 553, 471, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 540, 521, 510, 529, 525, 516, 524, 521, 498, 479, 520, 495, 541, 523, 475, 6, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 541, 526, 528, 521, 520, 508, 509, 470, 523, 508, 478, 524, 556, 486, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8182, 8249, 8129, 8191, 8206, 8222, 8197] avg=8192.0000 max=8249 max_rank=2 min=8129 min_rank=3 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[504, 545, 512, 533, 507, 487, 485, 496, 534, 501, 519, 484, 523, 493, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 518, 482, 540, 475, 527, 511, 504, 528, 519, 539, 480, 509, 508, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 528, 511, 449, 499, 522, 478, 520, 536, 555, 529, 523, 492, 482, 541, 535, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 529, 523, 521, 530, 535, 492, 509, 483, 526, 539, 511, 480, 507, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 535, 485, 476, 504, 486, 497, 485, 501, 492, 534, 578, 527, 494, 533, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 487, 497, 475, 506, 516, 476, 528, 491, 510, 494, 478, 529, 530, 486, 543, 37, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 541, 527, 534, 508, 527, 525, 492, 494, 521, 518, 494, 484, 519, 497, 479, 6, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 513, 529, 578, 484, 477, 542, 526, 504, 558, 513, 523, 521, 546, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8134, 8243, 8192, 8192, 8091, 8208, 8309] avg=8192.0000 max=8309 max_rank=7 min=8091 min_rank=5 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[496, 480, 537, 538, 515, 514, 546, 498, 490, 506, 481, 507, 505, 533, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 493, 507, 512, 516, 482, 491, 471, 496, 528, 519, 528, 542, 511, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 535, 479, 507, 495, 492, 532, 520, 541, 538, 537, 544, 492, 523, 519, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 476, 497, 493, 503, 534, 529, 557, 528, 504, 524, 476, 534, 523, 487, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 521, 537, 495, 462, 506, 502, 493, 512, 506, 498, 491, 500, 469, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 529, 512, 521, 491, 517, 499, 523, 544, 484, 511, 490, 526, 507, 509, 533, 37, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 531, 507, 485, 514, 485, 496, 479, 526, 530, 505, 539, 477, 503, 535, 490, 6, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 502, 499, 519, 535, 484, 537, 542, 491, 506, 501, 532, 504, 535, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8140, 8310, 8207, 8077, 8295, 8140, 8180] avg=8192.0000 max=8310 max_rank=2 min=8077 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[546, 531, 518, 496, 531, 525, 506, 528, 515, 557, 534, 480, 467, 528, 511, 470, 517, 519, 531, 540, 480, 509, 510, 535, 502, 509, 551, 525, 508, 500, 490, 480, 525, 497, 506, 508, 525, 509, 526, 504, 548, 502, 522, 527, 517, 490, 491, 483, 521, 532, 491, 523, 493, 507, 528, 467, 525, 498, 489, 469, 491, 542, 540, 544, 532, 508, 488, 519, 525, 527, 483, 507, 485, 503, 531, 555, 525, 530, 500, 542, 539, 532, 497, 510, 530, 495, 506, 477, 489, 512, 496, 540, 523, 510, 493, 517, 555, 519, 502, 548, 500, 491, 515, 543, 468, 513, 506, 514, 525, 516, 470, 528, 491, 501, 477, 513, 545, 492, 491, 513, 504, 504, 492, 495, 538, 493, 544, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[521, 526, 517, 543, 499, 513, 521, 509, 488, 498, 517, 518, 513, 525, 506, 510, 520, 514, 515, 511, 523, 520, 526, 506, 525, 516, 509, 523, 495, 501, 568, 503, 542, 477, 557, 537, 484, 516, 495, 489, 480, 492, 518, 473, 495, 507, 513, 504, 562, 566, 512, 514, 550, 487, 529, 495, 510, 482, 538, 473, 490, 513, 501, 535, 543, 505, 500, 523, 555, 493, 542, 504, 507, 522, 500, 539, 529, 478, 512, 493, 497, 508, 519, 503, 534, 509, 527, 506, 486, 489, 494, 529, 542, 547, 522, 516, 468, 520, 502, 523, 507, 532, 560, 492, 541, 534, 493, 506, 471, 474, 525, 542, 543, 504, 462, 494, 451, 540, 507, 449, 532, 470, 508, 532, 532, 497, 519, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[500, 474, 534, 498, 520, 517, 498, 532, 489, 545, 514, 536, 499, 479, 512, 505, 494, 522, 521, 474, 509, 500, 506, 479, 544, 518, 523, 491, 498, 529, 525, 497, 532, 521, 502, 522, 532, 533, 537, 508, 476, 497, 517, 529, 512, 517, 536, 511, 496, 531, 556, 513, 484, 519, 509, 501, 499, 509, 486, 515, 505, 541, 511, 470, 493, 523, 486, 511, 491, 501, 533, 523, 511, 537, 517, 506, 527, 515, 522, 502, 533, 541, 556, 570, 528, 515, 545, 507, 518, 528, 491, 503, 539, 476, 524, 495, 510, 453, 507, 509, 503, 515, 490, 543, 514, 475, 516, 543, 469, 508, 521, 534, 487, 563, 499, 480, 530, 498, 520, 518, 518, 499, 497, 498, 530, 472, 507, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[490, 510, 540, 510, 519, 521, 517, 517, 435, 526, 498, 524, 509, 475, 544, 518, 482, 497, 486, 486, 469, 549, 557, 534, 503, 506, 490, 493, 516, 496, 525, 488, 505, 485, 542, 495, 546, 530, 552, 522, 528, 527, 519, 509, 481, 539, 517, 460, 515, 527, 498, 516, 523, 497, 508, 521, 545, 533, 544, 499, 529, 518, 518, 477, 573, 509, 589, 520, 507, 513, 482, 529, 551, 490, 525, 528, 508, 496, 520, 522, 508, 491, 481, 497, 529, 474, 504, 545, 519, 571, 514, 537, 478, 533, 530, 491, 481, 504, 508, 470, 493, 479, 517, 504, 476, 496, 491, 495, 520, 514, 525, 538, 553, 517, 517, 517, 524, 518, 543, 498, 544, 463, 519, 478, 494, 500, 487, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[541, 502, 517, 507, 484, 523, 514, 505, 545, 528, 535, 495, 583, 486, 516, 456, 482, 543, 503, 472, 503, 534, 567, 522, 507, 524, 498, 520, 470, 506, 508, 535, 522, 496, 531, 500, 489, 508, 509, 519, 516, 516, 489, 509, 512, 528, 505, 531, 520, 513, 513, 532, 518, 507, 513, 535, 494, 489, 487, 504, 524, 508, 492, 528, 484, 481, 510, 486, 514, 491, 546, 495, 535, 530, 513, 472, 562, 518, 534, 458, 505, 516, 503, 503, 494, 529, 499, 482, 514, 533, 528, 494, 513, 497, 540, 452, 487, 528, 533, 556, 540, 530, 519, 533, 495, 509, 500, 501, 536, 509, 512, 487, 507, 500, 487, 521, 505, 567, 515, 522, 529, 525, 508, 517, 489, 529, 517, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[484, 499, 478, 516, 514, 491, 545, 503, 539, 516, 453, 509, 503, 472, 530, 531, 531, 485, 524, 516, 500, 484, 511, 513, 483, 492, 506, 508, 565, 540, 534, 507, 513, 520, 509, 528, 521, 518, 493, 512, 530, 525, 515, 489, 530, 515, 512, 497, 505, 499, 545, 509, 506, 494, 511, 544, 504, 487, 488, 514, 479, 537, 510, 517, 513, 540, 503, 536, 491, 487, 498, 497, 492, 544, 527, 502, 499, 520, 535, 487, 523, 494, 524, 483, 517, 519, 517, 502, 527, 502, 510, 518, 508, 511, 498, 496, 552, 518, 523, 494, 540, 520, 520, 528, 517, 529, 516, 496, 512, 480, 528, 530, 519, 512, 513, 522, 515, 513, 534, 523, 501, 527, 510, 509, 501, 539, 497, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[480, 533, 545, 508, 510, 526, 492, 518, 523, 517, 529, 523, 546, 483, 539, 482, 516, 480, 519, 485, 524, 548, 500, 482, 536, 493, 526, 487, 511, 442, 544, 507, 493, 543, 533, 568, 541, 510, 494, 484, 538, 463, 515, 527, 476, 502, 510, 507, 530, 494, 487, 502, 546, 498, 543, 504, 531, 500, 472, 508, 490, 549, 541, 537, 511, 530, 509, 488, 520, 508, 521, 490, 522, 516, 513, 567, 497, 523, 560, 521, 495, 524, 557, 500, 534, 519, 493, 486, 512, 516, 556, 529, 526, 529, 513, 530, 476, 555, 489, 516, 538, 471, 485, 513, 517, 504, 495, 512, 492, 510, 495, 472, 500, 528, 520, 507, 501, 514, 471, 516, 505, 474, 510, 511, 499, 513, 522, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[460, 484, 534, 465, 515, 501, 530, 510, 522, 529, 539, 479, 485, 508, 543, 474, 490, 498, 539, 479, 508, 538, 494, 529, 533, 531, 522, 514, 528, 507, 549, 531, 514, 512, 561, 500, 482, 506, 518, 542, 523, 514, 522, 477, 492, 540, 515, 510, 535, 562, 511, 580, 512, 498, 508, 513, 515, 509, 508, 494, 512, 498, 538, 508, 499, 482, 488, 556, 501, 485, 508, 524, 503, 509, 520, 509, 532, 507, 503, 497, 531, 518, 476, 490, 478, 521, 528, 505, 491, 482, 479, 508, 499, 508, 509, 505, 508, 512, 540, 521, 525, 480, 511, 477, 493, 536, 514, 530, 527, 549, 541, 460, 545, 519, 508, 517, 507, 512, 509, 546, 525, 524, 555, 503, 508, 522, 467, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 526, 517, 543, 499, 513, 521, 509, 488, 498, 517, 518, 513, 525, 506, 510, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 515, 511, 523, 520, 526, 506, 525, 516, 509, 523, 495, 501, 568, 503, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 477, 545, 537, 484, 516, 495, 489, 480, 492, 518, 473, 495, 507, 513, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 547, 512, 514, 550, 487, 529, 495, 510, 482, 538, 473, 490, 513, 501, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 505, 500, 523, 555, 493, 542, 504, 507, 522, 500, 539, 529, 478, 482, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 508, 519, 503, 534, 509, 527, 506, 486, 489, 494, 526, 542, 547, 522, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 520, 502, 523, 507, 532, 560, 492, 541, 534, 493, 506, 471, 474, 525, 542, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 504, 462, 494, 451, 540, 507, 449, 532, 470, 508, 532, 532, 497, 519, 498, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8282, 8067, 8238, 8215, 8225, 8211, 8060] avg=8192.0000 max=8282 max_rank=1 min=8060 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[546, 531, 518, 496, 531, 525, 506, 528, 515, 557, 534, 480, 467, 528, 511, 470, 11, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 531, 540, 480, 509, 510, 535, 502, 509, 551, 525, 508, 500, 490, 480, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 497, 486, 508, 525, 509, 526, 504, 548, 502, 522, 527, 517, 490, 491, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 491, 523, 493, 507, 528, 467, 525, 498, 489, 469, 491, 542, 540, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 508, 488, 519, 525, 527, 483, 507, 485, 503, 531, 555, 525, 530, 464, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 532, 497, 510, 530, 495, 506, 477, 489, 512, 496, 534, 523, 510, 493, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 519, 502, 548, 500, 491, 515, 543, 468, 513, 506, 514, 525, 516, 470, 528, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 477, 513, 545, 492, 491, 513, 504, 504, 492, 495, 538, 493, 544, 515, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8221, 8160, 8133, 8224, 8160, 8241, 8133] avg=8192.0000 max=8264 max_rank=0 min=8133 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 510, 540, 510, 519, 521, 517, 517, 435, 526, 498, 524, 509, 475, 544, 518, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 497, 486, 486, 469, 549, 557, 534, 503, 506, 490, 493, 516, 496, 525, 488, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 485, 530, 495, 546, 530, 552, 522, 528, 527, 519, 509, 481, 539, 517, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 498, 516, 523, 497, 508, 521, 545, 533, 544, 499, 529, 518, 518, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 509, 589, 520, 507, 513, 482, 529, 551, 490, 525, 528, 508, 496, 490, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 491, 481, 497, 529, 474, 504, 545, 519, 571, 514, 534, 478, 533, 530, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 508, 470, 493, 479, 517, 504, 476, 496, 491, 495, 520, 514, 525, 538, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 517, 517, 517, 524, 518, 543, 498, 544, 463, 519, 478, 494, 500, 487, 534, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8084, 8245, 8250, 8332, 8199, 8031, 8228] avg=8192.0000 max=8332 max_rank=4 min=8031 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[500, 474, 534, 498, 520, 517, 498, 532, 489, 545, 514, 536, 499, 479, 512, 505, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 522, 521, 474, 509, 500, 506, 479, 544, 518, 523, 491, 498, 529, 525, 497, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 521, 491, 522, 532, 533, 537, 508, 476, 497, 517, 529, 512, 517, 536, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 556, 513, 484, 519, 509, 501, 499, 509, 486, 515, 505, 541, 511, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 523, 486, 511, 491, 501, 533, 523, 511, 537, 517, 506, 527, 515, 492, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 541, 556, 570, 528, 515, 545, 507, 518, 528, 491, 500, 539, 476, 524, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 453, 507, 509, 503, 515, 490, 543, 514, 475, 516, 543, 469, 508, 521, 534, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 563, 499, 480, 530, 498, 520, 518, 518, 499, 497, 498, 530, 472, 507, 534, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8137, 8271, 8126, 8168, 8366, 8131, 8172] avg=8192.0000 max=8366 max_rank=5 min=8126 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[480, 533, 545, 508, 510, 526, 492, 518, 523, 517, 529, 523, 546, 483, 539, 482, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 480, 519, 485, 524, 548, 500, 482, 536, 493, 526, 487, 511, 442, 544, 507, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 543, 522, 568, 541, 510, 494, 484, 538, 463, 515, 527, 476, 502, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 477, 487, 502, 546, 498, 543, 504, 531, 500, 472, 508, 490, 549, 541, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 530, 509, 488, 520, 508, 521, 490, 522, 516, 513, 567, 497, 523, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 557, 500, 534, 519, 493, 486, 512, 516, 556, 526, 526, 529, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 555, 489, 516, 538, 471, 485, 513, 517, 504, 495, 512, 492, 510, 495, 472, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 528, 520, 507, 501, 514, 471, 516, 505, 474, 510, 511, 499, 513, 522, 500, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8107, 8193, 8215, 8264, 8316, 8060, 8115] avg=8192.0000 max=8316 max_rank=5 min=8060 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[541, 502, 517, 507, 484, 523, 514, 505, 545, 528, 535, 495, 583, 486, 516, 456, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 543, 503, 472, 503, 534, 567, 522, 507, 524, 498, 520, 470, 506, 508, 535, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 496, 520, 500, 489, 508, 509, 519, 516, 516, 489, 509, 512, 528, 505, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 496, 513, 532, 518, 507, 513, 535, 494, 489, 487, 504, 524, 508, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 481, 510, 486, 514, 491, 546, 495, 535, 530, 513, 472, 562, 518, 503, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 503, 503, 494, 529, 499, 482, 514, 533, 528, 491, 513, 497, 540, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 528, 533, 556, 540, 530, 519, 533, 495, 509, 500, 501, 536, 509, 512, 487, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 487, 521, 505, 567, 515, 522, 529, 525, 508, 517, 489, 529, 517, 504, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8201, 8169, 8160, 8098, 8099, 8295, 8265] avg=8192.0000 max=8295 max_rank=6 min=8098 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[484, 499, 478, 516, 514, 491, 545, 503, 539, 516, 453, 509, 503, 472, 530, 531, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 485, 524, 516, 500, 484, 511, 513, 483, 492, 506, 508, 565, 540, 534, 507, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 520, 498, 528, 521, 518, 493, 512, 530, 525, 515, 489, 530, 515, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 482, 545, 509, 506, 494, 511, 544, 504, 487, 488, 514, 479, 537, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 540, 503, 536, 491, 487, 498, 497, 492, 544, 527, 502, 499, 520, 504, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 524, 483, 517, 519, 517, 502, 527, 502, 510, 515, 508, 511, 498, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 518, 523, 494, 540, 520, 520, 528, 517, 529, 516, 496, 512, 480, 528, 530, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 512, 513, 522, 515, 513, 534, 523, 501, 527, 510, 509, 501, 539, 497, 520, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8206, 8216, 8132, 8140, 8146, 8323, 8278] avg=8192.0000 max=8323 max_rank=6 min=8095 min_rank=0 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[460, 484, 534, 465, 515, 501, 530, 510, 522, 529, 539, 479, 485, 508, 543, 474, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 498, 539, 479, 508, 538, 494, 529, 533, 531, 522, 514, 528, 507, 549, 531, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 512, 548, 500, 482, 506, 518, 542, 523, 514, 522, 477, 492, 540, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 543, 511, 580, 512, 498, 508, 513, 515, 509, 508, 494, 512, 498, 538, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 482, 488, 556, 501, 485, 508, 524, 503, 509, 520, 509, 532, 507, 475, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 518, 476, 490, 478, 521, 528, 505, 491, 482, 479, 505, 499, 508, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 540, 521, 525, 480, 511, 477, 493, 536, 514, 530, 527, 549, 541, 460, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 508, 517, 507, 512, 509, 546, 525, 524, 555, 503, 508, 522, 467, 497, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8092, 8298, 8215, 8282, 8095, 8025, 8244, 8285] avg=8192.0000 max=8298 max_rank=1 min=8025 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 3968, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[495, 502, 517, 549, 521, 525, 454, 481, 510, 520, 528, 486, 532, 490, 499, 527, 552, 526, 558, 517, 533, 538, 531, 491, 523, 492, 542, 524, 491, 478, 503, 507, 509, 521, 527, 494, 506, 500, 548, 528, 520, 494, 502, 521, 495, 495, 502, 528, 511, 494, 485, 518, 510, 563, 518, 505, 505, 503, 545, 507, 515, 495, 515, 455, 503, 522, 491, 529, 512, 466, 498, 533, 508, 534, 558, 474, 522, 541, 517, 545, 569, 489, 486, 493, 501, 562, 473, 513, 498, 488, 513, 480, 535, 524, 547, 496, 528, 517, 495, 504, 517, 510, 544, 528, 528, 507, 544, 508, 477, 523, 544, 504, 511, 537, 531, 513, 514, 482, 509, 481, 491, 515, 452, 495, 492, 524, 520, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[516, 517, 509, 488, 490, 507, 523, 502, 505, 548, 479, 530, 518, 516, 526, 510, 518, 513, 560, 536, 489, 492, 498, 485, 507, 499, 527, 503, 491, 512, 494, 496, 454, 527, 544, 492, 511, 498, 528, 504, 513, 490, 498, 492, 504, 516, 491, 527, 539, 529, 530, 523, 520, 519, 472, 532, 473, 521, 536, 517, 504, 470, 525, 496, 496, 569, 558, 531, 542, 513, 494, 513, 526, 532, 522, 529, 512, 484, 474, 511, 552, 467, 533, 520, 518, 428, 531, 521, 515, 506, 476, 470, 500, 505, 494, 523, 499, 506, 537, 534, 508, 580, 478, 507, 519, 497, 524, 480, 526, 539, 490, 503, 538, 509, 511, 461, 542, 563, 535, 495, 524, 493, 528, 494, 557, 527, 528, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[482, 519, 487, 530, 498, 477, 522, 535, 524, 521, 522, 529, 537, 458, 509, 485, 521, 510, 473, 516, 524, 513, 517, 505, 531, 502, 490, 520, 527, 531, 518, 475, 525, 522, 532, 528, 499, 527, 443, 482, 548, 516, 535, 550, 525, 519, 491, 481, 451, 494, 502, 521, 513, 539, 493, 492, 513, 509, 539, 519, 504, 533, 525, 498, 556, 522, 510, 494, 547, 515, 507, 532, 484, 488, 475, 512, 532, 500, 543, 494, 470, 506, 493, 475, 521, 526, 506, 463, 490, 474, 540, 474, 538, 524, 536, 496, 495, 522, 532, 465, 463, 496, 542, 495, 534, 508, 531, 544, 527, 489, 514, 535, 563, 512, 566, 497, 519, 519, 521, 499, 486, 551, 512, 537, 548, 572, 537, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[537, 489, 544, 461, 576, 509, 541, 537, 507, 541, 525, 516, 495, 490, 504, 510, 504, 518, 537, 531, 551, 498, 517, 533, 511, 511, 512, 490, 524, 504, 541, 489, 562, 527, 513, 508, 499, 514, 518, 506, 525, 504, 512, 535, 506, 483, 522, 536, 511, 512, 512, 523, 541, 512, 534, 553, 475, 505, 541, 544, 524, 478, 523, 506, 521, 534, 534, 511, 525, 498, 497, 476, 551, 482, 544, 488, 502, 474, 533, 475, 467, 519, 475, 479, 511, 543, 507, 480, 501, 488, 465, 528, 541, 509, 493, 509, 478, 530, 526, 526, 518, 522, 496, 502, 565, 524, 475, 481, 473, 525, 467, 532, 522, 518, 465, 521, 523, 534, 502, 516, 529, 512, 474, 503, 500, 536, 491, 473] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[492, 538, 485, 498, 491, 512, 502, 493, 508, 479, 469, 466, 488, 485, 513, 521, 504, 519, 502, 520, 525, 519, 512, 489, 500, 489, 519, 522, 493, 492, 495, 519, 535, 536, 515, 511, 520, 548, 522, 528, 514, 493, 514, 520, 525, 523, 519, 499, 503, 526, 510, 475, 534, 534, 498, 551, 527, 480, 526, 526, 507, 495, 541, 510, 494, 528, 559, 519, 498, 496, 491, 521, 505, 502, 533, 519, 526, 501, 533, 493, 530, 488, 515, 524, 518, 531, 500, 497, 546, 514, 517, 514, 509, 511, 488, 457, 529, 504, 518, 526, 516, 542, 508, 525, 533, 554, 528, 481, 529, 471, 528, 523, 508, 501, 527, 549, 547, 515, 504, 502, 519, 500, 491, 539, 509, 498, 494, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[497, 468, 534, 534, 558, 511, 522, 502, 510, 531, 494, 520, 465, 510, 518, 551, 514, 472, 488, 510, 535, 487, 532, 529, 509, 539, 528, 554, 499, 525, 525, 514, 524, 516, 517, 521, 499, 478, 504, 494, 492, 516, 537, 536, 507, 519, 499, 500, 484, 533, 533, 471, 519, 511, 501, 495, 562, 493, 501, 535, 500, 497, 515, 511, 487, 561, 516, 509, 524, 507, 510, 514, 476, 524, 552, 557, 522, 509, 515, 549, 506, 499, 488, 503, 510, 504, 501, 504, 544, 522, 506, 482, 521, 528, 530, 545, 498, 512, 525, 548, 537, 539, 507, 509, 463, 478, 540, 463, 539, 517, 497, 518, 488, 474, 466, 551, 529, 532, 513, 493, 520, 486, 475, 511, 504, 473, 507, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[489, 526, 510, 515, 514, 543, 511, 500, 531, 485, 497, 533, 478, 504, 504, 512, 536, 528, 476, 530, 455, 496, 498, 503, 516, 521, 566, 526, 470, 498, 503, 504, 462, 541, 524, 504, 503, 513, 517, 511, 502, 499, 507, 512, 507, 513, 473, 492, 518, 529, 505, 524, 534, 548, 561, 511, 512, 514, 522, 512, 542, 505, 514, 485, 514, 550, 468, 521, 517, 448, 502, 518, 496, 500, 539, 485, 541, 528, 520, 477, 533, 474, 523, 506, 513, 521, 550, 486, 502, 518, 522, 532, 532, 486, 545, 467, 499, 493, 473, 491, 527, 502, 551, 512, 512, 503, 492, 522, 478, 515, 560, 551, 544, 492, 487, 515, 487, 505, 516, 529, 534, 508, 554, 514, 529, 550, 549, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[497, 534, 522, 503, 504, 534, 504, 547, 535, 518, 500, 506, 537, 520, 514, 492, 503, 526, 496, 481, 492, 540, 513, 502, 533, 514, 516, 476, 506, 526, 532, 482, 510, 495, 532, 526, 554, 515, 482, 540, 495, 483, 487, 503, 536, 496, 510, 535, 520, 518, 493, 541, 499, 504, 500, 555, 532, 524, 548, 516, 487, 518, 517, 525, 515, 495, 517, 504, 468, 546, 543, 507, 470, 521, 523, 579, 471, 505, 527, 516, 493, 506, 489, 510, 500, 502, 484, 478, 501, 519, 506, 528, 492, 488, 490, 493, 501, 550, 546, 510, 529, 482, 523, 486, 523, 533, 479, 504, 476, 466, 512, 522, 531, 545, 529, 546, 499, 501, 492, 486, 504, 496, 542, 518, 509, 535, 533, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 502, 517, 549, 521, 525, 454, 481, 510, 520, 528, 486, 532, 490, 499, 527, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 526, 558, 517, 533, 538, 531, 491, 523, 492, 542, 524, 491, 478, 503, 507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 521, 527, 494, 506, 500, 548, 528, 520, 494, 502, 521, 495, 495, 502, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 494, 485, 518, 510, 563, 518, 505, 505, 503, 508, 507, 515, 495, 515, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 491, 529, 512, 466, 498, 533, 508, 534, 558, 474, 522, 541, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 489, 486, 493, 501, 562, 473, 513, 498, 488, 513, 480, 535, 524, 547, 496, 37, 35, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 517, 495, 504, 517, 510, 544, 528, 528, 507, 544, 508, 477, 523, 544, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 537, 531, 513, 514, 482, 509, 481, 491, 515, 452, 495, 492, 489, 520, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8308, 8190, 8107, 8223, 8256, 8264, 8027] avg=8192.0000 max=8308 max_rank=1 min=8027 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 517, 509, 488, 490, 507, 523, 502, 505, 548, 479, 530, 518, 516, 526, 510, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 560, 536, 489, 492, 498, 485, 507, 499, 527, 503, 491, 512, 494, 496, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 527, 544, 492, 511, 498, 528, 504, 513, 490, 498, 492, 504, 516, 491, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 529, 530, 523, 520, 519, 472, 532, 473, 521, 500, 517, 504, 470, 525, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 535, 558, 531, 542, 513, 494, 513, 526, 532, 522, 529, 512, 484, 474, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 467, 533, 520, 518, 428, 531, 521, 515, 506, 476, 470, 500, 505, 494, 523, 36, 35, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 537, 534, 508, 580, 478, 507, 519, 497, 524, 480, 526, 539, 490, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 509, 511, 461, 542, 563, 535, 495, 524, 493, 528, 494, 557, 492, 528, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8122, 8089, 8170, 8272, 8149, 8213, 8310] avg=8192.0000 max=8310 max_rank=7 min=8089 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[537, 489, 544, 461, 576, 509, 541, 537, 507, 541, 525, 516, 495, 490, 504, 510, 14, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 537, 531, 551, 498, 517, 533, 511, 511, 512, 490, 524, 504, 541, 489, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 527, 513, 508, 499, 514, 518, 506, 525, 504, 512, 535, 506, 483, 522, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 512, 512, 523, 541, 512, 534, 553, 475, 505, 505, 544, 524, 478, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 502, 534, 511, 525, 498, 497, 476, 551, 482, 544, 488, 502, 474, 533, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 519, 475, 479, 511, 543, 507, 480, 501, 488, 465, 528, 541, 509, 493, 509, 36, 36, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 530, 526, 526, 518, 522, 496, 502, 565, 524, 475, 481, 473, 525, 467, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 465, 521, 523, 534, 502, 516, 529, 512, 474, 503, 500, 500, 491, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8273, 8270, 8258, 8113, 8105, 8126, 8083] avg=8192.0000 max=8308 max_rank=0 min=8083 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[497, 468, 534, 534, 558, 511, 522, 502, 510, 531, 494, 520, 465, 510, 518, 551, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 472, 488, 510, 535, 487, 532, 529, 509, 539, 528, 554, 499, 525, 525, 514, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 516, 517, 521, 499, 478, 504, 494, 492, 516, 537, 536, 507, 519, 499, 500, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 533, 533, 471, 519, 511, 501, 495, 562, 493, 463, 535, 500, 497, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 519, 516, 509, 524, 507, 510, 514, 476, 524, 552, 557, 522, 509, 515, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 499, 488, 503, 510, 504, 501, 504, 544, 522, 506, 482, 521, 528, 530, 545, 38, 35, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 525, 548, 537, 539, 507, 509, 463, 478, 540, 463, 539, 517, 497, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 474, 466, 551, 529, 532, 513, 493, 520, 486, 475, 511, 504, 438, 507, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8268, 8163, 8123, 8290, 8290, 8162, 7981] avg=8192.0000 max=8290 max_rank=4 min=7981 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[482, 519, 487, 530, 498, 477, 522, 535, 524, 521, 522, 529, 537, 458, 509, 485, 13, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 473, 516, 524, 513, 517, 505, 531, 502, 490, 520, 527, 531, 518, 475, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 532, 528, 499, 527, 443, 482, 548, 516, 535, 550, 525, 519, 491, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 494, 502, 521, 513, 539, 493, 492, 513, 509, 503, 519, 504, 533, 525, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 492, 510, 494, 547, 515, 507, 532, 484, 488, 475, 512, 532, 500, 543, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 506, 493, 475, 521, 526, 506, 463, 490, 474, 540, 474, 538, 524, 536, 496, 36, 38, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 532, 465, 463, 496, 542, 495, 534, 508, 531, 544, 527, 489, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 512, 566, 497, 519, 519, 521, 499, 486, 551, 512, 537, 548, 534, 537, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8175, 8223, 8109, 8181, 8123, 8177, 8387] avg=8192.0000 max=8387 max_rank=7 min=8109 min_rank=3 imbalance_ratio=1.023804 max_load_reduction=57149 max_load_reduction_pct=0.872025 moved_to_echo=32868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[497, 534, 522, 503, 504, 534, 504, 547, 535, 518, 500, 506, 537, 520, 514, 492, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 496, 481, 492, 540, 513, 502, 533, 514, 516, 476, 506, 526, 532, 482, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 495, 532, 526, 554, 515, 482, 540, 495, 483, 487, 503, 536, 496, 510, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 518, 493, 541, 499, 504, 500, 555, 532, 524, 511, 516, 487, 518, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 466, 517, 504, 468, 546, 543, 507, 470, 521, 523, 579, 471, 505, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 506, 489, 510, 500, 502, 484, 478, 501, 519, 506, 528, 492, 488, 490, 493, 37, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 550, 546, 510, 529, 482, 523, 486, 523, 533, 479, 504, 476, 466, 512, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 545, 529, 546, 499, 501, 492, 486, 504, 496, 542, 518, 509, 499, 533, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 8140, 8199, 8260, 8178, 8068, 8128, 8271] avg=8192.0000 max=8292 max_rank=0 min=8068 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[489, 526, 510, 515, 514, 543, 511, 500, 531, 485, 497, 533, 478, 504, 504, 512, 14, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 528, 476, 530, 455, 496, 498, 503, 516, 521, 566, 526, 470, 498, 503, 504, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 541, 524, 504, 503, 513, 517, 511, 502, 499, 507, 512, 507, 513, 473, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 505, 524, 534, 548, 561, 511, 512, 514, 487, 512, 542, 505, 514, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 518, 468, 521, 517, 448, 502, 518, 496, 500, 539, 485, 541, 528, 520, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 474, 523, 506, 513, 521, 550, 486, 502, 518, 522, 532, 532, 486, 545, 467, 35, 37, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 473, 491, 527, 502, 551, 512, 512, 503, 492, 522, 478, 515, 560, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 492, 487, 515, 487, 505, 516, 529, 534, 508, 554, 514, 529, 513, 549, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8129, 8080, 8301, 8092, 8300, 8165, 8290] avg=8192.0000 max=8301 max_rank=3 min=8080 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 538, 485, 498, 491, 512, 502, 493, 508, 479, 469, 466, 488, 485, 513, 521, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 519, 502, 520, 525, 519, 512, 489, 500, 489, 519, 522, 493, 492, 495, 519, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 536, 515, 511, 520, 548, 522, 528, 514, 493, 514, 520, 525, 523, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 510, 475, 534, 534, 498, 551, 527, 480, 491, 526, 507, 495, 541, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 497, 559, 519, 498, 496, 491, 521, 505, 502, 533, 519, 526, 501, 533, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 488, 515, 524, 518, 531, 500, 497, 546, 514, 517, 514, 509, 511, 488, 457, 35, 33, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 518, 526, 516, 542, 508, 525, 533, 554, 528, 481, 529, 471, 528, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 527, 549, 547, 515, 504, 502, 519, 500, 491, 539, 509, 465, 494, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7965, 8121, 8322, 8208, 8187, 8245, 8301, 8187] avg=8192.0000 max=8322 max_rank=2 min=7965 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 3968, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 3968, 4096, 4096, 4096, 4224, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 384, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4352, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[492, 519, 535, 518, 502, 519, 520, 511, 499, 515, 507, 504, 472, 489, 489, 557, 498, 523, 520, 523, 511, 512, 511, 521, 554, 513, 527, 483, 515, 537, 498, 498, 510, 502, 514, 527, 491, 543, 562, 502, 482, 465, 516, 519, 524, 525, 514, 466, 507, 465, 512, 516, 506, 508, 535, 520, 509, 517, 553, 517, 516, 500, 516, 499, 498, 532, 519, 527, 511, 499, 529, 497, 508, 488, 496, 516, 527, 531, 464, 519, 481, 510, 557, 522, 514, 510, 524, 550, 503, 533, 557, 496, 505, 470, 528, 517, 483, 485, 513, 529, 471, 501, 514, 492, 471, 523, 520, 537, 530, 545, 526, 504, 532, 515, 506, 501, 473, 516, 508, 493, 522, 502, 519, 508, 507, 531, 535, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[513, 527, 509, 511, 509, 549, 522, 508, 520, 527, 521, 488, 542, 529, 510, 487, 513, 548, 497, 542, 439, 481, 467, 462, 498, 481, 501, 509, 525, 520, 486, 532, 491, 494, 499, 496, 528, 507, 524, 528, 496, 515, 507, 489, 554, 514, 519, 507, 479, 524, 523, 498, 500, 535, 487, 500, 529, 528, 550, 523, 499, 527, 508, 540, 510, 507, 509, 513, 531, 544, 526, 499, 533, 516, 532, 516, 519, 513, 520, 491, 486, 533, 530, 488, 530, 544, 487, 508, 537, 490, 550, 489, 525, 534, 488, 512, 507, 540, 506, 530, 518, 521, 531, 539, 503, 519, 481, 520, 492, 478, 518, 504, 517, 537, 503, 484, 455, 514, 488, 514, 513, 481, 544, 463, 549, 542, 490, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[554, 504, 503, 537, 485, 494, 520, 501, 519, 489, 548, 494, 512, 537, 497, 528, 504, 525, 537, 491, 535, 525, 515, 537, 530, 495, 500, 517, 518, 515, 507, 507, 470, 500, 507, 525, 558, 500, 516, 537, 500, 528, 498, 504, 529, 503, 516, 530, 493, 492, 513, 520, 541, 522, 501, 522, 483, 539, 507, 498, 513, 531, 530, 506, 483, 560, 491, 492, 512, 485, 514, 467, 464, 543, 503, 534, 503, 503, 527, 502, 488, 528, 477, 547, 460, 477, 488, 481, 480, 503, 494, 521, 505, 504, 532, 524, 512, 526, 490, 503, 512, 515, 495, 515, 501, 492, 463, 520, 523, 534, 519, 512, 535, 554, 539, 502, 524, 537, 534, 543, 543, 512, 541, 532, 516, 499, 481, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[537, 504, 536, 548, 502, 539, 514, 494, 499, 529, 498, 478, 521, 545, 521, 530, 556, 512, 506, 496, 486, 491, 508, 530, 517, 512, 498, 505, 484, 511, 506, 503, 542, 502, 527, 499, 532, 483, 494, 518, 478, 536, 497, 495, 513, 499, 500, 530, 509, 500, 531, 506, 531, 503, 482, 487, 515, 537, 539, 504, 485, 541, 522, 510, 515, 530, 502, 486, 533, 513, 492, 542, 498, 533, 506, 501, 525, 503, 497, 540, 518, 500, 510, 478, 512, 533, 506, 506, 537, 506, 488, 535, 510, 512, 541, 517, 486, 513, 498, 494, 523, 496, 536, 497, 520, 500, 493, 561, 526, 518, 501, 505, 492, 526, 533, 464, 522, 468, 526, 509, 509, 548, 487, 511, 509, 485, 528, 564] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[530, 518, 501, 528, 570, 519, 505, 494, 532, 490, 493, 512, 541, 508, 521, 451, 508, 497, 501, 536, 504, 482, 526, 507, 535, 507, 543, 526, 507, 498, 523, 498, 527, 502, 490, 500, 544, 485, 465, 511, 476, 474, 522, 511, 533, 495, 545, 483, 514, 541, 482, 488, 499, 546, 565, 524, 477, 522, 506, 533, 550, 500, 537, 531, 524, 522, 511, 527, 526, 531, 525, 507, 525, 529, 517, 508, 485, 515, 549, 448, 500, 492, 473, 473, 509, 539, 489, 505, 506, 512, 509, 534, 543, 510, 546, 541, 535, 510, 484, 497, 508, 477, 509, 520, 483, 503, 490, 507, 506, 487, 579, 536, 517, 492, 492, 490, 540, 536, 505, 549, 502, 482, 549, 469, 510, 509, 515, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[495, 511, 481, 487, 482, 510, 513, 526, 503, 511, 560, 477, 507, 520, 544, 466, 520, 496, 517, 481, 517, 506, 481, 479, 494, 530, 541, 523, 516, 542, 499, 527, 528, 529, 480, 527, 541, 539, 524, 489, 535, 509, 531, 503, 495, 538, 519, 547, 548, 481, 508, 536, 531, 491, 481, 518, 494, 537, 514, 526, 513, 472, 541, 470, 481, 502, 520, 500, 513, 506, 477, 520, 529, 517, 523, 522, 529, 524, 525, 486, 556, 509, 523, 515, 519, 517, 489, 497, 546, 521, 517, 517, 498, 503, 539, 495, 501, 488, 503, 494, 534, 514, 530, 520, 511, 504, 494, 549, 534, 541, 486, 502, 508, 511, 525, 474, 561, 515, 506, 506, 524, 509, 533, 481, 499, 512, 478, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[527, 517, 507, 500, 550, 522, 520, 544, 518, 530, 505, 532, 472, 505, 495, 512, 471, 501, 472, 514, 532, 489, 503, 510, 501, 494, 487, 494, 498, 480, 487, 522, 503, 540, 521, 469, 547, 484, 539, 532, 533, 504, 521, 505, 525, 503, 509, 549, 499, 514, 514, 522, 501, 504, 526, 497, 537, 535, 537, 500, 488, 511, 482, 503, 501, 509, 525, 525, 522, 506, 498, 489, 538, 485, 523, 499, 489, 514, 478, 501, 522, 509, 533, 516, 516, 533, 513, 508, 491, 500, 484, 550, 534, 556, 522, 532, 500, 536, 483, 479, 529, 515, 507, 533, 483, 505, 550, 542, 519, 514, 495, 491, 541, 517, 504, 511, 496, 491, 532, 499, 517, 512, 524, 518, 505, 538, 520, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[516, 509, 549, 530, 491, 488, 490, 498, 509, 485, 515, 514, 508, 507, 540, 502, 510, 494, 522, 491, 480, 520, 497, 504, 518, 491, 543, 505, 489, 518, 518, 510, 508, 532, 476, 549, 530, 529, 513, 559, 494, 576, 508, 532, 479, 538, 514, 551, 518, 500, 482, 460, 520, 509, 501, 537, 494, 482, 509, 505, 530, 497, 519, 511, 519, 520, 483, 498, 504, 487, 491, 471, 513, 503, 519, 558, 528, 481, 507, 534, 585, 486, 501, 501, 514, 478, 516, 502, 509, 549, 536, 472, 539, 509, 555, 482, 553, 494, 541, 476, 526, 502, 456, 484, 506, 508, 537, 511, 509, 468, 552, 550, 524, 514, 515, 540, 505, 532, 537, 495, 485, 490, 473, 517, 549, 545, 534, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 527, 509, 511, 509, 549, 522, 508, 520, 527, 521, 488, 542, 529, 510, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 548, 497, 542, 439, 481, 467, 462, 498, 481, 501, 509, 525, 520, 486, 532, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 494, 499, 496, 500, 507, 524, 528, 496, 515, 507, 489, 554, 514, 519, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 524, 523, 498, 500, 535, 487, 500, 529, 528, 545, 523, 499, 527, 508, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 509, 513, 531, 544, 526, 499, 533, 516, 532, 516, 519, 513, 520, 491, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 533, 530, 488, 530, 544, 487, 508, 537, 490, 550, 489, 525, 534, 467, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 540, 506, 530, 518, 521, 531, 539, 503, 519, 481, 520, 492, 478, 518, 504, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 537, 503, 484, 455, 514, 488, 514, 513, 481, 544, 463, 549, 542, 490, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8060, 8140, 8245, 8297, 8210, 8229, 8096] avg=8192.0000 max=8297 max_rank=4 min=8060 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 517, 507, 500, 550, 522, 520, 544, 518, 530, 505, 532, 472, 505, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 501, 472, 514, 532, 489, 503, 510, 501, 494, 487, 494, 498, 480, 487, 522, 33, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 540, 521, 469, 518, 484, 539, 532, 533, 504, 521, 505, 525, 503, 509, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 514, 514, 522, 501, 504, 526, 497, 537, 535, 532, 500, 488, 511, 482, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 525, 525, 522, 506, 498, 489, 538, 485, 523, 499, 489, 514, 478, 501, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 509, 533, 516, 516, 533, 513, 508, 491, 500, 484, 550, 534, 556, 500, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 536, 483, 479, 529, 515, 507, 533, 483, 505, 550, 542, 519, 514, 495, 491, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 517, 504, 511, 496, 491, 532, 499, 517, 512, 524, 518, 505, 538, 520, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8016, 8255, 8165, 8121, 8297, 8204, 8236] avg=8192.0000 max=8297 max_rank=5 min=8016 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[519, 504, 536, 548, 502, 539, 514, 494, 499, 529, 498, 478, 521, 545, 521, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 512, 506, 496, 486, 491, 508, 530, 517, 512, 498, 505, 484, 511, 506, 503, 39, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 502, 527, 499, 499, 483, 494, 518, 478, 536, 497, 495, 513, 499, 500, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 531, 506, 531, 503, 482, 487, 515, 537, 531, 504, 485, 541, 522, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 530, 502, 486, 533, 513, 492, 542, 498, 533, 506, 501, 525, 503, 497, 540, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 510, 478, 512, 533, 506, 506, 537, 506, 488, 535, 510, 512, 514, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 513, 498, 494, 523, 496, 536, 497, 520, 500, 493, 561, 526, 518, 501, 505, 27, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 526, 533, 464, 522, 468, 526, 509, 509, 548, 487, 511, 509, 485, 528, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8190, 8112, 8194, 8241, 8182, 8198, 8142] avg=8192.0000 max=8277 max_rank=0 min=8112 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[482, 511, 481, 487, 482, 510, 513, 526, 503, 511, 560, 477, 507, 520, 544, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 496, 517, 481, 517, 506, 481, 479, 494, 530, 541, 523, 516, 542, 499, 527, 30, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 529, 480, 527, 513, 539, 524, 489, 535, 509, 531, 503, 495, 538, 519, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 481, 508, 536, 531, 491, 481, 518, 494, 537, 509, 526, 513, 472, 541, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 502, 520, 500, 513, 506, 477, 520, 529, 517, 523, 522, 529, 524, 525, 486, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 509, 523, 515, 519, 517, 489, 497, 546, 521, 517, 517, 498, 503, 516, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 488, 503, 494, 534, 514, 530, 520, 511, 504, 494, 549, 534, 541, 486, 502, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 511, 525, 474, 561, 515, 506, 506, 524, 509, 533, 481, 499, 512, 478, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8080, 8226, 8306, 8156, 8192, 8238, 8229, 8109] avg=8192.0000 max=8306 max_rank=2 min=8080 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[479, 519, 535, 518, 502, 519, 520, 511, 499, 515, 507, 504, 472, 489, 489, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 523, 520, 523, 511, 512, 511, 521, 554, 513, 527, 483, 515, 537, 498, 498, 32, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 502, 514, 527, 465, 543, 562, 502, 482, 465, 516, 519, 524, 525, 514, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 465, 512, 516, 506, 508, 535, 520, 509, 517, 548, 517, 516, 500, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 532, 519, 527, 511, 499, 529, 497, 508, 488, 496, 516, 527, 531, 464, 519, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 510, 557, 522, 514, 510, 524, 550, 503, 533, 557, 496, 505, 470, 506, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 485, 513, 529, 471, 501, 514, 492, 471, 523, 520, 537, 530, 545, 526, 504, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 515, 506, 501, 473, 516, 508, 493, 522, 502, 519, 508, 507, 531, 535, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8301, 8136, 8191, 8179, 8255, 8167, 8172] avg=8192.0000 max=8301 max_rank=1 min=8135 min_rank=0 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[516, 518, 501, 528, 570, 519, 505, 494, 532, 490, 493, 512, 541, 508, 521, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 501, 536, 504, 482, 526, 507, 535, 507, 543, 526, 507, 498, 523, 498, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 490, 500, 516, 485, 465, 511, 476, 474, 522, 511, 533, 495, 545, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 541, 482, 488, 499, 546, 565, 524, 477, 522, 501, 533, 550, 500, 537, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 511, 527, 526, 531, 525, 507, 525, 529, 517, 508, 485, 515, 549, 448, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 492, 473, 473, 509, 539, 489, 505, 506, 512, 509, 534, 543, 510, 523, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 510, 484, 497, 508, 477, 509, 520, 483, 503, 490, 507, 506, 487, 579, 536, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 492, 492, 490, 540, 536, 505, 549, 502, 482, 549, 469, 510, 509, 515, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8257, 8035, 8310, 8268, 8158, 8155, 8154] avg=8192.0000 max=8310 max_rank=3 min=8035 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[539, 504, 503, 537, 485, 494, 520, 501, 519, 489, 548, 494, 512, 537, 497, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 537, 491, 535, 525, 515, 537, 530, 495, 500, 517, 518, 515, 507, 507, 31, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 500, 507, 525, 529, 500, 516, 537, 500, 528, 498, 504, 529, 503, 516, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 513, 520, 541, 522, 501, 522, 483, 539, 502, 498, 513, 531, 530, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 560, 491, 492, 512, 485, 514, 467, 464, 543, 503, 534, 503, 503, 527, 502, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 528, 477, 547, 460, 477, 488, 481, 480, 503, 494, 521, 505, 504, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 526, 490, 503, 512, 515, 495, 515, 501, 492, 463, 520, 523, 534, 519, 512, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 554, 539, 502, 524, 537, 534, 543, 543, 512, 541, 532, 516, 499, 481, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8317, 8192, 8206, 8103, 7986, 8156, 8369] avg=8192.0000 max=8369 max_rank=7 min=7986 min_rank=5 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 509, 549, 530, 491, 488, 490, 498, 509, 485, 515, 514, 508, 507, 540, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 494, 522, 491, 480, 520, 497, 504, 518, 491, 543, 505, 489, 518, 518, 510, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 532, 476, 549, 502, 529, 513, 559, 494, 576, 508, 532, 479, 538, 514, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 482, 460, 520, 509, 501, 537, 494, 482, 504, 505, 530, 497, 519, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 483, 498, 504, 487, 491, 471, 513, 503, 519, 558, 528, 481, 507, 534, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 486, 501, 501, 514, 478, 516, 502, 509, 549, 536, 472, 539, 509, 531, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 494, 541, 476, 526, 502, 456, 484, 506, 508, 537, 511, 509, 468, 552, 550, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 515, 540, 505, 532, 537, 495, 485, 490, 473, 517, 549, 545, 534, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8169, 8360, 8069, 8135, 8210, 8198, 8258] avg=8192.0000 max=8360 max_rank=2 min=8069 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4352, 4224, 4096, 4224, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[506, 512, 534, 471, 497, 535, 513, 474, 482, 490, 531, 505, 513, 489, 546, 490, 511, 552, 494, 532, 523, 547, 481, 507, 510, 528, 508, 498, 503, 526, 599, 495, 522, 503, 502, 507, 503, 546, 503, 510, 516, 548, 487, 503, 487, 500, 529, 505, 508, 496, 478, 528, 519, 500, 512, 490, 486, 530, 514, 479, 481, 508, 520, 505, 480, 534, 500, 508, 478, 510, 534, 495, 516, 491, 509, 510, 549, 497, 512, 503, 510, 509, 504, 515, 531, 535, 514, 514, 479, 529, 529, 517, 517, 508, 498, 505, 516, 558, 546, 470, 520, 516, 532, 532, 561, 535, 450, 563, 523, 516, 512, 524, 508, 527, 496, 491, 511, 520, 507, 528, 529, 491, 524, 514, 486, 530, 525, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[553, 525, 523, 504, 513, 505, 466, 527, 508, 532, 522, 512, 522, 494, 508, 520, 510, 534, 510, 481, 481, 500, 490, 503, 528, 520, 535, 520, 492, 455, 546, 514, 525, 512, 466, 488, 500, 527, 537, 510, 532, 505, 487, 537, 570, 524, 568, 488, 516, 544, 495, 508, 522, 490, 522, 527, 529, 520, 504, 531, 537, 491, 515, 500, 500, 506, 489, 489, 525, 502, 516, 548, 514, 513, 517, 540, 517, 498, 508, 478, 532, 498, 549, 507, 516, 495, 509, 493, 509, 513, 506, 538, 489, 536, 500, 523, 486, 524, 515, 468, 491, 487, 524, 521, 483, 495, 564, 516, 480, 553, 507, 521, 522, 498, 469, 513, 483, 501, 493, 466, 580, 504, 494, 518, 504, 572, 493, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[531, 481, 536, 477, 517, 471, 504, 524, 535, 485, 517, 494, 520, 497, 486, 469, 494, 516, 521, 514, 496, 534, 493, 519, 497, 530, 454, 510, 523, 514, 550, 515, 521, 510, 522, 507, 510, 530, 487, 554, 480, 524, 498, 502, 513, 502, 560, 479, 480, 523, 481, 477, 490, 545, 526, 522, 492, 498, 541, 509, 490, 477, 523, 513, 519, 554, 469, 522, 573, 520, 523, 511, 513, 507, 505, 493, 519, 525, 551, 541, 471, 530, 512, 534, 564, 505, 504, 477, 526, 486, 475, 524, 480, 526, 525, 510, 511, 527, 553, 505, 480, 542, 494, 556, 507, 500, 531, 468, 493, 518, 502, 473, 557, 514, 503, 519, 530, 520, 562, 488, 501, 530, 542, 517, 506, 501, 544, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[484, 525, 528, 440, 504, 500, 490, 539, 507, 561, 490, 521, 542, 503, 526, 511, 548, 471, 500, 512, 531, 534, 510, 521, 524, 501, 470, 503, 485, 509, 518, 521, 501, 535, 545, 522, 523, 541, 549, 527, 538, 513, 517, 528, 490, 532, 545, 498, 520, 556, 524, 490, 511, 486, 514, 466, 495, 525, 493, 494, 517, 505, 532, 539, 506, 517, 516, 528, 484, 547, 533, 512, 527, 508, 486, 508, 484, 530, 516, 515, 533, 506, 560, 504, 495, 478, 516, 545, 513, 532, 511, 501, 512, 512, 482, 499, 498, 471, 517, 481, 526, 543, 519, 529, 517, 510, 493, 503, 497, 531, 541, 522, 514, 518, 487, 516, 529, 512, 471, 504, 527, 441, 522, 492, 514, 498, 494, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[487, 525, 472, 552, 527, 516, 516, 483, 522, 537, 547, 531, 522, 512, 561, 531, 514, 535, 518, 530, 487, 501, 490, 485, 513, 550, 470, 545, 500, 526, 505, 496, 514, 477, 535, 509, 527, 471, 513, 450, 491, 504, 503, 481, 532, 514, 481, 505, 522, 522, 486, 526, 531, 508, 527, 508, 558, 530, 487, 523, 474, 531, 459, 519, 505, 537, 501, 486, 566, 515, 542, 489, 503, 528, 508, 473, 476, 550, 528, 493, 518, 516, 496, 520, 521, 488, 518, 559, 495, 500, 539, 565, 516, 505, 505, 526, 517, 509, 475, 511, 498, 515, 483, 565, 535, 503, 489, 516, 487, 538, 532, 497, 535, 518, 507, 511, 490, 525, 494, 536, 480, 513, 507, 483, 520, 521, 505, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[508, 515, 522, 501, 526, 531, 482, 504, 554, 480, 494, 500, 539, 528, 526, 548, 535, 497, 502, 468, 501, 509, 503, 527, 514, 511, 514, 502, 536, 522, 492, 571, 547, 499, 488, 495, 495, 493, 498, 553, 521, 482, 493, 495, 513, 509, 551, 512, 496, 497, 527, 524, 527, 536, 488, 526, 539, 515, 522, 520, 527, 530, 556, 513, 530, 503, 478, 532, 548, 508, 507, 515, 505, 528, 513, 491, 501, 534, 525, 518, 522, 534, 493, 504, 509, 512, 502, 509, 500, 493, 493, 465, 507, 506, 537, 524, 515, 534, 481, 498, 502, 516, 518, 469, 551, 547, 495, 494, 474, 499, 534, 509, 511, 484, 508, 491, 525, 478, 484, 516, 530, 476, 568, 526, 478, 496, 535, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[503, 514, 510, 542, 502, 548, 499, 509, 558, 505, 527, 499, 482, 534, 490, 517, 513, 505, 555, 512, 507, 510, 515, 517, 520, 484, 514, 513, 504, 504, 490, 487, 555, 553, 542, 514, 526, 484, 461, 484, 504, 508, 511, 521, 522, 525, 518, 506, 481, 512, 553, 523, 480, 482, 495, 522, 547, 480, 506, 504, 505, 522, 502, 501, 511, 536, 530, 528, 498, 530, 531, 526, 574, 487, 499, 493, 495, 532, 495, 495, 491, 494, 553, 512, 488, 537, 522, 482, 500, 521, 485, 501, 492, 532, 515, 537, 516, 500, 512, 504, 502, 490, 531, 507, 519, 512, 527, 499, 510, 515, 495, 518, 507, 484, 508, 542, 501, 533, 537, 491, 505, 520, 502, 529, 502, 481, 549, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[519, 473, 491, 492, 511, 532, 483, 492, 508, 518, 542, 527, 507, 543, 531, 471, 558, 502, 519, 528, 529, 504, 545, 536, 509, 505, 513, 503, 493, 497, 528, 536, 529, 533, 545, 496, 490, 502, 530, 501, 491, 506, 521, 508, 514, 506, 493, 541, 476, 495, 497, 499, 559, 502, 478, 539, 505, 519, 509, 492, 514, 522, 507, 548, 505, 554, 520, 517, 533, 502, 498, 509, 516, 505, 514, 501, 498, 492, 487, 522, 514, 466, 545, 491, 524, 441, 529, 528, 528, 520, 520, 502, 516, 504, 561, 505, 499, 516, 453, 512, 509, 520, 532, 541, 506, 546, 517, 487, 467, 494, 537, 488, 537, 512, 517, 505, 509, 540, 491, 496, 522, 504, 527, 544, 475, 505, 512, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[506, 512, 534, 471, 497, 535, 513, 474, 482, 490, 531, 505, 513, 489, 546, 490, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 552, 494, 532, 523, 547, 481, 507, 510, 528, 508, 498, 503, 526, 592, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 502, 507, 503, 546, 503, 510, 516, 548, 487, 503, 487, 500, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 478, 528, 519, 500, 512, 490, 486, 530, 514, 479, 481, 508, 520, 505, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 512, 500, 508, 478, 510, 534, 495, 516, 491, 509, 510, 549, 497, 512, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 504, 515, 531, 535, 514, 514, 479, 529, 529, 517, 517, 508, 498, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 558, 546, 470, 520, 516, 532, 532, 561, 535, 450, 563, 523, 516, 512, 524, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 496, 491, 511, 520, 507, 528, 529, 491, 524, 514, 486, 530, 525, 508, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8307, 8159, 8072, 8104, 8215, 8384, 8205] avg=8192.0000 max=8384 max_rank=6 min=8072 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32908 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[531, 481, 536, 477, 517, 471, 504, 524, 535, 485, 517, 494, 520, 497, 486, 469, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 516, 521, 514, 496, 534, 493, 519, 497, 530, 454, 510, 523, 514, 544, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 522, 507, 510, 530, 487, 554, 480, 524, 498, 502, 513, 502, 548, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 481, 477, 490, 545, 526, 522, 492, 498, 541, 509, 490, 477, 523, 513, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 469, 522, 573, 520, 523, 511, 513, 507, 505, 493, 519, 525, 551, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 530, 512, 534, 564, 505, 504, 477, 526, 486, 475, 524, 480, 526, 525, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 527, 553, 505, 480, 542, 494, 556, 507, 500, 531, 468, 493, 518, 502, 473, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 514, 503, 519, 530, 520, 562, 488, 501, 530, 542, 517, 506, 501, 544, 538, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8174, 8187, 8106, 8322, 8150, 8169, 8382] avg=8192.0000 max=8382 max_rank=7 min=8046 min_rank=0 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=33023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[553, 525, 523, 504, 513, 505, 466, 527, 508, 532, 522, 512, 522, 494, 508, 520, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 534, 510, 481, 481, 500, 490, 503, 528, 520, 535, 520, 492, 455, 525, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 512, 466, 488, 500, 527, 537, 510, 532, 505, 487, 537, 570, 524, 546, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 544, 495, 508, 522, 490, 522, 527, 529, 520, 504, 531, 537, 491, 515, 500, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 479, 489, 489, 525, 502, 516, 548, 514, 513, 517, 540, 517, 498, 508, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 498, 549, 507, 516, 495, 509, 493, 509, 513, 506, 538, 489, 536, 500, 523, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 524, 515, 468, 491, 487, 524, 521, 483, 495, 564, 516, 480, 553, 507, 521, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 498, 469, 513, 483, 501, 493, 466, 580, 504, 494, 518, 504, 572, 493, 538, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8098, 8254, 8272, 8133, 8219, 8154, 8165] avg=8192.0000 max=8272 max_rank=3 min=8098 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[503, 514, 510, 542, 502, 548, 499, 509, 558, 505, 527, 499, 482, 534, 490, 517, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 555, 512, 507, 510, 515, 517, 520, 484, 514, 513, 504, 504, 485, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 553, 542, 514, 526, 484, 461, 484, 504, 508, 511, 521, 522, 525, 506, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 512, 553, 523, 480, 482, 495, 522, 547, 480, 506, 504, 505, 522, 502, 501, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 530, 528, 498, 530, 531, 526, 574, 487, 499, 493, 495, 532, 495, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 494, 553, 512, 488, 537, 522, 482, 500, 521, 485, 501, 492, 532, 515, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 500, 512, 504, 502, 490, 531, 507, 519, 512, 527, 499, 510, 515, 495, 518, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 484, 508, 542, 501, 533, 537, 491, 505, 520, 502, 529, 502, 481, 549, 528, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8145, 8222, 8133, 8238, 8162, 8166, 8229] avg=8192.0000 max=8241 max_rank=0 min=8133 min_rank=3 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[487, 525, 472, 552, 527, 516, 516, 483, 522, 537, 547, 531, 522, 512, 561, 531, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 535, 518, 530, 487, 501, 490, 485, 513, 550, 470, 545, 500, 526, 500, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 477, 535, 509, 527, 471, 513, 450, 491, 504, 503, 481, 532, 514, 470, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 522, 486, 526, 531, 508, 527, 508, 558, 530, 487, 523, 474, 531, 459, 519, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 501, 486, 566, 515, 542, 489, 503, 528, 508, 473, 476, 550, 528, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 496, 520, 521, 488, 518, 559, 495, 500, 539, 565, 516, 505, 505, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 475, 511, 498, 515, 483, 565, 535, 503, 489, 516, 487, 538, 532, 497, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 518, 507, 511, 490, 525, 494, 536, 480, 513, 507, 483, 520, 521, 505, 510, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8343, 8160, 7996, 8229, 8178, 8287, 8178, 8165] avg=8192.0000 max=8343 max_rank=0 min=7996 min_rank=2 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[484, 525, 528, 440, 504, 500, 490, 539, 507, 561, 490, 521, 542, 503, 526, 511, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 471, 500, 512, 531, 534, 510, 521, 524, 501, 470, 503, 485, 509, 513, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 535, 545, 522, 523, 541, 549, 527, 538, 513, 517, 528, 490, 532, 533, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 556, 524, 490, 511, 486, 514, 466, 495, 525, 493, 494, 517, 505, 532, 539, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 496, 516, 528, 484, 547, 533, 512, 527, 508, 486, 508, 484, 530, 516, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 560, 504, 495, 478, 516, 545, 513, 532, 511, 501, 512, 512, 482, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 471, 517, 481, 526, 543, 519, 529, 517, 510, 493, 503, 497, 531, 541, 522, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 518, 487, 516, 529, 512, 471, 504, 527, 441, 522, 492, 514, 498, 494, 483, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8153, 8392, 8184, 8196, 8199, 8207, 8032] avg=8192.0000 max=8392 max_rank=2 min=8032 min_rank=7 imbalance_ratio=1.024414 max_load_reduction=57144 max_load_reduction_pct=0.871948 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 515, 522, 501, 526, 531, 482, 504, 554, 480, 494, 500, 539, 528, 526, 548, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 497, 502, 468, 501, 509, 503, 527, 514, 511, 514, 502, 536, 522, 487, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 499, 488, 495, 495, 493, 498, 553, 521, 482, 493, 495, 513, 509, 539, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 527, 524, 527, 536, 488, 526, 539, 515, 522, 520, 527, 530, 556, 513, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 482, 478, 532, 548, 508, 507, 515, 505, 528, 513, 491, 501, 534, 525, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 534, 493, 504, 509, 512, 502, 509, 500, 493, 493, 465, 507, 506, 537, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 534, 481, 498, 502, 516, 518, 469, 551, 547, 495, 494, 474, 499, 534, 509, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 484, 508, 491, 525, 478, 484, 516, 530, 476, 568, 526, 478, 496, 535, 499, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8199, 8132, 8360, 8215, 8110, 8145, 8115] avg=8192.0000 max=8360 max_rank=3 min=8110 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[519, 473, 491, 492, 511, 532, 483, 492, 508, 518, 542, 527, 507, 543, 531, 471, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 502, 519, 528, 529, 504, 545, 536, 509, 505, 513, 503, 493, 497, 523, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 533, 545, 496, 490, 502, 530, 501, 491, 506, 521, 508, 514, 506, 481, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 495, 497, 499, 559, 502, 478, 539, 505, 519, 509, 492, 514, 522, 507, 548, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 520, 517, 533, 502, 498, 509, 516, 505, 514, 501, 498, 492, 487, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 466, 545, 491, 524, 441, 529, 528, 528, 520, 520, 502, 516, 504, 561, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 516, 453, 512, 509, 520, 532, 541, 506, 546, 517, 487, 467, 494, 537, 488, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 512, 517, 505, 509, 540, 491, 496, 522, 504, 527, 544, 475, 505, 512, 537, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8300, 8194, 8180, 8150, 8194, 8133, 8243] avg=8192.0000 max=8300 max_rank=1 min=8133 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 3968, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[455, 506, 458, 529, 512, 498, 555, 508, 491, 480, 526, 515, 502, 544, 514, 487, 497, 527, 521, 496, 496, 489, 514, 519, 506, 508, 539, 542, 524, 526, 479, 512, 528, 523, 573, 502, 533, 510, 529, 533, 491, 547, 488, 487, 532, 543, 487, 534, 511, 527, 473, 519, 550, 483, 522, 509, 551, 472, 497, 532, 517, 524, 513, 515, 539, 499, 527, 512, 497, 536, 529, 532, 499, 520, 510, 492, 540, 519, 507, 486, 521, 461, 489, 512, 523, 494, 509, 532, 520, 520, 517, 479, 492, 530, 485, 524, 561, 490, 483, 493, 512, 502, 494, 501, 551, 516, 526, 493, 478, 514, 523, 496, 556, 492, 484, 524, 461, 529, 527, 497, 513, 540, 545, 509, 531, 516, 495, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[499, 531, 492, 504, 497, 470, 535, 503, 486, 500, 539, 536, 542, 527, 475, 476, 497, 480, 529, 537, 496, 526, 547, 464, 540, 511, 540, 519, 540, 582, 491, 524, 487, 470, 504, 526, 535, 516, 525, 498, 528, 504, 519, 499, 526, 516, 541, 516, 515, 519, 501, 513, 495, 499, 490, 503, 490, 519, 524, 494, 503, 545, 514, 521, 519, 505, 498, 531, 481, 518, 530, 521, 503, 518, 479, 505, 529, 520, 515, 510, 510, 513, 474, 495, 537, 512, 511, 485, 558, 509, 543, 524, 496, 519, 527, 487, 495, 535, 505, 496, 527, 512, 499, 526, 462, 496, 519, 527, 532, 526, 521, 530, 500, 520, 531, 541, 483, 503, 520, 532, 482, 487, 545, 447, 514, 511, 526, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[466, 494, 543, 553, 504, 550, 452, 546, 510, 516, 506, 524, 522, 513, 527, 522, 508, 507, 499, 543, 518, 536, 503, 538, 515, 502, 557, 507, 504, 474, 490, 508, 493, 473, 463, 510, 527, 519, 526, 508, 480, 499, 480, 549, 552, 525, 495, 466, 504, 500, 487, 502, 529, 512, 519, 533, 512, 522, 490, 503, 499, 541, 534, 500, 520, 517, 518, 494, 505, 525, 524, 546, 571, 529, 540, 514, 519, 518, 479, 507, 476, 483, 544, 546, 519, 519, 472, 509, 507, 518, 520, 528, 490, 511, 478, 537, 516, 510, 447, 497, 507, 529, 536, 503, 500, 512, 498, 486, 548, 562, 537, 492, 540, 530, 531, 501, 502, 518, 525, 496, 496, 483, 526, 505, 474, 536, 492, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[469, 498, 521, 519, 487, 500, 487, 529, 532, 475, 514, 529, 530, 552, 481, 542, 561, 478, 489, 521, 486, 553, 535, 501, 528, 519, 520, 536, 508, 536, 513, 515, 486, 510, 527, 486, 504, 487, 513, 512, 548, 491, 491, 488, 514, 506, 526, 486, 495, 523, 528, 536, 530, 495, 537, 555, 480, 498, 510, 517, 475, 485, 531, 505, 512, 522, 529, 548, 503, 544, 496, 525, 518, 499, 505, 496, 496, 520, 517, 546, 489, 578, 557, 520, 552, 521, 497, 508, 525, 544, 506, 513, 538, 511, 511, 498, 503, 474, 525, 511, 529, 536, 470, 539, 511, 490, 489, 461, 514, 503, 496, 478, 525, 507, 468, 490, 470, 482, 546, 522, 519, 521, 520, 516, 532, 525, 473, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[498, 512, 524, 533, 531, 534, 503, 519, 512, 508, 529, 491, 532, 502, 548, 485, 516, 507, 490, 470, 513, 509, 552, 508, 516, 475, 535, 513, 480, 513, 475, 542, 491, 547, 510, 521, 529, 526, 510, 516, 527, 525, 496, 529, 531, 516, 528, 505, 532, 531, 495, 561, 526, 519, 511, 492, 517, 489, 494, 492, 508, 509, 542, 557, 500, 521, 492, 492, 512, 518, 510, 482, 484, 510, 504, 496, 507, 499, 527, 513, 509, 494, 511, 502, 537, 509, 486, 507, 508, 503, 524, 500, 491, 506, 521, 487, 508, 529, 493, 532, 519, 533, 527, 503, 480, 483, 536, 527, 523, 519, 512, 509, 497, 528, 523, 463, 515, 561, 552, 511, 499, 480, 503, 558, 501, 499, 487, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[500, 515, 551, 479, 518, 546, 486, 480, 520, 526, 505, 488, 506, 523, 544, 509, 498, 524, 510, 503, 500, 531, 508, 490, 528, 494, 494, 507, 509, 464, 496, 500, 524, 502, 461, 537, 530, 515, 550, 494, 486, 505, 485, 518, 464, 497, 481, 512, 537, 531, 514, 532, 521, 552, 527, 495, 514, 477, 458, 527, 475, 482, 571, 485, 508, 505, 519, 506, 519, 507, 521, 541, 513, 478, 479, 529, 579, 535, 556, 540, 513, 518, 530, 490, 504, 527, 497, 528, 525, 480, 480, 562, 564, 534, 508, 441, 493, 493, 489, 528, 516, 499, 527, 530, 540, 536, 499, 494, 546, 524, 504, 545, 532, 482, 529, 529, 511, 493, 495, 530, 539, 496, 538, 529, 508, 481, 516, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[482, 530, 512, 524, 503, 461, 493, 522, 502, 545, 524, 487, 490, 514, 521, 486, 486, 487, 506, 464, 486, 494, 472, 504, 502, 506, 539, 550, 537, 490, 559, 521, 523, 516, 522, 478, 532, 514, 489, 513, 478, 515, 532, 536, 513, 474, 500, 528, 492, 510, 493, 558, 490, 527, 491, 490, 531, 518, 509, 542, 489, 508, 536, 550, 581, 514, 489, 489, 520, 522, 516, 473, 517, 502, 533, 470, 502, 539, 514, 542, 533, 487, 526, 512, 484, 484, 536, 488, 568, 526, 515, 552, 536, 506, 537, 497, 490, 516, 526, 514, 528, 519, 532, 533, 455, 525, 505, 485, 481, 526, 535, 507, 542, 510, 536, 487, 518, 503, 531, 487, 540, 519, 507, 493, 502, 521, 541, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[477, 548, 533, 505, 515, 505, 519, 500, 478, 532, 497, 473, 469, 504, 488, 511, 501, 470, 522, 526, 510, 529, 503, 493, 531, 486, 532, 483, 508, 463, 532, 512, 497, 498, 538, 518, 477, 540, 551, 543, 511, 513, 541, 551, 522, 481, 536, 569, 540, 515, 468, 499, 519, 472, 524, 500, 502, 531, 529, 530, 514, 505, 498, 520, 527, 480, 497, 512, 525, 477, 515, 482, 497, 500, 494, 488, 562, 475, 508, 542, 482, 486, 493, 533, 594, 509, 519, 547, 541, 527, 529, 536, 537, 507, 506, 458, 526, 528, 530, 524, 529, 497, 487, 536, 508, 546, 526, 485, 558, 491, 505, 520, 529, 478, 483, 528, 547, 492, 503, 497, 500, 505, 513, 520, 504, 499, 497, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[455, 506, 458, 529, 512, 498, 555, 508, 491, 480, 526, 515, 502, 544, 514, 487, 25, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 527, 521, 496, 496, 489, 514, 519, 506, 508, 539, 542, 524, 526, 479, 512, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 523, 573, 502, 533, 510, 521, 533, 491, 547, 488, 487, 532, 543, 487, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 527, 473, 519, 550, 483, 522, 509, 551, 472, 497, 532, 517, 524, 504, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 499, 527, 512, 497, 536, 529, 532, 499, 520, 510, 492, 515, 519, 507, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 461, 489, 512, 523, 494, 509, 532, 498, 520, 517, 479, 492, 530, 485, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 490, 483, 493, 512, 502, 494, 501, 551, 516, 526, 493, 478, 514, 523, 496, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 492, 484, 524, 461, 529, 527, 497, 513, 540, 545, 509, 531, 516, 495, 502, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8211, 8332, 8206, 8219, 8086, 8135, 8227] avg=8192.0000 max=8332 max_rank=2 min=8086 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[499, 531, 492, 504, 497, 470, 535, 503, 486, 500, 539, 536, 542, 527, 475, 476, 24, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 480, 529, 537, 496, 526, 547, 464, 540, 511, 540, 519, 540, 582, 491, 524, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 470, 504, 526, 535, 516, 517, 498, 528, 504, 519, 499, 526, 516, 541, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 519, 501, 513, 495, 499, 490, 503, 490, 519, 524, 494, 503, 545, 505, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 498, 531, 481, 518, 530, 521, 503, 518, 479, 505, 505, 520, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 474, 495, 537, 512, 511, 485, 535, 509, 543, 524, 496, 519, 527, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 535, 505, 496, 527, 512, 499, 526, 462, 496, 519, 527, 532, 526, 521, 530, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 520, 531, 541, 483, 503, 520, 532, 482, 487, 545, 447, 514, 511, 526, 514, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8339, 8202, 8136, 8158, 8177, 8210, 8162] avg=8192.0000 max=8339 max_rank=1 min=8136 min_rank=3 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[466, 494, 543, 553, 504, 550, 452, 546, 510, 516, 506, 524, 522, 513, 527, 522, 27, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 499, 543, 518, 536, 503, 538, 515, 502, 557, 507, 504, 474, 490, 508, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 473, 463, 510, 527, 519, 512, 508, 480, 499, 480, 549, 552, 525, 495, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 500, 487, 502, 529, 512, 519, 533, 512, 522, 490, 503, 499, 541, 514, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 518, 494, 505, 525, 524, 546, 571, 529, 540, 514, 492, 518, 479, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 483, 544, 546, 519, 519, 472, 509, 479, 518, 520, 528, 490, 511, 478, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 447, 497, 507, 529, 536, 503, 500, 512, 498, 486, 548, 562, 537, 492, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 530, 531, 501, 502, 518, 525, 496, 496, 483, 526, 505, 474, 536, 492, 509, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8233, 8051, 8167, 8299, 8129, 8184, 8180] avg=8192.0000 max=8299 max_rank=4 min=8051 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[469, 498, 521, 519, 487, 500, 487, 529, 532, 475, 514, 529, 530, 552, 481, 542, 23, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 478, 489, 521, 486, 553, 535, 501, 528, 519, 520, 536, 508, 536, 513, 515, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 510, 527, 486, 504, 487, 505, 512, 548, 491, 491, 488, 514, 506, 526, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 528, 536, 530, 495, 537, 555, 480, 498, 510, 517, 475, 485, 522, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 522, 529, 548, 503, 544, 496, 525, 518, 499, 505, 496, 473, 520, 517, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 578, 557, 520, 552, 521, 497, 508, 503, 544, 506, 513, 538, 511, 511, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 474, 525, 511, 529, 536, 470, 539, 511, 490, 489, 461, 514, 503, 496, 478, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 507, 468, 490, 470, 482, 546, 522, 519, 521, 520, 516, 532, 525, 473, 508, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8315, 8067, 8191, 8253, 8346, 8031, 8130] avg=8192.0000 max=8346 max_rank=5 min=8031 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[498, 512, 524, 533, 531, 534, 503, 519, 512, 508, 529, 491, 532, 502, 548, 485, 23, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 490, 470, 513, 509, 552, 508, 516, 475, 535, 513, 480, 513, 475, 542, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 547, 510, 521, 529, 526, 502, 516, 527, 525, 496, 529, 531, 516, 528, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 531, 495, 561, 526, 519, 511, 492, 517, 489, 494, 492, 508, 509, 533, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 521, 492, 492, 512, 518, 510, 482, 484, 510, 504, 496, 484, 499, 527, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 511, 502, 537, 509, 486, 507, 487, 503, 524, 500, 491, 506, 521, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 529, 493, 532, 519, 533, 527, 503, 480, 483, 536, 527, 523, 519, 512, 509, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 528, 523, 463, 515, 561, 552, 511, 499, 480, 503, 558, 501, 499, 487, 507, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8299, 8129, 8299, 8266, 8044, 8074, 8235, 8190] avg=8192.0000 max=8299 max_rank=0 min=8044 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[482, 530, 512, 524, 503, 461, 493, 522, 502, 545, 524, 487, 490, 514, 521, 486, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 487, 506, 464, 486, 494, 472, 504, 502, 506, 539, 550, 537, 490, 559, 521, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 522, 478, 532, 514, 482, 513, 478, 515, 532, 536, 513, 474, 500, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 510, 493, 558, 490, 527, 491, 490, 531, 518, 509, 542, 489, 508, 527, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 514, 489, 489, 520, 522, 516, 473, 517, 502, 533, 470, 479, 539, 514, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 487, 526, 512, 484, 484, 536, 488, 545, 526, 515, 552, 536, 506, 537, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 516, 526, 514, 528, 519, 532, 533, 455, 525, 505, 485, 481, 526, 535, 507, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 536, 487, 518, 503, 531, 487, 540, 519, 507, 493, 502, 521, 541, 516, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8119, 8156, 8225, 8200, 8264, 8178, 8259] avg=8192.0000 max=8264 max_rank=5 min=8119 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 515, 551, 479, 518, 546, 486, 480, 520, 526, 505, 488, 506, 523, 544, 509, 27, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 510, 503, 500, 531, 508, 490, 528, 494, 494, 507, 509, 464, 496, 500, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 502, 461, 537, 530, 515, 541, 494, 486, 505, 485, 518, 464, 497, 481, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 531, 514, 532, 521, 552, 527, 495, 514, 477, 458, 527, 475, 482, 561, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 505, 519, 506, 519, 507, 521, 541, 513, 478, 479, 529, 552, 535, 556, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 530, 490, 504, 527, 497, 528, 503, 480, 480, 562, 564, 534, 508, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 489, 528, 516, 499, 527, 530, 540, 536, 499, 494, 546, 524, 504, 545, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 482, 529, 529, 511, 493, 495, 530, 539, 496, 538, 529, 508, 481, 516, 518, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8073, 8052, 8188, 8308, 8179, 8265, 8233] avg=8192.0000 max=8308 max_rank=4 min=8052 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32985 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[477, 548, 533, 505, 515, 505, 519, 500, 478, 532, 497, 473, 469, 504, 488, 511, 26, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 470, 522, 526, 510, 529, 503, 493, 531, 486, 532, 483, 508, 463, 532, 512, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 538, 518, 477, 540, 542, 543, 511, 513, 541, 551, 522, 481, 536, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 515, 468, 499, 519, 472, 524, 500, 502, 531, 529, 530, 514, 505, 489, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 480, 497, 512, 525, 477, 515, 482, 497, 500, 494, 488, 536, 475, 508, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 486, 493, 533, 594, 509, 519, 547, 518, 527, 529, 536, 537, 507, 506, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 530, 524, 529, 497, 487, 536, 508, 546, 526, 485, 558, 491, 505, 520, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 478, 483, 528, 547, 492, 503, 497, 500, 505, 513, 520, 504, 499, 497, 553, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8117, 8377, 8157, 8055, 8281, 8298, 8155] avg=8192.0000 max=8377 max_rank=2 min=8055 min_rank=4 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4352, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4352, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[503, 511, 543, 504, 527, 512, 502, 523, 531, 523, 520, 484, 538, 499, 514, 509, 460, 469, 482, 526, 539, 523, 483, 532, 550, 486, 513, 550, 509, 470, 554, 500, 559, 524, 511, 529, 505, 503, 498, 488, 515, 523, 495, 484, 496, 497, 489, 535, 522, 481, 524, 517, 550, 521, 555, 488, 513, 523, 469, 537, 506, 503, 501, 488, 523, 511, 531, 456, 521, 513, 470, 470, 516, 505, 542, 507, 504, 496, 535, 479, 514, 514, 512, 540, 496, 503, 486, 515, 516, 501, 534, 523, 508, 510, 507, 505, 510, 536, 504, 472, 489, 556, 533, 511, 494, 533, 501, 546, 533, 527, 497, 509, 558, 507, 490, 508, 529, 513, 496, 498, 517, 565, 550, 513, 534, 519, 491, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[533, 555, 512, 540, 521, 505, 499, 481, 471, 524, 543, 557, 509, 477, 513, 510, 504, 534, 494, 513, 506, 521, 538, 514, 516, 494, 509, 497, 539, 506, 504, 519, 530, 505, 503, 511, 534, 497, 534, 513, 524, 514, 546, 542, 499, 535, 496, 558, 505, 533, 530, 529, 489, 474, 570, 503, 502, 510, 543, 476, 514, 542, 511, 507, 511, 506, 515, 525, 488, 474, 526, 500, 471, 487, 510, 504, 523, 511, 516, 530, 507, 521, 500, 528, 552, 535, 500, 502, 522, 529, 515, 512, 523, 466, 475, 498, 537, 473, 520, 467, 517, 502, 499, 556, 507, 498, 544, 479, 516, 512, 485, 522, 490, 508, 470, 522, 568, 469, 520, 519, 507, 473, 514, 477, 537, 518, 477, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[493, 503, 494, 498, 533, 508, 543, 507, 506, 540, 500, 481, 501, 532, 508, 556, 518, 495, 508, 538, 530, 538, 491, 496, 502, 516, 472, 542, 519, 529, 520, 489, 515, 482, 515, 469, 504, 487, 502, 541, 499, 497, 497, 469, 516, 523, 505, 545, 505, 513, 498, 550, 512, 480, 511, 546, 521, 494, 519, 529, 506, 507, 488, 526, 525, 530, 511, 502, 558, 493, 521, 534, 513, 522, 545, 508, 536, 539, 487, 469, 509, 492, 516, 537, 524, 537, 519, 521, 482, 503, 484, 509, 515, 507, 478, 502, 510, 505, 534, 492, 504, 528, 520, 518, 537, 521, 535, 490, 507, 523, 525, 534, 487, 538, 504, 517, 500, 525, 504, 541, 511, 536, 507, 494, 475, 509, 492, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[497, 536, 505, 563, 509, 526, 499, 496, 500, 499, 521, 517, 537, 494, 529, 535, 490, 506, 516, 489, 534, 467, 547, 527, 495, 493, 481, 486, 532, 522, 555, 538, 495, 513, 516, 555, 506, 512, 512, 497, 511, 512, 524, 529, 517, 490, 470, 505, 539, 510, 511, 515, 518, 497, 503, 522, 505, 496, 503, 561, 505, 484, 534, 486, 506, 504, 498, 537, 537, 478, 548, 532, 507, 532, 494, 529, 499, 500, 503, 465, 513, 513, 504, 527, 517, 472, 514, 484, 508, 501, 456, 537, 492, 512, 550, 533, 515, 499, 518, 499, 520, 549, 512, 496, 500, 530, 527, 517, 530, 513, 548, 495, 489, 542, 543, 502, 496, 519, 510, 485, 501, 496, 502, 511, 507, 544, 528, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[499, 499, 518, 553, 520, 529, 473, 553, 518, 533, 479, 544, 516, 563, 524, 486, 506, 497, 520, 493, 505, 521, 511, 494, 520, 485, 503, 508, 535, 513, 580, 499, 499, 524, 539, 530, 496, 470, 527, 529, 528, 502, 508, 531, 519, 536, 508, 493, 511, 490, 536, 521, 494, 536, 509, 549, 514, 522, 501, 566, 507, 497, 484, 537, 478, 520, 521, 517, 524, 546, 522, 494, 509, 485, 489, 516, 480, 484, 515, 478, 501, 524, 520, 561, 510, 515, 496, 475, 504, 533, 509, 548, 493, 506, 510, 498, 514, 523, 514, 509, 491, 505, 490, 507, 499, 538, 503, 495, 524, 494, 523, 508, 489, 540, 508, 508, 526, 502, 528, 497, 484, 495, 511, 515, 492, 526, 471, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[552, 512, 497, 538, 482, 520, 510, 488, 507, 532, 484, 523, 479, 500, 534, 511, 534, 474, 495, 487, 506, 493, 509, 492, 530, 529, 514, 498, 539, 510, 530, 539, 564, 515, 512, 521, 520, 543, 499, 524, 536, 508, 486, 488, 529, 527, 467, 472, 508, 525, 516, 504, 486, 542, 501, 524, 500, 511, 537, 509, 497, 523, 496, 544, 506, 472, 519, 496, 521, 509, 572, 554, 500, 497, 522, 508, 511, 546, 516, 506, 504, 542, 522, 508, 495, 504, 476, 530, 538, 508, 490, 493, 514, 523, 533, 497, 526, 524, 494, 476, 568, 528, 499, 526, 504, 548, 477, 492, 513, 473, 504, 533, 504, 494, 518, 504, 489, 500, 476, 469, 499, 478, 519, 551, 514, 567, 558, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[510, 512, 498, 501, 498, 483, 460, 521, 488, 494, 481, 519, 523, 531, 496, 543, 532, 481, 468, 537, 536, 511, 518, 535, 512, 516, 510, 499, 510, 518, 488, 485, 517, 513, 469, 519, 518, 507, 524, 526, 479, 494, 522, 513, 498, 508, 519, 471, 559, 515, 484, 528, 479, 524, 535, 494, 533, 516, 517, 526, 563, 514, 534, 519, 503, 478, 497, 506, 536, 497, 495, 510, 493, 544, 508, 513, 546, 538, 499, 531, 519, 525, 505, 504, 502, 529, 539, 510, 474, 506, 532, 509, 523, 519, 525, 513, 532, 496, 513, 515, 508, 508, 522, 503, 500, 512, 531, 524, 504, 533, 503, 529, 522, 488, 538, 498, 573, 502, 514, 515, 530, 517, 489, 519, 489, 484, 518, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[531, 549, 559, 465, 496, 518, 507, 544, 501, 482, 509, 498, 523, 488, 559, 505, 521, 527, 505, 476, 492, 478, 499, 525, 509, 550, 513, 472, 527, 509, 533, 471, 502, 529, 511, 532, 498, 494, 519, 536, 489, 511, 501, 513, 483, 515, 491, 481, 462, 536, 505, 519, 552, 518, 504, 491, 514, 509, 509, 546, 511, 522, 563, 568, 506, 499, 521, 504, 453, 500, 495, 499, 478, 525, 478, 510, 512, 471, 512, 523, 523, 506, 544, 527, 543, 466, 488, 498, 499, 509, 526, 522, 521, 529, 543, 532, 541, 526, 503, 505, 548, 500, 544, 549, 490, 532, 509, 518, 510, 476, 511, 494, 518, 538, 531, 509, 489, 498, 542, 513, 511, 488, 514, 548, 495, 532, 495, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[503, 489, 543, 504, 527, 512, 502, 523, 531, 523, 520, 484, 538, 499, 514, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 469, 482, 526, 539, 523, 483, 532, 550, 486, 513, 550, 509, 470, 554, 500, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 524, 511, 529, 505, 503, 498, 488, 515, 523, 495, 484, 496, 497, 489, 535, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 524, 517, 550, 521, 555, 488, 513, 523, 469, 480, 506, 503, 501, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 531, 456, 521, 513, 470, 470, 516, 505, 542, 507, 504, 496, 535, 479, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 512, 540, 496, 503, 486, 515, 516, 501, 534, 523, 508, 510, 507, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 504, 472, 489, 556, 533, 511, 494, 508, 501, 546, 533, 527, 497, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 507, 490, 508, 529, 513, 496, 498, 517, 565, 550, 513, 534, 519, 491, 496, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8166, 8170, 8141, 8127, 8185, 8226, 8300] avg=8192.0000 max=8300 max_rank=7 min=8127 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[533, 531, 512, 540, 521, 505, 499, 481, 471, 524, 543, 557, 509, 477, 513, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 534, 494, 513, 506, 521, 538, 514, 516, 494, 509, 497, 539, 506, 504, 519, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 503, 511, 534, 497, 534, 513, 524, 514, 546, 542, 499, 535, 496, 558, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 533, 530, 529, 489, 474, 570, 503, 502, 510, 543, 425, 514, 542, 511, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 515, 525, 488, 474, 526, 500, 471, 487, 510, 504, 523, 511, 516, 530, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 521, 500, 528, 552, 535, 500, 502, 522, 529, 515, 512, 523, 466, 475, 498, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 473, 520, 467, 517, 502, 499, 556, 507, 474, 544, 479, 516, 512, 485, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 508, 470, 522, 568, 469, 520, 519, 507, 473, 514, 477, 537, 518, 477, 514, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8227, 8359, 8187, 8140, 8186, 8110, 8101] avg=8192.0000 max=8359 max_rank=2 min=8101 min_rank=7 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 467, 518, 553, 520, 529, 473, 553, 518, 533, 479, 544, 516, 563, 524, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 497, 520, 493, 505, 521, 511, 494, 520, 485, 503, 508, 535, 513, 580, 499, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 539, 530, 496, 470, 527, 529, 528, 502, 508, 531, 519, 536, 508, 493, 19, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 490, 536, 521, 494, 536, 509, 549, 514, 522, 501, 497, 507, 497, 484, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 520, 521, 517, 524, 546, 522, 494, 509, 485, 489, 516, 480, 484, 515, 478, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 524, 520, 561, 510, 515, 496, 475, 504, 533, 509, 548, 493, 506, 510, 498, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 523, 514, 509, 491, 505, 490, 507, 499, 503, 503, 495, 524, 494, 523, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 540, 508, 508, 526, 502, 528, 497, 484, 495, 511, 515, 492, 526, 471, 516, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8220, 8268, 8205, 8133, 8205, 8102, 8128] avg=8192.0000 max=8275 max_rank=0 min=8102 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[531, 525, 559, 465, 496, 518, 507, 544, 501, 482, 509, 498, 523, 488, 559, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 505, 476, 492, 478, 499, 525, 509, 550, 513, 472, 527, 509, 533, 471, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 529, 511, 532, 498, 494, 519, 536, 489, 511, 501, 513, 483, 515, 491, 481, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 536, 505, 519, 552, 518, 504, 491, 514, 509, 509, 488, 511, 522, 563, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 499, 521, 504, 453, 500, 495, 499, 478, 525, 478, 510, 512, 471, 512, 523, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 506, 544, 527, 543, 466, 488, 498, 499, 509, 526, 522, 521, 529, 543, 532, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 526, 503, 505, 548, 500, 544, 549, 490, 507, 509, 518, 510, 476, 511, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 538, 531, 509, 489, 498, 542, 513, 511, 488, 514, 548, 495, 532, 495, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8127, 8124, 8271, 8035, 8277, 8231, 8261] avg=8192.0000 max=8277 max_rank=5 min=8035 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 490, 498, 501, 498, 483, 460, 521, 488, 494, 481, 519, 523, 531, 496, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 481, 468, 537, 536, 511, 518, 535, 512, 516, 510, 499, 510, 518, 488, 485, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 513, 469, 519, 518, 507, 524, 526, 479, 494, 522, 513, 498, 508, 519, 471, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 515, 484, 528, 479, 524, 535, 494, 533, 516, 517, 470, 563, 514, 534, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 478, 497, 506, 536, 497, 495, 510, 493, 544, 508, 513, 546, 538, 499, 531, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 525, 505, 504, 502, 529, 539, 510, 474, 506, 532, 509, 523, 519, 525, 513, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 496, 513, 515, 508, 508, 522, 503, 500, 487, 531, 524, 504, 533, 503, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 488, 538, 498, 573, 502, 514, 515, 530, 517, 489, 519, 489, 484, 518, 528, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 8176, 8116, 8284, 8241, 8235, 8208, 8240] avg=8192.0000 max=8284 max_rank=3 min=8036 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[493, 481, 494, 498, 533, 508, 543, 507, 506, 540, 500, 481, 501, 532, 508, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 495, 508, 538, 530, 538, 491, 496, 502, 516, 472, 542, 519, 529, 520, 489, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 482, 515, 469, 504, 487, 502, 541, 499, 497, 497, 469, 516, 523, 505, 545, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 498, 550, 512, 480, 511, 546, 521, 494, 519, 473, 506, 507, 488, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 530, 511, 502, 558, 493, 521, 534, 513, 522, 545, 508, 536, 539, 487, 469, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 492, 516, 537, 524, 537, 519, 521, 482, 503, 484, 509, 515, 507, 478, 502, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 505, 534, 492, 504, 528, 520, 518, 537, 496, 535, 490, 507, 523, 525, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 538, 504, 517, 500, 525, 504, 541, 511, 536, 507, 494, 475, 509, 492, 508, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8223, 8085, 8149, 8340, 8136, 8258, 8164] avg=8192.0000 max=8340 max_rank=4 min=8085 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[552, 490, 497, 538, 482, 520, 510, 488, 507, 532, 484, 523, 479, 500, 534, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 474, 495, 487, 506, 493, 509, 492, 530, 529, 514, 498, 539, 510, 530, 539, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 515, 512, 521, 520, 543, 499, 524, 536, 508, 486, 488, 529, 527, 467, 472, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 516, 504, 486, 542, 501, 524, 500, 511, 537, 455, 497, 523, 496, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 472, 519, 496, 521, 509, 572, 554, 500, 497, 522, 508, 511, 546, 516, 506, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 542, 522, 508, 495, 504, 476, 530, 538, 508, 490, 493, 514, 523, 533, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 524, 494, 476, 568, 528, 499, 526, 504, 521, 477, 492, 513, 473, 504, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 494, 518, 504, 489, 500, 476, 469, 499, 478, 519, 551, 514, 567, 558, 497, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8199, 8231, 8169, 8301, 8178, 8158, 8153] avg=8192.0000 max=8301 max_rank=4 min=8147 min_rank=0 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[497, 513, 505, 563, 509, 526, 499, 496, 500, 499, 521, 517, 537, 494, 529, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 506, 516, 489, 534, 467, 547, 527, 495, 493, 481, 486, 532, 522, 555, 538, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 513, 516, 555, 506, 512, 512, 497, 511, 512, 524, 529, 517, 490, 470, 505, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 510, 511, 515, 518, 497, 503, 522, 505, 496, 503, 502, 505, 484, 534, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 504, 498, 537, 537, 478, 548, 532, 507, 532, 494, 529, 499, 500, 503, 465, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 504, 527, 517, 472, 514, 484, 508, 501, 456, 537, 492, 512, 550, 533, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 518, 499, 520, 549, 512, 496, 500, 505, 527, 517, 530, 513, 548, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 542, 543, 502, 496, 519, 510, 485, 501, 496, 502, 511, 507, 544, 528, 497, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8198, 8183, 8130, 8219, 8134, 8243, 8189] avg=8192.0000 max=8243 max_rank=6 min=8130 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4352, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 3840, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[465, 497, 522, 454, 504, 529, 519, 514, 533, 476, 513, 527, 506, 530, 547, 502, 540, 475, 518, 498, 571, 521, 542, 522, 531, 494, 506, 495, 514, 519, 508, 539, 506, 491, 469, 475, 498, 518, 502, 522, 488, 520, 511, 534, 510, 528, 544, 523, 496, 537, 498, 491, 481, 516, 512, 504, 527, 504, 504, 520, 506, 524, 451, 507, 533, 541, 506, 539, 509, 492, 540, 485, 532, 501, 518, 503, 478, 534, 548, 523, 535, 521, 539, 492, 541, 533, 525, 541, 538, 523, 476, 528, 517, 530, 490, 496, 503, 527, 467, 544, 517, 516, 487, 498, 500, 530, 491, 537, 535, 527, 513, 495, 543, 525, 490, 496, 528, 471, 517, 486, 526, 484, 493, 553, 519, 488, 484, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[493, 553, 530, 507, 475, 524, 542, 499, 536, 560, 506, 532, 513, 471, 495, 502, 528, 534, 508, 493, 513, 521, 515, 470, 551, 506, 504, 477, 506, 497, 519, 518, 541, 503, 489, 509, 484, 484, 488, 512, 486, 497, 523, 517, 494, 528, 518, 504, 510, 521, 504, 514, 510, 512, 491, 520, 499, 537, 467, 517, 513, 488, 466, 558, 509, 503, 508, 499, 504, 503, 507, 513, 547, 497, 525, 487, 520, 500, 496, 529, 548, 527, 487, 544, 495, 544, 517, 532, 491, 522, 521, 486, 528, 510, 531, 502, 485, 514, 524, 519, 524, 557, 505, 522, 548, 526, 533, 551, 519, 537, 474, 523, 489, 493, 519, 476, 535, 492, 519, 501, 557, 497, 495, 497, 499, 529, 562, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[511, 504, 517, 518, 484, 486, 543, 524, 510, 498, 489, 537, 484, 466, 519, 531, 527, 520, 501, 516, 522, 515, 544, 499, 504, 517, 491, 516, 525, 492, 517, 543, 489, 495, 517, 507, 511, 510, 511, 486, 499, 497, 517, 525, 528, 505, 533, 498, 498, 552, 517, 510, 486, 500, 554, 520, 544, 479, 478, 515, 546, 556, 509, 508, 506, 517, 522, 509, 515, 535, 519, 481, 516, 540, 495, 505, 498, 520, 531, 530, 496, 511, 511, 533, 503, 502, 502, 518, 501, 480, 515, 514, 551, 507, 528, 502, 448, 522, 503, 533, 499, 494, 486, 538, 500, 480, 547, 507, 527, 499, 489, 554, 499, 534, 512, 485, 525, 532, 528, 503, 537, 516, 523, 525, 486, 482, 490, 550] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[530, 554, 491, 537, 509, 470, 533, 518, 496, 526, 501, 512, 482, 522, 536, 526, 524, 498, 489, 563, 541, 548, 497, 524, 560, 452, 492, 538, 490, 537, 491, 498, 509, 518, 547, 537, 507, 537, 503, 507, 476, 481, 511, 513, 499, 509, 507, 488, 494, 491, 479, 536, 505, 516, 516, 530, 543, 500, 529, 504, 502, 534, 503, 519, 463, 523, 532, 518, 555, 505, 493, 468, 506, 500, 495, 523, 534, 528, 506, 491, 539, 547, 511, 484, 477, 514, 547, 477, 535, 525, 487, 477, 524, 487, 536, 523, 495, 511, 524, 516, 471, 531, 521, 518, 492, 550, 476, 497, 520, 485, 500, 518, 530, 532, 501, 505, 493, 534, 505, 522, 540, 528, 484, 536, 492, 527, 510, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[514, 535, 477, 567, 486, 515, 536, 510, 517, 488, 511, 513, 488, 558, 492, 486, 524, 508, 508, 510, 552, 538, 543, 525, 522, 532, 513, 536, 520, 530, 470, 545, 521, 476, 530, 537, 514, 525, 492, 467, 518, 509, 506, 499, 538, 496, 507, 523, 500, 483, 495, 524, 504, 499, 509, 498, 534, 529, 507, 543, 500, 543, 524, 468, 492, 567, 513, 528, 495, 495, 480, 521, 529, 509, 490, 508, 520, 533, 493, 484, 510, 508, 509, 526, 485, 506, 508, 500, 511, 494, 512, 552, 519, 492, 516, 525, 519, 512, 518, 445, 500, 533, 490, 518, 518, 498, 479, 478, 511, 503, 492, 520, 504, 507, 553, 527, 496, 517, 511, 510, 521, 537, 504, 528, 506, 522, 519, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[492, 482, 543, 503, 515, 500, 517, 477, 536, 494, 484, 554, 499, 526, 499, 477, 468, 507, 519, 501, 485, 510, 495, 481, 510, 498, 525, 544, 530, 507, 520, 529, 524, 479, 507, 517, 545, 528, 519, 497, 495, 522, 502, 524, 526, 483, 504, 523, 477, 510, 524, 515, 504, 522, 533, 483, 492, 507, 520, 483, 528, 516, 504, 568, 507, 553, 519, 516, 486, 536, 516, 492, 563, 498, 510, 480, 525, 513, 509, 517, 509, 494, 555, 565, 518, 532, 498, 505, 506, 488, 533, 530, 545, 504, 491, 513, 495, 520, 539, 506, 495, 506, 470, 526, 515, 536, 537, 538, 519, 536, 510, 522, 531, 493, 553, 485, 521, 459, 494, 511, 492, 551, 509, 519, 559, 499, 469, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[510, 522, 540, 499, 547, 506, 500, 479, 510, 507, 528, 535, 483, 504, 515, 480, 523, 483, 532, 516, 506, 504, 514, 494, 493, 533, 556, 530, 543, 512, 540, 484, 537, 505, 514, 496, 504, 499, 511, 512, 520, 499, 484, 513, 537, 487, 507, 496, 530, 491, 530, 513, 522, 527, 520, 521, 520, 527, 470, 506, 505, 547, 548, 527, 513, 519, 519, 542, 491, 506, 528, 515, 508, 496, 500, 491, 529, 514, 513, 527, 526, 514, 477, 495, 497, 530, 477, 487, 466, 513, 501, 499, 494, 502, 498, 499, 524, 523, 527, 494, 487, 493, 536, 544, 507, 555, 518, 532, 524, 513, 474, 513, 473, 533, 506, 532, 522, 516, 503, 493, 513, 514, 532, 511, 506, 537, 532, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[528, 526, 531, 476, 495, 519, 525, 516, 489, 532, 525, 509, 530, 558, 538, 537, 532, 495, 526, 517, 534, 518, 524, 541, 565, 498, 519, 493, 530, 517, 535, 565, 526, 526, 489, 497, 500, 529, 515, 537, 524, 516, 516, 529, 509, 502, 467, 491, 484, 497, 511, 548, 481, 532, 507, 507, 514, 461, 478, 519, 470, 546, 501, 493, 501, 505, 539, 489, 527, 476, 543, 494, 484, 507, 513, 513, 510, 501, 512, 516, 541, 474, 518, 455, 486, 532, 515, 526, 460, 543, 520, 529, 548, 487, 540, 534, 461, 478, 530, 528, 467, 498, 497, 517, 531, 525, 507, 514, 548, 503, 538, 494, 552, 494, 510, 492, 496, 534, 509, 502, 458, 520, 515, 517, 519, 499, 487, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[465, 497, 522, 454, 504, 529, 519, 514, 533, 476, 513, 527, 506, 530, 547, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 475, 518, 498, 571, 521, 542, 522, 460, 494, 506, 495, 514, 519, 508, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 491, 469, 475, 498, 518, 502, 522, 488, 520, 511, 534, 510, 528, 544, 523, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 537, 498, 491, 481, 516, 512, 504, 527, 504, 504, 520, 506, 524, 451, 507, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 541, 506, 539, 509, 492, 540, 485, 532, 501, 518, 503, 478, 534, 548, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 521, 539, 492, 541, 533, 525, 541, 538, 523, 476, 528, 508, 530, 490, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 527, 467, 544, 517, 516, 487, 498, 500, 530, 491, 537, 535, 527, 513, 495, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 525, 490, 496, 528, 471, 517, 486, 526, 484, 493, 553, 519, 488, 484, 491, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8222, 8188, 8097, 8282, 8316, 8190, 8103] avg=8192.0000 max=8316 max_rank=5 min=8097 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 553, 530, 507, 475, 524, 542, 499, 536, 560, 506, 532, 513, 471, 495, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 534, 508, 493, 513, 521, 515, 470, 477, 506, 504, 477, 506, 497, 519, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 503, 489, 509, 484, 484, 488, 512, 486, 497, 523, 517, 494, 528, 518, 504, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 504, 514, 510, 512, 491, 520, 499, 537, 467, 517, 513, 488, 466, 558, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 503, 508, 499, 504, 503, 507, 513, 547, 497, 525, 487, 520, 500, 496, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 527, 487, 544, 495, 544, 517, 532, 491, 522, 521, 486, 519, 510, 531, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 514, 524, 519, 524, 557, 505, 522, 548, 526, 533, 551, 519, 537, 474, 523, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 493, 519, 476, 535, 492, 519, 501, 557, 497, 495, 497, 499, 529, 562, 481, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8086, 8128, 8147, 8147, 8276, 8364, 8150] avg=8192.0000 max=8364 max_rank=6 min=8086 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32937 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 554, 491, 537, 509, 470, 533, 518, 496, 526, 501, 512, 482, 522, 536, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 498, 489, 563, 541, 548, 497, 524, 484, 452, 492, 538, 490, 537, 491, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 547, 537, 507, 537, 503, 507, 476, 481, 511, 513, 499, 509, 507, 488, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 491, 479, 536, 505, 516, 516, 530, 543, 500, 529, 504, 502, 534, 503, 519, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 523, 532, 518, 555, 505, 493, 468, 506, 500, 495, 523, 534, 528, 506, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 547, 511, 484, 477, 514, 547, 477, 535, 525, 487, 477, 515, 487, 536, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 511, 524, 516, 471, 531, 521, 518, 492, 550, 476, 497, 520, 485, 500, 518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 532, 501, 505, 493, 534, 505, 522, 540, 528, 484, 536, 492, 527, 510, 507, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8166, 8201, 8222, 8140, 8181, 8128, 8255] avg=8192.0000 max=8255 max_rank=7 min=8128 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 535, 477, 567, 486, 515, 536, 510, 517, 488, 511, 513, 488, 558, 492, 486, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 508, 508, 510, 552, 538, 543, 525, 439, 532, 513, 536, 520, 530, 470, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 476, 530, 537, 514, 525, 492, 467, 518, 509, 506, 499, 538, 496, 507, 523, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 483, 495, 524, 504, 499, 509, 498, 534, 529, 507, 543, 500, 543, 524, 468, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 557, 513, 528, 495, 495, 480, 521, 529, 509, 490, 508, 520, 533, 493, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 508, 509, 526, 485, 506, 508, 500, 511, 494, 512, 552, 501, 492, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 512, 518, 445, 500, 533, 490, 518, 518, 498, 479, 478, 511, 503, 492, 520, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 553, 527, 496, 517, 511, 510, 521, 537, 504, 528, 506, 522, 519, 523, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8293, 8210, 8183, 8147, 8155, 8046, 8303] avg=8192.0000 max=8303 max_rank=7 min=8046 min_rank=6 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 504, 517, 518, 484, 486, 543, 524, 510, 498, 489, 537, 484, 466, 519, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 520, 501, 516, 522, 515, 544, 499, 436, 517, 491, 516, 525, 492, 517, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 495, 517, 507, 511, 510, 511, 486, 499, 497, 517, 525, 528, 505, 533, 498, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 552, 517, 510, 486, 500, 554, 520, 544, 479, 478, 515, 546, 556, 509, 508, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 522, 509, 515, 535, 519, 481, 516, 540, 495, 505, 498, 520, 531, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 511, 533, 503, 502, 502, 518, 501, 480, 515, 514, 542, 507, 528, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 522, 503, 533, 499, 494, 486, 538, 500, 480, 547, 507, 527, 499, 489, 554, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 512, 485, 525, 532, 528, 503, 537, 516, 523, 525, 486, 482, 490, 550, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8181, 8175, 8290, 8239, 8165, 8129, 8236] avg=8192.0000 max=8290 max_rank=3 min=8121 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[492, 482, 543, 503, 515, 500, 517, 477, 536, 494, 484, 554, 499, 526, 499, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 507, 519, 501, 485, 510, 495, 481, 441, 498, 525, 544, 530, 507, 520, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 507, 517, 545, 528, 519, 497, 495, 522, 502, 524, 526, 483, 504, 523, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 510, 524, 515, 504, 522, 533, 483, 492, 507, 520, 483, 528, 516, 504, 568, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 553, 519, 516, 486, 536, 516, 492, 563, 498, 510, 480, 525, 513, 509, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 555, 565, 518, 532, 498, 505, 506, 488, 533, 530, 536, 504, 491, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 520, 539, 506, 495, 506, 470, 526, 515, 536, 537, 538, 519, 536, 510, 522, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 493, 553, 485, 521, 459, 494, 511, 492, 551, 509, 519, 559, 499, 469, 487, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8060, 8242, 8205, 8240, 8277, 8273, 8141] avg=8192.0000 max=8277 max_rank=5 min=8060 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32931 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[510, 522, 540, 499, 547, 506, 500, 479, 510, 507, 528, 535, 483, 504, 515, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 483, 532, 516, 506, 504, 514, 494, 426, 533, 556, 530, 543, 512, 540, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 505, 514, 496, 504, 499, 511, 512, 520, 499, 484, 513, 537, 487, 507, 496, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 491, 530, 513, 522, 527, 520, 521, 520, 527, 470, 506, 505, 547, 548, 527, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 519, 542, 491, 506, 528, 515, 508, 496, 500, 491, 529, 514, 513, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 477, 495, 497, 530, 477, 487, 466, 513, 501, 499, 486, 502, 498, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 523, 527, 494, 487, 493, 536, 544, 507, 555, 518, 532, 524, 513, 474, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 533, 506, 532, 522, 516, 503, 493, 513, 514, 532, 511, 506, 537, 532, 510, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8196, 8167, 8322, 8211, 7967, 8267, 8241] avg=8192.0000 max=8322 max_rank=3 min=7967 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[528, 526, 531, 476, 495, 519, 525, 516, 489, 532, 525, 509, 530, 558, 538, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 495, 526, 517, 534, 518, 524, 541, 488, 498, 519, 493, 530, 517, 535, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 489, 497, 500, 529, 515, 537, 524, 516, 516, 529, 509, 502, 467, 491, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 497, 511, 548, 481, 532, 507, 507, 514, 461, 478, 519, 470, 546, 501, 493, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 505, 539, 489, 527, 476, 543, 494, 484, 507, 513, 513, 510, 501, 512, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 474, 518, 455, 486, 532, 515, 526, 460, 543, 520, 529, 539, 487, 540, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 478, 530, 528, 467, 498, 497, 517, 531, 525, 507, 514, 548, 503, 538, 494, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 494, 510, 492, 496, 534, 509, 502, 458, 520, 515, 517, 519, 499, 487, 493, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8334, 8332, 8225, 8070, 8130, 8199, 8139, 8107] avg=8192.0000 max=8334 max_rank=0 min=8070 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3712, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 3968, 4224, 4096, 4352, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[539, 542, 469, 525, 494, 508, 458, 537, 488, 524, 480, 511, 485, 477, 483, 518, 487, 516, 507, 480, 571, 514, 488, 505, 514, 500, 474, 500, 504, 536, 469, 513, 522, 534, 507, 470, 518, 528, 536, 517, 488, 523, 495, 484, 509, 533, 529, 502, 510, 544, 507, 534, 516, 508, 532, 505, 506, 527, 529, 487, 494, 532, 525, 486, 523, 487, 496, 509, 519, 519, 521, 539, 519, 549, 515, 479, 480, 476, 483, 514, 556, 535, 522, 509, 510, 542, 517, 530, 532, 529, 513, 511, 548, 534, 498, 562, 516, 517, 506, 526, 518, 511, 539, 522, 515, 502, 512, 513, 548, 503, 518, 498, 500, 501, 526, 517, 538, 503, 500, 531, 537, 474, 497, 520, 480, 518, 485, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[472, 514, 527, 488, 505, 504, 486, 500, 528, 497, 542, 506, 546, 508, 506, 524, 554, 524, 505, 527, 556, 500, 549, 511, 493, 508, 510, 490, 504, 542, 559, 540, 537, 497, 536, 522, 514, 511, 535, 514, 516, 493, 543, 520, 525, 497, 524, 489, 548, 521, 528, 509, 516, 534, 545, 513, 487, 512, 521, 497, 502, 478, 502, 468, 499, 510, 510, 515, 474, 535, 527, 523, 523, 510, 530, 492, 491, 504, 526, 516, 511, 513, 552, 520, 538, 511, 478, 553, 490, 552, 493, 498, 501, 514, 492, 464, 510, 523, 495, 512, 499, 463, 514, 496, 490, 525, 533, 508, 515, 500, 518, 510, 522, 485, 510, 510, 511, 531, 502, 492, 499, 508, 500, 492, 525, 520, 487, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[506, 503, 480, 531, 496, 495, 492, 490, 491, 513, 538, 509, 476, 478, 540, 524, 550, 549, 514, 482, 492, 538, 512, 512, 490, 511, 537, 505, 493, 497, 530, 496, 541, 482, 510, 495, 473, 538, 542, 532, 548, 510, 496, 507, 510, 494, 525, 500, 551, 512, 532, 540, 498, 528, 504, 543, 500, 493, 531, 497, 525, 518, 504, 524, 531, 543, 517, 516, 500, 479, 490, 508, 517, 516, 502, 508, 547, 523, 533, 530, 531, 513, 524, 447, 504, 524, 496, 482, 510, 483, 455, 535, 525, 537, 549, 543, 489, 521, 526, 502, 484, 519, 511, 531, 533, 503, 502, 501, 502, 496, 502, 517, 513, 491, 525, 535, 478, 524, 518, 512, 482, 510, 533, 486, 522, 542, 490, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[463, 527, 521, 541, 513, 507, 533, 534, 538, 487, 466, 502, 528, 529, 497, 487, 513, 535, 540, 558, 514, 519, 519, 503, 551, 511, 508, 519, 513, 526, 541, 535, 493, 482, 507, 496, 515, 516, 528, 507, 521, 519, 516, 513, 539, 510, 494, 516, 552, 446, 527, 479, 498, 503, 541, 514, 516, 523, 525, 493, 506, 524, 562, 518, 485, 499, 515, 526, 543, 524, 470, 527, 516, 516, 575, 470, 535, 485, 531, 504, 519, 488, 516, 508, 498, 490, 542, 516, 480, 517, 466, 495, 478, 482, 504, 508, 493, 517, 534, 520, 476, 503, 525, 520, 544, 521, 500, 528, 542, 497, 526, 477, 571, 507, 501, 464, 506, 507, 519, 539, 497, 498, 516, 500, 482, 491, 496, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[504, 510, 513, 532, 506, 538, 481, 498, 542, 509, 468, 494, 532, 499, 493, 463, 539, 525, 522, 503, 503, 510, 505, 504, 503, 531, 547, 520, 525, 516, 508, 523, 534, 507, 517, 525, 470, 502, 487, 500, 542, 495, 515, 493, 511, 517, 555, 477, 536, 497, 495, 516, 529, 498, 507, 459, 503, 503, 514, 533, 518, 526, 491, 519, 550, 493, 494, 521, 552, 533, 499, 502, 521, 498, 510, 506, 521, 551, 492, 540, 491, 512, 518, 479, 519, 510, 501, 525, 564, 513, 512, 528, 492, 497, 530, 496, 519, 510, 491, 514, 526, 498, 498, 504, 502, 512, 509, 519, 487, 542, 520, 496, 483, 511, 523, 493, 501, 551, 479, 529, 499, 515, 539, 520, 537, 515, 547, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[473, 493, 525, 476, 513, 493, 507, 531, 473, 515, 502, 495, 529, 517, 474, 521, 503, 526, 516, 494, 508, 493, 511, 512, 501, 509, 540, 556, 541, 504, 514, 503, 539, 517, 532, 543, 547, 500, 487, 491, 540, 493, 545, 480, 524, 481, 539, 520, 487, 491, 521, 510, 475, 520, 548, 528, 534, 529, 541, 481, 515, 482, 511, 546, 511, 528, 525, 540, 471, 535, 498, 495, 487, 494, 505, 533, 530, 521, 458, 520, 500, 521, 497, 552, 520, 521, 526, 492, 486, 494, 497, 549, 492, 519, 558, 504, 532, 535, 528, 507, 508, 544, 523, 484, 536, 523, 522, 484, 498, 513, 492, 463, 501, 501, 527, 555, 509, 529, 511, 525, 472, 482, 512, 559, 492, 490, 538, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[466, 516, 494, 495, 545, 548, 524, 544, 550, 540, 525, 538, 507, 494, 481, 528, 506, 516, 514, 552, 480, 498, 523, 551, 554, 512, 477, 498, 480, 534, 542, 539, 550, 513, 524, 524, 458, 535, 512, 529, 515, 499, 541, 501, 473, 484, 539, 507, 511, 513, 529, 506, 510, 514, 505, 468, 494, 529, 496, 500, 550, 510, 542, 485, 534, 491, 536, 509, 497, 497, 538, 502, 526, 493, 522, 561, 488, 513, 503, 517, 473, 548, 527, 499, 526, 507, 510, 521, 512, 498, 512, 528, 506, 489, 541, 501, 520, 501, 467, 512, 519, 555, 509, 470, 534, 518, 519, 533, 473, 535, 511, 514, 526, 513, 508, 466, 503, 516, 496, 469, 494, 509, 524, 532, 500, 457, 474, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[546, 507, 541, 514, 491, 492, 522, 524, 544, 494, 513, 487, 498, 510, 512, 500, 517, 524, 530, 521, 509, 521, 524, 521, 534, 543, 498, 476, 479, 536, 559, 538, 532, 479, 481, 520, 487, 487, 526, 526, 502, 497, 517, 495, 528, 550, 499, 484, 519, 531, 447, 498, 515, 526, 532, 523, 520, 518, 552, 518, 513, 499, 490, 503, 506, 517, 514, 489, 560, 465, 501, 482, 478, 518, 520, 530, 489, 543, 470, 528, 517, 515, 496, 494, 528, 475, 508, 482, 534, 512, 515, 517, 518, 556, 521, 521, 526, 479, 494, 540, 494, 505, 510, 508, 510, 519, 503, 483, 519, 475, 529, 514, 518, 540, 540, 522, 508, 543, 477, 528, 522, 502, 491, 488, 501, 553, 511, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[539, 542, 469, 525, 494, 508, 458, 537, 488, 524, 480, 511, 485, 477, 483, 518, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 516, 507, 480, 571, 514, 488, 505, 514, 500, 474, 500, 504, 536, 398, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 534, 507, 470, 518, 528, 536, 517, 488, 523, 495, 484, 509, 533, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 544, 507, 534, 516, 508, 514, 505, 506, 527, 529, 487, 494, 532, 525, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 487, 496, 509, 519, 519, 521, 539, 519, 549, 512, 479, 480, 476, 483, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 535, 522, 509, 510, 542, 517, 530, 532, 529, 513, 511, 548, 534, 489, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 506, 526, 518, 511, 539, 522, 515, 502, 512, 513, 548, 503, 518, 498, 9, 5, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 501, 526, 517, 538, 503, 500, 531, 537, 474, 497, 520, 480, 518, 485, 516, 16, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8093, 8007, 8190, 8224, 8125, 8439, 8282, 8176] avg=8192.0000 max=8439 max_rank=5 min=8007 min_rank=1 imbalance_ratio=1.030151 max_load_reduction=57097 max_load_reduction_pct=0.871231 moved_to_echo=33022 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[472, 514, 527, 488, 505, 504, 486, 500, 528, 497, 542, 506, 546, 508, 506, 524, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 524, 505, 527, 556, 500, 549, 511, 493, 508, 510, 490, 504, 542, 474, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 536, 522, 514, 511, 535, 514, 516, 493, 543, 520, 525, 497, 524, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 521, 528, 509, 516, 534, 527, 513, 487, 512, 521, 497, 502, 478, 502, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 510, 510, 515, 474, 535, 527, 523, 523, 510, 527, 492, 491, 504, 526, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 552, 520, 538, 511, 478, 553, 490, 552, 493, 498, 501, 514, 483, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 523, 495, 512, 499, 463, 514, 496, 490, 525, 533, 508, 515, 500, 518, 510, 9, 5, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 485, 510, 510, 511, 531, 502, 492, 499, 508, 500, 492, 525, 520, 487, 487, 19, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8287, 8268, 8163, 8182, 8171, 8129, 8117] avg=8192.0000 max=8287 max_rank=1 min=8117 min_rank=7 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[506, 503, 480, 531, 496, 495, 492, 490, 491, 513, 538, 509, 476, 478, 540, 524, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 549, 514, 482, 492, 538, 512, 512, 490, 511, 537, 505, 493, 497, 450, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 482, 510, 495, 473, 538, 542, 532, 548, 510, 496, 507, 510, 494, 525, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 512, 532, 540, 498, 528, 488, 543, 500, 493, 531, 497, 525, 518, 504, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 543, 517, 516, 500, 479, 490, 508, 517, 516, 499, 508, 547, 523, 533, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 513, 524, 447, 504, 524, 496, 482, 510, 483, 455, 535, 525, 537, 539, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 521, 526, 502, 484, 519, 511, 531, 533, 503, 502, 501, 502, 496, 502, 517, 10, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 491, 525, 535, 478, 524, 518, 512, 482, 510, 533, 486, 522, 542, 490, 545, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8128, 8198, 8284, 8257, 8148, 8157, 8240] avg=8192.0000 max=8284 max_rank=3 min=8124 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[463, 527, 521, 541, 513, 507, 533, 534, 538, 487, 466, 502, 528, 529, 497, 487, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 535, 540, 558, 514, 519, 519, 503, 551, 511, 508, 519, 513, 526, 450, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 482, 507, 496, 515, 516, 528, 507, 521, 519, 516, 513, 539, 510, 494, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 446, 527, 479, 498, 503, 518, 514, 516, 523, 525, 493, 506, 524, 562, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 499, 515, 526, 543, 524, 470, 527, 516, 516, 567, 470, 535, 485, 531, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 488, 516, 508, 498, 490, 542, 516, 480, 517, 466, 495, 478, 482, 491, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 534, 520, 476, 503, 525, 520, 544, 521, 500, 528, 542, 497, 526, 477, 13, 11, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 507, 501, 464, 506, 507, 519, 539, 497, 498, 516, 500, 482, 491, 496, 514, 23, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8314, 8161, 8204, 8213, 7994, 8259, 8150] avg=8192.0000 max=8314 max_rank=1 min=7994 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32616 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[504, 510, 513, 532, 506, 538, 481, 498, 542, 509, 468, 494, 532, 499, 493, 463, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 525, 522, 503, 503, 510, 505, 504, 503, 531, 547, 520, 525, 516, 431, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 517, 525, 470, 502, 487, 500, 542, 495, 515, 493, 511, 517, 555, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 497, 495, 516, 529, 498, 491, 459, 503, 503, 514, 533, 518, 526, 491, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 493, 494, 521, 552, 533, 499, 502, 521, 498, 507, 506, 521, 551, 492, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 518, 479, 519, 510, 501, 525, 564, 513, 512, 528, 492, 497, 521, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 491, 514, 526, 498, 498, 504, 502, 512, 509, 519, 487, 542, 520, 496, 9, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 511, 523, 493, 501, 551, 479, 529, 499, 515, 539, 520, 537, 515, 547, 520, 17, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8207, 8142, 8128, 8280, 8178, 8164, 8295] avg=8192.0000 max=8295 max_rank=7 min=8128 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[473, 493, 525, 476, 513, 493, 507, 531, 473, 515, 502, 495, 529, 517, 474, 521, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 516, 494, 508, 493, 511, 512, 501, 509, 540, 556, 541, 504, 437, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 532, 543, 547, 500, 487, 491, 540, 493, 545, 480, 524, 481, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 491, 521, 510, 475, 520, 530, 528, 534, 529, 541, 481, 515, 482, 511, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 528, 525, 540, 471, 535, 498, 495, 487, 494, 502, 533, 530, 521, 458, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 521, 497, 552, 520, 521, 526, 492, 486, 494, 497, 549, 492, 519, 548, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 535, 528, 507, 508, 544, 523, 484, 536, 523, 522, 484, 498, 513, 492, 463, 10, 5, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 527, 555, 509, 529, 511, 525, 472, 482, 512, 559, 492, 490, 538, 497, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8154, 8273, 8201, 8148, 8218, 8211, 8234] avg=8192.0000 max=8273 max_rank=2 min=8097 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[466, 516, 494, 495, 545, 548, 524, 544, 550, 540, 525, 538, 507, 494, 481, 528, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 514, 552, 480, 498, 523, 551, 554, 512, 477, 498, 480, 534, 460, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 513, 524, 524, 458, 535, 512, 529, 515, 499, 541, 501, 473, 484, 539, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 529, 506, 510, 514, 489, 468, 494, 529, 496, 500, 550, 510, 542, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 491, 536, 509, 497, 497, 538, 502, 526, 493, 519, 561, 488, 513, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 548, 527, 499, 526, 507, 510, 521, 512, 498, 512, 528, 506, 489, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 467, 512, 519, 555, 509, 470, 534, 518, 519, 533, 473, 535, 511, 514, 10, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 513, 508, 466, 503, 516, 496, 469, 494, 509, 524, 532, 500, 457, 474, 497, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8359, 8194, 8199, 8146, 8224, 8188, 8208, 8018] avg=8192.0000 max=8359 max_rank=0 min=8018 min_rank=7 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[546, 507, 541, 514, 491, 492, 522, 524, 544, 494, 513, 487, 498, 510, 512, 500, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 524, 530, 521, 509, 521, 524, 521, 534, 543, 498, 476, 479, 536, 474, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 479, 481, 520, 487, 487, 526, 526, 502, 497, 517, 495, 528, 550, 499, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 447, 498, 515, 526, 514, 523, 520, 518, 552, 518, 513, 499, 490, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 514, 489, 560, 465, 501, 482, 478, 518, 517, 530, 489, 543, 470, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 496, 494, 528, 475, 508, 482, 534, 512, 515, 517, 518, 556, 512, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 479, 494, 540, 494, 505, 510, 508, 510, 519, 503, 483, 519, 475, 529, 514, 9, 5, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 540, 540, 522, 508, 543, 477, 528, 522, 502, 491, 488, 501, 553, 511, 526, 19, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8245, 8105, 8186, 8107, 8200, 8126, 8306] avg=8192.0000 max=8306 max_rank=7 min=8105 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 3584, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[515, 539, 498, 473, 489, 552, 482, 517, 518, 536, 480, 531, 498, 510, 567, 483, 518, 526, 510, 516, 511, 503, 485, 499, 527, 509, 509, 513, 506, 536, 555, 513, 485, 477, 530, 501, 531, 526, 519, 547, 514, 535, 498, 502, 487, 474, 521, 535, 533, 531, 502, 504, 536, 493, 517, 523, 505, 487, 537, 547, 512, 495, 485, 477, 499, 512, 514, 521, 513, 500, 513, 519, 513, 529, 514, 482, 492, 527, 516, 506, 519, 499, 510, 469, 515, 552, 517, 492, 527, 495, 524, 518, 545, 485, 502, 475, 497, 535, 516, 503, 515, 506, 503, 503, 551, 514, 551, 512, 526, 538, 510, 521, 489, 488, 468, 538, 504, 519, 490, 470, 488, 532, 510, 518, 553, 519, 516, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[528, 509, 517, 478, 475, 514, 519, 550, 529, 504, 496, 477, 487, 510, 538, 502, 493, 520, 524, 511, 499, 551, 521, 560, 490, 499, 525, 513, 510, 513, 514, 504, 497, 491, 493, 561, 520, 516, 525, 510, 521, 509, 477, 505, 494, 526, 523, 503, 490, 487, 505, 508, 484, 511, 530, 506, 534, 536, 502, 516, 511, 469, 514, 529, 516, 498, 513, 486, 520, 529, 494, 511, 549, 530, 513, 507, 545, 515, 557, 498, 504, 508, 502, 473, 523, 464, 498, 504, 522, 529, 523, 523, 498, 548, 505, 550, 512, 513, 509, 507, 532, 516, 536, 529, 492, 516, 537, 503, 504, 491, 480, 503, 504, 533, 524, 513, 499, 520, 521, 511, 485, 531, 535, 498, 533, 500, 523, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[512, 497, 485, 505, 484, 499, 477, 494, 557, 543, 487, 511, 503, 558, 469, 508, 531, 490, 494, 542, 474, 460, 511, 509, 513, 510, 538, 469, 546, 513, 493, 480, 472, 502, 558, 533, 543, 519, 505, 525, 514, 549, 517, 486, 487, 532, 531, 523, 491, 485, 493, 527, 495, 517, 477, 493, 571, 515, 461, 493, 523, 536, 511, 517, 496, 514, 501, 539, 534, 521, 556, 497, 541, 551, 525, 504, 512, 503, 497, 525, 536, 490, 551, 531, 519, 524, 508, 490, 517, 494, 501, 509, 498, 494, 498, 508, 505, 540, 496, 544, 494, 474, 519, 485, 540, 533, 523, 526, 467, 513, 530, 524, 487, 530, 519, 516, 504, 517, 535, 536, 493, 482, 491, 537, 541, 526, 526, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[505, 489, 519, 525, 522, 481, 474, 533, 539, 540, 498, 539, 521, 529, 516, 490, 510, 542, 503, 506, 511, 478, 494, 477, 532, 482, 539, 487, 537, 527, 539, 512, 491, 523, 520, 513, 531, 522, 497, 498, 483, 510, 554, 540, 505, 521, 519, 528, 496, 501, 523, 545, 482, 493, 492, 510, 489, 496, 481, 530, 505, 514, 480, 520, 506, 546, 497, 504, 541, 481, 491, 493, 518, 489, 464, 564, 529, 514, 522, 478, 518, 514, 490, 519, 501, 499, 525, 518, 503, 483, 511, 524, 501, 524, 516, 502, 520, 499, 505, 551, 550, 531, 502, 503, 530, 551, 524, 573, 510, 508, 498, 546, 575, 513, 459, 524, 528, 510, 510, 534, 507, 475, 498, 476, 532, 463, 532, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[506, 586, 508, 500, 510, 498, 542, 510, 526, 531, 467, 515, 514, 499, 462, 537, 467, 478, 532, 515, 549, 514, 529, 513, 490, 566, 484, 533, 509, 549, 549, 496, 503, 516, 532, 545, 510, 520, 533, 539, 520, 510, 493, 496, 495, 488, 521, 524, 502, 505, 496, 508, 517, 513, 495, 487, 532, 464, 491, 479, 533, 496, 502, 514, 494, 508, 540, 510, 503, 505, 502, 520, 503, 539, 557, 507, 541, 529, 482, 545, 533, 522, 487, 524, 507, 519, 506, 503, 521, 470, 492, 529, 524, 523, 489, 528, 534, 528, 517, 524, 505, 498, 487, 479, 527, 495, 505, 523, 508, 496, 535, 538, 493, 484, 505, 513, 520, 506, 518, 505, 539, 483, 513, 485, 500, 509, 521, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[505, 529, 542, 526, 525, 537, 522, 522, 477, 493, 555, 540, 498, 500, 529, 494, 514, 518, 502, 451, 520, 498, 492, 518, 558, 476, 580, 540, 538, 534, 490, 462, 512, 476, 487, 525, 528, 541, 518, 513, 536, 523, 514, 505, 525, 544, 521, 524, 501, 497, 531, 535, 499, 522, 520, 488, 497, 514, 524, 513, 513, 483, 530, 501, 515, 494, 534, 513, 494, 532, 514, 529, 505, 532, 503, 484, 506, 532, 465, 547, 508, 546, 504, 479, 509, 517, 453, 481, 501, 511, 539, 526, 498, 500, 484, 507, 519, 527, 521, 545, 497, 502, 501, 513, 513, 504, 498, 538, 510, 483, 511, 521, 509, 500, 523, 479, 518, 514, 497, 545, 524, 483, 509, 484, 515, 510, 502, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[531, 496, 483, 545, 499, 505, 557, 467, 476, 519, 508, 546, 476, 493, 522, 528, 528, 493, 527, 516, 520, 514, 461, 523, 509, 497, 499, 476, 507, 534, 514, 533, 529, 540, 536, 497, 496, 555, 566, 500, 499, 551, 495, 485, 510, 532, 509, 515, 485, 495, 506, 516, 543, 504, 512, 495, 498, 482, 540, 491, 533, 513, 494, 525, 539, 491, 534, 522, 513, 520, 540, 502, 496, 495, 509, 521, 477, 488, 495, 475, 515, 486, 503, 478, 503, 539, 540, 495, 497, 536, 511, 503, 507, 523, 546, 497, 499, 522, 509, 506, 512, 457, 531, 491, 515, 486, 492, 511, 522, 521, 508, 496, 498, 501, 531, 550, 519, 548, 548, 517, 538, 548, 540, 502, 509, 517, 552, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[523, 531, 493, 534, 492, 529, 494, 479, 516, 537, 497, 536, 482, 511, 515, 524, 462, 474, 506, 492, 522, 510, 529, 525, 504, 508, 509, 498, 500, 517, 558, 481, 509, 495, 532, 519, 520, 522, 491, 522, 513, 502, 562, 521, 472, 521, 498, 509, 504, 486, 485, 503, 526, 520, 505, 525, 511, 499, 506, 512, 537, 505, 546, 525, 484, 488, 533, 545, 496, 544, 487, 511, 535, 532, 491, 494, 512, 524, 516, 471, 499, 541, 480, 501, 499, 496, 538, 518, 544, 500, 522, 473, 531, 529, 525, 500, 510, 547, 549, 511, 506, 494, 519, 535, 497, 493, 479, 503, 503, 531, 498, 532, 525, 509, 494, 510, 543, 539, 490, 529, 536, 575, 518, 531, 506, 495, 475, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 489, 519, 525, 522, 481, 474, 533, 539, 540, 498, 539, 521, 529, 516, 490, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 542, 503, 506, 511, 478, 494, 477, 532, 482, 539, 487, 537, 527, 539, 512, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 523, 520, 513, 531, 470, 497, 498, 483, 510, 554, 540, 505, 521, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 523, 545, 482, 493, 492, 510, 489, 496, 481, 530, 505, 514, 480, 520, 52, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 546, 497, 504, 541, 481, 491, 493, 518, 475, 464, 564, 529, 514, 522, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 514, 490, 519, 501, 499, 525, 518, 503, 483, 511, 524, 501, 524, 516, 502, 17, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 471, 505, 551, 550, 531, 502, 503, 530, 551, 524, 573, 510, 508, 498, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 513, 459, 524, 528, 510, 510, 534, 507, 475, 498, 476, 512, 463, 532, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8186, 8203, 8120, 8123, 8183, 8373, 8122] avg=8192.0000 max=8373 max_rank=6 min=8120 min_rank=3 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 539, 498, 473, 489, 552, 482, 517, 518, 536, 480, 531, 498, 510, 567, 483, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 526, 510, 516, 511, 503, 485, 499, 527, 509, 509, 513, 506, 536, 555, 513, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 477, 530, 501, 531, 473, 519, 547, 514, 535, 498, 502, 487, 474, 521, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 531, 502, 504, 536, 493, 517, 523, 505, 487, 537, 547, 512, 495, 485, 477, 53, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 512, 514, 521, 513, 500, 513, 519, 513, 513, 514, 482, 492, 527, 516, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 510, 469, 515, 552, 517, 492, 527, 495, 524, 518, 545, 485, 502, 475, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 505, 516, 503, 515, 506, 503, 503, 551, 514, 551, 512, 526, 538, 510, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 488, 468, 538, 504, 519, 490, 470, 488, 532, 510, 518, 533, 519, 516, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8247, 8129, 8249, 8154, 8180, 8271, 8111] avg=8192.0000 max=8271 max_rank=6 min=8111 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[528, 509, 517, 478, 475, 514, 519, 550, 529, 504, 496, 477, 487, 510, 538, 502, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 524, 511, 499, 551, 521, 560, 490, 499, 525, 513, 510, 513, 514, 504, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 491, 493, 561, 520, 464, 525, 510, 521, 509, 477, 505, 494, 526, 523, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 487, 505, 508, 484, 511, 530, 506, 534, 536, 502, 516, 511, 469, 514, 529, 52, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 498, 513, 486, 520, 529, 494, 511, 549, 514, 513, 507, 545, 515, 557, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 508, 502, 473, 523, 464, 498, 504, 522, 529, 523, 523, 498, 548, 505, 550, 17, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 484, 509, 507, 532, 516, 536, 529, 492, 516, 537, 503, 504, 491, 480, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 533, 524, 513, 499, 520, 521, 511, 485, 531, 535, 498, 513, 500, 523, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8258, 8119, 8196, 8265, 8209, 8151, 8198] avg=8192.0000 max=8265 max_rank=4 min=8119 min_rank=2 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[506, 586, 508, 500, 510, 498, 542, 510, 526, 531, 467, 515, 514, 499, 462, 537, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 478, 532, 515, 549, 514, 529, 513, 490, 566, 484, 533, 509, 549, 549, 496, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 532, 545, 510, 463, 533, 539, 520, 510, 493, 496, 495, 488, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 505, 496, 508, 517, 513, 495, 487, 532, 464, 491, 479, 533, 496, 502, 514, 57, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 540, 510, 503, 505, 502, 520, 503, 515, 557, 507, 541, 529, 482, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 522, 487, 524, 507, 519, 506, 503, 521, 470, 492, 529, 524, 523, 489, 528, 22, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 489, 517, 524, 505, 498, 487, 479, 527, 495, 505, 523, 508, 496, 535, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 484, 505, 513, 520, 506, 518, 505, 539, 483, 513, 485, 476, 509, 521, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8292, 8188, 8108, 8261, 8218, 8160, 8088] avg=8192.0000 max=8292 max_rank=1 min=8088 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[512, 497, 485, 505, 484, 499, 477, 494, 557, 543, 487, 511, 503, 558, 469, 508, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 490, 494, 542, 474, 460, 511, 509, 513, 510, 538, 469, 546, 513, 493, 480, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 502, 558, 533, 543, 467, 505, 525, 514, 549, 517, 486, 487, 532, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 485, 493, 527, 495, 517, 477, 493, 571, 515, 461, 493, 523, 536, 511, 517, 52, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 514, 501, 539, 534, 521, 556, 497, 541, 535, 525, 504, 512, 503, 497, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 490, 551, 531, 519, 524, 508, 490, 517, 494, 501, 509, 498, 494, 498, 508, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 496, 544, 494, 474, 519, 485, 540, 533, 523, 526, 467, 513, 530, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 530, 519, 516, 504, 517, 535, 536, 493, 482, 491, 537, 521, 526, 526, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8084, 8244, 8169, 8300, 8204, 8183, 8256] avg=8192.0000 max=8300 max_rank=4 min=8084 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32943 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[531, 496, 483, 545, 499, 505, 557, 467, 476, 519, 508, 546, 476, 493, 522, 528, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 493, 527, 516, 520, 514, 461, 523, 509, 497, 499, 476, 507, 534, 514, 533, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 540, 536, 497, 496, 499, 566, 500, 499, 551, 495, 485, 510, 532, 509, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 495, 506, 516, 543, 504, 512, 495, 498, 482, 540, 491, 533, 513, 494, 525, 56, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 491, 534, 522, 513, 520, 540, 502, 496, 481, 509, 521, 477, 488, 495, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 503, 478, 503, 539, 540, 495, 497, 536, 511, 503, 507, 523, 546, 497, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 509, 506, 512, 457, 531, 491, 515, 486, 492, 511, 522, 521, 508, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 501, 531, 550, 519, 548, 548, 517, 538, 548, 540, 502, 490, 517, 552, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8161, 8259, 8200, 8103, 8213, 8049, 8394] avg=8192.0000 max=8394 max_rank=7 min=8049 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=57142 max_load_reduction_pct=0.871918 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 529, 542, 526, 525, 537, 522, 522, 477, 493, 555, 540, 498, 500, 529, 494, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 518, 502, 451, 520, 498, 492, 518, 558, 476, 580, 540, 538, 534, 490, 462, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 476, 487, 525, 528, 487, 518, 513, 536, 523, 514, 505, 525, 544, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 497, 531, 535, 499, 522, 520, 488, 497, 514, 524, 513, 513, 483, 530, 501, 54, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 494, 534, 513, 494, 532, 514, 529, 505, 516, 503, 484, 506, 532, 465, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 546, 504, 479, 509, 517, 453, 481, 501, 511, 539, 526, 498, 500, 484, 507, 18, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 497, 521, 545, 497, 502, 501, 513, 513, 504, 498, 538, 510, 483, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 523, 479, 518, 514, 497, 545, 524, 483, 509, 484, 496, 510, 502, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8202, 8238, 8234, 8183, 8098, 8173, 8107] avg=8192.0000 max=8301 max_rank=0 min=8098 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[523, 531, 493, 534, 492, 529, 494, 479, 516, 537, 497, 536, 482, 511, 515, 524, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 474, 506, 492, 522, 510, 529, 525, 504, 508, 509, 498, 500, 517, 558, 481, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 532, 519, 520, 470, 491, 522, 513, 502, 562, 521, 472, 521, 498, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 486, 485, 503, 526, 520, 505, 525, 511, 499, 506, 512, 537, 505, 546, 525, 52, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 488, 533, 545, 496, 544, 487, 511, 535, 516, 491, 494, 512, 524, 516, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 541, 480, 501, 499, 496, 538, 518, 544, 500, 522, 473, 531, 529, 525, 500, 18, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 549, 511, 506, 494, 519, 535, 497, 493, 479, 503, 503, 531, 498, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 494, 510, 543, 539, 490, 529, 536, 575, 518, 531, 487, 495, 475, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8106, 8156, 8260, 8147, 8231, 8176, 8260] avg=8192.0000 max=8260 max_rank=3 min=8106 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 3840, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[519, 551, 568, 509, 495, 491, 518, 541, 545, 537, 518, 537, 511, 510, 522, 558, 523, 523, 469, 525, 470, 504, 480, 517, 504, 499, 527, 484, 517, 524, 507, 528, 518, 485, 529, 496, 480, 523, 499, 494, 486, 522, 495, 470, 537, 507, 564, 500, 504, 491, 515, 508, 514, 503, 517, 507, 538, 519, 480, 517, 491, 509, 538, 494, 492, 540, 488, 528, 517, 539, 541, 515, 519, 537, 525, 513, 463, 531, 542, 511, 509, 511, 510, 521, 545, 507, 558, 502, 519, 506, 518, 516, 496, 575, 486, 505, 505, 497, 476, 490, 505, 518, 495, 517, 477, 545, 502, 513, 498, 520, 490, 524, 526, 519, 481, 537, 482, 537, 511, 500, 483, 511, 450, 507, 528, 506, 533, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[472, 531, 508, 509, 537, 512, 501, 500, 497, 485, 539, 567, 513, 532, 492, 509, 514, 479, 550, 474, 479, 491, 534, 516, 504, 556, 534, 488, 519, 470, 498, 526, 542, 543, 463, 487, 456, 509, 483, 488, 520, 484, 502, 492, 493, 480, 543, 519, 522, 541, 540, 500, 529, 530, 520, 481, 515, 516, 527, 531, 526, 522, 486, 502, 523, 537, 531, 527, 573, 493, 537, 509, 480, 484, 520, 524, 522, 501, 524, 507, 530, 562, 514, 537, 510, 491, 508, 538, 531, 495, 506, 512, 521, 538, 520, 519, 534, 495, 464, 511, 510, 546, 520, 520, 536, 523, 512, 526, 489, 519, 529, 496, 483, 481, 488, 493, 484, 517, 510, 502, 551, 509, 493, 480, 504, 529, 502, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[522, 470, 515, 525, 493, 543, 516, 515, 506, 483, 558, 540, 536, 536, 477, 554, 561, 497, 492, 511, 509, 454, 540, 517, 508, 495, 510, 498, 553, 494, 502, 544, 557, 514, 530, 501, 507, 534, 528, 489, 515, 525, 512, 517, 539, 544, 511, 512, 477, 496, 527, 497, 527, 517, 516, 504, 515, 546, 508, 508, 483, 492, 510, 539, 535, 526, 504, 554, 529, 508, 510, 494, 481, 496, 533, 511, 477, 521, 492, 547, 485, 492, 530, 531, 529, 526, 482, 505, 473, 534, 508, 524, 509, 507, 517, 526, 486, 488, 516, 481, 483, 528, 492, 512, 515, 542, 476, 487, 517, 511, 503, 534, 470, 524, 503, 504, 487, 507, 500, 498, 511, 469, 519, 522, 490, 525, 538, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[566, 517, 522, 492, 517, 557, 501, 534, 522, 517, 500, 514, 503, 472, 472, 524, 527, 505, 540, 535, 505, 512, 567, 489, 540, 513, 502, 474, 543, 503, 487, 535, 547, 521, 540, 501, 497, 508, 489, 491, 516, 518, 505, 523, 475, 484, 529, 507, 503, 513, 520, 521, 557, 495, 552, 446, 494, 551, 472, 505, 513, 506, 498, 577, 544, 518, 496, 499, 491, 501, 485, 507, 504, 511, 513, 504, 495, 520, 527, 533, 518, 486, 495, 511, 499, 508, 524, 502, 517, 519, 531, 535, 522, 533, 526, 504, 515, 552, 547, 501, 531, 503, 525, 490, 511, 505, 488, 499, 551, 524, 501, 505, 483, 520, 517, 510, 499, 478, 529, 487, 508, 500, 528, 504, 504, 467, 497, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[476, 485, 506, 509, 521, 526, 575, 495, 482, 511, 513, 546, 515, 548, 507, 510, 535, 516, 497, 507, 492, 522, 493, 534, 541, 468, 465, 501, 478, 516, 495, 517, 501, 489, 477, 509, 510, 515, 489, 531, 520, 500, 553, 521, 492, 515, 499, 524, 538, 489, 469, 518, 492, 509, 495, 507, 491, 485, 486, 490, 532, 490, 538, 498, 532, 504, 535, 507, 504, 506, 516, 532, 520, 524, 550, 493, 507, 536, 538, 502, 562, 490, 547, 512, 520, 528, 529, 529, 518, 506, 545, 506, 534, 538, 487, 552, 505, 521, 537, 533, 501, 500, 506, 473, 556, 514, 507, 496, 496, 490, 514, 526, 496, 532, 554, 524, 512, 463, 492, 473, 533, 541, 495, 475, 540, 518, 485, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[487, 482, 475, 539, 494, 521, 504, 512, 577, 515, 534, 518, 536, 537, 507, 538, 491, 540, 526, 477, 523, 508, 511, 458, 492, 572, 494, 519, 485, 514, 501, 489, 480, 544, 487, 547, 506, 502, 513, 525, 492, 547, 549, 486, 524, 515, 479, 495, 487, 537, 531, 481, 526, 536, 526, 506, 534, 516, 489, 558, 524, 511, 511, 522, 541, 493, 551, 483, 522, 498, 523, 496, 548, 536, 497, 552, 530, 529, 508, 479, 499, 476, 517, 486, 514, 493, 499, 494, 519, 486, 518, 486, 505, 514, 497, 518, 510, 490, 493, 546, 513, 510, 502, 531, 493, 530, 507, 479, 530, 488, 542, 532, 520, 510, 481, 506, 539, 513, 513, 499, 499, 515, 530, 504, 541, 488, 509, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[507, 555, 495, 509, 486, 551, 539, 503, 514, 504, 512, 509, 505, 499, 544, 515, 513, 517, 480, 500, 542, 481, 517, 534, 549, 534, 488, 509, 529, 534, 496, 507, 520, 528, 539, 516, 533, 505, 529, 477, 498, 477, 506, 531, 535, 483, 478, 565, 485, 495, 520, 520, 506, 485, 505, 530, 486, 498, 522, 506, 516, 509, 512, 520, 511, 547, 508, 526, 508, 532, 561, 461, 464, 495, 543, 540, 511, 504, 519, 501, 488, 500, 525, 556, 504, 518, 521, 517, 539, 488, 529, 471, 547, 502, 526, 492, 545, 478, 496, 529, 491, 492, 489, 489, 493, 503, 504, 517, 524, 511, 522, 475, 520, 479, 522, 530, 528, 531, 517, 509, 490, 534, 493, 504, 478, 546, 519, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[486, 543, 496, 483, 513, 523, 487, 535, 499, 497, 513, 490, 499, 518, 539, 530, 496, 504, 544, 540, 514, 521, 492, 508, 502, 497, 505, 482, 523, 559, 494, 493, 540, 520, 490, 505, 487, 501, 487, 504, 536, 543, 510, 505, 507, 513, 474, 521, 510, 551, 507, 518, 501, 498, 514, 539, 518, 507, 503, 560, 507, 497, 507, 507, 484, 493, 512, 495, 479, 546, 505, 500, 545, 545, 505, 496, 542, 503, 547, 495, 508, 541, 541, 538, 540, 525, 512, 513, 520, 487, 519, 506, 494, 498, 542, 492, 507, 532, 497, 514, 473, 508, 519, 513, 507, 494, 497, 482, 489, 520, 519, 498, 493, 518, 487, 552, 522, 526, 472, 525, 497, 520, 523, 548, 522, 525, 562, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[522, 470, 515, 525, 493, 543, 516, 515, 506, 483, 558, 540, 536, 536, 477, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 497, 492, 511, 509, 454, 540, 517, 508, 495, 510, 498, 553, 494, 502, 544, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 514, 530, 501, 507, 534, 528, 489, 515, 525, 512, 517, 539, 544, 511, 512, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 496, 527, 497, 527, 517, 516, 504, 515, 546, 508, 508, 483, 492, 510, 539, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 526, 504, 554, 529, 508, 510, 494, 481, 496, 533, 511, 477, 521, 441, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 492, 530, 531, 529, 526, 482, 505, 473, 534, 508, 524, 509, 459, 517, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 488, 516, 481, 483, 528, 492, 512, 515, 542, 476, 487, 517, 511, 503, 534, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 524, 503, 504, 487, 507, 500, 498, 511, 469, 519, 522, 490, 525, 538, 531, 36, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8215, 8364, 8166, 8167, 8130, 8125, 8146] avg=8192.0000 max=8364 max_rank=2 min=8125 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[476, 485, 506, 509, 521, 526, 575, 495, 482, 511, 513, 546, 515, 548, 507, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 497, 507, 492, 522, 493, 534, 541, 468, 465, 501, 478, 516, 495, 517, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 489, 477, 509, 510, 515, 489, 531, 520, 500, 553, 521, 492, 515, 499, 524, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 469, 518, 492, 509, 495, 507, 491, 485, 486, 490, 532, 490, 538, 498, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 535, 507, 504, 506, 516, 532, 520, 524, 550, 493, 507, 536, 483, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 490, 547, 512, 520, 528, 529, 529, 518, 506, 545, 506, 534, 488, 487, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 537, 533, 501, 500, 506, 473, 556, 514, 507, 496, 496, 490, 514, 526, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 532, 554, 524, 512, 463, 492, 473, 533, 541, 495, 475, 540, 518, 485, 545, 39, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8109, 8175, 8031, 8251, 8353, 8225, 8228] avg=8192.0000 max=8353 max_rank=5 min=8031 min_rank=3 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=33057 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[519, 551, 568, 509, 495, 491, 518, 541, 545, 537, 518, 537, 511, 510, 522, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 469, 525, 470, 504, 480, 517, 504, 499, 527, 484, 517, 524, 507, 528, 23, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 485, 529, 496, 480, 523, 499, 494, 486, 522, 495, 470, 537, 507, 564, 500, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 491, 515, 508, 514, 503, 517, 507, 538, 519, 480, 517, 491, 509, 538, 494, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 540, 488, 528, 517, 539, 541, 515, 519, 537, 525, 513, 463, 531, 478, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 511, 510, 521, 545, 507, 558, 502, 519, 506, 518, 516, 496, 505, 486, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 497, 476, 490, 505, 518, 495, 517, 477, 545, 502, 513, 498, 520, 490, 524, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 519, 481, 537, 482, 537, 511, 500, 483, 511, 450, 507, 528, 506, 533, 487, 44, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8355, 8144, 8143, 8154, 8237, 8214, 8130, 8159] avg=8192.0000 max=8355 max_rank=0 min=8130 min_rank=6 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[566, 517, 522, 492, 517, 557, 501, 534, 522, 517, 500, 514, 503, 472, 472, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 505, 540, 535, 505, 512, 567, 489, 540, 513, 502, 474, 543, 503, 487, 535, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 521, 540, 501, 497, 508, 489, 491, 516, 518, 505, 523, 475, 484, 529, 507, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 520, 521, 557, 495, 552, 446, 494, 551, 472, 505, 513, 506, 498, 577, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 518, 496, 499, 491, 501, 485, 507, 504, 511, 513, 504, 495, 520, 473, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 495, 511, 499, 508, 524, 502, 517, 519, 531, 535, 522, 483, 526, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 552, 547, 501, 531, 503, 525, 490, 511, 505, 488, 499, 551, 524, 501, 505, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 520, 517, 510, 499, 478, 529, 487, 508, 500, 528, 504, 504, 467, 497, 498, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8309, 8181, 8227, 8094, 8180, 8299, 8079] avg=8192.0000 max=8309 max_rank=1 min=8079 min_rank=7 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[472, 531, 508, 509, 537, 512, 501, 500, 497, 485, 539, 567, 513, 532, 492, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 479, 550, 474, 479, 491, 534, 516, 504, 556, 534, 488, 519, 470, 498, 526, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 543, 463, 487, 456, 509, 483, 488, 520, 484, 502, 492, 493, 480, 543, 519, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 541, 540, 500, 529, 530, 520, 481, 515, 516, 527, 531, 526, 522, 486, 502, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 537, 531, 527, 573, 493, 537, 509, 480, 484, 520, 524, 522, 501, 471, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 562, 514, 537, 510, 491, 508, 538, 531, 495, 506, 512, 521, 488, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 495, 464, 511, 510, 546, 520, 520, 536, 523, 512, 526, 489, 519, 529, 496, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 481, 488, 493, 484, 517, 510, 502, 551, 509, 493, 480, 504, 529, 502, 528, 38, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8163, 8034, 8292, 8239, 8282, 8280, 8103] avg=8192.0000 max=8292 max_rank=3 min=8034 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 555, 495, 509, 486, 551, 539, 503, 514, 504, 512, 509, 505, 499, 544, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 517, 480, 500, 542, 481, 517, 534, 549, 534, 488, 509, 529, 534, 496, 507, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 539, 516, 533, 505, 529, 477, 498, 477, 506, 531, 535, 483, 478, 565, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 495, 520, 520, 506, 485, 505, 530, 486, 498, 522, 506, 516, 509, 512, 520, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 547, 508, 526, 508, 532, 561, 461, 464, 495, 543, 540, 511, 504, 466, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 500, 525, 556, 504, 518, 521, 517, 539, 488, 529, 471, 547, 455, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 478, 496, 529, 491, 492, 489, 489, 493, 503, 504, 517, 524, 511, 522, 475, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 479, 522, 530, 528, 531, 517, 509, 490, 534, 493, 504, 478, 546, 519, 512, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8260, 8248, 8119, 8178, 8176, 8108, 8262] avg=8192.0000 max=8262 max_rank=7 min=8108 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[487, 482, 475, 539, 494, 521, 504, 512, 577, 515, 534, 518, 536, 537, 507, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 540, 526, 477, 523, 508, 511, 458, 492, 572, 494, 519, 485, 514, 501, 489, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 544, 487, 547, 506, 502, 513, 525, 492, 547, 549, 486, 524, 515, 479, 495, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 537, 531, 481, 526, 536, 526, 506, 534, 516, 489, 558, 524, 511, 511, 522, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 493, 551, 483, 522, 498, 523, 496, 548, 536, 497, 552, 530, 529, 456, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 476, 517, 486, 514, 493, 499, 494, 519, 486, 518, 486, 505, 465, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 493, 546, 513, 510, 502, 531, 493, 530, 507, 479, 530, 488, 542, 532, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 510, 481, 506, 539, 513, 513, 499, 499, 515, 530, 504, 541, 488, 509, 504, 37, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8131, 8220, 8299, 8234, 7972, 8248, 8220] avg=8192.0000 max=8299 max_rank=3 min=7972 min_rank=5 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[486, 543, 496, 483, 513, 523, 487, 535, 499, 497, 513, 490, 499, 518, 539, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 504, 544, 540, 514, 521, 492, 508, 502, 497, 505, 482, 523, 559, 494, 493, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 520, 490, 505, 487, 501, 487, 504, 536, 543, 510, 505, 507, 513, 474, 521, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 551, 507, 518, 501, 498, 514, 539, 518, 507, 503, 560, 507, 497, 507, 507, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 493, 512, 495, 479, 546, 505, 500, 545, 545, 505, 496, 542, 503, 491, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 541, 541, 538, 540, 525, 512, 513, 520, 487, 519, 506, 494, 451, 542, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 532, 497, 514, 473, 508, 519, 513, 507, 494, 497, 482, 489, 520, 519, 498, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 518, 487, 552, 522, 526, 472, 525, 497, 520, 523, 548, 522, 525, 562, 495, 40, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8205, 8171, 8248, 8136, 8229, 8121, 8339] avg=8192.0000 max=8339 max_rank=7 min=8087 min_rank=0 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 3840, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 3840, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[516, 500, 503, 509, 565, 537, 506, 486, 536, 525, 501, 503, 519, 514, 511, 516, 558, 517, 497, 507, 503, 518, 518, 474, 527, 484, 508, 532, 560, 528, 517, 545, 516, 481, 502, 462, 487, 501, 544, 494, 515, 482, 541, 521, 492, 502, 480, 498, 529, 549, 531, 521, 495, 553, 508, 503, 489, 475, 474, 479, 546, 488, 533, 529, 497, 485, 475, 546, 525, 533, 516, 488, 504, 517, 478, 502, 502, 514, 517, 551, 534, 540, 489, 511, 504, 507, 514, 478, 489, 522, 513, 531, 553, 534, 527, 510, 507, 530, 518, 487, 530, 494, 512, 498, 478, 542, 514, 518, 538, 519, 509, 534, 510, 516, 470, 491, 521, 499, 536, 512, 533, 484, 537, 491, 477, 502, 534, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[526, 534, 526, 516, 500, 523, 542, 533, 494, 521, 501, 537, 509, 501, 494, 521, 517, 494, 469, 539, 517, 515, 487, 508, 521, 517, 491, 513, 475, 513, 521, 517, 517, 507, 504, 480, 499, 554, 505, 529, 533, 525, 512, 494, 490, 523, 512, 488, 550, 542, 544, 481, 524, 536, 542, 501, 513, 506, 525, 530, 508, 504, 525, 493, 463, 511, 489, 540, 543, 470, 563, 517, 518, 539, 533, 480, 495, 523, 516, 525, 504, 516, 537, 536, 513, 512, 452, 494, 506, 482, 487, 480, 547, 488, 531, 496, 519, 495, 504, 507, 460, 474, 514, 494, 568, 556, 515, 499, 501, 485, 525, 485, 482, 550, 525, 490, 506, 495, 532, 458, 558, 499, 484, 537, 546, 557, 508, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[514, 486, 524, 555, 516, 527, 549, 487, 520, 484, 524, 495, 467, 532, 492, 514, 547, 512, 530, 478, 504, 533, 493, 498, 536, 501, 511, 523, 510, 517, 500, 474, 531, 483, 460, 486, 508, 510, 541, 536, 540, 490, 506, 506, 506, 502, 496, 503, 505, 535, 530, 538, 518, 491, 544, 495, 503, 512, 497, 529, 493, 538, 509, 529, 522, 542, 508, 525, 517, 484, 511, 519, 499, 537, 565, 489, 476, 556, 481, 526, 472, 477, 518, 515, 494, 513, 482, 526, 501, 509, 494, 516, 487, 560, 531, 508, 542, 517, 473, 489, 509, 496, 512, 504, 494, 496, 514, 528, 506, 489, 508, 499, 493, 502, 533, 541, 525, 518, 521, 563, 484, 527, 522, 518, 544, 510, 527, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[491, 511, 555, 489, 506, 522, 506, 509, 532, 482, 475, 474, 489, 512, 515, 514, 543, 537, 516, 477, 505, 510, 496, 517, 529, 553, 522, 537, 517, 501, 517, 505, 525, 508, 517, 501, 504, 504, 510, 480, 490, 532, 547, 535, 514, 501, 480, 519, 523, 497, 538, 543, 527, 495, 489, 509, 537, 496, 527, 527, 523, 504, 500, 555, 531, 526, 504, 491, 494, 528, 503, 498, 495, 500, 562, 522, 483, 529, 524, 514, 503, 517, 515, 527, 492, 492, 495, 518, 519, 497, 499, 495, 527, 516, 521, 512, 530, 537, 536, 516, 503, 525, 478, 488, 485, 504, 538, 535, 519, 493, 523, 494, 545, 480, 460, 486, 496, 530, 521, 534, 526, 536, 509, 456, 495, 542, 540, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[494, 525, 499, 503, 540, 499, 521, 570, 499, 491, 528, 504, 486, 506, 527, 485, 537, 547, 470, 519, 492, 536, 506, 528, 522, 491, 513, 500, 523, 529, 514, 556, 496, 512, 511, 523, 490, 523, 485, 529, 558, 502, 504, 521, 509, 546, 566, 492, 513, 485, 526, 502, 485, 482, 521, 526, 561, 492, 499, 481, 512, 486, 514, 497, 510, 506, 490, 519, 501, 460, 518, 524, 497, 548, 513, 494, 540, 555, 548, 460, 536, 498, 489, 512, 517, 488, 498, 541, 538, 532, 501, 532, 529, 491, 537, 521, 495, 477, 541, 501, 563, 487, 485, 498, 508, 509, 535, 514, 531, 533, 525, 517, 484, 501, 474, 486, 504, 523, 518, 511, 490, 507, 483, 500, 520, 497, 530, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[552, 484, 520, 513, 508, 568, 502, 515, 491, 523, 518, 530, 534, 526, 515, 522, 520, 546, 496, 488, 491, 491, 501, 516, 483, 503, 504, 550, 523, 491, 521, 479, 532, 488, 557, 481, 521, 510, 502, 534, 486, 487, 520, 525, 525, 489, 514, 511, 478, 516, 503, 527, 502, 501, 507, 525, 531, 534, 469, 488, 520, 488, 508, 500, 494, 492, 530, 538, 498, 538, 502, 526, 526, 519, 540, 482, 521, 473, 496, 537, 511, 537, 500, 525, 544, 503, 552, 522, 546, 497, 476, 486, 489, 513, 522, 525, 498, 514, 481, 490, 486, 554, 491, 519, 503, 479, 504, 527, 508, 516, 543, 531, 514, 569, 505, 509, 514, 486, 511, 492, 497, 516, 510, 517, 525, 520, 536, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[539, 534, 550, 510, 497, 519, 491, 509, 521, 499, 500, 543, 522, 537, 510, 497, 509, 540, 509, 518, 513, 487, 493, 524, 519, 512, 509, 507, 510, 528, 516, 547, 517, 520, 525, 531, 522, 503, 535, 497, 495, 548, 490, 505, 525, 507, 540, 495, 493, 501, 495, 513, 519, 520, 498, 537, 502, 520, 508, 568, 529, 505, 491, 495, 492, 494, 509, 497, 498, 503, 501, 528, 511, 469, 512, 476, 495, 512, 519, 480, 478, 559, 516, 508, 523, 517, 484, 517, 483, 540, 524, 515, 507, 513, 471, 528, 520, 531, 505, 531, 525, 546, 461, 492, 528, 541, 488, 501, 519, 519, 541, 473, 492, 500, 534, 479, 500, 480, 493, 531, 518, 529, 504, 514, 511, 516, 533, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[479, 567, 508, 500, 464, 466, 553, 536, 517, 516, 478, 500, 486, 556, 515, 521, 525, 475, 497, 523, 564, 492, 524, 525, 508, 483, 477, 525, 531, 530, 550, 505, 467, 478, 498, 483, 460, 516, 509, 518, 527, 514, 499, 533, 472, 539, 545, 491, 510, 506, 540, 539, 499, 514, 513, 517, 475, 526, 500, 480, 531, 510, 500, 514, 502, 501, 554, 502, 509, 508, 501, 498, 517, 490, 522, 496, 485, 490, 518, 512, 540, 503, 510, 552, 525, 518, 509, 492, 548, 513, 540, 486, 505, 492, 516, 529, 572, 537, 486, 524, 515, 517, 496, 509, 465, 528, 535, 503, 525, 525, 530, 513, 546, 519, 507, 492, 533, 505, 546, 538, 529, 521, 517, 471, 485, 546, 496, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 486, 500, 555, 516, 527, 549, 487, 520, 484, 524, 495, 467, 532, 492, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 512, 530, 478, 504, 533, 493, 498, 536, 501, 511, 523, 510, 517, 500, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 483, 460, 486, 508, 510, 541, 536, 540, 490, 506, 506, 506, 502, 496, 503, 24, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 535, 519, 538, 518, 491, 544, 495, 503, 512, 497, 529, 493, 538, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 542, 508, 525, 517, 484, 511, 519, 499, 537, 565, 489, 476, 556, 481, 526, 12, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 477, 518, 513, 494, 513, 482, 526, 501, 509, 494, 516, 487, 560, 531, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 517, 473, 489, 509, 496, 512, 504, 494, 496, 514, 528, 506, 489, 508, 499, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 533, 541, 525, 518, 521, 563, 484, 527, 522, 518, 544, 510, 515, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8145, 8144, 8255, 8281, 8101, 8083, 8365] avg=8192.0000 max=8365 max_rank=7 min=8083 min_rank=6 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[494, 525, 476, 503, 540, 499, 521, 570, 499, 491, 528, 504, 486, 506, 527, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 547, 470, 519, 492, 536, 506, 528, 522, 491, 513, 500, 523, 529, 514, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 511, 523, 490, 523, 485, 529, 558, 502, 504, 521, 509, 546, 566, 492, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 485, 515, 502, 485, 482, 521, 526, 561, 492, 499, 481, 512, 486, 514, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 490, 519, 501, 460, 518, 524, 497, 548, 513, 494, 540, 555, 548, 460, 12, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 489, 510, 517, 488, 498, 541, 538, 532, 501, 532, 529, 491, 537, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 477, 541, 501, 563, 487, 485, 498, 508, 509, 535, 514, 531, 533, 525, 517, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 474, 486, 504, 523, 518, 511, 490, 507, 483, 500, 520, 497, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8261, 8306, 8071, 8207, 8258, 8226, 8053] avg=8192.0000 max=8306 max_rank=2 min=8053 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[526, 534, 502, 516, 500, 523, 542, 533, 494, 521, 501, 537, 509, 501, 494, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 469, 539, 517, 515, 487, 508, 521, 517, 491, 513, 475, 513, 521, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 504, 480, 499, 554, 505, 529, 533, 525, 512, 494, 490, 523, 512, 488, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 542, 533, 481, 524, 536, 542, 501, 513, 506, 525, 530, 508, 504, 525, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 511, 489, 540, 543, 470, 563, 517, 518, 539, 533, 480, 495, 523, 516, 525, 12, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 537, 534, 513, 512, 452, 494, 506, 482, 487, 480, 547, 488, 531, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 495, 504, 507, 460, 474, 514, 494, 568, 556, 515, 499, 501, 485, 525, 485, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 550, 525, 490, 506, 495, 532, 458, 558, 499, 484, 537, 546, 557, 496, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8093, 8211, 8313, 8249, 8079, 8108, 8229] avg=8192.0000 max=8313 max_rank=3 min=8079 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[491, 511, 526, 489, 506, 522, 506, 509, 532, 482, 475, 474, 489, 512, 515, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 516, 477, 505, 510, 496, 517, 529, 553, 522, 537, 517, 501, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 508, 517, 501, 504, 504, 510, 480, 490, 532, 547, 535, 514, 501, 480, 519, 29, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 520, 543, 527, 495, 489, 509, 537, 496, 527, 527, 523, 504, 500, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 526, 504, 491, 494, 528, 503, 498, 495, 500, 562, 522, 483, 529, 524, 514, 17, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 515, 523, 492, 492, 495, 518, 519, 497, 499, 495, 527, 516, 521, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 537, 536, 516, 503, 525, 478, 488, 485, 504, 538, 535, 519, 493, 523, 494, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 480, 460, 486, 496, 530, 521, 534, 526, 536, 509, 456, 495, 542, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 8252, 8215, 8272, 8240, 8141, 8218, 8145] avg=8192.0000 max=8272 max_rank=3 min=8053 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[516, 500, 480, 509, 565, 537, 506, 486, 536, 525, 501, 503, 519, 514, 511, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 517, 497, 507, 503, 518, 518, 474, 527, 484, 508, 532, 560, 528, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 481, 502, 462, 487, 501, 544, 494, 515, 482, 541, 521, 492, 502, 480, 498, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 549, 520, 521, 495, 553, 508, 503, 489, 475, 474, 479, 546, 488, 533, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 485, 475, 546, 525, 533, 516, 488, 504, 517, 478, 502, 502, 514, 517, 551, 12, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 540, 489, 509, 504, 507, 514, 478, 489, 522, 513, 531, 553, 534, 527, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 518, 487, 530, 494, 512, 498, 478, 542, 514, 518, 538, 519, 509, 534, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 470, 491, 521, 499, 536, 512, 533, 484, 537, 491, 477, 502, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8270, 8057, 8191, 8175, 8254, 8235, 8130] avg=8192.0000 max=8270 max_rank=1 min=8057 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[552, 484, 496, 513, 508, 568, 502, 515, 491, 523, 518, 530, 534, 526, 515, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 546, 496, 488, 491, 491, 501, 516, 483, 503, 504, 550, 523, 491, 521, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 488, 557, 481, 521, 510, 502, 534, 486, 487, 520, 525, 525, 489, 514, 511, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 516, 493, 527, 502, 501, 507, 525, 531, 534, 469, 488, 520, 488, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 530, 538, 498, 538, 502, 526, 526, 519, 540, 482, 521, 473, 496, 537, 13, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 537, 500, 523, 544, 503, 552, 522, 546, 497, 476, 486, 489, 513, 522, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 514, 481, 490, 486, 554, 491, 519, 503, 479, 504, 527, 508, 516, 543, 531, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 569, 505, 509, 514, 486, 511, 492, 497, 516, 510, 517, 525, 520, 523, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8082, 8221, 8087, 8236, 8246, 8151, 8216] avg=8192.0000 max=8297 max_rank=0 min=8082 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[539, 534, 525, 510, 497, 519, 491, 509, 521, 499, 500, 543, 522, 537, 510, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 540, 509, 518, 513, 487, 493, 524, 519, 512, 509, 507, 510, 528, 516, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 520, 525, 531, 522, 503, 535, 497, 495, 548, 490, 505, 525, 507, 540, 495, 25, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 501, 485, 513, 519, 520, 498, 537, 502, 520, 508, 568, 529, 505, 491, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 509, 497, 498, 503, 501, 528, 511, 469, 512, 476, 495, 512, 519, 480, 12, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 559, 516, 506, 523, 517, 484, 517, 483, 540, 524, 515, 507, 513, 471, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 531, 505, 531, 525, 546, 461, 492, 528, 541, 488, 501, 519, 519, 541, 473, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 500, 534, 479, 500, 480, 493, 531, 518, 529, 504, 514, 511, 516, 521, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8220, 8295, 8184, 8019, 8181, 8228, 8156] avg=8192.0000 max=8295 max_rank=2 min=8019 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[479, 567, 485, 500, 464, 466, 553, 536, 517, 516, 478, 500, 486, 556, 515, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 475, 497, 523, 564, 492, 524, 525, 508, 483, 477, 525, 531, 530, 550, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 478, 498, 483, 460, 516, 509, 518, 527, 514, 499, 533, 472, 539, 545, 491, 23, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 529, 539, 499, 514, 513, 517, 475, 526, 500, 480, 531, 510, 500, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 554, 502, 509, 508, 501, 498, 517, 490, 522, 496, 485, 490, 518, 512, 12, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 503, 510, 550, 525, 518, 509, 492, 548, 513, 540, 486, 505, 492, 516, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 537, 486, 524, 515, 517, 496, 509, 465, 528, 535, 503, 525, 525, 530, 513, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 519, 507, 492, 533, 505, 546, 538, 529, 521, 517, 471, 485, 546, 484, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8213, 8087, 8163, 8129, 8276, 8287, 8242] avg=8192.0000 max=8287 max_rank=6 min=8087 min_rank=2 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4352, 3968, 4096, 4224, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True + [2026-08-26 16:29:46] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 411.4 | throughput per GPU (TFLOP/s/GPU): 439.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.340202E+01 | loss scale: 1.0 | grad norm: 19.505 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[520, 504, 526, 483, 541, 510, 522, 515, 504, 497, 483, 511, 509, 538, 540, 529, 516, 499, 475, 485, 502, 474, 520, 477, 511, 523, 526, 535, 469, 526, 519, 539, 527, 516, 522, 531, 527, 505, 472, 560, 536, 527, 501, 510, 522, 552, 555, 479, 517, 502, 498, 551, 482, 448, 461, 524, 502, 521, 495, 517, 510, 496, 515, 482, 502, 537, 517, 468, 458, 548, 525, 538, 527, 498, 497, 558, 516, 487, 520, 517, 520, 534, 507, 552, 494, 554, 468, 522, 527, 494, 531, 506, 521, 492, 520, 492, 501, 509, 502, 525, 515, 526, 524, 488, 491, 497, 522, 530, 486, 514, 526, 516, 520, 527, 509, 475, 541, 505, 520, 511, 519, 512, 523, 535, 524, 485, 518, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[498, 499, 520, 523, 511, 540, 490, 504, 493, 529, 525, 509, 493, 531, 502, 506, 553, 564, 528, 498, 496, 493, 527, 475, 520, 540, 489, 473, 545, 511, 465, 485, 524, 525, 507, 532, 543, 493, 467, 510, 488, 493, 468, 529, 494, 513, 530, 510, 507, 517, 498, 483, 540, 538, 484, 486, 529, 510, 506, 519, 530, 496, 486, 536, 504, 511, 518, 526, 526, 507, 485, 512, 479, 552, 506, 516, 491, 522, 512, 504, 525, 522, 514, 499, 544, 523, 512, 501, 516, 489, 496, 505, 561, 499, 489, 523, 517, 533, 526, 551, 524, 518, 509, 508, 528, 478, 552, 536, 485, 544, 514, 508, 507, 504, 525, 525, 516, 499, 525, 547, 518, 513, 482, 530, 520, 498, 479, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[488, 506, 510, 500, 533, 487, 532, 514, 526, 471, 519, 539, 535, 484, 467, 492, 505, 508, 497, 482, 527, 495, 505, 508, 555, 507, 519, 520, 567, 494, 516, 508, 512, 491, 482, 484, 504, 542, 538, 514, 494, 501, 505, 527, 527, 532, 506, 519, 497, 479, 547, 549, 509, 482, 529, 461, 474, 493, 500, 503, 512, 531, 503, 503, 492, 500, 510, 533, 506, 542, 493, 458, 505, 499, 547, 498, 522, 514, 510, 556, 482, 552, 524, 507, 491, 553, 491, 483, 546, 487, 511, 513, 540, 538, 572, 522, 531, 530, 477, 538, 525, 493, 516, 529, 540, 525, 531, 481, 561, 537, 495, 517, 499, 503, 500, 489, 518, 490, 557, 549, 521, 506, 489, 471, 520, 510, 512, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[526, 503, 512, 484, 531, 497, 508, 521, 502, 481, 539, 489, 557, 483, 521, 499, 493, 546, 485, 488, 474, 500, 517, 495, 537, 477, 476, 521, 490, 512, 497, 517, 510, 491, 518, 555, 530, 528, 509, 518, 513, 534, 503, 523, 499, 482, 481, 477, 479, 557, 515, 558, 515, 522, 521, 533, 525, 525, 509, 513, 514, 498, 484, 512, 518, 511, 532, 564, 500, 518, 537, 523, 510, 488, 490, 503, 496, 545, 478, 503, 480, 479, 544, 504, 558, 518, 512, 553, 477, 525, 553, 496, 497, 510, 531, 502, 534, 514, 505, 510, 508, 489, 510, 543, 551, 521, 480, 541, 513, 520, 518, 505, 543, 495, 502, 499, 511, 540, 511, 487, 481, 504, 540, 540, 516, 530, 505, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[540, 497, 513, 512, 475, 504, 511, 522, 519, 533, 497, 514, 476, 501, 521, 524, 525, 511, 545, 526, 520, 513, 549, 521, 494, 517, 510, 510, 537, 529, 505, 544, 478, 543, 522, 516, 545, 523, 527, 526, 486, 515, 489, 518, 492, 498, 462, 528, 452, 506, 527, 528, 531, 489, 507, 501, 497, 524, 537, 487, 534, 516, 455, 502, 498, 525, 514, 525, 497, 489, 532, 525, 520, 497, 533, 545, 522, 523, 493, 539, 490, 497, 484, 537, 521, 524, 517, 497, 521, 527, 485, 527, 493, 473, 504, 475, 538, 492, 523, 496, 514, 524, 520, 500, 533, 493, 514, 479, 502, 520, 546, 488, 540, 495, 555, 507, 464, 510, 495, 532, 517, 502, 517, 518, 514, 519, 497, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[478, 547, 500, 506, 512, 505, 527, 531, 507, 478, 498, 534, 532, 510, 508, 522, 496, 480, 549, 490, 537, 516, 493, 533, 486, 533, 509, 554, 553, 471, 543, 509, 507, 487, 514, 515, 550, 516, 516, 495, 482, 510, 545, 503, 551, 508, 509, 519, 508, 515, 511, 480, 501, 511, 502, 505, 532, 506, 512, 527, 537, 501, 510, 536, 518, 480, 541, 507, 489, 518, 522, 505, 551, 487, 493, 500, 472, 488, 488, 542, 511, 536, 489, 497, 557, 488, 503, 518, 518, 495, 561, 524, 500, 520, 509, 492, 540, 497, 506, 511, 500, 502, 543, 542, 535, 535, 538, 523, 458, 526, 521, 488, 530, 516, 529, 504, 455, 463, 510, 514, 513, 483, 520, 494, 482, 532, 514, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[522, 525, 503, 513, 540, 494, 527, 511, 497, 519, 500, 535, 483, 497, 493, 478, 515, 533, 518, 550, 493, 511, 510, 503, 539, 516, 492, 518, 498, 509, 489, 497, 560, 529, 498, 501, 528, 505, 488, 503, 512, 531, 527, 519, 465, 464, 480, 538, 503, 503, 527, 484, 506, 538, 480, 524, 456, 522, 534, 476, 562, 507, 482, 515, 515, 539, 529, 509, 519, 528, 508, 540, 577, 532, 536, 505, 500, 530, 536, 528, 493, 512, 451, 545, 545, 502, 497, 561, 537, 512, 507, 492, 497, 526, 496, 483, 503, 494, 497, 553, 548, 474, 498, 480, 529, 502, 520, 516, 490, 502, 524, 505, 492, 486, 532, 520, 464, 538, 531, 517, 503, 526, 510, 523, 572, 497, 518, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[515, 547, 503, 528, 481, 511, 519, 536, 522, 510, 492, 530, 538, 464, 523, 515, 529, 489, 502, 513, 529, 487, 529, 508, 513, 535, 501, 533, 525, 519, 502, 493, 509, 501, 537, 509, 500, 541, 514, 514, 510, 488, 519, 544, 542, 537, 520, 473, 548, 485, 526, 547, 517, 510, 540, 511, 518, 493, 480, 449, 517, 535, 510, 503, 506, 536, 531, 527, 510, 541, 488, 500, 480, 497, 500, 526, 503, 521, 512, 504, 507, 505, 500, 548, 522, 498, 484, 542, 536, 484, 521, 511, 488, 508, 497, 489, 536, 494, 480, 531, 492, 528, 515, 510, 486, 494, 533, 503, 503, 500, 511, 528, 514, 508, 526, 490, 503, 507, 552, 523, 495, 533, 529, 518, 525, 483, 510, 466] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[498, 499, 520, 523, 511, 540, 490, 504, 493, 529, 525, 509, 493, 531, 502, 506, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 564, 528, 498, 496, 493, 527, 475, 520, 540, 489, 473, 545, 511, 465, 485, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 507, 532, 533, 493, 467, 510, 488, 493, 468, 529, 494, 513, 530, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 498, 483, 540, 538, 484, 486, 529, 510, 506, 519, 530, 496, 486, 536, 36, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 518, 526, 526, 507, 485, 512, 479, 552, 506, 516, 491, 522, 512, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 514, 499, 538, 523, 512, 501, 516, 489, 496, 505, 561, 499, 489, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 533, 526, 515, 524, 518, 509, 508, 528, 478, 552, 536, 485, 544, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 525, 525, 516, 499, 525, 547, 518, 513, 482, 530, 520, 498, 479, 502, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8166, 8116, 8215, 8143, 8212, 8295, 8198] avg=8192.0000 max=8295 max_rank=6 min=8116 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[488, 506, 510, 500, 533, 487, 532, 514, 526, 471, 519, 539, 535, 484, 467, 492, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 497, 482, 527, 495, 505, 508, 555, 507, 519, 520, 567, 494, 516, 508, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 491, 482, 484, 496, 542, 538, 514, 494, 501, 505, 527, 527, 532, 506, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 479, 547, 549, 509, 482, 529, 461, 474, 493, 500, 503, 512, 531, 503, 503, 35, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 500, 510, 533, 506, 542, 493, 458, 505, 499, 547, 498, 522, 514, 510, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 552, 524, 507, 486, 553, 491, 483, 546, 487, 511, 513, 540, 538, 572, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 530, 477, 503, 525, 493, 516, 529, 540, 525, 531, 481, 561, 537, 495, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 503, 500, 489, 518, 490, 557, 549, 521, 506, 489, 471, 520, 510, 512, 513, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8216, 8170, 8123, 8154, 8307, 8291, 8154] avg=8192.0000 max=8307 max_rank=5 min=8121 min_rank=0 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[520, 504, 526, 483, 541, 510, 522, 515, 504, 497, 483, 511, 509, 538, 540, 529, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 475, 485, 502, 474, 520, 477, 511, 523, 526, 535, 469, 526, 519, 539, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 516, 522, 531, 510, 505, 472, 560, 536, 527, 501, 510, 522, 552, 555, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 498, 551, 482, 448, 461, 524, 502, 521, 495, 517, 510, 496, 515, 482, 38, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 517, 468, 458, 548, 525, 538, 527, 498, 497, 558, 516, 487, 520, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 534, 507, 552, 482, 554, 468, 522, 527, 494, 531, 506, 521, 492, 520, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 502, 487, 515, 526, 524, 488, 491, 497, 522, 530, 486, 514, 526, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 527, 509, 475, 541, 505, 520, 511, 519, 512, 523, 535, 524, 485, 518, 502, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8103, 8325, 8078, 8177, 8222, 8134, 8241] avg=8192.0000 max=8325 max_rank=2 min=8078 min_rank=3 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[540, 497, 513, 512, 475, 504, 511, 522, 519, 533, 497, 514, 476, 501, 521, 524, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 511, 545, 526, 520, 513, 549, 521, 494, 517, 510, 510, 537, 529, 505, 544, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 543, 522, 516, 535, 523, 527, 526, 486, 515, 489, 518, 492, 498, 462, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 506, 527, 528, 531, 489, 507, 501, 497, 524, 537, 487, 534, 516, 455, 502, 33, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 514, 525, 497, 489, 532, 525, 520, 497, 533, 545, 522, 523, 493, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 497, 484, 537, 515, 524, 517, 497, 521, 527, 485, 527, 493, 473, 504, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 492, 523, 463, 514, 524, 520, 500, 533, 493, 514, 479, 502, 520, 546, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 495, 555, 507, 464, 510, 495, 532, 517, 502, 517, 518, 514, 519, 497, 547, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8360, 8158, 8141, 8247, 8066, 8149, 8237] avg=8192.0000 max=8360 max_rank=1 min=8066 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 503, 512, 484, 531, 497, 508, 521, 502, 481, 539, 489, 557, 483, 521, 499, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 546, 485, 488, 474, 500, 517, 495, 537, 477, 476, 521, 490, 512, 497, 517, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 518, 555, 520, 528, 509, 518, 513, 534, 503, 523, 499, 482, 481, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 557, 515, 558, 515, 522, 521, 533, 525, 525, 509, 513, 514, 498, 484, 512, 34, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 532, 564, 500, 518, 537, 523, 510, 488, 490, 503, 496, 545, 478, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 479, 544, 504, 552, 518, 512, 553, 477, 525, 553, 496, 497, 510, 531, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 514, 505, 476, 508, 489, 510, 543, 551, 521, 480, 541, 513, 520, 518, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 495, 502, 499, 511, 540, 511, 487, 481, 504, 540, 540, 516, 530, 505, 486, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8029, 8161, 8328, 8188, 8233, 8228, 8198] avg=8192.0000 max=8328 max_rank=3 min=8029 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[478, 547, 500, 506, 512, 505, 527, 531, 507, 478, 498, 534, 532, 510, 508, 522, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 480, 549, 490, 537, 516, 493, 533, 486, 533, 509, 554, 553, 471, 543, 509, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 514, 515, 540, 516, 516, 495, 482, 510, 545, 503, 551, 508, 509, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 515, 511, 480, 501, 511, 502, 505, 532, 506, 512, 527, 537, 501, 510, 536, 34, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 480, 541, 507, 489, 518, 522, 505, 551, 487, 493, 500, 472, 488, 488, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 536, 489, 497, 551, 488, 503, 518, 518, 495, 561, 524, 500, 520, 509, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 497, 506, 477, 500, 502, 543, 542, 535, 535, 538, 523, 458, 526, 521, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 516, 529, 504, 455, 463, 510, 514, 513, 483, 520, 494, 482, 532, 514, 525, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8256, 8217, 8243, 8071, 8212, 8231, 8092] avg=8192.0000 max=8256 max_rank=1 min=8071 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[522, 525, 503, 513, 540, 494, 527, 511, 497, 519, 500, 535, 483, 497, 493, 478, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 518, 550, 493, 511, 510, 503, 539, 516, 492, 518, 498, 509, 489, 497, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 529, 498, 501, 519, 505, 488, 503, 512, 531, 527, 519, 465, 464, 480, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 527, 484, 506, 538, 480, 524, 456, 522, 534, 476, 562, 507, 482, 515, 36, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 539, 529, 509, 519, 528, 508, 540, 577, 532, 536, 505, 500, 530, 536, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 512, 451, 545, 539, 502, 497, 561, 537, 512, 507, 492, 497, 526, 496, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 494, 497, 517, 548, 474, 498, 480, 529, 502, 520, 516, 490, 502, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 486, 532, 520, 464, 538, 531, 517, 503, 526, 510, 523, 572, 497, 518, 490, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8195, 8139, 8170, 8402, 8150, 8099, 8227] avg=8192.0000 max=8402 max_rank=4 min=8099 min_rank=6 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[515, 547, 503, 528, 481, 511, 519, 536, 522, 510, 492, 530, 538, 464, 523, 515, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 489, 502, 513, 529, 487, 529, 508, 513, 535, 501, 533, 525, 519, 502, 493, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 501, 537, 509, 492, 541, 514, 514, 510, 488, 519, 544, 542, 537, 520, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 485, 526, 547, 517, 510, 540, 511, 518, 493, 480, 449, 517, 535, 510, 503, 35, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 536, 531, 527, 510, 541, 488, 500, 480, 497, 500, 526, 503, 521, 512, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 505, 500, 548, 516, 498, 484, 542, 536, 484, 521, 511, 488, 508, 497, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 494, 480, 496, 492, 528, 515, 510, 486, 494, 533, 503, 503, 500, 511, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 526, 490, 503, 507, 552, 523, 495, 533, 529, 518, 525, 483, 510, 466, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8211, 8250, 8238, 8154, 8134, 8109, 8189] avg=8192.0000 max=8251 max_rank=0 min=8109 min_rank=6 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 3968, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[452, 530, 532, 495, 497, 521, 531, 575, 537, 470, 512, 525, 539, 494, 503, 516, 467, 523, 516, 506, 476, 468, 504, 573, 500, 519, 514, 506, 529, 498, 532, 506, 519, 518, 512, 508, 507, 516, 503, 531, 486, 536, 542, 497, 486, 546, 516, 537, 519, 521, 522, 531, 503, 509, 523, 511, 504, 522, 498, 513, 528, 474, 507, 469, 512, 504, 473, 521, 484, 516, 545, 506, 477, 518, 524, 522, 503, 489, 485, 544, 476, 502, 502, 519, 495, 521, 527, 524, 566, 524, 525, 512, 525, 512, 529, 528, 513, 485, 494, 511, 520, 510, 528, 526, 508, 492, 524, 540, 503, 518, 510, 473, 484, 529, 502, 498, 512, 505, 499, 513, 466, 547, 523, 509, 521, 535, 519, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[511, 540, 532, 515, 475, 495, 525, 541, 496, 486, 490, 511, 517, 501, 537, 517, 535, 507, 539, 466, 498, 500, 505, 533, 535, 545, 530, 551, 481, 531, 478, 558, 509, 479, 495, 542, 499, 499, 501, 554, 502, 499, 494, 492, 509, 494, 509, 514, 537, 483, 517, 523, 528, 555, 524, 498, 488, 474, 505, 510, 499, 518, 490, 507, 527, 449, 524, 516, 522, 548, 549, 519, 518, 530, 478, 479, 530, 514, 514, 526, 540, 515, 489, 518, 506, 477, 498, 497, 504, 501, 505, 501, 520, 512, 545, 533, 521, 506, 497, 529, 534, 506, 538, 496, 514, 524, 490, 533, 505, 554, 487, 541, 526, 506, 499, 483, 478, 527, 516, 525, 504, 522, 525, 560, 458, 516, 487, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[543, 553, 512, 483, 494, 537, 511, 522, 518, 502, 530, 496, 464, 492, 553, 511, 488, 551, 472, 507, 561, 539, 488, 493, 508, 500, 529, 513, 518, 508, 529, 517, 525, 543, 485, 524, 482, 496, 500, 506, 530, 483, 531, 478, 518, 518, 501, 517, 493, 513, 491, 487, 540, 509, 506, 491, 518, 517, 514, 493, 555, 550, 482, 534, 509, 535, 486, 505, 503, 529, 528, 521, 585, 521, 514, 473, 509, 494, 498, 473, 523, 524, 494, 485, 522, 489, 550, 504, 505, 513, 535, 476, 501, 519, 533, 516, 515, 528, 511, 534, 493, 512, 541, 538, 572, 528, 495, 518, 504, 531, 507, 499, 501, 488, 505, 526, 486, 502, 465, 509, 483, 526, 511, 488, 534, 501, 554, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[499, 547, 522, 523, 534, 533, 569, 538, 530, 486, 508, 533, 500, 539, 538, 525, 504, 502, 507, 482, 510, 497, 534, 509, 521, 485, 505, 550, 504, 516, 530, 472, 540, 566, 498, 503, 493, 492, 524, 529, 519, 497, 517, 499, 489, 508, 537, 473, 533, 516, 539, 528, 525, 497, 521, 562, 549, 524, 497, 530, 494, 510, 508, 522, 510, 539, 493, 534, 452, 484, 539, 549, 520, 506, 509, 497, 496, 504, 521, 534, 514, 506, 485, 540, 520, 479, 509, 497, 537, 528, 491, 475, 505, 478, 519, 504, 485, 479, 487, 487, 496, 503, 503, 510, 515, 461, 504, 570, 492, 494, 537, 487, 541, 502, 533, 527, 534, 504, 491, 513, 450, 513, 508, 519, 480, 538, 495, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[510, 503, 513, 523, 514, 499, 551, 487, 506, 485, 529, 516, 507, 569, 509, 557, 505, 559, 522, 521, 510, 485, 519, 482, 524, 487, 490, 478, 495, 545, 504, 481, 487, 529, 537, 468, 519, 564, 505, 485, 528, 524, 493, 507, 532, 523, 524, 511, 533, 485, 532, 493, 519, 474, 522, 497, 526, 513, 520, 472, 499, 550, 504, 572, 474, 563, 475, 518, 490, 511, 535, 504, 511, 474, 527, 496, 507, 483, 494, 523, 518, 486, 569, 509, 477, 535, 518, 483, 565, 528, 476, 498, 527, 559, 507, 539, 470, 501, 501, 523, 561, 483, 499, 488, 500, 515, 496, 487, 521, 466, 506, 534, 482, 499, 495, 522, 501, 528, 533, 511, 530, 516, 509, 515, 565, 515, 538, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[531, 544, 519, 502, 520, 528, 487, 504, 547, 497, 467, 562, 505, 502, 529, 499, 501, 502, 504, 535, 482, 480, 522, 495, 498, 559, 505, 459, 497, 558, 521, 534, 530, 517, 517, 516, 536, 530, 517, 509, 499, 504, 483, 507, 501, 507, 525, 486, 512, 549, 498, 524, 479, 499, 486, 529, 498, 491, 517, 564, 496, 491, 519, 506, 491, 513, 484, 482, 550, 538, 514, 507, 526, 501, 482, 489, 557, 512, 493, 483, 480, 522, 501, 486, 506, 520, 476, 491, 510, 564, 534, 494, 508, 524, 536, 511, 512, 505, 515, 538, 465, 531, 520, 505, 522, 514, 491, 491, 546, 531, 542, 486, 501, 522, 551, 467, 496, 510, 496, 530, 528, 566, 527, 564, 523, 503, 526, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[527, 502, 494, 536, 529, 498, 524, 560, 488, 502, 532, 539, 517, 498, 528, 513, 456, 494, 511, 500, 512, 548, 516, 474, 509, 497, 490, 513, 531, 530, 540, 552, 462, 510, 572, 513, 545, 488, 501, 540, 508, 509, 488, 509, 458, 499, 508, 525, 542, 537, 474, 518, 509, 534, 526, 536, 506, 495, 518, 499, 475, 518, 514, 485, 535, 515, 530, 498, 496, 468, 514, 488, 510, 512, 514, 508, 505, 536, 499, 501, 490, 509, 542, 533, 488, 500, 538, 524, 524, 522, 502, 500, 521, 489, 505, 547, 485, 528, 486, 505, 506, 518, 490, 521, 562, 524, 500, 488, 523, 504, 523, 527, 508, 499, 496, 498, 539, 476, 526, 520, 530, 526, 542, 493, 495, 521, 546, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[495, 490, 528, 486, 498, 498, 505, 463, 498, 502, 552, 500, 518, 511, 528, 569, 523, 523, 498, 537, 551, 495, 530, 474, 528, 477, 518, 507, 473, 519, 471, 535, 481, 521, 550, 507, 519, 488, 544, 529, 509, 520, 529, 524, 529, 487, 499, 485, 502, 499, 521, 464, 507, 544, 535, 479, 481, 517, 508, 513, 542, 531, 501, 506, 496, 496, 488, 497, 487, 487, 536, 513, 510, 540, 531, 523, 502, 556, 523, 488, 540, 533, 496, 508, 517, 538, 476, 523, 499, 492, 529, 501, 500, 531, 496, 562, 505, 512, 540, 535, 519, 491, 485, 475, 510, 510, 527, 494, 541, 516, 531, 508, 510, 520, 529, 511, 492, 566, 482, 526, 541, 526, 482, 527, 481, 501, 544, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[452, 530, 532, 495, 497, 521, 531, 575, 537, 470, 512, 525, 539, 494, 448, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 523, 516, 506, 476, 468, 504, 573, 500, 519, 514, 506, 529, 498, 532, 506, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 512, 508, 507, 516, 503, 531, 486, 536, 542, 497, 486, 546, 516, 537, 5, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 521, 522, 531, 503, 509, 523, 511, 504, 522, 498, 513, 528, 474, 507, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 473, 521, 484, 516, 545, 506, 477, 518, 524, 522, 503, 489, 485, 544, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 502, 502, 519, 495, 521, 527, 524, 566, 524, 525, 512, 525, 512, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 485, 494, 511, 520, 510, 528, 526, 508, 492, 524, 540, 503, 518, 510, 473, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 529, 502, 498, 512, 505, 499, 513, 466, 542, 523, 509, 521, 535, 519, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8156, 8270, 8151, 8157, 8273, 8169, 8186] avg=8192.0000 max=8273 max_rank=5 min=8151 min_rank=3 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 544, 519, 502, 520, 528, 487, 504, 547, 497, 467, 562, 505, 502, 471, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 502, 504, 535, 482, 480, 522, 495, 498, 559, 505, 459, 497, 558, 521, 534, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 517, 517, 516, 536, 530, 517, 509, 499, 504, 483, 507, 501, 507, 525, 486, 5, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 549, 498, 524, 479, 499, 486, 529, 498, 491, 517, 564, 496, 491, 519, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 513, 484, 482, 550, 538, 514, 507, 526, 501, 482, 489, 557, 512, 493, 483, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 522, 501, 486, 506, 520, 476, 491, 510, 564, 534, 494, 508, 524, 536, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 515, 538, 465, 531, 520, 505, 522, 514, 491, 491, 546, 531, 542, 486, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 522, 551, 467, 496, 510, 496, 530, 528, 561, 527, 564, 523, 503, 526, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8172, 8194, 8155, 8158, 8150, 8227, 8295] avg=8192.0000 max=8295 max_rank=7 min=8150 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[511, 540, 532, 515, 475, 495, 525, 541, 496, 486, 490, 511, 517, 501, 468, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 507, 539, 466, 498, 500, 505, 533, 535, 545, 530, 551, 481, 531, 478, 558, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 479, 495, 542, 499, 499, 501, 554, 502, 499, 494, 492, 509, 494, 509, 514, 9, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 483, 517, 523, 528, 555, 524, 498, 488, 474, 505, 510, 499, 518, 490, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 449, 524, 516, 522, 548, 549, 519, 518, 530, 478, 479, 530, 514, 514, 526, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 515, 489, 518, 506, 477, 498, 497, 504, 501, 505, 501, 520, 512, 545, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 506, 497, 529, 534, 506, 538, 496, 514, 524, 490, 533, 505, 554, 487, 541, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 506, 499, 483, 478, 527, 516, 525, 504, 513, 525, 560, 458, 516, 487, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8318, 8114, 8150, 8282, 8136, 8296, 8120] avg=8192.0000 max=8318 max_rank=1 min=8114 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[499, 547, 522, 523, 534, 533, 569, 538, 530, 486, 508, 533, 500, 539, 478, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 507, 482, 510, 497, 534, 509, 521, 485, 505, 550, 504, 516, 530, 472, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 566, 498, 503, 493, 492, 524, 529, 519, 497, 517, 499, 489, 508, 537, 473, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 516, 539, 528, 525, 497, 521, 562, 549, 524, 497, 530, 494, 510, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 539, 493, 534, 452, 484, 539, 549, 520, 506, 509, 497, 496, 504, 521, 534, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 485, 540, 520, 479, 509, 497, 537, 528, 491, 475, 505, 478, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 479, 487, 487, 496, 503, 503, 510, 515, 461, 504, 570, 492, 494, 537, 487, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 502, 533, 527, 534, 504, 491, 513, 450, 509, 508, 519, 480, 538, 495, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8364, 8149, 8193, 8352, 8224, 8074, 8023, 8157] avg=8192.0000 max=8364 max_rank=0 min=8023 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[543, 553, 512, 483, 494, 537, 511, 522, 518, 502, 530, 496, 464, 492, 492, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 551, 472, 507, 561, 539, 488, 493, 508, 500, 529, 513, 518, 508, 529, 517, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 543, 485, 524, 482, 496, 500, 506, 530, 483, 531, 478, 518, 518, 501, 517, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 513, 491, 487, 540, 509, 506, 491, 518, 517, 514, 493, 555, 550, 482, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 535, 486, 505, 503, 529, 528, 521, 585, 521, 514, 473, 509, 494, 498, 473, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 524, 494, 485, 522, 489, 550, 504, 505, 513, 535, 476, 501, 519, 533, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 528, 511, 534, 493, 512, 541, 538, 572, 528, 495, 518, 504, 531, 507, 499, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 488, 505, 526, 486, 502, 465, 509, 483, 522, 511, 488, 534, 501, 554, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8242, 8146, 8190, 8221, 8176, 8339, 8062] avg=8192.0000 max=8339 max_rank=6 min=8062 min_rank=7 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[510, 503, 513, 523, 514, 499, 551, 487, 506, 485, 529, 516, 507, 569, 452, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 559, 522, 521, 510, 485, 519, 482, 524, 487, 490, 478, 495, 545, 504, 481, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 529, 537, 468, 519, 564, 505, 485, 528, 524, 493, 507, 532, 523, 524, 511, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 485, 532, 493, 519, 474, 522, 497, 526, 513, 520, 472, 499, 550, 504, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 563, 475, 518, 490, 511, 535, 504, 511, 474, 527, 496, 507, 483, 494, 523, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 569, 509, 477, 535, 518, 483, 565, 528, 476, 498, 527, 559, 507, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 501, 501, 523, 561, 483, 499, 488, 500, 515, 496, 487, 521, 466, 506, 534, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 499, 495, 522, 501, 528, 533, 511, 530, 512, 509, 515, 565, 515, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8127, 8245, 8208, 8120, 8280, 8065, 8270] avg=8192.0000 max=8280 max_rank=5 min=8065 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[527, 502, 494, 536, 529, 498, 524, 560, 488, 502, 532, 539, 517, 498, 470, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 494, 511, 500, 512, 548, 516, 474, 509, 497, 490, 513, 531, 530, 540, 552, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 510, 572, 513, 545, 488, 501, 540, 508, 509, 488, 509, 458, 499, 508, 525, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 537, 474, 518, 509, 534, 526, 536, 506, 495, 518, 499, 475, 518, 514, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 515, 530, 498, 496, 468, 514, 488, 510, 512, 514, 508, 505, 536, 499, 501, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 509, 542, 533, 488, 500, 538, 524, 524, 522, 502, 500, 521, 489, 505, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 528, 486, 505, 506, 518, 490, 521, 562, 524, 500, 488, 523, 504, 523, 527, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 499, 496, 498, 539, 476, 526, 520, 530, 522, 542, 493, 495, 521, 546, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8193, 8144, 8183, 8165, 8220, 8204, 8198] avg=8192.0000 max=8229 max_rank=0 min=8144 min_rank=2 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 490, 528, 486, 498, 498, 505, 463, 498, 502, 552, 500, 518, 511, 470, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 498, 537, 551, 495, 530, 474, 528, 477, 518, 507, 473, 519, 471, 535, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 521, 550, 507, 519, 488, 544, 529, 509, 520, 529, 524, 529, 487, 499, 485, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 521, 464, 507, 544, 535, 479, 481, 517, 508, 513, 542, 531, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 496, 488, 497, 487, 487, 536, 513, 510, 540, 531, 523, 502, 556, 523, 488, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 533, 496, 508, 517, 538, 476, 523, 499, 492, 529, 501, 500, 531, 496, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 512, 540, 535, 519, 491, 485, 475, 510, 510, 527, 494, 541, 516, 531, 508, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 529, 511, 492, 566, 482, 526, 541, 522, 482, 527, 481, 501, 544, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8179, 8230, 8147, 8209, 8227, 8213, 8248] avg=8192.0000 max=8248 max_rank=7 min=8083 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 3968, 4096, 4096, 4096, 4352, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[506, 497, 478, 522, 481, 480, 509, 505, 498, 516, 534, 475, 535, 518, 509, 536, 487, 529, 527, 518, 494, 539, 501, 482, 538, 529, 494, 518, 516, 511, 537, 500, 546, 500, 491, 519, 494, 516, 564, 503, 532, 520, 528, 527, 504, 505, 537, 497, 491, 541, 508, 527, 504, 531, 487, 511, 504, 551, 514, 485, 505, 544, 492, 501, 507, 506, 512, 515, 483, 503, 553, 545, 516, 533, 529, 501, 494, 493, 519, 477, 478, 525, 508, 500, 537, 533, 486, 512, 486, 554, 515, 505, 518, 557, 498, 486, 504, 513, 489, 556, 502, 500, 502, 507, 541, 524, 524, 510, 477, 496, 538, 538, 509, 528, 507, 489, 503, 499, 528, 494, 498, 510, 498, 496, 493, 489, 559, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[504, 501, 531, 519, 481, 551, 507, 530, 517, 545, 546, 492, 551, 527, 517, 475, 477, 510, 503, 481, 489, 557, 501, 479, 513, 488, 499, 515, 515, 556, 531, 510, 501, 496, 491, 530, 485, 520, 517, 503, 484, 495, 513, 538, 480, 517, 467, 508, 499, 524, 531, 498, 514, 520, 496, 518, 518, 558, 512, 532, 521, 516, 518, 536, 528, 462, 498, 519, 538, 521, 506, 497, 515, 529, 523, 491, 488, 512, 512, 494, 510, 487, 493, 526, 530, 490, 532, 536, 524, 515, 520, 503, 489, 535, 516, 519, 474, 500, 527, 518, 514, 521, 477, 508, 494, 529, 481, 522, 520, 503, 482, 479, 499, 495, 527, 535, 505, 491, 521, 545, 487, 560, 563, 518, 537, 520, 524, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[537, 555, 531, 461, 555, 509, 518, 502, 532, 495, 511, 466, 485, 505, 489, 524, 522, 480, 533, 507, 485, 465, 492, 474, 498, 542, 505, 504, 475, 496, 547, 534, 508, 518, 497, 503, 518, 494, 551, 570, 491, 502, 486, 507, 572, 511, 554, 528, 500, 510, 524, 507, 516, 524, 517, 550, 531, 543, 526, 479, 500, 505, 489, 497, 518, 474, 525, 520, 523, 521, 508, 501, 539, 507, 525, 538, 523, 519, 503, 479, 503, 497, 523, 494, 515, 481, 514, 513, 476, 481, 467, 524, 501, 504, 509, 534, 524, 521, 527, 534, 516, 522, 545, 516, 501, 487, 508, 541, 519, 534, 502, 540, 506, 523, 491, 531, 500, 530, 542, 538, 487, 509, 488, 523, 517, 507, 487, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[516, 502, 544, 526, 515, 481, 529, 507, 543, 497, 523, 501, 514, 499, 508, 512, 497, 513, 512, 481, 484, 514, 522, 509, 516, 521, 514, 485, 508, 503, 499, 516, 478, 534, 569, 525, 481, 489, 540, 519, 494, 504, 527, 529, 534, 531, 479, 498, 529, 488, 501, 458, 494, 532, 504, 499, 511, 535, 481, 552, 543, 521, 470, 499, 500, 490, 506, 513, 488, 461, 545, 579, 528, 542, 553, 540, 527, 493, 497, 526, 550, 491, 487, 492, 533, 514, 490, 487, 503, 523, 517, 497, 490, 532, 519, 522, 503, 530, 518, 497, 512, 500, 561, 513, 515, 537, 506, 544, 497, 507, 519, 519, 497, 458, 516, 525, 515, 516, 515, 519, 485, 502, 496, 494, 546, 510, 533, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[476, 505, 535, 515, 528, 515, 481, 489, 472, 533, 484, 504, 525, 505, 536, 497, 516, 510, 513, 544, 503, 528, 502, 521, 546, 505, 499, 507, 535, 517, 525, 571, 528, 506, 489, 465, 499, 493, 520, 525, 512, 470, 509, 508, 550, 486, 519, 564, 556, 481, 499, 468, 480, 515, 499, 496, 496, 496, 543, 537, 504, 534, 526, 524, 522, 496, 516, 486, 526, 529, 528, 508, 495, 521, 495, 508, 491, 472, 519, 524, 487, 469, 495, 509, 559, 522, 482, 509, 524, 549, 516, 543, 524, 533, 515, 474, 519, 505, 521, 460, 526, 505, 507, 529, 470, 517, 527, 481, 513, 509, 498, 565, 514, 524, 556, 530, 508, 501, 515, 506, 565, 544, 478, 529, 517, 484, 516, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[535, 467, 499, 512, 490, 481, 507, 514, 514, 540, 484, 520, 511, 542, 507, 499, 545, 499, 507, 493, 554, 523, 527, 522, 512, 495, 494, 554, 553, 488, 544, 533, 503, 524, 506, 500, 519, 485, 480, 481, 532, 530, 501, 490, 512, 490, 507, 514, 537, 540, 489, 514, 549, 522, 533, 471, 485, 482, 506, 500, 526, 552, 522, 500, 506, 477, 477, 523, 528, 504, 516, 521, 517, 528, 515, 532, 513, 521, 495, 518, 470, 472, 510, 507, 554, 546, 492, 529, 502, 553, 455, 543, 505, 516, 574, 489, 525, 493, 490, 514, 519, 508, 502, 498, 525, 540, 522, 544, 483, 530, 530, 505, 570, 542, 554, 495, 504, 510, 483, 502, 510, 505, 493, 492, 493, 509, 490, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[493, 525, 510, 496, 484, 535, 493, 505, 529, 496, 507, 525, 533, 529, 511, 515, 518, 526, 479, 510, 532, 507, 473, 497, 529, 506, 506, 509, 530, 518, 507, 542, 517, 475, 546, 486, 544, 510, 500, 504, 502, 490, 510, 519, 519, 544, 551, 509, 545, 541, 520, 523, 525, 546, 499, 536, 507, 538, 487, 535, 497, 517, 479, 522, 511, 489, 529, 503, 504, 532, 503, 485, 493, 494, 550, 513, 517, 504, 519, 533, 524, 496, 489, 522, 470, 471, 478, 512, 490, 527, 544, 550, 537, 524, 509, 516, 503, 499, 483, 462, 560, 538, 466, 508, 485, 480, 505, 483, 525, 539, 498, 469, 568, 523, 525, 499, 508, 503, 476, 551, 515, 520, 517, 520, 496, 520, 507, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[521, 524, 499, 509, 522, 492, 532, 529, 530, 455, 520, 501, 512, 545, 510, 537, 506, 500, 491, 507, 487, 554, 530, 571, 506, 521, 524, 509, 517, 522, 501, 545, 524, 516, 520, 546, 491, 556, 529, 521, 492, 503, 498, 522, 516, 489, 514, 478, 520, 564, 537, 466, 479, 488, 513, 521, 512, 510, 536, 535, 486, 494, 495, 498, 527, 550, 513, 540, 507, 498, 531, 503, 463, 504, 483, 488, 499, 489, 509, 521, 536, 529, 506, 485, 501, 527, 517, 500, 527, 539, 515, 485, 509, 486, 530, 509, 500, 490, 512, 489, 535, 522, 541, 539, 540, 491, 479, 507, 486, 492, 506, 511, 484, 488, 513, 514, 519, 538, 523, 503, 506, 487, 544, 491, 523, 509, 525, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 497, 478, 522, 481, 480, 509, 505, 498, 516, 534, 475, 535, 518, 509, 536, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 529, 527, 518, 494, 539, 501, 482, 538, 529, 494, 518, 516, 511, 537, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 500, 491, 519, 494, 516, 564, 503, 532, 520, 528, 527, 504, 505, 537, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 541, 508, 527, 504, 531, 487, 511, 504, 531, 514, 485, 505, 544, 492, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 506, 512, 515, 483, 503, 553, 545, 516, 533, 529, 501, 494, 493, 519, 477, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 525, 508, 500, 537, 533, 486, 512, 486, 554, 515, 505, 518, 557, 498, 486, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 489, 556, 502, 500, 502, 507, 541, 524, 524, 510, 477, 496, 538, 538, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 528, 487, 489, 503, 499, 528, 494, 498, 510, 498, 496, 493, 489, 559, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8207, 8284, 8176, 8194, 8215, 8236, 8113] avg=8192.0000 max=8284 max_rank=2 min=8111 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 502, 544, 526, 515, 481, 529, 507, 543, 497, 523, 501, 514, 499, 508, 512, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 513, 512, 481, 484, 514, 522, 509, 516, 521, 514, 485, 508, 503, 499, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 534, 569, 525, 481, 489, 540, 519, 494, 504, 527, 529, 534, 531, 479, 498, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 488, 501, 458, 494, 532, 504, 499, 511, 509, 481, 552, 543, 521, 470, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 490, 506, 513, 488, 461, 545, 579, 528, 542, 553, 540, 527, 493, 497, 526, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 491, 487, 492, 533, 514, 490, 487, 503, 523, 517, 497, 490, 532, 519, 522, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 530, 518, 497, 512, 500, 561, 513, 515, 537, 506, 544, 497, 507, 519, 519, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 458, 486, 525, 515, 516, 515, 519, 485, 502, 496, 494, 546, 510, 533, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8071, 8233, 8091, 8301, 8170, 8301, 8134] avg=8192.0000 max=8301 max_rank=4 min=8071 min_rank=1 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[504, 501, 531, 519, 481, 551, 507, 530, 517, 545, 546, 492, 551, 527, 517, 475, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 510, 503, 481, 489, 557, 501, 479, 513, 488, 499, 515, 515, 556, 531, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 496, 491, 530, 485, 520, 517, 503, 484, 495, 513, 538, 480, 517, 467, 508, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 531, 498, 514, 520, 496, 518, 518, 538, 512, 532, 521, 516, 518, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 462, 498, 519, 538, 521, 506, 497, 515, 529, 523, 491, 488, 512, 512, 494, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 493, 526, 530, 490, 532, 536, 524, 515, 520, 503, 489, 535, 516, 519, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 500, 527, 518, 514, 521, 477, 508, 494, 529, 481, 522, 520, 503, 482, 479, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 495, 506, 535, 505, 491, 521, 545, 487, 560, 563, 518, 537, 520, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8110, 8046, 8291, 8141, 8242, 8065, 8334] avg=8192.0000 max=8334 max_rank=7 min=8046 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[537, 555, 531, 461, 555, 509, 518, 502, 532, 495, 511, 466, 485, 505, 489, 524, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 480, 533, 507, 485, 465, 492, 474, 498, 542, 505, 504, 475, 496, 547, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 518, 497, 503, 518, 494, 551, 570, 491, 502, 486, 507, 572, 511, 554, 528, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 524, 507, 516, 524, 517, 550, 531, 523, 526, 479, 500, 505, 489, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 474, 525, 520, 523, 521, 508, 501, 539, 507, 525, 538, 523, 519, 503, 479, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 523, 494, 515, 481, 514, 513, 476, 481, 467, 524, 501, 504, 509, 534, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 521, 527, 534, 516, 522, 545, 516, 501, 487, 508, 541, 519, 534, 502, 540, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 472, 531, 500, 530, 542, 538, 487, 509, 488, 523, 517, 507, 487, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8044, 8311, 8198, 8232, 8052, 8352, 8159] avg=8192.0000 max=8352 max_rank=6 min=8044 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 524, 499, 509, 522, 492, 532, 529, 530, 455, 520, 501, 512, 545, 510, 537, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 500, 491, 507, 487, 554, 530, 571, 506, 521, 524, 509, 517, 522, 501, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 516, 520, 546, 491, 556, 529, 521, 492, 503, 498, 522, 516, 489, 514, 478, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 564, 537, 466, 479, 488, 513, 521, 512, 492, 536, 535, 486, 494, 495, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 550, 513, 540, 507, 498, 531, 503, 463, 504, 483, 488, 499, 489, 509, 521, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 529, 506, 485, 501, 527, 517, 500, 527, 539, 515, 485, 509, 486, 530, 509, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 490, 512, 489, 535, 522, 541, 539, 540, 491, 479, 507, 486, 492, 506, 511, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 488, 493, 514, 519, 538, 523, 503, 506, 487, 544, 491, 523, 509, 525, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8276, 8216, 8136, 8134, 8218, 8154, 8152] avg=8192.0000 max=8276 max_rank=1 min=8134 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[535, 467, 499, 512, 490, 481, 507, 514, 514, 540, 484, 520, 511, 542, 507, 499, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 499, 507, 493, 554, 523, 527, 522, 512, 495, 494, 554, 553, 488, 544, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 506, 500, 519, 485, 480, 481, 532, 530, 501, 490, 512, 490, 507, 514, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 540, 489, 514, 549, 522, 533, 471, 485, 465, 506, 500, 526, 552, 522, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 477, 477, 523, 528, 504, 516, 521, 517, 528, 515, 532, 513, 521, 495, 518, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 472, 510, 507, 554, 546, 492, 529, 502, 553, 455, 543, 505, 516, 574, 489, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 493, 490, 514, 519, 508, 502, 498, 525, 540, 522, 544, 483, 530, 530, 505, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 542, 532, 495, 504, 510, 483, 502, 510, 505, 493, 492, 493, 509, 490, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8328, 8075, 8211, 8200, 8235, 8242, 8111] avg=8192.0000 max=8328 max_rank=1 min=8075 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[493, 525, 510, 496, 484, 535, 493, 505, 529, 496, 507, 525, 533, 529, 511, 515, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 526, 479, 510, 532, 507, 473, 497, 529, 506, 506, 509, 530, 518, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 546, 486, 544, 510, 500, 504, 502, 490, 510, 519, 519, 544, 551, 509, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 541, 520, 523, 525, 546, 499, 536, 507, 518, 487, 535, 497, 517, 479, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 489, 529, 503, 504, 532, 503, 485, 493, 494, 550, 513, 517, 504, 519, 533, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 489, 522, 470, 471, 478, 512, 490, 527, 544, 550, 537, 524, 509, 516, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 499, 483, 462, 560, 538, 466, 508, 485, 480, 505, 483, 525, 539, 498, 469, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 523, 504, 499, 508, 503, 476, 551, 515, 520, 517, 520, 496, 520, 507, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8174, 8227, 8297, 8188, 8176, 8019, 8256] avg=8192.0000 max=8297 max_rank=3 min=8019 min_rank=6 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[476, 505, 535, 515, 528, 515, 481, 489, 472, 533, 484, 504, 525, 505, 536, 497, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 513, 544, 503, 528, 502, 521, 546, 505, 499, 507, 535, 517, 525, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 489, 465, 499, 493, 520, 525, 512, 470, 509, 508, 550, 486, 519, 564, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 481, 499, 468, 480, 515, 499, 496, 496, 478, 543, 537, 504, 534, 526, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 496, 516, 486, 526, 529, 528, 508, 495, 521, 495, 508, 491, 472, 519, 524, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 469, 495, 509, 559, 522, 482, 509, 524, 549, 516, 543, 524, 533, 515, 474, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 521, 460, 526, 505, 507, 529, 470, 517, 527, 481, 513, 509, 498, 565, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 524, 534, 530, 508, 501, 515, 506, 565, 544, 478, 529, 517, 484, 516, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8326, 8144, 8136, 8146, 8228, 8167, 8277] avg=8192.0000 max=8326 max_rank=1 min=8112 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4352, 4096, 4224, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[537, 513, 521, 519, 536, 524, 494, 500, 525, 520, 515, 527, 518, 470, 512, 515, 533, 469, 534, 496, 515, 490, 542, 497, 525, 506, 510, 526, 513, 549, 483, 501, 497, 567, 503, 510, 505, 505, 494, 523, 535, 537, 501, 503, 531, 477, 472, 500, 489, 517, 488, 505, 517, 476, 490, 542, 499, 521, 544, 528, 482, 492, 492, 499, 534, 534, 513, 511, 515, 501, 526, 514, 504, 514, 500, 471, 511, 551, 512, 533, 535, 526, 477, 517, 515, 521, 569, 524, 492, 555, 507, 527, 506, 482, 563, 503, 475, 492, 500, 530, 524, 518, 508, 496, 512, 480, 501, 498, 536, 473, 513, 523, 501, 533, 524, 506, 492, 483, 525, 533, 529, 500, 516, 509, 523, 514, 535, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[491, 503, 547, 509, 519, 501, 473, 515, 532, 488, 511, 497, 538, 504, 516, 505, 512, 492, 484, 482, 486, 484, 483, 509, 542, 510, 504, 519, 544, 534, 523, 480, 511, 496, 504, 526, 452, 530, 506, 528, 512, 531, 491, 504, 526, 527, 513, 513, 510, 519, 514, 494, 473, 497, 533, 512, 490, 517, 485, 542, 524, 546, 504, 472, 555, 511, 556, 483, 511, 504, 526, 510, 501, 503, 460, 496, 506, 522, 501, 511, 529, 514, 534, 522, 478, 541, 542, 501, 530, 498, 524, 524, 462, 505, 560, 526, 520, 488, 507, 519, 481, 506, 543, 516, 500, 538, 558, 491, 536, 492, 521, 515, 555, 559, 504, 515, 497, 547, 528, 547, 500, 504, 509, 514, 509, 501, 513, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[499, 532, 516, 544, 528, 498, 504, 517, 504, 543, 495, 491, 547, 511, 542, 471, 512, 463, 482, 502, 542, 536, 529, 490, 474, 508, 512, 494, 494, 544, 535, 497, 531, 524, 519, 466, 548, 512, 521, 491, 481, 512, 517, 535, 513, 496, 519, 563, 502, 510, 538, 544, 508, 500, 507, 498, 503, 492, 509, 512, 488, 480, 485, 519, 497, 529, 506, 538, 467, 467, 458, 480, 487, 552, 538, 503, 515, 503, 523, 459, 512, 495, 525, 529, 529, 546, 526, 541, 541, 513, 510, 502, 519, 540, 501, 516, 517, 495, 503, 477, 517, 529, 503, 529, 537, 514, 547, 497, 506, 500, 539, 515, 510, 527, 544, 507, 519, 545, 501, 531, 493, 515, 534, 520, 501, 483, 536, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[502, 523, 510, 512, 518, 504, 504, 546, 483, 525, 529, 526, 480, 485, 517, 482, 544, 481, 523, 531, 504, 504, 548, 497, 484, 523, 527, 552, 513, 465, 535, 505, 522, 492, 516, 508, 511, 498, 518, 506, 527, 516, 518, 516, 549, 536, 482, 493, 546, 505, 516, 498, 536, 566, 496, 489, 493, 522, 526, 505, 509, 522, 492, 505, 516, 510, 506, 529, 519, 510, 493, 505, 531, 518, 522, 496, 525, 499, 493, 533, 530, 511, 553, 545, 460, 512, 536, 483, 506, 511, 521, 525, 466, 512, 490, 504, 467, 443, 552, 475, 506, 504, 483, 549, 520, 557, 550, 496, 530, 498, 574, 507, 502, 519, 481, 481, 502, 512, 489, 513, 565, 504, 491, 522, 468, 494, 585, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[562, 520, 506, 526, 490, 488, 530, 508, 518, 483, 547, 559, 514, 526, 559, 517, 523, 507, 520, 500, 494, 452, 515, 507, 521, 501, 532, 517, 528, 545, 531, 538, 568, 495, 485, 535, 505, 500, 510, 498, 510, 515, 508, 509, 520, 523, 549, 512, 508, 497, 494, 501, 486, 518, 493, 500, 561, 502, 494, 556, 528, 515, 511, 509, 532, 514, 544, 468, 498, 529, 493, 523, 481, 501, 495, 467, 521, 526, 511, 512, 497, 503, 454, 475, 511, 510, 524, 482, 536, 534, 533, 536, 507, 548, 469, 487, 534, 545, 502, 488, 552, 510, 508, 502, 488, 511, 451, 537, 505, 503, 479, 536, 486, 530, 512, 516, 489, 522, 490, 500, 477, 539, 541, 525, 490, 498, 516, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[491, 520, 498, 490, 513, 521, 497, 467, 490, 500, 493, 500, 487, 537, 528, 546, 486, 531, 487, 523, 485, 547, 520, 520, 509, 529, 503, 489, 493, 533, 523, 508, 516, 547, 522, 482, 505, 503, 532, 483, 515, 504, 558, 504, 534, 566, 513, 526, 475, 534, 508, 462, 521, 516, 494, 528, 509, 496, 498, 520, 538, 510, 536, 501, 552, 535, 534, 550, 556, 550, 513, 525, 493, 555, 500, 513, 550, 509, 479, 499, 480, 517, 505, 504, 517, 438, 531, 507, 530, 518, 516, 546, 497, 510, 512, 481, 522, 510, 481, 561, 464, 480, 481, 554, 491, 499, 517, 480, 494, 495, 512, 460, 543, 525, 533, 543, 516, 500, 526, 492, 523, 532, 495, 529, 509, 517, 535, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[492, 510, 499, 502, 514, 512, 509, 482, 515, 520, 491, 529, 490, 512, 502, 524, 499, 493, 479, 543, 581, 493, 500, 518, 525, 555, 521, 489, 492, 455, 527, 525, 487, 498, 519, 536, 506, 515, 544, 499, 502, 527, 542, 504, 534, 544, 516, 550, 517, 511, 469, 563, 499, 532, 519, 520, 501, 528, 463, 485, 506, 478, 486, 474, 522, 473, 513, 522, 516, 527, 549, 481, 484, 535, 531, 494, 523, 501, 509, 512, 536, 522, 536, 471, 495, 533, 471, 526, 508, 501, 526, 511, 499, 531, 535, 533, 516, 518, 513, 524, 526, 504, 491, 534, 512, 519, 508, 550, 499, 496, 518, 482, 535, 502, 517, 539, 488, 554, 479, 503, 504, 527, 526, 509, 471, 511, 520, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[515, 474, 524, 551, 511, 503, 495, 516, 532, 538, 533, 508, 533, 546, 520, 535, 523, 503, 535, 521, 478, 498, 467, 479, 479, 511, 512, 476, 477, 530, 480, 510, 524, 527, 483, 545, 524, 503, 496, 510, 504, 517, 496, 514, 519, 550, 539, 485, 526, 526, 509, 526, 503, 497, 534, 515, 515, 536, 529, 507, 476, 511, 512, 495, 502, 473, 485, 542, 503, 484, 504, 504, 520, 507, 515, 485, 503, 525, 525, 497, 552, 522, 546, 547, 516, 546, 444, 494, 511, 518, 482, 518, 544, 519, 558, 460, 481, 505, 535, 509, 525, 488, 524, 519, 478, 500, 535, 520, 505, 561, 485, 509, 511, 521, 529, 484, 516, 497, 540, 492, 534, 533, 502, 520, 519, 546, 507, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[537, 513, 521, 519, 536, 524, 494, 500, 525, 520, 515, 527, 518, 470, 498, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 469, 534, 496, 515, 490, 542, 497, 525, 506, 510, 526, 513, 549, 483, 501, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 567, 503, 510, 505, 505, 494, 523, 535, 537, 501, 503, 486, 477, 472, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 517, 488, 505, 517, 476, 490, 542, 499, 521, 544, 528, 482, 492, 492, 499, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 534, 513, 511, 515, 501, 526, 514, 504, 514, 500, 471, 511, 551, 512, 533, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 526, 477, 517, 515, 521, 569, 524, 492, 555, 507, 489, 506, 482, 563, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 492, 500, 530, 524, 518, 508, 496, 512, 480, 501, 498, 536, 473, 513, 523, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 533, 524, 506, 492, 483, 525, 533, 529, 500, 516, 509, 523, 514, 497, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8228, 8115, 8132, 8258, 8281, 8110, 8180] avg=8192.0000 max=8281 max_rank=5 min=8110 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32829 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[499, 532, 516, 544, 528, 498, 504, 517, 504, 543, 495, 491, 547, 511, 528, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 463, 482, 502, 542, 536, 529, 490, 474, 508, 512, 494, 494, 544, 535, 497, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 524, 519, 466, 548, 512, 521, 491, 481, 512, 517, 535, 470, 496, 519, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 510, 538, 544, 508, 500, 507, 498, 503, 492, 509, 512, 488, 480, 485, 519, 43, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 529, 506, 538, 467, 467, 458, 480, 487, 552, 538, 503, 515, 503, 523, 459, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 495, 525, 529, 529, 546, 526, 541, 541, 513, 510, 466, 519, 540, 501, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 503, 477, 517, 529, 503, 529, 537, 514, 547, 497, 506, 500, 539, 515, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 527, 544, 507, 519, 545, 501, 531, 493, 515, 534, 520, 501, 483, 498, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8153, 8205, 8144, 8036, 8309, 8254, 8207] avg=8192.0000 max=8309 max_rank=5 min=8036 min_rank=4 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32806 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[491, 503, 547, 509, 519, 501, 473, 515, 532, 488, 511, 497, 538, 504, 499, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 492, 484, 482, 486, 484, 483, 509, 542, 510, 504, 519, 544, 534, 523, 480, 35, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 496, 504, 526, 452, 530, 506, 528, 512, 531, 491, 504, 477, 527, 513, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 519, 514, 494, 473, 497, 533, 512, 490, 517, 485, 542, 524, 546, 504, 472, 49, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 511, 556, 483, 511, 504, 526, 510, 501, 503, 460, 496, 506, 522, 501, 511, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 514, 534, 522, 478, 541, 542, 501, 530, 498, 524, 478, 462, 505, 560, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 488, 507, 519, 481, 506, 543, 516, 500, 538, 558, 491, 536, 492, 521, 515, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 559, 504, 515, 497, 547, 528, 547, 500, 504, 509, 514, 509, 501, 467, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8134, 8121, 8192, 8173, 8244, 8266, 8274] avg=8192.0000 max=8274 max_rank=7 min=8121 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32957 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[562, 520, 506, 526, 490, 488, 530, 508, 518, 483, 547, 559, 514, 526, 544, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 520, 500, 494, 452, 515, 507, 521, 501, 532, 517, 528, 545, 531, 538, 31, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 495, 485, 535, 505, 500, 510, 498, 510, 515, 508, 509, 476, 523, 549, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 494, 501, 486, 518, 493, 500, 561, 502, 494, 556, 528, 515, 511, 509, 44, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 514, 544, 468, 498, 529, 493, 523, 481, 501, 495, 467, 521, 526, 511, 512, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 503, 454, 475, 511, 510, 524, 482, 536, 534, 533, 498, 507, 548, 469, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 545, 502, 488, 552, 510, 508, 502, 488, 511, 451, 537, 505, 503, 479, 536, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 512, 516, 489, 522, 490, 500, 477, 539, 541, 525, 490, 498, 479, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8269, 8198, 8223, 8130, 8068, 8182, 8128] avg=8192.0000 max=8338 max_rank=0 min=8068 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32508 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 523, 510, 512, 518, 504, 504, 546, 483, 525, 529, 526, 480, 485, 503, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 481, 523, 531, 504, 504, 548, 497, 484, 523, 527, 552, 513, 465, 535, 505, 35, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 492, 516, 508, 511, 498, 518, 506, 527, 516, 518, 516, 503, 536, 482, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 505, 516, 498, 536, 566, 496, 489, 493, 522, 526, 505, 509, 522, 492, 505, 46, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 506, 529, 519, 510, 493, 505, 531, 518, 522, 496, 525, 499, 493, 533, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 511, 553, 545, 460, 512, 536, 483, 506, 511, 521, 487, 466, 512, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 443, 552, 475, 506, 504, 483, 549, 520, 557, 550, 496, 530, 498, 574, 507, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 519, 481, 481, 502, 512, 489, 513, 565, 504, 491, 522, 468, 494, 543, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8278, 8162, 8279, 8219, 8127, 8242, 8097] avg=8192.0000 max=8279 max_rank=3 min=8097 min_rank=7 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32685 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[491, 520, 498, 490, 513, 521, 497, 467, 490, 500, 493, 500, 487, 537, 514, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 531, 487, 523, 485, 547, 520, 520, 509, 529, 503, 489, 493, 533, 523, 508, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 547, 522, 482, 505, 503, 532, 483, 515, 504, 558, 504, 489, 566, 513, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 534, 508, 462, 521, 516, 494, 528, 509, 496, 498, 520, 538, 510, 536, 501, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 535, 534, 550, 556, 550, 513, 525, 493, 555, 500, 513, 550, 509, 479, 499, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 517, 505, 504, 517, 438, 531, 507, 530, 518, 516, 507, 497, 510, 512, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 510, 481, 561, 464, 480, 481, 554, 491, 499, 517, 480, 494, 495, 512, 460, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 525, 533, 543, 516, 500, 526, 492, 523, 532, 495, 529, 509, 517, 497, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8064, 8225, 8265, 8197, 8427, 8070, 8033, 8255] avg=8192.0000 max=8427 max_rank=4 min=8033 min_rank=6 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=32785 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[492, 510, 499, 502, 514, 512, 509, 482, 515, 520, 491, 529, 490, 512, 489, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 479, 543, 581, 493, 500, 518, 525, 555, 521, 489, 492, 455, 527, 525, 31, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 498, 519, 536, 506, 515, 544, 499, 502, 527, 542, 504, 489, 544, 516, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 469, 563, 499, 532, 519, 520, 501, 528, 463, 485, 506, 478, 486, 474, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 473, 513, 522, 516, 527, 549, 481, 484, 535, 531, 494, 523, 501, 509, 512, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 522, 536, 471, 495, 533, 471, 526, 508, 501, 526, 474, 499, 531, 535, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 513, 524, 526, 504, 491, 534, 512, 519, 508, 550, 499, 496, 518, 482, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 517, 539, 488, 554, 479, 503, 504, 527, 526, 509, 471, 511, 483, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8233, 8278, 8102, 8205, 8197, 8240, 8191] avg=8192.0000 max=8278 max_rank=2 min=8090 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32833 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 474, 524, 551, 511, 503, 495, 516, 532, 538, 533, 508, 533, 546, 506, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 535, 521, 478, 498, 467, 479, 479, 511, 512, 476, 477, 530, 480, 510, 30, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 483, 545, 524, 503, 496, 510, 504, 517, 496, 514, 475, 550, 539, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 509, 526, 503, 497, 534, 515, 515, 536, 529, 507, 476, 511, 512, 495, 44, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 473, 485, 542, 503, 484, 504, 504, 520, 507, 515, 485, 503, 525, 525, 497, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 522, 546, 547, 516, 546, 444, 494, 511, 518, 482, 481, 544, 519, 558, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 505, 535, 509, 525, 488, 524, 519, 478, 500, 535, 520, 505, 561, 485, 509, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 521, 529, 484, 516, 497, 540, 492, 534, 533, 502, 520, 519, 546, 471, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8016, 8192, 8267, 8088, 8240, 8209, 8204] avg=8192.0000 max=8320 max_rank=0 min=8016 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32741 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 3968, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[516, 481, 498, 541, 502, 496, 508, 487, 537, 513, 540, 508, 486, 535, 500, 545, 536, 476, 528, 532, 517, 547, 521, 485, 489, 534, 519, 534, 524, 470, 503, 513, 532, 499, 492, 492, 505, 515, 511, 529, 502, 552, 514, 464, 500, 521, 513, 522, 542, 512, 529, 530, 519, 524, 522, 546, 510, 519, 512, 525, 517, 486, 526, 510, 502, 492, 482, 495, 507, 490, 494, 526, 531, 476, 533, 521, 505, 497, 501, 504, 523, 540, 489, 517, 512, 512, 484, 536, 516, 518, 516, 491, 523, 517, 529, 491, 544, 503, 476, 474, 513, 513, 512, 537, 506, 497, 497, 516, 528, 495, 517, 514, 518, 536, 509, 533, 553, 537, 468, 502, 505, 500, 504, 528, 501, 511, 518, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[477, 511, 468, 511, 527, 553, 496, 497, 479, 515, 499, 502, 477, 515, 524, 478, 523, 510, 517, 508, 520, 510, 525, 499, 476, 490, 523, 514, 495, 505, 527, 550, 519, 518, 533, 522, 525, 536, 495, 531, 511, 480, 489, 501, 513, 524, 498, 512, 553, 516, 509, 554, 528, 494, 504, 477, 503, 529, 535, 507, 543, 493, 505, 532, 528, 514, 495, 506, 512, 528, 498, 507, 516, 511, 540, 499, 469, 504, 548, 526, 519, 523, 546, 485, 527, 498, 529, 500, 477, 492, 524, 498, 523, 503, 507, 510, 505, 523, 526, 477, 516, 528, 529, 500, 520, 498, 496, 515, 533, 492, 519, 495, 506, 498, 524, 508, 545, 552, 520, 522, 521, 528, 485, 549, 505, 546, 498, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[542, 542, 500, 501, 522, 501, 528, 483, 543, 544, 501, 604, 517, 488, 520, 555, 533, 477, 525, 489, 490, 511, 503, 530, 525, 527, 489, 515, 481, 463, 465, 497, 498, 494, 466, 505, 521, 514, 540, 520, 503, 476, 489, 574, 524, 528, 525, 494, 481, 526, 528, 515, 547, 486, 520, 534, 488, 484, 483, 521, 513, 490, 498, 531, 463, 514, 492, 510, 539, 507, 527, 505, 512, 506, 555, 517, 456, 516, 487, 508, 485, 503, 546, 520, 495, 570, 488, 491, 476, 538, 491, 534, 491, 504, 528, 518, 515, 547, 513, 524, 534, 491, 515, 489, 527, 542, 533, 519, 494, 468, 519, 480, 500, 507, 502, 537, 491, 510, 515, 521, 488, 508, 557, 564, 527, 535, 509, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[517, 521, 542, 499, 508, 540, 539, 483, 493, 519, 543, 512, 515, 523, 516, 537, 479, 525, 549, 550, 502, 532, 486, 508, 516, 514, 508, 527, 499, 529, 541, 491, 512, 504, 503, 489, 525, 550, 501, 517, 511, 513, 536, 493, 529, 517, 476, 488, 489, 510, 495, 538, 496, 544, 474, 527, 514, 497, 533, 508, 508, 505, 544, 519, 483, 480, 516, 511, 513, 521, 532, 501, 518, 513, 501, 480, 516, 485, 503, 508, 477, 506, 524, 526, 514, 467, 558, 517, 528, 489, 494, 513, 483, 538, 521, 488, 487, 517, 492, 511, 504, 493, 527, 512, 505, 487, 530, 490, 527, 488, 529, 540, 441, 545, 516, 542, 504, 507, 492, 515, 491, 511, 510, 521, 562, 535, 525, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[506, 530, 525, 524, 547, 493, 490, 538, 499, 546, 489, 518, 485, 461, 499, 545, 539, 509, 541, 513, 524, 514, 534, 505, 493, 505, 511, 488, 532, 551, 484, 490, 521, 491, 506, 497, 519, 516, 510, 502, 459, 496, 502, 505, 506, 511, 483, 516, 509, 518, 500, 536, 510, 478, 504, 496, 536, 520, 533, 516, 542, 539, 504, 508, 528, 525, 524, 503, 522, 530, 468, 534, 495, 523, 544, 547, 507, 505, 473, 489, 522, 481, 535, 540, 530, 461, 507, 497, 529, 557, 543, 495, 479, 517, 527, 513, 501, 486, 517, 515, 527, 526, 530, 507, 494, 484, 486, 513, 489, 544, 500, 508, 506, 459, 511, 483, 510, 550, 539, 523, 499, 534, 530, 545, 527, 520, 486, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[477, 491, 533, 475, 502, 510, 509, 537, 543, 492, 527, 567, 471, 504, 521, 520, 495, 519, 510, 520, 473, 511, 486, 529, 492, 489, 505, 522, 471, 528, 517, 536, 550, 508, 558, 477, 520, 509, 484, 521, 495, 528, 506, 489, 545, 536, 513, 446, 511, 484, 517, 509, 527, 485, 466, 508, 480, 497, 523, 491, 482, 484, 517, 533, 535, 500, 540, 534, 520, 526, 493, 474, 536, 510, 552, 545, 526, 518, 500, 501, 536, 611, 495, 489, 498, 511, 502, 512, 485, 495, 508, 540, 512, 535, 538, 518, 509, 526, 555, 507, 482, 556, 485, 521, 498, 515, 516, 520, 499, 521, 502, 485, 514, 549, 505, 546, 527, 517, 522, 530, 502, 457, 530, 517, 496, 522, 527, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[498, 500, 528, 497, 471, 518, 516, 526, 492, 515, 534, 541, 519, 511, 507, 533, 513, 505, 539, 469, 474, 522, 520, 536, 529, 494, 514, 489, 555, 521, 486, 561, 517, 539, 533, 498, 528, 508, 523, 495, 476, 473, 504, 527, 499, 511, 485, 555, 509, 509, 527, 519, 526, 487, 525, 533, 499, 520, 520, 492, 516, 512, 491, 489, 534, 541, 534, 536, 497, 489, 499, 495, 500, 560, 481, 530, 492, 495, 489, 526, 459, 482, 511, 522, 514, 548, 545, 546, 513, 539, 542, 545, 485, 505, 511, 526, 557, 515, 502, 504, 537, 486, 503, 464, 509, 527, 483, 494, 516, 549, 484, 533, 511, 461, 489, 517, 465, 523, 520, 509, 576, 526, 533, 500, 482, 495, 497, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[516, 507, 510, 529, 482, 500, 538, 504, 518, 483, 520, 470, 503, 494, 513, 503, 483, 486, 544, 498, 509, 508, 528, 546, 529, 501, 495, 533, 503, 523, 534, 486, 506, 534, 521, 508, 502, 485, 596, 496, 508, 531, 474, 500, 508, 507, 519, 528, 516, 516, 480, 508, 487, 529, 482, 504, 520, 497, 497, 527, 489, 539, 488, 479, 508, 512, 487, 514, 575, 482, 505, 516, 507, 495, 574, 521, 530, 530, 563, 505, 525, 470, 500, 487, 548, 520, 499, 536, 556, 490, 516, 496, 551, 495, 479, 504, 528, 503, 509, 524, 468, 499, 528, 475, 471, 534, 523, 518, 488, 526, 538, 484, 553, 526, 522, 529, 487, 559, 527, 536, 556, 477, 490, 530, 514, 493, 520, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[506, 530, 525, 524, 547, 493, 490, 538, 499, 546, 489, 512, 485, 461, 499, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 509, 541, 513, 524, 514, 534, 505, 493, 505, 511, 488, 532, 551, 484, 490, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 506, 497, 519, 516, 510, 502, 459, 496, 502, 505, 506, 511, 483, 516, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 500, 536, 510, 478, 504, 496, 536, 520, 533, 516, 542, 539, 504, 508, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 525, 524, 503, 522, 530, 468, 534, 495, 523, 544, 547, 507, 505, 473, 489, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 519, 540, 530, 461, 507, 497, 529, 557, 543, 495, 479, 517, 527, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 486, 517, 515, 527, 526, 530, 507, 494, 484, 486, 513, 489, 544, 500, 508, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 459, 511, 483, 510, 490, 539, 523, 499, 534, 530, 545, 527, 520, 486, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8241, 8066, 8259, 8221, 8217, 8161, 8182] avg=8192.0000 max=8259 max_rank=3 min=8066 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 481, 498, 541, 502, 496, 508, 487, 537, 513, 540, 502, 486, 535, 500, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 476, 528, 532, 517, 547, 521, 485, 489, 534, 519, 534, 524, 470, 503, 513, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 499, 492, 492, 505, 515, 511, 529, 502, 552, 514, 464, 500, 521, 513, 522, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 512, 529, 530, 519, 524, 522, 546, 510, 519, 512, 525, 517, 486, 526, 510, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 492, 482, 495, 507, 490, 494, 526, 531, 476, 533, 521, 505, 497, 501, 504, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 540, 474, 517, 512, 512, 484, 536, 516, 518, 516, 491, 523, 517, 529, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 503, 476, 474, 513, 513, 512, 537, 506, 497, 497, 516, 528, 495, 517, 514, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 536, 509, 533, 553, 478, 468, 502, 505, 500, 504, 528, 501, 511, 518, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8236, 8188, 8338, 8060, 8199, 8176, 8152] avg=8192.0000 max=8338 max_rank=3 min=8060 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32587 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[542, 542, 500, 501, 522, 501, 528, 483, 543, 544, 501, 591, 517, 488, 520, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 477, 525, 489, 490, 511, 503, 530, 525, 527, 489, 515, 481, 463, 465, 497, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 466, 505, 521, 514, 540, 520, 503, 476, 489, 574, 524, 528, 525, 494, 29, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 526, 528, 515, 547, 486, 520, 534, 488, 484, 483, 521, 513, 490, 498, 531, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 514, 492, 510, 539, 507, 527, 505, 512, 506, 555, 517, 456, 516, 487, 508, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 503, 515, 520, 495, 570, 488, 491, 476, 538, 491, 534, 491, 504, 528, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 547, 513, 524, 534, 491, 515, 489, 527, 542, 533, 519, 494, 468, 519, 480, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 507, 502, 537, 491, 446, 515, 521, 488, 508, 557, 564, 527, 535, 509, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8378, 8035, 8207, 8159, 8122, 8147, 8245, 8243] avg=8192.0000 max=8378 max_rank=0 min=8035 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[477, 511, 468, 511, 527, 553, 496, 497, 479, 515, 499, 496, 477, 515, 524, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 517, 508, 520, 510, 525, 499, 476, 490, 523, 514, 495, 505, 527, 550, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 533, 522, 525, 536, 495, 531, 511, 480, 489, 501, 513, 524, 498, 512, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 516, 509, 554, 528, 494, 504, 477, 503, 529, 535, 507, 543, 493, 505, 532, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 514, 495, 506, 512, 528, 498, 507, 516, 511, 540, 499, 469, 504, 548, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 523, 529, 485, 527, 498, 529, 500, 477, 492, 524, 498, 523, 503, 507, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 523, 526, 477, 516, 528, 529, 500, 520, 498, 496, 515, 533, 492, 519, 495, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 498, 524, 508, 545, 491, 520, 522, 521, 528, 485, 549, 505, 546, 498, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8200, 8233, 8293, 8205, 8144, 8207, 8231] avg=8192.0000 max=8293 max_rank=3 min=8023 min_rank=0 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[517, 521, 542, 499, 508, 540, 539, 483, 493, 519, 543, 506, 515, 523, 516, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 525, 549, 550, 502, 532, 486, 508, 516, 514, 508, 527, 499, 529, 541, 491, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 503, 489, 525, 550, 501, 517, 511, 513, 536, 493, 529, 517, 476, 488, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 510, 495, 538, 496, 544, 474, 527, 514, 497, 533, 508, 508, 505, 544, 519, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 480, 516, 511, 513, 521, 532, 501, 518, 513, 501, 480, 516, 485, 503, 508, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 506, 508, 526, 514, 467, 558, 517, 528, 489, 494, 513, 483, 538, 521, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 517, 492, 511, 504, 493, 527, 512, 505, 487, 530, 490, 527, 488, 529, 540, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 545, 516, 542, 504, 451, 492, 515, 491, 511, 510, 521, 562, 535, 525, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8264, 8188, 8211, 8085, 8127, 8171, 8189] avg=8192.0000 max=8301 max_rank=0 min=8085 min_rank=4 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 500, 528, 497, 471, 518, 516, 526, 492, 515, 534, 535, 519, 511, 507, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 539, 469, 474, 522, 520, 536, 529, 494, 514, 489, 555, 521, 486, 561, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 539, 533, 498, 528, 508, 523, 495, 476, 473, 504, 527, 499, 511, 485, 555, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 527, 519, 526, 487, 525, 533, 499, 520, 520, 492, 516, 512, 491, 489, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 541, 534, 536, 497, 489, 499, 495, 500, 560, 481, 530, 492, 495, 489, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 482, 495, 522, 514, 548, 545, 546, 513, 539, 542, 545, 485, 505, 511, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 515, 502, 504, 537, 486, 503, 464, 509, 527, 483, 494, 516, 549, 484, 533, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 461, 489, 517, 465, 465, 520, 509, 576, 526, 533, 500, 482, 495, 497, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8235, 8196, 8184, 8202, 8277, 8196, 8046] avg=8192.0000 max=8277 max_rank=5 min=8046 min_rank=7 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[477, 491, 533, 475, 502, 510, 509, 537, 543, 492, 527, 561, 471, 504, 521, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 519, 510, 520, 473, 511, 486, 529, 492, 489, 505, 522, 471, 528, 517, 536, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 508, 558, 477, 520, 509, 484, 521, 495, 528, 506, 489, 545, 536, 513, 446, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 484, 517, 509, 527, 485, 466, 508, 480, 497, 523, 491, 482, 484, 517, 533, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 540, 534, 520, 526, 493, 474, 536, 510, 552, 545, 526, 518, 500, 501, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 611, 479, 489, 498, 511, 502, 512, 485, 495, 508, 540, 512, 535, 538, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 555, 507, 482, 556, 485, 521, 498, 515, 516, 520, 499, 521, 502, 485, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 549, 505, 546, 527, 461, 522, 530, 502, 457, 530, 517, 496, 522, 527, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8111, 8209, 8024, 8314, 8269, 8229, 8207] avg=8192.0000 max=8314 max_rank=4 min=8024 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=33019 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 507, 510, 529, 482, 500, 538, 504, 518, 483, 520, 465, 503, 494, 513, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 486, 544, 498, 509, 508, 528, 546, 529, 501, 495, 533, 503, 523, 534, 486, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 534, 521, 508, 502, 485, 596, 496, 508, 531, 474, 500, 508, 507, 519, 528, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 480, 508, 487, 529, 482, 504, 520, 497, 497, 527, 489, 539, 488, 479, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 487, 514, 575, 482, 505, 516, 507, 495, 574, 521, 530, 530, 563, 505, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 470, 484, 487, 548, 520, 499, 536, 556, 490, 516, 496, 551, 495, 479, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 503, 509, 524, 468, 499, 528, 475, 471, 534, 523, 518, 488, 526, 538, 484, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 526, 522, 529, 487, 498, 527, 536, 556, 477, 490, 530, 514, 493, 520, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8214, 8249, 8068, 8327, 8156, 8151, 8286] avg=8192.0000 max=8327 max_rank=4 min=8068 min_rank=3 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4352, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 3840, 4224, 4224, 4224, 4096, 4224, 4352, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[499, 486, 481, 528, 531, 526, 555, 539, 513, 530, 498, 552, 517, 509, 518, 477, 522, 514, 542, 499, 517, 530, 490, 513, 510, 517, 491, 491, 528, 514, 491, 498, 519, 500, 532, 521, 492, 498, 527, 533, 526, 500, 525, 506, 459, 536, 545, 534, 521, 506, 495, 459, 506, 521, 498, 464, 520, 477, 540, 509, 480, 528, 533, 485, 467, 559, 495, 539, 508, 502, 505, 479, 513, 501, 525, 531, 520, 497, 535, 524, 545, 519, 490, 517, 484, 530, 540, 484, 482, 490, 478, 512, 551, 526, 522, 528, 532, 522, 473, 501, 500, 501, 551, 544, 448, 493, 532, 480, 459, 519, 531, 554, 502, 520, 501, 538, 499, 517, 534, 519, 517, 489, 528, 510, 531, 524, 505, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[477, 500, 559, 497, 535, 491, 488, 563, 517, 520, 519, 504, 499, 526, 520, 499, 523, 522, 534, 495, 493, 520, 548, 496, 497, 505, 514, 544, 535, 492, 553, 504, 535, 516, 551, 459, 529, 530, 511, 533, 548, 536, 498, 515, 503, 546, 495, 511, 501, 516, 515, 497, 521, 514, 529, 496, 496, 515, 498, 490, 524, 509, 514, 525, 534, 494, 517, 563, 523, 508, 545, 516, 510, 532, 503, 497, 499, 460, 497, 529, 465, 493, 498, 517, 502, 500, 512, 484, 538, 533, 514, 488, 507, 542, 473, 502, 485, 516, 524, 484, 499, 506, 549, 515, 539, 468, 497, 510, 474, 523, 479, 528, 497, 530, 530, 483, 523, 522, 521, 472, 508, 506, 517, 538, 519, 508, 532, 474] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[493, 563, 548, 509, 490, 506, 522, 492, 531, 533, 492, 512, 511, 490, 506, 485, 529, 531, 534, 520, 503, 524, 524, 522, 527, 494, 511, 498, 468, 531, 495, 531, 528, 553, 558, 490, 504, 513, 500, 506, 512, 531, 498, 497, 554, 522, 508, 525, 501, 501, 536, 500, 515, 500, 475, 552, 506, 510, 520, 505, 493, 494, 526, 497, 561, 535, 524, 509, 530, 491, 520, 521, 507, 530, 507, 489, 526, 498, 497, 533, 526, 525, 492, 485, 519, 487, 491, 510, 519, 516, 531, 514, 500, 458, 515, 499, 498, 532, 524, 490, 508, 527, 499, 506, 517, 521, 517, 499, 508, 530, 509, 518, 549, 493, 507, 475, 488, 524, 457, 487, 497, 504, 533, 503, 490, 561, 508, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[525, 509, 508, 497, 493, 514, 522, 511, 518, 494, 530, 497, 511, 499, 545, 517, 514, 564, 500, 523, 512, 506, 547, 505, 546, 516, 493, 517, 508, 522, 493, 515, 520, 523, 493, 475, 486, 546, 471, 494, 541, 537, 525, 534, 528, 525, 514, 524, 531, 496, 487, 490, 519, 537, 500, 511, 515, 514, 526, 569, 534, 487, 467, 513, 528, 513, 466, 553, 533, 505, 509, 493, 472, 515, 520, 528, 534, 531, 543, 517, 500, 534, 502, 532, 473, 465, 499, 533, 511, 517, 526, 513, 523, 482, 526, 490, 521, 517, 477, 541, 519, 482, 545, 481, 526, 532, 497, 499, 521, 550, 501, 499, 516, 511, 541, 528, 483, 507, 484, 522, 460, 511, 499, 528, 492, 510, 486, 461] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[510, 500, 518, 534, 550, 520, 488, 515, 492, 520, 496, 541, 543, 494, 502, 510, 503, 514, 558, 534, 525, 518, 528, 538, 528, 506, 482, 573, 543, 516, 544, 513, 510, 476, 510, 476, 552, 518, 528, 536, 503, 526, 521, 507, 513, 486, 506, 530, 513, 489, 534, 520, 502, 506, 498, 515, 488, 487, 494, 525, 555, 542, 525, 542, 514, 514, 489, 540, 554, 564, 481, 538, 516, 512, 512, 505, 495, 507, 504, 523, 517, 527, 505, 515, 500, 526, 553, 528, 517, 504, 493, 501, 524, 506, 513, 473, 526, 516, 484, 514, 474, 499, 543, 508, 481, 496, 481, 476, 503, 488, 534, 520, 470, 502, 521, 466, 502, 517, 486, 518, 480, 482, 515, 533, 494, 481, 481, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[533, 545, 487, 504, 521, 476, 488, 525, 511, 520, 487, 504, 498, 508, 505, 469, 550, 481, 484, 518, 483, 535, 508, 504, 500, 521, 553, 534, 546, 510, 518, 479, 524, 550, 533, 499, 522, 519, 513, 517, 511, 484, 562, 493, 530, 526, 500, 531, 541, 507, 498, 551, 476, 526, 531, 525, 511, 511, 486, 536, 463, 524, 529, 514, 492, 470, 498, 508, 550, 544, 504, 530, 485, 477, 538, 531, 536, 519, 512, 493, 535, 523, 506, 489, 512, 513, 491, 495, 474, 533, 494, 538, 550, 440, 498, 501, 522, 562, 533, 509, 507, 506, 528, 501, 533, 514, 503, 513, 480, 511, 512, 512, 489, 514, 581, 532, 517, 520, 489, 530, 459, 486, 536, 486, 512, 491, 490, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[522, 524, 522, 491, 512, 465, 498, 505, 525, 529, 511, 540, 446, 538, 499, 523, 481, 528, 527, 510, 502, 516, 535, 545, 508, 471, 505, 482, 493, 505, 506, 516, 496, 539, 536, 537, 552, 556, 533, 502, 523, 491, 475, 553, 493, 555, 503, 547, 507, 518, 497, 511, 510, 516, 488, 523, 498, 490, 501, 511, 516, 504, 506, 467, 562, 500, 518, 518, 527, 491, 517, 538, 533, 555, 509, 472, 537, 509, 496, 523, 520, 505, 522, 478, 535, 534, 528, 528, 512, 471, 552, 495, 484, 523, 529, 488, 528, 514, 502, 502, 548, 494, 469, 482, 524, 499, 470, 507, 528, 484, 533, 515, 478, 523, 533, 519, 511, 537, 510, 484, 513, 541, 492, 523, 507, 508, 523, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[509, 537, 522, 533, 521, 528, 530, 476, 534, 493, 512, 502, 530, 500, 555, 491, 499, 530, 521, 538, 502, 524, 484, 490, 556, 503, 587, 528, 516, 501, 565, 503, 546, 500, 520, 502, 518, 521, 498, 503, 527, 471, 511, 536, 535, 502, 539, 485, 487, 480, 503, 507, 525, 483, 495, 555, 513, 526, 522, 449, 462, 500, 496, 522, 504, 506, 505, 542, 531, 526, 498, 528, 510, 512, 530, 513, 493, 517, 477, 508, 525, 503, 579, 502, 475, 516, 519, 501, 536, 532, 525, 550, 471, 488, 517, 508, 502, 527, 508, 524, 511, 502, 495, 536, 513, 526, 520, 512, 530, 456, 530, 487, 500, 497, 493, 481, 505, 546, 500, 511, 521, 503, 509, 524, 482, 479, 502, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[493, 557, 548, 509, 490, 506, 522, 492, 531, 533, 492, 512, 511, 490, 506, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 531, 464, 520, 503, 524, 524, 522, 527, 494, 511, 498, 468, 531, 495, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 553, 468, 490, 504, 513, 500, 506, 512, 531, 498, 497, 554, 522, 508, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 536, 500, 515, 500, 475, 552, 506, 510, 520, 505, 493, 494, 526, 497, 45, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 535, 524, 464, 530, 491, 520, 521, 507, 530, 507, 489, 526, 498, 497, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 492, 485, 519, 487, 491, 510, 519, 516, 531, 514, 500, 458, 515, 499, 31, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 532, 524, 490, 508, 527, 499, 506, 517, 521, 517, 499, 508, 530, 509, 518, 25, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 493, 507, 475, 488, 524, 457, 487, 497, 504, 533, 503, 490, 561, 508, 537, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8172, 8209, 8189, 8233, 8124, 8242, 8190] avg=8192.0000 max=8242 max_rank=6 min=8124 min_rank=5 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[499, 481, 481, 528, 531, 526, 555, 539, 513, 530, 498, 552, 517, 509, 518, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 472, 499, 517, 530, 490, 513, 510, 517, 491, 491, 528, 514, 491, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 447, 521, 492, 498, 527, 533, 526, 500, 525, 506, 459, 536, 545, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 506, 495, 459, 506, 521, 498, 464, 520, 477, 540, 509, 480, 528, 533, 485, 45, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 559, 495, 491, 508, 502, 505, 479, 513, 501, 525, 531, 520, 497, 535, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 490, 517, 484, 530, 540, 484, 482, 490, 478, 512, 551, 526, 522, 528, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 522, 473, 501, 500, 501, 551, 544, 448, 493, 532, 480, 459, 519, 531, 554, 25, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 501, 538, 499, 517, 534, 519, 517, 489, 528, 510, 531, 524, 505, 543, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8097, 8168, 8099, 8152, 8236, 8180, 8350] avg=8192.0000 max=8350 max_rank=7 min=8097 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 495, 518, 534, 550, 520, 488, 515, 492, 520, 496, 541, 543, 494, 502, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 485, 534, 525, 518, 528, 538, 528, 506, 482, 573, 543, 516, 544, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 476, 428, 476, 552, 518, 528, 536, 503, 526, 521, 507, 513, 486, 506, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 489, 534, 520, 502, 506, 498, 515, 488, 487, 494, 525, 555, 542, 525, 542, 47, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 489, 492, 554, 564, 481, 538, 516, 512, 512, 505, 495, 507, 504, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 527, 505, 515, 500, 526, 553, 528, 517, 504, 493, 501, 524, 506, 513, 473, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 516, 484, 514, 474, 499, 543, 508, 481, 496, 481, 476, 503, 488, 534, 520, 26, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 502, 521, 466, 502, 517, 486, 518, 480, 482, 515, 533, 494, 481, 481, 486, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8350, 8116, 8294, 8220, 8240, 8084, 8004] avg=8192.0000 max=8350 max_rank=1 min=8004 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 504, 508, 497, 493, 514, 522, 511, 518, 494, 530, 497, 511, 499, 545, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 564, 435, 523, 512, 506, 547, 505, 546, 516, 493, 517, 508, 522, 493, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 414, 475, 486, 546, 471, 494, 541, 537, 525, 534, 528, 525, 514, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 496, 487, 490, 519, 537, 500, 511, 515, 514, 526, 569, 534, 487, 467, 513, 42, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 466, 503, 533, 505, 509, 493, 472, 515, 520, 528, 534, 531, 543, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 534, 502, 532, 473, 465, 499, 533, 511, 517, 526, 513, 523, 482, 526, 490, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 477, 541, 519, 482, 545, 481, 526, 532, 497, 499, 521, 550, 501, 499, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 541, 528, 483, 507, 484, 522, 460, 511, 499, 528, 492, 510, 486, 461, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8216, 8157, 8249, 8210, 8165, 8247, 8107] avg=8192.0000 max=8249 max_rank=3 min=8107 min_rank=7 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[533, 535, 487, 504, 521, 476, 488, 525, 511, 520, 487, 504, 498, 508, 505, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 481, 415, 518, 483, 535, 508, 504, 500, 521, 553, 534, 546, 510, 518, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 550, 442, 499, 522, 519, 513, 517, 511, 484, 562, 493, 530, 526, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 507, 498, 551, 476, 526, 531, 525, 511, 511, 486, 536, 463, 524, 529, 514, 43, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 470, 498, 455, 550, 544, 504, 530, 485, 477, 538, 531, 536, 519, 512, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 523, 506, 489, 512, 513, 491, 495, 474, 533, 494, 538, 550, 440, 498, 501, 35, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 562, 533, 509, 507, 506, 528, 501, 533, 514, 503, 513, 480, 511, 512, 512, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 514, 581, 532, 517, 520, 489, 530, 459, 486, 536, 486, 512, 491, 490, 531, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8155, 8223, 8287, 8134, 8137, 8290, 8239] avg=8192.0000 max=8290 max_rank=6 min=8071 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 531, 522, 533, 521, 528, 530, 476, 534, 493, 512, 502, 530, 500, 555, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 530, 453, 538, 502, 524, 484, 490, 556, 503, 587, 528, 516, 501, 565, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 500, 437, 502, 518, 521, 498, 503, 527, 471, 511, 536, 535, 502, 539, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 480, 503, 507, 525, 483, 495, 555, 513, 526, 522, 449, 462, 500, 496, 522, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 505, 494, 531, 526, 498, 528, 510, 512, 530, 513, 493, 517, 477, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 503, 579, 502, 475, 516, 519, 501, 536, 532, 525, 550, 471, 488, 517, 508, 33, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 527, 508, 524, 511, 502, 495, 536, 513, 526, 520, 512, 530, 456, 530, 487, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 493, 481, 505, 546, 500, 511, 521, 503, 509, 524, 482, 479, 502, 498, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8279, 8131, 8081, 8152, 8286, 8218, 8122] avg=8192.0000 max=8286 max_rank=5 min=8081 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 519, 522, 491, 512, 465, 498, 505, 525, 529, 511, 540, 446, 538, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 528, 459, 510, 502, 516, 535, 545, 508, 471, 505, 482, 493, 505, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 539, 450, 537, 552, 556, 533, 502, 523, 491, 475, 553, 493, 555, 503, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 497, 511, 510, 516, 488, 523, 498, 490, 501, 511, 516, 504, 506, 467, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 500, 518, 471, 527, 491, 517, 538, 533, 555, 509, 472, 537, 509, 496, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 522, 478, 535, 534, 528, 528, 512, 471, 552, 495, 484, 523, 529, 488, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 514, 502, 502, 548, 494, 469, 482, 524, 499, 470, 507, 528, 484, 533, 515, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 523, 533, 519, 511, 537, 510, 484, 513, 541, 492, 523, 507, 508, 523, 492, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8062, 8305, 8119, 8258, 8241, 8138, 8268] avg=8192.0000 max=8305 max_rank=2 min=8062 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[477, 495, 559, 497, 535, 491, 488, 563, 517, 520, 519, 504, 499, 526, 520, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 464, 495, 493, 520, 548, 496, 497, 505, 514, 544, 535, 492, 553, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 462, 459, 529, 530, 511, 533, 548, 536, 498, 515, 503, 546, 495, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 515, 497, 521, 514, 529, 496, 496, 515, 498, 490, 524, 509, 514, 525, 45, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 494, 517, 513, 523, 508, 545, 516, 510, 532, 503, 497, 499, 460, 497, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 493, 498, 517, 502, 500, 512, 484, 538, 533, 514, 488, 507, 542, 473, 502, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 516, 524, 484, 499, 506, 549, 515, 539, 468, 497, 510, 474, 523, 479, 528, 25, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 530, 530, 483, 523, 522, 521, 472, 508, 506, 517, 538, 519, 508, 532, 474, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8205, 8227, 8218, 8177, 8107, 8137, 8256] avg=8192.0000 max=8256 max_rank=7 min=8107 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3712, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 3584, 3968, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 3968, 4352, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[539, 511, 547, 515, 546, 538, 476, 475, 545, 535, 502, 532, 538, 517, 533, 501, 518, 515, 513, 521, 522, 480, 520, 499, 530, 540, 514, 481, 530, 543, 493, 500, 508, 532, 522, 521, 486, 508, 532, 499, 506, 518, 536, 537, 490, 512, 523, 530, 492, 498, 532, 538, 463, 479, 534, 493, 512, 538, 480, 530, 526, 514, 512, 503, 515, 509, 502, 487, 498, 491, 524, 498, 514, 541, 478, 508, 515, 524, 487, 499, 504, 495, 536, 493, 480, 488, 481, 475, 519, 508, 511, 534, 502, 474, 516, 521, 515, 519, 478, 532, 479, 520, 495, 489, 511, 478, 563, 504, 574, 518, 522, 509, 500, 525, 508, 503, 516, 535, 547, 509, 507, 530, 504, 540, 498, 505, 509, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[497, 523, 496, 522, 476, 519, 528, 518, 539, 500, 524, 552, 539, 477, 514, 520, 545, 479, 520, 527, 501, 520, 476, 525, 482, 480, 499, 486, 521, 512, 471, 515, 518, 516, 497, 526, 501, 468, 502, 527, 509, 464, 535, 496, 509, 514, 499, 504, 529, 505, 479, 526, 532, 503, 500, 509, 535, 549, 516, 543, 511, 525, 516, 509, 574, 506, 483, 499, 483, 517, 512, 531, 544, 525, 490, 522, 528, 523, 521, 536, 535, 480, 502, 484, 536, 494, 523, 483, 467, 517, 518, 530, 537, 521, 544, 479, 540, 471, 554, 485, 505, 557, 540, 523, 535, 556, 516, 510, 512, 475, 487, 487, 510, 516, 535, 516, 496, 558, 521, 468, 502, 471, 512, 495, 535, 503, 528, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[491, 500, 507, 505, 511, 522, 518, 498, 540, 535, 500, 530, 522, 502, 519, 529, 493, 513, 512, 551, 552, 488, 514, 510, 483, 509, 511, 530, 514, 498, 555, 543, 501, 501, 529, 520, 499, 511, 514, 468, 460, 520, 518, 571, 530, 498, 528, 489, 486, 545, 507, 486, 497, 489, 537, 527, 519, 525, 477, 518, 501, 488, 519, 544, 484, 489, 492, 501, 535, 506, 505, 477, 505, 509, 522, 495, 518, 551, 533, 516, 509, 486, 504, 547, 514, 508, 507, 514, 495, 526, 505, 535, 488, 510, 551, 536, 529, 516, 510, 487, 497, 495, 519, 521, 505, 487, 515, 524, 507, 559, 460, 542, 520, 487, 530, 515, 509, 496, 549, 497, 516, 473, 510, 499, 503, 504, 541, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[481, 522, 497, 471, 523, 512, 547, 498, 493, 504, 493, 518, 481, 491, 519, 502, 547, 481, 498, 511, 553, 513, 498, 529, 542, 549, 540, 516, 517, 506, 528, 532, 463, 488, 541, 521, 525, 524, 485, 491, 503, 518, 521, 462, 529, 529, 527, 539, 468, 548, 556, 541, 489, 469, 513, 505, 537, 529, 518, 499, 505, 497, 510, 539, 535, 500, 526, 484, 479, 464, 524, 485, 521, 490, 527, 512, 531, 550, 477, 521, 541, 522, 506, 528, 554, 515, 498, 476, 510, 548, 538, 534, 477, 521, 507, 500, 519, 510, 495, 526, 512, 503, 480, 499, 547, 532, 532, 486, 509, 521, 504, 513, 528, 491, 512, 508, 492, 510, 534, 529, 484, 519, 477, 531, 489, 529, 531, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[519, 534, 490, 494, 529, 513, 531, 477, 523, 513, 529, 511, 491, 495, 495, 521, 544, 519, 548, 510, 496, 506, 488, 535, 547, 491, 555, 527, 482, 528, 515, 509, 500, 477, 518, 509, 538, 495, 520, 517, 492, 460, 461, 500, 531, 500, 487, 527, 508, 513, 544, 525, 518, 530, 479, 500, 533, 495, 532, 507, 510, 481, 487, 520, 519, 507, 534, 519, 530, 508, 508, 508, 500, 509, 534, 510, 534, 497, 507, 536, 518, 501, 489, 530, 536, 529, 494, 536, 493, 524, 534, 494, 542, 529, 527, 533, 510, 506, 507, 533, 513, 517, 493, 523, 491, 519, 503, 530, 543, 534, 479, 530, 486, 520, 488, 467, 537, 485, 485, 511, 513, 542, 530, 471, 507, 484, 495, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[514, 490, 502, 545, 512, 538, 512, 519, 498, 562, 530, 506, 535, 521, 500, 513, 514, 519, 493, 514, 537, 486, 536, 503, 518, 502, 527, 542, 477, 543, 512, 498, 484, 475, 551, 500, 530, 514, 515, 489, 472, 530, 543, 503, 514, 533, 543, 521, 493, 533, 498, 458, 511, 482, 475, 499, 495, 505, 555, 507, 491, 516, 523, 530, 475, 516, 533, 574, 494, 527, 504, 531, 512, 557, 524, 522, 480, 536, 537, 505, 503, 510, 487, 487, 505, 500, 536, 478, 542, 493, 491, 482, 496, 517, 493, 485, 518, 526, 510, 498, 532, 519, 541, 522, 505, 534, 513, 518, 509, 512, 542, 505, 506, 506, 533, 469, 489, 494, 512, 541, 514, 498, 494, 532, 477, 512, 477, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[545, 515, 541, 506, 534, 539, 493, 481, 539, 499, 479, 488, 486, 525, 506, 512, 529, 492, 469, 487, 507, 527, 520, 514, 554, 562, 490, 493, 514, 508, 528, 506, 505, 519, 545, 498, 519, 543, 573, 526, 502, 528, 469, 522, 502, 520, 524, 474, 491, 513, 505, 571, 471, 486, 518, 551, 513, 529, 567, 527, 507, 499, 514, 503, 476, 489, 505, 485, 577, 492, 521, 493, 487, 530, 517, 481, 530, 544, 520, 515, 487, 505, 473, 559, 504, 500, 494, 496, 528, 524, 533, 496, 505, 501, 510, 542, 487, 535, 509, 505, 515, 530, 518, 496, 525, 506, 510, 513, 491, 478, 506, 513, 522, 504, 482, 521, 514, 534, 495, 495, 528, 523, 511, 512, 514, 477, 510, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[511, 534, 525, 493, 525, 535, 520, 553, 530, 505, 509, 487, 547, 542, 493, 526, 549, 513, 546, 533, 522, 505, 505, 490, 536, 514, 513, 517, 518, 476, 514, 520, 464, 501, 522, 498, 480, 522, 573, 505, 493, 504, 503, 530, 507, 515, 471, 511, 539, 496, 536, 456, 502, 505, 484, 485, 511, 536, 517, 532, 519, 521, 496, 537, 528, 547, 538, 493, 483, 526, 561, 547, 555, 517, 444, 523, 536, 492, 511, 511, 529, 507, 490, 505, 493, 520, 454, 482, 503, 475, 494, 494, 501, 515, 521, 548, 497, 538, 525, 506, 496, 492, 499, 495, 504, 513, 529, 503, 495, 476, 532, 512, 532, 506, 530, 551, 524, 531, 494, 527, 483, 523, 524, 505, 498, 494, 492, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[497, 523, 496, 522, 476, 480, 528, 518, 539, 500, 524, 552, 539, 477, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 479, 520, 527, 501, 520, 476, 525, 482, 480, 499, 486, 521, 512, 471, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 497, 526, 501, 468, 502, 527, 509, 464, 535, 496, 509, 514, 499, 504, 21, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 505, 479, 526, 532, 503, 500, 509, 535, 549, 516, 543, 511, 525, 516, 509, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 506, 483, 499, 483, 517, 512, 531, 544, 525, 490, 522, 528, 502, 521, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 480, 502, 484, 536, 494, 523, 483, 467, 517, 518, 530, 537, 521, 544, 479, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 471, 554, 485, 505, 557, 540, 523, 535, 556, 508, 510, 512, 475, 487, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 535, 516, 496, 558, 521, 468, 502, 471, 512, 495, 535, 503, 528, 498, 35, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8012, 8115, 8290, 8273, 8193, 8245, 8203] avg=8192.0000 max=8290 max_rank=3 min=8012 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[539, 511, 547, 515, 546, 498, 476, 475, 545, 535, 502, 532, 538, 517, 533, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 515, 513, 521, 522, 480, 520, 499, 530, 540, 514, 481, 530, 543, 493, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 532, 522, 521, 486, 508, 532, 499, 506, 518, 536, 537, 490, 512, 523, 530, 21, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 498, 532, 538, 463, 479, 534, 493, 512, 538, 480, 530, 526, 514, 512, 503, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 502, 487, 498, 491, 524, 498, 514, 541, 478, 508, 515, 503, 487, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 495, 536, 493, 480, 488, 481, 475, 519, 508, 511, 534, 502, 474, 516, 521, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 519, 478, 532, 479, 520, 495, 489, 511, 478, 555, 504, 574, 518, 522, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 525, 508, 503, 516, 535, 547, 509, 507, 530, 504, 540, 498, 505, 509, 494, 36, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8174, 8290, 8147, 8069, 8078, 8198, 8270] avg=8192.0000 max=8310 max_rank=0 min=8069 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[481, 522, 497, 471, 523, 474, 547, 498, 493, 504, 493, 518, 481, 491, 519, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 481, 498, 511, 553, 513, 498, 529, 542, 549, 540, 516, 517, 506, 528, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 488, 541, 521, 525, 524, 485, 491, 503, 518, 521, 462, 529, 529, 527, 539, 22, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 548, 556, 541, 489, 469, 513, 505, 537, 529, 518, 499, 505, 497, 510, 539, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 526, 484, 479, 464, 524, 485, 521, 490, 527, 512, 531, 528, 477, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 522, 506, 528, 554, 515, 498, 476, 510, 548, 538, 534, 477, 521, 507, 500, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 495, 526, 512, 503, 480, 499, 547, 532, 524, 486, 509, 521, 504, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 491, 512, 508, 492, 510, 534, 529, 484, 519, 477, 531, 489, 529, 531, 482, 34, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8014, 8313, 8197, 8226, 8104, 8318, 8180, 8184] avg=8192.0000 max=8318 max_rank=5 min=8014 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[519, 534, 490, 494, 529, 465, 531, 477, 523, 513, 529, 511, 491, 495, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 519, 548, 510, 496, 506, 488, 535, 547, 491, 555, 527, 482, 528, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 477, 518, 509, 538, 495, 520, 517, 492, 460, 461, 500, 531, 500, 487, 527, 24, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 513, 544, 525, 518, 530, 479, 500, 533, 495, 532, 507, 510, 481, 487, 520, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 534, 519, 530, 508, 508, 508, 500, 509, 534, 510, 534, 473, 507, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 489, 530, 536, 529, 494, 536, 493, 524, 534, 494, 542, 529, 527, 533, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 507, 533, 513, 517, 493, 523, 491, 519, 491, 530, 543, 534, 479, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 520, 488, 467, 537, 485, 485, 511, 513, 542, 530, 471, 507, 484, 495, 536, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8248, 8071, 8188, 8236, 8355, 8219, 8102] avg=8192.0000 max=8355 max_rank=5 min=8071 min_rank=2 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[491, 500, 507, 505, 511, 483, 518, 498, 540, 535, 500, 530, 522, 502, 519, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 513, 512, 551, 552, 488, 514, 510, 483, 509, 511, 530, 514, 498, 555, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 529, 520, 499, 511, 514, 468, 460, 520, 518, 571, 530, 498, 528, 489, 22, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 545, 507, 486, 497, 489, 537, 527, 519, 525, 477, 518, 501, 488, 519, 544, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 489, 492, 501, 535, 506, 505, 477, 505, 509, 522, 495, 518, 529, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 486, 504, 547, 514, 508, 507, 514, 495, 526, 505, 535, 488, 510, 551, 536, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 516, 510, 487, 497, 495, 519, 521, 505, 487, 507, 524, 507, 559, 460, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 530, 515, 509, 496, 549, 497, 516, 473, 510, 499, 503, 504, 541, 514, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8234, 8188, 8168, 8116, 8274, 8165, 8201] avg=8192.0000 max=8274 max_rank=5 min=8116 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[514, 490, 502, 545, 512, 498, 512, 519, 498, 562, 530, 506, 535, 521, 500, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 519, 493, 514, 537, 486, 536, 503, 518, 502, 527, 542, 477, 543, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 475, 551, 500, 530, 514, 515, 489, 472, 530, 543, 503, 514, 533, 543, 521, 21, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 533, 498, 458, 511, 482, 475, 499, 495, 505, 555, 507, 491, 516, 523, 530, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 516, 533, 574, 494, 527, 504, 531, 512, 557, 524, 522, 480, 515, 537, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 487, 487, 505, 500, 536, 478, 542, 493, 491, 482, 496, 517, 493, 485, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 526, 510, 498, 532, 519, 541, 522, 505, 534, 505, 518, 509, 512, 542, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 506, 533, 469, 489, 494, 512, 541, 514, 498, 494, 532, 477, 512, 477, 540, 36, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8177, 8247, 8074, 8306, 8045, 8296, 8134] avg=8192.0000 max=8306 max_rank=4 min=8045 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[545, 515, 541, 506, 534, 499, 493, 481, 539, 499, 479, 488, 486, 525, 506, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 492, 469, 487, 507, 527, 520, 514, 554, 562, 490, 493, 514, 508, 528, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 519, 545, 498, 519, 543, 573, 526, 502, 528, 469, 522, 502, 520, 524, 474, 22, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 513, 505, 571, 471, 486, 518, 551, 513, 529, 567, 527, 507, 499, 514, 503, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 489, 505, 485, 577, 492, 521, 493, 487, 530, 517, 481, 530, 522, 520, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 505, 473, 559, 504, 500, 494, 496, 528, 524, 533, 496, 505, 501, 510, 542, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 535, 509, 505, 515, 530, 518, 496, 525, 506, 502, 513, 491, 478, 506, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 482, 521, 514, 534, 495, 495, 528, 523, 511, 512, 514, 477, 510, 516, 36, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8154, 8300, 8268, 8140, 8199, 8129, 8198] avg=8192.0000 max=8300 max_rank=2 min=8129 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 534, 525, 493, 525, 495, 520, 553, 530, 505, 509, 487, 547, 542, 493, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 546, 533, 522, 505, 505, 490, 536, 514, 513, 517, 518, 476, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 501, 522, 498, 480, 522, 573, 505, 493, 504, 503, 530, 507, 515, 471, 511, 20, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 496, 536, 456, 502, 505, 484, 485, 511, 536, 517, 532, 519, 521, 496, 537, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 547, 538, 493, 483, 526, 561, 547, 555, 517, 444, 523, 536, 472, 511, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 490, 505, 493, 520, 454, 482, 503, 475, 494, 494, 501, 515, 521, 548, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 538, 525, 506, 496, 492, 499, 495, 504, 513, 521, 503, 495, 476, 532, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 506, 530, 551, 524, 531, 494, 527, 483, 523, 524, 505, 498, 494, 492, 490, 36, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8224, 8128, 8175, 8292, 8074, 8104, 8244] avg=8192.0000 max=8295 max_rank=0 min=8074 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4352, 4096, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[507, 519, 501, 517, 527, 530, 497, 553, 533, 541, 498, 560, 472, 552, 527, 511, 490, 522, 491, 512, 521, 491, 509, 556, 474, 477, 512, 509, 551, 496, 501, 509, 528, 491, 473, 483, 468, 475, 517, 489, 505, 529, 505, 522, 525, 519, 546, 474, 497, 496, 512, 529, 485, 482, 519, 519, 503, 533, 533, 513, 547, 535, 522, 520, 515, 491, 496, 526, 538, 563, 515, 481, 502, 518, 507, 480, 521, 502, 518, 507, 473, 492, 490, 508, 508, 581, 500, 535, 493, 501, 492, 516, 516, 464, 499, 511, 521, 534, 491, 518, 520, 545, 500, 504, 507, 529, 555, 482, 489, 516, 500, 502, 513, 501, 536, 501, 550, 493, 516, 538, 553, 503, 535, 483, 521, 506, 535, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[504, 506, 534, 511, 444, 539, 492, 515, 515, 508, 493, 527, 493, 504, 516, 526, 514, 469, 552, 532, 500, 539, 498, 485, 514, 520, 505, 505, 531, 498, 552, 520, 483, 532, 548, 504, 495, 506, 511, 525, 487, 497, 507, 527, 517, 512, 496, 507, 513, 537, 468, 544, 531, 516, 530, 508, 496, 474, 513, 457, 487, 497, 513, 520, 520, 534, 504, 528, 506, 519, 524, 541, 537, 540, 539, 531, 485, 527, 497, 500, 539, 486, 513, 526, 505, 542, 491, 506, 527, 588, 500, 517, 489, 527, 525, 496, 493, 524, 542, 512, 507, 506, 473, 524, 518, 515, 524, 521, 500, 544, 495, 517, 513, 488, 531, 504, 507, 488, 507, 506, 522, 489, 525, 517, 506, 481, 502, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[543, 513, 551, 500, 496, 535, 544, 518, 495, 533, 545, 511, 460, 526, 522, 502, 515, 496, 525, 538, 469, 526, 516, 498, 523, 485, 488, 516, 523, 487, 522, 493, 514, 516, 531, 510, 531, 498, 526, 515, 492, 522, 552, 527, 511, 503, 512, 491, 504, 475, 486, 491, 519, 487, 516, 513, 497, 572, 524, 517, 529, 531, 528, 514, 548, 548, 517, 515, 501, 480, 523, 510, 491, 508, 526, 540, 529, 497, 509, 506, 494, 513, 495, 504, 509, 533, 508, 520, 547, 513, 495, 535, 493, 510, 468, 482, 488, 515, 541, 522, 518, 521, 542, 500, 447, 503, 504, 509, 531, 525, 497, 516, 501, 479, 545, 489, 507, 531, 495, 521, 489, 491, 513, 510, 539, 469, 531, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[530, 516, 505, 512, 475, 537, 531, 489, 508, 538, 555, 503, 548, 509, 539, 487, 514, 520, 523, 535, 478, 471, 526, 513, 504, 522, 472, 507, 547, 536, 532, 497, 494, 508, 517, 500, 519, 526, 510, 504, 507, 486, 505, 532, 499, 491, 558, 501, 507, 531, 499, 491, 488, 519, 530, 476, 475, 532, 470, 509, 494, 525, 520, 525, 482, 472, 474, 519, 496, 525, 467, 508, 531, 540, 527, 480, 516, 473, 516, 539, 516, 491, 530, 525, 554, 545, 539, 546, 505, 500, 516, 499, 479, 484, 501, 483, 532, 512, 503, 511, 516, 532, 507, 501, 500, 507, 510, 496, 546, 548, 487, 511, 512, 527, 526, 487, 487, 523, 538, 541, 507, 518, 505, 549, 504, 525, 534, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[506, 521, 490, 506, 531, 489, 538, 543, 511, 493, 515, 474, 508, 466, 473, 520, 507, 541, 524, 516, 508, 503, 521, 526, 483, 541, 518, 492, 522, 508, 511, 499, 485, 532, 509, 532, 516, 545, 506, 533, 543, 528, 481, 528, 509, 492, 547, 542, 507, 539, 505, 531, 524, 518, 520, 519, 509, 510, 503, 500, 539, 519, 531, 506, 470, 489, 529, 532, 542, 500, 492, 486, 511, 468, 538, 503, 491, 506, 523, 486, 522, 495, 567, 507, 519, 478, 513, 527, 536, 505, 531, 518, 505, 483, 491, 460, 477, 515, 497, 513, 526, 557, 511, 511, 542, 536, 512, 502, 461, 506, 498, 521, 504, 521, 478, 513, 507, 522, 501, 497, 553, 520, 525, 536, 514, 495, 476, 554] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[518, 507, 570, 466, 494, 494, 523, 506, 522, 513, 481, 503, 483, 564, 512, 502, 496, 510, 491, 498, 512, 545, 477, 548, 560, 529, 510, 521, 523, 511, 487, 500, 538, 507, 479, 542, 484, 531, 477, 500, 513, 542, 499, 491, 510, 473, 511, 494, 526, 529, 509, 500, 527, 516, 506, 518, 485, 526, 509, 506, 507, 491, 493, 502, 489, 531, 480, 564, 513, 488, 540, 480, 503, 518, 517, 536, 515, 517, 487, 499, 507, 523, 545, 546, 542, 518, 487, 496, 520, 496, 517, 516, 526, 482, 517, 528, 501, 529, 504, 517, 493, 503, 481, 504, 508, 509, 540, 571, 508, 524, 507, 485, 482, 536, 552, 520, 545, 542, 521, 503, 534, 483, 530, 523, 510, 491, 526, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[512, 502, 490, 528, 496, 551, 509, 540, 493, 493, 483, 533, 472, 505, 552, 505, 491, 551, 537, 510, 520, 489, 551, 487, 495, 487, 507, 463, 501, 528, 514, 545, 513, 495, 518, 483, 506, 519, 495, 519, 545, 516, 505, 541, 481, 529, 495, 523, 514, 556, 511, 524, 553, 499, 500, 512, 509, 494, 498, 526, 551, 533, 484, 487, 510, 515, 530, 512, 495, 515, 529, 493, 461, 520, 501, 536, 473, 509, 515, 501, 473, 520, 522, 508, 500, 526, 535, 448, 525, 501, 508, 519, 492, 509, 486, 534, 512, 522, 514, 515, 491, 542, 563, 534, 544, 507, 507, 541, 505, 515, 513, 552, 576, 494, 481, 518, 530, 486, 536, 506, 539, 499, 533, 524, 460, 510, 480, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[511, 540, 483, 498, 515, 549, 532, 507, 526, 536, 501, 512, 514, 503, 513, 499, 501, 552, 446, 525, 543, 514, 543, 480, 551, 521, 516, 537, 502, 494, 528, 528, 508, 549, 523, 538, 506, 547, 497, 504, 500, 547, 485, 528, 522, 507, 510, 512, 486, 497, 525, 554, 514, 519, 550, 507, 532, 474, 497, 506, 512, 493, 494, 537, 500, 503, 505, 522, 529, 510, 524, 515, 518, 506, 508, 502, 536, 465, 525, 504, 533, 482, 492, 474, 517, 512, 550, 540, 506, 498, 503, 514, 487, 518, 516, 503, 502, 514, 509, 473, 526, 480, 475, 498, 466, 511, 547, 502, 525, 526, 522, 517, 502, 530, 499, 498, 503, 527, 494, 500, 553, 520, 488, 493, 541, 471, 512, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[507, 519, 501, 517, 527, 511, 497, 553, 533, 541, 498, 560, 472, 552, 527, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 522, 491, 512, 521, 491, 509, 556, 474, 477, 512, 509, 548, 496, 501, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 491, 473, 483, 468, 475, 517, 489, 505, 529, 505, 522, 525, 519, 546, 474, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 496, 512, 529, 485, 482, 519, 519, 503, 533, 533, 513, 547, 535, 522, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 491, 496, 526, 538, 563, 515, 481, 502, 518, 507, 480, 521, 502, 518, 507, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 492, 490, 508, 508, 581, 500, 535, 493, 501, 492, 516, 516, 464, 499, 511, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 534, 491, 518, 520, 545, 500, 504, 507, 529, 555, 482, 489, 502, 500, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 536, 501, 550, 493, 516, 538, 542, 503, 535, 483, 521, 506, 535, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8118, 8054, 8246, 8201, 8099, 8199, 8293] avg=8192.0000 max=8326 max_rank=0 min=8054 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[504, 506, 534, 511, 444, 519, 492, 515, 515, 508, 493, 527, 493, 504, 516, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 469, 552, 532, 500, 539, 498, 485, 514, 520, 505, 505, 528, 498, 552, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 532, 548, 504, 495, 506, 511, 525, 487, 497, 507, 527, 517, 512, 496, 507, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 468, 544, 531, 516, 530, 508, 496, 474, 513, 457, 487, 497, 513, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 534, 504, 528, 506, 519, 524, 541, 537, 540, 539, 531, 485, 527, 497, 500, 20, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 486, 513, 526, 505, 542, 491, 506, 527, 588, 500, 517, 489, 527, 525, 496, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 524, 542, 512, 507, 506, 473, 524, 518, 515, 524, 521, 500, 528, 495, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 488, 531, 504, 507, 488, 507, 506, 512, 489, 525, 517, 506, 481, 502, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8231, 8158, 8105, 8355, 8298, 8199, 8083] avg=8192.0000 max=8355 max_rank=4 min=8083 min_rank=7 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[543, 513, 551, 500, 496, 512, 544, 518, 495, 533, 545, 511, 460, 526, 522, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 496, 525, 538, 469, 526, 516, 498, 523, 485, 488, 516, 509, 487, 522, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 516, 531, 510, 531, 498, 526, 515, 492, 522, 552, 527, 511, 503, 512, 491, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 475, 486, 491, 519, 487, 516, 513, 497, 572, 524, 517, 529, 531, 528, 514, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 548, 517, 515, 501, 480, 523, 510, 491, 508, 526, 540, 529, 497, 509, 506, 23, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 513, 495, 504, 509, 533, 508, 520, 547, 513, 495, 535, 493, 510, 468, 482, 17, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 515, 541, 522, 518, 521, 542, 500, 447, 503, 504, 509, 531, 501, 497, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 479, 545, 489, 507, 531, 495, 521, 471, 491, 513, 510, 539, 469, 531, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8106, 8262, 8211, 8278, 8149, 8155, 8104] avg=8192.0000 max=8278 max_rank=4 min=8104 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 507, 570, 466, 494, 476, 523, 506, 522, 513, 481, 503, 483, 564, 512, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 491, 498, 512, 545, 477, 548, 560, 529, 510, 521, 521, 511, 487, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 507, 479, 542, 484, 531, 477, 500, 513, 542, 499, 491, 510, 473, 511, 494, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 529, 509, 500, 527, 516, 506, 518, 485, 526, 509, 506, 507, 491, 493, 502, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 531, 480, 564, 513, 488, 540, 480, 503, 518, 517, 536, 515, 517, 487, 499, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 523, 545, 546, 542, 518, 487, 496, 520, 496, 517, 516, 526, 482, 517, 528, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 529, 504, 517, 493, 503, 481, 504, 508, 509, 540, 571, 508, 509, 507, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 536, 552, 520, 545, 542, 521, 503, 523, 483, 530, 523, 510, 491, 526, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8216, 8095, 8151, 8197, 8287, 8169, 8281] avg=8192.0000 max=8287 max_rank=5 min=8095 min_rank=2 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 516, 505, 512, 475, 518, 531, 489, 508, 538, 555, 503, 548, 509, 539, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 520, 523, 535, 478, 471, 526, 513, 504, 522, 472, 507, 544, 536, 532, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 517, 500, 519, 526, 510, 504, 507, 486, 505, 532, 499, 491, 558, 501, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 531, 499, 491, 488, 519, 530, 476, 475, 532, 470, 509, 494, 525, 520, 525, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 472, 474, 519, 496, 525, 467, 508, 531, 540, 527, 480, 516, 473, 516, 539, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 491, 530, 525, 554, 545, 539, 546, 505, 500, 516, 499, 479, 484, 501, 483, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 503, 511, 516, 532, 507, 501, 500, 507, 510, 496, 546, 532, 487, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 526, 487, 487, 523, 538, 541, 497, 518, 505, 549, 504, 525, 534, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8194, 8161, 8092, 8087, 8234, 8203, 8302] avg=8192.0000 max=8302 max_rank=7 min=8087 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[506, 521, 490, 506, 531, 471, 538, 543, 511, 493, 515, 474, 508, 466, 473, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 541, 524, 516, 508, 503, 521, 526, 483, 541, 518, 492, 520, 508, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 532, 509, 532, 516, 545, 506, 533, 543, 528, 481, 528, 509, 492, 547, 542, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 539, 505, 531, 524, 518, 520, 519, 509, 510, 503, 500, 539, 519, 531, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 489, 529, 532, 542, 500, 492, 486, 511, 468, 538, 503, 491, 506, 523, 486, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 567, 507, 519, 478, 513, 527, 536, 505, 531, 518, 505, 483, 491, 460, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 515, 497, 513, 526, 557, 511, 511, 542, 536, 512, 502, 461, 492, 498, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 478, 513, 507, 522, 501, 497, 542, 520, 525, 536, 514, 495, 476, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8218, 8332, 8281, 8086, 8177, 8171, 8205] avg=8192.0000 max=8332 max_rank=2 min=8066 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 502, 490, 528, 496, 531, 509, 540, 493, 493, 483, 533, 472, 505, 552, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 551, 537, 510, 520, 489, 551, 487, 495, 487, 507, 463, 499, 528, 514, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 495, 518, 483, 506, 519, 495, 519, 545, 516, 505, 541, 481, 529, 495, 523, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 556, 511, 524, 553, 499, 500, 512, 509, 494, 498, 526, 551, 533, 484, 487, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 530, 512, 495, 515, 529, 493, 461, 520, 501, 536, 473, 509, 515, 501, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 520, 522, 508, 500, 526, 535, 448, 525, 501, 508, 519, 492, 509, 486, 534, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 522, 514, 515, 491, 542, 563, 534, 544, 507, 507, 541, 505, 501, 513, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 494, 481, 518, 530, 486, 536, 506, 528, 499, 533, 524, 460, 510, 480, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8174, 8187, 8252, 8137, 8126, 8363, 8153] avg=8192.0000 max=8363 max_rank=6 min=8126 min_rank=5 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 540, 483, 498, 515, 529, 532, 507, 526, 536, 501, 512, 514, 503, 513, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 552, 446, 525, 543, 514, 543, 480, 551, 521, 516, 537, 500, 494, 528, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 549, 523, 538, 506, 547, 497, 504, 500, 547, 485, 528, 522, 507, 510, 512, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 497, 525, 554, 514, 519, 550, 507, 532, 474, 497, 506, 512, 493, 494, 537, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 505, 522, 529, 510, 524, 515, 518, 506, 508, 502, 536, 465, 525, 504, 20, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 482, 492, 474, 517, 512, 550, 540, 506, 498, 503, 514, 487, 518, 516, 503, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 509, 473, 526, 480, 475, 498, 466, 511, 547, 502, 525, 510, 522, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 530, 499, 498, 503, 527, 494, 500, 542, 520, 488, 493, 541, 471, 512, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8279, 8287, 8198, 8195, 8166, 8077, 8115] avg=8192.0000 max=8287 max_rank=2 min=8077 min_rank=6 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4352, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[495, 507, 489, 494, 567, 505, 502, 502, 498, 538, 505, 515, 503, 476, 524, 511, 511, 508, 510, 538, 516, 488, 517, 490, 517, 478, 540, 529, 537, 536, 539, 487, 478, 499, 506, 550, 529, 509, 500, 522, 528, 524, 538, 521, 550, 456, 513, 543, 525, 546, 502, 499, 502, 511, 494, 529, 490, 511, 507, 496, 556, 498, 506, 488, 521, 527, 480, 515, 526, 525, 524, 515, 522, 506, 535, 501, 492, 517, 514, 544, 524, 520, 507, 502, 498, 491, 516, 512, 522, 498, 496, 514, 473, 525, 475, 558, 542, 510, 494, 469, 520, 514, 497, 480, 529, 533, 516, 505, 536, 518, 498, 516, 470, 521, 516, 552, 487, 513, 541, 474, 514, 527, 467, 513, 538, 520, 505, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[513, 550, 485, 534, 509, 481, 506, 481, 548, 476, 546, 518, 519, 518, 513, 502, 516, 528, 513, 502, 531, 522, 518, 510, 484, 538, 496, 514, 515, 521, 485, 497, 491, 502, 463, 515, 502, 512, 508, 544, 525, 526, 509, 502, 543, 531, 505, 484, 518, 534, 498, 508, 563, 492, 537, 478, 543, 502, 532, 517, 470, 544, 491, 474, 501, 506, 522, 569, 539, 470, 509, 495, 496, 492, 493, 498, 542, 496, 525, 532, 535, 512, 536, 515, 549, 523, 507, 517, 506, 475, 504, 513, 533, 553, 530, 505, 488, 507, 526, 496, 513, 504, 506, 530, 514, 514, 481, 506, 482, 464, 556, 530, 567, 488, 520, 482, 528, 508, 509, 480, 489, 522, 511, 474, 557, 505, 519, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[486, 465, 490, 483, 548, 516, 525, 538, 534, 552, 501, 498, 520, 532, 499, 520, 495, 496, 500, 506, 536, 517, 540, 531, 527, 508, 526, 504, 516, 531, 512, 528, 509, 526, 485, 540, 531, 511, 526, 511, 508, 509, 510, 496, 538, 535, 514, 484, 502, 554, 520, 516, 497, 502, 508, 521, 533, 503, 554, 521, 491, 500, 515, 483, 488, 511, 564, 520, 497, 495, 507, 557, 493, 517, 512, 543, 509, 476, 511, 510, 519, 491, 468, 532, 509, 510, 503, 546, 513, 493, 471, 521, 507, 468, 534, 510, 501, 501, 515, 488, 497, 499, 519, 529, 518, 540, 531, 530, 548, 502, 524, 512, 493, 489, 510, 528, 480, 537, 496, 505, 509, 509, 502, 493, 498, 492, 494, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[565, 504, 522, 503, 530, 522, 516, 488, 537, 534, 521, 516, 483, 491, 484, 490, 538, 509, 527, 529, 527, 471, 537, 507, 508, 492, 535, 543, 485, 517, 526, 533, 496, 565, 513, 512, 491, 502, 526, 526, 483, 507, 499, 523, 516, 490, 493, 519, 485, 511, 507, 457, 516, 494, 538, 494, 532, 547, 518, 510, 488, 506, 527, 491, 462, 512, 496, 503, 480, 494, 522, 533, 488, 513, 542, 505, 522, 553, 510, 519, 516, 506, 482, 499, 507, 500, 472, 465, 515, 528, 477, 484, 554, 519, 497, 515, 526, 504, 490, 498, 515, 519, 471, 548, 527, 520, 530, 514, 495, 538, 501, 542, 535, 532, 548, 511, 547, 510, 503, 510, 516, 531, 502, 527, 513, 535, 515, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[500, 539, 493, 501, 541, 548, 523, 518, 532, 518, 517, 549, 479, 506, 571, 479, 516, 583, 497, 529, 479, 531, 518, 518, 506, 509, 507, 506, 493, 521, 487, 474, 539, 497, 505, 538, 513, 467, 516, 544, 521, 483, 462, 498, 546, 521, 507, 512, 528, 462, 490, 543, 498, 504, 485, 506, 520, 495, 525, 520, 515, 507, 513, 480, 524, 501, 500, 538, 505, 505, 508, 513, 510, 535, 497, 513, 515, 511, 501, 516, 502, 524, 511, 526, 506, 519, 530, 514, 534, 542, 511, 508, 499, 516, 510, 495, 533, 516, 541, 507, 511, 503, 511, 512, 512, 523, 488, 506, 464, 526, 486, 483, 491, 470, 529, 489, 499, 495, 538, 538, 556, 505, 539, 503, 536, 503, 495, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[553, 533, 525, 467, 527, 554, 538, 502, 532, 486, 496, 530, 518, 540, 502, 520, 544, 486, 520, 528, 519, 475, 508, 495, 534, 514, 519, 497, 503, 499, 489, 549, 501, 482, 459, 539, 495, 537, 479, 551, 497, 528, 541, 488, 538, 515, 511, 508, 488, 486, 506, 471, 519, 526, 510, 525, 546, 480, 489, 515, 518, 520, 566, 501, 548, 498, 467, 505, 526, 546, 505, 554, 502, 526, 526, 498, 484, 511, 542, 545, 493, 515, 489, 496, 508, 487, 515, 488, 506, 530, 519, 497, 498, 494, 511, 480, 548, 479, 491, 496, 492, 566, 538, 504, 500, 491, 512, 515, 495, 526, 538, 555, 487, 510, 519, 533, 477, 528, 519, 526, 540, 522, 480, 514, 519, 480, 515, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[488, 503, 484, 552, 531, 501, 457, 527, 502, 493, 496, 536, 523, 514, 521, 476, 507, 492, 447, 528, 543, 538, 515, 504, 541, 525, 535, 483, 539, 526, 532, 516, 509, 507, 552, 546, 480, 492, 541, 511, 529, 524, 497, 499, 502, 483, 486, 524, 490, 507, 531, 517, 527, 536, 523, 503, 546, 509, 464, 531, 466, 498, 542, 559, 496, 540, 507, 501, 523, 522, 492, 558, 518, 527, 487, 483, 531, 520, 523, 525, 496, 526, 534, 533, 499, 453, 471, 521, 474, 515, 520, 470, 527, 470, 504, 523, 513, 515, 496, 521, 551, 503, 514, 528, 490, 493, 511, 516, 480, 525, 505, 508, 503, 482, 498, 514, 517, 493, 545, 538, 492, 511, 487, 549, 497, 518, 585, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[489, 475, 523, 512, 491, 464, 511, 531, 526, 546, 509, 541, 516, 533, 510, 523, 548, 544, 495, 532, 554, 514, 545, 485, 506, 546, 493, 518, 518, 530, 543, 501, 532, 497, 498, 523, 499, 537, 510, 479, 508, 469, 486, 556, 516, 511, 503, 523, 504, 501, 546, 501, 483, 486, 513, 515, 482, 523, 528, 536, 508, 481, 576, 496, 518, 494, 519, 499, 488, 487, 530, 481, 531, 494, 511, 529, 517, 519, 535, 527, 488, 499, 499, 543, 524, 492, 516, 503, 508, 458, 528, 465, 489, 490, 536, 517, 528, 475, 488, 529, 520, 491, 489, 483, 508, 534, 512, 544, 533, 525, 513, 523, 541, 484, 495, 483, 508, 527, 504, 519, 563, 515, 512, 491, 507, 530, 535, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[495, 507, 489, 494, 548, 505, 502, 502, 498, 538, 505, 515, 503, 476, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 510, 538, 462, 488, 517, 490, 517, 478, 540, 529, 537, 536, 539, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 499, 506, 550, 529, 509, 500, 522, 528, 524, 538, 521, 550, 456, 513, 543, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 546, 502, 499, 502, 511, 494, 529, 490, 511, 507, 496, 556, 498, 506, 488, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 480, 515, 526, 525, 524, 515, 522, 506, 535, 501, 492, 517, 514, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 507, 502, 498, 491, 516, 512, 522, 498, 496, 514, 473, 525, 475, 558, 54, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 494, 469, 520, 514, 497, 480, 529, 533, 516, 505, 536, 518, 498, 516, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 521, 516, 552, 487, 513, 541, 474, 511, 527, 467, 513, 538, 520, 505, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8187, 8271, 8179, 8241, 8204, 8179, 8163] avg=8192.0000 max=8271 max_rank=2 min=8112 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 539, 493, 501, 523, 548, 523, 518, 532, 518, 517, 549, 479, 506, 571, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 583, 497, 529, 428, 531, 518, 518, 506, 509, 507, 506, 493, 521, 487, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 497, 505, 538, 513, 467, 516, 544, 521, 483, 462, 498, 546, 521, 507, 512, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 462, 490, 543, 498, 504, 485, 506, 520, 495, 525, 520, 515, 507, 513, 480, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 501, 500, 538, 505, 505, 508, 513, 510, 535, 497, 513, 515, 511, 501, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 511, 526, 506, 519, 530, 514, 534, 542, 511, 508, 499, 516, 510, 495, 51, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 516, 541, 507, 511, 503, 511, 512, 512, 523, 488, 506, 464, 526, 486, 483, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 470, 529, 489, 499, 495, 538, 538, 553, 505, 539, 503, 536, 503, 495, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8123, 8174, 8109, 8170, 8316, 8124, 8224] avg=8192.0000 max=8316 max_rank=5 min=8109 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 550, 485, 534, 486, 481, 506, 481, 548, 476, 546, 518, 519, 518, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 528, 513, 502, 472, 522, 518, 510, 484, 538, 496, 514, 515, 521, 485, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 502, 463, 515, 502, 512, 508, 544, 525, 526, 509, 502, 543, 531, 505, 484, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 498, 508, 563, 492, 537, 478, 543, 502, 532, 517, 470, 544, 491, 474, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 522, 569, 539, 470, 509, 495, 496, 492, 493, 498, 542, 496, 525, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 512, 536, 515, 549, 523, 507, 517, 506, 475, 504, 513, 533, 553, 530, 505, 59, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 526, 496, 513, 504, 506, 530, 514, 514, 481, 506, 482, 464, 556, 530, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 488, 520, 482, 528, 508, 509, 480, 479, 522, 511, 474, 557, 505, 519, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8131, 8175, 8224, 8157, 8393, 8121, 8159] avg=8192.0000 max=8393 max_rank=5 min=8121 min_rank=6 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[565, 504, 522, 503, 512, 522, 516, 488, 537, 534, 521, 516, 483, 491, 484, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 509, 527, 529, 471, 471, 537, 507, 508, 492, 535, 543, 485, 517, 526, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 565, 513, 512, 491, 502, 526, 526, 483, 507, 499, 523, 516, 490, 493, 519, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 511, 507, 457, 516, 494, 538, 494, 532, 547, 518, 510, 488, 506, 527, 491, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 512, 496, 503, 480, 494, 522, 533, 488, 513, 542, 505, 522, 553, 510, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 482, 499, 507, 500, 472, 465, 515, 528, 477, 484, 554, 519, 497, 515, 56, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 504, 490, 498, 515, 519, 471, 548, 527, 520, 530, 514, 495, 538, 501, 542, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 532, 548, 511, 547, 510, 503, 510, 513, 531, 502, 527, 513, 535, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8228, 8166, 8139, 8132, 8110, 8240, 8333] avg=8192.0000 max=8333 max_rank=7 min=8110 min_rank=5 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[486, 465, 490, 483, 530, 516, 525, 538, 534, 552, 501, 498, 520, 532, 499, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 500, 506, 479, 517, 540, 531, 527, 508, 526, 504, 516, 531, 512, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 485, 540, 531, 511, 526, 511, 508, 509, 510, 496, 538, 535, 514, 484, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 554, 520, 516, 497, 502, 508, 521, 533, 503, 554, 521, 491, 500, 515, 483, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 511, 564, 520, 497, 495, 507, 557, 493, 517, 512, 543, 509, 476, 511, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 491, 468, 532, 509, 510, 503, 546, 513, 493, 471, 521, 507, 468, 534, 510, 57, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 515, 488, 497, 499, 519, 529, 518, 540, 531, 530, 548, 502, 524, 512, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 489, 510, 528, 480, 537, 496, 505, 506, 509, 502, 493, 498, 492, 494, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8216, 8238, 8238, 8188, 8170, 8256, 8041] avg=8192.0000 max=8256 max_rank=6 min=8041 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 503, 484, 552, 513, 501, 457, 527, 502, 493, 496, 536, 523, 514, 521, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 492, 447, 528, 486, 538, 515, 504, 541, 525, 535, 483, 539, 526, 532, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 507, 552, 546, 480, 492, 541, 511, 529, 524, 497, 499, 502, 483, 486, 524, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 507, 531, 517, 527, 536, 523, 503, 546, 509, 464, 531, 466, 498, 542, 559, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 540, 507, 501, 523, 522, 492, 558, 518, 527, 487, 483, 531, 520, 523, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 526, 534, 533, 499, 453, 471, 521, 474, 515, 520, 470, 527, 470, 504, 523, 57, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 515, 496, 521, 551, 503, 514, 528, 490, 493, 511, 516, 480, 525, 505, 508, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 482, 498, 514, 517, 493, 545, 538, 489, 511, 487, 549, 497, 518, 585, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8214, 8187, 8267, 8231, 8111, 8171, 8269] avg=8192.0000 max=8269 max_rank=7 min=8086 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[553, 533, 525, 467, 509, 554, 538, 502, 532, 486, 496, 530, 518, 540, 502, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 486, 520, 528, 464, 475, 508, 495, 534, 514, 519, 497, 503, 499, 489, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 482, 459, 539, 495, 537, 479, 551, 497, 528, 541, 488, 538, 515, 511, 508, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 486, 506, 471, 519, 526, 510, 525, 546, 480, 489, 515, 518, 520, 566, 501, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 498, 467, 505, 526, 546, 505, 554, 502, 526, 526, 498, 484, 511, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 515, 489, 496, 508, 487, 515, 488, 506, 530, 519, 497, 498, 494, 511, 480, 55, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 479, 491, 496, 492, 566, 538, 504, 500, 491, 512, 515, 495, 526, 538, 555, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 510, 519, 533, 477, 528, 519, 526, 537, 522, 480, 514, 519, 480, 515, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8124, 8174, 8184, 8260, 8100, 8248, 8141] avg=8192.0000 max=8305 max_rank=0 min=8100 min_rank=5 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[489, 475, 523, 512, 475, 464, 511, 531, 526, 546, 509, 541, 516, 533, 510, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 544, 495, 532, 495, 514, 545, 485, 506, 546, 493, 518, 518, 530, 543, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 498, 523, 499, 537, 510, 479, 508, 469, 486, 556, 516, 511, 503, 523, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 546, 501, 483, 486, 513, 515, 482, 523, 528, 536, 508, 481, 576, 496, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 494, 519, 499, 488, 487, 530, 481, 531, 494, 511, 529, 517, 519, 535, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 499, 499, 543, 524, 492, 516, 503, 508, 458, 528, 465, 489, 490, 536, 517, 59, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 475, 488, 529, 520, 491, 489, 483, 508, 534, 512, 544, 533, 525, 513, 523, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 484, 495, 483, 508, 527, 504, 519, 560, 515, 512, 491, 507, 530, 535, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8313, 8151, 8196, 8157, 8132, 8197, 8206] avg=8192.0000 max=8313 max_rank=1 min=8132 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 3840, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4352, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[501, 507, 498, 516, 515, 526, 512, 534, 541, 516, 550, 509, 508, 537, 510, 486, 494, 509, 538, 549, 509, 509, 518, 493, 520, 506, 516, 512, 499, 486, 513, 513, 514, 497, 494, 500, 526, 525, 511, 514, 505, 488, 527, 514, 530, 572, 489, 502, 517, 521, 478, 508, 510, 501, 518, 521, 527, 553, 529, 514, 520, 520, 487, 506, 516, 497, 478, 544, 510, 524, 494, 494, 512, 503, 539, 504, 478, 551, 514, 513, 472, 493, 519, 512, 561, 511, 536, 498, 526, 489, 540, 540, 489, 498, 509, 504, 513, 532, 465, 512, 503, 485, 505, 497, 498, 510, 460, 513, 521, 562, 501, 569, 494, 549, 528, 556, 517, 525, 527, 473, 502, 482, 491, 488, 487, 510, 510, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[521, 497, 508, 528, 499, 514, 452, 488, 510, 511, 484, 509, 501, 550, 521, 532, 491, 441, 535, 489, 521, 533, 538, 543, 503, 499, 525, 531, 524, 491, 486, 505, 516, 515, 520, 460, 505, 513, 516, 510, 464, 507, 470, 525, 520, 503, 498, 494, 518, 536, 496, 516, 518, 508, 528, 521, 469, 485, 495, 502, 528, 507, 482, 528, 554, 564, 502, 497, 540, 509, 495, 572, 496, 524, 528, 520, 511, 528, 542, 468, 553, 555, 506, 467, 511, 497, 544, 527, 480, 514, 517, 496, 512, 491, 490, 526, 547, 482, 555, 543, 499, 467, 506, 539, 528, 508, 491, 504, 518, 523, 516, 525, 469, 546, 534, 525, 532, 527, 493, 507, 489, 498, 581, 487, 493, 566, 534, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[511, 505, 531, 551, 513, 517, 535, 505, 486, 504, 514, 515, 471, 538, 488, 536, 519, 486, 516, 548, 506, 497, 502, 527, 463, 557, 528, 486, 534, 523, 500, 525, 492, 517, 509, 537, 511, 518, 506, 554, 527, 522, 524, 473, 510, 516, 575, 523, 501, 498, 477, 517, 534, 523, 480, 500, 512, 538, 496, 506, 519, 484, 484, 484, 495, 524, 496, 519, 478, 531, 493, 500, 526, 534, 556, 514, 472, 503, 486, 585, 487, 512, 510, 505, 505, 472, 532, 522, 559, 488, 486, 500, 488, 529, 543, 530, 538, 522, 554, 484, 534, 467, 479, 493, 520, 537, 498, 552, 493, 501, 511, 501, 532, 510, 510, 531, 498, 493, 508, 513, 506, 508, 561, 516, 509, 495, 494, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[517, 502, 534, 495, 502, 508, 517, 484, 474, 477, 530, 499, 518, 553, 512, 455, 527, 541, 541, 560, 514, 505, 487, 505, 531, 522, 522, 534, 495, 472, 534, 542, 508, 506, 535, 534, 490, 509, 497, 519, 543, 490, 545, 505, 575, 477, 501, 500, 502, 520, 487, 484, 489, 520, 512, 497, 500, 533, 499, 524, 518, 487, 542, 475, 539, 501, 495, 502, 521, 534, 502, 503, 533, 540, 489, 509, 511, 523, 524, 524, 487, 496, 525, 508, 519, 501, 520, 483, 488, 521, 489, 528, 478, 522, 516, 514, 506, 538, 480, 514, 511, 531, 523, 532, 495, 539, 458, 536, 514, 553, 479, 493, 498, 540, 495, 490, 534, 505, 513, 507, 500, 507, 501, 542, 527, 547, 504, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[513, 486, 502, 507, 496, 517, 522, 498, 502, 501, 528, 540, 558, 495, 504, 522, 498, 531, 523, 560, 496, 486, 562, 503, 516, 508, 516, 517, 518, 530, 512, 506, 494, 507, 511, 512, 536, 489, 508, 501, 502, 495, 479, 515, 468, 474, 542, 519, 512, 506, 489, 532, 496, 542, 498, 507, 544, 526, 482, 482, 531, 503, 512, 517, 526, 548, 493, 516, 554, 539, 538, 510, 503, 461, 543, 467, 524, 542, 506, 476, 549, 497, 507, 521, 538, 520, 559, 543, 467, 537, 532, 512, 465, 513, 529, 480, 472, 553, 501, 525, 563, 518, 521, 507, 519, 493, 481, 491, 500, 542, 481, 468, 538, 467, 509, 511, 549, 494, 555, 488, 516, 502, 514, 505, 519, 518, 491, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[490, 503, 543, 493, 501, 552, 531, 508, 517, 496, 548, 503, 496, 502, 527, 503, 497, 541, 494, 491, 500, 504, 517, 522, 490, 529, 505, 523, 518, 502, 531, 511, 482, 504, 553, 508, 478, 531, 506, 482, 528, 501, 507, 539, 514, 497, 475, 518, 468, 534, 476, 521, 533, 534, 509, 499, 504, 528, 538, 510, 525, 520, 491, 510, 513, 497, 534, 474, 513, 512, 509, 498, 547, 505, 516, 530, 543, 536, 511, 523, 545, 475, 506, 540, 529, 514, 507, 521, 532, 517, 513, 507, 541, 506, 481, 527, 559, 523, 518, 523, 522, 531, 531, 508, 502, 486, 500, 524, 544, 559, 499, 497, 485, 522, 515, 497, 478, 481, 480, 480, 486, 500, 509, 511, 513, 549, 481, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[505, 469, 547, 525, 528, 533, 508, 499, 518, 487, 536, 524, 550, 544, 508, 511, 524, 543, 529, 517, 489, 503, 499, 476, 525, 522, 536, 524, 467, 523, 481, 490, 464, 527, 522, 485, 499, 525, 547, 514, 507, 538, 500, 491, 511, 524, 505, 527, 513, 496, 477, 499, 520, 556, 486, 543, 459, 565, 505, 511, 496, 495, 519, 492, 515, 485, 515, 565, 491, 507, 541, 518, 508, 486, 490, 536, 495, 539, 496, 497, 508, 510, 494, 490, 509, 519, 491, 517, 496, 516, 552, 460, 486, 525, 515, 531, 500, 494, 523, 477, 512, 554, 514, 500, 524, 516, 545, 518, 512, 501, 496, 555, 499, 538, 503, 508, 537, 475, 542, 494, 522, 532, 473, 511, 512, 530, 514, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[570, 535, 501, 524, 469, 537, 541, 497, 497, 516, 508, 512, 490, 536, 533, 505, 489, 471, 526, 488, 499, 492, 497, 497, 548, 526, 523, 526, 521, 497, 509, 520, 533, 538, 488, 492, 507, 464, 507, 517, 551, 527, 501, 531, 484, 507, 478, 483, 545, 514, 516, 517, 552, 519, 527, 522, 530, 516, 496, 522, 488, 543, 537, 530, 508, 523, 488, 497, 478, 490, 511, 552, 485, 488, 506, 492, 519, 562, 481, 558, 484, 535, 515, 521, 526, 475, 515, 539, 455, 513, 529, 524, 538, 530, 495, 509, 524, 520, 501, 494, 518, 518, 534, 505, 456, 493, 494, 515, 468, 525, 503, 517, 502, 514, 541, 534, 510, 510, 545, 534, 507, 500, 540, 483, 497, 517, 496, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[517, 502, 534, 495, 502, 508, 517, 484, 474, 477, 530, 499, 518, 538, 512, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 541, 541, 551, 514, 505, 487, 505, 531, 522, 522, 534, 495, 472, 534, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 506, 535, 534, 490, 509, 497, 519, 543, 490, 545, 505, 575, 477, 501, 500, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 487, 484, 489, 520, 512, 497, 500, 533, 499, 524, 518, 487, 542, 475, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 501, 495, 502, 521, 534, 502, 503, 533, 540, 489, 509, 511, 490, 524, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 496, 525, 508, 519, 501, 520, 483, 488, 521, 489, 528, 478, 522, 516, 514, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 538, 480, 514, 511, 531, 523, 532, 495, 539, 458, 536, 514, 549, 479, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 540, 495, 490, 534, 505, 513, 507, 500, 507, 501, 542, 527, 547, 504, 547, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8062, 8323, 8267, 8106, 8217, 8099, 8198, 8264] avg=8192.0000 max=8323 max_rank=1 min=8062 min_rank=0 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[521, 497, 508, 528, 499, 514, 452, 488, 510, 511, 484, 509, 501, 535, 521, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 441, 535, 481, 521, 533, 538, 543, 503, 499, 525, 531, 524, 491, 486, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 520, 460, 505, 513, 516, 510, 464, 507, 470, 525, 520, 503, 498, 494, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 536, 496, 516, 518, 508, 528, 521, 469, 485, 495, 502, 528, 507, 482, 528, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 564, 502, 497, 540, 509, 495, 572, 496, 524, 528, 520, 511, 495, 542, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 555, 506, 467, 511, 497, 544, 527, 480, 514, 517, 496, 512, 491, 490, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 482, 555, 543, 499, 467, 506, 539, 528, 508, 491, 504, 518, 519, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 546, 534, 525, 532, 527, 493, 507, 489, 498, 581, 487, 493, 566, 534, 515, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8147, 8069, 8154, 8317, 8190, 8247, 8302] avg=8192.0000 max=8317 max_rank=4 min=8069 min_rank=2 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=33056 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 507, 498, 516, 515, 526, 512, 534, 541, 516, 550, 509, 508, 522, 510, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 509, 538, 540, 509, 509, 518, 493, 520, 506, 516, 512, 499, 486, 513, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 497, 494, 500, 526, 525, 511, 514, 505, 488, 527, 514, 530, 572, 489, 502, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 521, 478, 508, 510, 501, 518, 521, 527, 553, 529, 514, 520, 520, 487, 506, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 497, 478, 544, 510, 524, 494, 494, 512, 503, 539, 504, 478, 516, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 493, 519, 512, 561, 511, 536, 498, 526, 489, 540, 540, 489, 498, 509, 504, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 532, 465, 512, 503, 485, 505, 497, 498, 510, 460, 513, 521, 558, 501, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 549, 528, 556, 517, 525, 527, 473, 502, 482, 491, 488, 487, 510, 510, 495, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8175, 8243, 8247, 8136, 8201, 8142, 8141] avg=8192.0000 max=8251 max_rank=0 min=8136 min_rank=4 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 505, 531, 551, 513, 517, 535, 505, 486, 504, 514, 515, 471, 520, 488, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 486, 516, 523, 506, 497, 502, 527, 463, 557, 528, 486, 534, 523, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 517, 509, 537, 511, 518, 506, 554, 527, 522, 524, 473, 510, 516, 575, 523, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 498, 477, 517, 534, 523, 480, 500, 512, 538, 496, 506, 519, 484, 484, 484, 18, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 496, 519, 478, 531, 493, 500, 526, 534, 556, 514, 472, 466, 486, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 512, 510, 505, 505, 472, 532, 522, 559, 488, 486, 500, 488, 529, 543, 530, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 522, 554, 484, 534, 467, 479, 493, 520, 537, 498, 552, 493, 494, 511, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 510, 510, 531, 498, 493, 508, 513, 506, 508, 561, 516, 509, 495, 494, 484, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8192, 8350, 8080, 8175, 8179, 8177, 8181] avg=8192.0000 max=8350 max_rank=2 min=8080 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 486, 502, 507, 496, 517, 522, 498, 502, 501, 528, 540, 558, 482, 504, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 531, 523, 551, 496, 486, 562, 503, 516, 508, 516, 517, 518, 530, 512, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 507, 511, 512, 536, 489, 508, 501, 502, 495, 479, 515, 468, 474, 542, 519, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 506, 489, 532, 496, 542, 498, 507, 544, 526, 482, 482, 531, 503, 512, 517, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 548, 493, 516, 554, 539, 538, 510, 503, 461, 543, 467, 524, 508, 506, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 497, 507, 521, 538, 520, 559, 543, 467, 537, 532, 512, 465, 513, 529, 480, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 553, 501, 525, 563, 518, 521, 507, 519, 493, 481, 491, 500, 538, 481, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 467, 509, 511, 549, 494, 555, 488, 516, 502, 514, 505, 519, 518, 491, 506, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8273, 8086, 8194, 8212, 8273, 8131, 8189] avg=8192.0000 max=8273 max_rank=1 min=8086 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[490, 503, 543, 493, 501, 552, 531, 508, 517, 496, 548, 503, 496, 488, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 541, 494, 482, 500, 504, 517, 522, 490, 529, 505, 523, 518, 502, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 504, 553, 508, 478, 531, 506, 482, 528, 501, 507, 539, 514, 497, 475, 518, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 534, 476, 521, 533, 534, 509, 499, 504, 528, 538, 510, 525, 520, 491, 510, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 497, 534, 474, 513, 512, 509, 498, 547, 505, 516, 530, 543, 502, 511, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 475, 506, 540, 529, 514, 507, 521, 532, 517, 513, 507, 541, 506, 481, 527, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 523, 518, 523, 522, 531, 531, 508, 502, 486, 500, 524, 544, 555, 499, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 522, 515, 497, 478, 481, 480, 480, 486, 500, 509, 511, 513, 549, 481, 490, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8166, 8157, 8216, 8227, 8265, 8322, 7984] avg=8192.0000 max=8322 max_rank=6 min=7984 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 469, 547, 525, 528, 533, 508, 499, 518, 487, 536, 524, 550, 529, 508, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 543, 529, 508, 489, 503, 499, 476, 525, 522, 536, 524, 467, 523, 481, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 527, 522, 485, 499, 525, 547, 514, 507, 538, 500, 491, 511, 524, 505, 527, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 496, 477, 499, 520, 556, 486, 543, 459, 565, 505, 511, 496, 495, 519, 492, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 485, 515, 565, 491, 507, 541, 518, 508, 486, 490, 536, 495, 505, 496, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 510, 494, 490, 509, 519, 491, 517, 496, 516, 552, 460, 486, 525, 515, 531, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 494, 523, 477, 512, 554, 514, 500, 524, 516, 545, 518, 512, 498, 496, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 538, 503, 508, 537, 475, 542, 494, 522, 532, 473, 511, 512, 530, 514, 544, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8139, 8220, 8149, 8150, 8122, 8238, 8241] avg=8192.0000 max=8277 max_rank=0 min=8122 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[570, 535, 501, 524, 469, 537, 541, 497, 497, 516, 508, 512, 490, 522, 533, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 471, 526, 480, 499, 492, 497, 497, 548, 526, 523, 526, 521, 497, 509, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 538, 488, 492, 507, 464, 507, 517, 551, 527, 501, 531, 484, 507, 478, 483, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 514, 516, 517, 552, 519, 527, 522, 530, 516, 496, 522, 488, 543, 537, 530, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 523, 488, 497, 478, 490, 511, 552, 485, 488, 506, 492, 519, 526, 481, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 535, 515, 521, 526, 475, 515, 539, 455, 513, 529, 524, 538, 530, 495, 509, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 501, 494, 518, 518, 534, 505, 456, 493, 494, 515, 468, 521, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 541, 534, 510, 510, 545, 534, 507, 500, 540, 483, 497, 517, 496, 498, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8121, 8144, 8390, 8102, 8207, 8081, 8234] avg=8192.0000 max=8390 max_rank=3 min=8081 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4096, 4352, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4352, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[537, 497, 489, 526, 512, 508, 523, 506, 551, 483, 524, 525, 514, 559, 507, 487, 475, 497, 495, 512, 471, 510, 514, 492, 533, 500, 524, 521, 503, 502, 517, 561, 540, 503, 504, 513, 511, 495, 519, 511, 526, 555, 539, 537, 511, 516, 476, 518, 481, 525, 493, 511, 494, 490, 509, 537, 541, 474, 516, 538, 493, 526, 512, 524, 491, 524, 496, 515, 505, 497, 511, 533, 493, 521, 525, 487, 503, 525, 486, 518, 523, 510, 511, 510, 502, 517, 514, 539, 484, 547, 514, 525, 513, 519, 505, 489, 507, 509, 504, 469, 551, 499, 500, 539, 466, 476, 519, 542, 540, 492, 497, 496, 508, 496, 541, 516, 516, 517, 496, 505, 554, 506, 478, 538, 520, 538, 539, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[487, 506, 508, 486, 521, 508, 502, 529, 478, 498, 524, 492, 499, 499, 487, 504, 485, 489, 501, 470, 527, 499, 535, 531, 517, 565, 541, 514, 488, 506, 512, 504, 509, 473, 516, 511, 552, 529, 529, 523, 538, 477, 491, 563, 491, 494, 473, 529, 511, 468, 538, 488, 484, 523, 546, 525, 511, 530, 506, 552, 495, 481, 548, 452, 522, 549, 486, 513, 543, 501, 506, 494, 545, 538, 504, 516, 490, 539, 484, 529, 533, 490, 488, 504, 495, 556, 504, 522, 556, 517, 531, 510, 544, 527, 513, 529, 527, 490, 535, 530, 506, 449, 536, 530, 481, 514, 468, 513, 483, 521, 527, 533, 517, 496, 516, 529, 515, 542, 522, 500, 500, 520, 524, 529, 504, 510, 516, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[467, 511, 521, 516, 518, 524, 554, 542, 507, 495, 530, 522, 513, 563, 489, 535, 544, 495, 539, 488, 473, 521, 504, 481, 516, 488, 492, 531, 481, 557, 495, 492, 556, 517, 494, 488, 543, 517, 498, 501, 514, 510, 509, 518, 521, 533, 506, 493, 518, 501, 504, 515, 497, 496, 523, 514, 538, 518, 508, 502, 461, 524, 521, 537, 534, 522, 520, 529, 511, 474, 521, 546, 503, 493, 548, 493, 500, 508, 544, 505, 484, 500, 516, 501, 496, 480, 529, 491, 505, 506, 530, 512, 497, 524, 493, 512, 523, 538, 514, 528, 510, 512, 511, 514, 536, 522, 503, 476, 514, 503, 496, 536, 570, 497, 545, 524, 500, 517, 516, 508, 527, 491, 503, 501, 512, 479, 480, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[495, 504, 506, 485, 507, 557, 527, 511, 493, 550, 529, 516, 493, 537, 545, 557, 490, 500, 511, 489, 506, 502, 554, 539, 540, 513, 485, 517, 529, 487, 512, 517, 498, 499, 508, 511, 479, 495, 510, 511, 515, 550, 508, 488, 495, 516, 501, 499, 493, 521, 509, 488, 511, 492, 470, 509, 528, 533, 513, 535, 515, 528, 489, 511, 464, 486, 555, 534, 520, 489, 516, 507, 519, 544, 533, 487, 524, 550, 530, 519, 546, 520, 464, 472, 523, 514, 481, 549, 540, 521, 456, 521, 510, 547, 476, 525, 487, 516, 541, 511, 522, 530, 501, 490, 503, 513, 515, 520, 497, 512, 507, 548, 508, 564, 477, 506, 501, 522, 521, 486, 504, 513, 510, 455, 486, 507, 519, 571] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[498, 503, 509, 489, 539, 485, 506, 510, 565, 512, 505, 541, 502, 522, 470, 519, 495, 508, 525, 536, 518, 476, 535, 521, 474, 518, 477, 506, 543, 516, 534, 523, 525, 495, 500, 531, 502, 509, 527, 481, 489, 516, 518, 556, 490, 480, 540, 499, 514, 528, 533, 548, 533, 540, 537, 483, 499, 525, 525, 507, 524, 523, 516, 512, 518, 463, 492, 502, 493, 488, 552, 513, 520, 532, 518, 485, 526, 554, 512, 492, 473, 495, 499, 513, 527, 500, 496, 523, 498, 492, 487, 516, 504, 500, 468, 538, 490, 531, 500, 516, 476, 499, 502, 487, 487, 531, 491, 521, 504, 524, 500, 557, 521, 533, 557, 533, 527, 484, 510, 503, 520, 523, 525, 573, 528, 503, 512, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[509, 520, 530, 553, 521, 529, 510, 506, 470, 514, 502, 543, 549, 511, 488, 483, 492, 532, 516, 524, 536, 520, 512, 505, 525, 499, 551, 510, 501, 509, 516, 493, 526, 542, 480, 544, 483, 461, 477, 522, 507, 501, 531, 535, 520, 523, 523, 540, 513, 523, 503, 488, 507, 515, 541, 491, 510, 538, 533, 493, 517, 487, 537, 537, 527, 494, 488, 552, 523, 506, 518, 520, 492, 550, 513, 560, 514, 486, 537, 526, 480, 489, 467, 474, 490, 551, 509, 486, 525, 530, 483, 525, 504, 493, 552, 496, 497, 512, 507, 472, 535, 480, 501, 506, 528, 512, 513, 493, 562, 544, 503, 488, 523, 507, 505, 516, 512, 509, 517, 535, 507, 488, 511, 495, 456, 504, 492, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[522, 514, 534, 516, 528, 494, 507, 507, 506, 476, 500, 490, 493, 531, 481, 516, 479, 490, 551, 518, 516, 483, 538, 532, 534, 507, 469, 520, 480, 533, 506, 552, 510, 500, 501, 484, 522, 510, 537, 506, 505, 551, 487, 490, 506, 512, 544, 526, 502, 530, 537, 523, 514, 525, 485, 536, 517, 486, 508, 509, 542, 506, 522, 503, 533, 527, 531, 530, 560, 506, 534, 536, 509, 519, 469, 512, 501, 528, 516, 525, 492, 503, 489, 515, 532, 510, 531, 517, 499, 514, 572, 481, 493, 517, 510, 517, 546, 509, 528, 508, 520, 524, 477, 482, 530, 506, 481, 513, 516, 506, 499, 508, 517, 489, 515, 512, 529, 554, 511, 490, 511, 521, 489, 490, 469, 495, 496, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[495, 542, 484, 540, 501, 467, 513, 516, 522, 510, 498, 521, 536, 519, 502, 501, 518, 502, 527, 493, 447, 529, 514, 532, 523, 517, 488, 502, 487, 565, 507, 550, 522, 556, 482, 482, 495, 520, 526, 511, 516, 518, 482, 522, 544, 540, 497, 552, 566, 521, 511, 494, 536, 503, 470, 492, 490, 510, 554, 517, 489, 492, 492, 538, 519, 508, 524, 502, 516, 496, 519, 507, 497, 488, 547, 497, 493, 494, 527, 507, 517, 476, 513, 528, 520, 519, 503, 533, 521, 507, 518, 500, 548, 510, 532, 485, 511, 502, 493, 560, 510, 538, 489, 507, 529, 494, 493, 465, 495, 481, 526, 531, 476, 520, 532, 485, 514, 506, 558, 528, 520, 506, 508, 492, 521, 550, 510, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[537, 497, 489, 526, 512, 508, 523, 506, 551, 483, 524, 525, 514, 552, 507, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 497, 495, 512, 471, 510, 514, 492, 533, 500, 524, 521, 503, 502, 517, 561, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 503, 504, 513, 511, 495, 519, 511, 526, 555, 539, 530, 511, 516, 476, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 525, 493, 511, 494, 490, 509, 537, 541, 474, 503, 538, 493, 526, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 524, 496, 515, 505, 497, 511, 533, 493, 482, 525, 487, 503, 525, 486, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 511, 510, 502, 517, 514, 539, 484, 547, 514, 525, 513, 519, 505, 489, 13, 9, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 504, 469, 551, 499, 500, 539, 466, 476, 519, 542, 540, 492, 497, 496, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 532, 516, 516, 517, 496, 505, 554, 506, 478, 538, 520, 538, 539, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8137, 8267, 8151, 8091, 8251, 8142, 8256] avg=8192.0000 max=8267 max_rank=2 min=8091 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[467, 511, 521, 516, 518, 524, 554, 542, 507, 495, 530, 522, 513, 556, 489, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 495, 539, 488, 473, 521, 504, 481, 516, 488, 492, 531, 481, 557, 495, 492, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 517, 494, 488, 543, 517, 498, 501, 514, 510, 509, 511, 521, 533, 506, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 504, 515, 497, 496, 523, 514, 538, 518, 495, 502, 461, 524, 521, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 522, 520, 529, 511, 474, 521, 546, 503, 456, 548, 493, 500, 508, 544, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 500, 516, 501, 496, 480, 529, 491, 505, 506, 530, 512, 497, 524, 493, 512, 13, 9, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 538, 514, 528, 510, 512, 511, 514, 536, 522, 503, 476, 514, 503, 496, 536, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 497, 536, 524, 500, 517, 516, 508, 527, 491, 503, 501, 512, 479, 480, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8107, 8211, 8164, 8214, 8105, 8270, 8165] avg=8192.0000 max=8300 max_rank=0 min=8105 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[498, 503, 509, 489, 539, 485, 506, 510, 565, 512, 505, 541, 502, 516, 470, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 525, 536, 518, 476, 535, 521, 474, 518, 477, 506, 543, 516, 534, 523, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 495, 500, 531, 502, 509, 527, 481, 489, 516, 518, 548, 490, 480, 540, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 528, 533, 548, 533, 540, 537, 483, 499, 525, 512, 507, 524, 523, 516, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 463, 492, 502, 493, 488, 552, 513, 520, 491, 518, 485, 526, 554, 512, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 495, 499, 513, 527, 500, 496, 523, 498, 492, 487, 516, 504, 500, 468, 538, 13, 9, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 531, 500, 516, 476, 499, 502, 487, 487, 531, 491, 521, 504, 524, 500, 557, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 533, 548, 533, 527, 484, 510, 503, 520, 523, 525, 573, 528, 503, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8214, 8150, 8334, 8119, 8060, 8153, 8337] avg=8192.0000 max=8337 max_rank=7 min=8060 min_rank=5 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[487, 506, 508, 486, 521, 508, 502, 529, 478, 498, 524, 492, 499, 489, 487, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 489, 501, 470, 527, 499, 535, 531, 517, 565, 541, 514, 488, 506, 512, 504, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 473, 516, 511, 552, 529, 529, 523, 538, 477, 491, 547, 491, 494, 473, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 468, 538, 488, 484, 523, 546, 525, 511, 530, 490, 552, 495, 481, 548, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 549, 486, 513, 543, 501, 506, 494, 545, 490, 504, 516, 490, 539, 484, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 490, 488, 504, 495, 556, 504, 522, 556, 517, 531, 510, 544, 527, 513, 529, 16, 11, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 490, 535, 530, 506, 449, 536, 530, 481, 514, 468, 513, 483, 521, 527, 533, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 505, 529, 515, 542, 522, 500, 500, 520, 524, 529, 504, 510, 516, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8018, 8201, 8182, 8142, 8211, 8363, 8183, 8236] avg=8192.0000 max=8363 max_rank=5 min=8018 min_rank=0 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32993 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 504, 506, 485, 507, 557, 527, 511, 493, 550, 529, 516, 493, 531, 545, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 500, 511, 489, 506, 502, 554, 539, 540, 513, 485, 517, 529, 487, 512, 517, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 508, 511, 479, 495, 510, 511, 515, 550, 508, 481, 495, 516, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 521, 509, 488, 511, 492, 470, 509, 528, 533, 500, 535, 515, 528, 489, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 486, 555, 534, 520, 489, 516, 507, 519, 503, 533, 487, 524, 550, 530, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 520, 464, 472, 523, 514, 481, 549, 540, 521, 456, 521, 510, 547, 476, 525, 13, 8, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 516, 541, 511, 522, 530, 501, 490, 503, 513, 515, 520, 497, 512, 507, 548, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 564, 469, 506, 501, 522, 521, 486, 504, 513, 510, 455, 486, 507, 519, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8200, 8076, 8132, 8236, 8194, 8250, 8142] avg=8192.0000 max=8306 max_rank=0 min=8076 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[509, 520, 530, 553, 521, 529, 510, 506, 470, 514, 502, 543, 549, 505, 488, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 532, 516, 524, 536, 520, 512, 505, 525, 499, 551, 510, 501, 509, 516, 493, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 542, 480, 544, 483, 461, 477, 522, 507, 501, 531, 528, 520, 523, 523, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 523, 503, 488, 507, 515, 541, 491, 510, 538, 520, 493, 517, 487, 537, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 494, 488, 552, 523, 506, 518, 520, 492, 508, 513, 560, 514, 486, 537, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 489, 467, 474, 490, 551, 509, 486, 525, 530, 483, 525, 504, 493, 552, 496, 13, 8, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 507, 472, 535, 480, 501, 506, 528, 512, 513, 493, 562, 544, 503, 488, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 497, 516, 512, 509, 517, 535, 507, 488, 511, 495, 456, 504, 492, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8250, 8208, 8220, 8264, 8083, 8191, 8088] avg=8192.0000 max=8264 max_rank=4 min=8083 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[522, 514, 534, 516, 528, 494, 507, 507, 506, 476, 500, 490, 493, 525, 481, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 490, 551, 518, 516, 483, 538, 532, 534, 507, 469, 520, 480, 533, 506, 552, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 500, 501, 484, 522, 510, 537, 506, 505, 551, 487, 483, 506, 512, 544, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 530, 537, 523, 514, 525, 485, 536, 517, 486, 495, 509, 542, 506, 522, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 527, 531, 530, 560, 506, 534, 536, 509, 480, 469, 512, 501, 528, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 503, 489, 515, 532, 510, 531, 517, 499, 514, 572, 481, 493, 517, 510, 517, 13, 8, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 509, 528, 508, 520, 524, 477, 482, 530, 506, 481, 513, 516, 506, 499, 508, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 489, 507, 512, 529, 554, 511, 490, 511, 521, 489, 490, 469, 495, 496, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8217, 8184, 8232, 8297, 8220, 8189, 8088] avg=8192.0000 max=8297 max_rank=4 min=8088 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 542, 484, 540, 501, 467, 513, 516, 522, 510, 498, 521, 536, 513, 502, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 502, 527, 493, 447, 529, 514, 532, 523, 517, 488, 502, 487, 565, 507, 550, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 556, 482, 482, 495, 520, 526, 511, 516, 518, 482, 515, 544, 540, 497, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 521, 511, 494, 536, 503, 470, 492, 490, 510, 540, 517, 489, 492, 492, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 524, 502, 516, 496, 519, 507, 497, 451, 547, 497, 493, 494, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 476, 513, 528, 520, 519, 503, 533, 521, 507, 518, 500, 548, 510, 532, 485, 14, 9, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 493, 560, 510, 538, 489, 507, 529, 494, 493, 465, 495, 481, 526, 531, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 520, 523, 485, 514, 506, 558, 528, 520, 506, 508, 492, 521, 550, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8210, 8258, 8161, 8104, 8260, 8158, 8224] avg=8192.0000 max=8260 max_rank=5 min=8104 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 3968, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[509, 475, 534, 487, 503, 494, 505, 484, 509, 501, 562, 520, 509, 501, 499, 502, 528, 536, 495, 520, 495, 548, 528, 524, 513, 519, 548, 547, 518, 473, 505, 533, 471, 514, 524, 505, 458, 524, 516, 527, 538, 501, 534, 501, 525, 520, 521, 563, 482, 521, 484, 507, 522, 492, 498, 481, 564, 497, 532, 502, 506, 494, 503, 542, 529, 511, 503, 545, 514, 511, 527, 533, 491, 498, 489, 507, 523, 538, 503, 490, 526, 524, 516, 487, 516, 509, 489, 484, 542, 502, 501, 520, 540, 484, 519, 512, 517, 523, 536, 495, 512, 557, 508, 498, 542, 525, 497, 493, 502, 515, 498, 460, 509, 494, 527, 535, 505, 495, 566, 522, 494, 505, 515, 488, 470, 519, 517, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[538, 487, 470, 548, 527, 490, 525, 505, 518, 522, 498, 501, 480, 529, 564, 553, 502, 549, 506, 526, 485, 527, 507, 511, 506, 548, 503, 501, 496, 534, 529, 497, 524, 492, 542, 515, 492, 497, 495, 525, 540, 538, 510, 518, 500, 479, 524, 499, 522, 531, 534, 501, 499, 504, 554, 498, 523, 489, 499, 507, 481, 467, 499, 505, 532, 522, 504, 515, 523, 494, 540, 521, 493, 473, 496, 511, 514, 551, 533, 501, 521, 505, 533, 524, 534, 499, 532, 546, 518, 512, 525, 503, 483, 517, 488, 482, 516, 556, 517, 473, 538, 555, 487, 468, 524, 567, 500, 509, 498, 517, 542, 455, 488, 528, 505, 518, 498, 508, 530, 536, 516, 501, 479, 507, 496, 492, 526, 456] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[480, 479, 500, 509, 533, 512, 492, 547, 452, 505, 477, 475, 523, 510, 534, 514, 507, 487, 514, 459, 536, 522, 511, 488, 523, 525, 563, 451, 505, 497, 530, 483, 456, 485, 517, 512, 499, 554, 552, 501, 520, 523, 466, 506, 498, 508, 552, 537, 521, 505, 511, 514, 545, 522, 502, 501, 529, 521, 507, 547, 525, 519, 471, 522, 498, 534, 522, 481, 525, 511, 507, 474, 512, 496, 510, 545, 496, 512, 519, 519, 465, 520, 540, 527, 503, 545, 489, 529, 495, 477, 562, 485, 512, 479, 516, 524, 522, 527, 495, 520, 505, 562, 547, 498, 538, 518, 533, 478, 501, 584, 531, 510, 511, 514, 510, 562, 494, 481, 519, 527, 546, 489, 490, 531, 528, 533, 510, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[520, 465, 536, 557, 498, 470, 503, 539, 502, 559, 541, 473, 508, 475, 535, 532, 517, 491, 533, 496, 515, 509, 566, 502, 486, 496, 497, 540, 494, 563, 517, 498, 550, 519, 521, 501, 505, 521, 506, 494, 547, 536, 493, 499, 521, 497, 502, 506, 482, 499, 513, 516, 548, 503, 486, 489, 514, 514, 535, 536, 482, 490, 525, 533, 534, 510, 527, 552, 537, 492, 505, 520, 478, 555, 497, 517, 501, 527, 525, 510, 522, 470, 462, 520, 491, 541, 523, 500, 522, 499, 510, 498, 505, 535, 506, 554, 500, 532, 524, 513, 495, 515, 554, 506, 503, 532, 522, 481, 481, 537, 542, 475, 496, 521, 511, 525, 491, 473, 562, 520, 504, 494, 476, 501, 479, 507, 503, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[527, 544, 490, 480, 479, 509, 502, 507, 519, 504, 491, 543, 528, 550, 533, 528, 520, 526, 505, 520, 491, 515, 521, 520, 538, 502, 518, 529, 527, 490, 514, 475, 511, 500, 539, 499, 533, 477, 516, 527, 504, 491, 539, 521, 482, 516, 519, 499, 520, 527, 498, 495, 521, 506, 470, 500, 495, 508, 528, 520, 525, 477, 513, 500, 510, 540, 502, 486, 541, 524, 537, 504, 518, 523, 529, 521, 513, 518, 495, 525, 514, 537, 506, 513, 463, 499, 516, 533, 509, 516, 532, 502, 538, 532, 489, 502, 504, 487, 505, 466, 497, 471, 573, 501, 520, 501, 514, 534, 524, 491, 536, 502, 530, 536, 523, 494, 532, 500, 498, 482, 534, 517, 514, 479, 515, 539, 502, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[514, 508, 510, 522, 504, 514, 523, 500, 499, 504, 494, 469, 520, 490, 536, 478, 518, 520, 502, 517, 515, 515, 542, 523, 512, 480, 489, 514, 501, 515, 483, 539, 505, 531, 538, 531, 519, 556, 519, 511, 502, 513, 520, 558, 510, 536, 514, 498, 534, 510, 483, 486, 495, 521, 455, 481, 492, 483, 515, 512, 515, 501, 544, 463, 535, 501, 511, 573, 503, 503, 495, 514, 509, 517, 509, 491, 484, 491, 509, 513, 544, 487, 551, 475, 487, 503, 488, 522, 490, 505, 506, 531, 468, 544, 495, 447, 568, 539, 534, 479, 552, 519, 522, 521, 552, 525, 525, 506, 516, 556, 498, 514, 519, 517, 522, 534, 521, 544, 501, 510, 520, 517, 503, 519, 516, 529, 480, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[505, 502, 497, 538, 534, 467, 557, 523, 543, 513, 500, 529, 524, 519, 539, 494, 517, 540, 459, 510, 514, 511, 518, 512, 507, 502, 526, 499, 482, 569, 521, 464, 520, 472, 504, 498, 532, 521, 502, 524, 534, 517, 509, 533, 518, 565, 528, 499, 511, 502, 538, 521, 501, 543, 507, 515, 472, 493, 546, 538, 480, 483, 508, 513, 478, 515, 481, 516, 492, 498, 505, 497, 509, 503, 538, 442, 529, 507, 516, 475, 527, 522, 510, 514, 481, 513, 495, 551, 510, 480, 518, 507, 517, 496, 536, 504, 515, 521, 513, 507, 482, 497, 501, 540, 524, 509, 507, 542, 531, 495, 534, 505, 493, 505, 500, 522, 524, 498, 533, 520, 495, 532, 523, 539, 491, 532, 557, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[498, 490, 529, 538, 514, 538, 486, 503, 520, 506, 540, 540, 488, 502, 494, 492, 477, 512, 516, 523, 501, 534, 517, 499, 502, 496, 495, 522, 512, 500, 529, 509, 463, 510, 510, 503, 511, 487, 488, 536, 514, 540, 555, 538, 472, 528, 494, 479, 535, 537, 518, 509, 501, 493, 561, 488, 507, 506, 478, 544, 511, 474, 468, 515, 538, 557, 490, 541, 502, 514, 531, 519, 520, 555, 522, 525, 563, 529, 525, 510, 454, 512, 509, 534, 503, 528, 531, 538, 517, 487, 510, 509, 520, 531, 512, 522, 504, 505, 510, 526, 516, 478, 492, 471, 465, 519, 540, 524, 519, 499, 528, 512, 503, 498, 536, 523, 494, 507, 558, 520, 480, 499, 503, 456, 539, 527, 500, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[509, 475, 534, 487, 503, 494, 505, 484, 509, 501, 562, 520, 509, 501, 499, 502, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 536, 495, 520, 495, 548, 524, 524, 513, 519, 548, 547, 518, 473, 505, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 514, 524, 505, 458, 524, 516, 527, 509, 501, 534, 501, 525, 520, 521, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 521, 484, 507, 522, 492, 498, 481, 564, 497, 532, 502, 506, 494, 503, 542, 38, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 511, 503, 517, 514, 511, 527, 533, 491, 498, 489, 507, 523, 538, 503, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 524, 516, 487, 516, 509, 489, 484, 542, 502, 501, 520, 540, 484, 519, 512, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 523, 536, 495, 512, 557, 508, 498, 542, 525, 497, 493, 502, 515, 460, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 527, 535, 505, 495, 566, 522, 494, 505, 515, 488, 470, 519, 517, 521, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8326, 8213, 8181, 8184, 8192, 8140, 8187] avg=8192.0000 max=8326 max_rank=1 min=8113 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[480, 479, 500, 509, 533, 512, 492, 547, 452, 505, 477, 475, 523, 510, 534, 514, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 514, 459, 536, 522, 507, 488, 523, 525, 563, 451, 505, 497, 530, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 485, 517, 512, 499, 554, 552, 501, 491, 523, 466, 506, 498, 508, 552, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 511, 514, 545, 522, 502, 501, 529, 521, 507, 547, 525, 519, 471, 522, 41, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 534, 522, 456, 525, 511, 507, 474, 512, 496, 510, 545, 496, 512, 519, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 520, 540, 527, 503, 545, 489, 529, 495, 477, 562, 485, 512, 479, 516, 524, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 527, 495, 520, 505, 562, 547, 498, 538, 518, 533, 478, 501, 584, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 510, 562, 494, 481, 519, 527, 546, 489, 490, 531, 528, 533, 510, 502, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8097, 8157, 8319, 8136, 8188, 8328, 8252] avg=8192.0000 max=8328 max_rank=6 min=8059 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[538, 487, 470, 548, 527, 490, 525, 505, 518, 522, 498, 501, 480, 529, 564, 553, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 549, 506, 526, 485, 527, 504, 511, 506, 548, 503, 501, 496, 534, 529, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 492, 542, 515, 492, 497, 495, 525, 511, 538, 510, 518, 500, 479, 524, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 531, 534, 501, 499, 504, 554, 498, 523, 489, 499, 507, 481, 467, 499, 505, 42, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 522, 504, 488, 523, 494, 540, 521, 493, 473, 496, 511, 514, 551, 533, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 533, 524, 534, 499, 532, 546, 518, 512, 525, 503, 483, 517, 488, 482, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 556, 517, 473, 538, 555, 487, 468, 524, 567, 500, 509, 498, 517, 500, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 528, 505, 518, 498, 508, 530, 536, 516, 501, 479, 507, 496, 492, 526, 456, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8224, 8161, 8171, 8196, 8243, 8180, 8088] avg=8192.0000 max=8273 max_rank=0 min=8088 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[527, 544, 490, 480, 479, 509, 502, 507, 519, 504, 491, 543, 528, 550, 533, 528, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 505, 520, 491, 515, 515, 520, 538, 502, 518, 529, 527, 490, 514, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 500, 539, 499, 533, 477, 516, 527, 467, 491, 539, 521, 482, 516, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 527, 498, 495, 521, 506, 470, 500, 495, 508, 528, 520, 525, 477, 513, 500, 45, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 540, 502, 447, 541, 524, 537, 504, 518, 523, 529, 521, 513, 518, 495, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 537, 506, 513, 463, 499, 516, 533, 509, 516, 532, 502, 538, 532, 489, 502, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 487, 505, 466, 497, 471, 573, 501, 520, 501, 514, 534, 524, 491, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 536, 523, 494, 532, 500, 498, 482, 534, 517, 514, 479, 515, 539, 502, 507, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8205, 8136, 8168, 8247, 8230, 8081, 8214] avg=8192.0000 max=8255 max_rank=0 min=8081 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[520, 465, 536, 557, 498, 470, 503, 539, 502, 559, 541, 473, 508, 475, 535, 532, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 491, 533, 496, 515, 509, 562, 502, 486, 496, 497, 540, 494, 563, 517, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 519, 521, 501, 505, 521, 506, 494, 517, 536, 493, 499, 521, 497, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 499, 513, 516, 548, 503, 486, 489, 514, 514, 535, 536, 482, 490, 525, 533, 42, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 527, 523, 537, 492, 505, 520, 478, 555, 497, 517, 501, 527, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 470, 462, 520, 491, 541, 523, 500, 522, 499, 510, 498, 505, 535, 506, 554, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 532, 524, 513, 495, 515, 554, 506, 503, 532, 522, 481, 481, 537, 500, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 521, 511, 525, 491, 473, 562, 520, 504, 494, 476, 501, 479, 507, 503, 500, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8216, 8188, 8224, 8258, 8180, 8170, 8068] avg=8192.0000 max=8258 max_rank=4 min=8068 min_rank=7 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 508, 510, 522, 504, 514, 523, 500, 499, 504, 494, 469, 520, 490, 536, 478, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 520, 502, 517, 515, 515, 538, 523, 512, 480, 489, 514, 501, 515, 483, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 538, 531, 519, 556, 519, 511, 475, 513, 520, 558, 510, 536, 514, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 483, 486, 495, 521, 455, 481, 492, 483, 515, 512, 515, 501, 544, 463, 38, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 501, 511, 543, 503, 503, 495, 514, 509, 517, 509, 491, 484, 491, 509, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 487, 551, 475, 487, 503, 488, 522, 490, 505, 506, 531, 468, 544, 495, 447, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 539, 534, 479, 552, 519, 522, 521, 552, 525, 525, 506, 516, 556, 460, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 517, 522, 534, 521, 544, 501, 510, 520, 517, 503, 519, 516, 529, 480, 536, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8181, 8334, 8043, 8128, 8064, 8388, 8293] avg=8192.0000 max=8388 max_rank=6 min=8043 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[505, 502, 497, 538, 534, 467, 557, 523, 543, 513, 500, 529, 524, 519, 539, 494, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 540, 459, 510, 514, 511, 514, 512, 507, 502, 526, 499, 482, 569, 521, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 472, 504, 498, 532, 521, 502, 524, 505, 517, 509, 533, 518, 565, 528, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 538, 521, 501, 543, 507, 515, 472, 493, 546, 538, 480, 483, 508, 513, 41, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 481, 489, 492, 498, 505, 497, 509, 503, 538, 442, 529, 507, 516, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 522, 510, 514, 481, 513, 495, 551, 510, 480, 518, 507, 517, 496, 536, 504, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 521, 513, 507, 482, 497, 501, 540, 524, 509, 507, 542, 531, 495, 493, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 500, 522, 524, 498, 533, 520, 495, 532, 523, 539, 491, 532, 557, 485, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8302, 8147, 8247, 8228, 7974, 8202, 8182, 8254] avg=8192.0000 max=8302 max_rank=0 min=7974 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[498, 490, 529, 538, 514, 538, 486, 503, 520, 506, 540, 540, 488, 502, 494, 492, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 512, 516, 523, 501, 534, 513, 499, 502, 496, 495, 522, 512, 500, 529, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 510, 510, 503, 511, 487, 488, 536, 486, 540, 555, 538, 472, 528, 494, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 537, 518, 509, 501, 493, 561, 488, 507, 506, 478, 544, 511, 474, 468, 515, 41, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 557, 490, 513, 502, 514, 531, 519, 520, 555, 522, 525, 563, 529, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 512, 509, 534, 503, 528, 531, 538, 517, 487, 510, 509, 520, 531, 512, 522, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 510, 526, 516, 478, 492, 471, 465, 519, 540, 524, 519, 499, 487, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 498, 536, 523, 494, 507, 558, 520, 480, 499, 503, 456, 539, 527, 500, 532, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8140, 8100, 8202, 8413, 8237, 8067, 8180] avg=8192.0000 max=8413 max_rank=4 min=8067 min_rank=6 imbalance_ratio=1.026978 max_load_reduction=57123 max_load_reduction_pct=0.871628 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4352, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 3968, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4096, 3968, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 3968, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4352, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True + [2026-08-26 16:29:47] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 419.7 | throughput per GPU (TFLOP/s/GPU): 430.8 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.040114E+01 | loss scale: 1.0 | grad norm: 88.395 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[532, 555, 511, 553, 502, 472, 538, 512, 537, 501, 508, 491, 564, 496, 521, 517, 519, 479, 521, 492, 467, 513, 470, 507, 546, 512, 517, 531, 513, 521, 520, 518, 524, 545, 510, 493, 548, 553, 525, 502, 531, 518, 534, 516, 513, 528, 511, 488, 544, 531, 498, 520, 522, 527, 513, 486, 506, 531, 516, 500, 511, 497, 505, 533, 525, 475, 491, 474, 505, 486, 532, 490, 533, 494, 519, 531, 484, 520, 553, 513, 536, 502, 496, 513, 464, 572, 474, 494, 521, 486, 491, 499, 540, 507, 469, 527, 527, 505, 550, 495, 516, 507, 511, 502, 502, 482, 519, 530, 494, 518, 511, 472, 523, 538, 489, 544, 521, 506, 487, 476, 520, 518, 514, 501, 509, 478, 502, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[553, 517, 468, 560, 540, 519, 509, 503, 507, 535, 523, 487, 502, 548, 501, 506, 541, 533, 513, 539, 502, 523, 564, 453, 495, 518, 451, 497, 501, 530, 517, 529, 486, 502, 517, 488, 502, 498, 471, 507, 514, 529, 555, 554, 508, 505, 529, 535, 528, 511, 526, 532, 492, 502, 505, 519, 521, 476, 512, 496, 509, 523, 504, 501, 524, 536, 553, 510, 520, 520, 518, 499, 517, 527, 493, 561, 523, 491, 486, 515, 516, 520, 499, 517, 526, 521, 490, 548, 521, 474, 488, 481, 492, 481, 539, 491, 538, 528, 495, 500, 510, 518, 508, 515, 482, 492, 559, 478, 524, 512, 526, 502, 485, 523, 541, 518, 502, 511, 529, 504, 504, 486, 498, 497, 513, 517, 489, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[505, 467, 492, 500, 506, 523, 546, 503, 514, 525, 516, 538, 487, 483, 530, 557, 500, 512, 537, 523, 536, 523, 469, 550, 491, 538, 473, 569, 469, 527, 520, 492, 562, 517, 531, 483, 486, 536, 531, 506, 514, 501, 534, 526, 508, 523, 443, 465, 556, 561, 503, 523, 495, 545, 509, 476, 488, 502, 532, 538, 519, 552, 537, 522, 508, 545, 510, 507, 496, 504, 512, 526, 520, 509, 515, 500, 497, 496, 520, 533, 475, 486, 513, 516, 503, 480, 520, 505, 489, 522, 509, 509, 513, 525, 540, 495, 535, 502, 505, 487, 493, 509, 481, 521, 529, 479, 523, 514, 471, 522, 515, 529, 538, 504, 515, 519, 521, 508, 512, 529, 512, 482, 536, 512, 493, 488, 515, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[470, 494, 545, 585, 521, 514, 522, 536, 528, 507, 549, 544, 502, 510, 448, 517, 516, 530, 523, 482, 495, 513, 506, 532, 512, 527, 507, 494, 502, 502, 513, 498, 499, 515, 509, 497, 517, 558, 558, 488, 525, 520, 537, 520, 542, 480, 514, 522, 532, 504, 547, 478, 522, 547, 531, 500, 503, 482, 508, 523, 476, 491, 508, 469, 568, 462, 481, 483, 528, 507, 508, 503, 508, 531, 562, 496, 529, 500, 501, 491, 500, 494, 525, 506, 519, 498, 526, 487, 499, 487, 485, 560, 484, 491, 506, 507, 477, 548, 497, 509, 548, 480, 513, 504, 530, 488, 532, 519, 505, 504, 493, 543, 519, 500, 547, 503, 510, 546, 544, 532, 511, 503, 513, 498, 527, 489, 500, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[486, 473, 495, 497, 481, 497, 532, 534, 526, 537, 486, 488, 544, 521, 495, 549, 479, 527, 520, 514, 515, 495, 498, 489, 532, 503, 520, 508, 504, 532, 520, 498, 539, 506, 516, 520, 483, 501, 508, 503, 479, 525, 503, 571, 517, 521, 511, 528, 480, 511, 546, 514, 582, 518, 532, 511, 499, 530, 572, 530, 515, 491, 554, 504, 471, 506, 503, 495, 530, 525, 535, 497, 506, 488, 483, 511, 543, 539, 555, 492, 538, 500, 517, 500, 472, 569, 489, 527, 503, 490, 513, 513, 477, 466, 475, 472, 510, 533, 510, 523, 487, 503, 532, 542, 501, 503, 507, 476, 514, 512, 501, 525, 527, 524, 488, 521, 522, 521, 513, 551, 529, 513, 514, 504, 481, 533, 491, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[546, 479, 500, 533, 536, 487, 494, 535, 502, 488, 559, 511, 531, 519, 548, 512, 506, 510, 498, 476, 496, 519, 500, 501, 524, 511, 512, 491, 487, 512, 545, 518, 496, 537, 503, 496, 524, 522, 467, 490, 505, 524, 530, 498, 534, 519, 530, 507, 514, 495, 548, 500, 539, 530, 511, 519, 508, 550, 497, 523, 526, 497, 471, 513, 521, 530, 537, 558, 488, 517, 532, 532, 523, 532, 460, 523, 532, 498, 554, 484, 496, 514, 487, 496, 532, 530, 486, 501, 517, 502, 509, 502, 479, 486, 488, 525, 504, 496, 523, 506, 509, 505, 498, 504, 550, 481, 486, 541, 541, 499, 531, 522, 506, 513, 521, 503, 528, 497, 549, 489, 484, 497, 549, 526, 468, 497, 510, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[517, 520, 523, 553, 501, 483, 519, 480, 503, 518, 528, 516, 539, 526, 482, 512, 511, 519, 524, 506, 551, 521, 495, 461, 511, 524, 525, 493, 516, 511, 527, 519, 538, 539, 539, 535, 536, 563, 519, 499, 488, 513, 518, 519, 462, 510, 508, 522, 507, 560, 532, 524, 528, 507, 557, 514, 490, 525, 511, 528, 486, 477, 519, 507, 524, 506, 505, 485, 491, 525, 510, 502, 501, 510, 503, 494, 540, 490, 488, 501, 527, 537, 496, 528, 531, 526, 496, 522, 491, 512, 511, 508, 501, 518, 524, 516, 512, 477, 504, 530, 526, 524, 515, 485, 493, 506, 537, 489, 501, 497, 493, 477, 525, 551, 499, 505, 520, 535, 550, 527, 496, 481, 466, 475, 509, 493, 486, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[518, 513, 523, 508, 509, 532, 532, 524, 476, 507, 493, 491, 511, 527, 520, 515, 522, 476, 537, 526, 521, 503, 520, 495, 515, 499, 492, 536, 469, 507, 531, 488, 513, 535, 527, 507, 476, 502, 478, 515, 505, 485, 484, 496, 537, 512, 496, 507, 522, 484, 542, 507, 527, 544, 493, 517, 483, 485, 480, 517, 521, 502, 501, 500, 532, 504, 515, 502, 524, 505, 544, 513, 491, 520, 509, 493, 529, 522, 524, 528, 510, 510, 495, 574, 494, 481, 500, 544, 499, 500, 500, 527, 523, 559, 499, 502, 543, 580, 512, 514, 545, 524, 509, 501, 516, 490, 503, 505, 525, 469, 535, 533, 545, 519, 497, 505, 509, 516, 484, 551, 509, 531, 521, 520, 497, 509, 492, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[505, 467, 492, 457, 506, 523, 546, 503, 514, 525, 516, 538, 487, 483, 530, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 537, 523, 536, 523, 469, 550, 491, 538, 473, 569, 469, 527, 520, 492, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 517, 531, 483, 486, 503, 531, 506, 514, 501, 534, 526, 508, 523, 443, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 561, 461, 523, 495, 545, 509, 476, 488, 502, 532, 538, 519, 552, 537, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 545, 510, 507, 496, 504, 504, 526, 520, 509, 515, 500, 497, 496, 520, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 486, 513, 516, 503, 480, 520, 505, 489, 522, 509, 509, 513, 525, 540, 495, 43, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 505, 487, 493, 509, 481, 521, 529, 479, 523, 514, 471, 522, 515, 529, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 504, 515, 519, 521, 508, 512, 529, 512, 482, 536, 512, 493, 488, 515, 494, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8251, 8133, 8316, 8190, 8173, 8135, 8189] avg=8192.0000 max=8316 max_rank=3 min=8133 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[553, 517, 468, 511, 540, 519, 509, 503, 507, 535, 523, 487, 502, 548, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 533, 513, 539, 502, 523, 564, 453, 495, 518, 451, 497, 501, 530, 517, 529, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 517, 488, 502, 469, 471, 507, 514, 529, 555, 554, 508, 505, 529, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 511, 481, 532, 492, 502, 505, 519, 521, 476, 512, 496, 509, 523, 504, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 536, 553, 510, 520, 520, 510, 499, 517, 527, 493, 561, 523, 491, 486, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 520, 499, 517, 526, 521, 490, 548, 521, 474, 488, 481, 492, 481, 539, 491, 49, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 528, 495, 500, 510, 518, 508, 515, 482, 492, 559, 478, 524, 512, 526, 502, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 523, 541, 518, 502, 511, 529, 504, 504, 486, 498, 497, 513, 517, 489, 494, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8228, 8171, 8112, 8285, 8185, 8205, 8121] avg=8192.0000 max=8285 max_rank=4 min=8112 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[532, 555, 511, 501, 502, 472, 538, 512, 537, 501, 508, 491, 564, 496, 521, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 479, 521, 492, 467, 513, 470, 507, 546, 512, 517, 531, 513, 521, 520, 518, 18, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 545, 510, 493, 548, 507, 525, 502, 531, 518, 534, 516, 513, 528, 511, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 531, 447, 520, 522, 527, 513, 486, 506, 531, 516, 500, 511, 497, 505, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 475, 491, 474, 505, 486, 520, 490, 533, 494, 519, 531, 484, 520, 553, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 502, 496, 513, 464, 572, 474, 494, 521, 486, 491, 499, 540, 507, 469, 527, 52, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 505, 550, 495, 516, 507, 511, 502, 502, 482, 519, 530, 494, 518, 511, 472, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 538, 489, 544, 521, 506, 487, 476, 520, 518, 514, 501, 509, 478, 502, 518, 12, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8179, 8293, 8189, 8113, 8176, 8165, 8163] avg=8192.0000 max=8293 max_rank=2 min=8113 min_rank=4 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[470, 494, 545, 534, 521, 514, 522, 536, 528, 507, 549, 544, 502, 510, 448, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 530, 523, 482, 495, 513, 506, 532, 512, 527, 507, 494, 502, 502, 513, 498, 13, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 515, 509, 497, 517, 524, 558, 488, 525, 520, 537, 520, 542, 480, 514, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 501, 478, 522, 547, 531, 500, 503, 482, 508, 523, 476, 491, 508, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 462, 481, 483, 528, 507, 500, 503, 508, 531, 562, 496, 529, 500, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 494, 525, 506, 519, 498, 526, 487, 499, 487, 485, 560, 484, 491, 506, 507, 51, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 548, 497, 509, 548, 480, 513, 504, 530, 488, 532, 519, 505, 504, 493, 543, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 547, 503, 510, 546, 544, 532, 511, 503, 513, 498, 527, 489, 500, 506, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8176, 8267, 8075, 8150, 8158, 8210, 8259] avg=8192.0000 max=8267 max_rank=2 min=8075 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[486, 473, 495, 454, 481, 497, 532, 534, 526, 537, 486, 488, 544, 521, 495, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 527, 520, 514, 515, 495, 498, 489, 532, 503, 520, 508, 504, 532, 520, 498, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 506, 516, 520, 483, 472, 508, 503, 479, 525, 503, 571, 517, 521, 511, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 511, 500, 514, 582, 518, 532, 511, 499, 530, 572, 530, 515, 491, 554, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 506, 503, 495, 530, 525, 527, 497, 506, 488, 483, 511, 543, 539, 555, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 500, 517, 500, 472, 569, 489, 527, 503, 490, 513, 513, 477, 466, 475, 472, 43, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 533, 510, 523, 487, 503, 532, 542, 501, 503, 507, 476, 514, 512, 501, 525, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 524, 488, 521, 522, 521, 513, 551, 529, 513, 514, 504, 481, 533, 491, 510, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8176, 8202, 8343, 8171, 8097, 8197, 8252] avg=8192.0000 max=8343 max_rank=3 min=8097 min_rank=5 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[517, 520, 523, 505, 501, 483, 519, 480, 503, 518, 528, 516, 539, 526, 482, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 524, 506, 551, 521, 495, 461, 511, 524, 525, 493, 516, 511, 527, 519, 13, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 539, 539, 535, 536, 528, 519, 499, 488, 513, 518, 519, 462, 510, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 560, 487, 524, 528, 507, 557, 514, 490, 525, 511, 528, 486, 477, 519, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 505, 485, 491, 525, 502, 502, 501, 510, 503, 494, 540, 490, 488, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 537, 496, 528, 531, 526, 496, 522, 491, 512, 511, 508, 501, 518, 524, 516, 48, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 477, 504, 530, 526, 524, 515, 485, 493, 506, 537, 489, 501, 497, 493, 477, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 551, 499, 505, 520, 535, 550, 527, 496, 481, 466, 475, 509, 493, 486, 519, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8238, 8273, 8227, 8067, 8324, 8087, 8148] avg=8192.0000 max=8324 max_rank=5 min=8067 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 513, 523, 464, 509, 532, 532, 524, 476, 507, 493, 491, 511, 527, 520, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 476, 537, 526, 521, 503, 520, 495, 515, 499, 492, 536, 469, 507, 531, 488, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 535, 527, 507, 476, 473, 478, 515, 505, 485, 484, 496, 537, 512, 496, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 484, 496, 507, 527, 544, 493, 517, 483, 485, 480, 517, 521, 502, 501, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 515, 502, 524, 505, 536, 513, 491, 520, 509, 493, 529, 522, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 510, 495, 574, 494, 481, 500, 544, 499, 500, 500, 527, 523, 559, 499, 502, 44, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 580, 512, 514, 545, 524, 509, 501, 516, 490, 503, 505, 525, 469, 535, 533, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 497, 505, 509, 516, 484, 551, 509, 531, 521, 520, 497, 509, 492, 519, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8159, 8046, 8079, 8247, 8294, 8322, 8234] avg=8192.0000 max=8322 max_rank=6 min=8046 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=33097 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[546, 479, 500, 487, 536, 487, 494, 535, 502, 488, 559, 511, 531, 519, 548, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 498, 476, 496, 519, 500, 501, 524, 511, 512, 491, 487, 512, 545, 518, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 537, 503, 496, 524, 491, 467, 490, 505, 524, 530, 498, 534, 519, 530, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 495, 502, 500, 539, 530, 511, 519, 508, 550, 497, 523, 526, 497, 471, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 530, 537, 558, 488, 517, 524, 532, 523, 532, 460, 523, 532, 498, 554, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 514, 487, 496, 532, 530, 486, 501, 517, 502, 509, 502, 479, 486, 488, 525, 46, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 496, 523, 506, 509, 505, 498, 504, 550, 481, 486, 541, 541, 499, 531, 522, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 521, 503, 528, 497, 549, 489, 484, 497, 549, 526, 468, 497, 510, 523, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8129, 8151, 8195, 8313, 8129, 8215, 8170] avg=8192.0000 max=8313 max_rank=4 min=8129 min_rank=1 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 3968, 4224, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 384, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[540, 545, 562, 524, 510, 535, 474, 492, 501, 533, 491, 510, 530, 491, 468, 510, 498, 507, 548, 484, 440, 510, 499, 500, 550, 505, 511, 519, 522, 591, 529, 506, 512, 530, 526, 559, 544, 517, 480, 526, 538, 506, 498, 538, 511, 501, 501, 496, 535, 509, 534, 503, 504, 486, 512, 461, 501, 554, 528, 486, 509, 480, 526, 498, 529, 456, 533, 484, 485, 462, 472, 533, 477, 545, 503, 541, 520, 468, 528, 552, 514, 533, 495, 530, 553, 513, 547, 504, 523, 537, 463, 535, 504, 491, 495, 487, 468, 528, 565, 486, 537, 497, 492, 511, 522, 496, 514, 492, 534, 521, 498, 517, 535, 513, 554, 507, 512, 526, 499, 503, 511, 515, 513, 506, 485, 511, 542, 470] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[502, 526, 497, 497, 498, 502, 539, 493, 517, 540, 497, 525, 509, 462, 535, 494, 508, 504, 525, 528, 492, 497, 516, 538, 520, 541, 516, 507, 491, 483, 524, 479, 510, 517, 507, 491, 531, 524, 515, 533, 478, 527, 497, 529, 560, 507, 518, 525, 507, 523, 498, 497, 548, 516, 529, 530, 464, 517, 514, 498, 509, 476, 497, 482, 534, 532, 539, 550, 514, 495, 505, 502, 502, 513, 535, 513, 513, 571, 507, 494, 451, 515, 530, 531, 496, 486, 507, 503, 498, 515, 526, 563, 572, 490, 512, 508, 528, 506, 537, 538, 507, 503, 470, 503, 477, 497, 527, 496, 491, 490, 534, 551, 494, 517, 509, 498, 511, 550, 471, 496, 549, 502, 509, 490, 541, 506, 524, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[494, 500, 526, 539, 517, 547, 467, 542, 534, 524, 515, 542, 530, 514, 520, 522, 530, 508, 510, 495, 497, 481, 507, 499, 537, 529, 542, 530, 531, 547, 514, 536, 522, 504, 495, 492, 505, 510, 479, 575, 510, 535, 510, 550, 543, 505, 509, 504, 526, 496, 536, 513, 521, 531, 470, 474, 497, 491, 525, 495, 513, 498, 528, 483, 493, 480, 535, 512, 505, 476, 516, 515, 509, 483, 544, 473, 546, 505, 507, 525, 524, 519, 555, 525, 497, 524, 492, 479, 480, 507, 507, 536, 486, 503, 510, 518, 524, 496, 542, 508, 512, 488, 463, 502, 532, 485, 510, 511, 500, 507, 483, 519, 504, 513, 496, 492, 490, 512, 523, 514, 542, 517, 527, 532, 509, 509, 509, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[531, 508, 497, 515, 511, 520, 523, 548, 524, 492, 514, 530, 551, 518, 501, 485, 521, 511, 451, 502, 515, 542, 478, 536, 494, 532, 496, 508, 515, 520, 538, 518, 509, 498, 518, 473, 492, 515, 510, 504, 514, 534, 498, 508, 540, 499, 510, 510, 508, 525, 517, 515, 512, 528, 517, 532, 527, 531, 546, 537, 518, 495, 492, 510, 499, 549, 482, 535, 522, 502, 481, 536, 551, 475, 513, 519, 498, 505, 533, 479, 554, 533, 549, 540, 481, 494, 508, 520, 513, 482, 514, 493, 511, 507, 495, 521, 514, 496, 463, 515, 512, 516, 518, 512, 514, 507, 550, 512, 540, 505, 481, 526, 472, 514, 515, 498, 503, 518, 511, 483, 478, 559, 506, 491, 525, 495, 511, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[508, 523, 556, 540, 522, 520, 498, 478, 515, 540, 497, 520, 530, 512, 502, 516, 513, 483, 537, 528, 510, 532, 509, 511, 548, 523, 516, 542, 518, 506, 483, 506, 492, 522, 495, 498, 509, 528, 478, 500, 508, 490, 506, 520, 497, 516, 493, 534, 517, 496, 521, 495, 532, 540, 521, 500, 478, 505, 505, 501, 527, 512, 507, 534, 541, 539, 528, 493, 508, 506, 529, 494, 512, 506, 533, 514, 521, 487, 534, 512, 501, 507, 484, 502, 550, 485, 530, 556, 521, 485, 466, 521, 492, 484, 507, 482, 517, 526, 480, 530, 539, 539, 520, 530, 528, 493, 503, 503, 512, 484, 497, 520, 540, 463, 555, 494, 496, 513, 491, 522, 498, 555, 499, 498, 518, 503, 515, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[492, 519, 547, 533, 523, 484, 519, 481, 506, 481, 539, 524, 503, 533, 540, 550, 520, 524, 518, 485, 522, 553, 505, 503, 563, 497, 529, 503, 538, 527, 540, 529, 509, 491, 525, 531, 486, 491, 545, 518, 530, 510, 508, 505, 514, 499, 538, 489, 518, 507, 484, 517, 523, 514, 487, 478, 490, 478, 484, 534, 479, 493, 555, 531, 518, 547, 495, 485, 517, 522, 489, 513, 537, 490, 516, 517, 520, 552, 518, 509, 516, 494, 490, 507, 514, 498, 514, 466, 520, 544, 545, 492, 498, 497, 534, 488, 505, 490, 494, 483, 509, 520, 455, 528, 514, 504, 520, 532, 530, 518, 543, 469, 507, 525, 535, 511, 542, 493, 511, 527, 477, 519, 538, 500, 500, 513, 504, 467] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[520, 520, 534, 480, 458, 510, 462, 512, 512, 533, 574, 518, 520, 543, 518, 535, 492, 533, 481, 565, 541, 515, 497, 519, 506, 517, 497, 516, 499, 516, 518, 516, 547, 533, 498, 496, 522, 526, 504, 516, 528, 552, 538, 537, 542, 526, 517, 520, 486, 502, 495, 529, 537, 504, 498, 509, 535, 483, 524, 515, 495, 477, 487, 513, 518, 503, 492, 531, 456, 508, 520, 494, 478, 500, 520, 528, 497, 509, 510, 524, 534, 532, 532, 507, 513, 516, 453, 484, 512, 489, 464, 509, 494, 534, 521, 537, 506, 522, 522, 529, 543, 527, 468, 527, 522, 493, 551, 467, 521, 497, 511, 508, 520, 493, 518, 543, 518, 499, 489, 554, 517, 514, 520, 480, 481, 492, 489, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[530, 519, 488, 515, 493, 488, 514, 519, 492, 498, 501, 492, 522, 515, 515, 509, 465, 495, 462, 476, 496, 538, 536, 511, 529, 525, 493, 488, 517, 499, 489, 494, 518, 523, 552, 492, 527, 484, 525, 531, 504, 547, 469, 529, 514, 508, 536, 519, 510, 521, 509, 500, 485, 523, 529, 501, 522, 473, 559, 522, 510, 504, 553, 500, 510, 525, 510, 510, 493, 467, 485, 512, 483, 498, 500, 497, 526, 505, 550, 538, 544, 547, 524, 520, 503, 522, 488, 506, 457, 538, 492, 539, 534, 551, 527, 467, 499, 470, 502, 546, 560, 480, 500, 542, 523, 518, 520, 522, 543, 505, 496, 507, 515, 554, 500, 521, 527, 578, 527, 520, 490, 495, 488, 523, 499, 523, 523, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[502, 526, 461, 497, 498, 502, 539, 493, 517, 540, 497, 525, 509, 462, 535, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 504, 525, 528, 492, 497, 516, 538, 499, 541, 516, 507, 491, 483, 524, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 517, 507, 491, 531, 524, 515, 533, 478, 527, 497, 529, 514, 507, 518, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 523, 498, 497, 548, 516, 529, 530, 464, 517, 514, 498, 509, 476, 497, 482, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 539, 550, 514, 495, 505, 502, 502, 513, 535, 513, 513, 571, 507, 494, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 515, 530, 531, 496, 486, 507, 503, 498, 515, 526, 563, 572, 490, 512, 508, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 537, 538, 507, 503, 470, 503, 477, 497, 527, 496, 491, 490, 534, 551, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 517, 509, 498, 511, 550, 471, 496, 549, 502, 509, 490, 541, 506, 524, 516, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8148, 8223, 8148, 8333, 8220, 8173, 8194] avg=8192.0000 max=8333 max_rank=4 min=8097 min_rank=0 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[540, 545, 520, 524, 510, 535, 474, 492, 501, 533, 491, 510, 530, 491, 468, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 507, 548, 484, 440, 510, 499, 500, 527, 505, 511, 519, 522, 591, 529, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 526, 559, 544, 517, 480, 526, 538, 506, 498, 538, 469, 501, 501, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 509, 534, 503, 504, 486, 512, 461, 501, 554, 528, 486, 509, 480, 526, 498, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 456, 533, 484, 485, 462, 472, 533, 477, 545, 503, 541, 520, 468, 528, 552, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 533, 495, 530, 553, 513, 547, 504, 523, 537, 463, 535, 504, 491, 495, 487, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 528, 565, 486, 537, 497, 492, 511, 522, 496, 514, 492, 534, 521, 498, 517, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 513, 554, 507, 512, 526, 499, 503, 511, 515, 513, 506, 485, 511, 542, 470, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8196, 8241, 8165, 8104, 8244, 8198, 8214] avg=8192.0000 max=8244 max_rank=5 min=8104 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 500, 486, 539, 517, 547, 467, 542, 534, 524, 515, 542, 530, 514, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 508, 510, 495, 497, 481, 507, 499, 514, 529, 542, 530, 531, 547, 514, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 495, 492, 505, 510, 479, 575, 510, 535, 510, 550, 499, 505, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 536, 513, 521, 531, 470, 474, 497, 491, 525, 495, 513, 498, 528, 483, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 480, 535, 512, 505, 476, 516, 515, 509, 483, 544, 473, 546, 505, 507, 525, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 519, 555, 525, 497, 524, 492, 479, 480, 507, 507, 536, 486, 503, 510, 518, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 542, 508, 512, 488, 463, 502, 532, 485, 510, 511, 500, 507, 483, 519, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 496, 492, 490, 512, 523, 514, 542, 517, 527, 532, 509, 509, 509, 508, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8270, 8204, 8138, 8140, 8181, 8101, 8209] avg=8192.0000 max=8293 max_rank=0 min=8101 min_rank=6 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 508, 450, 515, 511, 520, 523, 548, 524, 492, 514, 530, 551, 518, 501, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 451, 502, 515, 542, 478, 536, 466, 532, 496, 508, 515, 520, 538, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 518, 473, 492, 515, 510, 504, 514, 534, 498, 508, 490, 499, 510, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 517, 515, 512, 528, 517, 532, 527, 531, 546, 537, 518, 495, 492, 510, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 549, 482, 535, 522, 502, 481, 536, 551, 475, 513, 519, 498, 505, 533, 479, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 533, 549, 540, 481, 494, 508, 520, 513, 482, 514, 493, 511, 507, 495, 521, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 496, 463, 515, 512, 516, 518, 512, 514, 507, 550, 512, 540, 505, 481, 526, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 514, 515, 498, 503, 518, 511, 483, 478, 559, 506, 491, 525, 495, 511, 495, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8149, 8082, 8355, 8199, 8236, 8205, 8089] avg=8192.0000 max=8355 max_rank=3 min=8082 min_rank=2 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[520, 520, 494, 480, 458, 510, 462, 512, 512, 533, 574, 518, 520, 543, 518, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 533, 481, 565, 541, 515, 497, 519, 485, 517, 497, 516, 499, 516, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 533, 498, 496, 522, 526, 504, 516, 528, 552, 538, 537, 498, 526, 517, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 495, 529, 537, 504, 498, 509, 535, 483, 524, 515, 495, 477, 487, 513, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 492, 531, 456, 508, 520, 494, 478, 500, 520, 528, 497, 509, 510, 524, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 532, 507, 513, 516, 453, 484, 512, 489, 464, 509, 494, 534, 521, 537, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 522, 522, 529, 543, 527, 468, 527, 522, 493, 551, 467, 521, 497, 511, 508, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 493, 518, 543, 518, 499, 489, 554, 517, 514, 520, 480, 481, 492, 489, 508, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8207, 8358, 8130, 8103, 8150, 8233, 8146] avg=8192.0000 max=8358 max_rank=2 min=8103 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 523, 514, 540, 522, 520, 498, 478, 515, 540, 497, 520, 530, 512, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 483, 537, 528, 510, 532, 509, 511, 525, 523, 516, 542, 518, 506, 483, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 495, 498, 509, 528, 478, 500, 508, 490, 506, 520, 456, 516, 493, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 521, 495, 532, 540, 521, 500, 478, 505, 505, 501, 527, 512, 507, 534, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 539, 528, 493, 508, 506, 529, 494, 512, 506, 533, 514, 521, 487, 534, 512, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 484, 502, 550, 485, 530, 556, 521, 485, 466, 521, 492, 484, 507, 482, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 526, 480, 530, 539, 539, 520, 530, 528, 493, 503, 503, 512, 484, 497, 520, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 463, 555, 494, 496, 513, 491, 522, 498, 555, 499, 498, 518, 503, 515, 506, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8242, 8045, 8229, 8273, 8093, 8241, 8178] avg=8192.0000 max=8273 max_rank=4 min=8045 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[492, 519, 506, 533, 523, 484, 519, 481, 506, 481, 539, 524, 503, 533, 540, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 518, 485, 522, 553, 505, 503, 540, 497, 529, 503, 538, 527, 540, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 525, 531, 486, 491, 545, 518, 530, 510, 508, 505, 472, 499, 538, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 484, 517, 523, 514, 487, 478, 490, 478, 484, 534, 479, 493, 555, 531, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 547, 495, 485, 517, 522, 489, 513, 537, 490, 516, 517, 520, 552, 518, 509, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 494, 490, 507, 514, 498, 514, 466, 520, 544, 545, 492, 498, 497, 534, 488, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 490, 494, 483, 509, 520, 455, 528, 514, 504, 520, 532, 530, 518, 543, 469, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 535, 511, 542, 493, 511, 527, 477, 519, 538, 500, 500, 513, 504, 467, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8333, 8147, 8111, 8261, 8136, 8134, 8181] avg=8192.0000 max=8333 max_rank=1 min=8111 min_rank=3 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[530, 519, 452, 515, 493, 488, 514, 519, 492, 498, 501, 492, 522, 515, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 495, 462, 476, 496, 538, 536, 511, 507, 525, 493, 488, 517, 499, 489, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 523, 552, 492, 527, 484, 525, 531, 504, 547, 469, 529, 472, 508, 536, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 509, 500, 485, 523, 529, 501, 522, 473, 559, 522, 510, 504, 553, 500, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 510, 510, 493, 467, 485, 512, 483, 498, 500, 497, 526, 505, 550, 538, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 547, 524, 520, 503, 522, 488, 506, 457, 538, 492, 539, 534, 551, 527, 467, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 470, 502, 546, 560, 480, 500, 542, 523, 518, 520, 522, 543, 505, 496, 507, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 554, 500, 521, 527, 578, 527, 520, 490, 495, 488, 523, 499, 523, 523, 530, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8074, 7991, 8236, 8260, 8123, 8276, 8251, 8325] avg=8192.0000 max=8325 max_rank=7 min=7991 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 3968, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[503, 550, 515, 460, 511, 503, 533, 513, 518, 520, 521, 458, 537, 504, 503, 520, 528, 540, 514, 545, 478, 495, 496, 516, 535, 537, 508, 492, 532, 543, 478, 527, 485, 517, 503, 526, 494, 519, 524, 505, 496, 507, 524, 530, 528, 501, 509, 508, 520, 507, 504, 545, 537, 503, 494, 539, 486, 504, 546, 522, 472, 540, 507, 517, 512, 501, 503, 488, 493, 515, 522, 470, 482, 520, 519, 460, 510, 508, 537, 492, 494, 526, 528, 502, 528, 472, 507, 499, 500, 518, 515, 505, 487, 494, 476, 500, 496, 547, 468, 519, 517, 521, 482, 515, 545, 530, 499, 509, 539, 506, 501, 509, 500, 515, 546, 518, 534, 516, 521, 505, 514, 554, 497, 538, 530, 538, 524, 548] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[512, 468, 528, 521, 502, 499, 495, 497, 478, 522, 521, 498, 496, 498, 509, 533, 489, 522, 524, 495, 500, 536, 515, 516, 493, 533, 531, 480, 503, 515, 525, 474, 537, 514, 485, 520, 520, 494, 514, 485, 518, 484, 518, 504, 517, 526, 491, 496, 492, 541, 533, 496, 513, 499, 567, 531, 509, 507, 535, 524, 494, 524, 472, 537, 534, 490, 515, 495, 531, 482, 514, 536, 515, 477, 468, 516, 522, 482, 508, 540, 489, 537, 520, 519, 550, 537, 528, 517, 545, 481, 484, 516, 514, 543, 490, 498, 500, 499, 534, 468, 512, 524, 546, 492, 506, 518, 516, 521, 533, 514, 516, 542, 538, 478, 521, 536, 537, 504, 516, 488, 538, 519, 517, 517, 515, 502, 496, 555] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[512, 482, 473, 517, 479, 492, 538, 476, 471, 512, 496, 537, 514, 533, 482, 496, 526, 521, 535, 462, 507, 515, 497, 529, 470, 497, 513, 490, 504, 491, 547, 517, 502, 503, 525, 466, 495, 521, 489, 551, 520, 498, 529, 506, 514, 517, 520, 549, 533, 487, 518, 515, 552, 514, 495, 515, 525, 499, 532, 520, 522, 538, 562, 539, 530, 533, 474, 494, 521, 539, 520, 497, 521, 486, 491, 505, 544, 487, 511, 489, 510, 566, 556, 527, 531, 483, 500, 520, 518, 525, 547, 496, 508, 520, 527, 522, 510, 491, 549, 519, 507, 455, 510, 513, 513, 525, 508, 536, 487, 556, 522, 561, 513, 485, 517, 496, 522, 487, 513, 514, 522, 516, 504, 461, 513, 506, 522, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[492, 540, 481, 500, 514, 482, 509, 562, 512, 539, 493, 519, 506, 507, 539, 512, 551, 523, 512, 494, 503, 518, 498, 493, 502, 471, 518, 497, 518, 483, 526, 479, 501, 529, 488, 498, 478, 493, 532, 531, 499, 513, 511, 487, 495, 495, 507, 566, 553, 541, 470, 547, 543, 519, 502, 538, 518, 510, 500, 514, 500, 528, 527, 496, 514, 494, 510, 490, 526, 511, 498, 532, 480, 534, 507, 498, 502, 552, 508, 487, 516, 493, 525, 484, 485, 519, 483, 511, 502, 480, 508, 502, 549, 503, 538, 483, 489, 549, 549, 538, 561, 539, 488, 507, 499, 515, 496, 494, 519, 552, 498, 516, 508, 567, 494, 506, 504, 459, 527, 502, 530, 528, 538, 513, 507, 530, 528, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[524, 515, 490, 519, 495, 521, 519, 486, 474, 574, 503, 510, 493, 536, 565, 488, 543, 499, 538, 531, 544, 535, 500, 521, 491, 504, 502, 526, 542, 491, 474, 512, 495, 496, 494, 474, 458, 503, 480, 509, 509, 490, 503, 473, 535, 525, 498, 520, 497, 486, 532, 520, 486, 516, 517, 510, 559, 499, 482, 510, 498, 519, 527, 542, 510, 545, 514, 513, 524, 543, 491, 499, 497, 520, 484, 539, 524, 503, 516, 488, 487, 528, 553, 526, 536, 481, 573, 499, 496, 477, 520, 528, 500, 470, 561, 563, 534, 521, 471, 533, 496, 504, 534, 541, 499, 505, 572, 478, 507, 563, 505, 514, 505, 507, 504, 512, 507, 530, 493, 525, 503, 512, 494, 488, 542, 508, 531, 463] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[509, 559, 520, 494, 499, 490, 497, 503, 499, 514, 512, 465, 525, 524, 534, 523, 520, 526, 501, 512, 537, 503, 502, 464, 484, 506, 532, 523, 539, 514, 496, 513, 556, 523, 507, 517, 540, 510, 536, 504, 499, 559, 505, 504, 515, 485, 538, 505, 515, 510, 497, 538, 491, 478, 505, 534, 505, 535, 507, 516, 519, 530, 521, 468, 485, 530, 485, 500, 507, 510, 535, 524, 509, 486, 487, 474, 496, 534, 491, 507, 514, 460, 493, 492, 497, 484, 518, 527, 554, 510, 535, 525, 511, 507, 543, 509, 520, 479, 546, 494, 542, 521, 517, 524, 517, 530, 537, 472, 565, 534, 515, 508, 519, 539, 509, 488, 478, 519, 480, 511, 541, 523, 493, 523, 539, 525, 487, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[477, 495, 505, 515, 487, 490, 539, 543, 548, 534, 487, 503, 500, 500, 512, 513, 506, 495, 552, 500, 495, 491, 525, 500, 527, 521, 548, 537, 522, 510, 488, 530, 556, 504, 523, 532, 537, 539, 530, 549, 537, 509, 512, 491, 531, 499, 503, 538, 539, 520, 507, 462, 517, 512, 482, 512, 528, 522, 488, 502, 494, 521, 504, 474, 503, 489, 516, 527, 507, 505, 519, 507, 552, 504, 533, 513, 519, 519, 504, 497, 501, 504, 548, 517, 483, 503, 491, 514, 507, 509, 520, 454, 516, 519, 503, 515, 514, 530, 542, 501, 515, 515, 532, 514, 501, 510, 538, 563, 489, 469, 506, 535, 508, 489, 517, 502, 525, 458, 518, 540, 520, 499, 492, 545, 459, 449, 537, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[520, 532, 509, 500, 505, 502, 492, 499, 520, 519, 522, 506, 518, 499, 520, 490, 507, 504, 495, 517, 511, 516, 557, 484, 487, 526, 532, 523, 507, 528, 490, 529, 477, 501, 510, 510, 527, 514, 491, 517, 528, 519, 547, 486, 511, 496, 500, 501, 512, 471, 501, 527, 539, 478, 514, 520, 522, 514, 527, 485, 530, 561, 507, 477, 551, 487, 513, 526, 485, 531, 530, 527, 502, 540, 508, 512, 518, 490, 511, 515, 521, 539, 494, 506, 509, 498, 513, 538, 503, 472, 540, 546, 508, 515, 529, 516, 576, 501, 495, 536, 531, 533, 480, 530, 518, 535, 529, 533, 533, 464, 481, 482, 480, 523, 511, 524, 493, 496, 497, 497, 490, 520, 527, 506, 534, 522, 494, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 468, 528, 521, 502, 499, 495, 497, 478, 522, 521, 498, 496, 498, 509, 533, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 522, 524, 495, 500, 536, 515, 516, 493, 533, 531, 480, 503, 515, 525, 474, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 514, 485, 520, 520, 494, 514, 485, 518, 484, 518, 504, 517, 526, 491, 496, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 541, 533, 496, 513, 499, 567, 531, 509, 507, 535, 524, 494, 490, 472, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 515, 495, 531, 482, 514, 536, 515, 477, 468, 516, 522, 482, 508, 540, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 537, 515, 519, 550, 537, 528, 517, 545, 481, 484, 516, 514, 543, 490, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 499, 534, 468, 512, 524, 546, 492, 506, 518, 442, 521, 533, 514, 516, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 478, 521, 536, 537, 504, 516, 488, 538, 519, 517, 517, 515, 502, 496, 555, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8154, 8137, 8240, 8169, 8263, 8167, 8281] avg=8192.0000 max=8281 max_rank=7 min=8125 min_rank=0 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 540, 481, 500, 514, 482, 509, 562, 512, 539, 493, 519, 506, 507, 539, 512, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 523, 512, 494, 503, 518, 498, 493, 502, 471, 518, 497, 518, 483, 526, 479, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 529, 488, 498, 478, 493, 532, 531, 499, 513, 511, 487, 495, 495, 507, 566, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 541, 470, 547, 543, 519, 502, 538, 518, 510, 500, 514, 500, 494, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 494, 510, 490, 526, 511, 498, 532, 480, 534, 507, 498, 502, 552, 508, 487, 25, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 520, 484, 485, 519, 483, 511, 502, 480, 508, 502, 549, 503, 538, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 549, 549, 538, 561, 539, 488, 507, 499, 515, 425, 494, 519, 552, 498, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 567, 494, 506, 504, 459, 527, 502, 530, 528, 538, 513, 507, 530, 528, 540, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8089, 8137, 8272, 8186, 8076, 8238, 8285] avg=8192.0000 max=8285 max_rank=7 min=8076 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 482, 473, 517, 479, 492, 538, 476, 471, 512, 496, 537, 514, 533, 482, 496, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 521, 535, 462, 507, 515, 497, 529, 470, 497, 513, 490, 504, 491, 547, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 503, 525, 466, 495, 521, 489, 551, 520, 498, 529, 506, 514, 517, 520, 549, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 487, 518, 515, 552, 514, 495, 515, 525, 499, 532, 520, 522, 503, 562, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 533, 474, 494, 521, 539, 520, 497, 521, 486, 491, 505, 544, 487, 511, 489, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 566, 551, 527, 531, 483, 500, 520, 518, 525, 547, 496, 508, 520, 527, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 549, 519, 507, 455, 510, 513, 513, 525, 435, 536, 487, 556, 522, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 485, 517, 496, 522, 487, 513, 514, 522, 516, 504, 461, 513, 506, 522, 483, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8057, 8124, 8220, 8331, 8186, 8351, 8189, 8078] avg=8192.0000 max=8351 max_rank=5 min=8057 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[503, 550, 515, 460, 511, 503, 533, 513, 518, 520, 521, 458, 537, 504, 503, 520, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 540, 514, 545, 478, 495, 496, 516, 535, 537, 508, 492, 532, 543, 478, 527, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 517, 503, 526, 494, 519, 524, 505, 496, 507, 524, 530, 528, 501, 509, 508, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 507, 504, 545, 537, 503, 494, 539, 486, 504, 546, 522, 472, 491, 507, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 501, 503, 488, 493, 515, 522, 470, 482, 520, 519, 460, 510, 508, 537, 492, 28, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 526, 515, 502, 528, 472, 507, 499, 500, 518, 515, 505, 487, 494, 476, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 547, 468, 519, 517, 521, 482, 515, 545, 530, 421, 509, 539, 506, 501, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 515, 546, 518, 534, 516, 521, 505, 514, 554, 497, 538, 530, 538, 524, 548, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8272, 8195, 8194, 8082, 8038, 8125, 8411] avg=8192.0000 max=8411 max_rank=7 min=8038 min_rank=5 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[524, 515, 490, 519, 495, 521, 519, 486, 474, 574, 503, 510, 493, 536, 565, 488, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 499, 538, 531, 544, 535, 500, 521, 491, 504, 502, 526, 542, 491, 474, 512, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 494, 474, 458, 503, 480, 509, 509, 490, 503, 473, 535, 525, 498, 520, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 486, 532, 520, 486, 516, 517, 510, 559, 499, 482, 510, 498, 485, 527, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 545, 514, 513, 524, 543, 491, 499, 497, 520, 484, 539, 524, 503, 516, 488, 29, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 528, 548, 526, 536, 481, 573, 499, 496, 477, 520, 528, 500, 470, 561, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 471, 533, 496, 504, 534, 541, 499, 505, 490, 478, 507, 563, 505, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 507, 504, 512, 507, 530, 493, 525, 503, 512, 494, 488, 542, 508, 531, 463, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8256, 7976, 8166, 8257, 8293, 8195, 8128] avg=8192.0000 max=8293 max_rank=5 min=7976 min_rank=2 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[477, 495, 505, 515, 487, 490, 539, 543, 548, 534, 487, 503, 500, 500, 512, 513, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 552, 500, 495, 491, 525, 500, 527, 521, 548, 537, 522, 510, 488, 530, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 504, 523, 532, 537, 539, 530, 549, 537, 509, 512, 491, 531, 499, 503, 538, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 520, 507, 462, 517, 512, 482, 512, 528, 522, 488, 502, 494, 487, 504, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 489, 516, 527, 507, 505, 519, 507, 552, 504, 533, 513, 519, 519, 504, 497, 27, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 504, 543, 517, 483, 503, 491, 514, 507, 509, 520, 454, 516, 519, 503, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 530, 542, 501, 515, 515, 532, 514, 501, 510, 461, 563, 489, 469, 506, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 489, 517, 502, 525, 458, 518, 540, 520, 499, 492, 545, 459, 449, 537, 517, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8250, 8404, 8050, 8259, 8099, 8197, 8079] avg=8192.0000 max=8404 max_rank=2 min=8050 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=57132 max_load_reduction_pct=0.871765 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 559, 520, 494, 499, 490, 497, 503, 499, 514, 512, 465, 525, 524, 534, 523, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 501, 512, 537, 503, 502, 464, 484, 506, 532, 523, 539, 514, 496, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 523, 507, 517, 540, 510, 536, 504, 499, 559, 505, 504, 515, 485, 538, 505, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 497, 538, 491, 478, 505, 534, 505, 535, 507, 516, 519, 496, 521, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 530, 485, 500, 507, 510, 535, 524, 509, 486, 487, 474, 496, 534, 491, 507, 27, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 460, 488, 492, 497, 484, 518, 527, 554, 510, 535, 525, 511, 507, 543, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 479, 546, 494, 542, 521, 517, 524, 517, 530, 460, 472, 565, 534, 515, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 539, 509, 488, 478, 519, 480, 511, 541, 523, 493, 523, 539, 525, 487, 491, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8175, 8317, 8135, 8105, 8174, 8244, 8169] avg=8192.0000 max=8317 max_rank=2 min=8105 min_rank=4 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[520, 532, 509, 500, 505, 502, 492, 499, 520, 519, 522, 506, 518, 499, 520, 490, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 495, 517, 511, 516, 557, 484, 487, 526, 532, 523, 507, 528, 490, 529, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 501, 510, 510, 527, 514, 491, 517, 528, 519, 547, 486, 511, 496, 500, 501, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 471, 501, 527, 539, 478, 514, 520, 522, 514, 527, 485, 530, 524, 507, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 487, 513, 526, 485, 531, 530, 527, 502, 540, 508, 512, 518, 490, 511, 515, 27, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 539, 489, 506, 509, 498, 513, 538, 503, 472, 540, 546, 508, 515, 529, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 501, 495, 536, 531, 533, 480, 530, 518, 535, 453, 533, 533, 464, 481, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 511, 524, 493, 496, 497, 497, 490, 520, 527, 506, 534, 522, 494, 486, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8216, 8150, 8148, 8292, 8242, 8181, 8105] avg=8192.0000 max=8292 max_rank=4 min=8105 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 4224, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3712, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[528, 505, 489, 490, 513, 477, 538, 508, 530, 517, 547, 485, 525, 505, 484, 544, 525, 505, 543, 547, 494, 491, 489, 483, 461, 509, 536, 508, 523, 526, 523, 475, 492, 515, 501, 501, 517, 512, 522, 502, 508, 524, 552, 514, 457, 528, 527, 545, 512, 535, 449, 490, 523, 516, 519, 504, 506, 517, 530, 491, 513, 441, 501, 469, 540, 534, 553, 534, 523, 487, 551, 509, 514, 516, 507, 503, 495, 491, 556, 473, 490, 541, 505, 478, 483, 495, 512, 533, 554, 500, 536, 534, 553, 545, 536, 491, 512, 493, 522, 504, 492, 509, 493, 509, 466, 534, 531, 520, 513, 528, 502, 565, 511, 532, 518, 534, 526, 507, 517, 507, 505, 488, 515, 492, 531, 536, 485, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[487, 515, 488, 537, 506, 505, 527, 544, 508, 492, 515, 510, 506, 517, 486, 491, 487, 514, 544, 498, 487, 536, 543, 556, 489, 487, 500, 474, 470, 485, 511, 498, 566, 547, 542, 523, 554, 526, 519, 522, 497, 509, 516, 486, 522, 521, 544, 525, 507, 475, 557, 510, 530, 525, 534, 514, 490, 506, 563, 521, 523, 508, 536, 575, 539, 488, 492, 502, 497, 502, 525, 508, 492, 497, 473, 461, 523, 497, 478, 501, 519, 490, 509, 540, 519, 539, 512, 529, 512, 528, 498, 499, 515, 500, 493, 526, 514, 487, 490, 525, 537, 511, 539, 487, 513, 491, 515, 513, 527, 489, 504, 496, 489, 545, 523, 507, 526, 505, 488, 538, 535, 514, 511, 518, 517, 517, 452, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[519, 516, 501, 518, 570, 526, 502, 522, 503, 495, 524, 496, 527, 519, 495, 470, 503, 482, 479, 503, 510, 502, 535, 540, 514, 506, 519, 488, 514, 552, 501, 489, 527, 509, 542, 512, 498, 529, 532, 533, 486, 531, 530, 502, 491, 516, 527, 514, 535, 501, 536, 468, 489, 511, 505, 525, 509, 504, 529, 495, 505, 532, 500, 496, 537, 515, 513, 544, 570, 531, 549, 511, 523, 492, 520, 495, 493, 497, 506, 488, 500, 523, 530, 534, 498, 470, 513, 475, 516, 525, 437, 526, 537, 513, 544, 489, 506, 474, 533, 479, 530, 530, 490, 510, 476, 557, 542, 508, 530, 508, 486, 489, 492, 522, 513, 498, 517, 481, 492, 511, 489, 528, 545, 534, 540, 518, 523, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[526, 523, 468, 473, 527, 537, 511, 518, 491, 505, 513, 545, 516, 516, 515, 485, 521, 519, 583, 494, 511, 539, 506, 492, 506, 527, 519, 496, 527, 511, 491, 505, 536, 510, 513, 535, 507, 500, 467, 499, 469, 484, 498, 525, 522, 493, 530, 509, 518, 518, 512, 504, 531, 499, 545, 513, 511, 493, 530, 494, 519, 506, 498, 485, 533, 547, 542, 518, 537, 473, 469, 514, 475, 489, 488, 530, 511, 525, 467, 541, 505, 542, 514, 526, 582, 520, 523, 497, 519, 538, 516, 525, 530, 499, 550, 526, 502, 484, 456, 490, 532, 523, 533, 523, 525, 510, 502, 507, 521, 521, 505, 505, 520, 517, 533, 534, 485, 519, 503, 510, 508, 469, 526, 460, 491, 495, 543, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[488, 521, 572, 521, 504, 538, 485, 537, 512, 533, 474, 501, 539, 529, 488, 517, 543, 541, 522, 511, 491, 476, 486, 505, 516, 492, 527, 546, 521, 521, 473, 498, 510, 520, 516, 532, 499, 503, 493, 531, 493, 512, 496, 503, 564, 491, 510, 557, 524, 512, 532, 520, 504, 528, 555, 542, 521, 548, 518, 514, 514, 516, 518, 509, 477, 491, 514, 518, 473, 520, 521, 520, 501, 504, 496, 514, 540, 526, 472, 502, 523, 513, 521, 466, 517, 504, 506, 521, 530, 499, 518, 484, 531, 521, 492, 504, 502, 514, 508, 506, 508, 519, 457, 489, 479, 514, 499, 501, 506, 506, 498, 570, 534, 502, 527, 507, 513, 517, 452, 503, 521, 490, 513, 539, 517, 535, 509, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[514, 499, 482, 507, 511, 510, 495, 527, 516, 502, 482, 514, 476, 492, 521, 501, 565, 509, 479, 512, 473, 517, 502, 475, 553, 490, 530, 515, 531, 507, 540, 518, 490, 489, 505, 515, 521, 529, 505, 486, 476, 504, 523, 517, 488, 541, 559, 502, 526, 527, 564, 469, 504, 512, 545, 500, 510, 549, 505, 526, 478, 527, 521, 515, 480, 550, 511, 494, 476, 517, 493, 489, 484, 522, 509, 533, 537, 545, 519, 505, 492, 530, 551, 515, 507, 491, 501, 506, 531, 528, 542, 537, 522, 471, 516, 544, 471, 524, 482, 505, 493, 546, 552, 511, 543, 507, 514, 512, 510, 513, 495, 525, 536, 513, 498, 517, 494, 523, 520, 535, 499, 497, 485, 528, 543, 501, 521, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[500, 515, 510, 553, 467, 514, 501, 509, 504, 534, 462, 507, 530, 521, 508, 522, 532, 514, 514, 500, 524, 511, 512, 486, 536, 482, 525, 503, 511, 525, 556, 518, 487, 487, 525, 519, 522, 522, 532, 523, 516, 518, 528, 504, 559, 509, 473, 528, 528, 521, 551, 494, 530, 532, 521, 515, 537, 513, 509, 486, 535, 516, 504, 511, 531, 518, 511, 526, 465, 512, 513, 489, 508, 484, 475, 534, 511, 503, 527, 519, 543, 507, 528, 504, 481, 525, 519, 513, 521, 545, 486, 504, 533, 482, 495, 479, 532, 493, 477, 500, 518, 510, 502, 489, 524, 475, 507, 526, 506, 540, 516, 485, 519, 483, 560, 501, 543, 486, 489, 490, 536, 513, 492, 525, 497, 487, 529, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[476, 511, 511, 520, 490, 528, 495, 543, 499, 497, 524, 516, 507, 507, 555, 493, 485, 515, 535, 528, 529, 518, 520, 484, 476, 522, 519, 481, 542, 494, 531, 501, 539, 520, 498, 530, 548, 533, 514, 526, 528, 499, 501, 471, 482, 530, 473, 506, 519, 530, 494, 513, 510, 544, 526, 501, 501, 517, 522, 501, 555, 499, 510, 525, 547, 534, 566, 532, 525, 501, 520, 496, 560, 497, 495, 507, 575, 527, 526, 522, 506, 501, 510, 515, 496, 533, 499, 495, 472, 508, 547, 510, 471, 508, 530, 520, 496, 525, 488, 524, 501, 479, 483, 494, 529, 486, 492, 516, 524, 544, 477, 507, 526, 465, 527, 514, 512, 532, 524, 464, 555, 511, 502, 469, 507, 505, 480, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 505, 489, 490, 513, 477, 538, 508, 530, 517, 547, 485, 525, 505, 484, 544, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 543, 547, 494, 491, 489, 483, 461, 509, 536, 508, 523, 526, 523, 475, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 501, 501, 517, 512, 522, 502, 508, 524, 552, 514, 457, 528, 527, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 535, 449, 490, 523, 516, 472, 504, 506, 517, 530, 491, 513, 441, 501, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 534, 553, 534, 523, 487, 551, 509, 514, 516, 507, 503, 495, 491, 556, 473, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 541, 505, 478, 483, 495, 512, 533, 554, 500, 536, 534, 523, 545, 536, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 522, 504, 492, 509, 493, 509, 466, 534, 531, 520, 513, 528, 502, 565, 47, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 518, 534, 526, 507, 517, 507, 505, 488, 515, 492, 531, 536, 485, 511, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8153, 8178, 7969, 8289, 8256, 8250, 8229] avg=8192.0000 max=8289 max_rank=4 min=7969 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=33024 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[488, 521, 572, 521, 504, 538, 485, 537, 512, 533, 474, 501, 539, 529, 488, 517, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 541, 522, 511, 491, 476, 486, 505, 516, 492, 527, 546, 521, 521, 473, 498, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 516, 532, 499, 503, 493, 531, 493, 512, 496, 503, 564, 491, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 532, 520, 504, 528, 504, 542, 521, 548, 518, 514, 514, 516, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 491, 514, 518, 473, 520, 521, 520, 501, 504, 496, 514, 540, 526, 472, 502, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 521, 466, 517, 504, 506, 521, 530, 499, 518, 484, 502, 521, 492, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 508, 506, 508, 519, 457, 489, 479, 514, 499, 501, 506, 506, 498, 570, 51, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 502, 527, 507, 513, 517, 452, 503, 521, 490, 513, 539, 517, 535, 509, 509, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8184, 8191, 8324, 8091, 8121, 8137, 8202] avg=8192.0000 max=8324 max_rank=3 min=8091 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[487, 515, 488, 537, 506, 505, 527, 544, 508, 492, 515, 510, 506, 517, 486, 491, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 514, 544, 498, 487, 536, 543, 556, 489, 487, 500, 474, 470, 485, 511, 498, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 547, 542, 523, 554, 526, 519, 522, 497, 509, 516, 486, 522, 521, 544, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 475, 557, 510, 530, 525, 481, 514, 490, 506, 563, 521, 523, 508, 536, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 488, 492, 502, 497, 502, 525, 508, 492, 497, 473, 461, 523, 497, 478, 501, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 490, 509, 540, 519, 539, 512, 529, 512, 528, 498, 499, 474, 500, 493, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 487, 490, 525, 537, 511, 539, 487, 513, 491, 515, 513, 527, 489, 504, 496, 53, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 545, 523, 507, 526, 505, 488, 538, 535, 514, 511, 518, 517, 517, 452, 504, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8098, 8372, 8321, 7983, 8187, 8206, 8206] avg=8192.0000 max=8372 max_rank=2 min=7983 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 523, 468, 473, 527, 537, 511, 518, 491, 505, 513, 545, 516, 516, 515, 485, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 583, 494, 511, 539, 506, 492, 506, 527, 519, 496, 527, 511, 491, 505, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 510, 513, 535, 507, 500, 467, 499, 469, 484, 498, 525, 522, 493, 530, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 512, 504, 531, 499, 495, 513, 511, 493, 530, 494, 519, 506, 498, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 547, 542, 518, 537, 473, 469, 514, 475, 489, 488, 530, 511, 525, 467, 541, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 542, 514, 526, 582, 520, 523, 497, 519, 538, 516, 525, 501, 499, 550, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 484, 456, 490, 532, 523, 533, 523, 525, 510, 502, 507, 521, 521, 505, 505, 50, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 533, 534, 485, 519, 503, 510, 508, 469, 526, 460, 491, 495, 543, 524, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8262, 8061, 8126, 8161, 8383, 8198, 8151] avg=8192.0000 max=8383 max_rank=5 min=8061 min_rank=2 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[476, 511, 511, 520, 490, 528, 495, 543, 499, 497, 524, 516, 507, 507, 555, 493, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 515, 535, 528, 529, 518, 520, 484, 476, 522, 519, 481, 542, 494, 531, 501, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 520, 498, 530, 548, 533, 514, 526, 528, 499, 501, 471, 482, 530, 473, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 494, 513, 510, 544, 478, 501, 501, 517, 522, 501, 555, 499, 510, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 534, 566, 532, 525, 501, 520, 496, 560, 497, 495, 507, 575, 527, 526, 522, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 501, 510, 515, 496, 533, 499, 495, 472, 508, 547, 510, 446, 508, 530, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 525, 488, 524, 501, 479, 483, 494, 529, 486, 492, 516, 524, 544, 477, 507, 48, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 465, 527, 514, 512, 532, 524, 464, 555, 511, 502, 469, 507, 505, 480, 510, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8193, 8162, 8219, 8432, 8096, 8122, 8115] avg=8192.0000 max=8432 max_rank=4 min=8096 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 499, 482, 507, 511, 510, 495, 527, 516, 502, 482, 514, 476, 492, 521, 501, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 509, 479, 512, 473, 517, 502, 475, 553, 490, 530, 515, 531, 507, 540, 518, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 489, 505, 515, 521, 529, 505, 486, 476, 504, 523, 517, 488, 541, 559, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 564, 469, 504, 512, 495, 500, 510, 549, 505, 526, 478, 527, 521, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 550, 511, 494, 476, 517, 493, 489, 484, 522, 509, 533, 537, 545, 519, 505, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 530, 551, 515, 507, 491, 501, 506, 531, 528, 542, 537, 494, 471, 516, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 524, 482, 505, 493, 546, 552, 511, 543, 507, 514, 512, 510, 513, 495, 525, 50, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 513, 498, 517, 494, 523, 520, 535, 499, 497, 485, 528, 543, 501, 521, 482, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8230, 8115, 8228, 8166, 8256, 8262, 8206] avg=8192.0000 max=8262 max_rank=6 min=8073 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 515, 510, 553, 467, 514, 501, 509, 504, 534, 462, 507, 530, 521, 508, 522, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 514, 514, 500, 524, 511, 512, 486, 536, 482, 525, 503, 511, 525, 556, 518, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 487, 525, 519, 522, 522, 532, 523, 516, 518, 528, 504, 559, 509, 473, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 551, 494, 530, 532, 473, 515, 537, 513, 509, 486, 535, 516, 504, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 518, 511, 526, 465, 512, 513, 489, 508, 484, 475, 534, 511, 503, 527, 519, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 507, 528, 504, 481, 525, 519, 513, 521, 545, 486, 504, 504, 482, 495, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 493, 477, 500, 518, 510, 502, 489, 524, 475, 507, 526, 506, 540, 516, 485, 48, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 483, 560, 501, 543, 486, 489, 490, 536, 513, 492, 525, 497, 487, 529, 534, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8264, 8214, 8255, 8128, 8136, 8158, 8198] avg=8192.0000 max=8264 max_rank=1 min=8128 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[519, 516, 501, 518, 570, 526, 502, 522, 503, 495, 524, 496, 527, 519, 495, 470, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 482, 479, 503, 510, 502, 535, 540, 514, 506, 519, 488, 514, 552, 501, 489, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 509, 542, 512, 498, 529, 532, 533, 486, 531, 530, 502, 491, 516, 527, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 501, 536, 468, 489, 511, 459, 525, 509, 504, 529, 495, 505, 532, 500, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 515, 513, 544, 570, 531, 549, 511, 523, 492, 520, 495, 493, 497, 506, 488, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 530, 534, 498, 470, 513, 475, 516, 525, 437, 526, 508, 513, 544, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 474, 533, 479, 530, 530, 490, 510, 476, 557, 542, 508, 530, 508, 486, 489, 46, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 513, 498, 517, 481, 492, 511, 489, 528, 545, 534, 540, 518, 523, 512, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8152, 8243, 8094, 8286, 8101, 8203, 8229] avg=8192.0000 max=8286 max_rank=4 min=8094 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 3968, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[513, 511, 501, 514, 509, 502, 502, 531, 494, 507, 503, 494, 488, 533, 493, 472, 507, 539, 496, 529, 503, 522, 514, 500, 505, 492, 516, 497, 496, 512, 507, 489, 546, 519, 492, 495, 509, 493, 536, 516, 500, 514, 522, 557, 489, 518, 512, 515, 483, 509, 510, 559, 507, 535, 542, 522, 530, 511, 525, 477, 510, 502, 537, 505, 482, 521, 499, 527, 478, 520, 493, 515, 518, 511, 528, 486, 525, 545, 551, 488, 541, 511, 474, 476, 530, 527, 504, 478, 515, 523, 519, 491, 519, 566, 492, 500, 538, 522, 494, 479, 490, 486, 531, 524, 556, 530, 543, 494, 525, 513, 543, 512, 477, 492, 504, 501, 510, 554, 548, 516, 505, 528, 489, 520, 523, 553, 524, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[533, 501, 522, 543, 531, 530, 541, 527, 515, 501, 497, 495, 525, 517, 532, 529, 553, 526, 514, 477, 504, 545, 538, 524, 511, 538, 522, 509, 537, 489, 484, 539, 504, 494, 508, 545, 556, 529, 513, 516, 512, 505, 512, 504, 561, 505, 556, 474, 526, 488, 527, 515, 518, 508, 472, 538, 520, 499, 532, 480, 480, 503, 519, 480, 503, 510, 525, 494, 475, 501, 522, 491, 509, 503, 519, 499, 482, 503, 508, 495, 490, 480, 543, 522, 526, 508, 503, 498, 542, 507, 522, 537, 496, 509, 520, 534, 505, 527, 524, 516, 527, 542, 475, 520, 494, 505, 509, 523, 490, 485, 517, 503, 506, 529, 498, 506, 493, 508, 488, 503, 479, 514, 525, 492, 513, 512, 527, 457] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[514, 472, 461, 501, 536, 562, 515, 537, 559, 509, 494, 492, 522, 500, 504, 518, 511, 547, 498, 493, 519, 526, 520, 507, 504, 504, 503, 543, 505, 503, 492, 501, 504, 500, 509, 492, 522, 511, 483, 495, 516, 530, 516, 506, 510, 493, 510, 501, 524, 497, 492, 554, 524, 506, 512, 453, 487, 483, 515, 517, 507, 518, 510, 526, 499, 514, 522, 535, 531, 490, 509, 505, 503, 483, 537, 543, 522, 520, 519, 493, 494, 548, 517, 509, 517, 545, 496, 473, 470, 517, 502, 498, 522, 539, 497, 538, 527, 509, 528, 509, 510, 496, 491, 546, 522, 534, 521, 518, 508, 526, 485, 500, 517, 530, 532, 510, 565, 492, 548, 488, 490, 534, 495, 502, 520, 511, 543, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[495, 518, 498, 557, 547, 486, 541, 552, 495, 506, 526, 562, 504, 495, 496, 507, 524, 536, 547, 511, 496, 519, 512, 486, 508, 516, 464, 511, 511, 528, 488, 500, 531, 494, 528, 512, 527, 517, 462, 518, 479, 532, 516, 532, 538, 502, 504, 532, 498, 472, 479, 472, 535, 516, 512, 503, 525, 485, 520, 527, 525, 518, 458, 487, 485, 516, 541, 505, 487, 502, 552, 497, 473, 547, 539, 560, 498, 504, 504, 509, 520, 505, 523, 539, 512, 532, 490, 515, 525, 515, 500, 524, 498, 471, 507, 564, 502, 505, 505, 511, 529, 559, 503, 522, 507, 503, 546, 534, 511, 498, 502, 508, 531, 516, 482, 529, 514, 515, 535, 508, 492, 496, 491, 472, 537, 512, 517, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[496, 509, 510, 524, 500, 513, 543, 528, 500, 505, 499, 484, 493, 503, 495, 503, 520, 472, 553, 476, 523, 526, 512, 519, 551, 483, 490, 515, 547, 541, 498, 512, 559, 488, 475, 513, 473, 510, 508, 505, 471, 532, 507, 480, 533, 530, 524, 515, 489, 508, 505, 502, 568, 506, 543, 582, 537, 469, 539, 518, 512, 560, 518, 496, 517, 501, 523, 519, 497, 506, 528, 447, 483, 523, 539, 544, 524, 518, 511, 474, 549, 530, 535, 472, 546, 553, 508, 490, 509, 494, 498, 511, 426, 530, 514, 551, 513, 500, 497, 538, 500, 497, 504, 513, 525, 510, 516, 466, 515, 503, 520, 526, 524, 505, 494, 509, 520, 563, 518, 462, 540, 480, 524, 527, 528, 462, 501, 548] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[524, 490, 494, 522, 549, 509, 525, 512, 507, 524, 527, 541, 481, 508, 567, 540, 542, 499, 524, 509, 495, 525, 507, 532, 501, 530, 491, 492, 542, 528, 500, 499, 514, 492, 557, 491, 520, 502, 496, 486, 532, 496, 521, 462, 506, 511, 533, 510, 504, 539, 526, 495, 489, 521, 527, 507, 526, 489, 490, 521, 527, 522, 518, 497, 491, 457, 517, 506, 528, 483, 530, 541, 522, 528, 489, 465, 550, 543, 525, 500, 526, 530, 527, 530, 506, 511, 466, 470, 531, 519, 454, 471, 507, 505, 518, 535, 525, 534, 532, 543, 512, 501, 492, 477, 506, 544, 532, 525, 505, 547, 512, 498, 511, 510, 497, 511, 493, 511, 532, 514, 493, 497, 547, 506, 499, 515, 473, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[546, 487, 498, 508, 470, 485, 472, 524, 528, 498, 533, 505, 529, 514, 491, 515, 543, 481, 501, 476, 481, 487, 502, 570, 519, 511, 559, 477, 496, 491, 505, 483, 535, 512, 517, 528, 507, 506, 499, 556, 491, 487, 524, 496, 508, 544, 509, 530, 530, 509, 518, 501, 521, 540, 492, 471, 558, 533, 510, 493, 512, 519, 495, 533, 504, 559, 505, 497, 516, 528, 461, 504, 496, 460, 502, 488, 494, 548, 508, 522, 485, 535, 525, 512, 537, 539, 527, 526, 499, 515, 530, 503, 534, 524, 482, 519, 483, 516, 543, 484, 495, 501, 495, 547, 533, 492, 528, 509, 552, 547, 513, 518, 485, 538, 483, 526, 528, 470, 498, 542, 480, 498, 524, 551, 530, 511, 539, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[530, 546, 464, 484, 550, 460, 509, 482, 535, 530, 504, 490, 513, 508, 464, 474, 513, 524, 550, 486, 491, 490, 525, 474, 532, 503, 533, 555, 539, 502, 499, 512, 536, 532, 534, 530, 494, 491, 519, 527, 518, 537, 522, 495, 510, 474, 505, 493, 508, 470, 530, 504, 542, 453, 543, 493, 519, 510, 469, 522, 542, 544, 523, 487, 464, 501, 538, 442, 527, 495, 519, 549, 477, 519, 493, 533, 499, 529, 537, 520, 509, 513, 514, 524, 507, 511, 512, 509, 517, 513, 521, 464, 551, 548, 536, 518, 480, 524, 511, 518, 455, 500, 524, 470, 479, 541, 506, 526, 511, 497, 565, 514, 531, 521, 533, 488, 543, 512, 538, 520, 506, 503, 515, 536, 490, 536, 542, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[533, 501, 522, 543, 531, 530, 541, 527, 515, 501, 497, 495, 525, 517, 532, 529, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 526, 514, 477, 504, 545, 538, 524, 511, 538, 522, 509, 537, 489, 484, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 494, 508, 545, 556, 529, 513, 516, 512, 505, 512, 504, 561, 505, 556, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 488, 527, 515, 518, 508, 472, 538, 520, 499, 532, 480, 480, 503, 519, 480, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 525, 494, 475, 501, 522, 491, 509, 503, 519, 499, 482, 503, 508, 495, 22, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 480, 543, 522, 526, 508, 503, 498, 542, 507, 522, 537, 496, 509, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 524, 516, 527, 542, 475, 520, 494, 505, 487, 523, 490, 485, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 498, 506, 493, 508, 482, 503, 479, 514, 525, 492, 513, 512, 527, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8347, 8310, 8291, 8112, 8075, 8217, 8140, 8044] avg=8192.0000 max=8347 max_rank=0 min=8044 min_rank=7 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[495, 518, 498, 557, 547, 486, 541, 552, 495, 506, 526, 562, 504, 495, 496, 507, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 536, 547, 511, 496, 519, 512, 486, 508, 516, 464, 511, 511, 528, 488, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 494, 528, 512, 527, 517, 462, 518, 479, 532, 516, 532, 538, 502, 504, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 472, 479, 472, 535, 516, 512, 503, 525, 485, 520, 527, 525, 518, 458, 487, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 516, 541, 505, 487, 502, 552, 497, 473, 547, 539, 560, 498, 504, 504, 509, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 523, 539, 512, 532, 490, 515, 525, 515, 500, 524, 498, 471, 507, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 505, 505, 511, 529, 559, 503, 522, 507, 503, 522, 534, 511, 498, 502, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 516, 482, 529, 514, 515, 527, 508, 492, 496, 491, 472, 537, 512, 517, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8157, 8221, 8040, 8258, 8218, 8221, 8126] avg=8192.0000 max=8295 max_rank=0 min=8040 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 511, 501, 514, 509, 502, 502, 531, 494, 507, 503, 494, 488, 533, 493, 472, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 539, 496, 529, 503, 522, 514, 500, 505, 492, 516, 497, 496, 512, 507, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 519, 492, 495, 509, 493, 536, 516, 500, 514, 522, 557, 489, 518, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 509, 510, 559, 507, 535, 542, 522, 530, 511, 525, 477, 510, 502, 537, 505, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 521, 499, 527, 478, 520, 493, 515, 518, 511, 528, 486, 525, 545, 551, 488, 24, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 511, 474, 476, 530, 527, 504, 478, 515, 523, 519, 491, 519, 566, 492, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 522, 494, 479, 490, 486, 531, 524, 556, 530, 519, 494, 525, 513, 543, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 492, 504, 501, 510, 554, 540, 516, 505, 528, 489, 520, 523, 553, 524, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8076, 8124, 8230, 8272, 8224, 8147, 8256, 8207] avg=8192.0000 max=8272 max_rank=3 min=8076 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 472, 461, 501, 536, 562, 515, 537, 559, 509, 494, 492, 522, 500, 504, 518, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 547, 498, 493, 519, 526, 520, 507, 504, 504, 503, 543, 505, 503, 492, 501, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 500, 509, 492, 522, 511, 483, 495, 516, 530, 516, 506, 510, 493, 510, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 497, 492, 554, 524, 506, 512, 453, 487, 483, 515, 517, 507, 518, 510, 526, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 514, 522, 535, 531, 490, 509, 505, 503, 483, 537, 543, 522, 520, 519, 493, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 548, 517, 509, 517, 545, 496, 473, 470, 517, 502, 498, 522, 539, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 509, 528, 509, 510, 496, 491, 546, 522, 534, 495, 518, 508, 526, 485, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 530, 532, 510, 565, 492, 534, 488, 490, 534, 495, 502, 520, 511, 543, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8181, 8088, 8138, 8269, 8154, 8204, 8290] avg=8192.0000 max=8290 max_rank=7 min=8088 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[546, 487, 498, 508, 470, 485, 472, 524, 528, 498, 533, 505, 529, 514, 491, 515, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 481, 501, 476, 481, 487, 502, 570, 519, 511, 559, 477, 496, 491, 505, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 517, 528, 507, 506, 499, 556, 491, 487, 524, 496, 508, 544, 509, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 509, 518, 501, 521, 540, 492, 471, 558, 533, 510, 493, 512, 519, 495, 533, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 559, 505, 497, 516, 528, 461, 504, 496, 460, 502, 488, 494, 548, 508, 522, 23, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 535, 525, 512, 537, 539, 527, 526, 499, 515, 530, 503, 534, 524, 482, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 516, 543, 484, 495, 501, 495, 547, 533, 492, 505, 509, 552, 547, 513, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 538, 483, 526, 528, 470, 492, 542, 480, 498, 524, 551, 530, 511, 539, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8082, 8246, 8242, 8129, 8272, 8233, 8221] avg=8192.0000 max=8272 max_rank=5 min=8082 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[524, 490, 494, 522, 549, 509, 525, 512, 507, 524, 527, 541, 481, 508, 567, 540, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 499, 524, 509, 495, 525, 507, 532, 501, 530, 491, 492, 542, 528, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 557, 491, 520, 502, 496, 486, 532, 496, 521, 462, 506, 511, 533, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 539, 526, 495, 489, 521, 527, 507, 526, 489, 490, 521, 527, 522, 518, 497, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 457, 517, 506, 528, 483, 530, 541, 522, 528, 489, 465, 550, 543, 525, 500, 23, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 530, 527, 530, 506, 511, 466, 470, 531, 519, 454, 471, 507, 505, 518, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 534, 532, 543, 512, 501, 492, 477, 506, 544, 509, 525, 505, 547, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 497, 511, 493, 511, 524, 514, 493, 497, 547, 506, 499, 515, 473, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8329, 8216, 8126, 8206, 8212, 8086, 8262, 8099] avg=8192.0000 max=8329 max_rank=0 min=8086 min_rank=5 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[496, 509, 510, 524, 500, 513, 543, 528, 500, 505, 499, 484, 493, 503, 495, 503, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 472, 553, 476, 523, 526, 512, 519, 551, 483, 490, 515, 547, 541, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 488, 475, 513, 473, 510, 508, 505, 471, 532, 507, 480, 533, 530, 524, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 508, 505, 502, 568, 506, 543, 582, 537, 469, 539, 518, 512, 560, 518, 496, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 501, 523, 519, 497, 506, 528, 447, 483, 523, 539, 544, 524, 518, 511, 474, 22, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 530, 535, 472, 546, 553, 508, 490, 509, 494, 498, 511, 426, 530, 514, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 497, 538, 500, 497, 504, 513, 525, 510, 494, 466, 515, 503, 520, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 505, 494, 509, 520, 563, 511, 462, 540, 480, 524, 527, 528, 462, 501, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8238, 8120, 8359, 8191, 8195, 8121, 8198] avg=8192.0000 max=8359 max_rank=3 min=8114 min_rank=0 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 546, 464, 484, 550, 460, 509, 482, 535, 530, 504, 490, 513, 508, 464, 474, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 550, 486, 491, 490, 525, 474, 532, 503, 533, 555, 539, 502, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 532, 534, 530, 494, 491, 519, 527, 518, 537, 522, 495, 510, 474, 505, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 470, 530, 504, 542, 453, 543, 493, 519, 510, 469, 522, 542, 544, 523, 487, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 501, 538, 442, 527, 495, 519, 549, 477, 519, 493, 533, 499, 529, 537, 520, 22, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 514, 524, 507, 511, 512, 509, 517, 513, 521, 464, 551, 548, 536, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 524, 511, 518, 455, 500, 524, 470, 479, 541, 484, 526, 511, 497, 565, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 521, 533, 488, 543, 512, 530, 520, 506, 503, 515, 536, 490, 536, 542, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8052, 8228, 8214, 8167, 8178, 8247, 8099, 8351] avg=8192.0000 max=8351 max_rank=7 min=8052 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4352, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[506, 497, 474, 532, 513, 564, 511, 491, 511, 499, 522, 502, 482, 509, 507, 500, 524, 547, 523, 512, 532, 531, 484, 493, 493, 528, 544, 551, 541, 477, 523, 504, 521, 513, 505, 503, 524, 503, 482, 542, 576, 500, 515, 495, 512, 505, 535, 503, 545, 557, 493, 536, 525, 504, 521, 493, 492, 497, 516, 486, 553, 548, 502, 500, 523, 513, 464, 510, 479, 505, 511, 494, 534, 516, 496, 511, 515, 502, 499, 535, 477, 520, 503, 481, 504, 528, 539, 515, 505, 522, 538, 513, 538, 547, 524, 527, 518, 472, 550, 499, 485, 519, 503, 490, 539, 522, 500, 488, 524, 512, 545, 501, 495, 506, 485, 502, 546, 505, 508, 492, 538, 489, 482, 529, 492, 503, 475, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[490, 488, 510, 513, 528, 502, 525, 506, 503, 522, 530, 487, 507, 481, 528, 526, 515, 522, 512, 558, 541, 548, 477, 492, 506, 506, 525, 493, 515, 536, 501, 461, 513, 489, 505, 575, 513, 526, 544, 533, 501, 504, 466, 558, 529, 510, 541, 507, 546, 516, 501, 502, 504, 519, 518, 516, 481, 485, 517, 512, 476, 487, 510, 527, 501, 539, 564, 501, 508, 525, 535, 483, 523, 561, 484, 521, 504, 536, 506, 522, 501, 472, 512, 512, 493, 516, 515, 506, 524, 517, 523, 459, 516, 513, 498, 499, 521, 457, 492, 543, 526, 508, 486, 519, 527, 517, 522, 528, 498, 532, 508, 516, 519, 489, 508, 473, 491, 498, 480, 513, 548, 531, 510, 514, 554, 522, 526, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[530, 500, 509, 499, 532, 522, 522, 532, 501, 553, 528, 533, 498, 508, 513, 496, 477, 533, 515, 522, 496, 536, 508, 503, 503, 524, 520, 480, 528, 512, 467, 514, 534, 501, 501, 523, 496, 502, 521, 521, 526, 536, 523, 541, 519, 521, 543, 519, 475, 532, 540, 492, 465, 485, 471, 525, 503, 497, 527, 461, 547, 503, 515, 524, 517, 444, 491, 566, 516, 530, 496, 535, 547, 495, 510, 468, 532, 524, 523, 482, 500, 533, 473, 493, 564, 517, 460, 519, 508, 531, 517, 498, 492, 506, 513, 509, 515, 509, 494, 476, 489, 523, 504, 518, 536, 513, 505, 524, 524, 511, 487, 491, 545, 521, 520, 521, 515, 483, 533, 551, 514, 505, 514, 491, 538, 539, 518, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[483, 523, 469, 537, 555, 513, 492, 554, 527, 519, 496, 514, 517, 487, 527, 503, 499, 531, 478, 465, 511, 497, 513, 484, 498, 521, 523, 530, 487, 517, 524, 527, 477, 506, 524, 475, 525, 520, 530, 507, 508, 543, 536, 512, 532, 521, 521, 499, 533, 465, 511, 549, 499, 539, 518, 544, 530, 539, 468, 512, 506, 500, 480, 547, 558, 509, 471, 518, 492, 520, 525, 494, 546, 523, 501, 518, 513, 526, 505, 554, 495, 530, 526, 477, 534, 493, 526, 531, 505, 513, 516, 479, 499, 505, 484, 508, 516, 505, 501, 501, 480, 486, 527, 504, 507, 539, 557, 558, 483, 515, 491, 540, 534, 526, 495, 532, 542, 500, 521, 507, 493, 500, 486, 493, 476, 538, 485, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[497, 499, 502, 478, 554, 505, 512, 547, 499, 538, 505, 515, 510, 508, 517, 530, 534, 508, 558, 499, 527, 527, 499, 540, 529, 473, 472, 543, 534, 471, 518, 553, 509, 508, 517, 511, 510, 488, 514, 509, 548, 485, 524, 498, 517, 503, 484, 486, 523, 521, 559, 527, 557, 504, 531, 551, 494, 506, 534, 474, 531, 464, 547, 545, 532, 500, 506, 480, 526, 481, 530, 525, 518, 523, 555, 520, 512, 513, 507, 533, 517, 495, 470, 484, 514, 512, 501, 516, 507, 525, 516, 529, 500, 534, 459, 523, 511, 489, 471, 495, 527, 494, 514, 491, 525, 475, 496, 513, 487, 472, 496, 537, 514, 536, 502, 517, 508, 495, 474, 495, 508, 544, 548, 508, 462, 541, 525, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[502, 503, 484, 496, 471, 480, 472, 517, 484, 540, 515, 526, 520, 522, 506, 451, 524, 482, 531, 520, 491, 478, 513, 505, 515, 504, 502, 506, 524, 519, 573, 533, 543, 488, 488, 533, 463, 492, 509, 517, 520, 514, 542, 482, 514, 534, 529, 503, 518, 521, 522, 482, 513, 520, 508, 505, 496, 480, 555, 510, 538, 521, 537, 477, 491, 512, 542, 553, 520, 476, 469, 553, 517, 507, 531, 497, 548, 537, 532, 539, 549, 503, 513, 536, 557, 498, 506, 509, 518, 565, 507, 512, 467, 548, 511, 521, 487, 510, 490, 510, 540, 508, 526, 500, 561, 489, 532, 479, 500, 511, 515, 505, 527, 466, 515, 497, 462, 508, 530, 501, 516, 513, 544, 451, 499, 532, 511, 574] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[488, 540, 461, 510, 533, 514, 504, 487, 497, 500, 472, 519, 507, 468, 562, 477, 500, 502, 515, 537, 545, 510, 523, 533, 559, 495, 493, 497, 530, 472, 495, 504, 513, 519, 498, 497, 528, 500, 485, 559, 512, 492, 509, 530, 488, 509, 545, 515, 475, 504, 511, 507, 522, 507, 556, 498, 534, 485, 528, 538, 539, 527, 526, 528, 482, 527, 506, 560, 498, 509, 516, 519, 498, 525, 545, 530, 496, 546, 515, 480, 546, 486, 484, 520, 494, 486, 529, 470, 511, 521, 464, 477, 529, 496, 507, 508, 526, 487, 511, 513, 526, 529, 514, 501, 503, 526, 542, 483, 562, 508, 477, 567, 576, 537, 508, 485, 548, 498, 512, 520, 500, 500, 525, 502, 466, 501, 528, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[545, 515, 499, 509, 477, 533, 509, 513, 522, 520, 518, 449, 496, 501, 544, 536, 484, 532, 529, 541, 563, 551, 495, 480, 485, 464, 568, 524, 503, 532, 543, 510, 514, 536, 509, 534, 536, 505, 504, 526, 529, 490, 506, 499, 489, 540, 524, 524, 515, 508, 521, 527, 542, 493, 556, 510, 505, 544, 550, 506, 539, 529, 530, 515, 532, 491, 504, 508, 504, 495, 494, 515, 500, 494, 505, 503, 482, 492, 513, 521, 507, 495, 542, 514, 505, 518, 518, 512, 516, 529, 529, 486, 504, 506, 510, 491, 521, 452, 535, 496, 507, 493, 505, 523, 501, 545, 513, 485, 524, 472, 510, 517, 514, 507, 500, 499, 505, 529, 459, 510, 493, 508, 517, 495, 488, 572, 473, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[490, 488, 510, 513, 528, 502, 525, 506, 503, 522, 530, 487, 507, 481, 528, 526, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 512, 558, 511, 548, 477, 492, 506, 506, 525, 493, 515, 536, 501, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 489, 505, 575, 513, 526, 544, 533, 501, 504, 466, 558, 529, 510, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 516, 501, 502, 504, 519, 518, 516, 481, 485, 517, 512, 432, 487, 510, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 539, 564, 475, 508, 525, 535, 483, 523, 561, 484, 521, 504, 536, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 472, 512, 512, 493, 516, 515, 506, 524, 517, 523, 459, 516, 513, 498, 499, 29, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 457, 492, 543, 526, 508, 486, 519, 527, 517, 522, 528, 498, 532, 508, 516, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 489, 508, 473, 491, 498, 480, 513, 548, 531, 510, 514, 554, 522, 526, 486, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8178, 8274, 8073, 8287, 8112, 8238, 8189] avg=8192.0000 max=8287 max_rank=4 min=8073 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[506, 497, 474, 532, 513, 564, 511, 491, 511, 499, 522, 502, 482, 509, 507, 500, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 547, 523, 512, 502, 531, 484, 493, 493, 528, 544, 551, 541, 477, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 513, 505, 503, 524, 503, 482, 542, 576, 500, 515, 495, 512, 505, 495, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 557, 493, 536, 525, 504, 521, 493, 492, 497, 516, 486, 502, 548, 502, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 464, 483, 479, 505, 511, 494, 534, 516, 496, 511, 515, 502, 499, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 520, 503, 481, 504, 528, 539, 515, 505, 522, 538, 513, 538, 547, 524, 527, 29, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 472, 550, 499, 485, 519, 503, 490, 539, 522, 500, 488, 524, 512, 545, 501, 33, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 485, 502, 546, 505, 508, 492, 538, 489, 482, 529, 492, 503, 475, 505, 27, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8277, 8194, 8217, 8080, 8317, 8206, 8080] avg=8192.0000 max=8317 max_rank=5 min=8080 min_rank=4 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[502, 503, 484, 496, 471, 480, 472, 517, 484, 540, 515, 526, 520, 522, 506, 451, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 482, 531, 520, 464, 478, 513, 505, 515, 504, 502, 506, 524, 519, 573, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 488, 488, 533, 463, 492, 509, 517, 520, 514, 542, 482, 514, 534, 490, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 522, 482, 513, 520, 508, 505, 496, 480, 555, 510, 488, 521, 537, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 542, 524, 520, 476, 469, 553, 517, 507, 531, 497, 548, 537, 532, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 503, 513, 536, 557, 498, 506, 509, 518, 565, 507, 512, 467, 548, 511, 521, 27, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 510, 490, 510, 540, 508, 526, 500, 561, 489, 532, 479, 500, 511, 515, 505, 32, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 466, 515, 497, 462, 508, 530, 501, 516, 513, 544, 451, 499, 532, 511, 574, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8033, 8193, 8132, 8153, 8295, 8354, 8201, 8175] avg=8192.0000 max=8354 max_rank=5 min=8033 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=33025 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[488, 540, 461, 510, 533, 514, 504, 487, 497, 500, 472, 519, 507, 468, 562, 477, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 515, 537, 508, 510, 523, 533, 559, 495, 493, 497, 530, 472, 495, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 498, 497, 528, 500, 485, 559, 512, 492, 509, 530, 488, 509, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 504, 511, 507, 522, 507, 556, 498, 534, 485, 528, 538, 483, 527, 526, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 527, 506, 527, 498, 509, 516, 519, 498, 525, 545, 530, 496, 546, 515, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 486, 484, 520, 494, 486, 529, 470, 511, 521, 464, 477, 529, 496, 507, 508, 33, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 487, 511, 513, 526, 529, 514, 501, 503, 526, 542, 483, 562, 508, 477, 567, 37, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 537, 508, 485, 548, 498, 512, 520, 500, 500, 525, 502, 466, 501, 528, 542, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8173, 8150, 8229, 8219, 8073, 8321, 8285] avg=8192.0000 max=8321 max_rank=6 min=8073 min_rank=5 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[530, 500, 509, 499, 532, 522, 522, 532, 501, 553, 528, 533, 498, 508, 513, 496, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 533, 515, 522, 469, 536, 508, 503, 503, 524, 520, 480, 528, 512, 467, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 501, 523, 496, 502, 521, 521, 526, 536, 523, 541, 519, 521, 503, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 532, 540, 492, 465, 485, 471, 525, 503, 497, 527, 461, 496, 503, 515, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 444, 491, 536, 516, 530, 496, 535, 547, 495, 510, 468, 532, 524, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 533, 473, 493, 564, 517, 460, 519, 508, 531, 517, 498, 492, 506, 513, 509, 27, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 494, 476, 489, 523, 504, 518, 536, 513, 505, 524, 524, 511, 487, 491, 33, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 521, 520, 521, 515, 483, 533, 551, 514, 505, 514, 491, 538, 539, 518, 497, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8321, 8111, 8287, 8011, 8146, 8167, 8158, 8335] avg=8192.0000 max=8335 max_rank=7 min=8011 min_rank=3 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[483, 523, 469, 537, 555, 513, 492, 554, 527, 519, 496, 514, 517, 487, 527, 503, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 531, 478, 465, 483, 497, 513, 484, 498, 521, 523, 530, 487, 517, 524, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 506, 524, 475, 525, 520, 530, 507, 508, 543, 536, 512, 532, 521, 482, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 465, 511, 549, 499, 539, 518, 544, 530, 539, 468, 512, 460, 500, 480, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 509, 471, 491, 492, 520, 525, 494, 546, 523, 501, 518, 513, 526, 505, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 530, 526, 477, 534, 493, 526, 531, 505, 513, 516, 479, 499, 505, 484, 508, 28, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 505, 501, 501, 480, 486, 527, 504, 507, 539, 557, 558, 483, 515, 491, 540, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 526, 495, 532, 542, 500, 521, 507, 493, 500, 486, 493, 476, 538, 485, 507, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8077, 8197, 8194, 8246, 8156, 8247, 8162] avg=8192.0000 max=8257 max_rank=0 min=8077 min_rank=1 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[497, 499, 502, 478, 554, 505, 512, 547, 499, 538, 505, 515, 510, 508, 517, 530, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 558, 499, 497, 527, 499, 540, 529, 473, 472, 543, 534, 471, 518, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 517, 511, 510, 488, 514, 509, 548, 485, 524, 498, 517, 503, 449, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 521, 559, 527, 557, 504, 531, 551, 494, 506, 534, 474, 482, 464, 547, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 500, 506, 455, 526, 481, 530, 525, 518, 523, 555, 520, 512, 513, 507, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 470, 484, 514, 512, 501, 516, 507, 525, 516, 529, 500, 534, 459, 523, 29, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 489, 471, 495, 527, 494, 514, 491, 525, 475, 496, 513, 487, 472, 496, 537, 29, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 536, 502, 517, 508, 495, 474, 495, 508, 544, 548, 508, 462, 541, 525, 523, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8255, 8076, 8319, 8236, 8137, 8028, 8226] avg=8192.0000 max=8319 max_rank=3 min=8028 min_rank=6 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[545, 515, 499, 509, 477, 533, 509, 513, 522, 520, 518, 449, 496, 501, 544, 536, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 532, 529, 541, 531, 551, 495, 480, 485, 464, 568, 524, 503, 532, 543, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 536, 509, 534, 536, 505, 504, 526, 529, 490, 506, 499, 489, 540, 485, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 521, 527, 542, 493, 556, 510, 505, 544, 550, 506, 489, 529, 530, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 491, 504, 482, 504, 495, 494, 515, 500, 494, 505, 503, 482, 492, 513, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 542, 514, 505, 518, 518, 512, 516, 529, 529, 486, 504, 506, 510, 491, 31, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 452, 535, 496, 507, 493, 505, 523, 501, 545, 513, 485, 524, 472, 510, 517, 32, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 507, 500, 499, 505, 529, 459, 510, 493, 508, 517, 495, 488, 572, 473, 488, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8272, 8226, 8340, 8027, 8220, 8137, 8084] avg=8192.0000 max=8340 max_rank=3 min=8027 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 3968, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3968, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4352, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 3840, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[518, 547, 509, 560, 492, 534, 511, 514, 479, 522, 473, 497, 528, 498, 542, 523, 550, 502, 523, 487, 516, 510, 467, 553, 479, 471, 528, 486, 534, 488, 544, 467, 501, 498, 530, 516, 509, 482, 520, 532, 492, 519, 508, 503, 527, 521, 545, 506, 549, 523, 483, 529, 505, 534, 514, 507, 513, 521, 547, 494, 537, 516, 520, 502, 504, 482, 543, 470, 510, 499, 553, 480, 544, 538, 514, 514, 514, 476, 518, 497, 517, 495, 499, 504, 520, 485, 507, 521, 532, 507, 527, 533, 523, 525, 497, 503, 511, 504, 526, 512, 492, 528, 507, 531, 503, 475, 535, 472, 522, 515, 499, 541, 493, 535, 514, 497, 501, 497, 540, 524, 517, 512, 491, 499, 498, 516, 520, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[544, 527, 527, 488, 522, 536, 556, 504, 531, 489, 514, 507, 508, 527, 485, 528, 492, 480, 523, 528, 525, 532, 508, 527, 544, 505, 532, 549, 510, 526, 509, 530, 498, 522, 474, 507, 507, 537, 537, 550, 508, 511, 507, 504, 514, 497, 538, 461, 502, 524, 488, 478, 488, 523, 483, 517, 547, 545, 504, 503, 485, 503, 496, 511, 513, 516, 465, 488, 532, 497, 484, 454, 529, 512, 512, 498, 495, 525, 507, 489, 492, 523, 481, 519, 498, 509, 474, 516, 518, 513, 512, 530, 533, 508, 557, 456, 507, 509, 509, 513, 545, 530, 500, 516, 463, 516, 519, 544, 513, 544, 567, 520, 506, 523, 493, 520, 517, 542, 511, 504, 527, 487, 520, 493, 489, 497, 521, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[525, 509, 526, 475, 491, 546, 520, 525, 506, 473, 511, 515, 523, 518, 495, 540, 521, 522, 492, 521, 522, 491, 510, 531, 498, 557, 517, 512, 507, 535, 535, 537, 504, 516, 536, 546, 516, 490, 535, 517, 486, 514, 556, 530, 508, 515, 495, 536, 485, 492, 500, 534, 518, 487, 509, 503, 527, 506, 492, 524, 536, 511, 568, 517, 508, 517, 533, 489, 534, 498, 522, 494, 475, 524, 531, 502, 505, 523, 488, 501, 497, 467, 504, 517, 481, 523, 473, 489, 506, 541, 541, 509, 511, 515, 459, 506, 519, 514, 496, 461, 505, 507, 507, 506, 504, 514, 529, 507, 521, 549, 514, 511, 502, 540, 516, 473, 516, 516, 515, 497, 508, 484, 529, 525, 537, 508, 495, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[485, 479, 494, 550, 552, 518, 535, 449, 487, 496, 485, 518, 491, 513, 537, 519, 489, 488, 532, 521, 486, 524, 505, 526, 549, 506, 521, 504, 544, 487, 518, 496, 514, 508, 493, 501, 526, 519, 538, 518, 565, 533, 493, 478, 499, 548, 506, 516, 533, 545, 513, 549, 493, 488, 528, 512, 488, 518, 527, 553, 498, 556, 507, 458, 493, 493, 484, 480, 478, 518, 538, 505, 516, 524, 529, 543, 477, 526, 530, 493, 508, 512, 561, 522, 499, 514, 539, 512, 542, 503, 521, 518, 514, 510, 515, 525, 515, 497, 504, 504, 479, 475, 522, 520, 548, 462, 517, 509, 518, 514, 495, 524, 477, 537, 515, 524, 501, 498, 532, 492, 535, 536, 483, 534, 503, 535, 480, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[528, 508, 506, 517, 484, 526, 492, 501, 490, 495, 502, 531, 515, 527, 530, 484, 518, 464, 474, 521, 498, 483, 487, 527, 521, 526, 500, 522, 503, 541, 514, 513, 488, 523, 487, 479, 521, 509, 504, 486, 559, 519, 555, 477, 498, 517, 546, 519, 533, 495, 560, 541, 481, 495, 520, 539, 504, 527, 516, 562, 505, 504, 490, 493, 503, 505, 515, 474, 524, 473, 541, 500, 540, 531, 523, 485, 516, 512, 531, 518, 512, 497, 560, 501, 530, 536, 521, 497, 511, 507, 537, 495, 527, 492, 531, 517, 492, 512, 536, 465, 498, 556, 543, 535, 520, 500, 466, 512, 514, 525, 544, 502, 515, 517, 499, 480, 486, 497, 534, 540, 509, 527, 509, 530, 526, 480, 541, 464] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[504, 528, 508, 549, 489, 472, 484, 484, 500, 530, 550, 485, 501, 506, 471, 501, 512, 523, 541, 537, 473, 505, 519, 494, 533, 508, 532, 521, 499, 507, 532, 502, 486, 558, 495, 501, 489, 507, 511, 514, 551, 521, 560, 536, 482, 505, 539, 518, 547, 546, 552, 543, 510, 500, 491, 523, 537, 506, 513, 512, 508, 533, 552, 522, 491, 527, 505, 517, 510, 517, 516, 476, 540, 516, 517, 493, 480, 545, 508, 501, 509, 480, 521, 522, 522, 511, 481, 505, 503, 512, 501, 492, 497, 526, 525, 474, 451, 511, 479, 536, 528, 549, 516, 503, 519, 503, 469, 513, 516, 491, 491, 494, 503, 483, 487, 522, 546, 514, 518, 505, 537, 531, 539, 493, 486, 543, 534, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[522, 501, 549, 520, 531, 510, 538, 550, 549, 510, 511, 483, 488, 495, 529, 510, 496, 471, 532, 528, 553, 517, 498, 493, 528, 489, 514, 483, 484, 497, 502, 490, 475, 509, 481, 513, 501, 533, 570, 501, 484, 490, 524, 474, 498, 490, 512, 550, 487, 495, 527, 456, 498, 478, 525, 514, 518, 537, 497, 525, 510, 526, 509, 573, 493, 505, 529, 499, 508, 534, 508, 495, 483, 512, 529, 519, 528, 504, 512, 542, 482, 549, 490, 504, 508, 537, 490, 518, 519, 515, 544, 528, 518, 524, 506, 539, 527, 498, 526, 500, 554, 546, 545, 522, 509, 517, 494, 539, 522, 497, 508, 515, 483, 485, 480, 504, 522, 508, 535, 510, 527, 499, 516, 491, 512, 529, 483, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[537, 513, 510, 526, 508, 510, 528, 513, 514, 501, 493, 495, 513, 547, 544, 517, 503, 526, 540, 466, 526, 536, 511, 541, 517, 490, 557, 491, 496, 511, 470, 514, 487, 502, 527, 526, 547, 547, 496, 524, 490, 514, 526, 493, 538, 518, 499, 488, 549, 468, 509, 567, 509, 512, 492, 528, 509, 480, 489, 495, 532, 489, 504, 453, 501, 484, 481, 474, 529, 536, 496, 515, 496, 494, 516, 481, 512, 514, 512, 526, 533, 484, 517, 536, 551, 547, 513, 501, 485, 475, 529, 488, 522, 517, 533, 489, 469, 485, 478, 524, 489, 497, 515, 511, 509, 511, 543, 508, 530, 512, 544, 555, 527, 499, 596, 461, 512, 509, 502, 526, 548, 519, 490, 524, 506, 531, 532, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 479, 494, 541, 552, 518, 535, 449, 487, 496, 485, 518, 491, 513, 537, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 488, 532, 521, 486, 524, 505, 526, 549, 506, 520, 504, 544, 487, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 493, 501, 526, 519, 538, 518, 565, 533, 472, 478, 499, 548, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 545, 513, 517, 493, 488, 528, 512, 488, 518, 527, 553, 498, 556, 507, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 484, 480, 478, 518, 538, 505, 516, 524, 529, 543, 477, 526, 530, 493, 32, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 561, 522, 499, 514, 539, 512, 542, 503, 521, 518, 514, 510, 515, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 504, 504, 479, 475, 522, 520, 548, 462, 517, 509, 518, 514, 495, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 537, 515, 524, 501, 498, 532, 492, 535, 536, 483, 534, 503, 535, 480, 484, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8195, 8234, 8234, 8185, 8315, 8103, 8171] avg=8192.0000 max=8315 max_rank=5 min=8099 min_rank=0 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[525, 509, 526, 468, 491, 546, 520, 525, 506, 473, 511, 515, 523, 518, 495, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 522, 492, 521, 522, 491, 510, 531, 498, 557, 516, 512, 507, 535, 535, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 536, 546, 516, 490, 535, 517, 486, 514, 532, 530, 508, 515, 495, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 492, 500, 503, 518, 487, 509, 503, 527, 506, 492, 524, 536, 511, 568, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 533, 489, 534, 498, 522, 494, 475, 524, 531, 502, 505, 523, 488, 501, 31, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 467, 504, 517, 481, 523, 473, 489, 506, 541, 541, 509, 511, 515, 459, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 514, 496, 461, 505, 507, 507, 506, 504, 514, 529, 507, 521, 549, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 540, 516, 473, 516, 516, 515, 497, 508, 484, 529, 525, 537, 508, 495, 513, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8307, 8276, 8178, 8203, 8039, 8164, 8178] avg=8192.0000 max=8307 max_rank=1 min=8039 min_rank=5 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[504, 528, 508, 540, 489, 472, 484, 484, 500, 530, 550, 485, 501, 506, 471, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 541, 537, 473, 505, 519, 494, 533, 508, 531, 521, 499, 507, 532, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 558, 495, 501, 489, 507, 511, 514, 551, 521, 536, 536, 482, 505, 539, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 546, 552, 511, 510, 500, 491, 523, 537, 506, 513, 512, 508, 533, 552, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 527, 505, 517, 510, 517, 516, 476, 540, 516, 517, 493, 480, 545, 508, 501, 32, 24, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 480, 521, 522, 522, 511, 481, 505, 503, 512, 501, 492, 497, 526, 525, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 511, 479, 536, 528, 549, 516, 503, 519, 503, 469, 513, 516, 491, 491, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 483, 487, 522, 546, 514, 518, 505, 537, 531, 539, 493, 486, 543, 534, 518, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 8237, 8249, 8363, 8220, 8081, 8069, 8264] avg=8192.0000 max=8363 max_rank=3 min=8053 min_rank=0 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[528, 508, 506, 501, 484, 526, 492, 501, 490, 495, 502, 531, 515, 527, 530, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 464, 474, 521, 498, 483, 487, 527, 521, 526, 494, 522, 503, 541, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 523, 487, 479, 521, 509, 504, 486, 559, 519, 527, 477, 498, 517, 546, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 495, 560, 505, 481, 495, 520, 539, 504, 527, 516, 562, 505, 504, 490, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 515, 474, 524, 473, 541, 500, 540, 531, 523, 485, 516, 512, 531, 518, 36, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 560, 501, 530, 536, 521, 497, 511, 507, 537, 495, 527, 492, 531, 517, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 512, 536, 465, 498, 549, 543, 535, 520, 500, 466, 512, 514, 525, 544, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 517, 499, 480, 486, 497, 534, 540, 509, 527, 509, 530, 526, 480, 541, 464, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8106, 8159, 8229, 8263, 8276, 8213, 8170] avg=8192.0000 max=8276 max_rank=5 min=8106 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 547, 509, 551, 492, 534, 511, 514, 479, 522, 473, 497, 528, 498, 542, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 502, 523, 487, 516, 510, 467, 553, 479, 471, 527, 486, 534, 488, 544, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 498, 530, 516, 509, 482, 520, 532, 492, 519, 486, 503, 527, 521, 545, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 523, 483, 498, 505, 534, 514, 507, 513, 521, 547, 494, 537, 516, 520, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 482, 543, 470, 510, 499, 553, 480, 544, 538, 514, 514, 514, 476, 518, 497, 31, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 499, 504, 520, 485, 507, 521, 532, 507, 527, 533, 523, 525, 497, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 526, 512, 492, 528, 507, 531, 503, 475, 535, 472, 522, 515, 499, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 535, 514, 497, 501, 497, 540, 524, 517, 512, 491, 499, 498, 516, 520, 503, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8104, 8187, 8263, 8214, 8195, 8173, 8162] avg=8192.0000 max=8263 max_rank=3 min=8104 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[544, 527, 527, 481, 522, 536, 556, 504, 531, 489, 514, 507, 508, 527, 485, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 480, 523, 528, 525, 532, 508, 527, 544, 505, 531, 549, 510, 526, 509, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 522, 474, 507, 507, 537, 537, 550, 508, 511, 485, 504, 514, 497, 538, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 488, 450, 488, 523, 483, 517, 547, 545, 504, 503, 485, 503, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 516, 465, 488, 532, 497, 484, 454, 529, 512, 512, 498, 495, 525, 507, 489, 28, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 523, 481, 519, 498, 509, 474, 516, 518, 513, 512, 530, 533, 508, 557, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 509, 513, 545, 530, 500, 516, 463, 516, 519, 544, 513, 544, 567, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 493, 520, 517, 542, 511, 504, 527, 487, 520, 493, 489, 497, 521, 534, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8319, 8150, 8069, 8070, 8139, 8315, 8188] avg=8192.0000 max=8319 max_rank=1 min=8069 min_rank=3 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 501, 549, 512, 531, 510, 538, 550, 549, 510, 511, 483, 488, 495, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 471, 532, 528, 553, 517, 498, 493, 528, 489, 513, 483, 484, 497, 502, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 509, 481, 513, 501, 533, 570, 501, 484, 490, 501, 474, 498, 490, 512, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 495, 527, 429, 498, 478, 525, 514, 518, 537, 497, 525, 510, 526, 509, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 529, 499, 508, 534, 508, 495, 483, 512, 529, 519, 528, 504, 512, 542, 27, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 549, 490, 504, 508, 537, 490, 518, 519, 515, 544, 528, 518, 524, 506, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 498, 526, 500, 554, 546, 545, 522, 509, 517, 494, 539, 522, 497, 508, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 485, 480, 504, 522, 508, 535, 510, 527, 499, 516, 491, 512, 529, 483, 511, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8074, 8082, 8148, 8254, 8271, 8319, 8100] avg=8192.0000 max=8319 max_rank=6 min=8074 min_rank=1 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[537, 513, 510, 518, 508, 510, 528, 513, 514, 501, 493, 495, 513, 547, 544, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 540, 466, 526, 536, 511, 541, 517, 490, 556, 491, 496, 511, 470, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 502, 527, 526, 547, 547, 496, 524, 490, 514, 503, 493, 538, 518, 499, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 468, 509, 534, 509, 512, 492, 528, 509, 480, 489, 495, 532, 489, 504, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 484, 481, 474, 529, 536, 496, 515, 496, 494, 516, 481, 512, 514, 512, 526, 33, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 484, 517, 536, 551, 547, 513, 501, 485, 475, 529, 488, 522, 517, 533, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 485, 478, 524, 489, 497, 515, 511, 509, 511, 543, 508, 530, 512, 544, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 499, 596, 461, 512, 509, 502, 526, 548, 519, 490, 524, 506, 531, 532, 516, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8194, 8199, 8052, 8127, 8220, 8180, 8303] avg=8192.0000 max=8303 max_rank=7 min=8052 min_rank=3 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 3968, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[522, 511, 538, 493, 481, 484, 532, 524, 525, 507, 489, 534, 492, 532, 529, 512, 526, 515, 528, 497, 508, 550, 469, 526, 492, 472, 546, 541, 503, 508, 505, 522, 512, 489, 509, 491, 534, 526, 491, 478, 509, 509, 482, 549, 512, 494, 487, 521, 458, 501, 483, 513, 536, 520, 505, 540, 516, 529, 510, 479, 502, 520, 542, 513, 505, 481, 500, 516, 511, 534, 468, 532, 546, 520, 550, 550, 505, 516, 492, 466, 540, 523, 519, 508, 504, 489, 502, 482, 490, 525, 505, 505, 512, 508, 510, 511, 546, 533, 486, 541, 505, 535, 518, 484, 533, 525, 521, 589, 524, 495, 532, 488, 497, 512, 530, 485, 501, 540, 509, 505, 502, 498, 515, 517, 534, 496, 526, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[481, 506, 497, 494, 524, 491, 484, 506, 544, 499, 559, 513, 487, 533, 534, 497, 521, 490, 481, 529, 499, 501, 521, 491, 505, 480, 515, 488, 492, 498, 508, 513, 518, 511, 470, 523, 545, 510, 507, 496, 559, 512, 498, 532, 489, 512, 533, 477, 488, 472, 521, 505, 490, 486, 531, 586, 525, 515, 478, 513, 525, 510, 502, 521, 503, 505, 502, 553, 478, 474, 526, 551, 555, 493, 524, 506, 509, 543, 515, 516, 539, 532, 523, 510, 509, 527, 520, 509, 534, 514, 495, 515, 540, 537, 531, 543, 516, 541, 535, 524, 481, 519, 523, 468, 480, 506, 492, 480, 520, 485, 538, 516, 498, 515, 492, 524, 500, 498, 500, 517, 504, 551, 534, 538, 501, 479, 524, 565] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[518, 509, 546, 486, 519, 526, 482, 463, 510, 513, 488, 502, 497, 527, 514, 487, 539, 491, 487, 514, 503, 496, 493, 494, 524, 496, 539, 527, 551, 482, 532, 499, 532, 538, 512, 521, 506, 532, 528, 506, 506, 505, 498, 508, 496, 549, 540, 508, 513, 525, 523, 489, 527, 534, 541, 536, 506, 512, 514, 529, 490, 479, 513, 520, 517, 474, 496, 509, 522, 530, 473, 577, 545, 478, 494, 534, 558, 490, 478, 516, 536, 509, 503, 511, 513, 567, 535, 521, 522, 453, 486, 495, 509, 528, 481, 483, 492, 547, 494, 511, 562, 494, 489, 500, 502, 494, 528, 544, 520, 546, 529, 524, 509, 491, 510, 532, 523, 485, 530, 472, 464, 508, 506, 521, 518, 511, 519, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[500, 534, 513, 533, 557, 526, 506, 517, 516, 525, 540, 485, 532, 494, 542, 499, 506, 529, 494, 517, 477, 530, 531, 532, 489, 495, 524, 513, 497, 505, 550, 488, 502, 512, 478, 489, 542, 477, 524, 508, 504, 512, 516, 554, 526, 501, 492, 484, 529, 513, 486, 539, 534, 526, 526, 488, 522, 480, 506, 500, 508, 510, 531, 521, 523, 530, 525, 505, 496, 522, 543, 511, 506, 504, 466, 509, 482, 516, 503, 526, 536, 499, 495, 512, 537, 503, 522, 467, 475, 507, 517, 538, 502, 509, 506, 506, 511, 524, 530, 550, 511, 510, 526, 480, 544, 528, 501, 519, 511, 519, 484, 490, 483, 513, 496, 486, 541, 516, 497, 509, 570, 476, 529, 512, 505, 509, 507, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[499, 542, 519, 514, 517, 519, 459, 507, 515, 538, 506, 506, 549, 547, 543, 494, 481, 509, 517, 523, 557, 500, 509, 535, 497, 509, 504, 509, 529, 473, 506, 489, 470, 525, 490, 509, 546, 475, 490, 497, 512, 505, 528, 501, 514, 526, 503, 485, 495, 510, 546, 518, 509, 474, 519, 468, 528, 513, 518, 482, 557, 504, 494, 532, 541, 512, 451, 541, 494, 505, 533, 487, 505, 509, 495, 506, 512, 532, 541, 538, 527, 476, 522, 504, 540, 540, 449, 528, 533, 520, 524, 526, 517, 533, 504, 536, 497, 543, 503, 508, 507, 487, 506, 528, 520, 523, 525, 500, 497, 522, 547, 475, 533, 500, 558, 531, 493, 490, 519, 510, 512, 501, 500, 513, 530, 494, 489, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[486, 539, 528, 509, 530, 547, 499, 510, 513, 504, 527, 469, 525, 503, 511, 552, 512, 567, 498, 501, 463, 512, 551, 500, 497, 507, 532, 491, 493, 535, 494, 496, 512, 503, 506, 497, 520, 525, 466, 500, 517, 520, 520, 538, 504, 493, 484, 488, 533, 518, 516, 495, 528, 557, 526, 498, 501, 526, 475, 500, 518, 530, 511, 537, 508, 485, 500, 514, 484, 500, 472, 510, 520, 487, 529, 523, 539, 490, 521, 516, 533, 527, 542, 524, 473, 513, 516, 512, 502, 522, 509, 498, 527, 524, 523, 514, 520, 515, 515, 510, 497, 511, 521, 504, 489, 503, 524, 533, 498, 549, 508, 552, 490, 480, 527, 534, 510, 525, 486, 499, 513, 519, 539, 492, 520, 499, 514, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[521, 545, 477, 506, 523, 503, 499, 501, 523, 554, 491, 512, 528, 552, 553, 505, 537, 544, 534, 498, 497, 492, 472, 509, 486, 480, 512, 550, 525, 455, 526, 510, 530, 506, 512, 499, 497, 483, 505, 512, 490, 493, 514, 514, 488, 540, 537, 516, 488, 482, 500, 547, 508, 518, 521, 507, 502, 455, 531, 486, 484, 461, 533, 574, 560, 487, 511, 508, 536, 515, 490, 523, 538, 542, 480, 511, 473, 488, 486, 531, 543, 555, 507, 478, 544, 516, 496, 487, 510, 528, 480, 517, 501, 522, 474, 514, 490, 488, 585, 520, 521, 508, 487, 510, 516, 560, 521, 496, 535, 496, 505, 545, 502, 525, 548, 525, 533, 507, 500, 500, 524, 509, 495, 544, 489, 507, 505, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[522, 472, 513, 503, 495, 478, 491, 510, 526, 490, 543, 556, 508, 517, 540, 532, 501, 501, 495, 522, 526, 518, 498, 498, 502, 528, 513, 503, 514, 521, 536, 481, 514, 505, 527, 504, 515, 498, 501, 562, 531, 508, 575, 493, 539, 543, 534, 529, 542, 510, 491, 552, 522, 519, 481, 516, 560, 500, 538, 514, 554, 454, 522, 465, 506, 514, 514, 502, 547, 524, 518, 527, 489, 503, 526, 504, 492, 503, 506, 480, 505, 529, 477, 513, 490, 503, 499, 501, 521, 492, 519, 507, 525, 499, 503, 502, 530, 494, 496, 511, 536, 530, 516, 535, 506, 532, 485, 541, 530, 495, 511, 518, 481, 511, 514, 512, 492, 515, 521, 518, 452, 548, 515, 486, 480, 470, 533, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 511, 538, 493, 481, 484, 532, 524, 525, 507, 489, 534, 492, 532, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 528, 497, 508, 550, 469, 526, 492, 472, 546, 541, 503, 508, 505, 522, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 489, 509, 491, 534, 526, 491, 478, 509, 509, 482, 549, 512, 494, 487, 521, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 501, 483, 513, 536, 520, 505, 540, 516, 529, 510, 479, 502, 520, 542, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 481, 500, 516, 511, 534, 468, 532, 546, 520, 550, 550, 505, 516, 492, 466, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 523, 519, 508, 504, 489, 502, 482, 490, 525, 505, 505, 512, 508, 510, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 533, 486, 541, 505, 535, 518, 484, 533, 525, 521, 528, 524, 495, 532, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 530, 485, 501, 540, 509, 505, 502, 498, 515, 517, 534, 496, 526, 516, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8260, 8113, 8162, 8203, 8127, 8294, 8200] avg=8192.0000 max=8294 max_rank=6 min=8113 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[481, 506, 497, 494, 524, 491, 484, 506, 544, 499, 559, 513, 487, 533, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 490, 481, 529, 499, 501, 521, 491, 505, 480, 515, 488, 492, 498, 508, 513, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 470, 523, 545, 510, 507, 496, 559, 512, 498, 532, 489, 512, 533, 477, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 472, 521, 505, 490, 486, 531, 586, 525, 515, 478, 513, 525, 510, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 502, 553, 478, 474, 526, 551, 555, 493, 524, 506, 509, 543, 515, 516, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 532, 523, 510, 509, 527, 520, 509, 534, 514, 495, 515, 540, 537, 531, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 541, 535, 524, 481, 519, 523, 468, 480, 506, 492, 430, 520, 485, 538, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 492, 524, 500, 498, 500, 517, 504, 551, 534, 538, 501, 479, 524, 565, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8074, 8211, 8163, 8264, 8372, 8074, 8258] avg=8192.0000 max=8372 max_rank=5 min=8074 min_rank=1 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32968 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[500, 534, 513, 533, 557, 526, 506, 517, 516, 525, 540, 485, 532, 494, 513, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 494, 517, 477, 530, 531, 532, 489, 495, 524, 513, 497, 505, 550, 488, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 512, 478, 489, 542, 477, 524, 508, 504, 512, 516, 554, 526, 501, 492, 484, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 513, 486, 539, 534, 526, 526, 488, 522, 480, 506, 500, 508, 510, 531, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 530, 525, 505, 496, 522, 543, 511, 506, 504, 466, 509, 482, 516, 503, 526, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 495, 512, 537, 503, 522, 467, 475, 507, 517, 538, 502, 509, 506, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 524, 530, 550, 511, 510, 526, 480, 544, 528, 501, 466, 511, 519, 484, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 513, 496, 486, 541, 516, 497, 509, 570, 476, 529, 512, 505, 509, 507, 515, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8222, 8140, 8214, 8178, 8125, 8185, 8182] avg=8192.0000 max=8290 max_rank=0 min=8125 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 509, 546, 486, 519, 526, 482, 463, 510, 513, 488, 502, 497, 527, 475, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 491, 487, 514, 503, 496, 493, 494, 524, 496, 539, 527, 551, 482, 532, 499, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 538, 512, 521, 506, 532, 528, 506, 506, 505, 498, 508, 496, 549, 540, 508, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 523, 489, 527, 534, 541, 536, 506, 512, 514, 529, 490, 479, 513, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 474, 496, 509, 522, 530, 473, 577, 545, 478, 494, 534, 558, 490, 478, 516, 11, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 503, 511, 513, 567, 535, 521, 522, 453, 486, 495, 509, 528, 481, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 547, 494, 511, 562, 494, 489, 500, 502, 494, 528, 480, 520, 546, 529, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 510, 532, 523, 485, 530, 472, 464, 508, 506, 521, 518, 511, 519, 528, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 8219, 8312, 8243, 8214, 8141, 8212, 8147] avg=8192.0000 max=8312 max_rank=2 min=8048 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[499, 542, 519, 514, 517, 519, 459, 507, 515, 538, 506, 506, 549, 547, 514, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 509, 517, 523, 557, 500, 509, 535, 497, 509, 504, 509, 529, 473, 506, 489, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 525, 490, 509, 546, 475, 490, 497, 512, 505, 528, 501, 514, 526, 503, 485, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 510, 546, 518, 509, 474, 519, 468, 528, 513, 518, 482, 557, 504, 494, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 512, 451, 541, 494, 505, 533, 487, 505, 509, 495, 506, 512, 532, 541, 538, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 476, 522, 504, 540, 540, 449, 528, 533, 520, 524, 526, 517, 533, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 543, 503, 508, 507, 487, 506, 528, 520, 523, 525, 448, 497, 522, 547, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 500, 558, 531, 493, 490, 519, 510, 512, 501, 500, 513, 530, 494, 489, 530, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8191, 8095, 8162, 8213, 8273, 8136, 8221] avg=8192.0000 max=8273 max_rank=5 min=8095 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 545, 477, 506, 523, 503, 499, 501, 523, 554, 491, 512, 528, 552, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 544, 534, 498, 497, 492, 472, 509, 486, 480, 512, 550, 525, 455, 526, 510, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 506, 512, 499, 497, 483, 505, 512, 490, 493, 514, 514, 488, 540, 537, 516, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 482, 500, 547, 508, 518, 521, 507, 502, 455, 531, 486, 484, 461, 533, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 487, 511, 508, 536, 515, 490, 523, 538, 542, 480, 511, 473, 488, 486, 531, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 555, 507, 478, 544, 516, 496, 487, 510, 528, 480, 517, 501, 522, 474, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 488, 585, 520, 521, 508, 487, 510, 516, 560, 521, 445, 535, 496, 505, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 548, 525, 533, 507, 500, 500, 524, 509, 495, 544, 489, 507, 505, 536, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8170, 8155, 8092, 8190, 8166, 8232, 8267] avg=8192.0000 max=8267 max_rank=7 min=8092 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[486, 539, 528, 509, 530, 547, 499, 510, 513, 504, 527, 469, 525, 503, 484, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 567, 498, 501, 463, 512, 551, 500, 497, 507, 532, 491, 493, 535, 494, 496, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 503, 506, 497, 520, 525, 466, 500, 517, 520, 520, 538, 504, 493, 484, 488, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 516, 495, 528, 557, 526, 498, 501, 526, 475, 500, 518, 530, 511, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 485, 500, 514, 484, 500, 472, 510, 520, 487, 529, 523, 539, 490, 521, 516, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 527, 542, 524, 473, 513, 516, 512, 502, 522, 509, 498, 527, 524, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 515, 515, 510, 497, 511, 521, 504, 489, 503, 524, 477, 498, 549, 508, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 480, 527, 534, 510, 525, 486, 499, 513, 519, 539, 492, 520, 499, 514, 520, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8196, 8112, 8264, 8109, 8253, 8193, 8184] avg=8192.0000 max=8264 max_rank=3 min=8109 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 472, 513, 503, 495, 478, 491, 510, 526, 490, 543, 556, 508, 517, 511, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 495, 522, 526, 518, 498, 498, 502, 528, 513, 503, 514, 521, 536, 481, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 505, 527, 504, 515, 498, 501, 562, 531, 508, 575, 493, 539, 543, 534, 529, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 491, 552, 522, 519, 481, 516, 560, 500, 538, 514, 554, 454, 522, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 514, 502, 547, 524, 518, 527, 489, 503, 526, 504, 492, 503, 506, 480, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 529, 477, 513, 490, 503, 499, 501, 521, 492, 519, 507, 525, 499, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 494, 496, 511, 536, 530, 516, 535, 506, 532, 485, 485, 530, 495, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 511, 514, 512, 492, 515, 521, 518, 452, 548, 515, 486, 480, 470, 533, 511, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8204, 8398, 8236, 8165, 8079, 8210, 8077] avg=8192.0000 max=8398 max_rank=2 min=8077 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 3840, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 3968, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[542, 485, 520, 527, 525, 491, 517, 501, 552, 509, 511, 494, 537, 540, 503, 499, 482, 513, 507, 540, 488, 551, 521, 528, 573, 502, 515, 492, 499, 515, 462, 520, 517, 501, 488, 516, 518, 505, 501, 548, 481, 493, 519, 524, 529, 522, 523, 512, 516, 531, 502, 497, 525, 496, 500, 501, 515, 496, 502, 500, 510, 487, 535, 528, 474, 498, 505, 532, 537, 468, 503, 513, 506, 514, 502, 536, 504, 498, 485, 516, 531, 497, 498, 483, 520, 483, 501, 536, 508, 495, 528, 517, 502, 477, 511, 551, 501, 519, 487, 483, 464, 524, 547, 533, 519, 504, 524, 529, 512, 518, 509, 548, 552, 509, 508, 501, 504, 486, 552, 522, 519, 543, 499, 549, 527, 506, 509, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[513, 546, 496, 513, 504, 486, 493, 490, 507, 506, 480, 510, 546, 493, 513, 508, 538, 493, 542, 493, 499, 531, 505, 517, 487, 500, 498, 475, 572, 524, 522, 528, 525, 537, 493, 535, 482, 509, 538, 496, 494, 520, 507, 503, 493, 538, 501, 511, 516, 528, 486, 517, 519, 477, 493, 516, 519, 540, 506, 513, 532, 502, 512, 506, 522, 489, 493, 548, 492, 512, 495, 530, 538, 515, 509, 480, 508, 503, 527, 473, 540, 510, 509, 550, 469, 495, 518, 547, 538, 463, 554, 554, 497, 512, 510, 530, 479, 468, 495, 547, 482, 494, 499, 495, 509, 530, 504, 518, 533, 502, 516, 502, 518, 497, 543, 546, 524, 532, 527, 545, 518, 499, 516, 489, 523, 530, 518, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[480, 539, 536, 533, 508, 531, 505, 526, 502, 558, 493, 496, 504, 511, 524, 511, 491, 506, 536, 533, 485, 528, 506, 509, 514, 504, 474, 479, 491, 482, 522, 530, 478, 532, 526, 515, 535, 518, 526, 508, 496, 543, 517, 510, 515, 495, 518, 508, 504, 515, 579, 529, 546, 530, 540, 514, 518, 531, 490, 502, 546, 529, 481, 523, 502, 510, 501, 518, 493, 522, 514, 515, 502, 547, 498, 458, 503, 473, 538, 529, 462, 487, 508, 516, 517, 522, 491, 518, 508, 491, 482, 535, 507, 526, 516, 506, 533, 499, 513, 509, 503, 491, 545, 520, 530, 530, 538, 518, 503, 512, 497, 508, 499, 509, 505, 518, 493, 507, 498, 503, 505, 533, 458, 539, 528, 504, 492, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[472, 526, 541, 525, 497, 503, 534, 483, 530, 532, 527, 524, 516, 511, 549, 498, 522, 488, 529, 528, 527, 513, 514, 507, 530, 515, 508, 465, 499, 520, 496, 513, 566, 521, 509, 519, 521, 510, 499, 525, 512, 526, 557, 500, 518, 474, 501, 538, 515, 489, 534, 540, 509, 525, 500, 504, 471, 485, 477, 522, 502, 483, 519, 528, 493, 510, 519, 502, 492, 532, 524, 500, 511, 493, 529, 474, 500, 488, 541, 485, 504, 538, 525, 520, 518, 515, 499, 478, 468, 528, 492, 466, 539, 503, 519, 518, 502, 500, 550, 555, 546, 508, 505, 515, 560, 536, 487, 483, 515, 477, 507, 505, 490, 528, 504, 516, 515, 541, 508, 550, 527, 490, 515, 491, 519, 522, 484, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[484, 529, 489, 504, 544, 488, 487, 501, 496, 532, 495, 501, 520, 484, 479, 512, 507, 463, 522, 496, 501, 537, 509, 482, 522, 482, 507, 512, 492, 535, 536, 518, 522, 493, 545, 526, 557, 487, 524, 488, 472, 542, 515, 523, 561, 495, 515, 501, 505, 516, 507, 464, 529, 500, 516, 499, 549, 518, 529, 519, 491, 502, 516, 525, 553, 525, 498, 519, 505, 521, 503, 525, 518, 550, 536, 508, 543, 505, 555, 446, 493, 478, 503, 508, 520, 549, 524, 503, 508, 537, 565, 529, 493, 456, 525, 536, 449, 524, 507, 520, 535, 526, 535, 460, 474, 528, 554, 515, 519, 485, 484, 496, 517, 560, 528, 489, 518, 518, 551, 514, 511, 488, 489, 574, 485, 520, 487, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[481, 534, 522, 493, 479, 514, 539, 495, 521, 530, 524, 506, 508, 510, 530, 499, 496, 516, 516, 494, 552, 507, 515, 503, 498, 546, 515, 501, 503, 528, 504, 508, 514, 500, 553, 548, 493, 506, 473, 526, 533, 528, 503, 529, 474, 492, 516, 527, 514, 505, 504, 553, 445, 473, 509, 527, 493, 518, 489, 506, 557, 508, 517, 511, 497, 511, 482, 476, 518, 531, 541, 490, 508, 527, 519, 497, 525, 500, 517, 521, 512, 517, 527, 490, 525, 547, 514, 539, 553, 504, 496, 491, 455, 533, 512, 528, 526, 482, 500, 517, 548, 541, 514, 518, 487, 514, 530, 530, 542, 496, 511, 515, 492, 503, 517, 544, 487, 485, 523, 524, 526, 509, 489, 495, 514, 483, 504, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[521, 493, 525, 520, 505, 517, 506, 495, 508, 508, 502, 524, 520, 494, 525, 486, 522, 505, 529, 529, 525, 503, 501, 493, 544, 526, 484, 482, 506, 518, 490, 547, 488, 509, 507, 514, 483, 504, 469, 493, 466, 505, 517, 520, 554, 521, 502, 543, 525, 480, 530, 517, 540, 556, 562, 466, 512, 513, 477, 518, 508, 521, 540, 544, 558, 507, 519, 530, 529, 484, 497, 515, 511, 491, 532, 542, 533, 507, 525, 519, 517, 544, 507, 510, 488, 506, 497, 461, 510, 501, 502, 522, 528, 486, 484, 490, 490, 541, 499, 557, 554, 511, 499, 515, 489, 516, 498, 537, 538, 545, 537, 505, 522, 480, 486, 499, 520, 506, 494, 572, 483, 518, 461, 498, 521, 519, 488, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[504, 514, 516, 485, 506, 506, 494, 465, 498, 523, 489, 483, 502, 514, 490, 491, 505, 521, 531, 505, 533, 496, 486, 470, 541, 484, 524, 495, 489, 503, 532, 531, 504, 566, 534, 508, 549, 529, 531, 485, 473, 518, 506, 522, 491, 543, 465, 480, 573, 529, 486, 505, 561, 515, 493, 542, 500, 506, 511, 520, 538, 526, 516, 529, 497, 482, 532, 481, 490, 499, 515, 537, 532, 489, 508, 494, 497, 482, 510, 515, 540, 493, 509, 535, 506, 505, 537, 525, 542, 529, 487, 503, 478, 497, 501, 518, 506, 511, 529, 530, 509, 499, 501, 540, 463, 542, 557, 517, 508, 505, 535, 491, 506, 513, 504, 526, 527, 524, 557, 527, 546, 530, 510, 503, 523, 512, 512, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[542, 485, 520, 527, 525, 491, 517, 501, 552, 509, 511, 494, 537, 540, 503, 499, 32, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 513, 507, 540, 488, 551, 521, 528, 573, 502, 515, 492, 499, 515, 462, 520, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 501, 488, 498, 518, 505, 501, 548, 481, 493, 519, 524, 529, 522, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 531, 502, 497, 525, 496, 500, 501, 515, 496, 502, 500, 510, 487, 535, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 498, 505, 532, 537, 468, 503, 513, 506, 514, 502, 536, 504, 498, 485, 516, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 497, 498, 483, 520, 483, 501, 536, 508, 495, 528, 517, 502, 477, 511, 551, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 519, 487, 459, 464, 524, 547, 533, 519, 504, 524, 529, 512, 518, 509, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 509, 508, 501, 504, 486, 552, 490, 519, 543, 499, 549, 527, 506, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8226, 8179, 8112, 8123, 8156, 8197, 8255] avg=8192.0000 max=8288 max_rank=0 min=8112 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[480, 539, 536, 533, 508, 531, 505, 526, 502, 558, 493, 496, 504, 511, 524, 511, 31, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 506, 536, 533, 485, 528, 506, 509, 514, 504, 474, 479, 491, 482, 522, 530, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 532, 526, 497, 535, 518, 526, 508, 496, 543, 517, 510, 515, 495, 518, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 515, 579, 529, 546, 530, 540, 514, 518, 531, 490, 502, 546, 529, 481, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 510, 501, 518, 493, 522, 514, 515, 502, 547, 498, 458, 503, 473, 538, 529, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 487, 508, 516, 517, 522, 491, 518, 508, 491, 482, 535, 507, 526, 516, 506, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 499, 513, 484, 503, 491, 545, 520, 530, 530, 538, 518, 503, 512, 497, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 505, 518, 493, 507, 498, 472, 505, 533, 458, 539, 528, 504, 492, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8109, 8222, 8348, 8155, 8110, 8224, 8077] avg=8192.0000 max=8348 max_rank=3 min=8077 min_rank=7 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[472, 526, 541, 525, 497, 503, 534, 483, 530, 532, 527, 524, 516, 511, 549, 498, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 488, 529, 528, 527, 513, 514, 507, 530, 515, 508, 465, 499, 520, 496, 513, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 521, 509, 501, 521, 510, 499, 525, 512, 526, 557, 500, 518, 474, 501, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 489, 534, 540, 509, 525, 500, 504, 471, 485, 477, 522, 502, 483, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 510, 519, 502, 492, 532, 524, 500, 511, 493, 529, 474, 500, 488, 541, 485, 26, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 538, 525, 520, 518, 515, 499, 478, 468, 528, 492, 466, 539, 503, 519, 518, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 550, 527, 546, 508, 505, 515, 560, 536, 487, 483, 515, 477, 507, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 528, 504, 516, 515, 541, 508, 516, 527, 490, 515, 491, 519, 522, 484, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8195, 8278, 8074, 8126, 8148, 8223, 8187] avg=8192.0000 max=8305 max_rank=0 min=8074 min_rank=3 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[481, 534, 522, 493, 479, 514, 539, 495, 521, 530, 524, 506, 508, 510, 530, 499, 36, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 516, 494, 552, 507, 515, 503, 498, 546, 515, 501, 503, 528, 504, 508, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 553, 517, 493, 506, 473, 526, 533, 528, 503, 529, 474, 492, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 505, 504, 553, 445, 473, 509, 527, 493, 518, 489, 506, 557, 508, 517, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 511, 482, 476, 518, 531, 541, 490, 508, 527, 519, 497, 525, 500, 517, 521, 30, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 527, 490, 525, 547, 514, 539, 553, 504, 496, 491, 455, 533, 512, 528, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 482, 500, 486, 548, 541, 514, 518, 487, 514, 530, 530, 542, 496, 511, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 503, 517, 544, 487, 485, 523, 488, 526, 509, 489, 495, 514, 483, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8231, 8184, 8092, 8199, 8267, 8240, 8095] avg=8192.0000 max=8267 max_rank=5 min=8092 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 546, 496, 513, 504, 486, 493, 490, 507, 506, 480, 510, 546, 493, 513, 508, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 493, 542, 493, 499, 531, 505, 517, 487, 500, 498, 475, 572, 524, 522, 528, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 537, 493, 516, 482, 509, 538, 496, 494, 520, 507, 503, 493, 538, 501, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 528, 486, 517, 519, 477, 493, 516, 519, 540, 506, 513, 532, 502, 512, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 489, 493, 548, 492, 512, 495, 530, 538, 515, 509, 480, 508, 503, 527, 473, 25, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 510, 509, 550, 469, 495, 518, 547, 538, 463, 554, 554, 497, 512, 510, 530, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 468, 495, 520, 482, 494, 499, 495, 509, 530, 504, 518, 533, 502, 516, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 497, 543, 546, 524, 532, 527, 511, 518, 499, 516, 489, 523, 530, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8244, 8163, 8154, 8166, 8315, 8046, 8307] avg=8192.0000 max=8315 max_rank=5 min=8046 min_rank=6 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[484, 529, 489, 504, 544, 488, 487, 501, 496, 532, 495, 501, 520, 484, 479, 512, 32, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 463, 522, 496, 501, 537, 509, 482, 522, 482, 507, 512, 492, 535, 536, 518, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 493, 545, 507, 557, 487, 524, 488, 472, 542, 515, 523, 561, 495, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 507, 464, 529, 500, 516, 499, 549, 518, 529, 519, 491, 502, 516, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 525, 498, 519, 505, 521, 503, 525, 518, 550, 536, 508, 543, 505, 555, 446, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 478, 503, 508, 520, 549, 524, 503, 508, 537, 565, 529, 493, 456, 525, 536, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 524, 507, 495, 535, 526, 535, 460, 474, 528, 554, 515, 519, 485, 484, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 560, 528, 489, 518, 518, 551, 482, 511, 488, 489, 574, 485, 520, 487, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8080, 8140, 8247, 8156, 8342, 8246, 8086, 8239] avg=8192.0000 max=8342 max_rank=4 min=8080 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[521, 493, 525, 520, 505, 517, 506, 495, 508, 508, 502, 524, 520, 494, 525, 486, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 505, 529, 529, 525, 503, 501, 493, 544, 526, 484, 482, 506, 518, 490, 547, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 509, 507, 496, 483, 504, 469, 493, 466, 505, 517, 520, 554, 521, 502, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 480, 530, 517, 540, 556, 562, 466, 512, 513, 477, 518, 508, 521, 540, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 507, 519, 530, 529, 484, 497, 515, 511, 491, 532, 542, 533, 507, 525, 519, 27, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 544, 507, 510, 488, 506, 497, 461, 510, 501, 502, 522, 528, 486, 484, 490, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 541, 499, 529, 554, 511, 499, 515, 489, 516, 498, 537, 538, 545, 537, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 480, 486, 499, 520, 506, 494, 537, 483, 518, 461, 498, 521, 519, 488, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8225, 8077, 8279, 8333, 8071, 8303, 8061] avg=8192.0000 max=8333 max_rank=4 min=8061 min_rank=7 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[504, 514, 516, 485, 506, 506, 494, 465, 498, 523, 489, 483, 502, 514, 490, 491, 33, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 531, 505, 533, 496, 486, 470, 541, 484, 524, 495, 489, 503, 532, 531, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 566, 534, 490, 549, 529, 531, 485, 473, 518, 506, 522, 491, 543, 465, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 529, 486, 505, 561, 515, 493, 542, 500, 506, 511, 520, 538, 526, 516, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 482, 532, 481, 490, 499, 515, 537, 532, 489, 508, 494, 497, 482, 510, 515, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 493, 509, 535, 506, 505, 537, 525, 542, 529, 487, 503, 478, 497, 501, 518, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 511, 529, 504, 509, 499, 501, 540, 463, 542, 557, 517, 508, 505, 535, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 504, 526, 527, 524, 557, 494, 546, 530, 510, 503, 523, 512, 512, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8016, 8166, 8186, 8321, 8092, 8223, 8217, 8315] avg=8192.0000 max=8321 max_rank=3 min=8016 min_rank=0 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[502, 530, 523, 526, 517, 492, 469, 541, 503, 502, 520, 521, 504, 473, 512, 542, 543, 542, 497, 525, 502, 483, 528, 500, 514, 496, 517, 517, 553, 519, 493, 524, 530, 519, 506, 537, 486, 508, 562, 521, 550, 532, 484, 495, 575, 504, 523, 504, 519, 532, 468, 547, 514, 508, 505, 524, 495, 545, 475, 501, 517, 481, 478, 540, 491, 514, 518, 512, 541, 513, 547, 540, 471, 546, 541, 482, 515, 498, 553, 497, 509, 495, 490, 490, 532, 539, 491, 520, 491, 524, 537, 501, 500, 462, 475, 498, 507, 509, 506, 491, 522, 503, 491, 517, 531, 494, 517, 492, 490, 522, 540, 550, 497, 541, 500, 496, 492, 547, 531, 492, 456, 466, 497, 507, 512, 538, 513, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[504, 503, 500, 542, 518, 515, 488, 516, 508, 493, 532, 514, 515, 524, 494, 478, 518, 525, 514, 536, 512, 560, 539, 513, 527, 497, 537, 537, 528, 532, 521, 543, 541, 525, 557, 522, 498, 531, 513, 532, 491, 541, 522, 493, 504, 493, 517, 491, 519, 506, 509, 521, 503, 518, 508, 532, 512, 498, 480, 464, 501, 516, 526, 512, 501, 547, 529, 491, 502, 521, 492, 498, 485, 510, 519, 489, 519, 497, 520, 535, 480, 480, 512, 526, 514, 503, 469, 473, 520, 502, 537, 529, 516, 504, 525, 481, 516, 529, 493, 521, 532, 524, 513, 498, 503, 440, 484, 518, 548, 537, 456, 511, 543, 497, 483, 519, 515, 512, 502, 497, 481, 514, 522, 524, 544, 512, 529, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[535, 501, 481, 498, 519, 516, 499, 531, 526, 540, 542, 512, 514, 510, 558, 483, 530, 514, 527, 485, 501, 479, 500, 514, 515, 501, 521, 498, 508, 519, 489, 497, 528, 504, 537, 519, 562, 447, 537, 517, 500, 512, 499, 497, 503, 542, 542, 512, 486, 526, 524, 491, 493, 488, 506, 492, 513, 511, 532, 539, 518, 559, 471, 505, 497, 514, 515, 506, 513, 573, 497, 499, 495, 507, 517, 509, 537, 531, 521, 513, 547, 487, 530, 491, 503, 538, 522, 501, 501, 479, 530, 539, 509, 494, 542, 508, 542, 530, 517, 511, 505, 488, 458, 488, 503, 515, 525, 512, 523, 523, 531, 498, 507, 530, 487, 495, 498, 509, 524, 521, 513, 509, 472, 515, 483, 507, 529, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[493, 509, 551, 495, 545, 501, 504, 475, 508, 518, 489, 519, 505, 521, 551, 469, 522, 519, 526, 520, 454, 488, 511, 504, 508, 553, 537, 512, 495, 486, 499, 487, 506, 517, 543, 518, 509, 504, 475, 551, 499, 587, 521, 516, 477, 529, 493, 539, 515, 551, 503, 510, 488, 531, 532, 484, 509, 539, 500, 498, 501, 532, 533, 551, 511, 514, 528, 489, 517, 529, 525, 523, 469, 536, 521, 521, 478, 471, 492, 485, 505, 508, 497, 513, 463, 518, 495, 536, 473, 532, 492, 512, 524, 491, 517, 524, 545, 488, 497, 513, 523, 506, 527, 509, 471, 568, 540, 504, 509, 512, 553, 493, 489, 489, 528, 447, 519, 510, 556, 514, 517, 501, 545, 490, 533, 563, 508, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[505, 548, 508, 508, 513, 539, 521, 510, 496, 516, 513, 513, 498, 497, 477, 491, 501, 536, 483, 494, 509, 515, 501, 484, 485, 486, 542, 521, 511, 520, 476, 495, 493, 486, 550, 569, 501, 539, 528, 489, 496, 521, 493, 498, 523, 527, 523, 521, 482, 504, 573, 527, 568, 533, 513, 517, 499, 542, 532, 539, 550, 505, 516, 511, 534, 517, 487, 539, 533, 514, 522, 510, 509, 488, 477, 485, 513, 497, 501, 475, 505, 505, 533, 472, 496, 501, 489, 491, 498, 523, 524, 526, 521, 551, 525, 522, 513, 550, 549, 539, 533, 510, 515, 485, 506, 519, 534, 503, 545, 468, 543, 501, 501, 529, 505, 491, 484, 493, 503, 497, 500, 523, 467, 499, 508, 524, 505, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[496, 500, 522, 522, 500, 519, 505, 510, 499, 537, 523, 531, 505, 510, 516, 539, 541, 512, 546, 525, 490, 489, 489, 526, 496, 514, 520, 475, 462, 490, 461, 525, 498, 506, 505, 512, 491, 497, 519, 481, 514, 559, 532, 552, 511, 498, 497, 509, 488, 493, 523, 490, 508, 560, 506, 516, 527, 525, 508, 504, 514, 526, 530, 549, 495, 498, 502, 538, 522, 513, 518, 543, 527, 500, 509, 537, 493, 499, 544, 522, 493, 526, 542, 520, 552, 520, 517, 501, 498, 484, 508, 505, 497, 533, 523, 531, 535, 559, 486, 517, 540, 531, 493, 539, 496, 511, 500, 505, 531, 484, 498, 485, 494, 500, 515, 484, 553, 497, 483, 498, 502, 501, 499, 540, 453, 500, 550, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[536, 519, 543, 492, 566, 495, 507, 499, 488, 510, 513, 496, 530, 482, 515, 509, 495, 512, 511, 522, 525, 519, 507, 512, 481, 503, 516, 531, 569, 535, 535, 501, 534, 466, 478, 553, 512, 504, 522, 487, 481, 494, 480, 509, 499, 506, 489, 546, 518, 498, 513, 513, 510, 501, 531, 495, 531, 528, 493, 539, 490, 551, 531, 501, 503, 522, 512, 490, 520, 513, 521, 518, 498, 505, 528, 521, 526, 531, 491, 523, 502, 515, 492, 535, 526, 502, 562, 514, 510, 498, 505, 503, 520, 531, 516, 511, 524, 512, 543, 450, 538, 511, 509, 532, 499, 533, 517, 549, 538, 476, 480, 501, 465, 532, 522, 524, 493, 483, 471, 523, 466, 511, 497, 503, 537, 508, 533, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[498, 518, 536, 522, 566, 502, 521, 501, 506, 512, 537, 492, 485, 512, 563, 503, 519, 519, 494, 494, 464, 506, 507, 552, 515, 539, 482, 503, 538, 493, 513, 534, 486, 492, 483, 513, 475, 516, 515, 503, 489, 521, 559, 518, 533, 535, 498, 502, 543, 476, 539, 545, 465, 504, 491, 503, 519, 516, 484, 518, 498, 535, 537, 529, 536, 478, 487, 514, 525, 535, 534, 502, 505, 487, 521, 515, 518, 550, 508, 501, 469, 504, 490, 518, 489, 550, 520, 524, 508, 547, 540, 534, 532, 511, 502, 517, 537, 530, 512, 487, 528, 508, 512, 480, 497, 506, 531, 525, 531, 515, 529, 488, 531, 515, 519, 492, 484, 480, 509, 542, 512, 488, 479, 488, 526, 465, 525, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[504, 503, 500, 542, 511, 515, 488, 516, 508, 493, 532, 514, 515, 524, 494, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 514, 536, 512, 560, 539, 513, 527, 497, 537, 537, 528, 532, 521, 543, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 525, 557, 522, 498, 531, 513, 532, 491, 510, 522, 493, 504, 493, 517, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 509, 521, 503, 518, 508, 532, 512, 498, 480, 464, 501, 481, 526, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 547, 529, 491, 502, 515, 492, 498, 485, 510, 519, 489, 519, 497, 520, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 480, 512, 526, 514, 503, 469, 473, 520, 502, 537, 529, 516, 504, 525, 481, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 529, 493, 521, 514, 524, 513, 498, 503, 440, 484, 518, 548, 537, 456, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 497, 483, 519, 515, 512, 502, 497, 481, 514, 522, 524, 544, 512, 529, 514, 35, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8446, 8240, 8090, 8149, 8090, 8105, 8279] avg=8192.0000 max=8446 max_rank=1 min=8090 min_rank=3 imbalance_ratio=1.031006 max_load_reduction=57090 max_load_reduction_pct=0.871124 moved_to_echo=32623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 530, 523, 526, 510, 492, 469, 541, 503, 502, 520, 521, 504, 473, 512, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 542, 497, 525, 502, 483, 528, 500, 514, 496, 517, 517, 553, 519, 493, 524, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 519, 506, 537, 486, 508, 562, 521, 550, 501, 484, 495, 575, 504, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 532, 468, 547, 514, 508, 505, 524, 495, 545, 475, 501, 517, 448, 478, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 514, 518, 512, 541, 507, 547, 540, 471, 546, 541, 482, 515, 498, 553, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 490, 490, 532, 539, 491, 520, 491, 524, 537, 501, 500, 462, 475, 498, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 506, 491, 504, 503, 491, 517, 531, 494, 517, 492, 490, 522, 540, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 541, 500, 496, 492, 547, 531, 492, 456, 466, 497, 507, 512, 538, 513, 521, 33, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8260, 8305, 8116, 8273, 8073, 8164, 8175] avg=8192.0000 max=8305 max_rank=2 min=8073 min_rank=5 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[535, 501, 481, 498, 512, 516, 499, 531, 526, 540, 542, 512, 514, 510, 558, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 527, 485, 501, 479, 500, 514, 515, 501, 521, 498, 508, 519, 489, 497, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 504, 537, 519, 562, 447, 537, 517, 500, 483, 499, 497, 503, 542, 542, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 526, 524, 491, 493, 488, 506, 492, 513, 511, 532, 539, 518, 521, 471, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 515, 506, 513, 567, 497, 499, 495, 507, 517, 509, 537, 531, 521, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 487, 530, 491, 503, 538, 522, 501, 501, 479, 530, 539, 509, 494, 542, 508, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 530, 517, 511, 489, 488, 458, 488, 503, 515, 525, 512, 523, 523, 531, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 487, 495, 498, 509, 524, 521, 513, 509, 472, 515, 483, 507, 529, 528, 38, 29, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8105, 8229, 8116, 8238, 8239, 8153, 8198] avg=8192.0000 max=8258 max_rank=0 min=8105 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[496, 500, 522, 522, 488, 519, 505, 510, 499, 537, 523, 531, 505, 510, 516, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 512, 546, 525, 490, 489, 489, 526, 496, 514, 520, 475, 462, 490, 461, 525, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 506, 505, 512, 491, 497, 519, 481, 514, 523, 532, 552, 511, 498, 497, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 493, 523, 490, 508, 560, 506, 516, 527, 525, 508, 504, 514, 486, 530, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 498, 502, 538, 522, 505, 518, 543, 527, 500, 509, 537, 493, 499, 544, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 526, 542, 520, 552, 520, 517, 501, 498, 484, 508, 505, 497, 533, 523, 531, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 559, 486, 517, 517, 531, 493, 539, 496, 511, 500, 505, 531, 484, 498, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 500, 515, 484, 553, 497, 483, 498, 502, 501, 499, 540, 453, 500, 550, 504, 40, 36, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8071, 8145, 8227, 8252, 8276, 8187, 8156] avg=8192.0000 max=8276 max_rank=5 min=8071 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[493, 509, 551, 495, 537, 501, 504, 475, 508, 518, 489, 519, 505, 521, 551, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 519, 526, 520, 454, 488, 511, 504, 508, 553, 537, 512, 495, 486, 499, 487, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 543, 518, 509, 504, 475, 551, 499, 553, 521, 516, 477, 529, 493, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 551, 503, 510, 488, 531, 532, 484, 509, 539, 500, 498, 501, 496, 533, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 528, 489, 517, 523, 525, 523, 469, 536, 521, 521, 478, 471, 492, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 497, 513, 463, 518, 495, 536, 473, 532, 492, 512, 524, 491, 517, 524, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 488, 497, 513, 505, 506, 527, 509, 471, 568, 540, 504, 509, 512, 553, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 489, 528, 447, 519, 510, 556, 514, 517, 501, 545, 490, 533, 563, 508, 525, 36, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8128, 8250, 8241, 8103, 8120, 8240, 8309] avg=8192.0000 max=8309 max_rank=7 min=8103 min_rank=4 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 548, 508, 508, 506, 539, 521, 510, 496, 516, 513, 513, 498, 497, 477, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 483, 494, 509, 515, 501, 484, 485, 486, 542, 521, 511, 520, 476, 495, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 486, 550, 569, 501, 539, 528, 489, 496, 491, 493, 498, 523, 527, 523, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 504, 573, 527, 568, 533, 513, 517, 499, 542, 532, 539, 550, 470, 516, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 487, 539, 533, 508, 522, 510, 509, 488, 477, 485, 513, 497, 501, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 505, 533, 472, 496, 501, 489, 491, 498, 523, 524, 526, 521, 551, 525, 522, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 550, 549, 539, 515, 510, 515, 485, 506, 519, 534, 503, 545, 468, 543, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 529, 505, 491, 484, 493, 503, 497, 500, 523, 467, 499, 508, 524, 505, 531, 35, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8066, 8227, 8376, 8095, 8201, 8295, 8130] avg=8192.0000 max=8376 max_rank=3 min=8066 min_rank=1 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 518, 536, 522, 558, 502, 521, 501, 506, 512, 537, 492, 485, 512, 563, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 519, 494, 494, 464, 506, 507, 552, 515, 539, 482, 503, 538, 493, 513, 534, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 492, 483, 513, 475, 516, 515, 503, 489, 491, 559, 518, 533, 535, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 476, 539, 545, 465, 504, 491, 503, 519, 516, 484, 518, 498, 498, 537, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 478, 487, 514, 525, 529, 534, 502, 505, 487, 521, 515, 518, 550, 508, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 504, 490, 518, 489, 550, 520, 524, 508, 547, 540, 534, 532, 511, 502, 517, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 512, 487, 510, 508, 512, 480, 497, 506, 531, 525, 531, 515, 529, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 515, 519, 492, 484, 480, 509, 542, 512, 488, 479, 488, 526, 465, 525, 508, 37, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8179, 8108, 8165, 8210, 8275, 8198, 8135] avg=8192.0000 max=8275 max_rank=5 min=8108 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[536, 519, 543, 492, 558, 495, 507, 499, 488, 510, 513, 496, 530, 482, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 512, 511, 522, 525, 519, 507, 512, 481, 503, 516, 531, 569, 535, 535, 501, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 466, 478, 553, 512, 504, 522, 487, 481, 466, 480, 509, 499, 506, 489, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 498, 513, 513, 510, 501, 531, 495, 531, 528, 493, 539, 490, 513, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 522, 512, 490, 520, 507, 521, 518, 498, 505, 528, 521, 526, 531, 491, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 492, 535, 526, 502, 562, 514, 510, 498, 505, 503, 520, 531, 516, 511, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 543, 450, 520, 511, 509, 532, 499, 533, 517, 549, 538, 476, 480, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 532, 522, 524, 493, 483, 471, 523, 466, 511, 497, 503, 537, 508, 533, 515, 38, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8281, 8032, 8205, 8216, 8262, 8194, 8154] avg=8192.0000 max=8281 max_rank=1 min=8032 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[543, 501, 504, 483, 511, 524, 499, 483, 498, 472, 501, 502, 569, 531, 521, 515, 505, 526, 540, 478, 510, 515, 506, 501, 547, 473, 488, 495, 502, 496, 561, 538, 503, 524, 458, 552, 514, 535, 513, 536, 483, 525, 483, 538, 470, 519, 527, 492, 477, 487, 512, 512, 506, 495, 520, 515, 502, 495, 534, 482, 531, 499, 512, 522, 500, 471, 554, 554, 514, 514, 517, 518, 507, 510, 508, 531, 504, 541, 527, 502, 510, 498, 488, 515, 537, 540, 526, 527, 511, 545, 497, 513, 503, 538, 525, 508, 519, 492, 494, 509, 500, 525, 579, 508, 526, 550, 567, 512, 536, 511, 502, 483, 539, 508, 520, 507, 507, 512, 502, 447, 506, 472, 528, 501, 536, 500, 514, 460] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[506, 523, 505, 506, 510, 509, 495, 483, 466, 506, 522, 569, 511, 522, 508, 489, 485, 540, 508, 503, 485, 497, 520, 510, 497, 496, 529, 474, 533, 490, 507, 542, 500, 514, 545, 551, 500, 507, 494, 508, 529, 529, 524, 507, 523, 474, 480, 475, 487, 501, 543, 534, 514, 537, 495, 590, 500, 513, 491, 520, 516, 521, 508, 523, 517, 504, 482, 509, 489, 508, 503, 524, 526, 489, 504, 534, 515, 516, 533, 525, 510, 537, 502, 490, 521, 535, 516, 484, 520, 536, 513, 512, 522, 499, 513, 527, 522, 481, 525, 510, 533, 486, 483, 501, 546, 518, 546, 484, 521, 550, 520, 518, 512, 505, 511, 515, 508, 532, 497, 522, 511, 462, 520, 507, 543, 490, 525, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[554, 532, 500, 499, 488, 547, 525, 528, 556, 469, 512, 526, 486, 520, 469, 479, 537, 490, 509, 559, 523, 573, 507, 472, 517, 523, 477, 537, 480, 516, 521, 491, 524, 558, 471, 529, 497, 500, 464, 508, 467, 497, 514, 512, 516, 512, 499, 455, 537, 507, 535, 538, 544, 514, 488, 491, 544, 478, 500, 487, 511, 521, 495, 512, 503, 501, 512, 502, 483, 530, 499, 497, 517, 514, 499, 529, 535, 535, 510, 545, 480, 535, 521, 528, 496, 507, 505, 486, 478, 493, 562, 516, 528, 497, 538, 519, 522, 508, 517, 588, 502, 512, 496, 516, 526, 499, 495, 513, 536, 515, 523, 507, 506, 495, 498, 517, 553, 488, 485, 496, 522, 516, 547, 504, 492, 503, 528, 564] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[490, 498, 523, 522, 531, 504, 516, 489, 497, 526, 523, 511, 468, 499, 533, 508, 514, 530, 497, 492, 516, 514, 486, 482, 526, 510, 549, 500, 482, 529, 530, 517, 509, 484, 482, 522, 491, 544, 487, 527, 477, 512, 527, 502, 504, 503, 541, 527, 547, 527, 565, 505, 495, 470, 531, 505, 502, 520, 502, 465, 531, 507, 526, 512, 496, 497, 509, 538, 519, 555, 497, 503, 537, 469, 533, 504, 484, 471, 529, 511, 507, 484, 500, 522, 540, 499, 505, 543, 513, 553, 529, 501, 509, 567, 503, 516, 513, 546, 504, 496, 527, 519, 504, 535, 503, 528, 540, 493, 505, 514, 480, 531, 480, 531, 497, 512, 496, 511, 530, 527, 516, 508, 480, 500, 544, 541, 509, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[516, 505, 500, 491, 510, 448, 476, 480, 453, 516, 568, 517, 518, 481, 489, 489, 488, 532, 547, 511, 526, 509, 533, 491, 555, 502, 492, 527, 525, 523, 530, 504, 486, 481, 508, 535, 566, 476, 495, 534, 539, 508, 501, 500, 581, 502, 521, 482, 511, 513, 496, 515, 516, 515, 500, 493, 525, 518, 507, 538, 496, 514, 515, 541, 499, 489, 491, 486, 505, 480, 513, 499, 512, 518, 493, 521, 467, 525, 516, 504, 518, 533, 518, 526, 510, 510, 525, 483, 508, 492, 523, 544, 484, 545, 565, 564, 511, 531, 508, 501, 527, 525, 489, 552, 516, 492, 526, 532, 503, 496, 472, 514, 505, 495, 563, 510, 516, 495, 528, 518, 516, 540, 487, 528, 523, 501, 512, 558] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[562, 497, 509, 512, 529, 488, 429, 521, 538, 533, 549, 501, 507, 534, 475, 480, 519, 490, 498, 519, 503, 520, 550, 520, 497, 542, 539, 518, 552, 532, 519, 494, 503, 556, 549, 510, 517, 490, 538, 493, 536, 519, 509, 506, 504, 490, 508, 478, 496, 499, 482, 460, 515, 536, 526, 502, 533, 530, 513, 545, 483, 531, 477, 511, 506, 487, 517, 530, 493, 529, 512, 505, 486, 537, 524, 516, 506, 488, 509, 519, 511, 503, 550, 488, 528, 499, 536, 529, 464, 501, 514, 486, 518, 511, 502, 506, 510, 499, 494, 503, 511, 536, 528, 509, 507, 517, 522, 522, 495, 532, 496, 500, 509, 483, 485, 501, 519, 497, 551, 521, 515, 519, 541, 515, 540, 520, 540, 468] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[499, 505, 502, 507, 487, 503, 487, 538, 526, 514, 550, 508, 534, 517, 478, 498, 536, 545, 531, 518, 505, 487, 556, 524, 504, 532, 516, 568, 514, 507, 469, 535, 519, 506, 495, 517, 446, 502, 513, 487, 545, 514, 526, 518, 501, 529, 514, 566, 513, 512, 546, 522, 526, 489, 530, 512, 525, 509, 491, 495, 463, 526, 526, 536, 496, 533, 483, 514, 540, 474, 482, 539, 481, 520, 532, 516, 498, 505, 495, 517, 525, 504, 522, 531, 520, 519, 482, 509, 536, 511, 531, 467, 504, 459, 511, 558, 532, 452, 516, 549, 502, 491, 504, 506, 525, 507, 551, 454, 490, 512, 511, 514, 534, 495, 504, 506, 522, 521, 507, 503, 476, 512, 504, 502, 568, 493, 512, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[495, 481, 496, 459, 530, 500, 504, 514, 528, 509, 499, 494, 523, 511, 515, 555, 484, 542, 474, 517, 590, 510, 497, 519, 494, 473, 538, 495, 503, 556, 508, 500, 533, 506, 550, 540, 515, 498, 490, 475, 542, 494, 516, 507, 504, 511, 508, 536, 544, 527, 509, 501, 501, 498, 541, 531, 510, 498, 550, 517, 493, 508, 539, 487, 469, 532, 525, 498, 562, 489, 502, 489, 537, 471, 503, 519, 536, 509, 550, 529, 521, 511, 564, 503, 555, 494, 500, 481, 510, 516, 489, 518, 547, 490, 443, 510, 537, 519, 485, 524, 522, 522, 534, 493, 521, 487, 504, 528, 511, 524, 514, 505, 516, 486, 514, 561, 498, 538, 504, 472, 536, 470, 523, 509, 510, 490, 503, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[506, 523, 505, 506, 510, 509, 495, 483, 466, 506, 522, 569, 511, 522, 508, 489, 41, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 499, 508, 503, 485, 497, 520, 510, 497, 496, 529, 474, 533, 490, 507, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 545, 551, 500, 507, 494, 508, 529, 529, 524, 507, 523, 474, 480, 475, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 501, 531, 534, 514, 537, 495, 590, 500, 513, 491, 520, 516, 521, 508, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 504, 482, 509, 489, 508, 503, 524, 526, 489, 504, 534, 515, 516, 533, 525, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 537, 502, 490, 521, 535, 516, 484, 520, 536, 513, 512, 522, 499, 513, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 525, 510, 533, 486, 483, 501, 546, 518, 514, 484, 521, 550, 520, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 511, 515, 508, 532, 497, 522, 511, 462, 520, 507, 543, 490, 525, 518, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8075, 8184, 8281, 8203, 8201, 8212, 8183] avg=8192.0000 max=8281 max_rank=3 min=8075 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[516, 505, 500, 491, 510, 448, 476, 480, 453, 516, 568, 517, 518, 481, 489, 489, 40, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 492, 547, 511, 526, 509, 533, 491, 555, 502, 492, 527, 525, 523, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 481, 508, 535, 566, 476, 495, 534, 539, 508, 501, 500, 581, 502, 521, 482, 14, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 485, 515, 516, 515, 500, 493, 525, 518, 507, 538, 496, 514, 515, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 489, 491, 486, 505, 480, 513, 499, 512, 518, 493, 521, 467, 525, 516, 504, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 533, 518, 526, 510, 510, 525, 483, 508, 492, 523, 544, 484, 545, 565, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 531, 508, 501, 527, 525, 489, 552, 516, 492, 495, 532, 503, 496, 472, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 563, 510, 516, 495, 528, 518, 516, 540, 487, 528, 523, 501, 512, 558, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8255, 8240, 8202, 8041, 8309, 8164, 8300] avg=8192.0000 max=8309 max_rank=5 min=8025 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[490, 498, 523, 522, 531, 504, 516, 489, 497, 526, 523, 511, 468, 499, 533, 508, 40, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 490, 497, 492, 516, 514, 486, 482, 526, 510, 549, 500, 482, 529, 530, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 484, 482, 522, 491, 544, 487, 527, 477, 512, 527, 502, 504, 503, 541, 527, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 527, 553, 505, 495, 470, 531, 505, 502, 520, 502, 465, 531, 507, 526, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 509, 538, 519, 555, 497, 503, 537, 469, 533, 504, 484, 471, 529, 511, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 484, 500, 522, 540, 499, 505, 543, 513, 553, 529, 501, 509, 567, 503, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 546, 504, 496, 527, 519, 504, 535, 503, 528, 508, 493, 505, 514, 480, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 531, 497, 512, 496, 511, 530, 527, 516, 508, 480, 500, 544, 541, 509, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8134, 8163, 8198, 8177, 8255, 8206, 8199] avg=8192.0000 max=8255 max_rank=5 min=8134 min_rank=1 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[543, 501, 504, 483, 511, 524, 499, 483, 498, 472, 501, 502, 569, 531, 521, 515, 44, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 482, 540, 478, 510, 515, 506, 501, 547, 473, 488, 495, 502, 496, 561, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 458, 552, 514, 535, 513, 536, 483, 525, 483, 538, 470, 519, 527, 492, 19, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 487, 494, 512, 506, 495, 520, 515, 502, 495, 534, 482, 531, 499, 512, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 471, 554, 554, 514, 514, 517, 518, 507, 510, 508, 531, 504, 541, 527, 502, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 498, 488, 515, 537, 540, 526, 527, 511, 545, 497, 513, 503, 538, 525, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 492, 494, 509, 500, 525, 579, 508, 526, 550, 526, 512, 536, 511, 502, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 508, 520, 507, 507, 512, 502, 447, 506, 472, 528, 501, 536, 500, 514, 460, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8137, 8203, 8083, 8303, 8240, 8272, 8068] avg=8192.0000 max=8303 max_rank=4 min=8068 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[554, 532, 500, 499, 488, 547, 525, 528, 556, 469, 512, 526, 486, 520, 469, 479, 37, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 453, 509, 559, 523, 573, 507, 472, 517, 523, 477, 537, 480, 516, 521, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 558, 471, 529, 497, 500, 464, 508, 467, 497, 514, 512, 516, 512, 499, 455, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 524, 538, 544, 514, 488, 491, 544, 478, 500, 487, 511, 521, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 501, 512, 502, 483, 530, 499, 497, 517, 514, 499, 529, 535, 535, 510, 545, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 535, 521, 528, 496, 507, 505, 486, 478, 493, 562, 516, 528, 497, 538, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 517, 588, 502, 512, 496, 516, 526, 499, 466, 513, 536, 515, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 498, 517, 553, 488, 485, 496, 522, 516, 547, 504, 492, 503, 528, 564, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8195, 8046, 8191, 8233, 8153, 8246, 8219] avg=8192.0000 max=8253 max_rank=0 min=8046 min_rank=2 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[499, 505, 502, 507, 487, 503, 487, 538, 526, 514, 550, 508, 534, 517, 478, 498, 41, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 504, 531, 518, 505, 487, 556, 524, 504, 532, 516, 568, 514, 507, 469, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 495, 517, 446, 502, 513, 487, 545, 514, 526, 518, 501, 529, 514, 566, 14, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 512, 534, 522, 526, 489, 530, 512, 525, 509, 491, 495, 463, 526, 526, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 533, 483, 514, 540, 474, 482, 539, 481, 520, 532, 516, 498, 505, 495, 517, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 504, 522, 531, 520, 519, 482, 509, 536, 511, 531, 467, 504, 459, 511, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 452, 516, 549, 502, 491, 504, 506, 525, 507, 519, 454, 490, 512, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 495, 504, 506, 522, 521, 507, 503, 476, 512, 504, 502, 568, 493, 512, 528, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8306, 8223, 8209, 8150, 8150, 8084, 8192] avg=8192.0000 max=8306 max_rank=1 min=8084 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 481, 496, 459, 530, 500, 504, 514, 528, 509, 499, 494, 523, 511, 515, 555, 41, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 474, 517, 590, 510, 497, 519, 494, 473, 538, 495, 503, 556, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 550, 540, 515, 498, 490, 475, 542, 494, 516, 507, 504, 511, 508, 536, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 527, 498, 501, 501, 498, 541, 531, 510, 498, 550, 517, 493, 508, 539, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 532, 525, 498, 562, 489, 502, 489, 537, 471, 503, 519, 536, 509, 550, 529, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 564, 503, 555, 494, 500, 481, 510, 516, 489, 518, 547, 490, 443, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 519, 485, 524, 522, 522, 534, 493, 521, 487, 475, 528, 511, 524, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 486, 514, 561, 498, 538, 504, 472, 536, 470, 523, 509, 510, 490, 503, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8159, 8248, 8243, 8242, 8117, 8201, 8147] avg=8192.0000 max=8248 max_rank=2 min=8117 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[562, 497, 509, 512, 529, 488, 429, 521, 538, 533, 549, 501, 507, 534, 475, 480, 37, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 453, 498, 519, 503, 520, 550, 520, 497, 542, 539, 518, 552, 532, 519, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 556, 549, 510, 517, 490, 538, 493, 536, 519, 509, 506, 504, 490, 508, 478, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 499, 472, 460, 515, 536, 526, 502, 533, 530, 513, 545, 483, 531, 477, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 487, 517, 530, 493, 529, 512, 505, 486, 537, 524, 516, 506, 488, 509, 519, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 503, 550, 488, 528, 499, 536, 529, 464, 501, 514, 486, 518, 511, 502, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 499, 494, 503, 511, 536, 528, 509, 507, 517, 492, 522, 495, 532, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 483, 485, 501, 519, 497, 551, 521, 515, 519, 541, 515, 540, 520, 540, 468, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8275, 8229, 8129, 8187, 8111, 8151, 8228] avg=8192.0000 max=8275 max_rank=1 min=8111 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4352, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[508, 509, 531, 480, 524, 534, 496, 519, 511, 531, 499, 477, 509, 522, 494, 524, 510, 514, 531, 533, 505, 530, 569, 551, 493, 516, 530, 522, 502, 480, 476, 538, 517, 498, 503, 475, 492, 558, 494, 574, 473, 491, 533, 510, 550, 508, 527, 480, 502, 512, 515, 534, 506, 497, 494, 558, 518, 515, 522, 524, 505, 505, 531, 514, 552, 527, 504, 492, 500, 486, 522, 490, 504, 499, 510, 519, 493, 507, 534, 519, 495, 507, 520, 511, 548, 493, 503, 496, 492, 529, 477, 515, 493, 528, 492, 504, 541, 495, 522, 521, 484, 507, 517, 520, 533, 506, 537, 523, 521, 502, 542, 502, 543, 527, 517, 504, 486, 520, 481, 472, 509, 492, 494, 454, 520, 525, 526, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[515, 506, 530, 524, 511, 526, 507, 527, 524, 525, 516, 491, 527, 530, 502, 556, 520, 555, 491, 515, 519, 538, 473, 487, 539, 494, 510, 514, 505, 525, 557, 492, 451, 540, 466, 549, 510, 485, 528, 522, 492, 511, 527, 509, 524, 545, 506, 555, 491, 511, 504, 501, 499, 499, 485, 496, 507, 502, 499, 544, 544, 530, 499, 500, 486, 483, 495, 521, 533, 492, 487, 516, 516, 505, 497, 501, 536, 485, 477, 547, 508, 507, 528, 528, 510, 524, 500, 507, 507, 531, 508, 473, 482, 513, 472, 516, 516, 519, 528, 502, 530, 523, 524, 537, 510, 513, 524, 544, 505, 544, 507, 482, 510, 499, 509, 531, 517, 507, 486, 510, 510, 498, 459, 520, 553, 518, 516, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[517, 452, 523, 537, 498, 478, 501, 484, 483, 513, 506, 508, 493, 499, 532, 517, 526, 504, 535, 486, 503, 506, 527, 576, 502, 546, 511, 493, 484, 529, 491, 511, 499, 535, 516, 513, 545, 514, 475, 509, 552, 484, 558, 462, 503, 499, 519, 540, 506, 511, 510, 497, 517, 520, 506, 513, 503, 517, 505, 500, 529, 502, 493, 496, 510, 505, 555, 515, 500, 520, 537, 508, 526, 516, 534, 487, 504, 501, 494, 542, 525, 518, 492, 497, 540, 544, 533, 479, 502, 474, 535, 545, 503, 525, 506, 526, 514, 481, 511, 516, 500, 492, 502, 540, 530, 512, 503, 536, 501, 497, 530, 504, 522, 528, 500, 465, 493, 530, 547, 507, 514, 515, 523, 508, 531, 500, 536, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[536, 513, 525, 522, 513, 455, 517, 516, 468, 505, 509, 523, 513, 509, 494, 503, 525, 472, 559, 517, 518, 490, 520, 495, 497, 494, 506, 500, 522, 515, 542, 500, 511, 527, 535, 511, 507, 485, 485, 557, 518, 515, 489, 485, 525, 482, 513, 523, 536, 492, 519, 502, 542, 530, 495, 489, 524, 533, 508, 510, 497, 544, 512, 502, 490, 550, 504, 505, 531, 554, 488, 533, 502, 480, 523, 546, 527, 533, 542, 484, 517, 539, 532, 526, 490, 535, 470, 534, 525, 485, 503, 488, 498, 536, 510, 555, 531, 506, 511, 497, 515, 505, 523, 532, 507, 503, 537, 514, 534, 551, 463, 490, 523, 497, 508, 463, 535, 546, 477, 517, 539, 493, 483, 529, 447, 521, 508, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[519, 517, 492, 521, 562, 515, 507, 518, 476, 522, 525, 534, 508, 509, 537, 487, 562, 543, 512, 530, 499, 519, 513, 496, 502, 532, 493, 509, 510, 519, 513, 522, 509, 536, 557, 474, 504, 516, 537, 503, 467, 502, 472, 515, 532, 557, 520, 492, 509, 473, 495, 457, 516, 488, 537, 554, 529, 516, 536, 518, 517, 518, 509, 521, 497, 519, 467, 519, 552, 504, 490, 491, 496, 501, 500, 534, 500, 527, 468, 484, 510, 521, 526, 530, 529, 537, 507, 521, 499, 524, 560, 573, 503, 483, 489, 533, 501, 528, 520, 514, 516, 491, 503, 489, 506, 500, 524, 552, 498, 502, 533, 498, 515, 509, 513, 478, 536, 525, 521, 456, 508, 550, 502, 481, 496, 491, 507, 470] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[528, 506, 578, 548, 509, 501, 528, 527, 543, 476, 574, 484, 519, 471, 509, 552, 554, 516, 509, 496, 477, 527, 486, 475, 520, 523, 535, 528, 507, 484, 542, 493, 523, 524, 533, 472, 516, 505, 498, 493, 480, 507, 495, 514, 505, 496, 494, 493, 496, 513, 504, 524, 537, 528, 506, 532, 500, 492, 462, 551, 529, 538, 539, 477, 528, 490, 456, 487, 558, 517, 492, 535, 530, 530, 480, 523, 507, 548, 482, 487, 501, 508, 477, 510, 499, 491, 515, 518, 495, 501, 496, 470, 533, 453, 512, 508, 519, 518, 535, 528, 515, 493, 510, 535, 527, 519, 510, 509, 532, 524, 531, 539, 519, 514, 547, 510, 524, 483, 526, 542, 527, 480, 544, 501, 517, 520, 501, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[536, 556, 457, 499, 573, 504, 522, 500, 523, 502, 509, 509, 490, 518, 565, 526, 510, 508, 484, 474, 506, 534, 494, 536, 487, 536, 551, 484, 555, 524, 530, 528, 485, 519, 519, 510, 530, 496, 552, 520, 510, 527, 543, 504, 482, 520, 533, 507, 507, 529, 512, 516, 547, 493, 508, 521, 527, 500, 489, 508, 513, 505, 513, 495, 505, 514, 512, 538, 536, 529, 509, 487, 525, 532, 534, 491, 521, 526, 520, 522, 508, 490, 494, 481, 508, 497, 467, 516, 511, 521, 513, 473, 466, 504, 538, 516, 531, 505, 525, 526, 520, 515, 541, 501, 478, 487, 514, 506, 457, 497, 507, 528, 509, 481, 492, 515, 576, 497, 516, 513, 511, 533, 497, 510, 499, 494, 509, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[494, 499, 499, 533, 559, 514, 537, 471, 555, 513, 498, 526, 538, 514, 478, 527, 495, 498, 506, 507, 513, 504, 502, 509, 528, 517, 477, 518, 519, 540, 480, 541, 490, 515, 490, 512, 512, 495, 535, 498, 545, 447, 502, 489, 527, 533, 501, 529, 472, 476, 573, 542, 546, 520, 540, 535, 475, 562, 472, 528, 509, 509, 499, 533, 550, 539, 519, 523, 504, 502, 525, 527, 502, 489, 469, 447, 509, 501, 508, 530, 519, 566, 538, 514, 483, 474, 539, 511, 538, 537, 497, 531, 536, 554, 505, 508, 464, 485, 493, 477, 527, 485, 526, 480, 517, 529, 508, 520, 518, 538, 507, 498, 508, 516, 501, 502, 484, 520, 540, 547, 538, 464, 517, 501, 449, 524, 527, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[515, 506, 530, 524, 481, 526, 507, 527, 524, 525, 516, 491, 527, 530, 502, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 555, 491, 515, 519, 538, 473, 487, 539, 494, 510, 514, 505, 525, 557, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 540, 466, 549, 510, 485, 528, 522, 492, 511, 527, 509, 524, 545, 506, 555, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 511, 504, 501, 490, 499, 485, 496, 507, 502, 499, 544, 544, 530, 499, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 483, 495, 521, 533, 492, 487, 516, 516, 505, 497, 501, 536, 485, 477, 547, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 528, 528, 510, 524, 500, 507, 507, 531, 508, 473, 482, 513, 472, 516, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 528, 502, 530, 523, 524, 537, 510, 513, 524, 524, 505, 544, 507, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 499, 509, 531, 517, 507, 486, 510, 510, 498, 459, 520, 553, 518, 516, 512, 30, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8205, 8236, 8102, 8089, 8132, 8288, 8197] avg=8192.0000 max=8288 max_rank=6 min=8089 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[517, 452, 523, 537, 468, 478, 501, 484, 483, 513, 506, 508, 493, 499, 532, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 504, 535, 486, 503, 506, 527, 576, 502, 546, 511, 493, 484, 529, 491, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 535, 516, 513, 545, 514, 475, 509, 552, 484, 558, 462, 503, 499, 519, 540, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 511, 510, 497, 507, 520, 506, 513, 503, 517, 505, 500, 529, 502, 493, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 505, 555, 515, 500, 520, 537, 508, 526, 516, 534, 487, 504, 501, 494, 542, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 492, 497, 540, 544, 533, 479, 502, 474, 535, 545, 503, 525, 506, 526, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 481, 511, 516, 500, 492, 502, 540, 530, 512, 503, 517, 501, 497, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 500, 465, 493, 530, 547, 507, 514, 515, 523, 508, 531, 500, 536, 531, 30, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8011, 8201, 8238, 8115, 8267, 8262, 8150, 8292] avg=8192.0000 max=8292 max_rank=7 min=8011 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[508, 509, 531, 480, 488, 534, 496, 519, 511, 531, 499, 477, 509, 522, 494, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 514, 531, 533, 505, 530, 569, 551, 493, 516, 530, 522, 502, 480, 476, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 498, 503, 475, 492, 558, 494, 574, 473, 491, 533, 510, 550, 508, 527, 480, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 512, 515, 534, 493, 497, 494, 558, 518, 515, 522, 524, 505, 505, 531, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 527, 504, 492, 500, 486, 522, 490, 504, 499, 510, 519, 493, 507, 534, 519, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 507, 520, 511, 548, 493, 503, 496, 492, 529, 477, 515, 493, 528, 492, 504, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 495, 522, 521, 484, 507, 517, 520, 533, 506, 537, 495, 521, 502, 542, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 527, 517, 504, 486, 520, 481, 472, 509, 492, 494, 454, 520, 525, 526, 529, 36, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8264, 8201, 8239, 8179, 8124, 8245, 8152] avg=8192.0000 max=8264 max_rank=1 min=8124 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[519, 517, 492, 521, 529, 515, 507, 518, 476, 522, 525, 534, 508, 509, 537, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 543, 512, 530, 499, 519, 513, 496, 502, 532, 493, 509, 510, 519, 513, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 536, 557, 474, 504, 516, 537, 503, 467, 502, 472, 515, 532, 557, 520, 492, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 473, 495, 457, 506, 488, 537, 554, 529, 516, 536, 518, 517, 518, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 519, 467, 519, 552, 504, 490, 491, 496, 501, 500, 534, 500, 527, 468, 484, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 526, 530, 529, 537, 507, 521, 499, 524, 560, 573, 503, 483, 489, 533, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 520, 514, 516, 491, 503, 489, 506, 500, 524, 532, 498, 502, 533, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 513, 478, 536, 525, 521, 456, 508, 550, 502, 481, 496, 491, 507, 470, 33, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8243, 8209, 8183, 8062, 8364, 8155, 8104] avg=8192.0000 max=8364 max_rank=5 min=8062 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[528, 506, 578, 548, 479, 501, 528, 527, 543, 476, 574, 484, 519, 471, 509, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 509, 496, 477, 527, 486, 475, 520, 523, 535, 528, 507, 484, 542, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 524, 533, 472, 516, 505, 498, 493, 480, 507, 495, 514, 505, 496, 494, 493, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 504, 524, 527, 528, 506, 532, 500, 492, 462, 551, 529, 538, 539, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 490, 456, 487, 558, 517, 492, 535, 530, 530, 480, 523, 507, 548, 482, 487, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 508, 477, 510, 499, 491, 515, 518, 495, 501, 496, 470, 533, 453, 512, 508, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 535, 528, 515, 493, 510, 535, 527, 519, 510, 490, 532, 524, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 514, 547, 510, 524, 483, 526, 542, 527, 480, 544, 501, 517, 520, 501, 499, 30, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8141, 8063, 8218, 8163, 8006, 8325, 8297] avg=8192.0000 max=8325 max_rank=6 min=8006 min_rank=5 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[536, 513, 525, 522, 482, 455, 517, 516, 468, 505, 509, 523, 513, 509, 494, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 472, 559, 517, 518, 490, 520, 495, 497, 494, 506, 500, 522, 515, 542, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 527, 535, 511, 507, 485, 485, 557, 518, 515, 489, 485, 525, 482, 513, 523, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 492, 519, 502, 532, 530, 495, 489, 524, 533, 508, 510, 497, 544, 512, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 550, 504, 505, 531, 554, 488, 533, 502, 480, 523, 546, 527, 533, 542, 484, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 539, 532, 526, 490, 535, 470, 534, 525, 485, 503, 488, 498, 536, 510, 555, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 506, 511, 497, 515, 505, 523, 532, 507, 503, 537, 495, 534, 551, 463, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 508, 463, 535, 546, 477, 517, 539, 493, 483, 529, 447, 521, 508, 500, 31, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8143, 8183, 8225, 8305, 8261, 8200, 8129] avg=8192.0000 max=8305 max_rank=4 min=8090 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[494, 499, 499, 533, 526, 514, 537, 471, 555, 513, 498, 526, 538, 514, 478, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 498, 506, 507, 513, 504, 502, 509, 528, 517, 477, 518, 519, 540, 480, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 515, 490, 512, 512, 495, 535, 498, 545, 447, 502, 489, 527, 533, 501, 529, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 476, 573, 542, 536, 520, 540, 535, 475, 562, 472, 528, 509, 509, 499, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 539, 519, 523, 504, 502, 525, 527, 502, 489, 469, 447, 509, 501, 508, 530, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 566, 538, 514, 483, 474, 539, 511, 538, 537, 497, 531, 536, 554, 505, 508, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 485, 493, 477, 527, 485, 526, 480, 517, 529, 508, 501, 518, 538, 507, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 516, 501, 502, 484, 520, 540, 547, 538, 464, 517, 501, 449, 524, 527, 512, 33, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8126, 8135, 8281, 8157, 8367, 8053, 8195] avg=8192.0000 max=8367 max_rank=5 min=8053 min_rank=6 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[536, 556, 457, 499, 539, 504, 522, 500, 523, 502, 509, 509, 490, 518, 565, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 508, 484, 474, 506, 534, 494, 536, 487, 536, 551, 484, 555, 524, 530, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 519, 519, 510, 530, 496, 552, 520, 510, 527, 543, 504, 482, 520, 533, 507, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 529, 512, 516, 537, 493, 508, 521, 527, 500, 489, 508, 513, 505, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 512, 538, 536, 529, 509, 487, 525, 532, 534, 491, 521, 526, 520, 522, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 494, 481, 508, 497, 467, 516, 511, 521, 513, 473, 466, 504, 538, 516, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 505, 525, 526, 520, 515, 541, 501, 478, 487, 514, 488, 457, 497, 507, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 481, 492, 515, 576, 497, 516, 513, 511, 533, 497, 510, 499, 494, 509, 472, 34, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8213, 8271, 8173, 8314, 8020, 8120, 8170] avg=8192.0000 max=8314 max_rank=4 min=8020 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True + [2026-08-26 16:29:47] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 416.7 | throughput per GPU (TFLOP/s/GPU): 434.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.013064E+01 | loss scale: 1.0 | grad norm: 26.899 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[529, 524, 520, 538, 526, 508, 528, 511, 555, 542, 492, 517, 513, 559, 505, 500, 533, 548, 486, 521, 535, 506, 541, 512, 526, 487, 558, 490, 525, 469, 535, 510, 487, 510, 473, 548, 498, 510, 504, 504, 500, 540, 518, 496, 500, 564, 499, 505, 512, 493, 504, 479, 475, 534, 529, 493, 527, 530, 556, 497, 521, 540, 541, 496, 470, 535, 542, 501, 498, 497, 500, 501, 485, 495, 541, 502, 515, 493, 533, 515, 517, 507, 476, 520, 557, 479, 502, 502, 472, 500, 522, 513, 502, 524, 510, 454, 546, 513, 500, 520, 523, 501, 487, 512, 503, 502, 495, 497, 482, 477, 482, 539, 533, 486, 508, 557, 537, 527, 533, 509, 557, 468, 499, 468, 540, 536, 511, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[525, 508, 505, 483, 526, 468, 566, 526, 525, 521, 551, 536, 525, 522, 508, 550, 473, 522, 506, 504, 491, 504, 505, 506, 507, 488, 523, 531, 485, 495, 533, 490, 513, 534, 521, 484, 505, 506, 489, 498, 468, 500, 474, 556, 547, 523, 533, 506, 452, 518, 508, 517, 521, 539, 538, 525, 482, 479, 504, 514, 530, 494, 522, 503, 538, 491, 520, 497, 530, 532, 508, 523, 528, 543, 525, 519, 549, 527, 531, 535, 481, 506, 533, 522, 505, 488, 505, 500, 491, 523, 542, 513, 500, 548, 461, 539, 509, 447, 543, 533, 499, 497, 529, 527, 478, 523, 514, 502, 498, 520, 518, 512, 511, 487, 480, 499, 492, 528, 518, 491, 518, 533, 504, 511, 511, 499, 533, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[526, 490, 501, 510, 449, 527, 548, 526, 492, 505, 505, 512, 502, 522, 512, 519, 536, 479, 492, 485, 514, 493, 498, 505, 537, 555, 510, 506, 539, 543, 522, 556, 508, 539, 517, 507, 458, 538, 494, 520, 513, 539, 507, 505, 488, 505, 503, 520, 523, 484, 523, 516, 520, 530, 511, 500, 483, 514, 530, 548, 516, 500, 498, 501, 503, 496, 518, 491, 519, 517, 519, 471, 523, 509, 549, 524, 474, 490, 502, 463, 578, 525, 492, 535, 550, 511, 516, 544, 544, 542, 518, 494, 519, 514, 508, 509, 513, 519, 523, 536, 491, 509, 519, 493, 514, 530, 528, 499, 520, 535, 493, 533, 535, 520, 496, 513, 478, 542, 483, 523, 512, 486, 479, 479, 488, 493, 498, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[492, 525, 527, 487, 481, 535, 527, 493, 526, 494, 465, 499, 520, 514, 527, 494, 480, 522, 486, 495, 542, 510, 486, 521, 470, 532, 518, 505, 513, 544, 548, 543, 487, 526, 502, 537, 507, 497, 515, 498, 515, 504, 489, 485, 510, 511, 528, 540, 497, 504, 500, 531, 517, 500, 495, 494, 523, 507, 509, 546, 479, 504, 528, 487, 515, 512, 511, 499, 512, 501, 454, 492, 544, 496, 537, 493, 515, 515, 539, 536, 540, 446, 519, 536, 528, 509, 485, 533, 540, 493, 515, 500, 478, 530, 539, 511, 527, 531, 501, 527, 497, 503, 548, 518, 546, 500, 485, 543, 461, 492, 466, 504, 507, 528, 529, 528, 545, 505, 518, 500, 514, 515, 573, 494, 505, 574, 540, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[525, 533, 520, 519, 510, 543, 524, 527, 524, 548, 526, 498, 508, 512, 497, 510, 545, 522, 471, 519, 489, 515, 538, 505, 515, 504, 485, 543, 558, 499, 501, 505, 526, 491, 490, 507, 505, 542, 486, 477, 505, 523, 470, 489, 488, 510, 492, 504, 510, 525, 538, 529, 510, 488, 482, 552, 507, 492, 527, 502, 519, 534, 544, 551, 507, 495, 515, 518, 499, 518, 519, 491, 500, 501, 545, 504, 547, 472, 523, 521, 511, 496, 540, 514, 491, 494, 521, 500, 535, 508, 518, 522, 477, 505, 506, 477, 520, 503, 526, 534, 471, 535, 527, 491, 520, 531, 497, 489, 511, 508, 498, 505, 518, 495, 489, 490, 497, 487, 571, 558, 463, 517, 492, 516, 509, 517, 541, 567] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[553, 513, 559, 496, 489, 496, 512, 544, 494, 524, 518, 498, 517, 535, 504, 533, 518, 498, 540, 509, 501, 522, 514, 475, 501, 482, 534, 553, 512, 537, 493, 488, 548, 475, 532, 534, 482, 473, 516, 514, 509, 477, 517, 535, 513, 488, 499, 512, 521, 496, 482, 515, 480, 518, 527, 494, 543, 524, 507, 513, 510, 510, 487, 513, 504, 536, 504, 466, 495, 540, 553, 511, 498, 500, 538, 504, 490, 508, 533, 506, 511, 507, 505, 485, 499, 530, 504, 508, 542, 519, 524, 543, 540, 472, 520, 494, 514, 542, 540, 525, 515, 492, 538, 477, 561, 524, 534, 525, 481, 492, 505, 497, 521, 530, 487, 464, 497, 518, 514, 502, 541, 546, 498, 475, 502, 530, 497, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[526, 445, 508, 504, 487, 511, 560, 518, 532, 520, 504, 547, 557, 527, 510, 519, 518, 486, 524, 475, 493, 530, 509, 511, 552, 493, 539, 542, 496, 506, 555, 542, 513, 537, 526, 495, 504, 519, 480, 516, 507, 515, 522, 524, 482, 511, 501, 548, 508, 525, 518, 493, 527, 530, 484, 494, 517, 496, 510, 528, 529, 498, 455, 516, 520, 519, 505, 507, 492, 549, 496, 572, 511, 507, 508, 499, 503, 541, 488, 480, 552, 524, 498, 539, 507, 480, 520, 495, 519, 512, 492, 488, 524, 486, 521, 513, 494, 507, 547, 508, 518, 510, 540, 523, 515, 506, 489, 524, 537, 497, 514, 482, 540, 520, 531, 533, 452, 524, 539, 502, 469, 493, 532, 484, 507, 500, 483, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[550, 543, 511, 517, 518, 505, 541, 465, 496, 546, 558, 532, 525, 516, 516, 515, 519, 523, 567, 492, 488, 534, 507, 512, 544, 456, 525, 541, 494, 509, 519, 531, 508, 506, 517, 524, 513, 483, 519, 484, 498, 539, 509, 496, 553, 490, 496, 515, 503, 507, 532, 548, 514, 467, 504, 494, 512, 514, 514, 517, 525, 532, 495, 562, 554, 496, 491, 514, 505, 505, 537, 499, 493, 517, 547, 518, 467, 508, 496, 512, 511, 509, 491, 495, 503, 519, 524, 524, 494, 534, 505, 486, 525, 492, 494, 546, 508, 474, 509, 532, 493, 490, 534, 489, 529, 467, 488, 498, 489, 515, 522, 533, 550, 533, 463, 495, 481, 542, 495, 511, 510, 544, 522, 470, 488, 521, 512, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[529, 524, 520, 538, 526, 508, 449, 511, 555, 542, 492, 517, 513, 559, 505, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 548, 486, 521, 535, 506, 541, 512, 526, 487, 558, 466, 525, 469, 535, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 510, 473, 548, 498, 510, 504, 504, 500, 540, 518, 496, 500, 564, 499, 505, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 504, 479, 475, 534, 529, 493, 527, 530, 556, 497, 521, 540, 541, 496, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 535, 542, 501, 498, 497, 500, 501, 485, 495, 541, 502, 515, 493, 533, 515, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 476, 520, 557, 479, 502, 502, 472, 500, 522, 513, 502, 524, 510, 454, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 513, 500, 520, 523, 501, 487, 512, 503, 502, 495, 497, 482, 477, 482, 539, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 486, 508, 557, 537, 527, 533, 509, 557, 468, 499, 468, 540, 536, 511, 476, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8258, 8208, 8230, 8130, 8066, 8102, 8254] avg=8192.0000 max=8288 max_rank=0 min=8066 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[492, 525, 527, 487, 481, 535, 448, 493, 526, 494, 465, 499, 520, 514, 527, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 522, 486, 495, 542, 510, 486, 521, 470, 532, 518, 481, 513, 544, 548, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 526, 502, 537, 507, 497, 515, 498, 515, 504, 489, 485, 510, 511, 528, 540, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 504, 500, 531, 517, 500, 495, 494, 523, 507, 509, 546, 479, 504, 528, 487, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 511, 499, 512, 501, 454, 492, 544, 496, 537, 493, 515, 515, 539, 536, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 446, 519, 536, 528, 509, 485, 533, 540, 493, 515, 500, 478, 530, 539, 511, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 531, 501, 527, 497, 503, 548, 518, 546, 500, 485, 543, 461, 492, 466, 504, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 529, 528, 545, 505, 518, 500, 514, 515, 573, 494, 505, 574, 540, 546, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8027, 8191, 8203, 8124, 8178, 8211, 8172, 8430] avg=8192.0000 max=8430 max_rank=7 min=8027 min_rank=0 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32991 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 508, 505, 483, 526, 468, 480, 526, 525, 521, 551, 536, 525, 522, 508, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 522, 506, 504, 491, 504, 505, 506, 507, 488, 523, 505, 485, 495, 533, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 534, 521, 484, 505, 506, 489, 498, 468, 500, 474, 556, 547, 523, 533, 506, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 518, 508, 517, 521, 539, 538, 525, 482, 479, 504, 514, 530, 494, 522, 503, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 491, 520, 497, 530, 532, 508, 523, 528, 543, 525, 519, 549, 527, 531, 535, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 506, 533, 522, 505, 488, 505, 500, 491, 523, 542, 513, 500, 548, 461, 539, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 447, 543, 533, 499, 497, 529, 527, 478, 523, 514, 502, 498, 520, 518, 512, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 487, 480, 499, 492, 528, 518, 491, 518, 533, 504, 511, 511, 499, 533, 508, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8037, 8213, 8149, 8403, 8167, 8174, 8134] avg=8192.0000 max=8403 max_rank=4 min=8037 min_rank=1 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 490, 501, 510, 449, 527, 451, 526, 492, 505, 505, 512, 502, 522, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 479, 492, 485, 514, 493, 498, 505, 537, 555, 510, 468, 539, 543, 522, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 539, 517, 507, 458, 538, 494, 520, 513, 539, 507, 505, 488, 505, 503, 520, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 484, 523, 516, 520, 530, 511, 500, 483, 514, 530, 548, 516, 500, 498, 501, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 496, 518, 491, 519, 517, 519, 471, 523, 509, 549, 524, 474, 490, 502, 463, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 525, 492, 535, 550, 511, 516, 544, 544, 542, 518, 494, 519, 514, 508, 509, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 523, 536, 491, 509, 519, 493, 514, 530, 528, 499, 520, 535, 493, 533, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 520, 496, 513, 478, 542, 483, 523, 512, 486, 479, 479, 488, 493, 498, 515, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 8232, 8220, 8203, 8079, 8412, 8284, 8057] avg=8192.0000 max=8412 max_rank=5 min=8049 min_rank=0 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 533, 520, 519, 510, 543, 445, 527, 524, 548, 526, 498, 508, 512, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 522, 471, 519, 489, 515, 538, 505, 515, 504, 485, 517, 558, 499, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 491, 490, 507, 505, 542, 486, 477, 505, 523, 470, 489, 488, 510, 492, 504, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 538, 529, 510, 488, 482, 552, 507, 492, 527, 502, 519, 534, 544, 551, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 515, 518, 499, 518, 519, 491, 500, 501, 545, 504, 547, 472, 523, 521, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 496, 540, 514, 491, 494, 521, 500, 535, 508, 518, 522, 477, 505, 506, 477, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 503, 526, 534, 471, 535, 527, 491, 520, 531, 497, 489, 511, 508, 498, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 495, 489, 490, 497, 487, 571, 558, 463, 517, 492, 516, 509, 517, 541, 567, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8188, 8057, 8313, 8182, 8125, 8189, 8237] avg=8192.0000 max=8313 max_rank=3 min=8057 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[553, 513, 559, 496, 489, 496, 434, 544, 494, 524, 518, 498, 517, 535, 504, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 498, 540, 509, 501, 522, 514, 475, 501, 482, 534, 526, 512, 537, 493, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 475, 532, 534, 482, 473, 516, 514, 509, 477, 517, 535, 513, 488, 499, 512, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 496, 482, 515, 480, 518, 527, 494, 543, 524, 507, 513, 510, 510, 487, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 536, 504, 466, 495, 540, 553, 511, 498, 500, 538, 504, 490, 508, 533, 506, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 507, 505, 485, 499, 530, 504, 508, 542, 519, 524, 543, 540, 472, 520, 494, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 542, 540, 525, 515, 492, 538, 477, 561, 524, 534, 525, 481, 492, 505, 497, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 530, 487, 464, 497, 518, 514, 502, 541, 546, 498, 475, 502, 530, 497, 537, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8150, 8175, 8143, 8194, 8213, 8285, 8169] avg=8192.0000 max=8285 max_rank=6 min=8143 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[550, 543, 511, 517, 518, 505, 459, 465, 496, 546, 558, 532, 525, 516, 516, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 523, 567, 492, 488, 534, 507, 512, 544, 456, 525, 515, 494, 509, 519, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 506, 517, 524, 513, 483, 519, 484, 498, 539, 509, 496, 553, 490, 496, 515, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 507, 532, 548, 514, 467, 504, 494, 512, 514, 514, 517, 525, 532, 495, 562, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 496, 491, 514, 505, 505, 537, 499, 493, 517, 547, 518, 467, 508, 496, 512, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 491, 495, 503, 519, 524, 524, 494, 534, 505, 486, 525, 492, 494, 546, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 474, 509, 532, 493, 490, 534, 489, 529, 467, 488, 498, 489, 515, 522, 533, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 533, 463, 495, 481, 542, 495, 511, 510, 544, 522, 470, 488, 521, 512, 513, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8235, 8204, 8243, 8166, 8162, 8094, 8160] avg=8192.0000 max=8272 max_rank=0 min=8094 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 445, 508, 504, 487, 511, 474, 518, 532, 520, 504, 547, 557, 527, 510, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 524, 475, 493, 530, 509, 511, 552, 493, 539, 516, 496, 506, 555, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 526, 495, 504, 519, 480, 516, 507, 515, 522, 524, 482, 511, 501, 548, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 518, 493, 527, 530, 484, 494, 517, 496, 510, 528, 529, 498, 455, 516, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 519, 505, 507, 492, 549, 496, 572, 511, 507, 508, 499, 503, 541, 488, 480, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 524, 498, 539, 507, 480, 520, 495, 519, 512, 492, 488, 524, 486, 521, 513, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 507, 547, 508, 518, 510, 540, 523, 515, 506, 489, 524, 537, 497, 514, 482, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 520, 531, 533, 452, 524, 539, 502, 469, 493, 532, 484, 507, 500, 483, 475, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8245, 8256, 8131, 8204, 8180, 8236, 8095] avg=8192.0000 max=8256 max_rank=2 min=8095 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4352, 4096, 4224, 4096, 4096, 4096, 3712, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[521, 484, 505, 495, 524, 543, 508, 537, 493, 562, 484, 568, 518, 476, 521, 499, 540, 541, 516, 493, 511, 532, 470, 499, 539, 541, 543, 499, 512, 532, 506, 499, 497, 517, 483, 519, 514, 533, 515, 533, 522, 508, 513, 529, 496, 514, 505, 510, 517, 494, 526, 512, 518, 523, 475, 491, 526, 516, 508, 528, 530, 471, 505, 548, 504, 522, 505, 494, 505, 515, 507, 455, 486, 529, 514, 531, 490, 496, 510, 545, 505, 507, 506, 524, 487, 513, 520, 522, 519, 524, 530, 507, 508, 526, 489, 459, 531, 500, 527, 534, 514, 498, 526, 532, 536, 502, 477, 534, 494, 469, 532, 548, 494, 530, 527, 513, 508, 493, 482, 506, 512, 499, 505, 500, 520, 482, 536, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[521, 519, 515, 544, 477, 508, 502, 526, 492, 483, 539, 478, 516, 508, 488, 508, 521, 498, 487, 553, 544, 524, 530, 509, 494, 516, 482, 517, 479, 507, 491, 479, 499, 504, 495, 482, 489, 524, 520, 521, 500, 528, 544, 495, 501, 501, 516, 518, 497, 498, 516, 512, 547, 514, 507, 505, 517, 491, 494, 526, 533, 509, 466, 518, 534, 508, 518, 516, 515, 546, 499, 528, 505, 504, 540, 503, 524, 520, 543, 502, 506, 519, 548, 518, 505, 517, 504, 536, 524, 489, 549, 497, 514, 542, 504, 557, 493, 479, 458, 500, 513, 506, 531, 508, 573, 503, 526, 493, 505, 520, 489, 499, 516, 502, 489, 501, 522, 542, 525, 517, 556, 515, 508, 507, 537, 510, 526, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[513, 519, 530, 511, 477, 471, 507, 524, 512, 501, 539, 495, 542, 519, 513, 511, 499, 529, 456, 540, 524, 524, 488, 554, 468, 524, 518, 496, 530, 507, 458, 534, 525, 525, 568, 519, 513, 493, 505, 544, 525, 485, 528, 485, 492, 544, 537, 515, 524, 536, 524, 520, 558, 511, 513, 536, 505, 527, 479, 523, 483, 528, 530, 531, 557, 476, 513, 536, 505, 523, 491, 478, 486, 501, 508, 484, 523, 495, 523, 534, 495, 491, 500, 518, 528, 455, 551, 522, 483, 522, 502, 495, 549, 513, 525, 510, 508, 497, 519, 526, 510, 490, 523, 538, 507, 529, 490, 565, 493, 491, 497, 463, 505, 530, 519, 518, 510, 499, 505, 494, 488, 494, 487, 498, 568, 476, 504, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[496, 486, 514, 482, 512, 535, 500, 534, 540, 466, 487, 510, 510, 500, 513, 516, 491, 477, 551, 480, 489, 465, 542, 505, 543, 522, 489, 550, 482, 504, 522, 521, 496, 497, 539, 452, 550, 513, 471, 515, 484, 521, 550, 536, 524, 520, 494, 530, 512, 576, 480, 529, 467, 528, 527, 505, 531, 512, 495, 503, 494, 520, 491, 513, 501, 573, 521, 474, 505, 489, 539, 477, 544, 531, 554, 486, 540, 535, 530, 527, 469, 484, 492, 531, 544, 498, 532, 481, 485, 491, 521, 532, 513, 501, 490, 490, 521, 494, 534, 535, 534, 531, 522, 496, 513, 532, 512, 505, 500, 530, 511, 519, 491, 539, 565, 554, 510, 491, 491, 512, 533, 487, 517, 493, 508, 520, 523, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[526, 503, 555, 536, 506, 520, 554, 496, 537, 493, 484, 495, 544, 496, 475, 514, 497, 514, 509, 508, 526, 526, 488, 503, 520, 510, 559, 519, 538, 543, 482, 521, 554, 505, 485, 497, 512, 488, 529, 527, 539, 508, 516, 506, 514, 532, 551, 502, 523, 495, 522, 503, 520, 527, 456, 526, 537, 497, 503, 487, 481, 554, 489, 532, 486, 529, 520, 526, 469, 526, 536, 550, 528, 483, 489, 505, 551, 480, 543, 502, 493, 535, 528, 509, 478, 565, 490, 507, 500, 521, 554, 495, 494, 513, 526, 479, 530, 487, 525, 503, 492, 508, 485, 462, 515, 471, 492, 507, 481, 528, 552, 513, 533, 492, 506, 482, 487, 481, 502, 487, 521, 528, 518, 515, 542, 520, 550, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[517, 530, 516, 483, 456, 488, 504, 506, 468, 531, 565, 545, 531, 503, 491, 533, 535, 491, 521, 507, 540, 492, 485, 521, 488, 509, 488, 545, 557, 490, 539, 492, 528, 552, 521, 528, 480, 508, 473, 529, 522, 558, 530, 515, 554, 477, 497, 466, 479, 516, 533, 521, 503, 522, 528, 539, 519, 515, 527, 491, 498, 512, 508, 516, 523, 521, 480, 543, 537, 549, 475, 514, 480, 497, 482, 510, 522, 498, 487, 538, 514, 519, 489, 517, 501, 555, 463, 510, 530, 514, 505, 524, 523, 524, 539, 488, 520, 481, 495, 511, 448, 528, 502, 518, 565, 564, 526, 523, 532, 511, 510, 503, 520, 489, 490, 511, 505, 500, 527, 504, 488, 483, 484, 522, 498, 527, 532, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[518, 508, 492, 516, 497, 517, 518, 509, 463, 528, 524, 482, 540, 507, 519, 542, 537, 506, 532, 526, 509, 519, 519, 513, 484, 512, 532, 500, 513, 476, 522, 484, 474, 481, 525, 526, 533, 504, 526, 506, 505, 520, 499, 520, 494, 490, 521, 483, 484, 508, 523, 514, 513, 537, 514, 510, 540, 515, 515, 526, 522, 520, 545, 516, 539, 517, 537, 501, 548, 519, 542, 508, 495, 484, 503, 482, 537, 486, 504, 529, 521, 518, 530, 499, 496, 502, 528, 508, 543, 505, 496, 531, 511, 509, 515, 491, 507, 528, 524, 536, 536, 518, 506, 478, 522, 525, 512, 492, 512, 492, 532, 519, 495, 528, 526, 462, 495, 551, 469, 485, 500, 514, 514, 509, 485, 515, 518, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[528, 506, 553, 530, 521, 506, 516, 525, 528, 540, 520, 485, 447, 531, 529, 476, 526, 528, 539, 510, 500, 522, 486, 497, 522, 518, 504, 508, 488, 527, 538, 455, 516, 509, 538, 507, 551, 521, 515, 539, 500, 562, 519, 558, 502, 515, 483, 518, 518, 483, 533, 531, 509, 519, 537, 518, 508, 512, 474, 489, 493, 499, 529, 536, 499, 506, 531, 521, 507, 539, 490, 529, 481, 496, 488, 535, 511, 522, 502, 483, 518, 505, 509, 467, 528, 498, 498, 493, 501, 497, 545, 495, 502, 533, 475, 474, 507, 476, 494, 508, 541, 489, 563, 535, 541, 467, 482, 515, 511, 519, 530, 535, 534, 491, 467, 518, 516, 535, 533, 501, 525, 513, 529, 511, 470, 508, 511, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 519, 515, 544, 477, 508, 502, 526, 492, 483, 539, 478, 516, 508, 488, 508, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 498, 487, 553, 544, 524, 530, 509, 494, 516, 482, 517, 479, 507, 491, 479, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 504, 495, 482, 489, 524, 520, 489, 500, 528, 544, 495, 501, 501, 516, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 516, 512, 547, 514, 507, 505, 517, 491, 494, 526, 533, 509, 466, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 518, 516, 515, 536, 499, 528, 505, 504, 540, 503, 524, 520, 543, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 519, 548, 518, 505, 517, 504, 536, 524, 489, 549, 497, 514, 542, 504, 557, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 479, 458, 500, 513, 506, 531, 508, 573, 503, 526, 493, 505, 520, 489, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 489, 501, 522, 542, 525, 517, 556, 515, 508, 507, 537, 510, 526, 491, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8135, 8105, 8128, 8295, 8354, 8097, 8290] avg=8192.0000 max=8354 max_rank=5 min=8097 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=33036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 484, 505, 495, 524, 543, 508, 537, 493, 562, 484, 568, 518, 476, 521, 499, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 541, 516, 493, 511, 532, 470, 499, 539, 541, 543, 499, 512, 532, 506, 499, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 483, 519, 514, 533, 515, 500, 522, 508, 513, 529, 496, 514, 505, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 494, 526, 512, 518, 523, 475, 491, 526, 516, 508, 528, 530, 471, 505, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 505, 494, 505, 506, 507, 455, 486, 529, 514, 531, 490, 496, 510, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 507, 506, 524, 487, 513, 520, 522, 519, 524, 530, 507, 508, 526, 489, 459, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 500, 527, 534, 514, 498, 526, 532, 536, 502, 477, 534, 494, 469, 532, 548, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 530, 527, 513, 508, 493, 482, 506, 512, 499, 505, 500, 520, 482, 536, 514, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8277, 8175, 8165, 8099, 8172, 8255, 8148] avg=8192.0000 max=8277 max_rank=1 min=8099 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 519, 530, 511, 477, 471, 507, 524, 512, 501, 539, 495, 542, 519, 513, 511, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 529, 456, 540, 524, 524, 488, 554, 468, 524, 518, 496, 530, 507, 458, 534, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 525, 568, 519, 513, 493, 505, 505, 525, 485, 528, 485, 492, 544, 537, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 536, 524, 520, 558, 511, 513, 536, 505, 527, 479, 523, 483, 528, 530, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 476, 513, 536, 505, 504, 491, 478, 486, 501, 508, 484, 523, 495, 523, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 491, 500, 518, 528, 455, 551, 522, 483, 522, 502, 495, 549, 513, 525, 510, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 519, 526, 510, 490, 523, 538, 507, 529, 490, 565, 493, 491, 497, 463, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 530, 519, 518, 510, 499, 505, 494, 488, 494, 487, 498, 568, 476, 504, 539, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8156, 8264, 8298, 8114, 8191, 8150, 8165] avg=8192.0000 max=8298 max_rank=3 min=8114 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 530, 516, 483, 456, 488, 504, 506, 468, 531, 565, 545, 531, 503, 491, 533, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 491, 521, 507, 540, 492, 485, 521, 488, 509, 488, 545, 557, 490, 539, 492, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 552, 521, 528, 480, 508, 473, 496, 522, 558, 530, 515, 554, 477, 497, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 516, 533, 521, 503, 522, 528, 539, 519, 515, 527, 491, 498, 512, 508, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 521, 480, 543, 537, 539, 475, 514, 480, 497, 482, 510, 522, 498, 487, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 519, 489, 517, 501, 555, 463, 510, 530, 514, 505, 524, 523, 524, 539, 488, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 481, 495, 511, 448, 528, 502, 518, 565, 564, 526, 523, 532, 511, 510, 503, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 490, 511, 505, 500, 527, 504, 488, 483, 484, 522, 498, 527, 532, 516, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8204, 8205, 8206, 8146, 8239, 8238, 8123] avg=8192.0000 max=8239 max_rank=5 min=8123 min_rank=7 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[528, 506, 553, 530, 521, 506, 516, 525, 528, 540, 520, 485, 447, 531, 529, 476, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 539, 510, 500, 522, 486, 497, 522, 518, 504, 508, 488, 527, 538, 455, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 509, 538, 507, 551, 521, 515, 506, 500, 562, 519, 558, 502, 515, 483, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 483, 533, 531, 509, 519, 537, 518, 508, 512, 474, 489, 493, 499, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 531, 521, 507, 529, 490, 529, 481, 496, 488, 535, 511, 522, 502, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 509, 467, 528, 498, 498, 493, 501, 497, 545, 495, 502, 533, 475, 474, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 476, 494, 508, 541, 489, 563, 535, 541, 467, 482, 515, 511, 519, 530, 535, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 491, 467, 518, 516, 535, 533, 501, 525, 513, 529, 511, 470, 508, 511, 533, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8172, 8320, 8166, 8130, 8063, 8214, 8222] avg=8192.0000 max=8320 max_rank=2 min=8063 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 486, 514, 482, 512, 535, 500, 534, 540, 466, 487, 510, 510, 500, 513, 516, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 477, 551, 480, 489, 465, 542, 505, 543, 522, 489, 550, 482, 504, 522, 521, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 539, 452, 550, 513, 471, 483, 484, 521, 550, 536, 524, 520, 494, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 576, 480, 529, 467, 528, 527, 505, 531, 512, 495, 503, 494, 520, 491, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 573, 521, 474, 505, 480, 539, 477, 544, 531, 554, 486, 540, 535, 530, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 484, 492, 531, 544, 498, 532, 481, 485, 491, 521, 532, 513, 501, 490, 490, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 494, 534, 535, 534, 531, 522, 496, 513, 532, 512, 505, 500, 530, 511, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 539, 565, 554, 510, 491, 491, 512, 533, 487, 517, 493, 508, 520, 523, 524, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8137, 8160, 8162, 8317, 8078, 8290, 8284] avg=8192.0000 max=8317 max_rank=4 min=8078 min_rank=5 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32969 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 503, 555, 536, 506, 520, 554, 496, 537, 493, 484, 495, 544, 496, 475, 514, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 509, 508, 526, 526, 488, 503, 520, 510, 559, 519, 538, 543, 482, 521, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 505, 485, 497, 512, 488, 529, 494, 539, 508, 516, 506, 514, 532, 551, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 495, 522, 503, 520, 527, 456, 526, 537, 497, 503, 487, 481, 554, 489, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 529, 520, 526, 469, 516, 536, 550, 528, 483, 489, 505, 551, 480, 543, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 535, 528, 509, 478, 565, 490, 507, 500, 521, 554, 495, 494, 513, 526, 479, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 487, 525, 503, 492, 508, 485, 462, 515, 471, 492, 507, 481, 528, 552, 513, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 492, 506, 482, 487, 481, 502, 487, 521, 528, 518, 515, 542, 520, 550, 497, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8267, 8232, 8130, 8213, 8212, 8052, 8188] avg=8192.0000 max=8267 max_rank=1 min=8052 min_rank=6 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 508, 492, 516, 497, 517, 518, 509, 463, 528, 524, 482, 540, 507, 519, 542, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 506, 532, 526, 509, 519, 519, 513, 484, 512, 532, 500, 513, 476, 522, 484, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 481, 525, 526, 533, 504, 526, 474, 505, 520, 499, 520, 494, 490, 521, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 523, 514, 513, 537, 514, 510, 540, 515, 515, 526, 522, 520, 545, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 517, 537, 501, 548, 510, 542, 508, 495, 484, 503, 482, 537, 486, 504, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 518, 530, 499, 496, 502, 528, 508, 543, 505, 496, 531, 511, 509, 515, 491, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 524, 536, 536, 518, 506, 478, 522, 525, 512, 492, 512, 492, 532, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 528, 526, 462, 495, 551, 469, 485, 500, 514, 514, 509, 485, 515, 518, 524, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8188, 8075, 8281, 8222, 8227, 8240, 8116] avg=8192.0000 max=8281 max_rank=3 min=8075 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 3968, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[536, 499, 523, 477, 529, 458, 485, 537, 562, 537, 523, 486, 517, 495, 535, 491, 555, 511, 510, 526, 524, 541, 535, 489, 470, 468, 512, 528, 490, 560, 517, 488, 468, 525, 471, 536, 504, 522, 482, 516, 507, 500, 477, 522, 498, 499, 508, 523, 507, 537, 522, 500, 469, 526, 521, 533, 508, 540, 547, 509, 523, 487, 533, 543, 484, 495, 523, 522, 570, 500, 499, 540, 496, 515, 504, 513, 541, 508, 516, 491, 484, 504, 537, 514, 496, 547, 518, 530, 529, 512, 480, 547, 547, 506, 481, 550, 501, 482, 533, 510, 515, 511, 520, 491, 538, 463, 539, 544, 519, 498, 493, 491, 476, 530, 520, 508, 515, 509, 528, 518, 544, 461, 511, 485, 505, 497, 486, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[514, 510, 528, 516, 560, 504, 508, 520, 459, 497, 485, 517, 512, 469, 532, 529, 507, 512, 508, 505, 515, 526, 536, 489, 515, 486, 517, 465, 462, 536, 545, 493, 548, 499, 554, 476, 491, 506, 541, 540, 491, 465, 506, 526, 526, 559, 506, 534, 498, 459, 548, 523, 500, 499, 505, 493, 510, 557, 520, 505, 498, 522, 506, 510, 513, 495, 511, 524, 510, 533, 501, 515, 532, 507, 535, 502, 519, 527, 495, 500, 497, 543, 532, 519, 517, 520, 517, 490, 513, 496, 507, 505, 485, 544, 487, 547, 490, 510, 501, 462, 515, 501, 476, 505, 526, 498, 532, 502, 521, 547, 497, 539, 514, 491, 504, 504, 510, 503, 515, 515, 531, 539, 522, 525, 505, 521, 539, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[490, 512, 540, 509, 542, 486, 505, 520, 495, 534, 483, 496, 488, 499, 551, 515, 526, 548, 528, 475, 510, 503, 513, 481, 505, 553, 521, 516, 521, 515, 522, 494, 530, 485, 508, 538, 502, 532, 503, 508, 473, 558, 523, 524, 496, 519, 510, 517, 517, 510, 518, 519, 537, 567, 485, 535, 525, 533, 531, 506, 542, 512, 487, 557, 501, 477, 527, 503, 553, 482, 500, 457, 539, 498, 520, 504, 536, 538, 517, 498, 506, 551, 537, 471, 518, 518, 506, 488, 485, 504, 513, 464, 523, 493, 499, 496, 475, 542, 482, 513, 517, 502, 463, 495, 520, 474, 527, 486, 560, 516, 501, 513, 503, 530, 522, 529, 525, 542, 522, 521, 489, 508, 476, 521, 534, 492, 499, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[484, 547, 461, 547, 533, 528, 515, 487, 520, 483, 515, 488, 496, 536, 531, 554, 523, 497, 559, 541, 474, 489, 514, 500, 499, 518, 522, 517, 493, 492, 539, 520, 494, 517, 519, 488, 491, 486, 526, 521, 495, 537, 518, 512, 474, 521, 542, 525, 520, 494, 545, 491, 529, 506, 498, 505, 540, 513, 512, 483, 543, 503, 503, 516, 508, 471, 462, 530, 516, 459, 506, 534, 489, 519, 494, 520, 534, 494, 529, 518, 530, 512, 557, 531, 511, 552, 508, 514, 541, 488, 514, 487, 522, 534, 471, 527, 524, 549, 512, 504, 522, 503, 527, 480, 499, 496, 517, 496, 523, 484, 504, 475, 513, 488, 536, 489, 528, 544, 513, 490, 536, 539, 527, 503, 508, 533, 500, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[492, 499, 481, 496, 519, 530, 502, 489, 514, 514, 485, 528, 517, 496, 503, 507, 527, 515, 531, 566, 481, 512, 513, 499, 463, 493, 533, 481, 529, 512, 537, 495, 486, 549, 508, 523, 505, 510, 500, 530, 498, 502, 540, 524, 515, 534, 517, 533, 539, 510, 515, 528, 518, 524, 538, 467, 517, 494, 525, 507, 500, 525, 503, 515, 479, 467, 508, 527, 487, 545, 528, 518, 470, 451, 521, 514, 508, 550, 503, 525, 521, 525, 493, 501, 489, 525, 572, 552, 503, 475, 477, 503, 490, 492, 468, 505, 511, 521, 521, 537, 512, 528, 480, 500, 549, 497, 524, 514, 497, 520, 528, 509, 557, 497, 548, 499, 506, 520, 482, 533, 526, 536, 539, 530, 509, 537, 508, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[492, 506, 511, 525, 510, 529, 515, 533, 518, 500, 499, 481, 478, 496, 523, 474, 529, 488, 482, 541, 481, 522, 522, 529, 565, 489, 521, 499, 500, 462, 489, 509, 491, 548, 534, 492, 542, 508, 502, 523, 498, 533, 546, 535, 496, 491, 474, 506, 490, 526, 521, 513, 515, 502, 504, 530, 506, 516, 534, 506, 510, 503, 505, 488, 504, 509, 544, 511, 501, 527, 510, 465, 473, 549, 497, 555, 471, 494, 480, 499, 530, 529, 507, 535, 544, 512, 516, 527, 536, 497, 520, 534, 490, 535, 500, 538, 512, 554, 542, 522, 560, 500, 529, 502, 530, 549, 510, 542, 556, 485, 474, 530, 548, 493, 497, 493, 518, 504, 477, 521, 505, 529, 497, 540, 502, 507, 473, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[516, 483, 469, 527, 473, 518, 486, 551, 515, 521, 541, 516, 532, 497, 502, 502, 512, 538, 498, 514, 507, 481, 531, 502, 486, 506, 525, 560, 507, 508, 509, 524, 507, 533, 465, 490, 506, 547, 525, 515, 496, 500, 494, 482, 517, 491, 512, 551, 495, 497, 544, 519, 550, 515, 483, 502, 537, 488, 533, 489, 535, 489, 473, 519, 530, 500, 509, 475, 510, 512, 513, 517, 515, 528, 532, 482, 468, 516, 521, 535, 493, 542, 521, 555, 502, 558, 480, 538, 519, 504, 512, 536, 530, 471, 498, 516, 529, 530, 479, 472, 525, 522, 535, 543, 476, 530, 508, 553, 541, 509, 527, 475, 533, 494, 483, 525, 513, 525, 504, 494, 506, 509, 479, 546, 495, 546, 511, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[529, 499, 501, 489, 517, 566, 528, 492, 516, 469, 458, 511, 521, 516, 515, 548, 545, 509, 499, 508, 535, 523, 515, 481, 492, 498, 518, 491, 528, 557, 492, 465, 482, 501, 482, 506, 479, 524, 539, 542, 506, 511, 529, 500, 536, 505, 504, 494, 495, 499, 546, 476, 546, 517, 508, 532, 522, 524, 490, 521, 498, 508, 471, 536, 508, 460, 513, 546, 502, 537, 519, 468, 552, 498, 495, 514, 500, 499, 530, 555, 521, 467, 495, 534, 526, 512, 491, 519, 519, 527, 479, 511, 501, 534, 507, 536, 505, 503, 508, 538, 503, 515, 505, 526, 518, 529, 531, 545, 524, 514, 479, 543, 525, 470, 510, 522, 480, 511, 529, 518, 531, 516, 577, 466, 509, 532, 504, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[484, 547, 461, 547, 533, 528, 515, 487, 520, 483, 515, 488, 496, 536, 531, 554, 21, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 559, 541, 474, 489, 514, 500, 499, 518, 522, 517, 493, 492, 539, 520, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 517, 519, 488, 491, 486, 526, 521, 495, 537, 518, 512, 474, 521, 542, 525, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 494, 513, 491, 529, 506, 498, 505, 540, 513, 512, 483, 543, 503, 503, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 471, 462, 530, 516, 459, 506, 534, 489, 519, 494, 520, 534, 494, 529, 518, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 512, 557, 531, 511, 523, 508, 514, 541, 488, 514, 487, 522, 534, 471, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 549, 512, 504, 522, 503, 527, 480, 499, 496, 517, 496, 509, 484, 504, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 488, 536, 489, 528, 544, 513, 490, 536, 539, 527, 503, 508, 533, 500, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8207, 8172, 8169, 8123, 8270, 8101, 8231] avg=8192.0000 max=8270 max_rank=5 min=8101 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 510, 528, 516, 560, 504, 508, 520, 459, 497, 485, 517, 512, 469, 532, 529, 20, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 508, 505, 515, 526, 536, 489, 515, 486, 517, 465, 462, 536, 545, 493, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 499, 554, 476, 491, 506, 541, 540, 491, 465, 506, 526, 526, 559, 506, 534, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 459, 516, 523, 500, 499, 505, 493, 510, 557, 520, 505, 498, 522, 506, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 495, 511, 524, 510, 533, 501, 515, 532, 507, 535, 502, 519, 527, 495, 500, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 543, 532, 519, 517, 493, 517, 490, 513, 496, 507, 505, 485, 544, 487, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 510, 501, 462, 515, 501, 476, 505, 526, 498, 532, 502, 507, 547, 497, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 491, 504, 504, 510, 503, 515, 515, 531, 539, 522, 525, 505, 521, 539, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8127, 8274, 8121, 8258, 8192, 8108, 8259] avg=8192.0000 max=8274 max_rank=2 min=8108 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[490, 512, 540, 509, 542, 486, 505, 520, 495, 534, 483, 496, 488, 499, 551, 515, 24, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 548, 528, 475, 510, 503, 513, 481, 505, 553, 521, 516, 521, 515, 522, 494, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 485, 508, 538, 502, 532, 503, 508, 473, 558, 523, 524, 496, 519, 510, 517, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 483, 519, 537, 567, 485, 535, 525, 533, 531, 506, 542, 512, 487, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 477, 527, 503, 553, 482, 500, 457, 539, 498, 520, 504, 536, 538, 517, 498, 35, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 551, 537, 471, 518, 483, 506, 488, 485, 504, 513, 464, 523, 493, 499, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 542, 482, 513, 517, 502, 463, 495, 520, 474, 527, 486, 538, 516, 501, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 530, 522, 529, 525, 542, 522, 521, 489, 508, 476, 521, 534, 492, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8250, 8237, 8346, 8196, 8037, 8064, 8196] avg=8192.0000 max=8346 max_rank=3 min=8037 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[536, 499, 523, 477, 529, 458, 485, 537, 562, 537, 523, 486, 517, 495, 535, 491, 21, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 511, 510, 526, 524, 541, 535, 489, 470, 468, 512, 528, 490, 560, 517, 488, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 525, 471, 536, 504, 522, 482, 516, 507, 500, 477, 522, 498, 499, 508, 523, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 537, 491, 500, 469, 526, 521, 533, 508, 540, 547, 509, 523, 487, 533, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 495, 523, 522, 570, 500, 499, 540, 496, 515, 504, 513, 541, 508, 516, 491, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 504, 537, 514, 496, 518, 518, 530, 529, 512, 480, 547, 547, 506, 481, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 482, 533, 510, 515, 511, 520, 491, 538, 463, 539, 544, 505, 498, 493, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 530, 520, 508, 515, 509, 528, 518, 544, 461, 511, 485, 505, 497, 486, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8234, 8064, 8274, 8256, 8253, 8134, 8094] avg=8192.0000 max=8274 max_rank=3 min=8064 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 499, 501, 489, 517, 566, 528, 492, 516, 469, 458, 511, 521, 516, 515, 548, 20, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 499, 508, 535, 523, 515, 481, 492, 498, 518, 491, 528, 557, 492, 465, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 482, 506, 479, 524, 539, 542, 506, 511, 529, 500, 536, 505, 504, 494, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 499, 514, 476, 546, 517, 508, 532, 522, 524, 490, 521, 498, 508, 471, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 460, 513, 546, 502, 537, 519, 468, 552, 498, 495, 514, 500, 499, 530, 555, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 467, 495, 534, 526, 485, 491, 519, 519, 527, 479, 511, 501, 534, 507, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 508, 538, 503, 515, 505, 526, 518, 529, 531, 545, 510, 514, 479, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 470, 510, 522, 480, 511, 529, 518, 531, 516, 577, 466, 509, 532, 504, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8166, 8146, 8157, 8235, 8152, 8272, 8196] avg=8192.0000 max=8272 max_rank=6 min=8146 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 506, 511, 525, 510, 529, 515, 533, 518, 500, 499, 481, 478, 496, 523, 474, 20, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 488, 482, 541, 481, 522, 522, 529, 565, 489, 521, 499, 500, 462, 489, 509, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 548, 534, 492, 542, 508, 502, 523, 498, 533, 546, 535, 496, 491, 474, 506, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 526, 490, 513, 515, 502, 504, 530, 506, 516, 534, 506, 510, 503, 505, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 544, 511, 501, 527, 510, 465, 473, 549, 497, 555, 471, 494, 480, 499, 31, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 529, 507, 535, 544, 485, 516, 527, 536, 497, 520, 534, 490, 535, 500, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 554, 542, 522, 560, 500, 529, 502, 530, 549, 510, 542, 541, 485, 474, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 493, 497, 493, 518, 504, 477, 521, 505, 529, 497, 540, 502, 507, 473, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8138, 8226, 8138, 8127, 8323, 8382, 8074] avg=8192.0000 max=8382 max_rank=6 min=8074 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 483, 469, 527, 473, 518, 486, 551, 515, 521, 541, 516, 532, 497, 502, 502, 22, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 498, 514, 507, 481, 531, 502, 486, 506, 525, 560, 507, 508, 509, 524, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 533, 465, 490, 506, 547, 525, 515, 496, 500, 494, 482, 517, 491, 512, 551, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 512, 519, 550, 515, 483, 502, 537, 488, 533, 489, 535, 489, 473, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 500, 509, 475, 510, 512, 513, 517, 515, 528, 532, 482, 468, 516, 521, 535, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 542, 521, 555, 502, 528, 480, 538, 519, 504, 512, 536, 530, 471, 498, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 530, 479, 472, 525, 522, 535, 543, 476, 530, 508, 553, 527, 509, 527, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 494, 483, 525, 513, 525, 504, 494, 506, 509, 479, 546, 495, 546, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8218, 8137, 8136, 8203, 8245, 8240, 8168] avg=8192.0000 max=8245 max_rank=5 min=8136 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 499, 481, 496, 519, 530, 502, 489, 514, 514, 485, 528, 517, 496, 503, 507, 20, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 515, 531, 566, 481, 512, 513, 499, 463, 493, 533, 481, 529, 512, 537, 495, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 549, 508, 523, 505, 510, 500, 530, 498, 502, 540, 524, 515, 534, 517, 533, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 510, 485, 528, 518, 524, 538, 467, 517, 494, 525, 507, 500, 525, 503, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 467, 508, 527, 487, 545, 528, 518, 470, 451, 521, 514, 508, 550, 503, 525, 30, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 525, 493, 501, 489, 498, 572, 552, 503, 475, 477, 503, 490, 492, 468, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 521, 521, 537, 512, 528, 480, 500, 549, 497, 524, 514, 484, 520, 528, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 497, 548, 499, 506, 520, 482, 533, 526, 536, 539, 530, 509, 537, 508, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8196, 8280, 8195, 8138, 8064, 8235, 8318] avg=8192.0000 max=8318 max_rank=7 min=8064 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[502, 525, 492, 506, 536, 494, 530, 504, 499, 520, 494, 514, 542, 522, 536, 490, 514, 501, 506, 525, 506, 526, 544, 498, 507, 525, 504, 562, 513, 531, 499, 531, 523, 523, 498, 497, 484, 501, 516, 501, 500, 529, 520, 501, 486, 521, 517, 530, 486, 527, 532, 504, 523, 522, 514, 492, 495, 566, 558, 505, 517, 495, 511, 529, 534, 480, 519, 541, 524, 470, 504, 520, 524, 543, 526, 474, 507, 511, 479, 515, 534, 517, 483, 492, 523, 535, 531, 504, 489, 530, 503, 496, 500, 510, 517, 506, 543, 520, 477, 507, 508, 492, 507, 530, 531, 446, 502, 570, 509, 485, 518, 488, 530, 493, 506, 553, 514, 509, 440, 493, 528, 511, 555, 522, 492, 504, 464, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[490, 538, 489, 523, 495, 520, 496, 485, 496, 521, 542, 515, 509, 523, 524, 476, 521, 498, 519, 517, 559, 483, 484, 506, 503, 499, 519, 536, 514, 526, 546, 492, 514, 510, 513, 510, 527, 512, 545, 500, 506, 519, 533, 489, 492, 517, 555, 519, 555, 530, 509, 490, 463, 471, 487, 544, 509, 527, 476, 537, 483, 530, 474, 534, 505, 492, 544, 505, 526, 522, 521, 502, 520, 509, 550, 497, 481, 550, 524, 496, 544, 511, 483, 534, 469, 494, 523, 520, 500, 482, 505, 536, 521, 574, 464, 527, 507, 512, 503, 504, 543, 539, 505, 531, 499, 527, 508, 496, 550, 491, 500, 476, 517, 518, 533, 500, 498, 506, 500, 499, 516, 499, 515, 522, 541, 482, 513, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[549, 520, 496, 494, 517, 496, 533, 520, 479, 536, 505, 505, 498, 548, 532, 566, 510, 518, 509, 544, 525, 476, 533, 532, 456, 522, 520, 522, 478, 478, 467, 491, 448, 527, 508, 497, 493, 540, 485, 524, 541, 502, 542, 544, 566, 470, 499, 465, 514, 547, 551, 529, 505, 496, 532, 516, 525, 511, 524, 529, 534, 501, 491, 513, 541, 495, 538, 491, 506, 498, 502, 519, 508, 464, 505, 495, 505, 502, 523, 572, 471, 527, 528, 466, 534, 471, 480, 500, 512, 512, 510, 511, 527, 543, 499, 518, 520, 508, 482, 522, 492, 487, 494, 525, 504, 510, 511, 534, 505, 531, 536, 510, 520, 497, 533, 532, 469, 526, 499, 539, 501, 506, 540, 535, 514, 532, 496, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[515, 501, 543, 476, 496, 532, 526, 539, 489, 514, 511, 548, 538, 509, 548, 492, 520, 475, 516, 528, 533, 491, 504, 472, 501, 474, 503, 495, 475, 464, 477, 502, 528, 530, 492, 471, 505, 503, 469, 528, 533, 518, 523, 526, 553, 531, 535, 490, 526, 548, 490, 557, 522, 509, 514, 524, 483, 535, 508, 498, 513, 518, 525, 519, 514, 544, 537, 520, 513, 518, 496, 505, 526, 478, 498, 498, 497, 499, 545, 507, 538, 534, 521, 532, 488, 498, 522, 575, 498, 520, 482, 531, 478, 504, 532, 494, 470, 518, 509, 524, 509, 513, 546, 531, 507, 484, 509, 520, 519, 491, 504, 535, 514, 514, 501, 536, 524, 515, 519, 512, 529, 492, 487, 473, 526, 499, 526, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[525, 537, 480, 523, 519, 549, 490, 504, 516, 537, 526, 505, 503, 506, 489, 494, 542, 487, 522, 511, 531, 484, 527, 520, 530, 533, 480, 522, 527, 476, 517, 489, 459, 516, 503, 507, 501, 489, 526, 537, 537, 464, 527, 501, 516, 511, 512, 516, 565, 534, 520, 477, 488, 518, 493, 490, 543, 505, 504, 495, 512, 512, 486, 515, 522, 495, 511, 511, 505, 526, 497, 532, 513, 528, 495, 511, 531, 476, 498, 494, 537, 496, 493, 502, 497, 563, 489, 537, 506, 516, 525, 492, 554, 461, 515, 549, 502, 524, 494, 540, 493, 514, 542, 507, 537, 531, 530, 507, 501, 495, 535, 489, 497, 511, 515, 551, 486, 544, 514, 503, 513, 520, 481, 512, 556, 533, 477, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[523, 527, 521, 520, 514, 482, 525, 506, 497, 506, 493, 511, 502, 488, 520, 489, 518, 529, 515, 522, 512, 485, 506, 503, 500, 497, 513, 535, 497, 498, 497, 520, 537, 527, 517, 501, 518, 540, 557, 524, 495, 499, 484, 532, 478, 476, 486, 502, 485, 485, 512, 493, 480, 525, 541, 511, 547, 556, 503, 551, 536, 530, 508, 498, 498, 506, 552, 506, 485, 505, 496, 542, 495, 540, 507, 549, 531, 517, 514, 498, 500, 511, 500, 494, 511, 488, 512, 481, 550, 512, 509, 531, 521, 504, 500, 513, 489, 469, 511, 539, 510, 518, 523, 523, 520, 525, 522, 492, 569, 484, 502, 497, 519, 518, 515, 573, 439, 520, 508, 518, 500, 519, 506, 470, 577, 555, 499, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[522, 492, 483, 509, 506, 540, 534, 527, 522, 510, 505, 497, 547, 485, 494, 532, 526, 528, 542, 532, 509, 514, 499, 513, 522, 528, 497, 492, 529, 487, 536, 498, 493, 537, 525, 449, 515, 525, 522, 512, 557, 533, 550, 504, 490, 499, 517, 505, 492, 507, 528, 544, 515, 449, 474, 491, 476, 509, 552, 488, 494, 519, 543, 524, 506, 490, 537, 501, 534, 487, 513, 521, 543, 527, 491, 512, 526, 494, 488, 543, 518, 517, 507, 529, 550, 491, 516, 535, 484, 484, 518, 522, 508, 515, 489, 546, 463, 498, 511, 523, 557, 475, 466, 548, 515, 518, 491, 508, 529, 497, 506, 495, 517, 476, 485, 504, 501, 488, 502, 527, 529, 533, 543, 492, 494, 555, 528, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[536, 551, 507, 509, 483, 533, 535, 521, 515, 537, 555, 508, 510, 497, 528, 549, 471, 497, 542, 498, 520, 507, 509, 483, 493, 519, 472, 544, 519, 492, 468, 520, 505, 550, 503, 527, 523, 523, 532, 489, 522, 524, 505, 490, 492, 510, 503, 522, 527, 506, 509, 519, 471, 499, 474, 537, 514, 486, 471, 518, 504, 518, 533, 520, 485, 505, 497, 483, 530, 486, 504, 570, 508, 500, 481, 477, 478, 502, 527, 488, 474, 530, 528, 544, 495, 567, 522, 521, 511, 482, 521, 500, 488, 498, 509, 562, 511, 475, 509, 535, 513, 568, 567, 465, 473, 492, 508, 535, 526, 518, 551, 507, 526, 472, 551, 503, 485, 511, 514, 557, 504, 517, 513, 524, 509, 546, 499, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 489, 492, 506, 536, 494, 530, 504, 499, 520, 494, 514, 542, 522, 536, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 501, 506, 525, 506, 526, 544, 498, 507, 525, 504, 562, 513, 531, 499, 531, 36, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 498, 497, 484, 501, 516, 501, 500, 529, 520, 501, 486, 521, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 527, 532, 504, 523, 522, 514, 492, 495, 554, 558, 505, 517, 495, 511, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 480, 519, 541, 524, 470, 504, 520, 524, 543, 526, 474, 507, 511, 479, 515, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 483, 492, 523, 535, 531, 504, 489, 530, 503, 496, 500, 510, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 520, 477, 507, 508, 492, 507, 530, 531, 446, 502, 570, 509, 485, 518, 488, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 493, 506, 534, 514, 509, 440, 493, 528, 511, 555, 522, 492, 504, 464, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8337, 8147, 8264, 8189, 8167, 8140, 8122] avg=8192.0000 max=8337 max_rank=1 min=8122 min_rank=7 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32618 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[490, 501, 489, 523, 495, 520, 496, 485, 496, 521, 542, 515, 509, 523, 524, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 498, 519, 517, 559, 483, 484, 506, 503, 499, 519, 536, 514, 526, 546, 492, 37, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 513, 510, 527, 512, 545, 500, 506, 519, 533, 489, 492, 517, 555, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 530, 509, 490, 463, 471, 487, 544, 509, 516, 476, 537, 483, 530, 474, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 492, 544, 505, 526, 522, 521, 502, 520, 509, 550, 497, 481, 550, 524, 496, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 511, 483, 534, 469, 494, 523, 520, 500, 482, 505, 536, 521, 574, 464, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 503, 504, 543, 539, 505, 531, 499, 527, 508, 496, 550, 491, 500, 476, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 518, 533, 483, 498, 506, 500, 499, 516, 499, 515, 522, 541, 482, 513, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8267, 8261, 8108, 8260, 8184, 8198, 8153] avg=8192.0000 max=8267 max_rank=1 min=8105 min_rank=0 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 459, 483, 509, 506, 540, 534, 527, 522, 510, 505, 497, 547, 485, 494, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 542, 532, 509, 514, 499, 513, 522, 528, 497, 492, 529, 487, 536, 498, 33, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 537, 525, 449, 515, 525, 522, 512, 557, 533, 550, 504, 490, 499, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 507, 528, 544, 515, 449, 474, 491, 476, 499, 552, 488, 494, 519, 543, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 490, 537, 501, 534, 487, 513, 521, 543, 527, 491, 512, 526, 494, 488, 543, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 517, 507, 529, 550, 491, 516, 535, 484, 484, 518, 522, 508, 515, 489, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 498, 511, 523, 557, 475, 466, 548, 515, 518, 491, 508, 529, 497, 506, 495, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 476, 485, 487, 501, 488, 502, 527, 529, 533, 543, 492, 494, 555, 528, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8293, 8233, 8095, 8229, 8226, 8106, 8182] avg=8192.0000 max=8293 max_rank=1 min=8095 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 514, 507, 509, 483, 533, 535, 521, 515, 537, 555, 508, 510, 497, 528, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 497, 542, 498, 520, 507, 509, 483, 493, 519, 472, 544, 519, 492, 468, 520, 37, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 550, 503, 527, 523, 523, 532, 489, 522, 524, 505, 490, 492, 510, 503, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 506, 509, 519, 471, 499, 474, 537, 514, 476, 471, 518, 504, 518, 533, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 505, 497, 483, 530, 486, 504, 570, 508, 500, 481, 477, 478, 502, 527, 488, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 530, 528, 544, 495, 567, 522, 521, 511, 482, 521, 500, 488, 498, 509, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 475, 509, 535, 513, 568, 567, 465, 473, 492, 508, 535, 526, 518, 551, 507, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 472, 551, 486, 485, 511, 514, 557, 504, 517, 513, 524, 509, 546, 499, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8337, 8099, 8220, 8096, 8037, 8249, 8259, 8239] avg=8192.0000 max=8337 max_rank=0 min=8037 min_rank=4 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 467, 543, 476, 496, 532, 526, 539, 489, 514, 511, 548, 538, 509, 548, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 475, 516, 528, 533, 491, 504, 472, 501, 474, 503, 495, 475, 464, 477, 502, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 530, 492, 471, 505, 503, 469, 528, 533, 518, 523, 526, 553, 531, 535, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 548, 490, 557, 522, 509, 514, 524, 483, 523, 508, 498, 513, 518, 525, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 544, 537, 520, 513, 518, 496, 505, 526, 478, 498, 498, 497, 499, 545, 507, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 534, 521, 532, 488, 498, 522, 575, 498, 520, 482, 531, 478, 504, 532, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 518, 509, 524, 509, 513, 546, 531, 507, 484, 509, 520, 519, 491, 504, 535, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 501, 519, 524, 515, 519, 512, 529, 492, 487, 473, 526, 499, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 7972, 8235, 8277, 8212, 8244, 8196, 8157] avg=8192.0000 max=8277 max_rank=3 min=7972 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[525, 500, 480, 523, 519, 549, 490, 504, 516, 537, 526, 505, 503, 506, 489, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 487, 522, 511, 531, 484, 527, 520, 530, 533, 480, 522, 527, 476, 517, 489, 37, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 516, 503, 507, 501, 489, 526, 537, 537, 464, 527, 501, 516, 511, 512, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 534, 520, 477, 488, 518, 493, 490, 543, 495, 504, 495, 512, 512, 486, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 511, 511, 505, 526, 497, 532, 513, 528, 495, 511, 531, 476, 498, 494, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 496, 493, 502, 497, 563, 489, 537, 506, 516, 525, 492, 554, 461, 515, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 494, 540, 493, 514, 542, 507, 537, 531, 530, 507, 501, 495, 535, 489, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 511, 515, 532, 486, 544, 514, 503, 513, 520, 481, 512, 556, 533, 477, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8244, 8122, 8147, 8162, 8229, 8247, 8219] avg=8192.0000 max=8247 max_rank=6 min=8122 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[549, 481, 496, 494, 517, 496, 533, 520, 479, 536, 505, 505, 498, 548, 532, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 518, 509, 544, 525, 476, 533, 532, 456, 522, 520, 522, 478, 478, 467, 491, 39, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 521, 508, 497, 493, 540, 485, 524, 541, 502, 542, 544, 566, 470, 499, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 547, 551, 529, 505, 496, 532, 516, 525, 493, 524, 529, 534, 501, 491, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 495, 538, 491, 506, 498, 502, 519, 508, 464, 505, 495, 505, 502, 523, 572, 13, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 527, 528, 466, 534, 471, 480, 500, 512, 512, 510, 511, 527, 543, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 482, 522, 492, 487, 494, 525, 504, 510, 511, 534, 505, 531, 536, 510, 7, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 497, 533, 507, 469, 526, 499, 539, 501, 506, 540, 535, 514, 532, 496, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8132, 8145, 8300, 8188, 8103, 8190, 8223] avg=8192.0000 max=8300 max_rank=3 min=8103 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[523, 491, 521, 520, 514, 482, 525, 506, 497, 506, 493, 511, 502, 488, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 515, 522, 512, 485, 506, 503, 500, 497, 513, 535, 497, 498, 497, 520, 36, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 527, 517, 501, 518, 540, 557, 524, 495, 499, 484, 532, 478, 476, 486, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 485, 512, 493, 480, 525, 541, 511, 547, 544, 503, 551, 536, 530, 508, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 506, 552, 506, 485, 505, 496, 542, 495, 540, 507, 549, 531, 517, 514, 498, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 500, 494, 511, 488, 512, 481, 550, 512, 509, 531, 521, 504, 500, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 469, 511, 539, 510, 518, 523, 523, 520, 525, 522, 492, 569, 484, 502, 497, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 515, 554, 439, 520, 508, 518, 500, 519, 506, 470, 577, 555, 499, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8088, 8192, 8173, 8249, 8259, 8134, 8200, 8241] avg=8192.0000 max=8259 max_rank=4 min=8088 min_rank=0 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 3968, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4352, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[503, 513, 489, 520, 520, 506, 485, 524, 516, 498, 503, 486, 524, 503, 481, 501, 502, 529, 534, 490, 532, 509, 498, 489, 491, 503, 504, 555, 527, 506, 503, 548, 509, 496, 522, 517, 521, 525, 531, 525, 523, 524, 509, 534, 504, 492, 517, 534, 535, 482, 510, 545, 540, 520, 507, 565, 521, 517, 518, 502, 448, 494, 554, 473, 487, 546, 517, 472, 552, 524, 539, 542, 562, 531, 513, 528, 473, 497, 474, 459, 516, 538, 520, 520, 505, 498, 516, 506, 522, 544, 510, 517, 556, 514, 497, 484, 504, 508, 511, 508, 510, 490, 478, 506, 496, 484, 491, 500, 520, 525, 513, 493, 492, 500, 508, 523, 549, 540, 498, 519, 511, 509, 490, 528, 484, 511, 528, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[516, 530, 570, 521, 484, 515, 500, 499, 525, 546, 491, 506, 496, 524, 525, 494, 476, 510, 502, 506, 543, 510, 504, 480, 487, 545, 508, 514, 486, 503, 536, 522, 529, 525, 525, 502, 541, 534, 513, 544, 489, 529, 494, 490, 490, 532, 526, 564, 495, 513, 478, 463, 472, 513, 499, 501, 542, 513, 463, 543, 500, 494, 511, 496, 510, 526, 500, 543, 511, 542, 497, 506, 477, 581, 487, 527, 481, 503, 529, 519, 539, 500, 527, 474, 520, 498, 537, 528, 532, 485, 497, 515, 513, 493, 513, 478, 527, 514, 485, 556, 488, 525, 533, 491, 505, 526, 520, 529, 529, 461, 508, 517, 520, 510, 543, 476, 496, 482, 521, 540, 530, 509, 529, 548, 536, 505, 500, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[505, 534, 503, 509, 548, 498, 493, 507, 548, 493, 502, 539, 505, 495, 523, 552, 496, 527, 549, 524, 533, 511, 465, 479, 493, 495, 517, 513, 478, 523, 500, 530, 509, 533, 500, 541, 515, 524, 514, 482, 497, 530, 541, 542, 498, 473, 515, 525, 514, 508, 508, 501, 539, 549, 519, 492, 520, 543, 502, 516, 508, 489, 540, 490, 491, 460, 491, 533, 532, 515, 530, 513, 539, 520, 513, 509, 518, 558, 566, 495, 482, 528, 497, 544, 516, 523, 541, 472, 523, 507, 538, 487, 523, 490, 466, 511, 524, 511, 539, 499, 504, 486, 511, 508, 523, 430, 517, 494, 524, 472, 499, 497, 527, 500, 565, 501, 511, 479, 543, 503, 514, 513, 535, 517, 496, 507, 511, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[529, 525, 530, 519, 483, 512, 502, 497, 506, 484, 525, 485, 509, 481, 493, 561, 558, 509, 492, 486, 520, 489, 465, 500, 520, 515, 500, 509, 492, 560, 511, 501, 526, 471, 509, 501, 498, 531, 517, 501, 505, 541, 516, 472, 523, 503, 509, 563, 512, 488, 530, 475, 481, 528, 529, 521, 497, 511, 497, 520, 499, 494, 525, 486, 500, 520, 511, 490, 547, 535, 472, 513, 524, 537, 515, 563, 501, 550, 469, 522, 492, 526, 526, 560, 524, 496, 523, 497, 563, 497, 505, 516, 486, 512, 528, 505, 518, 491, 547, 523, 485, 491, 526, 519, 491, 545, 529, 498, 508, 515, 536, 496, 498, 513, 527, 480, 462, 555, 539, 516, 543, 531, 514, 496, 534, 566, 479, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[483, 473, 520, 508, 475, 521, 522, 502, 540, 457, 486, 513, 511, 530, 523, 494, 519, 501, 523, 532, 508, 522, 513, 528, 490, 511, 513, 489, 527, 526, 496, 524, 542, 535, 525, 494, 476, 526, 484, 505, 548, 530, 499, 486, 499, 552, 518, 524, 537, 486, 534, 518, 507, 522, 482, 506, 523, 488, 532, 560, 535, 499, 534, 516, 522, 554, 508, 514, 545, 495, 494, 448, 495, 553, 543, 521, 494, 506, 535, 553, 524, 527, 561, 482, 497, 535, 531, 521, 508, 544, 520, 508, 521, 512, 502, 506, 521, 516, 498, 551, 535, 484, 471, 531, 452, 501, 502, 485, 482, 473, 499, 487, 527, 523, 499, 499, 479, 530, 530, 523, 526, 505, 507, 503, 526, 518, 466, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[426, 512, 519, 540, 534, 529, 496, 475, 483, 522, 494, 533, 523, 511, 514, 521, 506, 481, 503, 513, 518, 494, 522, 520, 506, 511, 537, 540, 514, 484, 494, 488, 497, 517, 511, 514, 517, 525, 467, 530, 548, 510, 534, 498, 505, 550, 555, 504, 497, 484, 545, 471, 538, 496, 528, 517, 530, 489, 503, 487, 498, 500, 505, 552, 485, 509, 503, 525, 525, 528, 542, 557, 507, 511, 541, 512, 504, 530, 539, 523, 539, 520, 527, 517, 496, 530, 520, 481, 477, 490, 500, 529, 486, 498, 519, 505, 547, 526, 497, 506, 493, 548, 529, 513, 484, 508, 459, 561, 488, 520, 445, 543, 530, 493, 507, 536, 522, 514, 509, 523, 495, 488, 477, 523, 533, 512, 520, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[510, 555, 502, 506, 509, 500, 494, 524, 477, 485, 507, 484, 502, 503, 566, 516, 504, 518, 494, 497, 536, 522, 512, 518, 506, 533, 534, 527, 481, 507, 539, 522, 568, 524, 506, 502, 508, 498, 564, 520, 534, 510, 491, 516, 523, 501, 508, 519, 511, 526, 507, 497, 490, 483, 521, 524, 500, 520, 507, 520, 503, 494, 500, 492, 513, 503, 520, 487, 519, 485, 456, 484, 487, 530, 501, 514, 511, 536, 520, 503, 537, 544, 468, 534, 517, 539, 506, 498, 558, 534, 519, 480, 509, 510, 530, 488, 542, 486, 484, 502, 545, 499, 509, 495, 548, 497, 544, 541, 531, 494, 499, 493, 540, 483, 477, 523, 534, 504, 525, 463, 528, 496, 511, 504, 530, 539, 523, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[512, 437, 542, 491, 505, 483, 554, 463, 498, 524, 531, 515, 522, 498, 527, 504, 492, 480, 478, 526, 514, 499, 536, 542, 527, 539, 544, 502, 528, 527, 489, 508, 500, 497, 514, 522, 511, 470, 541, 513, 483, 509, 512, 493, 534, 552, 536, 496, 523, 541, 505, 523, 509, 520, 487, 539, 493, 456, 492, 515, 523, 513, 474, 513, 507, 504, 505, 477, 529, 444, 478, 493, 490, 509, 540, 560, 506, 516, 519, 521, 535, 511, 526, 491, 512, 517, 520, 506, 515, 502, 508, 532, 525, 498, 536, 536, 487, 526, 503, 493, 555, 494, 534, 532, 501, 488, 570, 479, 551, 534, 497, 496, 557, 522, 496, 517, 520, 543, 540, 510, 545, 504, 505, 490, 506, 506, 513, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[529, 525, 530, 519, 483, 512, 502, 497, 506, 484, 525, 485, 509, 481, 493, 561, 32, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 509, 492, 486, 520, 489, 465, 500, 520, 515, 500, 509, 492, 560, 511, 501, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 471, 509, 501, 498, 531, 517, 501, 505, 541, 516, 472, 523, 503, 509, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 530, 475, 481, 528, 529, 521, 497, 511, 497, 520, 499, 494, 525, 486, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 520, 511, 490, 547, 535, 472, 513, 524, 505, 515, 563, 501, 550, 469, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 526, 526, 560, 524, 496, 523, 497, 530, 497, 505, 516, 486, 512, 528, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 491, 547, 523, 485, 491, 526, 519, 491, 545, 529, 498, 508, 515, 536, 496, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 513, 527, 480, 462, 555, 510, 516, 543, 531, 514, 496, 534, 566, 479, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8138, 8121, 8137, 8237, 8223, 8274, 8217] avg=8192.0000 max=8274 max_rank=6 min=8121 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32951 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[503, 513, 489, 520, 520, 506, 485, 524, 516, 498, 503, 486, 524, 503, 481, 501, 32, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 529, 534, 490, 532, 509, 498, 489, 491, 503, 504, 555, 527, 506, 503, 548, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 496, 522, 517, 521, 525, 531, 525, 523, 524, 509, 534, 504, 492, 517, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 482, 510, 545, 540, 520, 507, 565, 521, 517, 518, 502, 448, 494, 554, 473, 24, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 546, 517, 472, 552, 524, 539, 542, 562, 499, 513, 528, 473, 497, 474, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 538, 520, 520, 505, 498, 516, 506, 491, 544, 510, 517, 556, 514, 497, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 508, 511, 508, 510, 490, 478, 506, 496, 484, 491, 500, 520, 525, 513, 493, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 500, 508, 523, 549, 540, 472, 519, 511, 509, 490, 528, 484, 511, 528, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8230, 8221, 8271, 8184, 8232, 8090, 8188] avg=8192.0000 max=8271 max_rank=3 min=8090 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[426, 512, 519, 540, 534, 529, 496, 475, 483, 522, 494, 533, 523, 511, 514, 521, 30, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 481, 503, 513, 518, 494, 522, 520, 506, 511, 537, 540, 514, 484, 494, 488, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 511, 514, 517, 525, 467, 530, 548, 510, 534, 498, 505, 550, 555, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 484, 545, 471, 538, 496, 528, 517, 530, 489, 503, 487, 498, 500, 505, 552, 22, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 509, 503, 525, 525, 528, 542, 557, 507, 481, 541, 512, 504, 530, 539, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 520, 527, 517, 496, 530, 520, 481, 449, 490, 500, 529, 486, 498, 519, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 526, 497, 506, 493, 548, 529, 513, 484, 508, 459, 561, 488, 520, 445, 543, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 493, 507, 536, 522, 514, 482, 523, 495, 488, 477, 523, 533, 512, 520, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8141, 8224, 8179, 8311, 8106, 8217, 8182] avg=8192.0000 max=8311 max_rank=4 min=8106 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[483, 473, 520, 508, 475, 521, 522, 502, 540, 457, 486, 513, 511, 530, 523, 494, 37, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 501, 523, 532, 508, 522, 513, 528, 490, 511, 513, 489, 527, 526, 496, 524, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 535, 525, 494, 476, 526, 484, 505, 548, 530, 499, 486, 499, 552, 518, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 486, 534, 518, 507, 522, 482, 506, 523, 488, 532, 560, 535, 499, 534, 516, 26, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 554, 508, 514, 545, 495, 494, 448, 495, 516, 543, 521, 494, 506, 535, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 561, 482, 497, 535, 531, 521, 472, 544, 520, 508, 521, 512, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 498, 551, 535, 484, 471, 531, 452, 501, 502, 485, 482, 473, 499, 487, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 523, 499, 499, 479, 530, 494, 523, 526, 505, 507, 503, 526, 518, 466, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8237, 8173, 8326, 8243, 8263, 8045, 8131] avg=8192.0000 max=8326 max_rank=3 min=8045 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[505, 534, 503, 509, 548, 498, 493, 507, 548, 493, 502, 539, 505, 495, 523, 552, 31, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 527, 549, 524, 533, 511, 465, 479, 493, 495, 517, 513, 478, 523, 500, 530, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 500, 541, 515, 524, 514, 482, 497, 530, 541, 542, 498, 473, 515, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 508, 501, 539, 549, 519, 492, 520, 543, 502, 516, 508, 489, 540, 490, 24, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 460, 491, 533, 532, 515, 530, 513, 539, 489, 513, 509, 518, 558, 566, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 528, 497, 544, 516, 523, 541, 472, 492, 507, 538, 487, 523, 490, 466, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 511, 539, 499, 504, 486, 511, 508, 523, 430, 517, 494, 524, 472, 499, 497, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 500, 565, 501, 511, 479, 514, 503, 514, 513, 535, 517, 496, 507, 511, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8144, 8178, 8280, 8252, 8117, 8090, 8174] avg=8192.0000 max=8301 max_rank=0 min=8090 min_rank=6 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[512, 437, 542, 491, 505, 483, 554, 463, 498, 524, 531, 515, 522, 498, 527, 504, 30, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 480, 478, 526, 514, 499, 536, 542, 527, 539, 544, 502, 528, 527, 489, 508, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 514, 522, 511, 470, 541, 513, 483, 509, 512, 493, 534, 552, 536, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 541, 505, 523, 509, 520, 487, 539, 493, 456, 492, 515, 523, 513, 474, 513, 24, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 505, 477, 529, 444, 478, 493, 490, 479, 540, 560, 506, 516, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 526, 491, 512, 517, 520, 506, 484, 502, 508, 532, 525, 498, 536, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 526, 503, 493, 555, 494, 534, 532, 501, 488, 570, 479, 551, 534, 497, 496, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 522, 496, 517, 520, 543, 511, 510, 545, 504, 505, 490, 506, 506, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8242, 8126, 8168, 8068, 8239, 8289, 8253] avg=8192.0000 max=8289 max_rank=6 min=8068 min_rank=4 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 555, 502, 506, 509, 500, 494, 524, 477, 485, 507, 484, 502, 503, 566, 516, 32, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 494, 497, 536, 522, 512, 518, 506, 533, 534, 527, 481, 507, 539, 522, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 524, 506, 502, 508, 498, 564, 520, 534, 510, 491, 516, 523, 501, 508, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 526, 507, 497, 490, 483, 521, 524, 500, 520, 507, 520, 503, 494, 500, 492, 26, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 503, 520, 487, 519, 485, 456, 484, 487, 498, 501, 514, 511, 536, 520, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 544, 468, 534, 517, 539, 506, 498, 525, 534, 519, 480, 509, 510, 530, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 486, 484, 502, 545, 499, 509, 495, 548, 497, 544, 541, 531, 494, 499, 493, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 483, 477, 523, 534, 504, 497, 463, 528, 496, 511, 504, 530, 539, 523, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8261, 8231, 8138, 8037, 8238, 8261, 8182] avg=8192.0000 max=8261 max_rank=1 min=8037 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 530, 570, 521, 484, 515, 500, 499, 525, 546, 491, 506, 496, 524, 525, 494, 35, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 510, 502, 506, 543, 510, 504, 480, 487, 545, 508, 514, 486, 503, 536, 522, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 525, 525, 502, 541, 534, 513, 544, 489, 529, 494, 490, 490, 532, 526, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 513, 478, 463, 472, 513, 499, 501, 542, 513, 463, 543, 500, 494, 511, 496, 24, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 526, 500, 543, 511, 542, 497, 506, 477, 546, 487, 527, 481, 503, 529, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 500, 527, 474, 520, 498, 537, 528, 501, 485, 497, 515, 513, 493, 513, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 514, 485, 556, 488, 525, 533, 491, 505, 526, 520, 529, 529, 461, 508, 517, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 510, 543, 476, 496, 482, 493, 540, 530, 509, 529, 548, 536, 505, 500, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8143, 8262, 8037, 8204, 8118, 8270, 8209] avg=8192.0000 max=8293 max_rank=0 min=8037 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[528, 499, 549, 517, 488, 465, 550, 529, 491, 528, 470, 480, 533, 527, 523, 506, 474, 519, 530, 513, 519, 558, 476, 513, 536, 521, 522, 529, 532, 512, 499, 499, 502, 492, 513, 533, 485, 544, 446, 494, 473, 524, 547, 523, 502, 524, 563, 497, 527, 502, 507, 521, 507, 537, 497, 519, 532, 508, 550, 494, 477, 527, 524, 522, 482, 510, 532, 511, 530, 514, 520, 534, 528, 514, 491, 491, 494, 512, 532, 515, 520, 489, 503, 539, 505, 562, 493, 526, 514, 515, 500, 532, 533, 446, 547, 526, 501, 532, 496, 510, 481, 496, 519, 519, 485, 516, 515, 501, 513, 490, 476, 513, 494, 496, 520, 501, 511, 517, 514, 500, 497, 483, 535, 556, 518, 510, 522, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[500, 517, 530, 482, 509, 519, 485, 550, 476, 512, 470, 492, 495, 550, 569, 506, 509, 518, 534, 531, 482, 496, 489, 506, 522, 552, 531, 513, 523, 551, 540, 494, 515, 546, 506, 507, 532, 553, 519, 528, 451, 531, 526, 527, 508, 500, 523, 517, 492, 534, 509, 503, 509, 525, 516, 487, 495, 474, 484, 529, 504, 505, 488, 459, 494, 526, 515, 513, 526, 484, 497, 540, 495, 497, 516, 476, 474, 499, 504, 548, 523, 487, 518, 528, 483, 512, 512, 485, 524, 514, 555, 550, 533, 536, 491, 517, 532, 488, 500, 486, 533, 504, 494, 531, 481, 496, 500, 513, 528, 560, 533, 516, 530, 555, 491, 472, 503, 536, 481, 518, 516, 512, 499, 499, 518, 536, 504, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[545, 500, 512, 507, 503, 515, 503, 506, 528, 508, 526, 529, 506, 498, 492, 504, 521, 504, 488, 551, 469, 494, 503, 556, 481, 509, 530, 502, 525, 510, 514, 537, 528, 506, 584, 522, 514, 518, 513, 527, 527, 523, 493, 491, 469, 533, 493, 510, 494, 481, 506, 486, 505, 519, 510, 512, 515, 507, 570, 524, 483, 536, 511, 530, 524, 520, 525, 527, 531, 526, 487, 512, 498, 508, 536, 526, 484, 548, 512, 531, 515, 507, 540, 501, 466, 507, 523, 564, 510, 493, 463, 521, 497, 513, 512, 524, 546, 487, 514, 516, 483, 513, 497, 489, 497, 495, 469, 505, 535, 539, 500, 535, 522, 517, 474, 515, 504, 541, 503, 461, 536, 501, 457, 528, 494, 528, 551, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[519, 544, 475, 489, 520, 518, 544, 502, 488, 510, 472, 503, 503, 502, 472, 548, 524, 527, 520, 536, 508, 489, 484, 540, 515, 526, 522, 499, 523, 528, 524, 532, 514, 541, 512, 537, 503, 540, 491, 538, 501, 504, 475, 471, 551, 500, 497, 518, 536, 501, 498, 506, 514, 559, 513, 494, 519, 503, 511, 472, 477, 566, 510, 539, 535, 497, 511, 505, 495, 496, 459, 504, 523, 509, 530, 515, 483, 538, 461, 503, 491, 500, 534, 518, 479, 536, 564, 521, 508, 531, 487, 515, 528, 545, 526, 538, 496, 551, 495, 547, 546, 467, 508, 497, 525, 505, 532, 490, 540, 501, 508, 481, 530, 500, 494, 509, 521, 554, 480, 524, 525, 481, 498, 499, 537, 471, 489, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[511, 494, 509, 516, 525, 523, 504, 518, 527, 525, 481, 523, 547, 541, 512, 509, 485, 527, 533, 538, 531, 511, 509, 519, 497, 530, 530, 502, 529, 531, 495, 480, 531, 530, 506, 508, 472, 558, 492, 522, 526, 481, 491, 542, 517, 493, 509, 499, 512, 531, 469, 526, 551, 483, 515, 506, 489, 505, 508, 485, 502, 494, 483, 523, 533, 515, 489, 464, 529, 517, 502, 513, 535, 547, 538, 514, 504, 498, 507, 513, 501, 535, 528, 446, 543, 539, 517, 563, 484, 539, 513, 510, 512, 540, 499, 506, 502, 533, 483, 509, 525, 546, 535, 498, 493, 538, 515, 498, 508, 463, 486, 495, 478, 506, 491, 540, 492, 485, 491, 576, 509, 521, 509, 504, 516, 487, 514, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[494, 510, 485, 501, 500, 490, 481, 502, 480, 508, 548, 510, 478, 506, 509, 505, 525, 477, 539, 507, 550, 487, 531, 493, 511, 530, 511, 500, 511, 503, 560, 512, 530, 473, 526, 520, 525, 478, 504, 529, 554, 458, 451, 532, 553, 491, 502, 522, 522, 540, 495, 517, 509, 515, 484, 492, 523, 513, 537, 523, 533, 487, 540, 530, 470, 530, 504, 536, 534, 504, 476, 517, 477, 543, 501, 453, 508, 507, 472, 477, 519, 510, 548, 514, 533, 521, 526, 510, 520, 518, 501, 538, 512, 519, 530, 522, 503, 516, 524, 512, 492, 506, 538, 533, 518, 503, 494, 517, 505, 528, 537, 514, 509, 516, 505, 534, 470, 527, 507, 527, 548, 519, 520, 525, 516, 520, 527, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[486, 521, 518, 552, 475, 539, 534, 503, 508, 518, 543, 516, 507, 514, 504, 465, 497, 528, 511, 487, 493, 499, 477, 525, 512, 571, 534, 493, 517, 461, 496, 512, 528, 505, 518, 480, 515, 538, 509, 532, 547, 515, 489, 567, 544, 500, 499, 530, 516, 480, 525, 471, 527, 513, 534, 482, 524, 503, 514, 504, 555, 535, 487, 537, 531, 495, 547, 485, 505, 514, 527, 531, 505, 515, 522, 489, 492, 506, 530, 497, 503, 509, 502, 515, 513, 526, 482, 517, 519, 493, 496, 544, 490, 513, 513, 496, 557, 531, 476, 492, 513, 513, 523, 521, 548, 495, 510, 540, 524, 489, 493, 507, 484, 508, 486, 488, 511, 509, 500, 551, 538, 493, 495, 521, 510, 518, 506, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[509, 564, 508, 499, 502, 510, 508, 484, 512, 524, 508, 491, 526, 468, 518, 536, 512, 509, 504, 488, 512, 506, 509, 528, 472, 487, 520, 508, 511, 484, 504, 511, 470, 493, 478, 509, 524, 514, 573, 528, 495, 528, 497, 506, 535, 486, 536, 493, 510, 509, 504, 496, 527, 520, 543, 537, 480, 535, 501, 520, 491, 523, 531, 539, 521, 554, 484, 503, 456, 521, 560, 540, 532, 436, 518, 497, 504, 548, 500, 511, 523, 507, 522, 539, 507, 504, 522, 517, 506, 514, 555, 529, 531, 492, 505, 526, 508, 522, 505, 554, 503, 542, 515, 506, 490, 512, 523, 474, 517, 518, 536, 494, 520, 509, 482, 483, 537, 502, 528, 496, 493, 507, 473, 487, 522, 538, 572, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[519, 544, 475, 489, 520, 518, 544, 502, 488, 510, 472, 503, 503, 502, 472, 548, 29, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 520, 536, 508, 489, 484, 540, 515, 507, 522, 499, 523, 528, 524, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 541, 512, 537, 503, 520, 491, 538, 501, 504, 475, 471, 551, 500, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 501, 498, 506, 514, 559, 513, 494, 519, 503, 511, 472, 477, 566, 510, 539, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 497, 511, 505, 495, 496, 459, 504, 523, 509, 530, 515, 483, 538, 461, 503, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 500, 534, 518, 479, 536, 564, 521, 508, 531, 487, 455, 528, 545, 526, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 551, 495, 547, 546, 467, 508, 497, 525, 505, 532, 490, 540, 501, 508, 481, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 500, 494, 509, 521, 554, 480, 524, 525, 481, 498, 499, 537, 471, 489, 533, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8278, 8173, 8224, 8095, 8261, 8207, 8159] avg=8192.0000 max=8278 max_rank=1 min=8095 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[511, 494, 509, 516, 525, 523, 504, 518, 527, 525, 481, 523, 547, 541, 512, 509, 29, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 527, 533, 538, 531, 511, 509, 519, 497, 511, 530, 502, 529, 531, 495, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 530, 506, 508, 472, 537, 492, 522, 526, 481, 491, 542, 517, 493, 509, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 531, 469, 526, 551, 483, 515, 506, 489, 505, 508, 485, 502, 494, 483, 523, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 515, 489, 464, 529, 517, 502, 513, 535, 547, 538, 514, 504, 498, 507, 513, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 535, 528, 446, 543, 539, 517, 563, 484, 539, 513, 450, 512, 540, 499, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 533, 483, 509, 525, 546, 535, 498, 493, 538, 515, 498, 508, 463, 486, 495, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 506, 491, 540, 492, 485, 491, 576, 509, 521, 509, 504, 516, 487, 514, 526, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8228, 8156, 8088, 8249, 8215, 8145, 8160] avg=8192.0000 max=8295 max_rank=0 min=8088 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[528, 499, 549, 517, 488, 465, 550, 529, 491, 528, 470, 480, 533, 527, 523, 506, 30, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 519, 530, 513, 519, 558, 476, 513, 536, 502, 522, 529, 532, 512, 499, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 492, 513, 533, 485, 524, 446, 494, 473, 524, 547, 523, 502, 524, 563, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 507, 521, 507, 537, 497, 519, 532, 508, 550, 494, 477, 527, 524, 522, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 510, 532, 511, 530, 514, 520, 534, 528, 514, 491, 491, 494, 512, 532, 515, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 503, 539, 505, 562, 493, 526, 514, 515, 500, 470, 533, 446, 547, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 496, 510, 481, 496, 519, 519, 485, 516, 515, 501, 513, 490, 476, 513, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 496, 520, 501, 511, 517, 514, 500, 497, 483, 535, 556, 518, 510, 522, 491, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8233, 8142, 8257, 8242, 8188, 8081, 8179] avg=8192.0000 max=8257 max_rank=3 min=8081 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 517, 530, 482, 509, 519, 485, 550, 476, 512, 470, 492, 495, 550, 569, 506, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 534, 531, 482, 496, 489, 506, 522, 532, 531, 513, 523, 551, 540, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 546, 506, 507, 532, 533, 519, 528, 451, 531, 526, 527, 508, 500, 523, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 534, 509, 503, 509, 525, 516, 487, 495, 474, 484, 529, 504, 505, 488, 459, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 526, 515, 513, 526, 484, 497, 540, 495, 497, 516, 476, 474, 499, 504, 548, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 487, 518, 528, 483, 512, 512, 485, 524, 514, 555, 486, 533, 536, 491, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 488, 500, 486, 533, 504, 494, 531, 481, 496, 500, 513, 528, 560, 533, 516, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 555, 491, 472, 503, 536, 481, 518, 516, 512, 499, 499, 518, 536, 504, 544, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8271, 8269, 8019, 8137, 8204, 8214, 8228] avg=8192.0000 max=8271 max_rank=1 min=8019 min_rank=3 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[494, 510, 485, 501, 500, 490, 481, 502, 480, 508, 548, 510, 478, 506, 509, 505, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 477, 539, 507, 550, 487, 531, 493, 511, 511, 511, 500, 511, 503, 560, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 473, 526, 520, 525, 461, 504, 529, 554, 458, 451, 532, 553, 491, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 540, 495, 517, 509, 515, 484, 492, 523, 513, 537, 523, 533, 487, 540, 530, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 530, 504, 536, 534, 504, 476, 517, 477, 543, 501, 453, 508, 507, 472, 477, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 548, 514, 533, 521, 526, 510, 520, 518, 501, 475, 512, 519, 530, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 524, 512, 492, 506, 538, 533, 518, 503, 494, 517, 505, 528, 537, 514, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 505, 534, 470, 527, 507, 527, 548, 519, 520, 525, 516, 520, 527, 514, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8039, 8228, 8131, 8265, 8041, 8278, 8258, 8296] avg=8192.0000 max=8296 max_rank=7 min=8039 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[545, 500, 512, 507, 503, 515, 503, 506, 528, 508, 526, 529, 506, 498, 492, 504, 34, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 488, 551, 469, 494, 503, 556, 481, 474, 530, 502, 525, 510, 514, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 584, 522, 514, 490, 513, 527, 527, 523, 493, 491, 469, 533, 493, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 481, 506, 486, 505, 519, 510, 512, 515, 507, 570, 524, 483, 536, 511, 530, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 525, 527, 531, 526, 487, 512, 498, 508, 536, 526, 484, 548, 512, 531, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 507, 540, 501, 466, 507, 523, 564, 510, 493, 463, 451, 497, 513, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 487, 514, 516, 483, 513, 497, 489, 497, 495, 469, 505, 535, 539, 500, 535, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 474, 515, 504, 541, 503, 461, 536, 501, 457, 528, 494, 528, 551, 517, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8159, 8223, 8198, 8331, 8086, 8143, 8174] avg=8192.0000 max=8331 max_rank=4 min=8086 min_rank=5 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[486, 521, 518, 552, 475, 539, 534, 503, 508, 518, 543, 516, 507, 514, 504, 465, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 528, 511, 487, 493, 499, 477, 525, 512, 550, 534, 493, 517, 461, 496, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 518, 480, 515, 518, 509, 532, 547, 515, 489, 567, 544, 500, 499, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 480, 525, 471, 527, 513, 534, 482, 524, 503, 514, 504, 555, 535, 487, 537, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 495, 547, 485, 505, 514, 527, 531, 505, 515, 522, 489, 492, 506, 530, 497, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 509, 502, 515, 513, 526, 482, 517, 519, 493, 496, 480, 490, 513, 513, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 531, 476, 492, 513, 513, 523, 521, 548, 495, 510, 540, 524, 489, 493, 507, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 486, 488, 511, 509, 500, 551, 538, 493, 495, 521, 510, 518, 506, 525, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8092, 8296, 8213, 8224, 8067, 8252, 8157] avg=8192.0000 max=8296 max_rank=2 min=8067 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 564, 508, 499, 502, 510, 508, 484, 512, 524, 508, 491, 526, 468, 518, 536, 30, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 504, 488, 512, 506, 509, 528, 472, 469, 520, 508, 511, 484, 504, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 493, 478, 509, 524, 495, 573, 528, 495, 528, 497, 506, 535, 486, 536, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 504, 496, 527, 520, 543, 537, 480, 535, 501, 520, 491, 523, 531, 539, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 554, 484, 503, 456, 521, 560, 540, 532, 436, 518, 497, 504, 548, 500, 511, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 522, 539, 507, 504, 522, 517, 506, 514, 555, 467, 531, 492, 505, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 522, 505, 554, 503, 542, 515, 506, 490, 512, 523, 474, 517, 518, 536, 494, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 482, 483, 537, 502, 528, 496, 493, 507, 473, 487, 522, 538, 572, 521, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8047, 8146, 8272, 8217, 8237, 8236, 8183] avg=8192.0000 max=8272 max_rank=3 min=8047 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 3968, 3968, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 3840, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[484, 562, 519, 497, 527, 510, 532, 517, 497, 487, 516, 517, 501, 496, 522, 516, 521, 549, 509, 488, 518, 498, 517, 490, 524, 551, 502, 497, 561, 491, 509, 539, 503, 490, 544, 510, 526, 466, 508, 529, 512, 531, 510, 503, 492, 500, 550, 531, 524, 519, 521, 517, 525, 489, 512, 479, 481, 493, 476, 498, 500, 536, 494, 493, 496, 484, 493, 526, 478, 462, 503, 541, 521, 512, 515, 508, 507, 512, 526, 475, 554, 492, 498, 542, 508, 532, 538, 536, 511, 501, 493, 562, 520, 506, 522, 506, 522, 484, 530, 505, 523, 506, 525, 518, 495, 518, 543, 493, 486, 533, 559, 515, 518, 481, 506, 566, 504, 509, 536, 495, 486, 487, 543, 475, 515, 521, 513, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[508, 503, 492, 512, 517, 504, 531, 500, 529, 541, 502, 518, 511, 509, 519, 485, 519, 509, 494, 489, 506, 508, 557, 502, 514, 514, 538, 561, 495, 494, 514, 550, 527, 509, 502, 524, 518, 510, 486, 516, 538, 509, 504, 544, 520, 502, 515, 521, 517, 525, 505, 514, 490, 533, 526, 500, 504, 473, 455, 520, 522, 520, 519, 560, 510, 465, 508, 535, 516, 497, 560, 524, 487, 507, 555, 509, 478, 545, 465, 518, 510, 536, 514, 513, 473, 494, 488, 537, 518, 461, 515, 539, 538, 484, 534, 527, 483, 512, 500, 531, 497, 480, 509, 512, 531, 543, 554, 513, 505, 508, 524, 467, 545, 480, 504, 551, 507, 470, 520, 495, 502, 512, 492, 521, 511, 514, 482, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[535, 521, 501, 491, 479, 507, 538, 492, 523, 508, 535, 498, 508, 482, 522, 540, 515, 516, 476, 490, 500, 543, 501, 496, 536, 499, 498, 492, 484, 565, 520, 551, 529, 536, 481, 500, 538, 500, 512, 490, 507, 475, 480, 461, 512, 501, 533, 503, 505, 493, 504, 516, 486, 511, 485, 555, 513, 549, 479, 539, 531, 531, 516, 518, 498, 497, 534, 499, 472, 476, 506, 516, 486, 528, 512, 494, 496, 505, 510, 502, 560, 519, 519, 513, 524, 511, 526, 502, 551, 489, 519, 515, 534, 524, 518, 518, 487, 527, 567, 505, 531, 521, 506, 540, 538, 532, 538, 480, 521, 497, 544, 536, 547, 520, 476, 526, 515, 502, 436, 480, 515, 484, 503, 522, 533, 541, 513, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[485, 535, 509, 482, 478, 467, 503, 494, 500, 509, 522, 521, 483, 509, 506, 508, 505, 524, 518, 535, 494, 519, 520, 466, 501, 516, 510, 540, 518, 521, 516, 521, 509, 512, 501, 504, 508, 545, 525, 532, 520, 512, 535, 521, 554, 519, 503, 537, 485, 533, 530, 519, 477, 512, 538, 532, 496, 514, 494, 546, 490, 564, 487, 541, 533, 516, 496, 490, 541, 519, 523, 522, 509, 519, 514, 517, 530, 487, 506, 496, 492, 543, 538, 532, 500, 520, 493, 538, 557, 492, 531, 517, 521, 498, 496, 495, 505, 476, 485, 500, 535, 493, 521, 514, 492, 502, 519, 514, 480, 531, 506, 516, 514, 531, 493, 476, 508, 503, 523, 515, 518, 462, 534, 551, 503, 492, 517, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[485, 519, 513, 485, 505, 536, 498, 524, 511, 498, 504, 509, 512, 495, 495, 503, 522, 551, 512, 504, 537, 480, 492, 536, 554, 519, 494, 493, 540, 553, 527, 469, 500, 516, 507, 515, 494, 538, 496, 493, 532, 532, 508, 521, 492, 478, 531, 496, 501, 516, 531, 554, 495, 496, 487, 526, 513, 502, 514, 491, 493, 513, 508, 528, 526, 549, 525, 526, 515, 519, 504, 521, 502, 519, 473, 519, 492, 534, 520, 507, 473, 509, 515, 512, 571, 518, 509, 492, 472, 481, 509, 528, 504, 506, 513, 545, 511, 496, 527, 536, 504, 506, 513, 521, 508, 529, 535, 504, 497, 515, 529, 494, 573, 576, 524, 529, 490, 501, 514, 514, 496, 518, 478, 523, 501, 487, 490, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[515, 547, 511, 533, 521, 550, 535, 497, 464, 484, 485, 492, 520, 510, 535, 524, 540, 525, 512, 481, 493, 505, 479, 501, 510, 521, 532, 553, 537, 497, 514, 549, 473, 516, 485, 469, 454, 505, 506, 487, 533, 523, 535, 521, 510, 518, 479, 496, 522, 518, 483, 491, 550, 511, 531, 497, 528, 511, 535, 497, 510, 518, 469, 517, 497, 502, 536, 506, 488, 499, 477, 485, 524, 517, 532, 503, 549, 465, 549, 506, 551, 560, 539, 547, 484, 556, 533, 478, 535, 497, 525, 495, 551, 520, 529, 503, 498, 537, 503, 493, 532, 502, 496, 544, 514, 522, 505, 494, 528, 502, 477, 498, 516, 493, 530, 499, 499, 530, 541, 536, 510, 475, 493, 547, 488, 503, 510, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[536, 534, 516, 513, 514, 566, 525, 473, 493, 510, 468, 548, 497, 526, 552, 507, 484, 489, 498, 500, 523, 493, 548, 485, 535, 529, 491, 476, 475, 481, 549, 503, 543, 503, 498, 517, 523, 476, 540, 478, 534, 559, 512, 508, 470, 512, 511, 483, 541, 516, 488, 539, 517, 501, 455, 567, 474, 524, 483, 558, 476, 484, 509, 498, 531, 510, 502, 522, 490, 524, 528, 487, 496, 528, 516, 499, 502, 497, 513, 538, 539, 513, 495, 512, 524, 522, 495, 509, 527, 540, 503, 530, 517, 517, 520, 520, 500, 489, 544, 527, 482, 487, 515, 542, 500, 518, 474, 525, 531, 543, 499, 505, 550, 501, 484, 524, 525, 532, 546, 515, 510, 500, 521, 535, 492, 509, 483, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[518, 483, 498, 505, 501, 509, 524, 493, 478, 508, 533, 510, 505, 524, 526, 480, 492, 499, 517, 511, 521, 504, 543, 491, 560, 498, 499, 529, 502, 531, 520, 513, 550, 512, 486, 490, 495, 512, 494, 510, 519, 523, 536, 529, 510, 508, 529, 536, 503, 502, 526, 508, 493, 508, 511, 474, 536, 539, 514, 503, 437, 530, 496, 500, 482, 532, 508, 457, 535, 504, 530, 532, 498, 521, 510, 533, 494, 527, 520, 513, 479, 514, 510, 499, 503, 513, 547, 550, 524, 505, 500, 543, 541, 505, 515, 501, 555, 535, 521, 556, 514, 511, 537, 482, 498, 529, 493, 534, 521, 507, 499, 520, 525, 499, 493, 529, 524, 503, 527, 508, 523, 486, 492, 498, 487, 485, 529, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[535, 521, 501, 491, 479, 507, 538, 492, 523, 508, 535, 498, 508, 482, 522, 540, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 516, 476, 490, 500, 543, 501, 496, 509, 499, 498, 492, 484, 565, 520, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 536, 481, 500, 538, 500, 512, 490, 507, 475, 480, 461, 512, 501, 533, 503, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 493, 504, 516, 486, 511, 485, 555, 513, 549, 479, 539, 531, 531, 516, 518, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 534, 499, 472, 476, 506, 516, 486, 528, 512, 494, 496, 505, 510, 502, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 519, 519, 513, 524, 511, 526, 502, 551, 489, 519, 436, 534, 524, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 527, 567, 505, 531, 521, 506, 540, 538, 507, 538, 480, 521, 497, 544, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 520, 476, 526, 515, 502, 436, 480, 515, 484, 503, 522, 533, 541, 513, 529, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8155, 8073, 8260, 8065, 8263, 8345, 8162] avg=8192.0000 max=8345 max_rank=6 min=8065 min_rank=4 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 519, 513, 485, 505, 536, 498, 524, 511, 498, 504, 509, 512, 495, 495, 503, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 551, 512, 504, 537, 480, 492, 536, 526, 519, 494, 493, 540, 553, 527, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 516, 507, 515, 494, 538, 496, 493, 532, 532, 508, 521, 492, 478, 531, 496, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 531, 554, 495, 496, 487, 526, 513, 502, 514, 491, 493, 513, 508, 528, 18, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 549, 525, 526, 515, 519, 504, 521, 502, 519, 473, 519, 492, 534, 520, 507, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 509, 515, 512, 571, 518, 509, 492, 472, 481, 509, 447, 504, 506, 513, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 496, 527, 536, 504, 506, 513, 521, 508, 504, 535, 504, 497, 515, 529, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 576, 524, 529, 490, 501, 514, 514, 496, 518, 478, 523, 501, 487, 490, 497, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8255, 8164, 8198, 8286, 8076, 8200, 8231] avg=8192.0000 max=8286 max_rank=4 min=8076 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 535, 509, 482, 478, 467, 503, 494, 500, 509, 522, 521, 483, 509, 506, 508, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 524, 518, 535, 494, 519, 520, 466, 476, 516, 510, 540, 518, 521, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 501, 504, 508, 545, 525, 532, 520, 512, 535, 521, 554, 519, 503, 537, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 533, 530, 519, 477, 512, 538, 532, 496, 514, 494, 546, 490, 564, 487, 541, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 516, 496, 490, 541, 519, 523, 522, 509, 519, 514, 517, 530, 487, 506, 496, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 543, 538, 532, 500, 520, 493, 538, 557, 492, 531, 438, 521, 498, 496, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 476, 485, 500, 535, 493, 521, 514, 492, 477, 519, 514, 480, 531, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 493, 476, 508, 503, 523, 515, 518, 462, 534, 551, 503, 492, 517, 496, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8044, 8199, 8352, 8286, 8252, 8184, 8064, 8155] avg=8192.0000 max=8352 max_rank=2 min=8044 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[484, 562, 519, 497, 527, 510, 532, 517, 497, 487, 516, 517, 501, 496, 522, 516, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 549, 509, 488, 518, 498, 517, 490, 490, 551, 502, 497, 561, 491, 509, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 490, 544, 510, 526, 466, 508, 529, 512, 531, 510, 503, 492, 500, 550, 531, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 519, 521, 517, 525, 489, 512, 479, 481, 493, 476, 498, 500, 536, 494, 493, 20, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 484, 493, 526, 478, 462, 503, 541, 521, 512, 515, 508, 507, 512, 526, 475, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 492, 498, 542, 508, 532, 538, 536, 511, 501, 493, 464, 520, 506, 522, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 484, 530, 505, 523, 506, 525, 518, 495, 484, 543, 493, 486, 533, 559, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 481, 506, 566, 504, 509, 536, 495, 486, 487, 543, 475, 515, 521, 513, 520, 14, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8230, 8225, 8094, 8100, 8223, 8221, 8203] avg=8192.0000 max=8240 max_rank=0 min=8094 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 483, 498, 505, 501, 509, 524, 493, 478, 508, 533, 510, 505, 524, 526, 480, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 499, 517, 511, 521, 504, 543, 491, 532, 498, 499, 529, 502, 531, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 512, 486, 490, 495, 512, 494, 510, 519, 523, 536, 529, 510, 508, 529, 536, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 502, 526, 508, 493, 508, 511, 474, 536, 539, 514, 503, 437, 530, 496, 500, 18, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 532, 508, 457, 535, 504, 530, 532, 498, 521, 510, 533, 494, 527, 520, 513, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 514, 510, 499, 503, 513, 547, 550, 524, 505, 500, 459, 541, 505, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 535, 521, 556, 514, 511, 537, 482, 498, 504, 493, 534, 521, 507, 499, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 499, 493, 529, 524, 503, 527, 508, 523, 486, 492, 498, 487, 485, 529, 527, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8202, 8254, 8111, 8232, 8165, 8287, 8155] avg=8192.0000 max=8287 max_rank=6 min=8111 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[536, 534, 516, 513, 514, 566, 525, 473, 493, 510, 468, 548, 497, 526, 552, 507, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 489, 498, 500, 523, 493, 548, 485, 508, 529, 491, 476, 475, 481, 549, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 498, 517, 523, 476, 540, 478, 534, 559, 512, 508, 470, 512, 511, 483, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 516, 488, 539, 517, 501, 455, 567, 474, 524, 483, 558, 476, 484, 509, 498, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 502, 522, 490, 524, 528, 487, 496, 528, 516, 499, 502, 497, 513, 538, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 513, 495, 512, 524, 522, 495, 509, 527, 540, 503, 449, 517, 517, 520, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 489, 544, 527, 482, 487, 515, 542, 500, 493, 474, 525, 531, 543, 499, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 501, 484, 524, 525, 532, 546, 515, 510, 500, 521, 535, 492, 509, 483, 528, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8312, 8032, 8182, 8159, 8218, 8202, 8156, 8275] avg=8192.0000 max=8312 max_rank=0 min=8032 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[515, 547, 511, 533, 521, 550, 535, 497, 464, 484, 485, 492, 520, 510, 535, 524, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 525, 512, 481, 493, 505, 479, 501, 485, 521, 532, 553, 537, 497, 514, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 516, 485, 469, 454, 505, 506, 487, 533, 523, 535, 521, 510, 518, 479, 496, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 483, 491, 550, 511, 531, 497, 528, 511, 535, 497, 510, 518, 469, 517, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 536, 506, 488, 499, 477, 485, 524, 517, 532, 503, 549, 465, 549, 506, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 560, 539, 547, 484, 556, 533, 478, 535, 497, 525, 418, 551, 520, 529, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 537, 503, 493, 532, 502, 496, 544, 514, 497, 505, 494, 528, 502, 477, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 530, 499, 499, 530, 541, 536, 510, 475, 493, 547, 488, 503, 510, 513, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8224, 8025, 8216, 8168, 8326, 8120, 8202] avg=8192.0000 max=8326 max_rank=5 min=8025 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[508, 503, 492, 512, 517, 504, 531, 500, 529, 541, 502, 518, 511, 509, 519, 485, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 509, 494, 489, 506, 508, 557, 502, 489, 514, 538, 561, 495, 494, 514, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 509, 502, 524, 518, 510, 486, 516, 538, 509, 504, 544, 520, 502, 515, 521, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 505, 514, 490, 533, 526, 500, 504, 473, 455, 520, 522, 520, 519, 560, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 465, 508, 535, 516, 497, 560, 524, 487, 507, 555, 509, 478, 545, 465, 518, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 514, 513, 473, 494, 488, 537, 518, 461, 515, 455, 538, 484, 534, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 512, 500, 531, 497, 480, 509, 512, 531, 517, 554, 513, 505, 508, 524, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 480, 504, 551, 507, 470, 520, 495, 502, 512, 492, 521, 511, 514, 482, 528, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8239, 8261, 8212, 8215, 8097, 8143, 8153] avg=8192.0000 max=8261 max_rank=2 min=8097 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 3584, 4352, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 3968, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4352, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[519, 520, 502, 553, 486, 540, 505, 502, 519, 494, 510, 514, 525, 508, 545, 522, 493, 513, 527, 478, 506, 532, 534, 499, 523, 488, 489, 499, 502, 547, 522, 531, 522, 519, 508, 506, 481, 545, 495, 526, 515, 495, 526, 505, 496, 499, 552, 540, 506, 509, 528, 517, 579, 497, 491, 514, 558, 522, 504, 521, 484, 510, 530, 535, 525, 536, 528, 484, 513, 483, 514, 556, 518, 551, 468, 513, 552, 524, 516, 539, 490, 524, 509, 483, 475, 498, 473, 504, 487, 504, 489, 498, 495, 492, 527, 494, 514, 496, 499, 498, 506, 490, 506, 504, 551, 517, 505, 523, 525, 506, 516, 485, 498, 502, 525, 524, 536, 515, 516, 489, 486, 500, 516, 531, 496, 476, 519, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[498, 489, 500, 511, 530, 546, 542, 466, 509, 512, 549, 470, 530, 503, 514, 466, 493, 504, 520, 507, 482, 525, 480, 514, 563, 529, 526, 523, 537, 542, 535, 491, 537, 490, 469, 515, 448, 520, 518, 518, 493, 509, 542, 536, 536, 503, 503, 516, 497, 496, 466, 533, 578, 518, 539, 562, 501, 537, 513, 550, 500, 503, 506, 529, 475, 506, 511, 489, 527, 499, 518, 524, 530, 498, 504, 511, 519, 516, 526, 488, 516, 516, 513, 515, 523, 511, 519, 510, 541, 492, 490, 488, 539, 465, 449, 519, 509, 517, 507, 486, 469, 490, 506, 532, 532, 546, 555, 527, 486, 483, 504, 505, 524, 479, 551, 524, 516, 507, 507, 532, 493, 522, 527, 510, 512, 510, 501, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[552, 516, 580, 515, 516, 509, 506, 514, 477, 501, 496, 517, 553, 514, 519, 509, 523, 512, 515, 506, 509, 461, 486, 506, 482, 520, 504, 529, 477, 530, 473, 544, 529, 542, 496, 512, 522, 544, 515, 494, 525, 530, 519, 566, 514, 480, 519, 517, 547, 506, 505, 502, 487, 485, 516, 553, 500, 517, 520, 468, 514, 545, 542, 519, 482, 501, 501, 475, 524, 495, 475, 499, 517, 536, 513, 518, 514, 512, 515, 525, 487, 539, 486, 464, 523, 548, 500, 491, 533, 498, 530, 513, 478, 501, 512, 509, 515, 501, 529, 517, 506, 502, 538, 557, 510, 541, 493, 511, 486, 485, 495, 494, 508, 551, 495, 531, 541, 524, 488, 512, 473, 503, 546, 503, 518, 520, 493, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[514, 516, 487, 511, 524, 535, 494, 534, 525, 539, 531, 525, 526, 525, 521, 453, 540, 537, 559, 517, 518, 500, 511, 549, 528, 506, 491, 519, 508, 499, 515, 525, 495, 523, 561, 488, 503, 507, 510, 535, 521, 512, 481, 469, 456, 489, 530, 502, 494, 504, 474, 518, 520, 519, 512, 515, 539, 482, 512, 464, 527, 511, 531, 532, 516, 474, 495, 462, 524, 526, 520, 498, 510, 526, 488, 531, 529, 525, 506, 520, 520, 495, 531, 522, 528, 497, 558, 520, 506, 533, 489, 513, 536, 488, 545, 505, 538, 489, 553, 506, 519, 518, 493, 504, 488, 485, 499, 521, 490, 556, 504, 510, 498, 504, 517, 489, 499, 489, 468, 523, 485, 530, 536, 510, 530, 492, 541, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[489, 552, 507, 490, 518, 522, 558, 487, 535, 505, 538, 464, 502, 551, 511, 493, 531, 471, 522, 527, 514, 489, 504, 465, 485, 516, 483, 534, 521, 456, 506, 536, 525, 526, 548, 508, 485, 532, 515, 523, 502, 499, 520, 502, 484, 517, 454, 501, 514, 525, 509, 506, 495, 535, 493, 501, 488, 486, 503, 546, 551, 510, 548, 544, 507, 552, 518, 525, 518, 486, 512, 512, 487, 524, 500, 521, 506, 513, 522, 536, 500, 511, 517, 496, 510, 499, 495, 505, 516, 508, 543, 490, 540, 539, 479, 543, 524, 513, 545, 510, 487, 536, 504, 513, 506, 501, 487, 532, 567, 537, 526, 520, 510, 506, 527, 486, 496, 543, 458, 528, 515, 522, 493, 477, 504, 515, 530, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[500, 522, 517, 526, 509, 508, 514, 538, 521, 480, 506, 557, 489, 480, 521, 534, 506, 515, 525, 536, 463, 550, 517, 514, 525, 515, 537, 539, 499, 512, 500, 486, 491, 511, 514, 511, 487, 535, 469, 477, 470, 510, 504, 503, 549, 526, 533, 512, 530, 502, 509, 498, 526, 495, 521, 502, 523, 538, 530, 508, 526, 542, 538, 520, 521, 507, 517, 549, 516, 556, 529, 508, 496, 481, 502, 489, 505, 500, 511, 508, 487, 559, 520, 506, 509, 505, 539, 519, 507, 498, 510, 548, 467, 533, 546, 508, 500, 512, 496, 570, 540, 516, 482, 523, 454, 511, 520, 467, 482, 528, 497, 494, 504, 521, 481, 523, 465, 514, 512, 526, 477, 499, 509, 494, 548, 498, 546, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[521, 509, 503, 490, 490, 500, 510, 469, 518, 528, 525, 510, 546, 529, 496, 521, 470, 501, 499, 503, 536, 480, 498, 528, 534, 480, 451, 555, 456, 512, 521, 516, 485, 520, 524, 519, 488, 514, 510, 521, 503, 518, 510, 481, 508, 513, 558, 511, 525, 551, 449, 484, 540, 508, 527, 545, 474, 532, 489, 547, 511, 497, 512, 542, 482, 513, 539, 512, 542, 539, 544, 521, 519, 540, 475, 512, 462, 487, 523, 504, 537, 537, 540, 523, 498, 509, 518, 492, 533, 500, 540, 506, 477, 535, 505, 529, 498, 508, 497, 516, 524, 520, 549, 513, 504, 487, 505, 502, 485, 497, 517, 474, 520, 508, 526, 547, 505, 515, 548, 546, 533, 520, 517, 478, 526, 512, 520, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[507, 474, 493, 520, 533, 502, 486, 510, 548, 521, 498, 525, 478, 442, 524, 531, 504, 514, 527, 524, 532, 525, 525, 494, 523, 520, 494, 539, 539, 511, 492, 520, 488, 536, 484, 535, 544, 495, 482, 559, 514, 497, 509, 505, 537, 526, 521, 504, 524, 488, 533, 517, 525, 510, 532, 497, 501, 499, 531, 546, 507, 525, 498, 501, 513, 473, 569, 519, 507, 508, 520, 496, 485, 490, 519, 526, 504, 533, 486, 491, 475, 506, 526, 523, 540, 521, 529, 521, 513, 476, 502, 497, 493, 502, 505, 480, 488, 509, 477, 508, 530, 503, 532, 526, 561, 477, 555, 475, 542, 531, 494, 566, 499, 525, 479, 528, 532, 515, 498, 484, 491, 536, 488, 524, 522, 495, 509, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[519, 520, 502, 553, 486, 527, 505, 502, 519, 494, 510, 514, 525, 508, 545, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 527, 478, 506, 532, 534, 499, 523, 488, 489, 499, 502, 547, 522, 531, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 519, 508, 506, 481, 545, 495, 526, 515, 495, 526, 505, 496, 499, 552, 540, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 528, 517, 511, 497, 491, 514, 558, 522, 504, 521, 484, 510, 530, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 536, 528, 484, 513, 483, 514, 556, 518, 551, 468, 513, 552, 524, 516, 539, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 524, 509, 483, 475, 498, 473, 504, 487, 504, 489, 498, 495, 492, 527, 494, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 496, 499, 498, 506, 490, 506, 504, 551, 517, 505, 523, 525, 506, 516, 485, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 525, 524, 536, 515, 516, 489, 486, 500, 516, 531, 496, 476, 519, 522, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8190, 8241, 8237, 8324, 7976, 8157, 8160] avg=8192.0000 max=8324 max_rank=4 min=7976 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[552, 516, 580, 515, 516, 496, 506, 514, 477, 501, 496, 517, 553, 514, 519, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 512, 515, 506, 509, 461, 486, 506, 482, 520, 504, 529, 477, 530, 473, 544, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 542, 496, 512, 522, 544, 515, 494, 525, 530, 519, 566, 514, 480, 519, 517, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 506, 505, 502, 431, 485, 516, 553, 500, 517, 520, 468, 514, 545, 542, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 501, 475, 524, 495, 475, 499, 517, 536, 513, 518, 514, 512, 515, 525, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 539, 486, 464, 523, 548, 500, 491, 533, 498, 530, 513, 478, 501, 512, 509, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 529, 517, 506, 502, 538, 557, 510, 541, 493, 511, 486, 485, 495, 494, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 551, 495, 531, 541, 524, 488, 512, 473, 503, 546, 503, 518, 520, 493, 515, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8084, 8333, 8170, 8106, 8140, 8193, 8229] avg=8192.0000 max=8333 max_rank=2 min=8084 min_rank=1 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 516, 487, 511, 524, 522, 494, 534, 525, 539, 531, 525, 526, 525, 521, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 537, 559, 517, 518, 500, 511, 549, 528, 506, 491, 519, 508, 499, 515, 525, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 561, 488, 503, 507, 510, 535, 521, 512, 481, 469, 456, 489, 530, 502, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 504, 474, 518, 460, 519, 512, 515, 539, 482, 512, 464, 527, 511, 531, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 474, 495, 462, 524, 526, 520, 498, 510, 526, 488, 531, 529, 525, 506, 520, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 495, 531, 522, 528, 497, 558, 520, 506, 533, 489, 513, 536, 488, 545, 505, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 553, 506, 519, 518, 493, 504, 488, 485, 499, 521, 490, 556, 504, 510, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 517, 489, 499, 489, 468, 523, 485, 530, 536, 510, 530, 492, 541, 498, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8329, 8092, 8094, 8154, 8316, 8187, 8117] avg=8192.0000 max=8329 max_rank=1 min=8092 min_rank=2 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[498, 489, 500, 511, 530, 532, 542, 466, 509, 512, 549, 470, 530, 503, 514, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 504, 520, 507, 482, 525, 480, 514, 563, 529, 526, 523, 537, 542, 535, 491, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 490, 469, 515, 448, 520, 518, 518, 493, 509, 542, 536, 536, 503, 503, 516, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 496, 466, 533, 510, 518, 539, 562, 501, 537, 513, 550, 500, 503, 506, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 506, 511, 489, 527, 499, 518, 524, 530, 498, 504, 511, 519, 516, 526, 488, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 513, 515, 523, 511, 519, 510, 541, 492, 490, 488, 539, 465, 449, 519, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 507, 486, 469, 490, 506, 532, 532, 546, 555, 527, 486, 483, 504, 505, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 551, 524, 516, 507, 507, 532, 493, 522, 527, 510, 512, 510, 501, 533, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8279, 8164, 8260, 8145, 8140, 8170, 8257] avg=8192.0000 max=8279 max_rank=1 min=8121 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[489, 552, 507, 490, 518, 496, 558, 487, 535, 505, 538, 464, 502, 551, 511, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 471, 522, 527, 514, 489, 504, 465, 485, 516, 483, 534, 521, 456, 506, 536, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 548, 508, 485, 532, 515, 523, 502, 499, 520, 502, 484, 517, 454, 501, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 525, 509, 506, 422, 535, 493, 501, 488, 486, 503, 546, 551, 510, 548, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 552, 518, 525, 518, 486, 512, 512, 487, 524, 500, 521, 506, 513, 522, 536, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 517, 496, 510, 499, 495, 505, 516, 508, 543, 490, 540, 539, 479, 543, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 545, 510, 487, 536, 504, 513, 506, 501, 487, 532, 567, 537, 526, 520, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 527, 486, 496, 543, 458, 528, 515, 522, 493, 477, 504, 515, 530, 511, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8071, 8155, 8181, 8247, 8224, 8325, 8137] avg=8192.0000 max=8325 max_rank=6 min=8071 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 509, 503, 490, 490, 487, 510, 469, 518, 528, 525, 510, 546, 529, 496, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 501, 499, 503, 536, 480, 498, 528, 534, 480, 451, 555, 456, 512, 521, 516, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 520, 524, 519, 488, 514, 510, 521, 503, 518, 510, 481, 508, 513, 558, 511, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 551, 449, 484, 479, 508, 527, 545, 474, 532, 489, 547, 511, 497, 512, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 513, 539, 512, 542, 539, 544, 521, 519, 540, 475, 512, 462, 487, 523, 504, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 537, 540, 523, 498, 509, 518, 492, 533, 500, 540, 506, 477, 535, 505, 529, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 497, 516, 524, 520, 549, 513, 504, 487, 505, 502, 485, 497, 517, 474, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 526, 547, 505, 515, 548, 546, 533, 520, 517, 478, 526, 512, 520, 505, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8047, 8193, 8172, 8218, 8310, 8110, 8334] avg=8192.0000 max=8334 max_rank=7 min=8047 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[500, 522, 517, 526, 509, 495, 514, 538, 521, 480, 506, 557, 489, 480, 521, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 525, 536, 463, 550, 517, 514, 525, 515, 537, 539, 499, 512, 500, 486, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 511, 514, 511, 487, 535, 469, 477, 470, 510, 504, 503, 549, 526, 533, 512, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 502, 509, 498, 466, 495, 521, 502, 523, 538, 530, 508, 526, 542, 538, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 507, 517, 549, 516, 556, 529, 508, 496, 481, 502, 489, 505, 500, 511, 508, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 559, 520, 506, 509, 505, 539, 519, 507, 498, 510, 548, 467, 533, 546, 508, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 496, 570, 540, 516, 482, 523, 454, 511, 520, 467, 482, 528, 497, 494, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 481, 523, 465, 514, 512, 526, 477, 499, 509, 494, 548, 498, 546, 500, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8246, 8112, 8248, 8199, 8291, 8106, 8125] avg=8192.0000 max=8291 max_rank=5 min=8106 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[507, 474, 493, 520, 533, 489, 486, 510, 548, 521, 498, 525, 478, 442, 524, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 527, 524, 532, 525, 525, 494, 523, 520, 494, 539, 539, 511, 492, 520, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 536, 484, 535, 544, 495, 482, 559, 514, 497, 509, 505, 537, 526, 521, 504, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 488, 533, 517, 465, 510, 532, 497, 501, 499, 531, 546, 507, 525, 498, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 473, 569, 519, 507, 508, 520, 496, 485, 490, 519, 526, 504, 533, 486, 491, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 506, 526, 523, 540, 521, 529, 521, 513, 476, 502, 497, 493, 502, 505, 480, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 509, 477, 508, 530, 503, 532, 526, 561, 477, 555, 475, 542, 531, 494, 566, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 525, 479, 528, 532, 515, 498, 484, 491, 536, 488, 524, 522, 495, 509, 544, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8079, 8290, 8246, 8174, 8143, 8139, 8288, 8177] avg=8192.0000 max=8290 max_rank=1 min=8079 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 3840, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4352, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[517, 527, 523, 489, 543, 485, 512, 459, 523, 484, 503, 480, 527, 486, 523, 550, 478, 526, 485, 542, 484, 524, 521, 532, 455, 537, 509, 485, 544, 524, 492, 500, 510, 507, 514, 506, 554, 533, 512, 493, 516, 542, 470, 511, 538, 482, 520, 499, 517, 525, 527, 555, 562, 542, 511, 522, 510, 553, 498, 507, 513, 538, 489, 516, 541, 526, 458, 510, 553, 516, 513, 504, 490, 520, 524, 515, 522, 512, 539, 523, 497, 505, 530, 524, 497, 492, 503, 536, 501, 511, 515, 491, 478, 496, 482, 512, 529, 546, 495, 528, 493, 528, 517, 523, 522, 502, 514, 493, 518, 509, 456, 515, 528, 505, 528, 499, 525, 493, 502, 522, 500, 478, 509, 543, 486, 480, 514, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[498, 560, 521, 509, 524, 521, 526, 535, 540, 505, 502, 556, 513, 508, 513, 495, 543, 502, 529, 506, 509, 506, 480, 502, 519, 483, 515, 534, 524, 499, 513, 514, 516, 507, 478, 513, 531, 488, 468, 520, 510, 511, 525, 533, 489, 540, 488, 493, 543, 503, 520, 468, 497, 533, 478, 529, 512, 495, 527, 535, 509, 493, 511, 520, 496, 502, 459, 502, 506, 487, 509, 490, 519, 527, 532, 537, 490, 476, 527, 546, 542, 493, 517, 535, 484, 525, 519, 502, 498, 484, 474, 501, 510, 533, 488, 534, 529, 490, 496, 529, 516, 533, 506, 530, 536, 530, 521, 474, 502, 529, 521, 510, 468, 540, 530, 528, 526, 525, 515, 538, 503, 511, 508, 487, 537, 510, 538, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[533, 503, 517, 502, 526, 499, 558, 528, 504, 542, 521, 475, 533, 501, 492, 551, 485, 478, 502, 507, 514, 512, 516, 527, 466, 499, 505, 488, 494, 522, 503, 501, 536, 483, 546, 484, 578, 504, 538, 512, 477, 476, 502, 519, 530, 522, 475, 506, 459, 499, 525, 497, 538, 558, 498, 524, 475, 517, 493, 499, 515, 503, 557, 502, 500, 529, 516, 539, 477, 504, 492, 519, 513, 527, 539, 557, 517, 499, 509, 473, 542, 522, 523, 455, 512, 532, 501, 516, 542, 492, 472, 495, 523, 543, 536, 543, 519, 522, 533, 508, 531, 566, 528, 507, 536, 486, 518, 507, 462, 499, 490, 479, 507, 517, 537, 515, 498, 520, 539, 514, 522, 562, 494, 479, 536, 476, 513, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[502, 524, 512, 471, 479, 503, 501, 492, 557, 529, 493, 566, 531, 534, 538, 482, 483, 545, 534, 545, 496, 543, 504, 546, 478, 530, 532, 486, 452, 499, 502, 480, 497, 516, 515, 514, 492, 511, 517, 478, 484, 517, 475, 490, 514, 473, 508, 533, 488, 492, 545, 506, 554, 492, 528, 521, 547, 494, 495, 538, 509, 539, 481, 484, 490, 504, 519, 537, 537, 491, 505, 460, 526, 562, 512, 557, 476, 472, 547, 501, 526, 511, 504, 525, 542, 502, 537, 519, 513, 549, 517, 505, 503, 566, 504, 489, 548, 533, 470, 512, 517, 482, 540, 475, 488, 550, 484, 500, 476, 480, 494, 568, 510, 490, 556, 506, 517, 540, 482, 510, 570, 533, 492, 511, 542, 520, 543, 473] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[521, 504, 515, 474, 503, 498, 527, 512, 543, 487, 540, 497, 499, 499, 537, 487, 511, 525, 495, 506, 488, 501, 540, 494, 481, 482, 515, 552, 530, 542, 483, 511, 499, 481, 494, 535, 522, 491, 493, 554, 506, 499, 513, 519, 508, 535, 502, 536, 540, 530, 513, 527, 527, 493, 535, 521, 520, 485, 500, 493, 521, 532, 476, 503, 541, 512, 503, 533, 527, 507, 465, 500, 541, 479, 523, 498, 502, 498, 510, 525, 481, 529, 493, 504, 522, 520, 495, 528, 527, 528, 490, 552, 546, 567, 462, 502, 494, 536, 484, 526, 533, 544, 503, 523, 496, 523, 521, 515, 526, 525, 486, 505, 494, 530, 542, 502, 512, 528, 515, 478, 502, 539, 526, 490, 491, 518, 493, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[527, 550, 476, 484, 503, 505, 503, 499, 497, 513, 498, 559, 532, 491, 498, 483, 489, 503, 555, 493, 497, 526, 527, 528, 529, 496, 528, 515, 545, 477, 499, 487, 513, 513, 540, 500, 551, 514, 516, 488, 461, 499, 495, 526, 499, 486, 523, 485, 502, 564, 503, 531, 523, 523, 527, 478, 512, 508, 546, 530, 499, 494, 512, 493, 522, 516, 491, 496, 502, 480, 508, 535, 543, 504, 502, 521, 496, 487, 559, 501, 527, 546, 500, 499, 504, 523, 480, 473, 527, 554, 486, 507, 509, 516, 535, 473, 541, 504, 528, 527, 525, 518, 497, 535, 494, 505, 521, 497, 530, 508, 503, 556, 513, 481, 505, 506, 559, 529, 508, 482, 541, 545, 528, 524, 492, 514, 511, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[524, 520, 489, 468, 501, 509, 484, 517, 467, 503, 510, 501, 556, 503, 529, 502, 477, 522, 536, 519, 554, 512, 496, 516, 527, 510, 536, 507, 486, 508, 519, 495, 536, 517, 490, 503, 474, 514, 540, 469, 499, 504, 477, 511, 517, 494, 546, 499, 495, 486, 517, 511, 484, 501, 488, 508, 542, 526, 512, 512, 489, 446, 518, 481, 481, 520, 500, 513, 493, 575, 511, 509, 521, 491, 478, 520, 482, 512, 532, 540, 535, 468, 513, 499, 509, 533, 515, 522, 511, 550, 501, 510, 526, 517, 546, 522, 534, 520, 511, 542, 560, 551, 525, 541, 514, 482, 530, 512, 487, 503, 494, 551, 506, 545, 501, 521, 498, 492, 491, 511, 534, 538, 541, 532, 508, 516, 543, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[507, 541, 515, 554, 506, 515, 520, 489, 575, 525, 468, 479, 544, 518, 521, 527, 514, 496, 550, 483, 507, 507, 518, 510, 494, 511, 495, 493, 481, 518, 504, 512, 506, 527, 491, 485, 553, 537, 488, 509, 494, 510, 506, 492, 531, 490, 477, 505, 496, 489, 496, 522, 502, 525, 511, 561, 507, 506, 478, 530, 531, 530, 522, 537, 525, 505, 518, 475, 538, 489, 546, 509, 502, 508, 483, 491, 502, 520, 544, 523, 518, 564, 533, 545, 508, 532, 535, 473, 518, 547, 546, 519, 511, 480, 513, 516, 511, 478, 493, 496, 454, 552, 491, 558, 491, 488, 480, 508, 540, 507, 476, 526, 523, 517, 526, 517, 517, 462, 532, 532, 535, 522, 498, 507, 481, 509, 485, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 527, 523, 489, 543, 485, 512, 459, 523, 484, 503, 480, 519, 486, 523, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 526, 485, 542, 484, 524, 521, 532, 455, 537, 509, 485, 544, 524, 492, 500, 18, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 514, 506, 554, 533, 512, 493, 516, 542, 470, 511, 538, 482, 520, 499, 41, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 527, 555, 547, 542, 511, 522, 510, 553, 498, 507, 513, 538, 489, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 526, 458, 510, 553, 516, 513, 504, 490, 520, 524, 515, 522, 512, 539, 523, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 505, 530, 524, 497, 492, 503, 536, 501, 511, 515, 491, 478, 465, 482, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 546, 495, 528, 493, 510, 517, 523, 522, 502, 514, 493, 518, 509, 456, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 528, 499, 525, 493, 502, 522, 500, 437, 509, 543, 486, 480, 514, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8185, 8264, 8370, 8275, 8039, 8170, 8110] avg=8192.0000 max=8370 max_rank=3 min=8039 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 503, 517, 502, 526, 499, 558, 528, 504, 542, 521, 475, 525, 501, 492, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 478, 502, 507, 514, 512, 516, 527, 466, 499, 505, 488, 494, 522, 503, 501, 19, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 483, 546, 484, 578, 504, 538, 512, 477, 476, 502, 519, 530, 522, 475, 506, 48, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 499, 525, 497, 524, 558, 498, 524, 475, 517, 493, 499, 515, 503, 557, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 529, 516, 539, 477, 504, 492, 519, 513, 527, 539, 557, 517, 499, 509, 473, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 522, 523, 455, 512, 532, 501, 516, 542, 492, 472, 495, 523, 509, 536, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 522, 533, 508, 531, 547, 528, 507, 536, 486, 518, 507, 462, 499, 490, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 537, 515, 498, 520, 539, 514, 522, 514, 494, 479, 536, 476, 513, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8067, 8254, 8145, 8219, 8215, 8172, 8187] avg=8192.0000 max=8277 max_rank=0 min=8067 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[498, 560, 521, 509, 524, 521, 526, 535, 540, 505, 502, 556, 501, 508, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 502, 529, 506, 509, 506, 480, 502, 519, 483, 515, 534, 524, 499, 513, 514, 23, 21, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 478, 513, 531, 488, 468, 520, 510, 511, 525, 533, 489, 540, 488, 493, 47, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 520, 468, 478, 533, 478, 529, 512, 495, 527, 535, 509, 493, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 502, 459, 502, 506, 487, 509, 490, 519, 527, 532, 537, 490, 476, 527, 546, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 493, 517, 535, 484, 525, 519, 502, 498, 484, 474, 501, 510, 490, 488, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 490, 496, 529, 516, 510, 506, 530, 536, 530, 521, 474, 502, 529, 521, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 540, 530, 528, 526, 525, 515, 538, 503, 464, 508, 487, 537, 510, 538, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8314, 8237, 8179, 8154, 8121, 8096, 8229, 8206] avg=8192.0000 max=8314 max_rank=0 min=8096 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[521, 504, 515, 474, 503, 498, 527, 512, 543, 487, 540, 497, 491, 499, 537, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 525, 495, 506, 488, 501, 540, 494, 481, 482, 515, 552, 530, 542, 483, 511, 18, 17, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 481, 494, 535, 522, 491, 493, 554, 506, 499, 513, 519, 508, 535, 502, 536, 46, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 530, 513, 527, 513, 493, 535, 521, 520, 485, 500, 493, 521, 532, 476, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 512, 503, 533, 527, 507, 465, 500, 541, 479, 523, 498, 502, 498, 510, 525, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 529, 493, 504, 522, 520, 495, 528, 527, 528, 490, 552, 546, 531, 462, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 536, 484, 526, 533, 526, 503, 523, 496, 523, 521, 515, 526, 525, 486, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 530, 542, 502, 512, 528, 515, 478, 502, 493, 526, 490, 491, 518, 493, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8204, 8252, 8202, 8173, 8210, 8222, 8138] avg=8192.0000 max=8252 max_rank=2 min=8135 min_rank=0 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[502, 524, 512, 471, 479, 503, 501, 492, 557, 529, 493, 566, 523, 534, 538, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 545, 534, 545, 496, 543, 504, 546, 478, 530, 532, 486, 452, 499, 502, 480, 16, 17, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 516, 515, 514, 492, 511, 517, 478, 484, 517, 475, 490, 514, 473, 508, 533, 46, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 492, 545, 506, 539, 492, 528, 521, 547, 494, 495, 538, 509, 539, 481, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 504, 519, 537, 537, 491, 505, 460, 526, 562, 512, 557, 476, 472, 547, 501, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 511, 504, 525, 542, 502, 537, 519, 513, 549, 517, 505, 503, 530, 504, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 533, 470, 512, 517, 466, 540, 475, 488, 550, 484, 500, 476, 480, 494, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 556, 506, 517, 540, 482, 510, 570, 487, 492, 511, 542, 520, 543, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8202, 8099, 8198, 8205, 8276, 8101, 8249] avg=8192.0000 max=8276 max_rank=5 min=8099 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[524, 520, 489, 468, 501, 509, 484, 517, 467, 503, 510, 501, 548, 503, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 522, 536, 519, 554, 512, 496, 516, 527, 510, 536, 507, 486, 508, 519, 495, 18, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 517, 490, 503, 474, 514, 540, 469, 499, 504, 477, 511, 517, 494, 546, 499, 46, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 486, 517, 511, 472, 501, 488, 508, 542, 526, 512, 512, 489, 446, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 520, 500, 513, 493, 575, 511, 509, 521, 491, 478, 520, 482, 512, 532, 540, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 468, 513, 499, 509, 533, 515, 522, 511, 550, 501, 510, 526, 485, 546, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 520, 511, 542, 560, 533, 525, 541, 514, 482, 530, 512, 487, 503, 494, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 545, 501, 521, 498, 492, 491, 511, 534, 492, 541, 532, 508, 516, 543, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8265, 8153, 8004, 8186, 8245, 8339, 8269] avg=8192.0000 max=8339 max_rank=6 min=8004 min_rank=3 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=33039 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[507, 541, 515, 554, 506, 515, 520, 489, 575, 525, 468, 479, 536, 518, 521, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 496, 550, 483, 507, 507, 518, 510, 494, 511, 495, 493, 481, 518, 504, 512, 18, 14, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 527, 491, 485, 553, 537, 488, 509, 494, 510, 506, 492, 531, 490, 477, 505, 45, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 489, 496, 522, 490, 525, 511, 561, 507, 506, 478, 530, 531, 530, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 518, 475, 538, 489, 546, 509, 502, 508, 483, 491, 502, 520, 544, 523, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 564, 533, 545, 508, 532, 535, 473, 518, 547, 546, 519, 511, 450, 513, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 478, 493, 496, 454, 534, 491, 558, 491, 488, 480, 508, 540, 507, 476, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 517, 526, 517, 517, 462, 532, 532, 535, 477, 498, 507, 481, 509, 485, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8137, 8162, 8231, 8186, 8328, 8031, 8165] avg=8192.0000 max=8328 max_rank=5 min=8031 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[527, 550, 476, 484, 503, 505, 503, 499, 497, 513, 498, 559, 524, 491, 498, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 503, 555, 493, 497, 526, 527, 528, 529, 496, 528, 515, 545, 477, 499, 487, 17, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 540, 500, 551, 514, 516, 488, 461, 499, 495, 526, 499, 486, 523, 485, 47, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 564, 503, 531, 510, 523, 527, 478, 512, 508, 546, 530, 499, 494, 512, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 491, 496, 502, 480, 508, 535, 543, 504, 502, 521, 496, 487, 559, 501, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 546, 500, 499, 504, 523, 480, 473, 527, 554, 486, 507, 509, 484, 535, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 504, 528, 527, 525, 501, 497, 535, 494, 505, 521, 497, 530, 508, 503, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 481, 505, 506, 559, 529, 508, 482, 541, 498, 528, 524, 492, 514, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8239, 8173, 8232, 8171, 8127, 8272, 8212] avg=8192.0000 max=8272 max_rank=6 min=8110 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4352, 4096, 3968, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4352, 4096, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4352, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 3968, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[526, 485, 533, 517, 480, 520, 500, 541, 491, 497, 514, 493, 529, 465, 533, 498, 516, 537, 510, 530, 477, 480, 567, 489, 491, 534, 484, 521, 513, 526, 561, 548, 514, 526, 533, 492, 516, 482, 489, 511, 516, 470, 489, 528, 491, 513, 503, 516, 532, 473, 506, 508, 504, 510, 519, 522, 543, 537, 458, 511, 495, 538, 492, 516, 555, 543, 531, 468, 476, 486, 527, 515, 499, 504, 517, 495, 480, 524, 500, 527, 489, 535, 515, 485, 541, 486, 542, 535, 502, 571, 498, 524, 501, 523, 515, 528, 518, 513, 531, 489, 498, 515, 491, 502, 511, 509, 531, 521, 527, 543, 499, 515, 543, 528, 514, 550, 497, 541, 488, 498, 485, 509, 513, 540, 511, 500, 463, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[481, 515, 478, 513, 468, 530, 511, 484, 499, 519, 518, 493, 539, 492, 524, 470, 562, 520, 506, 476, 487, 454, 483, 500, 509, 507, 510, 548, 517, 526, 524, 531, 543, 525, 516, 541, 507, 518, 548, 528, 511, 476, 509, 518, 504, 556, 514, 554, 499, 508, 511, 492, 516, 498, 517, 534, 523, 493, 502, 497, 520, 528, 490, 566, 504, 497, 496, 526, 511, 543, 472, 486, 507, 475, 527, 535, 534, 498, 497, 506, 489, 510, 511, 523, 548, 510, 481, 487, 506, 509, 537, 496, 554, 485, 536, 510, 528, 517, 522, 503, 526, 495, 497, 498, 548, 456, 547, 481, 543, 537, 519, 536, 509, 498, 545, 520, 489, 519, 522, 500, 491, 492, 506, 523, 572, 500, 529, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[502, 508, 533, 508, 523, 516, 530, 530, 538, 508, 472, 527, 504, 492, 527, 488, 512, 518, 497, 557, 491, 492, 516, 538, 516, 501, 517, 521, 463, 542, 519, 478, 525, 530, 506, 516, 489, 496, 519, 493, 476, 500, 539, 510, 509, 530, 496, 502, 491, 533, 516, 552, 521, 507, 498, 521, 500, 511, 535, 520, 521, 531, 507, 555, 501, 595, 529, 520, 507, 464, 504, 507, 528, 517, 513, 491, 495, 507, 503, 520, 528, 477, 506, 489, 501, 505, 548, 510, 507, 545, 510, 527, 506, 502, 516, 562, 528, 487, 452, 548, 485, 549, 509, 500, 519, 489, 521, 508, 454, 505, 497, 530, 525, 473, 493, 508, 518, 509, 538, 525, 506, 524, 501, 517, 530, 503, 523, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[574, 509, 525, 535, 544, 550, 521, 455, 529, 512, 546, 512, 492, 553, 499, 520, 560, 490, 543, 538, 508, 486, 479, 523, 525, 525, 513, 499, 561, 548, 509, 489, 538, 500, 471, 513, 511, 534, 506, 503, 511, 519, 544, 512, 504, 512, 508, 499, 515, 510, 491, 506, 500, 535, 475, 553, 512, 498, 518, 539, 543, 498, 490, 539, 526, 513, 461, 486, 505, 489, 559, 490, 505, 523, 519, 539, 531, 551, 487, 521, 458, 513, 473, 463, 504, 541, 504, 502, 521, 479, 508, 524, 485, 532, 499, 471, 474, 512, 522, 518, 499, 544, 484, 536, 525, 519, 481, 496, 504, 524, 493, 546, 520, 485, 511, 496, 520, 514, 533, 514, 528, 500, 500, 512, 485, 499, 491, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[548, 516, 533, 565, 461, 509, 545, 507, 517, 509, 523, 511, 479, 485, 494, 525, 544, 561, 513, 490, 512, 494, 491, 496, 508, 529, 499, 527, 484, 507, 489, 481, 513, 483, 526, 544, 518, 473, 497, 509, 517, 541, 503, 526, 512, 490, 492, 462, 506, 513, 536, 481, 525, 484, 512, 494, 494, 529, 524, 503, 500, 522, 517, 507, 502, 515, 498, 491, 514, 527, 504, 511, 478, 517, 539, 511, 515, 499, 486, 525, 508, 529, 520, 518, 573, 522, 536, 546, 506, 537, 545, 504, 542, 497, 487, 518, 529, 507, 493, 506, 469, 539, 526, 491, 520, 526, 505, 537, 499, 520, 527, 536, 505, 484, 509, 531, 476, 504, 513, 508, 485, 531, 558, 504, 518, 511, 513, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[504, 491, 507, 528, 521, 503, 521, 477, 501, 514, 504, 494, 536, 564, 522, 506, 488, 494, 545, 493, 536, 532, 499, 512, 502, 505, 500, 514, 528, 506, 540, 486, 550, 491, 481, 513, 466, 516, 539, 494, 513, 521, 544, 506, 551, 545, 514, 502, 502, 472, 493, 510, 522, 511, 492, 486, 513, 519, 496, 509, 515, 513, 550, 485, 530, 502, 488, 469, 498, 537, 494, 520, 506, 538, 503, 557, 493, 524, 557, 503, 505, 527, 503, 511, 487, 495, 530, 512, 523, 489, 524, 516, 498, 510, 515, 510, 498, 511, 509, 512, 515, 487, 507, 518, 510, 501, 531, 553, 512, 537, 521, 498, 534, 487, 546, 512, 559, 476, 556, 534, 507, 515, 551, 492, 516, 500, 479, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[500, 499, 532, 526, 480, 506, 538, 507, 471, 516, 511, 499, 473, 523, 496, 512, 516, 537, 549, 516, 508, 481, 524, 501, 508, 502, 501, 471, 518, 508, 519, 524, 531, 522, 514, 549, 534, 520, 500, 542, 493, 531, 466, 515, 501, 474, 490, 517, 494, 542, 528, 489, 517, 525, 482, 529, 547, 507, 523, 468, 514, 495, 554, 481, 531, 567, 537, 485, 501, 519, 535, 529, 513, 498, 476, 498, 502, 509, 501, 525, 499, 534, 526, 531, 486, 480, 511, 497, 497, 500, 521, 532, 526, 519, 516, 484, 520, 499, 550, 498, 499, 542, 533, 510, 487, 532, 538, 490, 507, 497, 509, 521, 554, 530, 546, 487, 489, 526, 491, 487, 493, 532, 518, 538, 548, 471, 491, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[499, 505, 516, 514, 555, 527, 503, 516, 514, 472, 549, 501, 527, 514, 496, 508, 504, 550, 489, 553, 513, 488, 527, 501, 475, 496, 470, 538, 502, 495, 497, 519, 487, 489, 494, 530, 532, 504, 465, 524, 519, 534, 523, 537, 478, 504, 505, 507, 508, 494, 516, 498, 485, 510, 502, 533, 518, 503, 552, 519, 492, 548, 534, 499, 537, 510, 481, 506, 493, 525, 552, 500, 522, 477, 490, 509, 506, 536, 521, 529, 525, 531, 505, 525, 506, 519, 487, 534, 504, 579, 533, 502, 489, 528, 495, 530, 490, 481, 516, 520, 499, 514, 537, 516, 475, 521, 525, 532, 500, 512, 484, 502, 497, 508, 524, 522, 509, 498, 508, 529, 515, 522, 539, 529, 497, 533, 523, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[574, 509, 525, 535, 544, 550, 521, 455, 529, 512, 546, 512, 492, 553, 499, 520, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 490, 543, 538, 508, 486, 479, 523, 525, 525, 513, 499, 561, 548, 509, 489, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 500, 471, 513, 511, 534, 506, 503, 511, 519, 544, 512, 504, 512, 508, 499, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 491, 506, 500, 535, 475, 553, 512, 498, 518, 539, 543, 498, 490, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 513, 461, 486, 505, 489, 559, 490, 505, 523, 519, 539, 531, 551, 487, 521, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 513, 473, 463, 504, 541, 504, 502, 521, 462, 508, 524, 485, 532, 499, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 512, 522, 518, 499, 544, 484, 536, 525, 519, 481, 496, 504, 524, 493, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 485, 500, 496, 520, 514, 533, 514, 528, 500, 500, 512, 485, 499, 491, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8384, 8297, 8193, 8222, 8216, 7960, 8177, 8087] avg=8192.0000 max=8384 max_rank=0 min=7960 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[548, 516, 533, 565, 461, 509, 545, 507, 517, 509, 523, 511, 479, 485, 494, 525, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 561, 513, 490, 512, 494, 491, 496, 508, 529, 499, 527, 484, 507, 489, 481, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 483, 526, 544, 518, 473, 497, 509, 517, 541, 503, 526, 512, 490, 492, 462, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 536, 481, 525, 484, 512, 494, 494, 529, 524, 503, 500, 522, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 498, 491, 514, 527, 504, 511, 478, 517, 539, 511, 515, 499, 486, 525, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 529, 520, 518, 573, 522, 536, 546, 506, 518, 545, 504, 542, 497, 487, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 493, 506, 469, 539, 526, 491, 520, 526, 505, 537, 499, 520, 527, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 484, 498, 531, 476, 504, 513, 508, 485, 531, 558, 504, 518, 511, 513, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8126, 8115, 8147, 8144, 8369, 8230, 8170] avg=8192.0000 max=8369 max_rank=5 min=8115 min_rank=2 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[504, 491, 507, 528, 521, 503, 521, 477, 501, 514, 504, 494, 536, 564, 522, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 494, 545, 493, 536, 532, 499, 512, 502, 505, 500, 514, 528, 506, 540, 486, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 491, 481, 513, 466, 516, 539, 494, 513, 521, 544, 506, 551, 545, 514, 502, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 472, 493, 510, 522, 511, 492, 486, 513, 519, 496, 509, 515, 513, 550, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 502, 488, 469, 498, 537, 494, 520, 506, 538, 503, 557, 493, 524, 557, 503, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 503, 511, 487, 495, 530, 512, 523, 472, 524, 516, 498, 510, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 511, 509, 512, 515, 487, 507, 518, 510, 501, 531, 553, 512, 537, 521, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 487, 534, 512, 559, 476, 556, 534, 507, 515, 551, 492, 516, 500, 479, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8181, 8254, 8088, 8230, 8138, 8220, 8223] avg=8192.0000 max=8254 max_rank=2 min=8088 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[481, 515, 478, 513, 468, 530, 511, 484, 499, 519, 518, 493, 539, 492, 524, 470, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 520, 506, 476, 487, 454, 483, 500, 509, 507, 510, 548, 517, 526, 524, 531, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 525, 516, 541, 507, 518, 548, 528, 511, 476, 509, 518, 504, 556, 514, 554, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 511, 492, 516, 498, 517, 534, 523, 493, 502, 497, 520, 524, 490, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 496, 526, 511, 543, 472, 486, 507, 475, 527, 535, 534, 498, 497, 506, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 510, 511, 523, 548, 510, 481, 487, 506, 482, 537, 496, 554, 485, 536, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 517, 522, 503, 526, 495, 497, 498, 548, 456, 547, 481, 543, 537, 519, 536, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 520, 520, 489, 519, 522, 500, 491, 492, 506, 523, 572, 500, 529, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 8166, 8383, 8190, 8129, 8165, 8259, 8196] avg=8192.0000 max=8383 max_rank=2 min=8048 min_rank=0 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[526, 485, 533, 517, 480, 520, 500, 541, 491, 497, 514, 493, 529, 465, 533, 498, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 537, 510, 530, 477, 480, 567, 489, 491, 534, 484, 521, 513, 526, 561, 548, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 526, 533, 492, 516, 482, 489, 511, 516, 470, 489, 528, 491, 513, 503, 516, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 473, 506, 508, 504, 510, 519, 522, 543, 537, 458, 511, 495, 538, 492, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 543, 531, 468, 476, 486, 527, 515, 499, 504, 517, 495, 480, 524, 500, 527, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 535, 515, 485, 541, 486, 542, 535, 502, 551, 498, 524, 501, 523, 515, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 531, 489, 498, 515, 491, 502, 511, 509, 531, 521, 527, 543, 499, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 528, 503, 550, 497, 541, 488, 498, 485, 509, 513, 540, 511, 500, 463, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8285, 8098, 8164, 8160, 8270, 8213, 8216] avg=8192.0000 max=8285 max_rank=1 min=8098 min_rank=2 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 508, 533, 508, 523, 516, 530, 530, 538, 508, 472, 527, 504, 492, 527, 488, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 497, 557, 491, 492, 516, 538, 516, 501, 517, 521, 463, 542, 519, 478, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 530, 506, 516, 489, 496, 519, 493, 476, 500, 539, 510, 509, 530, 496, 502, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 533, 516, 552, 521, 507, 498, 521, 500, 511, 535, 520, 521, 531, 507, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 595, 529, 520, 507, 464, 504, 507, 528, 517, 513, 491, 495, 507, 503, 520, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 477, 506, 489, 501, 505, 548, 510, 507, 526, 510, 527, 506, 502, 516, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 487, 452, 548, 485, 549, 509, 500, 519, 489, 521, 508, 454, 505, 497, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 473, 482, 508, 518, 509, 538, 525, 506, 524, 501, 517, 530, 503, 523, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8179, 8145, 8319, 8213, 8220, 8081, 8165] avg=8192.0000 max=8319 max_rank=3 min=8081 min_rank=6 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[499, 505, 516, 514, 555, 527, 503, 516, 514, 472, 549, 501, 527, 514, 496, 508, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 550, 489, 553, 513, 488, 527, 501, 475, 496, 470, 538, 502, 495, 497, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 489, 494, 530, 532, 504, 465, 524, 519, 534, 523, 537, 478, 504, 505, 507, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 494, 516, 498, 485, 510, 502, 533, 518, 503, 552, 519, 492, 548, 534, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 481, 506, 493, 525, 552, 500, 522, 477, 490, 509, 506, 536, 521, 529, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 531, 505, 525, 506, 519, 487, 534, 504, 559, 533, 502, 489, 528, 495, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 481, 516, 520, 499, 514, 537, 516, 475, 521, 525, 532, 500, 512, 484, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 508, 512, 522, 509, 498, 508, 529, 515, 522, 539, 529, 497, 533, 523, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8118, 8141, 8211, 8207, 8272, 8124, 8238] avg=8192.0000 max=8272 max_rank=5 min=8118 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[500, 499, 532, 526, 480, 506, 538, 507, 471, 516, 511, 499, 473, 523, 496, 512, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 537, 549, 516, 508, 481, 524, 501, 508, 502, 501, 471, 518, 508, 519, 524, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 522, 514, 549, 534, 520, 500, 542, 493, 531, 466, 515, 501, 474, 490, 517, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 542, 528, 489, 517, 525, 482, 529, 547, 507, 523, 468, 514, 495, 554, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 567, 537, 485, 501, 519, 535, 529, 513, 498, 476, 498, 502, 509, 501, 525, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 526, 531, 486, 480, 511, 497, 497, 483, 521, 532, 526, 519, 516, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 550, 498, 499, 542, 533, 510, 487, 532, 538, 490, 507, 497, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 530, 534, 487, 489, 526, 491, 487, 493, 532, 518, 538, 548, 471, 491, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8184, 8207, 8195, 8237, 8142, 8232, 8241] avg=8192.0000 max=8241 max_rank=7 min=8098 min_rank=0 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32852 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4352, 4096, 3968, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[514, 508, 509, 520, 517, 469, 474, 524, 490, 507, 531, 496, 470, 522, 490, 531, 545, 534, 518, 485, 514, 540, 549, 527, 495, 484, 465, 490, 508, 543, 537, 555, 536, 461, 484, 510, 503, 517, 518, 520, 479, 492, 510, 567, 523, 497, 524, 533, 520, 492, 487, 550, 500, 525, 506, 500, 543, 524, 478, 528, 538, 498, 503, 525, 513, 513, 487, 504, 535, 509, 544, 507, 493, 546, 515, 531, 512, 528, 505, 491, 488, 497, 534, 490, 530, 512, 495, 513, 476, 525, 520, 496, 504, 509, 509, 475, 506, 529, 508, 491, 506, 514, 527, 557, 499, 501, 541, 534, 534, 541, 488, 531, 503, 534, 442, 515, 539, 513, 516, 497, 503, 524, 537, 452, 515, 528, 544, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[495, 485, 495, 489, 497, 517, 512, 491, 516, 513, 534, 518, 518, 509, 513, 529, 495, 540, 516, 519, 488, 507, 524, 510, 501, 502, 496, 521, 492, 506, 548, 531, 475, 474, 555, 502, 505, 485, 522, 530, 486, 484, 492, 460, 519, 502, 492, 523, 489, 498, 512, 497, 528, 538, 513, 535, 554, 518, 516, 496, 507, 501, 552, 491, 510, 524, 501, 540, 516, 523, 489, 489, 517, 508, 499, 514, 539, 472, 515, 515, 488, 520, 532, 516, 502, 518, 501, 524, 521, 481, 514, 531, 505, 553, 539, 508, 543, 490, 509, 519, 521, 535, 513, 523, 501, 485, 572, 520, 510, 523, 542, 522, 489, 510, 517, 516, 492, 513, 516, 508, 483, 535, 503, 536, 528, 508, 500, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[495, 565, 535, 514, 481, 504, 526, 552, 526, 562, 504, 547, 544, 516, 497, 496, 532, 512, 497, 499, 491, 543, 539, 505, 483, 539, 548, 519, 511, 520, 516, 495, 531, 513, 468, 529, 507, 487, 538, 481, 472, 518, 535, 523, 480, 518, 510, 501, 551, 514, 471, 500, 503, 551, 528, 506, 507, 494, 490, 498, 533, 523, 517, 506, 495, 530, 552, 522, 492, 500, 534, 481, 499, 555, 476, 518, 466, 520, 529, 562, 450, 550, 520, 533, 479, 517, 501, 499, 511, 493, 467, 494, 494, 510, 477, 502, 493, 464, 487, 524, 498, 520, 507, 485, 488, 515, 534, 514, 522, 479, 477, 493, 482, 558, 495, 524, 523, 534, 541, 511, 533, 537, 551, 496, 542, 520, 537, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[507, 532, 485, 504, 512, 500, 521, 503, 525, 511, 522, 500, 476, 504, 507, 522, 502, 507, 505, 487, 532, 503, 534, 531, 506, 507, 476, 514, 513, 505, 504, 527, 511, 523, 484, 552, 498, 520, 497, 524, 510, 508, 509, 568, 501, 497, 502, 527, 530, 499, 534, 452, 511, 534, 487, 524, 482, 492, 513, 486, 484, 499, 510, 533, 528, 531, 492, 525, 532, 517, 539, 572, 496, 529, 531, 509, 478, 481, 499, 512, 504, 517, 515, 516, 500, 522, 526, 523, 460, 457, 532, 540, 493, 503, 520, 519, 493, 535, 524, 534, 525, 493, 530, 551, 520, 499, 543, 535, 531, 539, 521, 491, 502, 510, 541, 537, 471, 539, 491, 510, 536, 489, 476, 489, 524, 513, 520, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[512, 549, 520, 519, 484, 472, 509, 503, 529, 484, 524, 505, 503, 530, 490, 543, 525, 536, 499, 535, 482, 542, 519, 526, 527, 496, 483, 495, 489, 537, 510, 505, 464, 525, 553, 527, 476, 513, 529, 527, 489, 515, 500, 537, 489, 552, 484, 491, 557, 540, 509, 516, 506, 466, 482, 506, 511, 513, 523, 553, 519, 516, 476, 483, 490, 514, 538, 511, 535, 489, 484, 559, 527, 510, 535, 523, 499, 481, 490, 491, 490, 500, 532, 560, 476, 541, 494, 527, 540, 520, 494, 531, 498, 531, 499, 515, 482, 501, 504, 486, 532, 511, 519, 472, 475, 525, 554, 516, 504, 499, 517, 500, 501, 465, 500, 506, 522, 489, 504, 538, 519, 569, 534, 530, 531, 531, 523, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[483, 545, 497, 503, 513, 553, 507, 498, 486, 515, 502, 485, 512, 501, 540, 520, 467, 502, 512, 475, 543, 516, 481, 502, 522, 530, 523, 469, 542, 514, 535, 527, 516, 518, 474, 486, 529, 513, 491, 503, 509, 504, 536, 519, 535, 524, 532, 510, 513, 478, 501, 520, 547, 514, 511, 514, 518, 530, 534, 537, 549, 518, 549, 492, 503, 519, 529, 492, 486, 482, 529, 532, 521, 550, 502, 522, 494, 514, 502, 516, 499, 524, 528, 493, 462, 544, 530, 495, 490, 522, 502, 541, 491, 470, 482, 493, 504, 507, 536, 477, 509, 535, 508, 501, 520, 505, 512, 553, 511, 525, 515, 518, 531, 546, 526, 493, 478, 496, 513, 518, 503, 504, 503, 490, 577, 489, 524, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[504, 511, 524, 500, 542, 481, 504, 551, 500, 510, 513, 526, 484, 492, 513, 502, 503, 512, 460, 511, 530, 502, 463, 517, 484, 526, 518, 534, 530, 500, 499, 529, 483, 473, 532, 501, 539, 544, 490, 507, 533, 553, 483, 527, 545, 505, 550, 497, 489, 522, 518, 531, 559, 465, 517, 572, 463, 486, 508, 498, 491, 484, 502, 523, 510, 525, 497, 525, 532, 498, 506, 516, 517, 521, 506, 508, 504, 517, 507, 511, 515, 503, 494, 533, 516, 518, 504, 512, 549, 545, 523, 498, 541, 514, 516, 514, 479, 534, 509, 506, 502, 495, 487, 461, 493, 502, 517, 495, 542, 535, 535, 513, 518, 507, 541, 517, 497, 523, 518, 504, 533, 476, 478, 489, 520, 522, 544, 584] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[511, 508, 509, 507, 499, 512, 490, 490, 527, 536, 456, 501, 480, 523, 529, 498, 506, 481, 535, 487, 531, 456, 507, 490, 539, 536, 485, 513, 481, 529, 518, 558, 487, 496, 551, 496, 526, 508, 526, 547, 511, 530, 506, 501, 492, 507, 519, 493, 512, 524, 485, 521, 528, 546, 523, 529, 540, 529, 495, 526, 516, 479, 523, 499, 474, 529, 495, 489, 513, 543, 559, 531, 489, 519, 510, 506, 485, 522, 533, 495, 501, 519, 535, 506, 512, 482, 512, 527, 491, 493, 532, 489, 505, 466, 547, 533, 527, 515, 530, 492, 540, 482, 504, 545, 512, 498, 512, 531, 515, 484, 510, 518, 511, 478, 500, 526, 519, 510, 514, 548, 501, 519, 518, 528, 521, 569, 499, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[514, 508, 509, 520, 517, 469, 474, 524, 490, 507, 531, 496, 470, 522, 490, 531, 16, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 534, 518, 485, 514, 540, 549, 527, 495, 484, 465, 490, 508, 543, 537, 555, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 461, 484, 510, 503, 517, 518, 520, 479, 492, 510, 567, 523, 497, 524, 533, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 492, 487, 550, 500, 525, 506, 490, 543, 524, 478, 528, 538, 498, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 487, 504, 535, 509, 544, 507, 493, 530, 515, 531, 512, 528, 505, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 497, 534, 490, 530, 512, 495, 513, 476, 525, 520, 496, 504, 509, 509, 475, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 508, 491, 506, 514, 527, 557, 499, 501, 527, 534, 534, 541, 488, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 534, 442, 515, 539, 513, 516, 497, 503, 524, 537, 452, 463, 528, 544, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8290, 8196, 8207, 8217, 8110, 8293, 8119] avg=8192.0000 max=8293 max_rank=6 min=8104 min_rank=0 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 565, 535, 514, 481, 504, 526, 552, 526, 562, 504, 547, 544, 516, 497, 496, 17, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 497, 499, 491, 543, 539, 505, 483, 539, 548, 519, 511, 520, 516, 495, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 513, 468, 529, 507, 487, 538, 481, 472, 518, 535, 523, 480, 518, 510, 501, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 514, 471, 500, 503, 551, 528, 496, 507, 494, 490, 498, 533, 523, 517, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 530, 552, 522, 492, 500, 534, 481, 499, 538, 476, 518, 466, 520, 529, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 550, 520, 533, 479, 517, 501, 499, 511, 493, 467, 494, 494, 510, 477, 502, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 464, 487, 524, 498, 520, 507, 485, 488, 515, 520, 514, 522, 479, 477, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 558, 495, 524, 523, 534, 541, 511, 533, 537, 551, 496, 487, 520, 537, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8398, 8250, 8133, 8182, 8214, 8036, 7986, 8337] avg=8192.0000 max=8398 max_rank=0 min=7986 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32573 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 532, 485, 504, 512, 500, 521, 503, 525, 511, 522, 500, 476, 504, 507, 522, 16, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 505, 487, 532, 503, 534, 531, 506, 507, 476, 514, 513, 505, 504, 527, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 523, 484, 552, 498, 520, 497, 524, 510, 508, 509, 568, 501, 497, 502, 527, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 534, 452, 511, 534, 487, 514, 482, 492, 513, 486, 484, 499, 510, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 531, 492, 525, 532, 517, 539, 572, 496, 513, 531, 509, 478, 481, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 515, 516, 500, 522, 526, 523, 460, 457, 532, 540, 493, 503, 520, 519, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 535, 524, 534, 525, 493, 530, 551, 520, 499, 529, 535, 531, 539, 521, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 510, 541, 537, 471, 539, 491, 510, 536, 489, 476, 489, 472, 513, 520, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8154, 8253, 8060, 8255, 8184, 8350, 8117] avg=8192.0000 max=8350 max_rank=6 min=8060 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 485, 495, 489, 497, 517, 512, 491, 516, 513, 534, 518, 518, 509, 513, 529, 15, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 540, 516, 519, 488, 507, 524, 510, 501, 502, 496, 521, 492, 506, 548, 531, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 474, 555, 502, 505, 485, 522, 530, 486, 484, 492, 460, 519, 502, 492, 523, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 498, 512, 497, 528, 538, 513, 524, 554, 518, 516, 496, 507, 501, 552, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 501, 540, 516, 523, 489, 489, 517, 493, 499, 514, 539, 472, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 520, 532, 516, 502, 518, 501, 524, 521, 481, 514, 531, 505, 553, 539, 508, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 490, 509, 519, 521, 535, 513, 523, 501, 485, 557, 520, 510, 523, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 510, 517, 516, 492, 513, 516, 508, 483, 535, 503, 536, 475, 508, 500, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8197, 8030, 8234, 8156, 8291, 8313, 8153] avg=8192.0000 max=8313 max_rank=6 min=8030 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[512, 549, 520, 519, 484, 472, 509, 503, 529, 484, 524, 505, 503, 530, 490, 543, 18, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 536, 499, 535, 482, 542, 519, 526, 527, 496, 483, 495, 489, 537, 510, 505, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 525, 553, 527, 476, 513, 529, 527, 489, 515, 500, 537, 489, 552, 484, 491, 18, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 540, 509, 516, 506, 466, 482, 486, 511, 513, 523, 553, 519, 516, 476, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 514, 538, 511, 535, 489, 484, 559, 527, 492, 535, 523, 499, 481, 490, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 500, 532, 560, 476, 541, 494, 527, 540, 520, 494, 531, 498, 531, 499, 515, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 504, 486, 532, 511, 519, 472, 475, 525, 534, 516, 504, 499, 517, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 465, 500, 506, 522, 489, 504, 538, 519, 569, 534, 530, 469, 531, 523, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8212, 8203, 8156, 8158, 8289, 8077, 8224] avg=8192.0000 max=8289 max_rank=5 min=8077 min_rank=6 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[483, 545, 497, 503, 513, 553, 507, 498, 486, 515, 502, 485, 512, 501, 540, 520, 17, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 502, 512, 475, 543, 516, 481, 502, 522, 530, 523, 469, 542, 514, 535, 527, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 474, 486, 529, 513, 491, 503, 509, 504, 536, 519, 535, 524, 532, 510, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 478, 501, 520, 547, 514, 511, 504, 518, 530, 534, 537, 549, 518, 549, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 519, 529, 492, 486, 482, 529, 532, 521, 533, 502, 522, 494, 514, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 528, 493, 462, 544, 530, 495, 490, 522, 502, 541, 491, 470, 482, 493, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 536, 477, 509, 535, 508, 501, 520, 505, 499, 553, 511, 525, 515, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 546, 526, 493, 478, 496, 513, 518, 503, 504, 503, 490, 519, 489, 524, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8161, 8220, 8315, 8176, 8107, 8223, 8139] avg=8192.0000 max=8315 max_rank=3 min=8107 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[504, 511, 524, 500, 542, 481, 504, 551, 500, 510, 513, 526, 484, 492, 513, 502, 16, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 512, 460, 511, 530, 502, 463, 517, 484, 526, 518, 534, 530, 500, 499, 529, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 473, 532, 501, 539, 544, 490, 507, 533, 553, 483, 527, 545, 505, 550, 497, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 522, 518, 531, 559, 465, 517, 561, 463, 486, 508, 498, 491, 484, 502, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 497, 525, 532, 498, 506, 516, 517, 505, 506, 508, 504, 517, 507, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 503, 494, 533, 516, 518, 504, 512, 549, 545, 523, 498, 541, 514, 516, 514, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 534, 509, 506, 502, 495, 487, 461, 493, 502, 504, 495, 542, 535, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 541, 517, 497, 523, 518, 504, 533, 476, 478, 489, 468, 522, 544, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8119, 8284, 8117, 8184, 8332, 8092, 8219] avg=8192.0000 max=8332 max_rank=5 min=8092 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[511, 508, 509, 507, 499, 512, 490, 490, 527, 536, 456, 501, 480, 523, 529, 498, 16, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 481, 535, 487, 531, 456, 507, 490, 539, 536, 485, 513, 481, 529, 518, 558, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 496, 551, 496, 526, 508, 526, 547, 511, 530, 506, 501, 492, 507, 519, 493, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 485, 521, 528, 546, 523, 519, 540, 529, 495, 526, 516, 479, 523, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 529, 495, 489, 513, 543, 559, 531, 489, 503, 510, 506, 485, 522, 533, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 519, 535, 506, 512, 482, 512, 527, 491, 493, 532, 489, 505, 466, 547, 533, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 515, 530, 492, 540, 482, 504, 545, 512, 498, 499, 531, 515, 484, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 478, 500, 526, 519, 510, 514, 548, 501, 519, 518, 528, 469, 569, 499, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8153, 8217, 8265, 8176, 8187, 8202, 8228] avg=8192.0000 max=8265 max_rank=3 min=8108 min_rank=0 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 3840, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[522, 523, 534, 489, 560, 510, 475, 514, 500, 539, 523, 503, 516, 487, 476, 502, 510, 483, 512, 477, 519, 526, 547, 515, 534, 526, 501, 537, 532, 488, 524, 520, 501, 499, 497, 523, 518, 525, 542, 466, 519, 509, 523, 525, 497, 525, 564, 512, 532, 518, 530, 541, 557, 513, 492, 515, 496, 510, 521, 536, 530, 550, 476, 535, 493, 481, 514, 497, 539, 534, 475, 521, 501, 507, 500, 504, 488, 509, 509, 506, 505, 521, 515, 561, 526, 517, 503, 479, 461, 509, 521, 525, 488, 484, 514, 458, 503, 521, 462, 576, 521, 494, 540, 478, 507, 519, 505, 501, 479, 523, 509, 501, 548, 529, 512, 512, 479, 462, 530, 500, 509, 524, 516, 537, 544, 489, 522, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[510, 495, 528, 490, 493, 497, 515, 510, 517, 518, 535, 475, 548, 539, 564, 528, 508, 497, 516, 533, 511, 485, 514, 512, 507, 522, 514, 517, 529, 501, 503, 509, 552, 480, 490, 507, 508, 522, 516, 483, 510, 540, 520, 520, 483, 537, 509, 518, 505, 531, 554, 503, 502, 474, 538, 532, 538, 559, 503, 499, 473, 505, 517, 480, 521, 515, 516, 508, 583, 529, 495, 482, 477, 501, 518, 507, 513, 475, 495, 517, 513, 501, 513, 465, 485, 549, 479, 517, 498, 502, 511, 567, 546, 530, 515, 486, 539, 486, 510, 522, 529, 539, 494, 499, 482, 501, 487, 554, 508, 517, 545, 527, 504, 511, 486, 520, 482, 546, 492, 519, 467, 513, 508, 511, 506, 520, 495, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[507, 490, 501, 530, 515, 495, 521, 555, 517, 510, 523, 524, 561, 528, 529, 510, 544, 529, 508, 492, 502, 523, 510, 483, 508, 531, 493, 500, 540, 486, 524, 520, 503, 553, 521, 513, 487, 505, 484, 523, 478, 508, 518, 498, 499, 522, 508, 520, 502, 525, 563, 506, 492, 518, 557, 522, 488, 494, 476, 541, 497, 496, 502, 497, 506, 496, 538, 520, 517, 472, 476, 478, 537, 519, 524, 500, 508, 537, 540, 513, 531, 503, 510, 506, 508, 524, 553, 545, 512, 484, 457, 525, 525, 533, 526, 519, 524, 492, 546, 534, 506, 495, 548, 527, 501, 535, 505, 516, 505, 503, 498, 502, 471, 527, 532, 519, 526, 515, 467, 499, 492, 521, 501, 497, 475, 463, 545, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[511, 532, 530, 505, 529, 483, 527, 537, 542, 522, 523, 505, 529, 517, 483, 503, 476, 527, 455, 469, 505, 510, 520, 506, 538, 478, 483, 488, 531, 521, 503, 487, 536, 496, 495, 506, 529, 499, 522, 496, 530, 543, 516, 504, 542, 493, 540, 528, 543, 553, 521, 508, 506, 499, 534, 500, 545, 533, 529, 529, 522, 527, 515, 518, 504, 498, 518, 505, 501, 512, 537, 522, 520, 503, 538, 547, 471, 511, 524, 488, 522, 490, 476, 498, 541, 507, 538, 523, 493, 512, 483, 510, 471, 519, 525, 481, 495, 527, 484, 539, 545, 496, 535, 524, 520, 514, 541, 505, 521, 490, 533, 488, 529, 479, 483, 496, 525, 496, 512, 517, 475, 532, 519, 520, 521, 459, 508, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[546, 493, 566, 524, 519, 517, 507, 510, 521, 514, 480, 508, 552, 532, 467, 548, 518, 512, 503, 521, 537, 491, 484, 477, 540, 527, 503, 489, 525, 542, 506, 504, 503, 503, 483, 494, 528, 500, 492, 484, 490, 477, 507, 537, 542, 540, 455, 532, 503, 543, 514, 494, 533, 498, 504, 538, 516, 509, 523, 503, 453, 516, 520, 522, 534, 521, 512, 503, 488, 543, 507, 500, 478, 536, 495, 521, 513, 509, 554, 496, 487, 525, 508, 474, 508, 520, 505, 568, 506, 487, 549, 530, 533, 497, 492, 499, 509, 471, 523, 507, 522, 512, 521, 510, 533, 534, 504, 530, 541, 530, 555, 525, 507, 504, 492, 533, 513, 533, 490, 517, 498, 517, 510, 448, 489, 488, 523, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[500, 505, 531, 501, 495, 535, 466, 499, 505, 537, 521, 475, 510, 513, 509, 516, 524, 471, 488, 517, 516, 531, 537, 484, 535, 552, 504, 510, 489, 480, 506, 494, 540, 484, 535, 498, 540, 514, 515, 533, 517, 489, 502, 517, 534, 530, 445, 482, 448, 493, 507, 507, 506, 527, 516, 499, 497, 506, 474, 512, 525, 533, 496, 500, 540, 535, 508, 557, 498, 499, 491, 505, 513, 485, 503, 512, 523, 515, 498, 510, 548, 531, 575, 475, 501, 508, 548, 518, 537, 534, 525, 549, 483, 481, 521, 519, 536, 508, 540, 551, 515, 507, 514, 546, 493, 495, 528, 488, 537, 507, 494, 500, 520, 524, 495, 515, 512, 488, 525, 520, 503, 509, 532, 553, 478, 531, 524, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[521, 550, 503, 503, 480, 494, 529, 551, 479, 513, 508, 459, 533, 490, 542, 567, 539, 506, 514, 519, 504, 510, 516, 508, 501, 488, 488, 480, 524, 530, 497, 495, 520, 519, 492, 522, 497, 515, 509, 530, 504, 502, 538, 557, 538, 514, 533, 505, 537, 528, 540, 529, 488, 496, 514, 503, 542, 521, 492, 542, 544, 519, 550, 567, 495, 513, 540, 504, 486, 466, 514, 506, 493, 510, 498, 503, 499, 494, 517, 503, 536, 487, 488, 505, 514, 534, 515, 540, 474, 514, 494, 437, 519, 504, 512, 520, 523, 489, 555, 516, 516, 530, 515, 538, 526, 518, 531, 506, 509, 479, 508, 466, 517, 536, 519, 478, 502, 482, 527, 501, 497, 502, 530, 497, 507, 518, 498, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[492, 458, 479, 480, 512, 519, 483, 509, 520, 562, 525, 502, 519, 517, 525, 526, 501, 516, 543, 536, 510, 500, 484, 491, 540, 518, 512, 493, 521, 485, 514, 535, 512, 493, 525, 530, 511, 550, 531, 520, 522, 494, 554, 493, 559, 519, 529, 476, 462, 511, 533, 502, 506, 496, 514, 514, 478, 538, 525, 528, 494, 482, 534, 546, 524, 529, 520, 518, 492, 491, 548, 501, 518, 498, 526, 484, 515, 498, 478, 556, 522, 500, 498, 508, 515, 538, 496, 528, 521, 563, 477, 528, 521, 533, 505, 533, 510, 487, 513, 474, 502, 522, 503, 535, 487, 497, 494, 512, 501, 553, 502, 523, 564, 535, 508, 464, 510, 502, 515, 553, 524, 508, 514, 470, 475, 503, 529, 457] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[510, 495, 528, 490, 493, 497, 515, 510, 517, 518, 535, 475, 515, 539, 564, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 516, 533, 511, 485, 514, 512, 507, 522, 514, 517, 529, 501, 503, 509, 19, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 480, 490, 507, 508, 522, 516, 483, 510, 540, 520, 520, 464, 537, 509, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 503, 503, 502, 474, 538, 532, 538, 559, 503, 499, 473, 505, 517, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 515, 516, 508, 583, 529, 495, 482, 477, 501, 518, 507, 513, 475, 495, 517, 18, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 513, 465, 485, 549, 479, 517, 498, 502, 511, 567, 546, 530, 515, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 486, 510, 491, 529, 539, 494, 499, 482, 501, 487, 554, 508, 517, 545, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 486, 520, 482, 546, 492, 519, 467, 513, 508, 511, 506, 520, 495, 540, 51, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8210, 8176, 8162, 8187, 8177, 8208, 8187] avg=8192.0000 max=8229 max_rank=0 min=8162 min_rank=3 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[522, 523, 534, 489, 560, 510, 475, 514, 500, 539, 523, 503, 485, 487, 476, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 483, 512, 477, 519, 526, 547, 515, 534, 526, 501, 537, 532, 488, 524, 520, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 499, 497, 523, 518, 525, 542, 466, 519, 509, 523, 525, 477, 525, 564, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 518, 481, 541, 557, 513, 492, 515, 496, 510, 521, 536, 530, 550, 476, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 481, 514, 497, 539, 534, 475, 521, 501, 507, 500, 504, 488, 509, 509, 506, 20, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 515, 561, 526, 517, 503, 479, 461, 509, 521, 525, 488, 484, 514, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 521, 462, 541, 521, 494, 540, 478, 507, 519, 505, 501, 479, 523, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 529, 512, 512, 479, 462, 530, 500, 509, 524, 516, 537, 544, 489, 522, 498, 49, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8286, 8225, 8303, 8114, 8087, 8104, 8275] avg=8192.0000 max=8303 max_rank=3 min=8087 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[546, 493, 566, 524, 519, 517, 507, 510, 521, 514, 480, 508, 519, 532, 467, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 503, 521, 537, 491, 484, 477, 540, 527, 503, 489, 525, 542, 506, 504, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 483, 494, 528, 500, 492, 484, 490, 477, 507, 537, 520, 540, 455, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 543, 467, 494, 533, 498, 504, 538, 516, 509, 523, 503, 453, 516, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 512, 503, 488, 543, 507, 500, 478, 536, 495, 521, 513, 509, 554, 496, 18, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 525, 508, 474, 508, 520, 505, 568, 506, 487, 549, 530, 533, 497, 492, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 471, 523, 476, 522, 512, 521, 510, 533, 534, 504, 530, 541, 530, 555, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 492, 533, 513, 533, 490, 517, 498, 517, 510, 448, 489, 488, 523, 510, 47, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8214, 8045, 8142, 8245, 8188, 8296, 8135] avg=8192.0000 max=8296 max_rank=6 min=8045 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[511, 532, 530, 505, 529, 483, 527, 537, 542, 522, 523, 505, 489, 517, 483, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 527, 455, 469, 505, 510, 520, 506, 538, 478, 483, 488, 531, 521, 503, 487, 26, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 496, 495, 506, 529, 499, 522, 496, 530, 543, 516, 504, 514, 493, 540, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 553, 469, 508, 506, 499, 534, 500, 545, 533, 529, 529, 522, 527, 515, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 518, 505, 501, 512, 537, 522, 520, 503, 538, 547, 471, 511, 524, 488, 23, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 476, 498, 541, 507, 538, 523, 493, 512, 483, 510, 471, 519, 525, 481, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 527, 484, 499, 545, 496, 535, 524, 520, 514, 541, 505, 521, 490, 533, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 479, 483, 496, 525, 496, 512, 517, 475, 532, 519, 520, 521, 459, 508, 488, 52, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8040, 8247, 8330, 8242, 8091, 8217, 8131] avg=8192.0000 max=8330 max_rank=3 min=8040 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[507, 490, 501, 530, 515, 495, 521, 555, 517, 510, 523, 524, 527, 528, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 529, 508, 492, 502, 523, 510, 483, 508, 531, 493, 500, 540, 486, 524, 520, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 553, 521, 513, 487, 505, 484, 523, 478, 508, 518, 498, 479, 522, 508, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 511, 506, 492, 518, 557, 522, 488, 494, 476, 541, 497, 496, 502, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 496, 538, 520, 517, 472, 476, 478, 537, 519, 524, 500, 508, 537, 540, 513, 19, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 503, 510, 506, 508, 524, 553, 545, 512, 484, 457, 525, 525, 533, 526, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 492, 546, 502, 506, 495, 548, 527, 501, 535, 505, 516, 505, 503, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 527, 532, 519, 526, 515, 467, 499, 492, 521, 501, 497, 475, 463, 545, 482, 52, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8226, 8120, 8124, 8217, 8261, 8205, 8101] avg=8192.0000 max=8282 max_rank=0 min=8101 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[500, 505, 531, 501, 495, 535, 466, 499, 505, 537, 521, 475, 479, 513, 509, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 471, 488, 517, 516, 531, 537, 484, 535, 552, 504, 510, 489, 480, 506, 494, 21, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 484, 535, 498, 540, 514, 515, 533, 517, 489, 502, 517, 513, 530, 445, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 493, 460, 507, 506, 527, 516, 499, 497, 506, 474, 512, 525, 533, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 535, 508, 557, 498, 499, 491, 505, 513, 485, 503, 512, 523, 515, 498, 510, 19, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 531, 575, 475, 501, 508, 548, 518, 537, 534, 525, 549, 483, 481, 521, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 508, 540, 518, 515, 507, 514, 546, 493, 495, 528, 488, 537, 507, 494, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 495, 515, 512, 488, 525, 520, 503, 509, 532, 553, 478, 531, 524, 526, 47, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8173, 8154, 7999, 8227, 8353, 8226, 8317] avg=8192.0000 max=8353 max_rank=5 min=7999 min_rank=3 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=33123 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[492, 458, 479, 480, 512, 519, 483, 509, 520, 562, 525, 502, 488, 517, 525, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 543, 536, 510, 500, 484, 491, 540, 518, 512, 493, 521, 485, 514, 535, 22, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 525, 530, 511, 550, 531, 520, 522, 494, 554, 493, 537, 519, 529, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 511, 484, 502, 506, 496, 514, 514, 478, 538, 525, 528, 494, 482, 534, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 529, 520, 518, 492, 491, 548, 501, 518, 498, 526, 484, 515, 498, 478, 556, 17, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 500, 498, 508, 515, 538, 496, 528, 521, 563, 477, 528, 521, 533, 505, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 513, 445, 502, 522, 503, 535, 487, 497, 494, 512, 501, 553, 502, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 535, 508, 464, 510, 502, 515, 553, 524, 508, 514, 470, 475, 503, 529, 457, 49, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8233, 8296, 8114, 8229, 8286, 8086, 8195] avg=8192.0000 max=8296 max_rank=2 min=8086 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[521, 550, 503, 503, 480, 494, 529, 551, 479, 513, 508, 459, 501, 490, 542, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 506, 514, 519, 504, 510, 516, 508, 501, 488, 488, 480, 524, 530, 497, 495, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 519, 492, 522, 497, 515, 509, 530, 504, 502, 538, 557, 516, 514, 533, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 528, 490, 529, 488, 496, 514, 503, 542, 521, 492, 542, 544, 519, 550, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 513, 540, 504, 486, 466, 514, 506, 493, 510, 498, 503, 499, 494, 517, 503, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 487, 488, 505, 514, 534, 515, 540, 474, 514, 494, 437, 519, 504, 512, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 489, 555, 485, 516, 530, 515, 538, 526, 518, 531, 506, 509, 479, 508, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 536, 519, 478, 502, 482, 527, 501, 497, 502, 530, 497, 507, 518, 498, 518, 50, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8154, 8273, 8362, 8075, 8093, 8194, 8195] avg=8192.0000 max=8362 max_rank=3 min=8075 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True + [2026-08-26 16:29:47] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 413.8 | throughput per GPU (TFLOP/s/GPU): 437.0 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.024487E+01 | loss scale: 1.0 | grad norm: 6.230 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[531, 537, 537, 506, 488, 552, 521, 532, 517, 536, 506, 474, 508, 482, 494, 497, 537, 516, 523, 512, 511, 512, 540, 506, 476, 487, 537, 517, 548, 508, 545, 532, 480, 466, 500, 488, 498, 499, 522, 514, 512, 542, 526, 535, 522, 511, 513, 511, 512, 486, 525, 555, 538, 502, 511, 510, 517, 497, 499, 557, 467, 505, 514, 539, 515, 493, 469, 510, 527, 491, 505, 532, 526, 519, 545, 527, 538, 531, 529, 502, 497, 494, 511, 521, 518, 487, 465, 522, 458, 512, 499, 496, 545, 503, 487, 524, 507, 484, 503, 521, 490, 516, 538, 500, 511, 544, 526, 526, 482, 473, 463, 497, 514, 510, 513, 536, 546, 528, 477, 495, 539, 530, 496, 495, 486, 544, 523, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[472, 523, 502, 514, 492, 497, 523, 493, 514, 510, 522, 502, 534, 467, 518, 503, 530, 575, 525, 530, 527, 501, 521, 506, 574, 531, 517, 489, 509, 473, 541, 523, 498, 500, 486, 489, 505, 523, 501, 492, 492, 504, 550, 544, 493, 523, 497, 500, 505, 511, 508, 511, 513, 526, 481, 543, 511, 510, 512, 520, 508, 465, 504, 500, 511, 535, 510, 498, 484, 484, 523, 558, 490, 509, 532, 508, 482, 553, 508, 518, 490, 500, 489, 540, 521, 537, 532, 538, 517, 500, 509, 551, 461, 513, 534, 489, 526, 547, 516, 501, 507, 560, 513, 524, 572, 505, 475, 518, 533, 497, 531, 494, 520, 509, 507, 482, 524, 540, 527, 486, 505, 504, 504, 487, 490, 533, 507, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[502, 551, 528, 525, 536, 487, 520, 502, 471, 525, 530, 514, 488, 551, 530, 522, 510, 500, 496, 534, 487, 508, 559, 523, 519, 477, 546, 536, 509, 558, 540, 529, 485, 486, 547, 506, 492, 520, 477, 516, 498, 510, 490, 516, 547, 508, 549, 538, 495, 508, 492, 520, 516, 489, 504, 456, 527, 494, 560, 504, 508, 556, 520, 525, 491, 529, 551, 512, 523, 498, 470, 491, 482, 540, 484, 471, 498, 502, 523, 528, 528, 508, 474, 529, 488, 501, 537, 520, 535, 495, 501, 494, 529, 477, 517, 504, 541, 495, 505, 493, 483, 510, 502, 478, 489, 538, 497, 542, 514, 519, 500, 500, 520, 524, 521, 518, 504, 510, 537, 533, 494, 536, 511, 531, 499, 510, 478, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[484, 542, 513, 483, 504, 509, 510, 549, 518, 537, 526, 510, 487, 536, 509, 493, 546, 489, 503, 507, 513, 521, 504, 537, 498, 514, 542, 505, 485, 517, 508, 528, 464, 497, 509, 499, 503, 514, 489, 532, 520, 551, 546, 511, 484, 544, 499, 506, 542, 567, 513, 485, 504, 522, 525, 525, 482, 496, 499, 480, 522, 534, 491, 507, 517, 461, 526, 516, 509, 493, 538, 520, 484, 513, 516, 553, 521, 506, 519, 495, 502, 537, 486, 492, 492, 565, 514, 497, 472, 519, 567, 511, 508, 494, 508, 483, 556, 520, 520, 490, 527, 517, 555, 495, 546, 533, 515, 479, 483, 544, 510, 499, 525, 492, 518, 563, 496, 522, 487, 497, 507, 538, 506, 468, 519, 519, 473, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[481, 512, 460, 475, 544, 512, 506, 489, 473, 543, 553, 471, 541, 490, 510, 506, 493, 543, 493, 469, 519, 495, 515, 538, 507, 521, 521, 506, 583, 514, 529, 537, 510, 490, 540, 529, 481, 507, 547, 531, 544, 537, 502, 552, 503, 497, 506, 508, 539, 534, 549, 546, 521, 497, 506, 530, 527, 520, 510, 542, 476, 557, 503, 528, 542, 523, 496, 503, 504, 508, 531, 503, 518, 505, 490, 536, 496, 545, 517, 532, 545, 485, 508, 508, 488, 482, 506, 534, 525, 488, 518, 516, 520, 526, 509, 521, 512, 470, 527, 540, 495, 542, 456, 489, 489, 501, 495, 494, 513, 526, 519, 520, 490, 469, 496, 517, 499, 470, 503, 491, 518, 506, 528, 513, 531, 507, 479, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[500, 515, 529, 529, 510, 498, 490, 493, 502, 526, 466, 512, 483, 494, 501, 502, 521, 503, 528, 554, 538, 537, 496, 491, 526, 518, 520, 470, 502, 493, 493, 526, 501, 525, 549, 486, 531, 490, 515, 494, 524, 514, 543, 497, 524, 501, 537, 523, 532, 526, 536, 493, 534, 506, 532, 501, 495, 497, 538, 555, 504, 551, 525, 492, 523, 487, 564, 498, 533, 536, 520, 504, 501, 485, 563, 489, 521, 505, 523, 530, 553, 465, 500, 485, 512, 512, 510, 506, 518, 466, 522, 533, 526, 514, 499, 539, 512, 502, 517, 517, 511, 551, 485, 528, 468, 509, 477, 469, 526, 508, 505, 530, 507, 495, 482, 502, 511, 489, 514, 518, 538, 506, 482, 520, 516, 516, 509, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[525, 513, 486, 548, 531, 552, 508, 534, 523, 505, 507, 513, 544, 492, 521, 494, 508, 490, 554, 517, 498, 482, 514, 505, 534, 501, 515, 527, 526, 479, 519, 488, 536, 473, 523, 502, 498, 487, 510, 517, 533, 534, 545, 540, 513, 549, 474, 502, 517, 504, 511, 500, 490, 528, 513, 519, 507, 513, 495, 536, 535, 469, 509, 484, 501, 530, 502, 514, 498, 490, 489, 510, 521, 474, 523, 518, 490, 529, 519, 523, 478, 515, 485, 526, 525, 530, 501, 518, 524, 514, 463, 509, 520, 524, 479, 520, 519, 483, 476, 521, 584, 541, 504, 548, 492, 510, 506, 536, 477, 539, 514, 490, 505, 483, 492, 514, 516, 512, 506, 510, 512, 505, 499, 549, 515, 547, 530, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[547, 508, 526, 468, 505, 513, 515, 513, 485, 499, 523, 526, 484, 553, 489, 515, 517, 490, 496, 521, 513, 503, 513, 523, 504, 533, 524, 541, 484, 498, 496, 501, 504, 540, 525, 514, 489, 536, 526, 498, 481, 492, 488, 489, 490, 485, 527, 545, 493, 537, 546, 551, 483, 542, 516, 509, 567, 534, 515, 496, 510, 532, 518, 492, 497, 532, 518, 545, 500, 531, 482, 501, 531, 523, 507, 467, 520, 498, 496, 535, 469, 488, 504, 554, 523, 503, 502, 515, 523, 561, 517, 501, 497, 499, 527, 562, 490, 517, 554, 489, 518, 525, 483, 482, 519, 512, 514, 510, 505, 514, 494, 502, 544, 527, 514, 528, 507, 481, 479, 506, 504, 483, 504, 488, 533, 528, 551, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[531, 537, 537, 506, 488, 552, 521, 532, 517, 536, 506, 474, 508, 482, 494, 497, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 523, 512, 511, 512, 540, 506, 476, 487, 474, 517, 548, 508, 545, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 466, 500, 488, 498, 499, 522, 514, 512, 542, 526, 535, 522, 511, 513, 511, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 486, 525, 555, 538, 502, 511, 510, 517, 497, 499, 510, 467, 505, 514, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 493, 469, 510, 527, 491, 505, 532, 526, 519, 545, 527, 538, 526, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 511, 521, 518, 487, 465, 522, 458, 512, 499, 496, 545, 503, 487, 524, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 484, 503, 521, 490, 516, 538, 500, 511, 544, 526, 526, 482, 473, 463, 497, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 513, 536, 546, 528, 477, 495, 539, 530, 496, 495, 486, 544, 523, 527, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8244, 8144, 8187, 8254, 8074, 8103, 8292] avg=8192.0000 max=8292 max_rank=7 min=8074 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[472, 523, 502, 514, 492, 497, 523, 493, 514, 510, 522, 502, 534, 467, 518, 503, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 575, 525, 530, 527, 501, 521, 506, 574, 531, 457, 489, 509, 473, 541, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 500, 486, 489, 505, 523, 501, 492, 492, 504, 550, 544, 493, 523, 497, 500, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 508, 511, 513, 526, 481, 543, 511, 510, 512, 476, 508, 465, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 535, 510, 498, 484, 484, 523, 558, 490, 509, 532, 508, 482, 548, 508, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 500, 489, 540, 521, 537, 532, 538, 517, 500, 509, 551, 461, 513, 534, 489, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 547, 516, 501, 507, 560, 513, 524, 572, 505, 475, 518, 533, 497, 531, 494, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 507, 482, 524, 540, 527, 486, 505, 504, 504, 487, 490, 533, 507, 485, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8312, 8102, 8084, 8198, 8254, 8340, 8142] avg=8192.0000 max=8340 max_rank=6 min=8084 min_rank=3 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[502, 551, 528, 525, 536, 487, 520, 502, 471, 525, 530, 514, 488, 551, 530, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 500, 496, 534, 487, 508, 559, 523, 519, 477, 482, 536, 509, 558, 540, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 486, 547, 506, 492, 520, 477, 516, 498, 510, 490, 516, 547, 508, 549, 538, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 492, 520, 516, 489, 504, 456, 527, 494, 560, 461, 508, 556, 520, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 529, 551, 512, 523, 498, 470, 491, 482, 540, 484, 471, 498, 497, 523, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 508, 474, 529, 488, 501, 537, 520, 535, 495, 501, 494, 529, 477, 517, 504, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 495, 505, 493, 483, 510, 502, 478, 489, 538, 497, 542, 514, 519, 500, 500, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 521, 518, 504, 510, 537, 533, 494, 536, 511, 531, 499, 510, 478, 502, 29, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8267, 8190, 8131, 8088, 8172, 8126, 8262] avg=8192.0000 max=8300 max_rank=0 min=8088 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[484, 542, 513, 483, 504, 509, 510, 549, 518, 537, 526, 510, 487, 536, 509, 493, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 489, 503, 507, 513, 521, 504, 537, 498, 514, 470, 505, 485, 517, 508, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 497, 509, 499, 503, 514, 489, 532, 520, 551, 546, 511, 484, 544, 499, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 567, 513, 485, 504, 522, 525, 525, 482, 496, 499, 426, 522, 534, 491, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 461, 526, 516, 509, 493, 538, 520, 484, 513, 516, 553, 521, 490, 519, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 486, 492, 492, 565, 514, 497, 472, 519, 567, 511, 508, 494, 508, 483, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 520, 520, 490, 527, 517, 555, 495, 546, 533, 515, 479, 483, 544, 510, 499, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 492, 518, 563, 496, 522, 487, 497, 507, 538, 506, 468, 519, 519, 473, 494, 33, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8145, 8177, 8140, 8171, 8186, 8312, 8167] avg=8192.0000 max=8312 max_rank=6 min=8140 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[500, 515, 529, 529, 510, 498, 490, 493, 502, 526, 466, 512, 483, 494, 501, 502, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 528, 554, 538, 537, 496, 491, 526, 518, 459, 470, 502, 493, 493, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 525, 549, 486, 531, 490, 515, 494, 524, 514, 543, 497, 524, 501, 537, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 536, 493, 534, 506, 532, 501, 495, 497, 538, 508, 504, 551, 525, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 487, 564, 498, 533, 536, 520, 504, 501, 485, 563, 489, 521, 500, 523, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 465, 500, 485, 512, 512, 510, 506, 518, 466, 522, 533, 526, 514, 499, 539, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 502, 517, 517, 511, 551, 485, 528, 468, 509, 477, 469, 526, 508, 505, 530, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 482, 502, 511, 489, 514, 518, 538, 506, 482, 520, 516, 516, 509, 537, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8155, 8259, 8270, 8277, 8194, 8137, 8174] avg=8192.0000 max=8277 max_rank=4 min=8070 min_rank=0 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[481, 512, 460, 475, 544, 512, 506, 489, 473, 543, 553, 471, 541, 490, 510, 506, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 543, 493, 469, 519, 495, 515, 538, 507, 521, 459, 506, 583, 514, 529, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 540, 529, 481, 507, 547, 531, 544, 537, 502, 552, 503, 497, 506, 508, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 534, 549, 546, 521, 497, 506, 530, 527, 520, 510, 497, 476, 557, 503, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 523, 496, 503, 504, 508, 531, 503, 518, 505, 490, 536, 496, 540, 517, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 485, 508, 508, 488, 482, 506, 534, 525, 488, 518, 516, 520, 526, 509, 521, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 470, 527, 540, 495, 542, 456, 489, 489, 501, 495, 494, 513, 526, 519, 520, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 469, 496, 517, 499, 470, 503, 491, 518, 506, 528, 513, 531, 507, 479, 485, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8221, 8289, 8340, 8244, 8213, 8109, 8035] avg=8192.0000 max=8340 max_rank=3 min=8035 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 513, 486, 548, 531, 552, 508, 534, 523, 505, 507, 513, 544, 492, 521, 494, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 554, 517, 498, 482, 514, 505, 534, 501, 455, 527, 526, 479, 519, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 473, 523, 502, 498, 487, 510, 517, 533, 534, 545, 540, 513, 549, 474, 502, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 504, 511, 500, 490, 528, 513, 519, 507, 513, 495, 491, 535, 469, 509, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 530, 502, 514, 498, 490, 489, 510, 521, 474, 523, 518, 490, 524, 519, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 485, 526, 525, 530, 501, 518, 524, 514, 463, 509, 520, 524, 479, 520, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 483, 476, 521, 584, 541, 504, 548, 492, 510, 506, 536, 477, 539, 514, 490, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 483, 492, 514, 516, 512, 506, 510, 512, 505, 499, 549, 515, 547, 530, 520, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8097, 8241, 8085, 8126, 8164, 8261, 8247] avg=8192.0000 max=8315 max_rank=0 min=8085 min_rank=3 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[547, 508, 526, 468, 505, 513, 515, 513, 485, 499, 523, 526, 484, 553, 489, 515, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 490, 496, 521, 513, 503, 513, 523, 504, 533, 462, 541, 484, 498, 496, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 540, 525, 514, 489, 536, 526, 498, 481, 492, 488, 489, 490, 485, 527, 545, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 537, 546, 551, 483, 542, 516, 509, 567, 534, 515, 454, 510, 532, 518, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 532, 518, 545, 500, 531, 482, 501, 531, 523, 507, 467, 520, 493, 496, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 488, 504, 554, 523, 503, 502, 515, 523, 561, 517, 501, 497, 499, 527, 562, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 517, 554, 489, 518, 525, 483, 482, 519, 512, 514, 510, 505, 514, 494, 502, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 527, 514, 528, 507, 481, 479, 506, 504, 483, 504, 488, 533, 528, 551, 507, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8095, 8134, 8299, 8178, 8279, 8148, 8217] avg=8192.0000 max=8299 max_rank=3 min=8095 min_rank=1 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 3840, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3840, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4352, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 3968, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[490, 485, 496, 510, 520, 491, 495, 531, 540, 484, 549, 528, 491, 514, 533, 472, 498, 550, 504, 512, 531, 512, 534, 466, 527, 527, 501, 486, 506, 522, 495, 528, 527, 484, 523, 475, 541, 509, 518, 534, 529, 516, 540, 509, 554, 532, 504, 534, 553, 533, 511, 556, 504, 481, 523, 555, 529, 502, 491, 519, 488, 491, 525, 492, 532, 490, 507, 475, 496, 513, 516, 499, 525, 541, 511, 485, 521, 515, 495, 487, 497, 502, 513, 481, 556, 512, 531, 498, 524, 532, 594, 475, 526, 499, 457, 524, 498, 505, 501, 486, 477, 525, 529, 511, 513, 495, 509, 494, 524, 539, 518, 506, 547, 498, 516, 481, 534, 499, 517, 546, 515, 524, 524, 502, 500, 493, 488, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[523, 531, 520, 542, 494, 505, 527, 481, 499, 538, 484, 468, 491, 507, 536, 498, 515, 498, 477, 493, 531, 521, 488, 526, 514, 531, 497, 531, 517, 524, 534, 475, 512, 489, 553, 498, 523, 548, 485, 489, 525, 500, 553, 496, 543, 499, 527, 510, 500, 498, 501, 547, 550, 488, 467, 533, 510, 500, 540, 533, 508, 505, 504, 536, 498, 521, 508, 517, 502, 482, 535, 522, 541, 481, 508, 523, 483, 539, 500, 515, 523, 514, 530, 495, 490, 506, 534, 518, 497, 535, 514, 520, 533, 495, 513, 542, 479, 539, 485, 501, 516, 515, 524, 500, 510, 490, 538, 512, 544, 519, 509, 541, 470, 486, 543, 535, 496, 501, 502, 567, 509, 465, 506, 511, 494, 525, 505, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[507, 489, 506, 511, 491, 522, 526, 509, 519, 511, 459, 471, 537, 508, 496, 527, 527, 530, 525, 534, 507, 498, 483, 517, 508, 518, 492, 552, 497, 459, 494, 502, 504, 529, 496, 525, 521, 523, 521, 507, 505, 475, 473, 503, 535, 512, 544, 509, 506, 554, 520, 502, 480, 554, 495, 489, 543, 524, 529, 505, 500, 477, 530, 537, 506, 510, 547, 504, 526, 469, 530, 508, 493, 493, 519, 554, 497, 511, 528, 520, 519, 478, 531, 479, 518, 506, 496, 485, 538, 509, 516, 521, 547, 509, 478, 483, 540, 526, 516, 499, 538, 479, 506, 494, 534, 521, 537, 509, 516, 496, 551, 485, 497, 548, 519, 487, 508, 515, 515, 508, 519, 512, 527, 538, 555, 539, 488, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[532, 506, 529, 511, 506, 558, 546, 493, 505, 512, 496, 527, 528, 512, 522, 518, 498, 520, 506, 486, 502, 507, 512, 503, 498, 473, 464, 490, 498, 537, 510, 566, 503, 526, 553, 532, 510, 499, 536, 506, 506, 517, 497, 505, 508, 540, 547, 504, 530, 542, 511, 567, 492, 508, 567, 512, 484, 550, 513, 527, 510, 491, 506, 504, 495, 520, 518, 494, 537, 482, 516, 539, 511, 515, 515, 505, 504, 528, 555, 526, 530, 488, 500, 491, 500, 464, 495, 519, 518, 491, 525, 505, 495, 484, 522, 528, 524, 514, 502, 501, 516, 546, 528, 516, 501, 520, 465, 516, 507, 503, 518, 481, 491, 486, 513, 498, 505, 520, 501, 483, 524, 553, 527, 483, 481, 544, 495, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[517, 513, 535, 531, 520, 503, 505, 516, 506, 546, 517, 526, 534, 555, 527, 529, 519, 498, 494, 507, 536, 491, 488, 519, 478, 478, 512, 496, 502, 550, 484, 511, 522, 549, 509, 484, 482, 515, 538, 483, 508, 546, 523, 511, 498, 499, 509, 516, 485, 524, 506, 490, 529, 490, 475, 496, 534, 518, 517, 491, 500, 497, 499, 503, 520, 548, 491, 498, 510, 532, 500, 483, 523, 483, 490, 527, 505, 528, 515, 515, 495, 532, 516, 528, 481, 509, 502, 508, 516, 534, 504, 493, 520, 512, 505, 524, 522, 497, 486, 544, 532, 509, 490, 531, 539, 502, 546, 543, 519, 507, 481, 508, 488, 485, 512, 500, 494, 519, 545, 522, 543, 487, 524, 544, 522, 498, 542, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[500, 507, 497, 526, 494, 527, 522, 510, 546, 472, 506, 524, 509, 507, 468, 543, 482, 532, 525, 462, 492, 528, 527, 493, 577, 534, 521, 521, 510, 451, 497, 515, 499, 492, 529, 547, 523, 480, 536, 515, 485, 494, 508, 482, 506, 503, 517, 559, 509, 512, 527, 539, 530, 481, 543, 531, 512, 495, 519, 456, 505, 523, 521, 464, 551, 558, 500, 492, 543, 509, 531, 497, 551, 502, 533, 532, 495, 486, 521, 523, 491, 499, 471, 508, 545, 538, 530, 493, 502, 509, 533, 518, 548, 523, 498, 497, 512, 536, 531, 497, 534, 500, 533, 510, 514, 519, 479, 520, 514, 475, 518, 488, 536, 521, 497, 515, 547, 501, 493, 503, 490, 515, 515, 483, 477, 543, 523, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[495, 478, 470, 508, 502, 525, 495, 493, 538, 539, 523, 505, 547, 537, 509, 510, 533, 474, 491, 494, 495, 498, 486, 509, 496, 496, 475, 547, 499, 507, 542, 548, 494, 541, 511, 553, 494, 521, 532, 516, 491, 512, 496, 503, 498, 493, 522, 481, 470, 527, 512, 498, 536, 499, 536, 517, 522, 503, 502, 493, 555, 508, 521, 481, 545, 497, 516, 515, 497, 526, 512, 501, 509, 552, 507, 534, 564, 535, 452, 502, 482, 543, 507, 508, 490, 501, 509, 534, 508, 493, 514, 508, 521, 481, 512, 498, 556, 554, 557, 483, 538, 525, 554, 511, 522, 514, 527, 508, 485, 512, 537, 496, 524, 485, 525, 514, 511, 493, 508, 515, 548, 534, 525, 501, 483, 488, 494, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[480, 490, 498, 546, 505, 500, 501, 531, 524, 506, 495, 527, 484, 503, 527, 551, 546, 512, 512, 511, 503, 500, 507, 524, 493, 536, 486, 520, 482, 526, 538, 509, 537, 497, 479, 524, 526, 517, 515, 487, 525, 546, 505, 532, 513, 497, 518, 538, 530, 478, 535, 535, 496, 519, 515, 498, 526, 526, 514, 499, 496, 546, 495, 521, 532, 511, 523, 477, 506, 526, 542, 506, 509, 523, 530, 505, 525, 527, 467, 529, 561, 484, 508, 500, 494, 533, 515, 510, 491, 530, 503, 518, 514, 492, 525, 506, 496, 525, 496, 540, 488, 514, 498, 483, 488, 488, 466, 535, 492, 564, 546, 483, 549, 516, 518, 516, 470, 526, 486, 527, 497, 483, 525, 500, 496, 513, 507, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 485, 496, 510, 520, 491, 495, 531, 540, 484, 549, 528, 491, 514, 533, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 550, 504, 512, 531, 512, 534, 466, 527, 527, 501, 486, 506, 522, 495, 528, 35, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 484, 523, 475, 541, 509, 518, 534, 529, 516, 540, 509, 554, 532, 469, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 533, 511, 540, 504, 481, 523, 555, 529, 502, 491, 519, 488, 491, 525, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 490, 507, 475, 496, 513, 490, 499, 525, 541, 511, 485, 521, 515, 495, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 513, 481, 556, 512, 531, 498, 524, 532, 594, 475, 526, 499, 457, 524, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 489, 501, 486, 477, 525, 529, 511, 513, 495, 509, 494, 524, 539, 518, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 498, 516, 481, 534, 499, 517, 546, 515, 524, 524, 502, 500, 493, 488, 483, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8250, 8294, 8237, 8082, 8246, 8114, 8184] avg=8192.0000 max=8294 max_rank=2 min=8082 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[507, 489, 506, 511, 491, 522, 526, 509, 519, 511, 459, 471, 537, 508, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 530, 525, 534, 507, 498, 483, 517, 508, 518, 492, 552, 497, 459, 494, 502, 38, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 529, 496, 525, 521, 523, 521, 507, 505, 475, 473, 503, 535, 512, 506, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 554, 520, 488, 480, 554, 495, 489, 543, 524, 529, 505, 500, 477, 530, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 547, 504, 526, 469, 504, 508, 493, 493, 519, 554, 497, 511, 528, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 478, 531, 479, 518, 506, 496, 485, 538, 509, 516, 521, 547, 509, 478, 483, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 509, 516, 499, 538, 479, 506, 494, 534, 521, 537, 509, 516, 496, 551, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 548, 519, 487, 508, 515, 515, 508, 519, 512, 527, 538, 555, 539, 488, 527, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8197, 8144, 8231, 8189, 8139, 8230, 8317] avg=8192.0000 max=8317 max_rank=7 min=8089 min_rank=0 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 531, 520, 542, 494, 505, 527, 481, 499, 538, 484, 468, 491, 507, 536, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 498, 477, 493, 531, 521, 488, 526, 514, 531, 497, 531, 517, 524, 534, 475, 37, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 489, 553, 498, 523, 548, 485, 489, 525, 500, 553, 496, 543, 499, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 498, 501, 531, 550, 488, 467, 533, 510, 500, 540, 533, 508, 505, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 521, 508, 517, 502, 482, 509, 522, 541, 481, 508, 523, 483, 539, 500, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 514, 530, 495, 490, 506, 534, 518, 497, 535, 514, 520, 533, 495, 513, 542, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 522, 485, 501, 516, 515, 524, 500, 510, 490, 538, 512, 544, 519, 509, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 486, 543, 535, 496, 501, 502, 567, 509, 465, 506, 511, 494, 525, 505, 479, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8225, 8213, 8204, 8149, 8285, 8205, 8111] avg=8192.0000 max=8285 max_rank=5 min=8111 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 513, 535, 524, 520, 503, 505, 516, 506, 546, 517, 526, 534, 555, 527, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 498, 494, 507, 536, 491, 488, 519, 478, 478, 512, 496, 502, 550, 484, 511, 39, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 549, 509, 484, 482, 515, 538, 483, 508, 546, 523, 511, 498, 499, 470, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 524, 506, 471, 529, 490, 475, 496, 534, 518, 517, 491, 500, 497, 499, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 548, 491, 498, 510, 532, 467, 483, 523, 483, 490, 527, 505, 528, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 532, 516, 528, 481, 509, 502, 508, 516, 534, 504, 493, 520, 512, 505, 524, 18, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 477, 486, 544, 532, 509, 490, 531, 539, 502, 546, 543, 519, 507, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 485, 512, 500, 494, 519, 545, 522, 543, 487, 524, 544, 522, 498, 542, 519, 19, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8373, 8121, 8153, 8035, 8135, 8211, 8236, 8272] avg=8192.0000 max=8373 max_rank=0 min=8035 min_rank=3 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[532, 506, 529, 511, 506, 558, 546, 493, 505, 512, 496, 527, 528, 512, 522, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 520, 506, 486, 502, 507, 512, 503, 498, 473, 464, 490, 498, 537, 510, 566, 38, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 553, 532, 510, 499, 536, 506, 506, 517, 497, 505, 508, 540, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 542, 511, 551, 492, 508, 567, 512, 484, 550, 513, 527, 510, 491, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 520, 518, 494, 537, 482, 490, 539, 511, 515, 515, 505, 504, 528, 555, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 488, 500, 491, 500, 464, 495, 519, 518, 491, 525, 505, 495, 484, 522, 528, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 498, 502, 501, 516, 546, 528, 516, 501, 520, 465, 516, 507, 503, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 486, 513, 498, 505, 520, 501, 483, 524, 553, 527, 483, 481, 544, 495, 485, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8124, 8251, 8298, 8234, 8080, 8142, 8106] avg=8192.0000 max=8301 max_rank=0 min=8080 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[500, 507, 497, 526, 494, 527, 522, 510, 546, 472, 506, 524, 509, 507, 468, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 532, 525, 462, 492, 528, 527, 493, 577, 534, 521, 521, 510, 451, 497, 515, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 492, 529, 547, 523, 480, 536, 515, 485, 494, 508, 482, 506, 503, 481, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 527, 523, 530, 481, 543, 531, 512, 495, 519, 456, 505, 523, 521, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 558, 500, 492, 543, 509, 505, 497, 551, 502, 533, 532, 495, 486, 521, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 499, 471, 508, 545, 538, 530, 493, 502, 509, 533, 518, 548, 523, 498, 497, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 531, 497, 534, 500, 533, 510, 514, 519, 479, 520, 514, 475, 518, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 521, 497, 515, 547, 501, 493, 503, 490, 515, 515, 483, 477, 543, 523, 503, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8219, 8139, 8151, 8298, 8229, 8163, 8179] avg=8192.0000 max=8298 max_rank=4 min=8139 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 478, 470, 508, 502, 525, 495, 493, 538, 539, 523, 505, 547, 537, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 474, 491, 494, 495, 498, 486, 509, 496, 496, 475, 547, 499, 507, 542, 548, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 541, 511, 553, 494, 521, 532, 516, 491, 512, 496, 503, 498, 493, 486, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 527, 512, 484, 536, 499, 536, 517, 522, 503, 502, 493, 555, 508, 521, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 497, 516, 515, 497, 526, 486, 501, 509, 552, 507, 534, 564, 535, 452, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 543, 507, 508, 490, 501, 509, 534, 508, 493, 514, 508, 521, 481, 512, 498, 17, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 536, 557, 483, 538, 525, 554, 511, 522, 514, 527, 508, 485, 512, 537, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 485, 525, 514, 511, 493, 508, 515, 548, 534, 525, 501, 483, 488, 494, 534, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8142, 8122, 8166, 8238, 8136, 8361, 8197] avg=8192.0000 max=8361 max_rank=6 min=8122 min_rank=2 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32932 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[480, 490, 498, 546, 505, 500, 501, 531, 524, 506, 495, 527, 484, 503, 527, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 512, 512, 511, 503, 500, 507, 524, 493, 536, 486, 520, 482, 526, 538, 509, 36, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 497, 479, 524, 526, 517, 515, 487, 525, 546, 505, 532, 513, 497, 482, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 478, 535, 520, 496, 519, 515, 498, 526, 526, 514, 499, 496, 546, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 511, 523, 477, 506, 526, 515, 506, 509, 523, 530, 505, 525, 527, 467, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 484, 508, 500, 494, 533, 515, 510, 491, 530, 503, 518, 514, 492, 525, 506, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 508, 496, 540, 488, 514, 498, 483, 488, 488, 466, 535, 492, 564, 546, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 516, 518, 516, 470, 526, 486, 527, 497, 483, 525, 500, 496, 513, 507, 525, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8258, 8220, 8214, 8211, 8210, 8085, 8170] avg=8192.0000 max=8258 max_rank=1 min=8085 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 3968, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[499, 518, 525, 532, 513, 506, 541, 498, 498, 534, 455, 484, 505, 488, 527, 508, 505, 502, 529, 515, 504, 505, 506, 514, 512, 553, 495, 522, 529, 516, 530, 488, 489, 502, 520, 507, 526, 535, 508, 504, 477, 511, 517, 506, 522, 542, 491, 487, 509, 523, 530, 508, 508, 528, 475, 535, 533, 479, 507, 512, 482, 479, 515, 518, 509, 555, 537, 512, 510, 496, 536, 477, 548, 527, 515, 530, 520, 541, 521, 516, 470, 503, 490, 498, 502, 486, 514, 506, 519, 532, 531, 510, 537, 527, 571, 510, 498, 522, 516, 526, 505, 499, 524, 516, 512, 519, 516, 514, 528, 498, 519, 489, 541, 524, 500, 529, 552, 476, 514, 478, 508, 486, 480, 528, 516, 518, 485, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[554, 504, 499, 514, 469, 495, 520, 499, 526, 518, 555, 542, 515, 485, 468, 518, 515, 486, 489, 485, 499, 523, 506, 508, 508, 539, 517, 514, 502, 507, 506, 509, 482, 543, 524, 526, 537, 503, 498, 505, 478, 498, 493, 491, 516, 504, 551, 532, 535, 488, 460, 521, 527, 538, 526, 553, 508, 510, 521, 539, 559, 507, 506, 517, 521, 478, 523, 552, 490, 473, 499, 493, 432, 542, 508, 538, 437, 548, 460, 534, 563, 511, 569, 498, 593, 514, 501, 481, 550, 514, 499, 480, 488, 536, 521, 503, 490, 521, 505, 492, 500, 516, 511, 528, 529, 511, 481, 540, 510, 556, 530, 526, 506, 539, 535, 492, 504, 524, 505, 521, 482, 490, 492, 494, 526, 489, 538, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[550, 541, 529, 505, 550, 521, 496, 516, 514, 539, 512, 481, 523, 495, 501, 494, 529, 519, 493, 508, 530, 541, 475, 520, 487, 508, 515, 480, 526, 497, 514, 441, 524, 518, 500, 513, 518, 535, 493, 508, 509, 505, 469, 520, 518, 498, 522, 552, 512, 501, 502, 505, 504, 484, 527, 470, 508, 512, 560, 553, 506, 537, 519, 522, 503, 513, 483, 515, 516, 533, 484, 503, 549, 537, 537, 480, 534, 520, 534, 510, 518, 514, 503, 504, 494, 470, 498, 547, 477, 546, 527, 478, 487, 520, 515, 502, 542, 507, 537, 467, 497, 518, 520, 501, 498, 531, 490, 520, 542, 485, 528, 541, 507, 538, 495, 520, 473, 542, 527, 489, 523, 517, 493, 510, 521, 512, 504, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[531, 473, 543, 521, 510, 497, 531, 497, 520, 530, 480, 556, 512, 512, 487, 528, 509, 558, 532, 511, 512, 497, 498, 485, 515, 504, 513, 498, 518, 537, 540, 512, 563, 460, 524, 471, 519, 483, 494, 501, 516, 507, 487, 471, 549, 517, 520, 531, 479, 519, 529, 471, 520, 552, 523, 507, 472, 562, 526, 544, 468, 514, 479, 519, 536, 505, 520, 524, 522, 558, 517, 485, 466, 556, 505, 530, 515, 512, 508, 522, 469, 552, 473, 540, 524, 538, 510, 502, 503, 519, 504, 546, 489, 530, 512, 509, 528, 527, 494, 506, 527, 503, 547, 494, 496, 511, 508, 527, 462, 523, 517, 484, 489, 488, 512, 466, 484, 547, 532, 502, 520, 504, 485, 470, 551, 532, 538, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[553, 471, 484, 540, 503, 513, 526, 513, 559, 498, 494, 502, 541, 495, 494, 511, 461, 517, 468, 533, 516, 523, 547, 492, 538, 546, 453, 529, 502, 542, 471, 501, 541, 509, 468, 480, 503, 533, 538, 511, 544, 482, 510, 521, 521, 512, 470, 489, 554, 526, 521, 521, 504, 516, 533, 536, 532, 509, 505, 510, 514, 499, 516, 526, 472, 457, 505, 518, 517, 517, 491, 503, 492, 518, 568, 515, 482, 480, 514, 529, 512, 505, 519, 500, 496, 516, 546, 563, 465, 505, 519, 506, 500, 473, 514, 551, 507, 503, 536, 525, 509, 505, 528, 530, 520, 516, 502, 456, 543, 504, 474, 523, 513, 547, 507, 540, 534, 502, 520, 498, 510, 543, 535, 503, 515, 525, 492, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[508, 540, 486, 534, 511, 524, 474, 496, 488, 460, 541, 516, 533, 505, 515, 488, 509, 539, 491, 504, 534, 501, 549, 550, 494, 539, 535, 526, 549, 483, 509, 529, 550, 520, 500, 493, 533, 500, 510, 486, 527, 561, 461, 519, 513, 539, 538, 547, 505, 487, 494, 517, 542, 503, 481, 467, 502, 501, 545, 509, 529, 528, 532, 545, 539, 504, 504, 462, 517, 531, 511, 508, 554, 508, 520, 543, 507, 528, 496, 517, 505, 518, 469, 442, 506, 509, 507, 481, 508, 498, 509, 539, 531, 481, 522, 530, 567, 481, 524, 494, 488, 521, 489, 509, 518, 530, 472, 487, 521, 477, 500, 520, 528, 459, 489, 493, 505, 537, 515, 539, 528, 510, 542, 530, 486, 511, 525, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[502, 521, 509, 477, 478, 514, 514, 499, 482, 511, 540, 528, 524, 520, 529, 522, 517, 496, 533, 469, 496, 526, 484, 525, 506, 517, 512, 501, 499, 540, 489, 512, 481, 530, 501, 473, 511, 539, 505, 545, 467, 520, 558, 463, 527, 484, 512, 540, 484, 512, 525, 505, 485, 535, 507, 545, 525, 545, 540, 506, 534, 528, 534, 470, 529, 537, 529, 509, 485, 510, 579, 469, 524, 525, 535, 550, 515, 558, 493, 501, 504, 540, 492, 491, 484, 477, 552, 503, 521, 493, 505, 543, 546, 514, 510, 519, 500, 495, 488, 531, 500, 498, 504, 470, 517, 519, 487, 502, 452, 518, 472, 503, 511, 530, 540, 523, 541, 488, 503, 511, 496, 505, 542, 539, 510, 558, 503, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[509, 515, 471, 516, 474, 489, 499, 545, 571, 536, 514, 557, 485, 546, 505, 533, 483, 505, 514, 500, 504, 487, 508, 474, 521, 525, 526, 520, 496, 471, 536, 476, 507, 519, 505, 514, 494, 501, 532, 562, 476, 479, 510, 504, 497, 520, 537, 532, 537, 574, 527, 524, 490, 478, 510, 516, 549, 502, 535, 523, 487, 532, 503, 516, 454, 494, 554, 474, 501, 529, 512, 507, 526, 518, 503, 552, 498, 534, 537, 546, 502, 521, 537, 544, 479, 507, 524, 494, 509, 535, 473, 495, 489, 563, 511, 485, 492, 512, 458, 455, 521, 492, 507, 519, 523, 515, 516, 554, 541, 509, 512, 522, 494, 483, 561, 505, 489, 506, 523, 526, 495, 517, 518, 509, 542, 478, 491, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[552, 504, 499, 514, 469, 495, 520, 499, 526, 518, 555, 542, 515, 485, 468, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 489, 485, 499, 523, 506, 508, 508, 539, 517, 514, 502, 507, 506, 509, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 543, 524, 526, 537, 503, 498, 505, 478, 498, 493, 491, 516, 504, 551, 532, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 488, 460, 521, 527, 538, 526, 553, 508, 510, 470, 539, 559, 507, 506, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 478, 523, 552, 490, 473, 499, 493, 432, 542, 508, 503, 437, 548, 460, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 511, 569, 498, 593, 514, 501, 481, 550, 514, 499, 480, 488, 536, 521, 503, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 521, 505, 492, 500, 516, 511, 528, 529, 511, 481, 540, 510, 556, 530, 526, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 539, 535, 492, 504, 524, 505, 521, 482, 490, 492, 494, 522, 489, 538, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8141, 8196, 8264, 7993, 8330, 8286, 8147] avg=8192.0000 max=8330 max_rank=5 min=7993 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[548, 541, 529, 505, 550, 521, 496, 516, 514, 539, 512, 481, 523, 495, 501, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 493, 508, 530, 541, 475, 520, 487, 508, 515, 480, 526, 497, 514, 441, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 500, 513, 518, 535, 493, 508, 509, 505, 469, 520, 518, 498, 522, 552, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 501, 502, 505, 504, 484, 527, 470, 508, 512, 506, 553, 506, 537, 519, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 483, 515, 516, 533, 484, 503, 549, 537, 537, 449, 534, 520, 534, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 514, 503, 504, 494, 470, 498, 547, 477, 546, 527, 478, 487, 520, 515, 502, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 507, 537, 467, 497, 518, 520, 501, 498, 531, 490, 520, 542, 485, 528, 541, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 538, 495, 520, 473, 542, 527, 489, 523, 517, 493, 510, 517, 512, 504, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8109, 8215, 8168, 8220, 8109, 8267, 8183] avg=8192.0000 max=8267 max_rank=6 min=8109 min_rank=1 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[491, 518, 525, 532, 513, 506, 541, 498, 498, 534, 455, 484, 505, 488, 527, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 502, 529, 515, 504, 505, 506, 514, 512, 553, 495, 522, 529, 516, 530, 488, 22, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 502, 520, 507, 526, 535, 508, 504, 477, 511, 517, 506, 522, 542, 491, 487, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 523, 530, 508, 508, 528, 475, 535, 533, 479, 451, 512, 482, 479, 515, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 555, 537, 512, 510, 496, 536, 477, 548, 527, 515, 481, 520, 541, 521, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 503, 490, 498, 502, 486, 514, 506, 519, 532, 531, 510, 537, 527, 571, 510, 7, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 522, 516, 526, 505, 499, 524, 516, 512, 519, 516, 514, 528, 498, 519, 489, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 524, 500, 529, 552, 476, 514, 478, 508, 486, 480, 528, 509, 518, 485, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8261, 8164, 8085, 8301, 8228, 8243, 8131] avg=8192.0000 max=8301 max_rank=4 min=8085 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 473, 543, 521, 510, 497, 531, 497, 520, 530, 480, 556, 512, 512, 487, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 558, 532, 511, 512, 497, 498, 485, 515, 504, 513, 498, 518, 537, 540, 512, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 460, 524, 471, 519, 483, 494, 501, 516, 507, 487, 471, 549, 517, 520, 531, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 519, 529, 471, 520, 552, 523, 507, 472, 562, 475, 544, 468, 514, 479, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 505, 520, 524, 522, 558, 517, 485, 466, 556, 505, 495, 515, 512, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 552, 473, 540, 524, 538, 510, 502, 503, 519, 504, 546, 489, 530, 512, 509, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 527, 494, 506, 527, 503, 547, 494, 496, 511, 508, 527, 462, 523, 517, 484, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 488, 512, 466, 484, 547, 532, 502, 520, 504, 485, 470, 547, 532, 538, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8267, 8128, 8133, 8246, 8229, 8194, 8113] avg=8192.0000 max=8267 max_rank=1 min=8113 min_rank=7 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[551, 471, 484, 540, 503, 513, 526, 513, 559, 498, 494, 502, 541, 495, 494, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 517, 468, 533, 516, 523, 547, 492, 538, 546, 453, 529, 502, 542, 471, 501, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 509, 468, 480, 503, 533, 538, 511, 544, 482, 510, 521, 521, 512, 470, 489, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 526, 521, 521, 504, 516, 533, 536, 532, 509, 456, 510, 514, 499, 516, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 457, 505, 518, 517, 517, 491, 503, 492, 518, 568, 482, 482, 480, 514, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 519, 500, 496, 516, 546, 563, 465, 505, 519, 506, 500, 473, 514, 551, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 536, 525, 509, 505, 528, 530, 520, 516, 502, 456, 543, 504, 474, 523, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 547, 507, 540, 534, 502, 520, 498, 510, 543, 535, 503, 512, 525, 492, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8165, 8146, 8273, 8045, 8198, 8220, 8294] avg=8192.0000 max=8294 max_rank=7 min=8045 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[500, 521, 509, 477, 478, 514, 514, 499, 482, 511, 540, 528, 524, 520, 529, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 533, 469, 496, 526, 484, 525, 506, 517, 512, 501, 499, 540, 489, 512, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 530, 501, 473, 511, 539, 505, 545, 467, 520, 558, 463, 527, 484, 512, 540, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 512, 525, 505, 485, 535, 507, 545, 525, 545, 488, 506, 534, 528, 534, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 537, 529, 509, 485, 510, 579, 469, 524, 525, 535, 515, 515, 558, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 540, 492, 491, 484, 477, 552, 503, 521, 493, 505, 543, 546, 514, 510, 519, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 495, 488, 531, 500, 498, 504, 470, 517, 519, 487, 502, 452, 518, 472, 503, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 530, 540, 523, 541, 488, 503, 511, 496, 505, 542, 539, 507, 558, 503, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8150, 8171, 8228, 8313, 8202, 7997, 8307] avg=8192.0000 max=8313 max_rank=4 min=7997 min_rank=6 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[507, 515, 471, 516, 474, 489, 499, 545, 571, 536, 514, 557, 485, 546, 505, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 505, 514, 500, 504, 487, 508, 474, 521, 525, 526, 520, 496, 471, 536, 476, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 519, 505, 514, 494, 501, 532, 562, 476, 479, 510, 504, 497, 520, 537, 532, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 574, 527, 524, 490, 478, 510, 516, 549, 502, 483, 523, 487, 532, 503, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 494, 554, 474, 501, 529, 512, 507, 526, 518, 503, 517, 498, 534, 537, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 521, 537, 544, 479, 507, 524, 494, 509, 535, 473, 495, 489, 563, 511, 485, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 512, 458, 455, 521, 492, 507, 519, 523, 515, 516, 554, 541, 509, 512, 522, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 483, 561, 505, 489, 506, 523, 526, 495, 517, 518, 509, 538, 478, 491, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8074, 8204, 8251, 8204, 8177, 8189, 8174] avg=8192.0000 max=8263 max_rank=0 min=8074 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 540, 486, 534, 511, 524, 474, 496, 488, 460, 541, 516, 533, 505, 515, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 539, 491, 504, 534, 501, 549, 550, 494, 539, 535, 526, 549, 483, 509, 529, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 520, 500, 493, 533, 500, 510, 486, 527, 561, 461, 519, 513, 539, 538, 547, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 494, 517, 542, 503, 481, 467, 502, 501, 492, 509, 529, 528, 532, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 504, 462, 517, 531, 511, 508, 554, 508, 520, 508, 507, 528, 496, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 469, 442, 506, 509, 507, 481, 508, 498, 509, 539, 531, 481, 522, 530, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 481, 524, 494, 488, 521, 489, 509, 518, 530, 472, 487, 521, 477, 500, 520, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 459, 489, 493, 505, 537, 515, 539, 528, 510, 542, 530, 483, 511, 525, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8369, 8312, 8134, 8214, 8063, 8140, 8187] avg=8192.0000 max=8369 max_rank=1 min=8063 min_rank=5 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 3968, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4352, 4224, 3968, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 3840, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[502, 509, 519, 488, 504, 493, 517, 519, 546, 531, 473, 499, 505, 494, 548, 482, 508, 471, 540, 546, 532, 503, 492, 506, 541, 512, 509, 523, 520, 495, 491, 507, 510, 515, 504, 522, 489, 544, 498, 493, 496, 496, 522, 538, 522, 510, 541, 520, 554, 494, 532, 485, 511, 495, 493, 520, 499, 536, 527, 495, 505, 518, 522, 520, 476, 498, 487, 525, 535, 493, 481, 512, 499, 498, 535, 503, 520, 495, 538, 520, 512, 511, 542, 506, 523, 529, 529, 561, 494, 525, 514, 518, 527, 516, 534, 525, 516, 465, 467, 515, 527, 520, 531, 512, 533, 527, 512, 507, 537, 494, 522, 544, 511, 495, 521, 487, 471, 521, 508, 519, 502, 525, 473, 547, 483, 517, 487, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[518, 546, 525, 509, 502, 507, 500, 531, 500, 496, 553, 495, 510, 521, 545, 549, 470, 504, 494, 500, 530, 470, 497, 501, 460, 525, 500, 521, 497, 509, 489, 519, 515, 524, 541, 553, 531, 512, 516, 493, 482, 523, 494, 505, 528, 483, 500, 519, 531, 511, 529, 505, 538, 500, 483, 506, 474, 537, 544, 537, 488, 524, 529, 476, 503, 496, 544, 530, 525, 542, 536, 492, 495, 507, 515, 475, 506, 491, 518, 518, 528, 497, 522, 526, 559, 503, 522, 543, 530, 571, 475, 484, 501, 579, 516, 501, 490, 507, 500, 529, 503, 485, 505, 485, 529, 507, 511, 510, 518, 535, 505, 485, 531, 521, 508, 516, 514, 492, 532, 468, 524, 498, 500, 479, 543, 518, 532, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[523, 540, 488, 492, 494, 493, 559, 518, 496, 475, 496, 530, 503, 509, 480, 499, 506, 509, 520, 511, 489, 541, 497, 485, 517, 529, 502, 536, 480, 532, 523, 541, 487, 533, 485, 522, 479, 501, 505, 519, 485, 541, 502, 494, 511, 508, 482, 503, 506, 530, 500, 479, 477, 519, 484, 508, 510, 506, 521, 487, 531, 515, 528, 536, 496, 459, 516, 537, 496, 500, 522, 515, 534, 548, 501, 531, 513, 503, 512, 539, 541, 494, 549, 497, 532, 497, 512, 502, 497, 525, 533, 536, 519, 515, 564, 492, 479, 487, 547, 546, 517, 524, 521, 494, 521, 516, 518, 512, 558, 491, 511, 495, 523, 520, 539, 536, 508, 526, 528, 519, 510, 503, 511, 540, 528, 477, 475, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[494, 517, 528, 503, 517, 517, 511, 533, 509, 504, 534, 512, 498, 514, 487, 512, 503, 492, 482, 547, 483, 515, 491, 557, 506, 503, 496, 521, 507, 506, 533, 473, 510, 524, 532, 516, 495, 524, 487, 496, 501, 525, 512, 512, 507, 480, 545, 484, 529, 512, 513, 518, 531, 459, 515, 505, 522, 503, 534, 521, 505, 524, 492, 545, 538, 531, 558, 498, 479, 546, 520, 530, 502, 510, 545, 510, 536, 537, 513, 514, 518, 508, 524, 515, 485, 559, 514, 501, 506, 473, 465, 496, 530, 478, 516, 498, 485, 531, 527, 513, 492, 510, 531, 528, 533, 550, 494, 486, 502, 500, 527, 497, 518, 506, 547, 494, 503, 485, 534, 529, 524, 509, 489, 500, 542, 535, 489, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[518, 475, 506, 496, 501, 524, 475, 515, 532, 521, 529, 522, 509, 544, 527, 503, 524, 526, 540, 529, 468, 531, 523, 488, 524, 518, 516, 526, 504, 538, 494, 503, 517, 515, 563, 525, 542, 512, 483, 510, 514, 535, 536, 503, 523, 539, 501, 457, 520, 528, 515, 474, 471, 511, 490, 484, 545, 504, 499, 513, 523, 555, 505, 493, 516, 517, 495, 519, 528, 528, 463, 513, 485, 502, 516, 497, 498, 567, 525, 495, 491, 534, 509, 503, 548, 541, 514, 528, 491, 499, 501, 501, 511, 498, 514, 513, 497, 495, 516, 498, 507, 528, 516, 516, 482, 533, 485, 485, 517, 537, 520, 519, 522, 503, 490, 547, 496, 482, 528, 502, 555, 501, 509, 476, 490, 522, 537, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[521, 523, 524, 475, 462, 516, 500, 481, 542, 513, 521, 515, 502, 529, 531, 532, 502, 492, 520, 583, 496, 519, 505, 502, 498, 515, 449, 527, 539, 504, 503, 518, 506, 505, 493, 525, 542, 508, 505, 517, 476, 506, 519, 514, 497, 505, 526, 507, 473, 488, 462, 508, 558, 522, 498, 494, 541, 494, 535, 545, 538, 517, 532, 495, 546, 496, 465, 539, 546, 502, 505, 554, 460, 533, 537, 513, 511, 517, 498, 537, 522, 500, 492, 514, 487, 523, 523, 528, 491, 532, 494, 508, 515, 535, 491, 522, 482, 528, 530, 535, 505, 527, 501, 523, 523, 515, 536, 471, 481, 525, 509, 543, 471, 532, 547, 487, 534, 526, 473, 525, 508, 491, 510, 503, 508, 502, 517, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[545, 533, 540, 500, 489, 522, 483, 501, 469, 511, 495, 513, 491, 540, 525, 473, 536, 478, 494, 540, 500, 483, 512, 510, 547, 504, 510, 530, 504, 504, 513, 512, 534, 516, 507, 534, 492, 492, 528, 496, 450, 533, 537, 516, 482, 520, 513, 520, 548, 516, 511, 549, 530, 519, 517, 529, 516, 503, 531, 484, 509, 479, 514, 479, 497, 510, 478, 493, 516, 518, 527, 537, 531, 537, 531, 553, 481, 533, 509, 496, 518, 542, 541, 517, 504, 476, 528, 518, 492, 529, 528, 502, 492, 553, 493, 473, 508, 496, 507, 502, 473, 535, 532, 480, 491, 501, 516, 488, 532, 490, 554, 520, 537, 489, 497, 517, 522, 542, 535, 506, 523, 474, 536, 522, 512, 508, 523, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[494, 503, 501, 578, 514, 524, 538, 500, 542, 529, 512, 504, 537, 495, 511, 474, 508, 476, 539, 510, 492, 504, 521, 495, 507, 544, 526, 531, 543, 534, 521, 496, 497, 508, 539, 501, 519, 483, 488, 512, 501, 525, 505, 528, 536, 517, 503, 510, 504, 490, 519, 506, 498, 505, 503, 497, 527, 528, 498, 444, 517, 507, 485, 522, 522, 512, 490, 486, 500, 478, 515, 501, 528, 524, 492, 524, 504, 504, 527, 475, 563, 536, 528, 484, 484, 504, 493, 464, 504, 538, 538, 544, 491, 507, 527, 477, 522, 520, 510, 516, 528, 517, 514, 551, 492, 509, 516, 539, 514, 500, 500, 523, 485, 527, 499, 473, 502, 557, 517, 505, 516, 532, 536, 527, 537, 522, 516, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 546, 525, 509, 502, 507, 500, 531, 500, 496, 553, 495, 510, 521, 545, 549, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 504, 494, 500, 530, 470, 497, 501, 460, 525, 500, 521, 497, 509, 489, 519, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 524, 541, 553, 531, 512, 516, 493, 482, 523, 494, 505, 528, 483, 500, 519, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 511, 529, 505, 538, 500, 483, 506, 474, 537, 544, 537, 488, 524, 529, 476, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 496, 544, 530, 525, 542, 536, 492, 495, 507, 502, 475, 506, 491, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 481, 526, 559, 503, 522, 543, 530, 571, 475, 484, 501, 579, 516, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 507, 500, 529, 503, 485, 505, 485, 529, 507, 511, 510, 516, 535, 505, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 521, 508, 516, 514, 492, 532, 468, 524, 498, 500, 479, 543, 518, 532, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8007, 8233, 8229, 8180, 8316, 8102, 8158] avg=8192.0000 max=8316 max_rank=5 min=8007 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 509, 519, 488, 504, 493, 517, 519, 546, 531, 473, 499, 505, 494, 548, 482, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 471, 540, 546, 532, 503, 492, 506, 541, 512, 509, 523, 520, 495, 491, 507, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 504, 522, 489, 544, 498, 493, 496, 496, 522, 538, 522, 510, 541, 520, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 494, 532, 485, 511, 495, 493, 520, 499, 536, 527, 495, 505, 518, 522, 520, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 498, 487, 525, 535, 493, 481, 512, 499, 498, 521, 503, 520, 495, 538, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 511, 500, 506, 523, 529, 529, 561, 494, 525, 514, 518, 527, 516, 534, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 465, 467, 515, 527, 520, 531, 512, 533, 527, 512, 507, 535, 494, 522, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 495, 521, 487, 471, 521, 508, 519, 502, 525, 473, 547, 483, 517, 487, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8218, 8235, 8223, 8101, 8324, 8227, 8075] avg=8192.0000 max=8324 max_rank=5 min=8075 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[521, 523, 524, 475, 462, 516, 500, 481, 542, 513, 521, 515, 502, 529, 531, 532, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 492, 520, 583, 496, 519, 505, 502, 498, 515, 449, 527, 539, 504, 503, 518, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 505, 493, 525, 542, 508, 505, 517, 476, 506, 519, 514, 497, 505, 526, 507, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 488, 462, 508, 558, 522, 498, 494, 541, 494, 535, 545, 538, 517, 532, 495, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 496, 465, 539, 546, 502, 505, 554, 460, 533, 523, 513, 511, 517, 498, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 500, 453, 514, 487, 523, 523, 528, 491, 532, 494, 508, 515, 535, 491, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 528, 530, 535, 505, 527, 501, 523, 523, 515, 536, 471, 479, 525, 509, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 532, 547, 487, 534, 526, 473, 525, 508, 491, 510, 503, 508, 502, 517, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8166, 8216, 8245, 8138, 8232, 8156] avg=8192.0000 max=8245 max_rank=4 min=8138 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[523, 540, 488, 492, 494, 493, 559, 518, 496, 475, 496, 530, 503, 509, 480, 499, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 520, 511, 489, 541, 497, 485, 517, 529, 502, 536, 480, 532, 523, 541, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 533, 485, 522, 479, 501, 505, 519, 485, 541, 502, 494, 511, 508, 482, 503, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 530, 500, 479, 477, 519, 484, 508, 510, 506, 521, 487, 531, 515, 528, 536, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 459, 516, 537, 496, 500, 522, 515, 534, 548, 488, 531, 513, 503, 512, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 494, 506, 497, 532, 497, 512, 502, 497, 525, 533, 536, 519, 515, 564, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 487, 547, 546, 517, 524, 521, 494, 521, 516, 518, 512, 556, 491, 511, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 520, 539, 536, 508, 526, 528, 519, 510, 503, 511, 540, 528, 477, 475, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8240, 8071, 8155, 8209, 8262, 8235, 8265] avg=8192.0000 max=8265 max_rank=7 min=8071 min_rank=2 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 475, 506, 496, 501, 524, 475, 515, 532, 521, 529, 522, 509, 544, 527, 503, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 526, 540, 529, 468, 531, 523, 488, 524, 518, 516, 526, 504, 538, 494, 503, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 563, 525, 542, 512, 483, 510, 514, 535, 536, 503, 523, 539, 501, 457, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 515, 474, 471, 511, 490, 484, 545, 504, 499, 513, 523, 555, 505, 493, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 495, 519, 528, 528, 463, 513, 485, 502, 503, 497, 498, 567, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 534, 470, 503, 548, 541, 514, 528, 491, 499, 501, 501, 511, 498, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 516, 498, 507, 528, 516, 516, 482, 533, 485, 485, 515, 537, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 490, 547, 496, 482, 528, 502, 555, 501, 509, 476, 490, 522, 537, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8272, 8289, 8146, 8151, 8157, 8149, 8171] avg=8192.0000 max=8289 max_rank=2 min=8146 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[494, 517, 528, 503, 517, 517, 511, 533, 509, 504, 534, 512, 498, 514, 487, 512, 7, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 482, 547, 483, 515, 491, 557, 506, 503, 496, 521, 507, 506, 533, 473, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 532, 516, 495, 524, 487, 496, 501, 525, 512, 512, 507, 480, 545, 484, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 513, 518, 531, 459, 515, 505, 522, 503, 534, 521, 505, 524, 492, 545, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 531, 558, 498, 479, 546, 520, 530, 502, 510, 526, 510, 536, 537, 513, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 472, 515, 485, 559, 514, 501, 506, 473, 465, 496, 530, 478, 516, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 531, 527, 513, 492, 510, 531, 528, 533, 550, 494, 486, 495, 500, 527, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 506, 547, 494, 503, 485, 534, 529, 518, 509, 489, 500, 542, 535, 489, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8140, 8171, 8249, 8348, 8034, 8199, 8188] avg=8192.0000 max=8348 max_rank=4 min=8034 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[545, 533, 540, 500, 489, 522, 483, 501, 469, 511, 495, 513, 491, 540, 525, 473, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 478, 494, 540, 500, 483, 512, 510, 547, 504, 510, 530, 504, 504, 513, 512, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 516, 507, 534, 492, 492, 528, 496, 450, 533, 537, 516, 482, 520, 513, 520, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 516, 511, 549, 530, 519, 517, 529, 516, 503, 531, 484, 509, 479, 514, 479, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 478, 493, 516, 518, 527, 537, 531, 537, 517, 553, 481, 533, 509, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 542, 499, 517, 504, 476, 528, 518, 492, 529, 528, 502, 492, 553, 493, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 507, 502, 473, 535, 532, 480, 491, 501, 516, 488, 530, 490, 554, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 489, 497, 517, 522, 542, 535, 506, 523, 474, 536, 522, 512, 508, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8199, 8185, 8251, 8233, 8164, 8123, 8247] avg=8192.0000 max=8251 max_rank=3 min=8123 min_rank=6 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[494, 503, 501, 578, 514, 524, 538, 500, 542, 529, 512, 504, 537, 495, 511, 474, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 476, 539, 510, 492, 504, 521, 495, 507, 544, 526, 531, 543, 534, 521, 496, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 508, 539, 501, 519, 483, 488, 512, 501, 525, 505, 528, 536, 517, 503, 510, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 490, 519, 506, 498, 505, 503, 497, 527, 528, 498, 444, 517, 507, 485, 522, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 490, 486, 500, 478, 515, 501, 528, 524, 479, 524, 504, 504, 527, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 536, 487, 484, 484, 504, 493, 464, 504, 538, 538, 544, 491, 507, 527, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 510, 516, 528, 517, 514, 551, 492, 509, 516, 539, 512, 500, 500, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 527, 499, 473, 502, 557, 517, 505, 516, 532, 536, 527, 537, 522, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8268, 8186, 8067, 8069, 8141, 8269, 8276] avg=8192.0000 max=8276 max_rank=7 min=8067 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4224, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[549, 499, 548, 517, 513, 460, 487, 517, 487, 505, 486, 509, 525, 486, 524, 513, 521, 507, 512, 508, 500, 493, 525, 537, 524, 514, 533, 533, 507, 517, 543, 497, 538, 517, 506, 564, 506, 539, 535, 510, 523, 527, 507, 513, 514, 480, 517, 530, 512, 538, 524, 509, 509, 548, 514, 543, 517, 462, 515, 482, 507, 528, 544, 514, 467, 493, 541, 515, 506, 515, 508, 472, 514, 507, 517, 514, 498, 505, 471, 509, 493, 479, 500, 492, 516, 513, 516, 496, 497, 536, 508, 522, 487, 531, 574, 498, 539, 496, 483, 466, 487, 531, 509, 469, 512, 529, 501, 540, 464, 510, 525, 513, 499, 522, 506, 514, 546, 529, 493, 492, 523, 519, 549, 541, 512, 485, 495, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[508, 542, 545, 517, 536, 532, 513, 533, 505, 524, 498, 501, 521, 474, 499, 522, 523, 526, 484, 496, 477, 502, 523, 514, 518, 521, 493, 517, 468, 482, 508, 506, 536, 535, 502, 483, 503, 569, 520, 505, 499, 506, 491, 523, 495, 514, 523, 512, 485, 531, 503, 497, 528, 469, 495, 512, 494, 473, 510, 524, 517, 532, 507, 498, 491, 511, 544, 482, 530, 511, 508, 525, 499, 528, 488, 494, 520, 529, 485, 487, 542, 521, 516, 517, 536, 511, 509, 523, 540, 540, 529, 544, 532, 507, 525, 548, 488, 532, 524, 528, 480, 496, 551, 485, 521, 508, 528, 537, 489, 497, 503, 509, 483, 507, 496, 548, 553, 527, 536, 497, 506, 488, 525, 509, 491, 502, 523, 478] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[521, 511, 531, 508, 476, 502, 547, 505, 521, 527, 545, 520, 537, 519, 553, 504, 513, 529, 496, 515, 475, 524, 518, 522, 511, 510, 526, 527, 534, 536, 498, 473, 536, 537, 533, 496, 498, 475, 567, 526, 493, 489, 477, 511, 480, 480, 513, 545, 528, 497, 466, 502, 506, 497, 440, 500, 453, 504, 506, 508, 489, 476, 530, 516, 542, 523, 489, 511, 509, 468, 529, 486, 519, 505, 498, 528, 524, 559, 519, 508, 523, 536, 502, 515, 509, 536, 497, 519, 490, 473, 517, 523, 505, 537, 509, 516, 497, 518, 539, 492, 503, 510, 487, 507, 533, 523, 512, 519, 490, 489, 553, 523, 513, 505, 486, 513, 521, 539, 523, 513, 507, 498, 501, 572, 529, 526, 531, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[482, 558, 505, 500, 534, 524, 497, 507, 531, 529, 506, 498, 511, 521, 535, 518, 513, 512, 518, 501, 533, 518, 492, 518, 517, 496, 500, 518, 504, 511, 487, 498, 525, 485, 511, 515, 493, 515, 536, 530, 560, 521, 542, 512, 505, 530, 494, 527, 485, 476, 508, 527, 525, 520, 483, 501, 532, 539, 506, 512, 485, 541, 513, 497, 502, 514, 570, 521, 573, 509, 497, 526, 470, 490, 493, 508, 513, 526, 509, 496, 531, 483, 530, 526, 479, 525, 521, 528, 501, 478, 491, 515, 522, 492, 493, 555, 524, 508, 530, 529, 506, 512, 484, 459, 528, 519, 511, 507, 521, 493, 518, 529, 481, 534, 521, 500, 544, 499, 505, 484, 500, 510, 474, 512, 489, 529, 503, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[531, 491, 497, 538, 537, 498, 519, 490, 526, 479, 496, 457, 525, 492, 494, 538, 495, 473, 543, 512, 564, 489, 496, 510, 519, 545, 511, 500, 502, 569, 518, 538, 461, 488, 515, 524, 504, 523, 539, 557, 553, 507, 504, 507, 507, 518, 521, 520, 497, 509, 525, 488, 530, 516, 519, 515, 509, 489, 526, 492, 511, 548, 524, 498, 525, 542, 557, 514, 515, 497, 532, 534, 499, 522, 518, 511, 463, 516, 495, 487, 496, 500, 499, 494, 506, 508, 473, 497, 481, 513, 493, 498, 544, 521, 535, 523, 524, 518, 517, 506, 476, 509, 489, 514, 464, 519, 503, 506, 508, 475, 513, 503, 540, 483, 531, 508, 552, 553, 511, 527, 565, 540, 496, 532, 505, 508, 498, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[502, 525, 512, 522, 494, 503, 470, 514, 502, 501, 509, 520, 514, 517, 484, 521, 542, 510, 528, 497, 498, 505, 520, 542, 536, 529, 537, 492, 528, 547, 498, 486, 487, 550, 542, 496, 539, 533, 520, 511, 537, 502, 494, 533, 546, 489, 541, 484, 512, 497, 462, 514, 539, 481, 500, 528, 544, 542, 520, 504, 494, 528, 519, 549, 545, 510, 509, 521, 501, 500, 492, 561, 465, 525, 493, 524, 549, 495, 498, 535, 475, 516, 478, 492, 513, 501, 500, 492, 470, 493, 500, 550, 543, 512, 499, 508, 500, 540, 510, 530, 526, 512, 526, 481, 554, 514, 523, 509, 471, 498, 490, 481, 542, 517, 503, 481, 502, 539, 529, 484, 512, 532, 516, 485, 509, 517, 485, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[518, 507, 503, 516, 502, 497, 517, 510, 540, 499, 536, 482, 514, 505, 520, 531, 576, 532, 472, 516, 480, 498, 539, 480, 505, 540, 538, 549, 502, 497, 485, 514, 516, 508, 498, 531, 562, 528, 528, 525, 517, 536, 506, 525, 495, 511, 513, 501, 486, 504, 486, 507, 511, 503, 523, 510, 502, 495, 497, 531, 521, 528, 503, 487, 519, 516, 518, 550, 524, 511, 512, 466, 499, 473, 481, 510, 532, 515, 537, 529, 529, 492, 503, 512, 506, 506, 534, 504, 556, 520, 489, 498, 525, 463, 516, 514, 524, 483, 549, 510, 485, 505, 497, 526, 531, 505, 546, 467, 473, 530, 511, 490, 500, 522, 500, 475, 492, 523, 494, 562, 518, 511, 497, 504, 514, 508, 534, 577] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[576, 520, 524, 506, 519, 523, 481, 559, 518, 522, 522, 474, 494, 509, 502, 555, 504, 517, 510, 528, 521, 510, 517, 470, 504, 484, 508, 511, 493, 529, 461, 490, 515, 540, 519, 496, 500, 545, 518, 526, 528, 529, 483, 540, 509, 508, 499, 513, 540, 505, 473, 533, 532, 509, 515, 522, 504, 543, 487, 495, 536, 521, 549, 521, 472, 542, 489, 502, 475, 521, 478, 483, 455, 524, 493, 520, 518, 499, 504, 497, 504, 538, 541, 496, 491, 550, 558, 525, 535, 495, 483, 470, 529, 500, 498, 517, 515, 499, 535, 526, 547, 507, 502, 537, 495, 511, 526, 467, 536, 546, 528, 524, 487, 526, 553, 563, 526, 507, 504, 508, 491, 518, 479, 499, 498, 483, 475, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[531, 491, 497, 538, 537, 498, 519, 490, 526, 479, 496, 457, 525, 492, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 473, 543, 512, 564, 489, 496, 510, 519, 545, 511, 500, 502, 569, 518, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 488, 515, 524, 504, 523, 466, 557, 553, 507, 504, 507, 507, 518, 521, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 525, 488, 530, 516, 519, 515, 509, 489, 526, 492, 511, 548, 524, 498, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 542, 557, 514, 515, 497, 532, 534, 499, 522, 518, 511, 463, 516, 495, 487, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 500, 499, 494, 506, 508, 473, 497, 481, 513, 493, 498, 544, 521, 535, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 517, 506, 476, 509, 489, 514, 464, 519, 503, 506, 508, 475, 513, 503, 20, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 483, 531, 508, 527, 553, 511, 527, 565, 540, 496, 532, 505, 508, 498, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8284, 8175, 8236, 8265, 8086, 8078, 8323] avg=8192.0000 max=8323 max_rank=7 min=8078 min_rank=6 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[549, 499, 548, 517, 513, 460, 487, 517, 487, 505, 486, 509, 525, 486, 524, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 507, 512, 508, 500, 493, 525, 537, 524, 514, 533, 533, 507, 517, 543, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 517, 506, 564, 506, 539, 462, 510, 523, 527, 507, 513, 514, 480, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 524, 509, 509, 548, 514, 543, 517, 462, 515, 482, 507, 528, 544, 514, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 493, 541, 515, 506, 515, 508, 472, 514, 507, 517, 514, 498, 505, 471, 509, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 479, 500, 492, 516, 513, 516, 496, 497, 536, 508, 522, 487, 531, 574, 498, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 496, 483, 466, 487, 531, 509, 469, 512, 529, 501, 540, 464, 510, 525, 513, 20, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 522, 506, 514, 521, 529, 493, 492, 523, 519, 549, 541, 512, 485, 495, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8271, 8253, 8306, 8090, 8163, 8108, 8239] avg=8192.0000 max=8306 max_rank=3 min=8090 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32600 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[482, 558, 505, 500, 534, 524, 497, 507, 531, 529, 506, 498, 511, 521, 535, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 512, 518, 501, 533, 518, 492, 518, 517, 496, 500, 518, 504, 511, 487, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 485, 511, 515, 493, 515, 463, 530, 560, 521, 542, 512, 505, 530, 494, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 476, 508, 527, 525, 520, 483, 501, 532, 539, 506, 512, 485, 541, 513, 497, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 570, 521, 573, 509, 497, 526, 470, 490, 493, 508, 513, 526, 509, 496, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 483, 530, 526, 479, 525, 521, 528, 501, 478, 491, 515, 522, 492, 493, 555, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 508, 530, 529, 506, 512, 484, 459, 528, 519, 511, 507, 521, 493, 518, 529, 20, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 534, 521, 500, 519, 499, 505, 484, 500, 510, 474, 512, 489, 529, 503, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8136, 8228, 8190, 8255, 8175, 8212, 8103] avg=8192.0000 max=8255 max_rank=4 min=8103 min_rank=7 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[521, 511, 531, 508, 476, 502, 547, 505, 521, 527, 545, 520, 537, 519, 553, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 496, 515, 475, 524, 518, 522, 511, 510, 526, 527, 534, 536, 498, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 537, 533, 496, 498, 475, 490, 526, 493, 489, 477, 511, 480, 480, 513, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 466, 502, 506, 497, 440, 500, 453, 504, 506, 508, 489, 476, 530, 516, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 523, 489, 511, 509, 468, 529, 486, 519, 505, 498, 528, 524, 559, 519, 508, 35, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 536, 502, 515, 509, 536, 497, 519, 490, 473, 517, 523, 505, 537, 509, 516, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 518, 539, 492, 503, 510, 487, 507, 533, 523, 512, 519, 490, 489, 553, 523, 19, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 486, 513, 498, 539, 523, 513, 507, 498, 501, 572, 529, 526, 531, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8207, 8079, 7960, 8256, 8211, 8227, 8286] avg=8192.0000 max=8310 max_rank=0 min=7960 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32980 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 525, 512, 522, 494, 503, 470, 514, 502, 501, 509, 520, 514, 517, 484, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 528, 497, 498, 505, 520, 542, 536, 529, 537, 492, 528, 547, 498, 486, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 550, 542, 496, 539, 533, 444, 511, 537, 502, 494, 533, 546, 489, 541, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 462, 514, 539, 481, 500, 528, 544, 542, 520, 504, 494, 528, 519, 549, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 510, 509, 521, 501, 500, 492, 561, 465, 525, 493, 524, 549, 495, 498, 535, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 516, 478, 492, 513, 501, 500, 492, 470, 493, 500, 550, 543, 512, 499, 508, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 540, 510, 530, 526, 512, 526, 481, 554, 514, 523, 509, 471, 498, 490, 481, 20, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 517, 503, 481, 474, 539, 529, 484, 512, 532, 516, 485, 509, 517, 485, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 8300, 8228, 8275, 8265, 8049, 8202, 8136] avg=8192.0000 max=8300 max_rank=1 min=8049 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 507, 503, 516, 502, 497, 517, 510, 540, 499, 536, 482, 514, 505, 520, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 532, 472, 516, 480, 498, 539, 480, 505, 540, 538, 549, 502, 497, 485, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 498, 531, 562, 528, 457, 525, 517, 536, 506, 525, 495, 511, 513, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 504, 486, 507, 511, 503, 523, 510, 502, 495, 497, 531, 521, 528, 503, 487, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 516, 518, 550, 524, 511, 512, 466, 499, 473, 481, 510, 532, 515, 537, 529, 32, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 492, 503, 512, 506, 506, 534, 504, 556, 520, 489, 498, 525, 463, 516, 514, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 483, 549, 510, 485, 505, 497, 526, 531, 505, 546, 467, 473, 530, 511, 490, 18, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 522, 500, 475, 470, 523, 494, 562, 518, 511, 497, 504, 514, 508, 534, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8223, 8229, 8133, 8228, 8172, 8164, 8209] avg=8192.0000 max=8229 max_rank=2 min=8133 min_rank=3 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 542, 545, 517, 536, 532, 513, 533, 505, 524, 498, 501, 521, 474, 499, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 484, 496, 477, 502, 523, 514, 518, 521, 493, 517, 468, 482, 508, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 535, 502, 483, 503, 569, 450, 505, 499, 506, 491, 523, 495, 514, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 531, 503, 497, 528, 469, 495, 512, 494, 473, 510, 524, 517, 532, 507, 498, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 511, 544, 482, 530, 511, 508, 525, 499, 528, 488, 494, 520, 529, 485, 487, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 521, 516, 517, 536, 511, 509, 523, 540, 540, 529, 544, 532, 507, 525, 548, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 532, 524, 528, 480, 496, 551, 485, 521, 508, 528, 537, 489, 497, 503, 509, 20, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 507, 496, 548, 528, 527, 536, 497, 506, 488, 525, 509, 491, 502, 523, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8058, 8146, 8113, 8169, 8445, 8210, 8144] avg=8192.0000 max=8445 max_rank=5 min=8058 min_rank=1 imbalance_ratio=1.030884 max_load_reduction=57091 max_load_reduction_pct=0.871140 moved_to_echo=32968 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[576, 520, 524, 506, 519, 523, 481, 559, 518, 522, 522, 474, 494, 509, 502, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 510, 528, 521, 510, 517, 470, 504, 484, 508, 511, 493, 529, 461, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 540, 519, 496, 500, 545, 448, 526, 528, 529, 483, 540, 509, 508, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 505, 473, 533, 532, 509, 515, 522, 504, 543, 487, 495, 536, 521, 549, 521, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 542, 489, 502, 475, 521, 478, 483, 455, 524, 493, 520, 518, 499, 504, 497, 32, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 538, 541, 496, 491, 550, 558, 525, 535, 495, 483, 470, 529, 500, 498, 517, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 535, 526, 547, 507, 502, 537, 495, 511, 526, 467, 536, 546, 528, 524, 19, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 526, 553, 563, 503, 507, 504, 508, 491, 518, 479, 499, 498, 483, 475, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8057, 8198, 8323, 8008, 8235, 8335, 8096] avg=8192.0000 max=8335 max_rank=6 min=8008 min_rank=4 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4352, 3712, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[546, 474, 536, 525, 505, 503, 505, 537, 524, 562, 534, 500, 481, 503, 536, 515, 518, 507, 535, 502, 530, 528, 499, 524, 503, 500, 523, 514, 539, 527, 486, 469, 502, 514, 528, 517, 495, 526, 507, 474, 537, 504, 481, 513, 482, 522, 521, 510, 498, 537, 514, 569, 539, 496, 558, 552, 528, 504, 501, 507, 495, 509, 527, 508, 515, 477, 502, 517, 509, 521, 490, 532, 507, 513, 511, 498, 518, 522, 512, 517, 503, 543, 500, 515, 465, 504, 529, 506, 527, 527, 518, 535, 531, 494, 489, 519, 523, 512, 482, 507, 498, 473, 505, 522, 499, 497, 494, 509, 526, 500, 499, 518, 527, 510, 511, 519, 502, 480, 485, 483, 505, 543, 494, 523, 497, 523, 497, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[524, 520, 485, 515, 566, 473, 496, 512, 494, 517, 536, 517, 541, 516, 520, 534, 519, 515, 534, 551, 564, 514, 529, 521, 540, 506, 490, 512, 527, 497, 502, 522, 523, 550, 510, 480, 529, 518, 501, 524, 495, 489, 486, 475, 528, 485, 486, 527, 520, 497, 504, 538, 516, 497, 496, 513, 485, 505, 494, 503, 507, 496, 500, 523, 474, 521, 490, 519, 502, 534, 524, 538, 499, 481, 517, 507, 476, 515, 521, 536, 493, 463, 518, 505, 517, 524, 553, 504, 512, 498, 510, 533, 527, 485, 511, 487, 492, 515, 500, 507, 495, 517, 508, 503, 504, 512, 506, 473, 571, 537, 556, 526, 492, 505, 523, 544, 501, 545, 525, 539, 501, 489, 543, 492, 501, 493, 508, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[532, 507, 525, 533, 531, 528, 525, 536, 495, 525, 491, 465, 516, 494, 533, 519, 532, 506, 520, 512, 477, 522, 504, 547, 469, 502, 507, 486, 496, 511, 511, 520, 531, 504, 475, 504, 526, 491, 498, 533, 510, 505, 516, 542, 535, 510, 520, 483, 510, 488, 506, 484, 533, 511, 551, 520, 500, 496, 523, 495, 492, 489, 508, 528, 500, 508, 542, 547, 484, 457, 500, 545, 526, 524, 501, 547, 523, 508, 507, 502, 508, 538, 496, 499, 495, 525, 559, 475, 503, 530, 552, 493, 531, 494, 506, 510, 515, 526, 510, 504, 518, 537, 535, 512, 502, 539, 507, 484, 556, 514, 516, 519, 526, 484, 506, 525, 506, 513, 513, 486, 483, 485, 504, 504, 544, 525, 500, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[535, 518, 500, 516, 496, 565, 501, 466, 489, 532, 517, 535, 528, 490, 495, 533, 511, 501, 580, 506, 554, 498, 489, 552, 509, 509, 528, 475, 568, 556, 528, 497, 509, 536, 529, 512, 516, 507, 548, 520, 524, 527, 532, 517, 471, 538, 475, 484, 498, 556, 504, 522, 537, 494, 518, 515, 515, 498, 481, 494, 499, 547, 501, 490, 509, 499, 523, 522, 493, 519, 519, 521, 467, 520, 483, 515, 511, 537, 507, 510, 516, 504, 526, 503, 530, 461, 525, 519, 520, 555, 478, 525, 484, 521, 492, 503, 527, 516, 527, 539, 494, 507, 506, 517, 479, 516, 494, 485, 514, 499, 541, 521, 495, 521, 501, 485, 476, 472, 557, 522, 505, 532, 524, 510, 479, 496, 491, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[509, 480, 512, 530, 549, 522, 526, 542, 527, 527, 460, 498, 477, 509, 498, 507, 498, 536, 477, 538, 528, 469, 538, 557, 501, 471, 507, 512, 519, 495, 510, 533, 520, 538, 484, 522, 485, 478, 524, 489, 490, 541, 488, 535, 524, 507, 503, 528, 502, 484, 553, 518, 508, 511, 517, 549, 484, 499, 503, 502, 521, 530, 519, 476, 514, 500, 491, 503, 525, 512, 535, 505, 526, 523, 470, 537, 533, 529, 525, 541, 504, 523, 521, 502, 498, 552, 524, 503, 506, 508, 508, 500, 523, 514, 532, 511, 474, 507, 539, 489, 487, 528, 563, 501, 483, 492, 509, 516, 520, 496, 526, 491, 515, 490, 506, 545, 488, 500, 479, 539, 505, 558, 510, 520, 549, 487, 527, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[491, 512, 526, 493, 493, 505, 501, 526, 463, 519, 498, 479, 494, 497, 516, 472, 519, 528, 529, 538, 532, 527, 555, 476, 505, 551, 494, 510, 469, 528, 494, 506, 501, 506, 491, 516, 524, 535, 516, 522, 523, 491, 535, 515, 515, 513, 513, 519, 539, 518, 535, 529, 543, 482, 528, 511, 498, 501, 504, 501, 543, 504, 520, 534, 480, 525, 493, 509, 495, 510, 495, 508, 515, 526, 528, 504, 530, 532, 487, 557, 494, 491, 505, 505, 526, 502, 498, 562, 541, 490, 543, 551, 517, 552, 520, 494, 515, 501, 486, 518, 549, 523, 486, 479, 515, 513, 518, 524, 529, 520, 490, 525, 500, 536, 494, 511, 459, 520, 488, 492, 521, 489, 537, 520, 477, 552, 478, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[483, 535, 486, 512, 494, 528, 516, 535, 508, 531, 558, 478, 494, 464, 509, 521, 489, 537, 509, 516, 478, 504, 475, 539, 534, 494, 508, 526, 524, 510, 489, 526, 531, 535, 491, 537, 487, 483, 487, 543, 502, 528, 493, 474, 516, 495, 490, 521, 490, 517, 493, 517, 512, 517, 547, 527, 464, 500, 513, 530, 546, 529, 498, 472, 510, 472, 520, 516, 514, 504, 539, 507, 563, 529, 528, 484, 512, 511, 485, 523, 486, 533, 513, 512, 518, 488, 516, 489, 515, 526, 524, 492, 534, 480, 537, 491, 503, 525, 536, 497, 483, 498, 561, 536, 492, 516, 544, 518, 528, 524, 479, 505, 495, 515, 529, 541, 538, 522, 497, 512, 540, 501, 560, 519, 492, 515, 567, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[515, 541, 535, 501, 521, 578, 501, 503, 518, 505, 489, 513, 557, 506, 521, 504, 499, 481, 500, 519, 510, 555, 512, 504, 543, 517, 568, 508, 495, 535, 530, 497, 504, 502, 542, 533, 508, 521, 516, 571, 518, 514, 506, 524, 523, 492, 510, 495, 498, 500, 525, 502, 544, 499, 483, 513, 510, 506, 490, 486, 518, 515, 503, 494, 541, 491, 477, 526, 489, 517, 522, 472, 559, 497, 472, 543, 488, 509, 507, 529, 532, 515, 568, 496, 502, 469, 483, 499, 506, 483, 500, 500, 536, 505, 505, 473, 479, 487, 513, 537, 488, 510, 483, 530, 531, 520, 470, 536, 491, 512, 555, 534, 494, 501, 521, 524, 506, 536, 509, 517, 518, 498, 511, 505, 477, 519, 539, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[535, 518, 500, 516, 496, 565, 501, 466, 489, 520, 517, 535, 528, 490, 495, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 501, 580, 506, 554, 498, 489, 504, 509, 509, 528, 475, 568, 556, 528, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 536, 529, 512, 516, 507, 548, 520, 524, 527, 532, 517, 471, 538, 475, 484, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 556, 504, 522, 537, 494, 518, 515, 515, 498, 481, 494, 499, 547, 501, 490, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 499, 523, 522, 493, 519, 519, 521, 467, 520, 483, 515, 511, 537, 507, 510, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 504, 526, 503, 530, 461, 525, 519, 520, 555, 478, 525, 484, 521, 492, 503, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 516, 527, 539, 494, 507, 506, 517, 479, 516, 494, 485, 514, 499, 541, 521, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 521, 501, 485, 476, 472, 557, 522, 505, 532, 524, 510, 479, 496, 491, 480, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8313, 8256, 8178, 8158, 8170, 8187, 8070] avg=8192.0000 max=8313 max_rank=1 min=8070 min_rank=7 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[532, 507, 525, 533, 531, 528, 525, 536, 495, 513, 491, 465, 516, 494, 533, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 506, 520, 512, 477, 522, 504, 500, 469, 502, 507, 486, 496, 511, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 475, 504, 526, 491, 498, 533, 510, 505, 516, 542, 535, 510, 520, 483, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 506, 484, 533, 511, 551, 520, 500, 496, 523, 495, 492, 489, 508, 528, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 542, 547, 484, 457, 500, 545, 526, 524, 501, 547, 523, 508, 507, 502, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 538, 496, 499, 495, 525, 559, 475, 503, 530, 552, 493, 531, 494, 506, 510, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 526, 510, 504, 518, 537, 535, 512, 502, 539, 507, 484, 556, 514, 516, 519, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 484, 506, 525, 506, 513, 513, 486, 483, 485, 504, 504, 544, 525, 500, 509, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8075, 8194, 8143, 8224, 8222, 8299, 8136] avg=8192.0000 max=8299 max_rank=6 min=8075 min_rank=1 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[546, 474, 536, 525, 505, 503, 505, 537, 524, 549, 534, 500, 481, 503, 536, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 535, 502, 530, 528, 499, 480, 503, 500, 523, 514, 539, 527, 486, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 528, 517, 495, 526, 507, 474, 537, 504, 481, 513, 482, 522, 521, 510, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 537, 514, 569, 539, 496, 558, 552, 528, 504, 501, 507, 495, 509, 527, 508, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 477, 502, 517, 509, 521, 490, 532, 507, 513, 511, 498, 518, 522, 512, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 543, 500, 515, 465, 504, 529, 506, 527, 527, 518, 535, 531, 494, 489, 519, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 512, 482, 507, 498, 473, 505, 522, 499, 497, 494, 509, 526, 500, 499, 518, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 510, 511, 519, 502, 480, 485, 483, 505, 543, 494, 523, 497, 523, 497, 542, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8160, 8143, 8351, 8164, 8213, 8069, 8163] avg=8192.0000 max=8351 max_rank=3 min=8069 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 520, 485, 515, 566, 473, 496, 512, 494, 505, 536, 517, 541, 516, 520, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 515, 534, 551, 564, 514, 529, 477, 540, 506, 490, 512, 527, 497, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 550, 510, 480, 529, 518, 501, 524, 495, 489, 486, 475, 528, 485, 486, 527, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 497, 504, 538, 516, 497, 496, 513, 485, 505, 494, 503, 507, 496, 500, 523, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 521, 490, 519, 502, 534, 524, 538, 499, 481, 517, 507, 476, 515, 521, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 463, 518, 505, 517, 524, 553, 504, 512, 498, 510, 533, 527, 485, 511, 487, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 500, 507, 495, 517, 508, 503, 504, 512, 506, 473, 571, 537, 556, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 505, 523, 544, 501, 545, 525, 539, 501, 489, 543, 492, 501, 493, 508, 510, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8299, 8116, 8103, 8157, 8147, 8227, 8233] avg=8192.0000 max=8299 max_rank=1 min=8103 min_rank=3 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[491, 512, 526, 493, 493, 505, 501, 526, 463, 507, 498, 479, 494, 497, 516, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 528, 529, 538, 532, 527, 555, 436, 505, 551, 494, 510, 469, 528, 494, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 491, 516, 524, 535, 516, 522, 523, 491, 535, 515, 515, 513, 513, 519, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 518, 535, 529, 543, 482, 528, 511, 498, 501, 504, 501, 543, 504, 520, 534, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 525, 493, 509, 495, 510, 495, 508, 515, 526, 528, 504, 530, 532, 487, 557, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 491, 505, 505, 526, 502, 498, 562, 541, 490, 543, 551, 517, 552, 520, 494, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 486, 518, 549, 523, 486, 479, 515, 513, 518, 524, 529, 520, 490, 525, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 536, 494, 511, 459, 520, 488, 492, 521, 489, 537, 520, 477, 552, 478, 515, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7973, 8221, 8244, 8298, 8197, 8298, 8196, 8109] avg=8192.0000 max=8298 max_rank=3 min=7973 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 480, 512, 530, 549, 522, 526, 542, 527, 515, 460, 498, 477, 509, 498, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 536, 477, 538, 528, 469, 538, 509, 501, 471, 507, 512, 519, 495, 510, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 538, 484, 522, 485, 478, 524, 489, 490, 541, 488, 535, 524, 507, 503, 528, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 484, 553, 518, 508, 511, 517, 549, 484, 499, 503, 502, 521, 530, 519, 476, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 491, 503, 525, 512, 535, 505, 526, 523, 470, 537, 533, 529, 525, 541, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 523, 521, 502, 498, 552, 524, 503, 506, 508, 508, 500, 523, 514, 532, 511, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 507, 539, 489, 487, 528, 563, 501, 483, 492, 509, 516, 520, 496, 526, 491, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 490, 506, 545, 488, 500, 479, 539, 505, 558, 510, 520, 549, 487, 527, 505, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8141, 8167, 8185, 8272, 8237, 8126, 8247] avg=8192.0000 max=8272 max_rank=4 min=8126 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[483, 535, 486, 512, 494, 528, 516, 535, 508, 510, 558, 478, 494, 464, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 537, 509, 516, 478, 504, 475, 478, 534, 494, 508, 526, 524, 510, 489, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 535, 491, 537, 487, 483, 487, 543, 502, 528, 493, 474, 516, 495, 490, 521, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 517, 493, 517, 512, 517, 547, 527, 464, 500, 513, 530, 546, 529, 498, 472, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 472, 520, 516, 514, 504, 539, 507, 563, 529, 528, 484, 512, 511, 485, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 533, 513, 512, 518, 488, 516, 489, 515, 526, 524, 492, 534, 480, 537, 491, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 525, 536, 497, 483, 498, 561, 536, 492, 516, 544, 518, 528, 524, 479, 505, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 529, 541, 538, 522, 497, 512, 540, 501, 560, 519, 492, 515, 567, 482, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8097, 8127, 8184, 8222, 8170, 8252, 8353] avg=8192.0000 max=8353 max_rank=7 min=8097 min_rank=1 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32997 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 541, 535, 501, 521, 578, 501, 503, 518, 494, 489, 513, 557, 506, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 481, 500, 519, 510, 555, 512, 461, 543, 517, 568, 508, 495, 535, 530, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 542, 533, 508, 521, 516, 571, 518, 514, 506, 524, 523, 492, 510, 495, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 500, 525, 502, 544, 499, 483, 513, 510, 506, 490, 486, 518, 515, 503, 494, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 491, 477, 526, 489, 517, 522, 472, 559, 497, 472, 543, 488, 509, 507, 529, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 515, 568, 496, 502, 469, 483, 499, 506, 483, 500, 500, 536, 505, 505, 473, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 487, 513, 537, 488, 510, 483, 530, 531, 520, 470, 536, 491, 512, 555, 534, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 501, 521, 524, 506, 536, 509, 517, 518, 498, 511, 505, 477, 519, 539, 528, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8230, 8289, 8094, 8142, 8079, 8180, 8225] avg=8192.0000 max=8297 max_rank=0 min=8079 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4352, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4352, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[525, 511, 538, 497, 537, 485, 501, 524, 498, 503, 524, 507, 526, 499, 531, 495, 566, 540, 505, 495, 527, 526, 503, 521, 506, 516, 532, 497, 497, 522, 538, 477, 526, 556, 521, 529, 479, 515, 509, 511, 495, 514, 541, 495, 504, 506, 548, 516, 518, 511, 510, 495, 544, 506, 507, 521, 524, 506, 525, 482, 488, 502, 539, 490, 536, 526, 503, 530, 510, 507, 494, 545, 503, 506, 540, 522, 468, 501, 539, 528, 499, 485, 496, 496, 496, 503, 560, 458, 530, 529, 534, 510, 526, 527, 488, 552, 497, 501, 490, 534, 514, 482, 483, 532, 529, 513, 494, 496, 491, 508, 517, 475, 556, 494, 474, 515, 513, 513, 486, 500, 538, 501, 501, 494, 528, 487, 532, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[488, 516, 522, 516, 528, 476, 500, 525, 501, 489, 505, 531, 522, 490, 500, 498, 550, 520, 532, 520, 513, 484, 510, 544, 516, 499, 514, 501, 477, 512, 520, 534, 486, 501, 544, 496, 533, 516, 514, 512, 481, 516, 507, 502, 513, 518, 527, 492, 513, 484, 507, 478, 513, 560, 514, 536, 516, 523, 534, 509, 536, 535, 531, 507, 496, 472, 520, 519, 455, 530, 526, 512, 521, 541, 474, 509, 522, 483, 537, 519, 499, 500, 498, 567, 560, 513, 531, 526, 510, 524, 519, 499, 491, 529, 535, 513, 522, 518, 536, 510, 481, 544, 545, 492, 497, 514, 545, 474, 500, 478, 519, 505, 557, 497, 488, 517, 538, 492, 477, 508, 502, 507, 526, 487, 500, 501, 514, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[539, 527, 502, 537, 514, 483, 497, 547, 547, 510, 539, 509, 546, 522, 519, 532, 506, 553, 484, 507, 548, 495, 468, 514, 527, 509, 510, 497, 491, 509, 515, 467, 483, 499, 483, 483, 510, 495, 508, 472, 489, 523, 506, 524, 545, 534, 549, 537, 528, 471, 494, 496, 509, 544, 545, 521, 519, 508, 500, 492, 525, 526, 523, 492, 518, 515, 547, 480, 557, 490, 502, 517, 532, 535, 513, 515, 558, 476, 517, 505, 512, 508, 520, 525, 500, 496, 520, 518, 525, 497, 505, 531, 486, 502, 529, 474, 479, 510, 488, 490, 522, 510, 521, 540, 469, 502, 546, 513, 515, 495, 502, 534, 484, 519, 542, 495, 483, 539, 528, 475, 508, 507, 520, 504, 542, 529, 503, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[484, 517, 564, 505, 511, 556, 486, 482, 476, 506, 500, 545, 465, 515, 539, 533, 482, 506, 499, 493, 505, 478, 538, 483, 505, 541, 496, 494, 504, 505, 494, 538, 519, 492, 504, 506, 507, 519, 484, 536, 507, 535, 451, 516, 527, 511, 488, 517, 501, 501, 510, 538, 484, 496, 513, 548, 510, 531, 524, 524, 532, 503, 515, 539, 520, 491, 502, 553, 490, 479, 551, 533, 533, 512, 510, 502, 520, 508, 518, 461, 528, 536, 546, 547, 486, 521, 513, 510, 512, 543, 522, 528, 523, 504, 503, 530, 521, 517, 536, 514, 493, 528, 540, 495, 515, 505, 512, 535, 533, 500, 442, 500, 502, 507, 484, 491, 523, 472, 523, 537, 511, 509, 523, 516, 540, 497, 525, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[517, 495, 485, 494, 520, 502, 484, 474, 523, 542, 532, 505, 527, 525, 490, 499, 495, 538, 548, 511, 499, 473, 512, 522, 539, 468, 533, 531, 513, 516, 517, 500, 504, 513, 510, 514, 576, 496, 526, 525, 521, 492, 531, 549, 514, 493, 489, 513, 532, 499, 490, 484, 478, 497, 543, 554, 537, 561, 535, 503, 502, 467, 523, 502, 519, 510, 500, 512, 499, 491, 516, 520, 521, 472, 527, 501, 531, 493, 500, 512, 538, 520, 465, 526, 526, 523, 491, 514, 501, 496, 485, 478, 547, 519, 475, 515, 581, 506, 491, 491, 476, 537, 520, 481, 506, 513, 509, 511, 504, 545, 505, 515, 501, 532, 543, 549, 507, 518, 484, 511, 486, 544, 512, 546, 509, 514, 517, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[511, 534, 516, 529, 579, 495, 510, 528, 523, 488, 483, 511, 529, 488, 495, 524, 499, 510, 461, 509, 491, 512, 487, 500, 544, 568, 506, 523, 518, 536, 476, 547, 486, 515, 491, 515, 533, 514, 530, 548, 536, 536, 533, 564, 528, 523, 541, 476, 541, 508, 530, 532, 480, 555, 481, 509, 528, 539, 501, 522, 519, 502, 533, 511, 481, 525, 480, 488, 518, 523, 480, 500, 496, 526, 536, 476, 515, 475, 512, 516, 516, 488, 514, 539, 555, 511, 498, 513, 523, 486, 505, 522, 511, 516, 548, 492, 490, 494, 501, 488, 521, 462, 480, 485, 455, 563, 494, 480, 509, 503, 502, 531, 503, 552, 534, 534, 521, 537, 501, 498, 500, 502, 524, 545, 529, 463, 474, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[480, 497, 518, 482, 522, 496, 526, 515, 485, 542, 531, 498, 534, 519, 532, 523, 524, 535, 518, 499, 513, 510, 530, 521, 483, 534, 519, 494, 480, 563, 497, 544, 499, 499, 530, 502, 531, 493, 532, 541, 562, 518, 522, 493, 512, 507, 524, 512, 537, 505, 522, 529, 526, 476, 520, 497, 541, 522, 504, 522, 487, 519, 493, 518, 486, 513, 514, 515, 495, 519, 512, 555, 481, 495, 509, 511, 512, 481, 525, 478, 549, 526, 497, 500, 489, 530, 526, 523, 492, 506, 494, 541, 508, 498, 532, 473, 520, 498, 531, 516, 492, 512, 487, 526, 550, 524, 483, 505, 512, 499, 504, 446, 474, 508, 562, 481, 511, 508, 503, 538, 532, 508, 516, 466, 535, 513, 513, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[516, 537, 492, 521, 528, 523, 513, 530, 521, 497, 555, 531, 504, 508, 481, 480, 510, 495, 571, 492, 494, 524, 497, 531, 484, 466, 492, 488, 524, 508, 511, 549, 544, 532, 536, 545, 540, 512, 463, 494, 549, 517, 528, 514, 541, 515, 485, 512, 501, 546, 539, 539, 517, 467, 490, 500, 498, 484, 541, 525, 527, 511, 551, 488, 513, 510, 532, 500, 502, 483, 489, 489, 505, 498, 507, 558, 521, 490, 518, 501, 536, 493, 506, 510, 507, 497, 492, 547, 525, 492, 516, 514, 548, 519, 501, 506, 505, 489, 504, 528, 511, 518, 535, 539, 519, 535, 514, 491, 489, 482, 487, 468, 517, 467, 498, 495, 506, 480, 521, 509, 559, 507, 477, 530, 553, 564, 482, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[488, 516, 522, 516, 514, 476, 500, 525, 501, 489, 505, 531, 522, 490, 500, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 520, 532, 520, 513, 484, 510, 544, 516, 499, 514, 501, 477, 512, 520, 534, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 501, 544, 496, 485, 516, 514, 512, 481, 516, 507, 502, 513, 518, 527, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 484, 507, 478, 513, 560, 514, 536, 516, 523, 534, 509, 536, 535, 490, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 472, 520, 519, 455, 530, 526, 512, 521, 541, 474, 509, 522, 483, 537, 519, 25, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 500, 498, 539, 560, 513, 531, 526, 510, 524, 519, 499, 491, 529, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 536, 510, 481, 544, 545, 492, 497, 514, 545, 474, 500, 478, 519, 505, 48, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 497, 488, 517, 538, 492, 477, 508, 502, 507, 526, 487, 500, 501, 514, 488, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8093, 8253, 8110, 8255, 8174, 8286, 8256, 8109] avg=8192.0000 max=8286 max_rank=5 min=8093 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[484, 517, 564, 505, 497, 556, 486, 482, 476, 506, 500, 545, 465, 515, 539, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 506, 499, 493, 505, 478, 538, 483, 505, 541, 496, 494, 504, 505, 494, 538, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 492, 504, 506, 461, 519, 484, 536, 507, 535, 451, 516, 527, 511, 488, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 510, 538, 484, 496, 513, 548, 510, 531, 524, 524, 532, 503, 476, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 491, 502, 553, 490, 479, 551, 533, 533, 512, 510, 502, 520, 508, 518, 461, 24, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 536, 546, 520, 486, 521, 513, 510, 512, 543, 522, 528, 523, 504, 503, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 536, 514, 493, 528, 540, 495, 515, 505, 512, 535, 533, 500, 442, 500, 46, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 484, 491, 523, 472, 523, 537, 511, 509, 523, 516, 540, 497, 525, 522, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8068, 8073, 8230, 8219, 8325, 8259, 8192] avg=8192.0000 max=8325 max_rank=5 min=8068 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32995 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[539, 527, 502, 537, 494, 483, 497, 547, 547, 510, 539, 509, 546, 522, 519, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 553, 484, 507, 548, 495, 468, 514, 527, 509, 510, 497, 491, 509, 515, 467, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 499, 483, 483, 459, 495, 508, 472, 489, 523, 506, 524, 545, 534, 549, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 471, 494, 496, 509, 544, 545, 521, 519, 508, 500, 492, 525, 526, 475, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 515, 547, 480, 557, 490, 502, 517, 532, 535, 513, 515, 558, 476, 517, 505, 27, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 520, 492, 500, 496, 520, 518, 525, 497, 505, 531, 486, 502, 529, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 510, 488, 490, 522, 510, 521, 540, 469, 502, 546, 513, 515, 495, 502, 534, 51, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 519, 542, 495, 483, 539, 528, 475, 508, 507, 520, 504, 542, 529, 503, 494, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8350, 8109, 8089, 8145, 8320, 8115, 8219, 8189] avg=8192.0000 max=8350 max_rank=0 min=8089 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[525, 511, 538, 497, 523, 485, 501, 524, 498, 503, 524, 507, 526, 499, 531, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 540, 505, 495, 527, 526, 503, 521, 506, 516, 532, 497, 497, 522, 538, 477, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 556, 521, 529, 436, 515, 509, 511, 495, 514, 541, 495, 504, 506, 548, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 510, 495, 544, 506, 507, 521, 524, 506, 525, 482, 488, 502, 498, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 526, 503, 530, 510, 507, 494, 545, 503, 506, 540, 522, 468, 501, 539, 528, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 485, 496, 471, 496, 503, 560, 458, 530, 529, 534, 510, 526, 527, 488, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 501, 490, 534, 514, 482, 483, 532, 529, 513, 494, 496, 491, 508, 517, 475, 43, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 494, 474, 515, 513, 513, 486, 500, 538, 501, 501, 494, 528, 487, 532, 499, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8275, 8222, 8127, 8293, 8164, 8127, 8141] avg=8192.0000 max=8293 max_rank=4 min=8127 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 534, 516, 529, 563, 495, 510, 528, 523, 488, 483, 511, 529, 488, 495, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 510, 461, 509, 491, 512, 487, 500, 544, 568, 506, 523, 518, 536, 476, 547, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 515, 491, 515, 485, 514, 530, 548, 536, 536, 533, 564, 528, 523, 541, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 508, 530, 532, 480, 555, 481, 509, 528, 539, 501, 522, 519, 502, 492, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 525, 480, 488, 518, 523, 480, 500, 496, 526, 536, 476, 515, 475, 512, 516, 24, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 488, 514, 512, 555, 511, 498, 513, 523, 486, 505, 522, 511, 516, 548, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 494, 501, 488, 521, 462, 480, 485, 455, 563, 494, 480, 509, 503, 502, 531, 48, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 552, 534, 534, 521, 537, 501, 498, 500, 502, 524, 545, 529, 463, 474, 487, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8195, 8321, 8250, 8084, 8210, 8034, 8215] avg=8192.0000 max=8321 max_rank=2 min=8034 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 495, 485, 494, 506, 502, 484, 474, 523, 542, 532, 505, 527, 525, 490, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 538, 548, 511, 499, 473, 512, 522, 539, 468, 533, 531, 513, 516, 517, 500, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 510, 514, 524, 496, 526, 525, 521, 492, 531, 549, 514, 493, 489, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 499, 490, 484, 478, 497, 543, 554, 537, 561, 535, 503, 502, 467, 483, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 500, 512, 499, 491, 516, 520, 521, 472, 527, 501, 531, 493, 500, 512, 23, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 520, 465, 500, 526, 523, 491, 514, 501, 496, 485, 478, 547, 519, 475, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 506, 491, 491, 476, 537, 520, 481, 506, 513, 509, 511, 504, 545, 505, 515, 52, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 543, 549, 507, 518, 484, 511, 486, 544, 512, 546, 509, 514, 517, 527, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8222, 8214, 8167, 8160, 8093, 8270, 8310] avg=8192.0000 max=8310 max_rank=7 min=8093 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[480, 497, 518, 482, 508, 496, 526, 515, 485, 542, 531, 498, 534, 519, 532, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 535, 518, 499, 513, 510, 530, 521, 483, 534, 519, 494, 480, 563, 497, 544, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 530, 502, 483, 493, 532, 541, 562, 518, 522, 493, 512, 507, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 505, 522, 529, 526, 476, 520, 497, 541, 522, 504, 522, 487, 519, 455, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 513, 514, 515, 495, 519, 512, 555, 481, 495, 509, 511, 512, 481, 525, 478, 22, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 526, 497, 475, 489, 530, 526, 523, 492, 506, 494, 541, 508, 498, 532, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 498, 531, 516, 492, 512, 487, 526, 550, 524, 483, 505, 512, 499, 504, 446, 48, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 508, 562, 481, 511, 508, 503, 538, 532, 508, 516, 466, 535, 513, 513, 519, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8271, 8229, 8180, 8135, 8159, 8179, 8197] avg=8192.0000 max=8271 max_rank=1 min=8135 min_rank=4 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 537, 492, 521, 514, 523, 513, 530, 521, 497, 555, 531, 504, 508, 481, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 495, 571, 492, 494, 524, 497, 531, 484, 466, 492, 488, 524, 508, 511, 549, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 532, 536, 545, 491, 512, 463, 494, 549, 517, 528, 514, 541, 515, 485, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 546, 539, 539, 517, 467, 490, 500, 498, 484, 541, 525, 527, 511, 509, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 532, 500, 502, 483, 489, 489, 505, 498, 507, 558, 521, 490, 518, 501, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 493, 506, 485, 507, 497, 492, 547, 525, 492, 516, 514, 548, 519, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 489, 504, 528, 511, 518, 535, 539, 519, 535, 514, 491, 489, 482, 487, 468, 49, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 467, 498, 495, 506, 480, 521, 509, 559, 507, 477, 530, 553, 564, 482, 508, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8143, 8278, 8182, 8151, 8184, 8192, 8183] avg=8192.0000 max=8278 max_rank=2 min=8143 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 3840, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4352, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[534, 526, 518, 525, 483, 543, 537, 527, 483, 487, 513, 543, 514, 506, 512, 530, 486, 549, 526, 504, 502, 523, 483, 541, 536, 546, 515, 528, 520, 487, 475, 546, 503, 545, 517, 479, 526, 513, 499, 458, 482, 570, 514, 478, 490, 546, 466, 506, 528, 506, 552, 493, 545, 471, 489, 526, 548, 500, 546, 496, 540, 491, 546, 512, 489, 525, 519, 543, 499, 513, 524, 499, 514, 521, 486, 502, 506, 511, 557, 541, 503, 520, 535, 509, 471, 464, 498, 553, 516, 476, 490, 473, 506, 481, 559, 492, 490, 507, 515, 519, 523, 517, 505, 498, 538, 519, 482, 478, 481, 487, 543, 520, 519, 503, 495, 509, 564, 535, 501, 541, 491, 502, 537, 477, 518, 509, 462, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[498, 540, 527, 525, 514, 553, 518, 489, 501, 509, 496, 544, 463, 511, 538, 535, 522, 528, 440, 510, 549, 481, 488, 494, 489, 531, 502, 514, 485, 529, 477, 547, 500, 507, 516, 505, 501, 510, 554, 494, 560, 527, 531, 457, 496, 452, 483, 522, 528, 474, 513, 523, 482, 548, 508, 507, 532, 548, 528, 511, 523, 516, 528, 479, 513, 510, 535, 494, 504, 556, 528, 475, 525, 551, 547, 522, 551, 512, 510, 486, 529, 512, 515, 551, 512, 520, 513, 482, 493, 456, 522, 505, 533, 487, 502, 483, 510, 527, 541, 509, 552, 495, 529, 533, 480, 487, 537, 497, 524, 525, 485, 491, 504, 462, 520, 521, 487, 510, 542, 504, 490, 508, 536, 535, 488, 517, 546, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[523, 498, 491, 517, 488, 495, 497, 533, 500, 531, 501, 525, 504, 509, 515, 507, 490, 513, 467, 489, 537, 476, 534, 516, 489, 511, 538, 515, 490, 506, 497, 538, 513, 536, 486, 540, 508, 540, 487, 539, 479, 481, 527, 497, 507, 527, 508, 519, 515, 568, 544, 493, 555, 523, 491, 540, 467, 492, 507, 515, 479, 501, 504, 502, 546, 470, 521, 525, 487, 505, 470, 521, 507, 504, 528, 535, 517, 534, 483, 529, 577, 529, 456, 490, 500, 501, 510, 520, 488, 501, 515, 532, 508, 515, 530, 502, 538, 528, 542, 500, 527, 470, 514, 521, 481, 531, 566, 493, 502, 516, 554, 521, 509, 533, 531, 520, 524, 547, 528, 493, 518, 542, 467, 487, 523, 522, 484, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[467, 497, 494, 496, 506, 532, 553, 513, 537, 507, 515, 520, 480, 515, 472, 528, 483, 526, 500, 530, 516, 527, 508, 506, 489, 481, 506, 481, 491, 522, 480, 490, 508, 540, 533, 497, 515, 507, 499, 543, 505, 483, 533, 491, 533, 507, 547, 522, 489, 485, 511, 526, 521, 528, 470, 529, 524, 505, 540, 486, 497, 527, 479, 509, 514, 506, 509, 487, 521, 507, 512, 520, 488, 534, 470, 499, 514, 502, 491, 495, 515, 527, 556, 547, 542, 526, 541, 519, 487, 488, 516, 501, 474, 526, 519, 530, 513, 520, 541, 511, 498, 496, 530, 531, 506, 511, 493, 545, 459, 532, 492, 552, 502, 500, 542, 482, 515, 518, 510, 538, 514, 545, 502, 534, 503, 574, 530, 557] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[526, 524, 566, 532, 522, 506, 503, 535, 511, 478, 505, 475, 511, 505, 526, 542, 507, 558, 490, 541, 509, 514, 496, 507, 496, 550, 489, 558, 508, 495, 529, 516, 493, 492, 507, 515, 466, 513, 531, 552, 517, 498, 515, 510, 534, 569, 511, 506, 555, 526, 554, 492, 538, 519, 517, 493, 560, 488, 512, 543, 534, 550, 474, 507, 495, 522, 515, 450, 557, 528, 514, 476, 489, 512, 484, 515, 509, 519, 507, 489, 504, 503, 489, 513, 506, 508, 513, 475, 503, 462, 525, 537, 488, 509, 510, 493, 519, 533, 518, 491, 505, 533, 473, 531, 500, 528, 522, 487, 494, 484, 497, 510, 545, 475, 510, 503, 492, 451, 568, 490, 564, 520, 533, 525, 482, 493, 537, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[538, 556, 529, 455, 533, 543, 519, 487, 479, 578, 503, 523, 484, 536, 533, 511, 491, 509, 512, 529, 528, 524, 516, 514, 491, 488, 485, 525, 519, 502, 558, 528, 508, 513, 505, 536, 490, 507, 514, 472, 493, 492, 549, 544, 518, 535, 486, 520, 516, 523, 487, 501, 516, 513, 490, 545, 522, 537, 507, 492, 568, 497, 503, 513, 497, 498, 536, 510, 506, 452, 520, 551, 540, 471, 472, 498, 556, 492, 505, 503, 545, 509, 509, 508, 493, 504, 528, 501, 464, 520, 513, 519, 480, 508, 548, 526, 526, 477, 513, 558, 519, 501, 502, 525, 502, 483, 484, 484, 511, 529, 524, 525, 518, 488, 545, 507, 470, 519, 543, 485, 518, 509, 510, 506, 496, 504, 490, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[483, 495, 541, 560, 482, 524, 508, 522, 512, 480, 515, 513, 529, 516, 496, 549, 470, 493, 509, 533, 487, 501, 505, 507, 520, 493, 497, 534, 464, 481, 564, 491, 519, 520, 525, 498, 544, 501, 499, 513, 508, 538, 552, 521, 518, 510, 502, 489, 501, 498, 485, 547, 524, 525, 518, 499, 502, 493, 505, 508, 476, 532, 527, 516, 475, 487, 489, 482, 539, 520, 518, 513, 519, 493, 562, 540, 500, 526, 536, 546, 511, 552, 500, 519, 537, 507, 522, 552, 517, 538, 571, 521, 518, 492, 524, 510, 496, 518, 499, 524, 528, 495, 491, 483, 508, 487, 494, 507, 484, 491, 530, 493, 494, 515, 508, 482, 495, 507, 537, 516, 506, 496, 529, 500, 513, 542, 523, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[521, 493, 509, 496, 496, 469, 542, 505, 537, 544, 508, 502, 555, 522, 515, 514, 536, 536, 515, 500, 530, 508, 495, 495, 573, 546, 511, 483, 494, 532, 503, 486, 475, 457, 488, 504, 510, 507, 509, 519, 525, 495, 495, 517, 489, 521, 525, 477, 496, 494, 517, 558, 507, 499, 517, 526, 518, 503, 514, 519, 551, 503, 515, 519, 484, 494, 586, 515, 522, 507, 483, 525, 521, 533, 488, 547, 477, 518, 514, 494, 552, 485, 554, 500, 516, 511, 517, 527, 465, 505, 510, 573, 507, 523, 506, 532, 530, 475, 531, 499, 525, 532, 501, 499, 495, 513, 496, 550, 537, 504, 517, 537, 532, 501, 514, 510, 465, 528, 526, 486, 496, 527, 486, 468, 536, 481, 497, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[534, 526, 518, 525, 483, 543, 537, 527, 483, 487, 513, 543, 514, 506, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 549, 526, 504, 502, 523, 483, 541, 536, 546, 515, 528, 520, 487, 475, 546, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 545, 517, 479, 526, 513, 499, 458, 482, 570, 514, 478, 490, 546, 466, 506, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 552, 493, 504, 471, 489, 526, 548, 500, 546, 496, 540, 491, 546, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 525, 519, 543, 499, 513, 524, 499, 514, 521, 486, 502, 506, 511, 557, 541, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 520, 535, 509, 471, 464, 498, 553, 516, 476, 490, 473, 506, 481, 559, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 507, 515, 519, 523, 517, 505, 498, 538, 519, 482, 478, 481, 487, 543, 520, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 503, 495, 509, 564, 535, 500, 541, 491, 502, 537, 477, 518, 509, 462, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8301, 8115, 8248, 8258, 8046, 8134, 8189] avg=8192.0000 max=8301 max_rank=1 min=8046 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32627 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 498, 491, 517, 488, 495, 497, 533, 500, 531, 501, 525, 504, 509, 515, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 513, 467, 489, 537, 476, 534, 516, 489, 511, 538, 515, 490, 506, 497, 538, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 536, 486, 540, 508, 540, 487, 539, 479, 481, 527, 497, 507, 527, 508, 519, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 568, 544, 493, 513, 523, 491, 540, 467, 492, 507, 515, 479, 501, 504, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 470, 521, 525, 487, 505, 470, 521, 507, 504, 528, 535, 517, 534, 483, 529, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 529, 456, 490, 500, 501, 510, 520, 488, 501, 515, 532, 508, 515, 530, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 528, 542, 500, 527, 470, 514, 521, 481, 531, 566, 493, 502, 516, 554, 521, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 531, 520, 524, 547, 527, 493, 518, 542, 467, 487, 523, 522, 484, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8141, 8216, 8154, 8191, 8174, 8316, 8245] avg=8192.0000 max=8316 max_rank=6 min=8099 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32926 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[498, 540, 527, 525, 514, 553, 518, 489, 501, 509, 496, 544, 463, 511, 538, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 440, 510, 549, 481, 488, 494, 489, 531, 502, 514, 485, 529, 477, 547, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 507, 516, 505, 501, 510, 554, 494, 560, 527, 531, 457, 496, 452, 483, 522, 21, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 474, 513, 523, 437, 548, 508, 507, 532, 548, 528, 511, 523, 516, 528, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 535, 494, 504, 556, 528, 475, 525, 551, 547, 522, 551, 512, 510, 486, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 515, 551, 512, 520, 513, 482, 493, 456, 522, 505, 533, 487, 502, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 527, 541, 509, 552, 495, 529, 533, 480, 487, 537, 497, 524, 525, 485, 491, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 462, 520, 521, 487, 510, 540, 504, 490, 508, 536, 535, 488, 517, 546, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8120, 8147, 8203, 8332, 8115, 8238, 8168] avg=8192.0000 max=8332 max_rank=4 min=8115 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32853 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[538, 556, 529, 455, 533, 543, 519, 487, 479, 578, 503, 523, 484, 536, 533, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 509, 512, 529, 528, 524, 516, 514, 491, 488, 485, 525, 519, 502, 558, 528, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 513, 505, 536, 490, 507, 514, 472, 493, 492, 549, 544, 518, 535, 486, 520, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 523, 487, 501, 477, 513, 490, 545, 522, 537, 507, 492, 568, 497, 503, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 536, 510, 506, 452, 520, 551, 540, 471, 472, 498, 556, 492, 505, 503, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 509, 508, 493, 504, 528, 501, 464, 520, 513, 519, 480, 508, 548, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 477, 513, 558, 519, 501, 502, 525, 502, 483, 484, 484, 511, 529, 524, 525, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 488, 545, 507, 470, 519, 542, 485, 518, 509, 510, 506, 496, 504, 490, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8251, 8204, 8191, 8116, 8175, 8175, 8152] avg=8192.0000 max=8272 max_rank=0 min=8116 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32618 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[526, 524, 566, 532, 522, 506, 503, 535, 511, 478, 505, 475, 511, 505, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 558, 490, 541, 509, 514, 496, 507, 496, 550, 489, 558, 508, 495, 529, 516, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 507, 515, 466, 513, 531, 552, 517, 498, 515, 510, 534, 569, 511, 506, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 526, 554, 492, 497, 519, 517, 493, 560, 488, 512, 543, 534, 550, 474, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 515, 450, 557, 528, 514, 476, 489, 512, 484, 515, 509, 519, 507, 489, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 489, 513, 506, 508, 513, 475, 503, 462, 525, 537, 488, 509, 510, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 533, 518, 491, 505, 533, 473, 531, 500, 528, 522, 487, 494, 484, 497, 510, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 475, 510, 503, 492, 451, 567, 490, 564, 520, 533, 525, 482, 493, 537, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8297, 8252, 8321, 8090, 8038, 8137, 8170] avg=8192.0000 max=8321 max_rank=3 min=8038 min_rank=5 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32435 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[467, 497, 494, 496, 506, 532, 553, 513, 537, 507, 515, 520, 480, 515, 472, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 526, 500, 530, 516, 527, 508, 506, 489, 481, 506, 481, 491, 522, 480, 490, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 540, 533, 497, 515, 507, 499, 543, 505, 483, 533, 491, 533, 507, 547, 522, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 485, 511, 526, 481, 528, 470, 529, 524, 505, 540, 486, 497, 527, 479, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 509, 487, 521, 507, 512, 520, 488, 534, 470, 499, 514, 502, 491, 495, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 527, 556, 547, 542, 526, 541, 519, 487, 488, 516, 501, 474, 526, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 520, 541, 511, 498, 496, 530, 531, 506, 511, 493, 545, 459, 532, 492, 552, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 542, 482, 515, 518, 509, 538, 514, 545, 502, 534, 503, 574, 530, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8069, 8286, 8086, 8078, 8314, 8242, 8365] avg=8192.0000 max=8365 max_rank=7 min=8069 min_rank=1 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32999 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[483, 495, 541, 560, 482, 524, 508, 522, 512, 480, 515, 513, 529, 516, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 493, 509, 533, 487, 501, 505, 507, 520, 493, 497, 534, 464, 481, 564, 491, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 525, 498, 544, 501, 499, 513, 508, 538, 552, 521, 518, 510, 502, 489, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 498, 485, 547, 484, 525, 518, 499, 502, 493, 505, 508, 476, 532, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 487, 489, 482, 539, 520, 518, 513, 519, 493, 562, 540, 500, 526, 536, 546, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 552, 500, 519, 537, 507, 522, 552, 517, 538, 571, 521, 518, 492, 524, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 518, 499, 524, 528, 495, 491, 483, 508, 487, 494, 507, 484, 491, 530, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 508, 482, 495, 507, 536, 516, 506, 496, 529, 500, 513, 542, 523, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8082, 8281, 8116, 8254, 8391, 8041, 8184] avg=8192.0000 max=8391 max_rank=5 min=8041 min_rank=6 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=32870 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 493, 509, 496, 496, 469, 542, 505, 537, 544, 508, 502, 555, 522, 515, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 536, 515, 500, 530, 508, 495, 495, 573, 546, 511, 483, 494, 532, 503, 486, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 457, 488, 504, 510, 507, 509, 519, 525, 495, 495, 517, 489, 521, 525, 477, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 517, 558, 468, 499, 517, 526, 518, 503, 514, 519, 551, 503, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 494, 586, 515, 522, 507, 483, 525, 521, 533, 488, 547, 477, 518, 514, 494, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 485, 554, 500, 516, 511, 517, 527, 465, 505, 510, 573, 507, 523, 506, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 475, 531, 499, 525, 532, 501, 499, 495, 513, 496, 550, 537, 504, 517, 537, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 501, 514, 510, 465, 528, 525, 486, 496, 527, 486, 468, 536, 481, 497, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8275, 8035, 8217, 8217, 8283, 8253, 8063] avg=8192.0000 max=8283 max_rank=5 min=8035 min_rank=2 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32816 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4352, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 3968, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4352, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 3968, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[481, 519, 511, 533, 524, 476, 506, 502, 508, 489, 499, 518, 501, 493, 495, 545, 504, 480, 499, 546, 550, 503, 524, 487, 508, 514, 479, 529, 543, 520, 528, 493, 540, 523, 509, 481, 536, 512, 506, 509, 468, 522, 517, 517, 498, 504, 551, 474, 495, 488, 523, 496, 511, 482, 563, 519, 499, 479, 530, 482, 521, 538, 552, 507, 547, 533, 526, 515, 464, 500, 527, 501, 535, 473, 555, 479, 537, 520, 517, 523, 518, 471, 510, 532, 499, 539, 522, 495, 517, 510, 494, 502, 513, 543, 504, 522, 519, 482, 509, 495, 550, 536, 547, 542, 542, 476, 545, 494, 521, 525, 510, 511, 530, 501, 517, 524, 508, 458, 513, 486, 503, 498, 510, 551, 513, 509, 537, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[505, 501, 479, 508, 505, 510, 535, 553, 561, 518, 506, 502, 534, 527, 510, 508, 492, 530, 528, 497, 546, 511, 530, 523, 515, 498, 517, 492, 508, 482, 524, 515, 526, 521, 492, 534, 525, 565, 517, 484, 490, 494, 521, 514, 498, 510, 494, 489, 497, 517, 518, 526, 536, 526, 532, 494, 471, 511, 491, 494, 527, 538, 513, 544, 527, 513, 472, 468, 503, 488, 532, 525, 521, 517, 540, 515, 505, 516, 481, 523, 539, 509, 507, 518, 506, 500, 543, 551, 532, 472, 513, 550, 474, 526, 527, 543, 502, 527, 498, 545, 490, 503, 496, 494, 566, 508, 488, 507, 472, 525, 501, 517, 477, 509, 499, 524, 499, 546, 492, 503, 497, 511, 474, 509, 480, 475, 542, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[502, 519, 515, 539, 536, 496, 498, 516, 520, 498, 504, 506, 497, 507, 506, 525, 529, 496, 490, 505, 494, 547, 515, 510, 511, 484, 498, 469, 513, 515, 540, 529, 540, 505, 530, 528, 503, 506, 487, 479, 496, 507, 506, 481, 503, 537, 516, 554, 516, 541, 492, 487, 520, 497, 524, 500, 518, 505, 527, 504, 508, 522, 514, 481, 481, 519, 502, 531, 543, 543, 525, 513, 568, 508, 550, 517, 479, 517, 490, 529, 515, 501, 463, 513, 539, 515, 538, 504, 528, 525, 506, 528, 505, 503, 497, 530, 513, 523, 493, 500, 503, 512, 489, 545, 477, 520, 507, 495, 509, 492, 514, 454, 508, 486, 496, 538, 509, 543, 532, 524, 528, 514, 527, 513, 503, 529, 528, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[500, 517, 496, 512, 551, 506, 492, 509, 534, 533, 514, 499, 483, 528, 568, 543, 499, 508, 538, 539, 497, 522, 506, 549, 518, 474, 489, 515, 537, 449, 515, 510, 487, 487, 517, 542, 512, 513, 506, 501, 456, 510, 510, 494, 530, 539, 559, 485, 513, 518, 498, 525, 510, 510, 501, 501, 537, 518, 532, 511, 490, 530, 534, 554, 544, 520, 480, 513, 460, 477, 489, 486, 493, 557, 505, 524, 506, 473, 507, 522, 504, 534, 497, 501, 533, 492, 534, 473, 522, 494, 494, 531, 558, 503, 465, 523, 567, 494, 535, 523, 524, 474, 506, 493, 484, 505, 478, 501, 529, 519, 480, 558, 504, 536, 529, 507, 485, 495, 477, 529, 503, 517, 506, 533, 568, 506, 545, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[460, 540, 525, 500, 524, 517, 456, 508, 525, 461, 543, 522, 495, 539, 485, 503, 506, 492, 536, 491, 542, 505, 534, 517, 460, 556, 490, 527, 516, 514, 547, 513, 479, 495, 518, 474, 517, 506, 535, 506, 496, 521, 537, 524, 532, 504, 498, 512, 518, 529, 504, 528, 543, 526, 533, 481, 524, 494, 540, 516, 488, 512, 509, 538, 528, 528, 508, 515, 519, 531, 516, 509, 546, 555, 491, 524, 521, 527, 508, 454, 515, 528, 507, 533, 519, 514, 531, 495, 485, 537, 502, 436, 511, 503, 459, 519, 461, 491, 514, 480, 522, 517, 526, 502, 501, 492, 511, 538, 560, 516, 525, 540, 525, 534, 535, 504, 518, 502, 504, 524, 548, 496, 494, 518, 505, 505, 496, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[516, 522, 497, 538, 499, 495, 545, 500, 528, 519, 495, 486, 551, 506, 516, 493, 538, 474, 521, 481, 533, 502, 524, 517, 488, 521, 554, 531, 496, 576, 496, 494, 542, 485, 484, 534, 521, 499, 496, 492, 501, 550, 509, 509, 515, 507, 528, 521, 507, 507, 530, 487, 510, 475, 467, 490, 545, 517, 537, 490, 524, 474, 509, 508, 536, 491, 517, 495, 499, 475, 500, 519, 519, 531, 556, 537, 522, 493, 532, 505, 494, 500, 511, 517, 524, 519, 537, 528, 485, 512, 526, 506, 458, 568, 515, 501, 538, 517, 565, 493, 520, 529, 474, 500, 506, 463, 520, 542, 496, 498, 501, 522, 484, 509, 493, 513, 512, 529, 516, 507, 542, 509, 535, 524, 511, 487, 515, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[519, 504, 536, 522, 495, 518, 497, 485, 503, 515, 506, 484, 527, 526, 495, 523, 511, 541, 532, 491, 518, 528, 509, 515, 475, 508, 460, 551, 501, 515, 557, 514, 508, 495, 515, 511, 560, 521, 489, 529, 523, 550, 524, 502, 511, 500, 529, 531, 563, 511, 478, 543, 495, 537, 542, 493, 451, 521, 539, 501, 543, 542, 509, 529, 516, 557, 501, 514, 487, 564, 508, 487, 511, 497, 528, 526, 484, 516, 507, 540, 480, 502, 449, 520, 468, 526, 512, 525, 518, 495, 503, 474, 483, 534, 509, 490, 511, 526, 510, 503, 530, 494, 466, 527, 515, 489, 514, 517, 518, 504, 492, 537, 528, 490, 495, 537, 502, 537, 523, 497, 509, 515, 530, 515, 532, 491, 476, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[528, 487, 530, 520, 488, 524, 507, 505, 531, 544, 530, 482, 487, 536, 492, 523, 528, 537, 514, 509, 521, 512, 550, 525, 499, 471, 526, 498, 523, 502, 503, 520, 490, 497, 526, 491, 553, 550, 532, 503, 510, 508, 538, 486, 490, 519, 525, 505, 488, 531, 504, 491, 528, 501, 447, 518, 536, 506, 525, 505, 502, 529, 477, 530, 502, 511, 513, 538, 468, 501, 514, 531, 537, 512, 507, 506, 515, 509, 503, 517, 487, 482, 459, 501, 509, 526, 497, 535, 511, 490, 481, 573, 549, 494, 524, 518, 534, 521, 543, 496, 513, 509, 489, 529, 502, 554, 506, 523, 524, 501, 541, 510, 504, 503, 519, 483, 486, 475, 486, 509, 528, 500, 509, 552, 492, 534, 496, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[502, 519, 515, 539, 536, 496, 498, 516, 520, 498, 504, 506, 497, 507, 506, 525, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 490, 505, 494, 547, 515, 510, 511, 484, 498, 469, 513, 515, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 505, 530, 528, 503, 506, 487, 479, 496, 507, 506, 481, 503, 537, 516, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 541, 492, 487, 520, 497, 524, 500, 518, 505, 516, 504, 508, 522, 514, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 519, 502, 531, 543, 543, 525, 513, 568, 508, 529, 517, 479, 517, 490, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 463, 513, 539, 515, 538, 504, 528, 525, 506, 528, 505, 503, 497, 530, 21, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 523, 493, 500, 503, 512, 489, 545, 477, 520, 507, 495, 509, 492, 514, 454, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 486, 496, 538, 509, 543, 532, 524, 528, 514, 527, 513, 503, 529, 528, 524, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8127, 8178, 8145, 8294, 8241, 8052, 8312] avg=8192.0000 max=8312 max_rank=7 min=8052 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[481, 519, 511, 533, 524, 476, 506, 502, 508, 489, 499, 518, 501, 493, 495, 545, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 480, 499, 546, 550, 503, 524, 487, 508, 514, 479, 529, 543, 520, 510, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 523, 509, 481, 536, 512, 506, 509, 468, 522, 517, 517, 498, 504, 551, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 488, 523, 496, 511, 482, 563, 519, 499, 479, 519, 482, 521, 538, 552, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 533, 526, 515, 464, 500, 527, 501, 535, 473, 534, 479, 537, 520, 517, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 471, 510, 532, 499, 539, 522, 495, 517, 510, 494, 502, 513, 543, 504, 522, 21, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 482, 509, 495, 550, 536, 547, 542, 542, 476, 545, 494, 521, 525, 510, 511, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 501, 517, 524, 508, 458, 513, 486, 503, 498, 510, 551, 513, 509, 537, 472, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8189, 8167, 8174, 8231, 8222, 8310, 8140] avg=8192.0000 max=8310 max_rank=6 min=8103 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[505, 501, 479, 508, 505, 510, 535, 553, 561, 518, 506, 502, 534, 527, 510, 508, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 530, 528, 497, 546, 511, 530, 523, 515, 498, 517, 492, 508, 482, 499, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 521, 492, 534, 525, 565, 517, 484, 490, 494, 521, 514, 498, 510, 494, 489, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 518, 526, 536, 526, 532, 494, 471, 511, 471, 494, 527, 538, 513, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 513, 472, 468, 503, 488, 532, 525, 521, 517, 514, 515, 505, 516, 481, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 509, 507, 518, 506, 500, 543, 551, 532, 472, 513, 550, 474, 526, 527, 543, 26, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 527, 498, 545, 490, 503, 496, 494, 566, 508, 488, 507, 472, 525, 501, 517, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 509, 499, 524, 499, 546, 492, 503, 497, 511, 474, 509, 480, 475, 542, 525, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8183, 8175, 8215, 8120, 8350, 8146, 8079] avg=8192.0000 max=8350 max_rank=5 min=8079 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 504, 536, 522, 495, 518, 497, 485, 503, 515, 506, 484, 527, 526, 495, 523, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 541, 532, 491, 518, 528, 509, 515, 475, 508, 460, 551, 501, 515, 539, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 515, 511, 560, 521, 489, 529, 523, 550, 524, 502, 511, 500, 529, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 511, 478, 543, 495, 537, 542, 493, 451, 521, 528, 501, 543, 542, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 557, 501, 514, 487, 564, 508, 487, 511, 497, 508, 526, 484, 516, 507, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 502, 449, 520, 468, 526, 512, 525, 518, 495, 503, 474, 483, 534, 509, 490, 20, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 526, 510, 503, 530, 494, 466, 527, 515, 489, 514, 517, 518, 504, 492, 537, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 490, 495, 537, 502, 537, 523, 497, 509, 515, 530, 515, 532, 491, 476, 499, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8208, 8298, 8286, 8223, 8018, 8159, 8186] avg=8192.0000 max=8298 max_rank=2 min=8018 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 517, 496, 512, 551, 506, 492, 509, 534, 533, 514, 499, 483, 528, 568, 543, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 538, 539, 497, 522, 506, 549, 518, 474, 489, 515, 537, 449, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 487, 517, 542, 512, 513, 506, 501, 456, 510, 510, 494, 530, 539, 559, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 498, 525, 510, 510, 501, 501, 537, 518, 521, 511, 490, 530, 534, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 520, 480, 513, 460, 477, 489, 486, 493, 557, 486, 524, 506, 473, 507, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 534, 497, 501, 533, 492, 534, 473, 522, 494, 494, 531, 558, 503, 465, 523, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 494, 535, 523, 524, 474, 506, 493, 484, 505, 478, 501, 529, 519, 480, 558, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 536, 529, 507, 485, 495, 477, 529, 503, 517, 506, 533, 568, 506, 545, 532, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8149, 8148, 8271, 8037, 8186, 8176, 8281] avg=8192.0000 max=8288 max_rank=0 min=8037 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[460, 540, 525, 500, 524, 517, 456, 508, 525, 461, 543, 522, 495, 539, 485, 503, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 492, 536, 491, 542, 505, 534, 517, 460, 556, 490, 527, 516, 514, 529, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 495, 518, 474, 517, 506, 535, 506, 496, 521, 537, 524, 532, 504, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 504, 528, 543, 526, 533, 481, 524, 494, 529, 516, 488, 512, 509, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 528, 508, 515, 519, 531, 516, 509, 546, 555, 472, 524, 521, 527, 508, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 528, 507, 533, 519, 514, 531, 495, 485, 537, 502, 436, 511, 503, 459, 519, 19, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 491, 514, 480, 522, 517, 526, 502, 501, 492, 511, 538, 560, 516, 525, 540, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 534, 535, 504, 518, 502, 504, 524, 548, 496, 494, 518, 505, 505, 496, 472, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8228, 8154, 8272, 8261, 8123, 8202, 8190] avg=8192.0000 max=8272 max_rank=3 min=8106 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[516, 522, 497, 538, 499, 495, 545, 500, 528, 519, 495, 486, 551, 506, 516, 493, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 474, 521, 481, 533, 502, 524, 517, 488, 521, 554, 531, 496, 576, 480, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 485, 484, 534, 521, 499, 496, 492, 501, 550, 509, 509, 515, 507, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 507, 530, 487, 510, 475, 467, 490, 545, 517, 526, 490, 524, 474, 509, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 491, 517, 495, 499, 475, 500, 519, 519, 531, 535, 537, 522, 493, 532, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 500, 511, 517, 524, 519, 537, 528, 485, 512, 526, 506, 458, 568, 515, 501, 21, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 517, 565, 493, 520, 529, 474, 500, 506, 463, 520, 542, 496, 498, 501, 522, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 509, 493, 513, 512, 529, 516, 507, 542, 509, 535, 524, 511, 487, 515, 516, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8230, 8193, 8066, 8206, 8231, 8190, 8211] avg=8192.0000 max=8231 max_rank=5 min=8066 min_rank=3 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 487, 530, 520, 488, 524, 507, 505, 531, 544, 530, 482, 487, 536, 492, 523, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 514, 509, 521, 512, 550, 525, 499, 471, 526, 498, 523, 502, 487, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 497, 526, 491, 553, 550, 532, 503, 510, 508, 538, 486, 490, 519, 525, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 531, 504, 491, 528, 501, 447, 518, 536, 506, 514, 505, 502, 529, 477, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 513, 538, 468, 501, 514, 531, 537, 512, 487, 506, 515, 509, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 482, 459, 501, 509, 526, 497, 535, 511, 490, 481, 573, 549, 494, 524, 518, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 543, 496, 513, 509, 489, 529, 502, 554, 506, 523, 524, 501, 541, 510, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 519, 483, 486, 475, 486, 509, 528, 500, 509, 552, 492, 534, 496, 552, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8222, 8223, 8107, 8164, 8165, 8301, 8137] avg=8192.0000 max=8301 max_rank=6 min=8107 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 4352, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4224, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 3968, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[510, 507, 490, 510, 502, 499, 491, 506, 517, 544, 506, 527, 521, 533, 565, 519, 530, 507, 510, 548, 525, 513, 535, 510, 484, 507, 519, 519, 498, 537, 475, 533, 478, 515, 492, 507, 534, 525, 519, 528, 511, 529, 527, 547, 510, 488, 533, 515, 513, 504, 518, 526, 527, 511, 479, 528, 496, 486, 536, 506, 493, 517, 502, 494, 502, 541, 506, 489, 523, 499, 496, 486, 550, 515, 520, 517, 492, 523, 528, 504, 534, 521, 554, 504, 523, 504, 529, 495, 519, 522, 496, 526, 497, 524, 483, 512, 490, 535, 505, 498, 522, 522, 489, 509, 519, 492, 490, 535, 481, 504, 500, 528, 515, 508, 551, 482, 506, 496, 489, 506, 540, 537, 485, 509, 500, 474, 506, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[502, 528, 506, 493, 490, 507, 508, 552, 530, 521, 506, 555, 497, 477, 531, 505, 544, 496, 525, 505, 522, 515, 484, 492, 486, 518, 510, 497, 537, 499, 540, 569, 494, 538, 519, 483, 520, 504, 484, 529, 500, 525, 525, 501, 542, 498, 540, 495, 522, 490, 513, 511, 479, 538, 492, 523, 529, 508, 496, 536, 525, 506, 498, 491, 541, 478, 517, 515, 511, 496, 526, 485, 536, 518, 543, 495, 527, 507, 519, 521, 508, 487, 529, 511, 520, 542, 502, 554, 501, 497, 513, 502, 489, 536, 536, 483, 465, 468, 520, 499, 458, 506, 531, 518, 497, 531, 500, 523, 486, 514, 524, 511, 487, 512, 507, 523, 531, 513, 487, 482, 538, 519, 496, 523, 525, 572, 534, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[541, 461, 508, 505, 470, 500, 515, 488, 525, 504, 511, 466, 517, 540, 501, 537, 495, 495, 508, 492, 515, 506, 547, 490, 500, 536, 520, 532, 487, 538, 566, 523, 536, 524, 501, 485, 512, 502, 515, 529, 466, 497, 528, 538, 498, 481, 572, 454, 528, 553, 545, 497, 534, 531, 505, 516, 495, 504, 498, 481, 514, 518, 522, 539, 498, 506, 504, 529, 479, 537, 499, 495, 509, 493, 522, 508, 525, 583, 490, 463, 470, 518, 522, 528, 507, 510, 522, 555, 515, 510, 524, 513, 509, 514, 536, 514, 518, 536, 532, 511, 508, 516, 494, 485, 527, 533, 506, 508, 521, 468, 515, 515, 514, 520, 529, 513, 498, 526, 503, 517, 494, 523, 507, 474, 519, 520, 486, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[515, 506, 506, 513, 483, 523, 511, 521, 516, 512, 508, 511, 512, 486, 527, 533, 537, 478, 525, 511, 462, 527, 520, 517, 551, 529, 515, 532, 493, 550, 481, 519, 471, 479, 520, 501, 515, 511, 490, 507, 512, 490, 502, 501, 517, 498, 497, 515, 515, 529, 537, 539, 512, 507, 511, 481, 525, 514, 542, 487, 525, 495, 505, 513, 505, 528, 510, 525, 519, 555, 515, 508, 535, 505, 486, 500, 495, 470, 495, 500, 487, 555, 524, 518, 515, 544, 525, 542, 508, 549, 516, 504, 510, 477, 507, 492, 484, 502, 514, 544, 535, 509, 468, 533, 509, 524, 506, 508, 542, 510, 522, 506, 511, 497, 500, 487, 502, 540, 517, 502, 509, 515, 523, 505, 536, 526, 551, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[489, 501, 478, 528, 509, 497, 462, 498, 501, 497, 511, 477, 525, 502, 532, 509, 499, 494, 539, 512, 561, 474, 551, 520, 534, 549, 537, 528, 510, 516, 535, 528, 497, 498, 497, 529, 573, 534, 536, 491, 505, 512, 521, 518, 520, 504, 495, 506, 522, 514, 519, 502, 510, 529, 508, 518, 526, 483, 502, 532, 497, 562, 527, 527, 512, 480, 461, 504, 519, 505, 534, 532, 520, 506, 560, 518, 509, 499, 504, 497, 511, 522, 458, 471, 526, 470, 489, 492, 542, 498, 490, 506, 501, 519, 514, 528, 490, 497, 528, 535, 541, 541, 491, 495, 516, 509, 534, 494, 507, 523, 543, 501, 513, 476, 483, 552, 487, 520, 518, 479, 524, 523, 511, 495, 535, 530, 518, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[465, 499, 506, 476, 454, 536, 497, 482, 455, 534, 528, 514, 517, 560, 518, 537, 509, 531, 515, 482, 491, 558, 519, 515, 510, 500, 508, 502, 505, 496, 492, 517, 501, 510, 544, 516, 533, 482, 494, 471, 507, 508, 556, 503, 575, 502, 523, 544, 511, 481, 502, 467, 486, 503, 494, 510, 531, 526, 510, 522, 499, 507, 535, 547, 513, 514, 512, 483, 493, 485, 535, 488, 549, 537, 499, 516, 495, 509, 515, 501, 549, 490, 504, 503, 463, 520, 539, 524, 498, 497, 550, 482, 516, 544, 497, 480, 498, 537, 497, 519, 511, 500, 533, 531, 495, 519, 537, 544, 515, 470, 515, 500, 530, 550, 510, 509, 474, 509, 542, 536, 561, 530, 510, 496, 544, 545, 511, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[502, 512, 525, 504, 472, 480, 536, 500, 512, 493, 539, 489, 493, 540, 515, 496, 493, 517, 483, 541, 484, 508, 494, 492, 516, 497, 537, 512, 555, 529, 545, 535, 500, 588, 472, 500, 521, 527, 503, 497, 540, 502, 526, 516, 523, 501, 501, 512, 510, 488, 504, 532, 494, 499, 519, 510, 544, 453, 529, 511, 541, 501, 490, 540, 469, 528, 524, 487, 511, 512, 497, 551, 504, 509, 501, 519, 570, 493, 525, 555, 473, 511, 548, 529, 532, 545, 478, 499, 516, 470, 508, 533, 491, 498, 494, 534, 518, 508, 525, 527, 497, 512, 530, 536, 541, 527, 488, 493, 555, 509, 514, 501, 525, 501, 505, 537, 500, 515, 486, 490, 496, 508, 502, 501, 499, 507, 520, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[515, 484, 515, 556, 486, 484, 508, 553, 524, 546, 514, 519, 525, 551, 511, 567, 495, 528, 492, 501, 500, 483, 508, 460, 500, 511, 518, 508, 473, 528, 517, 531, 494, 472, 514, 477, 522, 476, 537, 525, 503, 516, 519, 500, 551, 521, 581, 486, 472, 556, 533, 497, 477, 503, 476, 560, 503, 518, 500, 517, 521, 532, 481, 524, 496, 519, 530, 466, 483, 501, 489, 510, 495, 558, 543, 521, 499, 520, 507, 522, 449, 507, 512, 515, 529, 508, 523, 502, 505, 532, 499, 481, 516, 524, 535, 523, 538, 502, 505, 516, 506, 513, 543, 519, 491, 477, 519, 506, 509, 502, 486, 546, 530, 539, 495, 511, 534, 474, 538, 526, 499, 482, 527, 501, 515, 513, 566, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[510, 507, 490, 510, 502, 499, 491, 506, 517, 544, 506, 527, 521, 533, 565, 519, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 507, 510, 548, 525, 513, 535, 510, 484, 507, 519, 519, 498, 537, 475, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 492, 507, 534, 525, 519, 528, 511, 529, 527, 547, 510, 488, 532, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 504, 518, 526, 527, 511, 479, 528, 496, 486, 536, 506, 493, 517, 502, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 541, 506, 489, 523, 499, 496, 486, 550, 515, 520, 517, 492, 523, 528, 504, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 554, 504, 523, 504, 529, 495, 519, 522, 496, 526, 497, 524, 483, 512, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 535, 505, 498, 522, 522, 489, 509, 519, 492, 490, 535, 481, 504, 500, 528, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 551, 482, 506, 496, 489, 506, 540, 537, 485, 509, 500, 474, 495, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8216, 8257, 8134, 8202, 8246, 8122, 8081] avg=8192.0000 max=8278 max_rank=0 min=8081 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[541, 461, 508, 505, 470, 500, 515, 488, 525, 504, 511, 466, 517, 540, 501, 537, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 495, 508, 492, 515, 506, 547, 490, 500, 536, 520, 532, 487, 538, 566, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 524, 501, 485, 512, 502, 515, 529, 466, 497, 528, 538, 498, 481, 571, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 553, 545, 497, 534, 531, 505, 516, 495, 504, 498, 481, 514, 518, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 506, 504, 529, 479, 537, 499, 495, 509, 493, 522, 508, 525, 583, 490, 463, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 518, 522, 528, 507, 510, 522, 555, 515, 510, 524, 513, 509, 514, 536, 514, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 536, 532, 511, 508, 516, 494, 485, 527, 533, 506, 508, 521, 468, 515, 515, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 520, 529, 513, 498, 526, 503, 517, 494, 523, 507, 474, 519, 520, 475, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8217, 8137, 8278, 8151, 8270, 8196, 8168] avg=8192.0000 max=8278 max_rank=3 min=8119 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 512, 525, 504, 472, 480, 536, 500, 512, 493, 539, 489, 493, 540, 515, 496, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 483, 541, 484, 508, 494, 492, 516, 497, 537, 512, 555, 529, 545, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 588, 472, 500, 521, 527, 503, 497, 540, 502, 526, 516, 523, 501, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 504, 532, 494, 499, 519, 510, 544, 453, 529, 511, 541, 501, 490, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 528, 524, 487, 511, 512, 497, 551, 504, 509, 501, 519, 570, 493, 525, 555, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 511, 548, 529, 532, 545, 478, 499, 516, 470, 508, 533, 491, 498, 494, 534, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 525, 527, 497, 512, 530, 536, 541, 527, 488, 493, 555, 509, 514, 501, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 501, 505, 537, 500, 515, 486, 490, 496, 508, 502, 501, 499, 507, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8204, 8228, 8163, 8266, 8162, 8284, 8090] avg=8192.0000 max=8284 max_rank=6 min=8090 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[465, 499, 506, 476, 454, 536, 497, 482, 455, 534, 528, 514, 517, 560, 518, 537, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 531, 515, 482, 491, 558, 519, 515, 510, 500, 508, 502, 505, 496, 492, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 510, 544, 516, 533, 482, 494, 471, 507, 508, 556, 503, 575, 502, 519, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 481, 502, 467, 486, 503, 494, 510, 531, 526, 510, 522, 499, 507, 535, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 514, 512, 483, 493, 485, 535, 488, 549, 537, 499, 516, 495, 509, 515, 501, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 490, 504, 503, 463, 520, 539, 524, 498, 497, 550, 482, 516, 544, 497, 480, 4, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 537, 497, 519, 511, 500, 533, 531, 495, 519, 537, 544, 515, 470, 515, 500, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 550, 510, 509, 474, 509, 542, 536, 561, 530, 510, 496, 544, 545, 488, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8109, 8265, 8127, 8165, 8166, 8229, 8364] avg=8192.0000 max=8364 max_rank=7 min=8109 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 528, 506, 493, 490, 507, 508, 552, 530, 521, 506, 555, 497, 477, 531, 505, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 496, 525, 505, 522, 515, 484, 492, 486, 518, 510, 497, 537, 499, 540, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 538, 519, 483, 520, 504, 484, 529, 500, 525, 525, 501, 542, 498, 539, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 513, 511, 479, 538, 492, 523, 529, 508, 496, 536, 525, 506, 498, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 478, 517, 515, 511, 496, 526, 485, 536, 518, 543, 495, 527, 507, 519, 521, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 487, 529, 511, 520, 542, 502, 554, 501, 497, 513, 502, 489, 536, 536, 483, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 468, 520, 499, 458, 506, 531, 518, 497, 531, 500, 523, 486, 514, 524, 511, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 512, 507, 523, 531, 513, 487, 482, 538, 519, 496, 523, 525, 572, 521, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8203, 8196, 8155, 8247, 8213, 8055, 8226] avg=8192.0000 max=8247 max_rank=4 min=8055 min_rank=6 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[489, 501, 478, 528, 509, 497, 462, 498, 501, 497, 511, 477, 525, 502, 532, 509, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 494, 539, 512, 561, 474, 551, 520, 534, 549, 537, 528, 510, 516, 535, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 497, 529, 573, 534, 536, 491, 505, 512, 521, 518, 520, 504, 494, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 519, 502, 510, 529, 508, 518, 526, 483, 502, 532, 497, 562, 527, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 480, 461, 504, 519, 505, 534, 532, 520, 506, 560, 518, 509, 499, 504, 497, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 522, 458, 471, 526, 470, 489, 492, 542, 498, 490, 506, 501, 519, 514, 528, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 497, 528, 535, 541, 541, 491, 495, 516, 509, 534, 494, 507, 523, 543, 501, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 476, 483, 552, 487, 520, 518, 479, 524, 523, 511, 495, 535, 530, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8354, 8235, 8276, 8171, 8040, 8248, 8166] avg=8192.0000 max=8354 max_rank=1 min=8040 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 506, 506, 513, 483, 523, 511, 521, 516, 512, 508, 511, 512, 486, 527, 533, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 478, 525, 511, 462, 527, 520, 517, 551, 529, 515, 532, 493, 550, 481, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 479, 520, 501, 515, 511, 490, 507, 512, 490, 502, 501, 517, 498, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 529, 537, 539, 512, 507, 511, 481, 525, 514, 542, 487, 525, 495, 505, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 510, 525, 519, 555, 515, 508, 535, 505, 486, 500, 495, 470, 495, 500, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 555, 524, 518, 515, 544, 525, 542, 508, 549, 516, 504, 510, 477, 507, 492, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 502, 514, 544, 535, 509, 468, 533, 509, 524, 506, 508, 542, 510, 522, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 500, 487, 502, 540, 517, 502, 509, 515, 523, 505, 536, 526, 538, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8214, 8025, 8235, 8163, 8276, 8220, 8190] avg=8192.0000 max=8276 max_rank=5 min=8025 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 484, 515, 556, 486, 484, 508, 553, 524, 546, 514, 519, 525, 551, 511, 567, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 528, 492, 501, 500, 483, 508, 460, 500, 511, 518, 508, 473, 528, 517, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 472, 514, 477, 522, 476, 537, 525, 503, 516, 519, 500, 551, 521, 580, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 556, 533, 497, 477, 503, 476, 560, 503, 518, 500, 517, 521, 532, 481, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 530, 466, 483, 501, 489, 510, 495, 558, 543, 521, 499, 520, 507, 522, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 507, 512, 515, 529, 508, 523, 502, 505, 532, 499, 481, 516, 524, 535, 523, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 502, 505, 516, 506, 513, 543, 519, 491, 477, 519, 506, 509, 502, 486, 546, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 539, 495, 511, 534, 474, 538, 526, 499, 482, 527, 501, 515, 513, 553, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8389, 8019, 8193, 8168, 8171, 8163, 8182, 8251] avg=8192.0000 max=8389 max_rank=0 min=8019 min_rank=1 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4352, 4096, 4352, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[515, 510, 493, 477, 544, 507, 558, 499, 510, 553, 521, 481, 517, 509, 534, 521, 524, 509, 493, 533, 530, 497, 497, 523, 506, 465, 485, 501, 497, 521, 515, 499, 506, 523, 542, 533, 497, 550, 575, 516, 517, 524, 499, 528, 488, 531, 525, 509, 518, 480, 534, 509, 488, 494, 539, 524, 523, 461, 505, 489, 497, 529, 518, 504, 521, 520, 519, 530, 547, 532, 491, 516, 524, 518, 534, 563, 480, 487, 501, 497, 517, 535, 512, 555, 514, 541, 517, 511, 520, 507, 491, 510, 491, 514, 519, 491, 501, 506, 495, 492, 452, 541, 546, 500, 527, 489, 501, 495, 494, 499, 468, 516, 542, 518, 519, 518, 496, 511, 517, 487, 512, 491, 512, 527, 479, 522, 494, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[493, 482, 495, 494, 531, 514, 546, 534, 495, 466, 506, 491, 516, 546, 523, 525, 538, 498, 510, 529, 542, 498, 514, 516, 507, 511, 496, 524, 501, 517, 551, 512, 528, 530, 526, 508, 519, 521, 497, 494, 471, 510, 531, 502, 514, 523, 500, 521, 486, 490, 497, 483, 480, 528, 523, 564, 501, 519, 553, 507, 474, 563, 507, 526, 491, 509, 515, 574, 495, 505, 499, 541, 509, 494, 516, 532, 520, 514, 548, 537, 504, 507, 510, 493, 489, 498, 491, 515, 505, 533, 500, 537, 497, 505, 507, 521, 502, 498, 503, 515, 545, 520, 512, 519, 518, 504, 491, 497, 490, 523, 493, 497, 507, 554, 494, 495, 531, 514, 505, 482, 534, 491, 513, 495, 520, 486, 514, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[543, 572, 534, 541, 516, 514, 527, 513, 512, 526, 487, 512, 530, 507, 512, 525, 538, 507, 531, 500, 514, 540, 525, 482, 529, 526, 520, 516, 513, 525, 509, 461, 503, 555, 503, 513, 512, 482, 507, 547, 502, 509, 521, 541, 499, 522, 498, 514, 515, 475, 524, 536, 499, 523, 491, 543, 499, 506, 488, 524, 477, 525, 511, 488, 529, 533, 493, 501, 543, 549, 500, 548, 504, 483, 480, 505, 511, 506, 491, 475, 481, 455, 503, 553, 514, 512, 507, 487, 511, 517, 481, 513, 482, 485, 496, 517, 534, 538, 488, 536, 512, 544, 523, 499, 494, 509, 501, 482, 486, 506, 551, 516, 473, 488, 529, 471, 496, 526, 522, 528, 498, 514, 519, 517, 520, 530, 516, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[527, 507, 575, 525, 539, 506, 486, 499, 481, 497, 519, 555, 507, 513, 506, 523, 528, 539, 520, 516, 506, 464, 492, 539, 497, 518, 525, 532, 503, 526, 518, 506, 482, 510, 524, 530, 492, 522, 492, 500, 504, 514, 544, 497, 522, 493, 529, 523, 484, 508, 540, 486, 507, 522, 513, 480, 504, 514, 544, 516, 481, 546, 479, 508, 521, 476, 533, 459, 529, 501, 525, 484, 511, 536, 539, 533, 518, 542, 582, 514, 470, 531, 465, 504, 484, 517, 541, 529, 505, 510, 520, 504, 551, 506, 484, 498, 471, 505, 496, 484, 531, 561, 515, 523, 483, 471, 515, 517, 475, 505, 461, 530, 497, 547, 519, 521, 509, 494, 528, 515, 518, 569, 466, 504, 518, 543, 527, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[533, 490, 507, 522, 528, 495, 511, 483, 516, 508, 497, 490, 497, 498, 524, 503, 514, 552, 484, 518, 532, 534, 547, 517, 518, 490, 504, 475, 506, 474, 502, 443, 506, 530, 526, 524, 527, 574, 528, 506, 530, 515, 507, 516, 498, 504, 519, 499, 471, 543, 507, 486, 504, 489, 532, 506, 502, 517, 504, 522, 496, 510, 546, 519, 527, 492, 502, 498, 542, 541, 504, 507, 512, 489, 499, 523, 510, 528, 527, 528, 484, 503, 522, 535, 502, 530, 507, 486, 553, 531, 543, 530, 543, 511, 516, 497, 550, 504, 532, 486, 492, 523, 532, 537, 511, 532, 493, 540, 471, 533, 531, 512, 503, 534, 528, 500, 468, 512, 518, 454, 499, 492, 496, 509, 503, 515, 516, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[508, 494, 523, 515, 508, 524, 552, 512, 510, 508, 473, 500, 516, 500, 499, 517, 538, 511, 536, 501, 523, 502, 490, 530, 481, 496, 538, 533, 467, 505, 510, 524, 538, 484, 519, 514, 517, 525, 484, 517, 536, 536, 523, 513, 487, 508, 506, 527, 533, 509, 517, 530, 529, 549, 527, 515, 493, 456, 535, 534, 498, 530, 534, 527, 508, 477, 514, 502, 540, 549, 513, 535, 499, 491, 512, 518, 506, 496, 514, 477, 514, 523, 495, 513, 506, 497, 530, 503, 528, 567, 521, 516, 502, 498, 536, 530, 485, 496, 503, 471, 499, 512, 499, 522, 506, 497, 478, 503, 521, 487, 515, 518, 524, 466, 561, 494, 505, 502, 484, 523, 531, 495, 534, 542, 496, 512, 529, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[506, 519, 527, 508, 489, 517, 506, 564, 527, 479, 510, 524, 499, 511, 498, 504, 491, 534, 539, 520, 519, 497, 509, 483, 511, 560, 509, 519, 526, 498, 494, 553, 521, 499, 546, 513, 495, 511, 519, 530, 525, 535, 493, 515, 487, 529, 517, 462, 507, 518, 482, 517, 536, 514, 502, 521, 519, 493, 491, 522, 549, 518, 531, 530, 503, 536, 491, 520, 503, 535, 524, 509, 562, 487, 502, 524, 504, 491, 507, 552, 505, 508, 537, 503, 512, 516, 491, 487, 528, 473, 505, 505, 571, 529, 498, 516, 526, 485, 539, 505, 487, 466, 519, 525, 488, 520, 468, 523, 549, 458, 502, 494, 519, 499, 520, 500, 500, 526, 495, 529, 537, 501, 499, 542, 519, 468, 494, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[518, 507, 515, 523, 554, 516, 508, 497, 490, 510, 525, 530, 496, 533, 506, 505, 514, 504, 535, 515, 506, 493, 505, 519, 523, 525, 478, 521, 500, 473, 506, 495, 513, 525, 511, 549, 552, 508, 523, 490, 491, 506, 517, 503, 490, 514, 493, 514, 486, 495, 534, 516, 527, 456, 515, 509, 519, 536, 518, 548, 527, 486, 520, 513, 504, 502, 496, 502, 509, 489, 524, 526, 467, 500, 513, 512, 485, 521, 540, 522, 516, 503, 470, 515, 463, 514, 531, 517, 478, 497, 515, 542, 459, 498, 529, 537, 504, 563, 538, 542, 496, 497, 510, 529, 531, 521, 499, 547, 555, 541, 548, 559, 470, 487, 481, 537, 548, 490, 506, 524, 497, 507, 513, 530, 488, 485, 546, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[493, 482, 495, 494, 509, 514, 546, 534, 495, 466, 506, 491, 516, 546, 523, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 498, 510, 529, 542, 498, 514, 516, 507, 511, 496, 524, 501, 517, 551, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 530, 481, 508, 519, 521, 497, 494, 471, 510, 531, 502, 514, 523, 500, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 490, 497, 483, 480, 528, 523, 564, 501, 519, 553, 507, 474, 563, 507, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 509, 515, 574, 495, 505, 499, 541, 509, 494, 516, 503, 520, 514, 548, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 510, 493, 489, 498, 491, 515, 505, 533, 500, 537, 497, 505, 507, 521, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 498, 503, 515, 545, 520, 512, 519, 518, 504, 491, 497, 490, 523, 493, 497, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 554, 494, 495, 531, 514, 505, 482, 534, 491, 513, 495, 520, 486, 514, 546, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8269, 8150, 8206, 8270, 8137, 8164, 8205] avg=8192.0000 max=8270 max_rank=4 min=8135 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[515, 510, 493, 477, 521, 507, 558, 499, 510, 553, 521, 481, 517, 509, 534, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 509, 493, 533, 530, 497, 497, 523, 506, 465, 485, 501, 497, 521, 515, 499, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 496, 533, 497, 550, 575, 516, 517, 524, 499, 528, 488, 531, 525, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 480, 534, 509, 488, 494, 539, 524, 523, 461, 505, 489, 497, 529, 518, 504, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 520, 519, 530, 547, 532, 491, 516, 524, 518, 534, 532, 480, 487, 501, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 535, 512, 555, 514, 541, 517, 511, 520, 507, 491, 510, 491, 514, 519, 491, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 495, 492, 452, 541, 546, 500, 527, 489, 501, 495, 494, 499, 468, 516, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 519, 518, 496, 511, 517, 487, 512, 491, 512, 527, 479, 522, 494, 525, 13, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8100, 8317, 8117, 8249, 8271, 8060, 8196] avg=8192.0000 max=8317 max_rank=2 min=8060 min_rank=6 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[527, 507, 575, 525, 516, 506, 486, 499, 481, 497, 519, 555, 507, 513, 506, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 539, 520, 516, 506, 464, 492, 539, 497, 518, 525, 532, 503, 526, 518, 506, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 510, 479, 530, 492, 522, 492, 500, 504, 514, 544, 497, 522, 493, 529, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 540, 486, 507, 522, 513, 480, 504, 514, 544, 516, 481, 546, 479, 508, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 476, 533, 459, 529, 501, 525, 484, 511, 536, 539, 504, 518, 542, 582, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 531, 465, 504, 484, 517, 541, 529, 505, 510, 520, 504, 551, 506, 484, 498, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 505, 496, 484, 531, 561, 515, 523, 483, 471, 515, 517, 475, 505, 461, 530, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 547, 519, 521, 509, 494, 528, 515, 518, 569, 466, 504, 518, 543, 527, 492, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8234, 8133, 8137, 8274, 8144, 8080, 8292] avg=8192.0000 max=8292 max_rank=7 min=8080 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[533, 490, 507, 522, 494, 495, 511, 483, 516, 508, 497, 490, 497, 498, 524, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 552, 484, 518, 532, 534, 547, 517, 518, 490, 504, 475, 506, 474, 502, 443, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 530, 474, 524, 527, 574, 528, 506, 530, 515, 507, 516, 498, 504, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 543, 507, 486, 504, 489, 532, 506, 502, 517, 504, 522, 496, 510, 546, 519, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 492, 502, 498, 542, 541, 504, 507, 512, 489, 499, 486, 510, 528, 527, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 503, 522, 535, 502, 530, 507, 486, 553, 531, 543, 530, 543, 511, 516, 497, 21, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 504, 532, 486, 492, 523, 532, 537, 511, 532, 493, 540, 471, 533, 531, 512, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 534, 528, 500, 468, 512, 518, 454, 499, 492, 496, 509, 503, 515, 516, 513, 17, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 8119, 8257, 8162, 8192, 8325, 8320, 8093] avg=8192.0000 max=8325 max_rank=5 min=8068 min_rank=0 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[508, 494, 523, 515, 486, 524, 552, 512, 510, 508, 473, 500, 516, 500, 499, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 511, 536, 501, 523, 502, 490, 530, 481, 496, 538, 533, 467, 505, 510, 524, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 484, 474, 514, 517, 525, 484, 517, 536, 536, 523, 513, 487, 508, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 509, 517, 530, 529, 549, 527, 515, 493, 456, 535, 534, 498, 530, 534, 527, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 477, 514, 502, 540, 549, 513, 535, 499, 491, 512, 489, 506, 496, 514, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 523, 495, 513, 506, 497, 530, 503, 528, 567, 521, 516, 502, 498, 536, 530, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 496, 503, 471, 499, 512, 499, 522, 506, 497, 478, 503, 521, 487, 515, 518, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 466, 561, 494, 505, 502, 484, 523, 531, 495, 534, 542, 496, 512, 529, 502, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8190, 8189, 8321, 8122, 8304, 8049, 8224] avg=8192.0000 max=8321 max_rank=3 min=8049 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[543, 572, 534, 541, 494, 514, 527, 513, 512, 526, 487, 512, 530, 507, 512, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 507, 531, 500, 514, 540, 525, 482, 529, 526, 520, 516, 513, 525, 509, 461, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 555, 460, 513, 512, 482, 507, 547, 502, 509, 521, 541, 499, 522, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 475, 524, 536, 499, 523, 491, 543, 499, 506, 488, 524, 477, 525, 511, 488, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 533, 493, 501, 543, 549, 500, 548, 504, 483, 480, 478, 511, 506, 491, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 455, 503, 553, 514, 512, 507, 487, 511, 517, 481, 513, 482, 485, 496, 517, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 538, 488, 536, 512, 544, 523, 499, 494, 509, 501, 482, 486, 506, 551, 516, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 488, 529, 471, 496, 526, 522, 528, 498, 514, 519, 517, 520, 530, 516, 546, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 8241, 8185, 8129, 8124, 8038, 8254, 8216] avg=8192.0000 max=8349 max_rank=0 min=8038 min_rank=5 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[506, 519, 527, 508, 470, 517, 506, 564, 527, 479, 510, 524, 499, 511, 498, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 534, 539, 520, 519, 497, 509, 483, 511, 560, 509, 519, 526, 498, 494, 553, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 499, 499, 513, 495, 511, 519, 530, 525, 535, 493, 515, 487, 529, 517, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 482, 517, 536, 514, 502, 521, 519, 493, 491, 522, 549, 518, 531, 530, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 536, 491, 520, 503, 535, 524, 509, 562, 487, 502, 495, 504, 491, 507, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 537, 503, 512, 516, 491, 487, 528, 473, 505, 505, 571, 529, 498, 516, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 485, 539, 505, 487, 466, 519, 525, 488, 520, 468, 523, 549, 458, 502, 494, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 520, 500, 500, 526, 495, 529, 537, 501, 499, 542, 519, 468, 494, 503, 11, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8266, 8150, 8254, 8221, 8209, 8093, 8174] avg=8192.0000 max=8266 max_rank=1 min=8093 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[518, 507, 515, 523, 531, 516, 508, 497, 490, 510, 525, 530, 496, 533, 506, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 535, 515, 506, 493, 505, 519, 523, 525, 478, 521, 500, 473, 506, 495, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 467, 549, 552, 508, 523, 490, 491, 506, 517, 503, 490, 514, 493, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 495, 534, 516, 527, 456, 515, 509, 519, 536, 518, 548, 527, 486, 520, 513, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 496, 502, 509, 489, 524, 526, 467, 500, 513, 484, 485, 521, 540, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 503, 470, 515, 463, 514, 531, 517, 478, 497, 515, 542, 459, 498, 529, 537, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 563, 538, 542, 496, 497, 510, 529, 531, 521, 499, 547, 555, 541, 548, 559, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 487, 481, 537, 548, 490, 506, 524, 497, 507, 513, 530, 488, 485, 546, 502, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8117, 8155, 8210, 8084, 8108, 8516, 8136] avg=8192.0000 max=8516 max_rank=6 min=8084 min_rank=4 imbalance_ratio=1.039551 max_load_reduction=57020 max_load_reduction_pct=0.870056 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3840, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[515, 534, 527, 473, 514, 514, 504, 483, 505, 527, 505, 500, 528, 493, 473, 507, 553, 468, 517, 471, 517, 526, 504, 538, 493, 522, 489, 507, 532, 514, 492, 546, 538, 495, 529, 504, 500, 487, 516, 537, 526, 510, 506, 479, 483, 511, 505, 560, 563, 506, 500, 477, 536, 504, 529, 504, 527, 505, 523, 505, 514, 520, 498, 492, 536, 538, 522, 487, 489, 526, 501, 486, 502, 564, 533, 526, 494, 496, 567, 524, 533, 522, 503, 532, 539, 556, 483, 528, 518, 476, 501, 495, 484, 495, 557, 482, 518, 534, 528, 476, 515, 486, 510, 493, 483, 498, 547, 476, 539, 521, 541, 542, 563, 490, 478, 485, 528, 510, 549, 467, 523, 541, 491, 495, 515, 478, 509, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[518, 481, 500, 483, 527, 500, 512, 512, 529, 534, 503, 496, 534, 506, 535, 524, 506, 529, 489, 494, 537, 540, 500, 489, 519, 539, 523, 530, 502, 527, 492, 509, 508, 507, 514, 498, 524, 494, 474, 520, 530, 542, 473, 476, 517, 539, 522, 494, 519, 542, 513, 505, 557, 507, 529, 513, 508, 535, 456, 515, 517, 512, 497, 522, 489, 529, 517, 518, 521, 510, 527, 528, 513, 573, 538, 451, 539, 530, 476, 519, 462, 532, 501, 487, 518, 540, 516, 508, 525, 543, 491, 487, 513, 488, 494, 478, 527, 481, 502, 526, 509, 506, 521, 528, 497, 506, 516, 503, 548, 546, 500, 498, 455, 556, 490, 481, 484, 519, 468, 526, 514, 511, 522, 515, 565, 509, 533, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[483, 534, 509, 472, 514, 478, 535, 516, 485, 520, 483, 492, 489, 538, 516, 542, 524, 527, 514, 506, 501, 479, 485, 525, 531, 494, 519, 524, 500, 538, 521, 484, 522, 520, 497, 553, 507, 519, 542, 516, 533, 486, 560, 521, 503, 517, 512, 490, 519, 517, 473, 517, 550, 503, 539, 524, 530, 481, 525, 543, 532, 511, 511, 556, 527, 476, 508, 540, 544, 522, 573, 479, 507, 515, 513, 524, 550, 509, 564, 486, 499, 510, 517, 486, 502, 474, 487, 514, 471, 519, 505, 520, 495, 502, 520, 485, 523, 504, 497, 527, 528, 483, 502, 508, 527, 494, 506, 538, 478, 520, 487, 501, 505, 540, 513, 471, 497, 501, 510, 485, 490, 548, 528, 522, 491, 520, 517, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[520, 491, 502, 524, 511, 540, 519, 513, 500, 517, 500, 496, 503, 522, 519, 510, 538, 518, 529, 523, 497, 517, 492, 509, 521, 522, 519, 507, 475, 520, 503, 493, 540, 531, 515, 534, 517, 527, 530, 514, 514, 525, 524, 548, 497, 521, 518, 523, 518, 551, 520, 504, 504, 504, 509, 499, 503, 503, 559, 503, 526, 472, 521, 495, 537, 510, 524, 510, 536, 521, 525, 518, 497, 508, 536, 511, 534, 523, 511, 515, 504, 484, 516, 474, 530, 531, 501, 528, 516, 498, 533, 481, 515, 531, 526, 543, 513, 473, 493, 535, 517, 497, 520, 508, 538, 477, 481, 537, 513, 494, 521, 507, 444, 477, 488, 508, 521, 529, 522, 499, 501, 478, 523, 490, 469, 516, 505, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[524, 498, 534, 555, 522, 494, 504, 516, 490, 494, 559, 484, 536, 518, 486, 453, 509, 526, 535, 530, 480, 531, 517, 507, 492, 489, 500, 508, 484, 508, 506, 498, 472, 511, 525, 504, 496, 502, 487, 536, 494, 535, 528, 539, 499, 482, 488, 541, 521, 525, 536, 485, 509, 514, 522, 502, 577, 507, 517, 523, 527, 542, 495, 514, 534, 484, 494, 510, 501, 501, 515, 483, 511, 521, 519, 511, 560, 503, 518, 474, 500, 486, 492, 496, 566, 535, 511, 477, 543, 535, 532, 493, 492, 504, 511, 481, 525, 543, 527, 495, 507, 558, 534, 531, 501, 535, 540, 527, 486, 486, 492, 499, 514, 532, 514, 454, 494, 530, 521, 493, 554, 539, 498, 506, 488, 559, 517, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[502, 553, 491, 488, 524, 543, 521, 512, 553, 519, 476, 511, 540, 470, 547, 510, 477, 516, 520, 514, 484, 525, 519, 516, 532, 477, 495, 550, 490, 524, 526, 543, 504, 541, 471, 545, 512, 523, 498, 532, 514, 514, 513, 525, 546, 488, 518, 491, 505, 491, 514, 524, 501, 496, 497, 515, 524, 486, 504, 512, 515, 553, 502, 506, 503, 517, 524, 516, 520, 509, 508, 554, 525, 504, 472, 494, 528, 499, 491, 483, 531, 494, 511, 530, 501, 508, 526, 505, 501, 518, 521, 504, 554, 501, 488, 518, 540, 482, 528, 517, 513, 505, 521, 476, 553, 514, 542, 504, 524, 535, 526, 524, 513, 522, 508, 491, 524, 468, 466, 497, 473, 484, 519, 512, 490, 532, 476, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[541, 482, 466, 495, 502, 528, 502, 532, 473, 557, 544, 556, 504, 491, 529, 491, 490, 505, 518, 496, 499, 479, 555, 532, 515, 542, 501, 519, 541, 501, 501, 501, 561, 492, 500, 514, 535, 496, 530, 540, 514, 507, 479, 493, 513, 524, 514, 500, 509, 515, 500, 498, 556, 536, 505, 460, 541, 489, 488, 529, 514, 535, 535, 508, 513, 510, 493, 526, 513, 515, 511, 525, 531, 498, 512, 522, 521, 508, 534, 536, 505, 516, 511, 485, 521, 513, 530, 480, 503, 533, 510, 501, 511, 488, 460, 470, 521, 489, 546, 511, 504, 526, 533, 521, 517, 479, 520, 529, 512, 494, 537, 497, 527, 490, 507, 554, 525, 496, 512, 486, 497, 494, 504, 517, 507, 517, 510, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[470, 523, 522, 521, 486, 501, 511, 519, 533, 497, 493, 540, 525, 498, 529, 548, 511, 550, 523, 495, 543, 557, 496, 492, 499, 571, 495, 499, 533, 527, 476, 517, 524, 530, 469, 518, 551, 496, 513, 524, 518, 537, 550, 491, 526, 483, 513, 526, 551, 522, 501, 482, 535, 499, 526, 493, 528, 480, 492, 537, 513, 489, 512, 506, 530, 493, 521, 505, 506, 550, 495, 498, 522, 530, 527, 529, 510, 533, 536, 481, 506, 524, 505, 476, 465, 517, 484, 528, 524, 501, 496, 535, 483, 517, 563, 528, 565, 516, 487, 469, 545, 494, 509, 484, 490, 501, 486, 461, 508, 508, 492, 519, 510, 532, 541, 479, 472, 518, 505, 526, 498, 489, 510, 513, 549, 507, 498, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[515, 534, 527, 473, 514, 514, 504, 483, 505, 527, 505, 500, 528, 493, 473, 507, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 468, 517, 471, 517, 526, 504, 538, 493, 522, 489, 507, 532, 511, 492, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 495, 529, 504, 500, 487, 516, 498, 526, 510, 506, 479, 483, 511, 505, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 506, 500, 477, 502, 504, 529, 504, 527, 505, 523, 505, 514, 520, 498, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 538, 522, 487, 489, 526, 501, 486, 502, 564, 533, 526, 435, 496, 567, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 522, 503, 532, 539, 556, 483, 528, 518, 476, 501, 495, 484, 495, 557, 482, 35, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 528, 476, 515, 486, 510, 493, 483, 498, 547, 476, 539, 521, 541, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 490, 478, 485, 528, 510, 549, 467, 523, 541, 491, 495, 515, 478, 509, 532, 59, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8186, 8147, 8169, 8232, 8264, 8207, 8217] avg=8192.0000 max=8264 max_rank=5 min=8114 min_rank=0 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[518, 481, 500, 483, 527, 500, 512, 512, 529, 534, 503, 496, 534, 506, 535, 524, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 489, 494, 537, 540, 500, 489, 519, 539, 523, 530, 502, 522, 492, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 514, 498, 524, 494, 474, 482, 530, 542, 473, 476, 517, 539, 522, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 542, 513, 505, 521, 507, 529, 513, 508, 535, 456, 515, 517, 512, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 529, 517, 518, 521, 510, 527, 528, 513, 573, 538, 451, 475, 530, 476, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 532, 501, 487, 518, 540, 516, 508, 525, 543, 491, 487, 513, 488, 494, 478, 34, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 481, 502, 526, 509, 506, 521, 528, 497, 506, 516, 503, 548, 546, 500, 498, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 556, 490, 481, 484, 519, 468, 526, 514, 511, 522, 515, 565, 509, 533, 515, 64, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8220, 8094, 8211, 8214, 8143, 8215, 8231] avg=8192.0000 max=8231 max_rank=7 min=8094 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[520, 491, 502, 524, 511, 540, 519, 513, 500, 517, 500, 496, 503, 522, 519, 510, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 518, 529, 523, 497, 517, 492, 509, 521, 522, 519, 507, 475, 515, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 531, 515, 534, 517, 527, 530, 477, 514, 525, 524, 548, 497, 521, 518, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 551, 520, 504, 472, 504, 509, 499, 503, 503, 559, 503, 526, 472, 521, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 524, 510, 536, 521, 525, 518, 497, 508, 536, 511, 471, 523, 511, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 484, 516, 474, 530, 531, 501, 528, 516, 498, 533, 481, 515, 531, 526, 543, 33, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 473, 493, 535, 517, 497, 520, 508, 538, 477, 481, 537, 513, 494, 521, 507, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 477, 488, 508, 521, 529, 522, 499, 501, 478, 523, 490, 469, 516, 505, 476, 63, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8178, 8341, 8159, 8253, 8267, 8125, 8013] avg=8192.0000 max=8341 max_rank=2 min=8013 min_rank=7 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[483, 534, 509, 472, 514, 478, 535, 516, 485, 520, 483, 492, 489, 538, 516, 542, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 514, 506, 501, 479, 485, 525, 531, 494, 519, 524, 500, 533, 521, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 497, 553, 507, 519, 542, 479, 533, 486, 560, 521, 503, 517, 512, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 517, 473, 517, 514, 503, 539, 524, 530, 481, 525, 543, 532, 511, 511, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 476, 508, 540, 544, 522, 573, 479, 507, 515, 513, 524, 485, 509, 564, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 510, 517, 486, 502, 474, 487, 514, 471, 519, 505, 520, 495, 502, 520, 485, 33, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 504, 497, 527, 528, 483, 502, 508, 527, 494, 506, 538, 478, 520, 487, 501, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 540, 513, 471, 497, 501, 510, 485, 490, 548, 528, 522, 491, 520, 517, 525, 65, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8167, 8261, 8295, 8272, 8065, 8124, 8232] avg=8192.0000 max=8295 max_rank=3 min=8065 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[524, 498, 534, 555, 522, 494, 504, 516, 490, 494, 559, 484, 536, 518, 486, 453, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 535, 530, 480, 531, 517, 507, 492, 489, 500, 508, 484, 505, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 511, 525, 504, 496, 502, 487, 497, 494, 535, 528, 539, 499, 482, 488, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 525, 536, 485, 476, 514, 522, 502, 577, 507, 517, 523, 527, 542, 495, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 484, 494, 510, 501, 501, 515, 483, 511, 521, 519, 511, 494, 503, 518, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 486, 492, 496, 566, 535, 511, 477, 543, 535, 532, 493, 492, 504, 511, 481, 35, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 543, 527, 495, 507, 558, 534, 531, 501, 535, 540, 527, 486, 486, 492, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 532, 514, 454, 494, 530, 521, 493, 554, 539, 498, 506, 488, 559, 517, 502, 66, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8117, 8100, 8283, 8073, 8213, 8286, 8285] avg=8192.0000 max=8286 max_rank=6 min=8073 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[541, 482, 466, 495, 502, 528, 502, 532, 473, 557, 544, 556, 504, 491, 529, 491, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 518, 496, 499, 479, 555, 532, 515, 542, 501, 519, 541, 492, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 492, 500, 514, 535, 496, 530, 493, 514, 507, 479, 493, 513, 524, 514, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 515, 500, 498, 512, 536, 505, 460, 541, 489, 488, 529, 514, 535, 535, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 493, 526, 513, 515, 511, 525, 531, 498, 512, 522, 455, 508, 534, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 511, 485, 521, 513, 530, 480, 503, 533, 510, 501, 511, 488, 460, 470, 38, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 489, 546, 511, 504, 526, 533, 521, 517, 479, 520, 529, 512, 494, 537, 497, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 490, 507, 554, 525, 496, 512, 486, 497, 494, 504, 517, 507, 517, 510, 534, 66, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8186, 8165, 8174, 8202, 8105, 8239, 8252] avg=8192.0000 max=8252 max_rank=7 min=8105 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[502, 553, 491, 488, 524, 543, 521, 512, 553, 519, 476, 511, 540, 470, 547, 510, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 516, 520, 514, 484, 525, 519, 516, 532, 477, 495, 550, 490, 519, 526, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 541, 471, 545, 512, 523, 498, 494, 514, 514, 513, 525, 546, 488, 518, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 491, 514, 524, 469, 496, 497, 515, 524, 486, 504, 512, 515, 553, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 524, 516, 520, 509, 508, 554, 525, 504, 472, 494, 465, 499, 491, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 494, 511, 530, 501, 508, 526, 505, 501, 518, 521, 504, 554, 501, 488, 518, 34, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 482, 528, 517, 513, 505, 521, 476, 553, 514, 542, 504, 524, 535, 526, 524, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 508, 491, 524, 468, 466, 497, 473, 484, 519, 512, 490, 532, 476, 551, 63, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8203, 8197, 8113, 8084, 8268, 8305, 8093] avg=8192.0000 max=8305 max_rank=6 min=8084 min_rank=4 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[470, 523, 522, 521, 486, 501, 511, 519, 533, 497, 493, 540, 525, 498, 529, 548, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 550, 523, 495, 543, 557, 496, 492, 499, 571, 495, 499, 533, 522, 476, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 530, 469, 518, 551, 496, 513, 486, 518, 537, 550, 491, 526, 483, 513, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 522, 501, 482, 501, 499, 526, 493, 528, 480, 492, 537, 513, 489, 512, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 493, 521, 505, 506, 550, 495, 498, 522, 530, 527, 529, 450, 533, 536, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 524, 505, 476, 465, 517, 484, 528, 524, 501, 496, 535, 483, 517, 563, 528, 34, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 516, 487, 469, 545, 494, 509, 484, 490, 501, 486, 461, 508, 508, 492, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 532, 541, 479, 472, 518, 505, 526, 498, 489, 510, 513, 549, 507, 498, 502, 60, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8279, 8231, 8132, 8206, 8211, 8035, 8213] avg=8192.0000 max=8279 max_rank=1 min=8035 min_rank=6 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 3968, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 3840, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4352, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True + [2026-08-26 16:29:48] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 413.8 | throughput per GPU (TFLOP/s/GPU): 437.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.794577E+00 | loss scale: 1.0 | grad norm: 5.602 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[510, 496, 502, 545, 487, 513, 433, 513, 500, 475, 531, 524, 504, 534, 529, 495, 481, 504, 536, 523, 506, 532, 553, 498, 500, 504, 506, 517, 528, 502, 519, 483, 500, 530, 533, 525, 512, 479, 533, 517, 538, 533, 495, 499, 515, 495, 524, 518, 494, 514, 504, 518, 555, 528, 551, 524, 492, 490, 455, 552, 519, 523, 536, 566, 552, 555, 496, 515, 521, 488, 490, 551, 488, 485, 480, 515, 512, 495, 500, 537, 500, 482, 513, 520, 537, 468, 518, 481, 499, 507, 518, 523, 529, 539, 482, 527, 473, 492, 524, 514, 535, 522, 545, 538, 486, 523, 476, 492, 527, 491, 530, 544, 519, 531, 492, 497, 494, 488, 512, 517, 467, 553, 485, 522, 509, 527, 504, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[465, 541, 530, 517, 542, 521, 503, 512, 504, 515, 512, 513, 498, 509, 488, 500, 523, 482, 523, 500, 514, 512, 498, 538, 487, 523, 520, 459, 541, 523, 519, 508, 490, 536, 526, 510, 537, 504, 526, 576, 515, 506, 476, 508, 501, 531, 513, 529, 500, 494, 510, 528, 533, 550, 521, 495, 519, 511, 525, 508, 487, 534, 487, 557, 520, 533, 503, 495, 549, 501, 516, 508, 475, 485, 494, 529, 485, 450, 496, 514, 501, 515, 536, 503, 502, 491, 491, 516, 484, 529, 511, 544, 535, 541, 469, 538, 465, 473, 538, 502, 549, 521, 504, 515, 507, 510, 530, 505, 537, 479, 501, 507, 551, 496, 521, 510, 524, 528, 533, 517, 520, 541, 513, 500, 494, 511, 512, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[558, 546, 507, 491, 563, 489, 497, 495, 517, 508, 530, 492, 477, 533, 512, 502, 500, 523, 494, 527, 472, 505, 510, 521, 538, 515, 479, 488, 534, 509, 521, 518, 514, 451, 475, 513, 552, 537, 527, 496, 549, 512, 527, 490, 490, 470, 530, 488, 504, 532, 520, 511, 539, 539, 487, 464, 541, 525, 503, 538, 518, 503, 501, 491, 484, 526, 524, 521, 504, 478, 505, 519, 545, 512, 515, 518, 509, 511, 529, 513, 549, 493, 521, 534, 523, 529, 519, 521, 532, 528, 530, 516, 513, 544, 468, 508, 493, 503, 510, 510, 520, 511, 537, 553, 514, 502, 510, 504, 505, 506, 495, 523, 494, 495, 523, 458, 527, 527, 512, 524, 518, 500, 491, 474, 533, 474, 514, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[516, 525, 501, 499, 509, 524, 464, 491, 511, 500, 518, 476, 499, 541, 504, 509, 492, 502, 502, 507, 471, 549, 515, 573, 472, 516, 465, 497, 505, 531, 556, 486, 522, 533, 526, 528, 499, 514, 499, 505, 510, 519, 484, 513, 495, 521, 506, 477, 526, 545, 512, 493, 539, 534, 505, 466, 548, 540, 511, 538, 565, 504, 512, 549, 465, 524, 500, 487, 497, 547, 502, 507, 541, 519, 507, 531, 508, 496, 485, 509, 466, 520, 540, 510, 486, 539, 509, 494, 512, 531, 491, 530, 515, 472, 524, 529, 519, 504, 546, 538, 489, 552, 483, 498, 521, 503, 481, 525, 504, 555, 529, 507, 520, 523, 491, 530, 476, 553, 502, 528, 501, 503, 501, 561, 530, 492, 527, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[510, 526, 528, 519, 559, 500, 500, 484, 529, 514, 487, 510, 511, 513, 480, 507, 532, 530, 532, 473, 516, 463, 511, 507, 528, 516, 506, 489, 523, 509, 485, 544, 485, 509, 482, 553, 557, 481, 526, 548, 514, 520, 553, 524, 496, 501, 560, 514, 490, 505, 510, 510, 546, 503, 476, 541, 516, 515, 476, 494, 533, 503, 556, 501, 516, 487, 550, 531, 526, 545, 530, 522, 510, 513, 546, 488, 489, 486, 526, 521, 528, 509, 500, 507, 497, 530, 485, 526, 511, 504, 502, 496, 476, 466, 529, 487, 496, 524, 498, 478, 489, 530, 503, 497, 497, 525, 516, 520, 536, 503, 533, 523, 542, 517, 526, 511, 503, 496, 505, 519, 510, 497, 476, 519, 562, 513, 507, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[523, 452, 528, 510, 499, 490, 521, 524, 469, 464, 554, 526, 547, 495, 517, 477, 520, 497, 521, 525, 513, 515, 502, 473, 486, 468, 495, 510, 481, 517, 541, 494, 537, 507, 545, 510, 564, 516, 526, 504, 487, 517, 498, 513, 525, 486, 522, 532, 531, 509, 487, 499, 512, 500, 503, 511, 581, 514, 502, 511, 487, 515, 529, 522, 525, 500, 529, 517, 525, 509, 506, 505, 509, 553, 522, 492, 534, 519, 505, 515, 515, 550, 507, 533, 520, 545, 543, 530, 490, 477, 537, 501, 531, 504, 497, 523, 530, 510, 469, 480, 514, 502, 501, 474, 545, 482, 544, 511, 501, 558, 536, 515, 493, 502, 515, 534, 514, 522, 463, 512, 526, 501, 513, 485, 516, 506, 536, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[524, 520, 511, 512, 540, 509, 505, 522, 511, 540, 513, 521, 478, 559, 500, 525, 507, 525, 521, 475, 533, 503, 474, 506, 512, 495, 522, 484, 488, 510, 501, 483, 489, 501, 513, 509, 524, 523, 509, 530, 499, 462, 524, 508, 530, 500, 513, 540, 515, 488, 499, 505, 502, 466, 541, 505, 525, 538, 523, 480, 540, 516, 525, 495, 528, 533, 488, 563, 515, 508, 505, 505, 482, 538, 497, 568, 474, 509, 512, 513, 544, 543, 492, 465, 506, 502, 488, 528, 485, 539, 512, 514, 528, 516, 515, 529, 509, 511, 528, 494, 521, 510, 525, 498, 498, 514, 528, 532, 533, 481, 515, 501, 537, 523, 494, 505, 526, 500, 474, 487, 515, 507, 543, 524, 513, 535, 514, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[527, 542, 501, 545, 515, 504, 522, 517, 492, 500, 506, 509, 518, 533, 556, 511, 551, 527, 504, 453, 509, 508, 511, 555, 535, 471, 531, 486, 520, 518, 507, 508, 494, 530, 514, 483, 523, 514, 508, 523, 514, 495, 560, 487, 505, 527, 545, 498, 484, 503, 522, 488, 510, 503, 516, 492, 501, 550, 523, 489, 473, 505, 537, 534, 474, 504, 513, 512, 529, 535, 480, 490, 528, 490, 516, 529, 501, 507, 497, 511, 487, 514, 497, 550, 525, 467, 516, 535, 499, 518, 520, 525, 536, 531, 467, 526, 496, 495, 514, 520, 507, 516, 504, 503, 560, 531, 497, 522, 493, 513, 543, 499, 483, 497, 543, 485, 509, 524, 503, 504, 545, 501, 478, 509, 512, 536, 494, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[558, 546, 507, 491, 563, 489, 497, 495, 517, 508, 530, 492, 477, 533, 512, 502, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 494, 527, 472, 505, 510, 521, 538, 515, 479, 488, 534, 509, 521, 518, 41, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 451, 475, 513, 518, 537, 527, 496, 549, 512, 527, 490, 490, 470, 530, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 532, 520, 511, 498, 539, 487, 464, 541, 525, 503, 538, 518, 503, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 526, 524, 521, 504, 478, 505, 519, 545, 512, 515, 518, 509, 511, 529, 513, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 493, 521, 534, 523, 529, 519, 521, 532, 528, 530, 516, 513, 544, 468, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 503, 510, 510, 520, 511, 537, 553, 514, 502, 510, 504, 505, 506, 493, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 495, 523, 458, 527, 527, 512, 524, 518, 500, 491, 474, 533, 474, 514, 527, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8207, 8087, 8175, 8219, 8319, 8194, 8104] avg=8192.0000 max=8319 max_rank=5 min=8087 min_rank=2 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[465, 541, 530, 517, 542, 521, 503, 512, 504, 515, 512, 513, 498, 509, 488, 500, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 482, 523, 500, 514, 512, 498, 538, 487, 523, 520, 459, 541, 523, 519, 508, 41, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 536, 526, 510, 504, 504, 526, 576, 515, 506, 476, 508, 501, 531, 513, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 494, 510, 528, 492, 550, 521, 495, 519, 511, 525, 508, 487, 534, 487, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 533, 503, 495, 549, 501, 516, 508, 475, 485, 494, 529, 485, 450, 496, 514, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 536, 503, 502, 491, 491, 516, 484, 529, 511, 544, 535, 541, 469, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 473, 538, 502, 549, 521, 504, 515, 507, 510, 530, 505, 537, 479, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 496, 521, 510, 524, 528, 533, 517, 520, 541, 513, 500, 494, 511, 512, 480, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8223, 8251, 8218, 8060, 8196, 8141, 8264] avg=8192.0000 max=8264 max_rank=7 min=8060 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 496, 502, 545, 487, 513, 433, 513, 500, 475, 531, 524, 504, 534, 529, 495, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 536, 523, 506, 532, 553, 498, 500, 504, 506, 517, 528, 502, 519, 483, 42, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 530, 533, 525, 481, 479, 533, 517, 538, 533, 495, 499, 515, 495, 524, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 514, 504, 518, 513, 528, 551, 524, 492, 490, 455, 552, 519, 523, 536, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 555, 496, 515, 521, 488, 490, 551, 488, 485, 480, 515, 512, 495, 500, 537, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 482, 513, 520, 537, 468, 518, 481, 499, 507, 518, 523, 529, 539, 482, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 492, 524, 514, 535, 522, 545, 538, 486, 523, 476, 492, 527, 491, 528, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 492, 497, 494, 488, 512, 517, 467, 553, 485, 522, 509, 527, 504, 534, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8245, 8215, 8279, 8187, 8133, 8210, 8163] avg=8192.0000 max=8279 max_rank=3 min=8104 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[516, 525, 501, 499, 509, 524, 464, 491, 511, 500, 518, 476, 499, 541, 504, 509, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 502, 502, 507, 471, 549, 515, 573, 472, 516, 465, 497, 505, 531, 556, 486, 41, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 533, 526, 528, 469, 514, 499, 505, 510, 519, 484, 513, 495, 521, 506, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 545, 512, 493, 498, 534, 505, 466, 548, 540, 511, 538, 565, 504, 512, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 524, 500, 487, 497, 547, 502, 507, 541, 519, 507, 531, 508, 496, 485, 509, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 520, 540, 510, 486, 539, 509, 494, 512, 531, 491, 530, 515, 472, 524, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 504, 546, 538, 489, 552, 483, 498, 521, 503, 481, 525, 504, 555, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 491, 530, 476, 553, 502, 528, 501, 503, 501, 561, 530, 492, 527, 487, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8191, 8121, 8346, 8132, 8158, 8252, 8237] avg=8192.0000 max=8346 max_rank=3 min=8099 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 452, 528, 510, 499, 490, 521, 524, 469, 464, 554, 526, 547, 495, 517, 477, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 497, 521, 525, 513, 515, 502, 473, 486, 468, 495, 510, 481, 517, 541, 494, 44, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 545, 510, 518, 516, 526, 504, 487, 517, 498, 513, 525, 486, 522, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 487, 499, 468, 500, 503, 511, 581, 514, 502, 511, 487, 515, 529, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 500, 529, 517, 525, 509, 506, 505, 509, 553, 522, 492, 534, 519, 505, 515, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 550, 507, 533, 520, 545, 543, 530, 490, 477, 537, 501, 531, 504, 497, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 510, 469, 480, 514, 502, 501, 474, 545, 482, 544, 511, 501, 558, 530, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 515, 534, 514, 522, 463, 512, 526, 501, 513, 485, 516, 506, 536, 502, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8118, 8243, 8169, 8278, 8287, 8166, 8161] avg=8192.0000 max=8287 max_rank=5 min=8114 min_rank=0 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 526, 528, 519, 559, 500, 500, 484, 529, 514, 487, 510, 511, 513, 480, 507, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 530, 532, 473, 516, 463, 511, 507, 528, 516, 506, 489, 523, 509, 485, 544, 42, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 509, 482, 553, 523, 481, 526, 548, 514, 520, 553, 524, 496, 501, 560, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 510, 510, 504, 503, 476, 541, 516, 515, 476, 494, 533, 503, 556, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 487, 550, 531, 526, 545, 530, 522, 510, 513, 546, 488, 489, 486, 526, 521, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 509, 500, 507, 497, 530, 485, 526, 511, 504, 502, 496, 476, 466, 529, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 498, 478, 489, 530, 503, 497, 497, 525, 516, 520, 536, 503, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 517, 526, 511, 503, 496, 505, 519, 510, 497, 476, 519, 562, 513, 507, 487, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8218, 8289, 8133, 8292, 8044, 8166, 8203] avg=8192.0000 max=8292 max_rank=4 min=8044 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[527, 542, 501, 545, 515, 504, 522, 517, 492, 500, 506, 509, 518, 533, 556, 511, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 527, 504, 453, 509, 508, 511, 555, 535, 471, 531, 486, 520, 518, 507, 508, 39, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 530, 514, 483, 491, 514, 508, 523, 514, 495, 560, 487, 505, 527, 545, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 503, 522, 488, 471, 503, 516, 492, 501, 550, 523, 489, 473, 505, 537, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 504, 513, 512, 529, 535, 480, 490, 528, 490, 516, 529, 501, 507, 497, 511, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 514, 497, 550, 525, 467, 516, 535, 499, 518, 520, 525, 536, 531, 467, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 514, 520, 507, 516, 504, 503, 560, 531, 497, 522, 493, 513, 541, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 497, 543, 485, 509, 524, 503, 504, 545, 501, 478, 509, 512, 536, 494, 529, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8245, 8188, 8091, 8123, 8203, 8211, 8164] avg=8192.0000 max=8311 max_rank=0 min=8091 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[524, 520, 511, 512, 540, 509, 505, 522, 511, 540, 513, 521, 478, 559, 500, 525, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 521, 475, 533, 503, 474, 506, 512, 495, 522, 484, 488, 510, 501, 483, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 501, 513, 509, 492, 523, 509, 530, 499, 462, 524, 508, 530, 500, 513, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 488, 499, 505, 464, 466, 541, 505, 525, 538, 523, 480, 540, 516, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 533, 488, 563, 515, 508, 505, 505, 482, 538, 497, 568, 474, 509, 512, 513, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 543, 492, 465, 506, 502, 488, 528, 485, 539, 512, 514, 528, 516, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 511, 528, 494, 521, 510, 525, 498, 498, 514, 528, 532, 533, 481, 513, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 523, 494, 505, 526, 500, 474, 487, 515, 507, 543, 524, 513, 535, 514, 531, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8089, 8142, 8125, 8245, 8196, 8196, 8240] avg=8192.0000 max=8303 max_rank=0 min=8089 min_rank=1 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[514, 512, 501, 573, 503, 533, 559, 509, 511, 489, 509, 543, 470, 506, 512, 500, 521, 498, 509, 492, 515, 551, 496, 520, 510, 536, 490, 500, 540, 502, 531, 513, 524, 529, 511, 461, 505, 525, 531, 470, 495, 491, 470, 507, 520, 540, 491, 512, 522, 531, 504, 523, 524, 506, 486, 532, 532, 535, 479, 524, 521, 542, 490, 543, 478, 515, 480, 490, 519, 522, 508, 486, 544, 526, 498, 494, 530, 504, 564, 476, 492, 521, 525, 499, 495, 566, 509, 506, 515, 531, 523, 513, 529, 502, 563, 496, 520, 482, 518, 496, 506, 504, 500, 519, 504, 538, 474, 516, 502, 519, 529, 504, 491, 541, 495, 510, 513, 521, 505, 510, 532, 451, 529, 509, 532, 514, 484, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[513, 483, 499, 483, 523, 520, 522, 513, 516, 525, 576, 471, 519, 517, 496, 478, 481, 499, 527, 496, 522, 495, 519, 513, 528, 553, 548, 510, 506, 498, 454, 510, 483, 483, 516, 500, 526, 495, 505, 544, 527, 479, 506, 499, 541, 460, 506, 514, 500, 502, 525, 551, 516, 533, 541, 535, 505, 494, 485, 511, 499, 509, 523, 506, 539, 500, 496, 516, 505, 536, 541, 503, 475, 517, 532, 494, 535, 520, 508, 534, 537, 526, 503, 522, 508, 480, 487, 485, 517, 523, 542, 520, 520, 505, 515, 504, 559, 529, 525, 493, 495, 541, 487, 520, 517, 507, 489, 526, 499, 488, 513, 537, 546, 491, 510, 497, 498, 528, 530, 524, 489, 509, 539, 496, 523, 513, 537, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[494, 487, 506, 544, 549, 499, 535, 508, 522, 524, 502, 511, 510, 476, 512, 502, 502, 521, 493, 565, 498, 495, 490, 506, 564, 522, 509, 527, 503, 508, 553, 534, 525, 529, 554, 529, 504, 493, 515, 541, 517, 509, 488, 512, 515, 464, 495, 519, 519, 492, 506, 517, 500, 525, 517, 548, 522, 505, 517, 500, 495, 494, 506, 540, 491, 545, 501, 510, 484, 542, 516, 533, 538, 526, 473, 549, 538, 500, 493, 528, 519, 500, 524, 516, 490, 483, 505, 534, 507, 476, 489, 562, 484, 487, 530, 528, 439, 537, 493, 498, 487, 522, 505, 513, 509, 514, 520, 529, 527, 495, 503, 542, 483, 489, 510, 494, 541, 535, 537, 506, 527, 488, 498, 508, 489, 506, 486, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[536, 516, 520, 508, 507, 522, 502, 488, 543, 500, 498, 492, 518, 497, 511, 528, 494, 505, 510, 535, 501, 487, 501, 521, 512, 514, 473, 488, 519, 531, 536, 492, 543, 521, 482, 517, 519, 473, 490, 532, 509, 540, 537, 545, 477, 547, 522, 499, 491, 531, 464, 504, 502, 500, 513, 530, 503, 495, 561, 551, 522, 505, 509, 558, 521, 503, 525, 522, 499, 528, 489, 503, 465, 535, 509, 543, 501, 525, 531, 513, 513, 484, 481, 523, 561, 489, 527, 504, 523, 520, 524, 498, 498, 515, 522, 516, 494, 495, 516, 521, 506, 573, 511, 491, 548, 520, 492, 554, 547, 514, 510, 534, 475, 501, 519, 499, 484, 481, 488, 525, 464, 510, 470, 481, 532, 508, 521, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[493, 485, 511, 502, 486, 496, 533, 475, 542, 507, 490, 518, 496, 494, 521, 535, 510, 487, 526, 497, 509, 514, 508, 545, 503, 506, 526, 489, 538, 535, 487, 532, 473, 539, 513, 525, 524, 511, 516, 512, 504, 504, 504, 502, 560, 512, 544, 506, 515, 526, 511, 530, 511, 502, 531, 492, 490, 518, 535, 508, 499, 509, 521, 510, 542, 498, 484, 497, 486, 495, 519, 517, 510, 501, 535, 537, 506, 517, 532, 532, 519, 485, 488, 526, 535, 517, 513, 533, 524, 535, 521, 517, 504, 516, 493, 528, 499, 487, 495, 489, 507, 528, 497, 507, 521, 487, 493, 525, 518, 522, 513, 514, 485, 519, 557, 537, 476, 530, 502, 484, 518, 539, 486, 537, 493, 509, 544, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[449, 508, 486, 546, 502, 514, 529, 492, 492, 500, 517, 533, 489, 554, 471, 523, 499, 534, 490, 517, 538, 480, 534, 524, 492, 543, 494, 532, 505, 494, 521, 506, 552, 511, 509, 533, 517, 496, 532, 503, 518, 484, 525, 515, 491, 505, 549, 536, 502, 530, 529, 495, 538, 538, 519, 497, 494, 518, 517, 524, 466, 501, 539, 512, 527, 537, 514, 492, 531, 480, 522, 505, 543, 495, 492, 519, 505, 487, 489, 510, 486, 502, 503, 518, 504, 516, 548, 515, 524, 508, 547, 508, 488, 506, 495, 523, 518, 512, 495, 511, 495, 509, 533, 547, 519, 508, 530, 550, 507, 544, 497, 480, 510, 520, 537, 486, 506, 526, 543, 502, 517, 487, 550, 463, 508, 534, 475, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[508, 551, 511, 554, 491, 496, 488, 555, 522, 504, 513, 496, 538, 517, 512, 508, 514, 470, 508, 531, 550, 559, 529, 512, 525, 515, 491, 496, 526, 538, 536, 509, 522, 515, 531, 554, 487, 472, 507, 514, 514, 470, 514, 509, 524, 478, 579, 492, 515, 444, 511, 521, 529, 539, 508, 465, 529, 515, 544, 508, 504, 519, 518, 481, 502, 485, 529, 581, 504, 527, 499, 473, 487, 524, 503, 508, 513, 501, 478, 503, 550, 536, 520, 518, 548, 506, 511, 509, 506, 541, 485, 506, 520, 512, 539, 520, 512, 518, 522, 520, 533, 494, 509, 517, 458, 508, 480, 555, 522, 506, 491, 514, 495, 528, 520, 512, 475, 525, 479, 566, 470, 505, 472, 496, 510, 520, 509, 446] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[491, 509, 548, 522, 518, 509, 559, 524, 481, 459, 500, 507, 522, 547, 599, 481, 520, 499, 509, 506, 522, 522, 508, 485, 526, 518, 476, 487, 499, 527, 516, 504, 493, 503, 512, 485, 543, 505, 517, 474, 531, 542, 516, 478, 520, 539, 510, 513, 539, 504, 523, 502, 486, 541, 512, 488, 483, 505, 470, 464, 523, 531, 495, 513, 502, 532, 453, 547, 492, 517, 536, 506, 501, 505, 560, 498, 520, 526, 516, 506, 509, 522, 526, 484, 491, 527, 510, 528, 522, 540, 500, 534, 492, 525, 498, 514, 475, 507, 508, 476, 540, 511, 503, 502, 531, 518, 547, 506, 534, 472, 506, 494, 495, 506, 576, 507, 521, 516, 545, 503, 507, 466, 532, 540, 527, 517, 522, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 512, 501, 573, 503, 533, 559, 509, 511, 489, 509, 543, 470, 506, 512, 500, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 498, 509, 492, 515, 551, 496, 520, 510, 524, 490, 500, 540, 502, 531, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 529, 511, 461, 505, 525, 531, 470, 495, 491, 470, 507, 520, 540, 491, 512, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 531, 504, 523, 524, 496, 486, 532, 532, 535, 479, 524, 521, 542, 490, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 480, 488, 519, 522, 508, 486, 544, 526, 498, 494, 530, 504, 564, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, 525, 499, 495, 566, 509, 506, 515, 497, 523, 513, 529, 502, 563, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 482, 518, 496, 506, 504, 500, 519, 504, 538, 474, 516, 502, 519, 529, 504, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 541, 495, 510, 513, 521, 505, 510, 532, 451, 529, 509, 532, 514, 484, 505, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8212, 8084, 8284, 8132, 8251, 8139, 8185] avg=8192.0000 max=8284 max_rank=3 min=8084 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[536, 516, 520, 508, 507, 522, 502, 488, 543, 500, 498, 492, 518, 497, 511, 528, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 505, 510, 535, 501, 487, 501, 521, 512, 502, 473, 488, 519, 531, 536, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 521, 482, 517, 519, 473, 490, 532, 509, 540, 537, 545, 477, 547, 522, 499, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 531, 464, 504, 502, 490, 513, 530, 503, 495, 561, 551, 522, 505, 509, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 525, 520, 499, 528, 489, 503, 465, 535, 509, 543, 501, 525, 531, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 484, 481, 523, 561, 489, 527, 504, 523, 487, 524, 498, 498, 515, 522, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 495, 516, 521, 506, 573, 511, 491, 548, 520, 492, 554, 547, 514, 510, 534, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 501, 519, 499, 484, 481, 488, 525, 464, 510, 470, 481, 532, 508, 521, 545, 33, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8107, 8255, 8229, 8210, 8165, 8334, 8045] avg=8192.0000 max=8334 max_rank=6 min=8045 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 487, 506, 544, 549, 499, 535, 508, 522, 524, 502, 511, 510, 476, 512, 502, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 521, 493, 565, 498, 495, 490, 506, 564, 510, 509, 527, 503, 508, 553, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 529, 554, 529, 504, 493, 515, 541, 517, 509, 488, 512, 515, 464, 495, 519, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 492, 506, 517, 500, 515, 517, 548, 522, 505, 517, 500, 495, 494, 506, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 545, 501, 508, 484, 542, 516, 533, 538, 526, 473, 549, 538, 500, 493, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 524, 516, 490, 483, 505, 534, 507, 446, 489, 562, 484, 487, 530, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 537, 493, 498, 487, 522, 505, 513, 509, 514, 520, 529, 527, 495, 503, 542, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 489, 510, 494, 541, 535, 537, 506, 527, 488, 498, 508, 489, 506, 486, 522, 30, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8278, 8211, 8193, 8265, 8104, 8141, 8158] avg=8192.0000 max=8278 max_rank=1 min=8104 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 483, 499, 483, 523, 520, 522, 513, 516, 525, 576, 471, 519, 517, 496, 478, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 499, 527, 496, 522, 495, 519, 513, 528, 541, 548, 510, 506, 498, 454, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 483, 516, 500, 526, 495, 505, 544, 527, 479, 506, 499, 541, 460, 506, 514, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 525, 551, 516, 523, 541, 535, 505, 494, 485, 511, 499, 509, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 500, 496, 514, 505, 536, 541, 503, 475, 517, 532, 494, 535, 520, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 526, 503, 522, 508, 480, 487, 485, 517, 490, 542, 520, 520, 505, 515, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 529, 525, 493, 495, 541, 487, 520, 517, 507, 489, 526, 499, 488, 513, 537, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 491, 510, 497, 498, 528, 530, 524, 489, 509, 539, 496, 523, 513, 537, 504, 33, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8147, 8086, 8225, 8249, 8161, 8233, 8276] avg=8192.0000 max=8276 max_rank=7 min=8086 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[449, 508, 486, 546, 502, 514, 529, 492, 492, 500, 517, 533, 489, 554, 471, 523, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 490, 517, 538, 480, 534, 524, 492, 523, 494, 532, 505, 494, 521, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 511, 509, 533, 517, 496, 532, 503, 518, 484, 525, 515, 491, 505, 549, 536, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 530, 529, 495, 538, 519, 519, 497, 494, 518, 517, 524, 466, 501, 539, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 537, 514, 488, 531, 480, 522, 505, 543, 495, 492, 519, 505, 487, 489, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 503, 518, 504, 516, 548, 515, 524, 471, 547, 508, 488, 506, 495, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 495, 511, 495, 509, 533, 547, 519, 508, 530, 550, 507, 544, 497, 480, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 537, 486, 506, 526, 543, 502, 517, 487, 550, 463, 508, 534, 475, 475, 37, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8183, 8282, 8200, 8144, 8154, 8272, 8189] avg=8192.0000 max=8282 max_rank=2 min=8112 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[493, 485, 511, 502, 486, 496, 533, 475, 542, 507, 490, 518, 496, 494, 521, 535, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 526, 497, 509, 514, 508, 545, 503, 494, 526, 489, 538, 535, 487, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 539, 513, 525, 524, 511, 516, 512, 504, 504, 504, 502, 560, 512, 544, 506, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 526, 511, 530, 511, 492, 531, 492, 490, 518, 535, 508, 499, 509, 521, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 498, 484, 495, 486, 495, 519, 517, 510, 501, 535, 537, 506, 517, 532, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 485, 488, 526, 535, 517, 513, 533, 524, 501, 521, 517, 504, 516, 493, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 487, 495, 489, 507, 528, 497, 507, 521, 487, 493, 525, 518, 522, 513, 514, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 519, 557, 537, 476, 530, 502, 484, 518, 539, 486, 537, 493, 509, 544, 503, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8200, 8251, 8198, 8206, 8220, 8110, 8262] avg=8192.0000 max=8262 max_rank=7 min=8089 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 551, 511, 554, 491, 496, 488, 555, 522, 504, 513, 496, 538, 517, 512, 508, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 470, 508, 531, 550, 559, 529, 512, 525, 503, 491, 496, 526, 538, 536, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 515, 531, 554, 487, 472, 507, 514, 514, 470, 514, 509, 524, 478, 579, 492, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 444, 511, 521, 529, 528, 508, 465, 529, 515, 544, 508, 504, 519, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 485, 529, 579, 504, 527, 499, 473, 487, 524, 503, 508, 513, 501, 478, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 536, 520, 518, 548, 506, 511, 509, 506, 507, 485, 506, 520, 512, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 522, 520, 533, 494, 509, 517, 458, 508, 480, 555, 522, 506, 491, 514, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 528, 520, 512, 475, 525, 479, 566, 470, 505, 472, 496, 510, 520, 509, 446, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8297, 8184, 8139, 8115, 8293, 8168, 8071] avg=8192.0000 max=8297 max_rank=1 min=8071 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[491, 509, 548, 522, 518, 509, 559, 524, 481, 459, 500, 507, 522, 547, 599, 481, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 509, 506, 522, 522, 508, 485, 526, 506, 476, 487, 499, 527, 516, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 503, 512, 485, 543, 505, 517, 474, 531, 542, 516, 478, 520, 539, 510, 513, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 523, 502, 486, 530, 512, 488, 483, 505, 470, 464, 523, 531, 495, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 532, 453, 545, 492, 517, 536, 506, 501, 505, 560, 498, 520, 526, 516, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 522, 526, 484, 491, 527, 510, 528, 522, 506, 500, 534, 492, 525, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 507, 508, 476, 540, 511, 503, 502, 531, 518, 547, 506, 534, 472, 506, 494, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 576, 507, 521, 516, 545, 503, 507, 466, 532, 540, 527, 517, 522, 527, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8112, 8183, 8068, 8215, 8188, 8139, 8350] avg=8192.0000 max=8350 max_rank=7 min=8068 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4352, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4352, 4096, 4096, 4352, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[525, 500, 498, 513, 527, 511, 523, 513, 515, 511, 491, 512, 516, 520, 520, 521, 552, 526, 512, 517, 518, 525, 544, 478, 533, 489, 505, 525, 456, 549, 499, 515, 478, 539, 493, 489, 538, 452, 493, 510, 542, 534, 533, 450, 518, 543, 526, 510, 526, 546, 510, 528, 519, 475, 470, 490, 522, 536, 538, 509, 521, 487, 521, 526, 510, 555, 439, 497, 533, 522, 498, 508, 479, 539, 526, 539, 530, 524, 512, 467, 508, 531, 506, 546, 483, 505, 496, 525, 538, 540, 502, 539, 478, 507, 507, 495, 522, 512, 500, 526, 527, 517, 529, 502, 510, 491, 513, 568, 474, 489, 516, 488, 518, 482, 445, 531, 524, 492, 505, 509, 481, 521, 530, 515, 512, 494, 566, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[517, 508, 514, 518, 471, 490, 503, 500, 538, 500, 513, 486, 535, 495, 527, 541, 519, 505, 488, 503, 500, 508, 529, 516, 508, 552, 504, 531, 486, 485, 525, 526, 510, 549, 528, 525, 513, 485, 511, 542, 495, 502, 466, 507, 516, 501, 561, 558, 477, 489, 512, 505, 510, 497, 510, 541, 520, 525, 494, 548, 514, 514, 543, 500, 517, 492, 541, 485, 529, 510, 509, 518, 516, 479, 505, 492, 496, 534, 499, 523, 509, 469, 497, 512, 532, 551, 483, 524, 505, 505, 505, 506, 508, 488, 522, 511, 545, 542, 521, 534, 512, 516, 524, 522, 491, 464, 542, 484, 510, 515, 530, 504, 484, 527, 518, 501, 495, 542, 542, 522, 475, 521, 510, 500, 524, 531, 508, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[462, 555, 518, 471, 531, 525, 516, 517, 531, 502, 509, 519, 490, 492, 532, 539, 490, 542, 506, 532, 498, 520, 533, 529, 486, 481, 487, 541, 534, 501, 496, 506, 540, 546, 461, 548, 520, 536, 485, 531, 506, 517, 505, 513, 508, 525, 492, 504, 566, 505, 529, 494, 544, 522, 490, 495, 504, 505, 502, 508, 489, 531, 488, 495, 547, 508, 491, 558, 505, 507, 510, 531, 544, 479, 520, 524, 519, 505, 502, 492, 498, 538, 492, 532, 508, 467, 487, 493, 517, 515, 545, 549, 517, 517, 504, 485, 527, 535, 484, 524, 495, 537, 463, 530, 472, 531, 499, 532, 522, 529, 519, 480, 506, 528, 497, 544, 516, 484, 513, 562, 482, 542, 513, 487, 499, 525, 474, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[507, 557, 523, 507, 525, 506, 500, 494, 510, 544, 482, 534, 496, 542, 525, 507, 512, 506, 527, 541, 525, 505, 475, 493, 512, 508, 538, 502, 525, 516, 494, 523, 511, 534, 472, 506, 482, 480, 541, 534, 521, 495, 543, 533, 503, 472, 516, 509, 509, 498, 523, 552, 502, 542, 524, 526, 498, 506, 543, 501, 498, 520, 511, 541, 524, 529, 522, 495, 521, 526, 531, 524, 504, 563, 524, 534, 527, 520, 519, 472, 508, 519, 467, 481, 511, 500, 475, 509, 516, 522, 510, 495, 535, 518, 495, 518, 541, 506, 522, 537, 486, 499, 488, 505, 528, 507, 471, 503, 486, 520, 474, 573, 506, 469, 522, 510, 514, 549, 496, 508, 487, 499, 510, 521, 490, 473, 526, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[513, 507, 515, 479, 511, 512, 526, 468, 507, 482, 499, 493, 511, 507, 520, 509, 509, 509, 486, 520, 505, 500, 511, 509, 523, 511, 533, 553, 482, 522, 506, 511, 535, 532, 510, 543, 518, 580, 526, 528, 498, 517, 502, 518, 515, 469, 488, 522, 536, 514, 524, 517, 516, 531, 517, 509, 482, 515, 539, 521, 509, 481, 517, 522, 530, 532, 519, 528, 490, 520, 528, 505, 475, 493, 511, 512, 504, 490, 517, 511, 511, 492, 488, 523, 525, 529, 504, 503, 510, 477, 490, 538, 522, 548, 518, 475, 511, 490, 513, 538, 486, 537, 517, 543, 503, 541, 493, 516, 530, 496, 475, 516, 514, 499, 515, 534, 521, 506, 489, 494, 512, 481, 485, 559, 543, 482, 535, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[516, 492, 525, 538, 555, 518, 510, 497, 506, 519, 498, 503, 561, 542, 532, 518, 509, 493, 551, 486, 498, 528, 512, 522, 491, 527, 520, 544, 535, 503, 522, 503, 472, 514, 529, 500, 503, 474, 534, 520, 498, 542, 518, 499, 508, 510, 487, 514, 509, 533, 502, 481, 511, 502, 512, 500, 505, 471, 485, 515, 510, 525, 484, 476, 531, 540, 478, 525, 526, 543, 507, 524, 481, 485, 502, 504, 527, 489, 513, 515, 537, 505, 526, 544, 488, 498, 517, 510, 500, 488, 490, 526, 500, 506, 517, 509, 513, 519, 486, 568, 529, 492, 544, 502, 517, 519, 521, 547, 502, 467, 514, 502, 517, 515, 508, 489, 526, 551, 501, 526, 484, 519, 528, 493, 530, 525, 483, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[514, 475, 511, 529, 557, 509, 525, 489, 513, 501, 504, 464, 546, 546, 506, 514, 508, 508, 510, 455, 495, 493, 488, 511, 503, 506, 512, 557, 481, 493, 506, 523, 533, 507, 485, 534, 487, 492, 531, 542, 516, 499, 498, 526, 539, 519, 506, 497, 550, 519, 500, 496, 489, 493, 519, 558, 508, 487, 508, 552, 509, 513, 528, 517, 501, 525, 502, 496, 497, 529, 531, 520, 469, 495, 562, 483, 484, 509, 532, 505, 527, 501, 554, 491, 538, 449, 542, 534, 491, 481, 521, 511, 487, 530, 519, 530, 527, 485, 523, 489, 535, 516, 509, 561, 506, 511, 568, 528, 521, 541, 475, 510, 495, 479, 473, 527, 516, 546, 519, 490, 503, 501, 502, 492, 577, 524, 510, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[585, 481, 509, 496, 517, 536, 537, 513, 482, 533, 537, 511, 484, 502, 478, 488, 460, 513, 536, 525, 520, 541, 513, 534, 512, 495, 509, 488, 506, 505, 523, 518, 484, 549, 496, 523, 484, 561, 498, 570, 470, 505, 500, 568, 515, 530, 515, 509, 480, 490, 511, 511, 498, 489, 480, 517, 518, 478, 506, 533, 484, 496, 512, 548, 505, 576, 555, 478, 517, 495, 532, 511, 585, 508, 516, 521, 507, 511, 483, 496, 508, 508, 522, 519, 542, 479, 524, 550, 494, 515, 484, 510, 525, 518, 476, 502, 532, 489, 481, 527, 580, 509, 525, 467, 523, 487, 502, 538, 443, 519, 513, 547, 488, 530, 468, 516, 484, 534, 490, 475, 530, 528, 499, 537, 510, 549, 517, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[525, 500, 498, 513, 517, 511, 523, 513, 515, 511, 491, 512, 516, 520, 520, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 526, 512, 517, 518, 525, 544, 478, 533, 489, 505, 525, 456, 549, 499, 515, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 539, 493, 489, 538, 452, 493, 489, 542, 534, 533, 450, 518, 543, 526, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 546, 510, 528, 519, 475, 470, 490, 522, 536, 538, 509, 521, 487, 521, 526, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 439, 497, 533, 522, 498, 508, 479, 539, 526, 539, 530, 524, 512, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 531, 506, 546, 483, 505, 496, 525, 538, 540, 502, 539, 478, 507, 507, 495, 21, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 500, 507, 527, 517, 529, 502, 510, 491, 513, 568, 474, 489, 516, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 482, 445, 531, 524, 492, 505, 509, 481, 521, 530, 515, 512, 494, 566, 512, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8248, 8127, 8234, 8159, 8237, 8165, 8160] avg=8192.0000 max=8248 max_rank=1 min=8127 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[462, 555, 518, 471, 514, 525, 516, 517, 531, 502, 509, 519, 490, 492, 532, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 542, 506, 532, 498, 520, 533, 529, 486, 481, 487, 541, 534, 501, 496, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 546, 461, 548, 520, 536, 485, 505, 506, 517, 505, 513, 508, 525, 492, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 505, 529, 494, 544, 522, 490, 495, 504, 505, 502, 508, 489, 531, 488, 495, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 484, 491, 558, 505, 507, 510, 531, 544, 479, 520, 524, 519, 505, 502, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 538, 492, 532, 508, 467, 487, 493, 517, 515, 545, 549, 517, 517, 504, 485, 26, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 535, 484, 498, 495, 537, 463, 530, 472, 531, 499, 532, 522, 529, 519, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 497, 544, 516, 484, 513, 562, 482, 542, 513, 487, 499, 525, 474, 484, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8211, 8183, 8218, 8203, 8153, 8185] avg=8192.0000 max=8218 max_rank=4 min=8153 min_rank=6 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 492, 525, 538, 545, 518, 510, 497, 506, 519, 498, 503, 561, 542, 532, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 551, 486, 498, 528, 512, 522, 491, 527, 520, 544, 535, 503, 522, 503, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 514, 529, 500, 503, 474, 534, 498, 498, 542, 518, 499, 508, 510, 487, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 502, 481, 511, 502, 512, 500, 505, 471, 485, 515, 510, 525, 484, 476, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 521, 478, 525, 526, 543, 507, 524, 481, 485, 502, 504, 527, 489, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 505, 526, 544, 488, 498, 517, 510, 500, 488, 490, 526, 500, 506, 517, 509, 22, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 486, 547, 529, 492, 544, 502, 517, 519, 521, 547, 502, 467, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 508, 489, 526, 551, 501, 526, 484, 519, 528, 493, 530, 525, 483, 531, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8249, 8100, 8031, 8171, 8194, 8221, 8250] avg=8192.0000 max=8320 max_rank=0 min=8031 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[513, 507, 515, 479, 501, 512, 526, 468, 507, 482, 499, 493, 511, 507, 520, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 486, 520, 505, 500, 511, 509, 523, 511, 533, 553, 482, 522, 506, 511, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 532, 510, 543, 518, 580, 526, 506, 498, 517, 502, 518, 515, 469, 488, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 524, 517, 516, 531, 517, 509, 482, 515, 539, 521, 509, 481, 517, 522, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 513, 519, 528, 490, 520, 528, 505, 475, 493, 511, 512, 504, 490, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 488, 523, 525, 529, 504, 503, 510, 477, 490, 538, 522, 548, 518, 475, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 490, 513, 518, 486, 537, 517, 543, 503, 541, 493, 516, 530, 496, 475, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 499, 515, 534, 521, 506, 489, 494, 512, 481, 485, 559, 543, 482, 535, 544, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 8195, 8279, 8260, 8146, 8185, 8185, 8237] avg=8192.0000 max=8279 max_rank=2 min=8049 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[507, 557, 523, 507, 515, 506, 500, 494, 510, 544, 482, 534, 496, 542, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 506, 527, 541, 525, 505, 475, 493, 512, 508, 538, 502, 525, 516, 494, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 534, 472, 506, 482, 480, 541, 512, 521, 495, 543, 533, 503, 472, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 523, 552, 502, 542, 524, 526, 498, 506, 543, 501, 498, 520, 511, 541, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 522, 495, 521, 526, 531, 524, 504, 563, 524, 534, 527, 520, 519, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 519, 467, 481, 511, 500, 475, 509, 516, 522, 510, 495, 535, 518, 495, 518, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 506, 522, 517, 486, 499, 488, 505, 528, 507, 471, 503, 486, 520, 474, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 469, 522, 510, 514, 549, 496, 508, 487, 499, 510, 521, 490, 473, 526, 489, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8207, 8130, 8304, 8316, 8111, 8126, 8093] avg=8192.0000 max=8316 max_rank=4 min=8093 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[517, 508, 514, 518, 463, 490, 503, 500, 538, 500, 513, 486, 535, 495, 527, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 488, 503, 500, 508, 529, 516, 508, 552, 504, 531, 486, 485, 525, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 549, 528, 525, 513, 485, 511, 520, 495, 502, 466, 507, 516, 501, 561, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 489, 512, 505, 510, 497, 510, 541, 520, 525, 494, 548, 514, 514, 543, 500, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 541, 485, 529, 510, 509, 518, 516, 479, 505, 492, 496, 534, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 469, 497, 512, 532, 551, 483, 524, 505, 505, 505, 506, 508, 488, 522, 511, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 542, 521, 514, 512, 516, 524, 522, 491, 464, 542, 484, 510, 515, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 518, 501, 495, 542, 542, 522, 475, 521, 510, 500, 524, 531, 508, 499, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8189, 8247, 8207, 8128, 8159, 8236, 8222] avg=8192.0000 max=8247 max_rank=2 min=8128 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 475, 511, 529, 547, 509, 525, 489, 513, 501, 504, 464, 546, 546, 506, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 510, 455, 495, 493, 488, 511, 503, 506, 512, 557, 481, 493, 506, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 507, 485, 534, 487, 492, 531, 520, 516, 499, 498, 526, 539, 519, 506, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 519, 500, 496, 489, 493, 519, 558, 508, 487, 508, 552, 509, 513, 528, 517, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 502, 496, 497, 529, 531, 520, 469, 495, 562, 483, 484, 509, 532, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 501, 554, 491, 538, 449, 542, 534, 491, 481, 521, 511, 487, 530, 519, 530, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 485, 523, 471, 535, 516, 509, 561, 506, 511, 568, 528, 521, 541, 475, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 479, 473, 527, 516, 546, 519, 490, 503, 501, 502, 492, 577, 524, 510, 522, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8054, 8189, 8256, 8122, 8237, 8287, 8198] avg=8192.0000 max=8287 max_rank=6 min=8054 min_rank=1 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[585, 481, 509, 496, 507, 536, 537, 513, 482, 533, 537, 511, 484, 502, 478, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 513, 536, 525, 520, 541, 513, 534, 512, 495, 509, 488, 506, 505, 523, 518, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 549, 496, 523, 484, 561, 498, 546, 470, 505, 500, 568, 515, 530, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 490, 511, 511, 498, 489, 480, 517, 518, 478, 506, 533, 484, 496, 512, 548, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 556, 555, 478, 517, 495, 532, 511, 585, 508, 516, 521, 507, 511, 483, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 522, 519, 542, 479, 524, 550, 494, 515, 484, 510, 525, 518, 476, 502, 24, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 489, 481, 508, 580, 509, 525, 467, 523, 487, 502, 538, 443, 519, 513, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 530, 468, 516, 484, 534, 490, 475, 530, 528, 499, 537, 510, 549, 517, 512, 15, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8203, 8253, 8061, 8276, 8210, 8163, 8191] avg=8192.0000 max=8276 max_rank=4 min=8061 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4352, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4352, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[527, 522, 510, 498, 509, 490, 531, 526, 509, 498, 518, 515, 498, 516, 533, 516, 512, 550, 487, 504, 525, 539, 486, 535, 511, 509, 490, 523, 540, 551, 514, 510, 493, 525, 495, 536, 524, 503, 536, 505, 474, 493, 460, 507, 508, 518, 537, 522, 466, 533, 519, 531, 553, 529, 529, 551, 523, 534, 469, 519, 518, 505, 496, 489, 469, 516, 518, 467, 536, 505, 552, 488, 498, 518, 539, 527, 554, 543, 516, 524, 491, 482, 520, 503, 539, 473, 528, 484, 498, 469, 520, 531, 513, 476, 509, 514, 490, 512, 527, 472, 516, 518, 531, 489, 504, 514, 501, 568, 494, 505, 498, 520, 512, 524, 493, 561, 470, 485, 498, 505, 558, 513, 526, 513, 506, 490, 506, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[522, 541, 477, 509, 539, 496, 526, 511, 498, 530, 514, 506, 550, 540, 526, 482, 497, 556, 505, 470, 529, 477, 517, 496, 497, 490, 490, 552, 480, 530, 533, 519, 535, 492, 524, 531, 496, 525, 495, 543, 496, 517, 520, 509, 509, 522, 514, 539, 535, 524, 516, 581, 504, 529, 501, 499, 527, 522, 492, 486, 468, 454, 502, 536, 526, 504, 527, 516, 513, 520, 511, 509, 523, 521, 505, 514, 537, 545, 540, 468, 493, 524, 476, 537, 538, 515, 529, 501, 509, 504, 519, 517, 487, 468, 493, 581, 543, 523, 553, 494, 490, 511, 525, 465, 519, 524, 490, 473, 484, 494, 575, 497, 508, 492, 494, 498, 484, 505, 507, 532, 500, 495, 507, 490, 522, 508, 517, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[506, 523, 514, 529, 555, 520, 541, 526, 514, 516, 492, 506, 487, 498, 531, 522, 498, 495, 500, 517, 526, 505, 503, 537, 518, 534, 502, 507, 582, 508, 462, 526, 500, 496, 519, 525, 506, 507, 492, 481, 487, 524, 492, 503, 500, 535, 454, 526, 537, 505, 479, 500, 531, 527, 533, 501, 509, 529, 479, 498, 557, 508, 489, 530, 526, 517, 531, 540, 530, 487, 538, 500, 539, 508, 517, 514, 489, 503, 513, 504, 509, 510, 520, 524, 482, 522, 501, 518, 548, 519, 499, 508, 476, 500, 526, 479, 511, 537, 522, 471, 562, 535, 491, 515, 498, 535, 530, 523, 509, 452, 468, 503, 537, 519, 518, 548, 519, 500, 493, 523, 502, 484, 533, 516, 508, 476, 528, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[515, 466, 482, 518, 523, 519, 490, 534, 537, 509, 520, 473, 546, 530, 496, 515, 544, 527, 515, 492, 531, 544, 511, 500, 486, 515, 489, 486, 522, 539, 493, 501, 530, 517, 473, 514, 493, 503, 507, 535, 531, 513, 511, 453, 532, 494, 473, 497, 526, 515, 552, 513, 494, 518, 466, 549, 500, 537, 480, 459, 534, 477, 534, 542, 484, 516, 505, 544, 505, 510, 514, 517, 516, 514, 507, 503, 485, 496, 530, 507, 497, 521, 510, 538, 503, 487, 459, 524, 503, 549, 525, 499, 509, 554, 515, 544, 505, 520, 532, 532, 529, 519, 540, 505, 478, 519, 465, 508, 474, 481, 532, 540, 505, 503, 537, 538, 537, 575, 474, 511, 516, 511, 520, 512, 505, 541, 538, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[559, 509, 502, 510, 491, 539, 501, 497, 502, 531, 494, 525, 521, 525, 529, 498, 509, 516, 539, 510, 493, 567, 507, 557, 487, 513, 500, 504, 468, 522, 527, 501, 542, 523, 550, 542, 544, 547, 518, 489, 513, 513, 480, 527, 508, 547, 526, 491, 489, 499, 489, 490, 541, 465, 511, 558, 492, 483, 501, 503, 519, 524, 527, 496, 505, 509, 535, 518, 487, 525, 489, 509, 521, 502, 477, 516, 521, 523, 514, 498, 497, 496, 508, 534, 519, 520, 472, 529, 530, 496, 497, 532, 507, 503, 542, 495, 492, 524, 519, 483, 515, 510, 506, 512, 513, 487, 533, 488, 513, 476, 498, 507, 476, 508, 553, 518, 503, 508, 509, 502, 556, 525, 522, 494, 515, 505, 549, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[470, 518, 500, 534, 470, 509, 501, 493, 515, 493, 501, 528, 529, 531, 498, 519, 537, 523, 510, 524, 474, 531, 526, 497, 516, 516, 525, 496, 523, 522, 527, 516, 512, 540, 532, 493, 518, 539, 571, 540, 471, 491, 495, 528, 512, 531, 512, 529, 477, 503, 516, 516, 489, 509, 549, 542, 467, 497, 527, 520, 540, 477, 496, 503, 505, 518, 488, 514, 509, 518, 519, 545, 518, 496, 499, 528, 488, 492, 525, 534, 501, 521, 498, 507, 486, 511, 485, 504, 518, 513, 471, 473, 555, 546, 499, 508, 494, 482, 529, 521, 538, 541, 552, 544, 496, 500, 504, 516, 551, 517, 493, 505, 494, 491, 518, 490, 515, 561, 504, 475, 511, 528, 534, 482, 542, 493, 509, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[481, 540, 524, 521, 471, 540, 512, 517, 477, 482, 493, 512, 502, 569, 489, 516, 490, 461, 523, 520, 521, 491, 517, 528, 583, 487, 546, 488, 473, 523, 548, 530, 509, 491, 511, 526, 516, 512, 506, 508, 533, 490, 512, 525, 511, 507, 514, 509, 512, 527, 530, 516, 544, 557, 491, 525, 518, 520, 529, 488, 477, 514, 475, 482, 523, 528, 492, 502, 534, 511, 523, 485, 544, 572, 531, 527, 512, 507, 521, 478, 523, 519, 538, 496, 509, 482, 556, 501, 477, 482, 506, 501, 537, 508, 510, 522, 511, 493, 509, 525, 543, 490, 542, 505, 490, 491, 483, 495, 513, 518, 479, 505, 492, 554, 501, 512, 519, 546, 511, 527, 541, 537, 545, 502, 497, 494, 485, 464] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[509, 465, 471, 519, 522, 524, 528, 516, 510, 494, 550, 525, 485, 514, 514, 497, 522, 508, 497, 523, 533, 500, 508, 531, 485, 532, 533, 522, 474, 534, 524, 507, 509, 516, 522, 533, 476, 509, 508, 533, 541, 452, 526, 489, 513, 500, 501, 515, 494, 512, 472, 547, 509, 481, 526, 508, 504, 511, 498, 521, 517, 547, 517, 496, 490, 451, 536, 506, 501, 496, 540, 517, 525, 534, 520, 536, 506, 535, 563, 508, 525, 536, 515, 454, 507, 534, 513, 544, 514, 520, 525, 506, 506, 520, 515, 523, 507, 534, 494, 523, 475, 548, 502, 482, 484, 502, 484, 507, 548, 514, 527, 495, 505, 511, 545, 467, 524, 507, 505, 531, 487, 534, 525, 539, 520, 505, 517, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 541, 477, 509, 539, 496, 526, 511, 498, 530, 514, 506, 550, 537, 526, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 556, 505, 470, 529, 477, 517, 496, 497, 490, 490, 552, 480, 500, 533, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 492, 524, 531, 496, 525, 495, 543, 496, 517, 520, 509, 509, 522, 514, 539, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 524, 516, 581, 504, 529, 501, 499, 527, 522, 492, 486, 468, 454, 502, 536, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 504, 527, 516, 513, 520, 511, 509, 523, 521, 505, 514, 537, 545, 500, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 524, 476, 537, 538, 515, 529, 501, 509, 504, 519, 517, 487, 468, 493, 581, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 523, 553, 494, 490, 511, 525, 465, 519, 524, 490, 473, 484, 494, 575, 497, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 492, 494, 498, 484, 505, 507, 532, 500, 495, 503, 490, 522, 508, 517, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8108, 8268, 8188, 8239, 8217, 8198, 8054] avg=8192.0000 max=8268 max_rank=2 min=8054 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[506, 523, 514, 529, 555, 520, 541, 526, 514, 516, 492, 506, 487, 495, 531, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 500, 517, 526, 505, 503, 537, 518, 534, 502, 507, 582, 479, 462, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 496, 519, 525, 506, 507, 492, 481, 487, 524, 492, 503, 500, 535, 454, 526, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 505, 479, 500, 531, 527, 533, 501, 509, 529, 479, 498, 557, 508, 489, 530, 6, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 531, 540, 530, 487, 538, 500, 539, 508, 517, 514, 489, 503, 475, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 510, 520, 524, 482, 522, 501, 518, 548, 519, 499, 508, 476, 500, 526, 479, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 537, 522, 471, 562, 535, 491, 515, 498, 535, 530, 523, 509, 452, 468, 503, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 519, 518, 548, 519, 500, 493, 523, 502, 484, 528, 516, 508, 476, 528, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8191, 8048, 8225, 8218, 8166, 8198, 8213] avg=8192.0000 max=8277 max_rank=0 min=8048 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[481, 540, 524, 521, 471, 540, 512, 517, 477, 482, 493, 512, 502, 559, 489, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 461, 523, 520, 521, 491, 517, 528, 583, 487, 546, 488, 473, 487, 548, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 511, 526, 516, 512, 506, 508, 533, 490, 512, 525, 511, 507, 514, 509, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 530, 516, 544, 557, 491, 525, 518, 520, 529, 488, 477, 514, 475, 482, 8, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 528, 492, 502, 534, 511, 523, 485, 544, 572, 531, 527, 512, 507, 473, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 519, 538, 496, 509, 482, 556, 501, 477, 482, 506, 501, 537, 508, 510, 522, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 493, 509, 525, 543, 490, 542, 505, 490, 491, 483, 495, 513, 518, 479, 505, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 554, 501, 512, 519, 546, 511, 527, 541, 537, 535, 502, 497, 494, 485, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8193, 8186, 8227, 8242, 8203, 8132, 8217] avg=8192.0000 max=8242 max_rank=4 min=8132 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 465, 471, 519, 522, 524, 528, 516, 510, 494, 550, 525, 485, 511, 514, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 497, 523, 533, 500, 508, 531, 485, 532, 533, 522, 474, 503, 524, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 522, 533, 476, 509, 508, 533, 541, 452, 526, 489, 513, 500, 501, 515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 512, 472, 547, 509, 481, 526, 508, 504, 511, 498, 521, 517, 547, 517, 496, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 451, 536, 506, 501, 496, 540, 517, 525, 534, 520, 536, 506, 535, 522, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 536, 515, 454, 507, 534, 513, 544, 514, 520, 525, 506, 506, 520, 515, 523, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 534, 494, 523, 475, 548, 502, 482, 484, 502, 484, 507, 548, 514, 527, 495, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 545, 467, 524, 507, 505, 531, 487, 534, 521, 539, 520, 505, 517, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8202, 8144, 8172, 8223, 8284, 8165, 8206] avg=8192.0000 max=8284 max_rank=5 min=8140 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[559, 509, 502, 510, 491, 539, 501, 497, 502, 531, 494, 525, 521, 522, 529, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 539, 510, 493, 567, 507, 557, 487, 513, 500, 504, 468, 492, 527, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 523, 550, 542, 544, 547, 518, 489, 513, 513, 480, 527, 508, 547, 526, 491, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 499, 489, 490, 541, 465, 511, 558, 492, 483, 501, 503, 519, 524, 527, 496, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 535, 518, 487, 525, 489, 509, 521, 502, 477, 516, 521, 523, 476, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 496, 508, 534, 519, 520, 472, 529, 530, 496, 497, 532, 507, 503, 542, 495, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 524, 519, 483, 515, 510, 506, 512, 513, 487, 533, 488, 513, 476, 498, 507, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 508, 553, 518, 503, 508, 509, 502, 556, 525, 518, 494, 515, 505, 549, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8190, 8361, 8099, 8111, 8203, 8112, 8230] avg=8192.0000 max=8361 max_rank=2 min=8099 min_rank=3 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[470, 518, 500, 534, 470, 509, 501, 493, 515, 493, 501, 528, 529, 528, 498, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 523, 510, 524, 474, 531, 526, 497, 516, 516, 525, 496, 523, 492, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 540, 532, 493, 518, 539, 571, 540, 471, 491, 495, 528, 512, 531, 512, 529, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 503, 516, 516, 489, 509, 549, 542, 467, 497, 527, 520, 540, 477, 496, 503, 6, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 488, 514, 509, 518, 519, 545, 518, 496, 499, 528, 488, 492, 486, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 498, 507, 486, 511, 485, 504, 518, 513, 471, 473, 555, 546, 499, 508, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 482, 529, 521, 538, 541, 552, 544, 496, 500, 504, 516, 551, 517, 493, 505, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 491, 518, 490, 515, 561, 504, 475, 511, 528, 529, 482, 542, 493, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8233, 8315, 8141, 8157, 8122, 8320, 8142] avg=8192.0000 max=8320 max_rank=6 min=8106 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 522, 510, 498, 509, 490, 531, 526, 509, 498, 518, 515, 498, 513, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 550, 487, 504, 525, 539, 486, 535, 511, 509, 490, 523, 540, 520, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 525, 495, 536, 524, 503, 536, 505, 474, 493, 460, 507, 508, 518, 537, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 533, 519, 531, 553, 529, 529, 551, 523, 534, 469, 519, 518, 505, 496, 489, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 516, 518, 467, 536, 505, 552, 488, 498, 518, 539, 527, 554, 543, 478, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 482, 520, 503, 539, 473, 528, 484, 498, 469, 520, 531, 513, 476, 509, 514, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 512, 527, 472, 516, 518, 531, 489, 504, 514, 501, 568, 494, 505, 498, 520, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 493, 561, 470, 485, 498, 505, 558, 513, 522, 513, 506, 490, 506, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8255, 8137, 8276, 8232, 8077, 8195, 8151] avg=8192.0000 max=8276 max_rank=3 min=8077 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 466, 482, 518, 523, 519, 490, 534, 537, 509, 520, 473, 546, 527, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 527, 515, 492, 531, 544, 511, 500, 486, 515, 489, 486, 522, 508, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 517, 473, 514, 493, 503, 507, 535, 531, 513, 511, 453, 532, 494, 473, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 552, 513, 494, 518, 466, 549, 500, 537, 480, 459, 534, 477, 534, 542, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 516, 505, 544, 505, 510, 514, 517, 516, 514, 507, 503, 485, 496, 491, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 521, 510, 538, 503, 487, 459, 524, 503, 549, 525, 499, 509, 554, 515, 544, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 520, 532, 532, 529, 519, 540, 505, 478, 519, 465, 508, 474, 481, 532, 540, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 537, 538, 537, 575, 474, 511, 516, 511, 516, 512, 505, 541, 538, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8164, 8077, 8208, 8114, 8264, 8216, 8323] avg=8192.0000 max=8323 max_rank=7 min=8077 min_rank=2 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 3968, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4352, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[490, 513, 552, 501, 519, 486, 494, 540, 481, 562, 512, 474, 526, 480, 517, 513, 490, 516, 507, 469, 543, 561, 527, 490, 493, 528, 523, 549, 535, 476, 502, 502, 509, 490, 506, 489, 476, 495, 513, 492, 495, 533, 526, 506, 534, 500, 481, 515, 525, 502, 478, 476, 516, 516, 534, 537, 524, 513, 497, 513, 509, 481, 520, 527, 509, 493, 511, 492, 500, 534, 570, 526, 523, 512, 524, 486, 533, 506, 537, 539, 518, 481, 530, 527, 461, 485, 484, 506, 530, 525, 549, 542, 515, 531, 534, 520, 519, 482, 508, 475, 484, 578, 505, 498, 506, 495, 512, 531, 523, 518, 498, 500, 565, 475, 537, 533, 528, 527, 488, 519, 492, 511, 498, 522, 517, 534, 473, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[559, 527, 515, 521, 553, 554, 523, 506, 526, 529, 515, 490, 501, 500, 527, 491, 522, 525, 505, 532, 472, 516, 506, 517, 495, 551, 475, 504, 537, 547, 575, 527, 507, 519, 530, 529, 497, 519, 518, 455, 533, 507, 488, 497, 529, 523, 516, 524, 483, 515, 503, 509, 508, 517, 538, 538, 461, 475, 508, 518, 494, 478, 514, 525, 489, 533, 496, 514, 497, 486, 483, 492, 508, 486, 505, 523, 510, 495, 515, 525, 504, 527, 558, 496, 525, 507, 533, 533, 525, 496, 514, 512, 500, 509, 510, 494, 535, 506, 513, 487, 507, 490, 497, 507, 516, 532, 477, 511, 482, 515, 532, 508, 493, 487, 539, 507, 511, 527, 515, 482, 507, 499, 502, 529, 512, 531, 527, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[518, 539, 441, 495, 478, 483, 456, 557, 513, 521, 532, 493, 513, 522, 546, 523, 524, 471, 487, 506, 484, 473, 501, 526, 491, 462, 521, 541, 545, 495, 482, 485, 494, 502, 529, 521, 489, 511, 522, 492, 490, 498, 522, 481, 515, 534, 547, 534, 542, 512, 534, 522, 528, 477, 494, 494, 524, 542, 502, 529, 517, 516, 531, 492, 526, 463, 459, 540, 513, 511, 497, 496, 543, 558, 504, 504, 512, 497, 531, 518, 534, 557, 503, 476, 527, 507, 512, 487, 539, 508, 530, 511, 489, 516, 495, 486, 532, 502, 534, 530, 537, 521, 520, 512, 534, 531, 533, 536, 507, 509, 508, 551, 531, 539, 548, 508, 502, 518, 485, 510, 502, 516, 515, 514, 538, 493, 490, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[510, 520, 507, 508, 459, 506, 479, 447, 524, 544, 522, 523, 486, 523, 491, 521, 507, 510, 539, 486, 526, 541, 501, 520, 505, 550, 506, 482, 512, 504, 499, 512, 519, 520, 526, 497, 490, 508, 514, 579, 529, 563, 512, 482, 524, 543, 509, 501, 502, 528, 500, 514, 481, 502, 576, 522, 475, 510, 496, 556, 524, 523, 517, 556, 557, 516, 506, 495, 538, 525, 553, 520, 545, 476, 521, 551, 534, 487, 483, 497, 492, 509, 536, 480, 491, 531, 516, 546, 510, 514, 517, 504, 498, 507, 523, 561, 516, 496, 467, 503, 492, 515, 540, 533, 486, 541, 500, 478, 491, 472, 522, 508, 505, 501, 537, 484, 492, 533, 512, 498, 469, 533, 525, 476, 508, 531, 466, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[543, 532, 532, 500, 505, 522, 542, 486, 488, 495, 516, 497, 564, 521, 486, 517, 546, 521, 488, 501, 478, 527, 513, 485, 509, 529, 528, 522, 546, 544, 528, 554, 510, 511, 511, 535, 502, 507, 516, 512, 524, 518, 539, 498, 521, 471, 545, 521, 511, 520, 463, 464, 521, 536, 508, 518, 540, 505, 524, 474, 511, 482, 512, 544, 490, 493, 495, 487, 537, 498, 523, 538, 533, 540, 470, 538, 529, 533, 515, 496, 478, 505, 503, 486, 470, 502, 480, 498, 514, 470, 545, 575, 479, 524, 496, 529, 515, 532, 506, 486, 537, 517, 491, 544, 509, 519, 475, 479, 506, 490, 511, 513, 547, 523, 527, 492, 575, 538, 507, 466, 475, 471, 512, 500, 500, 508, 536, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[485, 490, 496, 537, 529, 568, 482, 537, 496, 493, 501, 488, 512, 503, 518, 490, 515, 521, 521, 522, 552, 531, 547, 547, 499, 530, 537, 496, 506, 513, 490, 460, 469, 530, 500, 499, 500, 507, 501, 553, 462, 543, 531, 519, 494, 517, 541, 477, 526, 507, 531, 515, 550, 534, 478, 519, 499, 510, 503, 468, 506, 511, 513, 510, 538, 521, 494, 506, 528, 511, 525, 508, 509, 528, 514, 515, 522, 515, 493, 515, 539, 494, 510, 541, 535, 482, 515, 515, 514, 494, 501, 497, 495, 532, 514, 490, 520, 507, 534, 520, 475, 488, 502, 496, 532, 551, 521, 483, 516, 501, 496, 489, 552, 515, 536, 512, 518, 508, 486, 489, 520, 530, 516, 479, 527, 518, 503, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[471, 500, 501, 498, 501, 505, 512, 506, 482, 515, 539, 488, 526, 523, 534, 525, 539, 504, 538, 503, 502, 522, 525, 510, 507, 514, 512, 500, 520, 491, 501, 486, 516, 505, 501, 496, 519, 499, 498, 459, 533, 526, 549, 518, 528, 487, 533, 519, 497, 483, 551, 529, 497, 505, 499, 505, 501, 537, 523, 504, 548, 489, 540, 512, 518, 480, 458, 512, 512, 464, 513, 524, 484, 497, 505, 542, 509, 529, 486, 540, 571, 483, 514, 532, 524, 483, 540, 498, 508, 504, 547, 548, 507, 517, 525, 484, 527, 513, 533, 538, 521, 556, 534, 506, 505, 503, 508, 523, 522, 528, 501, 539, 506, 503, 524, 527, 498, 491, 547, 463, 507, 498, 489, 525, 517, 484, 528, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[580, 509, 535, 510, 503, 491, 467, 518, 507, 503, 479, 518, 475, 536, 525, 508, 520, 503, 542, 521, 501, 509, 460, 529, 519, 507, 510, 519, 514, 491, 525, 499, 527, 519, 503, 541, 508, 506, 501, 520, 526, 493, 520, 482, 515, 514, 553, 475, 510, 491, 560, 506, 527, 517, 509, 508, 522, 532, 474, 530, 552, 495, 501, 508, 528, 487, 460, 527, 517, 489, 479, 521, 491, 463, 514, 539, 513, 511, 508, 572, 509, 536, 518, 538, 536, 532, 510, 518, 547, 556, 482, 534, 491, 478, 539, 534, 494, 505, 487, 531, 518, 517, 521, 517, 501, 496, 517, 468, 494, 536, 486, 502, 503, 518, 543, 523, 479, 525, 540, 515, 510, 530, 545, 477, 492, 510, 493, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 539, 441, 495, 478, 483, 456, 557, 513, 521, 532, 493, 513, 522, 546, 523, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 471, 487, 506, 484, 473, 501, 526, 491, 462, 521, 541, 530, 495, 482, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 502, 529, 521, 489, 511, 522, 492, 490, 498, 522, 481, 515, 534, 547, 534, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 512, 534, 522, 528, 477, 494, 494, 524, 542, 502, 529, 517, 516, 531, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 463, 459, 540, 513, 511, 497, 496, 543, 558, 504, 504, 512, 497, 531, 518, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 557, 503, 476, 527, 507, 512, 487, 539, 508, 530, 485, 489, 516, 495, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 502, 534, 530, 537, 521, 520, 512, 534, 531, 533, 536, 507, 509, 508, 551, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 539, 548, 508, 502, 518, 485, 510, 502, 516, 515, 514, 538, 493, 490, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 7979, 8182, 8254, 8178, 8151, 8412, 8229] avg=8192.0000 max=8412 max_rank=6 min=7979 min_rank=1 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[490, 513, 552, 501, 519, 486, 494, 540, 481, 562, 512, 474, 526, 480, 517, 513, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 516, 507, 469, 543, 561, 527, 490, 493, 528, 523, 549, 520, 476, 502, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 506, 489, 476, 495, 513, 492, 495, 533, 526, 506, 534, 500, 481, 515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 502, 478, 476, 516, 516, 534, 537, 524, 513, 497, 513, 509, 481, 520, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 511, 492, 500, 534, 570, 526, 523, 512, 524, 486, 533, 506, 537, 539, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 481, 530, 527, 461, 485, 484, 506, 530, 525, 549, 515, 515, 531, 534, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 482, 508, 475, 484, 578, 505, 498, 506, 495, 512, 531, 523, 518, 498, 500, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 475, 537, 533, 528, 527, 488, 519, 492, 511, 498, 522, 517, 534, 473, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8196, 8061, 8166, 8301, 8211, 8147, 8272] avg=8192.0000 max=8301 max_rank=4 min=8061 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32931 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[543, 532, 532, 500, 505, 522, 542, 486, 488, 495, 516, 497, 564, 521, 486, 517, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 521, 488, 501, 478, 527, 513, 485, 509, 529, 528, 522, 523, 544, 528, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 511, 535, 502, 507, 516, 512, 524, 518, 539, 498, 521, 471, 545, 521, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 520, 463, 464, 521, 536, 508, 518, 540, 505, 524, 474, 511, 482, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 493, 495, 487, 537, 498, 523, 538, 533, 540, 470, 538, 529, 533, 515, 496, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 505, 503, 486, 470, 502, 480, 498, 514, 470, 545, 538, 479, 524, 496, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 532, 506, 486, 537, 517, 491, 544, 509, 519, 475, 479, 506, 490, 511, 513, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 523, 526, 492, 575, 538, 507, 466, 475, 471, 512, 500, 500, 508, 536, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8296, 8249, 8118, 8232, 8017, 8153, 8197] avg=8192.0000 max=8296 max_rank=1 min=8017 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[559, 527, 515, 521, 553, 554, 523, 506, 526, 529, 515, 490, 501, 500, 527, 491, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 505, 532, 472, 516, 506, 517, 495, 551, 475, 504, 522, 547, 575, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 519, 530, 529, 497, 519, 518, 455, 533, 507, 488, 497, 529, 523, 516, 524, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 515, 503, 509, 508, 517, 538, 538, 461, 475, 508, 518, 494, 478, 514, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 533, 496, 514, 497, 486, 483, 492, 508, 486, 505, 523, 510, 495, 515, 525, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 527, 558, 496, 525, 507, 533, 533, 525, 496, 514, 486, 500, 509, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 506, 513, 487, 507, 490, 497, 507, 516, 532, 477, 511, 482, 515, 532, 508, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 487, 539, 507, 511, 527, 515, 482, 507, 499, 502, 529, 512, 531, 527, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8358, 8291, 8192, 8082, 8063, 8217, 8130, 8203] avg=8192.0000 max=8358 max_rank=0 min=8063 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[485, 490, 496, 537, 529, 568, 482, 537, 496, 493, 501, 488, 512, 503, 518, 490, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 521, 521, 522, 552, 531, 547, 547, 499, 530, 537, 496, 492, 513, 490, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 530, 500, 499, 500, 507, 501, 553, 462, 543, 531, 519, 494, 517, 541, 477, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 531, 515, 550, 534, 478, 519, 499, 510, 503, 468, 506, 511, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 521, 494, 506, 528, 511, 525, 508, 509, 528, 514, 515, 522, 515, 493, 515, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 494, 510, 541, 535, 482, 515, 515, 514, 494, 501, 472, 495, 532, 514, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 507, 534, 520, 475, 488, 502, 496, 532, 551, 521, 483, 516, 501, 496, 489, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 515, 536, 512, 518, 508, 486, 489, 520, 530, 516, 479, 527, 518, 503, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8273, 8144, 8178, 8248, 8143, 8145, 8260] avg=8192.0000 max=8273 max_rank=1 min=8143 min_rank=5 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 520, 507, 508, 459, 506, 479, 447, 524, 544, 522, 523, 486, 523, 491, 521, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 539, 486, 526, 541, 501, 520, 505, 550, 506, 482, 498, 504, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 526, 497, 490, 508, 514, 579, 529, 563, 512, 482, 524, 543, 509, 501, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 528, 500, 514, 481, 502, 576, 522, 475, 510, 496, 556, 524, 523, 517, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 516, 506, 495, 538, 525, 553, 520, 545, 476, 521, 551, 534, 487, 483, 497, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 509, 536, 480, 491, 531, 516, 546, 510, 514, 517, 478, 498, 507, 523, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 496, 467, 503, 492, 515, 540, 533, 486, 541, 500, 478, 491, 472, 522, 508, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 501, 537, 484, 492, 533, 512, 498, 469, 533, 525, 476, 508, 531, 466, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8186, 8317, 8280, 8310, 8209, 8074, 8069] avg=8192.0000 max=8317 max_rank=2 min=8069 min_rank=7 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[471, 500, 501, 498, 501, 505, 512, 506, 482, 515, 539, 488, 526, 523, 534, 525, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 538, 503, 502, 522, 525, 510, 507, 514, 512, 500, 506, 491, 501, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 505, 501, 496, 519, 499, 498, 459, 533, 526, 549, 518, 528, 487, 533, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 483, 551, 529, 497, 505, 499, 505, 501, 537, 523, 504, 548, 489, 540, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 480, 458, 512, 512, 464, 513, 524, 484, 497, 505, 542, 509, 529, 486, 540, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 483, 514, 532, 524, 483, 540, 498, 508, 504, 547, 520, 507, 517, 525, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 513, 533, 538, 521, 556, 534, 506, 505, 503, 508, 523, 522, 528, 501, 539, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 503, 524, 527, 498, 491, 547, 463, 507, 498, 489, 525, 517, 484, 528, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8160, 8187, 8218, 8079, 8257, 8371, 8115] avg=8192.0000 max=8371 max_rank=6 min=8079 min_rank=4 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[580, 509, 535, 510, 503, 491, 467, 518, 507, 503, 479, 518, 475, 536, 525, 508, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 503, 542, 521, 501, 509, 460, 529, 519, 507, 510, 519, 500, 491, 525, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 503, 541, 508, 506, 501, 520, 526, 493, 520, 482, 515, 514, 553, 475, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 560, 506, 527, 517, 509, 508, 522, 532, 474, 530, 552, 495, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 487, 460, 527, 517, 489, 479, 521, 491, 463, 514, 539, 513, 511, 508, 572, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 536, 518, 538, 536, 532, 510, 518, 547, 556, 482, 507, 491, 478, 539, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 505, 487, 531, 518, 517, 521, 517, 501, 496, 517, 468, 494, 536, 486, 502, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 518, 543, 523, 479, 525, 540, 515, 510, 530, 545, 477, 492, 510, 493, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8155, 8204, 8240, 8125, 8331, 8104, 8191] avg=8192.0000 max=8331 max_rank=5 min=8104 min_rank=6 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4352, 4096, 4224, 4224, 4096, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[502, 503, 477, 519, 550, 537, 545, 507, 519, 497, 482, 513, 473, 488, 493, 531, 488, 521, 529, 520, 523, 490, 525, 505, 523, 506, 493, 517, 500, 527, 478, 502, 521, 536, 493, 499, 498, 516, 488, 501, 484, 507, 500, 525, 514, 509, 494, 498, 515, 517, 497, 516, 536, 510, 552, 529, 530, 486, 543, 509, 539, 522, 513, 510, 522, 538, 479, 541, 495, 532, 512, 508, 537, 517, 513, 478, 505, 478, 484, 471, 501, 557, 535, 518, 532, 518, 497, 479, 495, 516, 525, 551, 533, 521, 523, 506, 516, 516, 514, 526, 542, 507, 525, 522, 482, 521, 504, 517, 463, 503, 530, 522, 514, 531, 502, 533, 547, 508, 522, 517, 471, 477, 552, 510, 513, 507, 505, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[513, 471, 531, 479, 526, 518, 517, 546, 517, 483, 501, 473, 505, 481, 523, 536, 529, 498, 546, 516, 469, 494, 515, 526, 488, 499, 508, 527, 514, 517, 518, 552, 501, 506, 516, 550, 498, 542, 495, 479, 465, 536, 489, 515, 498, 490, 492, 499, 509, 507, 544, 518, 493, 484, 502, 487, 501, 535, 514, 519, 516, 479, 500, 497, 531, 553, 493, 522, 477, 516, 512, 524, 518, 505, 537, 529, 515, 530, 499, 527, 509, 544, 505, 507, 496, 551, 538, 529, 518, 527, 516, 578, 474, 487, 513, 486, 483, 494, 541, 524, 520, 510, 564, 504, 508, 491, 494, 545, 515, 509, 496, 527, 507, 533, 483, 530, 535, 529, 529, 481, 507, 507, 492, 524, 509, 498, 519, 550] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[543, 480, 493, 497, 511, 503, 547, 515, 539, 538, 554, 544, 507, 539, 443, 480, 478, 477, 505, 532, 499, 501, 514, 491, 513, 480, 473, 493, 542, 523, 493, 514, 571, 518, 491, 529, 534, 510, 501, 503, 501, 477, 517, 499, 562, 490, 468, 494, 537, 521, 534, 499, 559, 549, 513, 516, 552, 511, 554, 509, 454, 495, 505, 536, 489, 533, 544, 491, 506, 492, 491, 484, 489, 562, 531, 525, 518, 505, 507, 554, 496, 494, 486, 496, 531, 525, 453, 519, 487, 531, 552, 547, 524, 511, 533, 497, 517, 505, 495, 534, 521, 511, 529, 552, 507, 550, 482, 491, 501, 524, 489, 474, 515, 512, 513, 504, 535, 522, 524, 495, 494, 510, 514, 526, 510, 513, 494, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[525, 520, 476, 511, 520, 507, 491, 504, 532, 538, 482, 500, 496, 507, 496, 498, 545, 551, 524, 558, 554, 512, 484, 542, 524, 500, 496, 495, 511, 533, 525, 518, 517, 507, 519, 513, 519, 490, 529, 551, 485, 496, 486, 535, 492, 481, 543, 500, 496, 522, 526, 509, 526, 523, 484, 516, 512, 509, 515, 501, 517, 536, 476, 491, 495, 471, 550, 519, 518, 544, 483, 531, 501, 519, 477, 521, 509, 464, 514, 499, 516, 522, 478, 534, 541, 518, 508, 493, 509, 518, 548, 551, 511, 533, 498, 498, 550, 495, 520, 525, 536, 488, 483, 506, 511, 499, 511, 487, 543, 489, 515, 496, 514, 524, 481, 530, 483, 533, 532, 532, 529, 510, 492, 505, 530, 503, 496, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[510, 483, 492, 560, 467, 540, 550, 497, 508, 522, 509, 484, 531, 481, 542, 487, 483, 527, 530, 543, 472, 483, 554, 510, 539, 537, 510, 515, 516, 495, 519, 526, 517, 531, 537, 508, 464, 513, 527, 518, 498, 491, 512, 513, 506, 530, 482, 508, 501, 521, 512, 531, 488, 485, 521, 526, 520, 471, 514, 515, 551, 474, 545, 513, 511, 504, 510, 515, 521, 503, 528, 549, 496, 521, 482, 503, 529, 502, 497, 484, 519, 517, 498, 503, 516, 498, 523, 542, 504, 517, 507, 508, 518, 519, 520, 472, 511, 523, 512, 531, 536, 538, 517, 529, 488, 521, 511, 483, 532, 528, 472, 502, 511, 482, 512, 480, 519, 531, 513, 535, 486, 498, 506, 502, 562, 532, 517, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[489, 507, 471, 512, 511, 470, 475, 510, 538, 517, 548, 535, 529, 521, 499, 499, 572, 518, 511, 478, 522, 534, 512, 499, 520, 496, 506, 532, 528, 474, 507, 518, 506, 498, 495, 483, 496, 504, 478, 477, 558, 536, 551, 517, 510, 522, 507, 505, 498, 525, 532, 545, 520, 530, 502, 508, 503, 527, 496, 495, 546, 522, 518, 497, 538, 549, 509, 520, 525, 510, 525, 474, 504, 477, 498, 508, 506, 506, 482, 490, 534, 532, 503, 532, 525, 546, 503, 555, 505, 513, 495, 519, 504, 493, 538, 500, 500, 514, 494, 494, 504, 532, 544, 519, 504, 491, 534, 517, 503, 529, 528, 519, 481, 560, 509, 494, 493, 485, 451, 510, 519, 486, 532, 523, 531, 508, 493, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[516, 527, 495, 508, 511, 501, 530, 521, 510, 544, 492, 513, 531, 512, 515, 514, 460, 483, 511, 481, 474, 520, 501, 500, 529, 518, 510, 535, 527, 515, 538, 524, 523, 526, 483, 510, 508, 560, 534, 489, 521, 536, 492, 484, 495, 495, 511, 507, 514, 481, 496, 526, 502, 556, 495, 504, 469, 519, 536, 528, 564, 499, 515, 545, 520, 515, 491, 515, 495, 532, 564, 486, 543, 476, 503, 518, 473, 501, 541, 469, 496, 463, 519, 520, 511, 509, 489, 516, 516, 519, 543, 546, 535, 545, 513, 515, 533, 493, 525, 513, 502, 487, 512, 490, 522, 527, 489, 513, 516, 526, 479, 519, 550, 493, 540, 485, 517, 492, 463, 537, 514, 565, 487, 505, 527, 507, 526, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[556, 544, 543, 504, 515, 532, 538, 515, 505, 534, 528, 510, 499, 486, 465, 499, 532, 542, 554, 471, 490, 514, 487, 521, 543, 476, 549, 501, 527, 479, 504, 482, 506, 515, 548, 484, 522, 497, 521, 514, 531, 502, 493, 484, 517, 497, 497, 492, 521, 467, 499, 538, 512, 534, 518, 530, 541, 506, 542, 532, 508, 544, 506, 497, 438, 505, 527, 515, 492, 483, 515, 520, 487, 526, 520, 515, 534, 495, 508, 526, 498, 513, 478, 535, 526, 535, 522, 521, 495, 513, 508, 526, 502, 479, 495, 513, 504, 521, 504, 493, 509, 488, 529, 512, 572, 510, 536, 501, 478, 486, 521, 561, 496, 533, 486, 492, 526, 511, 508, 543, 513, 519, 545, 512, 502, 507, 499, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 471, 531, 479, 526, 518, 517, 546, 517, 483, 501, 473, 505, 481, 523, 536, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 498, 545, 516, 469, 494, 515, 526, 488, 499, 508, 527, 514, 517, 518, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 516, 550, 498, 542, 495, 479, 465, 536, 489, 515, 498, 490, 492, 499, 53, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 507, 544, 518, 493, 484, 502, 487, 501, 535, 468, 519, 516, 479, 500, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 553, 493, 522, 477, 516, 512, 524, 518, 505, 537, 529, 515, 530, 499, 527, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 544, 505, 507, 496, 551, 538, 529, 518, 527, 516, 525, 474, 487, 513, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 494, 541, 524, 520, 510, 556, 504, 508, 491, 494, 545, 515, 509, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 533, 483, 530, 535, 529, 529, 481, 507, 507, 492, 524, 508, 498, 519, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8215, 8131, 8059, 8321, 8225, 8217, 8232] avg=8192.0000 max=8321 max_rank=4 min=8059 min_rank=3 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32995 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[489, 507, 471, 512, 511, 470, 475, 510, 538, 517, 548, 535, 529, 521, 499, 499, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 518, 510, 478, 522, 534, 512, 499, 520, 496, 506, 532, 528, 474, 507, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 498, 495, 483, 496, 504, 478, 477, 558, 536, 551, 517, 510, 522, 507, 505, 47, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 532, 545, 520, 530, 502, 508, 503, 527, 451, 495, 546, 522, 518, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 549, 509, 520, 525, 510, 525, 474, 504, 477, 498, 508, 506, 506, 482, 490, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 503, 532, 525, 546, 503, 555, 505, 513, 495, 472, 504, 493, 538, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 494, 494, 504, 532, 536, 519, 504, 491, 534, 517, 503, 529, 528, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 560, 509, 494, 493, 485, 451, 510, 519, 486, 532, 523, 530, 508, 493, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8226, 8197, 8219, 8153, 8250, 8218, 8126] avg=8192.0000 max=8250 max_rank=5 min=8126 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[543, 480, 493, 497, 511, 503, 547, 515, 539, 538, 554, 544, 507, 539, 443, 480, 11, 9, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 477, 501, 532, 499, 501, 514, 491, 513, 480, 473, 493, 542, 523, 493, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 518, 491, 529, 534, 510, 501, 503, 501, 477, 517, 499, 562, 490, 468, 494, 55, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 521, 534, 499, 559, 549, 513, 516, 552, 511, 492, 509, 454, 495, 505, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 533, 544, 491, 506, 492, 491, 484, 489, 562, 531, 525, 518, 505, 507, 554, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 486, 496, 531, 525, 453, 519, 487, 531, 552, 492, 524, 511, 533, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 495, 534, 521, 511, 518, 552, 507, 550, 482, 491, 501, 524, 489, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 513, 504, 535, 522, 524, 495, 494, 510, 514, 526, 508, 513, 494, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8024, 8231, 8282, 8263, 8127, 8171, 8179] avg=8192.0000 max=8282 max_rank=3 min=8024 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 483, 492, 560, 467, 540, 550, 497, 508, 522, 509, 484, 531, 481, 542, 487, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 527, 529, 543, 472, 483, 554, 510, 539, 537, 510, 515, 516, 495, 519, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 531, 537, 508, 464, 513, 527, 518, 498, 491, 512, 513, 506, 530, 482, 508, 46, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 512, 531, 488, 485, 521, 526, 520, 471, 468, 515, 551, 474, 545, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 510, 515, 521, 503, 528, 549, 496, 521, 482, 503, 529, 502, 497, 484, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 517, 498, 503, 516, 498, 523, 542, 504, 517, 507, 462, 518, 519, 520, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 523, 512, 531, 536, 538, 509, 529, 488, 521, 511, 483, 532, 528, 472, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 482, 512, 480, 519, 531, 513, 535, 486, 498, 506, 502, 561, 532, 517, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8258, 8208, 8142, 8188, 8135, 8226, 8200] avg=8192.0000 max=8258 max_rank=1 min=8135 min_rank=5 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[502, 503, 477, 519, 550, 537, 545, 507, 519, 497, 482, 513, 473, 488, 493, 531, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 521, 528, 520, 523, 490, 525, 505, 523, 506, 493, 517, 500, 527, 478, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 536, 493, 499, 498, 516, 488, 501, 484, 507, 500, 525, 514, 509, 494, 498, 50, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 517, 497, 516, 536, 510, 552, 529, 530, 486, 494, 509, 539, 522, 513, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 538, 479, 541, 495, 532, 512, 508, 537, 517, 513, 478, 505, 478, 484, 471, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 557, 535, 518, 532, 518, 497, 479, 495, 516, 525, 501, 533, 521, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 514, 526, 542, 507, 517, 522, 482, 521, 504, 517, 463, 503, 530, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 502, 533, 547, 508, 522, 517, 471, 477, 552, 510, 512, 507, 505, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8146, 8141, 8275, 8145, 8257, 8202, 8218] avg=8192.0000 max=8275 max_rank=3 min=8141 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 520, 476, 511, 520, 507, 491, 504, 532, 538, 482, 500, 496, 507, 496, 498, 7, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 551, 523, 558, 554, 512, 484, 542, 524, 500, 496, 495, 511, 533, 525, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 519, 513, 519, 490, 529, 551, 485, 496, 486, 535, 492, 481, 543, 500, 50, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 522, 526, 509, 526, 523, 484, 516, 512, 509, 469, 501, 517, 536, 476, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 471, 550, 519, 518, 544, 483, 531, 501, 519, 477, 521, 509, 464, 514, 499, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 522, 478, 534, 541, 518, 508, 493, 509, 518, 548, 501, 511, 533, 498, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 495, 520, 525, 536, 488, 476, 506, 511, 499, 511, 487, 543, 489, 515, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 524, 481, 530, 483, 533, 532, 532, 529, 510, 492, 505, 529, 503, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8371, 8220, 8113, 8148, 8226, 8147, 8193] avg=8192.0000 max=8371 max_rank=1 min=8113 min_rank=3 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[516, 527, 495, 508, 511, 501, 530, 521, 510, 544, 492, 513, 531, 512, 515, 514, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 483, 510, 481, 474, 520, 501, 500, 529, 518, 510, 535, 527, 515, 538, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 483, 510, 508, 560, 534, 489, 521, 536, 492, 484, 495, 495, 511, 507, 50, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 481, 496, 526, 502, 556, 495, 504, 469, 519, 487, 528, 564, 499, 515, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 515, 491, 515, 495, 532, 564, 486, 543, 476, 503, 518, 473, 501, 541, 469, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 463, 519, 520, 511, 509, 489, 516, 516, 519, 543, 496, 535, 545, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 525, 513, 502, 487, 504, 490, 522, 527, 489, 513, 516, 526, 479, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 493, 540, 485, 517, 492, 463, 537, 514, 565, 487, 505, 526, 507, 526, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8125, 8232, 8200, 8177, 8205, 8138, 8203] avg=8192.0000 max=8256 max_rank=0 min=8125 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[556, 544, 543, 504, 515, 532, 538, 515, 505, 534, 528, 510, 499, 486, 465, 499, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 542, 553, 471, 490, 514, 487, 521, 543, 476, 549, 501, 527, 479, 504, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 548, 484, 522, 497, 521, 514, 531, 502, 493, 484, 517, 497, 497, 492, 48, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 467, 499, 538, 512, 534, 518, 530, 541, 506, 493, 532, 508, 544, 506, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 505, 527, 515, 492, 483, 515, 520, 487, 526, 520, 515, 534, 495, 508, 526, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 513, 478, 535, 526, 535, 522, 521, 495, 513, 508, 478, 502, 479, 495, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 504, 493, 509, 488, 521, 512, 572, 510, 536, 501, 478, 486, 521, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 533, 486, 492, 526, 511, 508, 543, 513, 519, 545, 512, 501, 507, 499, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 8171, 8176, 8246, 8141, 8111, 8217, 8185] avg=8192.0000 max=8289 max_rank=0 min=8111 min_rank=5 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 3840, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[531, 532, 508, 492, 540, 516, 475, 506, 508, 545, 521, 517, 502, 541, 548, 491, 472, 493, 513, 554, 537, 518, 534, 469, 497, 552, 497, 536, 491, 528, 487, 532, 494, 515, 495, 493, 513, 505, 539, 511, 503, 510, 534, 478, 521, 537, 488, 524, 485, 505, 535, 499, 537, 549, 544, 522, 515, 480, 470, 496, 525, 528, 493, 488, 544, 508, 521, 506, 543, 503, 501, 526, 513, 487, 533, 527, 519, 503, 499, 541, 499, 529, 499, 476, 515, 495, 509, 491, 506, 518, 490, 503, 495, 488, 502, 523, 496, 550, 525, 519, 514, 540, 520, 498, 513, 489, 524, 527, 535, 530, 495, 524, 482, 474, 527, 541, 499, 497, 517, 499, 508, 477, 525, 489, 497, 547, 523, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[534, 525, 574, 550, 501, 479, 502, 499, 553, 560, 539, 525, 512, 502, 521, 539, 479, 472, 531, 474, 517, 508, 464, 508, 516, 489, 509, 531, 494, 494, 499, 545, 491, 483, 506, 536, 494, 510, 489, 501, 510, 521, 527, 514, 532, 511, 479, 515, 490, 519, 536, 521, 513, 535, 548, 524, 516, 540, 499, 513, 497, 566, 509, 491, 519, 481, 520, 494, 528, 513, 520, 519, 510, 507, 505, 547, 490, 541, 510, 536, 518, 496, 523, 508, 523, 488, 501, 503, 536, 500, 457, 509, 512, 523, 513, 510, 484, 522, 511, 534, 535, 523, 509, 522, 521, 522, 522, 539, 521, 491, 512, 510, 505, 487, 515, 514, 469, 489, 510, 504, 531, 495, 510, 507, 484, 522, 504, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[474, 541, 543, 495, 508, 512, 481, 506, 493, 518, 471, 518, 543, 514, 536, 495, 505, 481, 555, 529, 550, 498, 501, 515, 516, 486, 507, 509, 502, 507, 520, 532, 525, 503, 550, 512, 508, 531, 520, 517, 508, 519, 521, 460, 501, 494, 489, 527, 498, 516, 507, 535, 519, 556, 532, 484, 533, 509, 517, 522, 521, 513, 523, 543, 521, 500, 533, 521, 509, 525, 514, 513, 557, 516, 487, 499, 541, 507, 542, 536, 504, 517, 505, 500, 510, 528, 533, 497, 481, 530, 505, 527, 480, 503, 499, 489, 481, 475, 517, 504, 491, 471, 550, 503, 499, 497, 493, 502, 524, 502, 513, 500, 509, 525, 526, 538, 490, 534, 505, 489, 500, 551, 499, 551, 480, 510, 494, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[538, 519, 517, 475, 526, 524, 500, 530, 534, 480, 540, 446, 533, 535, 547, 506, 508, 521, 499, 472, 511, 533, 508, 478, 510, 542, 500, 510, 520, 521, 506, 490, 523, 515, 535, 531, 505, 521, 502, 505, 491, 539, 514, 519, 527, 506, 509, 519, 535, 508, 520, 514, 517, 519, 538, 508, 519, 494, 527, 521, 462, 495, 518, 508, 481, 523, 533, 477, 504, 509, 508, 521, 518, 488, 513, 524, 510, 499, 489, 493, 490, 508, 517, 481, 494, 522, 497, 504, 532, 474, 485, 542, 520, 522, 507, 518, 537, 525, 498, 544, 513, 527, 546, 541, 531, 512, 495, 520, 532, 489, 507, 495, 534, 505, 493, 509, 488, 533, 523, 528, 529, 522, 489, 516, 505, 486, 493, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[518, 510, 509, 515, 467, 508, 479, 522, 479, 501, 492, 480, 527, 477, 459, 493, 547, 537, 504, 475, 519, 503, 520, 550, 508, 488, 501, 528, 525, 524, 499, 493, 503, 491, 499, 514, 523, 530, 505, 465, 482, 483, 514, 503, 544, 510, 501, 536, 551, 539, 520, 528, 488, 481, 529, 487, 495, 518, 513, 522, 516, 495, 521, 484, 465, 546, 499, 489, 541, 514, 525, 516, 477, 529, 493, 520, 522, 527, 516, 493, 527, 541, 532, 509, 540, 514, 520, 498, 499, 527, 542, 511, 546, 503, 510, 503, 537, 512, 578, 521, 512, 505, 508, 522, 477, 535, 511, 502, 536, 558, 515, 513, 518, 496, 526, 499, 543, 543, 514, 468, 525, 535, 513, 560, 464, 486, 546, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[505, 517, 486, 519, 512, 522, 500, 504, 511, 544, 521, 529, 531, 546, 523, 493, 492, 515, 506, 497, 513, 497, 515, 519, 523, 490, 487, 522, 549, 505, 549, 489, 501, 535, 518, 541, 517, 510, 528, 526, 487, 510, 480, 504, 512, 551, 521, 531, 504, 522, 507, 523, 494, 508, 526, 511, 501, 537, 512, 490, 541, 514, 522, 519, 492, 544, 521, 537, 537, 494, 518, 517, 520, 545, 488, 510, 487, 519, 559, 460, 495, 545, 490, 497, 518, 532, 513, 507, 516, 547, 475, 494, 519, 495, 511, 504, 500, 506, 539, 492, 521, 548, 519, 499, 517, 489, 451, 518, 513, 469, 496, 506, 481, 491, 548, 509, 488, 519, 504, 507, 497, 515, 493, 537, 515, 506, 495, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[543, 550, 504, 522, 483, 500, 560, 522, 517, 525, 527, 504, 484, 481, 493, 487, 472, 557, 505, 481, 529, 533, 528, 532, 517, 527, 511, 502, 511, 492, 492, 516, 502, 517, 522, 507, 496, 488, 522, 521, 487, 520, 529, 507, 520, 541, 537, 513, 513, 467, 538, 501, 520, 526, 508, 528, 526, 522, 561, 517, 530, 454, 502, 521, 507, 496, 508, 520, 530, 492, 495, 540, 517, 509, 492, 508, 506, 495, 572, 524, 532, 531, 502, 491, 498, 485, 482, 514, 505, 490, 515, 496, 535, 509, 496, 552, 505, 501, 498, 517, 489, 466, 504, 511, 510, 520, 506, 514, 513, 509, 510, 517, 521, 477, 530, 527, 498, 481, 522, 518, 514, 524, 537, 503, 517, 553, 515, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[557, 520, 500, 470, 514, 517, 519, 519, 500, 523, 508, 518, 475, 519, 523, 530, 491, 522, 506, 500, 511, 541, 503, 538, 542, 540, 501, 494, 478, 532, 531, 518, 506, 510, 494, 511, 525, 524, 506, 504, 477, 475, 506, 544, 518, 460, 488, 494, 479, 481, 468, 555, 464, 476, 477, 523, 499, 513, 546, 526, 477, 539, 523, 519, 487, 512, 512, 550, 534, 529, 511, 516, 508, 528, 516, 489, 504, 478, 519, 514, 508, 509, 509, 490, 531, 509, 529, 504, 513, 501, 504, 528, 540, 538, 508, 501, 523, 503, 518, 503, 547, 522, 520, 553, 522, 532, 500, 528, 539, 512, 499, 528, 532, 533, 511, 482, 498, 504, 518, 509, 550, 512, 507, 477, 493, 540, 504, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[531, 512, 508, 492, 540, 516, 475, 506, 508, 545, 521, 517, 502, 541, 548, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 493, 513, 554, 537, 518, 534, 469, 497, 552, 497, 536, 491, 528, 487, 532, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 495, 493, 513, 505, 539, 511, 503, 510, 534, 478, 521, 537, 488, 524, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 505, 535, 499, 537, 549, 519, 522, 515, 480, 470, 496, 525, 528, 493, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 508, 521, 506, 515, 503, 501, 526, 513, 487, 533, 527, 519, 503, 499, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 529, 499, 476, 515, 495, 509, 491, 506, 518, 490, 503, 495, 488, 502, 523, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 550, 525, 519, 514, 540, 520, 498, 513, 489, 524, 527, 505, 530, 495, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 474, 527, 541, 499, 497, 517, 499, 508, 477, 525, 489, 497, 547, 523, 509, 18, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8224, 8180, 8146, 8246, 8078, 8269, 8140] avg=8192.0000 max=8269 max_rank=6 min=8078 min_rank=5 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[474, 521, 543, 495, 508, 512, 481, 506, 493, 518, 471, 518, 543, 514, 536, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 481, 555, 529, 550, 498, 501, 515, 516, 486, 507, 509, 502, 507, 520, 532, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 503, 550, 512, 508, 531, 520, 517, 508, 519, 521, 460, 501, 494, 489, 527, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 516, 507, 535, 519, 556, 508, 484, 533, 509, 517, 522, 521, 513, 523, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 500, 533, 521, 483, 525, 514, 513, 557, 516, 487, 499, 541, 507, 542, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 505, 500, 510, 528, 533, 497, 481, 530, 505, 527, 480, 503, 499, 489, 29, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 475, 517, 504, 491, 471, 550, 503, 499, 497, 493, 502, 495, 502, 513, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 526, 538, 490, 534, 505, 489, 500, 551, 499, 551, 480, 510, 494, 510, 17, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8227, 8205, 8304, 8295, 8146, 7993, 8238] avg=8192.0000 max=8304 max_rank=3 min=7993 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[534, 498, 574, 550, 501, 479, 502, 499, 553, 560, 539, 525, 512, 502, 521, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 472, 531, 474, 517, 508, 464, 508, 516, 489, 509, 531, 494, 494, 499, 545, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 483, 506, 536, 494, 510, 489, 501, 510, 521, 527, 514, 532, 511, 479, 515, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, 536, 521, 513, 535, 514, 524, 516, 540, 499, 513, 497, 566, 509, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 481, 520, 494, 493, 513, 520, 519, 510, 507, 505, 547, 490, 541, 510, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 496, 523, 508, 523, 488, 501, 503, 536, 500, 457, 509, 512, 523, 513, 510, 32, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 522, 511, 534, 535, 523, 509, 522, 521, 522, 522, 539, 489, 491, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 515, 514, 469, 489, 510, 504, 531, 495, 510, 507, 484, 522, 504, 471, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8388, 8048, 8147, 8283, 8205, 8167, 8246, 8052] avg=8192.0000 max=8388 max_rank=0 min=8048 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[543, 530, 504, 522, 483, 500, 560, 522, 517, 525, 527, 504, 484, 481, 493, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 557, 505, 481, 529, 533, 528, 532, 517, 527, 511, 502, 511, 492, 492, 516, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 517, 522, 507, 496, 488, 522, 521, 487, 520, 529, 507, 520, 541, 537, 513, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 467, 538, 501, 520, 526, 485, 528, 526, 522, 561, 517, 530, 454, 502, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 496, 508, 520, 502, 492, 495, 540, 517, 509, 492, 508, 506, 495, 572, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 531, 502, 491, 498, 485, 482, 514, 505, 490, 515, 496, 535, 509, 496, 552, 29, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 501, 498, 517, 489, 466, 504, 511, 510, 520, 506, 514, 484, 509, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 477, 530, 527, 498, 481, 522, 518, 514, 524, 537, 503, 517, 553, 515, 495, 18, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8219, 8248, 8211, 8183, 8172, 8061, 8260] avg=8192.0000 max=8260 max_rank=7 min=8061 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[505, 498, 486, 519, 512, 522, 500, 504, 511, 544, 521, 529, 531, 546, 523, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 506, 497, 513, 497, 515, 519, 523, 490, 487, 522, 549, 505, 549, 489, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 535, 518, 541, 517, 510, 528, 526, 487, 510, 480, 504, 512, 551, 521, 531, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 507, 523, 494, 508, 502, 511, 501, 537, 512, 490, 541, 514, 522, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 544, 521, 537, 509, 494, 518, 517, 520, 545, 488, 510, 487, 519, 559, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 545, 490, 497, 518, 532, 513, 507, 516, 547, 475, 494, 519, 495, 511, 504, 29, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 506, 539, 492, 521, 548, 519, 499, 517, 489, 451, 518, 484, 469, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 491, 548, 509, 488, 519, 504, 507, 497, 515, 493, 537, 515, 506, 495, 508, 18, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8181, 8292, 8207, 8220, 8197, 8054, 8141] avg=8192.0000 max=8292 max_rank=2 min=8054 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[538, 500, 517, 475, 526, 524, 500, 530, 534, 480, 540, 446, 533, 535, 547, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 499, 472, 511, 533, 508, 478, 510, 542, 500, 510, 520, 521, 506, 490, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 515, 535, 531, 505, 521, 502, 505, 491, 539, 514, 519, 527, 506, 509, 519, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 508, 520, 514, 517, 519, 513, 508, 519, 494, 527, 521, 462, 495, 518, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 533, 477, 478, 509, 508, 521, 518, 488, 513, 524, 510, 499, 489, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 508, 517, 481, 494, 522, 497, 504, 532, 474, 485, 542, 520, 522, 507, 518, 30, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 498, 544, 513, 527, 546, 541, 531, 512, 495, 520, 502, 489, 507, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 505, 493, 509, 488, 533, 523, 528, 529, 522, 489, 516, 505, 486, 493, 525, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8142, 8281, 8178, 8064, 8152, 8282, 8206] avg=8192.0000 max=8282 max_rank=6 min=8064 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[557, 501, 500, 470, 514, 517, 519, 519, 500, 523, 508, 518, 475, 519, 523, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 522, 506, 500, 511, 541, 503, 538, 542, 540, 501, 494, 478, 532, 531, 518, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 494, 511, 525, 524, 506, 504, 477, 475, 506, 544, 518, 460, 488, 494, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 481, 468, 555, 464, 476, 456, 523, 499, 513, 546, 526, 477, 539, 523, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 512, 512, 550, 506, 529, 511, 516, 508, 528, 516, 489, 504, 478, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 509, 509, 490, 531, 509, 529, 504, 513, 501, 504, 528, 540, 538, 508, 501, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 518, 503, 547, 522, 520, 553, 522, 532, 500, 528, 509, 512, 499, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 533, 511, 482, 498, 504, 518, 509, 550, 512, 507, 477, 493, 540, 504, 521, 18, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8261, 8060, 8044, 8179, 8262, 8319, 8218] avg=8192.0000 max=8319 max_rank=6 min=8044 min_rank=3 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32978 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 491, 509, 515, 467, 508, 479, 522, 479, 501, 492, 480, 527, 477, 459, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 537, 504, 475, 519, 503, 520, 550, 508, 488, 501, 528, 525, 524, 499, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 499, 514, 523, 530, 505, 465, 482, 483, 514, 503, 544, 510, 501, 536, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 539, 520, 528, 488, 481, 505, 487, 495, 518, 513, 522, 516, 495, 521, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 546, 499, 489, 513, 514, 525, 516, 477, 529, 493, 520, 522, 527, 516, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 541, 532, 509, 540, 514, 520, 498, 499, 527, 542, 511, 546, 503, 510, 503, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 512, 578, 521, 512, 505, 508, 522, 477, 535, 511, 502, 506, 558, 515, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 496, 526, 499, 543, 543, 514, 468, 525, 535, 513, 560, 464, 486, 546, 517, 18, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7917, 8234, 8123, 8163, 8144, 8362, 8312, 8281] avg=8192.0000 max=8362 max_rank=5 min=7917 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=33099 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[509, 517, 542, 475, 531, 509, 522, 511, 523, 504, 485, 525, 536, 523, 536, 490, 513, 541, 517, 478, 487, 507, 528, 486, 500, 456, 480, 501, 493, 491, 559, 505, 535, 519, 506, 465, 552, 497, 511, 506, 517, 539, 521, 545, 509, 501, 514, 501, 505, 489, 540, 449, 517, 507, 502, 508, 513, 503, 530, 544, 488, 536, 509, 508, 483, 476, 505, 506, 561, 531, 476, 533, 519, 533, 464, 500, 537, 509, 506, 512, 524, 510, 535, 494, 515, 520, 500, 488, 481, 492, 533, 568, 511, 485, 521, 552, 520, 484, 486, 517, 524, 530, 497, 501, 520, 518, 573, 511, 525, 491, 516, 523, 542, 521, 515, 494, 550, 512, 527, 531, 513, 501, 523, 494, 502, 517, 503, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[530, 544, 569, 501, 494, 534, 539, 467, 524, 489, 456, 524, 520, 532, 522, 508, 496, 511, 515, 502, 544, 517, 532, 477, 460, 506, 510, 508, 508, 526, 497, 521, 545, 502, 491, 443, 490, 541, 483, 535, 518, 497, 478, 533, 524, 517, 556, 480, 514, 495, 558, 530, 532, 493, 469, 549, 473, 554, 513, 481, 533, 529, 541, 501, 493, 513, 513, 525, 519, 500, 485, 541, 517, 539, 497, 479, 500, 515, 547, 513, 520, 501, 499, 472, 525, 545, 524, 498, 514, 535, 503, 534, 526, 503, 532, 479, 519, 536, 529, 510, 521, 466, 522, 543, 505, 477, 526, 483, 499, 479, 537, 544, 551, 523, 510, 513, 486, 483, 519, 521, 476, 533, 536, 496, 479, 481, 508, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[469, 526, 524, 453, 476, 483, 497, 521, 528, 557, 493, 502, 523, 562, 478, 499, 483, 493, 510, 531, 527, 551, 517, 524, 534, 460, 482, 491, 512, 509, 481, 487, 521, 511, 479, 503, 520, 512, 509, 475, 451, 496, 491, 546, 526, 541, 495, 491, 541, 514, 475, 490, 492, 497, 464, 512, 526, 528, 519, 498, 483, 521, 484, 510, 502, 558, 500, 547, 518, 532, 520, 526, 470, 503, 512, 553, 535, 535, 513, 508, 505, 550, 514, 535, 499, 462, 514, 475, 500, 518, 515, 494, 531, 511, 537, 504, 502, 527, 516, 569, 514, 526, 497, 521, 544, 526, 562, 506, 540, 543, 504, 547, 482, 530, 517, 482, 539, 523, 475, 573, 535, 497, 508, 559, 508, 521, 542, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[513, 507, 517, 517, 513, 517, 514, 479, 546, 533, 516, 486, 500, 502, 523, 527, 542, 472, 495, 516, 532, 486, 500, 508, 501, 476, 512, 537, 476, 501, 442, 481, 503, 490, 474, 503, 531, 509, 528, 543, 488, 515, 503, 531, 507, 495, 490, 521, 537, 497, 512, 510, 522, 511, 529, 518, 474, 529, 529, 508, 513, 485, 504, 519, 510, 504, 466, 510, 552, 519, 506, 536, 488, 528, 503, 507, 534, 522, 520, 525, 520, 530, 539, 475, 504, 474, 533, 515, 528, 516, 541, 517, 521, 525, 524, 525, 482, 522, 561, 544, 539, 491, 507, 520, 490, 536, 487, 536, 482, 545, 535, 549, 514, 509, 489, 534, 533, 521, 510, 538, 519, 487, 509, 484, 508, 496, 514, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[511, 527, 529, 531, 520, 511, 511, 529, 534, 489, 517, 508, 467, 526, 536, 529, 504, 527, 570, 471, 505, 463, 571, 490, 488, 488, 545, 447, 512, 516, 536, 486, 501, 526, 541, 518, 528, 527, 495, 549, 515, 521, 492, 496, 490, 526, 485, 504, 475, 481, 542, 511, 501, 522, 494, 506, 486, 503, 544, 482, 517, 512, 487, 505, 510, 514, 492, 530, 510, 508, 562, 502, 479, 544, 521, 502, 498, 513, 484, 516, 547, 521, 528, 513, 495, 480, 522, 527, 492, 521, 590, 502, 519, 546, 503, 520, 525, 552, 501, 514, 513, 508, 489, 499, 492, 561, 522, 500, 518, 555, 503, 493, 498, 487, 544, 492, 546, 489, 519, 527, 488, 519, 479, 498, 501, 498, 497, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[515, 518, 517, 532, 531, 544, 502, 514, 527, 573, 506, 513, 532, 540, 521, 484, 529, 497, 494, 491, 518, 533, 470, 480, 513, 514, 494, 501, 497, 489, 491, 492, 533, 542, 520, 499, 493, 518, 505, 477, 531, 492, 517, 497, 488, 536, 503, 513, 497, 497, 486, 498, 506, 506, 484, 507, 528, 502, 516, 485, 515, 487, 540, 511, 501, 547, 540, 484, 501, 552, 509, 499, 495, 491, 509, 503, 488, 513, 495, 518, 554, 513, 570, 542, 484, 552, 472, 526, 514, 542, 540, 497, 491, 537, 505, 504, 560, 531, 485, 509, 544, 512, 528, 508, 505, 494, 530, 488, 528, 523, 519, 505, 517, 516, 516, 529, 490, 514, 503, 488, 547, 536, 494, 502, 568, 479, 494, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[519, 533, 500, 526, 537, 474, 465, 486, 517, 526, 494, 522, 499, 479, 544, 497, 477, 499, 525, 463, 539, 507, 512, 506, 524, 519, 514, 495, 514, 518, 503, 529, 510, 511, 501, 540, 531, 526, 544, 520, 501, 540, 511, 537, 534, 499, 519, 508, 505, 514, 525, 578, 485, 528, 511, 513, 500, 531, 548, 550, 493, 518, 501, 531, 485, 536, 481, 469, 528, 504, 543, 521, 483, 509, 519, 542, 525, 502, 511, 494, 524, 523, 474, 495, 480, 518, 507, 526, 466, 519, 515, 510, 534, 520, 550, 495, 561, 524, 519, 541, 505, 480, 492, 501, 515, 505, 509, 491, 507, 561, 523, 505, 489, 515, 526, 512, 500, 470, 539, 475, 511, 517, 520, 497, 507, 509, 476, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[514, 544, 495, 503, 539, 491, 560, 548, 477, 510, 532, 531, 540, 530, 451, 506, 484, 500, 533, 529, 535, 504, 531, 498, 540, 548, 541, 523, 467, 515, 508, 505, 476, 490, 464, 490, 539, 488, 537, 470, 518, 547, 502, 519, 514, 488, 544, 494, 523, 530, 507, 517, 518, 534, 489, 545, 538, 508, 553, 512, 513, 504, 525, 463, 559, 501, 474, 528, 471, 474, 538, 534, 544, 528, 514, 499, 503, 493, 530, 541, 516, 518, 519, 532, 534, 534, 532, 506, 466, 481, 490, 509, 507, 516, 525, 497, 499, 497, 514, 513, 510, 526, 502, 536, 551, 518, 522, 503, 543, 499, 487, 480, 518, 458, 485, 513, 506, 531, 531, 484, 494, 486, 509, 508, 488, 513, 502, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[469, 491, 524, 453, 476, 483, 497, 521, 528, 557, 493, 502, 523, 562, 478, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 493, 510, 531, 527, 551, 517, 524, 534, 460, 482, 491, 512, 509, 481, 487, 82, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 479, 503, 520, 512, 509, 475, 451, 496, 491, 546, 526, 541, 495, 491, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 514, 475, 490, 492, 497, 464, 512, 526, 528, 519, 498, 483, 521, 484, 510, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 558, 500, 547, 518, 532, 520, 516, 470, 503, 512, 553, 535, 535, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 550, 514, 535, 499, 462, 514, 475, 500, 518, 472, 494, 531, 511, 537, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 527, 516, 569, 514, 526, 497, 521, 544, 526, 480, 506, 540, 543, 504, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 530, 517, 482, 539, 523, 475, 573, 535, 497, 508, 559, 508, 521, 542, 501, 25, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8056, 8192, 8089, 8069, 8322, 8121, 8362, 8325] avg=8192.0000 max=8362 max_rank=6 min=8056 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=33130 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[509, 483, 542, 475, 531, 509, 522, 511, 523, 504, 485, 525, 536, 523, 536, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 541, 517, 478, 487, 507, 528, 486, 500, 456, 480, 501, 493, 491, 559, 505, 83, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 506, 465, 552, 497, 511, 506, 517, 539, 521, 545, 509, 501, 514, 501, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 489, 540, 449, 517, 507, 502, 508, 513, 503, 530, 544, 488, 536, 509, 508, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 476, 505, 506, 561, 531, 476, 523, 519, 533, 464, 500, 537, 509, 506, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 535, 494, 515, 520, 500, 488, 481, 492, 489, 568, 511, 485, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 484, 486, 517, 524, 530, 497, 501, 520, 518, 490, 511, 525, 491, 516, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 521, 515, 494, 550, 512, 527, 531, 513, 501, 523, 494, 502, 517, 503, 509, 26, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8143, 8259, 8163, 8141, 8185, 8153, 8288] avg=8192.0000 max=8288 max_rank=7 min=8141 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 509, 569, 501, 494, 534, 539, 467, 524, 489, 456, 524, 520, 532, 522, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 515, 502, 544, 517, 532, 477, 460, 506, 510, 508, 508, 526, 497, 521, 76, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 502, 491, 443, 490, 541, 483, 535, 518, 497, 478, 533, 524, 517, 556, 480, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 495, 558, 530, 532, 493, 469, 549, 473, 554, 513, 481, 533, 529, 541, 501, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 513, 525, 519, 500, 485, 530, 517, 539, 497, 479, 500, 515, 547, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 499, 472, 525, 545, 524, 498, 514, 535, 461, 534, 526, 503, 532, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 536, 529, 510, 521, 466, 522, 543, 505, 477, 450, 483, 499, 479, 537, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 523, 510, 513, 486, 483, 519, 521, 476, 533, 536, 496, 479, 481, 508, 538, 25, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8223, 8155, 8281, 8185, 8168, 8120, 8186] avg=8192.0000 max=8281 max_rank=3 min=8120 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[513, 461, 517, 517, 513, 517, 514, 479, 546, 533, 516, 486, 500, 502, 523, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 472, 495, 516, 532, 486, 500, 508, 501, 476, 512, 537, 476, 501, 442, 481, 75, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 490, 474, 503, 531, 509, 528, 543, 488, 515, 503, 531, 507, 495, 490, 521, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 497, 512, 510, 522, 511, 529, 518, 474, 529, 529, 508, 513, 485, 504, 519, 15, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 466, 510, 552, 519, 506, 521, 488, 528, 503, 507, 534, 522, 520, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 530, 539, 475, 504, 474, 533, 515, 528, 516, 488, 517, 521, 525, 524, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 522, 561, 544, 539, 491, 507, 520, 490, 536, 412, 536, 482, 545, 535, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 509, 489, 534, 533, 521, 510, 538, 519, 487, 509, 484, 508, 496, 514, 513, 31, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8074, 8156, 8220, 8215, 8234, 8251, 8222] avg=8192.0000 max=8251 max_rank=6 min=8074 min_rank=1 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 492, 529, 531, 520, 511, 511, 529, 534, 489, 517, 508, 467, 526, 536, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 527, 570, 471, 505, 463, 571, 490, 488, 488, 545, 447, 512, 516, 536, 486, 76, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 541, 518, 528, 527, 495, 549, 515, 521, 492, 496, 490, 526, 485, 504, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 481, 542, 511, 501, 522, 494, 506, 486, 503, 544, 482, 517, 512, 487, 505, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 514, 492, 530, 510, 508, 562, 492, 479, 544, 521, 502, 498, 513, 484, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 521, 528, 513, 495, 480, 522, 527, 492, 521, 541, 502, 519, 546, 503, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 552, 501, 514, 513, 508, 489, 499, 492, 561, 446, 500, 518, 555, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 487, 544, 492, 546, 489, 519, 527, 488, 519, 479, 498, 501, 498, 497, 522, 29, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8215, 8236, 8083, 8175, 8277, 8169, 8141] avg=8192.0000 max=8277 max_rank=5 min=8083 min_rank=3 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 509, 495, 503, 539, 491, 560, 548, 477, 510, 532, 531, 540, 530, 451, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 500, 533, 529, 535, 504, 531, 498, 540, 548, 541, 523, 467, 515, 508, 505, 76, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 490, 464, 490, 539, 488, 537, 470, 518, 547, 502, 519, 514, 488, 544, 494, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 530, 507, 517, 518, 534, 489, 545, 538, 508, 553, 512, 513, 504, 525, 463, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 501, 474, 528, 471, 474, 538, 524, 544, 528, 514, 499, 503, 493, 530, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 519, 532, 534, 534, 532, 506, 466, 481, 449, 509, 507, 516, 525, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 497, 514, 513, 510, 526, 502, 536, 551, 518, 446, 503, 543, 499, 487, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 458, 485, 513, 506, 531, 531, 484, 494, 486, 509, 508, 488, 513, 502, 506, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8354, 8102, 8294, 8221, 8141, 8124, 8064] avg=8192.0000 max=8354 max_rank=1 min=8064 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[515, 484, 517, 532, 531, 544, 502, 514, 527, 573, 506, 513, 532, 540, 521, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 497, 494, 491, 518, 533, 470, 480, 513, 514, 494, 501, 497, 489, 491, 492, 77, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 542, 520, 499, 493, 518, 505, 477, 531, 492, 517, 497, 488, 536, 503, 513, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 497, 486, 498, 506, 506, 484, 507, 528, 502, 516, 485, 515, 487, 540, 511, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 547, 540, 484, 501, 552, 509, 489, 495, 491, 509, 503, 488, 513, 495, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 513, 570, 542, 484, 552, 472, 526, 514, 542, 494, 497, 491, 537, 505, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 531, 485, 509, 544, 512, 528, 508, 505, 494, 453, 488, 528, 523, 519, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 516, 516, 529, 490, 514, 503, 488, 547, 536, 494, 502, 568, 479, 494, 485, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8335, 8099, 8185, 8080, 8135, 8297, 8192, 8213] avg=8192.0000 max=8335 max_rank=0 min=8080 min_rank=3 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[519, 498, 500, 526, 537, 474, 465, 486, 517, 526, 494, 522, 499, 479, 544, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 499, 525, 463, 539, 507, 512, 506, 524, 519, 514, 495, 514, 518, 503, 529, 74, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 501, 540, 531, 526, 544, 520, 501, 540, 511, 537, 534, 499, 519, 508, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 525, 578, 485, 528, 511, 513, 500, 531, 548, 550, 493, 518, 501, 531, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 536, 481, 469, 528, 504, 543, 511, 483, 509, 519, 542, 525, 502, 511, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 523, 474, 495, 480, 518, 507, 526, 466, 519, 472, 510, 534, 520, 550, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 524, 519, 541, 505, 480, 492, 501, 515, 505, 435, 491, 507, 561, 523, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 515, 526, 512, 500, 470, 539, 475, 511, 517, 520, 497, 507, 509, 476, 501, 25, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8236, 8354, 8346, 8142, 8113, 8165, 8097] avg=8192.0000 max=8354 max_rank=2 min=8083 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 3968, 4224, 3968, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 3712, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 3968, 4224, 4096, 4224, 4352, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 4096, 4096, 4096, 4096, 4096, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[533, 477, 512, 530, 537, 527, 506, 503, 466, 524, 498, 515, 460, 505, 528, 530, 511, 524, 499, 527, 527, 523, 548, 538, 536, 488, 525, 517, 513, 509, 479, 449, 479, 554, 500, 504, 503, 538, 513, 537, 519, 515, 483, 523, 528, 530, 531, 549, 551, 513, 500, 500, 513, 481, 475, 535, 466, 514, 545, 498, 557, 517, 488, 485, 543, 508, 500, 518, 501, 509, 521, 492, 516, 519, 505, 469, 504, 541, 502, 504, 515, 505, 476, 498, 520, 509, 522, 500, 539, 510, 483, 507, 530, 498, 533, 565, 485, 540, 542, 494, 499, 488, 559, 462, 488, 489, 532, 497, 535, 504, 533, 507, 514, 527, 516, 522, 519, 520, 535, 491, 484, 480, 507, 502, 545, 518, 542, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[533, 520, 496, 499, 476, 527, 538, 489, 536, 490, 533, 497, 512, 538, 520, 499, 527, 504, 522, 501, 483, 505, 473, 502, 527, 502, 518, 486, 525, 487, 518, 499, 487, 522, 506, 533, 519, 472, 505, 501, 525, 502, 492, 524, 516, 488, 495, 501, 510, 486, 517, 494, 552, 504, 516, 514, 496, 480, 538, 507, 507, 487, 509, 501, 491, 504, 530, 530, 522, 525, 506, 514, 507, 512, 473, 556, 501, 530, 541, 504, 500, 565, 510, 496, 544, 507, 514, 536, 513, 563, 527, 520, 529, 497, 508, 505, 524, 510, 501, 500, 520, 521, 516, 528, 527, 491, 525, 501, 526, 501, 537, 505, 524, 531, 499, 488, 511, 505, 507, 525, 499, 549, 548, 512, 526, 521, 506, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[499, 521, 517, 493, 547, 520, 524, 498, 519, 533, 550, 504, 528, 507, 494, 502, 503, 484, 536, 529, 482, 490, 506, 497, 461, 524, 552, 503, 530, 515, 535, 518, 512, 523, 497, 533, 520, 508, 520, 471, 503, 487, 511, 480, 505, 494, 517, 519, 511, 551, 494, 529, 543, 483, 530, 465, 537, 474, 504, 529, 513, 538, 513, 515, 488, 508, 531, 540, 537, 487, 525, 494, 487, 516, 529, 540, 541, 494, 511, 483, 545, 484, 500, 530, 486, 479, 492, 495, 480, 538, 513, 502, 498, 517, 516, 524, 506, 490, 501, 517, 478, 500, 542, 527, 509, 516, 478, 545, 485, 548, 548, 564, 528, 543, 448, 495, 515, 564, 503, 502, 527, 507, 505, 534, 487, 537, 526, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[499, 504, 542, 483, 516, 532, 523, 538, 493, 522, 558, 459, 546, 520, 521, 525, 480, 489, 487, 510, 529, 569, 517, 525, 514, 515, 545, 476, 508, 522, 545, 529, 492, 481, 513, 508, 499, 486, 455, 485, 506, 551, 520, 508, 494, 513, 523, 532, 529, 531, 493, 508, 482, 505, 485, 516, 546, 536, 500, 554, 521, 523, 516, 509, 522, 476, 532, 499, 506, 526, 500, 499, 497, 485, 547, 526, 496, 505, 515, 483, 524, 503, 494, 491, 504, 507, 513, 535, 497, 496, 530, 508, 499, 511, 545, 524, 528, 467, 569, 525, 483, 503, 507, 499, 511, 500, 502, 519, 473, 509, 522, 534, 499, 500, 522, 479, 507, 492, 522, 556, 499, 528, 540, 512, 507, 511, 535, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[488, 524, 511, 542, 523, 514, 508, 476, 483, 525, 510, 507, 528, 508, 474, 518, 501, 475, 476, 535, 511, 507, 518, 507, 506, 496, 496, 506, 463, 508, 534, 539, 528, 493, 505, 552, 487, 510, 488, 506, 501, 502, 534, 509, 538, 518, 532, 529, 516, 540, 509, 507, 499, 536, 512, 506, 495, 517, 541, 530, 497, 547, 511, 550, 543, 505, 464, 517, 513, 516, 542, 509, 505, 504, 503, 467, 523, 512, 565, 519, 515, 512, 500, 514, 478, 484, 506, 506, 511, 481, 513, 551, 527, 573, 506, 556, 522, 487, 529, 528, 492, 520, 508, 511, 509, 533, 499, 525, 491, 491, 516, 502, 477, 511, 516, 477, 525, 524, 515, 530, 517, 559, 478, 544, 499, 530, 480, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[507, 521, 535, 538, 475, 540, 545, 538, 502, 506, 491, 498, 532, 540, 522, 514, 530, 547, 521, 500, 501, 503, 503, 468, 493, 528, 499, 497, 548, 527, 533, 469, 510, 517, 503, 534, 505, 495, 525, 507, 500, 478, 505, 492, 481, 511, 507, 533, 530, 528, 489, 494, 518, 534, 482, 510, 503, 525, 496, 545, 504, 501, 538, 534, 509, 490, 464, 489, 501, 520, 501, 512, 482, 508, 520, 505, 519, 524, 480, 499, 504, 509, 546, 553, 530, 505, 493, 500, 540, 510, 518, 482, 474, 549, 526, 479, 514, 480, 488, 527, 498, 544, 526, 536, 527, 541, 480, 515, 485, 535, 482, 504, 530, 524, 533, 484, 535, 536, 535, 547, 535, 526, 506, 518, 489, 500, 520, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[534, 499, 494, 519, 513, 508, 510, 504, 471, 530, 480, 529, 513, 524, 528, 500, 523, 491, 512, 504, 516, 498, 519, 519, 482, 492, 500, 498, 545, 504, 528, 533, 564, 470, 522, 526, 500, 505, 501, 538, 489, 522, 514, 509, 508, 502, 522, 508, 530, 528, 535, 522, 508, 504, 523, 517, 513, 503, 496, 515, 538, 550, 538, 546, 503, 504, 533, 532, 533, 509, 476, 497, 519, 521, 530, 516, 512, 500, 508, 514, 477, 469, 483, 510, 503, 522, 544, 495, 516, 545, 490, 517, 500, 500, 536, 527, 504, 508, 507, 532, 503, 519, 547, 546, 546, 496, 498, 509, 502, 487, 547, 538, 482, 494, 501, 473, 534, 491, 505, 500, 494, 508, 518, 491, 506, 490, 530, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[523, 498, 520, 520, 529, 560, 515, 488, 504, 506, 520, 536, 532, 511, 520, 514, 509, 509, 535, 495, 524, 511, 498, 522, 489, 508, 492, 532, 512, 477, 511, 519, 528, 558, 493, 542, 477, 507, 506, 523, 485, 510, 522, 478, 547, 508, 508, 478, 478, 526, 504, 491, 491, 531, 513, 534, 524, 494, 499, 504, 499, 531, 524, 498, 524, 496, 545, 502, 475, 529, 507, 539, 537, 486, 516, 556, 531, 523, 546, 502, 517, 525, 551, 559, 520, 524, 521, 536, 537, 510, 522, 533, 484, 523, 579, 498, 499, 544, 472, 472, 484, 476, 507, 505, 509, 483, 508, 523, 490, 498, 492, 531, 498, 518, 508, 515, 524, 505, 439, 523, 475, 492, 515, 524, 496, 518, 500, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 477, 512, 530, 537, 497, 506, 503, 466, 524, 498, 515, 460, 505, 528, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 524, 499, 527, 527, 523, 548, 538, 536, 488, 525, 517, 513, 509, 479, 449, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 554, 500, 504, 503, 538, 513, 537, 519, 515, 483, 523, 528, 530, 531, 549, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 483, 500, 500, 513, 481, 475, 535, 466, 514, 545, 498, 557, 517, 488, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 508, 500, 518, 501, 509, 521, 492, 516, 519, 505, 469, 504, 541, 502, 504, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 505, 476, 498, 520, 509, 522, 500, 539, 510, 483, 507, 530, 498, 495, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 540, 542, 494, 499, 488, 559, 462, 488, 489, 532, 497, 535, 504, 533, 507, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 527, 516, 522, 519, 520, 535, 491, 484, 480, 507, 502, 545, 518, 542, 490, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8245, 8339, 8108, 8159, 8172, 8168, 8224] avg=8192.0000 max=8339 max_rank=2 min=8108 min_rank=3 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[507, 521, 535, 538, 475, 510, 545, 538, 502, 506, 491, 498, 532, 540, 522, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 547, 521, 500, 501, 503, 503, 468, 493, 528, 499, 497, 548, 527, 533, 469, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 517, 503, 534, 505, 495, 525, 507, 500, 478, 505, 492, 481, 511, 507, 533, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 497, 489, 494, 518, 534, 482, 510, 503, 525, 496, 545, 504, 501, 538, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 464, 489, 501, 520, 501, 512, 482, 508, 520, 505, 519, 524, 480, 499, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 546, 553, 530, 505, 493, 500, 540, 510, 518, 482, 474, 549, 488, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 480, 488, 527, 498, 544, 526, 536, 527, 541, 480, 515, 485, 535, 482, 504, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 524, 533, 484, 535, 536, 535, 547, 535, 526, 506, 518, 489, 500, 520, 490, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8200, 8136, 8200, 8030, 8180, 8196, 8320] avg=8192.0000 max=8320 max_rank=7 min=8030 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[534, 499, 494, 519, 513, 480, 510, 504, 471, 530, 480, 529, 513, 524, 528, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 491, 512, 504, 516, 498, 519, 519, 482, 492, 500, 498, 545, 504, 528, 533, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 470, 522, 526, 500, 505, 501, 538, 489, 522, 514, 509, 508, 502, 522, 508, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 497, 535, 522, 508, 504, 523, 517, 513, 503, 496, 515, 538, 550, 538, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 533, 532, 533, 509, 476, 497, 519, 521, 530, 516, 512, 500, 508, 514, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 469, 483, 510, 503, 522, 544, 495, 516, 545, 490, 517, 500, 500, 497, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 508, 507, 532, 503, 519, 547, 546, 546, 496, 498, 509, 502, 487, 547, 538, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 494, 501, 473, 534, 491, 505, 500, 494, 508, 518, 491, 506, 490, 530, 503, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8197, 8234, 8335, 8214, 8095, 8302, 8031] avg=8192.0000 max=8335 max_rank=3 min=8031 min_rank=7 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 520, 496, 499, 476, 497, 538, 489, 536, 490, 533, 497, 512, 538, 520, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 504, 522, 501, 483, 505, 473, 502, 527, 502, 518, 486, 525, 487, 518, 499, 26, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 506, 533, 519, 472, 505, 501, 525, 502, 492, 524, 516, 488, 495, 501, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 457, 517, 494, 552, 504, 516, 514, 496, 480, 538, 507, 507, 487, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 504, 530, 530, 522, 525, 506, 514, 507, 512, 473, 556, 501, 530, 541, 504, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 565, 510, 496, 544, 507, 514, 536, 513, 563, 527, 520, 529, 497, 472, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 501, 500, 520, 521, 516, 528, 527, 491, 525, 501, 526, 501, 537, 505, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 531, 499, 488, 511, 505, 507, 525, 499, 549, 548, 512, 526, 521, 506, 484, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8109, 8120, 8089, 8253, 8298, 8247, 8247] avg=8192.0000 max=8298 max_rank=5 min=8089 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=33045 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[499, 504, 542, 483, 516, 490, 523, 538, 493, 522, 558, 459, 546, 520, 521, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 489, 487, 510, 529, 569, 517, 525, 514, 515, 545, 476, 508, 522, 545, 529, 31, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 481, 513, 508, 499, 486, 455, 485, 506, 551, 520, 508, 494, 513, 523, 532, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 493, 493, 508, 482, 505, 485, 516, 546, 536, 500, 554, 521, 523, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 476, 532, 499, 506, 526, 500, 499, 497, 485, 547, 526, 496, 505, 515, 483, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 503, 494, 491, 504, 507, 513, 535, 497, 496, 530, 508, 499, 511, 497, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 467, 569, 525, 483, 503, 507, 499, 511, 500, 502, 519, 473, 509, 522, 534, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 500, 522, 479, 507, 492, 522, 556, 499, 528, 540, 512, 507, 511, 535, 520, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8300, 8105, 8216, 8126, 8133, 8173, 8244] avg=8192.0000 max=8300 max_rank=1 min=8105 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 524, 511, 542, 523, 485, 508, 476, 483, 525, 510, 507, 528, 508, 474, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 475, 476, 535, 511, 507, 518, 507, 506, 496, 496, 506, 463, 508, 534, 539, 29, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 493, 505, 552, 487, 510, 488, 506, 501, 502, 534, 509, 538, 518, 532, 529, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 509, 507, 499, 536, 512, 506, 495, 517, 541, 530, 497, 547, 511, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 505, 464, 517, 513, 516, 542, 509, 505, 504, 503, 467, 523, 512, 565, 519, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 500, 514, 478, 484, 506, 506, 511, 481, 513, 551, 527, 573, 470, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 487, 529, 528, 492, 520, 508, 511, 509, 533, 499, 525, 491, 491, 516, 502, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 511, 516, 477, 525, 524, 515, 530, 517, 559, 478, 544, 499, 530, 480, 489, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8111, 8264, 8281, 8214, 8197, 8176, 8183] avg=8192.0000 max=8281 max_rank=3 min=8110 min_rank=0 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[499, 521, 517, 493, 547, 491, 524, 498, 519, 533, 550, 504, 528, 507, 494, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 484, 536, 529, 482, 490, 506, 497, 461, 524, 552, 503, 530, 515, 535, 518, 29, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 497, 533, 520, 508, 520, 471, 503, 487, 511, 480, 505, 494, 517, 519, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 494, 529, 543, 483, 530, 465, 537, 474, 504, 529, 513, 538, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 508, 531, 540, 537, 487, 525, 494, 487, 516, 529, 540, 541, 494, 511, 483, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 484, 500, 530, 486, 479, 492, 495, 480, 538, 513, 502, 498, 517, 480, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 490, 501, 517, 478, 500, 542, 527, 509, 516, 478, 545, 485, 548, 548, 564, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 543, 448, 495, 515, 564, 503, 502, 527, 507, 505, 534, 487, 537, 526, 501, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8198, 8132, 8197, 8218, 8063, 8267, 8234] avg=8192.0000 max=8267 max_rank=6 min=8063 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[523, 498, 520, 520, 529, 528, 515, 488, 504, 506, 520, 536, 532, 511, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 535, 495, 524, 511, 498, 522, 489, 508, 492, 532, 512, 477, 511, 519, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 558, 493, 542, 477, 507, 506, 523, 485, 510, 522, 478, 547, 508, 508, 478, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 495, 504, 491, 491, 531, 513, 534, 524, 494, 499, 504, 499, 531, 524, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 545, 502, 475, 529, 507, 539, 537, 486, 516, 556, 531, 523, 546, 502, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 551, 559, 520, 524, 521, 536, 537, 510, 522, 533, 484, 523, 538, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 544, 472, 472, 484, 476, 507, 505, 509, 483, 508, 523, 490, 498, 492, 531, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 518, 508, 515, 524, 505, 439, 523, 475, 492, 515, 524, 496, 518, 500, 490, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8176, 8206, 8110, 8322, 8398, 8007, 8053] avg=8192.0000 max=8398 max_rank=5 min=8007 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[497, 509, 492, 547, 501, 517, 439, 494, 493, 509, 528, 554, 476, 489, 528, 528, 507, 560, 521, 526, 548, 524, 531, 547, 476, 522, 504, 520, 492, 510, 495, 535, 485, 545, 501, 482, 531, 518, 489, 518, 496, 519, 484, 516, 514, 455, 568, 524, 484, 470, 499, 513, 524, 468, 509, 528, 556, 505, 482, 481, 501, 521, 548, 497, 508, 495, 511, 474, 492, 516, 517, 504, 529, 546, 507, 501, 520, 459, 508, 522, 534, 477, 494, 524, 507, 519, 560, 520, 525, 544, 509, 541, 491, 515, 531, 515, 501, 546, 464, 488, 527, 523, 502, 505, 509, 525, 530, 527, 514, 493, 513, 532, 476, 497, 505, 509, 521, 514, 516, 522, 507, 516, 527, 557, 526, 516, 511, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[503, 540, 493, 488, 482, 495, 521, 497, 505, 512, 514, 529, 470, 525, 530, 503, 469, 497, 547, 536, 533, 497, 529, 516, 531, 484, 515, 459, 544, 481, 511, 475, 496, 516, 516, 549, 533, 475, 496, 514, 517, 511, 487, 538, 503, 492, 488, 476, 495, 486, 474, 512, 548, 513, 517, 493, 498, 499, 503, 496, 518, 496, 485, 520, 537, 529, 502, 502, 542, 513, 550, 521, 525, 484, 537, 526, 515, 537, 489, 531, 540, 475, 506, 506, 506, 490, 489, 553, 514, 525, 511, 546, 517, 511, 513, 545, 523, 531, 516, 496, 495, 489, 532, 501, 494, 520, 541, 548, 485, 552, 535, 508, 538, 527, 488, 569, 504, 518, 530, 496, 527, 533, 528, 497, 483, 523, 522, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[522, 488, 505, 494, 506, 528, 510, 514, 560, 488, 517, 516, 527, 496, 547, 525, 516, 565, 489, 549, 491, 537, 506, 545, 542, 496, 484, 494, 534, 540, 511, 512, 473, 528, 534, 525, 524, 510, 520, 490, 503, 549, 492, 536, 522, 495, 489, 516, 517, 516, 520, 509, 515, 514, 495, 485, 500, 511, 525, 501, 530, 476, 525, 528, 509, 494, 518, 492, 475, 515, 525, 493, 488, 509, 512, 501, 526, 522, 524, 525, 484, 525, 534, 537, 467, 531, 523, 509, 492, 500, 503, 455, 468, 519, 517, 518, 512, 460, 514, 484, 490, 530, 512, 492, 540, 481, 519, 513, 541, 510, 539, 478, 515, 485, 542, 522, 489, 494, 492, 523, 539, 534, 544, 530, 497, 524, 515, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[523, 476, 532, 533, 519, 515, 481, 523, 507, 491, 540, 500, 482, 529, 520, 542, 536, 536, 491, 543, 497, 519, 523, 506, 477, 491, 515, 494, 519, 507, 463, 497, 485, 497, 509, 522, 516, 524, 507, 459, 505, 538, 504, 511, 509, 480, 517, 538, 520, 485, 522, 506, 538, 505, 518, 536, 533, 536, 515, 493, 513, 539, 466, 537, 535, 510, 515, 518, 505, 486, 527, 515, 500, 507, 540, 496, 527, 499, 511, 488, 517, 522, 513, 521, 500, 560, 517, 506, 507, 537, 500, 506, 467, 488, 517, 573, 513, 540, 492, 502, 474, 531, 492, 491, 510, 547, 518, 538, 505, 530, 525, 523, 532, 483, 556, 486, 493, 500, 522, 560, 530, 504, 502, 491, 515, 496, 516, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[525, 545, 526, 526, 496, 512, 491, 492, 544, 562, 505, 509, 503, 536, 520, 522, 462, 541, 492, 522, 551, 513, 528, 540, 529, 489, 494, 542, 508, 530, 515, 477, 492, 473, 478, 491, 514, 500, 476, 573, 521, 526, 488, 476, 526, 519, 469, 510, 537, 520, 506, 469, 525, 538, 493, 482, 506, 513, 566, 522, 500, 562, 474, 482, 515, 478, 512, 505, 542, 522, 530, 489, 541, 527, 500, 471, 489, 525, 543, 523, 524, 487, 496, 503, 475, 493, 536, 536, 513, 510, 527, 517, 535, 501, 501, 533, 581, 542, 499, 495, 505, 508, 496, 495, 476, 514, 530, 518, 462, 520, 538, 517, 503, 511, 517, 503, 518, 525, 476, 525, 507, 489, 491, 510, 515, 519, 544, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[478, 479, 523, 505, 546, 519, 530, 553, 501, 522, 541, 521, 534, 508, 488, 501, 491, 480, 539, 553, 504, 479, 508, 512, 525, 484, 526, 500, 497, 503, 530, 534, 544, 526, 522, 449, 537, 538, 510, 541, 511, 546, 517, 504, 483, 527, 526, 503, 511, 495, 494, 496, 467, 538, 520, 485, 501, 509, 519, 519, 524, 516, 520, 518, 514, 503, 521, 513, 522, 523, 512, 516, 492, 502, 519, 517, 507, 463, 511, 501, 509, 494, 473, 523, 513, 529, 524, 505, 489, 535, 516, 530, 521, 520, 441, 537, 528, 529, 484, 543, 527, 547, 510, 555, 517, 515, 512, 488, 452, 509, 516, 525, 522, 516, 539, 522, 515, 504, 510, 495, 504, 462, 523, 503, 509, 505, 523, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[522, 515, 549, 472, 529, 529, 485, 527, 499, 498, 548, 487, 505, 535, 481, 508, 534, 501, 532, 538, 505, 506, 502, 542, 514, 503, 473, 477, 488, 515, 531, 538, 540, 509, 492, 527, 512, 507, 492, 515, 498, 492, 496, 534, 486, 489, 556, 512, 534, 500, 516, 523, 485, 507, 460, 506, 470, 499, 522, 527, 520, 509, 504, 516, 511, 486, 462, 492, 503, 523, 534, 468, 507, 475, 539, 527, 492, 516, 540, 536, 513, 506, 490, 525, 506, 512, 499, 529, 529, 489, 520, 517, 547, 524, 510, 542, 476, 506, 559, 490, 462, 507, 545, 536, 499, 495, 480, 504, 542, 508, 521, 475, 526, 514, 504, 596, 543, 519, 534, 514, 524, 557, 550, 495, 514, 542, 470, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[492, 508, 541, 528, 500, 506, 499, 549, 504, 537, 553, 479, 505, 500, 492, 521, 514, 509, 476, 514, 485, 505, 545, 526, 500, 490, 522, 473, 501, 542, 542, 506, 469, 498, 521, 501, 531, 503, 535, 479, 486, 554, 530, 471, 491, 505, 528, 524, 535, 516, 540, 519, 538, 529, 512, 516, 526, 465, 529, 462, 489, 507, 519, 529, 512, 538, 498, 509, 522, 549, 524, 527, 508, 477, 544, 483, 567, 525, 483, 512, 484, 486, 501, 519, 533, 528, 512, 504, 510, 529, 501, 513, 540, 525, 517, 477, 486, 564, 495, 550, 519, 484, 523, 487, 523, 537, 512, 526, 532, 521, 490, 495, 491, 518, 463, 540, 516, 505, 533, 494, 526, 511, 513, 512, 516, 498, 494, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 540, 493, 488, 482, 495, 521, 497, 505, 512, 503, 529, 470, 525, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 497, 547, 527, 533, 497, 529, 516, 531, 484, 515, 459, 544, 481, 511, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 516, 549, 533, 475, 496, 514, 517, 511, 487, 538, 503, 492, 488, 476, 44, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 486, 474, 512, 548, 513, 517, 493, 498, 499, 503, 496, 518, 496, 485, 520, 13, 11, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 529, 502, 502, 542, 513, 550, 521, 525, 484, 537, 526, 515, 537, 489, 531, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 475, 506, 506, 506, 490, 489, 553, 514, 525, 511, 546, 517, 511, 513, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 516, 496, 495, 489, 532, 501, 494, 520, 541, 548, 485, 552, 535, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 527, 488, 525, 504, 518, 530, 496, 527, 533, 528, 497, 483, 523, 522, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8115, 8154, 8090, 8345, 8231, 8257, 8248] avg=8192.0000 max=8345 max_rank=4 min=8090 min_rank=3 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=33081 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[523, 476, 532, 533, 519, 515, 481, 523, 507, 491, 529, 500, 482, 529, 520, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 536, 491, 534, 497, 519, 523, 506, 477, 491, 515, 494, 519, 507, 463, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 497, 509, 522, 516, 524, 507, 459, 505, 538, 504, 511, 509, 480, 517, 538, 37, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 485, 522, 506, 538, 505, 518, 536, 533, 536, 515, 493, 513, 539, 466, 537, 14, 11, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 510, 515, 518, 505, 486, 527, 515, 500, 507, 540, 496, 527, 499, 511, 488, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 513, 521, 500, 560, 517, 506, 507, 537, 500, 506, 467, 488, 517, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 492, 502, 474, 531, 492, 491, 510, 547, 518, 538, 505, 530, 525, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 483, 556, 449, 493, 500, 522, 560, 530, 504, 502, 491, 515, 496, 516, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8105, 8161, 8300, 8184, 8234, 8222, 8128] avg=8192.0000 max=8300 max_rank=3 min=8105 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[525, 545, 526, 526, 496, 512, 491, 492, 544, 562, 494, 509, 503, 536, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 541, 492, 513, 551, 513, 528, 540, 529, 489, 494, 542, 508, 530, 515, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 473, 478, 491, 514, 500, 476, 573, 521, 526, 488, 476, 526, 519, 469, 510, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 520, 506, 469, 525, 538, 493, 482, 506, 513, 566, 522, 500, 562, 474, 482, 13, 11, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 478, 512, 505, 542, 522, 530, 489, 541, 527, 500, 471, 489, 525, 543, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 487, 496, 503, 475, 493, 536, 536, 513, 510, 527, 517, 535, 501, 501, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 533, 499, 495, 505, 508, 496, 495, 476, 514, 530, 518, 462, 520, 538, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 517, 464, 518, 525, 476, 525, 507, 489, 491, 510, 515, 519, 544, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8224, 8074, 8232, 8217, 8171, 8187, 8128] avg=8192.0000 max=8303 max_rank=0 min=8074 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[497, 509, 492, 547, 501, 517, 439, 494, 493, 509, 517, 554, 476, 489, 528, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 560, 521, 517, 548, 524, 531, 547, 476, 522, 504, 520, 492, 510, 495, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 545, 501, 482, 531, 518, 489, 518, 496, 519, 484, 516, 514, 455, 568, 524, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 470, 499, 513, 524, 468, 509, 528, 556, 505, 482, 481, 501, 521, 548, 497, 12, 11, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 511, 474, 492, 516, 517, 504, 529, 546, 507, 501, 520, 459, 508, 522, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 477, 494, 524, 507, 519, 560, 520, 525, 544, 509, 541, 491, 515, 531, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 537, 464, 488, 527, 523, 502, 505, 509, 525, 530, 527, 514, 493, 513, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 497, 505, 470, 521, 514, 516, 522, 507, 516, 527, 557, 526, 516, 511, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8309, 8187, 8122, 8114, 8291, 8190, 8233] avg=8192.0000 max=8309 max_rank=1 min=8090 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[478, 479, 523, 505, 546, 519, 530, 553, 501, 522, 526, 521, 534, 508, 488, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 480, 539, 539, 504, 479, 508, 512, 525, 484, 526, 500, 497, 503, 530, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 526, 522, 449, 537, 538, 510, 541, 511, 546, 517, 504, 483, 527, 526, 503, 44, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 495, 494, 496, 467, 538, 520, 485, 501, 509, 519, 519, 524, 516, 520, 518, 18, 15, 14, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 503, 521, 513, 522, 523, 512, 516, 492, 502, 519, 517, 507, 463, 511, 501, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 473, 523, 513, 529, 524, 505, 489, 535, 516, 530, 521, 520, 441, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 484, 543, 527, 547, 510, 555, 517, 515, 512, 488, 452, 509, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 539, 478, 515, 504, 510, 495, 504, 462, 523, 503, 509, 505, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8151, 8334, 8186, 8145, 8135, 8241, 8110] avg=8192.0000 max=8334 max_rank=2 min=8110 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[522, 488, 505, 494, 506, 528, 510, 514, 560, 488, 506, 516, 527, 496, 547, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 565, 489, 539, 491, 537, 506, 545, 542, 496, 484, 494, 534, 540, 511, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 528, 534, 525, 524, 510, 520, 490, 503, 549, 492, 536, 522, 495, 489, 516, 40, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 516, 520, 509, 515, 514, 495, 485, 500, 511, 525, 501, 530, 476, 525, 528, 12, 11, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 518, 492, 475, 515, 525, 493, 488, 509, 512, 501, 526, 522, 524, 525, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 525, 534, 537, 467, 531, 523, 509, 492, 500, 503, 455, 468, 519, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 453, 514, 484, 490, 530, 512, 492, 540, 481, 519, 513, 541, 510, 539, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 485, 542, 482, 489, 494, 492, 523, 539, 534, 544, 530, 497, 524, 515, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8301, 8249, 8203, 8132, 8067, 8108, 8244] avg=8192.0000 max=8301 max_rank=1 min=8067 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[522, 515, 549, 472, 529, 529, 485, 527, 499, 498, 536, 487, 505, 535, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 532, 529, 505, 506, 502, 542, 514, 503, 473, 477, 488, 515, 531, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 509, 492, 527, 512, 507, 492, 515, 498, 492, 496, 534, 486, 489, 556, 512, 46, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 500, 516, 523, 485, 507, 460, 506, 470, 499, 522, 527, 520, 509, 504, 516, 13, 12, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 486, 462, 492, 503, 523, 534, 468, 507, 475, 539, 527, 492, 516, 540, 536, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 506, 490, 525, 506, 512, 499, 529, 529, 489, 520, 517, 547, 524, 510, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 497, 559, 490, 462, 507, 545, 536, 499, 495, 480, 504, 542, 508, 521, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 504, 550, 543, 519, 534, 514, 524, 557, 550, 495, 514, 542, 470, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8190, 8206, 8136, 8116, 8242, 8096, 8373] avg=8192.0000 max=8373 max_rank=7 min=8096 min_rank=6 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 508, 541, 528, 500, 506, 499, 549, 504, 537, 541, 479, 505, 500, 492, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 509, 476, 505, 485, 505, 545, 526, 500, 490, 522, 473, 501, 542, 542, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 498, 521, 501, 531, 503, 535, 479, 486, 554, 530, 471, 491, 505, 528, 524, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 540, 519, 538, 529, 512, 516, 526, 465, 529, 462, 489, 507, 519, 529, 11, 12, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 498, 509, 522, 549, 524, 527, 508, 477, 544, 483, 567, 525, 483, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 486, 501, 519, 533, 528, 512, 504, 510, 529, 501, 513, 540, 525, 517, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 555, 495, 550, 519, 484, 523, 487, 523, 537, 512, 526, 532, 521, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 518, 463, 498, 516, 505, 533, 494, 526, 511, 513, 512, 516, 498, 494, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8141, 8171, 8267, 8283, 8165, 8235, 8072] avg=8192.0000 max=8283 max_rank=4 min=8072 min_rank=7 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 3968, 4224, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4352, 4096, 3968, 4096, 3968, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[532, 517, 484, 515, 480, 511, 547, 519, 542, 518, 531, 541, 556, 522, 514, 534, 528, 500, 474, 455, 522, 496, 508, 492, 505, 491, 482, 494, 562, 515, 510, 540, 527, 521, 488, 480, 522, 467, 488, 511, 526, 474, 478, 514, 514, 511, 517, 489, 498, 515, 489, 522, 496, 541, 553, 485, 512, 476, 498, 491, 536, 506, 498, 552, 487, 548, 539, 543, 513, 475, 480, 514, 507, 514, 505, 528, 513, 512, 515, 534, 511, 503, 511, 505, 503, 543, 522, 499, 543, 539, 481, 517, 498, 496, 528, 516, 492, 520, 531, 518, 511, 502, 510, 501, 525, 481, 543, 512, 506, 533, 552, 489, 512, 565, 516, 522, 537, 492, 526, 542, 504, 494, 536, 487, 516, 504, 508, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[517, 527, 514, 503, 514, 478, 533, 503, 504, 497, 507, 527, 513, 498, 500, 511, 559, 520, 487, 530, 488, 498, 505, 516, 515, 509, 518, 530, 500, 505, 501, 528, 470, 534, 531, 504, 502, 470, 534, 521, 531, 512, 535, 503, 493, 461, 538, 518, 515, 516, 554, 531, 509, 540, 561, 507, 519, 516, 488, 533, 503, 511, 550, 529, 500, 495, 499, 534, 547, 546, 471, 543, 553, 499, 500, 504, 512, 508, 532, 525, 495, 496, 530, 517, 490, 536, 534, 524, 494, 482, 522, 512, 507, 493, 515, 520, 532, 460, 516, 510, 502, 518, 542, 497, 477, 509, 495, 494, 540, 510, 493, 489, 500, 502, 527, 482, 525, 496, 465, 496, 494, 489, 506, 562, 527, 518, 515, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[517, 551, 492, 461, 474, 523, 523, 463, 512, 509, 469, 508, 527, 513, 523, 494, 520, 551, 521, 552, 498, 516, 536, 531, 550, 519, 476, 507, 503, 496, 530, 548, 510, 504, 496, 500, 567, 508, 493, 507, 487, 522, 569, 507, 512, 506, 528, 532, 536, 514, 495, 510, 509, 536, 532, 493, 483, 525, 477, 479, 521, 537, 522, 481, 529, 506, 494, 535, 505, 506, 515, 506, 535, 522, 537, 479, 509, 541, 540, 507, 475, 517, 540, 534, 542, 470, 468, 511, 532, 508, 506, 548, 506, 524, 496, 530, 496, 496, 554, 492, 540, 530, 507, 511, 507, 476, 486, 502, 493, 524, 523, 496, 495, 508, 501, 508, 539, 525, 507, 502, 536, 512, 496, 493, 459, 516, 502, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[508, 519, 474, 523, 505, 502, 538, 505, 527, 521, 478, 498, 500, 504, 496, 478, 521, 486, 536, 522, 522, 527, 537, 512, 540, 507, 516, 538, 510, 523, 497, 488, 506, 512, 467, 461, 536, 541, 523, 507, 505, 515, 479, 537, 544, 512, 508, 542, 476, 503, 522, 514, 505, 528, 480, 529, 469, 534, 548, 488, 532, 508, 526, 505, 514, 529, 536, 520, 491, 515, 515, 489, 486, 481, 549, 496, 525, 508, 517, 545, 512, 513, 536, 539, 508, 465, 464, 493, 485, 506, 489, 521, 520, 503, 517, 503, 525, 506, 528, 525, 516, 487, 500, 536, 526, 491, 524, 522, 517, 514, 502, 508, 532, 529, 550, 540, 522, 532, 528, 493, 535, 505, 510, 521, 497, 516, 485, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[530, 504, 504, 548, 490, 513, 465, 522, 477, 501, 520, 489, 539, 531, 482, 540, 511, 491, 524, 511, 525, 523, 478, 528, 535, 519, 532, 497, 506, 511, 493, 485, 495, 531, 515, 522, 555, 497, 486, 515, 508, 494, 489, 510, 535, 501, 472, 498, 551, 471, 473, 513, 535, 526, 503, 519, 531, 515, 518, 521, 485, 541, 527, 515, 512, 532, 488, 508, 520, 516, 507, 497, 512, 493, 507, 526, 539, 522, 487, 543, 481, 550, 529, 491, 546, 547, 490, 501, 503, 529, 517, 502, 510, 482, 513, 472, 533, 509, 481, 486, 537, 541, 529, 512, 492, 531, 533, 530, 525, 480, 557, 467, 511, 551, 509, 519, 551, 502, 500, 506, 518, 501, 556, 522, 509, 519, 468, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[515, 487, 550, 534, 530, 455, 526, 518, 503, 539, 520, 520, 494, 513, 483, 523, 518, 460, 497, 515, 492, 474, 514, 497, 460, 500, 518, 585, 512, 518, 554, 486, 482, 510, 535, 502, 551, 507, 524, 496, 551, 506, 507, 546, 503, 528, 500, 490, 535, 499, 501, 497, 541, 551, 493, 534, 519, 508, 517, 557, 527, 526, 480, 541, 505, 557, 515, 516, 479, 515, 499, 505, 493, 518, 549, 518, 495, 525, 516, 530, 507, 500, 518, 516, 542, 486, 527, 488, 511, 495, 492, 463, 488, 483, 471, 552, 539, 492, 491, 495, 539, 497, 485, 478, 579, 534, 477, 523, 489, 503, 524, 505, 499, 540, 500, 499, 501, 505, 508, 524, 535, 508, 509, 551, 538, 503, 516, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[500, 495, 522, 499, 513, 519, 530, 529, 484, 546, 552, 521, 539, 487, 494, 472, 548, 507, 479, 475, 527, 489, 484, 527, 510, 516, 480, 543, 527, 482, 507, 499, 497, 520, 514, 552, 504, 535, 529, 491, 514, 553, 479, 560, 521, 502, 488, 548, 479, 544, 568, 517, 486, 506, 544, 478, 504, 506, 503, 504, 540, 468, 530, 501, 521, 510, 527, 542, 484, 484, 500, 521, 521, 529, 507, 515, 453, 517, 499, 486, 507, 515, 527, 544, 512, 472, 515, 527, 492, 517, 530, 482, 520, 519, 523, 468, 536, 577, 514, 511, 481, 553, 505, 503, 505, 518, 518, 477, 540, 504, 532, 535, 529, 507, 495, 521, 529, 499, 573, 529, 504, 478, 497, 497, 504, 506, 507, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[514, 484, 517, 527, 522, 524, 518, 523, 559, 528, 489, 490, 562, 527, 487, 534, 488, 505, 514, 513, 521, 525, 493, 484, 473, 483, 532, 482, 512, 495, 490, 523, 479, 488, 514, 522, 494, 553, 515, 477, 520, 540, 534, 473, 494, 504, 515, 489, 535, 508, 474, 526, 522, 520, 511, 540, 519, 555, 512, 487, 502, 490, 524, 501, 523, 525, 505, 518, 538, 502, 517, 499, 504, 525, 482, 501, 491, 483, 489, 519, 464, 518, 488, 551, 526, 495, 521, 500, 515, 493, 476, 539, 535, 549, 535, 503, 507, 557, 496, 515, 485, 530, 488, 543, 545, 543, 493, 536, 467, 511, 468, 517, 529, 484, 508, 525, 513, 521, 541, 559, 519, 536, 513, 493, 562, 508, 496, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[532, 517, 484, 515, 480, 511, 547, 519, 542, 518, 531, 541, 556, 522, 514, 534, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 474, 455, 522, 496, 508, 492, 505, 491, 482, 494, 562, 515, 510, 540, 24, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 521, 488, 480, 522, 467, 488, 511, 526, 474, 478, 514, 514, 511, 517, 489, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 489, 522, 496, 503, 553, 485, 512, 476, 498, 491, 536, 506, 498, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 548, 539, 528, 513, 475, 480, 514, 507, 514, 505, 528, 513, 512, 515, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 503, 511, 505, 503, 543, 522, 499, 543, 539, 481, 517, 498, 496, 528, 516, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 520, 531, 518, 511, 502, 510, 501, 525, 481, 543, 512, 506, 533, 552, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 565, 516, 522, 513, 492, 526, 542, 504, 494, 536, 487, 516, 504, 508, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 8100, 8043, 8130, 8212, 8252, 8222, 8212] avg=8192.0000 max=8365 max_rank=0 min=8043 min_rank=2 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[517, 527, 514, 503, 514, 478, 533, 503, 504, 497, 507, 527, 513, 498, 500, 511, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 520, 487, 530, 488, 498, 505, 516, 515, 509, 518, 530, 500, 505, 501, 528, 24, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 534, 531, 504, 502, 470, 534, 521, 531, 512, 535, 503, 493, 461, 538, 518, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 516, 554, 531, 509, 502, 561, 507, 519, 516, 488, 533, 503, 511, 550, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 495, 499, 519, 547, 546, 471, 543, 553, 499, 500, 504, 512, 508, 532, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 530, 517, 490, 536, 534, 524, 494, 482, 522, 512, 507, 493, 515, 520, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 460, 516, 510, 502, 518, 542, 497, 477, 509, 495, 494, 540, 510, 493, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 527, 482, 501, 496, 465, 496, 494, 489, 506, 562, 527, 518, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8235, 8173, 8344, 8253, 8204, 8080, 8099] avg=8192.0000 max=8344 max_rank=3 min=8080 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[508, 519, 474, 523, 505, 502, 538, 505, 527, 521, 478, 498, 500, 504, 496, 478, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 486, 536, 522, 522, 527, 537, 512, 540, 507, 516, 538, 510, 523, 497, 488, 27, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 467, 461, 536, 541, 523, 507, 505, 515, 479, 537, 544, 512, 508, 542, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 503, 522, 514, 505, 483, 480, 529, 469, 534, 548, 488, 532, 508, 526, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 536, 500, 491, 515, 515, 489, 486, 481, 549, 496, 525, 508, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 513, 536, 539, 508, 465, 464, 493, 485, 506, 489, 521, 520, 503, 517, 503, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 528, 525, 516, 487, 500, 536, 526, 491, 524, 522, 517, 514, 502, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 529, 550, 540, 495, 532, 528, 493, 535, 505, 510, 521, 497, 516, 485, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8316, 8216, 8122, 8196, 8114, 8218, 8272] avg=8192.0000 max=8316 max_rank=1 min=8082 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[530, 504, 504, 548, 490, 513, 465, 522, 477, 501, 520, 489, 539, 531, 482, 540, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 491, 524, 511, 525, 523, 478, 528, 535, 519, 532, 497, 506, 511, 493, 485, 25, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 531, 515, 522, 555, 497, 486, 515, 508, 494, 489, 510, 535, 501, 472, 498, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 471, 473, 513, 535, 489, 503, 519, 531, 515, 518, 521, 485, 541, 527, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 532, 488, 494, 520, 516, 507, 497, 512, 493, 507, 526, 539, 522, 487, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 550, 529, 491, 546, 547, 490, 501, 503, 529, 517, 502, 510, 482, 513, 472, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 509, 481, 486, 537, 541, 529, 512, 492, 531, 533, 530, 525, 480, 557, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 551, 509, 519, 526, 502, 500, 506, 518, 501, 556, 522, 509, 519, 468, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8196, 8138, 8207, 8195, 8199, 8239, 8205] avg=8192.0000 max=8239 max_rank=6 min=8138 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[517, 551, 492, 461, 474, 523, 523, 463, 512, 509, 469, 508, 527, 513, 523, 494, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 551, 521, 552, 498, 516, 536, 531, 550, 519, 476, 507, 503, 496, 530, 548, 24, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 496, 500, 567, 508, 493, 507, 487, 522, 569, 507, 512, 506, 528, 532, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 495, 510, 509, 499, 532, 493, 483, 525, 477, 479, 521, 537, 522, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 506, 494, 520, 505, 506, 515, 506, 535, 522, 537, 479, 509, 541, 540, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 517, 540, 534, 542, 470, 468, 511, 532, 508, 506, 548, 506, 524, 496, 530, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 496, 554, 492, 540, 530, 507, 511, 507, 476, 486, 502, 493, 524, 523, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 501, 508, 515, 525, 507, 502, 536, 512, 496, 493, 459, 516, 502, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8061, 8380, 8264, 8113, 8251, 8243, 8129, 8095] avg=8192.0000 max=8380 max_rank=1 min=8061 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[514, 484, 517, 527, 522, 524, 518, 523, 559, 528, 489, 490, 562, 527, 487, 534, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 505, 514, 513, 521, 525, 493, 484, 473, 483, 532, 482, 512, 495, 490, 523, 23, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 488, 514, 522, 494, 553, 515, 477, 520, 540, 534, 473, 494, 504, 515, 489, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 508, 474, 526, 522, 484, 511, 540, 519, 555, 512, 487, 502, 490, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 525, 505, 504, 538, 502, 517, 499, 504, 525, 482, 501, 491, 483, 489, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 518, 488, 551, 526, 495, 521, 500, 515, 493, 476, 539, 535, 549, 535, 503, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 557, 496, 515, 485, 530, 488, 543, 545, 543, 493, 536, 467, 511, 468, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 484, 508, 525, 490, 521, 541, 559, 519, 536, 513, 493, 562, 508, 496, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8058, 8126, 8190, 8107, 8243, 8197, 8308] avg=8192.0000 max=8308 max_rank=7 min=8058 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[515, 487, 550, 534, 530, 455, 526, 518, 503, 539, 520, 520, 494, 513, 483, 523, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 460, 497, 515, 492, 474, 514, 497, 460, 500, 518, 585, 512, 518, 554, 486, 23, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 510, 535, 502, 551, 507, 524, 496, 551, 506, 507, 546, 503, 528, 500, 490, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 499, 501, 497, 541, 512, 493, 534, 519, 508, 517, 557, 527, 526, 480, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 557, 515, 502, 479, 515, 499, 505, 493, 518, 549, 518, 495, 525, 516, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 518, 516, 542, 486, 527, 488, 511, 495, 492, 463, 488, 483, 471, 552, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 492, 491, 495, 539, 497, 485, 478, 579, 534, 477, 523, 489, 503, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 540, 500, 499, 478, 505, 508, 524, 535, 508, 509, 551, 538, 503, 516, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8125, 8253, 8287, 8221, 8077, 8146, 8215] avg=8192.0000 max=8287 max_rank=3 min=8077 min_rank=5 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[500, 495, 522, 499, 513, 519, 530, 529, 484, 546, 552, 521, 539, 487, 494, 472, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 507, 479, 475, 527, 489, 484, 527, 510, 516, 480, 543, 527, 482, 507, 499, 24, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 520, 514, 552, 504, 535, 529, 491, 514, 553, 479, 560, 521, 502, 488, 548, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 544, 568, 517, 486, 471, 544, 478, 504, 506, 503, 504, 540, 468, 530, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 527, 527, 484, 484, 500, 521, 521, 529, 507, 515, 453, 517, 499, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 527, 544, 512, 472, 515, 527, 492, 517, 530, 482, 520, 519, 523, 468, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 577, 514, 511, 481, 553, 505, 503, 505, 518, 518, 477, 540, 504, 532, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 495, 521, 505, 499, 573, 529, 504, 478, 497, 497, 504, 506, 507, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8126, 8323, 8143, 8101, 8204, 8305, 8130] avg=8192.0000 max=8323 max_rank=2 min=8101 min_rank=4 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[543, 515, 528, 536, 509, 507, 517, 554, 521, 489, 495, 519, 520, 506, 495, 489, 517, 476, 490, 519, 521, 521, 485, 569, 561, 536, 508, 483, 470, 548, 517, 502, 495, 504, 539, 499, 515, 524, 506, 487, 509, 524, 552, 505, 530, 487, 520, 465, 505, 490, 513, 490, 536, 489, 482, 531, 513, 535, 514, 490, 476, 533, 534, 540, 527, 488, 505, 509, 510, 466, 539, 493, 528, 507, 514, 496, 543, 498, 542, 526, 503, 526, 515, 530, 502, 516, 503, 554, 490, 496, 535, 516, 522, 465, 546, 480, 478, 504, 522, 502, 492, 492, 530, 532, 525, 535, 496, 485, 526, 518, 542, 547, 513, 462, 538, 513, 480, 541, 479, 506, 535, 507, 506, 529, 489, 517, 522, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[508, 526, 513, 510, 498, 520, 516, 523, 510, 534, 536, 512, 544, 515, 497, 496, 516, 489, 529, 507, 513, 509, 510, 528, 526, 505, 490, 497, 511, 505, 485, 555, 498, 564, 507, 541, 502, 508, 527, 509, 531, 537, 483, 474, 516, 512, 482, 522, 483, 524, 507, 494, 507, 530, 531, 522, 504, 525, 494, 526, 535, 512, 551, 513, 484, 501, 472, 508, 513, 525, 471, 553, 521, 475, 545, 526, 528, 523, 533, 492, 501, 484, 554, 522, 553, 505, 514, 482, 474, 511, 510, 479, 562, 487, 479, 493, 489, 545, 494, 516, 527, 499, 543, 506, 506, 543, 518, 524, 493, 505, 507, 484, 462, 515, 504, 538, 517, 504, 500, 469, 477, 515, 509, 504, 544, 554, 512, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[530, 525, 508, 508, 535, 532, 515, 512, 437, 488, 555, 531, 525, 561, 515, 539, 558, 523, 526, 476, 540, 538, 505, 546, 486, 510, 500, 527, 520, 500, 501, 501, 523, 511, 508, 510, 487, 500, 532, 495, 528, 507, 528, 507, 515, 516, 533, 547, 494, 511, 512, 524, 551, 541, 497, 514, 496, 480, 484, 546, 471, 479, 510, 508, 503, 516, 549, 503, 453, 491, 525, 483, 522, 516, 545, 516, 521, 514, 522, 509, 538, 490, 492, 472, 499, 499, 501, 518, 492, 510, 520, 514, 547, 490, 517, 531, 512, 529, 477, 521, 487, 472, 523, 547, 477, 495, 518, 501, 503, 508, 498, 549, 540, 497, 524, 515, 514, 525, 510, 520, 508, 495, 484, 544, 500, 480, 485, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[520, 458, 516, 513, 523, 559, 548, 481, 480, 484, 526, 511, 503, 532, 498, 537, 513, 544, 516, 517, 508, 546, 536, 542, 528, 544, 517, 526, 512, 507, 478, 515, 536, 493, 529, 494, 510, 531, 511, 494, 498, 535, 509, 490, 538, 513, 532, 505, 513, 477, 442, 507, 504, 481, 525, 521, 532, 490, 495, 537, 532, 527, 512, 484, 545, 511, 526, 518, 498, 566, 518, 503, 503, 491, 547, 521, 532, 500, 475, 525, 519, 484, 495, 511, 487, 548, 488, 518, 516, 507, 501, 480, 555, 462, 492, 506, 481, 488, 550, 539, 536, 486, 469, 509, 488, 509, 490, 504, 513, 513, 542, 504, 508, 520, 484, 522, 490, 503, 521, 449, 530, 506, 555, 514, 541, 543, 543, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[484, 514, 525, 527, 494, 475, 496, 503, 529, 494, 475, 507, 468, 549, 505, 512, 531, 512, 525, 526, 500, 529, 541, 503, 539, 533, 519, 497, 543, 513, 499, 501, 495, 523, 510, 518, 524, 493, 508, 493, 492, 565, 498, 526, 530, 505, 493, 493, 497, 540, 521, 522, 495, 492, 532, 506, 546, 510, 536, 495, 526, 495, 502, 481, 502, 533, 491, 498, 500, 546, 504, 508, 513, 504, 520, 525, 471, 501, 480, 491, 532, 515, 533, 538, 479, 510, 501, 485, 520, 504, 533, 528, 472, 517, 522, 521, 553, 514, 521, 523, 509, 494, 502, 484, 500, 513, 494, 540, 500, 501, 482, 485, 558, 516, 564, 551, 497, 527, 524, 497, 510, 524, 587, 479, 525, 523, 523, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[481, 532, 531, 494, 512, 529, 493, 493, 531, 523, 529, 516, 538, 533, 523, 534, 516, 484, 519, 495, 506, 480, 497, 495, 507, 515, 518, 506, 468, 516, 528, 486, 497, 523, 537, 537, 499, 499, 545, 544, 506, 517, 514, 555, 501, 523, 512, 515, 520, 523, 519, 504, 496, 523, 537, 503, 494, 544, 535, 532, 477, 485, 533, 487, 575, 504, 506, 549, 530, 527, 510, 499, 505, 501, 526, 501, 493, 467, 486, 497, 483, 497, 530, 501, 543, 466, 511, 561, 552, 516, 510, 503, 537, 517, 542, 494, 500, 547, 526, 550, 472, 499, 527, 521, 506, 455, 490, 525, 522, 504, 499, 452, 518, 523, 481, 510, 479, 489, 512, 512, 507, 522, 488, 528, 534, 489, 524, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[504, 559, 516, 552, 535, 473, 514, 507, 488, 506, 542, 489, 521, 558, 513, 463, 532, 517, 510, 485, 537, 507, 540, 472, 508, 527, 504, 537, 478, 446, 494, 519, 509, 509, 520, 524, 528, 507, 527, 511, 529, 479, 509, 468, 520, 518, 478, 522, 523, 511, 504, 495, 525, 486, 557, 525, 510, 511, 519, 551, 505, 522, 539, 523, 535, 473, 514, 533, 499, 506, 500, 469, 525, 533, 540, 503, 527, 531, 467, 504, 508, 505, 512, 475, 516, 494, 525, 504, 512, 519, 536, 549, 496, 523, 500, 498, 529, 491, 475, 526, 515, 514, 538, 517, 504, 509, 518, 506, 556, 496, 499, 481, 524, 484, 505, 507, 498, 543, 532, 509, 506, 528, 513, 509, 523, 519, 507, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[455, 483, 541, 515, 535, 481, 532, 518, 502, 496, 495, 521, 552, 535, 516, 503, 513, 558, 518, 535, 477, 566, 512, 517, 518, 487, 504, 516, 497, 514, 476, 500, 521, 504, 497, 499, 511, 511, 556, 494, 449, 473, 492, 525, 486, 484, 525, 540, 556, 481, 501, 498, 502, 498, 522, 488, 489, 520, 534, 470, 506, 524, 497, 507, 491, 521, 499, 505, 483, 532, 538, 528, 519, 536, 533, 516, 509, 518, 540, 482, 536, 529, 501, 506, 504, 511, 517, 499, 527, 561, 534, 562, 503, 499, 511, 507, 507, 475, 499, 530, 565, 544, 483, 476, 538, 550, 464, 484, 533, 523, 485, 526, 517, 502, 477, 479, 472, 512, 608, 550, 500, 529, 519, 494, 496, 521, 547, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[543, 515, 528, 536, 509, 507, 517, 554, 521, 489, 495, 519, 520, 478, 495, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 476, 490, 519, 521, 505, 485, 569, 561, 536, 508, 483, 470, 548, 517, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 504, 539, 499, 515, 524, 504, 487, 509, 524, 552, 505, 530, 487, 520, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 490, 513, 490, 536, 489, 482, 531, 513, 535, 514, 490, 476, 533, 534, 540, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 488, 505, 509, 510, 466, 539, 493, 528, 507, 514, 496, 543, 498, 542, 526, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 515, 530, 502, 516, 503, 554, 490, 496, 535, 516, 522, 465, 546, 480, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 504, 522, 502, 492, 492, 530, 532, 525, 535, 496, 485, 526, 518, 542, 547, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 462, 538, 513, 480, 541, 465, 506, 535, 507, 506, 529, 489, 517, 522, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8207, 8159, 8183, 8195, 8207, 8262, 8108] avg=8192.0000 max=8262 max_rank=6 min=8108 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[530, 525, 508, 508, 535, 532, 515, 512, 437, 488, 555, 531, 525, 530, 515, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 523, 526, 476, 540, 522, 505, 546, 486, 510, 500, 527, 520, 500, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 508, 510, 487, 500, 530, 495, 528, 507, 528, 507, 515, 516, 533, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 512, 524, 551, 541, 497, 514, 496, 480, 484, 546, 471, 479, 510, 508, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 549, 503, 453, 491, 525, 483, 522, 516, 545, 516, 521, 514, 522, 509, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 490, 492, 472, 499, 499, 501, 518, 492, 510, 520, 514, 547, 490, 517, 531, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 529, 477, 521, 487, 472, 523, 547, 477, 495, 518, 501, 503, 508, 498, 549, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 497, 524, 515, 514, 525, 495, 520, 508, 495, 484, 544, 500, 480, 485, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8241, 8245, 8130, 8192, 8139, 8156, 8148] avg=8192.0000 max=8285 max_rank=0 min=8130 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[455, 483, 541, 515, 535, 481, 532, 518, 502, 496, 495, 521, 552, 506, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 558, 518, 535, 477, 548, 512, 517, 518, 487, 504, 516, 497, 514, 476, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 497, 499, 511, 511, 554, 494, 449, 473, 492, 525, 486, 484, 525, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 481, 501, 498, 502, 498, 522, 488, 489, 520, 534, 470, 506, 524, 497, 507, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 521, 499, 505, 483, 532, 538, 528, 519, 536, 533, 516, 509, 518, 540, 482, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 529, 501, 506, 504, 511, 517, 499, 527, 561, 534, 562, 503, 499, 511, 507, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 475, 499, 530, 565, 544, 483, 476, 538, 550, 464, 484, 533, 523, 485, 526, 29, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 477, 479, 472, 512, 590, 550, 500, 529, 519, 494, 496, 521, 547, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8190, 8065, 8107, 8254, 8318, 8220, 8231] avg=8192.0000 max=8318 max_rank=5 min=8065 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=33023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[508, 526, 513, 510, 498, 520, 516, 523, 510, 534, 536, 512, 544, 487, 497, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 489, 529, 507, 513, 493, 510, 528, 526, 505, 490, 497, 511, 505, 485, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 564, 507, 541, 502, 508, 525, 509, 531, 537, 483, 474, 516, 512, 482, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 524, 507, 494, 507, 530, 531, 522, 504, 525, 494, 526, 535, 512, 551, 513, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 472, 508, 513, 525, 471, 553, 521, 475, 545, 526, 528, 523, 533, 492, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 484, 554, 522, 553, 505, 514, 482, 474, 511, 510, 479, 562, 487, 479, 493, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 545, 494, 516, 527, 499, 543, 506, 506, 543, 518, 524, 493, 505, 507, 484, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 515, 504, 538, 517, 504, 485, 469, 477, 515, 509, 504, 544, 554, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8159, 8211, 8270, 8174, 8119, 8235, 8138] avg=8192.0000 max=8270 max_rank=3 min=8119 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[484, 514, 525, 527, 494, 475, 496, 503, 529, 494, 475, 507, 468, 515, 505, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 525, 526, 500, 505, 541, 503, 539, 533, 519, 497, 543, 513, 499, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 510, 518, 524, 493, 504, 493, 492, 565, 498, 526, 530, 505, 493, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 540, 521, 522, 495, 492, 532, 506, 546, 510, 536, 495, 526, 495, 502, 481, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 533, 491, 498, 500, 546, 504, 508, 513, 504, 520, 525, 471, 501, 480, 491, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 515, 533, 538, 479, 510, 501, 485, 520, 504, 533, 528, 472, 517, 522, 521, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 514, 521, 523, 509, 494, 502, 484, 500, 513, 494, 540, 500, 501, 482, 485, 34, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 516, 564, 551, 497, 527, 502, 497, 510, 524, 587, 479, 525, 523, 523, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8287, 8162, 8214, 8095, 8222, 8161, 8372] avg=8192.0000 max=8372 max_rank=7 min=8023 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[504, 559, 516, 552, 535, 473, 514, 507, 488, 506, 542, 489, 521, 527, 513, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 517, 510, 485, 537, 491, 540, 472, 508, 527, 504, 537, 478, 446, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 520, 524, 528, 507, 525, 511, 529, 479, 509, 468, 520, 518, 478, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 504, 495, 525, 486, 557, 525, 510, 511, 519, 551, 505, 522, 539, 523, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 473, 514, 533, 499, 506, 500, 469, 525, 533, 540, 503, 527, 531, 467, 504, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 505, 512, 475, 516, 494, 525, 504, 512, 519, 536, 549, 496, 523, 500, 498, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 491, 475, 526, 515, 514, 538, 517, 504, 509, 518, 506, 556, 496, 499, 481, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 505, 507, 498, 543, 517, 509, 506, 528, 513, 509, 523, 519, 507, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8097, 8156, 8318, 8163, 8181, 8213, 8199] avg=8192.0000 max=8318 max_rank=3 min=8097 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[481, 532, 531, 494, 512, 529, 493, 493, 531, 523, 529, 516, 538, 504, 523, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 484, 519, 495, 506, 466, 497, 495, 507, 515, 518, 506, 468, 516, 528, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 523, 537, 537, 499, 499, 543, 544, 506, 517, 514, 555, 501, 523, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 519, 504, 496, 523, 537, 503, 494, 544, 535, 532, 477, 485, 533, 487, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 504, 506, 549, 530, 527, 510, 499, 505, 501, 526, 501, 493, 467, 486, 497, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 497, 530, 501, 543, 466, 511, 561, 552, 516, 510, 503, 537, 517, 542, 494, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 547, 526, 550, 472, 499, 527, 521, 506, 455, 490, 525, 522, 504, 499, 452, 29, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 523, 481, 510, 479, 489, 497, 512, 507, 522, 488, 528, 534, 489, 524, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8022, 8322, 8223, 8180, 8272, 8131, 8123] avg=8192.0000 max=8322 max_rank=2 min=8022 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[520, 458, 516, 513, 523, 559, 548, 481, 480, 484, 526, 511, 503, 503, 498, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 544, 516, 517, 508, 530, 536, 542, 528, 544, 517, 526, 512, 507, 478, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 493, 529, 494, 510, 531, 509, 494, 498, 535, 509, 490, 538, 513, 532, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 477, 442, 507, 504, 481, 525, 521, 532, 490, 495, 537, 532, 527, 512, 484, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 511, 526, 518, 498, 566, 518, 503, 503, 491, 547, 521, 532, 500, 475, 525, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 484, 495, 511, 487, 548, 488, 518, 516, 507, 501, 480, 555, 462, 492, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 488, 550, 539, 536, 486, 469, 509, 488, 509, 490, 504, 513, 513, 542, 504, 29, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 520, 484, 522, 490, 503, 506, 449, 530, 506, 555, 514, 541, 543, 543, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8333, 8216, 8091, 8283, 8078, 8158, 8217] avg=8192.0000 max=8333 max_rank=1 min=8078 min_rank=5 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4352, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 3968, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True + [2026-08-26 16:29:48] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 415.4 | throughput per GPU (TFLOP/s/GPU): 435.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.523711E+00 | loss scale: 1.0 | grad norm: 5.547 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[508, 515, 528, 495, 518, 513, 496, 519, 527, 552, 513, 555, 516, 560, 487, 550, 526, 476, 504, 549, 531, 494, 519, 527, 508, 533, 488, 481, 511, 503, 523, 506, 517, 543, 473, 513, 516, 498, 513, 528, 546, 553, 498, 493, 499, 510, 533, 516, 528, 479, 505, 521, 443, 473, 512, 507, 539, 498, 493, 480, 498, 511, 510, 463, 514, 518, 492, 479, 509, 462, 481, 497, 512, 516, 509, 535, 526, 511, 499, 535, 558, 532, 489, 505, 542, 533, 495, 473, 524, 519, 502, 524, 538, 572, 520, 482, 486, 507, 486, 520, 549, 526, 508, 502, 504, 517, 505, 514, 518, 509, 544, 497, 521, 539, 518, 524, 504, 554, 554, 547, 457, 498, 496, 524, 486, 509, 478, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[509, 500, 531, 515, 500, 521, 521, 521, 486, 478, 514, 481, 508, 509, 527, 516, 502, 497, 520, 511, 520, 510, 496, 524, 527, 522, 551, 489, 513, 510, 509, 526, 498, 502, 504, 520, 506, 533, 524, 501, 515, 531, 552, 522, 499, 510, 470, 514, 519, 506, 485, 475, 515, 538, 510, 505, 512, 507, 496, 520, 517, 520, 528, 555, 500, 506, 505, 492, 505, 493, 538, 525, 497, 539, 508, 485, 508, 520, 480, 529, 494, 480, 484, 543, 513, 530, 505, 529, 507, 511, 507, 531, 509, 548, 477, 521, 537, 512, 499, 513, 498, 481, 508, 504, 538, 512, 495, 544, 489, 472, 513, 518, 544, 505, 532, 530, 536, 482, 511, 512, 524, 510, 536, 513, 514, 522, 525, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[526, 508, 509, 510, 549, 514, 504, 490, 520, 519, 555, 497, 444, 534, 515, 515, 535, 499, 509, 464, 512, 528, 473, 514, 489, 519, 495, 509, 493, 531, 533, 506, 471, 515, 503, 526, 537, 548, 544, 518, 519, 546, 533, 519, 511, 495, 521, 509, 511, 483, 519, 503, 530, 473, 489, 522, 531, 482, 494, 516, 510, 533, 511, 525, 508, 529, 571, 513, 508, 518, 502, 539, 540, 525, 479, 547, 531, 518, 537, 531, 514, 522, 495, 514, 493, 524, 534, 505, 514, 509, 518, 474, 491, 485, 512, 509, 487, 465, 510, 488, 504, 472, 527, 546, 495, 526, 512, 496, 541, 534, 492, 499, 496, 528, 513, 532, 493, 507, 525, 513, 495, 522, 492, 503, 552, 498, 473, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[503, 530, 516, 508, 489, 485, 498, 499, 493, 536, 529, 489, 540, 458, 512, 491, 520, 528, 512, 484, 567, 527, 445, 520, 504, 543, 467, 513, 532, 502, 508, 538, 497, 526, 481, 489, 500, 502, 528, 490, 508, 530, 510, 543, 496, 490, 521, 512, 528, 525, 529, 465, 510, 562, 514, 523, 501, 527, 551, 498, 527, 496, 505, 550, 526, 502, 484, 531, 537, 521, 493, 515, 512, 515, 526, 515, 484, 572, 553, 552, 505, 521, 523, 507, 521, 482, 512, 539, 514, 499, 488, 536, 517, 512, 514, 468, 525, 475, 521, 523, 493, 476, 544, 489, 511, 529, 503, 511, 490, 504, 543, 544, 498, 535, 487, 509, 506, 531, 509, 498, 496, 482, 513, 511, 551, 495, 520, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[494, 511, 535, 507, 524, 522, 501, 517, 482, 498, 522, 508, 534, 522, 470, 527, 533, 493, 534, 521, 495, 487, 519, 526, 491, 525, 484, 480, 521, 484, 520, 475, 481, 491, 505, 537, 514, 536, 492, 529, 512, 518, 488, 507, 517, 490, 533, 511, 512, 515, 535, 531, 522, 510, 535, 546, 540, 531, 520, 507, 533, 496, 539, 526, 525, 479, 524, 486, 520, 508, 508, 517, 520, 494, 480, 502, 567, 529, 491, 522, 523, 495, 514, 506, 521, 492, 494, 512, 510, 478, 488, 487, 543, 512, 519, 516, 523, 540, 502, 512, 514, 531, 534, 486, 551, 503, 498, 515, 522, 535, 484, 492, 467, 484, 536, 491, 522, 508, 519, 529, 475, 526, 521, 534, 501, 559, 513, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[514, 542, 481, 532, 502, 507, 504, 521, 481, 499, 543, 461, 504, 526, 543, 500, 494, 484, 491, 510, 514, 509, 514, 480, 567, 506, 546, 519, 554, 454, 557, 498, 544, 501, 525, 499, 502, 502, 534, 497, 515, 479, 523, 506, 508, 510, 487, 480, 505, 514, 521, 503, 545, 513, 528, 460, 558, 544, 521, 465, 526, 482, 507, 499, 486, 521, 484, 504, 553, 521, 491, 534, 517, 531, 504, 524, 505, 491, 513, 558, 512, 527, 503, 495, 486, 518, 523, 527, 527, 532, 540, 508, 498, 541, 495, 504, 501, 518, 554, 514, 543, 507, 495, 557, 557, 526, 525, 509, 531, 499, 474, 488, 495, 525, 490, 496, 527, 482, 505, 531, 540, 504, 486, 501, 476, 516, 490, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[478, 532, 502, 534, 446, 495, 540, 515, 498, 519, 526, 490, 515, 536, 469, 519, 478, 492, 526, 513, 527, 510, 521, 521, 517, 526, 483, 525, 531, 481, 526, 500, 495, 478, 503, 542, 491, 507, 497, 508, 510, 541, 490, 524, 504, 536, 545, 488, 511, 533, 534, 531, 468, 497, 488, 544, 545, 541, 486, 541, 531, 509, 500, 510, 491, 527, 576, 534, 468, 544, 513, 514, 545, 436, 493, 497, 525, 491, 532, 510, 505, 481, 537, 517, 501, 511, 481, 513, 513, 513, 476, 518, 492, 521, 518, 491, 512, 497, 506, 493, 522, 534, 499, 522, 513, 540, 502, 503, 488, 553, 518, 471, 500, 539, 504, 508, 594, 540, 503, 543, 500, 520, 537, 513, 525, 531, 498, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[559, 494, 540, 541, 538, 461, 477, 524, 511, 516, 507, 491, 475, 514, 547, 551, 555, 503, 469, 532, 526, 515, 536, 481, 465, 526, 510, 531, 547, 522, 504, 527, 521, 508, 505, 516, 487, 542, 513, 527, 516, 482, 542, 552, 472, 516, 512, 468, 509, 550, 511, 539, 497, 498, 515, 513, 533, 513, 530, 485, 540, 491, 518, 526, 491, 512, 525, 519, 514, 527, 512, 493, 498, 510, 484, 504, 522, 513, 511, 486, 520, 506, 515, 487, 482, 499, 540, 518, 506, 508, 475, 511, 490, 522, 493, 535, 524, 547, 527, 497, 556, 496, 526, 480, 499, 529, 482, 500, 514, 564, 492, 483, 514, 513, 523, 511, 478, 519, 532, 492, 524, 495, 505, 529, 473, 506, 506, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[508, 515, 528, 495, 518, 513, 496, 519, 527, 552, 513, 555, 516, 560, 487, 550, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 476, 504, 549, 531, 494, 519, 527, 508, 533, 488, 481, 511, 503, 523, 506, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 543, 473, 513, 516, 498, 513, 528, 546, 553, 498, 493, 499, 510, 533, 516, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 479, 505, 521, 443, 473, 512, 507, 514, 498, 493, 480, 498, 511, 510, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 518, 492, 479, 509, 462, 481, 497, 512, 516, 509, 535, 526, 511, 499, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 532, 489, 505, 542, 533, 495, 473, 524, 519, 502, 524, 538, 572, 520, 482, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 507, 486, 520, 549, 526, 508, 502, 504, 517, 505, 514, 518, 509, 544, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 530, 518, 524, 504, 554, 554, 547, 457, 498, 496, 524, 486, 509, 478, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8359, 8191, 8253, 7935, 8078, 8336, 8192, 8192] avg=8192.0000 max=8359 max_rank=0 min=7935 min_rank=3 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 508, 509, 510, 549, 514, 504, 490, 520, 519, 555, 497, 444, 534, 515, 515, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 499, 509, 464, 512, 528, 473, 514, 489, 519, 495, 509, 493, 531, 533, 506, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 515, 503, 526, 537, 548, 544, 518, 519, 546, 533, 519, 511, 495, 521, 509, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 483, 519, 503, 530, 473, 489, 522, 506, 482, 494, 516, 510, 533, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 529, 571, 513, 508, 518, 502, 539, 540, 525, 479, 547, 531, 518, 537, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 522, 495, 514, 493, 524, 534, 505, 514, 509, 518, 474, 491, 485, 512, 509, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 465, 510, 488, 504, 472, 527, 546, 495, 526, 512, 496, 541, 534, 492, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 513, 532, 493, 507, 525, 513, 495, 522, 492, 503, 552, 498, 473, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8121, 8319, 8107, 8379, 8141, 8094, 8159] avg=8192.0000 max=8379 max_rank=4 min=8094 min_rank=6 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 542, 481, 532, 502, 507, 504, 521, 481, 499, 543, 461, 504, 526, 543, 500, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 484, 491, 510, 514, 509, 514, 480, 567, 506, 546, 519, 554, 454, 557, 498, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 501, 525, 499, 502, 502, 534, 497, 515, 479, 523, 506, 508, 510, 487, 480, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 521, 503, 545, 513, 528, 460, 529, 544, 521, 465, 526, 482, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 521, 484, 504, 553, 521, 491, 534, 517, 531, 504, 524, 505, 491, 513, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 503, 495, 486, 518, 523, 527, 527, 532, 540, 508, 498, 541, 495, 504, 29, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 554, 514, 543, 507, 495, 557, 557, 526, 525, 509, 531, 499, 474, 488, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 499, 490, 496, 527, 482, 505, 531, 540, 504, 486, 501, 476, 516, 490, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8213, 8121, 8162, 8207, 8271, 8305, 8079] avg=8192.0000 max=8305 max_rank=6 min=8079 min_rank=7 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[503, 530, 516, 508, 489, 485, 498, 499, 493, 536, 529, 489, 540, 458, 512, 491, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 512, 484, 567, 527, 445, 520, 504, 543, 467, 513, 532, 502, 508, 538, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, 481, 489, 500, 502, 528, 490, 508, 530, 510, 543, 496, 490, 521, 512, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 525, 529, 465, 510, 562, 514, 523, 478, 527, 551, 498, 527, 496, 505, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 502, 484, 531, 537, 521, 493, 515, 512, 515, 526, 515, 484, 572, 553, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 523, 507, 521, 482, 512, 539, 514, 499, 488, 536, 517, 512, 514, 468, 23, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 475, 521, 523, 493, 476, 544, 489, 511, 529, 503, 511, 490, 504, 543, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 526, 487, 509, 506, 531, 509, 498, 496, 482, 513, 511, 551, 495, 520, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8084, 8223, 8127, 8288, 8319, 8184, 8181, 8130] avg=8192.0000 max=8319 max_rank=4 min=8084 min_rank=0 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 511, 535, 507, 524, 522, 501, 517, 482, 498, 522, 508, 534, 522, 470, 527, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 534, 521, 495, 487, 519, 526, 491, 525, 484, 480, 521, 484, 520, 475, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 491, 505, 537, 514, 536, 492, 529, 512, 518, 488, 507, 517, 490, 533, 511, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 535, 531, 522, 510, 535, 546, 515, 531, 520, 507, 533, 496, 539, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 479, 524, 486, 520, 508, 508, 517, 520, 494, 480, 502, 567, 529, 491, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 495, 514, 506, 521, 492, 494, 512, 510, 478, 488, 487, 543, 512, 519, 516, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 540, 502, 512, 514, 531, 534, 486, 551, 503, 498, 515, 522, 535, 484, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 475, 536, 491, 522, 508, 519, 529, 475, 526, 521, 534, 501, 559, 513, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8100, 8165, 8373, 8155, 8138, 8242, 8182] avg=8192.0000 max=8373 max_rank=3 min=8100 min_rank=1 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[559, 494, 540, 541, 538, 461, 477, 524, 511, 516, 507, 491, 475, 514, 547, 551, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 503, 469, 532, 526, 515, 536, 481, 465, 526, 510, 531, 547, 522, 504, 527, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 508, 505, 516, 487, 542, 513, 527, 516, 482, 542, 552, 472, 516, 512, 468, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 550, 511, 539, 497, 498, 515, 513, 508, 513, 530, 485, 540, 491, 518, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 525, 519, 514, 527, 512, 493, 498, 510, 484, 504, 522, 513, 511, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 506, 515, 487, 482, 499, 540, 518, 506, 508, 475, 511, 490, 522, 493, 535, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 547, 527, 497, 556, 496, 526, 480, 499, 529, 482, 500, 514, 564, 492, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 523, 511, 478, 519, 532, 492, 524, 495, 505, 529, 473, 506, 506, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8260, 8183, 8243, 8106, 8134, 8216, 8141] avg=8192.0000 max=8260 max_rank=1 min=8106 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[509, 500, 531, 515, 500, 521, 521, 521, 486, 478, 514, 481, 508, 509, 527, 516, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 497, 520, 511, 520, 510, 496, 524, 527, 522, 551, 489, 513, 510, 509, 526, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 504, 520, 506, 533, 524, 501, 515, 531, 552, 522, 499, 510, 470, 514, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 485, 475, 515, 538, 510, 505, 488, 507, 496, 520, 517, 520, 528, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 506, 505, 492, 505, 493, 538, 525, 497, 539, 508, 485, 508, 520, 480, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 480, 484, 543, 513, 530, 505, 529, 507, 511, 507, 531, 509, 548, 477, 521, 24, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 512, 499, 513, 498, 481, 508, 504, 538, 512, 495, 544, 489, 472, 513, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 496, 532, 530, 536, 482, 511, 512, 524, 510, 536, 513, 514, 522, 525, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8239, 8205, 8184, 8113, 8216, 8133, 8302] avg=8192.0000 max=8302 max_rank=7 min=8113 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[478, 532, 502, 534, 446, 495, 540, 515, 498, 519, 526, 490, 515, 536, 469, 519, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 492, 526, 513, 527, 510, 521, 521, 517, 526, 483, 525, 531, 481, 526, 500, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 478, 503, 542, 491, 507, 497, 508, 510, 541, 490, 524, 504, 536, 545, 488, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 533, 534, 531, 468, 497, 488, 544, 520, 541, 486, 541, 531, 509, 500, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 527, 576, 534, 468, 544, 513, 514, 545, 436, 493, 497, 525, 491, 532, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 481, 537, 517, 501, 511, 481, 513, 513, 513, 476, 518, 492, 521, 518, 491, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 506, 493, 522, 534, 499, 522, 513, 540, 502, 503, 488, 553, 518, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 530, 504, 508, 594, 540, 503, 543, 500, 520, 537, 513, 525, 531, 498, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8189, 8163, 8244, 8179, 8116, 8173, 8351] avg=8192.0000 max=8351 max_rank=7 min=8116 min_rank=5 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[473, 568, 509, 500, 530, 492, 537, 545, 530, 494, 534, 544, 521, 521, 542, 538, 511, 518, 520, 521, 514, 496, 535, 488, 498, 500, 514, 527, 513, 521, 514, 494, 521, 545, 503, 541, 490, 500, 518, 511, 544, 480, 516, 493, 541, 519, 506, 515, 500, 492, 515, 487, 524, 474, 544, 506, 508, 500, 504, 525, 487, 475, 520, 483, 527, 525, 496, 533, 557, 486, 514, 522, 503, 500, 497, 531, 518, 515, 488, 519, 535, 535, 492, 536, 542, 501, 493, 497, 518, 517, 518, 513, 518, 490, 482, 495, 506, 513, 519, 495, 503, 518, 551, 517, 503, 501, 504, 498, 482, 539, 497, 521, 518, 513, 507, 502, 506, 506, 490, 493, 544, 498, 505, 499, 492, 502, 522, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[508, 520, 542, 524, 514, 532, 512, 501, 486, 490, 521, 490, 511, 508, 491, 496, 535, 511, 511, 496, 475, 550, 542, 522, 533, 484, 540, 520, 467, 515, 482, 510, 532, 536, 533, 521, 522, 553, 526, 511, 485, 525, 542, 484, 504, 522, 551, 479, 500, 508, 486, 508, 501, 494, 520, 510, 512, 515, 541, 471, 497, 564, 510, 490, 496, 519, 523, 491, 528, 511, 496, 521, 560, 479, 515, 498, 500, 552, 530, 493, 530, 492, 498, 505, 505, 518, 477, 482, 503, 486, 510, 533, 520, 549, 539, 499, 524, 484, 473, 513, 509, 508, 521, 500, 559, 505, 546, 508, 538, 545, 497, 533, 517, 472, 526, 489, 524, 495, 512, 516, 513, 485, 511, 505, 534, 505, 519, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[492, 517, 524, 508, 532, 533, 522, 486, 497, 504, 536, 529, 515, 501, 509, 509, 497, 485, 536, 529, 491, 522, 492, 519, 487, 509, 528, 508, 530, 511, 517, 491, 520, 512, 502, 472, 509, 588, 516, 504, 511, 520, 469, 491, 493, 518, 506, 468, 486, 470, 506, 507, 500, 552, 513, 465, 521, 523, 488, 514, 538, 506, 501, 530, 513, 511, 534, 462, 513, 498, 560, 502, 520, 475, 561, 510, 511, 532, 554, 509, 500, 497, 497, 509, 556, 514, 483, 493, 532, 520, 548, 508, 490, 532, 571, 544, 529, 480, 514, 532, 509, 499, 533, 564, 517, 526, 518, 557, 496, 549, 503, 503, 467, 522, 497, 507, 498, 514, 472, 489, 549, 485, 519, 522, 502, 475, 530, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[538, 516, 493, 530, 494, 531, 500, 531, 541, 509, 488, 521, 496, 482, 513, 511, 529, 505, 537, 505, 502, 478, 544, 509, 508, 538, 478, 541, 543, 530, 517, 499, 494, 521, 469, 491, 480, 522, 515, 509, 517, 506, 520, 510, 532, 529, 491, 532, 510, 524, 533, 465, 518, 490, 531, 528, 534, 518, 491, 529, 524, 515, 524, 512, 531, 514, 512, 490, 489, 495, 489, 521, 489, 511, 516, 567, 506, 495, 494, 495, 507, 509, 519, 520, 547, 494, 484, 476, 507, 500, 512, 518, 494, 524, 477, 516, 504, 513, 476, 547, 517, 524, 519, 437, 523, 514, 521, 514, 470, 486, 543, 527, 486, 548, 520, 521, 518, 490, 558, 518, 489, 536, 530, 506, 538, 512, 536, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[523, 533, 535, 497, 469, 476, 507, 507, 501, 536, 535, 516, 556, 490, 498, 481, 532, 565, 496, 532, 537, 500, 559, 510, 555, 506, 522, 481, 533, 493, 509, 478, 533, 485, 534, 535, 489, 508, 543, 518, 464, 478, 460, 446, 551, 462, 501, 478, 546, 522, 534, 515, 535, 554, 564, 470, 550, 542, 512, 460, 466, 514, 522, 518, 550, 518, 519, 505, 555, 487, 498, 485, 512, 534, 524, 476, 521, 482, 522, 481, 508, 542, 466, 477, 493, 527, 522, 524, 557, 474, 492, 507, 496, 514, 513, 494, 494, 494, 516, 541, 505, 505, 468, 521, 520, 528, 578, 511, 500, 529, 525, 547, 529, 502, 542, 506, 533, 486, 521, 528, 498, 517, 532, 500, 507, 488, 520, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[509, 533, 537, 527, 556, 512, 472, 516, 487, 522, 494, 488, 523, 526, 536, 492, 526, 507, 498, 550, 542, 540, 475, 491, 509, 458, 504, 500, 534, 558, 551, 504, 504, 506, 530, 535, 512, 501, 503, 466, 503, 485, 463, 515, 524, 525, 514, 481, 516, 501, 516, 526, 514, 506, 542, 500, 521, 549, 532, 491, 480, 521, 475, 511, 490, 482, 537, 498, 511, 501, 503, 511, 464, 511, 491, 501, 529, 485, 516, 513, 545, 524, 530, 553, 504, 547, 515, 554, 536, 484, 488, 507, 525, 499, 528, 553, 493, 513, 486, 466, 520, 530, 503, 501, 502, 474, 520, 506, 553, 518, 518, 486, 506, 509, 536, 528, 504, 542, 544, 511, 523, 530, 527, 523, 480, 500, 522, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[515, 507, 526, 522, 510, 546, 518, 500, 514, 497, 545, 536, 504, 539, 500, 478, 486, 494, 486, 546, 503, 473, 550, 551, 525, 533, 522, 539, 486, 496, 518, 484, 529, 528, 495, 533, 533, 500, 521, 500, 489, 494, 513, 471, 519, 481, 544, 532, 557, 526, 498, 503, 547, 515, 497, 543, 531, 526, 515, 516, 502, 508, 494, 494, 517, 525, 497, 500, 497, 536, 480, 485, 504, 513, 495, 505, 491, 561, 535, 584, 477, 495, 556, 499, 518, 511, 476, 511, 541, 499, 483, 538, 503, 475, 524, 506, 531, 529, 526, 538, 478, 516, 422, 498, 509, 479, 499, 506, 505, 536, 555, 523, 518, 512, 509, 541, 488, 512, 489, 522, 521, 486, 525, 517, 516, 479, 484, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[513, 548, 523, 495, 505, 524, 497, 503, 548, 496, 510, 488, 535, 476, 530, 523, 473, 529, 494, 489, 507, 524, 495, 549, 522, 495, 519, 544, 499, 525, 470, 511, 545, 507, 502, 493, 505, 502, 523, 488, 516, 487, 508, 518, 505, 528, 493, 521, 556, 477, 505, 551, 509, 489, 502, 505, 498, 466, 561, 485, 539, 527, 523, 538, 509, 532, 491, 517, 526, 517, 519, 486, 564, 555, 504, 544, 547, 490, 498, 494, 504, 516, 527, 473, 482, 499, 484, 508, 519, 524, 472, 510, 500, 531, 516, 527, 537, 518, 485, 502, 525, 519, 513, 511, 488, 517, 520, 531, 521, 517, 511, 542, 543, 470, 502, 538, 539, 491, 498, 512, 521, 482, 490, 487, 551, 509, 524, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 489, 542, 524, 514, 532, 512, 501, 486, 490, 521, 490, 511, 508, 491, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 511, 496, 475, 550, 524, 522, 533, 484, 540, 520, 467, 515, 482, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 536, 533, 521, 522, 553, 526, 511, 485, 525, 542, 484, 504, 522, 551, 479, 31, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 486, 508, 501, 494, 517, 510, 512, 515, 541, 471, 497, 564, 510, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 523, 491, 526, 511, 496, 521, 560, 479, 515, 498, 500, 552, 530, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 492, 498, 505, 505, 518, 477, 482, 503, 486, 510, 533, 520, 549, 539, 499, 8, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 473, 513, 509, 508, 521, 500, 559, 505, 546, 508, 538, 528, 497, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 472, 526, 489, 524, 495, 512, 516, 513, 485, 511, 505, 534, 505, 519, 500, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8175, 8369, 8124, 8210, 8159, 8246, 8138] avg=8192.0000 max=8369 max_rank=2 min=8115 min_rank=0 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[473, 534, 509, 500, 530, 492, 537, 545, 530, 494, 534, 544, 521, 521, 542, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 520, 521, 514, 496, 517, 488, 498, 500, 514, 527, 513, 521, 514, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 545, 503, 541, 490, 500, 518, 511, 544, 480, 516, 493, 541, 519, 506, 515, 34, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 492, 515, 487, 524, 474, 541, 506, 508, 500, 504, 525, 487, 475, 520, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 496, 533, 555, 486, 514, 522, 503, 500, 497, 531, 518, 515, 488, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 535, 492, 536, 542, 501, 493, 497, 518, 517, 518, 513, 518, 490, 482, 495, 8, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 519, 495, 503, 518, 551, 517, 503, 501, 504, 498, 482, 522, 497, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 507, 502, 506, 506, 490, 493, 544, 498, 505, 499, 492, 502, 522, 510, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 8166, 8289, 8041, 8229, 8195, 8150, 8122] avg=8192.0000 max=8344 max_rank=0 min=8041 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[538, 485, 493, 530, 494, 531, 500, 531, 541, 509, 488, 521, 496, 482, 513, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 505, 537, 505, 502, 478, 526, 509, 508, 538, 478, 541, 543, 530, 517, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 521, 469, 491, 480, 522, 515, 509, 517, 506, 520, 510, 532, 529, 491, 532, 31, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 533, 465, 518, 490, 528, 528, 534, 518, 491, 529, 524, 515, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 514, 512, 490, 488, 495, 489, 521, 489, 511, 516, 567, 506, 495, 494, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 519, 520, 547, 494, 484, 476, 507, 500, 512, 518, 494, 524, 477, 516, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 476, 547, 517, 524, 519, 437, 523, 514, 521, 514, 470, 471, 543, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 548, 520, 521, 518, 490, 558, 518, 489, 536, 530, 506, 538, 512, 536, 536, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8245, 8181, 8243, 8113, 8115, 8120, 8356] avg=8192.0000 max=8356 max_rank=7 min=8113 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[492, 483, 524, 508, 532, 533, 522, 486, 497, 504, 536, 529, 515, 501, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 485, 536, 529, 491, 522, 467, 519, 487, 509, 528, 508, 530, 511, 517, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 512, 502, 472, 509, 588, 516, 504, 511, 520, 469, 491, 493, 518, 506, 468, 34, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 470, 506, 507, 500, 552, 505, 465, 521, 523, 488, 514, 538, 506, 501, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 511, 534, 462, 508, 498, 560, 502, 520, 475, 561, 510, 511, 532, 554, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 497, 509, 556, 514, 483, 493, 532, 520, 548, 508, 490, 532, 571, 544, 11, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 480, 514, 532, 509, 499, 533, 564, 517, 526, 518, 557, 496, 525, 503, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 522, 497, 507, 498, 514, 472, 489, 549, 485, 519, 522, 502, 475, 530, 515, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8127, 8148, 8112, 8260, 8318, 8305, 8086] avg=8192.0000 max=8318 max_rank=5 min=8086 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32969 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[509, 501, 537, 527, 556, 512, 472, 516, 487, 522, 494, 488, 523, 526, 536, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 498, 550, 542, 540, 459, 491, 509, 458, 504, 500, 534, 558, 551, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 530, 535, 512, 501, 503, 466, 503, 485, 463, 515, 524, 525, 514, 481, 32, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 501, 516, 526, 514, 506, 539, 500, 521, 549, 532, 491, 480, 521, 475, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 482, 537, 498, 510, 501, 503, 511, 464, 511, 491, 501, 529, 485, 516, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 524, 530, 553, 504, 547, 515, 554, 536, 484, 488, 507, 525, 499, 528, 553, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 486, 466, 520, 530, 503, 501, 502, 474, 520, 506, 553, 503, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 536, 528, 504, 542, 544, 511, 523, 530, 527, 523, 480, 500, 522, 482, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8231, 8110, 8198, 8042, 8403, 8074, 8280] avg=8192.0000 max=8403 max_rank=5 min=8042 min_rank=4 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 501, 535, 497, 469, 476, 507, 507, 501, 536, 535, 516, 556, 490, 498, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 565, 496, 532, 537, 500, 540, 510, 555, 506, 522, 481, 533, 493, 509, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 485, 534, 535, 489, 508, 543, 518, 464, 478, 460, 446, 551, 462, 501, 478, 32, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 522, 534, 515, 535, 554, 561, 470, 550, 542, 512, 460, 466, 514, 522, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 518, 519, 505, 553, 487, 498, 485, 512, 534, 524, 476, 521, 482, 522, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 542, 466, 477, 493, 527, 522, 524, 557, 474, 492, 507, 496, 514, 513, 494, 7, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 494, 516, 541, 505, 505, 468, 521, 520, 528, 578, 511, 500, 513, 525, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 542, 506, 533, 486, 521, 528, 498, 517, 532, 500, 507, 488, 520, 493, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8289, 8030, 8321, 8167, 8118, 8266, 8217] avg=8192.0000 max=8321 max_rank=3 min=8030 min_rank=2 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[515, 477, 526, 522, 510, 546, 518, 500, 514, 497, 545, 536, 504, 539, 500, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 494, 486, 546, 503, 473, 532, 551, 525, 533, 522, 539, 486, 496, 518, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 495, 533, 533, 500, 521, 500, 489, 494, 513, 471, 519, 481, 544, 532, 30, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 526, 498, 503, 547, 515, 494, 543, 531, 526, 515, 516, 502, 508, 494, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 497, 500, 496, 536, 480, 485, 504, 513, 495, 505, 491, 561, 535, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 495, 556, 499, 518, 511, 476, 511, 541, 499, 483, 538, 503, 475, 524, 506, 8, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 529, 526, 538, 478, 516, 422, 498, 509, 479, 499, 506, 505, 519, 555, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 509, 541, 488, 512, 489, 522, 521, 486, 525, 517, 516, 479, 484, 527, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8174, 8224, 8269, 8224, 8124, 8133, 8161] avg=8192.0000 max=8269 max_rank=3 min=8124 min_rank=5 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 515, 523, 495, 505, 524, 497, 503, 548, 496, 510, 488, 535, 476, 530, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 529, 494, 489, 507, 524, 479, 549, 522, 495, 519, 544, 499, 525, 470, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 507, 502, 493, 505, 502, 523, 488, 516, 487, 508, 518, 505, 528, 493, 521, 33, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 477, 505, 551, 509, 489, 499, 505, 498, 466, 561, 485, 539, 527, 523, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 532, 491, 517, 524, 517, 519, 486, 564, 555, 504, 544, 547, 490, 498, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 527, 473, 482, 499, 484, 508, 519, 524, 472, 510, 500, 531, 516, 527, 7, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 518, 485, 502, 525, 519, 513, 511, 488, 517, 520, 531, 521, 502, 511, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 470, 502, 538, 539, 491, 498, 512, 521, 482, 490, 487, 551, 509, 524, 506, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8129, 8185, 8228, 8291, 8104, 8242, 8176] avg=8192.0000 max=8291 max_rank=4 min=8104 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 4224, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[515, 518, 514, 513, 519, 504, 500, 528, 486, 508, 502, 525, 502, 501, 527, 505, 493, 511, 532, 487, 526, 490, 489, 508, 493, 534, 551, 551, 494, 457, 495, 485, 498, 546, 491, 518, 521, 526, 528, 533, 541, 520, 484, 480, 553, 493, 521, 527, 507, 528, 507, 475, 503, 497, 509, 520, 535, 513, 494, 505, 516, 475, 462, 508, 537, 502, 515, 513, 516, 472, 519, 541, 506, 514, 547, 532, 524, 525, 549, 551, 513, 503, 499, 505, 532, 525, 544, 486, 466, 480, 521, 539, 517, 538, 534, 477, 462, 503, 566, 497, 530, 500, 537, 496, 508, 531, 536, 491, 533, 562, 513, 510, 511, 527, 543, 495, 505, 500, 461, 526, 504, 504, 490, 530, 483, 521, 511, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[499, 523, 544, 476, 523, 483, 502, 510, 462, 504, 473, 528, 557, 523, 535, 513, 512, 519, 519, 504, 501, 531, 542, 563, 512, 489, 482, 515, 521, 510, 535, 561, 546, 516, 505, 521, 518, 505, 483, 473, 476, 507, 551, 501, 530, 510, 508, 509, 541, 477, 512, 522, 525, 478, 511, 501, 496, 516, 512, 481, 488, 501, 472, 505, 508, 526, 543, 495, 550, 520, 490, 477, 523, 537, 491, 521, 504, 476, 530, 515, 527, 530, 500, 495, 570, 522, 488, 519, 519, 550, 524, 514, 460, 521, 519, 492, 492, 504, 488, 480, 499, 529, 496, 523, 513, 530, 483, 537, 553, 516, 529, 549, 511, 506, 498, 550, 529, 478, 534, 498, 514, 512, 502, 469, 538, 517, 510, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[548, 515, 481, 528, 518, 548, 490, 522, 468, 495, 484, 531, 492, 538, 508, 494, 486, 491, 498, 476, 523, 550, 503, 523, 525, 524, 515, 500, 524, 514, 515, 526, 523, 485, 494, 477, 518, 478, 545, 489, 511, 512, 505, 463, 520, 469, 514, 545, 539, 518, 510, 524, 480, 538, 520, 503, 518, 513, 547, 491, 539, 522, 524, 485, 479, 542, 509, 468, 516, 506, 507, 498, 527, 494, 547, 517, 500, 530, 477, 495, 534, 500, 525, 501, 488, 513, 510, 510, 539, 527, 492, 523, 542, 514, 522, 517, 485, 501, 512, 537, 519, 484, 549, 497, 517, 537, 490, 481, 526, 524, 502, 500, 536, 551, 537, 504, 499, 540, 512, 527, 514, 547, 513, 504, 506, 527, 502, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[484, 528, 513, 504, 550, 521, 486, 491, 528, 537, 555, 460, 500, 533, 468, 500, 512, 487, 498, 524, 503, 515, 504, 503, 542, 505, 527, 513, 514, 539, 472, 505, 522, 511, 497, 519, 501, 516, 496, 532, 523, 496, 522, 514, 500, 552, 490, 514, 498, 535, 511, 484, 475, 563, 521, 513, 510, 527, 536, 540, 487, 496, 511, 513, 530, 479, 538, 527, 519, 506, 514, 578, 550, 519, 480, 502, 563, 540, 493, 492, 547, 506, 540, 486, 511, 531, 492, 520, 533, 507, 462, 523, 495, 506, 496, 571, 499, 460, 533, 488, 475, 505, 503, 494, 485, 505, 508, 535, 501, 519, 477, 506, 518, 560, 473, 495, 497, 520, 539, 520, 532, 504, 517, 494, 504, 528, 517, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[497, 498, 521, 534, 515, 554, 498, 520, 522, 488, 529, 541, 525, 516, 530, 499, 501, 473, 520, 509, 520, 494, 485, 523, 538, 515, 511, 488, 513, 540, 535, 505, 532, 491, 513, 500, 519, 537, 541, 500, 503, 548, 556, 501, 529, 518, 498, 523, 507, 474, 495, 516, 496, 506, 519, 493, 518, 515, 490, 465, 500, 497, 537, 539, 509, 483, 498, 517, 512, 543, 511, 491, 477, 497, 512, 518, 541, 490, 543, 470, 466, 470, 538, 556, 525, 534, 491, 551, 538, 522, 504, 515, 515, 539, 483, 499, 538, 520, 477, 520, 498, 486, 516, 514, 523, 547, 493, 475, 500, 516, 476, 528, 531, 496, 503, 510, 477, 524, 549, 525, 503, 543, 514, 516, 500, 520, 516, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[500, 477, 520, 511, 505, 533, 528, 537, 506, 467, 512, 505, 478, 514, 518, 550, 539, 550, 525, 534, 506, 467, 524, 530, 496, 477, 530, 512, 482, 529, 546, 502, 483, 479, 535, 511, 534, 560, 519, 512, 522, 517, 477, 511, 522, 488, 507, 543, 484, 504, 492, 495, 499, 504, 504, 456, 492, 489, 534, 498, 489, 514, 525, 474, 548, 554, 502, 495, 519, 470, 540, 534, 534, 492, 534, 560, 494, 530, 503, 549, 528, 536, 480, 519, 487, 512, 502, 521, 500, 477, 514, 563, 537, 548, 492, 507, 534, 511, 576, 524, 495, 500, 527, 502, 493, 482, 509, 476, 527, 507, 528, 471, 520, 510, 499, 531, 476, 505, 533, 471, 514, 520, 544, 534, 505, 497, 534, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[484, 533, 537, 547, 499, 497, 547, 503, 489, 522, 487, 514, 467, 533, 500, 493, 546, 516, 529, 474, 537, 527, 497, 517, 508, 505, 496, 530, 476, 514, 498, 502, 508, 497, 513, 521, 524, 490, 531, 547, 515, 512, 508, 492, 505, 503, 485, 530, 560, 527, 521, 529, 495, 520, 499, 491, 529, 521, 530, 514, 525, 514, 512, 483, 517, 522, 521, 536, 483, 488, 478, 540, 530, 504, 499, 501, 489, 492, 496, 511, 503, 485, 517, 515, 540, 514, 519, 523, 529, 537, 512, 513, 538, 511, 505, 488, 516, 506, 503, 549, 519, 505, 507, 521, 483, 526, 525, 544, 523, 459, 502, 536, 564, 534, 488, 511, 504, 498, 487, 510, 492, 520, 544, 499, 493, 483, 515, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[501, 488, 560, 519, 552, 533, 494, 523, 484, 513, 511, 546, 489, 536, 496, 541, 512, 540, 538, 505, 540, 505, 496, 491, 502, 483, 519, 523, 505, 495, 494, 533, 538, 472, 498, 513, 477, 529, 501, 523, 477, 490, 530, 484, 521, 505, 538, 532, 515, 500, 483, 512, 508, 533, 509, 516, 511, 505, 516, 524, 521, 512, 548, 505, 555, 503, 537, 511, 531, 495, 540, 538, 565, 491, 475, 484, 557, 507, 504, 506, 493, 491, 541, 478, 503, 474, 498, 471, 518, 494, 512, 538, 508, 527, 494, 510, 539, 539, 514, 493, 514, 460, 547, 529, 523, 495, 493, 496, 542, 503, 506, 478, 495, 515, 483, 506, 514, 519, 519, 504, 500, 503, 519, 535, 530, 506, 520, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[548, 515, 481, 528, 518, 548, 490, 522, 468, 495, 484, 531, 492, 538, 508, 494, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 491, 498, 476, 523, 550, 503, 523, 525, 524, 515, 500, 524, 514, 515, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 485, 494, 477, 518, 478, 545, 489, 511, 512, 505, 463, 520, 469, 514, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 518, 510, 524, 480, 538, 520, 503, 518, 513, 547, 491, 539, 522, 524, 485, 43, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 542, 509, 468, 516, 506, 507, 498, 484, 494, 547, 517, 500, 530, 477, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 500, 525, 501, 488, 513, 510, 510, 539, 527, 492, 506, 542, 514, 522, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 501, 512, 537, 519, 484, 549, 497, 517, 537, 490, 481, 526, 524, 502, 500, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 530, 537, 504, 499, 540, 512, 527, 514, 547, 513, 504, 506, 527, 502, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8193, 8048, 8326, 8069, 8240, 8186, 8313] avg=8192.0000 max=8326 max_rank=3 min=8048 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[484, 528, 513, 504, 550, 521, 486, 491, 528, 537, 555, 460, 500, 533, 468, 500, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 487, 498, 524, 503, 515, 504, 503, 542, 505, 527, 513, 514, 539, 472, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 511, 497, 519, 501, 516, 496, 532, 523, 496, 522, 514, 500, 552, 490, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 535, 511, 484, 475, 563, 521, 513, 510, 527, 536, 540, 487, 496, 511, 513, 45, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 479, 538, 527, 519, 506, 514, 578, 505, 519, 480, 502, 563, 540, 493, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 506, 540, 486, 511, 531, 492, 520, 533, 507, 462, 506, 495, 506, 496, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 460, 533, 488, 475, 505, 503, 494, 485, 505, 508, 535, 501, 519, 477, 506, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 539, 473, 495, 497, 520, 539, 520, 532, 504, 517, 494, 504, 528, 517, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8163, 8205, 8277, 8285, 8209, 8018, 8220] avg=8192.0000 max=8285 max_rank=4 min=8018 min_rank=6 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[499, 523, 544, 476, 523, 483, 502, 510, 462, 504, 473, 528, 557, 523, 535, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 515, 504, 501, 531, 542, 563, 512, 489, 482, 515, 521, 510, 535, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 516, 505, 521, 518, 505, 483, 473, 476, 507, 551, 501, 530, 510, 508, 509, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 477, 512, 522, 525, 478, 511, 501, 496, 516, 512, 481, 488, 501, 472, 505, 47, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 526, 543, 495, 550, 520, 490, 477, 476, 537, 491, 521, 504, 476, 530, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 530, 500, 495, 570, 522, 488, 519, 519, 550, 524, 490, 460, 521, 519, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 488, 480, 499, 529, 496, 523, 513, 530, 483, 537, 553, 516, 529, 549, 19, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 479, 498, 550, 529, 478, 534, 498, 514, 512, 502, 469, 538, 517, 510, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8312, 8165, 8100, 8159, 8226, 8252, 8164] avg=8192.0000 max=8312 max_rank=1 min=8100 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[515, 518, 514, 513, 519, 504, 500, 528, 486, 508, 502, 525, 502, 501, 527, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 511, 532, 487, 526, 490, 489, 508, 493, 534, 551, 551, 494, 457, 495, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 546, 491, 518, 521, 526, 528, 533, 541, 520, 484, 480, 553, 493, 521, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 507, 475, 503, 497, 509, 520, 535, 513, 494, 505, 516, 475, 462, 508, 42, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 502, 515, 513, 516, 472, 519, 541, 464, 514, 547, 532, 524, 525, 549, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 503, 499, 505, 532, 525, 544, 486, 466, 480, 521, 522, 517, 538, 534, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 503, 566, 497, 530, 500, 537, 496, 508, 531, 536, 491, 533, 562, 513, 510, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 543, 495, 505, 500, 461, 526, 504, 504, 490, 530, 483, 521, 511, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8096, 8280, 8107, 8321, 8162, 8299, 8103] avg=8192.0000 max=8321 max_rank=4 min=8096 min_rank=1 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[500, 477, 520, 511, 505, 533, 528, 537, 506, 467, 512, 505, 478, 514, 518, 550, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 550, 525, 534, 506, 467, 524, 530, 496, 477, 530, 512, 482, 529, 546, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 479, 535, 511, 534, 560, 519, 512, 522, 517, 477, 511, 522, 488, 507, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 504, 492, 495, 499, 504, 504, 456, 492, 489, 534, 498, 489, 514, 525, 474, 44, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 554, 502, 495, 519, 470, 540, 534, 490, 492, 534, 560, 494, 530, 503, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 536, 480, 519, 487, 512, 502, 521, 500, 477, 514, 545, 537, 548, 492, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 576, 524, 495, 500, 527, 502, 493, 482, 509, 476, 527, 507, 528, 471, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 491, 499, 531, 476, 505, 533, 471, 514, 520, 544, 534, 505, 497, 534, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8249, 8220, 8008, 8314, 8205, 8187, 8191] avg=8192.0000 max=8314 max_rank=4 min=8008 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[501, 488, 560, 519, 552, 533, 494, 523, 484, 513, 511, 546, 489, 536, 496, 541, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 540, 538, 505, 540, 505, 496, 491, 502, 483, 519, 523, 505, 495, 494, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 472, 498, 513, 477, 529, 501, 523, 477, 490, 530, 484, 521, 505, 538, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 500, 483, 512, 508, 533, 509, 516, 511, 505, 516, 524, 521, 512, 548, 505, 47, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 503, 537, 511, 531, 495, 540, 538, 518, 491, 475, 484, 557, 507, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 491, 541, 478, 503, 474, 498, 471, 518, 494, 512, 521, 508, 527, 494, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 539, 514, 493, 514, 460, 547, 529, 523, 495, 493, 496, 542, 503, 506, 478, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 483, 506, 514, 519, 519, 504, 500, 503, 519, 535, 530, 506, 520, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8181, 8128, 8276, 8252, 8033, 8195, 8184] avg=8192.0000 max=8287 max_rank=0 min=8033 min_rank=5 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[497, 498, 521, 534, 515, 554, 498, 520, 522, 488, 529, 541, 525, 516, 530, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 473, 520, 509, 520, 494, 485, 523, 538, 515, 511, 488, 513, 540, 535, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 491, 513, 500, 519, 537, 541, 500, 503, 548, 556, 501, 529, 518, 498, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 474, 495, 516, 496, 506, 519, 493, 518, 515, 490, 465, 500, 497, 537, 539, 39, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 483, 498, 517, 512, 543, 511, 491, 438, 497, 512, 518, 541, 490, 543, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 470, 538, 556, 525, 534, 491, 551, 538, 522, 504, 498, 515, 539, 483, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 520, 477, 520, 498, 486, 516, 514, 523, 547, 493, 475, 500, 516, 476, 528, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 477, 503, 510, 477, 524, 549, 525, 503, 543, 514, 516, 500, 520, 516, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8170, 8309, 8117, 8073, 8229, 8151, 8199] avg=8192.0000 max=8309 max_rank=2 min=8073 min_rank=4 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[484, 533, 537, 547, 499, 497, 547, 503, 489, 522, 487, 514, 467, 533, 500, 493, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 516, 529, 474, 537, 527, 497, 517, 508, 505, 496, 530, 476, 514, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 513, 521, 524, 490, 531, 547, 515, 512, 508, 492, 505, 503, 485, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 527, 521, 529, 495, 520, 499, 491, 529, 521, 530, 514, 525, 514, 512, 483, 44, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 521, 536, 483, 488, 478, 540, 486, 504, 499, 501, 489, 492, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 485, 517, 515, 540, 514, 519, 523, 529, 537, 512, 496, 538, 511, 505, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 503, 549, 519, 505, 507, 521, 483, 526, 525, 544, 523, 459, 502, 536, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 515, 488, 511, 504, 498, 487, 510, 492, 520, 544, 499, 493, 483, 515, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8172, 8181, 8325, 8063, 8232, 8248, 8162] avg=8192.0000 max=8325 max_rank=3 min=8063 min_rank=4 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[526, 526, 536, 528, 512, 521, 514, 494, 514, 505, 476, 512, 495, 483, 478, 537, 505, 520, 503, 523, 508, 513, 526, 514, 537, 524, 487, 515, 506, 540, 527, 505, 521, 516, 543, 506, 516, 518, 516, 545, 522, 507, 519, 531, 483, 519, 473, 489, 516, 507, 489, 496, 489, 510, 543, 463, 491, 499, 509, 480, 528, 508, 508, 499, 484, 521, 542, 497, 527, 527, 512, 520, 512, 488, 531, 544, 526, 500, 500, 469, 498, 508, 509, 492, 516, 467, 499, 523, 478, 516, 571, 501, 503, 553, 491, 516, 505, 556, 526, 486, 518, 520, 509, 502, 493, 556, 502, 534, 493, 486, 499, 508, 513, 538, 516, 538, 525, 550, 471, 543, 502, 510, 520, 505, 561, 491, 495, 555] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[497, 495, 539, 446, 530, 506, 534, 526, 546, 530, 543, 519, 532, 546, 523, 502, 536, 515, 515, 542, 496, 553, 540, 513, 472, 505, 497, 515, 533, 479, 488, 485, 534, 562, 509, 518, 495, 498, 514, 499, 486, 519, 516, 522, 512, 499, 502, 499, 550, 482, 505, 489, 545, 500, 492, 532, 532, 502, 523, 503, 545, 525, 504, 483, 473, 496, 507, 520, 463, 542, 492, 501, 503, 518, 527, 532, 504, 503, 520, 520, 539, 491, 504, 501, 502, 505, 517, 551, 494, 510, 513, 561, 510, 543, 492, 508, 509, 452, 494, 522, 519, 476, 487, 493, 495, 498, 510, 555, 520, 514, 529, 529, 501, 527, 523, 498, 501, 558, 511, 543, 521, 538, 515, 484, 495, 486, 484, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[528, 504, 486, 528, 539, 506, 450, 514, 432, 519, 518, 503, 535, 529, 508, 534, 556, 568, 509, 520, 488, 486, 533, 495, 500, 525, 499, 511, 536, 508, 495, 511, 478, 502, 545, 529, 500, 522, 509, 497, 499, 489, 509, 498, 541, 523, 509, 516, 500, 510, 520, 490, 485, 508, 509, 469, 535, 497, 505, 491, 480, 551, 477, 551, 515, 503, 522, 519, 535, 499, 491, 473, 511, 523, 513, 520, 539, 475, 515, 516, 503, 510, 504, 524, 519, 514, 485, 526, 491, 532, 518, 509, 563, 514, 495, 522, 467, 544, 524, 490, 499, 533, 532, 522, 502, 510, 529, 522, 507, 508, 541, 517, 526, 530, 498, 495, 530, 521, 540, 482, 531, 474, 536, 521, 529, 559, 513, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[483, 539, 523, 500, 547, 526, 498, 540, 499, 498, 512, 521, 518, 520, 506, 491, 479, 482, 479, 500, 502, 542, 523, 526, 509, 514, 485, 514, 503, 535, 503, 488, 505, 486, 509, 519, 514, 536, 514, 479, 485, 491, 528, 496, 528, 475, 548, 533, 524, 481, 520, 518, 521, 559, 540, 515, 494, 497, 485, 505, 546, 482, 542, 516, 527, 503, 525, 488, 508, 515, 488, 501, 517, 527, 532, 523, 499, 507, 538, 466, 513, 493, 489, 475, 551, 498, 485, 516, 557, 535, 539, 536, 499, 503, 503, 526, 515, 506, 509, 533, 518, 558, 521, 486, 521, 523, 481, 532, 547, 516, 496, 511, 512, 509, 544, 517, 510, 536, 513, 511, 517, 500, 466, 544, 469, 531, 515, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[510, 529, 486, 530, 497, 515, 510, 521, 521, 518, 543, 509, 510, 484, 521, 524, 508, 540, 490, 549, 472, 497, 537, 502, 520, 537, 508, 530, 563, 481, 526, 491, 527, 525, 548, 521, 532, 458, 483, 556, 486, 534, 524, 527, 472, 517, 494, 531, 519, 507, 451, 484, 539, 499, 542, 512, 530, 512, 496, 470, 548, 506, 501, 557, 499, 490, 503, 485, 517, 516, 529, 548, 536, 523, 491, 479, 478, 497, 544, 526, 460, 517, 501, 567, 498, 522, 517, 519, 524, 491, 534, 544, 544, 484, 471, 507, 507, 504, 514, 497, 534, 491, 495, 484, 485, 509, 495, 501, 530, 529, 530, 514, 527, 508, 501, 523, 515, 505, 516, 481, 491, 502, 481, 509, 567, 499, 526, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[518, 482, 505, 496, 493, 494, 509, 508, 526, 506, 515, 544, 527, 508, 512, 499, 473, 519, 470, 519, 519, 520, 523, 510, 478, 532, 496, 541, 511, 503, 480, 487, 515, 491, 514, 508, 537, 537, 479, 515, 543, 508, 531, 533, 530, 534, 497, 519, 519, 517, 523, 509, 515, 530, 543, 485, 487, 556, 519, 529, 493, 499, 510, 559, 452, 539, 518, 548, 493, 475, 518, 506, 515, 504, 536, 522, 495, 498, 502, 494, 526, 491, 513, 510, 539, 514, 491, 525, 493, 542, 550, 492, 535, 513, 503, 495, 531, 503, 468, 448, 517, 516, 482, 477, 519, 519, 478, 535, 477, 532, 520, 484, 509, 524, 525, 544, 508, 477, 536, 554, 502, 556, 525, 552, 477, 548, 500, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[530, 537, 527, 499, 493, 536, 486, 477, 529, 537, 513, 515, 503, 531, 520, 513, 468, 544, 513, 559, 480, 490, 532, 510, 512, 491, 519, 514, 525, 555, 494, 527, 511, 506, 511, 529, 549, 528, 465, 472, 557, 530, 506, 502, 488, 529, 537, 511, 496, 471, 492, 515, 465, 520, 499, 535, 537, 515, 518, 569, 503, 480, 534, 490, 486, 483, 486, 544, 515, 515, 512, 488, 559, 486, 485, 502, 508, 482, 489, 501, 500, 530, 522, 512, 494, 529, 503, 511, 499, 541, 484, 533, 504, 485, 516, 518, 539, 505, 514, 524, 554, 503, 531, 509, 508, 527, 528, 519, 516, 498, 548, 501, 495, 497, 508, 538, 528, 518, 471, 503, 531, 498, 512, 499, 515, 502, 514, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[500, 523, 475, 497, 533, 544, 468, 523, 524, 504, 553, 492, 532, 516, 508, 569, 493, 534, 506, 523, 538, 504, 516, 506, 523, 515, 506, 521, 505, 526, 496, 520, 522, 540, 500, 503, 519, 525, 507, 543, 507, 502, 495, 494, 519, 489, 529, 527, 527, 483, 513, 531, 528, 527, 511, 512, 484, 477, 503, 486, 506, 512, 508, 506, 534, 490, 523, 503, 495, 526, 515, 514, 500, 499, 487, 523, 490, 513, 515, 533, 499, 535, 524, 523, 522, 475, 500, 515, 515, 520, 519, 525, 508, 521, 539, 535, 503, 552, 505, 539, 533, 521, 519, 510, 485, 522, 514, 487, 544, 491, 504, 487, 461, 512, 545, 510, 492, 470, 545, 515, 475, 516, 489, 488, 546, 467, 512, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 526, 536, 528, 512, 521, 514, 494, 514, 505, 458, 512, 495, 483, 478, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 520, 503, 521, 508, 513, 526, 514, 537, 524, 487, 515, 506, 540, 527, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 523, 506, 516, 518, 516, 545, 522, 507, 519, 531, 483, 519, 473, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 489, 496, 489, 510, 543, 463, 491, 499, 509, 480, 528, 508, 508, 499, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 521, 542, 497, 527, 527, 512, 520, 512, 488, 531, 544, 526, 500, 500, 469, 24, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 509, 492, 516, 467, 499, 523, 478, 516, 547, 501, 503, 553, 491, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 556, 526, 486, 518, 520, 509, 502, 493, 556, 502, 534, 493, 486, 499, 508, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 538, 516, 516, 525, 550, 471, 543, 502, 510, 520, 505, 561, 491, 495, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8251, 8204, 8064, 8251, 8117, 8199, 8311] avg=8192.0000 max=8311 max_rank=7 min=8064 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 482, 505, 496, 493, 494, 509, 508, 526, 506, 496, 544, 527, 508, 512, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 519, 470, 517, 519, 520, 523, 510, 478, 532, 496, 541, 511, 503, 480, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 491, 495, 508, 537, 537, 479, 515, 543, 508, 531, 533, 530, 534, 497, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 517, 523, 509, 515, 530, 543, 485, 487, 556, 519, 529, 493, 499, 510, 559, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 539, 518, 548, 493, 475, 518, 506, 515, 504, 536, 522, 495, 498, 502, 494, 23, 23, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 491, 513, 510, 539, 514, 491, 525, 493, 542, 527, 492, 535, 513, 503, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 503, 468, 448, 517, 516, 482, 477, 519, 519, 478, 535, 477, 532, 520, 484, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 524, 525, 521, 508, 477, 536, 554, 502, 556, 525, 552, 477, 548, 500, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8079, 8272, 8322, 8166, 8209, 8012, 8353] avg=8192.0000 max=8353 max_rank=7 min=8012 min_rank=6 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[497, 495, 539, 446, 530, 506, 534, 526, 546, 530, 513, 519, 532, 546, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 515, 515, 534, 496, 553, 540, 513, 472, 505, 497, 515, 533, 479, 488, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 562, 483, 518, 495, 498, 514, 499, 486, 519, 516, 522, 512, 499, 502, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 482, 505, 489, 545, 500, 492, 532, 532, 502, 523, 503, 545, 525, 504, 483, 20, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 496, 507, 520, 463, 542, 492, 501, 503, 518, 527, 532, 504, 503, 520, 520, 25, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 491, 504, 501, 502, 505, 517, 551, 494, 510, 488, 561, 510, 543, 492, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 452, 494, 522, 519, 476, 487, 493, 495, 498, 510, 555, 520, 514, 529, 529, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 527, 523, 473, 501, 558, 511, 543, 521, 538, 515, 484, 495, 486, 484, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8176, 8158, 8252, 8181, 8216, 8116, 8153] avg=8192.0000 max=8284 max_rank=0 min=8116 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[530, 537, 527, 499, 493, 536, 486, 477, 529, 537, 494, 515, 503, 531, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 544, 513, 557, 480, 490, 532, 510, 512, 491, 519, 514, 525, 555, 494, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 492, 529, 549, 528, 465, 472, 557, 530, 506, 502, 488, 529, 537, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 471, 492, 515, 465, 520, 499, 535, 537, 515, 518, 569, 503, 480, 534, 490, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 483, 486, 544, 515, 515, 512, 488, 559, 486, 485, 502, 508, 482, 489, 501, 20, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 530, 522, 512, 494, 529, 503, 511, 499, 541, 464, 533, 504, 485, 516, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 505, 514, 524, 554, 503, 531, 509, 508, 527, 528, 519, 516, 498, 548, 501, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 508, 516, 528, 518, 471, 503, 531, 498, 512, 499, 515, 502, 514, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8231, 8212, 8168, 8088, 8161, 8330, 8119] avg=8192.0000 max=8330 max_rank=6 min=8088 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 523, 475, 497, 533, 544, 468, 523, 524, 504, 532, 492, 532, 516, 508, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 534, 506, 521, 538, 504, 516, 506, 523, 515, 506, 521, 505, 526, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 540, 481, 503, 519, 525, 507, 543, 507, 502, 495, 494, 519, 489, 529, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 483, 513, 531, 528, 527, 511, 512, 484, 477, 503, 486, 506, 512, 508, 506, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 523, 503, 495, 526, 515, 514, 500, 499, 487, 523, 490, 513, 515, 533, 22, 21, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 535, 524, 523, 522, 475, 500, 515, 515, 520, 497, 525, 508, 521, 539, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 552, 505, 539, 533, 521, 519, 510, 485, 522, 514, 487, 544, 491, 504, 487, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 512, 545, 489, 492, 470, 545, 515, 475, 516, 489, 488, 546, 467, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8230, 8202, 8144, 8209, 8253, 8222, 8036] avg=8192.0000 max=8253 max_rank=5 min=8036 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[483, 539, 523, 500, 547, 526, 498, 540, 499, 498, 493, 521, 518, 520, 506, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 482, 479, 498, 502, 542, 523, 526, 509, 514, 485, 514, 503, 535, 503, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 486, 490, 519, 514, 536, 514, 479, 485, 491, 528, 496, 528, 475, 548, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 481, 520, 518, 521, 559, 540, 515, 494, 497, 485, 505, 546, 482, 542, 516, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 503, 525, 488, 508, 515, 488, 501, 517, 527, 532, 523, 499, 507, 538, 466, 23, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 493, 489, 475, 551, 498, 485, 516, 557, 535, 516, 536, 499, 503, 503, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 506, 509, 533, 518, 558, 521, 486, 521, 523, 481, 532, 547, 516, 496, 511, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 544, 496, 510, 536, 513, 511, 517, 500, 466, 544, 469, 531, 515, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8082, 8127, 8274, 8213, 8195, 8279, 8164] avg=8192.0000 max=8279 max_rank=6 min=8082 min_rank=1 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[510, 529, 486, 530, 497, 515, 510, 521, 521, 518, 522, 509, 510, 484, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 540, 490, 547, 472, 497, 537, 502, 520, 537, 508, 530, 563, 481, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 528, 521, 532, 458, 483, 556, 486, 534, 524, 527, 472, 517, 494, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 451, 484, 539, 499, 542, 512, 530, 512, 496, 470, 548, 506, 501, 557, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 490, 503, 485, 517, 516, 529, 548, 536, 523, 491, 479, 478, 497, 544, 526, 22, 22, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 517, 501, 567, 498, 522, 517, 519, 524, 491, 512, 544, 544, 484, 471, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 514, 497, 534, 491, 495, 484, 485, 509, 495, 501, 530, 529, 530, 514, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 508, 501, 501, 515, 505, 516, 481, 491, 502, 481, 509, 567, 499, 526, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8249, 8215, 8204, 8211, 8178, 8125, 8147] avg=8192.0000 max=8249 max_rank=1 min=8125 min_rank=6 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 504, 486, 528, 539, 506, 450, 514, 432, 519, 499, 503, 535, 529, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 568, 509, 518, 488, 486, 533, 495, 500, 525, 499, 511, 536, 508, 495, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 502, 525, 529, 500, 522, 509, 497, 499, 489, 509, 498, 541, 523, 509, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 520, 490, 485, 508, 509, 469, 535, 497, 505, 491, 480, 551, 477, 551, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 503, 522, 519, 535, 499, 491, 473, 511, 523, 513, 520, 539, 475, 515, 516, 22, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 504, 524, 519, 514, 485, 526, 491, 532, 496, 509, 563, 514, 495, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 544, 524, 490, 499, 533, 532, 522, 502, 510, 529, 522, 507, 508, 541, 517, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 530, 498, 474, 530, 521, 540, 482, 531, 474, 536, 521, 529, 559, 513, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8238, 8146, 8108, 8217, 8207, 8253, 8253] avg=8192.0000 max=8253 max_rank=6 min=8108 min_rank=3 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4352, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 3968, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[525, 477, 508, 522, 548, 505, 493, 500, 517, 503, 528, 539, 513, 512, 510, 556, 501, 503, 532, 504, 576, 543, 503, 519, 514, 500, 508, 459, 548, 511, 490, 514, 552, 518, 527, 487, 488, 524, 520, 491, 497, 535, 534, 468, 528, 504, 503, 507, 492, 530, 533, 488, 541, 518, 499, 529, 490, 507, 533, 489, 515, 516, 522, 557, 560, 549, 505, 511, 514, 496, 486, 498, 490, 478, 541, 523, 496, 475, 549, 498, 517, 463, 524, 520, 494, 555, 501, 495, 512, 478, 526, 477, 519, 476, 496, 496, 521, 502, 531, 503, 507, 517, 511, 523, 528, 515, 487, 500, 533, 486, 513, 529, 539, 496, 539, 560, 483, 504, 491, 520, 545, 482, 528, 502, 477, 522, 513, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[534, 523, 511, 508, 516, 475, 539, 514, 527, 464, 496, 497, 539, 485, 502, 519, 509, 529, 475, 544, 509, 547, 528, 540, 498, 507, 528, 550, 495, 531, 529, 465, 478, 528, 461, 509, 488, 507, 522, 514, 498, 469, 531, 539, 518, 525, 519, 502, 485, 557, 492, 545, 524, 513, 527, 542, 501, 509, 492, 513, 487, 488, 517, 570, 497, 498, 513, 494, 515, 515, 531, 538, 517, 500, 527, 540, 536, 503, 542, 506, 522, 504, 534, 510, 506, 501, 540, 509, 527, 508, 532, 484, 474, 505, 521, 515, 486, 522, 538, 504, 499, 488, 506, 499, 491, 521, 495, 541, 504, 506, 506, 485, 517, 497, 528, 518, 529, 512, 516, 505, 496, 486, 532, 536, 496, 475, 526, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[518, 483, 528, 508, 531, 523, 502, 535, 532, 533, 539, 545, 484, 517, 510, 520, 505, 522, 524, 528, 503, 508, 511, 538, 485, 538, 515, 540, 503, 489, 521, 522, 547, 512, 516, 497, 533, 498, 526, 506, 493, 471, 508, 516, 448, 506, 497, 494, 503, 555, 532, 508, 499, 502, 517, 512, 527, 484, 511, 548, 508, 512, 510, 512, 523, 531, 514, 505, 517, 521, 477, 541, 500, 521, 511, 530, 518, 524, 493, 496, 515, 511, 473, 478, 537, 534, 517, 517, 505, 498, 535, 524, 510, 503, 510, 526, 533, 457, 472, 499, 556, 516, 486, 531, 489, 545, 493, 535, 513, 515, 498, 489, 523, 512, 505, 464, 520, 488, 512, 511, 478, 536, 498, 525, 525, 491, 521, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[498, 533, 524, 540, 544, 495, 515, 516, 527, 523, 490, 492, 506, 530, 535, 497, 510, 498, 477, 532, 548, 488, 523, 463, 500, 499, 511, 535, 498, 520, 526, 498, 514, 480, 548, 530, 481, 502, 570, 519, 499, 515, 558, 506, 483, 513, 499, 477, 504, 484, 529, 526, 520, 530, 483, 535, 483, 504, 530, 515, 520, 506, 504, 487, 533, 491, 526, 485, 530, 537, 531, 489, 496, 478, 503, 532, 492, 497, 552, 481, 473, 474, 488, 529, 505, 475, 514, 493, 529, 509, 509, 534, 524, 513, 532, 541, 565, 479, 527, 502, 536, 487, 499, 521, 513, 523, 496, 541, 497, 549, 517, 494, 533, 532, 515, 490, 534, 512, 544, 490, 512, 501, 507, 505, 517, 535, 481, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[523, 514, 555, 552, 493, 539, 521, 513, 496, 495, 506, 579, 541, 515, 502, 526, 525, 519, 535, 466, 523, 540, 535, 540, 525, 491, 497, 486, 542, 484, 484, 499, 468, 506, 528, 504, 512, 507, 539, 487, 512, 514, 512, 533, 543, 488, 515, 516, 533, 519, 489, 505, 523, 495, 507, 506, 540, 522, 550, 510, 508, 513, 531, 505, 520, 489, 513, 494, 542, 488, 499, 480, 511, 512, 566, 493, 494, 517, 500, 512, 533, 501, 515, 475, 562, 521, 539, 476, 496, 525, 516, 512, 484, 550, 516, 456, 528, 497, 519, 476, 550, 525, 521, 530, 534, 497, 495, 491, 513, 519, 507, 489, 510, 509, 488, 526, 487, 510, 504, 491, 507, 483, 466, 459, 520, 484, 551, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[536, 526, 529, 506, 473, 498, 502, 492, 506, 516, 498, 527, 557, 534, 505, 528, 508, 478, 498, 500, 490, 512, 523, 522, 513, 539, 506, 512, 509, 485, 506, 512, 484, 571, 528, 523, 519, 521, 480, 506, 527, 495, 525, 535, 518, 515, 503, 542, 513, 525, 482, 518, 527, 474, 527, 507, 496, 533, 480, 489, 503, 528, 491, 507, 513, 519, 484, 528, 517, 516, 510, 481, 505, 558, 507, 515, 494, 545, 515, 515, 554, 496, 532, 526, 511, 513, 488, 506, 515, 519, 533, 517, 524, 499, 498, 502, 552, 520, 544, 508, 480, 488, 512, 530, 512, 552, 502, 509, 503, 524, 493, 461, 486, 530, 524, 500, 512, 506, 496, 517, 471, 526, 557, 490, 485, 515, 534, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[518, 504, 501, 471, 512, 512, 516, 507, 509, 547, 465, 486, 547, 511, 519, 518, 499, 493, 498, 516, 506, 538, 484, 521, 523, 537, 517, 529, 464, 497, 511, 525, 563, 519, 513, 515, 510, 536, 536, 518, 515, 516, 539, 502, 484, 508, 478, 513, 515, 533, 471, 466, 531, 518, 539, 492, 489, 468, 539, 475, 531, 531, 503, 520, 485, 503, 541, 522, 524, 511, 513, 507, 507, 487, 512, 509, 523, 492, 507, 499, 518, 535, 514, 549, 533, 502, 570, 466, 502, 516, 553, 515, 529, 544, 519, 477, 506, 535, 518, 547, 497, 506, 502, 506, 493, 510, 490, 520, 521, 490, 512, 519, 549, 542, 528, 494, 536, 495, 504, 502, 466, 474, 559, 508, 495, 522, 498, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[481, 518, 507, 513, 487, 525, 512, 515, 501, 518, 497, 489, 517, 517, 497, 516, 521, 508, 499, 522, 491, 533, 477, 486, 520, 511, 520, 532, 500, 526, 534, 480, 532, 509, 505, 493, 533, 551, 531, 518, 508, 518, 513, 528, 521, 508, 493, 494, 524, 485, 477, 507, 500, 472, 496, 519, 553, 519, 511, 527, 527, 517, 513, 539, 535, 523, 514, 532, 518, 511, 488, 482, 506, 489, 526, 512, 559, 525, 513, 508, 501, 485, 506, 520, 493, 506, 517, 525, 521, 518, 515, 519, 516, 503, 510, 520, 523, 522, 531, 508, 524, 526, 480, 533, 519, 537, 505, 485, 560, 514, 515, 481, 500, 523, 518, 539, 523, 488, 474, 500, 518, 498, 509, 528, 513, 558, 454, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[525, 477, 508, 522, 548, 505, 493, 500, 517, 503, 528, 539, 477, 512, 510, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 503, 532, 504, 576, 543, 503, 519, 514, 500, 508, 459, 548, 511, 490, 514, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 518, 527, 487, 488, 524, 519, 491, 497, 535, 534, 468, 528, 504, 503, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 530, 533, 488, 541, 518, 499, 529, 490, 507, 533, 489, 515, 516, 522, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 549, 505, 511, 514, 496, 486, 498, 490, 478, 540, 523, 496, 475, 549, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 463, 524, 520, 494, 555, 501, 495, 512, 478, 526, 477, 519, 476, 496, 496, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 502, 531, 503, 507, 517, 511, 523, 528, 515, 487, 500, 533, 486, 513, 529, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 496, 539, 560, 483, 504, 491, 520, 545, 482, 528, 502, 477, 522, 513, 488, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8227, 8182, 8253, 8168, 8053, 8212, 8221] avg=8192.0000 max=8253 max_rank=3 min=8053 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 514, 555, 552, 493, 539, 521, 513, 496, 495, 506, 579, 504, 515, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 519, 535, 466, 523, 540, 535, 540, 525, 491, 497, 486, 542, 484, 484, 499, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 506, 528, 504, 512, 507, 538, 487, 512, 514, 512, 533, 543, 488, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 519, 489, 505, 523, 495, 507, 506, 540, 522, 550, 510, 508, 513, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 513, 494, 542, 488, 499, 480, 511, 512, 565, 493, 494, 517, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 501, 515, 475, 562, 521, 539, 476, 496, 525, 516, 512, 484, 550, 516, 456, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 519, 476, 550, 525, 521, 530, 534, 497, 495, 491, 513, 519, 507, 489, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 488, 526, 487, 510, 504, 491, 507, 483, 466, 459, 520, 484, 551, 542, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8333, 8193, 8183, 8252, 8129, 8180, 8196, 8070] avg=8192.0000 max=8333 max_rank=0 min=8070 min_rank=7 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[534, 523, 511, 508, 516, 475, 539, 514, 527, 464, 496, 497, 502, 485, 502, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 529, 475, 544, 509, 547, 528, 540, 498, 507, 528, 550, 495, 531, 529, 465, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 528, 461, 509, 488, 507, 521, 514, 498, 469, 531, 539, 518, 525, 519, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 557, 492, 545, 524, 513, 527, 542, 501, 509, 492, 513, 487, 488, 517, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 513, 494, 515, 515, 531, 538, 517, 500, 526, 540, 536, 503, 542, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 534, 510, 506, 501, 540, 509, 527, 508, 532, 484, 474, 505, 521, 515, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 522, 538, 504, 499, 488, 506, 499, 491, 521, 495, 541, 504, 506, 506, 485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 497, 528, 518, 529, 512, 516, 505, 496, 486, 532, 536, 496, 475, 526, 509, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8286, 8107, 8256, 8271, 8196, 8097, 8211] avg=8192.0000 max=8286 max_rank=1 min=8097 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 504, 501, 471, 512, 512, 516, 507, 509, 547, 465, 486, 501, 511, 519, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 498, 516, 506, 538, 484, 521, 523, 537, 517, 529, 464, 497, 511, 525, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 519, 513, 515, 510, 536, 530, 518, 515, 516, 539, 502, 484, 508, 478, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 471, 466, 531, 518, 539, 492, 489, 468, 539, 475, 531, 531, 503, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 503, 541, 522, 524, 511, 513, 507, 507, 487, 504, 509, 523, 492, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 514, 549, 533, 502, 570, 466, 502, 516, 553, 515, 529, 544, 519, 477, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 535, 518, 547, 497, 506, 502, 506, 493, 510, 490, 520, 521, 490, 512, 519, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 542, 528, 494, 536, 495, 504, 502, 466, 474, 559, 508, 495, 522, 498, 521, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8171, 8259, 8107, 8134, 8351, 8185, 8232] avg=8192.0000 max=8351 max_rank=5 min=8097 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32902 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 533, 524, 540, 544, 495, 515, 516, 527, 523, 490, 492, 471, 530, 535, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 498, 477, 532, 548, 488, 523, 463, 500, 499, 511, 535, 498, 520, 526, 498, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 480, 548, 530, 481, 502, 569, 519, 499, 515, 558, 506, 483, 513, 499, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 484, 529, 526, 520, 530, 483, 535, 483, 504, 530, 515, 520, 506, 504, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 491, 526, 485, 530, 537, 531, 489, 496, 478, 502, 532, 492, 497, 552, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 474, 488, 529, 505, 475, 514, 493, 529, 509, 509, 534, 524, 513, 532, 541, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 479, 527, 502, 536, 487, 499, 521, 513, 523, 496, 541, 497, 549, 517, 494, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 532, 515, 490, 534, 512, 544, 490, 512, 501, 507, 505, 517, 535, 481, 542, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8128, 8193, 8156, 8152, 8145, 8251, 8281] avg=8192.0000 max=8281 max_rank=7 min=8128 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 483, 528, 508, 531, 523, 502, 535, 532, 533, 539, 545, 451, 517, 510, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 522, 524, 528, 503, 508, 511, 538, 485, 538, 515, 540, 503, 489, 521, 522, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 512, 516, 497, 533, 498, 525, 506, 493, 471, 508, 516, 448, 506, 497, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 555, 532, 508, 499, 502, 517, 512, 527, 484, 511, 548, 508, 512, 510, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 531, 514, 505, 517, 521, 477, 541, 500, 521, 510, 530, 518, 524, 493, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 473, 478, 537, 534, 517, 517, 505, 498, 535, 524, 510, 503, 510, 526, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 457, 472, 499, 556, 516, 486, 531, 489, 545, 493, 535, 513, 515, 498, 489, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 512, 505, 464, 520, 488, 512, 511, 478, 536, 498, 525, 525, 491, 521, 517, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8254, 8067, 8236, 8221, 8196, 8131, 8156] avg=8192.0000 max=8275 max_rank=0 min=8067 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[481, 518, 507, 513, 487, 525, 512, 515, 501, 518, 497, 489, 481, 517, 497, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 508, 499, 522, 491, 533, 477, 486, 520, 511, 520, 532, 500, 526, 534, 480, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 509, 505, 493, 533, 551, 530, 518, 508, 518, 513, 528, 521, 508, 493, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 485, 477, 507, 500, 472, 496, 519, 553, 519, 511, 527, 527, 517, 513, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 523, 514, 532, 518, 511, 488, 482, 506, 489, 525, 512, 559, 525, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 485, 506, 520, 493, 506, 517, 525, 521, 518, 515, 519, 516, 503, 510, 520, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 531, 508, 524, 526, 480, 533, 519, 537, 505, 485, 560, 514, 515, 481, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 518, 539, 523, 488, 474, 500, 518, 498, 509, 528, 513, 558, 454, 503, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8074, 8162, 8254, 8180, 8240, 8179, 8269, 8178] avg=8192.0000 max=8269 max_rank=6 min=8074 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[536, 526, 529, 506, 473, 498, 502, 492, 506, 516, 498, 527, 519, 534, 505, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 478, 498, 500, 490, 512, 523, 522, 513, 539, 506, 512, 509, 485, 506, 512, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 571, 528, 523, 519, 521, 479, 506, 527, 495, 525, 535, 518, 515, 503, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 482, 518, 527, 474, 527, 507, 496, 533, 480, 489, 503, 528, 491, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 484, 528, 517, 516, 510, 481, 505, 558, 506, 515, 494, 545, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 496, 532, 526, 511, 513, 488, 506, 515, 519, 533, 517, 524, 499, 498, 502, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 520, 544, 508, 480, 488, 512, 530, 512, 552, 502, 509, 503, 524, 493, 461, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 524, 500, 512, 506, 496, 517, 471, 526, 557, 490, 485, 515, 534, 504, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8115, 8291, 8096, 8221, 8236, 8195, 8187] avg=8192.0000 max=8291 max_rank=2 min=8096 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 3968, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[567, 484, 494, 509, 496, 540, 505, 494, 520, 489, 524, 515, 504, 500, 507, 536, 513, 507, 497, 508, 496, 497, 552, 502, 509, 515, 491, 528, 495, 521, 520, 488, 514, 500, 522, 516, 528, 504, 487, 518, 495, 512, 484, 527, 483, 510, 539, 503, 506, 516, 505, 538, 483, 512, 497, 512, 502, 508, 495, 514, 477, 505, 506, 517, 489, 512, 515, 516, 487, 536, 529, 526, 528, 472, 511, 498, 514, 535, 495, 494, 537, 548, 476, 532, 546, 510, 538, 519, 578, 475, 530, 495, 482, 553, 512, 473, 529, 531, 489, 498, 516, 483, 492, 524, 503, 530, 505, 488, 472, 532, 500, 522, 487, 525, 520, 538, 527, 511, 532, 548, 516, 519, 495, 518, 536, 509, 588, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[549, 488, 531, 499, 517, 532, 517, 498, 533, 507, 508, 544, 501, 508, 500, 514, 513, 513, 533, 519, 504, 503, 491, 497, 507, 513, 544, 514, 538, 514, 482, 512, 537, 488, 508, 484, 496, 519, 502, 504, 492, 513, 508, 521, 525, 497, 494, 518, 495, 520, 509, 500, 564, 522, 530, 512, 491, 489, 506, 460, 520, 527, 514, 485, 506, 513, 448, 524, 557, 507, 529, 515, 556, 492, 491, 545, 499, 562, 476, 472, 534, 532, 571, 499, 509, 480, 506, 526, 497, 509, 513, 506, 483, 519, 507, 509, 506, 507, 513, 518, 519, 535, 500, 517, 512, 496, 513, 543, 556, 522, 528, 523, 499, 493, 497, 535, 519, 497, 483, 490, 542, 550, 514, 540, 506, 483, 497, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[512, 498, 490, 518, 473, 513, 563, 503, 531, 466, 539, 523, 538, 554, 467, 503, 499, 514, 476, 511, 529, 515, 515, 482, 496, 519, 554, 486, 536, 509, 518, 518, 524, 545, 491, 559, 540, 553, 512, 502, 506, 500, 551, 486, 491, 500, 527, 477, 471, 510, 510, 567, 531, 529, 496, 539, 507, 537, 496, 540, 531, 501, 491, 499, 498, 526, 492, 512, 496, 537, 496, 544, 514, 500, 538, 508, 523, 509, 504, 520, 545, 509, 487, 510, 522, 527, 539, 497, 500, 514, 461, 510, 475, 472, 532, 540, 513, 520, 532, 491, 494, 532, 488, 498, 563, 471, 525, 491, 454, 513, 506, 525, 529, 504, 505, 534, 464, 531, 509, 503, 533, 511, 504, 489, 535, 497, 510, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[491, 504, 519, 481, 537, 507, 518, 498, 492, 522, 461, 491, 525, 546, 454, 530, 498, 504, 501, 482, 550, 547, 474, 466, 518, 509, 504, 479, 472, 485, 531, 500, 487, 464, 484, 565, 477, 508, 556, 501, 569, 513, 520, 503, 520, 518, 535, 544, 501, 512, 535, 541, 491, 515, 527, 550, 512, 533, 536, 480, 514, 508, 512, 526, 505, 491, 521, 529, 517, 499, 491, 543, 540, 521, 537, 503, 494, 510, 522, 503, 556, 501, 513, 570, 527, 529, 497, 507, 494, 487, 491, 533, 518, 534, 511, 488, 495, 557, 521, 519, 503, 493, 553, 550, 510, 479, 475, 509, 457, 490, 523, 504, 468, 513, 520, 503, 499, 501, 513, 518, 517, 516, 485, 523, 544, 572, 544, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[504, 545, 533, 495, 546, 519, 538, 537, 504, 539, 526, 481, 483, 492, 497, 450, 522, 468, 523, 489, 494, 514, 533, 494, 504, 482, 531, 554, 540, 511, 505, 520, 537, 537, 521, 521, 531, 517, 543, 504, 486, 512, 531, 542, 458, 498, 504, 474, 521, 531, 489, 487, 486, 506, 502, 549, 512, 516, 547, 493, 507, 520, 510, 513, 511, 553, 494, 522, 514, 556, 515, 530, 493, 500, 506, 520, 515, 501, 512, 514, 513, 510, 488, 542, 484, 524, 522, 487, 493, 522, 470, 514, 540, 450, 483, 499, 547, 528, 490, 508, 486, 518, 514, 506, 519, 539, 466, 516, 507, 544, 529, 494, 559, 519, 493, 516, 496, 512, 529, 535, 530, 512, 502, 507, 502, 503, 532, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[487, 514, 501, 499, 516, 510, 519, 498, 519, 496, 516, 564, 519, 509, 528, 518, 523, 506, 483, 502, 507, 524, 501, 537, 497, 515, 544, 530, 541, 495, 520, 537, 518, 525, 521, 500, 506, 530, 474, 513, 498, 520, 548, 539, 506, 494, 499, 510, 552, 506, 533, 508, 481, 492, 507, 499, 523, 525, 495, 481, 506, 519, 513, 479, 533, 517, 503, 534, 462, 529, 517, 494, 518, 535, 499, 529, 499, 483, 520, 531, 516, 516, 532, 504, 554, 575, 462, 526, 511, 510, 484, 506, 481, 501, 518, 516, 538, 502, 499, 504, 547, 531, 486, 499, 531, 553, 536, 520, 495, 515, 512, 488, 498, 543, 538, 499, 507, 518, 504, 498, 500, 486, 505, 519, 475, 483, 498, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[521, 474, 519, 483, 472, 519, 576, 499, 548, 515, 524, 522, 487, 519, 506, 497, 491, 504, 491, 522, 531, 531, 513, 502, 533, 526, 499, 509, 502, 528, 512, 507, 509, 516, 504, 506, 497, 536, 499, 506, 542, 484, 512, 513, 542, 488, 535, 522, 529, 511, 495, 533, 485, 522, 469, 472, 536, 526, 501, 525, 507, 519, 531, 548, 522, 536, 538, 527, 514, 525, 514, 504, 499, 521, 468, 515, 528, 519, 521, 567, 502, 515, 521, 520, 527, 529, 509, 501, 502, 513, 515, 517, 469, 537, 522, 462, 512, 537, 511, 539, 498, 497, 509, 543, 503, 485, 513, 514, 480, 510, 496, 520, 509, 545, 517, 503, 495, 500, 494, 500, 473, 541, 503, 483, 500, 468, 539, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[529, 498, 523, 505, 510, 493, 464, 523, 527, 486, 514, 543, 535, 547, 515, 483, 498, 556, 478, 510, 489, 518, 478, 530, 545, 515, 497, 482, 517, 533, 541, 502, 531, 529, 522, 487, 540, 486, 476, 498, 523, 526, 511, 475, 537, 469, 502, 537, 538, 557, 495, 478, 510, 492, 550, 509, 473, 495, 541, 493, 500, 522, 515, 532, 570, 549, 516, 517, 521, 496, 476, 498, 516, 514, 567, 501, 539, 480, 484, 494, 545, 529, 473, 531, 533, 487, 487, 486, 540, 544, 506, 532, 489, 502, 483, 511, 477, 562, 470, 553, 478, 497, 532, 480, 529, 502, 532, 533, 493, 518, 539, 525, 519, 501, 501, 501, 548, 495, 530, 517, 511, 487, 510, 522, 484, 527, 461, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[567, 484, 494, 509, 496, 540, 505, 494, 520, 489, 524, 515, 504, 500, 507, 536, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 507, 497, 508, 496, 497, 552, 502, 509, 515, 491, 528, 495, 521, 520, 488, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 522, 516, 528, 504, 487, 518, 495, 512, 483, 527, 483, 510, 539, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 505, 538, 483, 512, 497, 512, 502, 508, 495, 514, 477, 505, 506, 517, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 480, 515, 516, 487, 536, 529, 526, 528, 472, 511, 498, 514, 535, 495, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 548, 476, 532, 546, 510, 538, 519, 578, 475, 530, 495, 482, 553, 512, 473, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 531, 489, 498, 516, 483, 492, 524, 503, 530, 505, 488, 472, 532, 500, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 525, 520, 538, 527, 511, 532, 548, 516, 519, 495, 518, 536, 509, 585, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8159, 8141, 8097, 8125, 8309, 8114, 8400] avg=8192.0000 max=8400 max_rank=7 min=8097 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=57136 max_load_reduction_pct=0.871826 moved_to_echo=32948 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[512, 498, 490, 518, 473, 513, 563, 503, 531, 466, 539, 523, 538, 554, 467, 503, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 514, 476, 511, 529, 515, 515, 482, 496, 519, 554, 486, 536, 509, 518, 518, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 545, 491, 559, 540, 553, 512, 502, 506, 500, 550, 486, 491, 500, 527, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 510, 510, 567, 531, 529, 496, 539, 507, 537, 496, 540, 531, 501, 491, 499, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 492, 512, 496, 537, 496, 544, 514, 500, 538, 508, 523, 509, 504, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 487, 510, 522, 527, 539, 497, 500, 514, 461, 510, 475, 472, 532, 540, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 520, 532, 491, 494, 532, 488, 498, 563, 471, 525, 491, 454, 513, 506, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 505, 534, 464, 531, 509, 503, 533, 511, 504, 489, 535, 497, 508, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8197, 8263, 8258, 8185, 8145, 8116, 8174] avg=8192.0000 max=8263 max_rank=2 min=8116 min_rank=6 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[504, 545, 533, 495, 546, 519, 538, 537, 504, 539, 526, 481, 483, 492, 497, 450, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 468, 523, 489, 494, 514, 533, 494, 504, 482, 531, 554, 540, 511, 505, 520, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 537, 521, 521, 531, 517, 543, 504, 486, 512, 530, 542, 458, 498, 504, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 531, 489, 487, 486, 506, 502, 549, 512, 516, 547, 493, 507, 520, 510, 513, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 494, 522, 514, 556, 515, 530, 493, 500, 506, 520, 515, 501, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 488, 542, 484, 524, 522, 487, 493, 522, 470, 514, 540, 450, 483, 499, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 528, 490, 508, 486, 518, 514, 506, 519, 539, 466, 516, 507, 544, 529, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 519, 493, 516, 496, 512, 529, 535, 530, 512, 502, 507, 502, 503, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8206, 8215, 8192, 8221, 8046, 8211, 8248] avg=8192.0000 max=8248 max_rank=7 min=8046 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[491, 504, 519, 481, 537, 507, 518, 498, 492, 522, 461, 491, 525, 546, 454, 530, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 501, 482, 550, 547, 474, 466, 518, 509, 504, 479, 472, 485, 531, 500, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 464, 484, 565, 477, 508, 556, 501, 569, 513, 510, 503, 520, 518, 535, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 535, 541, 491, 515, 527, 550, 512, 533, 536, 480, 514, 508, 512, 526, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 449, 521, 529, 517, 499, 491, 543, 540, 521, 537, 503, 494, 510, 522, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 501, 513, 570, 527, 529, 497, 507, 494, 487, 491, 533, 518, 534, 511, 488, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 557, 521, 519, 503, 493, 553, 550, 510, 479, 475, 509, 457, 490, 523, 504, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 513, 520, 503, 499, 501, 513, 518, 517, 516, 485, 523, 544, 572, 535, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8043, 8254, 8305, 8184, 8265, 8144, 8254] avg=8192.0000 max=8305 max_rank=3 min=8043 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[549, 488, 531, 499, 517, 532, 517, 498, 533, 507, 508, 544, 501, 508, 500, 514, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 533, 519, 504, 503, 491, 497, 507, 513, 544, 514, 538, 514, 482, 512, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 488, 508, 484, 496, 519, 502, 504, 492, 513, 507, 521, 525, 497, 494, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 520, 509, 500, 564, 522, 530, 512, 491, 489, 506, 460, 520, 527, 514, 485, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 481, 448, 524, 557, 507, 529, 515, 556, 492, 491, 545, 499, 562, 476, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 571, 499, 509, 480, 506, 526, 497, 509, 513, 506, 483, 519, 507, 509, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 507, 513, 518, 519, 535, 500, 517, 512, 496, 513, 543, 556, 522, 528, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 497, 535, 519, 497, 483, 490, 542, 550, 514, 540, 506, 483, 495, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8217, 8105, 8147, 8160, 8205, 8308, 8141] avg=8192.0000 max=8308 max_rank=6 min=8105 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[529, 498, 523, 505, 510, 493, 464, 523, 527, 486, 514, 543, 535, 547, 515, 483, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 556, 478, 510, 489, 518, 478, 530, 545, 515, 497, 482, 517, 533, 541, 502, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 529, 522, 487, 540, 486, 476, 498, 523, 526, 510, 475, 537, 469, 502, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 557, 495, 478, 510, 492, 550, 509, 473, 495, 541, 493, 500, 522, 515, 532, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 516, 516, 517, 521, 496, 476, 498, 516, 514, 567, 501, 539, 480, 484, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 529, 473, 531, 533, 487, 487, 486, 540, 544, 506, 532, 489, 502, 483, 511, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 562, 470, 553, 478, 497, 532, 480, 529, 502, 532, 533, 493, 518, 539, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 501, 501, 501, 548, 495, 530, 517, 511, 487, 510, 522, 484, 527, 459, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8210, 8148, 8203, 8205, 8183, 8220, 8165] avg=8192.0000 max=8220 max_rank=6 min=8148 min_rank=2 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[521, 474, 519, 483, 472, 519, 576, 499, 548, 515, 524, 522, 487, 519, 506, 497, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 504, 491, 522, 531, 531, 513, 502, 533, 526, 499, 509, 502, 528, 512, 507, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 504, 506, 497, 536, 499, 506, 542, 484, 511, 513, 542, 488, 535, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 511, 495, 533, 485, 522, 469, 472, 536, 526, 501, 525, 507, 519, 531, 548, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 538, 527, 514, 525, 514, 504, 499, 521, 468, 515, 528, 519, 521, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 521, 520, 527, 529, 509, 501, 502, 513, 515, 517, 469, 537, 522, 462, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 537, 511, 539, 498, 497, 509, 543, 503, 485, 513, 514, 480, 510, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 545, 517, 503, 495, 500, 494, 500, 473, 541, 503, 483, 500, 468, 537, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8222, 8210, 8212, 8285, 8166, 8167, 8086] avg=8192.0000 max=8285 max_rank=4 min=8086 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[487, 514, 501, 499, 516, 510, 519, 498, 519, 496, 516, 564, 519, 509, 528, 518, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 506, 483, 502, 507, 524, 501, 537, 497, 515, 544, 530, 541, 495, 520, 537, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 521, 500, 506, 530, 474, 513, 498, 520, 547, 539, 506, 494, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 506, 533, 508, 481, 492, 507, 499, 523, 525, 495, 481, 506, 519, 513, 479, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 485, 503, 534, 462, 529, 517, 494, 518, 535, 499, 529, 499, 483, 520, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 532, 504, 554, 575, 462, 526, 511, 510, 484, 506, 481, 501, 518, 516, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 502, 499, 504, 547, 531, 486, 499, 531, 553, 536, 520, 495, 515, 512, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 543, 538, 499, 507, 518, 504, 498, 500, 486, 505, 519, 475, 483, 496, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8282, 8200, 8122, 8171, 8217, 8256, 8068] avg=8192.0000 max=8282 max_rank=1 min=8068 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4352, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 3968, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4352, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[520, 476, 547, 513, 510, 487, 487, 494, 495, 488, 514, 563, 490, 537, 483, 512, 515, 489, 495, 510, 527, 489, 534, 513, 517, 533, 524, 488, 491, 494, 514, 518, 545, 510, 516, 512, 495, 534, 470, 498, 600, 504, 479, 526, 530, 516, 526, 516, 534, 474, 518, 469, 486, 484, 545, 512, 530, 483, 517, 511, 491, 543, 495, 469, 570, 530, 501, 489, 485, 509, 504, 520, 501, 527, 495, 531, 511, 487, 536, 490, 518, 534, 467, 566, 520, 531, 491, 551, 496, 533, 534, 488, 506, 528, 554, 526, 523, 554, 492, 511, 490, 562, 502, 492, 517, 514, 474, 554, 537, 505, 483, 470, 509, 516, 550, 507, 494, 510, 514, 494, 519, 511, 514, 516, 479, 526, 534, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[496, 511, 509, 503, 489, 516, 529, 537, 518, 509, 540, 510, 524, 542, 507, 501, 519, 488, 517, 504, 497, 509, 539, 557, 511, 521, 496, 527, 530, 512, 516, 515, 516, 512, 516, 536, 513, 509, 489, 506, 504, 533, 537, 525, 540, 510, 529, 543, 479, 518, 519, 497, 479, 535, 498, 521, 541, 534, 511, 512, 537, 528, 519, 505, 483, 519, 512, 529, 459, 523, 507, 529, 541, 519, 522, 528, 513, 526, 467, 508, 516, 519, 504, 517, 488, 513, 540, 490, 504, 524, 492, 512, 504, 515, 496, 510, 518, 506, 488, 495, 520, 515, 534, 541, 495, 477, 480, 479, 542, 490, 518, 486, 515, 523, 470, 503, 487, 566, 491, 501, 487, 517, 491, 469, 545, 510, 529, 469] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[520, 501, 530, 493, 551, 537, 563, 519, 485, 501, 553, 505, 471, 503, 513, 525, 516, 497, 481, 534, 492, 530, 502, 517, 521, 518, 547, 481, 514, 478, 472, 487, 508, 495, 542, 532, 482, 500, 509, 482, 492, 511, 505, 535, 510, 478, 503, 503, 524, 473, 496, 479, 513, 505, 508, 537, 550, 491, 512, 550, 495, 488, 525, 522, 526, 532, 523, 526, 540, 481, 524, 483, 498, 516, 529, 521, 478, 529, 483, 469, 515, 478, 512, 514, 522, 507, 519, 513, 522, 543, 537, 494, 518, 539, 515, 528, 521, 510, 483, 513, 531, 522, 523, 520, 537, 524, 528, 497, 532, 513, 542, 479, 500, 497, 518, 514, 526, 482, 508, 513, 553, 549, 527, 516, 497, 497, 501, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[546, 492, 496, 527, 517, 473, 508, 523, 502, 512, 532, 496, 523, 497, 473, 516, 527, 534, 503, 538, 490, 501, 489, 509, 536, 516, 516, 499, 465, 530, 519, 496, 527, 500, 506, 549, 519, 529, 498, 538, 507, 496, 523, 522, 509, 494, 532, 514, 511, 486, 478, 564, 517, 537, 519, 525, 480, 501, 497, 500, 506, 526, 517, 516, 529, 522, 533, 468, 525, 488, 489, 511, 475, 513, 498, 502, 520, 486, 515, 497, 493, 538, 552, 513, 552, 489, 519, 472, 487, 489, 502, 515, 537, 508, 509, 492, 522, 564, 522, 506, 522, 489, 519, 511, 513, 502, 483, 545, 503, 508, 517, 509, 521, 526, 507, 504, 527, 553, 547, 506, 538, 502, 505, 500, 515, 516, 506, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[495, 533, 468, 534, 528, 486, 526, 517, 487, 514, 514, 522, 521, 512, 495, 479, 533, 526, 493, 502, 502, 505, 538, 490, 524, 504, 513, 442, 522, 465, 527, 509, 515, 511, 515, 464, 494, 529, 501, 526, 513, 514, 529, 497, 491, 504, 508, 469, 537, 486, 490, 527, 505, 515, 511, 519, 501, 483, 512, 498, 493, 493, 528, 538, 526, 520, 498, 536, 514, 523, 521, 533, 484, 522, 505, 514, 466, 501, 512, 514, 483, 552, 507, 525, 505, 530, 494, 495, 508, 496, 543, 480, 487, 489, 531, 523, 519, 504, 555, 525, 510, 488, 512, 508, 536, 503, 521, 543, 496, 529, 497, 561, 507, 530, 561, 521, 508, 501, 539, 569, 499, 528, 553, 503, 514, 565, 515, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[522, 502, 481, 516, 510, 464, 503, 517, 532, 512, 500, 523, 501, 470, 487, 520, 475, 516, 527, 483, 520, 521, 528, 552, 510, 495, 499, 500, 492, 504, 525, 518, 532, 554, 506, 522, 467, 504, 525, 490, 525, 531, 522, 508, 577, 476, 544, 499, 501, 518, 477, 504, 527, 457, 504, 560, 542, 496, 512, 505, 516, 497, 478, 477, 568, 501, 499, 549, 540, 522, 543, 516, 529, 478, 515, 456, 524, 515, 494, 498, 476, 534, 521, 538, 513, 519, 523, 499, 541, 535, 478, 494, 535, 482, 465, 500, 515, 524, 517, 538, 528, 526, 517, 551, 527, 507, 526, 511, 496, 531, 541, 515, 526, 507, 501, 493, 562, 498, 536, 506, 529, 541, 486, 533, 502, 510, 491, 467] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[516, 496, 512, 504, 492, 499, 526, 502, 541, 502, 554, 517, 527, 481, 511, 534, 469, 519, 500, 483, 498, 524, 506, 528, 551, 564, 487, 502, 492, 482, 517, 510, 491, 492, 527, 517, 519, 525, 482, 471, 518, 488, 536, 479, 509, 500, 543, 501, 555, 541, 509, 498, 497, 471, 540, 494, 471, 527, 508, 547, 522, 542, 487, 487, 507, 475, 523, 528, 508, 503, 488, 507, 485, 525, 502, 501, 516, 561, 519, 561, 478, 489, 529, 507, 485, 529, 513, 503, 545, 560, 511, 524, 546, 482, 513, 542, 529, 499, 518, 480, 470, 535, 506, 510, 526, 519, 513, 494, 514, 505, 503, 497, 519, 496, 482, 540, 549, 532, 506, 522, 533, 505, 522, 482, 509, 541, 528, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[483, 485, 507, 493, 508, 532, 503, 514, 477, 508, 496, 562, 488, 516, 558, 556, 490, 516, 513, 507, 485, 504, 556, 546, 519, 478, 505, 526, 511, 504, 495, 488, 536, 507, 504, 482, 529, 536, 549, 489, 510, 531, 500, 538, 518, 497, 521, 485, 523, 541, 539, 505, 517, 527, 513, 460, 504, 532, 555, 484, 494, 509, 508, 516, 477, 534, 508, 504, 537, 515, 527, 529, 515, 482, 525, 549, 494, 484, 504, 511, 529, 563, 485, 522, 513, 542, 510, 535, 501, 459, 488, 502, 526, 512, 477, 475, 540, 522, 534, 520, 543, 512, 502, 508, 475, 488, 506, 517, 505, 519, 510, 546, 514, 532, 514, 508, 506, 506, 464, 490, 516, 514, 532, 499, 521, 525, 500, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[496, 511, 509, 503, 489, 516, 529, 537, 518, 509, 540, 510, 524, 542, 507, 501, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 488, 517, 504, 497, 509, 539, 557, 511, 521, 496, 527, 530, 512, 516, 515, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 512, 516, 536, 513, 509, 489, 506, 504, 533, 537, 525, 540, 510, 518, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 518, 519, 497, 479, 535, 498, 521, 541, 534, 511, 512, 537, 528, 519, 505, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 519, 512, 529, 459, 523, 507, 529, 541, 519, 522, 528, 513, 526, 467, 508, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 504, 517, 488, 513, 540, 490, 504, 524, 492, 512, 504, 515, 496, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 506, 488, 495, 520, 515, 534, 541, 495, 477, 480, 479, 542, 490, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 523, 470, 503, 487, 566, 491, 501, 487, 517, 491, 469, 545, 460, 529, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8299, 8307, 8265, 8196, 8133, 8050, 8023] avg=8192.0000 max=8307 max_rank=2 min=8023 min_rank=7 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[546, 492, 496, 527, 517, 473, 508, 523, 502, 512, 532, 496, 523, 497, 473, 516, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 534, 503, 538, 490, 501, 489, 509, 536, 516, 516, 499, 465, 530, 519, 496, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 500, 506, 549, 519, 529, 498, 538, 507, 496, 523, 522, 509, 494, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 486, 478, 564, 517, 537, 519, 525, 480, 501, 497, 500, 506, 526, 517, 516, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 522, 533, 468, 525, 488, 489, 511, 475, 513, 498, 502, 520, 486, 515, 497, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 527, 552, 513, 552, 489, 519, 472, 487, 489, 502, 515, 537, 508, 509, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 564, 522, 506, 522, 489, 519, 511, 513, 502, 483, 545, 503, 508, 517, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 526, 507, 504, 527, 553, 547, 506, 538, 502, 505, 500, 515, 466, 506, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8209, 8251, 8212, 8082, 8156, 8201, 8269] avg=8192.0000 max=8269 max_rank=7 min=8082 min_rank=4 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 501, 530, 493, 551, 537, 563, 519, 485, 501, 553, 505, 471, 503, 513, 525, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 497, 481, 534, 492, 530, 502, 517, 521, 518, 547, 481, 514, 478, 472, 487, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 542, 532, 482, 500, 509, 482, 492, 511, 505, 535, 510, 478, 492, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 473, 496, 479, 513, 505, 508, 537, 550, 491, 512, 550, 495, 488, 525, 522, 23, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 532, 523, 526, 540, 481, 524, 483, 498, 516, 529, 521, 478, 529, 483, 469, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 468, 512, 514, 522, 507, 519, 513, 522, 543, 537, 494, 518, 539, 515, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 510, 483, 513, 531, 522, 523, 520, 537, 524, 528, 497, 532, 513, 542, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 518, 514, 526, 482, 508, 513, 553, 549, 527, 516, 497, 450, 501, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 8126, 8076, 8199, 8168, 8266, 8241, 8168] avg=8192.0000 max=8292 max_rank=0 min=8076 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 476, 547, 513, 510, 487, 487, 494, 495, 488, 514, 563, 490, 537, 483, 512, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 489, 495, 510, 527, 489, 534, 513, 517, 533, 524, 488, 491, 494, 514, 518, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 510, 516, 512, 495, 534, 470, 498, 600, 504, 479, 526, 530, 516, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 474, 518, 469, 486, 484, 545, 512, 530, 483, 517, 511, 491, 543, 495, 469, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 530, 501, 489, 485, 509, 504, 520, 501, 527, 495, 531, 511, 487, 536, 490, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 523, 467, 566, 520, 531, 491, 551, 496, 533, 534, 488, 506, 528, 554, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 554, 492, 511, 490, 562, 502, 492, 517, 514, 474, 554, 537, 505, 483, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 550, 507, 494, 510, 514, 494, 519, 511, 514, 516, 479, 475, 534, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8193, 8266, 8093, 8197, 8332, 8146, 8171] avg=8192.0000 max=8332 max_rank=5 min=8093 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 533, 468, 534, 528, 486, 526, 517, 487, 514, 514, 522, 521, 512, 495, 479, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 526, 493, 502, 502, 505, 538, 490, 524, 504, 513, 442, 522, 465, 527, 509, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 515, 464, 494, 529, 501, 526, 513, 514, 529, 497, 491, 504, 487, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 486, 490, 527, 505, 515, 511, 519, 501, 483, 512, 498, 493, 493, 528, 538, 27, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 520, 498, 536, 514, 523, 521, 533, 484, 522, 505, 514, 466, 501, 512, 514, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 538, 507, 525, 505, 530, 494, 495, 508, 496, 543, 480, 487, 489, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 504, 555, 525, 510, 488, 512, 508, 536, 503, 521, 543, 496, 529, 497, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 561, 521, 508, 501, 539, 569, 499, 528, 553, 503, 514, 505, 515, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8143, 8059, 8175, 8209, 8134, 8267, 8390] avg=8192.0000 max=8390 max_rank=7 min=8059 min_rank=2 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=33000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 502, 481, 516, 510, 464, 503, 517, 532, 512, 500, 523, 501, 470, 487, 520, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 516, 527, 483, 520, 521, 528, 552, 510, 495, 499, 500, 492, 504, 525, 518, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 554, 506, 522, 467, 504, 525, 490, 525, 531, 522, 508, 577, 476, 532, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 477, 504, 527, 457, 504, 560, 542, 496, 512, 505, 516, 497, 478, 477, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 501, 499, 549, 540, 522, 543, 516, 529, 478, 515, 456, 524, 515, 494, 498, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 523, 521, 538, 513, 519, 523, 499, 541, 535, 478, 494, 535, 482, 465, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 524, 517, 538, 528, 526, 517, 551, 527, 507, 526, 511, 496, 531, 541, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 501, 493, 562, 498, 536, 506, 529, 541, 486, 533, 502, 460, 491, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8206, 8270, 8103, 8258, 8142, 8336, 8138] avg=8192.0000 max=8336 max_rank=6 min=8083 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 496, 512, 504, 492, 499, 526, 502, 541, 502, 554, 517, 527, 481, 511, 534, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 519, 500, 483, 498, 524, 506, 528, 551, 564, 487, 502, 492, 482, 517, 510, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 492, 527, 517, 519, 525, 482, 471, 518, 488, 536, 479, 509, 500, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 541, 509, 498, 497, 471, 540, 494, 471, 527, 508, 547, 522, 542, 487, 487, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 475, 523, 528, 508, 503, 488, 507, 485, 525, 502, 501, 516, 561, 519, 561, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 479, 529, 507, 485, 529, 513, 503, 545, 560, 511, 524, 546, 482, 513, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 499, 518, 480, 470, 535, 506, 510, 526, 519, 513, 494, 514, 505, 503, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 482, 540, 549, 532, 506, 522, 533, 505, 522, 482, 509, 489, 528, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8175, 8086, 8229, 8219, 8246, 8083, 8261] avg=8192.0000 max=8261 max_rank=7 min=8083 min_rank=6 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[483, 485, 507, 493, 508, 532, 503, 514, 477, 508, 496, 562, 488, 516, 558, 556, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 516, 513, 507, 485, 504, 556, 546, 519, 478, 505, 526, 511, 504, 495, 488, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 507, 504, 482, 529, 536, 549, 489, 510, 531, 500, 538, 518, 497, 510, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 541, 539, 505, 517, 527, 513, 460, 504, 532, 555, 484, 494, 509, 508, 516, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 534, 508, 504, 537, 515, 527, 529, 515, 482, 525, 549, 494, 484, 504, 511, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 551, 485, 522, 513, 542, 510, 535, 501, 459, 488, 502, 526, 512, 477, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 522, 534, 520, 543, 512, 502, 508, 475, 488, 506, 517, 505, 519, 510, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 532, 514, 508, 506, 506, 464, 490, 516, 514, 532, 499, 521, 474, 500, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8185, 8221, 8260, 8207, 8127, 8212, 8116] avg=8192.0000 max=8260 max_rank=3 min=8116 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 3840, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[503, 525, 498, 495, 471, 519, 478, 524, 535, 505, 480, 530, 521, 512, 557, 528, 493, 481, 495, 509, 525, 530, 503, 536, 517, 498, 523, 509, 520, 501, 480, 549, 517, 533, 499, 530, 511, 508, 501, 518, 497, 538, 524, 525, 532, 482, 517, 510, 516, 533, 489, 473, 549, 481, 517, 517, 500, 503, 522, 510, 521, 511, 492, 508, 499, 532, 519, 526, 479, 507, 506, 497, 490, 493, 514, 538, 523, 491, 504, 550, 517, 517, 494, 495, 501, 509, 501, 521, 494, 481, 481, 536, 559, 517, 533, 494, 544, 521, 514, 481, 549, 511, 499, 495, 500, 542, 508, 498, 519, 522, 501, 494, 482, 521, 511, 486, 531, 576, 500, 510, 534, 508, 530, 533, 518, 509, 508, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[527, 532, 538, 502, 503, 504, 479, 523, 495, 536, 546, 542, 489, 525, 516, 492, 533, 538, 508, 509, 546, 529, 536, 524, 511, 498, 498, 511, 482, 513, 487, 515, 530, 524, 492, 481, 487, 509, 513, 543, 485, 530, 506, 521, 525, 507, 508, 537, 515, 521, 518, 540, 538, 555, 517, 547, 509, 472, 486, 471, 503, 539, 516, 522, 518, 521, 477, 469, 518, 558, 514, 529, 526, 502, 552, 505, 476, 495, 513, 534, 546, 496, 482, 530, 488, 510, 523, 518, 482, 514, 515, 565, 511, 492, 490, 470, 548, 496, 525, 514, 482, 558, 510, 481, 491, 495, 500, 488, 515, 488, 497, 494, 475, 469, 523, 510, 510, 540, 522, 493, 498, 519, 503, 523, 495, 544, 528, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[517, 512, 547, 510, 481, 511, 532, 525, 498, 543, 500, 532, 504, 525, 517, 544, 570, 504, 543, 522, 465, 492, 523, 535, 541, 519, 485, 505, 505, 556, 494, 515, 523, 549, 528, 538, 533, 518, 506, 488, 512, 527, 482, 519, 474, 508, 497, 509, 482, 545, 525, 530, 531, 488, 508, 521, 486, 496, 509, 541, 479, 506, 490, 485, 537, 542, 471, 503, 496, 541, 484, 484, 508, 554, 516, 495, 522, 509, 533, 499, 536, 519, 524, 512, 521, 467, 535, 521, 533, 521, 506, 526, 498, 516, 510, 512, 500, 481, 510, 537, 473, 496, 483, 526, 507, 475, 538, 535, 479, 502, 526, 476, 499, 534, 519, 511, 546, 480, 516, 479, 507, 495, 513, 534, 530, 479, 532, 462] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[516, 530, 501, 527, 481, 484, 511, 519, 480, 526, 511, 549, 505, 498, 499, 531, 543, 499, 522, 572, 517, 472, 513, 533, 521, 504, 507, 467, 512, 486, 485, 530, 549, 503, 479, 506, 520, 467, 471, 521, 570, 553, 476, 502, 508, 517, 493, 486, 504, 528, 539, 531, 501, 499, 513, 475, 504, 475, 523, 523, 529, 512, 522, 477, 510, 504, 526, 462, 518, 544, 483, 512, 487, 502, 505, 524, 509, 480, 530, 548, 502, 533, 506, 536, 535, 541, 511, 519, 549, 508, 532, 529, 483, 490, 558, 574, 520, 496, 528, 482, 538, 533, 503, 507, 513, 507, 475, 468, 538, 551, 529, 486, 492, 541, 480, 479, 529, 541, 524, 533, 494, 492, 491, 491, 523, 499, 523, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[537, 495, 524, 499, 530, 500, 523, 485, 505, 487, 532, 524, 527, 515, 494, 468, 499, 539, 535, 537, 482, 538, 519, 519, 513, 537, 537, 533, 504, 531, 508, 494, 500, 489, 533, 494, 497, 543, 515, 489, 514, 519, 513, 519, 494, 520, 483, 512, 511, 543, 515, 537, 509, 497, 531, 526, 516, 461, 483, 508, 506, 476, 495, 537, 527, 522, 517, 483, 501, 492, 510, 492, 514, 508, 509, 501, 493, 516, 518, 500, 542, 516, 485, 509, 517, 501, 534, 518, 494, 541, 519, 543, 507, 483, 521, 500, 508, 497, 546, 516, 507, 496, 518, 536, 550, 534, 555, 543, 488, 495, 469, 537, 487, 505, 493, 529, 489, 490, 487, 513, 515, 525, 549, 520, 506, 531, 504, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[506, 516, 525, 521, 516, 513, 506, 505, 501, 480, 484, 508, 514, 514, 518, 463, 520, 558, 510, 529, 541, 570, 483, 505, 528, 518, 494, 518, 522, 531, 545, 492, 534, 481, 456, 525, 532, 538, 470, 547, 480, 534, 512, 471, 534, 513, 522, 500, 452, 499, 499, 529, 536, 488, 513, 561, 548, 520, 515, 535, 552, 497, 496, 512, 457, 493, 495, 515, 508, 499, 544, 529, 521, 545, 528, 530, 499, 531, 519, 510, 523, 545, 494, 512, 486, 506, 521, 497, 536, 497, 535, 505, 507, 506, 498, 500, 544, 502, 504, 463, 502, 499, 492, 451, 518, 512, 475, 487, 528, 551, 525, 521, 569, 495, 488, 573, 513, 499, 529, 499, 525, 505, 503, 488, 473, 501, 525, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[587, 499, 561, 495, 500, 511, 536, 495, 495, 482, 509, 516, 529, 477, 519, 528, 481, 494, 496, 504, 541, 510, 469, 519, 505, 525, 544, 495, 462, 526, 542, 511, 491, 509, 539, 507, 491, 532, 541, 518, 511, 532, 562, 527, 493, 547, 537, 494, 473, 497, 526, 514, 520, 490, 493, 516, 511, 495, 514, 520, 504, 525, 547, 543, 518, 511, 532, 508, 483, 504, 487, 505, 465, 498, 522, 509, 545, 500, 521, 526, 516, 478, 525, 523, 494, 493, 505, 490, 523, 463, 501, 481, 513, 527, 496, 479, 524, 517, 465, 497, 486, 486, 523, 523, 529, 510, 540, 525, 530, 514, 486, 481, 503, 548, 521, 528, 533, 556, 529, 499, 561, 520, 505, 523, 490, 533, 502, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[541, 460, 542, 524, 552, 532, 522, 519, 524, 483, 512, 509, 536, 529, 503, 532, 462, 460, 532, 527, 525, 521, 524, 547, 492, 562, 506, 501, 467, 525, 535, 535, 508, 517, 477, 489, 532, 526, 512, 540, 521, 527, 532, 511, 473, 545, 492, 565, 517, 503, 527, 471, 502, 516, 500, 504, 529, 499, 514, 481, 493, 557, 526, 528, 555, 506, 509, 512, 507, 504, 524, 517, 491, 516, 493, 506, 505, 487, 473, 527, 519, 500, 506, 519, 516, 474, 494, 515, 482, 556, 505, 518, 542, 525, 531, 504, 526, 563, 514, 490, 514, 520, 522, 470, 518, 503, 463, 500, 487, 531, 478, 490, 541, 521, 537, 563, 510, 478, 475, 495, 512, 492, 478, 520, 481, 499, 509, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[503, 525, 481, 495, 471, 519, 478, 524, 535, 505, 480, 530, 521, 512, 557, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 481, 495, 509, 525, 530, 503, 490, 517, 498, 523, 509, 520, 501, 480, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 533, 499, 530, 511, 508, 501, 518, 497, 524, 524, 525, 532, 482, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 533, 489, 473, 549, 481, 517, 517, 500, 503, 522, 510, 521, 511, 492, 508, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 519, 526, 479, 507, 506, 497, 490, 493, 514, 538, 523, 491, 504, 550, 14, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 517, 494, 495, 501, 509, 501, 521, 494, 481, 481, 534, 559, 517, 533, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 521, 514, 481, 549, 511, 499, 495, 500, 542, 508, 498, 519, 522, 501, 494, 46, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 521, 511, 474, 531, 576, 500, 510, 534, 508, 530, 533, 518, 509, 508, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8123, 8228, 8152, 8194, 8148, 8253, 8274] avg=8192.0000 max=8274 max_rank=7 min=8123 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 512, 520, 510, 481, 511, 532, 525, 498, 543, 500, 532, 504, 525, 517, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 504, 543, 522, 465, 492, 523, 485, 541, 519, 485, 505, 505, 556, 494, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 549, 528, 538, 533, 518, 506, 488, 512, 509, 482, 519, 474, 508, 497, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 545, 525, 530, 531, 488, 508, 521, 486, 496, 509, 541, 479, 506, 490, 485, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 542, 471, 503, 496, 541, 484, 484, 508, 554, 516, 495, 522, 509, 533, 499, 18, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 519, 524, 512, 521, 467, 535, 521, 533, 521, 506, 520, 498, 516, 510, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 481, 510, 537, 473, 496, 483, 526, 507, 475, 538, 535, 479, 502, 526, 476, 50, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 519, 490, 546, 480, 516, 479, 507, 495, 513, 534, 530, 479, 532, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8224, 8193, 8140, 8234, 8251, 8108, 8115] avg=8192.0000 max=8271 max_rank=0 min=8108 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[537, 495, 506, 499, 530, 500, 523, 485, 505, 487, 532, 524, 527, 515, 494, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 539, 535, 537, 482, 538, 519, 475, 513, 537, 537, 533, 504, 531, 508, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 489, 533, 494, 497, 543, 515, 489, 514, 505, 513, 519, 494, 520, 483, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 543, 515, 537, 509, 497, 531, 526, 516, 461, 483, 508, 506, 476, 495, 537, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 522, 517, 483, 501, 492, 510, 492, 514, 508, 509, 501, 493, 516, 518, 500, 14, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 516, 485, 509, 517, 501, 534, 518, 494, 541, 519, 541, 507, 483, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 546, 516, 507, 496, 518, 536, 550, 534, 555, 543, 488, 495, 469, 537, 44, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 505, 493, 516, 489, 490, 487, 513, 515, 525, 549, 520, 506, 531, 504, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8281, 8120, 8161, 8130, 8228, 8349, 8140] avg=8192.0000 max=8349 max_rank=6 min=8120 min_rank=2 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[506, 516, 507, 521, 516, 513, 506, 505, 501, 480, 484, 508, 514, 514, 518, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 558, 510, 529, 541, 570, 483, 462, 528, 518, 494, 518, 522, 531, 545, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 481, 456, 525, 532, 538, 470, 547, 480, 520, 512, 471, 534, 513, 522, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 499, 499, 529, 536, 488, 513, 561, 548, 520, 515, 535, 552, 497, 496, 512, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 493, 495, 515, 508, 499, 544, 529, 521, 545, 528, 530, 499, 531, 519, 510, 14, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 545, 494, 512, 486, 506, 521, 497, 536, 497, 535, 503, 507, 506, 498, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 502, 504, 463, 502, 499, 492, 451, 518, 512, 475, 487, 528, 551, 525, 521, 43, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 495, 488, 559, 513, 499, 529, 499, 525, 505, 503, 488, 473, 501, 525, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8072, 8321, 8135, 8263, 8250, 8166, 8127, 8202] avg=8192.0000 max=8321 max_rank=1 min=8072 min_rank=0 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 530, 484, 527, 481, 484, 511, 519, 480, 526, 511, 549, 505, 498, 499, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 499, 522, 572, 517, 472, 513, 488, 521, 504, 507, 467, 512, 486, 485, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 503, 479, 506, 520, 467, 471, 521, 570, 538, 476, 502, 508, 517, 493, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 528, 539, 531, 501, 499, 513, 475, 504, 475, 523, 523, 529, 512, 522, 477, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 526, 462, 518, 544, 483, 512, 487, 502, 505, 524, 509, 480, 530, 548, 15, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 533, 506, 536, 535, 541, 511, 519, 549, 508, 532, 527, 483, 490, 558, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 496, 528, 482, 538, 533, 503, 507, 513, 507, 475, 468, 538, 551, 529, 486, 45, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 541, 480, 467, 529, 541, 524, 533, 494, 492, 491, 491, 523, 499, 523, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8138, 8106, 8165, 8171, 8404, 8228, 8173] avg=8192.0000 max=8404 max_rank=5 min=8106 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=57132 max_load_reduction_pct=0.871765 moved_to_echo=32976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[527, 532, 519, 502, 503, 504, 479, 523, 495, 536, 546, 542, 489, 525, 516, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 538, 508, 509, 546, 529, 536, 480, 511, 498, 498, 511, 482, 513, 487, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 524, 492, 481, 487, 509, 513, 543, 485, 516, 506, 521, 525, 507, 508, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 521, 518, 540, 538, 555, 517, 547, 509, 472, 486, 471, 503, 539, 516, 522, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 477, 469, 518, 558, 514, 529, 526, 502, 552, 505, 476, 495, 513, 534, 14, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 496, 482, 530, 488, 510, 523, 518, 482, 514, 515, 563, 511, 492, 490, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 496, 525, 514, 482, 558, 510, 481, 491, 495, 500, 488, 515, 488, 497, 494, 44, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 469, 523, 497, 510, 540, 522, 493, 498, 519, 503, 523, 495, 544, 528, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8194, 8184, 8279, 8235, 8130, 8136, 8148] avg=8192.0000 max=8279 max_rank=3 min=8130 min_rank=5 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[541, 460, 523, 524, 552, 532, 522, 519, 524, 483, 512, 509, 536, 529, 503, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 460, 532, 527, 525, 521, 524, 501, 492, 562, 506, 501, 467, 525, 535, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 477, 489, 532, 526, 512, 540, 521, 513, 532, 511, 473, 545, 492, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 503, 527, 471, 502, 516, 500, 504, 529, 499, 514, 481, 493, 557, 526, 528, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 506, 509, 512, 507, 504, 524, 517, 491, 516, 493, 506, 505, 487, 473, 527, 14, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 506, 519, 516, 474, 494, 515, 482, 556, 505, 516, 542, 525, 531, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 563, 514, 490, 514, 520, 522, 470, 518, 503, 463, 500, 487, 531, 478, 490, 46, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 521, 537, 549, 510, 478, 475, 495, 512, 492, 478, 520, 481, 499, 509, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8175, 8253, 8178, 8160, 8204, 8145, 8120] avg=8192.0000 max=8301 max_rank=0 min=8120 min_rank=7 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[587, 499, 542, 495, 500, 511, 536, 495, 495, 482, 509, 516, 529, 477, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 494, 496, 504, 541, 510, 469, 475, 505, 525, 544, 495, 462, 526, 542, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 509, 539, 507, 491, 532, 541, 518, 511, 518, 562, 527, 493, 547, 537, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 497, 526, 514, 520, 490, 493, 516, 511, 495, 514, 520, 504, 525, 547, 543, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 532, 508, 483, 504, 487, 505, 465, 498, 522, 509, 545, 500, 521, 526, 14, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 478, 525, 523, 494, 493, 505, 490, 523, 463, 501, 479, 513, 527, 496, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 517, 465, 497, 486, 486, 523, 523, 529, 510, 540, 525, 530, 514, 486, 481, 44, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 548, 521, 515, 533, 556, 529, 499, 561, 520, 505, 523, 490, 533, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8080, 8317, 8198, 8162, 8005, 8190, 8364] avg=8192.0000 max=8364 max_rank=7 min=8005 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4352, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[500, 487, 531, 527, 530, 506, 502, 481, 525, 507, 507, 490, 522, 468, 539, 507, 516, 520, 511, 479, 477, 551, 477, 537, 494, 511, 517, 500, 533, 490, 520, 519, 510, 501, 504, 501, 500, 522, 511, 500, 506, 525, 509, 497, 531, 506, 521, 534, 509, 517, 498, 508, 506, 529, 512, 513, 480, 493, 496, 503, 475, 541, 564, 446, 505, 503, 537, 518, 559, 529, 505, 524, 530, 529, 527, 507, 523, 478, 526, 556, 497, 512, 522, 514, 506, 487, 521, 467, 507, 509, 487, 498, 513, 532, 534, 541, 520, 505, 556, 515, 507, 553, 505, 529, 464, 547, 515, 541, 494, 499, 491, 503, 535, 530, 524, 544, 495, 499, 512, 514, 495, 510, 499, 528, 516, 546, 506, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[538, 505, 494, 561, 485, 492, 491, 534, 488, 549, 513, 522, 520, 518, 489, 508, 510, 504, 514, 510, 494, 516, 521, 524, 523, 526, 505, 516, 514, 514, 483, 523, 503, 527, 536, 478, 511, 547, 503, 539, 529, 500, 505, 500, 547, 504, 521, 533, 490, 509, 556, 514, 507, 489, 491, 510, 493, 476, 523, 526, 481, 479, 496, 501, 510, 518, 517, 488, 542, 535, 468, 510, 541, 514, 510, 526, 550, 527, 534, 573, 475, 489, 491, 522, 472, 493, 516, 541, 516, 495, 509, 483, 527, 526, 502, 497, 543, 544, 512, 543, 505, 503, 521, 510, 527, 550, 529, 486, 489, 535, 523, 495, 533, 503, 512, 523, 507, 487, 518, 511, 494, 491, 498, 532, 465, 491, 515, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[528, 532, 512, 501, 524, 489, 500, 489, 478, 494, 514, 526, 537, 501, 480, 501, 521, 530, 484, 532, 490, 505, 516, 505, 500, 539, 491, 572, 499, 497, 513, 520, 519, 523, 501, 519, 521, 473, 530, 494, 528, 477, 514, 506, 509, 467, 533, 503, 505, 528, 510, 494, 552, 492, 490, 486, 494, 533, 497, 546, 524, 487, 524, 526, 529, 523, 483, 477, 528, 504, 519, 511, 473, 503, 481, 496, 572, 509, 549, 544, 537, 479, 530, 501, 515, 485, 503, 544, 509, 524, 577, 525, 488, 505, 516, 544, 564, 529, 518, 508, 471, 525, 498, 475, 503, 487, 496, 516, 505, 535, 527, 530, 530, 504, 528, 493, 549, 494, 528, 485, 497, 527, 547, 525, 519, 489, 506, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[500, 537, 495, 483, 555, 467, 513, 532, 491, 516, 499, 554, 497, 527, 527, 506, 502, 509, 507, 535, 516, 529, 521, 476, 523, 501, 482, 504, 525, 512, 514, 509, 503, 500, 461, 524, 480, 513, 531, 511, 527, 511, 508, 493, 473, 511, 526, 521, 508, 536, 506, 535, 494, 509, 534, 508, 484, 522, 520, 497, 524, 497, 478, 519, 521, 519, 550, 528, 461, 480, 497, 516, 498, 518, 513, 471, 509, 513, 522, 511, 498, 536, 472, 504, 538, 521, 533, 533, 550, 549, 521, 524, 518, 519, 543, 502, 523, 528, 486, 513, 523, 535, 547, 525, 524, 517, 519, 519, 489, 509, 544, 526, 499, 480, 519, 515, 484, 524, 502, 482, 525, 478, 531, 469, 525, 509, 534, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[525, 538, 492, 546, 489, 534, 512, 541, 520, 514, 515, 483, 500, 483, 517, 475, 534, 543, 508, 504, 496, 478, 527, 482, 536, 535, 529, 485, 525, 512, 519, 485, 490, 488, 527, 486, 516, 498, 494, 537, 528, 553, 485, 484, 499, 514, 533, 522, 503, 536, 493, 489, 515, 517, 525, 537, 518, 499, 543, 500, 528, 468, 494, 543, 475, 529, 476, 473, 513, 507, 519, 520, 563, 482, 506, 513, 467, 533, 494, 516, 534, 535, 526, 507, 518, 502, 548, 526, 489, 485, 536, 527, 477, 530, 531, 519, 537, 493, 475, 523, 501, 521, 581, 550, 493, 524, 518, 480, 508, 486, 493, 500, 518, 535, 537, 503, 519, 507, 482, 503, 497, 506, 548, 528, 521, 525, 471, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[528, 526, 536, 518, 530, 502, 518, 536, 557, 501, 505, 508, 469, 526, 508, 507, 516, 473, 517, 497, 492, 531, 540, 526, 479, 529, 523, 498, 524, 507, 464, 495, 537, 538, 487, 506, 527, 499, 509, 493, 515, 524, 488, 495, 509, 529, 540, 523, 542, 510, 481, 512, 485, 513, 508, 494, 480, 502, 521, 534, 500, 508, 504, 511, 526, 544, 538, 512, 528, 532, 482, 498, 528, 483, 498, 494, 500, 482, 517, 515, 504, 525, 506, 530, 564, 507, 541, 519, 501, 505, 525, 471, 477, 514, 486, 529, 508, 526, 518, 522, 524, 517, 484, 530, 521, 519, 512, 513, 514, 507, 516, 539, 494, 515, 502, 507, 506, 555, 506, 556, 513, 493, 499, 507, 522, 486, 518, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[519, 500, 528, 514, 509, 523, 494, 527, 485, 536, 499, 523, 498, 533, 494, 536, 510, 546, 504, 511, 493, 558, 513, 526, 506, 483, 515, 457, 494, 510, 524, 490, 498, 548, 515, 511, 531, 514, 527, 477, 481, 503, 481, 517, 545, 476, 531, 511, 514, 478, 521, 535, 551, 514, 493, 505, 485, 534, 498, 513, 513, 546, 503, 484, 524, 507, 482, 507, 457, 529, 480, 506, 500, 511, 512, 491, 534, 520, 483, 541, 518, 563, 481, 497, 503, 506, 549, 491, 466, 485, 529, 521, 568, 547, 518, 494, 526, 502, 525, 514, 475, 532, 507, 564, 501, 493, 493, 557, 513, 531, 488, 519, 485, 533, 566, 520, 508, 533, 500, 544, 504, 489, 551, 525, 487, 518, 488, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[460, 488, 537, 515, 484, 530, 506, 519, 531, 525, 487, 550, 516, 530, 502, 513, 552, 512, 519, 558, 534, 532, 460, 487, 481, 527, 506, 529, 474, 513, 474, 534, 494, 511, 493, 505, 464, 504, 527, 511, 507, 547, 490, 513, 535, 502, 520, 488, 488, 547, 538, 528, 456, 518, 489, 509, 522, 510, 512, 552, 510, 489, 463, 490, 505, 503, 472, 496, 492, 505, 498, 505, 510, 510, 514, 492, 509, 542, 508, 504, 511, 490, 480, 546, 509, 525, 544, 505, 526, 498, 541, 492, 495, 517, 532, 544, 508, 497, 519, 569, 524, 511, 528, 498, 519, 525, 530, 522, 518, 462, 505, 524, 532, 498, 538, 494, 518, 509, 522, 524, 514, 536, 585, 512, 531, 509, 505, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 487, 531, 527, 530, 506, 502, 481, 525, 507, 507, 490, 522, 468, 539, 507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 520, 511, 479, 477, 551, 477, 537, 494, 511, 517, 500, 533, 490, 520, 519, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 501, 504, 501, 500, 522, 511, 500, 506, 525, 509, 497, 531, 506, 521, 534, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 498, 508, 506, 529, 512, 513, 480, 493, 496, 503, 475, 541, 564, 446, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 537, 518, 559, 529, 505, 524, 530, 529, 527, 507, 523, 478, 526, 556, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 522, 514, 506, 487, 486, 467, 507, 509, 487, 498, 513, 532, 534, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 505, 556, 515, 507, 553, 505, 529, 464, 547, 515, 541, 494, 499, 491, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 530, 524, 544, 495, 499, 512, 514, 495, 510, 486, 528, 516, 546, 506, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8171, 8203, 8142, 8365, 8112, 8185, 8227] avg=8192.0000 max=8365 max_rank=4 min=8112 min_rank=5 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32889 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[525, 538, 492, 546, 489, 534, 512, 541, 520, 514, 515, 483, 500, 483, 517, 475, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 543, 508, 504, 496, 478, 527, 482, 536, 535, 529, 485, 525, 512, 519, 485, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 488, 527, 486, 516, 498, 494, 537, 528, 553, 485, 484, 499, 514, 533, 522, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 536, 493, 489, 515, 517, 525, 537, 518, 499, 543, 500, 528, 468, 494, 543, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 529, 476, 473, 513, 507, 519, 520, 563, 482, 506, 513, 467, 533, 494, 516, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 535, 526, 507, 518, 502, 511, 526, 489, 485, 536, 527, 477, 530, 531, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 493, 475, 523, 501, 521, 581, 550, 493, 524, 518, 480, 508, 486, 493, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 537, 503, 519, 507, 482, 503, 497, 506, 533, 528, 521, 525, 471, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8219, 8181, 8261, 8095, 8253, 8122, 8218] avg=8192.0000 max=8261 max_rank=3 min=8095 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 526, 536, 518, 530, 502, 518, 536, 557, 501, 505, 508, 469, 526, 508, 507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 473, 517, 497, 492, 531, 540, 526, 479, 529, 523, 498, 524, 507, 464, 495, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 538, 487, 506, 527, 499, 509, 493, 515, 524, 488, 495, 509, 529, 540, 523, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 481, 512, 485, 513, 508, 494, 480, 502, 521, 534, 500, 508, 504, 511, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 544, 538, 512, 528, 532, 482, 498, 528, 483, 498, 494, 500, 482, 517, 515, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 506, 530, 564, 507, 504, 519, 501, 505, 525, 471, 477, 514, 486, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 526, 518, 522, 524, 517, 484, 530, 521, 519, 512, 513, 514, 507, 516, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 502, 507, 506, 555, 506, 556, 513, 493, 486, 507, 522, 486, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8131, 8245, 8156, 8186, 8167, 8212, 8162] avg=8192.0000 max=8277 max_rank=0 min=8131 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[538, 505, 494, 561, 485, 492, 491, 534, 488, 549, 513, 522, 520, 518, 489, 508, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 514, 510, 494, 516, 521, 524, 523, 526, 505, 516, 514, 514, 483, 523, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 527, 536, 478, 511, 547, 503, 539, 529, 500, 505, 500, 547, 504, 521, 533, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 509, 556, 514, 507, 489, 491, 510, 493, 476, 523, 526, 481, 479, 496, 501, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 518, 517, 488, 542, 535, 468, 510, 541, 514, 510, 526, 550, 527, 534, 573, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 489, 491, 522, 472, 493, 481, 541, 516, 495, 509, 483, 527, 526, 502, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 544, 512, 543, 505, 503, 521, 510, 527, 550, 529, 486, 489, 535, 523, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 503, 512, 523, 507, 487, 518, 511, 494, 491, 485, 532, 465, 491, 515, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8216, 8309, 8095, 8372, 8019, 8253, 8063] avg=8192.0000 max=8372 max_rank=4 min=8019 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 532, 512, 501, 524, 489, 500, 489, 478, 494, 514, 526, 537, 501, 480, 501, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 530, 484, 532, 490, 505, 516, 505, 500, 539, 491, 572, 499, 497, 513, 520, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 523, 501, 519, 521, 473, 530, 494, 528, 477, 514, 506, 509, 467, 533, 503, 21, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 510, 494, 552, 492, 490, 486, 494, 533, 497, 546, 524, 487, 524, 526, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 523, 483, 477, 528, 504, 519, 511, 473, 503, 481, 496, 572, 509, 549, 544, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 479, 530, 501, 515, 485, 463, 544, 509, 524, 577, 525, 488, 505, 516, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 529, 518, 508, 471, 525, 498, 475, 503, 487, 496, 516, 505, 535, 527, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 504, 528, 493, 549, 494, 528, 485, 497, 527, 520, 525, 519, 489, 506, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8240, 8151, 8248, 8214, 8242, 8114, 8214] avg=8192.0000 max=8248 max_rank=3 min=8113 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 537, 495, 483, 555, 467, 513, 532, 491, 516, 499, 554, 497, 527, 527, 506, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 509, 507, 535, 516, 529, 521, 476, 523, 501, 482, 504, 525, 512, 514, 509, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 500, 461, 524, 480, 513, 531, 511, 527, 511, 508, 493, 473, 511, 526, 521, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 536, 506, 535, 494, 509, 534, 508, 484, 522, 520, 497, 524, 497, 478, 519, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 550, 528, 461, 480, 497, 516, 498, 518, 513, 471, 509, 513, 522, 511, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 536, 472, 504, 538, 521, 498, 533, 550, 549, 521, 524, 518, 519, 543, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 528, 486, 513, 523, 535, 547, 525, 524, 517, 519, 519, 489, 509, 544, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 480, 519, 515, 484, 524, 502, 482, 525, 478, 516, 469, 525, 509, 534, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8185, 8118, 8223, 8136, 8326, 8268, 8078] avg=8192.0000 max=8326 max_rank=5 min=8078 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 500, 528, 514, 509, 523, 494, 527, 485, 536, 499, 523, 498, 533, 494, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 546, 504, 511, 493, 558, 513, 526, 506, 483, 515, 457, 494, 510, 524, 490, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 548, 515, 511, 531, 514, 527, 477, 481, 503, 481, 517, 545, 476, 531, 511, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 478, 521, 535, 551, 514, 493, 505, 485, 534, 498, 513, 513, 546, 503, 484, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 482, 507, 457, 529, 480, 506, 500, 511, 512, 491, 534, 520, 483, 541, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 563, 481, 497, 503, 506, 512, 491, 466, 485, 529, 521, 568, 547, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 502, 525, 514, 475, 532, 507, 564, 501, 493, 493, 557, 513, 531, 488, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 533, 566, 520, 508, 533, 500, 544, 504, 489, 536, 525, 487, 518, 488, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8161, 8192, 8239, 8093, 8199, 8181, 8250] avg=8192.0000 max=8250 max_rank=7 min=8093 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[460, 488, 537, 515, 484, 530, 506, 519, 531, 525, 487, 550, 516, 530, 502, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 512, 519, 558, 534, 532, 460, 487, 481, 527, 506, 529, 474, 513, 474, 534, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 493, 505, 464, 504, 527, 511, 507, 547, 490, 513, 535, 502, 520, 488, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 547, 538, 528, 456, 518, 489, 509, 522, 510, 512, 552, 510, 489, 463, 490, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 472, 496, 492, 505, 498, 505, 510, 510, 514, 492, 509, 542, 508, 504, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 490, 480, 546, 509, 525, 507, 505, 526, 498, 541, 492, 495, 517, 532, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 497, 519, 569, 524, 511, 528, 498, 519, 525, 530, 522, 518, 462, 505, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 498, 538, 494, 518, 509, 522, 524, 514, 536, 569, 512, 531, 509, 505, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8213, 8137, 8172, 8075, 8218, 8201, 8324] avg=8192.0000 max=8324 max_rank=7 min=8075 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3840, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4352, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4352, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[517, 490, 521, 530, 492, 550, 475, 512, 528, 505, 452, 508, 506, 505, 487, 499, 517, 535, 512, 559, 489, 496, 540, 492, 535, 504, 506, 515, 449, 494, 536, 500, 524, 504, 482, 533, 518, 514, 498, 495, 522, 490, 551, 523, 545, 514, 502, 548, 525, 488, 520, 507, 522, 502, 519, 525, 527, 552, 488, 478, 475, 501, 509, 481, 509, 520, 558, 514, 530, 507, 539, 481, 526, 510, 544, 516, 546, 544, 471, 501, 501, 528, 509, 506, 520, 481, 487, 487, 498, 537, 503, 526, 519, 517, 491, 504, 508, 516, 490, 487, 529, 512, 557, 502, 527, 539, 501, 502, 479, 532, 506, 504, 541, 539, 528, 541, 511, 503, 530, 505, 511, 511, 492, 497, 516, 527, 557, 468] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[513, 519, 510, 513, 539, 492, 519, 529, 496, 558, 520, 551, 501, 494, 488, 484, 491, 529, 542, 494, 475, 528, 507, 542, 525, 503, 568, 535, 510, 522, 509, 494, 501, 520, 530, 494, 469, 532, 478, 463, 504, 499, 537, 485, 528, 498, 524, 491, 499, 535, 482, 505, 487, 481, 477, 523, 501, 540, 497, 557, 517, 541, 506, 483, 517, 505, 506, 536, 480, 524, 509, 531, 515, 537, 472, 530, 514, 529, 541, 510, 528, 472, 514, 519, 504, 489, 505, 475, 499, 498, 527, 465, 478, 546, 511, 549, 502, 507, 504, 526, 537, 516, 559, 526, 487, 541, 511, 524, 463, 522, 500, 543, 491, 494, 527, 494, 562, 558, 505, 518, 501, 541, 518, 533, 486, 518, 511, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[522, 459, 513, 527, 462, 502, 502, 526, 513, 517, 509, 531, 496, 518, 488, 518, 515, 551, 542, 519, 524, 507, 525, 484, 503, 532, 537, 521, 513, 492, 515, 527, 512, 514, 560, 539, 510, 518, 519, 530, 480, 533, 529, 483, 529, 509, 473, 512, 519, 506, 534, 568, 544, 480, 516, 491, 493, 483, 510, 507, 497, 523, 522, 542, 520, 498, 537, 496, 511, 539, 537, 504, 464, 492, 546, 488, 491, 531, 515, 479, 550, 530, 543, 523, 502, 544, 534, 519, 490, 473, 474, 536, 511, 517, 489, 500, 495, 506, 500, 509, 556, 475, 499, 497, 530, 495, 498, 508, 532, 468, 486, 538, 491, 535, 536, 454, 487, 549, 497, 536, 488, 516, 510, 524, 475, 564, 506, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[500, 504, 506, 498, 527, 499, 530, 515, 499, 545, 528, 506, 526, 482, 524, 514, 513, 514, 518, 496, 501, 521, 494, 452, 489, 523, 519, 510, 524, 529, 489, 554, 525, 531, 534, 528, 511, 527, 533, 495, 497, 516, 508, 552, 527, 525, 592, 537, 521, 517, 524, 531, 553, 509, 521, 478, 525, 498, 511, 516, 486, 482, 526, 474, 517, 515, 473, 548, 480, 527, 512, 520, 517, 478, 517, 491, 491, 498, 527, 530, 520, 517, 492, 497, 476, 514, 481, 551, 499, 503, 517, 503, 521, 486, 514, 492, 501, 501, 507, 545, 502, 508, 504, 500, 517, 514, 474, 552, 486, 559, 528, 484, 544, 505, 496, 535, 492, 527, 531, 517, 515, 531, 508, 521, 437, 487, 496, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[501, 492, 522, 518, 463, 522, 536, 517, 511, 504, 500, 541, 492, 479, 505, 494, 539, 475, 500, 485, 502, 516, 507, 531, 534, 533, 495, 508, 516, 492, 519, 520, 524, 537, 499, 503, 491, 492, 517, 498, 484, 509, 513, 501, 549, 470, 552, 528, 484, 487, 512, 539, 533, 517, 533, 511, 506, 493, 518, 540, 501, 548, 515, 526, 511, 532, 506, 493, 514, 515, 504, 541, 479, 568, 507, 532, 527, 505, 508, 508, 544, 521, 493, 535, 488, 516, 508, 478, 470, 509, 515, 487, 509, 516, 497, 495, 512, 524, 504, 488, 484, 555, 506, 496, 496, 530, 526, 532, 520, 533, 499, 510, 502, 501, 474, 547, 516, 585, 500, 504, 512, 556, 521, 486, 515, 525, 523, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[515, 542, 498, 507, 510, 514, 493, 528, 534, 501, 505, 504, 504, 517, 461, 520, 509, 521, 517, 509, 522, 538, 516, 512, 519, 523, 490, 515, 514, 520, 501, 528, 503, 539, 512, 486, 506, 488, 502, 550, 497, 496, 519, 487, 536, 478, 528, 498, 535, 486, 488, 535, 507, 486, 494, 541, 476, 558, 490, 491, 477, 541, 490, 483, 526, 511, 521, 557, 505, 524, 517, 507, 541, 484, 521, 457, 545, 496, 509, 483, 515, 536, 541, 505, 548, 506, 553, 488, 498, 560, 537, 524, 525, 533, 509, 501, 543, 535, 500, 514, 496, 512, 500, 506, 516, 535, 471, 481, 497, 494, 488, 491, 485, 500, 488, 514, 519, 535, 567, 542, 521, 494, 480, 484, 517, 520, 546, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[515, 499, 488, 528, 531, 528, 531, 476, 535, 516, 508, 525, 498, 510, 530, 521, 535, 496, 496, 517, 527, 521, 467, 491, 518, 465, 514, 516, 525, 486, 483, 537, 481, 539, 504, 524, 536, 479, 531, 521, 514, 469, 524, 523, 542, 494, 533, 524, 493, 502, 480, 491, 531, 534, 532, 553, 495, 538, 524, 496, 520, 527, 533, 537, 511, 497, 487, 524, 499, 505, 494, 554, 497, 535, 517, 521, 537, 514, 501, 542, 523, 511, 526, 525, 512, 517, 529, 522, 509, 507, 491, 506, 480, 479, 545, 489, 538, 508, 498, 500, 500, 533, 552, 479, 549, 507, 528, 515, 503, 502, 530, 497, 489, 494, 474, 472, 495, 541, 476, 522, 510, 526, 499, 516, 497, 503, 501, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[530, 556, 517, 486, 532, 486, 495, 538, 499, 500, 479, 529, 513, 508, 515, 493, 528, 537, 528, 520, 553, 528, 473, 508, 549, 507, 533, 515, 545, 505, 487, 507, 543, 483, 498, 490, 512, 530, 505, 519, 496, 531, 537, 487, 445, 555, 482, 511, 512, 497, 480, 495, 537, 500, 504, 503, 526, 537, 515, 516, 505, 542, 509, 538, 505, 504, 485, 533, 516, 529, 489, 511, 498, 530, 543, 534, 574, 505, 476, 513, 555, 491, 519, 530, 567, 552, 548, 530, 508, 541, 478, 462, 460, 523, 495, 502, 504, 481, 516, 493, 474, 549, 493, 475, 501, 470, 491, 486, 500, 509, 532, 503, 522, 507, 548, 520, 522, 549, 494, 517, 510, 479, 504, 504, 513, 486, 544, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 519, 510, 513, 539, 492, 519, 529, 496, 558, 520, 551, 501, 494, 488, 484, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 529, 542, 494, 475, 528, 507, 542, 499, 503, 568, 535, 510, 522, 509, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 520, 530, 494, 469, 532, 478, 463, 504, 499, 523, 485, 528, 498, 524, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 535, 482, 505, 487, 481, 477, 523, 501, 540, 497, 557, 517, 541, 506, 483, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 506, 536, 480, 524, 509, 531, 515, 537, 472, 530, 483, 529, 541, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 472, 514, 519, 504, 489, 505, 475, 499, 498, 527, 465, 478, 546, 511, 549, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 504, 526, 537, 516, 559, 526, 487, 541, 511, 524, 463, 522, 500, 543, 31, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 494, 527, 494, 562, 542, 505, 518, 501, 541, 518, 533, 486, 518, 511, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8248, 8039, 8135, 8225, 8095, 8303, 8233] avg=8192.0000 max=8303 max_rank=6 min=8039 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[517, 490, 521, 530, 492, 550, 475, 512, 528, 505, 452, 508, 506, 505, 487, 499, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 535, 512, 559, 489, 496, 540, 492, 509, 504, 506, 515, 449, 494, 536, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 504, 482, 533, 518, 514, 498, 495, 522, 490, 536, 523, 545, 514, 502, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 488, 520, 507, 522, 502, 519, 525, 527, 552, 488, 478, 475, 501, 509, 481, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 520, 558, 514, 530, 507, 539, 481, 526, 510, 544, 516, 513, 544, 471, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 509, 506, 520, 481, 487, 487, 498, 537, 503, 526, 519, 517, 491, 504, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 516, 490, 487, 529, 512, 557, 502, 527, 539, 501, 502, 479, 532, 506, 504, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 539, 528, 541, 511, 489, 530, 505, 511, 511, 492, 497, 516, 527, 557, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8153, 8248, 8123, 8283, 8130, 8228, 8263] avg=8192.0000 max=8283 max_rank=4 min=8108 min_rank=0 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[522, 459, 513, 527, 462, 502, 502, 526, 513, 517, 509, 531, 496, 518, 488, 518, 21, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 551, 542, 519, 524, 507, 525, 484, 478, 532, 537, 521, 513, 492, 515, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 514, 560, 539, 510, 518, 519, 530, 480, 533, 515, 483, 529, 509, 473, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 534, 568, 544, 480, 516, 491, 493, 483, 510, 507, 497, 523, 522, 542, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 498, 537, 496, 511, 539, 537, 504, 464, 492, 546, 488, 461, 531, 515, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 530, 543, 523, 502, 544, 534, 519, 490, 473, 474, 536, 511, 517, 489, 500, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 500, 509, 556, 475, 499, 497, 530, 495, 498, 508, 532, 468, 486, 538, 30, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 535, 536, 454, 487, 534, 497, 536, 488, 516, 510, 524, 475, 564, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8282, 8236, 8239, 8118, 8250, 8126, 8151] avg=8192.0000 max=8282 max_rank=1 min=8118 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[500, 504, 506, 498, 527, 499, 530, 515, 499, 545, 528, 506, 526, 482, 524, 514, 25, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 514, 518, 496, 501, 521, 494, 452, 458, 523, 519, 510, 524, 529, 489, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 531, 534, 528, 511, 527, 533, 495, 497, 516, 485, 552, 527, 525, 592, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 524, 531, 553, 509, 521, 478, 525, 498, 511, 516, 486, 482, 526, 474, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 473, 548, 480, 527, 512, 520, 517, 478, 517, 491, 458, 498, 527, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 492, 497, 476, 514, 481, 551, 499, 503, 517, 503, 521, 486, 514, 492, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 507, 545, 502, 508, 504, 500, 517, 514, 474, 552, 486, 559, 528, 484, 33, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 505, 496, 535, 492, 504, 531, 517, 515, 531, 508, 521, 437, 487, 496, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8115, 8415, 8180, 8108, 8108, 8221, 8148] avg=8192.0000 max=8415 max_rank=2 min=8108 min_rank=4 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 492, 522, 518, 463, 522, 536, 517, 511, 504, 500, 541, 492, 479, 505, 494, 22, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 475, 500, 485, 502, 516, 507, 531, 508, 533, 495, 508, 516, 492, 519, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 537, 499, 503, 491, 492, 517, 498, 484, 509, 499, 501, 549, 470, 552, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 487, 512, 539, 533, 517, 533, 511, 506, 493, 518, 540, 501, 548, 515, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 506, 493, 514, 515, 504, 541, 479, 568, 507, 532, 495, 505, 508, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 521, 493, 535, 488, 516, 508, 478, 470, 509, 515, 487, 509, 516, 497, 495, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 504, 488, 484, 555, 506, 496, 496, 530, 526, 532, 520, 533, 499, 510, 32, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 474, 547, 516, 568, 500, 504, 512, 556, 521, 486, 515, 525, 523, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8146, 8153, 8267, 8218, 8097, 8251, 8274] avg=8192.0000 max=8274 max_rank=7 min=8097 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[530, 556, 517, 486, 532, 486, 495, 538, 499, 500, 479, 529, 513, 508, 515, 493, 23, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 528, 520, 553, 528, 473, 508, 522, 507, 533, 515, 545, 505, 487, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 483, 498, 490, 512, 530, 505, 519, 496, 531, 523, 487, 445, 555, 482, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 480, 495, 537, 500, 504, 503, 526, 537, 515, 516, 505, 542, 509, 538, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 485, 533, 516, 529, 489, 511, 498, 530, 543, 534, 539, 505, 476, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 491, 519, 530, 567, 552, 548, 530, 508, 541, 478, 462, 460, 523, 495, 502, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 481, 516, 493, 474, 549, 493, 475, 501, 470, 491, 486, 500, 509, 532, 503, 35, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 507, 548, 520, 522, 534, 494, 517, 510, 479, 504, 504, 513, 486, 544, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8296, 8110, 8220, 8210, 8276, 8016, 8199] avg=8192.0000 max=8296 max_rank=1 min=8016 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 542, 498, 507, 510, 514, 493, 528, 534, 501, 505, 504, 504, 517, 461, 520, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 521, 517, 509, 522, 538, 516, 512, 493, 523, 490, 515, 514, 520, 501, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 539, 512, 486, 506, 488, 502, 550, 497, 496, 505, 487, 536, 478, 528, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 486, 488, 535, 507, 486, 494, 541, 476, 558, 490, 491, 477, 541, 490, 483, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 511, 521, 557, 505, 524, 517, 507, 541, 484, 521, 457, 512, 496, 509, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 541, 505, 548, 506, 553, 488, 498, 560, 537, 524, 525, 533, 509, 501, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 535, 500, 514, 496, 512, 500, 506, 516, 535, 471, 481, 497, 494, 488, 491, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 500, 488, 514, 519, 520, 567, 542, 521, 494, 480, 484, 517, 520, 546, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8228, 8111, 8082, 8171, 8394, 8116, 8249] avg=8192.0000 max=8394 max_rank=5 min=8082 min_rank=3 imbalance_ratio=1.024658 max_load_reduction=57142 max_load_reduction_pct=0.871918 moved_to_echo=32930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 499, 488, 528, 531, 528, 531, 476, 535, 516, 508, 525, 498, 510, 530, 521, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 496, 496, 517, 527, 521, 467, 491, 492, 465, 514, 516, 525, 486, 483, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 539, 504, 524, 536, 479, 531, 521, 514, 469, 510, 523, 542, 494, 533, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 480, 491, 531, 534, 532, 553, 495, 538, 524, 496, 520, 527, 533, 537, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 487, 524, 499, 505, 494, 554, 497, 535, 517, 521, 505, 514, 501, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 526, 525, 512, 517, 529, 522, 509, 507, 491, 506, 480, 479, 545, 489, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 508, 498, 500, 500, 533, 552, 479, 549, 507, 528, 515, 503, 502, 530, 497, 32, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 494, 474, 472, 495, 526, 476, 522, 510, 526, 499, 516, 497, 503, 501, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8068, 8224, 8290, 8203, 8186, 8275, 8019] avg=8192.0000 max=8290 max_rank=3 min=8019 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4352, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 3968, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[479, 545, 506, 496, 479, 532, 471, 541, 502, 545, 517, 479, 539, 515, 510, 535, 475, 560, 496, 518, 498, 529, 505, 524, 564, 484, 536, 493, 498, 551, 534, 491, 492, 546, 497, 483, 505, 464, 506, 535, 498, 499, 509, 516, 530, 506, 512, 497, 470, 543, 519, 487, 537, 521, 499, 506, 517, 474, 537, 535, 509, 522, 544, 511, 496, 512, 495, 522, 493, 519, 539, 542, 486, 498, 512, 514, 550, 510, 467, 493, 547, 524, 493, 500, 526, 483, 513, 494, 533, 554, 500, 531, 500, 503, 484, 524, 504, 505, 501, 540, 517, 513, 520, 517, 502, 516, 490, 539, 527, 507, 480, 477, 510, 542, 524, 532, 512, 514, 537, 521, 502, 524, 507, 469, 517, 475, 552, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[541, 548, 520, 494, 534, 515, 485, 488, 484, 511, 541, 501, 485, 557, 563, 523, 526, 491, 498, 563, 518, 504, 509, 551, 561, 529, 532, 533, 471, 564, 476, 481, 534, 525, 502, 542, 503, 519, 568, 512, 508, 551, 519, 480, 528, 480, 494, 520, 508, 490, 503, 519, 521, 513, 460, 496, 505, 464, 529, 476, 527, 503, 498, 484, 469, 490, 484, 512, 523, 516, 505, 509, 531, 481, 471, 482, 520, 493, 456, 514, 488, 494, 503, 529, 517, 522, 503, 529, 505, 468, 545, 520, 507, 503, 547, 529, 525, 546, 544, 533, 509, 507, 511, 516, 510, 513, 542, 499, 505, 492, 484, 510, 535, 519, 517, 525, 512, 500, 529, 487, 521, 570, 476, 528, 540, 502, 514, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[535, 550, 532, 508, 497, 558, 484, 501, 519, 505, 512, 503, 476, 495, 471, 509, 513, 510, 544, 521, 503, 550, 497, 524, 559, 467, 540, 499, 495, 486, 494, 541, 492, 545, 476, 478, 495, 512, 519, 503, 548, 539, 500, 507, 516, 481, 512, 514, 468, 538, 514, 534, 496, 515, 511, 496, 539, 484, 475, 517, 512, 527, 485, 516, 507, 500, 461, 475, 505, 504, 490, 519, 495, 496, 481, 538, 482, 522, 582, 541, 509, 531, 516, 493, 550, 530, 495, 507, 515, 493, 503, 525, 516, 532, 522, 496, 499, 522, 492, 498, 540, 547, 523, 512, 528, 514, 546, 512, 513, 532, 519, 514, 494, 549, 540, 548, 477, 525, 542, 527, 507, 517, 518, 492, 509, 489, 519, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[548, 519, 480, 493, 532, 510, 543, 521, 517, 487, 526, 486, 521, 499, 518, 522, 530, 495, 527, 537, 506, 505, 482, 541, 529, 507, 509, 511, 499, 531, 525, 550, 513, 542, 501, 521, 557, 499, 547, 520, 536, 539, 485, 479, 526, 539, 477, 510, 517, 560, 486, 519, 496, 534, 533, 504, 516, 499, 521, 513, 477, 483, 518, 517, 469, 502, 505, 557, 537, 513, 479, 493, 520, 534, 524, 516, 475, 522, 526, 491, 477, 493, 546, 507, 513, 497, 526, 497, 477, 493, 528, 483, 473, 528, 560, 525, 519, 521, 444, 526, 520, 501, 500, 512, 503, 539, 504, 512, 517, 471, 523, 554, 524, 517, 474, 530, 521, 497, 500, 508, 525, 482, 491, 479, 500, 511, 537, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[489, 514, 516, 499, 511, 512, 502, 511, 519, 541, 484, 486, 505, 521, 517, 488, 535, 546, 542, 536, 451, 546, 466, 515, 514, 509, 532, 511, 514, 467, 556, 518, 528, 463, 499, 493, 509, 515, 540, 479, 541, 557, 516, 447, 506, 511, 528, 477, 513, 532, 519, 527, 559, 502, 530, 471, 505, 499, 474, 518, 515, 508, 523, 521, 501, 545, 518, 555, 507, 469, 500, 513, 521, 515, 509, 503, 493, 517, 506, 521, 500, 536, 555, 512, 533, 495, 523, 492, 488, 498, 496, 481, 507, 522, 505, 512, 468, 570, 538, 496, 510, 512, 517, 472, 563, 516, 542, 544, 477, 525, 525, 516, 497, 534, 500, 517, 512, 536, 478, 516, 528, 516, 494, 509, 505, 540, 535, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[519, 516, 530, 513, 493, 498, 525, 500, 493, 541, 483, 509, 519, 533, 509, 487, 491, 472, 518, 484, 510, 494, 526, 494, 506, 485, 488, 511, 472, 495, 485, 482, 503, 516, 498, 535, 521, 531, 505, 507, 512, 516, 452, 562, 537, 530, 500, 513, 497, 550, 529, 491, 536, 513, 543, 512, 481, 537, 522, 510, 531, 505, 500, 499, 527, 533, 530, 500, 496, 504, 517, 551, 513, 517, 515, 554, 496, 489, 501, 511, 530, 486, 525, 465, 532, 480, 508, 495, 541, 497, 556, 484, 545, 517, 537, 518, 501, 501, 520, 542, 473, 526, 533, 534, 525, 502, 491, 513, 556, 563, 499, 538, 526, 517, 527, 493, 461, 499, 552, 487, 512, 523, 512, 496, 533, 508, 504, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[503, 504, 524, 515, 483, 548, 499, 493, 510, 573, 478, 554, 514, 507, 565, 522, 510, 490, 528, 499, 512, 511, 508, 485, 539, 504, 513, 519, 546, 503, 565, 515, 491, 498, 513, 490, 515, 511, 498, 482, 487, 513, 489, 507, 516, 536, 507, 544, 521, 494, 483, 527, 522, 484, 505, 475, 553, 482, 475, 504, 500, 548, 544, 515, 512, 535, 495, 512, 523, 494, 504, 497, 513, 497, 517, 505, 492, 506, 526, 531, 480, 513, 520, 520, 550, 500, 502, 517, 521, 516, 500, 529, 498, 513, 535, 500, 527, 510, 530, 519, 522, 537, 500, 510, 535, 488, 472, 464, 535, 547, 506, 536, 532, 474, 493, 549, 491, 523, 501, 523, 518, 493, 510, 517, 518, 482, 527, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[532, 492, 522, 529, 532, 489, 529, 486, 508, 521, 493, 510, 477, 520, 499, 500, 498, 533, 499, 526, 530, 492, 488, 502, 518, 511, 502, 526, 534, 534, 497, 486, 539, 537, 540, 513, 513, 524, 477, 506, 515, 492, 477, 544, 536, 503, 518, 538, 516, 546, 514, 494, 516, 474, 509, 522, 541, 543, 521, 486, 525, 502, 513, 554, 524, 527, 520, 528, 508, 475, 563, 474, 508, 527, 502, 518, 523, 485, 546, 511, 529, 480, 509, 516, 488, 499, 552, 506, 516, 518, 487, 496, 499, 535, 515, 511, 543, 486, 523, 524, 509, 510, 515, 498, 504, 531, 529, 479, 478, 540, 503, 486, 484, 507, 499, 511, 521, 542, 526, 500, 543, 522, 480, 482, 492, 488, 478, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[489, 514, 516, 499, 511, 512, 502, 511, 519, 538, 484, 486, 505, 521, 517, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 546, 542, 536, 451, 546, 466, 515, 497, 509, 532, 511, 514, 467, 556, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 463, 499, 493, 509, 515, 540, 479, 541, 557, 516, 447, 506, 511, 528, 477, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 532, 519, 527, 559, 502, 530, 471, 505, 499, 474, 518, 515, 508, 523, 521, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 545, 518, 555, 507, 469, 500, 513, 521, 515, 509, 503, 493, 517, 506, 521, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 536, 555, 512, 533, 495, 523, 492, 488, 498, 496, 481, 507, 522, 505, 512, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 570, 538, 456, 510, 512, 517, 472, 563, 516, 542, 544, 477, 525, 525, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 534, 500, 517, 512, 536, 478, 516, 528, 516, 494, 509, 505, 540, 535, 482, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8241, 8110, 8228, 8231, 8157, 8251, 8206] avg=8192.0000 max=8251 max_rank=6 min=8110 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[503, 504, 524, 515, 483, 548, 499, 493, 510, 570, 478, 554, 514, 507, 565, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 528, 499, 512, 511, 508, 485, 521, 504, 513, 519, 546, 503, 565, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 513, 490, 515, 511, 498, 482, 487, 513, 489, 507, 516, 536, 507, 544, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 494, 483, 527, 522, 484, 505, 475, 553, 482, 475, 504, 500, 548, 544, 515, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 535, 495, 512, 523, 494, 504, 497, 513, 497, 517, 505, 492, 506, 526, 531, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 513, 520, 520, 550, 500, 502, 517, 521, 516, 500, 529, 498, 513, 535, 500, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 510, 530, 477, 522, 537, 500, 510, 535, 488, 472, 464, 535, 547, 506, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 474, 493, 549, 491, 523, 501, 523, 518, 493, 510, 517, 518, 482, 527, 506, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 8229, 8098, 8145, 8199, 8216, 8196, 8164] avg=8192.0000 max=8289 max_rank=0 min=8098 min_rank=2 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[548, 519, 480, 493, 532, 510, 543, 521, 517, 472, 526, 486, 521, 499, 518, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 495, 527, 537, 506, 505, 482, 541, 503, 507, 509, 511, 499, 531, 525, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 542, 501, 521, 557, 499, 547, 520, 536, 539, 485, 479, 526, 539, 477, 510, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 560, 486, 519, 496, 534, 533, 504, 516, 499, 521, 513, 477, 483, 518, 517, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 502, 505, 557, 537, 513, 479, 493, 520, 534, 524, 516, 475, 522, 526, 491, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 493, 546, 507, 513, 497, 526, 497, 477, 493, 528, 483, 473, 528, 560, 525, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 521, 444, 475, 520, 501, 500, 512, 503, 539, 504, 512, 517, 471, 523, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 517, 474, 530, 521, 497, 500, 508, 525, 482, 491, 479, 500, 511, 537, 498, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8258, 8296, 8214, 8208, 8130, 8115, 8108] avg=8192.0000 max=8296 max_rank=2 min=8108 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[519, 516, 530, 513, 493, 498, 525, 500, 493, 538, 483, 509, 519, 533, 509, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 472, 518, 484, 510, 494, 526, 494, 490, 485, 488, 511, 472, 495, 485, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 498, 535, 521, 531, 505, 507, 512, 516, 452, 562, 537, 530, 500, 513, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 550, 529, 491, 536, 513, 543, 512, 481, 537, 522, 510, 531, 505, 500, 499, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 533, 530, 500, 496, 504, 517, 551, 513, 517, 515, 554, 496, 489, 501, 511, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 486, 525, 465, 532, 480, 508, 495, 541, 497, 556, 484, 545, 517, 537, 518, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 520, 498, 473, 526, 533, 534, 525, 502, 491, 513, 556, 563, 499, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 527, 493, 461, 499, 552, 487, 512, 523, 512, 496, 533, 508, 504, 524, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 7897, 8239, 8268, 8296, 8218, 8273, 8180] avg=8192.0000 max=8296 max_rank=4 min=7897 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32967 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[479, 545, 506, 496, 479, 532, 471, 541, 502, 542, 517, 479, 539, 515, 510, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 560, 496, 518, 498, 529, 505, 524, 546, 484, 536, 493, 498, 551, 534, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 546, 497, 483, 505, 464, 506, 535, 498, 499, 509, 516, 530, 506, 512, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 543, 519, 487, 537, 521, 499, 506, 517, 474, 537, 535, 509, 522, 544, 511, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 495, 522, 493, 519, 539, 542, 486, 498, 512, 514, 550, 510, 467, 493, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 524, 493, 500, 526, 483, 513, 494, 533, 554, 500, 531, 500, 503, 484, 524, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 501, 496, 517, 513, 520, 517, 502, 516, 490, 539, 527, 507, 480, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 542, 524, 532, 512, 514, 537, 521, 502, 524, 507, 469, 517, 475, 552, 513, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8238, 8096, 8244, 8190, 8211, 8111, 8258] avg=8192.0000 max=8258 max_rank=7 min=8096 min_rank=2 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[535, 550, 532, 508, 497, 558, 484, 501, 519, 502, 512, 503, 476, 495, 471, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 544, 521, 503, 550, 497, 524, 541, 467, 540, 499, 495, 486, 494, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 545, 476, 478, 495, 512, 519, 503, 548, 539, 500, 507, 516, 481, 512, 514, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 538, 514, 534, 496, 515, 511, 496, 539, 484, 475, 517, 512, 527, 485, 516, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 461, 475, 505, 504, 490, 519, 495, 496, 481, 538, 482, 522, 582, 541, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 531, 516, 493, 550, 530, 495, 507, 515, 493, 503, 525, 516, 532, 522, 496, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 522, 492, 458, 540, 547, 523, 512, 528, 514, 546, 512, 513, 532, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 549, 540, 548, 477, 525, 542, 527, 507, 517, 518, 492, 509, 489, 519, 479, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8225, 8138, 8140, 8136, 8235, 8271, 8239] avg=8192.0000 max=8271 max_rank=6 min=8136 min_rank=4 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[532, 492, 522, 529, 532, 489, 529, 486, 508, 518, 493, 510, 477, 520, 499, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 533, 499, 526, 530, 492, 488, 502, 501, 511, 502, 526, 534, 534, 497, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 537, 540, 513, 513, 524, 477, 506, 515, 492, 477, 544, 536, 503, 518, 538, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 546, 514, 494, 516, 474, 509, 522, 541, 543, 521, 486, 525, 502, 513, 554, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 520, 528, 508, 475, 563, 474, 508, 527, 502, 518, 523, 485, 546, 511, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 480, 509, 516, 488, 499, 552, 506, 516, 518, 487, 496, 499, 535, 515, 511, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 486, 523, 482, 509, 510, 515, 498, 504, 531, 529, 479, 478, 540, 503, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 507, 499, 511, 521, 542, 526, 500, 543, 522, 480, 482, 492, 488, 478, 545, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8159, 8273, 8288, 8279, 8158, 8116, 8127] avg=8192.0000 max=8288 max_rank=3 min=8116 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[541, 548, 520, 494, 534, 515, 485, 488, 484, 508, 541, 501, 485, 557, 563, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 491, 498, 563, 518, 504, 509, 551, 543, 529, 532, 533, 471, 564, 476, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 525, 502, 542, 503, 519, 568, 512, 508, 551, 519, 480, 528, 480, 494, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 503, 519, 521, 513, 460, 496, 505, 464, 529, 476, 527, 503, 498, 484, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 490, 484, 512, 523, 516, 505, 509, 531, 481, 471, 482, 520, 493, 456, 514, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 494, 503, 529, 517, 522, 503, 529, 505, 468, 545, 520, 507, 503, 547, 529, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 546, 544, 490, 509, 507, 511, 516, 510, 513, 542, 499, 505, 492, 484, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 517, 525, 512, 500, 529, 487, 521, 570, 476, 528, 540, 502, 514, 472, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8289, 8286, 8009, 7997, 8211, 8203, 8254] avg=8192.0000 max=8289 max_rank=1 min=7997 min_rank=4 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4352, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 3840, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[487, 502, 501, 469, 491, 508, 559, 558, 546, 505, 525, 505, 483, 518, 509, 512, 522, 536, 528, 498, 512, 558, 543, 515, 531, 502, 502, 482, 502, 461, 517, 525, 517, 500, 509, 486, 504, 470, 519, 535, 501, 521, 562, 497, 471, 539, 529, 477, 561, 513, 480, 524, 524, 499, 501, 501, 485, 518, 527, 494, 504, 479, 528, 466, 493, 513, 505, 505, 514, 529, 527, 489, 520, 483, 533, 541, 528, 493, 512, 528, 505, 528, 535, 491, 503, 515, 498, 516, 526, 496, 509, 500, 511, 528, 555, 484, 495, 503, 499, 536, 553, 557, 554, 488, 523, 467, 498, 528, 514, 511, 482, 515, 504, 494, 549, 569, 502, 501, 545, 500, 518, 492, 476, 534, 529, 491, 513, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[502, 497, 522, 532, 520, 492, 515, 515, 524, 526, 484, 494, 498, 548, 510, 550, 465, 537, 517, 528, 550, 520, 523, 525, 519, 482, 542, 543, 500, 513, 519, 478, 541, 509, 529, 478, 541, 499, 466, 520, 482, 477, 520, 491, 501, 543, 498, 488, 495, 490, 539, 508, 507, 515, 511, 501, 494, 522, 507, 507, 518, 496, 516, 531, 523, 494, 550, 523, 503, 472, 530, 491, 502, 499, 522, 505, 525, 513, 505, 509, 492, 484, 524, 529, 518, 498, 555, 523, 533, 549, 490, 484, 507, 490, 502, 519, 478, 550, 505, 523, 536, 515, 525, 493, 466, 542, 527, 549, 528, 531, 542, 518, 481, 488, 527, 492, 487, 538, 499, 501, 516, 516, 492, 530, 522, 504, 516, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[534, 534, 502, 523, 514, 511, 462, 506, 535, 505, 524, 520, 506, 497, 516, 540, 489, 527, 490, 508, 541, 526, 472, 538, 526, 524, 505, 494, 514, 511, 514, 525, 500, 468, 520, 489, 543, 486, 544, 488, 505, 547, 491, 530, 547, 551, 498, 553, 511, 518, 496, 538, 539, 511, 496, 510, 528, 486, 542, 501, 536, 522, 490, 504, 498, 490, 512, 483, 541, 508, 487, 510, 510, 518, 475, 529, 523, 538, 520, 517, 526, 525, 477, 529, 525, 517, 489, 537, 527, 522, 533, 500, 560, 533, 473, 542, 512, 522, 517, 545, 484, 504, 520, 491, 513, 511, 529, 457, 473, 512, 556, 511, 501, 552, 523, 470, 455, 484, 489, 520, 482, 516, 526, 518, 487, 487, 477, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[492, 508, 521, 518, 484, 527, 495, 491, 510, 473, 541, 464, 526, 507, 543, 551, 478, 523, 500, 449, 484, 510, 522, 502, 528, 501, 513, 535, 516, 537, 520, 486, 489, 506, 532, 485, 558, 499, 542, 501, 501, 509, 518, 509, 473, 453, 530, 507, 532, 529, 527, 524, 499, 473, 509, 533, 547, 513, 525, 472, 497, 514, 570, 538, 503, 516, 520, 512, 508, 535, 517, 502, 519, 529, 513, 505, 497, 533, 502, 521, 488, 479, 517, 528, 514, 481, 530, 543, 514, 492, 516, 498, 483, 525, 523, 496, 532, 485, 514, 524, 525, 510, 519, 515, 516, 532, 538, 482, 491, 557, 528, 519, 491, 517, 521, 517, 495, 514, 497, 525, 517, 492, 515, 494, 531, 501, 568, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[542, 511, 510, 551, 510, 535, 537, 520, 533, 525, 490, 475, 500, 494, 533, 485, 496, 520, 494, 514, 546, 500, 528, 512, 520, 512, 532, 492, 537, 501, 518, 481, 519, 500, 530, 508, 510, 491, 527, 491, 503, 462, 493, 482, 525, 519, 507, 499, 494, 514, 505, 529, 511, 508, 559, 543, 556, 529, 438, 545, 529, 515, 532, 532, 509, 495, 510, 494, 522, 466, 531, 501, 552, 506, 479, 509, 489, 518, 511, 486, 515, 522, 524, 559, 509, 465, 479, 525, 543, 507, 528, 514, 497, 495, 502, 495, 530, 454, 529, 516, 523, 526, 539, 505, 494, 519, 508, 515, 519, 460, 487, 503, 509, 502, 573, 546, 507, 545, 512, 539, 510, 499, 525, 508, 524, 491, 475, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[532, 479, 528, 511, 500, 536, 494, 541, 519, 518, 514, 472, 522, 506, 506, 519, 506, 539, 488, 542, 518, 518, 494, 516, 519, 517, 501, 510, 492, 512, 514, 508, 529, 518, 480, 504, 539, 552, 537, 527, 522, 545, 495, 501, 532, 499, 498, 526, 527, 499, 532, 493, 539, 507, 515, 486, 494, 488, 520, 516, 521, 519, 494, 524, 518, 526, 534, 495, 504, 483, 496, 490, 484, 503, 545, 532, 531, 531, 516, 521, 494, 509, 516, 505, 495, 520, 513, 486, 507, 481, 527, 518, 516, 495, 504, 494, 516, 476, 549, 504, 500, 509, 487, 486, 506, 488, 524, 532, 536, 498, 540, 492, 545, 489, 498, 511, 540, 491, 484, 540, 509, 498, 519, 511, 483, 539, 517, 561] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[526, 529, 503, 513, 538, 498, 517, 505, 476, 533, 524, 508, 522, 528, 529, 529, 478, 521, 528, 499, 508, 481, 555, 538, 556, 510, 492, 487, 521, 542, 524, 537, 536, 502, 506, 538, 543, 491, 555, 483, 505, 526, 512, 529, 473, 479, 494, 514, 477, 485, 476, 529, 519, 521, 488, 503, 470, 485, 567, 519, 538, 504, 512, 501, 529, 522, 475, 524, 521, 494, 512, 507, 511, 475, 515, 540, 467, 529, 479, 522, 531, 504, 514, 493, 504, 507, 501, 528, 531, 510, 487, 482, 524, 556, 503, 517, 495, 518, 480, 518, 496, 520, 543, 531, 506, 558, 513, 515, 479, 511, 484, 484, 518, 563, 508, 535, 539, 517, 520, 506, 487, 512, 540, 499, 491, 508, 503, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[499, 468, 516, 528, 539, 496, 495, 525, 495, 547, 489, 455, 554, 527, 501, 501, 499, 532, 517, 522, 502, 510, 443, 520, 545, 518, 496, 493, 503, 522, 493, 563, 508, 514, 476, 547, 509, 488, 542, 539, 518, 525, 503, 521, 509, 554, 532, 495, 521, 529, 551, 487, 501, 544, 549, 549, 510, 486, 488, 491, 520, 521, 510, 531, 528, 532, 472, 523, 544, 516, 514, 517, 508, 447, 509, 520, 519, 497, 481, 530, 523, 488, 526, 495, 491, 547, 538, 502, 539, 476, 521, 536, 546, 503, 511, 504, 488, 488, 542, 484, 471, 480, 495, 517, 534, 508, 535, 509, 491, 501, 517, 530, 488, 507, 530, 572, 511, 493, 507, 497, 522, 512, 488, 491, 520, 488, 525, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[487, 502, 501, 469, 491, 508, 559, 558, 546, 505, 525, 505, 483, 518, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 536, 528, 498, 512, 558, 543, 515, 517, 502, 502, 482, 502, 461, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 500, 509, 486, 504, 470, 519, 535, 501, 521, 562, 497, 471, 539, 529, 477, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 513, 480, 524, 524, 499, 501, 501, 485, 518, 527, 494, 486, 479, 528, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 505, 505, 514, 529, 527, 489, 520, 483, 533, 541, 528, 493, 512, 528, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 535, 491, 503, 515, 498, 516, 526, 496, 509, 500, 511, 528, 555, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 503, 499, 536, 553, 557, 554, 488, 523, 467, 498, 528, 514, 511, 482, 515, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 494, 549, 569, 502, 501, 545, 500, 518, 492, 476, 534, 529, 491, 513, 530, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8220, 8149, 8086, 8237, 8200, 8226, 8252] avg=8192.0000 max=8252 max_rank=7 min=8086 min_rank=3 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32915 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[502, 497, 522, 532, 520, 492, 515, 515, 524, 526, 484, 494, 498, 548, 510, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 537, 517, 528, 550, 520, 523, 525, 505, 482, 542, 543, 500, 513, 519, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 509, 529, 478, 541, 499, 466, 520, 482, 477, 520, 491, 501, 543, 498, 488, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 490, 539, 508, 507, 515, 511, 501, 494, 522, 507, 507, 500, 496, 516, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 550, 523, 503, 472, 530, 491, 502, 499, 522, 505, 525, 513, 505, 509, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 484, 524, 529, 518, 498, 555, 523, 533, 549, 490, 484, 507, 490, 502, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 550, 505, 523, 536, 515, 525, 493, 466, 542, 527, 549, 528, 531, 542, 518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 488, 527, 492, 487, 538, 499, 501, 516, 516, 492, 530, 522, 504, 516, 506, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8247, 8096, 8139, 8190, 8197, 8331, 8120] avg=8192.0000 max=8331 max_rank=6 min=8096 min_rank=2 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32838 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[542, 511, 510, 551, 510, 535, 537, 520, 533, 525, 490, 475, 500, 494, 533, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 520, 494, 514, 546, 500, 528, 512, 500, 512, 532, 492, 537, 501, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 530, 508, 510, 491, 527, 491, 503, 462, 493, 482, 525, 519, 507, 499, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 514, 505, 529, 511, 508, 559, 543, 556, 529, 438, 545, 506, 515, 532, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 510, 494, 522, 466, 531, 501, 552, 506, 479, 509, 489, 518, 511, 486, 23, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 524, 559, 509, 465, 479, 525, 543, 507, 528, 514, 497, 495, 502, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 454, 529, 516, 523, 526, 539, 505, 494, 519, 508, 515, 519, 460, 487, 503, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 502, 573, 546, 507, 545, 512, 539, 510, 499, 525, 508, 524, 491, 475, 528, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8183, 8086, 8316, 8109, 8179, 8136, 8301] avg=8192.0000 max=8316 max_rank=3 min=8086 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32725 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[492, 508, 521, 518, 484, 527, 495, 491, 510, 473, 541, 464, 526, 507, 543, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 523, 500, 449, 484, 510, 522, 502, 514, 501, 513, 535, 516, 537, 520, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 506, 532, 485, 558, 499, 542, 501, 501, 509, 518, 509, 473, 453, 530, 507, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 529, 527, 524, 499, 473, 509, 533, 547, 513, 525, 472, 480, 514, 570, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 520, 512, 508, 535, 517, 502, 519, 529, 513, 505, 497, 533, 502, 521, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 479, 517, 528, 514, 481, 530, 543, 514, 492, 516, 498, 483, 525, 523, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 485, 514, 524, 525, 510, 519, 515, 516, 532, 538, 482, 491, 557, 528, 519, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 517, 521, 517, 495, 514, 497, 525, 517, 492, 515, 494, 531, 501, 568, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8090, 8125, 8285, 8255, 8127, 8290, 8226] avg=8192.0000 max=8290 max_rank=6 min=8090 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32898 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[532, 479, 528, 511, 500, 536, 494, 541, 519, 518, 514, 472, 522, 506, 506, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 539, 488, 542, 518, 518, 494, 516, 505, 517, 501, 510, 492, 512, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 518, 480, 504, 539, 552, 537, 527, 522, 545, 495, 501, 532, 499, 498, 526, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 499, 532, 493, 539, 507, 515, 486, 494, 488, 520, 516, 503, 519, 494, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 526, 534, 495, 504, 483, 496, 490, 484, 503, 545, 532, 531, 531, 516, 521, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 509, 516, 505, 495, 520, 513, 486, 507, 481, 527, 518, 516, 495, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 476, 549, 504, 500, 509, 487, 486, 506, 488, 524, 532, 536, 498, 540, 492, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 489, 498, 511, 540, 491, 484, 540, 509, 498, 519, 511, 483, 539, 517, 561, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8180, 8316, 8156, 8233, 8080, 8146, 8240] avg=8192.0000 max=8316 max_rank=2 min=8080 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32699 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[526, 529, 503, 513, 538, 498, 517, 505, 476, 533, 524, 508, 522, 528, 529, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 521, 528, 499, 508, 481, 555, 538, 541, 510, 492, 487, 521, 542, 524, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 502, 506, 538, 543, 491, 555, 483, 505, 526, 512, 529, 473, 479, 494, 514, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 485, 476, 529, 519, 521, 488, 503, 470, 485, 567, 519, 519, 504, 512, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 522, 475, 524, 521, 494, 512, 507, 511, 475, 515, 540, 467, 529, 479, 522, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 514, 493, 504, 507, 501, 528, 531, 510, 487, 482, 524, 556, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 518, 480, 518, 496, 520, 543, 531, 506, 558, 513, 515, 479, 511, 484, 484, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 563, 508, 535, 539, 517, 520, 506, 487, 512, 540, 499, 491, 508, 503, 490, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8262, 8199, 8075, 8147, 8192, 8154, 8241] avg=8192.0000 max=8266 max_rank=0 min=8075 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32734 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[534, 534, 502, 523, 514, 511, 462, 506, 535, 505, 524, 520, 506, 497, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 527, 490, 508, 541, 526, 472, 538, 512, 524, 505, 494, 514, 511, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 468, 520, 489, 543, 486, 544, 488, 505, 547, 491, 530, 547, 551, 498, 553, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 496, 538, 539, 511, 496, 510, 528, 486, 542, 501, 517, 522, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 490, 512, 483, 541, 508, 487, 510, 510, 518, 475, 529, 523, 538, 520, 517, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 477, 529, 525, 517, 489, 537, 527, 522, 533, 500, 560, 533, 473, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 522, 517, 545, 484, 504, 520, 491, 513, 511, 529, 457, 473, 512, 556, 511, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 552, 523, 470, 455, 484, 489, 520, 482, 516, 526, 518, 487, 487, 477, 497, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8190, 8273, 8209, 8184, 8315, 8160, 7989] avg=8192.0000 max=8315 max_rank=5 min=7989 min_rank=7 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32648 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[499, 468, 516, 528, 539, 496, 495, 525, 495, 547, 489, 455, 554, 527, 501, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 517, 522, 502, 510, 443, 520, 531, 518, 496, 493, 503, 522, 493, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 476, 547, 509, 488, 542, 539, 518, 525, 503, 521, 509, 554, 532, 495, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 529, 551, 487, 501, 544, 549, 549, 510, 486, 488, 491, 502, 521, 510, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 532, 472, 523, 544, 516, 514, 517, 508, 447, 509, 520, 519, 497, 481, 530, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 488, 526, 495, 491, 547, 538, 502, 539, 476, 521, 536, 546, 503, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 488, 542, 484, 471, 480, 495, 517, 534, 508, 535, 509, 491, 501, 517, 530, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 530, 572, 511, 493, 507, 497, 522, 512, 488, 491, 520, 488, 525, 511, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8164, 8292, 8270, 8181, 8246, 8093, 8167] avg=8192.0000 max=8292 max_rank=2 min=8093 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32687 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4352, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4352, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True + [2026-08-26 16:29:49] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 416.7 | throughput per GPU (TFLOP/s/GPU): 433.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.430557E+00 | loss scale: 1.0 | grad norm: 5.457 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[507, 498, 560, 520, 534, 510, 527, 487, 533, 550, 512, 553, 521, 511, 494, 526, 506, 516, 482, 504, 562, 526, 543, 511, 476, 510, 497, 512, 510, 517, 478, 503, 477, 482, 499, 493, 505, 487, 511, 487, 492, 512, 506, 505, 574, 546, 527, 500, 510, 522, 503, 499, 503, 491, 511, 497, 553, 549, 472, 492, 483, 504, 530, 521, 515, 504, 516, 530, 512, 512, 504, 502, 530, 533, 501, 543, 491, 523, 503, 507, 545, 508, 504, 482, 535, 515, 533, 521, 502, 536, 490, 467, 546, 506, 519, 545, 495, 524, 523, 518, 525, 531, 489, 501, 499, 498, 548, 532, 499, 471, 492, 456, 535, 488, 527, 516, 524, 520, 469, 513, 537, 503, 523, 477, 532, 522, 523, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[529, 502, 569, 504, 533, 493, 526, 517, 507, 552, 486, 480, 532, 496, 492, 528, 502, 495, 493, 510, 490, 501, 463, 531, 519, 486, 499, 468, 526, 562, 510, 537, 521, 479, 476, 499, 519, 493, 506, 468, 456, 512, 508, 532, 506, 544, 511, 499, 530, 467, 510, 520, 528, 483, 510, 515, 534, 519, 502, 517, 500, 572, 513, 544, 511, 497, 521, 509, 466, 525, 495, 505, 521, 502, 517, 527, 477, 532, 499, 523, 490, 533, 496, 488, 505, 554, 511, 491, 515, 565, 500, 547, 511, 537, 526, 530, 516, 506, 516, 519, 504, 507, 505, 548, 516, 544, 540, 544, 539, 525, 546, 528, 534, 524, 518, 498, 520, 523, 509, 474, 480, 511, 536, 467, 493, 526, 502, 461] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[541, 533, 492, 518, 519, 492, 491, 490, 554, 514, 476, 515, 477, 550, 531, 493, 521, 515, 539, 559, 474, 515, 492, 454, 578, 492, 546, 561, 518, 531, 521, 484, 495, 507, 541, 510, 481, 542, 523, 513, 502, 507, 506, 499, 474, 504, 503, 541, 512, 471, 536, 503, 493, 520, 496, 497, 506, 531, 521, 492, 531, 495, 487, 549, 537, 503, 486, 531, 490, 500, 534, 541, 517, 516, 534, 489, 514, 484, 535, 493, 511, 460, 515, 485, 497, 483, 538, 546, 509, 471, 497, 527, 478, 511, 555, 523, 544, 497, 494, 514, 513, 515, 548, 520, 534, 493, 513, 527, 516, 511, 507, 516, 512, 521, 506, 511, 502, 492, 489, 567, 504, 483, 538, 506, 528, 510, 536, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[482, 556, 503, 506, 492, 545, 491, 510, 538, 509, 515, 508, 511, 486, 511, 505, 564, 472, 536, 514, 522, 519, 512, 465, 538, 519, 508, 509, 513, 551, 502, 495, 550, 494, 517, 519, 500, 543, 492, 526, 526, 507, 511, 523, 556, 500, 535, 515, 482, 511, 508, 529, 478, 504, 502, 537, 492, 545, 501, 497, 501, 526, 500, 504, 499, 517, 499, 538, 492, 517, 512, 491, 503, 521, 545, 508, 485, 488, 535, 520, 524, 486, 494, 514, 504, 529, 529, 510, 506, 516, 516, 511, 515, 525, 504, 535, 458, 504, 553, 548, 532, 506, 513, 535, 502, 465, 485, 530, 503, 499, 457, 570, 529, 508, 513, 499, 492, 517, 502, 532, 501, 513, 492, 508, 503, 515, 491, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[514, 501, 495, 480, 507, 486, 537, 496, 530, 499, 468, 521, 509, 505, 515, 507, 488, 518, 501, 497, 552, 526, 497, 541, 558, 520, 517, 519, 511, 507, 497, 525, 549, 519, 511, 559, 531, 525, 508, 532, 522, 531, 546, 489, 485, 515, 538, 517, 491, 554, 505, 539, 487, 511, 542, 504, 482, 503, 506, 475, 522, 503, 514, 498, 508, 500, 471, 533, 484, 481, 483, 519, 523, 550, 496, 525, 554, 555, 518, 522, 469, 510, 532, 525, 511, 559, 503, 515, 520, 489, 537, 487, 507, 517, 507, 504, 532, 473, 495, 530, 489, 476, 487, 485, 505, 498, 517, 501, 493, 509, 500, 526, 497, 514, 539, 525, 505, 537, 487, 522, 467, 509, 540, 535, 507, 506, 503, 556] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[514, 507, 563, 484, 477, 472, 505, 537, 509, 530, 497, 509, 536, 503, 507, 513, 537, 526, 518, 513, 524, 518, 487, 538, 518, 515, 502, 519, 508, 511, 527, 490, 530, 515, 510, 527, 524, 545, 513, 503, 489, 504, 527, 501, 522, 565, 472, 552, 513, 458, 479, 522, 540, 513, 518, 522, 519, 503, 503, 519, 540, 495, 488, 502, 537, 530, 488, 508, 473, 525, 511, 540, 512, 501, 514, 547, 539, 488, 481, 498, 486, 549, 515, 524, 522, 488, 515, 501, 477, 531, 514, 555, 488, 488, 519, 533, 514, 553, 518, 457, 499, 493, 521, 479, 490, 481, 517, 575, 491, 562, 487, 517, 481, 504, 550, 460, 519, 504, 508, 516, 503, 501, 543, 550, 519, 502, 484, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[523, 472, 526, 503, 504, 533, 526, 528, 535, 528, 517, 506, 502, 516, 506, 504, 502, 502, 507, 469, 514, 513, 499, 533, 520, 475, 460, 488, 493, 516, 512, 522, 546, 510, 471, 529, 471, 521, 521, 525, 490, 496, 494, 478, 536, 521, 516, 499, 498, 524, 522, 509, 489, 513, 543, 506, 503, 501, 522, 535, 522, 486, 487, 506, 512, 528, 549, 509, 511, 538, 511, 535, 491, 515, 503, 464, 493, 538, 486, 503, 501, 463, 505, 474, 478, 518, 546, 493, 489, 517, 531, 525, 523, 522, 514, 526, 555, 549, 510, 508, 555, 552, 470, 548, 484, 497, 512, 515, 512, 500, 534, 498, 480, 488, 511, 551, 531, 553, 545, 514, 508, 533, 534, 522, 557, 521, 513, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[494, 509, 502, 506, 500, 530, 472, 524, 526, 502, 500, 504, 506, 513, 514, 562, 512, 517, 533, 527, 526, 562, 549, 539, 504, 551, 521, 517, 526, 527, 540, 517, 496, 473, 509, 545, 513, 549, 528, 518, 526, 523, 489, 496, 489, 507, 490, 495, 499, 538, 493, 526, 500, 491, 496, 501, 504, 533, 502, 522, 504, 518, 508, 492, 499, 556, 517, 502, 505, 505, 522, 555, 529, 470, 498, 515, 526, 504, 498, 530, 540, 515, 538, 521, 460, 477, 502, 509, 514, 547, 484, 523, 500, 541, 472, 492, 493, 544, 483, 438, 511, 477, 494, 552, 504, 488, 527, 533, 516, 501, 497, 519, 533, 511, 518, 540, 525, 536, 517, 483, 532, 513, 515, 495, 509, 464, 516, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[529, 502, 569, 504, 533, 493, 526, 517, 504, 552, 486, 480, 532, 496, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 493, 510, 490, 501, 463, 531, 519, 486, 499, 468, 526, 527, 510, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 479, 476, 499, 519, 493, 506, 468, 456, 512, 508, 532, 506, 544, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 467, 510, 520, 528, 483, 510, 515, 534, 519, 502, 517, 500, 572, 513, 544, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 521, 509, 466, 525, 495, 503, 521, 502, 517, 527, 477, 532, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 533, 496, 488, 505, 554, 511, 491, 515, 565, 500, 547, 511, 537, 526, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 516, 519, 504, 507, 505, 548, 516, 544, 540, 544, 539, 525, 546, 528, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 524, 518, 498, 520, 523, 509, 474, 480, 511, 531, 467, 493, 526, 502, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8057, 8029, 8304, 8125, 8299, 8408, 8071] avg=8192.0000 max=8408 max_rank=6 min=8029 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=57128 max_load_reduction_pct=0.871704 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[482, 556, 503, 506, 492, 545, 491, 510, 534, 509, 515, 508, 511, 486, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 472, 536, 514, 522, 519, 512, 465, 538, 519, 508, 509, 513, 517, 502, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 494, 517, 519, 500, 543, 492, 526, 526, 507, 511, 523, 556, 500, 535, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 511, 508, 529, 478, 504, 502, 537, 492, 545, 501, 497, 501, 526, 500, 504, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 499, 538, 492, 517, 512, 489, 503, 521, 545, 508, 485, 488, 535, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 486, 494, 514, 504, 529, 529, 510, 506, 516, 516, 511, 515, 525, 504, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 504, 553, 548, 532, 506, 513, 535, 502, 465, 485, 530, 503, 499, 457, 570, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 508, 513, 499, 492, 517, 502, 532, 501, 513, 487, 508, 503, 515, 491, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8205, 8314, 8156, 8168, 8218, 8166, 8145] avg=8192.0000 max=8314 max_rank=2 min=8145 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[541, 533, 492, 518, 519, 492, 491, 490, 550, 514, 476, 515, 477, 550, 531, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 515, 539, 559, 474, 515, 492, 454, 578, 492, 546, 561, 518, 498, 521, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 507, 541, 510, 481, 542, 523, 513, 502, 507, 506, 499, 474, 504, 503, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 471, 536, 503, 493, 520, 496, 497, 506, 531, 521, 492, 531, 495, 487, 549, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 503, 486, 531, 490, 500, 534, 539, 517, 516, 534, 489, 514, 484, 535, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 460, 515, 485, 497, 483, 538, 546, 509, 471, 497, 527, 478, 511, 555, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 497, 494, 514, 513, 515, 548, 520, 534, 493, 513, 527, 516, 511, 507, 516, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 521, 506, 511, 502, 492, 489, 567, 504, 483, 533, 506, 528, 510, 536, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8267, 8148, 8178, 8202, 8106, 8268, 8185] avg=8192.0000 max=8268 max_rank=6 min=8106 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 498, 560, 520, 534, 510, 527, 487, 529, 550, 512, 553, 521, 511, 494, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 482, 504, 562, 526, 543, 511, 476, 510, 497, 512, 510, 485, 478, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 482, 499, 493, 505, 487, 511, 487, 492, 512, 506, 505, 574, 546, 527, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 522, 503, 499, 503, 491, 511, 497, 553, 549, 472, 492, 483, 504, 530, 521, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 516, 530, 512, 512, 504, 500, 530, 533, 501, 543, 491, 523, 503, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 508, 504, 482, 535, 515, 533, 521, 502, 536, 490, 467, 546, 506, 519, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 523, 518, 525, 531, 489, 501, 499, 498, 548, 532, 499, 471, 492, 456, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 488, 527, 516, 524, 520, 469, 513, 537, 503, 518, 477, 532, 522, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8339, 8121, 8103, 8177, 8224, 8254, 8107, 8211] avg=8192.0000 max=8339 max_rank=0 min=8103 min_rank=2 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 501, 495, 480, 507, 486, 537, 496, 526, 499, 468, 521, 509, 505, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 501, 497, 552, 526, 497, 541, 558, 520, 517, 519, 511, 476, 497, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 519, 511, 559, 531, 525, 508, 532, 522, 531, 546, 489, 485, 515, 538, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 554, 505, 539, 487, 511, 542, 504, 482, 503, 506, 475, 522, 503, 514, 498, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 471, 533, 484, 481, 483, 517, 523, 550, 496, 525, 554, 555, 518, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 510, 532, 525, 511, 559, 503, 515, 520, 489, 537, 487, 507, 517, 507, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 473, 495, 530, 489, 476, 487, 485, 505, 498, 517, 501, 493, 509, 500, 526, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 539, 525, 505, 537, 487, 522, 467, 509, 535, 535, 507, 506, 503, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8243, 8377, 8172, 8220, 8192, 8022, 8244] avg=8192.0000 max=8377 max_rank=2 min=8022 min_rank=6 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 472, 526, 503, 504, 533, 526, 528, 531, 528, 517, 506, 502, 516, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 502, 507, 469, 514, 513, 499, 533, 520, 475, 460, 488, 493, 484, 512, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 510, 471, 529, 471, 521, 521, 525, 490, 496, 494, 478, 536, 521, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 522, 509, 489, 513, 543, 506, 503, 501, 522, 535, 522, 486, 487, 506, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 528, 549, 509, 511, 538, 511, 533, 491, 515, 503, 464, 493, 538, 486, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 463, 505, 474, 478, 518, 546, 493, 489, 517, 531, 525, 523, 522, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 549, 510, 508, 555, 552, 470, 548, 484, 497, 512, 515, 512, 500, 534, 498, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 488, 511, 551, 531, 553, 545, 514, 508, 533, 529, 522, 557, 521, 513, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 7993, 8124, 8203, 8184, 8125, 8305, 8377] avg=8192.0000 max=8377 max_rank=7 min=7993 min_rank=1 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32991 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 509, 502, 506, 500, 530, 472, 524, 519, 502, 500, 504, 506, 513, 514, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 533, 527, 526, 562, 549, 539, 504, 551, 521, 517, 526, 490, 540, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 473, 509, 545, 513, 545, 528, 518, 526, 523, 489, 496, 489, 507, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 538, 493, 526, 500, 491, 496, 501, 504, 533, 502, 522, 504, 518, 508, 492, 37, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 556, 517, 502, 505, 505, 522, 547, 529, 470, 498, 515, 526, 504, 498, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 515, 538, 521, 460, 477, 502, 509, 514, 547, 484, 523, 500, 541, 472, 492, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 544, 483, 438, 511, 477, 494, 552, 504, 488, 527, 533, 516, 501, 497, 519, 7, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 511, 518, 540, 525, 536, 517, 483, 532, 513, 497, 495, 509, 464, 516, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8431, 8142, 8175, 8223, 8137, 8101, 8170] avg=8192.0000 max=8431 max_rank=1 min=8101 min_rank=6 imbalance_ratio=1.029175 max_load_reduction=57105 max_load_reduction_pct=0.871353 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 507, 563, 484, 477, 472, 505, 537, 506, 530, 497, 509, 536, 503, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 526, 518, 513, 524, 518, 487, 538, 518, 515, 502, 519, 508, 479, 527, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 515, 510, 527, 524, 545, 513, 503, 489, 504, 527, 501, 522, 565, 472, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 458, 479, 522, 540, 513, 518, 522, 519, 503, 503, 519, 540, 495, 488, 502, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 488, 508, 473, 525, 511, 538, 512, 501, 514, 547, 539, 488, 481, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 549, 515, 524, 522, 488, 515, 501, 477, 531, 514, 555, 488, 488, 519, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 553, 518, 457, 499, 493, 521, 479, 490, 481, 517, 575, 491, 562, 487, 517, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 550, 460, 519, 504, 508, 516, 503, 501, 538, 550, 519, 502, 484, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8219, 8299, 8171, 8190, 8205, 8159, 8133] avg=8192.0000 max=8299 max_rank=2 min=8133 min_rank=7 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 3968, 4224, 4352, 4096, 4096, 4096, 4224, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[523, 473, 514, 488, 510, 506, 528, 524, 531, 520, 521, 507, 510, 520, 527, 558, 461, 541, 492, 546, 503, 544, 501, 488, 528, 463, 535, 525, 519, 502, 501, 535, 500, 462, 540, 479, 533, 495, 480, 507, 503, 516, 543, 511, 531, 469, 494, 541, 507, 535, 546, 523, 515, 505, 497, 536, 498, 510, 475, 501, 515, 464, 551, 523, 533, 492, 514, 487, 513, 489, 510, 522, 538, 492, 526, 532, 476, 496, 508, 524, 517, 530, 507, 535, 464, 524, 547, 523, 515, 470, 524, 503, 512, 494, 534, 490, 542, 517, 492, 485, 545, 492, 557, 510, 485, 513, 534, 529, 502, 551, 494, 506, 483, 521, 512, 523, 489, 534, 517, 505, 503, 498, 485, 546, 486, 491, 552, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[531, 545, 487, 508, 491, 544, 503, 531, 492, 539, 492, 516, 525, 529, 501, 506, 487, 508, 520, 523, 508, 491, 508, 529, 501, 495, 501, 487, 515, 481, 533, 485, 488, 474, 521, 507, 500, 536, 500, 536, 507, 492, 491, 525, 505, 502, 513, 566, 536, 540, 501, 514, 508, 547, 551, 516, 509, 495, 528, 536, 514, 532, 537, 468, 537, 510, 507, 502, 488, 507, 512, 479, 539, 486, 527, 514, 538, 579, 544, 501, 524, 471, 520, 480, 513, 498, 531, 531, 562, 532, 515, 455, 544, 518, 519, 548, 498, 488, 526, 474, 502, 550, 505, 481, 510, 488, 558, 499, 508, 512, 523, 538, 509, 494, 480, 487, 502, 549, 494, 505, 472, 515, 490, 505, 524, 511, 493, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[540, 495, 526, 515, 518, 539, 467, 558, 530, 515, 506, 491, 533, 519, 500, 513, 481, 560, 486, 541, 514, 530, 511, 551, 460, 508, 513, 476, 493, 474, 490, 540, 510, 500, 524, 503, 521, 515, 537, 536, 520, 523, 488, 501, 545, 528, 493, 505, 492, 524, 513, 460, 467, 503, 548, 506, 540, 521, 535, 521, 492, 525, 502, 520, 512, 549, 583, 514, 516, 476, 512, 531, 508, 489, 509, 513, 512, 490, 529, 516, 515, 516, 525, 503, 511, 514, 495, 520, 511, 499, 505, 508, 562, 485, 513, 523, 563, 505, 502, 509, 528, 537, 530, 487, 525, 478, 502, 506, 531, 500, 466, 509, 472, 510, 464, 498, 500, 496, 486, 501, 557, 530, 516, 485, 537, 557, 473, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[471, 504, 518, 505, 476, 554, 520, 540, 522, 532, 532, 523, 511, 506, 503, 525, 516, 499, 514, 539, 534, 522, 517, 507, 533, 541, 499, 475, 472, 506, 520, 510, 571, 502, 532, 506, 545, 474, 517, 514, 522, 511, 509, 510, 476, 510, 532, 543, 503, 545, 510, 511, 523, 486, 483, 491, 523, 502, 540, 488, 531, 492, 490, 504, 477, 521, 526, 512, 500, 517, 511, 529, 526, 465, 498, 518, 560, 529, 513, 504, 507, 503, 516, 531, 480, 532, 504, 525, 521, 476, 518, 489, 510, 496, 514, 539, 495, 496, 493, 509, 547, 536, 526, 483, 509, 504, 507, 524, 513, 521, 507, 542, 494, 528, 506, 523, 482, 515, 505, 527, 494, 514, 522, 508, 451, 541, 492, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[496, 540, 497, 494, 484, 512, 530, 492, 551, 529, 509, 474, 508, 519, 536, 517, 510, 548, 518, 499, 569, 494, 527, 519, 532, 474, 517, 515, 506, 533, 503, 530, 527, 525, 510, 488, 525, 483, 522, 474, 532, 484, 514, 504, 489, 540, 493, 577, 493, 488, 530, 501, 486, 486, 510, 522, 530, 528, 526, 547, 534, 515, 517, 482, 514, 517, 517, 529, 507, 497, 498, 498, 492, 548, 512, 469, 459, 518, 488, 529, 501, 489, 514, 495, 497, 506, 511, 548, 488, 501, 510, 525, 515, 516, 519, 491, 497, 502, 540, 535, 516, 529, 513, 529, 515, 524, 502, 485, 537, 499, 513, 552, 491, 515, 510, 532, 482, 483, 545, 546, 514, 517, 499, 495, 486, 525, 520, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[468, 532, 548, 527, 476, 523, 473, 468, 541, 528, 541, 508, 507, 508, 503, 541, 476, 479, 516, 515, 496, 507, 527, 508, 560, 526, 523, 516, 466, 554, 522, 498, 519, 508, 512, 514, 519, 529, 522, 505, 527, 527, 490, 506, 527, 523, 468, 470, 512, 515, 542, 497, 472, 500, 527, 538, 512, 495, 523, 493, 500, 476, 513, 507, 543, 493, 504, 513, 461, 535, 514, 505, 481, 522, 494, 554, 530, 487, 495, 494, 495, 492, 505, 548, 541, 486, 485, 492, 512, 546, 512, 526, 520, 486, 520, 510, 524, 514, 521, 498, 524, 527, 461, 548, 509, 533, 525, 519, 529, 495, 495, 492, 477, 527, 536, 572, 494, 471, 533, 508, 575, 497, 522, 531, 536, 569, 487, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[476, 514, 568, 521, 534, 492, 516, 528, 503, 517, 508, 500, 503, 507, 533, 509, 535, 523, 514, 521, 499, 511, 497, 562, 532, 510, 526, 519, 533, 543, 502, 510, 503, 520, 511, 489, 509, 543, 522, 502, 515, 519, 519, 516, 550, 489, 494, 506, 511, 472, 503, 520, 482, 522, 512, 522, 510, 517, 529, 502, 488, 485, 496, 482, 497, 486, 510, 519, 539, 510, 503, 520, 482, 481, 515, 499, 543, 499, 511, 487, 509, 527, 506, 519, 537, 501, 498, 518, 525, 521, 489, 522, 525, 489, 518, 552, 529, 521, 488, 502, 502, 501, 538, 517, 521, 483, 547, 530, 512, 506, 515, 518, 474, 505, 542, 501, 510, 515, 495, 473, 528, 521, 491, 485, 480, 507, 545, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[491, 500, 521, 527, 497, 514, 522, 504, 531, 526, 520, 482, 495, 497, 529, 514, 476, 539, 514, 498, 497, 503, 518, 505, 542, 492, 469, 511, 504, 499, 519, 525, 515, 484, 479, 514, 519, 503, 573, 549, 497, 507, 471, 499, 526, 506, 482, 525, 492, 514, 529, 494, 500, 525, 481, 475, 520, 498, 547, 519, 561, 544, 492, 510, 508, 523, 515, 505, 511, 536, 520, 535, 520, 507, 516, 519, 542, 508, 514, 493, 506, 548, 529, 505, 521, 519, 548, 505, 502, 537, 530, 491, 503, 532, 482, 476, 501, 522, 471, 541, 486, 510, 518, 539, 501, 526, 522, 548, 502, 528, 507, 548, 516, 493, 498, 509, 532, 510, 530, 501, 495, 543, 522, 522, 470, 512, 502, 474] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[540, 495, 526, 515, 518, 539, 467, 558, 530, 485, 506, 491, 533, 519, 500, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 560, 486, 541, 514, 530, 511, 551, 460, 508, 513, 476, 493, 474, 490, 540, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 500, 524, 503, 521, 515, 537, 536, 520, 523, 488, 501, 545, 528, 493, 505, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 524, 513, 460, 467, 503, 548, 506, 540, 521, 535, 521, 492, 525, 502, 520, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 549, 583, 514, 516, 476, 512, 531, 508, 489, 509, 513, 512, 490, 529, 516, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 516, 525, 503, 511, 514, 495, 520, 511, 499, 505, 508, 554, 485, 513, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 505, 502, 509, 528, 537, 530, 487, 525, 478, 502, 506, 531, 500, 466, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 510, 464, 498, 500, 496, 486, 501, 557, 530, 516, 485, 537, 557, 473, 501, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8130, 8253, 8186, 8266, 8197, 8146, 8123] avg=8192.0000 max=8266 max_rank=4 min=8123 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[468, 532, 548, 527, 476, 523, 473, 468, 541, 498, 541, 508, 507, 508, 503, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 479, 516, 515, 496, 507, 527, 508, 560, 526, 523, 516, 466, 554, 522, 498, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 512, 514, 519, 529, 522, 505, 527, 527, 490, 506, 527, 523, 468, 470, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 542, 497, 472, 500, 527, 538, 512, 495, 523, 493, 500, 476, 513, 507, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 493, 504, 513, 461, 535, 514, 505, 481, 522, 494, 554, 530, 487, 495, 494, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 492, 505, 548, 541, 486, 485, 492, 512, 546, 512, 526, 512, 486, 520, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 521, 498, 524, 527, 461, 548, 509, 533, 525, 519, 529, 495, 495, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 527, 536, 572, 494, 471, 533, 508, 575, 497, 522, 531, 536, 569, 487, 517, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8191, 8170, 8139, 8132, 8168, 8183, 8391] avg=8192.0000 max=8391 max_rank=7 min=8132 min_rank=4 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 545, 487, 508, 491, 544, 503, 531, 492, 499, 492, 516, 525, 529, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 508, 520, 523, 508, 491, 508, 529, 501, 495, 501, 487, 515, 481, 533, 485, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 474, 521, 507, 500, 536, 500, 536, 507, 492, 491, 525, 505, 502, 513, 566, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 540, 501, 514, 508, 547, 551, 516, 509, 495, 528, 536, 514, 532, 537, 468, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 507, 502, 488, 507, 512, 479, 539, 486, 527, 514, 538, 579, 544, 501, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 471, 520, 480, 513, 498, 531, 531, 562, 532, 515, 455, 528, 518, 519, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 488, 526, 474, 502, 550, 505, 481, 510, 488, 558, 499, 508, 512, 523, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 480, 487, 502, 549, 494, 505, 472, 515, 490, 505, 524, 511, 493, 508, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8075, 8172, 8354, 8280, 8245, 8122, 8088] avg=8192.0000 max=8354 max_rank=3 min=8075 min_rank=1 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[476, 514, 568, 521, 534, 492, 516, 528, 503, 487, 508, 500, 503, 507, 533, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 523, 514, 521, 499, 511, 497, 562, 532, 510, 526, 519, 533, 543, 502, 510, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 520, 511, 489, 509, 543, 522, 502, 515, 519, 519, 516, 550, 489, 494, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 472, 503, 520, 482, 522, 512, 522, 510, 517, 529, 502, 488, 485, 496, 482, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 486, 510, 519, 539, 510, 503, 520, 482, 481, 515, 499, 543, 499, 511, 487, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 506, 519, 537, 501, 498, 518, 525, 521, 489, 522, 517, 489, 518, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 521, 488, 502, 502, 501, 538, 517, 521, 483, 547, 530, 512, 506, 515, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 505, 542, 501, 510, 515, 495, 473, 528, 521, 491, 485, 480, 507, 545, 551, 33, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8339, 8211, 8070, 8108, 8248, 8197, 8164] avg=8192.0000 max=8339 max_rank=1 min=8070 min_rank=3 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 473, 514, 488, 510, 506, 528, 524, 531, 490, 521, 507, 510, 520, 527, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 541, 492, 546, 503, 544, 501, 488, 528, 463, 535, 525, 519, 502, 501, 535, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 462, 540, 479, 533, 495, 480, 507, 503, 516, 543, 511, 531, 469, 494, 541, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 535, 546, 523, 515, 505, 497, 536, 498, 510, 475, 501, 515, 464, 551, 523, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 492, 514, 487, 513, 489, 510, 522, 538, 492, 526, 532, 476, 496, 508, 524, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 530, 507, 535, 464, 524, 547, 523, 515, 470, 524, 503, 504, 494, 534, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 517, 492, 485, 545, 492, 557, 510, 485, 513, 534, 529, 502, 551, 494, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 521, 512, 523, 489, 534, 517, 505, 503, 498, 485, 546, 486, 491, 552, 547, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8186, 8108, 8218, 8159, 8181, 8222, 8232] avg=8192.0000 max=8232 max_rank=7 min=8108 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 540, 497, 494, 484, 512, 530, 492, 551, 499, 509, 474, 508, 519, 536, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 548, 518, 499, 569, 494, 527, 519, 532, 474, 517, 515, 506, 533, 503, 530, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 510, 488, 525, 483, 522, 474, 532, 484, 514, 504, 489, 540, 493, 577, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 488, 530, 501, 486, 486, 510, 522, 530, 528, 526, 547, 534, 515, 517, 482, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 517, 517, 529, 507, 497, 498, 498, 492, 548, 512, 469, 459, 518, 488, 529, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 489, 514, 495, 497, 506, 511, 548, 488, 501, 510, 525, 507, 516, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 540, 535, 516, 529, 513, 529, 515, 524, 502, 485, 537, 499, 513, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 510, 532, 482, 483, 545, 546, 514, 517, 499, 495, 486, 525, 520, 506, 35, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8296, 8191, 8212, 8099, 8118, 8253, 8209] avg=8192.0000 max=8296 max_rank=1 min=8099 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[471, 504, 518, 505, 476, 554, 520, 540, 522, 502, 532, 523, 511, 506, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 514, 539, 534, 522, 517, 507, 533, 541, 499, 475, 472, 506, 520, 510, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 502, 532, 506, 545, 474, 517, 514, 522, 511, 509, 510, 476, 510, 532, 543, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 545, 510, 511, 523, 486, 483, 491, 523, 502, 540, 488, 531, 492, 490, 504, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 521, 526, 512, 500, 517, 511, 529, 526, 465, 498, 518, 560, 529, 513, 504, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 516, 531, 480, 532, 504, 525, 521, 476, 518, 489, 502, 496, 514, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 493, 509, 547, 536, 526, 483, 509, 504, 507, 524, 513, 521, 507, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 528, 506, 523, 482, 515, 505, 527, 494, 514, 522, 508, 451, 541, 492, 513, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8206, 8278, 8139, 8213, 8153, 8178, 8157] avg=8192.0000 max=8278 max_rank=2 min=8139 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[491, 500, 521, 527, 497, 514, 522, 504, 531, 496, 520, 482, 495, 497, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 539, 514, 498, 497, 503, 518, 505, 542, 492, 469, 511, 504, 499, 519, 525, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 484, 479, 514, 519, 503, 573, 549, 497, 507, 471, 499, 526, 506, 482, 525, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 514, 529, 494, 500, 525, 481, 475, 520, 498, 547, 519, 561, 544, 492, 510, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 523, 515, 505, 511, 536, 520, 535, 520, 507, 516, 519, 542, 508, 514, 493, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 548, 529, 505, 521, 519, 548, 505, 502, 537, 530, 491, 495, 532, 482, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 522, 471, 541, 486, 510, 518, 539, 501, 526, 522, 548, 502, 528, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 498, 509, 532, 510, 530, 501, 495, 543, 522, 522, 470, 512, 502, 474, 35, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8113, 8153, 8218, 8279, 8226, 8235, 8172] avg=8192.0000 max=8279 max_rank=4 min=8113 min_rank=1 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[521, 520, 517, 522, 514, 479, 519, 474, 523, 521, 583, 498, 506, 502, 500, 548, 539, 494, 507, 523, 555, 513, 487, 522, 550, 488, 520, 496, 579, 535, 515, 532, 539, 498, 521, 476, 499, 494, 544, 499, 517, 514, 530, 498, 496, 489, 505, 490, 501, 486, 495, 520, 485, 541, 516, 497, 495, 464, 504, 509, 532, 513, 525, 502, 502, 501, 520, 504, 511, 555, 533, 486, 544, 505, 539, 539, 553, 526, 512, 494, 505, 492, 538, 462, 472, 519, 529, 506, 507, 522, 509, 497, 538, 486, 494, 545, 492, 522, 511, 494, 479, 530, 456, 525, 533, 504, 504, 506, 506, 529, 483, 494, 517, 511, 542, 545, 502, 504, 491, 504, 543, 476, 511, 470, 495, 541, 539, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[549, 495, 540, 499, 561, 534, 529, 500, 506, 547, 496, 497, 517, 527, 532, 535, 500, 508, 473, 476, 503, 503, 549, 513, 523, 503, 497, 520, 514, 537, 541, 467, 534, 508, 511, 504, 541, 492, 483, 513, 531, 497, 486, 487, 494, 542, 516, 503, 527, 481, 507, 509, 519, 531, 527, 517, 473, 506, 535, 522, 498, 511, 505, 479, 501, 507, 506, 470, 516, 533, 505, 483, 513, 543, 520, 500, 541, 560, 467, 506, 502, 483, 520, 524, 488, 500, 490, 492, 525, 526, 507, 487, 496, 483, 532, 517, 547, 522, 520, 504, 519, 546, 536, 500, 478, 514, 488, 511, 513, 519, 507, 548, 515, 501, 517, 506, 514, 521, 501, 549, 515, 536, 541, 520, 499, 510, 515, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[511, 505, 515, 502, 508, 504, 529, 570, 490, 466, 535, 538, 493, 585, 516, 482, 509, 491, 491, 493, 529, 523, 518, 496, 499, 544, 504, 512, 528, 479, 526, 498, 514, 516, 497, 477, 488, 511, 489, 542, 512, 548, 506, 504, 537, 506, 531, 510, 508, 501, 535, 528, 503, 496, 501, 539, 476, 494, 518, 526, 549, 504, 518, 506, 527, 478, 510, 505, 505, 493, 517, 540, 521, 515, 548, 482, 549, 530, 495, 507, 483, 518, 528, 515, 491, 500, 488, 500, 479, 517, 513, 558, 523, 489, 507, 505, 528, 501, 503, 510, 505, 535, 489, 491, 506, 499, 523, 493, 523, 457, 539, 524, 460, 530, 535, 537, 532, 511, 526, 532, 532, 513, 491, 517, 518, 540, 511, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[493, 497, 521, 498, 524, 546, 466, 494, 477, 535, 528, 484, 544, 497, 508, 541, 539, 525, 510, 480, 533, 501, 539, 540, 496, 498, 510, 509, 520, 482, 509, 482, 552, 505, 502, 484, 545, 535, 507, 513, 523, 521, 514, 518, 533, 521, 524, 499, 544, 513, 492, 512, 540, 489, 503, 509, 546, 524, 495, 531, 539, 504, 529, 490, 471, 539, 479, 489, 500, 508, 491, 470, 499, 535, 531, 497, 474, 517, 522, 480, 538, 525, 481, 516, 535, 570, 502, 531, 513, 482, 524, 508, 526, 499, 514, 518, 516, 526, 459, 492, 521, 509, 528, 498, 520, 534, 505, 532, 538, 544, 543, 491, 515, 497, 484, 504, 510, 530, 492, 508, 491, 523, 496, 520, 536, 522, 506, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[495, 507, 478, 531, 503, 513, 519, 532, 484, 509, 503, 529, 531, 523, 515, 494, 514, 529, 519, 519, 501, 496, 508, 492, 520, 465, 505, 523, 511, 535, 478, 483, 475, 526, 556, 529, 491, 508, 515, 550, 506, 502, 505, 513, 462, 509, 504, 543, 501, 503, 515, 532, 525, 494, 561, 509, 531, 503, 487, 545, 496, 518, 515, 511, 502, 503, 519, 484, 525, 512, 539, 531, 500, 525, 513, 472, 531, 514, 521, 492, 507, 525, 525, 548, 530, 541, 519, 521, 494, 519, 533, 508, 517, 481, 522, 511, 485, 528, 526, 521, 563, 521, 497, 480, 524, 508, 494, 500, 495, 489, 532, 504, 525, 512, 534, 494, 535, 538, 517, 491, 513, 517, 485, 478, 516, 509, 490, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[558, 527, 512, 475, 516, 560, 489, 484, 524, 501, 497, 531, 506, 527, 489, 528, 540, 538, 476, 512, 496, 480, 506, 520, 525, 498, 532, 509, 538, 561, 559, 538, 487, 474, 512, 463, 523, 526, 504, 515, 444, 552, 574, 534, 470, 534, 479, 548, 536, 518, 557, 504, 534, 584, 465, 520, 519, 510, 506, 531, 535, 486, 489, 530, 448, 523, 474, 536, 519, 498, 511, 496, 496, 507, 514, 531, 529, 483, 491, 504, 559, 514, 503, 492, 521, 529, 491, 527, 491, 491, 464, 540, 515, 501, 520, 466, 509, 492, 536, 531, 540, 518, 515, 456, 503, 533, 501, 472, 505, 551, 549, 526, 490, 487, 531, 516, 499, 477, 535, 555, 458, 502, 494, 507, 502, 501, 562, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[557, 510, 520, 473, 519, 563, 526, 501, 497, 511, 508, 502, 509, 545, 504, 490, 519, 499, 518, 494, 530, 483, 523, 494, 506, 521, 537, 544, 559, 498, 529, 491, 479, 515, 521, 507, 539, 490, 526, 504, 512, 473, 532, 489, 517, 501, 517, 544, 509, 475, 500, 532, 479, 539, 519, 496, 527, 504, 557, 503, 513, 504, 521, 497, 525, 517, 477, 530, 461, 503, 549, 526, 523, 503, 537, 499, 489, 547, 484, 541, 521, 509, 478, 502, 487, 463, 536, 522, 525, 497, 517, 493, 504, 536, 506, 508, 496, 518, 535, 497, 524, 550, 522, 533, 532, 505, 503, 472, 519, 500, 513, 496, 506, 529, 498, 530, 523, 541, 513, 531, 508, 495, 493, 495, 490, 511, 512, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[521, 495, 481, 489, 488, 496, 523, 502, 519, 493, 546, 512, 513, 536, 533, 528, 550, 547, 532, 540, 457, 505, 498, 484, 517, 498, 522, 572, 516, 490, 455, 543, 483, 546, 519, 510, 506, 496, 494, 520, 481, 494, 517, 534, 523, 542, 565, 512, 491, 543, 495, 469, 478, 522, 502, 484, 515, 502, 537, 502, 543, 521, 549, 502, 528, 489, 511, 518, 484, 530, 505, 515, 523, 548, 496, 486, 523, 530, 531, 502, 450, 517, 536, 512, 506, 506, 510, 490, 523, 526, 512, 512, 523, 531, 496, 496, 503, 533, 522, 484, 490, 535, 558, 509, 533, 532, 489, 482, 508, 512, 497, 460, 524, 510, 548, 527, 515, 548, 536, 501, 499, 492, 546, 503, 465, 487, 500, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[549, 495, 540, 499, 561, 534, 529, 500, 506, 547, 496, 497, 517, 494, 532, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 473, 476, 503, 503, 549, 513, 523, 503, 497, 520, 496, 537, 541, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 511, 504, 541, 492, 483, 513, 531, 497, 486, 487, 494, 542, 516, 503, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 481, 507, 509, 519, 531, 527, 517, 473, 506, 535, 522, 492, 511, 505, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 506, 470, 516, 533, 505, 483, 513, 543, 520, 500, 541, 560, 467, 506, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 483, 520, 524, 488, 500, 490, 492, 525, 526, 507, 487, 496, 483, 532, 517, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 522, 520, 504, 519, 546, 536, 500, 478, 514, 488, 511, 513, 519, 507, 548, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 513, 506, 514, 521, 501, 549, 515, 536, 541, 520, 499, 510, 515, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8331, 8109, 8149, 8141, 8187, 8105, 8277, 8237] avg=8192.0000 max=8331 max_rank=0 min=8105 min_rank=5 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 520, 517, 522, 514, 479, 519, 474, 523, 521, 583, 498, 506, 470, 500, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 494, 507, 523, 555, 513, 487, 522, 550, 488, 520, 496, 559, 535, 515, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 498, 521, 476, 499, 494, 544, 499, 517, 514, 530, 498, 496, 489, 505, 490, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 486, 495, 520, 485, 541, 516, 497, 495, 464, 504, 509, 526, 513, 525, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 520, 504, 511, 555, 533, 486, 544, 505, 539, 539, 553, 526, 512, 494, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 492, 538, 462, 472, 519, 529, 506, 507, 522, 509, 497, 538, 486, 494, 545, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 511, 494, 479, 530, 456, 525, 533, 504, 504, 506, 506, 529, 483, 494, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 538, 545, 502, 504, 491, 504, 543, 476, 511, 470, 495, 541, 539, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8335, 8116, 8079, 8342, 8153, 8073, 8223] avg=8192.0000 max=8342 max_rank=4 min=8073 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[558, 527, 512, 475, 516, 560, 489, 484, 524, 501, 497, 531, 506, 494, 489, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 538, 476, 512, 496, 480, 506, 520, 525, 498, 532, 509, 519, 561, 559, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 474, 512, 463, 523, 526, 504, 515, 444, 552, 574, 534, 470, 534, 479, 548, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 518, 557, 504, 534, 584, 465, 520, 519, 510, 506, 531, 529, 486, 489, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 523, 474, 536, 519, 498, 511, 496, 496, 507, 514, 531, 529, 483, 491, 504, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 514, 503, 492, 521, 529, 491, 527, 491, 491, 464, 540, 515, 501, 520, 466, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 492, 536, 531, 540, 518, 515, 456, 503, 533, 501, 472, 505, 551, 549, 526, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 487, 527, 516, 499, 477, 535, 555, 458, 502, 494, 507, 502, 501, 562, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8309, 8146, 8318, 8077, 8157, 8242, 8096] avg=8192.0000 max=8318 max_rank=3 min=8077 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[493, 497, 521, 498, 524, 546, 466, 494, 477, 535, 528, 484, 544, 456, 508, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 525, 510, 480, 533, 501, 539, 540, 496, 498, 510, 509, 496, 482, 509, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 505, 502, 484, 545, 535, 507, 513, 523, 521, 514, 518, 533, 521, 524, 499, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 513, 492, 512, 540, 489, 503, 509, 546, 524, 495, 531, 523, 504, 529, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 539, 479, 489, 500, 508, 491, 470, 499, 535, 531, 497, 474, 517, 522, 480, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 525, 481, 516, 535, 570, 502, 531, 513, 482, 524, 508, 526, 499, 514, 518, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 526, 459, 492, 521, 509, 528, 498, 520, 534, 505, 532, 538, 544, 543, 491, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 478, 504, 510, 530, 492, 508, 491, 523, 496, 520, 536, 522, 506, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8149, 8307, 8244, 8028, 8318, 8270, 8108] avg=8192.0000 max=8318 max_rank=5 min=8028 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 507, 478, 531, 503, 513, 519, 532, 484, 509, 503, 529, 531, 490, 515, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 519, 519, 501, 496, 508, 492, 520, 465, 505, 523, 493, 535, 478, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 526, 556, 529, 491, 508, 515, 550, 506, 502, 505, 513, 462, 509, 504, 543, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 503, 515, 532, 525, 494, 561, 509, 531, 503, 487, 545, 490, 518, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 503, 519, 484, 525, 512, 539, 531, 500, 525, 513, 472, 531, 514, 521, 492, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 525, 548, 530, 541, 519, 521, 494, 519, 533, 508, 517, 481, 522, 511, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 528, 526, 521, 563, 521, 497, 480, 524, 508, 494, 500, 495, 489, 532, 504, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 512, 530, 494, 535, 538, 517, 491, 513, 517, 485, 478, 516, 509, 490, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8080, 8201, 8240, 8199, 8334, 8172, 8177] avg=8192.0000 max=8334 max_rank=5 min=8080 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[511, 505, 515, 502, 508, 504, 529, 570, 490, 466, 535, 538, 493, 548, 516, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 491, 493, 529, 523, 518, 496, 499, 544, 504, 512, 509, 479, 526, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 516, 497, 477, 488, 511, 489, 542, 512, 548, 506, 504, 537, 506, 531, 510, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 535, 528, 503, 496, 501, 539, 476, 494, 518, 526, 543, 504, 518, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 478, 510, 505, 505, 493, 517, 540, 521, 515, 548, 482, 549, 530, 495, 507, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 528, 515, 491, 500, 488, 500, 479, 517, 513, 558, 523, 489, 507, 505, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 501, 503, 510, 505, 535, 489, 491, 506, 499, 523, 493, 523, 457, 539, 524, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 530, 531, 537, 532, 511, 526, 532, 532, 513, 491, 517, 518, 540, 511, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8121, 8195, 8196, 8239, 8151, 8131, 8291] avg=8192.0000 max=8291 max_rank=7 min=8121 min_rank=1 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[557, 510, 520, 473, 519, 563, 526, 501, 497, 511, 508, 502, 509, 511, 504, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 518, 494, 530, 483, 523, 494, 506, 521, 537, 544, 539, 498, 529, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 515, 521, 507, 539, 490, 526, 504, 512, 473, 532, 489, 517, 501, 517, 544, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 475, 500, 532, 479, 539, 519, 496, 527, 504, 557, 503, 507, 504, 521, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 517, 477, 530, 461, 503, 549, 526, 523, 503, 537, 499, 489, 547, 484, 541, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 509, 478, 502, 487, 463, 536, 522, 525, 497, 517, 493, 504, 536, 506, 508, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 518, 535, 497, 524, 550, 522, 533, 532, 505, 503, 472, 519, 500, 513, 496, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 495, 530, 523, 541, 513, 531, 508, 495, 493, 495, 490, 511, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8225, 8173, 8169, 8229, 8138, 8219, 8182] avg=8192.0000 max=8229 max_rank=4 min=8138 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 495, 481, 489, 488, 496, 523, 502, 519, 493, 546, 512, 513, 502, 533, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 547, 532, 540, 457, 505, 498, 484, 517, 498, 522, 572, 498, 490, 455, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 546, 519, 510, 506, 496, 494, 520, 481, 494, 517, 534, 523, 542, 565, 512, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 543, 495, 469, 478, 522, 502, 484, 515, 502, 537, 502, 537, 521, 549, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 489, 511, 518, 484, 530, 505, 515, 523, 548, 496, 486, 523, 530, 531, 502, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 517, 536, 512, 506, 506, 510, 490, 523, 526, 512, 512, 523, 531, 496, 496, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 533, 522, 484, 490, 535, 558, 509, 533, 532, 489, 482, 508, 512, 497, 460, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 544, 527, 515, 548, 536, 501, 499, 492, 546, 503, 465, 487, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8208, 8249, 8149, 8235, 8180, 8152, 8222] avg=8192.0000 max=8249 max_rank=2 min=8141 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4352, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[471, 485, 525, 533, 495, 489, 487, 518, 553, 537, 543, 501, 483, 487, 533, 477, 506, 560, 504, 491, 540, 512, 565, 523, 560, 497, 515, 521, 511, 551, 500, 508, 544, 494, 497, 531, 496, 489, 467, 543, 538, 488, 564, 565, 507, 507, 480, 531, 529, 515, 505, 510, 489, 536, 514, 531, 470, 524, 514, 492, 528, 505, 502, 515, 528, 473, 494, 534, 549, 493, 502, 482, 522, 546, 536, 487, 501, 497, 526, 494, 509, 532, 524, 459, 511, 484, 511, 506, 481, 479, 507, 527, 493, 512, 529, 510, 534, 485, 506, 470, 514, 490, 508, 491, 532, 489, 503, 498, 516, 572, 528, 494, 512, 490, 502, 540, 483, 534, 525, 546, 504, 528, 530, 520, 538, 515, 514, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[504, 512, 475, 475, 462, 442, 527, 547, 552, 525, 538, 550, 511, 521, 521, 470, 496, 480, 509, 523, 498, 515, 517, 544, 506, 522, 522, 475, 486, 526, 497, 523, 537, 507, 516, 507, 530, 500, 553, 498, 474, 546, 551, 535, 507, 524, 502, 539, 524, 544, 527, 503, 491, 575, 503, 503, 545, 567, 493, 538, 498, 507, 498, 492, 512, 487, 503, 511, 516, 481, 518, 489, 538, 492, 514, 552, 519, 503, 504, 526, 502, 507, 525, 531, 526, 519, 504, 514, 537, 471, 503, 477, 488, 513, 509, 509, 488, 482, 531, 499, 520, 505, 497, 506, 497, 544, 459, 513, 538, 557, 524, 517, 507, 475, 545, 483, 503, 481, 525, 513, 535, 528, 491, 542, 484, 523, 524, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[506, 539, 488, 522, 517, 527, 501, 520, 546, 506, 510, 559, 509, 471, 564, 508, 519, 482, 531, 500, 515, 487, 517, 506, 511, 516, 510, 567, 512, 507, 499, 491, 531, 537, 525, 480, 471, 485, 494, 509, 498, 494, 515, 516, 477, 515, 502, 518, 489, 528, 492, 526, 528, 499, 500, 515, 505, 478, 459, 503, 502, 536, 518, 551, 493, 496, 548, 503, 541, 528, 493, 506, 467, 532, 515, 502, 502, 535, 506, 507, 500, 501, 490, 518, 535, 497, 519, 521, 510, 492, 486, 533, 537, 510, 501, 535, 539, 519, 510, 512, 536, 536, 524, 517, 487, 511, 517, 496, 486, 542, 536, 478, 544, 543, 553, 513, 483, 482, 497, 551, 544, 540, 495, 512, 514, 492, 507, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[518, 521, 515, 497, 520, 523, 510, 526, 493, 560, 516, 522, 490, 534, 501, 485, 540, 518, 526, 525, 548, 512, 529, 517, 492, 492, 532, 505, 527, 497, 494, 521, 506, 535, 463, 534, 509, 530, 507, 525, 490, 526, 465, 512, 504, 543, 520, 510, 467, 481, 467, 533, 494, 507, 497, 497, 476, 510, 471, 513, 565, 528, 552, 520, 514, 503, 536, 521, 521, 530, 538, 513, 543, 520, 495, 517, 524, 492, 517, 512, 532, 501, 493, 506, 510, 492, 506, 504, 508, 520, 526, 487, 526, 501, 520, 524, 510, 529, 518, 498, 530, 528, 473, 491, 511, 583, 494, 546, 505, 493, 526, 483, 518, 490, 536, 489, 520, 487, 514, 507, 510, 502, 492, 510, 503, 484, 524, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[461, 473, 500, 490, 534, 487, 494, 518, 516, 553, 496, 515, 498, 517, 465, 548, 509, 488, 521, 508, 529, 523, 491, 497, 515, 472, 506, 580, 541, 512, 495, 516, 541, 522, 505, 535, 513, 500, 501, 532, 521, 538, 514, 513, 519, 487, 524, 501, 540, 533, 516, 501, 519, 499, 506, 533, 533, 501, 462, 517, 543, 490, 515, 502, 507, 521, 491, 500, 517, 512, 507, 553, 488, 532, 530, 473, 517, 534, 474, 504, 500, 505, 483, 519, 485, 478, 510, 465, 548, 505, 552, 523, 536, 546, 555, 514, 515, 509, 506, 494, 536, 480, 492, 494, 498, 517, 541, 520, 580, 495, 525, 527, 504, 544, 520, 529, 528, 509, 457, 506, 511, 502, 529, 513, 510, 506, 508, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[520, 516, 502, 520, 486, 477, 482, 514, 507, 485, 531, 497, 556, 536, 529, 508, 508, 486, 484, 528, 497, 502, 520, 536, 503, 526, 530, 539, 506, 509, 498, 499, 475, 537, 523, 507, 484, 495, 500, 502, 519, 494, 540, 522, 495, 519, 509, 533, 509, 530, 519, 479, 512, 490, 473, 515, 537, 523, 482, 522, 499, 519, 559, 531, 503, 480, 527, 511, 514, 506, 508, 487, 526, 524, 505, 493, 542, 535, 494, 508, 543, 537, 530, 525, 501, 535, 502, 514, 482, 489, 514, 507, 483, 526, 506, 504, 523, 504, 524, 461, 552, 497, 503, 495, 516, 543, 542, 486, 470, 543, 508, 577, 526, 522, 520, 516, 520, 539, 499, 485, 524, 482, 506, 490, 527, 547, 557, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[536, 542, 502, 509, 513, 513, 500, 556, 492, 480, 535, 504, 491, 483, 495, 529, 523, 503, 502, 512, 518, 537, 485, 485, 494, 515, 489, 532, 489, 502, 493, 534, 480, 508, 476, 505, 503, 477, 515, 516, 514, 515, 565, 500, 520, 541, 493, 481, 504, 490, 504, 505, 534, 539, 499, 529, 510, 506, 516, 524, 527, 515, 528, 512, 491, 522, 492, 539, 507, 517, 558, 519, 495, 494, 498, 514, 528, 529, 514, 563, 525, 518, 497, 448, 482, 521, 521, 552, 542, 515, 522, 503, 518, 485, 579, 504, 523, 486, 520, 494, 520, 520, 494, 464, 536, 518, 498, 540, 490, 552, 502, 529, 514, 526, 503, 526, 501, 516, 526, 513, 483, 505, 528, 512, 501, 522, 521, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[540, 498, 490, 486, 489, 553, 521, 503, 511, 496, 523, 527, 512, 507, 496, 515, 490, 471, 501, 502, 521, 492, 524, 523, 529, 495, 520, 514, 483, 485, 472, 501, 483, 504, 533, 529, 536, 539, 530, 540, 515, 550, 492, 509, 523, 532, 514, 529, 497, 514, 504, 545, 513, 525, 497, 494, 467, 480, 544, 557, 496, 540, 514, 524, 471, 509, 502, 533, 530, 524, 499, 490, 537, 517, 502, 492, 518, 530, 475, 505, 507, 505, 541, 490, 528, 508, 534, 501, 530, 489, 495, 516, 530, 497, 547, 525, 491, 515, 496, 512, 491, 555, 565, 499, 526, 458, 562, 537, 509, 496, 536, 513, 499, 527, 516, 459, 495, 541, 511, 496, 529, 564, 519, 556, 493, 492, 471, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[471, 485, 525, 533, 495, 489, 487, 518, 553, 537, 543, 501, 483, 487, 533, 477, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 560, 504, 491, 540, 512, 565, 523, 560, 497, 515, 521, 511, 551, 500, 508, 6, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 494, 497, 531, 496, 489, 467, 543, 538, 488, 543, 565, 507, 507, 480, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 515, 505, 510, 489, 536, 514, 531, 470, 524, 514, 492, 528, 505, 500, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 473, 494, 534, 549, 493, 502, 482, 522, 546, 536, 487, 501, 497, 526, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 532, 524, 459, 511, 484, 511, 506, 481, 479, 507, 527, 493, 512, 529, 510, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 485, 506, 470, 514, 490, 508, 491, 532, 489, 503, 498, 516, 553, 528, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 501, 540, 483, 534, 525, 546, 504, 528, 530, 520, 538, 515, 514, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8373, 8220, 8177, 8164, 8085, 8111, 8266] avg=8192.0000 max=8373 max_rank=1 min=8085 min_rank=5 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32626 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[504, 512, 475, 475, 462, 442, 527, 547, 552, 525, 538, 550, 511, 521, 521, 470, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 480, 509, 523, 498, 515, 517, 544, 506, 522, 522, 475, 486, 526, 497, 523, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 516, 507, 530, 500, 553, 498, 474, 546, 530, 535, 507, 524, 502, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 544, 527, 503, 491, 575, 503, 503, 545, 567, 493, 538, 498, 507, 496, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 487, 503, 511, 516, 481, 518, 489, 538, 492, 514, 552, 519, 503, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 525, 531, 526, 519, 504, 514, 537, 471, 503, 477, 488, 513, 509, 509, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 482, 531, 499, 520, 505, 497, 506, 497, 544, 459, 513, 538, 539, 524, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 475, 544, 483, 503, 481, 525, 513, 535, 528, 491, 542, 484, 523, 524, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8147, 8305, 8306, 8165, 8146, 8159, 8153] avg=8192.0000 max=8306 max_rank=3 min=8146 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32623 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 542, 502, 509, 513, 513, 500, 556, 492, 480, 535, 504, 491, 483, 495, 529, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 502, 512, 518, 537, 485, 485, 494, 515, 489, 532, 489, 502, 493, 534, 9, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 508, 476, 505, 503, 477, 515, 516, 514, 515, 541, 500, 520, 541, 493, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 490, 504, 505, 534, 539, 499, 529, 510, 506, 516, 524, 527, 515, 522, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 522, 492, 539, 507, 517, 558, 519, 495, 494, 498, 514, 528, 529, 514, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 497, 448, 482, 521, 521, 552, 542, 515, 522, 503, 518, 485, 579, 504, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 486, 520, 494, 520, 520, 494, 464, 536, 518, 498, 540, 490, 521, 502, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 526, 497, 526, 501, 516, 526, 513, 483, 505, 528, 512, 501, 522, 521, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8134, 8085, 8236, 8280, 8247, 8155, 8188] avg=8192.0000 max=8280 max_rank=4 min=8085 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32870 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[461, 473, 500, 490, 534, 487, 494, 518, 516, 553, 496, 515, 498, 517, 465, 548, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 488, 521, 508, 529, 523, 491, 497, 515, 472, 506, 580, 541, 512, 495, 516, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 522, 505, 535, 513, 500, 501, 532, 521, 538, 494, 513, 519, 487, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 533, 516, 501, 519, 499, 506, 533, 533, 501, 462, 517, 543, 490, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 521, 491, 500, 517, 512, 507, 553, 488, 532, 530, 473, 517, 534, 474, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 505, 483, 519, 485, 478, 510, 465, 548, 505, 552, 523, 536, 546, 555, 514, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 506, 494, 536, 480, 492, 494, 498, 517, 541, 520, 580, 478, 525, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 544, 519, 529, 528, 509, 457, 506, 511, 502, 529, 513, 510, 506, 508, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8211, 8246, 8208, 8160, 8234, 8212, 8178] avg=8192.0000 max=8246 max_rank=2 min=8087 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32784 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[506, 539, 488, 522, 517, 527, 501, 520, 546, 506, 510, 559, 509, 471, 564, 508, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 482, 531, 500, 515, 487, 517, 506, 511, 516, 510, 567, 512, 507, 499, 491, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 537, 525, 480, 471, 485, 494, 509, 498, 494, 495, 516, 477, 515, 502, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 528, 492, 526, 528, 499, 500, 515, 505, 478, 459, 503, 502, 536, 516, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 496, 548, 503, 541, 528, 493, 506, 467, 532, 515, 502, 502, 535, 506, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 501, 490, 518, 535, 497, 519, 521, 510, 492, 486, 533, 537, 510, 501, 535, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 519, 510, 512, 536, 536, 524, 517, 487, 511, 517, 496, 486, 524, 536, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 543, 552, 513, 483, 482, 497, 551, 544, 540, 495, 512, 514, 492, 507, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8178, 8047, 8127, 8174, 8196, 8228, 8271] avg=8192.0000 max=8315 max_rank=0 min=8047 min_rank=2 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32869 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 516, 502, 520, 486, 477, 482, 514, 507, 485, 531, 497, 556, 536, 529, 508, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 486, 484, 528, 497, 502, 520, 536, 503, 526, 530, 539, 506, 509, 498, 499, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 537, 523, 507, 484, 495, 500, 502, 519, 494, 519, 522, 495, 519, 509, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 530, 519, 479, 512, 490, 473, 515, 537, 523, 482, 522, 499, 519, 557, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 480, 527, 511, 514, 506, 508, 487, 526, 524, 505, 493, 542, 535, 494, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 537, 530, 525, 501, 535, 502, 514, 482, 489, 514, 507, 483, 526, 506, 504, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 504, 524, 461, 552, 497, 503, 495, 516, 543, 542, 486, 470, 525, 508, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 522, 519, 516, 520, 539, 499, 485, 524, 482, 506, 490, 527, 547, 557, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8179, 8133, 8197, 8163, 8209, 8226, 8240] avg=8192.0000 max=8240 max_rank=7 min=8133 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32838 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 521, 515, 497, 520, 523, 510, 526, 493, 560, 516, 522, 490, 534, 501, 485, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 518, 526, 525, 548, 512, 529, 517, 492, 492, 532, 505, 527, 497, 494, 521, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 535, 463, 534, 509, 530, 507, 525, 490, 526, 447, 512, 504, 543, 520, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 481, 467, 533, 494, 507, 497, 497, 476, 510, 471, 513, 565, 528, 550, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 503, 536, 521, 521, 530, 538, 513, 543, 520, 495, 517, 524, 492, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 501, 493, 506, 510, 492, 506, 504, 508, 520, 526, 487, 526, 501, 520, 524, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 529, 518, 498, 530, 528, 473, 491, 511, 583, 494, 546, 505, 476, 526, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 490, 535, 489, 520, 487, 514, 507, 510, 502, 492, 510, 503, 484, 524, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8283, 8161, 8076, 8296, 8166, 8201, 8102] avg=8192.0000 max=8296 max_rank=4 min=8076 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32765 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[540, 498, 490, 486, 489, 553, 521, 503, 511, 496, 523, 527, 512, 507, 496, 515, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 471, 501, 502, 521, 492, 524, 523, 529, 495, 520, 514, 483, 485, 472, 501, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 504, 533, 529, 536, 539, 530, 540, 515, 550, 473, 509, 523, 532, 514, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 504, 545, 513, 525, 497, 494, 467, 480, 544, 557, 496, 540, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 509, 502, 533, 530, 524, 499, 490, 537, 517, 502, 492, 518, 530, 475, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 505, 541, 490, 528, 508, 534, 501, 530, 489, 495, 516, 530, 497, 547, 525, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 496, 512, 491, 555, 565, 499, 526, 458, 562, 537, 509, 479, 536, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 527, 515, 459, 495, 541, 511, 496, 529, 564, 519, 556, 493, 492, 471, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8031, 8339, 8209, 8134, 8253, 8244, 8138] avg=8192.0000 max=8339 max_rank=2 min=8031 min_rank=1 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32769 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4096, 4352, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 3968, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4352, 4096, 4096, 3968, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[496, 531, 537, 493, 514, 514, 511, 531, 495, 498, 510, 495, 521, 499, 540, 542, 498, 528, 489, 491, 510, 512, 497, 515, 487, 522, 520, 498, 516, 542, 499, 508, 507, 545, 516, 500, 553, 501, 496, 523, 504, 512, 524, 483, 510, 479, 511, 547, 528, 489, 495, 517, 542, 507, 489, 527, 526, 507, 480, 528, 476, 486, 553, 522, 521, 519, 478, 499, 483, 519, 467, 537, 526, 511, 571, 506, 529, 551, 516, 534, 516, 498, 480, 552, 526, 533, 512, 530, 520, 496, 543, 484, 524, 519, 494, 520, 503, 517, 518, 535, 505, 500, 517, 504, 517, 524, 530, 512, 502, 515, 502, 501, 519, 486, 483, 533, 481, 515, 524, 501, 466, 507, 512, 511, 515, 526, 514, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[497, 519, 534, 522, 486, 504, 485, 492, 518, 510, 542, 527, 544, 474, 497, 518, 525, 521, 509, 550, 494, 505, 522, 508, 512, 525, 502, 554, 510, 537, 523, 517, 516, 505, 488, 497, 528, 495, 492, 521, 526, 515, 506, 504, 513, 494, 521, 518, 514, 532, 522, 522, 521, 528, 543, 505, 518, 532, 499, 498, 503, 510, 522, 501, 523, 516, 488, 521, 512, 528, 514, 459, 519, 518, 481, 514, 516, 547, 526, 523, 475, 504, 540, 463, 501, 480, 514, 512, 530, 498, 489, 530, 525, 478, 520, 483, 527, 508, 498, 521, 549, 493, 470, 490, 536, 533, 503, 532, 487, 483, 493, 498, 535, 513, 529, 523, 550, 539, 533, 464, 567, 525, 505, 480, 483, 464, 533, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[497, 517, 498, 508, 550, 505, 525, 540, 499, 510, 487, 513, 534, 529, 522, 509, 496, 495, 487, 566, 519, 476, 504, 491, 511, 472, 493, 535, 501, 525, 537, 517, 522, 528, 483, 506, 495, 509, 513, 495, 507, 505, 490, 475, 497, 509, 474, 527, 515, 495, 498, 514, 519, 539, 483, 502, 547, 521, 495, 524, 507, 500, 534, 531, 502, 514, 516, 509, 504, 523, 553, 492, 507, 500, 525, 553, 538, 518, 505, 497, 527, 529, 541, 506, 530, 516, 486, 522, 520, 480, 503, 505, 556, 533, 485, 515, 507, 525, 530, 515, 531, 548, 532, 538, 541, 489, 507, 506, 519, 489, 505, 490, 561, 473, 499, 497, 499, 519, 532, 490, 477, 500, 511, 537, 519, 515, 488, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[475, 501, 497, 536, 500, 461, 497, 508, 544, 524, 545, 544, 521, 518, 483, 556, 542, 518, 515, 547, 493, 502, 495, 499, 527, 521, 478, 508, 515, 475, 477, 464, 550, 531, 466, 492, 508, 498, 500, 557, 532, 523, 546, 506, 505, 501, 534, 522, 503, 530, 477, 485, 514, 478, 529, 518, 519, 502, 529, 521, 494, 544, 520, 558, 500, 545, 498, 519, 514, 508, 481, 505, 511, 514, 544, 484, 531, 510, 505, 528, 489, 519, 503, 501, 524, 502, 515, 530, 484, 544, 505, 566, 491, 520, 506, 500, 524, 525, 518, 475, 485, 547, 526, 498, 523, 511, 499, 511, 516, 480, 527, 484, 561, 487, 510, 514, 517, 529, 499, 516, 508, 496, 511, 530, 521, 466, 515, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[526, 533, 498, 520, 515, 507, 466, 510, 521, 496, 500, 531, 535, 499, 545, 536, 512, 457, 510, 528, 497, 496, 510, 489, 493, 532, 539, 526, 549, 474, 514, 518, 521, 528, 455, 530, 516, 538, 481, 520, 492, 496, 507, 517, 516, 516, 502, 529, 510, 514, 509, 566, 535, 519, 528, 495, 491, 513, 518, 491, 508, 492, 505, 512, 500, 518, 525, 534, 508, 521, 508, 543, 480, 535, 505, 505, 487, 510, 496, 475, 497, 527, 510, 519, 538, 506, 527, 520, 479, 480, 558, 527, 489, 507, 481, 552, 507, 505, 549, 531, 520, 501, 525, 477, 493, 488, 511, 521, 524, 518, 507, 531, 484, 491, 535, 526, 516, 518, 536, 524, 499, 512, 483, 517, 494, 510, 511, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[523, 493, 526, 504, 495, 518, 510, 516, 471, 506, 547, 495, 539, 533, 498, 512, 529, 472, 520, 526, 492, 541, 474, 470, 526, 506, 489, 539, 489, 500, 547, 490, 503, 505, 493, 525, 513, 514, 534, 502, 490, 491, 509, 558, 529, 544, 467, 510, 527, 516, 509, 535, 517, 498, 487, 562, 526, 533, 529, 485, 524, 490, 529, 489, 498, 483, 489, 497, 492, 530, 520, 516, 483, 519, 527, 509, 491, 511, 529, 493, 474, 511, 521, 486, 536, 518, 566, 511, 533, 512, 526, 524, 502, 543, 502, 535, 511, 516, 512, 454, 507, 504, 519, 556, 512, 487, 518, 517, 526, 528, 515, 509, 531, 507, 568, 528, 508, 506, 484, 517, 527, 515, 519, 510, 479, 480, 534, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[485, 471, 510, 500, 513, 498, 552, 511, 513, 514, 513, 548, 500, 476, 520, 497, 491, 511, 497, 481, 512, 531, 503, 509, 528, 496, 509, 498, 515, 512, 480, 497, 493, 529, 515, 478, 497, 513, 473, 544, 516, 534, 511, 499, 526, 517, 546, 521, 515, 553, 514, 522, 513, 508, 506, 497, 487, 527, 512, 544, 510, 516, 549, 488, 505, 504, 469, 509, 508, 528, 540, 484, 522, 503, 501, 518, 522, 564, 512, 539, 503, 523, 528, 534, 487, 500, 497, 525, 531, 516, 517, 532, 491, 475, 499, 552, 490, 538, 526, 536, 529, 496, 519, 503, 504, 524, 529, 505, 523, 522, 513, 499, 524, 475, 497, 499, 561, 505, 473, 498, 514, 524, 524, 518, 509, 532, 521, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[542, 493, 489, 502, 478, 548, 499, 484, 491, 519, 491, 540, 478, 506, 524, 507, 519, 535, 490, 504, 521, 518, 494, 508, 477, 523, 499, 492, 550, 444, 488, 490, 502, 526, 541, 507, 523, 528, 493, 568, 541, 522, 493, 510, 525, 488, 523, 518, 524, 532, 567, 518, 526, 506, 529, 472, 488, 537, 557, 487, 528, 535, 506, 476, 533, 502, 551, 537, 541, 486, 484, 536, 499, 554, 514, 483, 521, 511, 521, 488, 509, 527, 510, 504, 471, 504, 486, 537, 502, 520, 533, 505, 500, 538, 551, 528, 474, 500, 516, 473, 489, 500, 504, 516, 544, 516, 550, 525, 502, 521, 508, 553, 524, 517, 486, 509, 522, 510, 506, 511, 521, 504, 479, 494, 504, 522, 513, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[496, 531, 537, 493, 514, 514, 511, 531, 495, 498, 510, 495, 521, 499, 540, 542, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 528, 489, 491, 510, 512, 497, 515, 487, 522, 520, 498, 516, 542, 499, 508, 47, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 545, 516, 500, 553, 501, 496, 523, 504, 512, 524, 483, 510, 479, 511, 547, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 489, 495, 517, 542, 507, 489, 527, 526, 507, 480, 528, 476, 486, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 478, 499, 483, 519, 467, 537, 526, 511, 571, 506, 529, 537, 516, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 498, 480, 552, 526, 533, 512, 511, 520, 496, 543, 484, 524, 519, 494, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 518, 535, 505, 500, 517, 504, 511, 524, 530, 512, 502, 515, 502, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 486, 483, 533, 481, 515, 524, 501, 466, 507, 512, 511, 515, 526, 514, 485, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8198, 8212, 8125, 8253, 8228, 8196, 8092] avg=8192.0000 max=8253 max_rank=4 min=8092 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[485, 471, 510, 500, 513, 498, 552, 511, 513, 514, 513, 548, 500, 476, 520, 497, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 511, 497, 481, 512, 531, 503, 509, 528, 496, 509, 498, 515, 512, 480, 497, 47, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 529, 515, 478, 497, 513, 473, 544, 516, 534, 511, 499, 526, 517, 546, 521, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 553, 514, 522, 513, 508, 506, 497, 487, 527, 512, 544, 510, 516, 502, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 469, 509, 508, 528, 540, 484, 522, 503, 501, 518, 522, 550, 512, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 523, 528, 534, 487, 500, 497, 507, 531, 516, 517, 532, 491, 475, 499, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 538, 526, 536, 529, 496, 519, 503, 498, 524, 529, 505, 523, 522, 513, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 475, 497, 499, 561, 505, 473, 498, 514, 524, 524, 518, 509, 532, 521, 504, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8135, 8213, 8214, 8214, 8192, 8250, 8192] avg=8192.0000 max=8250 max_rank=6 min=8126 min_rank=0 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 493, 526, 504, 495, 518, 510, 516, 471, 506, 547, 495, 539, 533, 498, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 472, 520, 526, 492, 541, 474, 470, 526, 506, 489, 539, 489, 500, 547, 490, 45, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 493, 525, 513, 514, 534, 502, 490, 491, 509, 558, 529, 544, 467, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 516, 509, 535, 517, 498, 487, 562, 526, 533, 529, 485, 524, 490, 484, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 483, 489, 497, 492, 530, 520, 516, 483, 519, 527, 509, 491, 498, 529, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 511, 521, 486, 536, 518, 566, 493, 533, 512, 526, 524, 502, 543, 502, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 516, 512, 454, 507, 504, 519, 556, 506, 487, 518, 517, 526, 528, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 507, 568, 528, 508, 506, 484, 517, 527, 515, 519, 510, 479, 480, 534, 506, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8173, 8188, 8211, 8074, 8282, 8185, 8232] avg=8192.0000 max=8282 max_rank=5 min=8074 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[497, 517, 498, 508, 550, 505, 525, 540, 499, 510, 487, 513, 534, 529, 522, 509, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 487, 566, 519, 476, 504, 491, 511, 472, 493, 535, 501, 525, 537, 517, 45, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 483, 506, 495, 509, 513, 495, 507, 505, 490, 475, 497, 509, 474, 527, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 495, 498, 514, 519, 539, 483, 502, 547, 521, 495, 524, 507, 500, 489, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 516, 509, 504, 523, 553, 492, 507, 500, 525, 553, 538, 505, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 529, 541, 506, 530, 516, 486, 504, 520, 480, 503, 505, 556, 533, 485, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 530, 515, 531, 548, 532, 538, 535, 489, 507, 506, 519, 489, 505, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 473, 499, 497, 499, 519, 532, 490, 477, 500, 511, 537, 519, 515, 488, 510, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8188, 8036, 8179, 8243, 8236, 8266, 8140] avg=8192.0000 max=8266 max_rank=6 min=8036 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[497, 519, 534, 522, 486, 504, 485, 492, 518, 510, 542, 527, 544, 474, 497, 518, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 521, 509, 550, 494, 505, 522, 508, 512, 525, 502, 554, 510, 537, 523, 517, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 505, 488, 497, 528, 495, 492, 521, 526, 515, 506, 504, 513, 494, 521, 518, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 532, 522, 522, 521, 528, 543, 505, 518, 532, 499, 498, 503, 510, 478, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 488, 521, 512, 528, 514, 459, 519, 518, 481, 514, 516, 533, 526, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 504, 540, 463, 501, 480, 514, 494, 530, 498, 489, 530, 525, 478, 520, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 508, 498, 521, 549, 493, 470, 490, 530, 533, 503, 532, 487, 483, 493, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 513, 529, 523, 550, 539, 533, 464, 567, 525, 505, 480, 483, 464, 533, 533, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8376, 8140, 8226, 8191, 8024, 8115, 8290] avg=8192.0000 max=8376 max_rank=1 min=8024 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[475, 501, 497, 536, 500, 461, 497, 508, 544, 524, 545, 544, 521, 518, 483, 556, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 515, 547, 493, 502, 495, 499, 527, 521, 478, 508, 515, 475, 477, 464, 44, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 531, 466, 492, 508, 498, 500, 557, 532, 523, 546, 506, 505, 501, 534, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 530, 477, 485, 514, 478, 529, 518, 519, 502, 529, 521, 494, 544, 476, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 545, 498, 519, 514, 508, 481, 505, 511, 514, 544, 484, 531, 497, 505, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 519, 503, 501, 524, 502, 515, 511, 484, 544, 505, 566, 491, 520, 506, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 518, 475, 485, 547, 526, 498, 517, 511, 499, 511, 516, 480, 527, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 487, 510, 514, 517, 529, 499, 516, 508, 496, 511, 530, 521, 466, 515, 533, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8139, 8272, 8177, 8184, 8180, 8143, 8226] avg=8192.0000 max=8272 max_rank=2 min=8139 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[526, 533, 498, 520, 515, 507, 466, 510, 521, 496, 500, 531, 535, 499, 545, 536, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 457, 510, 528, 497, 496, 510, 489, 493, 532, 539, 526, 549, 474, 514, 518, 47, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 528, 455, 530, 516, 538, 481, 520, 492, 496, 507, 517, 516, 516, 502, 529, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 514, 509, 566, 535, 519, 528, 495, 491, 513, 518, 491, 508, 492, 458, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 518, 525, 534, 508, 521, 508, 543, 480, 535, 505, 505, 487, 489, 496, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 527, 510, 519, 538, 506, 527, 491, 479, 480, 558, 527, 489, 507, 481, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 505, 549, 531, 520, 501, 525, 477, 481, 488, 511, 521, 524, 518, 507, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 491, 535, 526, 516, 518, 536, 524, 499, 512, 483, 517, 494, 510, 511, 553, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8213, 8166, 8159, 8129, 8188, 8196, 8231] avg=8192.0000 max=8254 max_rank=0 min=8129 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[542, 493, 489, 502, 478, 548, 499, 484, 491, 519, 491, 540, 478, 506, 524, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 535, 490, 504, 521, 518, 494, 508, 477, 523, 499, 492, 550, 444, 488, 490, 43, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 526, 541, 507, 523, 528, 493, 568, 541, 522, 493, 510, 525, 488, 523, 518, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 532, 567, 518, 526, 506, 529, 472, 488, 537, 557, 487, 528, 535, 463, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 502, 551, 537, 541, 486, 484, 536, 499, 554, 514, 483, 521, 498, 521, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 510, 504, 471, 504, 486, 518, 502, 520, 533, 505, 500, 538, 551, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 500, 516, 473, 489, 500, 504, 516, 538, 516, 550, 525, 502, 521, 508, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 517, 486, 509, 522, 510, 506, 511, 521, 504, 479, 494, 504, 522, 513, 498, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8114, 8309, 8245, 8248, 8206, 8185, 8133] avg=8192.0000 max=8309 max_rank=2 min=8096 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4352, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4096, 4096, 4352, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[507, 503, 539, 504, 498, 534, 538, 511, 544, 515, 503, 514, 531, 548, 496, 535, 509, 525, 484, 515, 514, 511, 512, 526, 493, 557, 491, 478, 513, 576, 519, 527, 536, 483, 538, 526, 541, 513, 513, 503, 514, 506, 502, 483, 500, 482, 488, 488, 521, 529, 523, 537, 496, 505, 534, 538, 479, 500, 501, 494, 541, 475, 505, 478, 499, 532, 522, 469, 512, 522, 487, 527, 503, 507, 528, 515, 545, 531, 546, 462, 477, 507, 506, 503, 521, 529, 510, 487, 492, 486, 522, 509, 524, 542, 519, 514, 497, 559, 498, 515, 479, 502, 501, 496, 509, 493, 500, 493, 515, 510, 518, 500, 484, 524, 494, 508, 529, 496, 520, 501, 524, 531, 515, 531, 560, 506, 485, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[503, 506, 484, 517, 521, 522, 484, 472, 478, 442, 544, 495, 532, 473, 521, 522, 509, 493, 500, 536, 523, 500, 541, 484, 469, 500, 505, 541, 566, 523, 481, 502, 586, 531, 555, 518, 474, 519, 498, 531, 492, 516, 513, 526, 543, 485, 524, 491, 488, 524, 487, 507, 476, 511, 523, 551, 521, 533, 476, 515, 494, 520, 498, 485, 513, 526, 472, 540, 507, 555, 517, 486, 510, 485, 490, 503, 536, 518, 500, 524, 517, 499, 538, 500, 499, 513, 511, 525, 509, 471, 534, 536, 529, 556, 497, 547, 555, 533, 542, 511, 521, 554, 510, 493, 547, 468, 454, 552, 532, 516, 534, 508, 528, 507, 501, 505, 517, 510, 517, 507, 549, 526, 477, 447, 525, 502, 500, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[522, 510, 488, 539, 537, 509, 503, 472, 520, 533, 506, 509, 497, 497, 497, 529, 516, 536, 501, 511, 541, 502, 516, 512, 515, 564, 491, 509, 505, 527, 504, 490, 520, 490, 501, 561, 535, 508, 523, 518, 519, 535, 523, 534, 487, 497, 517, 512, 505, 536, 504, 543, 499, 505, 494, 487, 518, 501, 471, 484, 521, 527, 498, 483, 525, 524, 493, 502, 502, 530, 502, 487, 501, 507, 508, 517, 530, 460, 511, 525, 496, 512, 508, 511, 478, 538, 535, 521, 512, 520, 488, 528, 505, 518, 522, 458, 547, 520, 527, 504, 559, 526, 484, 519, 493, 454, 522, 553, 492, 501, 508, 517, 516, 480, 532, 538, 505, 520, 513, 518, 512, 493, 517, 528, 505, 538, 522, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[524, 496, 525, 515, 489, 524, 495, 497, 502, 548, 508, 518, 527, 500, 524, 493, 528, 458, 516, 509, 515, 483, 533, 512, 485, 531, 515, 578, 483, 510, 473, 504, 490, 530, 497, 492, 534, 535, 492, 505, 530, 509, 518, 543, 521, 527, 515, 539, 503, 507, 524, 532, 530, 506, 512, 532, 513, 537, 537, 483, 535, 512, 523, 495, 525, 514, 539, 476, 536, 517, 521, 525, 524, 509, 521, 530, 486, 494, 467, 507, 529, 546, 544, 522, 517, 550, 487, 482, 514, 524, 518, 523, 513, 506, 500, 528, 521, 492, 505, 517, 531, 475, 503, 547, 520, 518, 510, 540, 544, 496, 478, 478, 505, 535, 511, 487, 481, 525, 491, 478, 510, 498, 499, 497, 484, 509, 495, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[515, 523, 520, 508, 530, 523, 493, 495, 531, 495, 510, 494, 499, 507, 522, 513, 497, 543, 557, 501, 535, 510, 536, 525, 510, 526, 496, 477, 535, 573, 531, 523, 505, 524, 568, 492, 502, 500, 545, 515, 507, 488, 492, 516, 482, 520, 505, 483, 507, 521, 518, 504, 498, 503, 501, 483, 512, 491, 566, 507, 509, 527, 494, 525, 509, 493, 493, 534, 503, 533, 526, 514, 473, 499, 487, 554, 520, 504, 549, 483, 481, 542, 560, 526, 484, 514, 537, 528, 502, 511, 515, 503, 496, 532, 521, 490, 488, 510, 460, 504, 489, 481, 487, 507, 498, 495, 498, 556, 505, 527, 536, 511, 509, 480, 505, 535, 519, 497, 512, 506, 501, 473, 506, 525, 532, 506, 539, 560] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[536, 503, 486, 507, 509, 529, 488, 525, 502, 504, 515, 527, 494, 512, 475, 481, 513, 521, 551, 520, 472, 518, 573, 510, 528, 518, 521, 479, 486, 520, 514, 526, 486, 506, 492, 528, 483, 525, 507, 531, 580, 491, 498, 502, 533, 512, 527, 507, 515, 529, 532, 499, 503, 495, 541, 493, 508, 496, 503, 510, 468, 503, 467, 524, 545, 579, 507, 508, 503, 516, 495, 490, 482, 524, 523, 562, 539, 502, 487, 541, 537, 481, 495, 556, 505, 510, 544, 531, 520, 528, 515, 456, 516, 503, 494, 513, 486, 525, 531, 540, 547, 524, 494, 485, 535, 516, 472, 543, 488, 560, 508, 484, 520, 515, 510, 473, 537, 491, 490, 491, 544, 527, 493, 525, 512, 511, 479, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[485, 483, 521, 505, 496, 521, 520, 514, 496, 486, 510, 538, 480, 525, 512, 517, 537, 521, 488, 514, 538, 487, 528, 473, 508, 513, 515, 520, 505, 517, 539, 520, 519, 484, 493, 493, 547, 525, 500, 534, 475, 550, 521, 530, 496, 531, 496, 492, 528, 547, 465, 489, 533, 472, 522, 526, 502, 514, 502, 532, 523, 497, 511, 512, 490, 539, 497, 511, 506, 519, 511, 522, 531, 463, 522, 528, 524, 500, 484, 487, 479, 516, 492, 470, 526, 511, 537, 543, 540, 536, 490, 508, 528, 527, 518, 481, 501, 497, 536, 512, 493, 502, 488, 509, 493, 508, 540, 539, 514, 479, 534, 512, 525, 512, 553, 513, 551, 524, 516, 489, 523, 503, 541, 538, 542, 493, 503, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[562, 497, 519, 513, 493, 537, 530, 530, 485, 503, 534, 518, 540, 515, 487, 495, 497, 486, 498, 512, 497, 502, 529, 517, 516, 476, 500, 514, 536, 502, 556, 497, 547, 538, 517, 472, 515, 514, 509, 504, 508, 505, 520, 546, 486, 479, 488, 515, 492, 537, 515, 535, 501, 477, 514, 494, 500, 532, 521, 526, 499, 514, 551, 531, 526, 527, 547, 479, 524, 543, 510, 513, 497, 507, 530, 512, 544, 488, 497, 522, 492, 485, 486, 501, 534, 486, 541, 566, 544, 505, 557, 563, 496, 537, 492, 484, 538, 512, 472, 486, 489, 545, 541, 512, 501, 503, 495, 490, 467, 517, 556, 488, 500, 540, 525, 484, 512, 487, 524, 537, 522, 473, 456, 492, 515, 494, 545, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[507, 503, 539, 504, 498, 534, 538, 511, 544, 515, 503, 514, 531, 548, 496, 535, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 484, 515, 514, 511, 482, 526, 493, 557, 491, 478, 513, 576, 519, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 483, 538, 526, 541, 513, 513, 503, 514, 506, 502, 483, 500, 482, 488, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 529, 523, 537, 496, 505, 534, 538, 479, 500, 501, 494, 541, 475, 505, 478, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 522, 469, 512, 517, 487, 527, 503, 507, 528, 515, 545, 531, 546, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 507, 506, 503, 521, 529, 510, 487, 492, 486, 522, 509, 524, 510, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 559, 498, 515, 479, 502, 501, 496, 509, 493, 500, 493, 515, 510, 518, 500, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 524, 494, 508, 529, 496, 520, 501, 524, 531, 515, 531, 560, 506, 485, 546, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8345, 8220, 8099, 8185, 8202, 8116, 8104, 8265] avg=8192.0000 max=8345 max_rank=0 min=8099 min_rank=2 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 496, 525, 515, 489, 524, 495, 497, 502, 548, 508, 518, 527, 500, 524, 493, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 458, 516, 509, 515, 483, 502, 512, 485, 531, 515, 578, 483, 510, 473, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 530, 497, 492, 534, 535, 492, 505, 530, 509, 518, 543, 521, 527, 515, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 507, 524, 532, 530, 506, 512, 532, 513, 537, 537, 483, 535, 512, 523, 495, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 539, 476, 536, 512, 521, 525, 524, 509, 521, 530, 486, 494, 467, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 546, 544, 522, 517, 550, 487, 482, 514, 524, 518, 523, 513, 475, 500, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 492, 505, 517, 531, 475, 503, 547, 520, 518, 510, 540, 544, 496, 478, 478, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 535, 511, 487, 481, 525, 491, 478, 510, 498, 499, 497, 484, 509, 495, 486, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8102, 8262, 8311, 8186, 8272, 8193, 8001] avg=8192.0000 max=8311 max_rank=3 min=8001 min_rank=7 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 510, 488, 539, 537, 509, 503, 472, 520, 533, 506, 509, 497, 497, 497, 529, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 536, 501, 511, 541, 502, 486, 512, 515, 564, 491, 509, 505, 527, 504, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 490, 501, 561, 535, 508, 523, 518, 519, 535, 523, 534, 487, 497, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 536, 504, 543, 499, 505, 494, 487, 518, 501, 471, 484, 521, 527, 498, 483, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 524, 493, 502, 502, 525, 502, 487, 501, 507, 508, 517, 530, 460, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 508, 511, 478, 538, 535, 521, 512, 520, 488, 528, 505, 487, 522, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 520, 527, 504, 559, 526, 484, 519, 493, 454, 522, 553, 492, 501, 508, 517, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 480, 532, 538, 505, 520, 513, 518, 512, 493, 517, 528, 505, 538, 522, 535, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8210, 8264, 8105, 8119, 8119, 8245, 8282] avg=8192.0000 max=8282 max_rank=7 min=8105 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[503, 506, 484, 517, 521, 522, 484, 472, 478, 442, 544, 495, 532, 473, 521, 522, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 500, 536, 523, 500, 509, 484, 469, 500, 505, 541, 566, 523, 481, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 531, 555, 518, 474, 519, 498, 531, 492, 516, 513, 526, 543, 485, 524, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 524, 487, 507, 476, 511, 523, 551, 521, 533, 476, 515, 494, 520, 498, 485, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 472, 540, 507, 550, 517, 486, 510, 485, 490, 503, 536, 518, 500, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 499, 538, 500, 499, 513, 511, 525, 509, 471, 534, 536, 529, 522, 497, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 533, 542, 511, 521, 554, 510, 493, 547, 468, 454, 552, 532, 516, 534, 508, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 507, 501, 505, 517, 510, 517, 507, 549, 526, 477, 447, 525, 502, 500, 525, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8042, 8141, 8284, 8140, 8177, 8247, 8351, 8154] avg=8192.0000 max=8351 max_rank=6 min=8042 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 523, 520, 508, 530, 523, 493, 495, 531, 495, 510, 494, 499, 507, 522, 513, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 543, 557, 501, 535, 510, 504, 525, 510, 526, 496, 477, 535, 573, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 524, 568, 492, 502, 500, 545, 515, 507, 488, 492, 516, 482, 520, 505, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 521, 518, 504, 498, 503, 501, 483, 512, 491, 566, 507, 509, 527, 494, 525, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 493, 534, 503, 528, 526, 514, 473, 499, 487, 554, 520, 504, 549, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 542, 560, 526, 484, 514, 537, 528, 502, 511, 515, 503, 496, 500, 521, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 510, 460, 504, 489, 481, 487, 507, 498, 495, 498, 556, 505, 527, 536, 511, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 480, 505, 535, 519, 497, 512, 506, 501, 473, 506, 525, 532, 506, 539, 560, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8343, 8128, 8197, 8169, 8210, 8071, 8215] avg=8192.0000 max=8343 max_rank=1 min=8071 min_rank=6 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[562, 497, 519, 513, 493, 537, 530, 530, 485, 503, 534, 518, 540, 515, 487, 495, 28, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 486, 498, 512, 497, 502, 487, 517, 516, 476, 500, 514, 536, 502, 556, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 538, 517, 472, 515, 514, 509, 504, 508, 505, 520, 546, 486, 479, 488, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 537, 515, 535, 501, 477, 514, 494, 500, 532, 521, 526, 499, 514, 551, 531, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 547, 479, 524, 533, 510, 513, 497, 507, 530, 512, 544, 488, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 485, 486, 501, 534, 486, 541, 566, 544, 505, 557, 563, 496, 497, 492, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 512, 472, 486, 489, 545, 541, 512, 501, 503, 495, 490, 467, 517, 556, 488, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 540, 525, 484, 512, 487, 524, 537, 522, 473, 456, 492, 515, 494, 545, 488, 11, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8093, 8136, 8274, 8256, 8229, 8140, 8115] avg=8192.0000 max=8293 max_rank=0 min=8093 min_rank=1 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[485, 483, 521, 505, 496, 521, 520, 514, 496, 486, 510, 538, 480, 525, 512, 517, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 521, 488, 514, 538, 487, 497, 473, 508, 513, 515, 520, 505, 517, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 484, 493, 493, 547, 525, 500, 534, 475, 550, 521, 530, 496, 531, 496, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 547, 465, 489, 533, 472, 522, 526, 502, 514, 502, 532, 523, 497, 511, 512, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 539, 497, 511, 506, 514, 511, 522, 531, 463, 522, 528, 524, 500, 484, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 516, 492, 470, 526, 511, 537, 543, 540, 536, 490, 508, 528, 495, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 497, 536, 512, 493, 502, 488, 509, 493, 508, 540, 539, 514, 479, 534, 512, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 512, 553, 513, 551, 524, 516, 489, 523, 503, 541, 538, 542, 493, 503, 524, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8192, 8170, 8205, 8129, 8170, 8176, 8360] avg=8192.0000 max=8360 max_rank=7 min=8129 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[536, 503, 486, 507, 509, 529, 488, 525, 502, 504, 515, 527, 494, 512, 475, 481, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 521, 551, 520, 472, 518, 538, 510, 528, 518, 521, 479, 486, 520, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 506, 492, 528, 483, 525, 507, 531, 580, 491, 498, 502, 533, 512, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 529, 532, 499, 503, 495, 541, 493, 508, 496, 503, 510, 468, 503, 467, 524, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 579, 507, 508, 503, 511, 495, 490, 482, 524, 523, 562, 539, 502, 487, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 481, 495, 556, 505, 510, 544, 531, 520, 528, 515, 456, 516, 472, 494, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 525, 531, 540, 547, 524, 494, 485, 535, 516, 472, 543, 488, 560, 508, 484, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 515, 510, 473, 537, 491, 490, 491, 544, 527, 493, 525, 512, 511, 479, 516, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8235, 8193, 8119, 8298, 8173, 8256, 8144] avg=8192.0000 max=8298 max_rank=4 min=8118 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4352, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 3968, 4096, 4352, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4352, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4352, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[521, 528, 494, 499, 541, 501, 509, 523, 514, 506, 523, 556, 476, 497, 518, 531, 570, 499, 530, 505, 493, 533, 516, 496, 504, 500, 479, 542, 526, 496, 491, 514, 503, 518, 524, 517, 527, 499, 499, 535, 480, 543, 472, 500, 506, 499, 483, 495, 530, 550, 519, 511, 543, 518, 519, 512, 543, 556, 463, 519, 566, 511, 545, 456, 518, 495, 552, 529, 534, 503, 483, 499, 497, 525, 485, 526, 500, 514, 540, 551, 526, 519, 494, 524, 538, 524, 490, 506, 512, 479, 510, 506, 529, 533, 524, 520, 498, 503, 492, 485, 510, 491, 486, 478, 507, 532, 511, 467, 534, 472, 510, 480, 499, 551, 497, 485, 521, 517, 534, 525, 508, 524, 490, 523, 489, 560, 489, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[534, 510, 487, 532, 521, 535, 497, 515, 483, 486, 467, 510, 513, 501, 498, 522, 542, 482, 525, 505, 530, 542, 500, 508, 513, 519, 530, 469, 535, 515, 512, 517, 475, 539, 509, 512, 547, 530, 495, 511, 499, 518, 500, 551, 506, 477, 501, 498, 509, 506, 523, 501, 506, 533, 511, 519, 504, 509, 535, 494, 519, 551, 508, 484, 481, 478, 502, 507, 472, 501, 490, 515, 537, 526, 538, 511, 473, 529, 539, 549, 548, 513, 518, 554, 516, 561, 578, 512, 520, 493, 489, 510, 510, 493, 532, 536, 499, 524, 523, 480, 502, 475, 508, 500, 515, 476, 507, 527, 530, 527, 496, 490, 495, 492, 535, 485, 517, 479, 512, 477, 514, 508, 516, 542, 524, 555, 515, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[543, 483, 478, 558, 516, 506, 501, 505, 481, 507, 494, 520, 529, 503, 478, 506, 534, 489, 503, 529, 529, 516, 539, 493, 508, 506, 538, 510, 511, 522, 548, 494, 475, 536, 498, 528, 492, 487, 545, 515, 551, 520, 498, 538, 467, 539, 497, 504, 491, 515, 488, 535, 503, 488, 523, 476, 508, 571, 499, 492, 522, 519, 502, 528, 499, 476, 524, 537, 500, 495, 538, 543, 511, 496, 552, 515, 514, 514, 508, 498, 507, 528, 533, 536, 530, 513, 503, 476, 553, 492, 507, 526, 517, 524, 476, 523, 531, 526, 474, 517, 495, 495, 520, 525, 548, 519, 534, 469, 545, 487, 491, 516, 459, 492, 565, 522, 517, 459, 511, 544, 525, 490, 480, 566, 497, 515, 508, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[509, 518, 549, 476, 547, 493, 503, 530, 467, 491, 520, 533, 521, 508, 479, 495, 518, 525, 527, 483, 531, 474, 536, 541, 506, 540, 519, 530, 509, 524, 529, 494, 505, 481, 490, 498, 494, 543, 543, 525, 524, 500, 498, 515, 515, 501, 482, 542, 521, 503, 531, 500, 479, 537, 499, 511, 503, 491, 530, 520, 489, 506, 500, 498, 512, 511, 527, 474, 515, 487, 522, 520, 489, 520, 518, 495, 504, 528, 491, 528, 512, 532, 543, 495, 538, 532, 526, 493, 532, 513, 550, 496, 525, 522, 571, 496, 548, 519, 534, 511, 514, 542, 491, 514, 516, 516, 505, 461, 509, 496, 519, 494, 495, 506, 539, 520, 529, 485, 501, 515, 510, 496, 467, 516, 522, 511, 486, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[472, 513, 541, 514, 506, 519, 503, 525, 531, 508, 496, 512, 499, 522, 501, 510, 494, 499, 490, 495, 512, 518, 506, 500, 493, 518, 519, 525, 511, 522, 495, 507, 474, 500, 520, 556, 503, 518, 504, 478, 538, 482, 533, 499, 509, 547, 486, 506, 507, 509, 506, 506, 522, 539, 529, 511, 510, 507, 524, 545, 529, 544, 521, 516, 458, 500, 501, 549, 496, 528, 494, 551, 500, 526, 515, 520, 502, 531, 515, 535, 519, 487, 490, 541, 522, 532, 490, 494, 526, 494, 520, 501, 542, 504, 485, 502, 513, 530, 520, 524, 504, 525, 528, 529, 519, 489, 493, 545, 527, 513, 530, 500, 482, 524, 486, 550, 540, 514, 490, 525, 504, 538, 457, 453, 494, 534, 521, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[491, 532, 526, 512, 502, 541, 462, 556, 527, 485, 522, 522, 511, 537, 515, 478, 526, 538, 530, 539, 533, 492, 526, 516, 527, 490, 469, 498, 543, 504, 525, 521, 498, 487, 499, 509, 514, 582, 541, 499, 521, 501, 499, 497, 467, 471, 542, 548, 531, 527, 549, 508, 512, 477, 539, 542, 488, 530, 518, 513, 507, 488, 460, 495, 463, 518, 516, 538, 507, 509, 489, 521, 524, 526, 506, 491, 525, 490, 565, 532, 511, 519, 492, 520, 530, 562, 491, 524, 498, 513, 494, 496, 506, 475, 476, 488, 533, 531, 496, 512, 468, 482, 498, 525, 509, 512, 494, 507, 518, 501, 498, 498, 529, 541, 528, 508, 506, 526, 528, 569, 499, 525, 480, 493, 525, 509, 502, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[544, 517, 507, 495, 552, 514, 504, 510, 510, 509, 505, 534, 492, 491, 504, 506, 507, 487, 502, 512, 500, 501, 511, 507, 510, 464, 500, 534, 509, 539, 501, 538, 523, 537, 571, 489, 506, 508, 482, 535, 526, 481, 555, 510, 511, 561, 530, 466, 524, 520, 536, 493, 504, 531, 501, 513, 513, 500, 480, 507, 476, 489, 522, 490, 537, 510, 510, 501, 482, 516, 522, 502, 498, 528, 531, 518, 543, 510, 496, 547, 513, 485, 503, 508, 514, 500, 543, 502, 528, 529, 485, 496, 533, 479, 522, 515, 532, 525, 505, 519, 479, 531, 485, 544, 477, 467, 482, 525, 510, 519, 513, 539, 522, 516, 520, 509, 512, 494, 527, 513, 519, 524, 526, 476, 555, 529, 518, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[528, 532, 521, 567, 530, 522, 561, 540, 532, 493, 521, 509, 520, 510, 554, 515, 542, 511, 463, 492, 494, 493, 535, 505, 532, 534, 509, 520, 491, 491, 528, 515, 543, 488, 514, 508, 482, 498, 482, 550, 474, 503, 496, 506, 501, 473, 503, 476, 512, 539, 472, 476, 559, 546, 503, 490, 495, 506, 503, 500, 473, 520, 507, 483, 513, 513, 523, 523, 518, 499, 502, 495, 494, 480, 504, 479, 484, 533, 525, 496, 489, 513, 477, 561, 528, 513, 472, 530, 524, 536, 519, 519, 508, 560, 519, 516, 545, 550, 520, 533, 536, 513, 500, 509, 502, 493, 524, 529, 513, 498, 486, 516, 523, 506, 550, 518, 523, 487, 523, 496, 499, 518, 504, 502, 533, 494, 499, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[521, 528, 494, 499, 529, 501, 509, 523, 514, 506, 523, 556, 476, 497, 518, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 499, 530, 505, 493, 533, 516, 496, 504, 500, 479, 542, 526, 496, 491, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 518, 524, 517, 527, 499, 499, 535, 480, 543, 472, 500, 506, 499, 483, 495, 15, 12, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 550, 519, 511, 543, 518, 519, 512, 543, 556, 463, 519, 566, 511, 545, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 495, 552, 529, 534, 503, 483, 499, 497, 525, 485, 526, 500, 514, 540, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 519, 494, 478, 538, 524, 490, 506, 512, 479, 510, 506, 529, 533, 524, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 492, 485, 510, 491, 486, 478, 507, 532, 511, 467, 534, 472, 510, 480, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 551, 493, 485, 521, 517, 534, 525, 508, 524, 490, 523, 489, 560, 489, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8179, 8137, 8361, 8251, 8188, 7996, 8199] avg=8192.0000 max=8361 max_rank=3 min=7996 min_rank=6 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[543, 483, 478, 558, 504, 506, 501, 505, 481, 507, 494, 520, 529, 503, 478, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 503, 529, 529, 516, 539, 493, 508, 506, 538, 510, 511, 522, 548, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 536, 498, 528, 492, 487, 545, 515, 551, 520, 498, 538, 467, 539, 497, 504, 14, 12, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 488, 535, 503, 488, 523, 476, 508, 571, 499, 492, 522, 519, 502, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 476, 524, 537, 500, 495, 538, 543, 511, 496, 552, 515, 514, 514, 508, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 533, 489, 530, 513, 503, 476, 553, 492, 507, 526, 517, 524, 476, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 526, 474, 517, 495, 495, 520, 525, 548, 519, 534, 469, 545, 487, 491, 516, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 492, 561, 522, 517, 459, 511, 544, 525, 490, 480, 566, 497, 515, 508, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8255, 8226, 8160, 8220, 8197, 8233, 8149] avg=8192.0000 max=8255 max_rank=1 min=8096 min_rank=0 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[472, 513, 541, 514, 495, 519, 503, 525, 531, 508, 496, 512, 499, 522, 501, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 499, 490, 495, 512, 518, 506, 500, 493, 518, 519, 525, 511, 522, 495, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 500, 520, 556, 503, 518, 504, 478, 538, 482, 533, 499, 509, 547, 486, 506, 13, 11, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 506, 506, 522, 539, 529, 511, 510, 507, 524, 545, 529, 544, 521, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 500, 501, 549, 496, 528, 494, 551, 500, 526, 515, 520, 502, 531, 515, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 487, 490, 493, 522, 532, 490, 494, 526, 494, 520, 501, 542, 504, 485, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 530, 520, 524, 504, 525, 528, 529, 519, 489, 493, 545, 527, 513, 530, 500, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 524, 483, 550, 540, 514, 490, 525, 504, 538, 457, 453, 494, 534, 521, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8091, 8186, 8325, 8221, 8101, 8331, 8120] avg=8192.0000 max=8331 max_rank=6 min=8091 min_rank=1 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[534, 510, 487, 532, 509, 535, 497, 515, 483, 486, 467, 510, 513, 501, 498, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 482, 525, 505, 530, 542, 500, 508, 513, 519, 530, 469, 535, 515, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 539, 509, 512, 547, 530, 495, 511, 499, 518, 500, 551, 506, 477, 501, 498, 14, 12, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 506, 523, 501, 506, 533, 511, 519, 504, 509, 535, 494, 519, 551, 508, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 478, 502, 507, 472, 501, 490, 515, 537, 526, 538, 511, 473, 529, 539, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 513, 518, 504, 516, 561, 578, 512, 520, 493, 489, 510, 510, 493, 532, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 523, 480, 502, 475, 508, 500, 515, 476, 507, 527, 530, 527, 496, 490, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 492, 531, 485, 517, 479, 512, 477, 514, 508, 516, 542, 524, 555, 515, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8230, 8205, 8212, 8148, 8333, 8122, 8187] avg=8192.0000 max=8333 max_rank=5 min=8099 min_rank=0 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 518, 549, 476, 535, 493, 503, 530, 467, 491, 520, 533, 521, 508, 479, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 527, 483, 531, 474, 536, 541, 506, 540, 519, 530, 509, 524, 529, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 481, 490, 498, 494, 543, 543, 525, 524, 500, 498, 515, 515, 501, 482, 542, 14, 12, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 531, 500, 479, 537, 499, 511, 503, 491, 530, 520, 489, 506, 500, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 511, 527, 474, 515, 487, 522, 520, 489, 520, 518, 495, 504, 528, 491, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 532, 543, 451, 538, 532, 526, 493, 532, 513, 550, 496, 525, 522, 571, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 519, 534, 511, 514, 542, 491, 514, 516, 516, 505, 461, 509, 496, 519, 494, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 535, 520, 529, 485, 501, 515, 510, 496, 467, 516, 522, 511, 486, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8272, 8192, 8118, 8141, 8332, 8227, 8127] avg=8192.0000 max=8332 max_rank=5 min=8118 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[544, 517, 507, 495, 535, 514, 504, 510, 510, 509, 505, 534, 492, 491, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 487, 502, 512, 500, 501, 511, 507, 510, 464, 500, 534, 509, 539, 501, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 537, 571, 489, 506, 508, 482, 535, 526, 481, 555, 510, 511, 561, 530, 466, 17, 17, 10, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 536, 493, 504, 531, 501, 513, 513, 493, 480, 507, 476, 489, 522, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 510, 501, 482, 516, 522, 502, 498, 528, 531, 518, 543, 510, 496, 547, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 485, 503, 455, 514, 500, 543, 502, 528, 529, 485, 496, 533, 479, 522, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 505, 519, 479, 531, 485, 544, 477, 467, 482, 525, 510, 519, 513, 539, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 513, 509, 512, 494, 527, 513, 519, 524, 526, 476, 555, 529, 518, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8105, 8346, 8092, 8254, 8102, 8195, 8265] avg=8192.0000 max=8346 max_rank=2 min=8092 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[528, 532, 521, 567, 518, 522, 561, 540, 532, 493, 521, 509, 520, 510, 554, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 511, 463, 492, 494, 493, 535, 505, 532, 534, 509, 520, 491, 491, 528, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 488, 514, 508, 482, 498, 482, 550, 474, 503, 496, 506, 501, 473, 503, 476, 14, 12, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 539, 472, 476, 559, 546, 503, 490, 495, 506, 503, 500, 473, 520, 507, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 523, 523, 518, 499, 502, 495, 494, 480, 504, 479, 484, 533, 525, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 513, 477, 511, 528, 513, 472, 530, 524, 536, 519, 519, 508, 560, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 550, 520, 533, 536, 513, 500, 509, 502, 493, 524, 529, 513, 498, 486, 516, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 506, 546, 518, 523, 487, 523, 496, 499, 518, 504, 502, 533, 494, 499, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8443, 8141, 8034, 8084, 8081, 8234, 8310, 8209] avg=8192.0000 max=8443 max_rank=0 min=8034 min_rank=2 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[491, 532, 526, 512, 491, 541, 462, 556, 527, 485, 522, 522, 511, 537, 515, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 530, 539, 533, 492, 526, 516, 527, 490, 469, 498, 543, 504, 525, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 487, 499, 509, 514, 582, 541, 499, 521, 501, 499, 497, 467, 471, 542, 548, 14, 11, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 527, 549, 508, 512, 477, 539, 542, 488, 530, 518, 513, 507, 488, 460, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 518, 516, 538, 507, 509, 489, 521, 524, 526, 506, 491, 525, 490, 565, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 492, 474, 530, 562, 491, 524, 498, 513, 494, 496, 506, 475, 476, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 531, 496, 512, 468, 482, 498, 525, 509, 512, 494, 507, 518, 501, 498, 498, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 541, 524, 508, 506, 526, 528, 569, 499, 525, 480, 493, 525, 509, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8263, 8210, 8184, 8220, 8049, 8122, 8280] avg=8192.0000 max=8280 max_rank=7 min=8049 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[464, 530, 511, 537, 533, 540, 473, 553, 583, 533, 497, 503, 526, 537, 507, 527, 504, 511, 520, 504, 551, 516, 490, 527, 519, 529, 514, 474, 496, 470, 519, 514, 531, 504, 497, 492, 536, 527, 510, 457, 509, 529, 516, 490, 506, 504, 534, 552, 511, 481, 499, 529, 513, 553, 489, 532, 511, 495, 456, 466, 542, 506, 519, 541, 509, 496, 515, 503, 511, 499, 518, 491, 497, 536, 488, 498, 513, 516, 483, 501, 504, 505, 512, 487, 547, 515, 497, 506, 535, 495, 495, 511, 505, 537, 541, 512, 515, 504, 492, 467, 519, 481, 533, 507, 505, 505, 525, 531, 519, 529, 511, 507, 497, 530, 504, 487, 528, 517, 529, 516, 503, 527, 518, 499, 518, 536, 501, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[506, 529, 527, 532, 517, 488, 487, 568, 511, 520, 513, 540, 515, 527, 507, 485, 506, 510, 502, 470, 521, 520, 533, 514, 497, 530, 488, 488, 487, 529, 511, 451, 504, 517, 491, 508, 468, 541, 520, 562, 499, 534, 549, 480, 513, 506, 525, 488, 516, 507, 556, 497, 497, 527, 521, 474, 546, 517, 510, 480, 514, 507, 514, 537, 511, 541, 533, 482, 535, 487, 507, 519, 536, 526, 521, 499, 482, 507, 522, 477, 524, 484, 492, 495, 517, 510, 521, 558, 515, 502, 536, 521, 510, 502, 514, 527, 515, 466, 509, 496, 548, 533, 496, 509, 527, 518, 505, 489, 483, 521, 547, 508, 558, 526, 506, 514, 499, 506, 507, 521, 510, 533, 500, 494, 505, 529, 495, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[527, 531, 493, 495, 476, 566, 554, 496, 523, 520, 511, 476, 518, 508, 498, 520, 513, 507, 498, 510, 509, 511, 515, 482, 542, 513, 510, 492, 520, 516, 521, 529, 494, 492, 563, 500, 500, 531, 521, 542, 498, 553, 522, 488, 542, 512, 495, 536, 544, 524, 487, 505, 503, 531, 533, 482, 506, 503, 521, 505, 502, 450, 507, 522, 492, 493, 528, 478, 518, 536, 522, 529, 495, 499, 512, 486, 519, 475, 523, 504, 508, 495, 540, 538, 539, 516, 530, 492, 518, 492, 524, 511, 489, 518, 527, 532, 473, 462, 487, 471, 509, 497, 496, 531, 497, 475, 520, 490, 557, 537, 505, 516, 542, 518, 520, 524, 518, 468, 521, 599, 539, 519, 534, 512, 521, 512, 486, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[488, 505, 527, 485, 526, 517, 536, 513, 490, 508, 533, 486, 517, 545, 485, 510, 495, 538, 541, 488, 488, 533, 559, 535, 527, 551, 501, 546, 470, 510, 479, 519, 531, 527, 566, 513, 504, 504, 528, 502, 460, 535, 501, 530, 486, 497, 537, 500, 511, 532, 455, 509, 530, 537, 524, 512, 492, 537, 502, 543, 518, 468, 461, 542, 513, 484, 493, 538, 516, 494, 514, 491, 503, 530, 517, 533, 501, 499, 548, 548, 523, 517, 509, 525, 494, 482, 470, 513, 515, 519, 502, 560, 515, 512, 511, 520, 499, 527, 514, 488, 489, 488, 508, 505, 496, 505, 518, 579, 529, 508, 457, 518, 508, 488, 525, 508, 508, 498, 522, 544, 507, 499, 453, 506, 517, 522, 522, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[540, 591, 502, 528, 521, 482, 548, 506, 486, 546, 529, 527, 507, 499, 535, 509, 543, 503, 521, 515, 540, 481, 570, 494, 541, 506, 486, 463, 493, 512, 480, 507, 543, 515, 493, 536, 505, 532, 495, 537, 485, 478, 505, 484, 496, 543, 501, 528, 537, 560, 494, 513, 471, 562, 530, 518, 490, 477, 517, 493, 471, 511, 530, 485, 509, 497, 488, 506, 496, 475, 508, 521, 507, 517, 476, 511, 536, 497, 503, 494, 523, 466, 491, 560, 506, 480, 507, 507, 527, 541, 548, 536, 546, 479, 498, 515, 498, 512, 515, 462, 519, 505, 502, 515, 525, 536, 534, 495, 537, 528, 507, 527, 537, 533, 486, 491, 528, 513, 500, 514, 503, 499, 526, 498, 475, 571, 498, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[515, 492, 516, 510, 514, 494, 511, 520, 486, 536, 525, 482, 478, 489, 523, 494, 498, 492, 511, 519, 523, 483, 495, 530, 529, 500, 534, 490, 475, 515, 494, 522, 533, 515, 563, 465, 505, 538, 534, 485, 519, 507, 571, 536, 539, 497, 573, 545, 530, 553, 518, 543, 491, 507, 494, 483, 529, 518, 507, 485, 531, 520, 506, 492, 531, 510, 499, 452, 537, 486, 489, 537, 543, 511, 528, 476, 524, 545, 496, 466, 527, 502, 478, 534, 538, 523, 521, 514, 525, 514, 506, 521, 495, 558, 511, 529, 498, 512, 514, 490, 527, 532, 528, 510, 492, 515, 539, 480, 492, 522, 521, 543, 488, 469, 518, 480, 523, 533, 489, 514, 517, 497, 475, 492, 511, 521, 521, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[544, 516, 531, 510, 505, 509, 511, 519, 525, 489, 494, 501, 501, 533, 545, 515, 502, 513, 501, 496, 524, 523, 546, 539, 484, 509, 521, 496, 526, 496, 536, 471, 564, 534, 558, 525, 548, 548, 482, 521, 509, 538, 509, 491, 489, 471, 554, 525, 529, 493, 503, 514, 537, 518, 520, 517, 485, 513, 535, 547, 513, 523, 537, 524, 536, 535, 510, 509, 445, 518, 522, 486, 541, 550, 515, 504, 523, 496, 476, 497, 459, 478, 509, 483, 531, 490, 529, 468, 508, 513, 478, 538, 473, 472, 519, 511, 553, 513, 493, 515, 528, 498, 533, 504, 486, 551, 521, 498, 481, 497, 515, 506, 554, 520, 505, 480, 509, 483, 451, 535, 495, 537, 548, 467, 510, 544, 482, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[525, 526, 498, 549, 497, 534, 531, 490, 537, 517, 503, 532, 538, 492, 488, 534, 519, 510, 512, 465, 558, 517, 521, 519, 527, 501, 458, 508, 530, 510, 510, 514, 507, 496, 517, 517, 505, 522, 492, 519, 501, 547, 513, 534, 513, 495, 542, 516, 512, 514, 518, 541, 488, 523, 515, 497, 475, 521, 492, 513, 538, 492, 525, 510, 496, 487, 500, 525, 494, 541, 515, 523, 518, 468, 481, 491, 500, 502, 496, 497, 556, 537, 510, 528, 493, 520, 498, 469, 512, 534, 538, 490, 502, 499, 494, 510, 520, 494, 520, 518, 505, 532, 507, 509, 495, 495, 500, 529, 525, 534, 502, 532, 507, 524, 495, 492, 490, 529, 565, 490, 544, 499, 528, 502, 513, 539, 483, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[464, 474, 511, 537, 533, 540, 473, 553, 583, 533, 497, 503, 526, 537, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 520, 504, 551, 516, 490, 527, 519, 529, 514, 474, 496, 470, 519, 514, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 497, 492, 536, 527, 510, 457, 509, 529, 516, 490, 506, 504, 462, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 481, 499, 529, 513, 553, 489, 532, 511, 495, 456, 466, 542, 506, 519, 541, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 496, 515, 503, 511, 499, 518, 491, 497, 536, 488, 498, 513, 516, 483, 501, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 512, 487, 547, 515, 497, 506, 535, 495, 495, 508, 505, 537, 541, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 492, 467, 519, 481, 533, 507, 505, 505, 525, 531, 519, 529, 511, 507, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 530, 504, 487, 528, 517, 529, 516, 503, 527, 518, 499, 518, 536, 501, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8186, 8122, 8146, 8146, 8201, 8178, 8259] avg=8192.0000 max=8298 max_rank=0 min=8122 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[527, 475, 493, 495, 476, 566, 554, 496, 523, 520, 511, 476, 518, 508, 498, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 507, 498, 510, 509, 511, 515, 482, 542, 513, 510, 492, 520, 516, 521, 529, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 563, 500, 500, 531, 521, 542, 498, 553, 522, 488, 542, 512, 429, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 524, 487, 505, 503, 531, 533, 482, 506, 503, 521, 505, 502, 450, 507, 522, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 493, 528, 478, 518, 536, 522, 529, 495, 499, 512, 486, 519, 475, 523, 504, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 540, 538, 539, 516, 530, 492, 518, 492, 524, 508, 489, 518, 527, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 462, 487, 471, 509, 497, 496, 531, 497, 475, 520, 490, 557, 537, 505, 516, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 520, 524, 518, 468, 521, 599, 539, 519, 534, 512, 521, 512, 486, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8216, 8223, 8128, 8175, 8266, 8051, 8321] avg=8192.0000 max=8321 max_rank=7 min=8051 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[506, 473, 527, 532, 517, 488, 487, 568, 511, 520, 513, 540, 515, 527, 507, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 502, 470, 521, 520, 533, 514, 497, 530, 488, 488, 487, 529, 511, 451, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 491, 508, 468, 541, 520, 562, 499, 534, 549, 480, 513, 506, 455, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 556, 497, 497, 527, 521, 474, 546, 517, 510, 480, 514, 507, 514, 537, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 541, 533, 482, 535, 487, 507, 519, 536, 526, 521, 499, 482, 507, 522, 477, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 492, 495, 517, 510, 521, 558, 515, 502, 536, 518, 510, 502, 514, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 466, 509, 496, 548, 533, 496, 509, 527, 518, 505, 489, 483, 521, 547, 508, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 526, 506, 514, 499, 506, 507, 521, 510, 533, 500, 494, 505, 529, 495, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8085, 8135, 8223, 8255, 8225, 8198, 8199] avg=8192.0000 max=8255 max_rank=4 min=8085 min_rank=1 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[515, 431, 516, 510, 514, 494, 511, 520, 486, 536, 525, 482, 478, 489, 523, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 492, 511, 519, 523, 483, 495, 530, 529, 500, 534, 490, 475, 515, 494, 522, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 515, 563, 465, 505, 538, 534, 485, 519, 507, 571, 536, 539, 497, 491, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 553, 518, 543, 491, 507, 494, 483, 529, 518, 507, 485, 531, 520, 506, 492, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 499, 452, 537, 486, 489, 537, 543, 511, 528, 476, 524, 545, 496, 466, 82, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 478, 534, 538, 523, 521, 514, 525, 514, 506, 515, 495, 558, 511, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 514, 490, 527, 532, 528, 510, 492, 515, 539, 480, 492, 522, 521, 543, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 469, 518, 480, 523, 533, 489, 514, 517, 497, 475, 492, 511, 521, 521, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8024, 8139, 8343, 8213, 8214, 8290, 8244, 8069] avg=8192.0000 max=8343 max_rank=2 min=8024 min_rank=0 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[488, 452, 527, 485, 526, 517, 536, 513, 490, 508, 533, 486, 517, 545, 485, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 538, 541, 488, 488, 533, 559, 535, 527, 551, 501, 546, 470, 510, 479, 519, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 527, 566, 513, 504, 504, 528, 502, 460, 535, 501, 530, 486, 497, 465, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 455, 509, 530, 537, 524, 512, 492, 537, 502, 543, 518, 468, 461, 542, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 484, 493, 538, 516, 494, 514, 491, 503, 530, 517, 533, 501, 499, 548, 548, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 517, 509, 525, 494, 482, 470, 513, 515, 519, 502, 557, 515, 512, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 527, 514, 488, 489, 488, 508, 505, 496, 505, 518, 579, 529, 508, 457, 518, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 488, 525, 508, 508, 498, 522, 544, 507, 499, 453, 506, 517, 522, 522, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8307, 8149, 8176, 8294, 8184, 8154, 8154] avg=8192.0000 max=8307 max_rank=1 min=8118 min_rank=0 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[540, 529, 502, 528, 521, 482, 548, 506, 486, 546, 529, 527, 507, 499, 535, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 521, 515, 540, 481, 570, 494, 541, 506, 486, 463, 493, 512, 480, 507, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 515, 493, 536, 505, 532, 495, 537, 485, 478, 505, 484, 496, 543, 434, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 560, 494, 513, 471, 562, 530, 518, 490, 477, 517, 493, 471, 511, 530, 485, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 497, 488, 506, 496, 475, 508, 521, 507, 517, 476, 511, 536, 497, 503, 494, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 466, 491, 560, 506, 480, 507, 507, 527, 541, 548, 533, 546, 479, 498, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 515, 462, 519, 505, 502, 515, 525, 536, 534, 495, 537, 528, 507, 527, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 533, 486, 491, 528, 513, 500, 514, 503, 499, 526, 498, 475, 571, 498, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 8186, 8109, 8162, 8108, 8227, 8248, 8202] avg=8192.0000 max=8294 max_rank=0 min=8108 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[544, 462, 531, 510, 505, 509, 511, 519, 525, 489, 494, 501, 501, 533, 545, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 501, 496, 524, 523, 546, 539, 484, 509, 521, 496, 526, 496, 536, 471, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 534, 558, 525, 548, 548, 482, 521, 509, 538, 509, 491, 489, 471, 480, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 493, 503, 514, 537, 518, 520, 517, 485, 513, 535, 547, 513, 523, 537, 524, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 535, 510, 509, 445, 518, 522, 486, 541, 550, 515, 504, 523, 496, 476, 497, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 478, 509, 483, 531, 490, 529, 468, 508, 513, 478, 535, 473, 472, 519, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 513, 493, 515, 528, 498, 533, 504, 486, 551, 521, 498, 481, 497, 515, 506, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 520, 505, 480, 509, 483, 451, 535, 495, 537, 548, 467, 510, 544, 482, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8210, 8292, 8311, 8237, 7956, 8219, 8117] avg=8192.0000 max=8311 max_rank=3 min=7956 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[525, 471, 498, 549, 497, 534, 531, 490, 537, 517, 503, 532, 538, 492, 488, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 512, 465, 558, 517, 521, 519, 527, 501, 458, 508, 530, 510, 510, 514, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 496, 517, 517, 505, 522, 492, 519, 501, 547, 513, 534, 513, 495, 469, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 514, 518, 541, 488, 523, 515, 497, 475, 521, 492, 513, 538, 492, 525, 510, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 487, 500, 525, 494, 541, 515, 523, 518, 468, 481, 491, 500, 502, 496, 497, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 537, 510, 528, 493, 520, 498, 469, 512, 534, 538, 487, 502, 499, 494, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 494, 520, 518, 505, 532, 507, 509, 495, 495, 500, 529, 525, 534, 502, 532, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 524, 495, 492, 490, 529, 565, 490, 544, 499, 528, 502, 513, 539, 483, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8207, 8163, 8177, 8107, 8187, 8244, 8215] avg=8192.0000 max=8244 max_rank=6 min=8107 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3840, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4352, 4096, 4096, 4352, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 3712, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4096, 4352, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4352, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4352, 4224, 4224, 4096, 4096, 4096, 4352, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[537, 489, 518, 534, 507, 530, 538, 489, 510, 538, 544, 496, 527, 487, 476, 519, 529, 505, 479, 530, 528, 516, 548, 480, 479, 532, 490, 513, 528, 494, 549, 521, 551, 486, 519, 521, 495, 493, 529, 519, 520, 544, 504, 501, 491, 486, 511, 525, 515, 510, 515, 531, 484, 487, 519, 532, 512, 505, 507, 514, 538, 538, 532, 528, 500, 518, 482, 479, 498, 471, 522, 521, 499, 494, 496, 505, 520, 498, 515, 510, 503, 514, 525, 507, 512, 509, 570, 504, 484, 513, 506, 480, 512, 540, 486, 500, 507, 516, 563, 530, 511, 519, 503, 505, 500, 522, 490, 507, 503, 542, 534, 514, 497, 533, 524, 472, 520, 541, 498, 496, 492, 499, 516, 470, 556, 533, 503, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[499, 503, 517, 468, 535, 530, 543, 484, 496, 503, 527, 514, 535, 514, 504, 492, 537, 539, 494, 456, 519, 526, 498, 502, 524, 552, 524, 505, 531, 516, 497, 555, 496, 516, 513, 497, 480, 531, 534, 481, 496, 495, 516, 502, 530, 513, 517, 503, 518, 467, 525, 487, 505, 492, 529, 535, 500, 500, 494, 527, 490, 530, 501, 536, 496, 487, 478, 514, 535, 501, 506, 507, 493, 528, 561, 515, 481, 503, 519, 510, 474, 551, 496, 517, 501, 521, 500, 533, 483, 549, 513, 503, 528, 526, 515, 492, 518, 503, 481, 526, 503, 532, 507, 522, 510, 531, 491, 494, 556, 523, 491, 527, 523, 507, 498, 511, 508, 517, 486, 510, 500, 536, 527, 552, 492, 530, 533, 560] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[553, 513, 510, 528, 492, 546, 492, 519, 489, 520, 492, 554, 496, 490, 524, 555, 475, 512, 510, 542, 512, 505, 534, 491, 526, 510, 529, 501, 492, 519, 525, 523, 538, 499, 500, 543, 505, 495, 501, 503, 544, 505, 492, 504, 506, 506, 542, 500, 526, 523, 508, 516, 501, 453, 528, 500, 516, 525, 534, 499, 496, 519, 446, 490, 522, 481, 480, 520, 504, 505, 528, 550, 543, 560, 513, 524, 533, 527, 515, 526, 515, 486, 534, 504, 496, 518, 467, 512, 539, 528, 500, 459, 449, 514, 507, 505, 529, 529, 539, 518, 489, 505, 528, 508, 508, 503, 523, 485, 524, 523, 488, 521, 501, 548, 531, 492, 505, 509, 512, 520, 559, 521, 502, 516, 514, 505, 488, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[489, 477, 488, 512, 512, 529, 528, 528, 484, 506, 505, 513, 571, 501, 514, 529, 510, 530, 525, 500, 481, 514, 500, 559, 494, 540, 528, 509, 506, 539, 533, 550, 506, 516, 482, 513, 497, 523, 538, 511, 537, 501, 539, 479, 547, 520, 496, 530, 473, 494, 485, 498, 486, 480, 513, 550, 505, 480, 539, 536, 524, 522, 519, 511, 509, 481, 521, 515, 526, 474, 497, 562, 497, 509, 551, 544, 494, 513, 482, 483, 505, 469, 483, 557, 541, 512, 546, 493, 478, 566, 477, 520, 517, 537, 530, 479, 523, 502, 481, 512, 467, 505, 525, 538, 462, 543, 511, 522, 509, 523, 508, 468, 498, 524, 553, 540, 500, 520, 511, 522, 539, 501, 471, 489, 511, 488, 524, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[467, 519, 490, 496, 492, 504, 509, 518, 508, 488, 485, 522, 499, 545, 480, 531, 503, 510, 518, 526, 527, 504, 528, 549, 482, 517, 482, 514, 458, 479, 522, 493, 531, 519, 531, 508, 523, 480, 537, 527, 541, 462, 471, 494, 528, 465, 535, 458, 529, 530, 491, 507, 578, 513, 501, 501, 520, 533, 473, 485, 522, 497, 514, 521, 512, 542, 501, 560, 530, 501, 518, 522, 519, 512, 521, 541, 548, 538, 509, 506, 516, 511, 502, 516, 524, 477, 530, 518, 500, 514, 524, 499, 551, 554, 497, 509, 505, 518, 554, 506, 551, 524, 483, 512, 520, 527, 490, 510, 509, 543, 520, 515, 516, 502, 486, 498, 531, 489, 546, 492, 512, 510, 534, 468, 528, 472, 519, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[524, 538, 534, 535, 505, 507, 472, 523, 489, 467, 501, 509, 509, 519, 505, 513, 541, 512, 537, 542, 520, 511, 490, 499, 507, 547, 496, 549, 511, 540, 487, 524, 528, 494, 558, 515, 530, 525, 534, 509, 525, 526, 470, 500, 536, 498, 527, 480, 518, 534, 495, 536, 488, 494, 523, 545, 510, 505, 498, 515, 487, 528, 580, 534, 489, 520, 502, 515, 542, 521, 481, 562, 534, 514, 500, 481, 491, 507, 505, 505, 502, 514, 529, 536, 496, 502, 511, 488, 511, 462, 518, 506, 510, 504, 486, 497, 505, 497, 523, 503, 501, 498, 524, 523, 527, 533, 504, 504, 502, 511, 496, 519, 494, 506, 505, 517, 520, 522, 514, 523, 504, 488, 539, 530, 441, 497, 502, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[502, 516, 518, 538, 509, 518, 471, 529, 487, 510, 507, 539, 529, 490, 562, 534, 543, 525, 537, 495, 524, 525, 510, 519, 480, 532, 517, 531, 468, 495, 551, 524, 496, 545, 554, 540, 560, 503, 510, 492, 517, 557, 519, 511, 503, 529, 493, 495, 535, 512, 583, 496, 504, 526, 482, 530, 483, 526, 467, 504, 509, 522, 497, 462, 543, 514, 506, 528, 511, 544, 531, 513, 467, 530, 504, 479, 522, 523, 507, 561, 473, 516, 532, 512, 498, 529, 527, 506, 522, 495, 479, 520, 503, 474, 495, 527, 498, 547, 481, 494, 524, 538, 484, 510, 516, 503, 509, 481, 479, 467, 507, 527, 480, 506, 475, 517, 515, 485, 491, 509, 513, 522, 542, 490, 505, 519, 486, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[529, 535, 533, 545, 537, 524, 488, 485, 496, 539, 495, 541, 470, 510, 468, 551, 501, 533, 526, 537, 498, 536, 506, 477, 528, 467, 461, 496, 469, 482, 517, 544, 519, 506, 485, 509, 551, 508, 539, 491, 508, 509, 505, 490, 546, 494, 522, 474, 509, 487, 501, 505, 517, 523, 518, 515, 498, 512, 500, 519, 528, 509, 488, 540, 529, 545, 516, 473, 502, 482, 513, 515, 499, 510, 538, 523, 521, 503, 465, 542, 531, 503, 535, 495, 515, 546, 522, 510, 524, 486, 521, 475, 504, 530, 501, 529, 500, 508, 494, 523, 497, 518, 568, 495, 546, 500, 536, 563, 483, 511, 511, 527, 536, 466, 530, 498, 547, 547, 471, 515, 512, 516, 522, 484, 496, 491, 529, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[537, 489, 518, 534, 507, 530, 538, 489, 510, 538, 544, 496, 527, 487, 476, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 505, 479, 530, 528, 516, 548, 480, 479, 532, 490, 513, 528, 494, 549, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 486, 519, 521, 495, 493, 524, 519, 520, 544, 504, 501, 491, 486, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 515, 531, 484, 487, 519, 532, 512, 505, 507, 514, 538, 538, 532, 528, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 518, 482, 479, 498, 471, 522, 507, 499, 494, 496, 505, 520, 498, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 525, 507, 512, 509, 570, 504, 484, 513, 506, 480, 512, 540, 486, 500, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 516, 563, 530, 511, 519, 503, 505, 500, 514, 490, 507, 503, 542, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 533, 524, 472, 520, 541, 498, 496, 492, 499, 516, 470, 556, 533, 503, 505, 6, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8189, 8190, 8282, 8014, 8198, 8258, 8170] avg=8192.0000 max=8282 max_rank=3 min=8014 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[499, 503, 517, 468, 535, 530, 543, 484, 496, 503, 527, 514, 535, 514, 504, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 539, 494, 456, 519, 526, 498, 502, 524, 552, 524, 505, 531, 516, 497, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 513, 497, 480, 531, 529, 481, 496, 495, 516, 502, 530, 513, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 467, 525, 487, 505, 492, 529, 535, 500, 500, 494, 527, 490, 530, 501, 536, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 487, 478, 514, 535, 501, 506, 494, 493, 528, 561, 515, 481, 503, 519, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 551, 496, 517, 501, 521, 500, 533, 483, 549, 513, 503, 528, 526, 515, 492, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 481, 526, 503, 532, 507, 522, 510, 523, 491, 494, 556, 523, 491, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 498, 511, 508, 517, 486, 510, 500, 536, 527, 552, 492, 530, 533, 560, 6, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8241, 8115, 8150, 8121, 8237, 8207, 8304] avg=8192.0000 max=8304 max_rank=7 min=8115 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[553, 513, 510, 528, 492, 546, 492, 519, 489, 520, 492, 554, 496, 490, 524, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 512, 510, 542, 512, 505, 534, 491, 526, 510, 529, 501, 492, 519, 525, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 499, 500, 543, 505, 495, 497, 503, 544, 505, 492, 504, 506, 506, 542, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 523, 508, 516, 501, 453, 528, 500, 516, 525, 534, 499, 496, 519, 446, 490, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 480, 520, 504, 505, 528, 535, 543, 560, 513, 524, 533, 527, 515, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 534, 504, 496, 518, 467, 512, 539, 528, 500, 459, 449, 514, 507, 505, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 529, 539, 518, 489, 505, 528, 508, 508, 496, 523, 485, 524, 523, 488, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 548, 531, 492, 505, 509, 512, 520, 559, 521, 502, 516, 514, 505, 488, 487, 5, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8174, 8179, 8096, 8316, 8066, 8213, 8223] avg=8192.0000 max=8316 max_rank=4 min=8066 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[467, 519, 490, 496, 492, 504, 509, 518, 508, 488, 485, 522, 499, 545, 480, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 518, 526, 527, 504, 528, 549, 482, 517, 482, 514, 458, 479, 522, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 519, 531, 508, 523, 480, 529, 527, 541, 462, 471, 494, 528, 465, 535, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 530, 491, 507, 578, 513, 501, 501, 520, 533, 473, 485, 522, 497, 514, 521, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 542, 501, 560, 530, 501, 518, 499, 519, 512, 521, 541, 548, 538, 509, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 502, 516, 524, 477, 530, 518, 500, 514, 524, 499, 551, 554, 497, 509, 34, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 554, 506, 551, 524, 483, 512, 520, 514, 490, 510, 509, 543, 520, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 486, 498, 531, 489, 546, 492, 512, 510, 534, 468, 528, 472, 519, 534, 10, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8078, 8102, 8234, 8357, 8283, 8274, 8162] avg=8192.0000 max=8357 max_rank=4 min=8046 min_rank=0 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=33076 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[524, 538, 534, 535, 505, 507, 472, 523, 489, 467, 501, 509, 509, 519, 505, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 512, 537, 542, 520, 511, 490, 499, 507, 547, 496, 549, 511, 540, 487, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 494, 558, 515, 530, 525, 529, 509, 525, 526, 470, 500, 536, 498, 527, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 495, 536, 488, 494, 523, 545, 510, 505, 498, 515, 487, 528, 580, 534, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 520, 502, 515, 542, 521, 481, 547, 534, 514, 500, 481, 491, 507, 505, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 529, 536, 496, 502, 511, 488, 511, 462, 518, 506, 510, 504, 486, 497, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 497, 523, 503, 501, 498, 524, 523, 527, 525, 504, 504, 502, 511, 496, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 506, 505, 517, 520, 522, 514, 523, 504, 488, 539, 530, 441, 497, 502, 515, 6, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8281, 8250, 8306, 8154, 8105, 8162, 8132] avg=8192.0000 max=8306 max_rank=3 min=8105 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[489, 477, 488, 512, 512, 529, 528, 528, 484, 506, 505, 513, 571, 501, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 530, 525, 500, 481, 514, 500, 559, 494, 540, 528, 509, 506, 539, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 482, 513, 497, 523, 533, 511, 537, 501, 539, 479, 547, 520, 496, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 494, 485, 498, 486, 480, 513, 550, 505, 480, 539, 536, 524, 522, 519, 511, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 481, 521, 515, 526, 474, 497, 547, 497, 509, 551, 544, 494, 513, 482, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 469, 483, 557, 541, 512, 546, 493, 478, 566, 477, 520, 517, 537, 530, 479, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 502, 481, 512, 467, 505, 525, 538, 462, 535, 511, 522, 509, 523, 508, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 553, 540, 500, 520, 511, 522, 539, 501, 471, 489, 511, 488, 524, 524, 6, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8284, 8230, 8131, 8143, 8245, 8091, 8230] avg=8192.0000 max=8284 max_rank=1 min=8091 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[502, 516, 518, 538, 509, 518, 471, 529, 487, 510, 507, 539, 529, 490, 562, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 525, 537, 495, 524, 525, 510, 519, 480, 532, 517, 531, 468, 495, 551, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 545, 554, 540, 560, 503, 505, 492, 517, 557, 519, 511, 503, 529, 493, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 512, 583, 496, 504, 526, 482, 530, 483, 526, 467, 504, 509, 522, 497, 462, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 514, 506, 528, 511, 544, 531, 499, 467, 530, 504, 479, 522, 523, 507, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 516, 532, 512, 498, 529, 527, 506, 522, 495, 479, 520, 503, 474, 495, 527, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 547, 481, 494, 524, 538, 484, 510, 516, 496, 509, 481, 479, 467, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 506, 475, 517, 515, 485, 491, 509, 513, 522, 542, 490, 505, 519, 486, 528, 5, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8244, 8319, 8153, 8269, 8141, 8058, 8097] avg=8192.0000 max=8319 max_rank=2 min=8058 min_rank=6 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[529, 535, 533, 545, 537, 524, 488, 485, 496, 539, 495, 541, 470, 510, 468, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 533, 526, 537, 498, 536, 506, 477, 528, 467, 461, 496, 469, 482, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 485, 509, 551, 508, 534, 491, 508, 509, 505, 490, 546, 494, 522, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 501, 505, 517, 523, 518, 515, 498, 512, 500, 519, 528, 509, 488, 540, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 545, 516, 473, 502, 482, 513, 501, 499, 510, 538, 523, 521, 503, 465, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 503, 535, 495, 515, 546, 522, 510, 524, 486, 521, 475, 504, 530, 501, 529, 33, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 494, 523, 497, 518, 568, 495, 546, 493, 536, 563, 483, 511, 511, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 466, 530, 498, 547, 547, 471, 515, 512, 516, 522, 484, 496, 491, 529, 544, 5, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8045, 8151, 8184, 8162, 8261, 8273, 8218] avg=8192.0000 max=8273 max_rank=6 min=8045 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[492, 533, 484, 555, 491, 481, 480, 479, 525, 501, 497, 554, 517, 536, 513, 533, 499, 495, 489, 530, 545, 540, 505, 503, 501, 509, 505, 537, 515, 526, 496, 527, 482, 459, 516, 498, 489, 544, 515, 514, 554, 500, 500, 527, 502, 504, 522, 513, 530, 501, 543, 535, 543, 520, 525, 527, 480, 493, 510, 546, 496, 471, 489, 504, 505, 477, 503, 538, 480, 516, 525, 568, 516, 510, 540, 535, 484, 529, 533, 503, 546, 523, 530, 511, 563, 464, 512, 568, 506, 544, 549, 508, 466, 501, 517, 487, 518, 534, 530, 509, 482, 488, 479, 485, 490, 470, 514, 517, 488, 507, 465, 486, 514, 525, 495, 518, 518, 479, 558, 523, 541, 527, 471, 472, 541, 538, 521, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[480, 540, 520, 517, 507, 530, 527, 491, 488, 517, 529, 480, 522, 533, 520, 538, 518, 520, 510, 526, 507, 517, 505, 552, 540, 526, 435, 522, 497, 542, 493, 540, 514, 491, 508, 510, 522, 505, 518, 504, 497, 515, 545, 534, 487, 470, 501, 499, 491, 498, 541, 495, 542, 509, 552, 499, 500, 476, 543, 552, 550, 500, 515, 513, 519, 505, 509, 559, 495, 490, 473, 504, 515, 522, 525, 532, 514, 499, 517, 510, 484, 499, 532, 532, 529, 532, 487, 520, 488, 469, 500, 510, 534, 497, 517, 510, 523, 501, 499, 538, 489, 513, 519, 534, 526, 520, 500, 538, 488, 487, 528, 518, 505, 515, 525, 518, 501, 518, 515, 495, 493, 568, 505, 458, 489, 487, 503, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[527, 485, 544, 497, 534, 534, 517, 529, 506, 534, 553, 505, 488, 528, 479, 495, 517, 475, 544, 524, 548, 548, 529, 492, 543, 488, 500, 525, 497, 514, 544, 567, 494, 496, 500, 499, 517, 493, 513, 511, 475, 508, 448, 543, 528, 514, 537, 500, 536, 498, 509, 503, 545, 497, 513, 485, 514, 500, 519, 468, 490, 488, 508, 494, 520, 537, 522, 513, 480, 503, 477, 530, 519, 500, 490, 508, 506, 528, 494, 513, 519, 547, 547, 525, 491, 540, 501, 518, 506, 500, 498, 512, 502, 519, 476, 530, 488, 530, 529, 504, 498, 477, 512, 496, 528, 532, 544, 532, 520, 532, 533, 516, 480, 514, 521, 507, 539, 523, 521, 488, 544, 488, 519, 472, 520, 492, 518, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[512, 531, 505, 521, 492, 534, 506, 473, 490, 493, 546, 515, 537, 514, 536, 509, 510, 496, 528, 529, 517, 489, 474, 551, 535, 497, 536, 528, 479, 480, 524, 523, 498, 532, 553, 578, 462, 515, 523, 506, 523, 481, 532, 514, 496, 529, 485, 535, 537, 492, 513, 522, 518, 503, 486, 528, 498, 524, 527, 519, 532, 534, 508, 500, 470, 489, 529, 521, 502, 532, 522, 496, 513, 525, 499, 513, 514, 500, 502, 512, 491, 512, 502, 538, 498, 532, 522, 520, 481, 512, 474, 495, 500, 530, 504, 515, 529, 530, 482, 510, 484, 531, 521, 482, 490, 504, 479, 496, 524, 526, 500, 523, 512, 530, 476, 480, 559, 545, 473, 505, 547, 515, 508, 541, 550, 490, 509, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[507, 503, 535, 480, 523, 541, 514, 487, 506, 500, 531, 497, 492, 496, 493, 492, 562, 476, 494, 542, 483, 536, 547, 523, 520, 503, 518, 556, 469, 505, 548, 489, 545, 489, 505, 540, 508, 567, 518, 505, 512, 480, 512, 486, 534, 541, 489, 503, 526, 542, 514, 498, 523, 508, 499, 541, 489, 540, 497, 488, 543, 540, 473, 516, 520, 522, 487, 508, 499, 527, 510, 488, 513, 517, 496, 480, 526, 540, 552, 528, 526, 550, 510, 512, 486, 491, 511, 483, 522, 507, 515, 521, 524, 508, 518, 532, 522, 518, 494, 492, 537, 470, 515, 515, 518, 512, 502, 490, 499, 538, 515, 551, 503, 502, 491, 503, 526, 503, 503, 495, 502, 499, 520, 516, 450, 514, 515, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[484, 538, 516, 564, 467, 537, 492, 519, 518, 451, 505, 530, 474, 472, 513, 519, 525, 501, 498, 509, 525, 463, 495, 487, 491, 530, 494, 514, 516, 503, 542, 499, 507, 515, 536, 538, 499, 497, 528, 503, 511, 499, 519, 524, 492, 495, 497, 490, 494, 533, 510, 521, 519, 523, 522, 495, 499, 537, 512, 543, 516, 522, 521, 531, 517, 541, 505, 536, 510, 493, 524, 502, 473, 497, 535, 539, 500, 499, 531, 489, 518, 544, 511, 509, 546, 504, 526, 517, 526, 461, 507, 502, 494, 514, 480, 494, 521, 544, 525, 510, 519, 496, 528, 552, 581, 532, 486, 502, 508, 474, 500, 529, 482, 499, 525, 528, 526, 545, 526, 517, 504, 503, 506, 537, 515, 520, 495, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[524, 523, 515, 506, 513, 497, 464, 501, 537, 554, 523, 529, 471, 502, 528, 535, 508, 529, 530, 529, 481, 536, 523, 496, 505, 513, 503, 510, 537, 493, 521, 522, 476, 500, 586, 483, 485, 474, 542, 517, 506, 459, 529, 486, 527, 537, 546, 490, 504, 503, 497, 503, 521, 487, 546, 504, 514, 559, 508, 492, 517, 506, 478, 536, 487, 494, 539, 522, 522, 524, 531, 528, 503, 483, 546, 494, 518, 535, 478, 522, 522, 514, 520, 521, 512, 538, 493, 543, 520, 553, 490, 502, 528, 504, 544, 498, 522, 508, 489, 493, 449, 509, 480, 482, 497, 503, 538, 509, 528, 491, 522, 489, 534, 519, 491, 530, 522, 521, 511, 515, 516, 510, 488, 515, 484, 516, 546, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[509, 495, 506, 496, 536, 517, 533, 516, 507, 502, 489, 523, 496, 524, 534, 517, 494, 530, 524, 505, 500, 528, 502, 544, 503, 540, 502, 569, 538, 518, 540, 527, 530, 520, 515, 490, 528, 466, 511, 498, 540, 482, 541, 504, 503, 527, 516, 532, 529, 539, 511, 481, 525, 513, 500, 515, 508, 501, 495, 495, 499, 499, 525, 520, 504, 529, 491, 466, 508, 486, 502, 530, 536, 491, 461, 504, 540, 515, 496, 537, 493, 491, 546, 514, 480, 515, 536, 503, 500, 482, 482, 532, 499, 500, 520, 524, 487, 562, 517, 530, 479, 526, 529, 505, 531, 484, 524, 493, 507, 483, 495, 535, 473, 544, 514, 541, 514, 498, 499, 541, 481, 498, 518, 505, 565, 522, 496, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 533, 484, 555, 491, 481, 480, 479, 525, 501, 497, 554, 517, 536, 513, 533, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 495, 489, 530, 545, 540, 505, 503, 501, 509, 505, 481, 515, 526, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 459, 516, 498, 489, 544, 515, 514, 554, 500, 500, 527, 502, 504, 522, 513, 21, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 501, 543, 535, 529, 520, 525, 527, 480, 493, 510, 546, 496, 471, 489, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 477, 503, 538, 480, 516, 525, 568, 516, 510, 540, 535, 484, 529, 533, 503, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 523, 525, 511, 563, 464, 512, 568, 506, 544, 549, 508, 466, 501, 517, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 530, 509, 482, 488, 479, 485, 490, 470, 514, 517, 488, 507, 465, 486, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 525, 495, 518, 518, 479, 558, 523, 541, 527, 471, 472, 541, 538, 521, 531, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8166, 8164, 8199, 8272, 8290, 7997, 8273] avg=8192.0000 max=8290 max_rank=5 min=7997 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[480, 540, 520, 517, 507, 530, 527, 491, 488, 517, 529, 480, 522, 533, 520, 538, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 520, 510, 526, 507, 517, 505, 552, 540, 526, 435, 468, 497, 542, 493, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 491, 508, 510, 522, 505, 518, 504, 497, 515, 545, 534, 487, 470, 501, 499, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 541, 495, 528, 509, 552, 499, 500, 476, 543, 552, 550, 500, 515, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 509, 559, 495, 490, 473, 504, 515, 522, 525, 532, 514, 499, 517, 510, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 499, 527, 532, 529, 532, 487, 520, 488, 469, 500, 510, 534, 497, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 501, 499, 538, 489, 513, 519, 534, 526, 520, 500, 538, 488, 487, 528, 518, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 525, 518, 501, 518, 515, 495, 493, 568, 505, 458, 489, 487, 503, 507, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8196, 8144, 8262, 8198, 8135, 8255, 8103] avg=8192.0000 max=8262 max_rank=3 min=8103 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[484, 538, 516, 564, 467, 537, 492, 519, 518, 451, 505, 530, 474, 472, 513, 519, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 501, 498, 509, 525, 463, 495, 487, 491, 530, 494, 461, 516, 503, 542, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 536, 538, 499, 497, 528, 503, 511, 499, 519, 524, 492, 495, 497, 490, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 533, 510, 521, 506, 523, 522, 495, 499, 537, 512, 543, 516, 522, 521, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 541, 505, 536, 510, 493, 524, 502, 473, 497, 535, 539, 500, 499, 531, 489, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 544, 506, 509, 546, 504, 526, 517, 526, 461, 507, 502, 494, 514, 480, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 544, 525, 510, 519, 496, 528, 552, 581, 532, 486, 502, 508, 474, 500, 529, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 499, 525, 528, 526, 545, 526, 517, 504, 503, 506, 537, 515, 520, 495, 518, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8039, 8174, 8285, 8200, 8148, 8340, 8247] avg=8192.0000 max=8340 max_rank=6 min=8039 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[512, 531, 505, 521, 492, 534, 506, 473, 490, 493, 546, 515, 537, 514, 536, 509, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 496, 528, 529, 517, 489, 474, 551, 535, 497, 536, 465, 479, 480, 524, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 532, 553, 578, 462, 515, 523, 506, 523, 481, 532, 514, 496, 529, 485, 535, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 492, 513, 522, 489, 503, 486, 528, 498, 524, 527, 519, 532, 534, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 489, 529, 521, 502, 532, 522, 496, 513, 525, 499, 513, 514, 500, 502, 512, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 493, 538, 498, 532, 522, 520, 481, 512, 474, 495, 500, 530, 504, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 530, 482, 510, 484, 531, 521, 482, 490, 504, 479, 496, 524, 526, 500, 523, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 476, 480, 559, 545, 473, 505, 547, 515, 508, 541, 550, 490, 509, 507, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8133, 8297, 8212, 8152, 8117, 8149, 8251] avg=8192.0000 max=8297 max_rank=2 min=8117 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[527, 485, 544, 497, 534, 534, 517, 529, 506, 534, 553, 505, 488, 528, 479, 495, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 544, 524, 548, 548, 529, 492, 543, 488, 500, 471, 497, 514, 544, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 496, 500, 499, 517, 493, 513, 511, 475, 508, 448, 543, 528, 514, 537, 500, 20, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 509, 503, 530, 497, 513, 485, 514, 500, 519, 468, 490, 488, 508, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 537, 522, 513, 480, 503, 477, 530, 519, 500, 490, 508, 506, 528, 494, 513, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 547, 542, 525, 491, 540, 501, 518, 506, 500, 498, 512, 502, 519, 476, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 530, 529, 504, 498, 477, 512, 496, 528, 532, 544, 532, 520, 532, 533, 516, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 514, 521, 507, 539, 523, 521, 488, 544, 488, 519, 472, 520, 492, 518, 495, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8301, 8101, 8052, 8150, 8226, 8305, 8142] avg=8192.0000 max=8305 max_rank=6 min=8052 min_rank=3 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[507, 503, 535, 480, 523, 541, 514, 487, 506, 500, 531, 497, 492, 496, 493, 492, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 476, 494, 542, 483, 536, 547, 523, 520, 503, 518, 498, 469, 505, 548, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 489, 505, 540, 508, 567, 518, 505, 512, 480, 512, 486, 534, 541, 489, 503, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 542, 514, 498, 509, 508, 499, 541, 489, 540, 497, 488, 543, 540, 473, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 522, 487, 508, 499, 527, 510, 488, 513, 517, 496, 480, 526, 540, 552, 528, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 550, 505, 512, 486, 491, 511, 483, 522, 507, 515, 521, 524, 508, 518, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 494, 492, 537, 470, 515, 515, 518, 512, 502, 490, 499, 538, 515, 551, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 502, 491, 503, 526, 503, 503, 495, 502, 499, 520, 516, 450, 514, 515, 538, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8213, 8260, 8223, 8223, 8211, 8224, 8081] avg=8192.0000 max=8260 max_rank=2 min=8081 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 495, 506, 496, 536, 517, 533, 516, 507, 502, 489, 523, 496, 524, 534, 517, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 530, 524, 505, 500, 528, 502, 544, 503, 540, 502, 510, 538, 518, 540, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 520, 515, 490, 528, 466, 511, 498, 540, 482, 541, 504, 503, 527, 516, 532, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 539, 511, 481, 511, 513, 500, 515, 508, 501, 495, 495, 499, 499, 525, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 529, 491, 466, 508, 486, 502, 530, 536, 491, 461, 504, 540, 515, 496, 537, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 491, 541, 514, 480, 515, 536, 503, 500, 482, 482, 532, 499, 500, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 562, 517, 530, 479, 526, 529, 505, 531, 484, 524, 493, 507, 483, 495, 535, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 544, 514, 541, 514, 498, 499, 541, 481, 498, 518, 505, 565, 522, 496, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8305, 8229, 8141, 8106, 8112, 8224, 8215] avg=8192.0000 max=8305 max_rank=1 min=8106 min_rank=4 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 523, 515, 506, 513, 497, 464, 501, 537, 554, 523, 529, 471, 502, 528, 535, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 529, 530, 529, 481, 536, 523, 496, 505, 513, 503, 457, 537, 493, 521, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 500, 586, 483, 485, 474, 542, 517, 506, 459, 529, 486, 527, 537, 546, 490, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 497, 503, 508, 487, 546, 504, 514, 559, 508, 492, 517, 506, 478, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 494, 539, 522, 522, 524, 531, 528, 503, 483, 546, 494, 518, 535, 478, 522, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 515, 521, 512, 538, 493, 543, 520, 553, 490, 502, 528, 504, 544, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 489, 493, 449, 509, 480, 482, 497, 503, 538, 509, 528, 491, 522, 489, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 519, 491, 530, 522, 521, 511, 515, 516, 510, 488, 515, 484, 516, 546, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8183, 8167, 8162, 8235, 8297, 8042, 8224] avg=8192.0000 max=8297 max_rank=5 min=8042 min_rank=6 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 3840, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4352, 4096, 4096, 3968, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[547, 512, 484, 527, 525, 540, 499, 497, 497, 505, 515, 476, 476, 518, 545, 517, 502, 525, 503, 504, 500, 553, 506, 496, 500, 519, 506, 551, 529, 549, 525, 503, 518, 522, 505, 516, 484, 557, 531, 512, 518, 465, 537, 488, 517, 480, 487, 499, 502, 530, 512, 540, 512, 531, 498, 512, 533, 499, 481, 539, 486, 524, 513, 494, 552, 502, 552, 561, 498, 523, 496, 512, 545, 507, 500, 495, 493, 510, 513, 461, 484, 497, 531, 483, 529, 475, 564, 582, 479, 531, 527, 470, 508, 533, 508, 497, 503, 525, 509, 499, 501, 494, 511, 475, 491, 465, 532, 531, 528, 494, 512, 498, 514, 519, 505, 528, 513, 495, 494, 489, 520, 526, 505, 533, 540, 551, 488, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[500, 526, 533, 476, 517, 516, 488, 544, 498, 519, 528, 524, 493, 511, 498, 526, 475, 503, 534, 518, 527, 534, 495, 425, 499, 512, 544, 545, 527, 481, 521, 497, 504, 521, 503, 531, 527, 555, 512, 540, 510, 499, 506, 507, 509, 507, 503, 548, 498, 519, 555, 503, 480, 530, 495, 517, 510, 498, 531, 523, 467, 540, 510, 517, 481, 522, 551, 508, 485, 513, 495, 508, 521, 516, 513, 511, 522, 517, 528, 523, 532, 512, 476, 522, 521, 505, 512, 566, 528, 506, 531, 484, 533, 522, 485, 521, 455, 470, 496, 512, 508, 530, 513, 496, 542, 520, 523, 530, 583, 503, 472, 497, 517, 469, 573, 482, 457, 517, 455, 508, 516, 488, 495, 530, 526, 498, 539, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[503, 502, 521, 537, 505, 489, 562, 515, 462, 537, 490, 574, 499, 516, 497, 530, 517, 478, 532, 504, 486, 491, 527, 492, 508, 488, 503, 527, 518, 493, 488, 490, 540, 514, 512, 568, 513, 497, 506, 533, 543, 511, 496, 496, 505, 461, 540, 507, 565, 521, 516, 492, 512, 506, 506, 517, 488, 484, 503, 491, 540, 495, 538, 526, 487, 492, 517, 496, 535, 510, 509, 519, 488, 489, 529, 496, 490, 489, 496, 517, 530, 505, 505, 546, 556, 550, 541, 539, 478, 539, 494, 508, 488, 505, 483, 558, 544, 494, 521, 540, 497, 500, 515, 476, 499, 547, 529, 496, 501, 524, 525, 518, 479, 490, 562, 489, 524, 502, 547, 512, 504, 526, 523, 492, 505, 538, 520, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[495, 495, 478, 483, 537, 501, 530, 525, 509, 485, 517, 486, 521, 516, 533, 540, 485, 515, 498, 518, 453, 515, 494, 515, 512, 485, 502, 487, 501, 506, 520, 537, 543, 471, 513, 521, 517, 522, 544, 493, 508, 492, 529, 491, 555, 481, 521, 528, 549, 509, 503, 507, 521, 506, 525, 491, 488, 544, 512, 501, 529, 519, 501, 499, 510, 532, 490, 507, 486, 507, 510, 533, 486, 509, 517, 497, 528, 484, 521, 507, 509, 487, 545, 511, 509, 497, 518, 511, 511, 528, 530, 545, 519, 494, 511, 493, 529, 528, 502, 537, 549, 537, 541, 536, 495, 523, 514, 514, 536, 547, 509, 536, 546, 527, 552, 491, 522, 542, 499, 519, 529, 485, 497, 470, 521, 455, 514, 465] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[553, 503, 510, 532, 531, 473, 518, 488, 545, 569, 493, 486, 500, 494, 522, 485, 561, 484, 463, 470, 520, 522, 534, 517, 501, 485, 518, 503, 496, 528, 503, 525, 535, 500, 475, 487, 520, 498, 563, 480, 541, 519, 520, 485, 535, 506, 500, 461, 487, 538, 511, 559, 536, 503, 508, 506, 488, 508, 528, 530, 489, 489, 507, 499, 532, 507, 529, 490, 455, 504, 505, 502, 538, 517, 526, 536, 516, 511, 503, 500, 520, 502, 510, 553, 510, 471, 483, 535, 502, 535, 502, 497, 537, 484, 493, 505, 510, 504, 534, 512, 535, 512, 507, 529, 549, 533, 531, 514, 507, 515, 506, 487, 545, 552, 518, 541, 504, 529, 506, 506, 513, 529, 536, 491, 475, 517, 511, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[526, 522, 498, 518, 510, 544, 502, 519, 503, 516, 527, 477, 504, 495, 513, 538, 516, 460, 502, 519, 536, 582, 520, 495, 490, 548, 491, 500, 506, 525, 515, 525, 503, 514, 519, 513, 498, 541, 499, 497, 518, 532, 529, 461, 529, 539, 520, 514, 489, 557, 468, 520, 502, 496, 504, 508, 486, 490, 530, 509, 491, 524, 515, 538, 562, 507, 509, 520, 536, 515, 485, 524, 503, 545, 502, 491, 503, 536, 482, 531, 489, 525, 495, 484, 535, 568, 503, 525, 536, 500, 510, 474, 554, 513, 505, 553, 512, 513, 460, 524, 500, 542, 505, 487, 530, 532, 526, 496, 467, 561, 475, 480, 539, 523, 516, 507, 485, 537, 536, 493, 514, 470, 492, 473, 506, 512, 499, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[523, 492, 501, 529, 520, 498, 562, 532, 507, 500, 531, 503, 485, 494, 513, 508, 549, 539, 484, 493, 511, 515, 486, 495, 506, 517, 459, 551, 524, 534, 492, 502, 524, 539, 532, 472, 517, 538, 491, 507, 493, 526, 517, 526, 509, 499, 529, 505, 497, 520, 533, 507, 530, 527, 524, 480, 559, 545, 518, 489, 475, 526, 506, 494, 513, 531, 494, 521, 505, 515, 514, 509, 519, 537, 515, 508, 497, 554, 469, 512, 511, 502, 519, 514, 490, 508, 484, 518, 506, 540, 507, 492, 523, 518, 512, 541, 530, 499, 491, 497, 512, 530, 536, 489, 483, 507, 530, 561, 541, 495, 507, 496, 533, 503, 520, 508, 499, 464, 493, 521, 491, 526, 523, 533, 511, 492, 495, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[519, 527, 513, 495, 484, 503, 472, 483, 542, 514, 502, 515, 526, 494, 490, 509, 519, 503, 524, 530, 480, 539, 504, 539, 486, 484, 509, 538, 468, 486, 526, 480, 528, 505, 500, 499, 553, 515, 534, 520, 499, 509, 484, 529, 528, 541, 479, 488, 507, 529, 514, 522, 519, 484, 549, 521, 528, 521, 485, 504, 489, 516, 504, 521, 511, 550, 523, 537, 500, 487, 495, 489, 492, 497, 485, 506, 533, 509, 516, 511, 529, 542, 529, 503, 482, 524, 489, 498, 529, 512, 507, 489, 530, 541, 519, 499, 546, 518, 468, 553, 493, 514, 524, 518, 522, 524, 554, 498, 522, 551, 526, 488, 468, 483, 541, 535, 522, 553, 500, 508, 491, 516, 515, 506, 531, 512, 543, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[547, 512, 484, 527, 525, 540, 499, 497, 497, 505, 515, 476, 476, 518, 545, 517, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 503, 504, 500, 553, 506, 496, 500, 519, 506, 551, 529, 549, 525, 503, 37, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 522, 505, 516, 484, 540, 531, 512, 518, 465, 537, 488, 517, 480, 487, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 528, 512, 540, 512, 531, 498, 512, 533, 499, 481, 539, 486, 524, 513, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 502, 552, 561, 498, 523, 496, 512, 545, 507, 500, 495, 493, 510, 513, 461, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 497, 531, 483, 529, 475, 564, 545, 479, 531, 527, 470, 508, 533, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 525, 509, 499, 501, 494, 511, 475, 491, 465, 501, 531, 528, 494, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 519, 505, 528, 513, 495, 494, 489, 520, 526, 505, 533, 540, 551, 488, 537, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8324, 8119, 8204, 8237, 8161, 8037, 8266] avg=8192.0000 max=8324 max_rank=1 min=8037 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[500, 526, 533, 476, 517, 516, 488, 544, 498, 519, 528, 524, 493, 511, 498, 526, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 503, 534, 518, 527, 534, 495, 425, 499, 512, 544, 545, 527, 481, 521, 497, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 503, 531, 527, 538, 512, 540, 510, 499, 506, 507, 509, 507, 503, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 517, 555, 503, 480, 530, 495, 517, 510, 498, 531, 523, 467, 540, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 522, 551, 508, 485, 513, 495, 508, 521, 516, 513, 511, 522, 517, 528, 523, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 476, 522, 521, 505, 512, 530, 528, 506, 531, 484, 533, 522, 485, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 470, 496, 512, 508, 530, 513, 496, 542, 520, 492, 530, 583, 503, 472, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 469, 573, 482, 457, 517, 455, 508, 516, 488, 495, 530, 526, 498, 539, 537, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8189, 8265, 8191, 8231, 8220, 8119, 8116] avg=8192.0000 max=8265 max_rank=2 min=8116 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 495, 478, 483, 537, 501, 530, 525, 509, 485, 517, 486, 521, 516, 533, 540, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 515, 498, 518, 453, 515, 494, 515, 512, 485, 502, 487, 501, 506, 520, 537, 32, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 471, 513, 521, 517, 506, 544, 493, 508, 492, 529, 491, 555, 481, 521, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 507, 503, 507, 521, 506, 525, 491, 488, 544, 512, 501, 529, 519, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 532, 490, 507, 486, 507, 510, 533, 486, 509, 517, 497, 528, 484, 521, 507, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 545, 511, 509, 497, 518, 479, 511, 528, 530, 545, 519, 494, 511, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 502, 537, 549, 537, 541, 536, 495, 523, 483, 514, 536, 547, 509, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 527, 552, 491, 522, 542, 499, 519, 529, 485, 497, 470, 521, 455, 514, 465, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8091, 8213, 8202, 8141, 8186, 8402, 8143] avg=8192.0000 max=8402 max_rank=6 min=8091 min_rank=1 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[526, 522, 498, 518, 510, 544, 502, 519, 503, 516, 527, 477, 504, 495, 513, 538, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 460, 502, 519, 536, 582, 520, 495, 490, 548, 491, 500, 506, 525, 515, 525, 38, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 519, 513, 498, 514, 499, 497, 518, 532, 529, 461, 529, 539, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 548, 468, 520, 502, 496, 504, 508, 486, 490, 530, 509, 491, 524, 515, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 507, 509, 520, 536, 515, 485, 524, 503, 545, 502, 491, 503, 536, 482, 531, 18, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 525, 495, 484, 535, 568, 503, 487, 536, 500, 510, 474, 554, 513, 505, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 513, 460, 524, 500, 542, 505, 487, 530, 532, 486, 496, 467, 561, 475, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 523, 516, 507, 485, 537, 536, 493, 514, 470, 492, 473, 506, 512, 499, 509, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8290, 8199, 8118, 8274, 8231, 8070, 8124] avg=8192.0000 max=8290 max_rank=1 min=8070 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[503, 502, 521, 537, 505, 489, 562, 515, 462, 537, 490, 574, 499, 516, 497, 530, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 478, 532, 504, 486, 491, 527, 492, 508, 488, 503, 527, 518, 493, 488, 490, 34, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 514, 512, 568, 513, 482, 506, 533, 543, 511, 496, 496, 505, 461, 540, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 519, 516, 492, 512, 506, 506, 517, 488, 484, 503, 491, 540, 495, 538, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 492, 517, 496, 535, 510, 509, 519, 488, 489, 529, 496, 490, 489, 496, 517, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 505, 546, 556, 550, 541, 505, 478, 539, 494, 508, 488, 505, 483, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 494, 521, 540, 497, 500, 515, 476, 499, 547, 498, 496, 501, 524, 525, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 490, 562, 489, 524, 502, 547, 512, 504, 526, 523, 492, 505, 538, 520, 490, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8092, 8227, 8198, 8076, 8291, 8195, 8211] avg=8192.0000 max=8291 max_rank=5 min=8076 min_rank=4 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[553, 503, 510, 532, 531, 473, 518, 488, 545, 569, 493, 486, 500, 494, 522, 485, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 484, 463, 470, 520, 522, 534, 517, 501, 485, 518, 503, 496, 528, 503, 525, 34, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 475, 487, 520, 483, 563, 480, 541, 519, 520, 485, 535, 506, 500, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 536, 511, 559, 536, 503, 508, 506, 488, 508, 528, 530, 489, 489, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 507, 529, 490, 455, 504, 505, 502, 538, 517, 526, 536, 516, 511, 503, 500, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 502, 510, 553, 510, 471, 483, 501, 502, 535, 502, 497, 537, 484, 493, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 534, 512, 535, 512, 507, 529, 549, 533, 500, 514, 507, 515, 506, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 552, 518, 541, 504, 529, 506, 506, 513, 529, 536, 491, 475, 517, 511, 525, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8180, 8110, 8184, 8188, 8105, 8254, 8306] avg=8192.0000 max=8306 max_rank=7 min=8105 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[523, 492, 501, 529, 520, 498, 562, 532, 507, 500, 531, 503, 485, 494, 513, 508, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 539, 484, 493, 511, 515, 486, 495, 506, 517, 459, 551, 524, 534, 492, 502, 33, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 539, 532, 472, 517, 522, 491, 507, 493, 526, 517, 526, 509, 499, 529, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 518, 533, 507, 530, 527, 524, 480, 559, 545, 518, 489, 475, 526, 506, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 494, 521, 505, 515, 514, 509, 519, 537, 515, 508, 497, 554, 469, 512, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 519, 514, 490, 508, 484, 485, 506, 540, 507, 492, 523, 518, 512, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 491, 497, 512, 530, 536, 489, 483, 507, 499, 561, 541, 495, 507, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 503, 520, 508, 499, 464, 493, 521, 491, 526, 523, 533, 511, 492, 495, 513, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8206, 8208, 8228, 8230, 8152, 8173, 8134] avg=8192.0000 max=8230 max_rank=4 min=8134 min_rank=7 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[519, 527, 513, 495, 484, 503, 472, 483, 542, 514, 502, 515, 526, 494, 490, 509, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 503, 524, 530, 480, 539, 504, 539, 486, 484, 509, 538, 468, 486, 526, 480, 32, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 500, 499, 553, 499, 534, 520, 499, 509, 484, 529, 528, 541, 479, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 527, 514, 522, 519, 484, 549, 521, 528, 521, 485, 504, 489, 516, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 550, 523, 537, 500, 487, 495, 489, 492, 497, 485, 506, 533, 509, 516, 511, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 542, 529, 503, 482, 524, 489, 466, 529, 512, 507, 489, 530, 541, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 518, 468, 553, 493, 514, 524, 518, 522, 524, 521, 498, 522, 551, 526, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 483, 541, 535, 522, 553, 500, 508, 491, 516, 515, 506, 531, 512, 543, 503, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8164, 8195, 8211, 8159, 8190, 8286, 8236] avg=8192.0000 max=8286 max_rank=6 min=8095 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[535, 522, 499, 491, 521, 517, 540, 541, 476, 486, 461, 495, 541, 535, 477, 550, 530, 552, 520, 516, 515, 492, 487, 504, 503, 536, 486, 492, 499, 498, 512, 489, 482, 520, 527, 500, 519, 534, 511, 494, 553, 492, 530, 531, 497, 515, 496, 510, 549, 511, 493, 521, 458, 506, 518, 524, 533, 516, 509, 508, 482, 509, 527, 544, 489, 506, 514, 505, 549, 466, 525, 545, 537, 503, 522, 528, 499, 519, 492, 511, 522, 520, 529, 509, 472, 524, 509, 491, 518, 516, 531, 491, 521, 499, 520, 524, 525, 483, 501, 504, 525, 544, 530, 504, 491, 521, 528, 487, 492, 474, 568, 509, 516, 499, 500, 556, 511, 481, 493, 532, 545, 540, 503, 502, 517, 508, 479, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[499, 499, 508, 522, 541, 531, 480, 489, 513, 488, 495, 535, 538, 530, 529, 504, 526, 505, 509, 497, 489, 487, 522, 536, 501, 529, 478, 538, 531, 475, 512, 515, 538, 472, 566, 523, 524, 521, 499, 514, 516, 494, 488, 500, 537, 543, 526, 514, 545, 499, 501, 522, 497, 491, 469, 487, 521, 483, 505, 479, 527, 507, 511, 539, 495, 490, 516, 554, 509, 502, 550, 551, 484, 454, 529, 510, 479, 543, 470, 502, 531, 485, 492, 511, 520, 530, 531, 499, 489, 527, 511, 537, 541, 537, 561, 491, 547, 521, 490, 541, 510, 522, 519, 501, 525, 497, 480, 542, 515, 504, 501, 501, 519, 527, 517, 554, 501, 492, 520, 496, 502, 523, 506, 477, 483, 548, 517, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[519, 509, 483, 498, 485, 512, 504, 515, 493, 521, 526, 526, 518, 494, 508, 523, 506, 510, 523, 502, 515, 481, 520, 519, 557, 535, 523, 557, 560, 500, 507, 501, 520, 492, 516, 504, 458, 490, 508, 484, 506, 489, 469, 498, 512, 530, 530, 545, 515, 500, 515, 490, 515, 499, 455, 470, 482, 519, 519, 525, 496, 508, 520, 518, 520, 506, 553, 488, 471, 545, 519, 530, 514, 544, 512, 539, 498, 536, 473, 517, 521, 528, 508, 528, 513, 468, 529, 524, 531, 528, 501, 548, 515, 510, 522, 529, 448, 527, 529, 505, 490, 520, 485, 501, 529, 498, 507, 560, 560, 525, 510, 494, 525, 530, 497, 499, 525, 541, 503, 520, 501, 503, 547, 534, 472, 552, 504, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[462, 482, 523, 525, 517, 537, 544, 476, 484, 478, 513, 529, 517, 518, 528, 514, 514, 545, 535, 534, 530, 534, 491, 497, 533, 489, 482, 468, 575, 498, 500, 510, 529, 535, 556, 528, 487, 536, 546, 522, 522, 531, 532, 529, 506, 490, 527, 494, 533, 496, 534, 447, 501, 529, 492, 558, 511, 524, 533, 485, 582, 557, 477, 493, 529, 484, 526, 534, 558, 524, 518, 542, 500, 516, 466, 490, 492, 495, 472, 487, 528, 502, 520, 489, 484, 512, 532, 536, 509, 516, 501, 520, 517, 506, 492, 493, 533, 513, 494, 523, 541, 489, 500, 502, 509, 545, 485, 492, 533, 553, 501, 517, 529, 463, 521, 506, 510, 499, 517, 475, 480, 463, 490, 523, 515, 516, 490, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[555, 532, 532, 503, 485, 545, 489, 483, 557, 538, 522, 508, 557, 520, 526, 504, 518, 504, 532, 517, 515, 462, 505, 472, 483, 509, 536, 467, 523, 514, 500, 497, 513, 479, 507, 530, 501, 503, 476, 509, 498, 518, 521, 528, 491, 535, 494, 530, 513, 503, 509, 511, 482, 501, 511, 509, 532, 505, 508, 542, 505, 518, 507, 537, 553, 563, 495, 503, 498, 496, 533, 498, 558, 511, 519, 516, 552, 506, 521, 492, 523, 509, 505, 449, 504, 535, 517, 512, 524, 548, 463, 491, 503, 518, 536, 518, 503, 514, 505, 492, 515, 477, 482, 526, 542, 526, 515, 540, 539, 517, 492, 498, 541, 492, 493, 526, 489, 496, 500, 550, 512, 524, 495, 511, 514, 510, 495, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[530, 468, 512, 523, 537, 538, 483, 516, 498, 519, 458, 503, 524, 539, 474, 485, 498, 500, 523, 557, 515, 495, 492, 490, 525, 495, 522, 481, 511, 565, 522, 539, 497, 514, 543, 503, 523, 492, 558, 504, 493, 552, 521, 519, 509, 475, 495, 549, 535, 480, 500, 554, 596, 503, 517, 511, 525, 521, 506, 496, 485, 522, 517, 548, 493, 490, 536, 473, 489, 496, 503, 503, 489, 509, 566, 496, 532, 563, 487, 523, 511, 487, 536, 562, 463, 506, 508, 493, 541, 507, 522, 466, 521, 510, 467, 537, 518, 515, 531, 479, 527, 505, 519, 491, 496, 531, 481, 523, 528, 488, 514, 525, 498, 494, 478, 526, 519, 517, 504, 532, 509, 517, 514, 514, 519, 515, 479, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[457, 518, 529, 548, 501, 540, 581, 505, 473, 517, 510, 516, 506, 518, 544, 526, 528, 446, 528, 519, 536, 509, 519, 485, 500, 516, 515, 519, 544, 477, 490, 535, 510, 495, 487, 512, 504, 482, 507, 538, 503, 515, 559, 540, 539, 481, 494, 525, 486, 530, 527, 508, 501, 527, 527, 502, 508, 488, 531, 490, 507, 525, 526, 467, 509, 523, 528, 502, 509, 530, 547, 538, 507, 501, 516, 494, 518, 493, 505, 488, 475, 521, 512, 525, 506, 513, 509, 516, 493, 562, 522, 470, 519, 522, 507, 500, 499, 491, 543, 528, 509, 480, 506, 499, 510, 505, 482, 535, 501, 474, 504, 506, 495, 544, 540, 534, 522, 499, 531, 516, 553, 501, 503, 525, 526, 515, 509, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[523, 534, 509, 540, 531, 488, 508, 508, 483, 478, 490, 517, 503, 462, 478, 511, 533, 528, 530, 510, 532, 495, 489, 519, 512, 551, 536, 558, 513, 526, 490, 528, 488, 495, 512, 475, 479, 495, 535, 525, 530, 537, 481, 507, 523, 504, 513, 526, 499, 531, 541, 477, 468, 530, 511, 505, 516, 497, 483, 558, 527, 499, 487, 495, 514, 514, 511, 520, 495, 525, 506, 509, 499, 482, 505, 525, 514, 521, 517, 518, 497, 536, 528, 523, 517, 517, 492, 537, 534, 517, 503, 536, 537, 498, 505, 473, 507, 530, 508, 547, 468, 554, 545, 534, 540, 538, 509, 503, 507, 509, 505, 502, 530, 530, 521, 517, 493, 513, 480, 500, 474, 525, 548, 527, 471, 492, 526, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[499, 499, 508, 522, 541, 531, 480, 489, 513, 488, 495, 535, 538, 530, 529, 504, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 509, 497, 489, 487, 522, 536, 501, 529, 478, 538, 520, 475, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 472, 558, 523, 524, 521, 499, 514, 516, 494, 488, 500, 537, 543, 526, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 499, 501, 522, 497, 491, 469, 487, 521, 483, 505, 479, 527, 507, 511, 539, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 490, 516, 554, 509, 502, 550, 545, 484, 454, 529, 510, 479, 543, 470, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 485, 492, 511, 520, 530, 531, 499, 489, 512, 511, 537, 541, 537, 561, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 521, 490, 541, 510, 522, 519, 501, 525, 497, 480, 540, 515, 504, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 527, 517, 554, 501, 492, 520, 496, 502, 523, 506, 477, 483, 548, 517, 498, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8139, 8267, 8102, 8132, 8278, 8214, 8197] avg=8192.0000 max=8278 max_rank=5 min=8102 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[535, 522, 499, 491, 521, 517, 540, 541, 476, 486, 461, 495, 541, 535, 477, 550, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 552, 520, 516, 515, 492, 487, 504, 503, 536, 486, 492, 489, 498, 512, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 520, 519, 500, 519, 534, 511, 494, 553, 492, 530, 531, 497, 515, 496, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 511, 493, 521, 458, 506, 518, 524, 533, 516, 509, 508, 482, 509, 527, 544, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 506, 514, 505, 549, 466, 525, 539, 537, 503, 522, 528, 499, 519, 492, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 529, 509, 472, 524, 509, 491, 518, 501, 531, 491, 521, 499, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 483, 501, 504, 525, 544, 530, 504, 491, 521, 528, 486, 492, 474, 568, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 500, 556, 511, 481, 493, 532, 545, 540, 503, 502, 517, 508, 479, 525, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8121, 8203, 8227, 8204, 8181, 8185, 8223] avg=8192.0000 max=8227 max_rank=3 min=8121 min_rank=1 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[519, 509, 483, 498, 485, 512, 504, 515, 493, 521, 526, 526, 518, 494, 508, 523, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 523, 502, 515, 481, 520, 519, 557, 535, 523, 557, 549, 500, 507, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 492, 508, 504, 458, 490, 508, 484, 506, 489, 469, 498, 512, 530, 530, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 500, 515, 490, 515, 499, 455, 470, 482, 519, 519, 525, 496, 508, 520, 518, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 506, 553, 488, 471, 545, 519, 524, 514, 544, 512, 539, 498, 536, 473, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 528, 508, 528, 513, 468, 529, 524, 531, 513, 501, 548, 515, 510, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 527, 529, 505, 490, 520, 485, 501, 529, 498, 507, 558, 560, 525, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 530, 497, 499, 525, 541, 503, 520, 501, 503, 547, 534, 472, 552, 504, 480, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8305, 8043, 8065, 8259, 8288, 8186, 8250] avg=8192.0000 max=8305 max_rank=1 min=8043 min_rank=2 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[530, 468, 512, 523, 537, 538, 483, 516, 498, 519, 458, 503, 524, 539, 474, 485, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 500, 523, 557, 515, 495, 492, 490, 525, 495, 522, 481, 491, 565, 522, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 531, 503, 523, 492, 558, 504, 493, 552, 521, 519, 509, 475, 495, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 480, 500, 554, 596, 503, 517, 511, 525, 521, 506, 496, 485, 522, 517, 548, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 490, 536, 473, 489, 496, 503, 490, 489, 509, 566, 496, 532, 563, 487, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 487, 536, 562, 463, 506, 508, 493, 541, 487, 522, 466, 521, 510, 467, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 515, 531, 479, 527, 505, 519, 491, 496, 531, 481, 518, 528, 488, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 478, 526, 519, 517, 504, 532, 509, 517, 514, 514, 519, 515, 479, 545, 12, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8210, 8235, 8345, 8135, 8117, 8166, 8208] avg=8192.0000 max=8345 max_rank=3 min=8117 min_rank=5 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[462, 482, 523, 525, 517, 537, 544, 476, 484, 478, 513, 529, 517, 518, 528, 514, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 545, 535, 534, 530, 534, 491, 497, 533, 489, 482, 468, 564, 498, 500, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 535, 548, 528, 487, 536, 546, 522, 522, 531, 532, 529, 506, 490, 527, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 496, 534, 447, 501, 529, 492, 558, 511, 524, 533, 485, 582, 557, 477, 493, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 484, 526, 534, 558, 524, 518, 536, 500, 516, 466, 490, 492, 495, 472, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 502, 520, 489, 484, 512, 532, 536, 509, 501, 501, 520, 517, 506, 492, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 513, 494, 523, 541, 489, 500, 502, 509, 545, 485, 491, 533, 553, 501, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 463, 521, 506, 510, 499, 517, 475, 480, 463, 490, 523, 515, 516, 490, 515, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8224, 8362, 8271, 8127, 8142, 8229, 8029] avg=8192.0000 max=8362 max_rank=2 min=8029 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[457, 518, 529, 548, 501, 540, 581, 505, 473, 517, 510, 516, 506, 518, 544, 526, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 446, 528, 519, 536, 509, 519, 485, 500, 516, 515, 519, 533, 477, 490, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 495, 480, 512, 504, 482, 507, 538, 503, 515, 559, 540, 539, 481, 494, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 527, 508, 501, 527, 527, 502, 508, 488, 531, 490, 507, 525, 526, 467, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 523, 528, 502, 509, 530, 547, 532, 507, 501, 516, 494, 518, 493, 505, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 521, 512, 525, 506, 513, 509, 516, 493, 546, 522, 470, 519, 522, 507, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 491, 543, 528, 509, 480, 506, 499, 510, 505, 482, 533, 501, 474, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 544, 540, 534, 522, 499, 531, 516, 553, 501, 503, 525, 526, 515, 509, 475, 7, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8155, 8184, 8170, 8202, 8156, 8070, 8304] avg=8192.0000 max=8304 max_rank=7 min=8070 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[523, 534, 509, 540, 531, 488, 508, 508, 483, 478, 490, 517, 503, 462, 478, 511, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 528, 530, 510, 532, 495, 489, 519, 512, 551, 536, 558, 502, 526, 490, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 495, 504, 475, 479, 495, 535, 525, 530, 537, 481, 507, 523, 504, 513, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 531, 541, 477, 468, 530, 511, 505, 516, 497, 483, 558, 527, 499, 487, 495, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 511, 520, 495, 525, 506, 503, 499, 482, 505, 525, 514, 521, 517, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 536, 528, 523, 517, 517, 492, 537, 534, 502, 503, 536, 537, 498, 505, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 508, 547, 468, 554, 545, 534, 540, 538, 509, 502, 507, 509, 505, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 530, 521, 517, 493, 513, 480, 500, 474, 525, 548, 527, 471, 492, 526, 496, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 8339, 8117, 8143, 8169, 8235, 8305, 8160] avg=8192.0000 max=8339 max_rank=1 min=8068 min_rank=0 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[555, 532, 532, 503, 485, 545, 489, 483, 557, 538, 522, 508, 557, 520, 526, 504, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 504, 532, 517, 515, 462, 505, 472, 483, 509, 536, 467, 512, 514, 500, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 479, 499, 530, 501, 503, 476, 509, 498, 518, 521, 528, 491, 535, 494, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 503, 509, 511, 482, 501, 511, 509, 532, 505, 508, 542, 505, 518, 507, 537, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 563, 495, 503, 498, 496, 533, 492, 558, 511, 519, 516, 552, 506, 521, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 509, 505, 449, 504, 535, 517, 512, 524, 532, 463, 491, 503, 518, 536, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 505, 492, 515, 477, 482, 526, 542, 526, 515, 538, 539, 517, 492, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 492, 493, 526, 489, 496, 500, 550, 512, 524, 495, 511, 514, 510, 495, 500, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8362, 8043, 8125, 8213, 8308, 8139, 8181, 8165] avg=8192.0000 max=8362 max_rank=0 min=8043 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True + [2026-08-26 16:29:49] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 415.6 | throughput per GPU (TFLOP/s/GPU): 435.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.298552E+00 | loss scale: 1.0 | grad norm: 5.375 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[523, 546, 511, 516, 504, 534, 544, 511, 497, 482, 532, 557, 528, 516, 539, 524, 506, 482, 473, 541, 488, 510, 517, 512, 530, 520, 499, 523, 508, 493, 513, 497, 507, 510, 478, 490, 512, 506, 502, 522, 558, 524, 513, 490, 535, 541, 500, 524, 495, 538, 490, 517, 526, 520, 524, 498, 504, 514, 519, 518, 550, 503, 502, 524, 514, 479, 498, 499, 538, 520, 463, 491, 513, 509, 481, 517, 496, 522, 529, 537, 500, 519, 482, 516, 523, 455, 524, 552, 497, 511, 489, 515, 547, 495, 504, 513, 554, 510, 503, 509, 513, 476, 518, 506, 505, 519, 532, 495, 502, 489, 482, 473, 564, 501, 527, 483, 562, 524, 501, 528, 502, 492, 553, 504, 489, 523, 494, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[450, 530, 488, 510, 508, 518, 521, 517, 516, 536, 491, 504, 517, 530, 519, 520, 500, 552, 498, 531, 529, 525, 493, 547, 516, 494, 530, 482, 525, 537, 505, 525, 516, 475, 523, 529, 521, 483, 530, 502, 503, 466, 496, 504, 483, 507, 515, 502, 510, 497, 509, 484, 554, 521, 517, 502, 523, 501, 500, 467, 505, 524, 523, 497, 530, 522, 517, 541, 550, 511, 533, 477, 499, 501, 528, 541, 533, 525, 522, 535, 538, 489, 507, 507, 538, 509, 528, 507, 504, 501, 521, 497, 509, 516, 518, 509, 533, 490, 524, 533, 479, 480, 510, 521, 509, 511, 492, 503, 516, 492, 503, 534, 525, 505, 468, 506, 549, 535, 485, 502, 510, 504, 513, 502, 527, 521, 508, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[490, 481, 521, 485, 552, 529, 469, 510, 523, 513, 488, 518, 492, 529, 534, 514, 526, 540, 510, 520, 500, 507, 504, 521, 454, 486, 526, 497, 523, 504, 518, 512, 507, 513, 552, 498, 518, 500, 530, 502, 475, 486, 553, 462, 523, 524, 501, 497, 469, 499, 507, 518, 493, 545, 526, 547, 493, 528, 505, 505, 476, 512, 518, 538, 518, 503, 476, 535, 535, 470, 523, 495, 490, 504, 514, 538, 523, 495, 540, 513, 542, 518, 504, 501, 537, 537, 551, 509, 540, 522, 514, 495, 545, 516, 530, 505, 539, 490, 504, 481, 507, 531, 505, 523, 498, 482, 510, 555, 532, 553, 486, 490, 498, 503, 509, 534, 495, 532, 527, 512, 518, 519, 529, 519, 514, 469, 496, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[493, 489, 494, 529, 511, 516, 516, 531, 506, 511, 477, 491, 484, 516, 513, 521, 522, 538, 551, 508, 474, 512, 517, 510, 502, 541, 512, 534, 492, 520, 537, 540, 503, 471, 518, 538, 525, 562, 479, 526, 543, 552, 483, 484, 536, 509, 478, 493, 531, 524, 533, 524, 486, 546, 520, 520, 524, 528, 488, 478, 489, 512, 509, 497, 534, 502, 489, 464, 513, 494, 515, 526, 544, 561, 535, 532, 490, 472, 574, 550, 509, 498, 485, 541, 484, 542, 513, 487, 510, 538, 494, 510, 543, 532, 488, 495, 503, 491, 494, 525, 554, 486, 488, 485, 524, 532, 519, 507, 516, 505, 491, 474, 513, 531, 511, 523, 495, 492, 491, 554, 478, 513, 523, 511, 514, 539, 524, 449] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[518, 510, 533, 528, 554, 511, 514, 513, 492, 488, 503, 511, 489, 540, 526, 521, 520, 536, 517, 509, 502, 510, 513, 504, 509, 530, 522, 488, 488, 522, 493, 495, 498, 494, 493, 515, 490, 542, 518, 488, 487, 527, 511, 524, 525, 516, 514, 523, 523, 557, 525, 534, 511, 552, 475, 494, 513, 507, 514, 526, 498, 490, 486, 464, 521, 495, 542, 573, 464, 511, 516, 519, 539, 498, 476, 516, 497, 511, 499, 485, 521, 519, 540, 522, 547, 492, 509, 548, 502, 516, 502, 529, 495, 501, 502, 491, 476, 523, 496, 482, 498, 498, 548, 522, 485, 472, 457, 511, 505, 556, 529, 520, 520, 549, 532, 527, 489, 492, 484, 564, 553, 505, 489, 548, 513, 523, 517, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[514, 511, 499, 524, 525, 508, 482, 472, 483, 514, 510, 513, 542, 520, 538, 548, 524, 538, 519, 515, 525, 509, 518, 506, 509, 529, 506, 537, 499, 512, 516, 500, 486, 511, 521, 491, 511, 503, 567, 474, 500, 524, 571, 529, 545, 495, 506, 540, 548, 542, 482, 499, 487, 490, 499, 493, 539, 521, 537, 495, 521, 509, 518, 500, 547, 508, 535, 515, 470, 515, 505, 509, 509, 526, 494, 503, 545, 499, 508, 538, 504, 509, 538, 468, 539, 501, 496, 516, 511, 505, 524, 510, 526, 498, 532, 503, 538, 540, 521, 495, 504, 512, 521, 494, 540, 490, 484, 486, 488, 503, 498, 482, 469, 469, 525, 540, 517, 513, 508, 489, 499, 505, 515, 515, 495, 540, 512, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[522, 510, 504, 494, 482, 493, 530, 501, 524, 504, 521, 535, 589, 540, 483, 536, 531, 551, 485, 516, 517, 507, 556, 521, 497, 515, 496, 503, 496, 523, 480, 511, 518, 478, 473, 494, 529, 485, 512, 547, 511, 553, 515, 515, 495, 528, 476, 507, 513, 525, 487, 537, 489, 527, 517, 502, 543, 495, 491, 497, 526, 500, 523, 464, 525, 537, 497, 515, 487, 491, 548, 506, 500, 509, 512, 528, 502, 487, 503, 526, 518, 469, 513, 474, 526, 481, 505, 543, 551, 496, 528, 498, 540, 498, 530, 514, 510, 496, 533, 509, 481, 533, 545, 488, 529, 462, 557, 530, 525, 547, 469, 533, 494, 494, 521, 514, 497, 523, 512, 532, 549, 539, 502, 497, 489, 512, 491, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[545, 496, 526, 520, 506, 489, 501, 558, 529, 515, 522, 520, 491, 532, 511, 491, 540, 494, 465, 559, 476, 515, 521, 561, 516, 508, 499, 557, 519, 566, 490, 548, 501, 532, 523, 531, 524, 521, 516, 503, 522, 507, 517, 475, 542, 525, 510, 522, 519, 468, 506, 502, 529, 518, 433, 516, 502, 530, 473, 518, 506, 481, 496, 495, 483, 522, 529, 529, 486, 539, 524, 513, 500, 549, 528, 515, 519, 526, 502, 552, 541, 513, 522, 530, 529, 504, 504, 518, 468, 486, 550, 497, 493, 489, 476, 486, 501, 518, 487, 502, 506, 475, 524, 527, 505, 514, 509, 500, 541, 518, 500, 501, 495, 504, 527, 519, 524, 527, 482, 525, 510, 482, 515, 492, 521, 518, 463, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 546, 511, 516, 504, 534, 544, 511, 497, 482, 532, 557, 528, 489, 539, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 450, 473, 541, 488, 510, 517, 512, 530, 520, 499, 523, 508, 493, 513, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 478, 490, 512, 506, 502, 522, 558, 524, 513, 490, 535, 541, 500, 524, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 538, 490, 517, 526, 520, 524, 498, 504, 514, 519, 518, 550, 503, 502, 524, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 479, 498, 499, 538, 520, 463, 491, 513, 509, 481, 517, 496, 522, 529, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 519, 482, 516, 518, 455, 524, 552, 497, 511, 489, 515, 547, 495, 504, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 510, 503, 509, 513, 476, 518, 506, 505, 519, 532, 495, 502, 489, 482, 473, 32, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 501, 527, 483, 562, 524, 501, 524, 502, 492, 553, 504, 489, 523, 494, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8337, 8080, 8221, 8277, 8072, 8137, 8144, 8268] avg=8192.0000 max=8337 max_rank=0 min=8072 min_rank=4 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32621 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[450, 530, 488, 510, 508, 518, 521, 517, 516, 536, 491, 504, 517, 499, 519, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 498, 531, 529, 525, 493, 547, 516, 494, 530, 482, 525, 537, 505, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 475, 523, 529, 521, 483, 530, 502, 503, 466, 496, 504, 483, 507, 515, 502, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 497, 509, 484, 554, 521, 517, 502, 523, 501, 500, 467, 505, 524, 523, 497, 31, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 522, 517, 541, 550, 511, 533, 477, 499, 501, 528, 541, 533, 525, 522, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 507, 507, 527, 509, 528, 507, 504, 501, 521, 497, 509, 516, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 490, 524, 533, 479, 480, 510, 521, 509, 511, 492, 503, 516, 492, 503, 534, 42, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 468, 506, 549, 535, 485, 493, 510, 504, 513, 502, 527, 521, 508, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8247, 8073, 8179, 8323, 8187, 8202, 8181] avg=8192.0000 max=8323 max_rank=4 min=8073 min_rank=2 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[493, 489, 494, 529, 511, 516, 516, 531, 506, 511, 477, 491, 484, 489, 513, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 551, 508, 474, 512, 517, 510, 502, 541, 512, 534, 492, 520, 537, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 471, 518, 538, 525, 562, 479, 526, 543, 552, 483, 484, 536, 509, 478, 493, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 524, 533, 524, 486, 546, 520, 520, 524, 528, 488, 478, 489, 512, 509, 497, 27, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 502, 489, 464, 513, 494, 515, 526, 544, 561, 535, 532, 490, 472, 574, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 485, 541, 480, 542, 513, 487, 510, 538, 494, 510, 543, 532, 488, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 494, 525, 554, 486, 488, 485, 524, 532, 519, 507, 516, 505, 491, 474, 35, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 511, 523, 495, 492, 491, 549, 478, 513, 523, 511, 514, 539, 524, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8275, 8209, 8245, 8259, 8165, 8156, 8156] avg=8192.0000 max=8275 max_rank=1 min=8071 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[490, 481, 521, 485, 552, 529, 469, 510, 523, 513, 488, 518, 492, 502, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 510, 520, 500, 507, 504, 521, 454, 486, 526, 497, 523, 504, 518, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 513, 552, 498, 518, 500, 530, 502, 475, 486, 553, 462, 523, 524, 501, 497, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 499, 507, 518, 493, 545, 526, 547, 493, 528, 505, 505, 476, 512, 518, 538, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 476, 535, 535, 470, 523, 495, 490, 504, 514, 538, 523, 495, 540, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 504, 501, 532, 537, 551, 509, 540, 522, 514, 495, 545, 516, 530, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 490, 504, 481, 507, 531, 505, 523, 498, 482, 510, 555, 532, 553, 486, 490, 35, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 509, 534, 495, 532, 527, 508, 518, 519, 529, 519, 514, 469, 496, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8113, 8150, 8214, 8139, 8361, 8246, 8192] avg=8192.0000 max=8361 max_rank=5 min=8113 min_rank=1 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 511, 499, 524, 525, 508, 482, 472, 483, 514, 510, 513, 542, 493, 538, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 503, 519, 515, 525, 509, 518, 506, 509, 529, 506, 537, 499, 512, 516, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 511, 521, 491, 511, 503, 567, 474, 500, 524, 571, 529, 545, 495, 506, 540, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 542, 482, 499, 487, 490, 499, 493, 539, 521, 537, 495, 521, 509, 518, 500, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 508, 535, 515, 470, 515, 505, 509, 509, 526, 494, 503, 545, 499, 508, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 538, 468, 534, 501, 496, 516, 511, 505, 524, 510, 526, 498, 532, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 540, 521, 495, 504, 512, 521, 494, 540, 490, 484, 486, 488, 503, 498, 482, 35, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 469, 525, 540, 517, 513, 508, 485, 499, 505, 515, 515, 495, 540, 512, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8227, 8283, 8215, 8192, 8175, 8157, 8111] avg=8192.0000 max=8283 max_rank=2 min=8111 min_rank=7 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[522, 510, 504, 494, 482, 493, 530, 501, 524, 504, 521, 535, 589, 512, 483, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 515, 485, 516, 517, 507, 556, 521, 497, 515, 496, 503, 496, 523, 480, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 478, 473, 494, 529, 485, 512, 547, 511, 553, 515, 515, 495, 528, 476, 507, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 487, 537, 489, 527, 517, 502, 543, 495, 491, 497, 526, 500, 523, 464, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 537, 497, 515, 487, 491, 548, 506, 500, 509, 512, 528, 502, 487, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 469, 513, 474, 521, 481, 505, 543, 551, 496, 528, 498, 540, 498, 530, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 496, 533, 509, 481, 533, 545, 488, 529, 462, 557, 530, 525, 547, 469, 533, 36, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 494, 521, 514, 497, 523, 512, 528, 549, 539, 502, 497, 489, 512, 491, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8169, 8145, 8172, 8140, 8179, 8308, 8183] avg=8192.0000 max=8308 max_rank=6 min=8140 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 510, 533, 528, 554, 511, 514, 513, 492, 488, 503, 511, 489, 512, 526, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 517, 509, 502, 510, 513, 504, 509, 530, 522, 488, 488, 522, 493, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 493, 515, 490, 542, 518, 488, 487, 527, 511, 524, 525, 516, 514, 523, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 557, 525, 534, 511, 552, 475, 494, 513, 507, 514, 526, 498, 490, 486, 464, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 495, 542, 573, 464, 511, 516, 519, 539, 498, 476, 516, 497, 511, 499, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 540, 522, 542, 492, 509, 548, 502, 516, 502, 529, 495, 501, 502, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 523, 496, 482, 498, 498, 548, 522, 485, 472, 457, 511, 505, 556, 529, 520, 35, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 549, 532, 527, 489, 492, 484, 559, 553, 505, 489, 548, 513, 523, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8123, 8175, 8205, 8131, 8231, 8136, 8312] avg=8192.0000 max=8312 max_rank=7 min=8123 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[545, 496, 526, 520, 506, 489, 501, 558, 529, 515, 522, 520, 491, 504, 511, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 462, 465, 559, 476, 515, 521, 561, 516, 508, 499, 557, 519, 566, 490, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 523, 531, 524, 521, 516, 503, 522, 507, 517, 475, 542, 525, 510, 522, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 468, 506, 502, 529, 518, 433, 516, 502, 530, 473, 518, 506, 481, 496, 495, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 522, 529, 529, 486, 539, 524, 513, 500, 549, 528, 515, 519, 526, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 513, 522, 530, 524, 504, 504, 518, 468, 486, 550, 497, 493, 489, 476, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 487, 502, 506, 475, 524, 527, 505, 514, 509, 500, 541, 518, 500, 501, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 504, 527, 519, 524, 527, 482, 521, 510, 482, 515, 492, 521, 518, 463, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8302, 8280, 8029, 8280, 8101, 8187, 8133] avg=8192.0000 max=8302 max_rank=1 min=8029 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 3968, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[490, 528, 530, 544, 518, 517, 494, 520, 470, 541, 551, 516, 495, 521, 516, 472, 486, 529, 508, 499, 487, 505, 513, 543, 518, 530, 519, 511, 535, 534, 519, 519, 550, 515, 522, 514, 496, 539, 537, 517, 475, 511, 505, 526, 537, 500, 523, 494, 534, 487, 575, 506, 486, 496, 497, 508, 503, 449, 502, 512, 544, 512, 511, 497, 461, 539, 488, 475, 504, 520, 517, 549, 505, 527, 522, 501, 481, 519, 507, 518, 487, 503, 488, 520, 499, 528, 535, 541, 529, 509, 517, 540, 506, 495, 491, 539, 479, 502, 494, 476, 516, 484, 479, 510, 503, 484, 561, 535, 515, 505, 488, 532, 503, 497, 526, 461, 510, 517, 518, 560, 559, 549, 547, 526, 503, 502, 476, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[502, 533, 546, 471, 509, 554, 494, 495, 559, 512, 493, 505, 539, 491, 506, 496, 503, 528, 486, 480, 496, 513, 504, 523, 509, 509, 489, 505, 509, 514, 519, 521, 549, 495, 531, 532, 502, 491, 490, 535, 513, 486, 514, 479, 508, 506, 546, 551, 528, 537, 524, 496, 570, 521, 497, 497, 523, 533, 505, 539, 529, 530, 526, 506, 516, 489, 506, 515, 529, 486, 535, 532, 494, 480, 470, 473, 561, 478, 510, 504, 542, 516, 499, 516, 523, 476, 521, 532, 528, 509, 569, 514, 534, 501, 504, 509, 534, 490, 485, 512, 515, 519, 506, 497, 502, 463, 498, 491, 498, 530, 496, 530, 505, 511, 522, 513, 498, 525, 527, 498, 518, 497, 490, 530, 518, 501, 516, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[539, 516, 527, 547, 541, 475, 552, 488, 496, 539, 530, 468, 479, 537, 505, 471, 570, 523, 488, 531, 509, 517, 512, 520, 513, 537, 544, 504, 539, 499, 542, 521, 521, 543, 480, 492, 497, 509, 502, 517, 540, 491, 507, 502, 515, 528, 494, 490, 525, 526, 504, 481, 532, 487, 482, 530, 500, 488, 514, 530, 490, 513, 531, 484, 512, 527, 505, 531, 495, 518, 502, 487, 511, 524, 506, 522, 503, 551, 539, 489, 509, 500, 542, 495, 516, 540, 496, 538, 512, 525, 494, 504, 492, 524, 492, 480, 471, 546, 486, 517, 537, 477, 496, 526, 505, 475, 512, 528, 528, 491, 498, 524, 505, 517, 524, 495, 508, 517, 522, 501, 509, 507, 513, 513, 532, 522, 493, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[530, 514, 525, 541, 516, 489, 499, 506, 493, 484, 537, 512, 559, 502, 539, 506, 500, 523, 538, 530, 520, 513, 530, 501, 525, 486, 515, 471, 516, 511, 536, 522, 496, 512, 520, 484, 491, 517, 483, 519, 590, 528, 496, 485, 516, 459, 515, 530, 529, 550, 491, 498, 522, 530, 445, 515, 507, 490, 494, 509, 510, 490, 518, 507, 531, 508, 520, 509, 532, 531, 500, 505, 504, 483, 519, 516, 523, 490, 482, 544, 460, 561, 526, 531, 491, 532, 538, 540, 503, 529, 512, 532, 509, 555, 477, 525, 505, 523, 520, 481, 478, 503, 549, 504, 541, 514, 508, 519, 550, 502, 472, 498, 537, 503, 455, 534, 543, 536, 489, 498, 491, 523, 494, 504, 490, 528, 517, 474] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[524, 493, 506, 489, 525, 529, 474, 513, 507, 532, 544, 511, 530, 493, 531, 513, 497, 517, 535, 522, 568, 482, 526, 551, 515, 512, 539, 501, 476, 525, 530, 530, 534, 503, 559, 505, 518, 501, 517, 538, 456, 513, 553, 526, 523, 529, 522, 522, 483, 522, 529, 503, 521, 508, 529, 504, 490, 489, 495, 537, 519, 481, 505, 456, 516, 494, 522, 517, 498, 519, 532, 500, 496, 521, 519, 512, 505, 530, 510, 515, 495, 515, 519, 500, 503, 455, 510, 487, 500, 538, 521, 489, 533, 483, 508, 500, 482, 479, 566, 476, 502, 508, 552, 529, 495, 494, 509, 528, 502, 520, 503, 536, 508, 489, 550, 515, 489, 484, 525, 520, 501, 557, 504, 507, 496, 528, 521, 469] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[519, 553, 480, 527, 485, 482, 521, 470, 515, 496, 466, 521, 504, 475, 544, 527, 513, 533, 512, 509, 523, 491, 546, 516, 551, 492, 532, 506, 504, 482, 472, 483, 468, 505, 530, 478, 552, 480, 529, 512, 548, 512, 478, 530, 499, 487, 514, 525, 509, 500, 512, 486, 538, 499, 519, 521, 517, 468, 514, 503, 512, 531, 509, 511, 504, 517, 500, 523, 480, 523, 510, 526, 502, 519, 530, 547, 513, 525, 506, 521, 474, 538, 530, 516, 466, 490, 510, 543, 535, 519, 528, 511, 505, 498, 539, 468, 494, 537, 516, 534, 505, 527, 523, 507, 487, 521, 498, 532, 530, 549, 537, 510, 533, 537, 497, 529, 498, 454, 500, 536, 493, 523, 487, 505, 534, 535, 565, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[518, 508, 468, 503, 523, 498, 546, 518, 498, 510, 540, 532, 494, 532, 493, 539, 500, 485, 478, 509, 525, 501, 527, 492, 530, 505, 526, 536, 496, 502, 505, 492, 486, 519, 494, 499, 535, 505, 500, 516, 495, 509, 505, 488, 544, 503, 562, 508, 516, 519, 538, 504, 552, 505, 483, 540, 509, 553, 524, 509, 505, 564, 533, 506, 505, 529, 515, 491, 508, 488, 516, 519, 491, 540, 483, 507, 493, 505, 539, 502, 510, 509, 516, 516, 487, 480, 511, 497, 494, 502, 529, 569, 528, 486, 502, 498, 545, 533, 513, 499, 491, 506, 521, 554, 539, 477, 501, 549, 494, 487, 535, 497, 498, 496, 541, 496, 487, 527, 499, 550, 492, 530, 502, 487, 533, 513, 541, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[497, 505, 501, 481, 552, 512, 511, 515, 551, 519, 525, 510, 528, 525, 485, 548, 536, 523, 516, 511, 506, 534, 528, 573, 466, 536, 543, 503, 565, 509, 473, 503, 494, 492, 475, 503, 548, 482, 515, 503, 526, 502, 508, 504, 440, 493, 502, 491, 516, 527, 502, 499, 485, 541, 547, 508, 508, 494, 522, 537, 551, 485, 498, 499, 504, 514, 532, 471, 498, 505, 537, 498, 560, 502, 517, 523, 517, 494, 528, 510, 522, 508, 524, 505, 513, 547, 522, 493, 487, 521, 506, 481, 514, 487, 473, 519, 521, 517, 520, 498, 502, 538, 514, 521, 471, 519, 519, 516, 504, 503, 513, 493, 541, 521, 512, 519, 504, 546, 508, 521, 493, 504, 477, 552, 518, 507, 493, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 528, 530, 544, 518, 517, 494, 520, 470, 541, 527, 516, 495, 521, 516, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 529, 508, 499, 487, 505, 513, 490, 518, 530, 519, 511, 535, 534, 519, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 515, 522, 514, 496, 539, 537, 517, 475, 511, 505, 526, 537, 500, 523, 494, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 487, 575, 506, 486, 496, 497, 508, 503, 449, 502, 512, 544, 512, 511, 497, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 539, 488, 475, 504, 520, 517, 549, 505, 527, 522, 501, 481, 519, 507, 518, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 503, 488, 520, 499, 528, 535, 541, 529, 509, 517, 540, 506, 495, 491, 539, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 502, 494, 476, 516, 484, 479, 510, 503, 484, 561, 535, 515, 505, 488, 532, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 526, 461, 510, 517, 518, 560, 559, 525, 547, 526, 503, 502, 476, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8202, 8288, 8126, 8156, 8238, 8096, 8231] avg=8192.0000 max=8288 max_rank=2 min=8096 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[530, 514, 525, 541, 516, 489, 499, 506, 493, 484, 520, 512, 559, 502, 539, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 538, 530, 520, 513, 530, 460, 525, 486, 515, 471, 516, 511, 536, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 520, 484, 491, 517, 483, 519, 590, 528, 496, 485, 516, 459, 515, 530, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 550, 491, 498, 522, 530, 445, 515, 507, 490, 494, 509, 510, 490, 518, 507, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 520, 509, 532, 531, 500, 505, 504, 483, 519, 516, 523, 490, 482, 544, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 561, 526, 531, 491, 532, 538, 540, 503, 529, 512, 532, 509, 555, 477, 525, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 523, 520, 481, 478, 503, 549, 504, 541, 514, 508, 519, 550, 502, 472, 498, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 503, 455, 534, 543, 536, 489, 498, 491, 513, 494, 504, 490, 528, 517, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8196, 8160, 8109, 8210, 8327, 8193, 8106] avg=8192.0000 max=8327 max_rank=5 min=8106 min_rank=7 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 508, 468, 503, 523, 498, 546, 518, 498, 510, 523, 532, 494, 532, 493, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 485, 478, 509, 525, 501, 527, 451, 530, 505, 526, 536, 496, 502, 505, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 494, 499, 535, 505, 500, 516, 495, 509, 505, 488, 544, 503, 562, 508, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 538, 504, 552, 505, 483, 540, 509, 553, 524, 509, 505, 564, 533, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 529, 515, 491, 508, 488, 516, 519, 491, 540, 483, 507, 493, 505, 539, 502, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 516, 516, 487, 480, 511, 497, 494, 502, 529, 569, 528, 486, 502, 498, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 533, 513, 499, 491, 506, 521, 554, 539, 477, 501, 549, 494, 487, 535, 497, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 496, 541, 496, 487, 527, 499, 550, 492, 519, 502, 487, 533, 513, 541, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8068, 8187, 8364, 8145, 8140, 8267, 8162] avg=8192.0000 max=8364 max_rank=3 min=8068 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[539, 516, 527, 547, 541, 475, 552, 488, 496, 539, 513, 468, 479, 537, 505, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 523, 488, 531, 509, 517, 512, 477, 513, 537, 544, 504, 539, 499, 542, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 543, 480, 492, 497, 509, 502, 517, 540, 491, 507, 502, 515, 528, 494, 490, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 504, 481, 532, 487, 482, 530, 500, 488, 514, 530, 490, 513, 531, 484, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 505, 531, 495, 518, 502, 487, 511, 524, 506, 522, 503, 551, 539, 489, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 542, 495, 516, 540, 496, 538, 512, 525, 494, 504, 492, 524, 492, 480, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 546, 486, 517, 537, 477, 496, 526, 505, 475, 512, 528, 528, 491, 498, 524, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 517, 524, 495, 508, 517, 522, 501, 509, 497, 513, 513, 532, 522, 493, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8326, 8148, 8121, 8235, 8165, 8144, 8204] avg=8192.0000 max=8326 max_rank=1 min=8121 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[502, 533, 546, 471, 509, 554, 494, 495, 559, 512, 478, 505, 539, 491, 506, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 528, 486, 480, 496, 513, 504, 480, 509, 509, 489, 505, 509, 514, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 495, 531, 532, 502, 491, 490, 535, 513, 486, 514, 479, 508, 506, 546, 551, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 524, 496, 570, 521, 497, 497, 523, 533, 505, 539, 529, 530, 526, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 489, 506, 515, 529, 486, 535, 532, 494, 480, 470, 473, 561, 478, 510, 504, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 516, 499, 516, 523, 476, 521, 532, 528, 509, 569, 514, 534, 501, 504, 509, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 485, 512, 515, 519, 506, 497, 502, 463, 498, 491, 498, 530, 496, 530, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 522, 513, 498, 525, 527, 498, 518, 487, 490, 530, 518, 501, 516, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8065, 8247, 8365, 8090, 8299, 8093, 8187] avg=8192.0000 max=8365 max_rank=3 min=8065 min_rank=1 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[519, 553, 480, 527, 485, 482, 521, 470, 515, 496, 452, 521, 504, 475, 544, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 533, 512, 509, 523, 491, 546, 473, 551, 492, 532, 506, 504, 482, 472, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 505, 530, 478, 552, 480, 529, 512, 548, 512, 478, 530, 499, 487, 514, 525, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 512, 486, 538, 499, 519, 521, 517, 468, 514, 503, 512, 531, 509, 511, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 500, 523, 480, 523, 510, 526, 502, 519, 530, 547, 513, 525, 506, 521, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 538, 530, 516, 466, 490, 510, 543, 535, 519, 528, 511, 505, 498, 539, 468, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 537, 516, 534, 505, 527, 523, 507, 487, 521, 498, 532, 530, 549, 537, 510, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 537, 497, 529, 498, 454, 500, 536, 493, 513, 487, 505, 534, 535, 565, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8122, 8166, 8153, 8257, 8176, 8334, 8257] avg=8192.0000 max=8334 max_rank=6 min=8071 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=33024 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[524, 493, 506, 489, 525, 529, 474, 513, 507, 532, 527, 511, 530, 493, 531, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 535, 522, 568, 482, 526, 505, 515, 512, 539, 501, 476, 525, 530, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 503, 559, 505, 518, 501, 517, 538, 456, 513, 553, 526, 523, 529, 522, 522, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 522, 529, 503, 521, 508, 529, 504, 490, 489, 495, 537, 519, 481, 505, 456, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 494, 522, 517, 498, 519, 532, 500, 496, 521, 519, 512, 505, 530, 510, 515, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 519, 500, 503, 455, 510, 487, 500, 538, 521, 489, 533, 483, 508, 500, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 479, 566, 476, 502, 508, 552, 529, 495, 494, 509, 528, 502, 520, 503, 536, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 489, 550, 515, 489, 484, 525, 520, 501, 545, 504, 507, 496, 528, 521, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8280, 8340, 8075, 8220, 8063, 8210, 8151] avg=8192.0000 max=8340 max_rank=2 min=8063 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[497, 505, 501, 481, 552, 512, 511, 515, 551, 519, 508, 510, 528, 525, 485, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 523, 516, 511, 506, 534, 528, 525, 466, 536, 543, 503, 565, 509, 473, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 475, 503, 548, 482, 515, 503, 526, 502, 508, 504, 440, 493, 502, 491, 18, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 527, 502, 499, 485, 541, 547, 508, 508, 494, 522, 537, 551, 485, 498, 499, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 532, 471, 498, 505, 537, 498, 560, 502, 517, 523, 517, 494, 528, 510, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 524, 505, 513, 547, 522, 493, 487, 521, 506, 481, 514, 487, 473, 519, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 520, 498, 502, 538, 514, 521, 471, 519, 519, 516, 504, 503, 513, 493, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 521, 512, 519, 504, 546, 508, 521, 493, 494, 477, 552, 518, 507, 493, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8277, 8000, 8223, 8223, 8128, 8199, 8238] avg=8192.0000 max=8277 max_rank=1 min=8000 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4224, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[506, 478, 497, 517, 530, 501, 511, 452, 478, 502, 496, 514, 515, 493, 498, 526, 514, 499, 517, 535, 524, 559, 555, 477, 494, 512, 514, 511, 567, 482, 497, 513, 520, 456, 511, 488, 533, 513, 519, 547, 547, 521, 484, 525, 501, 537, 537, 502, 509, 534, 493, 496, 509, 557, 529, 492, 515, 510, 516, 479, 549, 500, 493, 514, 507, 477, 507, 501, 534, 519, 516, 483, 520, 525, 533, 549, 492, 509, 485, 480, 476, 532, 526, 515, 511, 498, 472, 551, 492, 516, 479, 528, 506, 486, 542, 492, 522, 510, 467, 516, 554, 551, 495, 557, 525, 500, 499, 489, 491, 499, 497, 501, 532, 511, 507, 529, 548, 526, 542, 564, 533, 508, 493, 511, 512, 498, 518, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[516, 523, 530, 480, 495, 523, 536, 506, 508, 521, 530, 491, 569, 525, 497, 503, 527, 482, 486, 479, 527, 514, 534, 504, 521, 497, 502, 516, 520, 504, 541, 519, 497, 500, 525, 503, 491, 476, 505, 543, 505, 466, 550, 519, 503, 511, 528, 522, 487, 527, 513, 477, 497, 514, 544, 525, 473, 499, 516, 517, 529, 519, 483, 476, 509, 507, 528, 487, 508, 519, 512, 528, 537, 528, 478, 502, 487, 535, 509, 526, 489, 528, 491, 520, 518, 514, 528, 535, 489, 511, 528, 495, 550, 551, 482, 533, 516, 488, 501, 550, 508, 481, 533, 484, 526, 495, 493, 536, 495, 491, 506, 528, 514, 501, 500, 518, 501, 518, 536, 546, 491, 537, 525, 527, 526, 516, 515, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[505, 541, 504, 525, 534, 537, 548, 532, 560, 520, 524, 501, 537, 525, 474, 502, 526, 477, 487, 489, 526, 512, 518, 521, 466, 517, 516, 537, 513, 529, 516, 478, 519, 498, 486, 523, 559, 514, 492, 490, 502, 531, 498, 534, 506, 498, 529, 526, 493, 536, 504, 519, 497, 495, 492, 526, 479, 495, 496, 549, 490, 528, 503, 518, 504, 507, 507, 502, 534, 543, 493, 511, 520, 496, 504, 534, 462, 483, 494, 492, 487, 505, 491, 472, 540, 510, 536, 506, 507, 493, 524, 507, 520, 512, 531, 512, 539, 540, 528, 495, 498, 477, 555, 498, 471, 530, 510, 518, 515, 547, 504, 552, 535, 509, 519, 516, 480, 464, 507, 518, 531, 514, 527, 520, 537, 514, 517, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[532, 492, 500, 516, 536, 463, 548, 531, 500, 510, 465, 508, 551, 523, 523, 490, 514, 506, 494, 544, 516, 478, 477, 487, 508, 512, 517, 517, 491, 507, 515, 491, 484, 499, 501, 503, 492, 506, 537, 503, 506, 519, 490, 530, 503, 528, 497, 508, 553, 468, 519, 513, 481, 504, 508, 511, 497, 503, 559, 515, 518, 503, 512, 553, 537, 473, 528, 506, 515, 499, 543, 521, 537, 551, 531, 525, 527, 515, 496, 513, 511, 518, 539, 501, 513, 523, 507, 507, 475, 521, 523, 492, 507, 491, 546, 517, 504, 518, 498, 535, 520, 484, 527, 490, 526, 528, 511, 518, 503, 508, 529, 535, 477, 519, 543, 549, 516, 484, 511, 512, 498, 527, 528, 535, 479, 502, 530, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[502, 511, 511, 518, 476, 513, 497, 520, 537, 540, 519, 526, 534, 498, 512, 518, 525, 468, 521, 519, 509, 508, 489, 493, 562, 504, 520, 462, 517, 520, 561, 508, 491, 558, 505, 477, 529, 524, 534, 497, 491, 527, 551, 516, 486, 503, 530, 526, 525, 502, 494, 554, 472, 542, 557, 511, 544, 534, 453, 487, 457, 482, 536, 481, 489, 522, 503, 509, 544, 480, 533, 532, 536, 465, 548, 507, 574, 482, 517, 508, 551, 510, 515, 498, 510, 521, 560, 514, 494, 488, 503, 544, 483, 502, 501, 506, 477, 521, 531, 509, 516, 499, 487, 488, 521, 492, 525, 478, 526, 524, 521, 462, 508, 526, 510, 524, 518, 504, 509, 562, 519, 519, 537, 457, 489, 491, 522, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[526, 490, 512, 564, 545, 476, 506, 472, 542, 490, 492, 487, 504, 491, 504, 546, 530, 491, 494, 465, 498, 487, 519, 517, 508, 504, 511, 524, 506, 497, 527, 531, 491, 557, 515, 534, 498, 516, 523, 472, 493, 535, 512, 499, 531, 521, 534, 493, 546, 505, 511, 541, 516, 578, 546, 528, 532, 515, 530, 513, 500, 523, 537, 485, 526, 525, 509, 524, 527, 504, 483, 524, 523, 521, 512, 524, 513, 493, 502, 520, 479, 515, 475, 535, 494, 505, 491, 505, 521, 529, 518, 490, 492, 504, 534, 529, 497, 509, 497, 510, 532, 486, 555, 511, 515, 533, 488, 520, 497, 473, 508, 526, 518, 525, 507, 544, 488, 510, 482, 484, 533, 519, 502, 468, 481, 514, 547, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[478, 510, 485, 553, 561, 492, 515, 550, 544, 487, 507, 503, 489, 490, 494, 508, 561, 526, 496, 492, 543, 523, 530, 542, 526, 477, 515, 496, 495, 478, 466, 556, 544, 516, 539, 500, 516, 485, 525, 519, 510, 507, 517, 514, 493, 558, 512, 557, 498, 528, 461, 535, 499, 487, 500, 505, 532, 496, 517, 516, 501, 540, 500, 502, 495, 515, 509, 518, 537, 501, 496, 566, 451, 546, 528, 500, 467, 490, 518, 469, 504, 493, 540, 521, 515, 500, 540, 491, 522, 496, 513, 568, 539, 542, 544, 512, 487, 520, 490, 559, 475, 505, 516, 510, 482, 511, 507, 511, 500, 523, 519, 464, 557, 512, 527, 484, 540, 534, 517, 488, 483, 483, 497, 543, 539, 483, 536, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[510, 534, 464, 518, 541, 484, 551, 516, 517, 501, 516, 521, 470, 536, 530, 479, 516, 504, 524, 491, 494, 543, 485, 489, 512, 528, 510, 485, 526, 526, 509, 491, 514, 494, 491, 514, 516, 499, 515, 531, 505, 548, 500, 476, 562, 520, 501, 505, 522, 543, 512, 476, 507, 504, 538, 495, 480, 505, 518, 552, 530, 541, 486, 511, 496, 479, 496, 537, 534, 526, 511, 494, 506, 521, 521, 507, 560, 480, 515, 460, 505, 509, 475, 501, 495, 541, 508, 510, 492, 498, 502, 503, 486, 545, 514, 466, 516, 518, 517, 471, 551, 504, 547, 538, 511, 507, 524, 559, 545, 493, 544, 489, 501, 520, 504, 499, 522, 528, 524, 545, 513, 530, 485, 549, 497, 528, 508, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[532, 492, 500, 516, 534, 463, 548, 531, 500, 510, 465, 508, 551, 523, 523, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 494, 544, 516, 478, 477, 487, 508, 512, 517, 517, 491, 507, 515, 491, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 499, 501, 503, 492, 506, 537, 503, 506, 519, 490, 530, 503, 513, 497, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 468, 519, 513, 481, 504, 508, 511, 497, 503, 559, 515, 518, 503, 512, 553, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 473, 528, 506, 515, 499, 543, 521, 537, 551, 531, 525, 527, 515, 496, 513, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 539, 501, 513, 523, 507, 507, 475, 521, 523, 492, 507, 491, 546, 517, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 498, 535, 520, 484, 527, 490, 526, 528, 511, 518, 503, 508, 529, 535, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 519, 543, 549, 516, 484, 511, 469, 498, 527, 528, 535, 479, 502, 530, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8103, 8091, 8220, 8325, 8203, 8242, 8166] avg=8192.0000 max=8325 max_rank=4 min=8091 min_rank=2 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 478, 497, 517, 528, 501, 511, 452, 478, 502, 496, 514, 515, 493, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 499, 517, 535, 524, 559, 555, 477, 494, 512, 514, 511, 567, 482, 497, 513, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 456, 511, 488, 533, 513, 519, 547, 547, 521, 484, 525, 501, 522, 537, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 534, 493, 496, 509, 557, 529, 492, 515, 510, 516, 479, 549, 500, 493, 514, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 477, 507, 501, 534, 519, 516, 483, 520, 525, 533, 549, 492, 509, 485, 480, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 532, 526, 515, 511, 498, 472, 551, 492, 516, 479, 528, 506, 486, 542, 492, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 510, 467, 516, 554, 551, 495, 557, 525, 500, 499, 489, 491, 499, 497, 501, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 511, 507, 529, 548, 526, 542, 516, 533, 508, 493, 511, 512, 498, 518, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8012, 8302, 8226, 8198, 8145, 8135, 8182, 8336] avg=8192.0000 max=8336 max_rank=7 min=8012 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[505, 541, 504, 525, 532, 537, 548, 532, 560, 520, 524, 501, 537, 525, 474, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 477, 487, 489, 526, 512, 518, 521, 466, 517, 516, 537, 513, 529, 516, 478, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 498, 486, 523, 559, 514, 492, 490, 502, 531, 498, 534, 506, 483, 529, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 536, 504, 519, 497, 495, 492, 526, 479, 495, 496, 549, 490, 528, 503, 518, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 507, 502, 534, 543, 493, 511, 520, 496, 504, 534, 462, 483, 494, 492, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 505, 491, 472, 540, 510, 536, 506, 507, 493, 524, 507, 520, 512, 531, 512, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 540, 528, 495, 498, 477, 555, 498, 471, 530, 510, 518, 515, 547, 504, 552, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 509, 519, 516, 480, 464, 507, 475, 531, 514, 527, 520, 537, 514, 517, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 8157, 8190, 8123, 8094, 8165, 8285, 8155] avg=8192.0000 max=8367 max_rank=0 min=8094 min_rank=4 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[526, 490, 512, 564, 538, 476, 506, 472, 542, 490, 492, 487, 504, 491, 504, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 491, 494, 465, 498, 487, 519, 517, 508, 504, 511, 524, 506, 497, 527, 531, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 557, 515, 534, 498, 516, 523, 472, 493, 535, 512, 499, 531, 495, 534, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 505, 511, 541, 516, 578, 546, 528, 532, 515, 530, 513, 500, 523, 537, 485, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 509, 524, 527, 504, 483, 524, 523, 521, 512, 524, 513, 493, 502, 520, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 515, 475, 535, 494, 505, 491, 505, 521, 529, 518, 490, 492, 504, 534, 529, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 497, 510, 532, 486, 555, 511, 515, 533, 488, 520, 497, 473, 508, 526, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 507, 544, 488, 510, 482, 432, 533, 519, 502, 468, 481, 514, 547, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8139, 8198, 8419, 8241, 8131, 8173, 8095] avg=8192.0000 max=8419 max_rank=3 min=8095 min_rank=7 imbalance_ratio=1.027710 max_load_reduction=57117 max_load_reduction_pct=0.871536 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[502, 511, 511, 518, 474, 513, 497, 520, 537, 540, 519, 526, 534, 498, 512, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 468, 521, 519, 509, 508, 489, 493, 562, 504, 520, 462, 517, 520, 561, 508, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 558, 505, 477, 529, 524, 534, 497, 491, 527, 551, 516, 486, 488, 530, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 502, 494, 554, 472, 542, 557, 511, 544, 534, 453, 487, 457, 482, 536, 481, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 522, 503, 509, 544, 480, 533, 532, 536, 465, 548, 507, 574, 482, 517, 508, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 510, 515, 498, 510, 521, 560, 514, 494, 488, 503, 544, 483, 502, 501, 506, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 521, 531, 509, 516, 499, 487, 488, 521, 492, 525, 478, 526, 524, 521, 462, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 526, 510, 524, 518, 504, 509, 514, 519, 519, 537, 457, 489, 491, 522, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8218, 8230, 8134, 8257, 8213, 8086, 8168] avg=8192.0000 max=8257 max_rank=4 min=8086 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 523, 530, 480, 493, 523, 536, 506, 508, 521, 530, 491, 569, 525, 497, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 482, 486, 479, 527, 514, 534, 504, 521, 497, 502, 516, 520, 504, 541, 519, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 500, 525, 503, 491, 476, 505, 543, 505, 466, 550, 519, 503, 496, 528, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 527, 513, 477, 497, 514, 544, 525, 473, 499, 516, 517, 529, 519, 483, 476, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 507, 528, 487, 508, 519, 512, 528, 537, 528, 478, 502, 487, 535, 509, 526, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 528, 491, 520, 518, 514, 528, 535, 489, 511, 528, 495, 550, 551, 482, 533, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 488, 501, 550, 508, 481, 533, 484, 526, 495, 493, 536, 495, 491, 506, 528, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 501, 500, 518, 501, 518, 536, 500, 491, 537, 525, 527, 526, 516, 515, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8204, 8129, 8099, 8208, 8275, 8139, 8231] avg=8192.0000 max=8275 max_rank=5 min=8099 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[510, 534, 464, 518, 539, 484, 551, 516, 517, 501, 516, 521, 470, 536, 530, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 504, 524, 491, 494, 543, 485, 489, 512, 528, 510, 485, 526, 526, 509, 491, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 494, 491, 514, 516, 499, 515, 531, 505, 548, 500, 476, 562, 505, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 543, 512, 476, 507, 504, 538, 495, 480, 505, 518, 552, 530, 541, 486, 511, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 479, 496, 537, 534, 526, 511, 494, 506, 521, 521, 507, 560, 480, 515, 460, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 475, 501, 495, 541, 508, 510, 492, 498, 502, 503, 486, 545, 514, 466, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 517, 471, 551, 504, 547, 538, 511, 507, 524, 559, 545, 493, 544, 489, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 520, 504, 499, 522, 528, 524, 499, 513, 530, 485, 549, 497, 528, 508, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8164, 8176, 8223, 8151, 8063, 8342, 8231] avg=8192.0000 max=8342 max_rank=6 min=8063 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[478, 510, 485, 553, 559, 492, 515, 550, 544, 487, 507, 503, 489, 490, 494, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 526, 496, 492, 543, 523, 530, 542, 526, 477, 515, 496, 495, 478, 466, 556, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 516, 539, 500, 516, 485, 525, 519, 510, 507, 517, 514, 493, 542, 512, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 528, 461, 535, 499, 487, 500, 505, 532, 496, 517, 516, 501, 540, 500, 502, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 509, 518, 537, 501, 496, 566, 451, 546, 528, 500, 467, 490, 518, 469, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 493, 540, 521, 515, 500, 540, 491, 522, 496, 513, 568, 539, 542, 544, 512, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 520, 490, 559, 475, 505, 516, 510, 482, 511, 507, 511, 500, 523, 519, 464, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 512, 527, 484, 540, 534, 517, 448, 483, 483, 497, 543, 539, 483, 536, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8249, 8296, 8120, 8115, 8351, 8087, 8154] avg=8192.0000 max=8351 max_rank=5 min=8087 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3968, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[522, 504, 539, 540, 539, 489, 474, 501, 519, 523, 520, 503, 545, 502, 518, 514, 539, 478, 542, 515, 532, 497, 521, 495, 514, 512, 523, 558, 503, 550, 495, 507, 527, 481, 501, 532, 514, 514, 500, 501, 495, 502, 524, 477, 511, 520, 508, 534, 520, 494, 518, 501, 531, 520, 514, 482, 515, 500, 528, 497, 457, 530, 486, 527, 492, 480, 516, 484, 490, 519, 555, 508, 515, 523, 546, 538, 515, 509, 504, 536, 476, 518, 538, 510, 512, 546, 492, 521, 520, 504, 527, 529, 476, 476, 530, 525, 490, 520, 502, 521, 500, 515, 514, 509, 487, 496, 558, 501, 479, 485, 523, 492, 503, 519, 528, 505, 527, 525, 501, 525, 528, 494, 537, 498, 530, 533, 466, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[536, 508, 509, 499, 518, 526, 479, 522, 523, 506, 536, 539, 460, 519, 506, 526, 539, 500, 500, 533, 487, 531, 511, 482, 515, 517, 505, 515, 486, 504, 538, 520, 500, 482, 486, 487, 524, 600, 499, 500, 468, 483, 545, 465, 500, 472, 513, 539, 489, 551, 552, 495, 533, 503, 506, 507, 484, 514, 499, 500, 525, 541, 506, 540, 489, 476, 499, 527, 524, 520, 496, 561, 514, 501, 516, 510, 506, 517, 501, 496, 540, 517, 498, 525, 536, 513, 525, 531, 518, 474, 476, 525, 490, 484, 502, 527, 505, 547, 481, 499, 534, 522, 518, 504, 518, 495, 523, 518, 532, 542, 507, 506, 538, 486, 522, 525, 500, 524, 505, 528, 524, 534, 514, 521, 532, 454, 540, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[513, 506, 531, 521, 500, 540, 509, 487, 499, 526, 540, 493, 516, 515, 505, 531, 514, 484, 474, 502, 521, 505, 497, 512, 523, 555, 528, 472, 488, 509, 525, 505, 538, 539, 502, 495, 509, 476, 520, 500, 507, 529, 484, 526, 523, 477, 463, 540, 500, 490, 515, 514, 527, 500, 501, 520, 552, 506, 526, 507, 506, 531, 494, 533, 505, 534, 533, 521, 540, 505, 539, 534, 498, 513, 533, 486, 527, 524, 484, 478, 507, 518, 510, 527, 520, 494, 501, 489, 511, 500, 503, 537, 536, 529, 460, 494, 470, 525, 525, 506, 539, 520, 516, 489, 545, 510, 508, 518, 506, 579, 549, 509, 505, 500, 508, 530, 518, 495, 513, 551, 509, 503, 517, 452, 472, 509, 521, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[527, 508, 563, 496, 464, 482, 528, 523, 538, 530, 516, 529, 489, 480, 480, 555, 497, 493, 506, 541, 501, 503, 484, 525, 560, 504, 548, 507, 528, 488, 503, 477, 530, 517, 510, 509, 505, 519, 523, 523, 492, 511, 490, 534, 519, 480, 511, 487, 517, 504, 478, 489, 538, 514, 537, 543, 509, 511, 486, 526, 500, 486, 516, 507, 494, 497, 533, 562, 495, 507, 522, 524, 524, 516, 478, 510, 498, 489, 536, 513, 497, 479, 577, 542, 456, 518, 449, 522, 519, 538, 485, 533, 552, 522, 549, 512, 492, 533, 565, 507, 502, 484, 533, 526, 491, 518, 555, 532, 509, 493, 530, 519, 513, 502, 529, 480, 514, 504, 508, 457, 511, 508, 489, 520, 475, 534, 540, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[491, 544, 528, 515, 494, 538, 514, 499, 509, 506, 519, 493, 528, 503, 561, 495, 529, 515, 495, 542, 506, 501, 520, 468, 543, 470, 503, 502, 522, 523, 534, 523, 512, 500, 527, 504, 526, 493, 544, 501, 497, 531, 525, 537, 532, 486, 527, 538, 521, 520, 523, 478, 525, 510, 470, 506, 480, 512, 554, 514, 553, 502, 517, 478, 488, 502, 512, 509, 543, 514, 459, 519, 519, 519, 465, 513, 469, 522, 514, 495, 487, 503, 508, 525, 503, 490, 503, 514, 500, 519, 498, 524, 508, 527, 481, 519, 528, 527, 512, 514, 527, 520, 549, 511, 489, 532, 557, 482, 481, 525, 498, 518, 515, 484, 525, 504, 520, 515, 529, 499, 525, 526, 497, 516, 492, 492, 526, 554] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[528, 487, 521, 550, 508, 516, 499, 557, 527, 552, 537, 520, 496, 528, 476, 503, 497, 498, 506, 486, 539, 520, 529, 516, 492, 484, 505, 547, 548, 507, 519, 516, 509, 493, 486, 517, 504, 515, 518, 513, 520, 535, 527, 534, 549, 519, 503, 517, 532, 525, 514, 502, 536, 487, 480, 550, 518, 498, 480, 498, 520, 517, 488, 519, 531, 508, 523, 483, 516, 496, 549, 492, 537, 506, 487, 495, 525, 502, 530, 483, 501, 509, 522, 470, 521, 520, 482, 508, 527, 534, 516, 506, 531, 515, 516, 522, 521, 484, 515, 487, 499, 505, 479, 524, 527, 497, 502, 499, 516, 508, 486, 535, 495, 515, 538, 554, 468, 494, 469, 465, 538, 502, 537, 505, 525, 508, 510, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[511, 532, 500, 491, 545, 546, 529, 505, 509, 496, 536, 487, 538, 517, 514, 500, 538, 473, 540, 510, 521, 501, 491, 540, 521, 511, 509, 513, 537, 496, 542, 520, 487, 538, 508, 545, 523, 499, 493, 532, 459, 514, 537, 533, 511, 554, 479, 523, 560, 472, 529, 538, 544, 468, 509, 478, 519, 529, 506, 537, 495, 500, 486, 520, 464, 522, 503, 528, 527, 516, 528, 519, 483, 495, 512, 484, 539, 493, 507, 527, 478, 509, 510, 481, 496, 525, 529, 508, 479, 543, 502, 526, 496, 513, 529, 500, 496, 534, 522, 489, 541, 518, 485, 537, 538, 505, 481, 486, 496, 524, 508, 498, 489, 487, 513, 514, 518, 498, 521, 511, 513, 522, 505, 500, 502, 494, 559, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[484, 507, 510, 507, 483, 508, 537, 528, 538, 530, 501, 490, 476, 546, 548, 491, 499, 496, 493, 570, 491, 507, 486, 520, 502, 497, 505, 530, 521, 511, 516, 480, 528, 521, 501, 549, 500, 519, 512, 514, 540, 547, 520, 526, 511, 515, 513, 472, 525, 508, 508, 520, 492, 532, 505, 519, 519, 524, 488, 490, 506, 530, 508, 522, 514, 530, 550, 488, 527, 527, 492, 533, 490, 482, 536, 511, 523, 496, 482, 501, 490, 488, 529, 531, 530, 544, 518, 554, 525, 506, 478, 554, 526, 483, 536, 488, 510, 532, 505, 505, 492, 489, 516, 538, 525, 488, 534, 540, 521, 498, 503, 491, 504, 516, 517, 490, 494, 493, 534, 495, 502, 527, 535, 478, 499, 505, 501, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 508, 509, 499, 518, 526, 479, 522, 523, 506, 493, 539, 460, 519, 506, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 500, 500, 533, 487, 531, 511, 482, 515, 517, 505, 515, 486, 504, 538, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 482, 486, 487, 524, 600, 499, 500, 468, 483, 545, 465, 498, 472, 513, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 551, 552, 495, 533, 503, 506, 507, 484, 514, 499, 500, 525, 541, 506, 540, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 476, 499, 527, 524, 520, 496, 561, 514, 501, 516, 510, 506, 517, 501, 496, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 517, 498, 525, 536, 513, 525, 531, 518, 474, 476, 525, 490, 484, 502, 527, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 547, 481, 499, 534, 522, 518, 504, 518, 495, 511, 518, 532, 542, 507, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 486, 522, 525, 500, 524, 505, 528, 524, 534, 514, 521, 532, 454, 540, 501, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8183, 8061, 8253, 8171, 8187, 8239, 8273] avg=8192.0000 max=8273 max_rank=7 min=8061 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 504, 539, 540, 539, 489, 474, 501, 519, 523, 478, 503, 545, 502, 518, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 478, 542, 515, 532, 497, 521, 495, 514, 512, 523, 558, 503, 550, 495, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 481, 501, 532, 514, 514, 500, 501, 495, 502, 524, 477, 509, 520, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 494, 518, 501, 531, 520, 514, 482, 515, 500, 528, 497, 457, 530, 486, 527, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 480, 516, 484, 490, 519, 555, 508, 515, 523, 546, 538, 515, 509, 504, 536, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 518, 538, 510, 512, 546, 492, 521, 520, 504, 527, 529, 476, 476, 530, 525, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 520, 502, 521, 500, 515, 514, 509, 487, 496, 544, 501, 479, 485, 523, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 519, 528, 505, 527, 525, 501, 525, 528, 494, 537, 498, 530, 533, 466, 501, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8281, 8139, 8129, 8248, 8207, 8078, 8244] avg=8192.0000 max=8281 max_rank=1 min=8078 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 487, 521, 550, 508, 516, 499, 557, 527, 552, 494, 520, 496, 528, 476, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 506, 486, 539, 520, 529, 516, 492, 484, 505, 547, 548, 507, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 486, 517, 504, 515, 518, 513, 520, 535, 527, 534, 547, 519, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 514, 502, 536, 487, 480, 550, 518, 498, 480, 498, 520, 517, 488, 519, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 523, 483, 516, 496, 549, 492, 537, 506, 487, 495, 525, 502, 530, 483, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 522, 470, 521, 520, 482, 508, 527, 534, 516, 506, 531, 515, 516, 522, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 484, 515, 487, 499, 505, 479, 524, 527, 497, 490, 499, 516, 508, 486, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 538, 554, 468, 494, 469, 465, 538, 502, 537, 505, 525, 508, 510, 529, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8209, 8257, 8172, 8181, 8206, 8072, 8177] avg=8192.0000 max=8262 max_rank=0 min=8072 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 506, 531, 521, 500, 540, 509, 487, 499, 526, 485, 493, 516, 515, 505, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 484, 474, 502, 521, 505, 497, 512, 523, 555, 528, 472, 488, 509, 525, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 539, 502, 495, 509, 476, 520, 500, 507, 529, 484, 526, 519, 477, 463, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 490, 515, 514, 527, 500, 501, 520, 552, 506, 526, 507, 506, 531, 494, 533, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 534, 533, 521, 540, 505, 539, 534, 498, 513, 533, 486, 527, 524, 484, 478, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 510, 527, 520, 494, 501, 489, 511, 500, 503, 537, 536, 529, 460, 494, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 525, 525, 506, 539, 520, 516, 489, 545, 510, 488, 518, 506, 579, 549, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 500, 508, 530, 518, 495, 513, 551, 509, 503, 517, 452, 472, 509, 521, 533, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8115, 8124, 8239, 8276, 8147, 8294, 8164] avg=8192.0000 max=8294 max_rank=6 min=8115 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 508, 563, 496, 464, 482, 528, 523, 538, 530, 474, 529, 489, 480, 480, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 493, 506, 541, 501, 503, 484, 525, 560, 504, 548, 507, 528, 488, 503, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 517, 510, 509, 505, 519, 523, 523, 492, 511, 490, 534, 517, 480, 511, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 504, 478, 489, 538, 514, 537, 543, 509, 511, 486, 526, 500, 486, 516, 507, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 497, 533, 562, 495, 507, 522, 524, 524, 516, 478, 510, 498, 489, 536, 513, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 479, 577, 542, 456, 518, 449, 522, 519, 538, 485, 533, 552, 522, 549, 512, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 533, 565, 507, 502, 484, 533, 526, 491, 518, 541, 532, 509, 493, 530, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 502, 529, 480, 514, 504, 508, 457, 511, 508, 489, 520, 475, 534, 540, 521, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8165, 8158, 8170, 8216, 8257, 8275, 8129] avg=8192.0000 max=8275 max_rank=6 min=8129 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[511, 532, 500, 491, 545, 546, 529, 505, 509, 496, 493, 487, 538, 517, 514, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 473, 540, 510, 521, 501, 491, 540, 521, 511, 509, 513, 537, 496, 542, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 538, 508, 545, 523, 499, 493, 532, 459, 514, 537, 533, 509, 554, 479, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 472, 529, 538, 544, 468, 509, 478, 519, 529, 506, 537, 495, 500, 486, 520, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 522, 503, 528, 527, 516, 528, 519, 483, 495, 512, 484, 539, 493, 507, 527, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 509, 510, 481, 496, 525, 529, 508, 479, 543, 502, 526, 496, 513, 529, 500, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 534, 522, 489, 541, 518, 485, 537, 538, 505, 470, 486, 496, 524, 508, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 487, 513, 514, 518, 498, 521, 511, 513, 522, 505, 500, 502, 494, 559, 517, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8263, 8233, 8197, 8165, 8130, 8147, 8188] avg=8192.0000 max=8263 max_rank=1 min=8130 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[491, 544, 528, 515, 494, 538, 514, 499, 509, 506, 477, 493, 528, 503, 561, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 515, 495, 542, 506, 501, 520, 468, 543, 470, 503, 502, 522, 523, 534, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 500, 527, 504, 526, 493, 544, 501, 497, 531, 525, 537, 530, 486, 527, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 520, 523, 478, 525, 510, 470, 506, 480, 512, 554, 514, 553, 502, 517, 478, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 502, 512, 509, 543, 514, 459, 519, 519, 519, 465, 513, 469, 522, 514, 495, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 503, 508, 525, 503, 490, 503, 514, 500, 519, 498, 524, 508, 527, 481, 519, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 527, 512, 514, 527, 520, 549, 511, 489, 532, 543, 482, 481, 525, 498, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 484, 525, 504, 520, 515, 529, 499, 525, 526, 497, 516, 492, 492, 526, 554, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8196, 8278, 8172, 8080, 8116, 8256, 8243] avg=8192.0000 max=8278 max_rank=2 min=8080 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[484, 507, 510, 507, 483, 508, 537, 528, 538, 530, 461, 490, 476, 546, 548, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 496, 493, 570, 491, 507, 486, 520, 502, 497, 505, 530, 521, 511, 516, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 501, 549, 500, 519, 512, 514, 540, 547, 520, 526, 509, 515, 513, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 508, 508, 520, 492, 532, 505, 519, 519, 524, 488, 490, 506, 530, 508, 522, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 530, 550, 488, 527, 527, 492, 533, 490, 482, 536, 511, 523, 496, 482, 501, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 488, 529, 531, 530, 544, 518, 554, 525, 506, 478, 554, 526, 483, 536, 488, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 532, 505, 505, 492, 489, 516, 538, 525, 488, 522, 540, 521, 498, 503, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 517, 490, 494, 493, 534, 495, 502, 527, 535, 478, 499, 505, 501, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8124, 8286, 8204, 8199, 8286, 8175, 8118] avg=8192.0000 max=8286 max_rank=2 min=8118 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4352, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[506, 578, 481, 507, 527, 544, 535, 517, 502, 541, 539, 502, 496, 489, 516, 523, 472, 506, 539, 494, 527, 508, 567, 508, 510, 501, 537, 496, 541, 522, 544, 486, 496, 494, 530, 493, 509, 493, 516, 479, 521, 506, 512, 474, 479, 483, 496, 506, 516, 506, 513, 489, 535, 504, 525, 455, 494, 523, 539, 495, 466, 513, 528, 504, 486, 517, 525, 474, 522, 477, 483, 541, 544, 524, 517, 502, 558, 529, 507, 512, 512, 524, 469, 520, 542, 504, 532, 514, 528, 477, 525, 535, 502, 503, 499, 478, 539, 482, 486, 543, 484, 564, 514, 505, 493, 546, 483, 515, 485, 529, 490, 517, 520, 517, 508, 508, 539, 557, 533, 477, 508, 497, 551, 545, 540, 510, 515, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[524, 495, 520, 492, 519, 505, 527, 497, 543, 477, 491, 479, 505, 481, 517, 500, 505, 509, 497, 472, 559, 489, 495, 517, 560, 486, 513, 519, 514, 481, 541, 519, 492, 507, 489, 522, 546, 471, 530, 488, 502, 495, 515, 498, 536, 520, 513, 450, 550, 535, 523, 512, 522, 519, 513, 489, 507, 489, 493, 528, 494, 555, 528, 501, 542, 537, 511, 505, 517, 512, 483, 526, 516, 531, 479, 535, 517, 531, 507, 504, 528, 507, 524, 505, 526, 516, 497, 473, 505, 529, 541, 511, 552, 529, 490, 471, 528, 521, 532, 552, 495, 504, 506, 491, 509, 493, 502, 513, 530, 525, 509, 497, 496, 523, 517, 523, 556, 525, 468, 534, 471, 515, 537, 531, 517, 526, 536, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[509, 500, 495, 521, 508, 485, 538, 528, 493, 525, 538, 533, 504, 475, 554, 511, 550, 475, 490, 503, 510, 546, 552, 526, 499, 510, 511, 495, 500, 549, 498, 523, 531, 536, 490, 473, 493, 547, 562, 491, 504, 504, 556, 504, 514, 458, 490, 510, 519, 534, 513, 515, 482, 507, 511, 485, 534, 508, 495, 544, 502, 500, 523, 489, 505, 518, 468, 497, 552, 505, 534, 553, 543, 520, 508, 542, 506, 518, 524, 512, 495, 509, 521, 529, 496, 547, 512, 533, 506, 514, 490, 508, 483, 502, 514, 526, 525, 527, 528, 519, 531, 543, 537, 507, 548, 506, 495, 505, 490, 530, 475, 489, 512, 499, 473, 491, 518, 472, 492, 509, 498, 499, 500, 472, 528, 487, 539, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[533, 533, 517, 467, 508, 505, 508, 506, 529, 552, 484, 539, 542, 476, 537, 506, 520, 489, 532, 511, 527, 535, 503, 499, 517, 476, 495, 507, 540, 479, 523, 502, 487, 531, 505, 534, 539, 561, 548, 533, 519, 512, 514, 520, 479, 497, 525, 525, 510, 508, 508, 512, 503, 528, 502, 479, 521, 532, 518, 487, 514, 521, 530, 482, 489, 515, 563, 495, 546, 523, 524, 508, 510, 525, 491, 535, 504, 523, 528, 541, 491, 485, 471, 543, 529, 487, 456, 489, 511, 525, 486, 498, 507, 478, 529, 520, 511, 532, 501, 528, 510, 515, 551, 528, 535, 508, 502, 506, 495, 500, 474, 506, 481, 496, 514, 509, 533, 508, 513, 518, 503, 518, 508, 517, 491, 463, 527, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[507, 507, 518, 522, 526, 471, 470, 545, 526, 489, 510, 536, 511, 503, 508, 545, 473, 526, 559, 536, 527, 495, 493, 509, 545, 506, 518, 539, 522, 473, 562, 532, 493, 492, 502, 576, 522, 521, 482, 535, 524, 502, 542, 511, 503, 537, 539, 530, 523, 502, 507, 496, 516, 487, 504, 460, 528, 460, 513, 511, 508, 544, 514, 507, 522, 532, 517, 570, 499, 508, 494, 510, 484, 487, 528, 571, 495, 540, 512, 482, 483, 530, 481, 512, 543, 503, 462, 471, 495, 495, 502, 525, 499, 529, 515, 487, 512, 528, 520, 503, 482, 490, 481, 495, 498, 493, 562, 513, 482, 547, 554, 518, 520, 544, 523, 567, 513, 530, 480, 489, 529, 508, 499, 511, 456, 518, 488, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[530, 544, 477, 504, 494, 496, 536, 518, 544, 521, 505, 507, 505, 526, 480, 564, 455, 517, 535, 520, 515, 488, 536, 544, 519, 505, 543, 503, 543, 505, 491, 502, 493, 519, 518, 506, 497, 523, 500, 503, 527, 495, 513, 505, 486, 518, 550, 493, 531, 530, 475, 504, 541, 520, 526, 529, 512, 506, 536, 482, 503, 512, 541, 507, 514, 516, 491, 559, 505, 526, 507, 494, 492, 551, 498, 496, 499, 491, 497, 517, 538, 519, 516, 532, 523, 501, 533, 514, 512, 543, 547, 529, 507, 490, 509, 516, 511, 515, 488, 491, 511, 458, 541, 521, 499, 501, 528, 506, 497, 523, 499, 544, 517, 542, 494, 482, 520, 529, 497, 535, 504, 492, 510, 487, 525, 516, 442, 456] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[528, 501, 504, 477, 532, 514, 519, 533, 502, 538, 518, 500, 521, 507, 528, 531, 496, 523, 536, 474, 462, 489, 517, 513, 494, 563, 517, 495, 553, 481, 513, 531, 535, 524, 532, 533, 497, 522, 512, 475, 496, 495, 505, 518, 524, 523, 535, 515, 490, 496, 496, 484, 544, 524, 545, 514, 484, 505, 505, 484, 478, 516, 534, 527, 501, 518, 499, 504, 495, 529, 463, 533, 537, 533, 518, 536, 480, 501, 507, 502, 491, 487, 503, 561, 564, 506, 523, 516, 501, 512, 523, 491, 531, 509, 525, 494, 477, 492, 473, 503, 501, 514, 485, 520, 496, 539, 540, 485, 529, 543, 479, 553, 512, 530, 537, 523, 526, 516, 516, 536, 472, 504, 517, 507, 536, 483, 541, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[529, 486, 527, 498, 555, 523, 523, 529, 484, 490, 561, 532, 525, 490, 541, 477, 529, 516, 505, 538, 511, 530, 505, 485, 471, 523, 500, 517, 481, 490, 519, 553, 537, 544, 517, 515, 516, 515, 528, 472, 488, 547, 536, 539, 531, 516, 515, 510, 515, 533, 525, 492, 508, 516, 481, 506, 525, 520, 538, 533, 495, 536, 527, 506, 540, 512, 501, 509, 496, 532, 483, 519, 536, 473, 511, 529, 497, 515, 510, 495, 520, 487, 499, 490, 499, 513, 495, 503, 542, 504, 537, 488, 516, 533, 500, 543, 534, 507, 528, 480, 498, 520, 543, 506, 491, 516, 503, 541, 487, 501, 523, 472, 489, 496, 497, 468, 515, 474, 497, 524, 525, 523, 487, 519, 492, 518, 505, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[524, 495, 520, 492, 519, 505, 527, 497, 543, 477, 491, 479, 505, 481, 485, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 497, 472, 559, 489, 495, 517, 560, 486, 513, 519, 495, 481, 541, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 507, 489, 522, 546, 471, 530, 488, 502, 495, 510, 498, 536, 520, 513, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 535, 523, 512, 522, 519, 513, 489, 507, 489, 493, 528, 494, 555, 528, 501, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 537, 511, 505, 517, 512, 483, 526, 516, 531, 479, 500, 517, 531, 507, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 507, 524, 505, 526, 516, 497, 473, 505, 529, 541, 511, 552, 529, 490, 471, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 532, 552, 495, 504, 506, 491, 509, 493, 502, 513, 530, 525, 509, 497, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 523, 517, 523, 556, 525, 468, 534, 471, 515, 537, 531, 517, 526, 536, 517, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8040, 8157, 8069, 8275, 8218, 8230, 8220, 8327] avg=8192.0000 max=8327 max_rank=7 min=8040 min_rank=0 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32995 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[509, 500, 495, 521, 508, 485, 538, 528, 493, 525, 538, 533, 504, 475, 521, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 475, 490, 503, 510, 546, 552, 526, 499, 510, 511, 495, 483, 549, 498, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 536, 490, 473, 493, 547, 562, 491, 504, 504, 551, 504, 514, 458, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 534, 513, 515, 482, 507, 511, 485, 534, 508, 495, 544, 502, 500, 523, 489, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 468, 497, 552, 505, 534, 553, 543, 520, 508, 507, 506, 518, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 521, 529, 496, 547, 512, 533, 506, 514, 490, 508, 483, 502, 514, 526, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 527, 528, 519, 531, 543, 537, 507, 548, 506, 495, 505, 490, 530, 475, 489, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 499, 473, 491, 518, 472, 492, 509, 498, 499, 500, 472, 528, 487, 539, 524, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8220, 8158, 8177, 8270, 8212, 8267, 8048] avg=8192.0000 max=8270 max_rank=4 min=8048 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[506, 578, 481, 507, 527, 544, 535, 517, 502, 541, 539, 502, 496, 489, 473, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 506, 539, 494, 527, 508, 567, 508, 510, 501, 537, 496, 514, 522, 544, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 530, 493, 509, 493, 516, 479, 521, 506, 505, 474, 479, 483, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 513, 489, 535, 504, 525, 455, 494, 523, 539, 495, 466, 513, 528, 504, 15, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 517, 525, 474, 522, 477, 483, 541, 544, 524, 517, 467, 558, 529, 507, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 469, 520, 542, 504, 532, 514, 528, 477, 525, 535, 502, 503, 499, 478, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 482, 486, 543, 484, 564, 514, 505, 493, 546, 483, 515, 485, 529, 490, 517, 12, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 508, 508, 539, 557, 533, 477, 508, 497, 551, 545, 540, 510, 515, 501, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8231, 7980, 8129, 8183, 8193, 8194, 8366] avg=8192.0000 max=8366 max_rank=7 min=7980 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[533, 533, 517, 467, 508, 505, 508, 506, 529, 552, 484, 539, 542, 476, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 532, 511, 527, 535, 503, 499, 517, 476, 495, 507, 521, 479, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 531, 505, 534, 539, 561, 548, 533, 519, 512, 509, 520, 479, 497, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 508, 508, 512, 503, 528, 502, 479, 521, 532, 518, 487, 514, 521, 530, 482, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 515, 563, 495, 546, 523, 524, 508, 510, 525, 491, 500, 504, 523, 528, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 485, 471, 543, 529, 487, 456, 489, 511, 525, 486, 498, 507, 478, 529, 520, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 501, 528, 510, 515, 551, 528, 535, 508, 502, 506, 495, 500, 474, 506, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 496, 514, 509, 533, 508, 513, 518, 503, 518, 508, 517, 491, 463, 527, 529, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8136, 8324, 8172, 8285, 8032, 8215, 8163] avg=8192.0000 max=8324 max_rank=2 min=8032 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[529, 486, 527, 498, 555, 523, 523, 529, 484, 490, 561, 532, 525, 490, 508, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 516, 505, 538, 511, 530, 505, 485, 471, 523, 500, 517, 464, 490, 519, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 544, 517, 515, 516, 515, 528, 472, 488, 547, 531, 539, 531, 516, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 525, 492, 508, 516, 481, 506, 525, 520, 538, 533, 495, 536, 527, 506, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 512, 501, 509, 496, 532, 483, 519, 536, 473, 511, 495, 497, 515, 510, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 499, 490, 499, 513, 495, 503, 542, 504, 537, 488, 516, 533, 500, 543, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 507, 528, 480, 498, 520, 543, 506, 491, 516, 503, 541, 487, 501, 523, 472, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 496, 497, 468, 515, 474, 497, 524, 525, 523, 487, 519, 492, 518, 505, 505, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8156, 8321, 8272, 8124, 8196, 8162, 8068] avg=8192.0000 max=8321 max_rank=2 min=8068 min_rank=7 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[507, 507, 518, 522, 526, 471, 470, 545, 526, 489, 510, 536, 511, 503, 477, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 526, 559, 536, 527, 495, 493, 509, 545, 506, 518, 539, 503, 473, 562, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 502, 576, 522, 521, 482, 535, 524, 502, 537, 511, 503, 537, 539, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 502, 507, 496, 516, 487, 504, 460, 528, 460, 513, 511, 508, 544, 514, 507, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 532, 517, 570, 499, 508, 494, 510, 484, 487, 528, 534, 495, 540, 512, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 530, 481, 512, 543, 503, 462, 471, 495, 495, 502, 525, 499, 529, 515, 487, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 528, 520, 503, 482, 490, 481, 495, 498, 493, 562, 513, 482, 547, 554, 518, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 544, 523, 567, 513, 530, 480, 489, 529, 508, 499, 511, 456, 518, 488, 500, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8296, 8306, 8097, 8214, 8057, 8191, 8212] avg=8192.0000 max=8306 max_rank=2 min=8057 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[528, 501, 504, 477, 532, 514, 519, 533, 502, 538, 518, 500, 521, 507, 496, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 523, 536, 474, 462, 489, 517, 513, 494, 563, 517, 495, 534, 481, 513, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 524, 532, 533, 497, 522, 512, 475, 496, 495, 501, 518, 524, 523, 535, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 496, 496, 484, 544, 524, 545, 514, 484, 505, 505, 484, 478, 516, 534, 527, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 499, 504, 495, 529, 463, 533, 537, 533, 518, 501, 480, 501, 507, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 487, 503, 561, 564, 506, 523, 516, 501, 512, 523, 491, 531, 509, 525, 494, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 492, 473, 503, 501, 514, 485, 520, 496, 539, 540, 485, 529, 543, 479, 553, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 537, 523, 526, 516, 516, 536, 472, 504, 517, 507, 536, 483, 541, 481, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8138, 8237, 8143, 8121, 8263, 8141, 8272] avg=8192.0000 max=8272 max_rank=7 min=8121 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 544, 477, 504, 494, 496, 536, 518, 544, 521, 505, 507, 505, 526, 451, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 517, 535, 520, 515, 488, 536, 544, 519, 505, 543, 503, 524, 505, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 519, 518, 506, 497, 523, 500, 503, 527, 495, 508, 505, 486, 518, 550, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 530, 475, 504, 541, 520, 526, 529, 512, 506, 536, 482, 503, 512, 541, 507, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 516, 491, 559, 505, 526, 507, 494, 492, 551, 498, 464, 499, 491, 497, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 519, 516, 532, 523, 501, 533, 514, 512, 543, 547, 529, 507, 490, 509, 516, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 515, 488, 491, 511, 458, 541, 521, 499, 501, 528, 506, 497, 523, 499, 544, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 542, 494, 482, 520, 529, 497, 535, 504, 492, 510, 487, 525, 516, 442, 456, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8202, 8141, 8271, 8121, 8353, 8146, 8080] avg=8192.0000 max=8353 max_rank=5 min=8080 min_rank=7 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 3968, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 3968, 4224, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[482, 531, 470, 509, 498, 513, 520, 527, 482, 476, 496, 504, 471, 514, 493, 528, 531, 510, 516, 511, 540, 538, 511, 547, 470, 523, 517, 509, 530, 507, 517, 521, 506, 502, 465, 533, 497, 489, 532, 519, 509, 526, 522, 512, 497, 532, 575, 477, 506, 528, 539, 529, 474, 486, 519, 541, 457, 559, 533, 522, 513, 552, 529, 532, 546, 546, 565, 508, 503, 502, 538, 509, 485, 514, 515, 487, 478, 515, 493, 550, 521, 525, 558, 500, 471, 484, 570, 543, 490, 484, 513, 492, 502, 505, 482, 520, 538, 470, 553, 490, 496, 538, 538, 462, 549, 529, 510, 491, 507, 501, 504, 505, 503, 526, 520, 496, 490, 543, 496, 488, 518, 448, 542, 469, 498, 500, 527, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[479, 514, 510, 533, 525, 497, 512, 513, 539, 509, 473, 532, 548, 488, 520, 531, 501, 533, 466, 511, 523, 501, 534, 491, 544, 491, 486, 537, 509, 475, 514, 488, 494, 510, 517, 500, 535, 527, 480, 486, 493, 515, 530, 552, 490, 516, 527, 520, 508, 466, 549, 542, 513, 493, 494, 499, 516, 539, 515, 537, 476, 498, 509, 478, 520, 549, 508, 472, 532, 513, 510, 517, 507, 510, 472, 535, 514, 491, 527, 488, 548, 516, 513, 558, 491, 531, 493, 523, 525, 486, 528, 501, 525, 486, 443, 503, 520, 502, 468, 525, 488, 515, 535, 531, 543, 494, 513, 513, 576, 497, 514, 496, 494, 521, 516, 529, 494, 544, 487, 550, 516, 507, 510, 583, 555, 493, 520, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[489, 506, 523, 543, 546, 518, 510, 502, 509, 511, 537, 528, 433, 509, 539, 516, 538, 482, 512, 526, 494, 461, 502, 505, 541, 515, 518, 520, 525, 541, 547, 515, 536, 516, 506, 482, 525, 497, 510, 528, 536, 519, 536, 514, 517, 483, 535, 509, 530, 547, 518, 528, 528, 504, 501, 488, 484, 495, 493, 489, 497, 485, 485, 532, 503, 518, 514, 503, 509, 509, 517, 531, 472, 556, 523, 507, 528, 478, 554, 477, 496, 495, 477, 469, 514, 547, 486, 545, 524, 534, 528, 492, 500, 503, 552, 529, 499, 513, 483, 510, 486, 545, 522, 543, 517, 482, 501, 550, 529, 485, 514, 546, 498, 527, 503, 513, 520, 508, 542, 547, 519, 455, 488, 493, 495, 526, 484, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[487, 508, 470, 530, 498, 459, 554, 486, 552, 508, 493, 500, 462, 546, 542, 495, 511, 497, 546, 536, 479, 530, 539, 530, 498, 493, 508, 517, 538, 496, 503, 508, 508, 494, 500, 537, 542, 521, 529, 513, 501, 505, 514, 527, 529, 515, 526, 534, 472, 514, 539, 513, 486, 555, 526, 479, 510, 503, 469, 539, 506, 518, 503, 531, 517, 511, 541, 490, 497, 489, 481, 513, 549, 509, 477, 508, 518, 468, 522, 482, 546, 548, 494, 536, 505, 520, 526, 523, 516, 519, 537, 500, 515, 544, 530, 461, 499, 526, 495, 515, 507, 533, 485, 496, 494, 496, 557, 539, 568, 517, 523, 523, 509, 537, 520, 468, 534, 504, 529, 520, 496, 480, 476, 476, 513, 523, 487, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[500, 534, 499, 482, 496, 524, 542, 504, 497, 476, 521, 534, 558, 529, 492, 480, 523, 493, 529, 481, 521, 482, 554, 479, 479, 523, 510, 526, 503, 454, 499, 510, 531, 508, 531, 510, 517, 531, 551, 488, 534, 531, 509, 507, 534, 515, 521, 511, 497, 498, 505, 541, 520, 516, 508, 514, 532, 506, 493, 474, 490, 511, 487, 477, 516, 513, 508, 503, 514, 526, 538, 495, 502, 511, 502, 527, 527, 523, 543, 520, 480, 515, 540, 534, 528, 525, 516, 528, 510, 544, 468, 515, 538, 495, 528, 487, 484, 492, 539, 518, 506, 540, 514, 481, 524, 502, 510, 528, 546, 507, 538, 490, 478, 475, 487, 472, 495, 548, 567, 519, 517, 528, 530, 470, 528, 557, 473, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[503, 513, 508, 508, 527, 505, 512, 574, 519, 512, 491, 508, 563, 505, 483, 490, 497, 526, 499, 513, 506, 534, 513, 539, 511, 527, 557, 516, 509, 574, 514, 511, 487, 502, 484, 505, 466, 544, 538, 521, 505, 542, 511, 492, 507, 478, 526, 541, 470, 504, 523, 520, 557, 476, 519, 523, 497, 542, 505, 489, 519, 511, 508, 520, 506, 506, 540, 495, 517, 495, 513, 544, 497, 499, 508, 525, 508, 481, 506, 526, 479, 508, 488, 502, 507, 521, 504, 522, 510, 527, 458, 524, 501, 516, 517, 517, 549, 497, 532, 547, 512, 495, 473, 554, 476, 490, 532, 510, 507, 549, 548, 524, 490, 529, 496, 517, 506, 560, 456, 484, 522, 484, 499, 515, 533, 488, 500, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[530, 518, 501, 519, 525, 527, 550, 520, 497, 498, 508, 503, 513, 514, 535, 524, 461, 516, 502, 564, 508, 565, 528, 524, 520, 521, 493, 515, 467, 511, 532, 494, 506, 505, 498, 544, 546, 565, 532, 508, 495, 496, 501, 476, 531, 495, 509, 494, 505, 487, 507, 514, 472, 481, 544, 496, 509, 496, 506, 533, 503, 535, 543, 486, 495, 516, 521, 481, 494, 531, 511, 539, 507, 497, 546, 491, 521, 512, 549, 497, 486, 535, 528, 485, 497, 507, 517, 529, 505, 484, 499, 537, 504, 546, 480, 527, 560, 503, 502, 468, 520, 518, 505, 538, 523, 489, 512, 511, 505, 498, 481, 542, 505, 505, 514, 491, 529, 504, 492, 539, 533, 506, 482, 505, 481, 497, 515, 568] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[513, 496, 507, 526, 535, 496, 502, 482, 520, 522, 514, 510, 535, 509, 528, 527, 502, 515, 453, 515, 466, 542, 492, 528, 488, 526, 510, 493, 528, 551, 521, 502, 479, 519, 547, 519, 514, 527, 511, 524, 495, 504, 518, 521, 545, 529, 532, 531, 475, 525, 519, 498, 518, 525, 498, 498, 477, 533, 502, 490, 506, 497, 510, 506, 507, 526, 512, 519, 477, 484, 530, 523, 480, 523, 517, 471, 481, 537, 517, 534, 522, 538, 536, 523, 508, 490, 489, 503, 506, 494, 504, 518, 482, 507, 516, 522, 546, 534, 478, 503, 545, 523, 528, 522, 554, 530, 523, 506, 519, 491, 559, 513, 505, 475, 497, 515, 510, 511, 496, 511, 525, 500, 515, 492, 522, 499, 536, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[482, 531, 470, 509, 498, 513, 520, 527, 482, 476, 496, 504, 471, 514, 493, 528, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 516, 511, 540, 538, 499, 547, 470, 523, 517, 509, 530, 507, 517, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 502, 465, 533, 497, 489, 532, 519, 509, 526, 522, 512, 497, 532, 524, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 539, 529, 474, 486, 519, 541, 457, 559, 533, 522, 513, 552, 529, 532, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 546, 565, 508, 503, 502, 538, 509, 485, 514, 515, 487, 478, 515, 493, 550, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 525, 558, 500, 471, 484, 570, 543, 490, 484, 513, 492, 502, 505, 482, 520, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 470, 553, 490, 496, 538, 538, 462, 549, 529, 510, 491, 456, 501, 504, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 520, 496, 490, 543, 496, 488, 518, 448, 542, 469, 498, 500, 527, 553, 41, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8026, 8286, 8142, 8363, 8262, 8162, 8130, 8165] avg=8192.0000 max=8363 max_rank=3 min=8026 min_rank=0 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[489, 506, 523, 543, 546, 518, 510, 502, 509, 511, 537, 528, 433, 509, 539, 516, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 482, 512, 526, 494, 461, 491, 505, 541, 515, 518, 520, 525, 541, 547, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 516, 506, 482, 525, 497, 510, 528, 536, 519, 536, 514, 517, 483, 487, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 547, 518, 528, 528, 504, 501, 488, 484, 495, 493, 489, 497, 485, 485, 532, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 518, 514, 503, 509, 509, 517, 531, 472, 556, 523, 507, 528, 478, 554, 477, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 477, 469, 514, 547, 486, 545, 524, 534, 528, 492, 500, 503, 552, 529, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 513, 483, 510, 486, 545, 522, 543, 517, 482, 501, 550, 476, 485, 514, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 527, 503, 513, 520, 508, 542, 547, 519, 455, 488, 493, 495, 526, 484, 489, 38, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8231, 8201, 8150, 8206, 8193, 8172, 8152] avg=8192.0000 max=8231 max_rank=0 min=8150 min_rank=3 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[479, 514, 510, 533, 525, 497, 512, 513, 539, 509, 473, 532, 548, 488, 520, 531, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 533, 466, 511, 523, 501, 522, 491, 544, 491, 486, 537, 509, 475, 514, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 510, 517, 500, 535, 527, 480, 486, 493, 515, 530, 552, 490, 516, 480, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 466, 549, 542, 513, 493, 494, 499, 516, 539, 515, 537, 476, 498, 509, 478, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 549, 508, 472, 532, 513, 510, 517, 507, 510, 472, 535, 514, 491, 527, 488, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 516, 513, 558, 491, 531, 493, 523, 525, 486, 528, 501, 525, 486, 443, 503, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 502, 468, 525, 488, 515, 535, 531, 543, 494, 513, 513, 518, 497, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 521, 516, 529, 494, 544, 487, 550, 516, 507, 510, 583, 555, 493, 520, 501, 38, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8092, 8145, 8182, 8173, 8172, 8172, 8366] avg=8192.0000 max=8366 max_rank=7 min=8092 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[503, 513, 508, 508, 527, 505, 512, 574, 519, 512, 491, 508, 563, 505, 483, 490, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, 499, 513, 506, 534, 493, 539, 511, 527, 557, 516, 509, 574, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 502, 484, 505, 466, 544, 538, 521, 505, 542, 511, 492, 507, 478, 470, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 504, 523, 520, 557, 476, 519, 523, 497, 542, 505, 489, 519, 511, 508, 520, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 506, 540, 495, 517, 495, 513, 544, 497, 499, 508, 525, 508, 481, 506, 526, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 508, 488, 502, 507, 521, 504, 522, 510, 527, 458, 524, 501, 516, 517, 517, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 497, 532, 547, 512, 495, 473, 554, 476, 490, 532, 510, 448, 549, 548, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 529, 496, 517, 506, 560, 456, 484, 522, 484, 499, 515, 533, 488, 500, 496, 42, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8326, 8093, 8230, 8178, 8106, 8236, 8129] avg=8192.0000 max=8326 max_rank=1 min=8093 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 534, 499, 482, 496, 524, 542, 504, 497, 476, 521, 534, 558, 529, 492, 480, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 493, 529, 481, 521, 482, 542, 479, 479, 523, 510, 526, 503, 454, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 531, 510, 517, 531, 551, 488, 534, 531, 509, 507, 534, 515, 475, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 505, 541, 520, 516, 508, 514, 532, 506, 493, 474, 490, 511, 487, 477, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 513, 508, 503, 514, 526, 538, 495, 502, 511, 502, 527, 527, 523, 543, 520, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 515, 540, 534, 528, 525, 516, 528, 510, 544, 468, 515, 538, 495, 528, 487, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 492, 539, 518, 506, 540, 514, 481, 524, 502, 510, 528, 491, 507, 538, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 475, 487, 472, 495, 548, 567, 519, 517, 528, 530, 470, 528, 557, 473, 522, 37, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8054, 8283, 8116, 8276, 8253, 8164, 8211] avg=8192.0000 max=8283 max_rank=2 min=8054 min_rank=1 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[487, 508, 470, 530, 498, 459, 554, 486, 552, 508, 493, 500, 462, 546, 542, 495, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 546, 536, 479, 530, 527, 530, 498, 493, 508, 517, 538, 496, 503, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 494, 500, 537, 542, 521, 529, 513, 501, 505, 514, 527, 529, 515, 480, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 514, 539, 513, 486, 555, 526, 479, 510, 503, 469, 539, 506, 518, 503, 531, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 541, 490, 497, 489, 481, 513, 549, 509, 477, 508, 518, 468, 522, 482, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 548, 494, 536, 505, 520, 526, 523, 516, 519, 537, 500, 515, 544, 530, 461, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 526, 495, 515, 507, 533, 485, 496, 494, 496, 557, 539, 511, 517, 523, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 537, 520, 468, 534, 504, 529, 520, 496, 480, 476, 476, 513, 523, 487, 522, 37, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8217, 8249, 8212, 8080, 8322, 8216, 8139] avg=8192.0000 max=8322 max_rank=5 min=8080 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 496, 507, 526, 535, 496, 502, 482, 520, 522, 514, 510, 535, 509, 528, 527, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 453, 515, 466, 542, 481, 528, 488, 526, 510, 493, 528, 551, 521, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 519, 547, 519, 514, 527, 511, 524, 495, 504, 518, 521, 545, 529, 484, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 525, 519, 498, 518, 525, 498, 498, 477, 533, 502, 490, 506, 497, 510, 506, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 526, 512, 519, 477, 484, 530, 523, 480, 523, 517, 471, 481, 537, 517, 534, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 538, 536, 523, 508, 490, 489, 503, 506, 494, 504, 518, 482, 507, 516, 522, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 534, 478, 503, 545, 523, 528, 522, 554, 530, 523, 506, 467, 491, 559, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 475, 497, 515, 510, 511, 496, 511, 525, 500, 515, 492, 522, 499, 536, 511, 38, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8121, 8267, 8122, 8145, 8160, 8322, 8165] avg=8192.0000 max=8322 max_rank=6 min=8121 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[530, 518, 501, 519, 525, 527, 550, 520, 497, 498, 508, 503, 513, 514, 535, 524, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 516, 502, 564, 508, 565, 516, 524, 520, 521, 493, 515, 467, 511, 532, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 505, 498, 544, 546, 565, 532, 508, 495, 496, 501, 476, 531, 495, 464, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 507, 514, 472, 481, 544, 496, 509, 496, 506, 533, 503, 535, 543, 486, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 516, 521, 481, 494, 531, 511, 539, 507, 497, 546, 491, 521, 512, 549, 497, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 535, 528, 485, 497, 507, 517, 529, 505, 484, 499, 537, 504, 546, 480, 527, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 503, 502, 468, 520, 518, 505, 538, 523, 489, 512, 511, 454, 498, 481, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 505, 514, 491, 529, 504, 492, 539, 533, 506, 482, 505, 481, 497, 515, 568, 36, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8209, 8156, 8161, 8216, 8168, 8124, 8209] avg=8192.0000 max=8293 max_rank=0 min=8124 min_rank=6 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 3840, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[482, 467, 514, 506, 548, 472, 519, 565, 497, 494, 512, 518, 492, 532, 529, 511, 488, 491, 505, 509, 476, 514, 524, 499, 460, 511, 506, 491, 511, 497, 514, 540, 550, 519, 527, 548, 517, 525, 525, 495, 496, 530, 572, 548, 494, 486, 554, 510, 483, 537, 509, 524, 493, 536, 542, 511, 487, 513, 517, 487, 534, 516, 496, 515, 544, 522, 518, 516, 522, 492, 501, 511, 514, 494, 501, 497, 490, 516, 526, 519, 520, 509, 500, 501, 550, 481, 535, 498, 508, 504, 577, 519, 495, 498, 535, 499, 513, 509, 491, 500, 526, 525, 500, 507, 535, 493, 509, 502, 491, 532, 499, 552, 478, 509, 514, 512, 517, 485, 510, 509, 575, 546, 513, 496, 477, 504, 502, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[542, 496, 548, 511, 511, 490, 523, 526, 535, 509, 505, 533, 497, 504, 513, 521, 491, 519, 555, 507, 488, 510, 525, 503, 525, 517, 521, 533, 499, 487, 517, 485, 495, 523, 522, 510, 547, 524, 535, 519, 529, 521, 538, 474, 490, 536, 508, 512, 471, 519, 467, 487, 506, 507, 482, 527, 501, 486, 518, 504, 491, 478, 503, 476, 505, 483, 531, 525, 533, 512, 519, 487, 526, 496, 481, 497, 545, 525, 481, 498, 544, 531, 515, 507, 545, 530, 516, 512, 508, 503, 523, 491, 498, 482, 516, 487, 502, 528, 496, 492, 547, 504, 510, 520, 501, 543, 478, 511, 506, 540, 486, 546, 547, 506, 513, 550, 496, 532, 518, 538, 529, 495, 534, 510, 491, 507, 547, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[524, 524, 515, 509, 519, 511, 507, 510, 477, 510, 508, 516, 470, 525, 529, 527, 496, 522, 483, 494, 533, 524, 489, 551, 478, 540, 474, 465, 518, 507, 524, 518, 507, 525, 510, 526, 488, 513, 530, 480, 526, 535, 492, 524, 490, 513, 486, 509, 543, 492, 491, 528, 482, 478, 530, 512, 515, 519, 486, 521, 527, 512, 520, 512, 506, 550, 501, 506, 502, 507, 489, 531, 509, 511, 501, 492, 538, 521, 510, 472, 542, 489, 525, 518, 483, 537, 518, 500, 496, 545, 489, 526, 532, 507, 517, 521, 493, 493, 493, 534, 523, 552, 490, 525, 535, 560, 479, 505, 505, 544, 562, 544, 497, 565, 536, 530, 501, 525, 509, 489, 540, 509, 533, 503, 474, 505, 472, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[518, 475, 532, 473, 496, 515, 513, 537, 522, 460, 495, 487, 526, 542, 516, 536, 522, 515, 520, 528, 502, 519, 514, 492, 506, 519, 504, 512, 518, 493, 531, 481, 534, 480, 504, 509, 524, 514, 492, 503, 505, 516, 507, 523, 517, 485, 578, 533, 499, 498, 512, 567, 496, 525, 531, 521, 482, 538, 508, 483, 501, 485, 503, 529, 534, 476, 498, 502, 511, 513, 484, 511, 503, 517, 509, 539, 523, 520, 503, 506, 554, 478, 532, 473, 493, 505, 517, 475, 494, 495, 541, 519, 508, 525, 474, 518, 488, 548, 538, 551, 484, 485, 525, 523, 552, 553, 513, 515, 531, 494, 514, 500, 537, 509, 519, 516, 514, 520, 512, 528, 514, 510, 508, 529, 481, 477, 550, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[516, 502, 474, 540, 508, 521, 551, 536, 495, 483, 472, 511, 555, 514, 491, 494, 478, 520, 541, 526, 509, 515, 498, 510, 515, 508, 542, 521, 478, 501, 479, 519, 503, 511, 465, 507, 517, 528, 514, 541, 557, 505, 518, 508, 505, 478, 513, 508, 519, 519, 519, 523, 525, 495, 547, 510, 488, 509, 514, 518, 510, 474, 502, 542, 506, 517, 531, 513, 512, 493, 509, 480, 510, 532, 512, 510, 527, 483, 522, 512, 500, 488, 520, 482, 489, 496, 500, 540, 525, 491, 496, 510, 511, 544, 500, 511, 494, 520, 524, 540, 493, 535, 521, 497, 534, 560, 532, 508, 517, 539, 524, 495, 488, 489, 505, 529, 527, 513, 491, 507, 528, 540, 507, 519, 522, 490, 512, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[498, 487, 490, 523, 519, 524, 519, 526, 528, 529, 521, 531, 502, 500, 521, 490, 511, 524, 517, 463, 540, 504, 530, 504, 551, 517, 511, 500, 489, 475, 510, 492, 507, 501, 498, 516, 498, 490, 498, 479, 497, 549, 545, 487, 467, 570, 526, 502, 516, 532, 542, 516, 562, 497, 563, 479, 510, 547, 547, 532, 505, 515, 507, 543, 565, 523, 519, 496, 505, 523, 493, 480, 509, 520, 542, 515, 526, 520, 539, 517, 510, 514, 516, 514, 522, 520, 553, 505, 475, 501, 520, 493, 498, 482, 483, 521, 523, 475, 502, 504, 492, 510, 531, 538, 500, 501, 518, 496, 535, 500, 484, 497, 492, 496, 516, 476, 517, 489, 467, 512, 518, 523, 529, 565, 498, 488, 512, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[505, 507, 517, 548, 497, 532, 519, 501, 507, 482, 500, 495, 534, 532, 509, 486, 461, 518, 483, 501, 530, 568, 524, 501, 510, 516, 524, 527, 506, 524, 491, 498, 521, 559, 495, 562, 536, 507, 490, 502, 515, 477, 529, 497, 544, 516, 513, 474, 492, 523, 521, 487, 511, 545, 546, 518, 500, 539, 502, 515, 521, 502, 490, 543, 518, 489, 531, 488, 493, 517, 503, 542, 522, 506, 526, 454, 535, 525, 484, 542, 532, 528, 485, 531, 511, 487, 510, 465, 531, 503, 541, 538, 541, 498, 496, 501, 488, 531, 550, 487, 557, 533, 486, 528, 543, 501, 510, 504, 475, 483, 517, 488, 476, 525, 529, 498, 505, 488, 462, 505, 506, 476, 519, 527, 497, 560, 532, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[473, 488, 517, 559, 532, 541, 492, 542, 523, 502, 505, 465, 525, 513, 531, 502, 551, 491, 516, 483, 534, 532, 522, 502, 500, 488, 507, 499, 515, 508, 476, 495, 524, 510, 508, 485, 487, 482, 548, 520, 561, 532, 470, 466, 505, 509, 542, 502, 535, 457, 520, 547, 487, 533, 495, 515, 554, 499, 494, 501, 511, 555, 520, 503, 502, 511, 495, 505, 503, 531, 490, 524, 470, 489, 513, 489, 469, 538, 491, 546, 470, 515, 541, 558, 493, 516, 530, 473, 541, 485, 516, 537, 549, 495, 514, 536, 533, 540, 503, 509, 543, 503, 496, 510, 477, 525, 531, 527, 464, 505, 532, 495, 503, 547, 477, 519, 543, 521, 550, 485, 483, 536, 515, 527, 558, 529, 520, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[482, 467, 514, 506, 548, 472, 519, 565, 497, 494, 512, 518, 492, 532, 529, 511, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 491, 505, 509, 476, 514, 524, 499, 460, 511, 506, 491, 511, 497, 514, 540, 37, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 519, 527, 548, 517, 525, 525, 495, 496, 530, 572, 548, 494, 486, 527, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 537, 509, 524, 493, 536, 538, 511, 487, 513, 517, 487, 534, 516, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 522, 518, 516, 522, 492, 501, 511, 514, 494, 501, 497, 490, 516, 526, 519, 14, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 500, 501, 550, 481, 535, 498, 508, 504, 577, 519, 495, 498, 535, 499, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 509, 491, 500, 526, 525, 500, 507, 535, 456, 509, 502, 491, 532, 499, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 509, 514, 512, 517, 485, 510, 509, 555, 546, 513, 496, 477, 504, 502, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8086, 8369, 8196, 8210, 8236, 8147, 8130] avg=8192.0000 max=8369 max_rank=2 min=8086 min_rank=1 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 475, 532, 473, 496, 515, 513, 537, 522, 460, 495, 487, 526, 542, 516, 536, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 515, 520, 528, 502, 519, 514, 492, 506, 519, 504, 512, 518, 493, 531, 481, 42, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 480, 504, 509, 524, 514, 492, 503, 505, 516, 507, 523, 517, 485, 549, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 498, 512, 567, 496, 525, 527, 521, 482, 538, 508, 483, 501, 485, 503, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 476, 498, 502, 511, 513, 484, 511, 503, 517, 509, 539, 523, 520, 503, 506, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 478, 532, 473, 493, 505, 517, 475, 494, 495, 541, 519, 508, 525, 474, 518, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 548, 538, 551, 484, 485, 525, 523, 552, 511, 513, 515, 531, 494, 514, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 509, 519, 516, 514, 520, 512, 528, 496, 510, 508, 529, 481, 477, 550, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8232, 8195, 8174, 8176, 8107, 8272, 8233] avg=8192.0000 max=8272 max_rank=6 min=8107 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[498, 487, 490, 523, 519, 524, 519, 526, 528, 529, 521, 531, 502, 500, 521, 490, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 524, 517, 463, 540, 504, 530, 504, 551, 517, 511, 500, 489, 475, 510, 492, 38, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 501, 498, 516, 498, 490, 498, 479, 497, 549, 545, 487, 467, 570, 500, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 532, 542, 516, 562, 497, 558, 479, 510, 547, 547, 532, 505, 515, 507, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 523, 519, 496, 505, 523, 493, 480, 509, 520, 542, 515, 526, 520, 539, 517, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 514, 516, 514, 522, 520, 553, 505, 475, 501, 520, 493, 498, 482, 483, 521, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 475, 502, 504, 492, 510, 531, 538, 500, 463, 518, 496, 535, 500, 484, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 496, 516, 476, 517, 489, 467, 512, 500, 523, 529, 565, 498, 488, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8189, 8104, 8408, 8317, 8133, 8068, 8104] avg=8192.0000 max=8408 max_rank=3 min=8068 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=57128 max_load_reduction_pct=0.871704 moved_to_echo=32622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[524, 524, 515, 509, 519, 511, 507, 510, 477, 510, 508, 516, 470, 525, 529, 527, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 522, 483, 494, 533, 524, 489, 551, 478, 540, 474, 465, 518, 507, 524, 518, 45, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 510, 526, 488, 513, 530, 480, 526, 535, 492, 524, 490, 513, 453, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 492, 491, 528, 482, 478, 518, 512, 515, 519, 486, 521, 527, 512, 520, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 550, 501, 506, 502, 507, 489, 531, 509, 511, 501, 492, 538, 521, 510, 472, 17, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 489, 525, 518, 483, 537, 518, 500, 496, 545, 489, 526, 532, 507, 517, 521, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 493, 534, 523, 552, 490, 525, 535, 515, 479, 505, 505, 544, 562, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 565, 536, 530, 501, 525, 509, 489, 514, 509, 533, 503, 474, 505, 472, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8177, 8121, 8156, 8179, 8258, 8292, 8163] avg=8192.0000 max=8292 max_rank=6 min=8121 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[542, 496, 548, 511, 511, 490, 523, 526, 535, 509, 505, 533, 497, 504, 513, 521, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 519, 555, 507, 488, 510, 525, 503, 525, 517, 521, 533, 499, 487, 517, 485, 41, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 522, 510, 547, 524, 535, 519, 529, 521, 538, 474, 490, 536, 483, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 519, 467, 487, 506, 507, 478, 527, 501, 486, 518, 504, 491, 478, 503, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 483, 531, 525, 533, 512, 519, 487, 526, 496, 481, 497, 545, 525, 481, 498, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 531, 515, 507, 545, 530, 516, 512, 508, 503, 523, 491, 498, 482, 516, 487, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 528, 496, 492, 547, 504, 510, 520, 501, 502, 478, 511, 506, 540, 486, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 506, 513, 550, 496, 532, 518, 538, 511, 495, 534, 510, 491, 507, 547, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8235, 8258, 7919, 8169, 8214, 8169, 8304] avg=8192.0000 max=8304 max_rank=7 min=7919 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 502, 474, 540, 508, 521, 551, 536, 495, 483, 472, 511, 555, 514, 491, 494, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 520, 541, 526, 509, 515, 498, 510, 515, 508, 542, 521, 478, 501, 479, 519, 42, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 465, 507, 517, 528, 514, 541, 557, 505, 518, 508, 505, 478, 488, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 519, 519, 523, 525, 495, 543, 510, 488, 509, 514, 518, 510, 474, 502, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 531, 513, 512, 493, 509, 480, 510, 532, 512, 510, 527, 483, 522, 512, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 488, 520, 482, 489, 496, 500, 540, 525, 491, 496, 510, 511, 544, 500, 511, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 520, 524, 540, 493, 535, 521, 497, 534, 518, 532, 508, 517, 539, 524, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 489, 505, 529, 527, 513, 491, 507, 510, 540, 507, 519, 522, 490, 512, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8214, 8153, 8210, 8194, 8109, 8291, 8198] avg=8192.0000 max=8291 max_rank=6 min=8109 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[505, 507, 517, 548, 497, 532, 519, 501, 507, 482, 500, 495, 534, 532, 509, 486, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 518, 483, 501, 530, 568, 524, 501, 510, 516, 524, 527, 506, 524, 491, 498, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 559, 495, 562, 536, 507, 490, 502, 515, 477, 529, 497, 544, 516, 488, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 523, 521, 487, 511, 545, 542, 518, 500, 539, 502, 515, 521, 502, 490, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 531, 488, 493, 517, 503, 542, 522, 506, 526, 454, 535, 525, 484, 542, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 528, 485, 531, 511, 487, 510, 465, 531, 503, 541, 538, 541, 498, 496, 501, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 531, 550, 487, 557, 533, 486, 528, 543, 463, 510, 504, 475, 483, 517, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 525, 529, 498, 505, 488, 462, 505, 488, 476, 519, 527, 497, 560, 532, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8232, 8212, 8251, 8200, 8204, 8143, 8119] avg=8192.0000 max=8251 max_rank=3 min=8119 min_rank=7 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[473, 488, 517, 559, 532, 541, 492, 542, 523, 502, 505, 465, 525, 513, 531, 502, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 491, 516, 483, 534, 532, 522, 502, 500, 488, 507, 499, 515, 508, 476, 495, 39, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 508, 485, 487, 482, 548, 520, 561, 532, 470, 466, 505, 509, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 457, 520, 547, 487, 533, 491, 515, 554, 499, 494, 501, 511, 555, 520, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 495, 505, 503, 531, 490, 524, 470, 489, 513, 489, 469, 538, 491, 546, 14, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 515, 541, 558, 493, 516, 530, 473, 541, 485, 516, 537, 549, 495, 514, 536, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 540, 503, 509, 543, 503, 496, 510, 477, 486, 531, 527, 464, 505, 532, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 547, 477, 519, 543, 521, 550, 485, 466, 536, 515, 527, 558, 529, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8171, 8124, 8222, 8091, 8275, 8154, 8285] avg=8192.0000 max=8285 max_rank=7 min=8091 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4224, 4224, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[511, 493, 475, 529, 508, 514, 531, 520, 521, 525, 507, 521, 578, 532, 544, 463, 596, 498, 484, 527, 514, 481, 558, 498, 508, 531, 499, 509, 540, 529, 506, 551, 508, 474, 525, 493, 498, 508, 502, 545, 523, 464, 488, 538, 471, 512, 514, 511, 493, 510, 504, 512, 532, 517, 509, 490, 548, 529, 526, 518, 544, 492, 469, 502, 581, 505, 507, 486, 532, 515, 524, 534, 528, 485, 543, 520, 495, 494, 504, 501, 529, 502, 505, 522, 496, 530, 506, 481, 523, 526, 498, 484, 506, 484, 511, 490, 500, 527, 517, 499, 542, 512, 530, 485, 501, 538, 521, 508, 503, 528, 489, 507, 501, 512, 487, 515, 494, 523, 509, 501, 497, 500, 481, 534, 524, 500, 533, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[551, 503, 500, 522, 518, 517, 508, 517, 497, 490, 453, 491, 507, 479, 533, 531, 510, 508, 503, 507, 504, 499, 515, 497, 511, 511, 510, 525, 485, 517, 568, 513, 546, 519, 515, 492, 511, 488, 487, 532, 525, 514, 539, 519, 528, 535, 487, 543, 535, 530, 498, 507, 496, 523, 544, 503, 528, 528, 476, 501, 537, 466, 497, 486, 521, 499, 484, 501, 495, 507, 520, 486, 542, 514, 509, 496, 508, 501, 493, 521, 487, 507, 515, 540, 516, 515, 519, 499, 504, 511, 551, 519, 505, 495, 509, 535, 539, 494, 559, 520, 493, 484, 527, 516, 484, 529, 492, 533, 516, 493, 518, 488, 542, 509, 509, 507, 549, 508, 486, 523, 509, 493, 494, 513, 576, 518, 535, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[516, 474, 513, 479, 525, 515, 520, 504, 475, 533, 512, 521, 500, 497, 540, 499, 538, 563, 498, 496, 495, 537, 507, 544, 505, 477, 496, 513, 528, 530, 511, 469, 494, 498, 514, 494, 501, 527, 494, 481, 536, 517, 552, 506, 512, 487, 521, 476, 504, 505, 522, 517, 523, 504, 520, 506, 540, 491, 482, 524, 536, 510, 504, 558, 507, 514, 527, 516, 478, 520, 526, 493, 548, 559, 521, 528, 478, 542, 522, 538, 536, 462, 533, 527, 511, 501, 510, 479, 527, 554, 497, 473, 477, 523, 494, 515, 512, 523, 514, 542, 488, 550, 532, 526, 494, 517, 490, 512, 517, 526, 506, 506, 494, 521, 501, 543, 521, 569, 487, 524, 503, 498, 479, 516, 505, 533, 477, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[525, 520, 512, 484, 515, 499, 476, 521, 541, 528, 553, 507, 486, 482, 510, 529, 526, 506, 517, 511, 476, 519, 489, 518, 538, 537, 519, 482, 474, 500, 511, 507, 482, 518, 512, 516, 530, 513, 524, 513, 479, 517, 486, 535, 515, 518, 542, 491, 481, 511, 507, 538, 550, 515, 529, 526, 470, 506, 526, 534, 513, 512, 505, 530, 547, 538, 537, 511, 506, 515, 512, 523, 527, 495, 523, 519, 503, 510, 515, 507, 519, 504, 492, 492, 500, 539, 517, 524, 534, 546, 527, 505, 504, 462, 519, 511, 553, 497, 518, 479, 523, 534, 555, 487, 492, 506, 497, 492, 488, 521, 478, 538, 523, 484, 449, 529, 518, 529, 529, 501, 480, 472, 556, 493, 509, 521, 538, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[557, 489, 523, 523, 511, 511, 525, 492, 503, 510, 485, 514, 519, 523, 520, 480, 514, 537, 515, 545, 476, 495, 525, 525, 561, 545, 516, 520, 482, 528, 465, 496, 494, 545, 549, 555, 525, 526, 498, 512, 482, 493, 530, 505, 554, 490, 496, 530, 519, 508, 526, 526, 518, 504, 523, 481, 526, 478, 477, 509, 500, 457, 501, 516, 501, 518, 523, 531, 473, 542, 507, 512, 518, 501, 518, 480, 503, 499, 514, 541, 516, 543, 501, 487, 545, 546, 486, 498, 520, 509, 520, 528, 503, 490, 506, 509, 504, 515, 499, 513, 504, 525, 538, 498, 533, 496, 518, 528, 494, 496, 497, 529, 512, 513, 513, 523, 544, 459, 507, 506, 531, 546, 499, 506, 496, 500, 483, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[496, 502, 536, 496, 495, 515, 519, 490, 523, 517, 536, 495, 503, 531, 507, 495, 545, 492, 477, 501, 504, 500, 543, 487, 538, 505, 475, 527, 533, 487, 464, 495, 500, 509, 536, 512, 529, 521, 490, 536, 523, 538, 517, 531, 505, 510, 530, 512, 531, 481, 504, 555, 464, 507, 490, 499, 511, 534, 498, 504, 510, 532, 486, 508, 480, 490, 488, 522, 563, 517, 521, 515, 525, 531, 544, 534, 500, 522, 499, 474, 534, 506, 544, 525, 483, 468, 528, 505, 505, 548, 519, 534, 507, 546, 494, 557, 504, 512, 499, 533, 485, 514, 512, 483, 485, 506, 534, 519, 490, 488, 525, 506, 539, 464, 465, 524, 512, 522, 484, 525, 549, 506, 563, 511, 515, 519, 530, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[536, 540, 545, 515, 466, 535, 484, 498, 524, 509, 510, 530, 521, 521, 522, 545, 526, 520, 444, 474, 494, 501, 543, 503, 553, 551, 504, 526, 489, 527, 530, 505, 518, 543, 502, 502, 508, 499, 501, 548, 524, 507, 489, 523, 548, 502, 495, 475, 502, 528, 499, 523, 513, 484, 513, 543, 507, 526, 510, 536, 506, 493, 484, 507, 518, 479, 521, 495, 530, 490, 501, 500, 544, 508, 505, 487, 567, 518, 518, 483, 530, 531, 533, 502, 542, 523, 493, 466, 519, 496, 486, 509, 534, 487, 518, 558, 461, 498, 521, 503, 509, 490, 501, 532, 542, 517, 522, 505, 507, 508, 514, 486, 551, 515, 556, 500, 529, 464, 488, 487, 520, 543, 519, 485, 516, 495, 535, 477] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[500, 502, 525, 518, 511, 519, 500, 553, 528, 512, 489, 524, 552, 527, 505, 507, 491, 517, 537, 547, 511, 497, 511, 542, 522, 531, 507, 524, 455, 534, 476, 494, 490, 498, 522, 505, 524, 533, 493, 534, 523, 515, 504, 516, 539, 497, 515, 500, 490, 509, 511, 562, 507, 502, 513, 494, 520, 526, 506, 479, 476, 511, 508, 495, 473, 504, 545, 523, 508, 537, 551, 499, 507, 498, 543, 509, 504, 495, 529, 487, 477, 510, 495, 517, 526, 524, 542, 528, 498, 471, 492, 550, 491, 493, 523, 498, 481, 532, 528, 514, 532, 505, 535, 527, 538, 555, 474, 540, 520, 535, 519, 530, 481, 457, 530, 492, 521, 502, 519, 497, 522, 508, 493, 468, 510, 490, 515, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[502, 493, 475, 529, 508, 514, 531, 520, 521, 525, 507, 521, 578, 532, 544, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 498, 484, 527, 514, 481, 558, 498, 508, 531, 499, 509, 540, 529, 506, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 474, 525, 493, 498, 508, 502, 534, 523, 464, 488, 538, 471, 512, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 510, 504, 512, 532, 517, 509, 490, 548, 529, 526, 518, 544, 492, 469, 502, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 505, 507, 486, 532, 515, 524, 534, 503, 485, 543, 520, 495, 494, 504, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 505, 522, 496, 530, 506, 481, 523, 526, 498, 484, 506, 484, 511, 490, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 527, 517, 499, 542, 512, 527, 485, 501, 538, 521, 508, 503, 528, 489, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 487, 515, 494, 523, 509, 501, 497, 500, 481, 534, 524, 500, 533, 501, 9, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8327, 8063, 8222, 8229, 8096, 8204, 8132] avg=8192.0000 max=8327 max_rank=1 min=8063 min_rank=2 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[507, 474, 513, 479, 525, 515, 520, 504, 475, 533, 512, 521, 500, 497, 540, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 563, 498, 496, 495, 537, 507, 544, 505, 477, 496, 513, 528, 530, 511, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 498, 514, 494, 501, 527, 494, 471, 536, 517, 552, 506, 512, 487, 521, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 522, 517, 523, 504, 520, 506, 540, 491, 482, 524, 536, 510, 504, 558, 26, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 514, 527, 516, 478, 520, 526, 493, 522, 559, 521, 528, 478, 542, 522, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 462, 533, 527, 511, 501, 510, 479, 527, 554, 497, 473, 477, 523, 494, 515, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 514, 542, 488, 550, 529, 526, 494, 517, 490, 512, 517, 526, 506, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 521, 501, 543, 521, 569, 487, 524, 503, 498, 479, 516, 505, 533, 477, 488, 9, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8205, 8100, 8274, 8291, 8122, 8252, 8178] avg=8192.0000 max=8291 max_rank=4 min=8100 min_rank=2 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[487, 502, 536, 496, 495, 515, 519, 490, 523, 517, 536, 495, 503, 531, 507, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 492, 477, 501, 504, 500, 543, 487, 538, 505, 475, 527, 533, 487, 464, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 509, 536, 512, 529, 521, 490, 525, 523, 538, 517, 531, 505, 510, 530, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 481, 504, 555, 464, 507, 490, 499, 511, 534, 498, 504, 510, 532, 486, 508, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 490, 488, 522, 563, 517, 521, 515, 500, 531, 544, 534, 500, 522, 499, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 506, 544, 525, 483, 468, 528, 505, 505, 548, 519, 534, 507, 546, 494, 557, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 512, 499, 533, 485, 514, 509, 483, 485, 506, 534, 519, 490, 488, 525, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 464, 465, 524, 512, 522, 484, 525, 549, 506, 563, 511, 515, 519, 530, 543, 9, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8071, 8288, 8141, 8200, 8306, 8092, 8291] avg=8192.0000 max=8306 max_rank=5 min=8071 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32889 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[541, 503, 500, 522, 518, 517, 508, 517, 497, 490, 453, 491, 507, 479, 533, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 503, 507, 504, 499, 515, 497, 511, 511, 510, 525, 485, 517, 568, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 519, 515, 492, 511, 488, 487, 521, 525, 514, 539, 519, 528, 535, 487, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 530, 498, 507, 496, 523, 544, 503, 528, 528, 476, 501, 537, 466, 497, 486, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 499, 484, 501, 495, 507, 520, 486, 517, 514, 509, 496, 508, 501, 493, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 507, 515, 540, 516, 515, 519, 499, 504, 511, 551, 519, 505, 495, 509, 535, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 494, 559, 520, 493, 484, 524, 516, 484, 529, 492, 533, 516, 493, 518, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 509, 509, 507, 549, 508, 486, 523, 509, 493, 494, 513, 576, 518, 535, 521, 10, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8181, 8269, 8182, 8072, 8230, 8182, 8313] avg=8192.0000 max=8313 max_rank=7 min=8072 min_rank=4 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[547, 489, 523, 523, 511, 511, 525, 492, 503, 510, 485, 514, 519, 523, 520, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 537, 515, 545, 476, 495, 525, 525, 561, 545, 516, 520, 482, 528, 465, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 545, 549, 555, 525, 526, 498, 501, 482, 493, 530, 505, 554, 490, 496, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 526, 526, 518, 504, 523, 481, 526, 478, 477, 509, 500, 457, 501, 516, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 523, 531, 473, 542, 507, 512, 494, 501, 518, 480, 503, 499, 514, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 543, 501, 487, 545, 546, 486, 498, 520, 509, 520, 528, 503, 490, 506, 509, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 515, 499, 513, 504, 525, 535, 498, 533, 496, 518, 528, 494, 496, 497, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 513, 513, 523, 544, 459, 507, 506, 531, 546, 499, 506, 496, 500, 483, 540, 10, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8243, 8273, 8095, 8157, 8210, 8184, 8199] avg=8192.0000 max=8273 max_rank=2 min=8095 min_rank=3 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 540, 545, 515, 466, 535, 484, 498, 524, 509, 510, 530, 521, 521, 522, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 444, 474, 494, 501, 543, 503, 553, 551, 504, 526, 489, 527, 530, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 543, 502, 502, 508, 499, 501, 530, 524, 507, 489, 523, 548, 502, 495, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 528, 499, 523, 513, 484, 513, 543, 507, 526, 510, 536, 506, 493, 484, 507, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 479, 521, 495, 530, 490, 501, 500, 516, 508, 505, 487, 567, 518, 518, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 531, 533, 502, 542, 523, 493, 466, 519, 496, 486, 509, 534, 487, 518, 558, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 498, 521, 503, 509, 490, 490, 532, 542, 517, 522, 505, 507, 508, 514, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 515, 556, 500, 529, 464, 488, 487, 520, 543, 519, 485, 516, 495, 535, 477, 13, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8186, 8166, 8207, 8136, 8237, 8105, 8211] avg=8192.0000 max=8288 max_rank=0 min=8105 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 520, 512, 484, 515, 499, 476, 521, 541, 528, 553, 507, 486, 482, 510, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 517, 511, 476, 519, 489, 518, 538, 537, 519, 482, 474, 500, 511, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 518, 512, 516, 530, 513, 524, 502, 479, 517, 486, 535, 515, 518, 542, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 511, 507, 538, 550, 515, 529, 526, 470, 506, 526, 534, 513, 512, 505, 530, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 538, 537, 511, 506, 515, 512, 523, 502, 495, 523, 519, 503, 510, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 504, 492, 492, 500, 539, 517, 524, 534, 546, 527, 505, 504, 462, 519, 511, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 497, 518, 479, 523, 534, 552, 487, 492, 506, 497, 492, 488, 521, 478, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 484, 449, 529, 518, 529, 529, 501, 480, 472, 556, 493, 509, 521, 538, 502, 9, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8128, 8180, 8280, 8263, 8198, 8155, 8153] avg=8192.0000 max=8280 max_rank=3 min=8128 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 502, 525, 518, 511, 519, 500, 553, 528, 512, 489, 524, 552, 527, 505, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 517, 537, 547, 511, 497, 511, 542, 522, 531, 507, 524, 455, 534, 476, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 498, 522, 505, 524, 533, 493, 523, 523, 515, 504, 516, 539, 497, 515, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 509, 511, 562, 507, 502, 513, 494, 520, 526, 506, 479, 476, 511, 508, 495, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 504, 545, 523, 508, 537, 551, 499, 483, 498, 543, 509, 504, 495, 529, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 510, 495, 517, 526, 524, 542, 528, 498, 471, 492, 550, 491, 493, 523, 498, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 532, 528, 514, 532, 505, 532, 527, 538, 555, 474, 540, 520, 535, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 457, 530, 492, 521, 502, 519, 497, 522, 508, 493, 468, 510, 490, 515, 534, 9, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8195, 8197, 8135, 8188, 8137, 8362, 8059] avg=8192.0000 max=8362 max_rank=6 min=8059 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4352, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4352, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4352, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[506, 520, 573, 503, 505, 497, 514, 524, 528, 481, 509, 510, 542, 538, 524, 502, 545, 544, 506, 533, 498, 518, 516, 507, 548, 522, 555, 487, 512, 481, 535, 491, 504, 514, 481, 518, 477, 488, 506, 512, 506, 555, 543, 524, 539, 536, 520, 490, 506, 522, 479, 510, 520, 563, 508, 502, 518, 481, 532, 487, 490, 499, 521, 478, 520, 503, 503, 542, 541, 500, 494, 526, 499, 541, 503, 480, 536, 477, 519, 526, 493, 502, 504, 496, 488, 500, 523, 485, 513, 507, 533, 520, 501, 510, 525, 536, 521, 486, 510, 504, 559, 536, 506, 502, 464, 513, 502, 505, 514, 518, 488, 529, 510, 522, 531, 557, 545, 472, 493, 502, 478, 487, 504, 515, 507, 525, 496, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[531, 535, 503, 515, 535, 537, 538, 515, 542, 495, 471, 505, 531, 516, 507, 539, 522, 492, 503, 512, 506, 505, 513, 510, 530, 532, 479, 526, 545, 493, 484, 527, 513, 486, 545, 518, 542, 526, 501, 552, 500, 494, 476, 486, 536, 528, 499, 502, 499, 537, 510, 496, 498, 535, 533, 512, 502, 523, 508, 494, 468, 528, 489, 517, 484, 489, 513, 522, 536, 534, 494, 556, 544, 477, 532, 506, 483, 517, 515, 527, 541, 461, 508, 525, 539, 509, 500, 496, 518, 532, 519, 496, 512, 493, 508, 494, 514, 521, 523, 494, 556, 507, 486, 488, 470, 522, 493, 545, 533, 499, 499, 546, 499, 501, 525, 519, 484, 512, 494, 491, 529, 491, 515, 533, 491, 498, 516, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[531, 476, 529, 510, 510, 521, 535, 509, 528, 565, 499, 498, 494, 510, 519, 513, 505, 490, 489, 497, 511, 539, 494, 543, 511, 490, 528, 515, 509, 494, 520, 501, 492, 534, 491, 534, 503, 508, 490, 522, 540, 508, 476, 511, 484, 457, 465, 510, 458, 525, 460, 509, 528, 519, 550, 517, 511, 552, 560, 468, 558, 489, 516, 533, 506, 523, 513, 491, 530, 509, 519, 545, 483, 521, 537, 537, 477, 519, 478, 484, 526, 536, 547, 497, 501, 514, 497, 494, 507, 537, 498, 501, 484, 485, 509, 548, 507, 495, 569, 502, 496, 515, 537, 533, 521, 482, 515, 514, 528, 526, 515, 494, 511, 515, 477, 531, 511, 540, 507, 505, 526, 524, 514, 527, 498, 510, 563, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[470, 490, 470, 574, 479, 537, 493, 500, 505, 506, 499, 562, 518, 499, 478, 512, 515, 514, 491, 520, 533, 552, 504, 511, 505, 532, 473, 508, 530, 536, 499, 499, 525, 513, 524, 517, 509, 517, 510, 519, 548, 485, 497, 520, 492, 509, 513, 536, 489, 509, 489, 513, 528, 519, 497, 502, 533, 502, 482, 505, 524, 533, 502, 554, 496, 486, 535, 501, 509, 498, 507, 494, 521, 526, 524, 519, 501, 489, 494, 548, 527, 527, 519, 550, 551, 515, 487, 534, 522, 515, 517, 520, 498, 532, 521, 533, 560, 490, 536, 529, 500, 510, 525, 507, 498, 509, 528, 491, 532, 503, 531, 470, 522, 498, 491, 528, 505, 475, 484, 520, 506, 503, 474, 463, 461, 514, 551, 577] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[488, 517, 485, 515, 520, 545, 526, 531, 516, 445, 523, 526, 505, 528, 512, 474, 518, 516, 518, 499, 527, 560, 528, 518, 519, 541, 501, 553, 501, 489, 525, 535, 523, 523, 542, 502, 499, 515, 476, 500, 513, 533, 502, 485, 519, 540, 530, 495, 500, 500, 524, 521, 544, 540, 480, 566, 498, 491, 512, 515, 512, 506, 472, 482, 507, 536, 520, 536, 514, 504, 518, 526, 511, 497, 488, 503, 498, 519, 506, 529, 517, 516, 544, 517, 512, 514, 487, 490, 499, 524, 526, 481, 536, 553, 537, 502, 482, 557, 488, 470, 534, 499, 503, 502, 507, 462, 473, 523, 574, 473, 516, 527, 487, 472, 473, 504, 500, 578, 483, 512, 528, 547, 527, 540, 468, 482, 528, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[461, 444, 505, 498, 513, 506, 549, 492, 525, 495, 497, 518, 517, 509, 503, 502, 533, 536, 519, 518, 502, 461, 474, 513, 539, 515, 509, 504, 502, 495, 515, 521, 515, 525, 525, 558, 518, 508, 532, 492, 517, 469, 482, 524, 511, 497, 531, 519, 513, 486, 487, 550, 487, 494, 522, 492, 501, 500, 534, 501, 516, 518, 523, 513, 502, 539, 501, 470, 516, 500, 524, 519, 526, 493, 528, 505, 537, 529, 527, 537, 487, 516, 505, 517, 503, 529, 501, 540, 545, 534, 480, 524, 519, 524, 486, 522, 535, 499, 494, 510, 503, 524, 525, 531, 514, 530, 462, 498, 508, 550, 529, 498, 507, 549, 548, 524, 512, 490, 514, 516, 515, 550, 501, 540, 507, 522, 488, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[537, 562, 507, 518, 536, 517, 497, 537, 548, 537, 467, 530, 578, 520, 536, 535, 480, 511, 484, 500, 509, 478, 543, 516, 517, 504, 533, 531, 530, 512, 468, 525, 506, 528, 471, 526, 532, 512, 496, 508, 515, 512, 508, 474, 535, 494, 524, 471, 474, 499, 525, 503, 498, 483, 477, 507, 484, 476, 545, 532, 456, 526, 520, 497, 504, 541, 498, 532, 499, 481, 490, 487, 502, 538, 505, 563, 544, 526, 521, 503, 504, 501, 527, 558, 472, 517, 536, 517, 532, 512, 490, 508, 486, 487, 516, 480, 534, 510, 548, 526, 496, 516, 480, 529, 549, 536, 511, 538, 508, 484, 476, 496, 520, 527, 519, 553, 505, 535, 470, 475, 510, 510, 542, 485, 502, 513, 544, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[550, 507, 551, 492, 533, 474, 475, 528, 500, 538, 530, 463, 484, 526, 520, 505, 541, 523, 510, 537, 475, 544, 527, 519, 481, 504, 494, 541, 503, 502, 518, 540, 489, 494, 495, 519, 507, 545, 522, 505, 512, 518, 505, 501, 496, 511, 502, 522, 511, 502, 474, 516, 539, 536, 505, 519, 531, 511, 540, 543, 501, 465, 517, 504, 537, 516, 520, 538, 493, 515, 510, 516, 490, 529, 492, 515, 524, 531, 548, 496, 478, 538, 528, 522, 470, 509, 538, 534, 543, 491, 505, 528, 503, 518, 510, 522, 541, 477, 518, 494, 478, 463, 503, 509, 483, 491, 465, 524, 552, 500, 510, 511, 519, 506, 533, 536, 522, 514, 520, 524, 492, 521, 523, 524, 487, 518, 489, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[506, 520, 573, 503, 505, 497, 514, 524, 501, 481, 509, 510, 542, 538, 524, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 544, 506, 533, 498, 518, 516, 507, 548, 522, 555, 464, 512, 481, 535, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 481, 518, 477, 488, 506, 512, 506, 555, 543, 524, 539, 536, 520, 490, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 522, 479, 510, 520, 563, 508, 502, 518, 481, 532, 487, 490, 499, 521, 478, 27, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 503, 503, 542, 541, 500, 494, 502, 499, 541, 503, 480, 536, 477, 519, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 504, 477, 488, 500, 523, 485, 513, 507, 533, 520, 501, 510, 525, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 486, 510, 504, 559, 536, 506, 502, 464, 513, 502, 505, 514, 518, 488, 529, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 522, 531, 557, 545, 472, 493, 502, 478, 487, 504, 515, 507, 525, 496, 486, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8275, 8235, 8154, 8186, 8117, 8176, 8144] avg=8192.0000 max=8275 max_rank=1 min=8117 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[470, 490, 470, 574, 479, 537, 493, 500, 479, 506, 499, 562, 518, 499, 478, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 514, 491, 520, 533, 552, 504, 511, 505, 532, 473, 484, 530, 536, 499, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 513, 524, 517, 509, 517, 510, 519, 548, 485, 497, 520, 492, 509, 513, 536, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 509, 489, 513, 528, 519, 497, 502, 533, 502, 482, 505, 524, 533, 502, 554, 26, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 486, 535, 501, 509, 498, 507, 472, 521, 526, 524, 519, 501, 489, 494, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 527, 519, 529, 551, 515, 487, 534, 522, 515, 517, 520, 498, 532, 521, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 490, 536, 529, 500, 510, 525, 507, 498, 509, 528, 491, 532, 503, 531, 470, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 498, 491, 528, 505, 475, 484, 520, 506, 503, 474, 463, 461, 514, 551, 577, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8198, 8255, 8219, 8126, 8347, 8238, 8087] avg=8192.0000 max=8347 max_rank=5 min=8066 min_rank=0 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[531, 535, 503, 515, 535, 537, 538, 515, 514, 495, 471, 505, 531, 516, 507, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 492, 503, 512, 506, 505, 513, 510, 530, 532, 479, 501, 545, 493, 484, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 486, 545, 518, 542, 526, 501, 552, 500, 494, 476, 486, 536, 528, 499, 502, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 537, 510, 496, 498, 535, 533, 512, 502, 523, 508, 494, 468, 528, 489, 517, 28, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 489, 513, 522, 536, 534, 494, 532, 544, 477, 532, 506, 483, 517, 515, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 461, 508, 505, 539, 509, 500, 496, 518, 532, 519, 496, 512, 493, 508, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 521, 523, 494, 556, 507, 486, 488, 470, 522, 493, 545, 533, 499, 499, 546, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 501, 525, 519, 484, 512, 494, 491, 529, 491, 515, 533, 491, 498, 516, 515, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8154, 8227, 8189, 8205, 8131, 8216, 8127] avg=8192.0000 max=8287 max_rank=0 min=8127 min_rank=7 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[531, 476, 529, 510, 510, 521, 535, 509, 496, 565, 499, 498, 494, 510, 519, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 490, 489, 497, 511, 539, 494, 543, 511, 490, 528, 483, 509, 494, 520, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 534, 491, 534, 503, 508, 490, 522, 540, 508, 476, 511, 484, 457, 465, 510, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 525, 460, 509, 528, 519, 550, 517, 511, 552, 560, 468, 558, 489, 516, 533, 32, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 513, 491, 530, 509, 519, 514, 483, 521, 537, 537, 477, 519, 478, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 536, 547, 468, 501, 514, 497, 494, 507, 537, 498, 501, 484, 485, 509, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 569, 502, 496, 515, 537, 533, 521, 482, 515, 514, 528, 526, 515, 494, 18, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 515, 477, 531, 511, 540, 507, 505, 526, 524, 514, 527, 498, 510, 563, 514, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8104, 8054, 8301, 8141, 8152, 8277, 8292] avg=8192.0000 max=8301 max_rank=3 min=8054 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[461, 444, 505, 498, 513, 506, 549, 492, 498, 495, 497, 518, 517, 509, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 536, 519, 518, 502, 461, 474, 513, 539, 515, 509, 481, 502, 495, 515, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 525, 525, 558, 518, 508, 532, 492, 517, 469, 482, 524, 511, 497, 531, 519, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 486, 487, 550, 487, 494, 522, 492, 501, 500, 534, 501, 516, 518, 523, 513, 27, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 539, 501, 470, 516, 500, 524, 496, 526, 493, 528, 505, 537, 529, 527, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 516, 505, 498, 503, 529, 501, 540, 545, 534, 480, 524, 519, 524, 486, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 499, 494, 510, 503, 524, 525, 531, 514, 530, 462, 498, 508, 550, 529, 498, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 549, 548, 524, 512, 490, 514, 516, 515, 550, 501, 540, 507, 522, 488, 508, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8007, 8133, 8245, 8175, 8230, 8213, 8228, 8305] avg=8192.0000 max=8305 max_rank=7 min=8007 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 517, 485, 515, 520, 545, 526, 531, 489, 445, 523, 526, 505, 528, 512, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 518, 499, 527, 560, 528, 518, 519, 541, 501, 527, 501, 489, 525, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 542, 502, 499, 515, 476, 500, 513, 533, 502, 485, 519, 540, 530, 495, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 500, 524, 521, 544, 540, 480, 566, 498, 491, 512, 515, 512, 506, 472, 482, 27, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 536, 520, 536, 514, 504, 518, 502, 511, 497, 488, 503, 498, 519, 506, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 516, 544, 498, 512, 514, 487, 490, 499, 524, 526, 481, 536, 553, 537, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 557, 488, 470, 534, 499, 503, 502, 507, 462, 473, 523, 574, 473, 516, 527, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 472, 473, 504, 500, 578, 483, 512, 528, 547, 527, 540, 468, 482, 528, 486, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8322, 8220, 8203, 8188, 8236, 8109, 8129] avg=8192.0000 max=8322 max_rank=1 min=8109 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[550, 507, 551, 492, 533, 474, 475, 528, 474, 538, 530, 463, 484, 526, 520, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 523, 510, 537, 475, 544, 527, 519, 481, 504, 494, 516, 503, 502, 518, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 494, 495, 519, 507, 545, 522, 505, 512, 518, 505, 501, 496, 511, 502, 522, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 474, 516, 539, 536, 505, 519, 531, 511, 540, 543, 501, 465, 517, 504, 26, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 520, 538, 493, 515, 510, 493, 490, 529, 492, 515, 524, 531, 548, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 538, 528, 503, 470, 509, 538, 534, 543, 491, 505, 528, 503, 518, 510, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 477, 518, 494, 478, 463, 503, 509, 483, 491, 465, 524, 552, 500, 510, 511, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 533, 536, 522, 514, 520, 524, 492, 521, 523, 524, 487, 518, 489, 490, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8234, 8166, 8252, 8247, 8218, 8037, 8232] avg=8192.0000 max=8252 max_rank=3 min=8037 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[537, 562, 507, 518, 536, 517, 497, 537, 519, 537, 467, 530, 578, 520, 536, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 511, 484, 500, 509, 478, 543, 516, 517, 504, 533, 506, 530, 512, 468, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 471, 526, 532, 512, 496, 508, 515, 512, 508, 474, 535, 494, 524, 471, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 499, 525, 503, 498, 483, 477, 507, 484, 476, 545, 532, 456, 526, 520, 497, 29, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 541, 498, 532, 499, 481, 490, 466, 502, 538, 505, 563, 544, 526, 521, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 527, 537, 472, 517, 536, 517, 532, 512, 490, 508, 486, 487, 516, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 548, 526, 496, 516, 480, 529, 549, 536, 511, 538, 508, 484, 476, 496, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 527, 519, 553, 505, 535, 470, 475, 510, 510, 542, 485, 502, 513, 544, 495, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8433, 8116, 8134, 8043, 8213, 8122, 8255, 8220] avg=8192.0000 max=8433 max_rank=0 min=8043 min_rank=3 imbalance_ratio=1.029419 max_load_reduction=57103 max_load_reduction_pct=0.871323 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4224, 4352, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3968, 4096, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4352, 4096, 4224, 3968, 4096, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[507, 515, 502, 523, 526, 506, 502, 506, 482, 534, 559, 513, 498, 529, 498, 505, 491, 474, 513, 523, 488, 506, 477, 528, 513, 512, 505, 526, 508, 506, 533, 496, 490, 506, 501, 517, 531, 492, 517, 484, 530, 536, 537, 523, 548, 494, 493, 514, 551, 510, 529, 481, 505, 522, 535, 503, 513, 499, 487, 496, 492, 522, 495, 521, 481, 503, 458, 535, 549, 535, 475, 481, 516, 517, 527, 512, 510, 516, 537, 508, 524, 528, 556, 488, 495, 462, 512, 493, 494, 553, 513, 499, 519, 515, 519, 502, 539, 507, 500, 490, 516, 563, 481, 534, 489, 506, 523, 535, 502, 540, 533, 512, 525, 562, 522, 529, 482, 533, 491, 498, 515, 519, 536, 476, 509, 540, 456, 563] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[515, 495, 555, 517, 519, 519, 500, 470, 522, 556, 537, 537, 521, 487, 463, 499, 515, 529, 492, 516, 506, 521, 529, 504, 469, 502, 502, 503, 527, 540, 508, 555, 512, 495, 506, 462, 460, 499, 515, 520, 530, 453, 480, 464, 494, 521, 521, 491, 506, 479, 534, 515, 514, 541, 484, 497, 518, 506, 544, 506, 528, 535, 515, 533, 519, 531, 541, 520, 531, 506, 508, 494, 525, 522, 527, 531, 502, 497, 484, 540, 501, 526, 503, 515, 514, 492, 508, 519, 511, 544, 544, 540, 540, 505, 526, 468, 556, 540, 481, 483, 502, 566, 523, 530, 523, 534, 476, 543, 546, 504, 480, 467, 543, 517, 539, 471, 493, 495, 529, 482, 534, 510, 505, 489, 501, 544, 464, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[512, 524, 495, 562, 493, 533, 542, 513, 493, 524, 524, 536, 504, 528, 496, 535, 482, 527, 489, 538, 515, 490, 522, 535, 533, 485, 538, 511, 525, 491, 506, 504, 498, 502, 511, 528, 533, 509, 554, 523, 487, 494, 466, 558, 504, 523, 475, 505, 532, 497, 501, 483, 503, 521, 492, 498, 489, 508, 515, 521, 505, 483, 495, 515, 513, 541, 528, 516, 524, 495, 534, 522, 500, 543, 520, 515, 491, 506, 527, 532, 466, 521, 525, 493, 471, 502, 533, 506, 506, 518, 512, 519, 565, 534, 540, 476, 442, 528, 535, 525, 495, 490, 533, 533, 547, 521, 535, 505, 502, 508, 479, 494, 502, 526, 473, 518, 523, 492, 509, 501, 509, 516, 523, 487, 483, 540, 518, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[492, 531, 538, 585, 494, 528, 524, 539, 508, 513, 496, 493, 489, 560, 532, 519, 481, 481, 526, 527, 504, 545, 559, 531, 507, 480, 497, 478, 509, 517, 488, 482, 483, 494, 495, 516, 532, 500, 521, 507, 516, 512, 524, 561, 497, 462, 527, 491, 493, 528, 498, 508, 480, 506, 533, 468, 501, 517, 547, 524, 513, 491, 540, 538, 522, 495, 506, 550, 516, 529, 514, 539, 525, 491, 485, 467, 524, 483, 521, 530, 486, 536, 531, 526, 523, 523, 498, 528, 537, 503, 515, 503, 520, 541, 510, 515, 542, 462, 530, 539, 538, 490, 508, 524, 484, 528, 525, 477, 497, 501, 530, 511, 489, 509, 533, 505, 515, 513, 488, 534, 510, 473, 494, 486, 503, 491, 533, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[481, 544, 522, 495, 545, 500, 512, 489, 527, 490, 502, 529, 498, 494, 476, 507, 483, 520, 484, 471, 491, 531, 498, 499, 512, 535, 538, 504, 533, 548, 489, 512, 535, 531, 525, 506, 512, 510, 526, 527, 535, 541, 450, 532, 524, 518, 507, 485, 543, 505, 501, 529, 508, 501, 497, 495, 495, 505, 518, 507, 460, 485, 545, 491, 532, 493, 512, 503, 515, 504, 530, 538, 470, 487, 521, 519, 542, 498, 477, 567, 526, 503, 511, 532, 510, 504, 531, 524, 533, 488, 518, 517, 518, 493, 500, 551, 510, 531, 525, 533, 511, 523, 478, 519, 499, 521, 490, 515, 550, 536, 464, 522, 513, 531, 510, 485, 533, 495, 547, 547, 529, 483, 535, 485, 540, 493, 509, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[562, 521, 536, 478, 469, 492, 506, 505, 514, 506, 505, 528, 471, 523, 504, 506, 537, 517, 500, 521, 499, 556, 509, 519, 529, 531, 513, 507, 503, 502, 485, 515, 514, 482, 505, 513, 544, 497, 474, 541, 486, 538, 477, 507, 482, 526, 519, 514, 495, 491, 502, 509, 480, 511, 517, 489, 525, 497, 514, 527, 527, 525, 502, 477, 496, 511, 530, 522, 537, 539, 531, 515, 502, 517, 526, 521, 524, 496, 517, 537, 494, 533, 518, 476, 495, 560, 520, 546, 508, 470, 519, 551, 491, 489, 528, 533, 526, 470, 518, 549, 512, 513, 493, 525, 519, 509, 492, 501, 476, 534, 516, 530, 488, 532, 519, 519, 507, 504, 526, 543, 505, 517, 548, 497, 519, 467, 521, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[493, 533, 516, 498, 499, 495, 501, 535, 483, 501, 504, 480, 486, 528, 516, 524, 540, 517, 538, 529, 491, 536, 470, 518, 537, 506, 485, 493, 523, 508, 524, 527, 502, 495, 483, 515, 511, 519, 492, 503, 518, 512, 538, 495, 525, 534, 511, 477, 514, 507, 562, 466, 490, 533, 483, 498, 514, 553, 500, 509, 496, 496, 517, 489, 521, 534, 504, 557, 486, 504, 507, 519, 487, 531, 531, 536, 525, 503, 509, 486, 518, 503, 516, 530, 531, 535, 531, 512, 496, 512, 489, 523, 525, 524, 539, 504, 484, 516, 489, 508, 511, 490, 522, 521, 520, 476, 545, 529, 548, 512, 533, 563, 463, 534, 511, 508, 501, 531, 517, 499, 530, 492, 523, 515, 526, 483, 506, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[509, 539, 495, 474, 509, 501, 509, 472, 495, 531, 528, 496, 528, 490, 514, 485, 478, 459, 542, 519, 536, 590, 541, 499, 524, 521, 511, 514, 522, 544, 519, 535, 497, 516, 520, 511, 517, 514, 505, 478, 523, 528, 530, 525, 511, 510, 497, 509, 513, 493, 496, 475, 499, 499, 543, 502, 522, 517, 511, 502, 515, 504, 502, 510, 528, 537, 531, 517, 518, 500, 509, 549, 509, 524, 549, 503, 497, 488, 540, 505, 512, 488, 497, 493, 536, 495, 545, 512, 495, 485, 521, 552, 520, 518, 517, 525, 495, 489, 457, 570, 541, 541, 497, 502, 509, 492, 520, 527, 506, 484, 535, 532, 524, 506, 509, 461, 545, 511, 549, 502, 500, 488, 476, 496, 487, 514, 524, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[507, 515, 502, 523, 526, 506, 502, 506, 482, 534, 559, 513, 498, 529, 498, 505, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 474, 513, 523, 488, 498, 477, 528, 513, 512, 505, 526, 508, 506, 533, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 506, 501, 517, 531, 492, 517, 484, 530, 536, 537, 523, 548, 494, 493, 514, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 510, 529, 481, 505, 522, 535, 503, 513, 499, 487, 496, 492, 522, 495, 521, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 503, 458, 475, 549, 535, 475, 481, 516, 517, 527, 512, 510, 516, 537, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 528, 556, 488, 495, 462, 512, 493, 494, 553, 513, 456, 519, 515, 519, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 507, 500, 465, 516, 563, 481, 534, 489, 506, 523, 535, 502, 540, 533, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 562, 522, 529, 482, 533, 491, 498, 515, 519, 536, 476, 509, 540, 456, 563, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8091, 8258, 8221, 8100, 8129, 8245, 8280] avg=8192.0000 max=8280 max_rank=7 min=8091 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 495, 555, 517, 519, 519, 500, 470, 522, 556, 537, 537, 521, 487, 463, 499, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 529, 492, 516, 506, 513, 529, 504, 469, 502, 502, 503, 527, 540, 508, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 495, 506, 462, 460, 499, 515, 520, 530, 453, 480, 464, 494, 521, 521, 491, 46, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 479, 534, 515, 514, 541, 484, 497, 518, 506, 544, 506, 528, 535, 515, 533, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 541, 462, 531, 506, 508, 494, 525, 522, 527, 531, 502, 497, 484, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 503, 515, 514, 492, 508, 519, 511, 544, 544, 494, 540, 505, 526, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 540, 481, 458, 502, 566, 523, 530, 523, 534, 476, 543, 546, 504, 480, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 517, 539, 471, 493, 495, 529, 482, 534, 510, 505, 489, 501, 544, 464, 524, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8210, 7971, 8313, 8220, 8210, 8229, 8164] avg=8192.0000 max=8313 max_rank=3 min=7971 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[481, 544, 522, 495, 545, 500, 512, 489, 527, 490, 502, 529, 498, 494, 476, 507, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 520, 484, 471, 491, 523, 498, 499, 512, 535, 538, 504, 533, 548, 489, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 531, 525, 506, 512, 510, 526, 527, 535, 541, 450, 532, 524, 518, 507, 485, 44, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 505, 501, 529, 508, 501, 497, 495, 495, 505, 518, 507, 460, 485, 545, 491, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 493, 512, 447, 515, 504, 530, 538, 470, 487, 521, 519, 542, 498, 477, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 503, 511, 532, 510, 504, 531, 524, 533, 488, 518, 473, 518, 493, 500, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 531, 525, 506, 511, 523, 478, 519, 499, 521, 490, 515, 550, 536, 464, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 510, 485, 533, 495, 547, 547, 529, 483, 535, 485, 540, 493, 509, 499, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8140, 8310, 8141, 8152, 8215, 8200, 8260] avg=8192.0000 max=8310 max_rank=2 min=8118 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[512, 524, 495, 562, 493, 533, 542, 513, 493, 524, 524, 536, 504, 528, 496, 535, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 527, 489, 538, 515, 483, 522, 535, 533, 485, 538, 511, 525, 491, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 511, 528, 533, 509, 554, 523, 487, 494, 466, 558, 504, 523, 475, 505, 44, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 501, 483, 503, 521, 492, 498, 489, 508, 515, 521, 505, 483, 495, 515, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 541, 528, 459, 524, 495, 534, 522, 500, 543, 520, 515, 491, 506, 527, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 521, 525, 493, 471, 502, 533, 506, 506, 518, 512, 475, 565, 534, 540, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 528, 535, 498, 495, 490, 533, 533, 547, 521, 535, 505, 502, 508, 479, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 526, 473, 518, 523, 492, 509, 501, 509, 516, 523, 487, 483, 540, 518, 517, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8184, 8216, 8115, 8250, 8143, 8145, 8163] avg=8192.0000 max=8320 max_rank=0 min=8115 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[562, 521, 536, 478, 469, 492, 506, 505, 514, 506, 505, 528, 471, 523, 504, 506, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 517, 500, 521, 499, 541, 509, 519, 529, 531, 513, 507, 503, 502, 485, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 482, 505, 513, 544, 497, 474, 541, 486, 538, 477, 507, 482, 526, 519, 514, 51, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 491, 502, 509, 480, 511, 517, 489, 525, 497, 514, 527, 527, 525, 502, 477, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 530, 455, 537, 539, 531, 515, 502, 517, 526, 521, 524, 496, 517, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 533, 518, 476, 495, 560, 520, 546, 508, 470, 519, 500, 491, 489, 528, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 470, 518, 515, 512, 513, 493, 525, 519, 509, 492, 501, 476, 534, 516, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 532, 519, 519, 507, 504, 526, 543, 505, 517, 548, 497, 519, 467, 521, 513, 28, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8228, 8181, 8150, 8254, 8180, 8149, 8257] avg=8192.0000 max=8257 max_rank=7 min=8137 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[493, 533, 516, 498, 499, 495, 501, 535, 483, 501, 504, 480, 486, 528, 516, 524, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 517, 538, 529, 491, 528, 470, 518, 537, 506, 485, 493, 523, 508, 524, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 483, 515, 511, 519, 492, 503, 518, 512, 538, 495, 525, 534, 511, 477, 45, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 507, 562, 466, 490, 533, 483, 498, 514, 553, 500, 509, 496, 496, 517, 489, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 534, 504, 495, 486, 504, 507, 519, 487, 531, 531, 536, 525, 503, 509, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 516, 530, 531, 535, 531, 512, 496, 512, 489, 478, 525, 524, 539, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 516, 489, 482, 511, 490, 522, 521, 520, 476, 545, 529, 548, 512, 533, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 534, 511, 508, 501, 531, 517, 499, 530, 492, 523, 515, 526, 483, 506, 511, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8234, 8177, 8189, 8178, 8243, 8241, 8175] avg=8192.0000 max=8243 max_rank=5 min=8099 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 539, 495, 474, 509, 501, 509, 472, 495, 531, 528, 496, 528, 490, 514, 485, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 459, 542, 519, 536, 581, 541, 499, 524, 521, 511, 514, 522, 544, 519, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 516, 520, 511, 517, 514, 505, 478, 523, 528, 530, 525, 511, 510, 497, 509, 47, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 493, 496, 475, 499, 499, 543, 502, 522, 517, 511, 502, 515, 504, 502, 510, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 531, 459, 518, 500, 509, 549, 509, 524, 549, 503, 497, 488, 540, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 497, 493, 536, 495, 545, 512, 495, 485, 521, 505, 520, 518, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 489, 457, 541, 541, 541, 497, 502, 509, 492, 520, 527, 506, 484, 535, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 509, 461, 545, 511, 549, 502, 500, 488, 476, 496, 487, 514, 524, 509, 27, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8345, 8240, 8161, 8246, 8164, 8168, 8129] avg=8192.0000 max=8345 max_rank=1 min=8083 min_rank=0 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 531, 538, 585, 494, 528, 524, 539, 508, 513, 496, 493, 489, 560, 532, 519, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 481, 526, 527, 504, 537, 559, 531, 507, 480, 497, 478, 509, 517, 488, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 494, 495, 516, 532, 500, 521, 507, 516, 512, 524, 561, 497, 462, 527, 491, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 528, 498, 508, 480, 506, 533, 468, 501, 517, 547, 524, 513, 491, 540, 538, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 506, 489, 516, 529, 514, 539, 525, 491, 485, 467, 524, 483, 521, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 536, 531, 526, 523, 523, 498, 528, 537, 503, 515, 460, 520, 541, 510, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 462, 530, 512, 538, 490, 508, 524, 484, 528, 525, 477, 497, 501, 530, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 509, 533, 505, 515, 513, 488, 534, 510, 473, 494, 486, 503, 491, 533, 506, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8348, 8104, 8183, 8246, 8136, 8252, 8159, 8108] avg=8192.0000 max=8348 max_rank=0 min=8104 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 3840, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[488, 516, 516, 482, 519, 521, 541, 503, 493, 535, 555, 503, 530, 518, 496, 520, 486, 526, 539, 499, 494, 512, 519, 485, 525, 497, 536, 487, 499, 496, 521, 528, 538, 522, 458, 517, 492, 539, 499, 515, 504, 570, 506, 528, 508, 511, 516, 533, 492, 492, 500, 520, 522, 530, 475, 516, 527, 471, 488, 515, 552, 530, 537, 515, 531, 551, 538, 468, 518, 529, 500, 506, 534, 471, 497, 495, 532, 495, 509, 500, 496, 518, 539, 506, 518, 483, 534, 554, 485, 550, 515, 511, 487, 513, 488, 493, 511, 504, 533, 517, 479, 506, 471, 500, 512, 521, 480, 551, 490, 499, 543, 507, 530, 536, 498, 520, 538, 506, 516, 559, 541, 503, 525, 505, 481, 509, 493, 465] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[502, 540, 510, 508, 571, 557, 541, 513, 497, 518, 447, 468, 525, 507, 493, 504, 553, 540, 530, 496, 506, 522, 503, 535, 496, 495, 544, 499, 502, 516, 557, 527, 532, 515, 499, 530, 552, 515, 505, 484, 515, 487, 537, 494, 469, 514, 549, 533, 517, 451, 477, 509, 477, 514, 524, 507, 541, 503, 520, 504, 512, 492, 522, 498, 476, 533, 489, 559, 491, 498, 481, 478, 515, 499, 559, 537, 520, 506, 537, 497, 493, 503, 531, 502, 528, 507, 505, 502, 503, 514, 519, 564, 521, 510, 509, 497, 532, 526, 497, 505, 530, 486, 504, 509, 494, 498, 505, 524, 510, 465, 483, 501, 508, 501, 526, 514, 488, 524, 523, 487, 524, 573, 492, 518, 491, 555, 517, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[524, 512, 511, 470, 531, 524, 513, 493, 551, 509, 474, 508, 556, 491, 550, 493, 503, 495, 492, 515, 530, 531, 517, 501, 525, 493, 488, 515, 508, 527, 519, 514, 530, 519, 498, 474, 484, 503, 513, 547, 503, 529, 540, 511, 494, 508, 495, 499, 548, 512, 520, 508, 569, 516, 504, 501, 528, 506, 528, 491, 513, 492, 494, 508, 517, 522, 487, 546, 538, 548, 541, 506, 508, 505, 559, 545, 501, 483, 563, 520, 484, 533, 509, 519, 523, 463, 493, 534, 515, 483, 516, 509, 499, 529, 538, 506, 547, 490, 505, 461, 489, 506, 467, 487, 491, 522, 540, 508, 493, 498, 545, 498, 536, 498, 502, 499, 517, 491, 559, 502, 521, 527, 493, 507, 531, 534, 505, 457] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[483, 514, 552, 517, 516, 503, 540, 482, 508, 500, 525, 510, 533, 526, 512, 530, 504, 498, 504, 479, 521, 492, 509, 537, 526, 519, 539, 511, 488, 523, 492, 518, 499, 519, 480, 515, 540, 511, 555, 566, 525, 499, 493, 485, 525, 505, 527, 496, 481, 495, 530, 475, 549, 503, 542, 548, 541, 508, 534, 487, 510, 537, 472, 523, 522, 537, 511, 487, 525, 489, 504, 503, 515, 532, 479, 507, 507, 508, 518, 525, 551, 518, 518, 529, 531, 464, 494, 529, 488, 474, 505, 504, 504, 532, 511, 523, 481, 501, 520, 494, 518, 529, 480, 504, 531, 471, 535, 515, 520, 533, 503, 523, 527, 480, 491, 486, 507, 496, 550, 494, 496, 530, 487, 512, 494, 541, 547, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[521, 466, 518, 526, 528, 530, 480, 503, 509, 528, 517, 528, 513, 486, 539, 485, 502, 541, 524, 504, 549, 490, 543, 503, 508, 523, 488, 508, 523, 545, 505, 479, 488, 471, 532, 496, 522, 512, 514, 488, 520, 493, 553, 557, 521, 501, 503, 526, 507, 512, 508, 501, 556, 529, 535, 525, 497, 518, 481, 534, 496, 570, 514, 496, 542, 488, 537, 525, 476, 530, 489, 511, 499, 479, 485, 520, 532, 454, 527, 498, 524, 522, 536, 511, 488, 525, 500, 494, 551, 529, 504, 488, 501, 530, 525, 507, 513, 546, 528, 519, 528, 523, 498, 530, 497, 556, 500, 478, 521, 483, 490, 509, 502, 483, 469, 523, 531, 536, 515, 529, 475, 517, 521, 489, 492, 535, 489, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[509, 502, 470, 547, 524, 529, 528, 551, 517, 517, 537, 525, 502, 498, 541, 521, 505, 525, 548, 453, 491, 514, 508, 494, 517, 492, 530, 513, 487, 530, 523, 506, 498, 531, 537, 520, 530, 519, 516, 494, 541, 495, 507, 507, 517, 529, 510, 510, 473, 512, 516, 523, 564, 523, 511, 473, 493, 546, 498, 499, 537, 468, 518, 542, 513, 475, 515, 486, 514, 488, 481, 502, 507, 482, 508, 504, 483, 506, 519, 524, 501, 521, 519, 519, 508, 500, 531, 503, 510, 525, 508, 516, 511, 539, 478, 533, 498, 510, 509, 508, 497, 499, 479, 503, 543, 535, 515, 527, 505, 518, 547, 517, 532, 484, 531, 537, 525, 504, 515, 504, 518, 468, 483, 503, 547, 513, 508, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[489, 500, 510, 514, 538, 536, 541, 493, 501, 542, 530, 579, 482, 495, 521, 499, 508, 499, 518, 524, 479, 525, 546, 497, 535, 468, 531, 503, 531, 512, 501, 508, 536, 517, 519, 526, 516, 493, 520, 522, 500, 503, 500, 519, 552, 461, 505, 497, 452, 513, 497, 489, 502, 503, 531, 481, 505, 490, 507, 512, 489, 555, 500, 519, 540, 514, 500, 514, 490, 537, 505, 497, 504, 522, 546, 519, 485, 492, 501, 543, 474, 504, 524, 526, 502, 525, 487, 498, 500, 564, 533, 503, 522, 516, 525, 523, 507, 561, 533, 554, 506, 539, 501, 510, 518, 570, 508, 527, 488, 481, 507, 493, 520, 524, 508, 499, 478, 491, 519, 528, 499, 486, 501, 561, 489, 506, 531, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[520, 472, 503, 522, 552, 564, 495, 474, 494, 497, 493, 551, 530, 475, 515, 490, 489, 526, 492, 529, 485, 510, 498, 505, 506, 519, 463, 507, 510, 500, 534, 553, 550, 464, 543, 516, 493, 518, 521, 522, 489, 523, 494, 494, 510, 534, 523, 485, 538, 524, 533, 558, 474, 502, 503, 534, 515, 520, 503, 539, 494, 511, 493, 498, 544, 490, 505, 531, 516, 511, 476, 500, 498, 513, 542, 534, 519, 520, 523, 524, 503, 494, 522, 546, 516, 537, 507, 483, 530, 478, 520, 523, 482, 497, 517, 492, 519, 527, 516, 492, 495, 554, 551, 513, 498, 551, 491, 513, 501, 468, 499, 537, 527, 497, 522, 512, 491, 479, 483, 504, 622, 498, 510, 499, 470, 557, 519, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[488, 516, 516, 482, 487, 521, 541, 503, 493, 535, 555, 503, 530, 518, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 526, 539, 499, 494, 512, 519, 485, 525, 497, 536, 487, 499, 496, 521, 528, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 522, 458, 517, 492, 539, 499, 515, 504, 570, 506, 528, 508, 511, 516, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 492, 500, 520, 522, 530, 475, 516, 527, 471, 488, 515, 552, 530, 537, 515, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 551, 538, 468, 518, 529, 500, 506, 534, 471, 497, 495, 532, 495, 509, 500, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 518, 537, 506, 518, 483, 534, 554, 485, 550, 515, 511, 487, 513, 488, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 533, 517, 479, 506, 471, 500, 512, 521, 480, 551, 490, 499, 543, 507, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 536, 498, 520, 538, 506, 516, 559, 541, 503, 525, 505, 481, 509, 493, 465, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8154, 8238, 8188, 8183, 8188, 8145, 8236] avg=8192.0000 max=8238 max_rank=2 min=8145 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32752 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[502, 540, 510, 508, 535, 557, 541, 513, 497, 518, 447, 468, 525, 507, 493, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 540, 530, 496, 506, 522, 503, 535, 496, 495, 544, 499, 502, 516, 557, 527, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 515, 499, 530, 552, 515, 505, 484, 515, 487, 537, 494, 469, 514, 549, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 451, 477, 509, 477, 514, 524, 507, 541, 503, 520, 504, 512, 492, 522, 498, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 533, 489, 559, 491, 498, 481, 478, 515, 499, 559, 537, 520, 506, 537, 497, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 503, 529, 502, 528, 507, 505, 502, 503, 514, 519, 564, 521, 510, 509, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 497, 505, 530, 486, 504, 509, 494, 498, 505, 524, 510, 465, 483, 501, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 526, 514, 488, 524, 523, 487, 524, 573, 492, 518, 491, 555, 517, 523, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8326, 8212, 8074, 8184, 8206, 8093, 8276] avg=8192.0000 max=8326 max_rank=1 min=8074 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32759 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[509, 502, 470, 547, 491, 529, 528, 551, 517, 517, 537, 525, 502, 498, 541, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 525, 548, 453, 491, 514, 508, 494, 517, 492, 530, 513, 487, 530, 523, 506, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 531, 537, 520, 530, 519, 516, 494, 541, 495, 507, 507, 517, 529, 510, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 512, 516, 523, 564, 523, 511, 473, 493, 546, 498, 499, 537, 468, 518, 542, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 475, 515, 486, 514, 488, 481, 502, 507, 482, 508, 504, 483, 506, 519, 524, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 517, 519, 508, 500, 531, 503, 510, 525, 508, 516, 511, 539, 478, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 510, 509, 508, 497, 499, 479, 503, 543, 535, 515, 527, 505, 518, 547, 517, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 484, 531, 537, 525, 504, 515, 504, 518, 468, 483, 503, 547, 513, 508, 514, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8141, 8244, 8202, 8015, 8220, 8232, 8197] avg=8192.0000 max=8285 max_rank=0 min=8015 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32664 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[483, 514, 552, 517, 476, 503, 540, 482, 508, 500, 525, 510, 533, 526, 512, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 504, 479, 521, 492, 509, 537, 526, 519, 539, 511, 488, 523, 492, 518, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 519, 480, 515, 540, 511, 555, 566, 525, 499, 493, 485, 525, 505, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 495, 530, 475, 549, 503, 542, 548, 541, 508, 534, 487, 510, 537, 472, 523, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 537, 511, 487, 525, 489, 504, 503, 515, 532, 479, 507, 507, 508, 518, 525, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 518, 512, 529, 531, 464, 494, 529, 488, 474, 505, 504, 504, 532, 511, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 501, 520, 494, 518, 529, 480, 504, 531, 471, 535, 515, 520, 533, 503, 523, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 480, 491, 486, 507, 496, 550, 494, 496, 530, 487, 512, 494, 541, 547, 510, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8175, 8206, 8246, 8183, 8169, 8183, 8163] avg=8192.0000 max=8246 max_rank=3 min=8163 min_rank=7 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32698 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[524, 512, 511, 470, 498, 524, 513, 493, 551, 509, 474, 508, 556, 491, 550, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 495, 492, 515, 530, 531, 517, 501, 525, 493, 488, 515, 508, 527, 519, 514, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 498, 474, 484, 503, 513, 547, 503, 529, 540, 511, 494, 508, 495, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 512, 520, 508, 569, 516, 504, 501, 528, 506, 528, 491, 513, 492, 494, 508, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 487, 546, 538, 548, 541, 506, 508, 505, 559, 545, 501, 483, 563, 520, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 533, 507, 519, 523, 463, 493, 534, 515, 483, 516, 509, 499, 529, 538, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 490, 505, 461, 489, 506, 467, 487, 491, 522, 540, 508, 493, 498, 545, 498, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 502, 499, 517, 491, 559, 502, 521, 527, 493, 507, 531, 534, 505, 457, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8178, 8129, 8244, 8398, 8151, 8069, 8190] avg=8192.0000 max=8398 max_rank=4 min=8069 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32808 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[521, 466, 518, 526, 495, 530, 480, 503, 509, 528, 517, 528, 513, 486, 539, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 541, 524, 504, 549, 490, 543, 503, 508, 523, 488, 508, 523, 545, 505, 479, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 471, 532, 496, 522, 512, 514, 488, 520, 493, 553, 557, 521, 501, 503, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 508, 501, 556, 529, 535, 525, 497, 518, 481, 534, 496, 570, 514, 496, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 488, 537, 525, 476, 530, 489, 511, 499, 479, 485, 520, 532, 454, 527, 498, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 534, 511, 488, 525, 500, 494, 551, 529, 504, 488, 501, 530, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 546, 528, 519, 528, 523, 498, 530, 497, 556, 500, 478, 521, 483, 490, 509, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 483, 469, 523, 531, 536, 515, 529, 475, 517, 521, 489, 492, 535, 489, 496, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8240, 8180, 8285, 8100, 8233, 8241, 8113] avg=8192.0000 max=8285 max_rank=3 min=8100 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32687 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[489, 500, 510, 514, 505, 536, 541, 493, 501, 542, 530, 579, 482, 495, 521, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 499, 518, 524, 479, 525, 546, 497, 535, 468, 531, 503, 531, 512, 501, 508, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 517, 519, 526, 516, 493, 520, 522, 500, 503, 500, 519, 552, 461, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 513, 497, 489, 502, 503, 531, 481, 505, 490, 507, 512, 489, 555, 500, 519, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 514, 500, 514, 490, 537, 505, 497, 504, 522, 546, 519, 485, 492, 501, 543, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 504, 522, 526, 502, 525, 487, 498, 500, 564, 533, 503, 522, 516, 525, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 561, 533, 554, 506, 539, 501, 510, 518, 570, 508, 527, 488, 481, 507, 493, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 508, 499, 478, 491, 519, 528, 499, 486, 501, 561, 489, 506, 531, 472, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8190, 8168, 8051, 8218, 8224, 8325, 8123] avg=8192.0000 max=8325 max_rank=6 min=8051 min_rank=3 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32890 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[520, 472, 503, 522, 518, 564, 495, 474, 494, 497, 493, 551, 530, 475, 515, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 526, 492, 529, 485, 510, 498, 505, 506, 519, 463, 507, 510, 500, 534, 553, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 464, 543, 516, 493, 518, 521, 522, 489, 523, 494, 494, 510, 534, 523, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 524, 533, 558, 474, 502, 503, 534, 515, 520, 503, 539, 494, 511, 493, 498, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 490, 505, 531, 516, 511, 476, 500, 498, 513, 542, 534, 519, 520, 523, 524, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 494, 520, 546, 516, 537, 507, 483, 530, 478, 520, 523, 482, 497, 517, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 527, 516, 492, 495, 554, 551, 513, 498, 551, 491, 513, 501, 468, 499, 537, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 497, 522, 512, 491, 479, 483, 504, 622, 498, 510, 499, 470, 557, 519, 537, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8132, 8159, 8246, 8255, 8145, 8248, 8238] avg=8192.0000 max=8255 max_rank=4 min=8113 min_rank=0 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32886 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4352, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4096, 3968, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4352, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[514, 480, 525, 563, 485, 529, 525, 499, 519, 518, 534, 510, 512, 505, 529, 502, 520, 521, 518, 534, 524, 549, 535, 497, 506, 529, 496, 537, 483, 531, 509, 502, 504, 511, 515, 512, 540, 519, 505, 494, 542, 500, 494, 536, 481, 498, 467, 524, 492, 504, 503, 501, 532, 510, 486, 522, 490, 500, 507, 554, 531, 516, 529, 550, 489, 477, 511, 503, 519, 468, 524, 511, 481, 524, 546, 524, 521, 554, 536, 479, 526, 528, 490, 536, 527, 501, 529, 516, 511, 512, 535, 477, 509, 502, 494, 516, 488, 524, 465, 516, 490, 509, 500, 527, 531, 498, 522, 490, 526, 519, 516, 531, 496, 486, 504, 487, 475, 526, 488, 525, 525, 463, 537, 519, 515, 510, 529, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[550, 522, 506, 499, 510, 488, 473, 501, 497, 540, 479, 555, 522, 538, 476, 528, 487, 466, 518, 498, 495, 534, 533, 523, 510, 548, 535, 465, 491, 516, 497, 479, 482, 505, 519, 526, 493, 506, 530, 485, 510, 501, 497, 532, 547, 515, 506, 494, 543, 478, 511, 524, 552, 523, 530, 514, 493, 500, 506, 520, 515, 526, 493, 492, 514, 514, 514, 515, 526, 502, 534, 522, 520, 517, 515, 501, 516, 518, 505, 489, 559, 549, 508, 529, 521, 524, 502, 491, 551, 551, 520, 528, 512, 531, 491, 511, 501, 526, 483, 529, 497, 531, 511, 485, 524, 495, 506, 565, 500, 494, 508, 494, 453, 490, 500, 475, 493, 496, 518, 507, 560, 480, 521, 532, 554, 490, 526, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[489, 461, 530, 517, 506, 481, 491, 535, 516, 474, 528, 512, 508, 533, 519, 522, 465, 515, 520, 512, 525, 498, 540, 532, 504, 491, 507, 533, 535, 512, 473, 527, 486, 480, 530, 516, 530, 482, 514, 479, 510, 498, 555, 526, 491, 515, 502, 514, 532, 527, 512, 494, 501, 528, 521, 511, 485, 496, 505, 541, 514, 532, 526, 525, 520, 476, 537, 525, 530, 484, 499, 493, 535, 509, 508, 498, 502, 525, 529, 536, 507, 487, 525, 487, 507, 481, 511, 519, 526, 527, 554, 495, 539, 505, 500, 522, 524, 525, 514, 476, 480, 535, 505, 504, 479, 520, 564, 555, 487, 492, 517, 510, 506, 530, 507, 532, 520, 522, 487, 492, 481, 533, 533, 539, 534, 528, 515, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[521, 471, 534, 488, 486, 504, 496, 522, 508, 522, 486, 527, 541, 483, 519, 490, 517, 515, 496, 511, 507, 517, 540, 488, 502, 515, 532, 541, 480, 518, 476, 516, 524, 542, 505, 473, 544, 475, 554, 474, 542, 525, 492, 495, 475, 516, 500, 499, 484, 508, 514, 510, 529, 557, 544, 543, 460, 502, 541, 518, 535, 468, 489, 540, 452, 530, 513, 517, 532, 520, 501, 548, 549, 470, 494, 506, 519, 504, 504, 526, 505, 487, 492, 524, 513, 478, 497, 524, 530, 532, 524, 521, 538, 488, 512, 520, 500, 547, 502, 524, 521, 499, 561, 478, 496, 525, 525, 490, 459, 517, 548, 551, 501, 492, 528, 556, 513, 515, 523, 537, 495, 531, 544, 494, 496, 503, 555, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[537, 517, 490, 502, 509, 516, 499, 532, 499, 478, 510, 507, 520, 541, 504, 503, 496, 511, 476, 506, 488, 537, 487, 513, 513, 515, 531, 504, 491, 506, 532, 500, 487, 532, 530, 523, 519, 539, 482, 519, 496, 519, 505, 451, 497, 518, 483, 525, 510, 530, 501, 549, 517, 500, 533, 514, 538, 534, 511, 522, 517, 496, 496, 547, 518, 524, 526, 529, 543, 529, 487, 493, 542, 503, 492, 513, 525, 492, 502, 576, 472, 505, 474, 499, 524, 540, 521, 482, 482, 511, 496, 498, 550, 541, 498, 475, 511, 506, 479, 491, 511, 502, 501, 523, 548, 533, 553, 556, 511, 487, 460, 543, 500, 478, 533, 504, 510, 530, 518, 541, 527, 512, 539, 519, 519, 507, 505, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[492, 464, 499, 507, 497, 510, 484, 518, 552, 515, 495, 544, 545, 527, 526, 553, 509, 487, 526, 540, 549, 496, 489, 474, 510, 539, 506, 515, 491, 517, 523, 540, 514, 497, 507, 530, 555, 493, 519, 508, 533, 539, 515, 521, 528, 525, 523, 511, 507, 501, 518, 495, 516, 503, 529, 557, 524, 526, 464, 469, 491, 518, 582, 479, 493, 526, 550, 491, 504, 502, 525, 552, 511, 526, 495, 537, 513, 514, 518, 530, 507, 474, 491, 528, 526, 468, 521, 481, 527, 492, 484, 534, 513, 504, 497, 513, 480, 470, 524, 495, 486, 519, 491, 504, 481, 533, 521, 492, 517, 545, 512, 522, 514, 508, 506, 494, 499, 542, 479, 489, 544, 529, 529, 507, 506, 527, 484, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[484, 469, 519, 521, 494, 509, 495, 481, 499, 502, 520, 516, 513, 520, 523, 534, 500, 526, 521, 543, 487, 537, 502, 473, 543, 488, 542, 508, 513, 513, 527, 502, 461, 531, 472, 543, 545, 494, 549, 523, 560, 510, 483, 478, 534, 560, 480, 528, 517, 496, 511, 513, 505, 549, 498, 498, 512, 512, 508, 473, 507, 524, 500, 508, 509, 495, 478, 483, 543, 479, 512, 490, 538, 531, 498, 517, 561, 559, 510, 512, 491, 492, 508, 535, 528, 525, 502, 504, 539, 527, 505, 461, 539, 513, 524, 513, 549, 523, 533, 507, 508, 517, 489, 497, 492, 504, 518, 533, 503, 521, 537, 498, 513, 526, 520, 519, 547, 534, 486, 499, 502, 466, 515, 511, 510, 479, 549, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[497, 479, 500, 499, 523, 522, 532, 492, 527, 500, 561, 478, 541, 494, 534, 496, 501, 521, 479, 528, 523, 508, 521, 523, 514, 513, 520, 516, 513, 491, 489, 486, 535, 521, 533, 526, 527, 513, 509, 522, 511, 564, 496, 514, 513, 509, 543, 478, 524, 512, 530, 456, 516, 527, 538, 513, 490, 500, 506, 521, 483, 495, 511, 514, 501, 476, 469, 560, 498, 511, 516, 523, 495, 481, 536, 504, 523, 497, 543, 539, 512, 480, 546, 520, 496, 544, 506, 535, 536, 508, 520, 467, 487, 488, 462, 503, 495, 519, 525, 489, 505, 513, 544, 493, 512, 514, 538, 520, 510, 547, 473, 508, 528, 528, 525, 564, 492, 491, 495, 505, 509, 529, 514, 526, 531, 517, 522, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[521, 471, 534, 488, 486, 504, 496, 522, 508, 522, 486, 527, 541, 483, 519, 490, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 496, 511, 507, 517, 540, 488, 502, 515, 532, 541, 480, 518, 476, 516, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 542, 505, 473, 541, 475, 554, 474, 542, 525, 492, 495, 475, 516, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 514, 510, 529, 537, 544, 543, 460, 502, 541, 518, 535, 468, 489, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 530, 513, 517, 506, 520, 501, 548, 549, 470, 494, 506, 519, 504, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 492, 524, 513, 478, 497, 524, 530, 532, 524, 521, 538, 488, 512, 520, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 547, 502, 524, 521, 499, 561, 478, 496, 525, 525, 490, 459, 517, 548, 551, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 492, 528, 556, 513, 515, 523, 537, 495, 531, 530, 494, 496, 503, 555, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8184, 8132, 8222, 8159, 8201, 8251, 8263] avg=8192.0000 max=8263 max_rank=7 min=8124 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[550, 522, 506, 499, 510, 488, 473, 501, 497, 540, 479, 555, 522, 538, 476, 528, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 466, 518, 498, 495, 534, 533, 523, 510, 548, 535, 465, 491, 516, 497, 479, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 505, 519, 526, 491, 506, 530, 485, 510, 501, 497, 532, 547, 515, 506, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 478, 511, 524, 552, 505, 530, 514, 493, 500, 506, 520, 515, 526, 493, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 514, 515, 500, 502, 534, 522, 520, 517, 515, 501, 516, 518, 505, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 549, 508, 529, 521, 524, 502, 491, 551, 551, 520, 528, 512, 531, 491, 511, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 483, 529, 497, 531, 511, 485, 524, 495, 506, 565, 500, 494, 508, 494, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 490, 500, 475, 493, 496, 518, 507, 560, 480, 509, 532, 554, 490, 526, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8106, 8146, 8202, 8196, 8393, 8155, 8128] avg=8192.0000 max=8393 max_rank=5 min=8106 min_rank=1 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 480, 525, 563, 485, 529, 525, 499, 519, 518, 534, 510, 512, 505, 529, 502, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 521, 518, 534, 524, 549, 535, 497, 506, 529, 496, 537, 483, 531, 509, 502, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 515, 512, 537, 519, 505, 494, 542, 500, 494, 536, 481, 498, 467, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 503, 501, 532, 492, 486, 522, 490, 500, 507, 554, 531, 516, 529, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 477, 511, 503, 493, 468, 524, 511, 481, 524, 546, 524, 521, 554, 536, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 490, 536, 527, 501, 529, 516, 511, 512, 535, 477, 509, 502, 494, 516, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 524, 465, 516, 490, 509, 500, 527, 531, 498, 522, 490, 526, 519, 516, 531, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 486, 504, 487, 475, 526, 488, 525, 525, 463, 524, 519, 515, 510, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8303, 8139, 8209, 8141, 8224, 8159, 8086] avg=8192.0000 max=8303 max_rank=1 min=8086 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[489, 461, 530, 517, 506, 481, 491, 535, 516, 474, 528, 512, 508, 533, 519, 522, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 515, 520, 512, 525, 498, 540, 532, 504, 491, 507, 533, 535, 512, 473, 527, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 480, 530, 516, 527, 482, 514, 479, 510, 498, 555, 526, 491, 515, 502, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 527, 512, 494, 501, 509, 521, 511, 485, 496, 505, 541, 514, 532, 526, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 476, 537, 525, 504, 484, 499, 493, 535, 509, 508, 498, 502, 525, 529, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 525, 487, 507, 481, 511, 519, 526, 527, 554, 495, 539, 505, 500, 522, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 514, 476, 480, 535, 505, 504, 479, 520, 564, 555, 487, 492, 517, 510, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 530, 507, 532, 520, 522, 487, 492, 481, 533, 521, 539, 534, 528, 515, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8200, 8125, 8231, 8180, 8208, 8194, 8250] avg=8192.0000 max=8250 max_rank=7 min=8125 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[537, 517, 490, 502, 509, 516, 499, 532, 499, 478, 510, 507, 520, 541, 504, 503, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 476, 506, 488, 537, 487, 513, 513, 515, 531, 504, 491, 506, 532, 500, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 532, 530, 523, 514, 539, 482, 519, 496, 519, 505, 451, 497, 518, 483, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 530, 501, 549, 517, 468, 533, 514, 538, 534, 511, 522, 517, 496, 496, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 524, 526, 529, 512, 529, 487, 493, 542, 503, 492, 513, 525, 492, 502, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 505, 474, 499, 524, 540, 521, 482, 482, 511, 496, 498, 550, 541, 498, 475, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 479, 491, 511, 502, 501, 523, 548, 533, 553, 556, 511, 487, 460, 543, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 478, 533, 504, 510, 530, 518, 541, 527, 512, 516, 519, 519, 507, 505, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8127, 8120, 8283, 8263, 8087, 8230, 8226] avg=8192.0000 max=8283 max_rank=3 min=8087 min_rank=5 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[492, 464, 499, 507, 497, 510, 484, 518, 552, 515, 495, 544, 545, 527, 526, 553, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 526, 540, 549, 496, 489, 474, 510, 539, 506, 515, 491, 517, 523, 540, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 497, 507, 530, 552, 493, 519, 508, 533, 539, 515, 521, 528, 525, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 501, 518, 495, 516, 485, 529, 557, 524, 526, 464, 469, 491, 518, 582, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 526, 550, 491, 479, 502, 525, 552, 511, 526, 495, 537, 513, 514, 518, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 474, 491, 528, 526, 468, 521, 481, 527, 492, 484, 534, 513, 504, 497, 513, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 470, 524, 495, 486, 519, 491, 504, 481, 533, 521, 492, 517, 545, 512, 522, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 506, 494, 499, 542, 479, 489, 544, 529, 517, 507, 506, 527, 484, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8222, 8315, 8161, 8262, 8075, 8099, 8149] avg=8192.0000 max=8315 max_rank=2 min=8075 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[497, 479, 500, 499, 523, 522, 532, 492, 527, 500, 561, 478, 541, 494, 534, 496, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 479, 528, 523, 508, 521, 523, 514, 513, 520, 516, 513, 491, 489, 486, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 521, 533, 526, 524, 513, 509, 522, 511, 564, 496, 514, 513, 509, 543, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 530, 456, 516, 508, 538, 513, 490, 500, 506, 521, 483, 495, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 476, 469, 560, 473, 511, 516, 523, 495, 481, 536, 504, 523, 497, 543, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 480, 546, 520, 496, 544, 506, 535, 536, 508, 520, 467, 487, 488, 462, 503, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 519, 525, 489, 505, 513, 544, 493, 512, 514, 538, 520, 510, 547, 473, 508, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 528, 525, 564, 492, 491, 495, 505, 509, 529, 502, 526, 531, 517, 522, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8157, 8311, 8117, 8147, 8126, 8212, 8266] avg=8192.0000 max=8311 max_rank=2 min=8117 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[484, 469, 519, 521, 494, 509, 495, 481, 499, 502, 520, 516, 513, 520, 523, 534, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 526, 521, 543, 487, 537, 502, 473, 543, 488, 542, 508, 513, 513, 527, 502, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 531, 472, 543, 542, 494, 549, 523, 560, 510, 483, 478, 534, 560, 480, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 511, 513, 505, 529, 498, 498, 512, 512, 508, 473, 507, 524, 500, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 478, 483, 516, 479, 512, 490, 538, 531, 498, 517, 561, 559, 510, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 492, 508, 535, 528, 525, 502, 504, 539, 527, 505, 461, 539, 513, 524, 513, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 523, 533, 507, 508, 517, 489, 497, 492, 504, 518, 533, 503, 521, 537, 498, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 520, 519, 547, 534, 486, 499, 502, 466, 503, 511, 510, 479, 549, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8237, 8248, 8111, 8188, 8222, 8236, 8168] avg=8192.0000 max=8248 max_rank=2 min=8111 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4352, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=81920 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=81920 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=81920 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4352, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 device_initiated_grouped_gemm=True + [2026-08-26 16:29:49] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 419.0 | throughput per GPU (TFLOP/s/GPU): 431.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.169026E+00 | loss scale: 1.0 | grad norm: 5.242 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:29:49 +W0826 16:29:50.560000 220093 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 16:29:50.561000 220093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 220187 closing signal SIGTERM +W0826 16:29:50.562000 220093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 220188 closing signal SIGTERM +W0826 16:29:50.565000 220093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 220189 closing signal SIGTERM +W0826 16:29:50.565000 220093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 220190 closing signal SIGTERM +W0826 16:29:50.567000 220093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 220191 closing signal SIGTERM +W0826 16:29:50.569000 220093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 220192 closing signal SIGTERM +W0826 16:29:50.570000 220093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 220193 closing signal SIGTERM +W0826 16:29:50.572000 220093 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 220194 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 220093 got signal: 15 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv index 269a5f8a073def22812adf911ba68a8cfae61493..aadd835b1d23c66067ea75e3d8e7f8cce7dd494c 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 2 1, 1 2, 1 3, 1 @@ -38,22 +38,22 @@ 5, 4 6, 4 7, 4 -0, 4 -1, 4 -2, 4 -3, 4 -4, 4 -5, 4 -6, 4 -7, 4 -0, 821 -1, 821 -2, 821 -3, 821 -4, 821 -5, 821 -6, 821 -7, 821 +0, 725 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 935 +1, 949 +2, 949 +3, 961 +4, 973 +5, 933 +6, 971 +7, 951 0, 1975 1, 1975 2, 1975 @@ -62,30 +62,30 @@ 5, 1975 6, 1975 7, 1975 -0, 46263 -1, 45927 -2, 46359 -3, 45975 -4, 45879 -5, 46071 -6, 47079 -7, 46263 -0, 48205 -1, 48219 -2, 48231 -3, 48209 -4, 47925 -5, 48153 -6, 48155 -7, 48221 -0, 49543 -1, 49543 -2, 49543 -3, 49543 -4, 49255 -5, 49543 -6, 49543 -7, 49543 +0, 47803 +1, 47803 +2, 47803 +3, 47803 +4, 47515 +5, 47803 +6, 47803 +7, 47803 +0, 49279 +1, 49279 +2, 49055 +3, 49279 +4, 48767 +5, 49055 +6, 49119 +7, 49279 +0, 49927 +1, 49927 +2, 49927 +3, 49927 +4, 49639 +5, 49927 +6, 49927 +7, 49927 0, 53697 1, 53697 2, 53697 @@ -102,14 +102,14 @@ 5, 54105 6, 54105 7, 54105 -0, 54409 -1, 54419 -2, 54411 -3, 54443 -4, 54111 -5, 54419 -6, 54413 -7, 54425 +0, 54429 +1, 54455 +2, 54439 +3, 54421 +4, 54163 +5, 54407 +6, 54431 +7, 54437 0, 55543 1, 55543 2, 55543 @@ -142,14 +142,14 @@ 5, 59799 6, 59799 7, 59799 -0, 105823 -1, 105869 -2, 105847 -3, 105885 -4, 105795 -5, 105829 -6, 105841 -7, 105823 +0, 105919 +1, 105959 +2, 105959 +3, 105959 +4, 105915 +5, 105957 +6, 105939 +7, 105957 0, 106307 1, 106347 2, 106347 @@ -182,54 +182,54 @@ 5, 106349 6, 106349 7, 106349 -0, 110361 -1, 108481 -2, 110401 -3, 110401 -4, 110381 +0, 109593 +1, 107713 +2, 107713 +3, 107713 +4, 107693 5, 107713 -6, 110401 -7, 108097 +6, 107713 +7, 107713 0, 111393 -1, 111049 +1, 111433 2, 111049 3, 111433 4, 111029 5, 111433 -6, 111433 -7, 111049 -0, 112059 -1, 111733 -2, 111737 -3, 112119 -4, 111717 -5, 112109 -6, 112135 -7, 111769 +6, 111049 +7, 111433 +0, 112087 +1, 112135 +2, 111769 +3, 112131 +4, 111733 +5, 112175 +6, 111787 +7, 112075 0, 141823 -1, 141559 +1, 141943 2, 141559 3, 141943 4, 141539 5, 141943 -6, 141943 -7, 141559 +6, 141559 +7, 141943 0, 141823 -1, 141559 +1, 141943 2, 141559 3, 141943 4, 141539 5, 141943 -6, 141943 -7, 141559 -0, 141823 -1, 141559 -2, 141559 -3, 141943 +6, 141559 +7, 141943 +0, 141821 +1, 141941 +2, 141557 +3, 141941 4, 141539 -5, 141943 -6, 141943 -7, 141559 +5, 141941 +6, 141559 +7, 141943 0, 3963 1, 3966 2, 3966 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log index 965ec44ac2571bee3cb6959d85fa3c217e9a72e5..5d4416357c5b53514100ce103c5ec86e89a2771c 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log @@ -1,5 +1,5 @@ -[16:16:59] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[16:16:59] WARN: token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +[16:26:07] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[16:26:07] WARN: token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) ============================================================== QWEN3_235B / echo (slots/rank=16 policy=matched) E=128 K=8 H'=1536 m=16 @@ -8,10 +8,17 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ b099fd84 ============================================================== -W0826 16:17:00.815000 203627 torch/distributed/run.py:766] -W0826 16:17:00.815000 203627 torch/distributed/run.py:766] ***************************************** -W0826 16:17:00.815000 203627 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 16:17:00.815000 203627 torch/distributed/run.py:766] ***************************************** +W0826 16:26:08.659000 213257 torch/distributed/run.py:766] +W0826 16:26:08.659000 213257 torch/distributed/run.py:766] ***************************************** +W0826 16:26:08.659000 213257 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:26:08.659000 213257 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -562,11 +569,6 @@ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled @@ -575,53 +577,51 @@ WARNING: one_logger package is required to enable e2e metrics tracking. please g warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 4[Gloo] Rank [Gloo] Rank 7 is connected to 6 is connected to 7 peer ranks. 5 is connected to 7Expected number of connected peer ranks is : is connected to 7 peer ranks. 77 peer ranks. Expected number of connected peer ranks is : - peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 7 -7 - -[Gloo] Rank [Gloo] Rank 5[Gloo] Rank is connected to 270 is connected to peer ranks. is connected to 7Expected number of connected peer ranks is : 7 peer ranks. 7 peer ranks. Expected number of connected peer ranks is : -Expected number of connected peer ranks is : 77 - [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to [Gloo] Rank 7 peer ranks. Expected number of connected peer ranks is : 67 is connected to -7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 - [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank1]:[W826 16:17:19.130645271 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 16:17:19.130711804 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 16:17:19.130756118 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 16:17:19.133355170 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 16:17:19.134509395 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 16:17:19.134514719 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 16:17:19.140706487 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W826 16:26:26.927648234 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:26:26.927726126 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 16:26:26.929805686 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 16:26:26.935392716 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:26:26.935395439 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 16:26:26.935420115 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 16:26:26.935596929 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.036 seconds +>>> done with dataset index builder. Compilation time: 0.034 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 16:17:19.289250136 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.877 seconds -time to initialize megatron (seconds): 7.241 -[after megatron is initialized] datetime: 2026-08-26 16:17:25 +[rank0]:[W826 16:26:27.100956222 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.712 seconds +time to initialize megatron (seconds): 7.616 +[after megatron is initialized] datetime: 2026-08-26 16:26:32 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -645,98 +645,98 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (206045184 elements, 206045184 padded size): module.output_layer.weight Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight module.decoder.final_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.layers.11.self_attention.q_layernorm.weight - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.self_attention.k_layernorm.weight Params for bucket 3 (71835904 elements, 71835904 padded size): - module.decoder.layers.10.self_attention.linear_qkv.weight - module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.mlp.router.weight module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.q_layernorm.weight Params for bucket 4 (71835904 elements, 71835904 padded size): - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.q_layernorm.weight - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.k_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight Params for bucket 5 (71835904 elements, 71835904 padded size): - module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.pre_mlp_layernorm.weight - module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight Params for bucket 6 (71835904 elements, 71835904 padded size): module.decoder.layers.7.mlp.router.weight module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.k_layernorm.weight module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.7.pre_mlp_layernorm.weight - module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.k_layernorm.weight module.decoder.layers.6.self_attention.q_layernorm.weight - module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight Params for bucket 8 (71835904 elements, 71835904 padded size): - module.decoder.layers.5.self_attention.k_layernorm.weight - module.decoder.layers.5.self_attention.linear_qkv.weight module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.self_attention.q_layernorm.weight - module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight Params for bucket 9 (71835904 elements, 71835904 padded size): - module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.weight - module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.4.self_attention.q_layernorm.weight - module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight Params for bucket 10 (71835904 elements, 71835904 padded size): module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.self_attention.linear_qkv.weight module.decoder.layers.3.self_attention.q_layernorm.weight - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_proj.weight Params for bucket 11 (71835904 elements, 71835904 padded size): - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.self_attention.linear_proj.weight Params for bucket 12 (71835904 elements, 71835904 padded size): - module.decoder.layers.1.self_attention.q_layernorm.weight - module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.weight module.decoder.layers.1.self_attention.k_layernorm.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight Params for bucket 13 (71835904 elements, 71835904 padded size): module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.q_layernorm.weight module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.q_layernorm.weight module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.0.self_attention.linear_qkv.weight @@ -768,13 +768,13 @@ Params for bucket 3 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight13 Params for bucket 4 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight12 - module.decoder.layers.11.mlp.experts.linear_fc1.weight11 module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 module.decoder.layers.11.mlp.experts.linear_fc1.weight9 Params for bucket 5 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight8 - module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 Params for bucket 6 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight4 @@ -797,12 +797,12 @@ Params for bucket 8 (44040192 elements, 44040192 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight5 Params for bucket 9 (50331648 elements, 50331648 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc1.weight14 - module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 Params for bucket 10 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight13 module.decoder.layers.10.mlp.experts.linear_fc1.weight11 @@ -815,14 +815,14 @@ Params for bucket 11 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight7 Params for bucket 12 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight5 - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 Params for bucket 13 (44040192 elements, 44040192 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight14 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight13 Params for bucket 14 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight12 @@ -833,33 +833,33 @@ Params for bucket 14 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight11 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 Params for bucket 15 (50331648 elements, 50331648 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 Params for bucket 16 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight14 - module.decoder.layers.9.mlp.experts.linear_fc1.weight13 - module.decoder.layers.9.mlp.experts.linear_fc1.weight12 module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 Params for bucket 17 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 module.decoder.layers.9.mlp.experts.linear_fc1.weight9 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 - module.decoder.layers.9.mlp.experts.linear_fc1.weight8 Params for bucket 18 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight5 Params for bucket 19 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 Params for bucket 20 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight14 module.decoder.layers.8.mlp.experts.linear_fc2.weight12 @@ -871,20 +871,20 @@ Params for bucket 20 (44040192 elements, 44040192 padded size): Params for bucket 21 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight6 module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 Params for bucket 22 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight0 - module.decoder.layers.8.mlp.experts.linear_fc1.weight14 module.decoder.layers.8.mlp.experts.linear_fc1.weight15 module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 Params for bucket 23 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight12 - module.decoder.layers.8.mlp.experts.linear_fc1.weight11 module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 module.decoder.layers.8.mlp.experts.linear_fc1.weight10 Params for bucket 24 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight8 @@ -894,35 +894,35 @@ Params for bucket 24 (50331648 elements, 50331648 padded size): Params for bucket 25 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 Params for bucket 26 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight14 module.decoder.layers.7.mlp.experts.linear_fc2.weight12 - module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 module.decoder.layers.7.mlp.experts.linear_fc2.weight11 Params for bucket 27 (44040192 elements, 44040192 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight10 - module.decoder.layers.7.mlp.experts.linear_fc2.weight8 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 Params for bucket 28 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight1 module.decoder.layers.7.mlp.experts.linear_fc1.weight15 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight14 Params for bucket 29 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 module.decoder.layers.7.mlp.experts.linear_fc1.weight12 - module.decoder.layers.7.mlp.experts.linear_fc1.weight10 Params for bucket 30 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight9 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 @@ -936,9 +936,9 @@ Params for bucket 31 (50331648 elements, 50331648 padded size): Params for bucket 32 (44040192 elements, 44040192 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight1 module.decoder.layers.6.mlp.experts.linear_fc2.weight14 - module.decoder.layers.6.mlp.experts.linear_fc2.weight15 - module.decoder.layers.6.mlp.experts.linear_fc2.weight13 module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 Params for bucket 33 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight12 module.decoder.layers.6.mlp.experts.linear_fc2.weight10 @@ -948,17 +948,17 @@ Params for bucket 33 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight9 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 module.decoder.layers.6.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc1.weight15 Params for bucket 35 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight14 - module.decoder.layers.6.mlp.experts.linear_fc1.weight12 module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 module.decoder.layers.6.mlp.experts.linear_fc1.weight13 Params for bucket 36 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight10 @@ -968,71 +968,71 @@ Params for bucket 36 (50331648 elements, 50331648 padded size): Params for bucket 37 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 - module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 Params for bucket 38 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.6.mlp.experts.linear_fc1.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 module.decoder.layers.5.mlp.experts.linear_fc2.weight14 - module.decoder.layers.5.mlp.experts.linear_fc2.weight10 module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 module.decoder.layers.5.mlp.experts.linear_fc2.weight8 - module.decoder.layers.5.mlp.experts.linear_fc2.weight11 - module.decoder.layers.5.mlp.experts.linear_fc2.weight13 - module.decoder.layers.5.mlp.experts.linear_fc2.weight9 Params for bucket 40 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 Params for bucket 41 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc1.weight14 - module.decoder.layers.5.mlp.experts.linear_fc1.weight15 module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 Params for bucket 42 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight12 - module.decoder.layers.5.mlp.experts.linear_fc1.weight11 module.decoder.layers.5.mlp.experts.linear_fc1.weight9 module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 Params for bucket 43 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc1.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight5 - module.decoder.layers.5.mlp.experts.linear_fc1.weight6 Params for bucket 44 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 Params for bucket 45 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight15 - module.decoder.layers.4.mlp.experts.linear_fc2.weight13 - module.decoder.layers.4.mlp.experts.linear_fc2.weight12 module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 Params for bucket 46 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight9 - module.decoder.layers.4.mlp.experts.linear_fc2.weight7 - module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight10 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight8 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc1.weight14 - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight15 Params for bucket 48 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight13 module.decoder.layers.4.mlp.experts.linear_fc1.weight12 @@ -1041,17 +1041,17 @@ Params for bucket 48 (50331648 elements, 50331648 padded size): Params for bucket 49 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight9 module.decoder.layers.4.mlp.experts.linear_fc1.weight6 - module.decoder.layers.4.mlp.experts.linear_fc1.weight8 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 Params for bucket 50 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 - module.decoder.layers.4.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 Params for bucket 51 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight14 module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight13 module.decoder.layers.3.mlp.experts.linear_fc2.weight15 Params for bucket 52 (44040192 elements, 44040192 padded size): @@ -1059,154 +1059,154 @@ Params for bucket 52 (44040192 elements, 44040192 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight10 module.decoder.layers.3.mlp.experts.linear_fc2.weight8 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 - module.decoder.layers.3.mlp.experts.linear_fc2.weight11 module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 Params for bucket 53 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 Params for bucket 54 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight14 - module.decoder.layers.3.mlp.experts.linear_fc1.weight12 module.decoder.layers.3.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 Params for bucket 55 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight10 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 - module.decoder.layers.3.mlp.experts.linear_fc1.weight9 module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 Params for bucket 56 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 Params for bucket 57 (44040192 elements, 44040192 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight15 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 Params for bucket 58 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight11 - module.decoder.layers.2.mlp.experts.linear_fc2.weight13 - module.decoder.layers.2.mlp.experts.linear_fc2.weight9 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 - module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 Params for bucket 59 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 Params for bucket 60 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight0 - module.decoder.layers.2.mlp.experts.linear_fc1.weight14 - module.decoder.layers.2.mlp.experts.linear_fc1.weight15 module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 Params for bucket 61 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight12 - module.decoder.layers.2.mlp.experts.linear_fc1.weight11 module.decoder.layers.2.mlp.experts.linear_fc1.weight9 module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 Params for bucket 62 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight8 - module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight6 module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 Params for bucket 63 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 Params for bucket 64 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight14 - module.decoder.layers.1.mlp.experts.linear_fc2.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight15 module.decoder.layers.1.mlp.experts.linear_fc2.weight13 module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 - module.decoder.layers.1.mlp.experts.linear_fc2.weight9 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc2.weight8 - module.decoder.layers.1.mlp.experts.linear_fc2.weight5 Params for bucket 66 (50331648 elements, 50331648 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc1.weight15 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 Params for bucket 67 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight13 - module.decoder.layers.1.mlp.experts.linear_fc1.weight12 - module.decoder.layers.1.mlp.experts.linear_fc1.weight10 module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight9 - module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 Params for bucket 69 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight2 Params for bucket 70 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight14 - module.decoder.layers.0.mlp.experts.linear_fc2.weight15 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 module.decoder.layers.0.mlp.experts.linear_fc2.weight13 Params for bucket 71 (44040192 elements, 44040192 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight12 - module.decoder.layers.0.mlp.experts.linear_fc2.weight10 module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 module.decoder.layers.0.mlp.experts.linear_fc2.weight6 - module.decoder.layers.0.mlp.experts.linear_fc2.weight9 module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 module.decoder.layers.0.mlp.experts.linear_fc2.weight7 Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.0.mlp.experts.linear_fc2.weight5 module.decoder.layers.0.mlp.experts.linear_fc2.weight3 module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 Params for bucket 73 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight14 - module.decoder.layers.0.mlp.experts.linear_fc1.weight12 module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 module.decoder.layers.0.mlp.experts.linear_fc1.weight11 Params for bucket 74 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight10 module.decoder.layers.0.mlp.experts.linear_fc1.weight9 - module.decoder.layers.0.mlp.experts.linear_fc1.weight8 module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 Params for bucket 75 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight4 - module.decoder.layers.0.mlp.experts.linear_fc1.weight5 Params for bucket 76 (37748736 elements, 37748736 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight2 - module.decoder.layers.0.mlp.experts.linear_fc1.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:17:25 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:26:33 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1214,7 +1214,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1227,30 +1227,30 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 16:17:25 +[after dataloaders are built] datetime: 2026-08-26 16:26:33 done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (629.22, 644.90) - train/valid/test-data-iterators-setup ..........: (52.88, 63.78) training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (610.49, 624.22) + train/valid/test-data-iterators-setup ..........: (52.21, 70.43) Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 16:17:25 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3319, 91, 9, 325, 141, 771, 10, 0, 113, 131, 1583, 574, 1908, 98, 291, 400, 141, 1705, 44, 11, 526, 73, 84, 1479, 991, 46, 37, 1484, 600, 1555, 38, 361, 613, 2423, 39, 56, 153, 263, 95, 37, 1024, 35, 915, 2986, 44, 2, 246, 1124, 281, 240, 1, 18, 3, 576, 40, 30, 25, 37, 47, 586, 1487, 17, 275, 149, 2130, 74, 13, 2, 1225, 15, 54, 1289, 715, 56, 2134, 152, 477, 2, 779, 1073, 45, 21, 2, 34, 38, 3, 657, 745, 18, 56, 1257, 6, 64, 194, 280, 124, 954, 126, 99, 1887, 72, 13, 71, 122, 338, 133, 579, 5356, 46, 166, 60, 69, 339, 75, 5, 33, 11, 515, 109, 306, 1787, 70, 1401, 352, 186, 418, 3, 3295] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2665, 133, 14, 471, 269, 921, 14, 52, 65, 336, 1992, 615, 1704, 128, 116, 437, 47, 1632, 291, 3, 442, 11, 160, 2050, 728, 41, 144, 1375, 716, 2000, 107, 271, 480, 1912, 60, 37, 102, 501, 132, 41, 1399, 67, 766, 2873, 85, 57, 156, 1053, 444, 246, 2, 44, 2, 326, 97, 16, 203, 15, 27, 598, 1585, 41, 434, 342, 2007, 8, 1, 12, 1186, 8, 83, 1486, 1110, 213, 1137, 96, 316, 19, 485, 778, 46, 2, 2, 57, 88, 24, 486, 839, 32, 43, 674, 2, 17, 340, 254, 78, 1434, 115, 34, 1751, 122, 0, 87, 102, 323, 122, 746, 4321, 4, 124, 80, 83, 903, 82, 67, 46, 8, 689, 261, 399, 909, 69, 1362, 557, 120, 550, 17, 4027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3717, 45, 34, 300, 95, 653, 52, 51, 104, 192, 1954, 454, 2109, 99, 125, 334, 421, 1669, 139, 48, 884, 11, 59, 1539, 1204, 46, 11, 1777, 700, 2391, 70, 239, 782, 1437, 128, 12, 147, 177, 94, 45, 626, 24, 911, 3248, 27, 0, 201, 749, 265, 152, 6, 29, 1, 164, 75, 47, 55, 33, 48, 504, 1107, 40, 260, 135, 2559, 19, 1, 15, 1338, 7, 43, 1005, 1350, 92, 1655, 231, 451, 10, 624, 918, 107, 20, 8, 54, 19, 11, 1470, 689, 17, 40, 1188, 53, 37, 117, 145, 28, 1156, 254, 48, 1829, 139, 4, 28, 65, 223, 134, 710, 4519, 63, 123, 56, 17, 829, 28, 23, 16, 27, 895, 161, 162, 1302, 83, 1006, 435, 211, 188, 22, 3404] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3582, 65, 17, 348, 173, 720, 6, 38, 93, 234, 1681, 554, 1835, 93, 233, 407, 142, 1767, 156, 15, 766, 17, 101, 1975, 1326, 75, 34, 1459, 303, 1912, 104, 216, 594, 1910, 110, 39, 109, 360, 58, 49, 1286, 52, 559, 2506, 31, 11, 243, 627, 184, 103, 0, 37, 1, 344, 78, 32, 81, 26, 23, 445, 1958, 51, 365, 198, 2233, 9, 3, 6, 1086, 21, 40, 1186, 763, 243, 1978, 168, 302, 7, 660, 925, 16, 10, 6, 46, 23, 2, 1051, 620, 14, 48, 1267, 4, 24, 145, 535, 142, 956, 143, 119, 2025, 93, 1, 53, 92, 209, 217, 753, 4732, 22, 151, 67, 75, 978, 43, 46, 88, 0, 405, 158, 486, 1208, 85, 1268, 431, 301, 219, 3, 3609] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +[before the start of training step] datetime: 2026-08-26 16:26:33 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3275, 77, 6, 177, 143, 360, 15, 19, 62, 166, 1593, 1553, 2007, 122, 228, 400, 246, 1894, 279, 10, 678, 29, 99, 1122, 518, 137, 56, 2124, 1089, 1664, 33, 110, 943, 2147, 54, 28, 94, 314, 68, 11, 745, 90, 518, 2679, 47, 21, 192, 487, 109, 347, 1, 61, 8, 693, 29, 8, 32, 24, 50, 523, 2127, 50, 559, 37, 1993, 45, 6, 5, 1362, 35, 12, 1426, 967, 121, 2164, 75, 535, 6, 627, 616, 46, 7, 0, 26, 191, 18, 568, 833, 16, 20, 925, 3, 12, 191, 265, 109, 1284, 218, 98, 3389, 66, 0, 41, 126, 759, 87, 349, 3809, 72, 111, 39, 68, 258, 23, 13, 40, 3, 359, 150, 626, 1393, 76, 1267, 386, 357, 121, 3, 3333] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2774, 81, 20, 440, 113, 825, 1, 7, 136, 273, 1522, 352, 1570, 94, 177, 387, 235, 1345, 284, 33, 919, 36, 29, 1509, 489, 70, 25, 1199, 452, 1511, 12, 376, 878, 2834, 125, 78, 175, 272, 175, 99, 1378, 40, 576, 2598, 65, 9, 257, 644, 184, 315, 3, 56, 9, 660, 54, 34, 188, 28, 21, 756, 1127, 28, 279, 236, 2031, 54, 1, 19, 1611, 7, 76, 1269, 1083, 94, 1524, 249, 482, 5, 368, 923, 23, 18, 2, 51, 52, 11, 519, 705, 39, 18, 862, 9, 112, 282, 241, 92, 2628, 39, 39, 1753, 126, 7, 88, 169, 431, 338, 490, 4414, 32, 231, 104, 65, 1090, 24, 28, 48, 8, 783, 114, 301, 1128, 63, 1940, 366, 244, 369, 12, 3755] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2556, 42, 35, 461, 182, 450, 13, 35, 64, 376, 2194, 396, 1708, 186, 108, 858, 303, 1598, 273, 11, 372, 22, 43, 1516, 794, 17, 33, 1125, 451, 2609, 106, 431, 707, 1920, 113, 21, 105, 709, 184, 163, 904, 31, 727, 2372, 48, 26, 191, 651, 282, 290, 7, 29, 5, 238, 118, 61, 192, 35, 34, 463, 1327, 32, 595, 411, 2335, 15, 6, 11, 1851, 5, 145, 1681, 1183, 218, 1290, 394, 290, 12, 543, 713, 27, 26, 3, 50, 82, 4, 722, 1085, 68, 60, 784, 10, 50, 298, 226, 124, 839, 179, 99, 1323, 158, 32, 190, 60, 360, 307, 584, 4653, 22, 198, 126, 25, 820, 36, 33, 57, 5, 1137, 216, 182, 809, 74, 1406, 416, 267, 185, 8, 4060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3582, 65, 17, 348, 173, 720, 6, 38, 93, 234, 1681, 554, 1835, 93, 233, 407, 142, 1767, 156, 15, 766, 17, 101, 1975, 1326, 75, 34, 1459, 303, 1912, 104, 216, 594, 1910, 110, 39, 109, 360, 58, 49, 1286, 52, 559, 2506, 31, 11, 243, 627, 184, 103, 0, 37, 1, 344, 78, 32, 81, 26, 23, 445, 1958, 51, 365, 198, 2233, 9, 3, 6, 1086, 21, 40, 1186, 763, 243, 1978, 168, 302, 7, 660, 925, 16, 10, 6, 46, 23, 2, 1051, 620, 14, 48, 1267, 4, 24, 145, 535, 142, 956, 143, 119, 2025, 93, 1, 53, 92, 209, 217, 753, 4732, 22, 151, 67, 75, 978, 43, 46, 88, 0, 405, 158, 486, 1208, 85, 1268, 431, 301, 219, 3, 3609] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3717, 45, 34, 300, 95, 653, 52, 51, 104, 192, 1954, 454, 2109, 99, 125, 334, 421, 1669, 139, 48, 884, 11, 59, 1539, 1204, 46, 11, 1777, 700, 2391, 70, 239, 782, 1437, 128, 12, 147, 177, 94, 45, 626, 24, 911, 3248, 27, 0, 201, 749, 265, 152, 6, 29, 1, 164, 75, 47, 55, 33, 48, 504, 1107, 40, 260, 135, 2559, 19, 1, 15, 1338, 7, 43, 1005, 1350, 92, 1655, 231, 451, 10, 624, 918, 107, 20, 8, 54, 19, 11, 1470, 689, 17, 40, 1188, 53, 37, 117, 145, 28, 1156, 254, 48, 1829, 139, 4, 28, 65, 223, 134, 710, 4519, 63, 123, 56, 17, 829, 28, 23, 16, 27, 895, 161, 162, 1302, 83, 1006, 435, 211, 188, 22, 3404] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3319, 91, 9, 325, 141, 771, 10, 0, 113, 131, 1583, 574, 1908, 98, 291, 400, 141, 1705, 44, 11, 526, 73, 84, 1479, 991, 46, 37, 1484, 600, 1555, 38, 361, 613, 2423, 39, 56, 153, 263, 95, 37, 1024, 35, 915, 2986, 44, 2, 246, 1124, 281, 240, 1, 18, 3, 576, 40, 30, 25, 37, 47, 586, 1487, 17, 275, 149, 2130, 74, 13, 2, 1225, 15, 54, 1289, 715, 56, 2134, 152, 477, 2, 779, 1073, 45, 21, 2, 34, 38, 3, 657, 745, 18, 56, 1257, 6, 64, 194, 280, 124, 954, 126, 99, 1887, 72, 13, 71, 122, 338, 133, 579, 5356, 46, 166, 60, 69, 339, 75, 5, 33, 11, 515, 109, 306, 1787, 70, 1401, 352, 186, 418, 3, 3295] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2665, 133, 14, 471, 269, 921, 14, 52, 65, 336, 1992, 615, 1704, 128, 116, 437, 47, 1632, 291, 3, 442, 11, 160, 2050, 728, 41, 144, 1375, 716, 2000, 107, 271, 480, 1912, 60, 37, 102, 501, 132, 41, 1399, 67, 766, 2873, 85, 57, 156, 1053, 444, 246, 2, 44, 2, 326, 97, 16, 203, 15, 27, 598, 1585, 41, 434, 342, 2007, 8, 1, 12, 1186, 8, 83, 1486, 1110, 213, 1137, 96, 316, 19, 485, 778, 46, 2, 2, 57, 88, 24, 486, 839, 32, 43, 674, 2, 17, 340, 254, 78, 1434, 115, 34, 1751, 122, 0, 87, 102, 323, 122, 746, 4321, 4, 124, 80, 83, 903, 82, 67, 46, 8, 689, 261, 399, 909, 69, 1362, 557, 120, 550, 17, 4027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3419, 64, 28, 251, 137, 848, 33, 12, 76, 106, 1605, 391, 1987, 36, 318, 356, 166, 2384, 247, 50, 858, 11, 90, 1556, 781, 75, 13, 1523, 630, 1004, 48, 475, 733, 2475, 39, 61, 153, 188, 102, 42, 857, 33, 447, 3171, 69, 18, 216, 775, 179, 153, 0, 131, 1, 297, 102, 27, 105, 40, 56, 854, 1205, 22, 184, 346, 1707, 36, 2, 36, 2173, 1, 57, 967, 1056, 103, 1069, 219, 529, 22, 800, 1137, 39, 71, 10, 38, 84, 13, 1114, 765, 4, 45, 998, 5, 30, 137, 359, 119, 1929, 240, 33, 2444, 41, 1, 81, 127, 311, 147, 585, 4953, 244, 217, 66, 64, 797, 128, 64, 44, 3, 621, 94, 235, 1047, 107, 1257, 386, 289, 307, 13, 2257] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2774, 81, 20, 440, 113, 825, 1, 7, 136, 273, 1522, 352, 1570, 94, 177, 387, 235, 1345, 284, 33, 919, 36, 29, 1509, 489, 70, 25, 1199, 452, 1511, 12, 376, 878, 2834, 125, 78, 175, 272, 175, 99, 1378, 40, 576, 2598, 65, 9, 257, 644, 184, 315, 3, 56, 9, 660, 54, 34, 188, 28, 21, 756, 1127, 28, 279, 236, 2031, 54, 1, 19, 1611, 7, 76, 1269, 1083, 94, 1524, 249, 482, 5, 368, 923, 23, 18, 2, 51, 52, 11, 519, 705, 39, 18, 862, 9, 112, 282, 241, 92, 2628, 39, 39, 1753, 126, 7, 88, 169, 431, 338, 490, 4414, 32, 231, 104, 65, 1090, 24, 28, 48, 8, 783, 114, 301, 1128, 63, 1940, 366, 244, 369, 12, 3755] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1611, 77, 6, 177, 143, 360, 15, 19, 62, 166, 1593, 1553, 2007, 122, 228, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1894, 279, 10, 678, 29, 99, 1122, 518, 137, 56, 2124, 1089, 135, 33, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 2147, 54, 28, 94, 314, 68, 11, 745, 90, 518, 1690, 47, 21, 192, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 347, 1, 61, 8, 693, 29, 8, 32, 24, 50, 523, 2127, 50, 559, 37, 1664, 989, 912, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 45, 6, 5, 1362, 35, 12, 1426, 967, 121, 2164, 75, 535, 6, 627, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 7, 0, 26, 191, 18, 568, 833, 16, 20, 925, 3, 12, 191, 265, 109, 1573, 1635, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 218, 98, 3389, 66, 0, 41, 126, 759, 87, 349, 2236, 72, 111, 39, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 23, 13, 40, 3, 359, 150, 626, 1393, 76, 1267, 386, 357, 121, 3, 2421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8539, 8559, 7449, 8820, 8360, 7370, 8943, 7496] avg=8192.0000 max=8943 max_rank=6 min=7370 min_rank=5 imbalance_ratio=1.091675 max_load_reduction=56593 max_load_reduction_pct=0.863541 moved_to_echo=32169 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1823, 45, 34, 300, 95, 653, 52, 51, 104, 192, 1954, 454, 2109, 99, 125, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 1669, 139, 48, 884, 11, 59, 1539, 1204, 46, 11, 1777, 700, 189, 70, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 1437, 128, 12, 147, 177, 94, 45, 626, 24, 911, 2044, 27, 0, 201, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 152, 6, 29, 1, 164, 75, 47, 55, 33, 48, 504, 1107, 40, 260, 135, 1894, 1204, 935, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 19, 1, 15, 1338, 7, 43, 1005, 1350, 92, 1655, 231, 451, 10, 624, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 20, 8, 54, 19, 11, 1470, 689, 17, 40, 1188, 53, 37, 117, 145, 28, 1871, 2103, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 254, 48, 1829, 139, 4, 28, 65, 223, 134, 710, 2648, 63, 123, 56, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 28, 23, 16, 27, 895, 161, 162, 1302, 83, 1006, 435, 211, 188, 22, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8424, 9006, 7404, 7814, 8215, 9319, 7497, 7857] avg=8192.0000 max=9319 max_rank=5 min=7404 min_rank=2 imbalance_ratio=1.137573 max_load_reduction=56217 max_load_reduction_pct=0.857803 moved_to_echo=32888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1311, 133, 14, 471, 269, 921, 14, 52, 65, 336, 1992, 615, 1704, 128, 116, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 1632, 291, 3, 442, 11, 160, 2050, 728, 41, 144, 1375, 716, 163, 107, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 1912, 60, 37, 102, 501, 132, 41, 1399, 67, 766, 1813, 85, 57, 156, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 246, 2, 44, 2, 326, 97, 16, 203, 15, 27, 598, 1585, 41, 434, 342, 1354, 1060, 1102, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 8, 1, 12, 1186, 8, 83, 1486, 1110, 213, 1137, 96, 316, 19, 485, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 2, 2, 57, 88, 24, 486, 839, 32, 43, 674, 2, 17, 340, 254, 78, 1785, 1646, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1434, 115, 34, 1751, 122, 0, 87, 102, 323, 122, 746, 2536, 4, 124, 80, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 82, 67, 46, 8, 689, 261, 399, 909, 69, 1362, 557, 120, 550, 17, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8578, 8181, 8661, 8655, 7299, 7535, 7663, 8964] avg=8192.0000 max=8964 max_rank=7 min=7299 min_rank=4 imbalance_ratio=1.094238 max_load_reduction=56572 max_load_reduction_pct=0.863220 moved_to_echo=31461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1365, 81, 20, 440, 113, 825, 1, 7, 136, 273, 1522, 352, 1570, 94, 177, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 1345, 284, 33, 919, 36, 29, 1509, 489, 70, 25, 1199, 452, 123, 12, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 2834, 125, 78, 175, 272, 175, 99, 1378, 40, 576, 1639, 65, 9, 257, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 315, 3, 56, 9, 660, 54, 34, 188, 28, 21, 756, 1127, 28, 279, 236, 1409, 959, 1028, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 54, 1, 19, 1611, 7, 76, 1269, 1083, 94, 1524, 249, 482, 5, 368, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 18, 2, 51, 52, 11, 519, 705, 39, 18, 862, 9, 112, 282, 241, 92, 1823, 1666, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 39, 39, 1753, 126, 7, 88, 169, 431, 338, 490, 2591, 32, 231, 104, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 24, 28, 48, 8, 783, 114, 301, 1128, 63, 1940, 366, 244, 369, 12, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7363, 7136, 9244, 7916, 8130, 7371, 9131, 9245] avg=8192.0000 max=9245 max_rank=7 min=7136 min_rank=1 imbalance_ratio=1.128540 max_load_reduction=56291 max_load_reduction_pct=0.858932 moved_to_echo=33877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1762, 65, 17, 348, 173, 720, 6, 38, 93, 234, 1681, 554, 1835, 93, 233, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 1767, 156, 15, 766, 17, 101, 1975, 1326, 75, 34, 1459, 303, 155, 104, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 1910, 110, 39, 109, 360, 58, 49, 1286, 52, 559, 1581, 31, 11, 243, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 103, 0, 37, 1, 344, 78, 32, 81, 26, 23, 445, 1958, 51, 365, 198, 1820, 925, 988, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 9, 3, 6, 1086, 21, 40, 1186, 763, 243, 1978, 168, 302, 7, 660, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 10, 6, 46, 23, 2, 1051, 620, 14, 48, 1267, 4, 24, 145, 535, 142, 1955, 1832, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 143, 119, 2025, 93, 1, 53, 92, 209, 217, 753, 2777, 22, 151, 67, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 43, 46, 88, 0, 405, 158, 486, 1208, 85, 1268, 431, 301, 219, 3, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8611, 7619, 8345, 7798, 8811, 7753, 8340] avg=8192.0000 max=8811 max_rank=5 min=7619 min_rank=2 imbalance_ratio=1.075562 max_load_reduction=56725 max_load_reduction_pct=0.865555 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1823, 45, 34, 300, 95, 653, 52, 51, 104, 192, 1954, 454, 2109, 99, 125, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 1669, 139, 48, 884, 11, 59, 1539, 1204, 46, 11, 1777, 700, 189, 70, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 1437, 128, 12, 147, 177, 94, 45, 626, 24, 911, 2044, 27, 0, 201, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 152, 6, 29, 1, 164, 75, 47, 55, 33, 48, 504, 1107, 40, 260, 135, 1894, 1204, 935, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 19, 1, 15, 1338, 7, 43, 1005, 1350, 92, 1655, 231, 451, 10, 624, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 20, 8, 54, 19, 11, 1470, 689, 17, 40, 1188, 53, 37, 117, 145, 28, 1871, 2103, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 254, 48, 1829, 139, 4, 28, 65, 223, 134, 710, 2648, 63, 123, 56, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 28, 23, 16, 27, 895, 161, 162, 1302, 83, 1006, 435, 211, 188, 22, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8424, 9006, 7404, 7814, 8215, 9319, 7497, 7857] avg=8192.0000 max=9319 max_rank=5 min=7404 min_rank=2 imbalance_ratio=1.137573 max_load_reduction=56217 max_load_reduction_pct=0.857803 moved_to_echo=32888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1682, 64, 28, 251, 137, 848, 33, 12, 76, 106, 1605, 391, 1987, 36, 318, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 2384, 247, 50, 858, 11, 90, 1556, 781, 75, 13, 1523, 630, 82, 48, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 2475, 39, 61, 153, 188, 102, 42, 857, 33, 447, 2001, 69, 18, 216, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 153, 0, 131, 1, 297, 102, 27, 105, 40, 56, 854, 1205, 22, 184, 346, 1737, 1170, 618, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 36, 2, 36, 2173, 1, 57, 967, 1056, 103, 1069, 219, 529, 22, 800, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 71, 10, 38, 84, 13, 1114, 765, 4, 45, 998, 5, 30, 137, 359, 119, 2046, 1400, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 240, 33, 2444, 41, 1, 81, 127, 311, 147, 585, 2907, 244, 217, 66, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 128, 64, 44, 3, 621, 94, 235, 1047, 107, 1257, 386, 289, 307, 13, 1639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7930, 8989, 8209, 7587, 8514, 7839, 9437, 7031] avg=8192.0000 max=9437 max_rank=6 min=7031 min_rank=7 imbalance_ratio=1.151978 max_load_reduction=56099 max_load_reduction_pct=0.856003 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1611, 77, 6, 177, 143, 360, 15, 19, 62, 166, 1593, 1553, 2007, 122, 228, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1894, 279, 10, 678, 29, 99, 1122, 518, 137, 56, 2124, 1089, 135, 33, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 2147, 54, 28, 94, 314, 68, 11, 745, 90, 518, 1690, 47, 21, 192, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 347, 1, 61, 8, 693, 29, 8, 32, 24, 50, 523, 2127, 50, 559, 37, 1664, 989, 912, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 45, 6, 5, 1362, 35, 12, 1426, 967, 121, 2164, 75, 535, 6, 627, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 7, 0, 26, 191, 18, 568, 833, 16, 20, 925, 3, 12, 191, 265, 109, 1573, 1635, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 218, 98, 3389, 66, 0, 41, 126, 759, 87, 349, 2236, 72, 111, 39, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 23, 13, 40, 3, 359, 150, 626, 1393, 76, 1267, 386, 357, 121, 3, 2421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8539, 8559, 7449, 8820, 8360, 7370, 8943, 7496] avg=8192.0000 max=8943 max_rank=6 min=7370 min_rank=5 imbalance_ratio=1.091675 max_load_reduction=56593 max_load_reduction_pct=0.863541 moved_to_echo=32169 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1311, 133, 14, 471, 269, 921, 14, 52, 65, 336, 1992, 615, 1704, 128, 116, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 1632, 291, 3, 442, 11, 160, 2050, 728, 41, 144, 1375, 716, 163, 107, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 1912, 60, 37, 102, 501, 132, 41, 1399, 67, 766, 1813, 85, 57, 156, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 246, 2, 44, 2, 326, 97, 16, 203, 15, 27, 598, 1585, 41, 434, 342, 1354, 1060, 1102, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 8, 1, 12, 1186, 8, 83, 1486, 1110, 213, 1137, 96, 316, 19, 485, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 2, 2, 57, 88, 24, 486, 839, 32, 43, 674, 2, 17, 340, 254, 78, 1785, 1646, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1434, 115, 34, 1751, 122, 0, 87, 102, 323, 122, 746, 2536, 4, 124, 80, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 82, 67, 46, 8, 689, 261, 399, 909, 69, 1362, 557, 120, 550, 17, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8578, 8181, 8661, 8655, 7299, 7535, 7663, 8964] avg=8192.0000 max=8964 max_rank=7 min=7299 min_rank=4 imbalance_ratio=1.094238 max_load_reduction=56572 max_load_reduction_pct=0.863220 moved_to_echo=31461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1633, 91, 9, 325, 141, 771, 10, 0, 113, 131, 1583, 574, 1908, 98, 291, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 1705, 44, 11, 526, 73, 84, 1479, 991, 46, 37, 1484, 600, 127, 38, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 2423, 39, 56, 153, 263, 95, 37, 1024, 35, 915, 1884, 44, 2, 246, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 240, 1, 18, 3, 576, 40, 30, 25, 37, 47, 586, 1487, 17, 275, 149, 1686, 1102, 902, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 74, 13, 2, 1225, 15, 54, 1289, 715, 56, 2134, 152, 477, 2, 779, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 21, 2, 34, 38, 3, 657, 745, 18, 56, 1257, 6, 64, 194, 280, 124, 2212, 1747, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 126, 99, 1887, 72, 13, 71, 122, 338, 133, 579, 3144, 46, 166, 60, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 75, 5, 33, 11, 515, 109, 306, 1787, 70, 1401, 352, 186, 418, 3, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 7747, 8953, 8059, 8443, 8374, 7879, 8003] avg=8192.0000 max=8953 max_rank=2 min=7747 min_rank=1 imbalance_ratio=1.092896 max_load_reduction=56583 max_load_reduction_pct=0.863388 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1257, 42, 35, 461, 182, 450, 13, 35, 64, 376, 2194, 396, 1708, 186, 108, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 1598, 273, 11, 372, 22, 43, 1516, 794, 17, 33, 1125, 451, 212, 106, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 1920, 113, 21, 105, 709, 184, 163, 904, 31, 727, 1497, 48, 26, 191, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 290, 7, 29, 5, 238, 118, 61, 192, 35, 34, 463, 1327, 32, 595, 411, 1299, 875, 1111, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 15, 6, 11, 1851, 5, 145, 1681, 1183, 218, 1290, 394, 290, 12, 543, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 26, 3, 50, 82, 4, 722, 1085, 68, 60, 784, 10, 50, 298, 226, 124, 1922, 1915, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 179, 99, 1323, 158, 32, 190, 60, 360, 307, 584, 2731, 22, 198, 126, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 36, 33, 57, 5, 1137, 216, 182, 809, 74, 1406, 416, 267, 185, 8, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 7307, 7997, 8340, 8777, 8917, 7233, 8600] avg=8192.0000 max=8917 max_rank=5 min=7233 min_rank=6 imbalance_ratio=1.088501 max_load_reduction=56619 max_load_reduction_pct=0.863937 moved_to_echo=33527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1365, 81, 20, 440, 113, 825, 1, 7, 136, 273, 1522, 352, 1570, 94, 177, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 1345, 284, 33, 919, 36, 29, 1509, 489, 70, 25, 1199, 452, 123, 12, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 2834, 125, 78, 175, 272, 175, 99, 1378, 40, 576, 1639, 65, 9, 257, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 315, 3, 56, 9, 660, 54, 34, 188, 28, 21, 756, 1127, 28, 279, 236, 1409, 959, 1028, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 54, 1, 19, 1611, 7, 76, 1269, 1083, 94, 1524, 249, 482, 5, 368, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 18, 2, 51, 52, 11, 519, 705, 39, 18, 862, 9, 112, 282, 241, 92, 1823, 1666, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 39, 39, 1753, 126, 7, 88, 169, 431, 338, 490, 2591, 32, 231, 104, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 24, 28, 48, 8, 783, 114, 301, 1128, 63, 1940, 366, 244, 369, 12, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7363, 7136, 9244, 7916, 8130, 7371, 9131, 9245] avg=8192.0000 max=9245 max_rank=7 min=7136 min_rank=1 imbalance_ratio=1.128540 max_load_reduction=56291 max_load_reduction_pct=0.858932 moved_to_echo=33877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1283,30 +1283,30 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[384, 256, 128, 384, 640, 128, 6656, 6400, 256, 384, 8064, 128, 384, 1792, 2432, 896, 15232, 13952, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[11264, 1408, 640, 16512, 896, 128, 640, 896, 3072, 1536, 4864, 21632, 512, 1408, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[6016, 512, 384, 384, 128, 5504, 1280, 2816, 9600, 640, 11008, 3456, 2048, 2432, 128, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3072, 384, 128, 128, 11904, 128, 512, 10368, 8320, 1152, 13056, 1664, 3456, 128, 4992, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[12544, 640, 256, 2816, 1280, 5632, 256, 256, 768, 1920, 14208, 4992, 14848, 896, 1664, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1792, 14080, 1792, 256, 5504, 256, 768, 12800, 6912, 512, 384, 12160, 4992, 1280, 640, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[384, 256, 128, 384, 640, 128, 6656, 6400, 256, 384, 8064, 128, 384, 1792, 2432, 896, 15232, 13952, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[5760, 17152, 768, 384, 1152, 2816, 1024, 512, 8320, 384, 5504, 14208, 512, 256, 1792, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1792, 14080, 1792, 256, 5504, 256, 768, 12800, 6912, 512, 384, 12160, 4992, 1280, 640, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[2048, 1920, 128, 512, 128, 3328, 640, 256, 896, 256, 384, 4736, 12032, 384, 3072, 1920, 12928, 8320, 7680, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[6016, 512, 384, 384, 128, 5504, 1280, 2816, 9600, 640, 11008, 3456, 2048, 2432, 128, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[11264, 1408, 640, 16512, 896, 128, 640, 896, 3072, 1536, 4864, 21632, 512, 1408, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[12544, 640, 256, 2816, 1280, 5632, 256, 256, 768, 1920, 14208, 4992, 14848, 896, 1664, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1323,30 +1323,30 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1331, 242, 0, 6709, 17, 0, 28, 2, 0, 480, 2779, 0, 0, 7, 0, 1, 0, 1, 0, 2008, 0, 0, 12, 0, 0, 0, 22, 119, 0, 0, 24, 101, 0, 0, 1, 0, 18, 243, 0, 0, 0, 2773, 4, 3, 0, 167, 0, 0, 0, 145, 0, 0, 681, 1, 202, 7580, 0, 7187, 550, 42, 0, 1664, 0, 2132, 224, 1, 0, 9, 1446, 8168, 10, 1, 0, 0, 0, 190, 0, 13, 20, 0, 0, 28, 0, 0, 0, 0, 0, 0, 5109, 0, 904, 0, 9, 0, 2149, 32, 0, 0, 9, 35, 0, 93, 0, 174, 28, 0, 0, 153, 0, 216, 165, 0, 0, 12, 0, 1, 667, 1, 138, 79, 0, 0, 196, 0, 7979, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[324, 775, 0, 3049, 0, 0, 2, 7, 0, 168, 8068, 0, 0, 48, 0, 1, 0, 25, 2, 42, 3, 0, 24, 0, 0, 0, 1, 443, 5, 0, 1, 7, 0, 0, 0, 0, 0, 99, 1, 1, 0, 31, 22, 23, 150, 0, 4, 1, 1, 11, 0, 0, 1265, 0, 3, 6109, 0, 242, 5403, 0, 0, 92, 0, 571, 19, 0, 0, 0, 565, 7899, 46, 0, 0, 0, 0, 68, 0, 0, 3, 0, 0, 43, 154, 573, 37, 0, 53, 0, 5181, 0, 3309, 0, 42, 0, 6072, 15, 0, 0, 0, 266, 0, 5079, 0, 0, 3, 26, 27, 0, 0, 612, 0, 0, 0, 26, 10, 0, 2, 0, 1458, 27, 0, 0, 3, 0, 6894, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[688, 184, 0, 2055, 0, 0, 5, 107, 0, 0, 3547, 0, 0, 269, 0, 61, 0, 109, 0, 1, 16, 0, 22, 0, 137, 4, 443, 700, 76, 1, 165, 119, 30, 0, 0, 1, 6, 0, 1, 0, 134, 0, 302, 1, 2, 0, 0, 4, 7, 969, 0, 1, 2935, 0, 6, 3686, 0, 166, 3327, 6612, 0, 36, 23, 5588, 0, 4, 0, 0, 1908, 8002, 95, 36, 3, 10, 0, 10, 3, 0, 3, 0, 0, 51, 126, 225, 0, 23, 29, 0, 7844, 0, 5135, 0, 9, 0, 7, 5, 27, 18, 2, 2, 4, 3366, 1, 0, 40, 0, 32, 422, 90, 0, 14, 0, 11, 4, 0, 0, 10, 2, 1437, 1, 2, 1, 0, 0, 3976, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1429, 392, 0, 7003, 12, 0, 3, 1, 34, 86, 6511, 0, 1, 63, 0, 0, 0, 0, 216, 355, 0, 1, 155, 0, 87, 1, 148, 2942, 6, 0, 1, 0, 12, 1, 0, 0, 1, 1555, 0, 6, 0, 42, 5, 2, 4, 15, 0, 0, 0, 283, 4, 9, 1365, 7, 27, 6376, 2, 271, 102, 1, 0, 15, 0, 597, 13, 0, 3, 0, 1450, 8182, 0, 0, 2, 0, 0, 743, 0, 0, 1, 0, 0, 1, 10, 2, 0, 0, 0, 11, 5948, 0, 915, 10, 17, 0, 2610, 5, 0, 0, 0, 0, 5, 2277, 12, 1, 3, 8, 0, 15, 1, 10, 10, 7, 0, 80, 0, 0, 0, 1, 7628, 0, 0, 1, 7, 0, 5390, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1144, 22, 2, 944, 0, 0, 7, 29, 0, 12, 7774, 0, 20, 5, 0, 99, 0, 3, 151, 911, 0, 0, 0, 0, 38, 0, 23, 57, 3, 0, 24, 248, 0, 0, 3, 15, 98, 3, 0, 0, 4, 262, 42, 25, 5, 184, 0, 0, 14, 23, 0, 0, 4475, 1, 310, 6856, 0, 1970, 362, 155, 0, 2834, 69, 869, 18, 0, 0, 0, 1314, 8136, 0, 0, 0, 52, 0, 66, 0, 17, 1, 0, 0, 143, 5, 0, 0, 0, 29, 0, 7982, 1, 871, 0, 57, 7, 1674, 284, 1, 36, 0, 0, 15, 486, 0, 0, 0, 0, 180, 2, 3, 0, 0, 0, 1, 166, 0, 0, 58, 0, 5496, 158, 6, 0, 1, 0, 8108, 67, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[171, 212, 0, 647, 0, 0, 0, 0, 0, 0, 7648, 0, 2, 0, 0, 5, 5, 193, 97, 524, 0, 0, 1862, 0, 5, 0, 48, 5753, 2, 0, 0, 1, 194, 0, 0, 0, 32, 26, 0, 1, 148, 6, 24, 0, 239, 9, 3, 0, 0, 86, 49, 0, 780, 66, 212, 5369, 3, 287, 438, 15, 0, 516, 0, 1486, 0, 0, 0, 0, 1578, 6956, 9, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 283, 796, 4, 1, 0, 1876, 2, 8169, 0, 1026, 1, 373, 1, 6852, 16, 0, 0, 0, 0, 0, 1949, 0, 0, 98, 18, 180, 19, 1, 360, 64, 2, 0, 0, 1, 2, 0, 0, 40, 0, 0, 11, 6, 0, 7650, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4190, 9, 13, 3708, 1, 0, 0, 2, 0, 2, 8052, 2, 0, 0, 1, 0, 0, 2, 0, 955, 0, 0, 0, 0, 1, 0, 50, 33, 10, 3, 34, 9, 0, 0, 1, 0, 0, 1, 6, 1, 3, 0, 0, 3, 1910, 0, 4, 0, 2, 1, 0, 0, 4502, 3, 156, 45, 0, 42, 1427, 4, 0, 143, 0, 2777, 15, 0, 0, 2, 75, 7645, 9, 1, 0, 0, 0, 0, 22, 0, 0, 57, 0, 13, 144, 5, 17, 183, 1012, 4, 7976, 0, 2294, 9, 41, 2, 2049, 743, 1, 0, 0, 0, 0, 5175, 0, 0, 0, 81, 1, 0, 0, 22, 0, 0, 0, 201, 20, 2, 1, 0, 1727, 0, 0, 0, 15, 25, 7821, 18, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4, 814, 0, 2, 0, 0, 44, 62, 0, 0, 7372, 0, 0, 2024, 1, 1253, 0, 0, 29, 45, 0, 38, 0, 0, 0, 0, 2, 290, 0, 1, 1, 10, 1, 22, 0, 1, 3, 0, 0, 0, 0, 3706, 0, 4, 4, 0, 1, 0, 3, 0, 0, 4, 7678, 0, 0, 3976, 0, 2454, 1356, 1995, 0, 2, 93, 4201, 0, 0, 0, 109, 119, 7847, 0, 1, 71, 53, 0, 107, 0, 3, 201, 0, 4, 2491, 108, 0, 2, 3, 440, 0, 6177, 10, 2119, 2, 4, 0, 12, 365, 0, 0, 0, 0, 18, 1843, 0, 1, 0, 24, 38, 304, 30, 0, 0, 0, 0, 0, 0, 0, 33, 0, 513, 2, 0, 0, 20, 0, 4711, 0, 0, 255] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[324, 775, 0, 3049, 0, 0, 2, 7, 0, 168, 8068, 0, 0, 48, 0, 1, 0, 25, 2, 42, 3, 0, 24, 0, 0, 0, 1, 443, 5, 0, 1, 7, 0, 0, 0, 0, 0, 99, 1, 1, 0, 31, 22, 23, 150, 0, 4, 1, 1, 11, 0, 0, 1265, 0, 3, 6109, 0, 242, 5403, 0, 0, 92, 0, 571, 19, 0, 0, 0, 565, 7899, 46, 0, 0, 0, 0, 68, 0, 0, 3, 0, 0, 43, 154, 573, 37, 0, 53, 0, 5181, 0, 3309, 0, 42, 0, 6072, 15, 0, 0, 0, 266, 0, 5079, 0, 0, 3, 26, 27, 0, 0, 612, 0, 0, 0, 26, 10, 0, 2, 0, 1458, 27, 0, 0, 3, 0, 6894, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[171, 212, 0, 647, 0, 0, 0, 0, 0, 0, 7648, 0, 2, 0, 0, 5, 5, 193, 97, 524, 0, 0, 1862, 0, 5, 0, 48, 5753, 2, 0, 0, 1, 194, 0, 0, 0, 32, 26, 0, 1, 148, 6, 24, 0, 239, 9, 3, 0, 0, 86, 49, 0, 780, 66, 212, 5369, 3, 287, 438, 15, 0, 516, 0, 1486, 0, 0, 0, 0, 1578, 6956, 9, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 283, 796, 4, 1, 0, 1876, 2, 8169, 0, 1026, 1, 373, 1, 6852, 16, 0, 0, 0, 0, 0, 1949, 0, 0, 98, 18, 180, 19, 1, 360, 64, 2, 0, 0, 1, 2, 0, 0, 40, 0, 0, 11, 6, 0, 7650, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1429, 392, 0, 7003, 12, 0, 3, 1, 34, 86, 6511, 0, 1, 63, 0, 0, 0, 0, 216, 355, 0, 1, 155, 0, 87, 1, 148, 2942, 6, 0, 1, 0, 12, 1, 0, 0, 1, 1555, 0, 6, 0, 42, 5, 2, 4, 15, 0, 0, 0, 283, 4, 9, 1365, 7, 27, 6376, 2, 271, 102, 1, 0, 15, 0, 597, 13, 0, 3, 0, 1450, 8182, 0, 0, 2, 0, 0, 743, 0, 0, 1, 0, 0, 1, 10, 2, 0, 0, 0, 11, 5948, 0, 915, 10, 17, 0, 2610, 5, 0, 0, 0, 0, 5, 2277, 12, 1, 3, 8, 0, 15, 1, 10, 10, 7, 0, 80, 0, 0, 0, 1, 7628, 0, 0, 1, 7, 0, 5390, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1331, 242, 0, 6709, 17, 0, 28, 2, 0, 480, 2779, 0, 0, 7, 0, 1, 0, 1, 0, 2008, 0, 0, 12, 0, 0, 0, 22, 119, 0, 0, 24, 101, 0, 0, 1, 0, 18, 243, 0, 0, 0, 2773, 4, 3, 0, 167, 0, 0, 0, 145, 0, 0, 681, 1, 202, 7580, 0, 7187, 550, 42, 0, 1664, 0, 2132, 224, 1, 0, 9, 1446, 8168, 10, 1, 0, 0, 0, 190, 0, 13, 20, 0, 0, 28, 0, 0, 0, 0, 0, 0, 5109, 0, 904, 0, 9, 0, 2149, 32, 0, 0, 9, 35, 0, 93, 0, 174, 28, 0, 0, 153, 0, 216, 165, 0, 0, 12, 0, 1, 667, 1, 138, 79, 0, 0, 196, 0, 7979, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1331, 242, 0, 6709, 17, 0, 28, 2, 0, 480, 1317, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2008, 0, 0, 12, 0, 0, 0, 22, 119, 0, 0, 24, 101, 2344, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 243, 0, 0, 0, 2773, 4, 3, 0, 167, 0, 0, 7580, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 139, 1, 202, 0, 0, 7187, 550, 42, 0, 1664, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1, 0, 9, 1446, 7112, 10, 1, 0, 0, 0, 190, 0, 13, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 1556, 0, 904, 0, 9, 0, 2149, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 35, 0, 93, 0, 174, 28, 0, 0, 153, 0, 216, 165, 0, 542, 1056, 1162, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 667, 1, 138, 79, 0, 0, 196, 0, 6817, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10134, 5735, 11998, 12062, 9026, 4678, 3991, 7912] avg=8192.0000 max=12062 max_rank=3 min=3991 min_rank=6 imbalance_ratio=1.472412 max_load_reduction=53474 max_load_reduction_pct=0.815948 moved_to_echo=25607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4190, 9, 13, 3708, 1, 0, 0, 2, 0, 2, 3805, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 955, 0, 0, 0, 0, 1, 0, 50, 33, 10, 3, 34, 9, 3665, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 6, 1, 3, 0, 0, 3, 1910, 0, 4, 0, 45, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 910, 3, 156, 0, 0, 42, 1427, 4, 0, 143, 0, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 2, 75, 6653, 9, 1, 0, 0, 0, 0, 22, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 144, 5, 17, 183, 1012, 4, 2417, 0, 2294, 9, 41, 2, 2049, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5175, 0, 0, 0, 81, 1, 0, 0, 22, 0, 0, 3592, 992, 1141, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 20, 2, 1, 0, 1727, 0, 0, 0, 15, 25, 6680, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11733, 7965, 3868, 5465, 6834, 8933, 12049, 8689] avg=8192.0000 max=12049 max_rank=6 min=3868 min_rank=2 imbalance_ratio=1.470825 max_load_reduction=53487 max_load_reduction_pct=0.816147 moved_to_echo=36505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[324, 775, 0, 3049, 0, 0, 2, 7, 0, 168, 3820, 0, 0, 48, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2, 42, 3, 0, 24, 0, 0, 0, 1, 443, 5, 0, 1, 7, 2377, 3207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 1, 1, 0, 31, 22, 23, 150, 0, 4, 1, 6109, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 257, 0, 3, 0, 0, 242, 5403, 0, 0, 92, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 565, 6878, 46, 0, 0, 0, 0, 68, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 154, 573, 37, 0, 53, 0, 1578, 0, 3309, 0, 42, 0, 6072, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 5079, 0, 0, 3, 26, 27, 0, 0, 612, 0, 0, 1008, 1021, 1004, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 10, 0, 2, 0, 1458, 27, 0, 0, 3, 0, 5890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6137, 7667, 6580, 7579, 11876, 10087, 7416] avg=8192.0000 max=11876 max_rank=5 min=6137 min_rank=1 imbalance_ratio=1.449707 max_load_reduction=53660 max_load_reduction_pct=0.818787 moved_to_echo=36958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1144, 22, 2, 944, 0, 0, 7, 29, 0, 12, 3681, 0, 20, 5, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 151, 911, 0, 0, 0, 0, 38, 0, 23, 57, 3, 0, 24, 248, 3663, 3090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 15, 98, 3, 0, 0, 4, 262, 42, 25, 5, 184, 0, 0, 6856, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 23, 0, 0, 909, 1, 310, 0, 0, 1970, 362, 155, 0, 2834, 69, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1314, 7084, 0, 0, 0, 52, 0, 66, 0, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 5, 0, 0, 0, 29, 0, 2429, 1, 871, 0, 57, 7, 1674, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 0, 0, 15, 486, 0, 0, 0, 0, 180, 2, 3, 0, 0, 0, 3566, 1052, 1181, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 166, 0, 0, 58, 0, 5496, 158, 6, 0, 1, 0, 6927, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 8211, 9387, 7516, 8552, 5500, 7525, 12880] avg=8192.0000 max=12880 max_rank=7 min=5500 min_rank=5 imbalance_ratio=1.572266 max_load_reduction=52656 max_load_reduction_pct=0.803467 moved_to_echo=34457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[324, 775, 0, 3049, 0, 0, 2, 7, 0, 168, 3820, 0, 0, 48, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2, 42, 3, 0, 24, 0, 0, 0, 1, 443, 5, 0, 1, 7, 2377, 3207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 1, 1, 0, 31, 22, 23, 150, 0, 4, 1, 6109, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 257, 0, 3, 0, 0, 242, 5403, 0, 0, 92, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 565, 6878, 46, 0, 0, 0, 0, 68, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 154, 573, 37, 0, 53, 0, 1578, 0, 3309, 0, 42, 0, 6072, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 5079, 0, 0, 3, 26, 27, 0, 0, 612, 0, 0, 1008, 1021, 1004, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 10, 0, 2, 0, 1458, 27, 0, 0, 3, 0, 5890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6137, 7667, 6580, 7579, 11876, 10087, 7416] avg=8192.0000 max=11876 max_rank=5 min=6137 min_rank=1 imbalance_ratio=1.449707 max_load_reduction=53660 max_load_reduction_pct=0.818787 moved_to_echo=36958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[171, 212, 0, 647, 0, 0, 0, 0, 0, 0, 3621, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 193, 97, 524, 0, 0, 1862, 0, 5, 0, 48, 5753, 2, 0, 0, 1, 3748, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 32, 26, 0, 1, 148, 6, 24, 0, 239, 9, 3, 0, 5369, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 49, 0, 159, 66, 212, 0, 3, 287, 438, 15, 0, 516, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1578, 6057, 9, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 796, 4, 1, 0, 1876, 2, 2487, 0, 1026, 1, 373, 1, 6852, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 0, 0, 98, 18, 180, 19, 1, 360, 64, 2, 621, 899, 1114, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 40, 0, 0, 11, 6, 0, 6536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4658, 15278, 7985, 3317, 7672, 13718, 6312, 6596] avg=8192.0000 max=15278 max_rank=1 min=3317 min_rank=3 imbalance_ratio=1.864990 max_load_reduction=50258 max_load_reduction_pct=0.766876 moved_to_echo=34298 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[688, 184, 0, 2055, 0, 0, 5, 107, 0, 0, 1681, 0, 0, 269, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 1, 16, 0, 22, 0, 137, 4, 443, 700, 76, 1, 165, 119, 3599, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 1, 6, 0, 1, 0, 134, 0, 302, 1, 2, 0, 0, 4, 3686, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 969, 0, 1, 597, 0, 6, 0, 0, 166, 3327, 6612, 0, 36, 23, 5588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1908, 6968, 95, 36, 3, 10, 0, 10, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 126, 225, 0, 23, 29, 0, 2388, 0, 5135, 0, 9, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 18, 2, 2, 4, 3366, 1, 0, 40, 0, 32, 422, 90, 0, 14, 0, 2338, 1034, 579, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4, 0, 0, 10, 2, 1437, 1, 2, 1, 0, 0, 3397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5050, 6801, 6024, 17332, 9040, 7998, 8426, 4865] avg=8192.0000 max=17332 max_rank=3 min=4865 min_rank=7 imbalance_ratio=2.115723 max_load_reduction=48204 max_load_reduction_pct=0.735535 moved_to_echo=30329 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[171, 212, 0, 647, 0, 0, 0, 0, 0, 0, 3621, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 193, 97, 524, 0, 0, 1862, 0, 5, 0, 48, 5753, 2, 0, 0, 1, 3748, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 32, 26, 0, 1, 148, 6, 24, 0, 239, 9, 3, 0, 5369, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 49, 0, 159, 66, 212, 0, 3, 287, 438, 15, 0, 516, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1578, 6057, 9, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 796, 4, 1, 0, 1876, 2, 2487, 0, 1026, 1, 373, 1, 6852, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 0, 0, 98, 18, 180, 19, 1, 360, 64, 2, 621, 899, 1114, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 40, 0, 0, 11, 6, 0, 6536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4658, 15278, 7985, 3317, 7672, 13718, 6312, 6596] avg=8192.0000 max=15278 max_rank=1 min=3317 min_rank=3 imbalance_ratio=1.864990 max_load_reduction=50258 max_load_reduction_pct=0.766876 moved_to_echo=34298 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 814, 0, 2, 0, 0, 44, 62, 0, 0, 3491, 0, 0, 2024, 1, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 45, 0, 38, 0, 0, 0, 0, 2, 290, 0, 1, 1, 10, 2834, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 22, 0, 1, 3, 0, 0, 0, 0, 3706, 0, 4, 4, 0, 1, 0, 3976, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 1560, 0, 0, 0, 0, 2454, 1356, 1995, 0, 2, 93, 4201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 119, 6833, 0, 1, 71, 53, 0, 107, 0, 3, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2491, 108, 0, 2, 3, 440, 0, 1881, 10, 2119, 2, 4, 0, 12, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1843, 0, 1, 0, 24, 38, 304, 30, 0, 0, 0, 6118, 1014, 686, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 513, 2, 0, 0, 20, 0, 4025, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7695, 6180, 9180, 11668, 7497, 7441, 11027, 4848] avg=8192.0000 max=11668 max_rank=3 min=4848 min_rank=7 imbalance_ratio=1.424316 max_load_reduction=53868 max_load_reduction_pct=0.821960 moved_to_echo=30813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1429, 392, 0, 7003, 12, 0, 3, 1, 34, 86, 3083, 0, 1, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 355, 0, 1, 155, 0, 87, 1, 148, 2942, 6, 0, 1, 0, 2729, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 1, 1555, 0, 6, 0, 42, 5, 2, 4, 15, 0, 0, 6376, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 4, 9, 278, 7, 27, 0, 2, 271, 102, 1, 0, 15, 0, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 0, 1450, 7124, 0, 0, 2, 0, 0, 743, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 2, 0, 0, 0, 11, 1811, 0, 915, 10, 17, 0, 2610, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2277, 12, 1, 3, 8, 0, 15, 1, 10, 10, 7, 1087, 1058, 785, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 1, 7628, 0, 0, 1, 7, 0, 4605, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12107, 9229, 9427, 1596, 9336, 5392, 6119, 12330] avg=8192.0000 max=12330 max_rank=7 min=1596 min_rank=3 imbalance_ratio=1.505127 max_load_reduction=53206 max_load_reduction_pct=0.811859 moved_to_echo=33177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1144, 22, 2, 944, 0, 0, 7, 29, 0, 12, 3681, 0, 20, 5, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 151, 911, 0, 0, 0, 0, 38, 0, 23, 57, 3, 0, 24, 248, 3663, 3090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 15, 98, 3, 0, 0, 4, 262, 42, 25, 5, 184, 0, 0, 6856, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 23, 0, 0, 909, 1, 310, 0, 0, 1970, 362, 155, 0, 2834, 69, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1314, 7084, 0, 0, 0, 52, 0, 66, 0, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 5, 0, 0, 0, 29, 0, 2429, 1, 871, 0, 57, 7, 1674, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 0, 0, 15, 486, 0, 0, 0, 0, 180, 2, 3, 0, 0, 0, 3566, 1052, 1181, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 166, 0, 0, 58, 0, 5496, 158, 6, 0, 1, 0, 6927, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 8211, 9387, 7516, 8552, 5500, 7525, 12880] avg=8192.0000 max=12880 max_rank=7 min=5500 min_rank=5 imbalance_ratio=1.572266 max_load_reduction=52656 max_load_reduction_pct=0.803467 moved_to_echo=34457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 814, 0, 2, 0, 0, 44, 62, 0, 0, 3491, 0, 0, 2024, 1, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 45, 0, 38, 0, 0, 0, 0, 2, 290, 0, 1, 1, 10, 2834, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 22, 0, 1, 3, 0, 0, 0, 0, 3706, 0, 4, 4, 0, 1, 0, 3976, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 1560, 0, 0, 0, 0, 2454, 1356, 1995, 0, 2, 93, 4201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 119, 6833, 0, 1, 71, 53, 0, 107, 0, 3, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2491, 108, 0, 2, 3, 440, 0, 1881, 10, 2119, 2, 4, 0, 12, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1843, 0, 1, 0, 24, 38, 304, 30, 0, 0, 0, 6118, 1014, 686, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 513, 2, 0, 0, 20, 0, 4025, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7695, 6180, 9180, 11668, 7497, 7441, 11027, 4848] avg=8192.0000 max=11668 max_rank=3 min=4848 min_rank=7 imbalance_ratio=1.424316 max_load_reduction=53868 max_load_reduction_pct=0.821960 moved_to_echo=30813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1371,103 +1371,103 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 384, 512, 4864, 128, 128, 2176, 0, 384, 128, 768, 10368, 128, 128, 256, 512, 24960, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 384, 128, 20352, 128, 256, 256, 256, 512, 1024, 128, 1280, 256, 128, 18944, 8192, 7680, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 1536, 128, 128, 4864, 128, 1024, 0, 128, 12672, 13056, 8832, 0, 5376, 256, 18304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 384, 128, 20352, 128, 256, 256, 256, 512, 1024, 128, 1280, 256, 128, 18944, 8192, 7680, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 3072, 1408, 896, 128, 256, 3456, 128, 16640, 128, 16640, 128, 640, 128, 21504, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[9344, 2688, 128, 24192, 128, 0, 128, 256, 128, 768, 24576, 128, 128, 2432, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[256, 128, 128, 128, 256, 2048, 128, 128, 384, 6912, 512, 128, 2432, 384, 128, 128, 40064, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 512, 128, 128, 896, 128, 18560, 384, 128, 128, 256, 128, 44928, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[384, 128, 128, 128, 8576, 54784, 256, 128, 128, 128, 0, 1280, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 8005, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 4, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 382, 10, 1, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 7914, 0, 1458, 0, 0, 69, 0, 5, 0, 2, 0, 0, 0, 0, 32, 0, 8103, 0, 0, 0, 77, 7449, 0, 0, 0, 2, 0, 5, 0, 0, 33, 88, 0, 0, 69, 0, 0, 127, 0, 0, 0, 1, 0, 22, 7126, 12, 0, 0, 7, 8188, 0, 10, 0, 0, 0, 2, 0, 0, 0, 7962, 11, 0, 4, 0, 60, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3, 17, 0, 0, 0, 8165, 24, 0, 1, 623, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 1, 7, 29, 3, 172, 7, 3, 0, 0, 0, 0, 0, 16, 0, 35, 17, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 3092, 0, 7906, 0, 0, 0, 12, 7106, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2049, 0, 2, 9, 8123, 2, 0, 0, 0, 1620, 0, 0, 0, 0, 0, 9, 2, 0, 39, 2954, 0, 0, 155, 0, 0, 176, 0, 0, 0, 0, 36, 2440, 7437, 0, 0, 0, 0, 3729, 0, 0, 13, 0, 0, 2, 0, 0, 0, 8166, 0, 0, 229, 15, 197, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7713, 1, 7, 0, 0, 3, 159, 0, 0, 0, 0, 0, 0, 812, 3, 0, 0, 43, 3, 0, 14, 0, 0, 0, 185, 0, 0, 0, 0, 27, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8, 171, 0, 0, 0, 0, 0, 550, 0, 7700, 0, 0, 0, 3, 8127, 0, 64, 1, 0, 272, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 7620, 101, 5, 2, 8, 8073, 0, 100, 0, 0, 0, 0, 0, 0, 2038, 0, 0, 3, 3652, 0, 1, 0, 0, 157, 0, 0, 4, 5, 1235, 2, 0, 0, 0, 7865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 119, 0, 471, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[10, 0, 3, 0, 2, 6231, 0, 0, 2, 6132, 0, 0, 0, 7, 65, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 27, 0, 1, 0, 7730, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 0, 1, 0, 0, 5, 0, 0, 1, 21, 0, 1069, 0, 1290, 2, 0, 0, 0, 8148, 0, 6, 348, 0, 0, 0, 3, 0, 44, 0, 0, 0, 0, 0, 43, 7140, 180, 0, 45, 16, 0, 69, 0, 0, 0, 0, 0, 0, 38, 0, 3361, 1, 8, 158, 0, 0, 0, 12, 32, 0, 0, 0, 5392, 4432, 0, 0, 3, 0, 4530, 0, 0, 50, 5, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 623, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[82, 1, 1, 0, 0, 7967, 0, 4, 0, 0, 0, 4, 1, 27, 0, 0, 0, 39, 4364, 30, 11, 0, 23, 0, 11, 0, 1, 0, 0, 66, 2, 26, 212, 0, 14, 0, 2, 0, 0, 0, 0, 17, 0, 3, 0, 0, 9, 0, 11, 313, 0, 0, 0, 0, 0, 1, 0, 7987, 0, 0, 4, 0, 8123, 0, 1974, 0, 0, 76, 5, 0, 0, 0, 0, 0, 4, 0, 2, 0, 7608, 0, 0, 5, 0, 0, 67, 90, 0, 0, 0, 459, 0, 0, 68, 0, 0, 0, 280, 0, 0, 445, 6, 1, 77, 0, 2560, 0, 5914, 0, 0, 7, 0, 7110, 54, 0, 12, 16, 0, 66, 475, 0, 57, 8124, 3, 331, 160, 117, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7713, 1, 7, 0, 0, 3, 159, 0, 0, 0, 0, 0, 0, 812, 3, 0, 0, 43, 3, 0, 14, 0, 0, 0, 185, 0, 0, 0, 0, 27, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8, 171, 0, 0, 0, 0, 0, 550, 0, 7700, 0, 0, 0, 3, 8127, 0, 64, 1, 0, 272, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 7620, 101, 5, 2, 8, 8073, 0, 100, 0, 0, 0, 0, 0, 0, 2038, 0, 0, 3, 3652, 0, 1, 0, 0, 157, 0, 0, 4, 5, 1235, 2, 0, 0, 0, 7865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 119, 0, 471, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 66, 0, 0, 7, 68, 0, 0, 0, 0, 31, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4949, 0, 0, 12, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7426, 0, 8145, 0, 0, 0, 1374, 6463, 0, 2564, 0, 0, 11, 0, 0, 0, 1, 0, 0, 6, 0, 0, 4, 8190, 205, 0, 0, 0, 4434, 0, 0, 0, 5, 0, 0, 0, 1, 529, 0, 0, 0, 3407, 0, 3, 19, 0, 0, 0, 0, 1, 0, 737, 0, 2, 0, 0, 6971, 3, 0, 697, 13, 0, 0, 0, 2, 2, 7831, 0, 0, 12, 1, 1263, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 5, 23, 0, 0, 26, 21, 0, 5092, 0, 14, 0, 0, 0, 0, 0, 152, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2, 1, 0, 0, 0, 0, 122, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3158, 0, 21, 0, 8, 7978, 0, 2249, 0, 0, 62, 0, 0, 0, 0, 0, 0, 12, 145, 0, 0, 5422, 1107, 0, 0, 0, 7606, 0, 1839, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 7, 1, 6, 0, 2, 0, 0, 0, 0, 7525, 5, 4688, 0, 85, 0, 0, 8181, 900, 0, 0, 0, 0, 17, 0, 0, 203, 5949, 0, 0, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 6172, 0, 0, 0, 3, 0, 0, 0, 13, 0, 0, 1, 1, 144, 6, 0, 0, 0, 0, 0, 6, 0, 138, 11, 2, 0, 1, 0, 3, 0, 0, 1, 0, 0, 8, 16, 0, 0, 0, 0, 0, 12, 11, 7, 71, 2, 0, 0, 0, 0, 50, 0, 6162, 1, 22, 0, 957, 7581, 0, 2757, 0, 0, 0, 0, 1, 0, 113, 0, 0, 323, 70, 0, 0, 8029, 2, 0, 0, 6, 8128, 0, 0, 0, 0, 24, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 5, 0, 1, 0, 251, 0, 8007, 0, 0, 0, 0, 8018, 2, 0, 0, 0, 0, 0, 243, 0, 0, 8135, 0, 1, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[82, 1, 1, 0, 0, 7967, 0, 4, 0, 0, 0, 4, 1, 27, 0, 0, 0, 39, 4364, 30, 11, 0, 23, 0, 11, 0, 1, 0, 0, 66, 2, 26, 212, 0, 14, 0, 2, 0, 0, 0, 0, 17, 0, 3, 0, 0, 9, 0, 11, 313, 0, 0, 0, 0, 0, 1, 0, 7987, 0, 0, 4, 0, 8123, 0, 1974, 0, 0, 76, 5, 0, 0, 0, 0, 0, 4, 0, 2, 0, 7608, 0, 0, 5, 0, 0, 67, 90, 0, 0, 0, 459, 0, 0, 68, 0, 0, 0, 280, 0, 0, 445, 6, 1, 77, 0, 2560, 0, 5914, 0, 0, 7, 0, 7110, 54, 0, 12, 16, 0, 66, 475, 0, 57, 8124, 3, 331, 160, 117, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 5, 23, 0, 0, 26, 21, 0, 5092, 0, 14, 0, 0, 0, 0, 0, 152, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2, 1, 0, 0, 0, 0, 122, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3158, 0, 21, 0, 8, 7978, 0, 2249, 0, 0, 62, 0, 0, 0, 0, 0, 0, 12, 145, 0, 0, 5422, 1107, 0, 0, 0, 7606, 0, 1839, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 7, 1, 6, 0, 2, 0, 0, 0, 0, 7525, 5, 4688, 0, 85, 0, 0, 8181, 900, 0, 0, 0, 0, 17, 0, 0, 203, 5949, 0, 0, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 17, 0, 0, 0, 8165, 24, 0, 1, 623, 0, 0, 0, 0, 0, 0, 675, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 1, 7, 29, 3, 172, 7, 3, 3054, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 35, 17, 0, 0, 0, 0, 0, 0, 0, 6932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 3092, 0, 7673, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2049, 0, 2, 9, 6628, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 0, 0, 0, 0, 9, 2, 0, 39, 2954, 0, 0, 979, 452, 233, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 176, 0, 0, 0, 0, 36, 2440, 7212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2, 0, 0, 0, 7714, 0, 0, 229, 15, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10024, 3459, 7000, 11651, 8700, 6513, 10019, 8170] avg=8192.0000 max=11651 max_rank=3 min=3459 min_rank=1 imbalance_ratio=1.422241 max_load_reduction=53885 max_load_reduction_pct=0.822220 moved_to_echo=33402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 8005, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 1483, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 4, 0, 0, 0, 0, 0, 75, 6705, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 10, 1, 0, 0, 0, 1, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 0, 69, 0, 5, 0, 2, 0, 0, 0, 0, 32, 0, 6611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 7449, 0, 0, 0, 2, 0, 5, 0, 0, 33, 88, 0, 0, 977, 440, 241, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 127, 0, 0, 0, 1, 0, 22, 6911, 12, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 7522, 11, 0, 4, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10015, 6990, 7726, 8343, 8177, 9527, 7149, 7609] avg=8192.0000 max=10015 max_rank=0 min=6990 min_rank=1 imbalance_ratio=1.222534 max_load_reduction=55521 max_load_reduction_pct=0.847183 moved_to_echo=32462 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 6172, 0, 0, 0, 3, 0, 0, 0, 13, 0, 0, 1452, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 144, 6, 0, 0, 0, 0, 0, 6, 0, 138, 11, 2, 0, 1, 6566, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 8, 16, 0, 0, 0, 0, 0, 12, 11, 7395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 71, 2, 0, 0, 0, 0, 50, 0, 5981, 1, 22, 0, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 0, 0, 0, 1, 0, 113, 0, 0, 323, 70, 0, 0, 6551, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8128, 0, 0, 0, 0, 24, 0, 0, 0, 7, 0, 0, 0, 968, 450, 181, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 0, 1, 0, 251, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 243, 0, 0, 7685, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 7062, 7446, 7091, 9817, 10006, 8029, 7934] avg=8192.0000 max=10006 max_rank=5 min=7062 min_rank=1 imbalance_ratio=1.221436 max_load_reduction=55530 max_load_reduction_pct=0.847321 moved_to_echo=35786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[82, 1, 1, 0, 0, 7967, 0, 4, 0, 0, 0, 4, 1, 27, 0, 0, 1288, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4364, 30, 11, 0, 23, 0, 11, 0, 1, 0, 0, 66, 2, 26, 5822, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 14, 0, 2, 0, 0, 0, 0, 17, 0, 3, 0, 0, 9, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 313, 0, 0, 0, 0, 0, 1, 0, 7752, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 0, 0, 76, 5, 0, 0, 0, 0, 0, 4, 0, 2, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 67, 90, 0, 0, 0, 459, 0, 0, 68, 0, 0, 0, 917, 449, 235, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 445, 6, 1, 77, 0, 2560, 0, 5735, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 12, 16, 0, 66, 475, 0, 57, 7675, 3, 331, 160, 117, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9859, 10595, 8180, 8081, 8268, 2469, 9111, 8973] avg=8192.0000 max=10595 max_rank=1 min=2469 min_rank=5 imbalance_ratio=1.293335 max_load_reduction=54941 max_load_reduction_pct=0.838333 moved_to_echo=28821 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7713, 1, 7, 0, 0, 3, 159, 0, 0, 0, 0, 1424, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 812, 3, 0, 0, 43, 3, 0, 14, 0, 0, 0, 185, 0, 0, 6441, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 171, 0, 0, 0, 0, 0, 550, 0, 7473, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 272, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 6218, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 8, 8073, 0, 100, 0, 0, 0, 0, 0, 0, 2038, 0, 0, 3, 918, 453, 227, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3652, 0, 1, 0, 0, 157, 0, 0, 4, 5, 1198, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7729, 0, 1, 119, 0, 471, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9791, 7700, 7965, 8205, 6660, 11864, 5019, 8332] avg=8192.0000 max=11864 max_rank=5 min=5019 min_rank=6 imbalance_ratio=1.448242 max_load_reduction=53672 max_load_reduction_pct=0.818970 moved_to_echo=31875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 5, 23, 0, 0, 26, 1482, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 5092, 0, 14, 0, 0, 0, 0, 0, 152, 0, 0, 0, 2, 1, 6699, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2, 1, 0, 0, 0, 0, 122, 7781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3065, 0, 21, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, 62, 0, 0, 0, 0, 0, 0, 12, 145, 0, 0, 4425, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 0, 1839, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 7, 653, 329, 93, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 2, 0, 0, 0, 0, 7525, 5, 4546, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 17, 0, 0, 203, 5620, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1951, 12178, 7912, 3112, 8000, 13470, 12170, 6743] avg=8192.0000 max=13470 max_rank=5 min=1951 min_rank=0 imbalance_ratio=1.644287 max_load_reduction=52066 max_load_reduction_pct=0.794464 moved_to_echo=40383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 17, 0, 0, 0, 8165, 24, 0, 1, 623, 0, 0, 0, 0, 0, 0, 675, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 1, 7, 29, 3, 172, 7, 3, 3054, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 35, 17, 0, 0, 0, 0, 0, 0, 0, 6932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 3092, 0, 7673, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2049, 0, 2, 9, 6628, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 0, 0, 0, 0, 9, 2, 0, 39, 2954, 0, 0, 979, 452, 233, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 176, 0, 0, 0, 0, 36, 2440, 7212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2, 0, 0, 0, 7714, 0, 0, 229, 15, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10024, 3459, 7000, 11651, 8700, 6513, 10019, 8170] avg=8192.0000 max=11651 max_rank=3 min=3459 min_rank=1 imbalance_ratio=1.422241 max_load_reduction=53885 max_load_reduction_pct=0.822220 moved_to_echo=33402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 6172, 0, 0, 0, 3, 0, 0, 0, 13, 0, 0, 1452, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 144, 6, 0, 0, 0, 0, 0, 6, 0, 138, 11, 2, 0, 1, 6566, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 8, 16, 0, 0, 0, 0, 0, 12, 11, 7395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 71, 2, 0, 0, 0, 0, 50, 0, 5981, 1, 22, 0, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 0, 0, 0, 1, 0, 113, 0, 0, 323, 70, 0, 0, 6551, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8128, 0, 0, 0, 0, 24, 0, 0, 0, 7, 0, 0, 0, 968, 450, 181, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 0, 1, 0, 251, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 243, 0, 0, 7685, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 7062, 7446, 7091, 9817, 10006, 8029, 7934] avg=8192.0000 max=10006 max_rank=5 min=7062 min_rank=1 imbalance_ratio=1.221436 max_load_reduction=55530 max_load_reduction_pct=0.847321 moved_to_echo=35786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 3, 0, 2, 6231, 0, 0, 2, 6132, 0, 0, 0, 7, 65, 0, 821, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 27, 0, 1, 0, 7730, 0, 0, 3709, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 0, 1, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 21, 0, 1069, 0, 1249, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 348, 0, 0, 0, 3, 0, 44, 0, 0, 0, 0, 0, 43, 5817, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 16, 0, 69, 0, 0, 0, 0, 0, 0, 38, 0, 3361, 1, 8, 865, 455, 41, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 12, 32, 0, 0, 0, 5392, 4295, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 5, 0, 0, 0, 0, 0, 7706, 0, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13731, 11669, 8036, 2347, 6441, 5036, 9892, 8384] avg=8192.0000 max=13731 max_rank=0 min=2347 min_rank=3 imbalance_ratio=1.676147 max_load_reduction=51805 max_load_reduction_pct=0.790482 moved_to_echo=29753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 66, 0, 0, 7, 68, 0, 0, 0, 0, 31, 46, 1263, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 5708, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4949, 0, 0, 12, 0, 1, 0, 0, 0, 6304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7426, 0, 7905, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 0, 11, 0, 0, 0, 1, 0, 0, 6, 0, 0, 4, 6682, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4434, 0, 0, 0, 5, 0, 0, 0, 1, 529, 0, 0, 0, 987, 433, 240, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 3, 19, 0, 0, 0, 0, 1, 0, 715, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 697, 13, 0, 0, 0, 2, 2, 7398, 0, 0, 12, 1, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2014, 5883, 11271, 16706, 9473, 6651, 4147, 9391] avg=8192.0000 max=16706 max_rank=3 min=2014 min_rank=0 imbalance_ratio=2.039307 max_load_reduction=48830 max_load_reduction_pct=0.745087 moved_to_echo=29662 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[82, 1, 1, 0, 0, 7967, 0, 4, 0, 0, 0, 4, 1, 27, 0, 0, 1288, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4364, 30, 11, 0, 23, 0, 11, 0, 1, 0, 0, 66, 2, 26, 5822, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 14, 0, 2, 0, 0, 0, 0, 17, 0, 3, 0, 0, 9, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 313, 0, 0, 0, 0, 0, 1, 0, 7752, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 0, 0, 76, 5, 0, 0, 0, 0, 0, 4, 0, 2, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 67, 90, 0, 0, 0, 459, 0, 0, 68, 0, 0, 0, 917, 449, 235, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 445, 6, 1, 77, 0, 2560, 0, 5735, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 12, 16, 0, 66, 475, 0, 57, 7675, 3, 331, 160, 117, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9859, 10595, 8180, 8081, 8268, 2469, 9111, 8973] avg=8192.0000 max=10595 max_rank=1 min=2469 min_rank=5 imbalance_ratio=1.293335 max_load_reduction=54941 max_load_reduction_pct=0.838333 moved_to_echo=28821 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 5, 23, 0, 0, 26, 1482, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 5092, 0, 14, 0, 0, 0, 0, 0, 152, 0, 0, 0, 2, 1, 6699, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2, 1, 0, 0, 0, 0, 122, 7781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3065, 0, 21, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, 62, 0, 0, 0, 0, 0, 0, 12, 145, 0, 0, 4425, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 0, 1839, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 7, 653, 329, 93, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 2, 0, 0, 0, 0, 7525, 5, 4546, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 17, 0, 0, 203, 5620, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1951, 12178, 7912, 3112, 8000, 13470, 12170, 6743] avg=8192.0000 max=13470 max_rank=5 min=1951 min_rank=0 imbalance_ratio=1.644287 max_load_reduction=52066 max_load_reduction_pct=0.794464 moved_to_echo=40383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 37376, 256, 2048, 0, 128, 128, 512, 128, 128, 5632, 6528, 128, 128, 7296, 3584, 1536, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 44416, 128, 128, 128, 6912, 128, 256, 128, 128, 128, 128, 9984, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 10496, 128, 128, 0, 128, 128, 128, 128, 256, 256, 128, 8192, 128, 128, 44800, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 1536, 512, 128, 128, 128, 0, 12288, 128, 49152, 128, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 10496, 128, 128, 0, 128, 128, 128, 128, 256, 256, 128, 8192, 128, 128, 44800, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[7808, 128, 128, 896, 128, 256, 128, 128, 10496, 7936, 38400, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11136, 384, 0, 512, 128, 128, 0, 256, 0, 0, 2432, 256, 128, 128, 49152, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[1024, 128, 896, 128, 0, 128, 768, 128, 384, 59136, 128, 384, 640, 256, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[384, 128, 128, 0, 128, 128, 128, 5120, 128, 128, 128, 128, 128, 0, 128, 256, 60032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[384, 128, 128, 0, 128, 128, 128, 5120, 128, 128, 128, 128, 128, 0, 128, 256, 60032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 0, 0, 0, 0, 49, 0, 7496, 2, 0, 0, 0, 0, 1, 0, 0, 7292, 0, 0, 1017, 3, 0, 0, 0, 0, 0, 12, 0, 89, 0, 0, 22, 0, 51, 332, 5911, 0, 0, 0, 92, 0, 0, 0, 0, 0, 154, 0, 275, 527, 0, 0, 0, 0, 0, 0, 61, 0, 331, 42, 0, 36, 0, 7915, 0, 0, 0, 0, 0, 0, 7639, 146, 0, 0, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 93, 0, 1122, 0, 0, 0, 0, 2, 0, 6, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 22, 3, 8043, 0, 0, 11, 1174, 0, 0, 0, 26, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 57, 0, 0, 6330, 0, 0, 39, 1937, 312, 599, 0, 0, 0, 0, 1, 7, 0, 15, 349, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 81, 42, 0, 0, 1, 13, 0, 0, 8022, 1, 0, 0, 3, 0, 0, 0, 0, 0, 290, 0, 0, 2, 6, 0, 0, 0, 0, 0, 15, 0, 1, 4, 5, 0, 0, 6708, 1770, 36, 0, 0, 0, 0, 5072, 0, 1883, 0, 0, 0, 0, 0, 0, 5, 571, 1, 5, 6741, 0, 4, 0, 1, 313, 0, 3998, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 8183, 0, 7551, 4487, 0, 0, 0, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[31, 0, 0, 0, 0, 1352, 4, 829, 0, 0, 0, 8037, 0, 2, 0, 0, 0, 0, 1, 0, 129, 0, 1, 0, 71, 0, 13, 0, 0, 0, 0, 2568, 1, 114, 53, 0, 12, 0, 0, 1446, 0, 0, 0, 699, 0, 0, 0, 12, 0, 7680, 0, 0, 2238, 0, 24, 0, 0, 2, 0, 2, 0, 1, 6474, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 8177, 0, 1566, 0, 5, 0, 8154, 444, 0, 0, 0, 0, 0, 5901, 28, 0, 0, 0, 0, 0, 618, 1, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 3, 0, 0, 0, 0, 2, 26, 0, 50, 0, 0, 8137, 14, 0, 561, 0, 0, 44, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4686, 0, 0, 3, 886, 0, 0, 5, 0, 5025, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 7121, 0, 4131, 6, 992, 0, 0, 0, 1, 0, 2, 55, 0, 193, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6782, 0, 0, 0, 2, 2, 0, 0, 0, 441, 0, 7828, 193, 1152, 49, 0, 661, 8105, 0, 0, 0, 0, 0, 0, 87, 1, 0, 7, 0, 0, 0, 67, 0, 0, 0, 1, 2, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 8181, 0, 0, 7340, 964, 0, 2, 84, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 93, 15, 0, 0, 125, 0, 309, 1175, 143, 306, 0, 2, 0, 2, 0, 0, 2, 0, 9, 370, 0, 0, 0, 0, 0, 0, 0, 0, 14, 217, 0, 0, 0, 0, 0, 0, 42, 17, 7737, 0, 765, 0, 8087, 0, 289, 0, 0, 0, 34, 0, 0, 783, 0, 73, 17, 8, 129, 26, 0, 0, 0, 7992, 0, 0, 2, 35, 143, 0, 0, 0, 2, 11, 5108, 0, 115, 33, 78, 35, 5591, 0, 0, 0, 0, 0, 0, 6763, 0, 24, 2, 45, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 1, 920, 0, 7954, 0, 9, 5429, 0, 0, 0, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[22, 0, 0, 69, 1, 556, 1, 7, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 1, 6511, 0, 0, 0, 0, 0, 0, 0, 4, 94, 0, 1, 0, 0, 3, 0, 0, 0, 0, 7388, 0, 0, 0, 120, 0, 0, 0, 0, 0, 73, 0, 1, 0, 7, 0, 9, 0, 0, 0, 0, 0, 1652, 12, 0, 7, 0, 42, 0, 0, 0, 33, 20, 0, 4212, 0, 0, 0, 0, 0, 8157, 1000, 0, 0, 5, 0, 8, 7680, 0, 0, 0, 0, 290, 57, 3666, 0, 0, 0, 6108, 0, 21, 0, 0, 0, 0, 0, 7, 0, 0, 166, 0, 0, 0, 0, 1099, 0, 63, 1, 0, 7983, 0, 0, 4, 243, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3106, 0, 8, 0, 8151, 0, 4414, 0, 0, 13, 162, 0, 0, 8, 0, 1896, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 966, 0, 1, 0, 0, 0, 4, 0, 8181, 0, 0, 0, 7149, 0, 0, 0, 0, 0, 2114, 0, 0, 1, 0, 0, 0, 0, 9, 1689, 0, 0, 0, 360, 0, 0, 0, 486, 864, 0, 0, 0, 0, 0, 6528, 31, 183, 0, 0, 0, 107, 0, 0, 1, 0, 0, 0, 2068, 0, 1, 0, 0, 0, 34, 7036, 0, 0, 0, 14, 0, 48, 0, 2, 24, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 7817, 0, 0, 2017, 6, 0, 0, 0, 13, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 0, 0, 0, 0, 49, 0, 7496, 2, 0, 0, 0, 0, 1, 0, 0, 7292, 0, 0, 1017, 3, 0, 0, 0, 0, 0, 12, 0, 89, 0, 0, 22, 0, 51, 332, 5911, 0, 0, 0, 92, 0, 0, 0, 0, 0, 154, 0, 275, 527, 0, 0, 0, 0, 0, 0, 61, 0, 331, 42, 0, 36, 0, 7915, 0, 0, 0, 0, 0, 0, 7639, 146, 0, 0, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 93, 0, 1122, 0, 0, 0, 0, 2, 0, 6, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 22, 3, 8043, 0, 0, 11, 1174, 0, 0, 0, 26, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[22, 0, 0, 69, 1, 556, 1, 7, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 1, 6511, 0, 0, 0, 0, 0, 0, 0, 4, 94, 0, 1, 0, 0, 3, 0, 0, 0, 0, 7388, 0, 0, 0, 120, 0, 0, 0, 0, 0, 73, 0, 1, 0, 7, 0, 9, 0, 0, 0, 0, 0, 1652, 12, 0, 7, 0, 42, 0, 0, 0, 33, 20, 0, 4212, 0, 0, 0, 0, 0, 8157, 1000, 0, 0, 5, 0, 8, 7680, 0, 0, 0, 0, 290, 57, 3666, 0, 0, 0, 6108, 0, 21, 0, 0, 0, 0, 0, 7, 0, 0, 166, 0, 0, 0, 0, 1099, 0, 63, 1, 0, 7983, 0, 0, 4, 243, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 93, 15, 0, 0, 125, 0, 309, 1175, 143, 306, 0, 2, 0, 2, 0, 0, 2, 0, 9, 370, 0, 0, 0, 0, 0, 0, 0, 0, 14, 217, 0, 0, 0, 0, 0, 0, 42, 17, 7737, 0, 765, 0, 8087, 0, 289, 0, 0, 0, 34, 0, 0, 783, 0, 73, 17, 8, 129, 26, 0, 0, 0, 7992, 0, 0, 2, 35, 143, 0, 0, 0, 2, 11, 5108, 0, 115, 33, 78, 35, 5591, 0, 0, 0, 0, 0, 0, 6763, 0, 24, 2, 45, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 1, 920, 0, 7954, 0, 9, 5429, 0, 0, 0, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2245, 0, 31, 402, 2, 887, 0, 0, 4, 6405, 0, 3, 0, 0, 0, 488, 1, 0, 56, 0, 1568, 0, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 5, 0, 2930, 6, 6, 0, 0, 0, 0, 1, 0, 51, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6329, 0, 2, 0, 6987, 0, 0, 578, 6, 0, 4, 9, 0, 0, 0, 30, 0, 73, 0, 0, 6, 0, 0, 0, 3410, 0, 1, 2, 0, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 983, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4525, 0, 23, 88, 0, 8046, 0, 0, 7812, 0, 0, 59, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 57, 0, 0, 6330, 0, 0, 39, 1937, 312, 599, 0, 0, 0, 0, 1, 7, 0, 15, 349, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 81, 42, 0, 0, 1, 13, 0, 0, 8022, 1, 0, 0, 3, 0, 0, 0, 0, 0, 290, 0, 0, 2, 6, 0, 0, 0, 0, 0, 15, 0, 1, 4, 5, 0, 0, 6708, 1770, 36, 0, 0, 0, 0, 5072, 0, 1883, 0, 0, 0, 0, 0, 0, 5, 571, 1, 5, 6741, 0, 4, 0, 1, 313, 0, 3998, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 8183, 0, 7551, 4487, 0, 0, 0, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 69, 1, 556, 1, 7, 0, 0, 0, 7057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6511, 0, 0, 0, 0, 0, 0, 0, 4, 94, 0, 1, 3805, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 6347, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1, 0, 7, 0, 9, 0, 0, 0, 0, 0, 1652, 12, 0, 1041, 978, 1075, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 42, 0, 0, 0, 33, 20, 0, 0, 0, 0, 0, 0, 0, 7179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 0, 0, 5, 0, 8, 6349, 0, 0, 0, 0, 290, 57, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 21, 0, 0, 0, 0, 0, 7, 0, 0, 166, 0, 0, 0, 4212, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 63, 1, 0, 2569, 0, 0, 4, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7713, 11079, 6470, 5516, 7281, 11375, 12123, 3979] avg=8192.0000 max=12123 max_rank=6 min=3979 min_rank=7 imbalance_ratio=1.479858 max_load_reduction=53413 max_load_reduction_pct=0.815018 moved_to_echo=34758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 0, 0, 0, 0, 49, 0, 6507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7292, 0, 0, 1017, 3, 0, 0, 0, 0, 0, 12, 0, 3828, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 22, 0, 51, 332, 5081, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 275, 527, 0, 0, 0, 0, 0, 0, 61, 0, 331, 42, 0, 830, 903, 989, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 7915, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 6661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6582, 0, 0, 93, 0, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7639, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3, 2597, 0, 0, 11, 1174, 0, 0, 0, 26, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6561, 12834, 5667, 4799, 14758, 7797, 9283, 3837] avg=8192.0000 max=14758 max_rank=4 min=3837 min_rank=7 imbalance_ratio=1.801514 max_load_reduction=50778 max_load_reduction_pct=0.774811 moved_to_echo=35675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 0, 0, 0, 1352, 4, 829, 0, 0, 0, 6977, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 129, 0, 1, 0, 71, 0, 13, 0, 0, 0, 0, 2568, 3872, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 114, 53, 0, 12, 0, 0, 1243, 0, 0, 0, 699, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 0, 0, 2238, 0, 24, 0, 0, 2, 0, 2, 0, 1, 6474, 0, 203, 974, 1060, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 1566, 0, 5, 0, 7180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, 0, 0, 4886, 28, 0, 0, 0, 0, 0, 618, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 3, 0, 0, 0, 0, 8177, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 26, 0, 50, 0, 0, 2628, 14, 0, 561, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9195, 7160, 2134, 19168, 8755, 5977, 9822, 3325] avg=8192.0000 max=19168 max_rank=3 min=2134 min_rank=2 imbalance_ratio=2.339844 max_load_reduction=46368 max_load_reduction_pct=0.707520 moved_to_echo=27879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 93, 15, 0, 0, 125, 0, 309, 1175, 143, 306, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 370, 0, 0, 0, 0, 0, 0, 0, 0, 14, 217, 0, 3785, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 17, 6651, 0, 765, 0, 8087, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 783, 0, 73, 17, 8, 129, 26, 0, 0, 0, 7992, 0, 1086, 668, 0, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 143, 0, 0, 0, 2, 11, 0, 0, 115, 33, 78, 35, 4923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5599, 0, 24, 2, 45, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 5108, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 920, 0, 2569, 0, 9, 5429, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2170, 4976, 15851, 11401, 5377, 9949, 6876, 8936] avg=8192.0000 max=15851 max_rank=2 min=2170 min_rank=0 imbalance_ratio=1.934937 max_load_reduction=49685 max_load_reduction_pct=0.758133 moved_to_echo=31138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4068, 0, 0, 3, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5025, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 3893, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 6122, 0, 4131, 6, 992, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 55, 0, 193, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6782, 0, 999, 968, 618, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 441, 0, 0, 193, 1152, 49, 0, 661, 7137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1, 0, 7, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7828, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 2643, 0, 0, 7340, 964, 0, 2, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4958, 8935, 11270, 9625, 9637, 148, 9870, 11093] avg=8192.0000 max=11270 max_rank=2 min=148 min_rank=5 imbalance_ratio=1.375732 max_load_reduction=54266 max_load_reduction_pct=0.828033 moved_to_echo=30748 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 57, 0, 0, 6330, 0, 0, 39, 1937, 312, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 15, 349, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 81, 3894, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 1, 13, 0, 0, 6896, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 2, 6, 0, 0, 0, 0, 0, 15, 0, 1, 4, 5, 1126, 0, 79, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6708, 1770, 36, 0, 0, 0, 0, 0, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 571, 1, 5, 5581, 0, 4, 0, 1, 313, 0, 3998, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2, 0, 0, 0, 0, 0, 5072, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 2643, 0, 7551, 4487, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9199, 4937, 6956, 2111, 10397, 10504, 6741, 14691] avg=8192.0000 max=14691 max_rank=7 min=2111 min_rank=3 imbalance_ratio=1.793335 max_load_reduction=50845 max_load_reduction_pct=0.775833 moved_to_echo=42333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4068, 0, 0, 3, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5025, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 3893, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 6122, 0, 4131, 6, 992, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 55, 0, 193, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6782, 0, 999, 968, 618, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 441, 0, 0, 193, 1152, 49, 0, 661, 7137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1, 0, 7, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7828, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 2643, 0, 0, 7340, 964, 0, 2, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4958, 8935, 11270, 9625, 9637, 148, 9870, 11093] avg=8192.0000 max=11270 max_rank=2 min=148 min_rank=5 imbalance_ratio=1.375732 max_load_reduction=54266 max_load_reduction_pct=0.828033 moved_to_echo=30748 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3106, 0, 8, 0, 8151, 0, 3832, 0, 0, 13, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1896, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 966, 3720, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7033, 0, 0, 0, 7149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2114, 0, 0, 1, 0, 0, 0, 0, 9, 1689, 0, 0, 0, 360, 0, 1148, 12, 582, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 864, 0, 0, 0, 0, 0, 0, 31, 183, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1713, 0, 1, 0, 0, 0, 34, 7036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 48, 0, 2, 24, 0, 0, 0, 0, 9, 0, 0, 1, 0, 6528, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 2525, 0, 0, 2017, 6, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15273, 6772, 14187, 6093, 1659, 8785, 8198, 4569] avg=8192.0000 max=15273 max_rank=0 min=1659 min_rank=4 imbalance_ratio=1.864380 max_load_reduction=50263 max_load_reduction_pct=0.766953 moved_to_echo=23211 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 0, 0, 0, 1352, 4, 829, 0, 0, 0, 6977, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 129, 0, 1, 0, 71, 0, 13, 0, 0, 0, 0, 2568, 3872, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 114, 53, 0, 12, 0, 0, 1243, 0, 0, 0, 699, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 0, 0, 2238, 0, 24, 0, 0, 2, 0, 2, 0, 1, 6474, 0, 203, 974, 1060, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 1566, 0, 5, 0, 7180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, 0, 0, 4886, 28, 0, 0, 0, 0, 0, 618, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 3, 0, 0, 0, 0, 8177, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 26, 0, 50, 0, 0, 2628, 14, 0, 561, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9195, 7160, 2134, 19168, 8755, 5977, 9822, 3325] avg=8192.0000 max=19168 max_rank=3 min=2134 min_rank=2 imbalance_ratio=2.339844 max_load_reduction=46368 max_load_reduction_pct=0.707520 moved_to_echo=27879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 69, 1, 556, 1, 7, 0, 0, 0, 7057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6511, 0, 0, 0, 0, 0, 0, 0, 4, 94, 0, 1, 3805, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 6347, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1, 0, 7, 0, 9, 0, 0, 0, 0, 0, 1652, 12, 0, 1041, 978, 1075, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 42, 0, 0, 0, 33, 20, 0, 0, 0, 0, 0, 0, 0, 7179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 0, 0, 5, 0, 8, 6349, 0, 0, 0, 0, 290, 57, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 21, 0, 0, 0, 0, 0, 7, 0, 0, 166, 0, 0, 0, 4212, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 63, 1, 0, 2569, 0, 0, 4, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7713, 11079, 6470, 5516, 7281, 11375, 12123, 3979] avg=8192.0000 max=12123 max_rank=6 min=3979 min_rank=7 imbalance_ratio=1.479858 max_load_reduction=53413 max_load_reduction_pct=0.815018 moved_to_echo=34758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 93, 15, 0, 0, 125, 0, 309, 1175, 143, 306, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 370, 0, 0, 0, 0, 0, 0, 0, 0, 14, 217, 0, 3785, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 17, 6651, 0, 765, 0, 8087, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 783, 0, 73, 17, 8, 129, 26, 0, 0, 0, 7992, 0, 1086, 668, 0, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 143, 0, 0, 0, 2, 11, 0, 0, 115, 33, 78, 35, 4923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5599, 0, 24, 2, 45, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 5108, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 920, 0, 2569, 0, 9, 5429, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2170, 4976, 15851, 11401, 5377, 9949, 6876, 8936] avg=8192.0000 max=15851 max_rank=2 min=2170 min_rank=0 imbalance_ratio=1.934937 max_load_reduction=49685 max_load_reduction_pct=0.758133 moved_to_echo=31138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 0, 0, 0, 0, 49, 0, 6507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7292, 0, 0, 1017, 3, 0, 0, 0, 0, 0, 12, 0, 3828, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 22, 0, 51, 332, 5081, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 275, 527, 0, 0, 0, 0, 0, 0, 61, 0, 331, 42, 0, 830, 903, 989, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 7915, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 6661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6582, 0, 0, 93, 0, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7639, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3, 2597, 0, 0, 11, 1174, 0, 0, 0, 26, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6561, 12834, 5667, 4799, 14758, 7797, 9283, 3837] avg=8192.0000 max=14758 max_rank=4 min=3837 min_rank=7 imbalance_ratio=1.801514 max_load_reduction=50778 max_load_reduction_pct=0.774811 moved_to_echo=35675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2245, 0, 31, 402, 2, 887, 0, 0, 4, 6405, 0, 3, 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 56, 0, 1568, 0, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 2, 3829, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 2930, 6, 6, 0, 0, 0, 0, 1, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6329, 0, 0, 8, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6987, 0, 0, 578, 6, 0, 4, 0, 0, 0, 0, 30, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2824, 0, 1, 2, 0, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4525, 0, 23, 88, 0, 2599, 0, 0, 7812, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10467, 8843, 3001, 6823, 7672, 11001, 2623, 15106] avg=8192.0000 max=15106 max_rank=7 min=2623 min_rank=6 imbalance_ratio=1.843994 max_load_reduction=50430 max_load_reduction_pct=0.769501 moved_to_echo=36402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 10368, 128, 384, 3968, 128, 128, 128, 128, 256, 1792, 128, 0, 2048, 28032, 128, 6528, 4608, 4480, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 5760, 0, 256, 1152, 128, 20864, 128, 7680, 27776, 2432, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 256, 128, 128, 128, 128, 3328, 39424, 128, 4992, 128, 17152, 0, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1536, 0, 128, 640, 128, 128, 33664, 128, 128, 128, 128, 1792, 128, 27904, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 23168, 0, 128, 4224, 128, 0, 128, 128, 128, 128, 256, 3712, 30720, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[2304, 128, 128, 512, 128, 12416, 128, 1280, 1280, 16768, 640, 29056, 128, 128, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 23168, 0, 128, 4224, 128, 0, 128, 128, 128, 128, 256, 3712, 30720, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 6144, 128, 512, 1152, 128, 128, 128, 0, 128, 128, 128, 256, 0, 128, 128, 44672, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 22272, 2816, 128, 640, 128, 512, 128, 0, 384, 4992, 128, 128, 768, 33280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 22272, 2816, 128, 640, 128, 512, 128, 0, 384, 4992, 128, 128, 768, 33280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True @@ -1475,30 +1475,30 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 3, 13, 5, 21, 0, 0, 0, 41, 9, 170, 0, 0, 0, 0, 0, 3, 7269, 7125, 0, 0, 7910, 0, 0, 1, 24, 1253, 1187, 2, 0, 0, 7365, 37, 24, 0, 10, 194, 58, 0, 0, 0, 498, 0, 0, 0, 0, 128, 2, 0, 0, 0, 240, 0, 1, 0, 14, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 41, 8, 0, 902, 4, 0, 0, 0, 8183, 0, 1, 33, 1796, 0, 0, 29, 0, 0, 1, 15, 40, 0, 242, 0, 28, 0, 0, 39, 0, 7125, 0, 0, 9, 0, 0, 5891, 0, 0, 6804, 0, 183, 0, 0, 66, 0, 0, 34, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 3259, 0, 0, 31, 0, 0, 4, 2923, 0, 3, 0, 0, 0, 5, 8182, 18, 0, 0, 0, 125, 0, 1, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1243, 0, 0, 5244, 0, 0, 0, 58, 0, 1, 0, 0, 0, 0, 0, 29, 0, 0, 18, 2218, 5415, 0, 0, 0, 0, 0, 0, 47, 2, 10, 1, 0, 17, 0, 0, 2097, 4698, 0, 0, 0, 21, 7453, 5, 0, 0, 0, 21, 0, 0, 0, 0, 6318, 63, 0, 65, 0, 0, 0, 0, 0, 0, 1866, 7845, 0, 0, 0, 0, 78, 1233, 0, 0, 4854, 0, 0, 0, 0, 0, 0, 0, 4, 34] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 8175, 7564, 12, 0, 249, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 3, 385, 68, 0, 21, 0, 28, 1, 1, 0, 6, 102, 671, 7, 0, 0, 7876, 1, 0, 0, 0, 19, 0, 0, 32, 0, 0, 5, 1, 1, 0, 0, 395, 0, 0, 232, 3, 0, 2, 0, 0, 0, 8161, 0, 0, 1, 8141, 0, 0, 0, 0, 0, 0, 608, 0, 0, 304, 12, 0, 1, 98, 0, 0, 6848, 0, 5, 0, 0, 0, 7819, 0, 0, 6321, 0, 0, 0, 795, 296, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 644, 0, 0, 0, 0, 0, 57, 0, 0, 0, 481, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 176, 0, 0, 0, 895, 454, 0, 6593, 8092, 33, 6, 0, 0, 2, 376, 0, 0, 0, 18, 10, 0, 231, 6, 0, 0, 4, 0, 0, 4741, 367, 0, 2, 0, 0, 0, 2, 4983, 137, 7866, 33, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 31, 0, 0, 0, 1, 0, 11, 8, 388, 100, 53, 0, 0, 7823, 1, 0, 2, 33, 0, 0, 117, 0, 0, 16, 0, 0, 0, 6745, 1, 0, 0, 0, 0, 0, 7499, 0, 0, 0, 2, 0, 5, 0, 0, 6371, 0, 100, 0, 0, 0, 0, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 1014, 0, 0, 394, 0, 0, 128, 0, 0, 0, 28, 0, 0, 0, 70, 0, 313, 7701, 3062, 0, 0, 0, 0, 50, 0, 2436, 0, 0, 0, 6, 243, 17, 0, 0, 0, 140, 0, 0, 6287, 0, 0, 0, 1, 0, 8, 0, 0, 0, 2378, 0, 0, 11, 3, 0, 0, 7365, 0, 0, 1365, 304, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 51, 7882, 2, 0, 3, 0, 7933, 0, 0, 0, 7, 0, 0, 2, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 2656, 0, 3, 0, 0, 1, 0, 0, 0, 5363, 0, 6, 3, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 10, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 1, 0, 325, 0, 0, 0, 0, 0, 3, 6910, 8, 5, 0, 0, 0, 1061, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 3, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 10, 6794, 39, 0, 0, 0, 2, 0, 0, 1, 0, 443, 6364, 0, 17, 28, 0, 57, 0, 14, 0, 6297, 321, 0, 0, 0, 0, 0, 1288, 1, 0, 6796, 0, 0, 0, 0, 8162, 0, 1, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5710, 0, 74, 0, 0, 11, 0, 4186, 0, 0, 0, 0, 0, 0, 8092, 33, 224, 0, 4, 0, 0, 0, 21, 2, 14, 0, 0, 0, 34, 0, 0, 3, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 309, 0, 6541, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 136, 0, 4, 0, 0, 0, 0, 0, 6, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4820, 0, 5, 3, 2, 0, 0, 0, 7, 0, 36, 7996, 0, 0, 0, 0, 0, 8186, 0, 0, 2071, 0, 0, 0, 1, 0, 0, 0, 28, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 1014, 0, 0, 394, 0, 0, 128, 0, 0, 0, 28, 0, 0, 0, 70, 0, 313, 7701, 3062, 0, 0, 0, 0, 50, 0, 2436, 0, 0, 0, 6, 243, 17, 0, 0, 0, 140, 0, 0, 6287, 0, 0, 0, 1, 0, 8, 0, 0, 0, 2378, 0, 0, 11, 3, 0, 0, 7365, 0, 0, 1365, 304, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 51, 7882, 2, 0, 3, 0, 7933, 0, 0, 0, 7, 0, 0, 2, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 2656, 0, 3, 0, 0, 1, 0, 0, 0, 5363, 0, 6, 3, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 8175, 7564, 12, 0, 249, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 3, 385, 68, 0, 21, 0, 28, 1, 1, 0, 6, 102, 671, 7, 0, 0, 7876, 1, 0, 0, 0, 19, 0, 0, 32, 0, 0, 5, 1, 1, 0, 0, 395, 0, 0, 232, 3, 0, 2, 0, 0, 0, 8161, 0, 0, 1, 8141, 0, 0, 0, 0, 0, 0, 608, 0, 0, 304, 12, 0, 1, 98, 0, 0, 6848, 0, 5, 0, 0, 0, 7819, 0, 0, 6321, 0, 0, 0, 795, 296, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 644, 0, 0, 0, 0, 0, 57, 0, 0, 0, 481, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 176, 0, 0, 0, 895, 454, 0, 6593, 8092, 33, 6, 0, 0, 2, 376, 0, 0, 0, 18, 10, 0, 231, 6, 0, 0, 4, 0, 0, 4741, 367, 0, 2, 0, 0, 0, 2, 4983, 137, 7866, 33, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 31, 0, 0, 0, 1, 0, 11, 8, 388, 100, 53, 0, 0, 7823, 1, 0, 2, 33, 0, 0, 117, 0, 0, 16, 0, 0, 0, 6745, 1, 0, 0, 0, 0, 0, 7499, 0, 0, 0, 2, 0, 5, 0, 0, 6371, 0, 100, 0, 0, 0, 0, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 65, 0, 0, 0, 66, 4, 0, 0, 2, 0, 0, 0, 18, 0, 8, 0, 0, 6458, 0, 271, 0, 0, 0, 0, 0, 0, 5261, 0, 0, 5, 18, 0, 0, 0, 1401, 6, 8, 0, 0, 2, 0, 0, 0, 0, 0, 0, 18, 0, 2, 1888, 0, 0, 0, 0, 0, 2, 0, 5392, 0, 1673, 0, 0, 0, 9, 8189, 0, 0, 0, 2, 0, 2, 0, 17, 0, 0, 0, 0, 21, 7, 0, 267, 0, 0, 0, 0, 8030, 0, 2, 0, 0, 0, 0, 7773, 0, 0, 1171, 0, 0, 0, 126, 0, 0, 1, 920, 0, 0, 8143, 0, 16, 0, 0, 0, 9, 79, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 3, 13, 5, 21, 5226, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 9, 170, 0, 0, 0, 0, 0, 3, 7269, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 1, 24, 1253, 1187, 2, 0, 0, 7365, 37, 24, 0, 10, 194, 2634, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 498, 0, 0, 0, 0, 128, 2, 0, 0, 0, 240, 0, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 3404, 1158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 8, 0, 902, 4, 0, 0, 0, 2957, 0, 1, 33, 1796, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 40, 0, 242, 0, 28, 0, 0, 39, 0, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 5891, 0, 0, 4541, 0, 183, 0, 0, 66, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6375, 10825, 21767, 2063, 4588, 5771, 3423, 10724] avg=8192.0000 max=21767 max_rank=2 min=2063 min_rank=3 imbalance_ratio=2.657104 max_load_reduction=43769 max_load_reduction_pct=0.667862 moved_to_echo=24506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 3, 13, 5, 21, 0, 0, 0, 41, 9, 170, 0, 0, 0, 0, 0, 3, 7269, 7125, 0, 0, 7910, 0, 0, 1, 24, 1253, 1187, 2, 0, 0, 7365, 37, 24, 0, 10, 194, 58, 0, 0, 0, 498, 0, 0, 0, 0, 128, 2, 0, 0, 0, 240, 0, 1, 0, 14, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 41, 8, 0, 902, 4, 0, 0, 0, 8183, 0, 1, 33, 1796, 0, 0, 29, 0, 0, 1, 15, 40, 0, 242, 0, 28, 0, 0, 39, 0, 7125, 0, 0, 9, 0, 0, 5891, 0, 0, 6804, 0, 183, 0, 0, 66, 0, 0, 34, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 3259, 0, 0, 31, 0, 0, 4, 2923, 0, 3, 0, 0, 0, 5, 8182, 18, 0, 0, 0, 125, 0, 1, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1243, 0, 0, 5244, 0, 0, 0, 58, 0, 1, 0, 0, 0, 0, 0, 29, 0, 0, 18, 2218, 5415, 0, 0, 0, 0, 0, 0, 47, 2, 10, 1, 0, 17, 0, 0, 2097, 4698, 0, 0, 0, 21, 7453, 5, 0, 0, 0, 21, 0, 0, 0, 0, 6318, 63, 0, 65, 0, 0, 0, 0, 0, 0, 1866, 7845, 0, 0, 0, 0, 78, 1233, 0, 0, 4854, 0, 0, 0, 0, 0, 0, 0, 4, 34] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 1014, 0, 0, 394, 5066, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 28, 0, 0, 0, 70, 0, 313, 3603, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 2436, 0, 0, 0, 6, 243, 17, 0, 0, 0, 140, 2847, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6287, 0, 0, 0, 1, 0, 8, 0, 0, 0, 2378, 0, 0, 11, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7365, 0, 0, 1365, 304, 0, 0, 0, 0, 0, 0, 328, 0, 1269, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 7882, 2, 0, 3, 0, 2867, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 1140, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3579, 0, 6, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6731, 7204, 6627, 9581, 11885, 10814, 9101, 3593] avg=8192.0000 max=11885 max_rank=4 min=3593 min_rank=7 imbalance_ratio=1.450806 max_load_reduction=53651 max_load_reduction_pct=0.818649 moved_to_echo=35393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5710, 5077, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 11, 0, 4186, 0, 0, 0, 0, 0, 0, 3785, 33, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 21, 2, 14, 0, 0, 0, 34, 0, 0, 3, 811, 2992, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 309, 0, 6541, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 136, 0, 3820, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4820, 0, 5, 3, 2, 0, 0, 0, 7, 0, 36, 3432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1383, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19690, 8313, 4223, 7197, 5326, 2883, 8306, 9598] avg=8192.0000 max=19690 max_rank=0 min=2883 min_rank=5 imbalance_ratio=2.403564 max_load_reduction=45846 max_load_reduction_pct=0.699554 moved_to_echo=26113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 3259, 4760, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4, 2923, 0, 3, 0, 0, 0, 5, 3828, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 1, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1243, 3025, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5244, 0, 0, 0, 58, 0, 1, 0, 0, 0, 0, 0, 29, 0, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2218, 5415, 0, 0, 0, 0, 0, 0, 47, 2, 10, 1, 0, 17, 3748, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 4698, 0, 0, 0, 21, 2693, 5, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6318, 63, 0, 65, 0, 0, 0, 0, 0, 0, 1866, 3367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1233, 0, 0, 3240, 0, 0, 0, 0, 0, 0, 0, 4, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8767, 6812, 5206, 6143, 12805, 9535, 11679, 4589] avg=8192.0000 max=12805 max_rank=4 min=4589 min_rank=7 imbalance_ratio=1.563110 max_load_reduction=52731 max_load_reduction_pct=0.804611 moved_to_echo=38608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 65, 0, 0, 0, 66, 4, 0, 0, 2, 0, 0, 0, 18, 5128, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 6458, 0, 271, 0, 0, 0, 0, 0, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 18, 0, 0, 0, 1401, 6, 8, 0, 0, 2, 0, 0, 0, 0, 0, 1945, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 2, 1888, 0, 0, 0, 0, 0, 2, 0, 5392, 0, 1673, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8189, 0, 0, 0, 2, 0, 2, 0, 17, 0, 0, 0, 0, 3891, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7, 0, 267, 0, 0, 0, 0, 2902, 0, 2, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 0, 126, 0, 0, 1, 920, 0, 0, 3494, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 79, 0, 5461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6041, 9198, 4740, 10343, 12965, 10972, 5728, 5549] avg=8192.0000 max=12965 max_rank=4 min=4740 min_rank=2 imbalance_ratio=1.582642 max_load_reduction=52571 max_load_reduction_pct=0.802170 moved_to_echo=35214 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 169, 5212, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 8175, 3539, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 3, 385, 2796, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 21, 0, 28, 1, 1, 0, 6, 102, 671, 7, 0, 0, 7876, 1, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 32, 0, 0, 5, 1, 1, 0, 0, 395, 0, 3272, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 3, 0, 2, 0, 0, 0, 2949, 0, 0, 1, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 304, 12, 0, 1, 98, 0, 0, 2939, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7819, 0, 0, 4218, 0, 0, 0, 795, 296, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 11728, 4500, 9839, 4954, 11328, 3967, 13129] avg=8192.0000 max=13129 max_rank=7 min=3967 min_rank=6 imbalance_ratio=1.602661 max_load_reduction=52407 max_load_reduction_pct=0.799667 moved_to_echo=33378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 65, 0, 0, 0, 66, 4, 0, 0, 2, 0, 0, 0, 18, 5128, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 6458, 0, 271, 0, 0, 0, 0, 0, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 18, 0, 0, 0, 1401, 6, 8, 0, 0, 2, 0, 0, 0, 0, 0, 1945, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 2, 1888, 0, 0, 0, 0, 0, 2, 0, 5392, 0, 1673, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8189, 0, 0, 0, 2, 0, 2, 0, 17, 0, 0, 0, 0, 3891, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7, 0, 267, 0, 0, 0, 0, 2902, 0, 2, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 0, 126, 0, 0, 1, 920, 0, 0, 3494, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 79, 0, 5461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6041, 9198, 4740, 10343, 12965, 10972, 5728, 5549] avg=8192.0000 max=12965 max_rank=4 min=4740 min_rank=2 imbalance_ratio=1.582642 max_load_reduction=52571 max_load_reduction_pct=0.802170 moved_to_echo=35214 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 644, 0, 0, 0, 0, 0, 57, 0, 0, 0, 481, 0, 0, 5001, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 176, 0, 0, 0, 895, 207, 0, 6593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 33, 6, 0, 0, 2, 376, 0, 0, 0, 18, 10, 0, 231, 6, 0, 171, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4741, 367, 0, 2, 0, 0, 0, 2, 4983, 137, 7866, 33, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 31, 0, 0, 0, 1, 3586, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 8, 388, 100, 53, 0, 0, 2822, 1, 0, 2, 33, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6745, 1, 0, 0, 0, 0, 0, 3210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 4242, 0, 100, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6886, 7882, 10005, 19204, 3696, 3535, 9972, 4356] avg=8192.0000 max=19204 max_rank=3 min=3535 min_rank=5 imbalance_ratio=2.344238 max_load_reduction=46332 max_load_reduction_pct=0.706970 moved_to_echo=21559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 1014, 0, 0, 394, 5066, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 28, 0, 0, 0, 70, 0, 313, 3603, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 2436, 0, 0, 0, 6, 243, 17, 0, 0, 0, 140, 2847, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6287, 0, 0, 0, 1, 0, 8, 0, 0, 0, 2378, 0, 0, 11, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7365, 0, 0, 1365, 304, 0, 0, 0, 0, 0, 0, 328, 0, 1269, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 7882, 2, 0, 3, 0, 2867, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 1140, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3579, 0, 6, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6731, 7204, 6627, 9581, 11885, 10814, 9101, 3593] avg=8192.0000 max=11885 max_rank=4 min=3593 min_rank=7 imbalance_ratio=1.450806 max_load_reduction=53651 max_load_reduction_pct=0.818649 moved_to_echo=35393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 568, 4339, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 325, 0, 0, 0, 0, 0, 3, 3232, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 2555, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 153, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 10, 2455, 39, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 443, 6364, 0, 17, 28, 0, 57, 0, 14, 0, 6297, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 1, 0, 4535, 0, 0, 0, 0, 8162, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4955, 3574, 8468, 1166, 9317, 10698, 13360, 13998] avg=8192.0000 max=13998 max_rank=7 min=1166 min_rank=3 imbalance_ratio=1.708740 max_load_reduction=51538 max_load_reduction_pct=0.786407 moved_to_echo=47373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 3259, 4760, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4, 2923, 0, 3, 0, 0, 0, 5, 3828, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 1, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1243, 3025, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5244, 0, 0, 0, 58, 0, 1, 0, 0, 0, 0, 0, 29, 0, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2218, 5415, 0, 0, 0, 0, 0, 0, 47, 2, 10, 1, 0, 17, 3748, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 4698, 0, 0, 0, 21, 2693, 5, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6318, 63, 0, 65, 0, 0, 0, 0, 0, 0, 1866, 3367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1233, 0, 0, 3240, 0, 0, 0, 0, 0, 0, 0, 4, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8767, 6812, 5206, 6143, 12805, 9535, 11679, 4589] avg=8192.0000 max=12805 max_rank=4 min=4589 min_rank=7 imbalance_ratio=1.563110 max_load_reduction=52731 max_load_reduction_pct=0.804611 moved_to_echo=38608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 3, 13, 5, 21, 5226, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 9, 170, 0, 0, 0, 0, 0, 3, 7269, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 1, 24, 1253, 1187, 2, 0, 0, 7365, 37, 24, 0, 10, 194, 2634, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 498, 0, 0, 0, 0, 128, 2, 0, 0, 0, 240, 0, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 3404, 1158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 8, 0, 902, 4, 0, 0, 0, 2957, 0, 1, 33, 1796, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 40, 0, 242, 0, 28, 0, 0, 39, 0, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 5891, 0, 0, 4541, 0, 183, 0, 0, 66, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6375, 10825, 21767, 2063, 4588, 5771, 3423, 10724] avg=8192.0000 max=21767 max_rank=2 min=2063 min_rank=3 imbalance_ratio=2.657104 max_load_reduction=43769 max_load_reduction_pct=0.667862 moved_to_echo=24506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[16256, 128, 256, 1152, 128, 5120, 1664, 128, 0, 128, 11392, 128, 128, 256, 128, 2816, 19072, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 9344, 0, 128, 0, 128, 128, 128, 0, 128, 1152, 512, 128, 10240, 39936, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 11648, 128, 7168, 384, 128, 128, 128, 256, 768, 128, 13184, 256, 24320, 128, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -1506,244 +1506,244 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 24448, 128, 0, 31232, 0, 384, 128, 896, 8576, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 20736, 7168, 128, 128, 7552, 128, 128, 128, 1152, 128, 8320, 20864, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 384, 2176, 14208, 128, 128, 128, 128, 22528, 128, 128, 128, 9984, 128, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 256, 128, 6528, 256, 7808, 128, 256, 0, 128, 128, 16768, 24064, 3200, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 256, 128, 6528, 256, 7808, 128, 256, 0, 128, 128, 16768, 24064, 3200, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 2041, 0, 0, 1, 0, 0, 0, 3, 4570, 0, 0, 0, 9, 0, 0, 0, 0, 4996, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 6482, 0, 0, 18, 0, 708, 0, 129, 0, 0, 0, 0, 0, 0, 8172, 0, 372, 10, 1, 0, 1603, 46, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 23, 0, 0, 7575, 0, 0, 0, 222, 0, 0, 0, 6544, 0, 0, 0, 0, 0, 7, 0, 4, 0, 0, 57, 5, 9, 0, 0, 5129, 0, 0, 7784, 1266, 0, 0, 0, 7205, 0, 0, 0, 0, 12, 0, 0, 2, 0, 0, 1, 0, 0, 16, 55, 0, 15, 0, 379, 0, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 153, 1, 0, 8019, 0, 2552, 1715, 0, 0, 236, 0, 0, 4486, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2, 33, 0, 0, 282, 4, 14, 0, 0, 7845, 0, 0, 0, 145, 3, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 63, 0, 1, 0, 0, 7900, 0, 0, 209, 0, 43, 1, 0, 1785, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 31, 305, 4626, 1, 20, 0, 3492, 0, 0, 48, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 6367, 10, 0, 0, 0, 0, 0, 0, 7177, 0, 7837] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 0, 0, 941, 0, 0, 5107, 0, 0, 0, 26, 0, 51, 1, 0, 8046, 2, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 357, 96, 0, 0, 0, 0, 0, 41, 197, 0, 0, 1103, 0, 0, 0, 6455, 0, 20, 0, 0, 0, 2391, 29, 0, 0, 0, 0, 12, 18, 237, 209, 0, 0, 0, 0, 0, 0, 92, 0, 0, 7955, 0, 149, 3, 0, 2, 0, 0, 8187, 0, 0, 0, 0, 1, 1080, 0, 0, 0, 2, 137, 0, 0, 0, 0, 59, 0, 0, 2701, 41, 37, 0, 952, 7614, 0, 83, 0, 59, 0, 0, 0, 7, 7, 0, 2351, 0, 0, 6980, 172, 0, 1, 0, 0, 0, 1498, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 7640, 0, 0, 0, 0, 0, 497, 0, 7190, 0, 0, 10, 9, 0, 0, 2, 0, 65, 0, 1, 0, 6606, 0, 0, 0, 2, 0, 0, 4, 0, 0, 6773, 0, 0, 3, 0, 8, 0, 1, 0, 0, 21, 0, 0, 0, 7997, 154, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 415, 0, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 14, 9, 6, 1, 8, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4240, 0, 0, 0, 0, 46, 0, 0, 0, 6230, 0, 0, 307, 0, 0, 0, 614, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 131] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 7969, 0, 1333, 0, 2407, 5, 0, 0, 0, 0, 2082, 0, 0, 0, 3, 0, 0, 20, 0, 0, 283, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2778, 1, 0, 0, 0, 6, 57, 0, 0, 0, 2, 7941, 22, 0, 725, 0, 331, 0, 70, 6321, 0, 6869, 3, 2, 0, 7, 0, 0, 58, 2605, 3203, 0, 90, 0, 20, 0, 0, 0, 0, 659, 0, 1, 8176, 5317, 0, 0, 0, 34, 0, 1, 0, 0, 984, 0, 0, 630, 0, 0, 0, 5, 0, 46, 0, 0, 0, 4236, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 0, 0, 941, 0, 0, 5107, 0, 0, 0, 26, 0, 51, 1, 0, 8046, 2, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 357, 96, 0, 0, 0, 0, 0, 41, 197, 0, 0, 1103, 0, 0, 0, 6455, 0, 20, 0, 0, 0, 2391, 29, 0, 0, 0, 0, 12, 18, 237, 209, 0, 0, 0, 0, 0, 0, 92, 0, 0, 7955, 0, 149, 3, 0, 2, 0, 0, 8187, 0, 0, 0, 0, 1, 1080, 0, 0, 0, 2, 137, 0, 0, 0, 0, 59, 0, 0, 2701, 41, 37, 0, 952, 7614, 0, 83, 0, 59, 0, 0, 0, 7, 7, 0, 2351, 0, 0, 6980, 172, 0, 1, 0, 0, 0, 1498, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 275, 2136, 7747, 0, 1, 15, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 16, 33, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 7630, 2, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 12, 0, 8, 1162, 0, 0, 0, 0, 0, 1612, 0, 0, 215, 0, 8153, 521, 2370, 3, 0, 0, 0, 0, 6466, 0, 668, 0, 0, 0, 0, 2, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 7018, 0, 6, 0, 326, 6255, 0, 2, 0, 55, 0, 0, 0, 0, 192, 0, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 70, 7112, 0, 0, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 736, 2197, 0, 0, 0, 2434, 591, 0, 3487, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 2, 1, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7870, 0, 0, 0, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 4, 0, 515, 0, 0, 0, 1, 8093, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 9, 0, 0, 0, 8167, 1, 0, 0, 0, 2, 0, 0, 97, 0, 0, 7, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 133, 0, 0, 75, 0, 0, 0, 3126, 0, 1827, 0, 0, 0, 8128, 0, 2, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 41, 0, 0, 17, 0, 0, 8180, 0, 0, 5994, 0, 0, 0, 0, 47, 0, 2244, 0, 0, 0, 0, 0, 2, 0, 0, 4666, 0, 0, 0, 0, 0, 0, 0, 364, 1236, 0, 0, 1043, 7908, 0, 0, 0, 6495, 0, 0, 0, 0, 0, 1, 0, 5633, 0, 1, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 275, 2136, 7747, 0, 1, 15, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 16, 33, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 7630, 2, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 12, 0, 8, 1162, 0, 0, 0, 0, 0, 1612, 0, 0, 215, 0, 8153, 521, 2370, 3, 0, 0, 0, 0, 6466, 0, 668, 0, 0, 0, 0, 2, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 7018, 0, 6, 0, 326, 6255, 0, 2, 0, 55, 0, 0, 0, 0, 192, 0, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2041, 0, 0, 1, 0, 0, 0, 3, 2462, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4996, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 4723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6482, 0, 0, 18, 0, 708, 0, 129, 0, 0, 0, 0, 0, 0, 6606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 10, 1, 0, 1603, 46, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4167, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 23, 0, 0, 1824, 0, 0, 0, 222, 0, 0, 0, 6544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 0, 57, 5, 9, 0, 0, 5129, 1746, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 1266, 0, 0, 0, 960, 0, 0, 0, 0, 12, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 55, 0, 15, 0, 379, 0, 0, 0, 0, 3, 1566, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4516, 9728, 13943, 7264, 8629, 9035, 10024, 2397] avg=8192.0000 max=13943 max_rank=2 min=2397 min_rank=7 imbalance_ratio=1.702026 max_load_reduction=51593 max_load_reduction_pct=0.787247 moved_to_echo=30085 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 153, 1, 0, 8019, 0, 2552, 1715, 0, 0, 236, 0, 0, 4486, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2, 33, 0, 0, 282, 4, 14, 0, 0, 7845, 0, 0, 0, 145, 3, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 63, 0, 1, 0, 0, 7900, 0, 0, 209, 0, 43, 1, 0, 1785, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 31, 305, 4626, 1, 20, 0, 3492, 0, 0, 48, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 6367, 10, 0, 0, 0, 0, 0, 0, 7177, 0, 7837] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 7640, 0, 0, 0, 0, 0, 497, 0, 7190, 0, 0, 10, 9, 0, 0, 2, 0, 65, 0, 1, 0, 6606, 0, 0, 0, 2, 0, 0, 4, 0, 0, 6773, 0, 0, 3, 0, 8, 0, 1, 0, 0, 21, 0, 0, 0, 7997, 154, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 415, 0, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 14, 9, 6, 1, 8, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4240, 0, 0, 0, 0, 46, 0, 0, 0, 6230, 0, 0, 307, 0, 0, 0, 614, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 131] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 70, 7112, 0, 0, 0, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 736, 4907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 2434, 591, 0, 3487, 0, 0, 0, 0, 0, 0, 6578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 2, 0, 0, 4681, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 0, 0, 0, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 1235, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 515, 0, 0, 0, 1, 1077, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796, 1564, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8924, 5711, 15287, 13946, 9642, 3631, 1779, 6616] avg=8192.0000 max=15287 max_rank=2 min=1779 min_rank=6 imbalance_ratio=1.866089 max_load_reduction=50249 max_load_reduction_pct=0.766739 moved_to_echo=21668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 0, 0, 941, 0, 0, 2751, 0, 0, 0, 26, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8046, 2, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 357, 4960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 41, 197, 0, 0, 1103, 0, 0, 0, 5218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 2391, 29, 0, 0, 0, 0, 12, 18, 237, 209, 0, 0, 4404, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 1916, 0, 149, 3, 0, 2, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1080, 0, 0, 0, 2, 137, 0, 0, 0, 0, 59, 1951, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2701, 41, 37, 0, 952, 1014, 0, 83, 0, 59, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2351, 0, 0, 6980, 172, 0, 1, 0, 0, 0, 1498, 0, 0, 0, 1237, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3791, 13371, 6655, 8399, 10349, 5426, 4894, 12651] avg=8192.0000 max=13371 max_rank=1 min=3791 min_rank=0 imbalance_ratio=1.632202 max_load_reduction=52165 max_load_reduction_pct=0.795975 moved_to_echo=33320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 153, 1, 0, 4320, 0, 2552, 1715, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4486, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2, 33, 0, 0, 282, 4, 14, 0, 0, 6341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 3, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2019, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 1, 0, 0, 1903, 0, 0, 209, 0, 43, 1, 0, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3063, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 305, 4626, 1, 20, 0, 466, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 6367, 10, 0, 0, 0, 0, 0, 0, 7177, 0, 7837, 1504, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8981, 9413, 6716, 3252, 4005, 4075, 5497, 23597] avg=8192.0000 max=23597 max_rank=7 min=3252 min_rank=3 imbalance_ratio=2.880493 max_load_reduction=41939 max_load_reduction_pct=0.639938 moved_to_echo=37174 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 9, 0, 0, 0, 4399, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 7, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 133, 0, 0, 75, 0, 0, 0, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 8128, 0, 2, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4574, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 17, 0, 0, 1970, 0, 0, 5994, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2244, 0, 0, 0, 0, 0, 2, 0, 0, 4666, 0, 0, 0, 0, 0, 3120, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 1236, 0, 0, 1043, 1053, 0, 0, 0, 6495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5633, 0, 1, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 599, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4413, 5213, 2738, 15686, 8069, 12313, 10191, 6913] avg=8192.0000 max=15686 max_rank=3 min=2738 min_rank=2 imbalance_ratio=1.914795 max_load_reduction=49850 max_load_reduction_pct=0.760651 moved_to_echo=37486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 275, 2136, 7747, 0, 1, 9, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 33, 0, 0, 0, 0, 0, 0, 0, 0, 5083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 6167, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 12, 0, 8, 1162, 0, 0, 0, 3618, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1612, 0, 0, 215, 0, 1964, 521, 2370, 3, 0, 0, 0, 0, 6466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 2, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 5, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7018, 0, 6, 0, 326, 833, 0, 2, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10200, 5132, 6553, 8524, 13151, 8705, 8240, 5031] avg=8192.0000 max=13151 max_rank=4 min=5031 min_rank=7 imbalance_ratio=1.605347 max_load_reduction=52385 max_load_reduction_pct=0.799332 moved_to_echo=35127 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2041, 0, 0, 1, 0, 0, 0, 3, 2462, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4996, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 4723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6482, 0, 0, 18, 0, 708, 0, 129, 0, 0, 0, 0, 0, 0, 6606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 10, 1, 0, 1603, 46, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4167, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 23, 0, 0, 1824, 0, 0, 0, 222, 0, 0, 0, 6544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 0, 57, 5, 9, 0, 0, 5129, 1746, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 1266, 0, 0, 0, 960, 0, 0, 0, 0, 12, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 55, 0, 15, 0, 379, 0, 0, 0, 0, 3, 1566, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4516, 9728, 13943, 7264, 8629, 9035, 10024, 2397] avg=8192.0000 max=13943 max_rank=2 min=2397 min_rank=7 imbalance_ratio=1.702026 max_load_reduction=51593 max_load_reduction_pct=0.787247 moved_to_echo=30085 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 7969, 0, 1333, 0, 1290, 5, 0, 0, 0, 0, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 20, 0, 0, 283, 0, 1, 0, 0, 3, 0, 4954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2778, 1, 0, 0, 3078, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 57, 0, 0, 0, 2, 1905, 22, 0, 725, 0, 331, 0, 70, 6321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6869, 3, 2, 0, 7, 0, 0, 58, 2605, 3203, 0, 90, 0, 20, 0, 923, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 1, 8176, 701, 0, 0, 0, 34, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 0, 0, 630, 0, 0, 0, 5, 0, 46, 0, 0, 0, 4236, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12682, 5264, 220, 6946, 9439, 15318, 9572, 6095] avg=8192.0000 max=15318 max_rank=5 min=220 min_rank=2 imbalance_ratio=1.869873 max_load_reduction=50218 max_load_reduction_pct=0.766266 moved_to_echo=40424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 70, 7112, 0, 0, 0, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 736, 4907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 2434, 591, 0, 3487, 0, 0, 0, 0, 0, 0, 6578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 2, 0, 0, 4681, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 0, 0, 0, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 1235, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 515, 0, 0, 0, 1, 1077, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796, 1564, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8924, 5711, 15287, 13946, 9642, 3631, 1779, 6616] avg=8192.0000 max=15287 max_rank=2 min=1779 min_rank=6 imbalance_ratio=1.866089 max_load_reduction=50249 max_load_reduction_pct=0.766739 moved_to_echo=21668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7640, 0, 0, 0, 0, 0, 497, 0, 3873, 0, 0, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 0, 1, 0, 6606, 0, 0, 0, 2, 0, 0, 4, 0, 0, 5024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6773, 0, 0, 3, 0, 8, 0, 1, 0, 0, 21, 0, 0, 0, 6464, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 415, 0, 5, 0, 0, 2, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 14, 9, 6, 1, 8, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4240, 0, 0, 0, 0, 46, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6230, 0, 0, 307, 0, 0, 0, 614, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 131, 1533, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12029, 11704, 13424, 1519, 2252, 7033, 15339, 2236] avg=8192.0000 max=15339 max_rank=6 min=1519 min_rank=3 imbalance_ratio=1.872437 max_load_reduction=50197 max_load_reduction_pct=0.765945 moved_to_echo=26860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 275, 2136, 7747, 0, 1, 9, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 33, 0, 0, 0, 0, 0, 0, 0, 0, 5083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 6167, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 12, 0, 8, 1162, 0, 0, 0, 3618, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1612, 0, 0, 215, 0, 1964, 521, 2370, 3, 0, 0, 0, 0, 6466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 2, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 5, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7018, 0, 6, 0, 326, 833, 0, 2, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10200, 5132, 6553, 8524, 13151, 8705, 8240, 5031] avg=8192.0000 max=13151 max_rank=4 min=5031 min_rank=7 imbalance_ratio=1.605347 max_load_reduction=52385 max_load_reduction_pct=0.799332 moved_to_echo=35127 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 9, 0, 0, 0, 4399, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 7, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 133, 0, 0, 75, 0, 0, 0, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 8128, 0, 2, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4574, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 17, 0, 0, 1970, 0, 0, 5994, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2244, 0, 0, 0, 0, 0, 2, 0, 0, 4666, 0, 0, 0, 0, 0, 3120, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 1236, 0, 0, 1043, 1053, 0, 0, 0, 6495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5633, 0, 1, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 599, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4413, 5213, 2738, 15686, 8069, 12313, 10191, 6913] avg=8192.0000 max=15686 max_rank=3 min=2738 min_rank=2 imbalance_ratio=1.914795 max_load_reduction=49850 max_load_reduction_pct=0.760651 moved_to_echo=37486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 153, 1, 0, 4320, 0, 2552, 1715, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4486, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2, 33, 0, 0, 282, 4, 14, 0, 0, 6341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 3, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2019, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 1, 0, 0, 1903, 0, 0, 209, 0, 43, 1, 0, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3063, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 305, 4626, 1, 20, 0, 466, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 6367, 10, 0, 0, 0, 0, 0, 0, 7177, 0, 7837, 1504, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8981, 9413, 6716, 3252, 4005, 4075, 5497, 23597] avg=8192.0000 max=23597 max_rank=7 min=3252 min_rank=3 imbalance_ratio=2.880493 max_load_reduction=41939 max_load_reduction_pct=0.639938 moved_to_echo=37174 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 128, 2432, 6272, 0, 16768, 256, 128, 128, 128, 384, 0, 1536, 11520, 0, 12800, 9472, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[2304, 128, 256, 128, 12160, 128, 2688, 8320, 0, 128, 512, 128, 4224, 384, 128, 128, 26624, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[16000, 0, 0, 128, 2560, 1408, 128, 4096, 0, 512, 1280, 128, 0, 0, 39936, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 9856, 128, 128, 128, 1152, 128, 128, 128, 2688, 18560, 128, 128, 0, 128, 5248, 14848, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 18688, 14080, 128, 128, 10880, 6144, 0, 128, 896, 14848, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 17664, 128, 128, 0, 6656, 128, 128, 384, 128, 128, 128, 128, 128, 1152, 39680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 18688, 14080, 128, 128, 10880, 6144, 0, 128, 896, 14848, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 9728, 128, 128, 384, 17280, 8960, 1920, 128, 20864, 128, 2560, 1792, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 1792, 0, 256, 256, 128, 15360, 640, 2560, 7040, 8064, 384, 128, 128, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 336, 0, 0, 8189, 0, 0, 0, 103, 7233, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4500, 0, 0, 1955, 0, 10, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 39, 136, 8189, 0, 29, 0, 0, 0, 0, 0, 0, 777, 0, 6732, 6426, 0, 8081, 0, 0, 33, 571, 0, 7595, 0, 144, 0, 435, 0, 1, 0, 0, 38, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1360, 1692] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 24, 0, 0, 8101, 0, 0, 0, 0, 0, 5, 0, 0, 0, 190, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 3, 0, 3, 8169, 0, 0, 4169, 0, 1, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 20, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 53, 0, 0, 0, 0, 394, 0, 23, 15, 0, 0, 52, 7726, 0, 1372, 0, 0, 0, 0, 72, 5992, 0, 0, 20, 0, 0, 0, 0, 0, 0, 7579, 0, 0, 43, 0, 715, 0, 0, 0, 4857, 5089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4814, 0, 1, 26, 2, 6, 11, 0, 318, 0, 4, 0, 0, 0, 2133, 0, 0, 2, 0, 0, 1, 0, 0, 2, 0, 17, 0, 0, 0, 228, 0, 0, 2, 0, 1, 0, 0, 0, 7825, 81, 0, 0, 0, 189, 0, 0, 7553, 0, 0, 0, 360, 0, 112, 0, 0, 7980, 0, 0, 0, 5, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 2, 15, 269, 8133, 0, 8155, 0, 0, 66, 0, 0, 5017, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 2508, 0, 11, 0, 6964, 1273] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 336, 0, 0, 8189, 0, 0, 0, 103, 7233, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4500, 0, 0, 1955, 0, 10, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 39, 136, 8189, 0, 29, 0, 0, 0, 0, 0, 0, 777, 0, 6732, 6426, 0, 8081, 0, 0, 33, 571, 0, 7595, 0, 144, 0, 435, 0, 1, 0, 0, 38, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1360, 1692] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 3, 1019, 7, 0, 0, 134, 0, 0, 0, 6138, 0, 0, 0, 86, 6603, 3, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 3, 0, 1, 1, 0, 301, 0, 0, 0, 0, 0, 0, 0, 5, 40, 0, 0, 0, 0, 4865, 0, 133, 0, 8095, 38, 0, 49, 0, 7954, 19, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 3, 21, 473, 0, 0, 3892, 0, 25, 0, 6494, 0, 1, 29, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 2613, 0, 0, 105, 0, 0, 3653, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 32, 3, 0, 4032, 121, 2, 7, 7561, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 1, 0, 0, 740, 20, 15, 0, 0, 0, 0, 0, 1, 0, 0, 10, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 8147, 7839, 0, 8191, 0, 1, 131, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8165, 12, 0, 1875, 0, 3, 157, 0, 1546, 8017, 0, 3, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 777, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[70, 1, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 13, 0, 9, 7961, 0, 1, 0, 5534, 0, 0, 0, 0, 0, 2, 0, 0, 0, 12, 2, 0, 0, 14, 0, 77, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1175, 4730, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 3509, 0, 1, 7548, 0, 0, 0, 1, 0, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8142, 0, 0, 0, 0, 40, 0, 0, 1, 137, 0, 1315, 10, 1, 77, 0, 0, 1193, 462, 0, 7681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 91, 0, 0, 0, 7771, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 1139, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 13, 0, 7141, 0, 0, 166, 0, 2296, 0, 0, 0, 0, 221, 0, 1, 0, 0, 14, 0, 694, 8, 36, 0, 465, 0, 0, 0, 5, 3, 0, 0, 106, 81, 0, 0, 0, 0, 0, 45, 178, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3592, 13, 0, 0, 0, 119, 130, 0, 0, 4268, 0, 0, 583, 0, 0, 0, 0, 0, 350, 0, 0, 10, 5920, 0, 7394, 0, 0, 0, 0, 13, 47, 2, 0, 0, 0, 0, 398, 0, 8043, 0, 0, 2643, 7031, 1, 17, 0, 0, 3583, 0, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 0, 7, 0, 0, 754, 7947] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 1, 0, 0, 740, 20, 15, 0, 0, 0, 0, 0, 1, 0, 0, 10, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 8147, 7839, 0, 8191, 0, 1, 131, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8165, 12, 0, 1875, 0, 3, 157, 0, 1546, 8017, 0, 3, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 777, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4814, 0, 1, 26, 2, 6, 11, 0, 318, 0, 4, 0, 0, 0, 2133, 0, 0, 2, 0, 0, 1, 0, 0, 2, 0, 17, 0, 0, 0, 228, 0, 0, 2, 0, 1, 0, 0, 0, 7825, 81, 0, 0, 0, 189, 0, 0, 7553, 0, 0, 0, 360, 0, 112, 0, 0, 7980, 0, 0, 0, 5, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 2, 15, 269, 8133, 0, 8155, 0, 0, 66, 0, 0, 5017, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 2508, 0, 11, 0, 6964, 1273] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1815, 0, 0, 0, 0, 3, 0, 0, 7854, 17, 1, 0, 501, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 125, 0, 0, 0, 0, 183, 0, 0, 0, 0, 16, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 16, 790, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4586, 0, 0, 0, 0, 3, 0, 807, 0, 0, 0, 0, 4733, 0, 4135, 0, 0, 0, 0, 0, 0, 3, 0, 8056, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 8159, 0, 0, 0, 7065, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 3, 1019, 7, 0, 0, 134, 0, 0, 0, 6138, 0, 0, 0, 86, 6603, 3, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 3, 0, 1, 1, 0, 301, 0, 0, 0, 0, 0, 0, 0, 5, 40, 0, 0, 0, 0, 4865, 0, 133, 0, 8095, 38, 0, 49, 0, 7954, 19, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 3, 21, 473, 0, 0, 3892, 0, 25, 0, 6494, 0, 1, 29, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 2613, 0, 0, 105, 0, 0, 3653, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 32, 3, 0, 4032, 121, 2, 7, 7561, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 336, 0, 0, 6740, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6584, 0, 0, 0, 103, 7233, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 847, 0, 6654, 418, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4500, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1517, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 39, 136, 8189, 0, 29, 0, 0, 0, 0, 0, 0, 2467, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 0, 6732, 3941, 0, 0, 0, 0, 33, 571, 0, 0, 0, 144, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 38, 0, 13, 0, 0, 0, 0, 0, 0, 0, 942, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8544, 13926, 7938, 4955, 2386, 12468, 12633, 2686] avg=8192.0000 max=13926 max_rank=1 min=2386 min_rank=4 imbalance_ratio=1.699951 max_load_reduction=51610 max_load_reduction_pct=0.787506 moved_to_echo=30173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 24, 0, 0, 8101, 0, 0, 0, 0, 0, 5, 0, 5317, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1130, 1494, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 3, 0, 3, 8169, 0, 0, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 20, 0, 0, 0, 0, 3235, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 53, 0, 0, 0, 0, 394, 0, 23, 15, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4737, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7579, 0, 0, 43, 0, 715, 0, 0, 0, 3363, 5089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13743, 2878, 2694, 9213, 11879, 3459, 4881, 16789] avg=8192.0000 max=16789 max_rank=7 min=2694 min_rank=2 imbalance_ratio=2.049438 max_load_reduction=48747 max_load_reduction_pct=0.743820 moved_to_echo=37008 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4814, 0, 1, 26, 2, 6, 11, 0, 4452, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 4, 0, 0, 0, 2133, 0, 0, 2, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 228, 0, 0, 2, 0, 1, 0, 0, 0, 7825, 81, 6714, 2142, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 7553, 0, 0, 0, 360, 0, 112, 0, 0, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 6192, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 3123, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 269, 4986, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 2508, 0, 11, 0, 4822, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10753, 2398, 17034, 10002, 7559, 3524, 5530, 8736] avg=8192.0000 max=17034 max_rank=2 min=2398 min_rank=1 imbalance_ratio=2.079346 max_load_reduction=48502 max_load_reduction_pct=0.740082 moved_to_echo=25349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1139, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 13, 0, 16, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5739, 0, 0, 166, 0, 2296, 0, 0, 0, 0, 221, 0, 1, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 8, 36, 0, 465, 0, 0, 0, 5, 3, 0, 0, 106, 81, 0, 6621, 234, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 178, 0, 0, 0, 0, 0, 0, 0, 1, 0, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 119, 130, 0, 0, 4268, 0, 0, 583, 0, 0, 0, 0, 2789, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 10, 5920, 0, 7394, 0, 0, 0, 0, 13, 47, 2, 0, 157, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 2643, 7031, 1, 0, 0, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 0, 7, 0, 0, 520, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2599, 8437, 8257, 1027, 7903, 15295, 13495, 8523] avg=8192.0000 max=15295 max_rank=5 min=1027 min_rank=3 imbalance_ratio=1.867065 max_load_reduction=50241 max_load_reduction_pct=0.766617 moved_to_echo=45216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 7116, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 1, 0, 0, 740, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 8147, 7839, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 131, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 6356, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 8, 0, 0, 0, 3, 157, 0, 1546, 0, 0, 3, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 777, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15522, 7024, 1881, 17962, 7391, 164, 14808, 784] avg=8192.0000 max=17962 max_rank=3 min=164 min_rank=5 imbalance_ratio=2.192627 max_load_reduction=47574 max_load_reduction_pct=0.725922 moved_to_echo=23147 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 24, 0, 0, 8101, 0, 0, 0, 0, 0, 5, 0, 5317, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1130, 1494, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 3, 0, 3, 8169, 0, 0, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 20, 0, 0, 0, 0, 3235, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 53, 0, 0, 0, 0, 394, 0, 23, 15, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4737, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7579, 0, 0, 43, 0, 715, 0, 0, 0, 3363, 5089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13743, 2878, 2694, 9213, 11879, 3459, 4881, 16789] avg=8192.0000 max=16789 max_rank=7 min=2694 min_rank=2 imbalance_ratio=2.049438 max_load_reduction=48747 max_load_reduction_pct=0.743820 moved_to_echo=37008 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1815, 0, 0, 0, 0, 3, 0, 0, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6315, 17, 1, 0, 501, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 3405, 2173, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 183, 0, 0, 0, 0, 16, 0, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 16, 790, 0, 0, 0, 0, 0, 0, 0, 6357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4586, 0, 0, 0, 0, 3, 0, 807, 0, 1817, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 8159, 0, 0, 0, 4892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2549, 7125, 13740, 2159, 7167, 8754, 10962, 13080] avg=8192.0000 max=13740 max_rank=2 min=2159 min_rank=3 imbalance_ratio=1.677246 max_load_reduction=51796 max_load_reduction_pct=0.790344 moved_to_echo=39963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 3, 1019, 7, 0, 0, 134, 0, 0, 0, 3243, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4935, 0, 0, 0, 86, 6603, 3, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 3, 0, 1, 1, 0, 301, 0, 0, 0, 0, 0, 0, 0, 2151, 2325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 40, 0, 0, 0, 0, 4865, 0, 133, 0, 8095, 38, 0, 49, 0, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 3, 21, 6172, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 0, 0, 3892, 0, 25, 0, 6494, 0, 1, 29, 0, 0, 0, 0, 0, 1, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 32, 3, 0, 4032, 121, 2, 7, 5236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4874, 11796, 5539, 15007, 6634, 12118, 112, 9456] avg=8192.0000 max=15007 max_rank=3 min=112 min_rank=6 imbalance_ratio=1.831909 max_load_reduction=50529 max_load_reduction_pct=0.771011 moved_to_echo=28320 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[70, 1, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 13, 0, 9, 6816, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6401, 0, 1, 0, 5534, 0, 0, 0, 0, 0, 2, 0, 0, 0, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 77, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1175, 4730, 63, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 3509, 0, 1, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 5857, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8142, 0, 0, 0, 0, 40, 0, 0, 1, 3, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 1315, 7, 1, 0, 0, 0, 1193, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 91, 0, 0, 0, 5381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6952, 11952, 8453, 5211, 14617, 9754, 3115, 5482] avg=8192.0000 max=14617 max_rank=4 min=3115 min_rank=6 imbalance_ratio=1.784302 max_load_reduction=50919 max_load_reduction_pct=0.776962 moved_to_echo=32968 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1815, 0, 0, 0, 0, 3, 0, 0, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6315, 17, 1, 0, 501, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 3405, 2173, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 183, 0, 0, 0, 0, 16, 0, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 16, 790, 0, 0, 0, 0, 0, 0, 0, 6357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4586, 0, 0, 0, 0, 3, 0, 807, 0, 1817, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 8159, 0, 0, 0, 4892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2549, 7125, 13740, 2159, 7167, 8754, 10962, 13080] avg=8192.0000 max=13740 max_rank=2 min=2159 min_rank=3 imbalance_ratio=1.677246 max_load_reduction=51796 max_load_reduction_pct=0.790344 moved_to_echo=39963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 7116, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 1, 0, 0, 740, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 8147, 7839, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 131, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 6356, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 8, 0, 0, 0, 3, 157, 0, 1546, 0, 0, 3, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 777, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15522, 7024, 1881, 17962, 7391, 164, 14808, 784] avg=8192.0000 max=17962 max_rank=3 min=164 min_rank=5 imbalance_ratio=2.192627 max_load_reduction=47574 max_load_reduction_pct=0.725922 moved_to_echo=23147 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1139, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 13, 0, 16, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5739, 0, 0, 166, 0, 2296, 0, 0, 0, 0, 221, 0, 1, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 8, 36, 0, 465, 0, 0, 0, 5, 3, 0, 0, 106, 81, 0, 6621, 234, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 178, 0, 0, 0, 0, 0, 0, 0, 1, 0, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 119, 130, 0, 0, 4268, 0, 0, 583, 0, 0, 0, 0, 2789, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 10, 5920, 0, 7394, 0, 0, 0, 0, 13, 47, 2, 0, 157, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 2643, 7031, 1, 0, 0, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 0, 7, 0, 0, 520, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2599, 8437, 8257, 1027, 7903, 15295, 13495, 8523] avg=8192.0000 max=15295 max_rank=5 min=1027 min_rank=3 imbalance_ratio=1.867065 max_load_reduction=50241 max_load_reduction_pct=0.766617 moved_to_echo=45216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 1536, 128, 128, 128, 1024, 128, 0, 384, 128, 128, 0, 0, 128, 18048, 4864, 28288, 11264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 256, 256, 128, 256, 12544, 256, 256, 256, 8576, 128, 24576, 7936, 128, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 1536, 128, 128, 128, 1024, 128, 0, 384, 128, 128, 0, 0, 128, 18048, 4864, 28288, 11264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 0, 128, 7680, 128, 128, 128, 128, 16384, 128, 128, 128, 25216, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1024, 128, 16640, 16896, 128, 0, 0, 128, 4224, 8064, 1664, 0, 128, 256, 16896, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[512, 384, 128, 3968, 384, 6144, 256, 34944, 0, 128, 128, 0, 512, 128, 896, 128, 10624, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 1152, 128, 0, 0, 128, 0, 128, 23936, 128, 128, 128, 256, 384, 128, 128, 33792, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[30336, 128, 256, 256, 6272, 25472, 2176, 256, 0, 768, 256, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 256, 128, 128, 256, 0, 896, 20992, 128, 0, 640, 128, 0, 128, 128, 38528, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 2, 119, 0, 0, 0, 0, 0, 3, 0, 0, 4322, 0, 0, 3526, 1, 0, 0, 0, 1, 0, 147, 2, 141, 0, 0, 0, 1219, 1445, 0, 0, 0, 0, 7774, 0, 0, 193, 12, 0, 0, 1, 3166, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4157, 0, 76, 7919, 0, 0, 0, 0, 0, 0, 5, 27, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1372, 0, 0, 0, 0, 2, 0, 3730, 406, 8155, 0, 0, 0, 0, 0, 0, 1, 675, 0, 0, 0, 81, 0, 2891, 0, 0, 8143, 0, 458, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 34, 1302, 0, 17, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 28, 0, 0, 0, 0, 7809, 0, 0, 0, 7005, 2, 0, 0, 0, 0, 114, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 6806, 0, 0, 642, 0, 0, 0, 0, 43, 121, 0, 0, 662, 0, 0, 0, 0, 8132, 0, 0, 7534, 3639, 0, 0, 1236, 0, 0, 2, 0, 0, 0, 172, 0, 0, 0, 0, 0, 4325, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 3, 8191] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 450, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 4, 0, 0, 5922, 0, 0, 0, 0, 0, 0, 5402, 0, 0, 8, 3749, 0, 0, 0, 244, 0, 7824, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 1, 3, 0, 0, 0, 7, 0, 1, 2, 1, 0, 0, 0, 1, 0, 2, 4754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 6, 0, 8176, 56, 0, 1, 5, 0, 11, 35, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8158, 0, 0, 7, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 1150, 3, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 154, 0, 0, 0, 51, 0, 0, 0, 1, 0, 0, 0, 6945, 0, 0, 0, 0, 6896, 0, 0, 633, 65, 0, 0, 3, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4721, 2, 7, 17, 0, 0, 0, 6249, 1, 0, 0, 0, 0, 0, 0, 0, 1012, 7281, 0, 0, 0, 0, 2, 0, 0, 0, 39, 0, 0, 0, 152, 4589, 73, 1381, 0, 0, 0, 11, 0, 0, 0, 0, 0, 7651, 0, 0, 0, 0, 8147, 0, 0, 51, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4, 8125, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1, 3306, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8091, 0, 1, 14, 0, 4596, 2, 1123, 0, 0, 0, 0, 0, 18, 0, 46, 1, 1479, 68, 0, 0, 0, 0, 1, 0, 10, 7994, 68, 0, 0, 0, 0, 7564, 853, 0, 0, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5, 7023, 0, 0, 309, 0, 0, 0, 72, 452, 0, 0, 0, 3, 0, 6928, 0, 8152, 0, 0, 24, 0, 0, 0, 0, 0, 96, 0, 0, 0, 6, 81, 0, 0, 0, 1, 2033, 0, 0, 20, 0, 36, 0, 0, 0, 0, 0, 0, 15, 22, 0, 0, 0, 4901, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 1150, 3, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 154, 0, 0, 0, 51, 0, 0, 0, 1, 0, 0, 0, 6945, 0, 0, 0, 0, 6896, 0, 0, 633, 65, 0, 0, 3, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4721, 2, 7, 17, 0, 0, 0, 6249, 1, 0, 0, 0, 0, 0, 0, 0, 1012, 7281, 0, 0, 0, 0, 2, 0, 0, 0, 39, 0, 0, 0, 152, 4589, 73, 1381, 0, 0, 0, 11, 0, 0, 0, 0, 0, 7651, 0, 0, 0, 0, 8147, 0, 0, 51, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4, 8125, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 34, 1302, 0, 17, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 28, 0, 0, 0, 0, 7809, 0, 0, 0, 7005, 2, 0, 0, 0, 0, 114, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 6806, 0, 0, 642, 0, 0, 0, 0, 43, 121, 0, 0, 662, 0, 0, 0, 0, 8132, 0, 0, 7534, 3639, 0, 0, 1236, 0, 0, 2, 0, 0, 0, 172, 0, 0, 0, 0, 0, 4325, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 3, 8191] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 2, 119, 0, 0, 0, 0, 0, 3, 0, 0, 4322, 0, 0, 3526, 1, 0, 0, 0, 1, 0, 147, 2, 141, 0, 0, 0, 1219, 1445, 0, 0, 0, 0, 7774, 0, 0, 193, 12, 0, 0, 1, 3166, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4157, 0, 76, 7919, 0, 0, 0, 0, 0, 0, 5, 27, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1372, 0, 0, 0, 0, 2, 0, 3730, 406, 8155, 0, 0, 0, 0, 0, 0, 1, 675, 0, 0, 0, 81, 0, 2891, 0, 0, 8143, 0, 458, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 443, 0, 4, 0, 0, 0, 0, 0, 7941, 0, 0, 17, 0, 6, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8186, 0, 0, 1233, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2, 7446, 0, 0, 0, 0, 17, 0, 0, 8182, 0, 0, 1, 3, 0, 0, 0, 297, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 0, 0, 0, 7, 0, 0, 0, 0, 6671, 5, 0, 0, 0, 0, 4150, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 0, 4393, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 6289, 14, 0, 0, 0, 0, 8155, 0, 0, 1316, 5025, 0, 9, 0, 5688, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 35, 7989, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 676, 0, 0, 4, 4256, 0, 0, 0, 0, 2857, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 28, 0, 0, 0, 0, 2, 2710, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 450, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 4, 0, 0, 5922, 0, 0, 0, 0, 0, 0, 5402, 0, 0, 8, 3749, 0, 0, 0, 244, 0, 7824, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 1, 3, 0, 0, 0, 7, 0, 1, 2, 1, 0, 0, 0, 1, 0, 2, 4754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 6, 0, 8176, 56, 0, 1, 5, 0, 11, 35, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8158, 0, 0, 7, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 3623, 0, 0, 0, 3918, 1, 0, 0, 0, 8180, 0, 36, 0, 0, 0, 404, 185, 3, 0, 12, 0, 0, 0, 0, 0, 0, 7929, 0, 241, 8110, 169, 0, 0, 0, 0, 0, 4346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 6248, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 10, 40, 0, 8137, 0, 0, 0, 0, 0, 1, 50, 0, 27, 0, 2286, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 2402, 1, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 941, 0, 0, 0, 122, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 119, 0, 0, 0, 0, 0, 3, 0, 0, 4322, 0, 0, 3526, 1, 3, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 147, 2, 141, 0, 0, 0, 1219, 1445, 0, 0, 0, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 9, 0, 0, 1, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2976, 0, 76, 7919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 27, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1924, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 0, 0, 0, 0, 2, 0, 3730, 406, 8155, 0, 0, 0, 0, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 675, 0, 0, 0, 81, 0, 2891, 0, 0, 8051, 0, 458, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 5923, 3369, 6368, 10972, 2125, 16547, 12164] avg=8192.0000 max=16547 max_rank=6 min=2125 min_rank=5 imbalance_ratio=2.019897 max_load_reduction=48989 max_load_reduction_pct=0.747513 moved_to_echo=41808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 34, 1302, 0, 2192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 28, 0, 0, 0, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4696, 2, 0, 0, 0, 0, 114, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6806, 0, 0, 642, 0, 0, 0, 0, 43, 121, 2062, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 5820, 0, 0, 7534, 3639, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 172, 0, 0, 0, 0, 0, 4325, 0, 0, 0, 0, 1934, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 3, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3542, 3032, 4906, 9791, 18891, 6683, 10318, 8373] avg=8192.0000 max=18891 max_rank=4 min=3032 min_rank=1 imbalance_ratio=2.306030 max_load_reduction=46645 max_load_reduction_pct=0.711746 moved_to_echo=44265 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 450, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 1173, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 4, 0, 0, 5922, 0, 0, 0, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2514, 0, 0, 0, 244, 0, 7824, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 1, 3, 0, 0, 0, 7, 0, 1, 2, 1205, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 6, 0, 8176, 56, 0, 1, 5, 0, 11, 35, 0, 1337, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8066, 0, 0, 7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10141, 13049, 12668, 6304, 3407, 9837, 2007, 8123] avg=8192.0000 max=13049 max_rank=1 min=2007 min_rank=6 imbalance_ratio=1.592896 max_load_reduction=52487 max_load_reduction_pct=0.800888 moved_to_echo=23374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1150, 3, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 20, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 51, 0, 0, 0, 1, 0, 0, 0, 6945, 0, 0, 0, 2634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 44, 0, 0, 3, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4721, 2, 7, 17, 0, 0, 0, 6249, 1, 0, 1846, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 5211, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 152, 4589, 73, 1381, 0, 0, 0, 11, 0, 0, 0, 0, 1707, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7651, 0, 0, 0, 0, 8147, 0, 0, 51, 0, 0, 0, 0, 0, 0, 2555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4, 8034, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1352, 9785, 691, 12844, 6225, 8176, 18404, 8059] avg=8192.0000 max=18404 max_rank=6 min=691 min_rank=2 imbalance_ratio=2.246582 max_load_reduction=47132 max_load_reduction_pct=0.719177 moved_to_echo=40864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 6289, 14, 0, 0, 0, 0, 8155, 0, 0, 1316, 5025, 0, 9, 0, 5688, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 35, 7989, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 676, 0, 0, 4, 4256, 0, 0, 0, 0, 2857, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 28, 0, 0, 0, 0, 2, 2710, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 3306, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8091, 0, 1, 14, 0, 2506, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4596, 2, 1123, 0, 0, 0, 0, 0, 18, 0, 46, 1, 1479, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 5350, 68, 0, 0, 0, 0, 7564, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 9, 0, 1784, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5019, 0, 0, 309, 0, 0, 0, 72, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6928, 0, 8152, 0, 0, 24, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 6, 81, 0, 0, 0, 1, 2033, 0, 0, 20, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 22, 0, 0, 0, 4843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14001, 7333, 13846, 2019, 5857, 15327, 2273, 4880] avg=8192.0000 max=15327 max_rank=5 min=2019 min_rank=3 imbalance_ratio=1.870972 max_load_reduction=50209 max_load_reduction_pct=0.766129 moved_to_echo=28337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1572, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 6289, 14, 0, 0, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 3369, 0, 9, 0, 5688, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 35, 5719, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 4, 4256, 0, 0, 0, 0, 2857, 0, 0, 0, 0, 2019, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 28, 0, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2710, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9832, 11403, 10534, 3929, 5863, 10057, 3159, 10759] avg=8192.0000 max=11403 max_rank=1 min=3159 min_rank=6 imbalance_ratio=1.391968 max_load_reduction=54133 max_load_reduction_pct=0.826004 moved_to_echo=29838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 34, 1302, 0, 2192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 28, 0, 0, 0, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4696, 2, 0, 0, 0, 0, 114, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6806, 0, 0, 642, 0, 0, 0, 0, 43, 121, 2062, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 5820, 0, 0, 7534, 3639, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 172, 0, 0, 0, 0, 0, 4325, 0, 0, 0, 0, 1934, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 3, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3542, 3032, 4906, 9791, 18891, 6683, 10318, 8373] avg=8192.0000 max=18891 max_rank=4 min=3032 min_rank=1 imbalance_ratio=2.306030 max_load_reduction=46645 max_load_reduction_pct=0.711746 moved_to_echo=44265 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 3623, 0, 0, 0, 3918, 1, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 36, 0, 0, 0, 404, 185, 3, 0, 12, 0, 0, 0, 0, 0, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 8110, 115, 0, 0, 0, 0, 0, 4346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 6248, 37, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8, 40, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 50, 0, 27, 0, 2286, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 2402, 1, 0, 123, 0, 0, 0, 0, 0, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 941, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7597, 11848, 12812, 6418, 8195, 4359, 5520, 8787] avg=8192.0000 max=12812 max_rank=2 min=4359 min_rank=5 imbalance_ratio=1.563965 max_load_reduction=52724 max_load_reduction_pct=0.804504 moved_to_echo=26861 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 119, 0, 0, 0, 0, 0, 3, 0, 0, 4322, 0, 0, 3526, 1, 3, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 147, 2, 141, 0, 0, 0, 1219, 1445, 0, 0, 0, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 9, 0, 0, 1, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2976, 0, 76, 7919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 27, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1924, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 0, 0, 0, 0, 2, 0, 3730, 406, 8155, 0, 0, 0, 0, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 675, 0, 0, 0, 81, 0, 2891, 0, 0, 8051, 0, 458, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 5923, 3369, 6368, 10972, 2125, 16547, 12164] avg=8192.0000 max=16547 max_rank=6 min=2125 min_rank=5 imbalance_ratio=2.019897 max_load_reduction=48989 max_load_reduction_pct=0.747513 moved_to_echo=41808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 443, 0, 4, 0, 0, 0, 0, 0, 7941, 0, 0, 17, 0, 2549, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1233, 5461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2, 7446, 0, 0, 0, 0, 17, 0, 0, 8182, 0, 2064, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 297, 5825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 0, 0, 0, 7, 0, 0, 0, 0, 6671, 5, 0, 0, 0, 2028, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4150, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 2, 2, 0, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 0, 4344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11003, 3163, 6710, 17863, 6126, 8972, 7308, 4391] avg=8192.0000 max=17863 max_rank=3 min=3163 min_rank=1 imbalance_ratio=2.180542 max_load_reduction=47673 max_load_reduction_pct=0.727432 moved_to_echo=26797 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1572, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 6289, 14, 0, 0, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 3369, 0, 9, 0, 5688, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 35, 5719, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 4, 4256, 0, 0, 0, 0, 2857, 0, 0, 0, 0, 2019, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 28, 0, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2710, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9832, 11403, 10534, 3929, 5863, 10057, 3159, 10759] avg=8192.0000 max=11403 max_rank=1 min=3159 min_rank=6 imbalance_ratio=1.391968 max_load_reduction=54133 max_load_reduction_pct=0.826004 moved_to_echo=29838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 450, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 1173, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 4, 0, 0, 5922, 0, 0, 0, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2514, 0, 0, 0, 244, 0, 7824, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 1, 3, 0, 0, 0, 7, 0, 1, 2, 1205, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 6, 0, 8176, 56, 0, 1, 5, 0, 11, 35, 0, 1337, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8066, 0, 0, 7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10141, 13049, 12668, 6304, 3407, 9837, 2007, 8123] avg=8192.0000 max=13049 max_rank=1 min=2007 min_rank=6 imbalance_ratio=1.592896 max_load_reduction=52487 max_load_reduction_pct=0.800888 moved_to_echo=23374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1150, 3, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 20, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 51, 0, 0, 0, 1, 0, 0, 0, 6945, 0, 0, 0, 2634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 44, 0, 0, 3, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4721, 2, 7, 17, 0, 0, 0, 6249, 1, 0, 1846, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 5211, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 152, 4589, 73, 1381, 0, 0, 0, 11, 0, 0, 0, 0, 1707, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7651, 0, 0, 0, 0, 8147, 0, 0, 51, 0, 0, 0, 0, 0, 0, 2555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4, 8034, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1352, 9785, 691, 12844, 6225, 8176, 18404, 8059] avg=8192.0000 max=18404 max_rank=6 min=691 min_rank=2 imbalance_ratio=2.246582 max_load_reduction=47132 max_load_reduction_pct=0.719177 moved_to_echo=40864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 13312, 0, 0, 128, 128, 18048, 128, 3840, 640, 10240, 0, 0, 128, 128, 128, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[13056, 128, 3200, 128, 128, 256, 5504, 384, 128, 0, 6016, 7552, 9984, 128, 0, 0, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 256, 11520, 21632, 128, 128, 128, 9216, 0, 19968, 3200, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 768, 128, 256, 14080, 4352, 17792, 128, 0, 128, 13952, 128, 128, 128, 0, 12928, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 128, 256, 37376, 128, 128, 768, 0, 128, 128, 6272, 8320, 128, 12160, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 256, 11520, 21632, 128, 128, 128, 9216, 0, 19968, 3200, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 4992, 16256, 512, 128, 128, 0, 3712, 128, 0, 0, 24832, 128, 128, 4864, 128, 10112, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 768, 0, 128, 0, 1408, 34048, 128, 128, 23936, 3712, 128, 128, 1408, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 8448, 2816, 0, 128, 1152, 128, 2944, 128, 128, 41600, 0, 512, 128, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4005, 2, 0, 0, 0, 3, 0, 1, 0, 0, 250, 0, 0, 0, 0, 4, 6192, 0, 0, 0, 4186, 0, 0, 0, 0, 0, 65, 235, 0, 0, 0, 24, 0, 0, 0, 0, 2214, 0, 0, 0, 8090, 0, 0, 0, 4272, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 1541, 0, 0, 5097, 0, 142, 0, 0, 0, 0, 0, 7879, 0, 0, 7885, 0, 18, 0, 0, 0, 0, 0, 0, 296, 0, 0, 155, 0, 258, 0, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 7161, 0, 0, 1659, 0, 0, 0, 0, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 1435, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 5986, 0, 0, 0, 777, 0, 8, 76, 0, 7, 69, 0, 0, 0, 0, 0, 4166, 0, 0, 0, 8, 2, 0, 0, 4, 0, 0, 8, 0, 534, 1728, 2, 0, 0, 0, 0, 7579, 0, 0, 1, 3419, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5619, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4742, 0, 0, 3376, 0, 2165, 0, 0, 0, 7631, 0, 0, 0, 11, 0, 0, 3461, 1, 0, 0, 0, 0, 0, 2765, 0, 0, 155, 0, 0, 6, 0, 43, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8187, 0, 0, 254, 0, 5, 0, 0, 0, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4710, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 33, 6306, 0, 0, 0, 140, 8147, 18, 0, 0, 6827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7657, 40, 39, 0, 0, 0, 0, 0, 0, 13, 7861, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 870, 477, 0, 0, 0, 3, 0, 7156, 0, 0, 0, 0, 3273, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 144, 0, 0, 109, 8169, 0, 0, 976, 287, 0, 0, 0, 0, 0, 0, 5, 0, 67, 125, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 580, 0, 0, 0, 30, 0, 0, 21, 0, 0, 0, 0, 0, 0, 2, 0, 4787, 48, 0, 0, 0, 0, 177, 4, 0, 0, 0, 0, 1, 0, 8186, 0, 2, 0, 0, 0, 240, 23, 0, 0, 6806, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 6, 321, 0, 37, 0, 688, 0, 8140, 0, 0, 0, 0, 118, 0, 22, 0, 7662, 3, 58, 0, 0, 7058, 0, 2, 3, 0, 3229, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 233, 0, 0, 0, 82, 3, 7296, 0, 0, 11, 0, 113, 0, 418, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 476, 0, 0, 0, 0, 1, 1267, 0, 1, 131, 0, 3, 0, 0, 7686, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 47, 0, 8, 0, 7630, 3, 3, 0, 4, 1, 23, 0, 0, 0, 280, 0, 0, 0, 0, 0, 10, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 52, 4468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 6417, 3, 2, 92, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 3535, 4, 0, 0, 0, 0, 0, 8027, 0, 0, 0, 2, 8035, 0, 12, 0, 516, 39, 0, 0, 0, 0, 0, 0, 27, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4, 2237, 0, 0, 1, 148, 0, 6, 8, 171, 0, 3966, 0, 0, 0, 1, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4, 3181, 0, 0, 45, 0, 0, 0, 0, 0, 0, 8051, 0, 12, 0, 4231, 0, 726, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 206, 255, 1, 5823, 13, 0, 11, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 10, 0, 0, 7245, 0, 0, 0, 2, 2, 7279, 0, 0, 0, 2, 0, 19, 0, 0, 0, 1, 8089, 0, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4005, 2, 0, 0, 0, 3, 0, 1, 0, 0, 250, 0, 0, 0, 0, 4, 6192, 0, 0, 0, 4186, 0, 0, 0, 0, 0, 65, 235, 0, 0, 0, 24, 0, 0, 0, 0, 2214, 0, 0, 0, 8090, 0, 0, 0, 4272, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 1541, 0, 0, 5097, 0, 142, 0, 0, 0, 0, 0, 7879, 0, 0, 7885, 0, 18, 0, 0, 0, 0, 0, 0, 296, 0, 0, 155, 0, 258, 0, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 7161, 0, 0, 1659, 0, 0, 0, 0, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 1435, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 24, 5392, 0, 9, 0, 0, 0, 0, 0, 0, 161, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8055, 0, 0, 0, 0, 0, 7258, 0, 0, 5, 8180, 0, 0, 0, 6268, 71, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 253, 0, 0, 7929, 0, 0, 0, 0, 0, 5, 0, 124, 0, 0, 0, 0, 70, 0, 0, 2134, 0, 0, 0, 572, 0, 3, 0, 0, 3, 0, 36, 0, 3, 0, 0, 0, 0, 2475, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 8019, 0, 0, 32, 0, 0, 0, 0, 0, 7777, 7, 0, 0, 0, 0, 0, 450, 0, 120] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4710, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 33, 6306, 0, 0, 0, 140, 8147, 18, 0, 0, 6827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7657, 40, 39, 0, 0, 0, 0, 0, 0, 13, 7861, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 870, 477, 0, 0, 0, 3, 0, 7156, 0, 0, 0, 0, 3273, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 144, 0, 0, 109, 8169, 0, 0, 976, 287, 0, 0, 0, 0, 0, 0, 5, 0, 67, 125, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 249, 1788, 0, 0, 0, 0, 2, 0, 0, 13, 96, 0, 0, 0, 636, 0, 6800, 0, 0, 345, 6718, 0, 0, 0, 0, 0, 14, 1033, 0, 212, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5812, 0, 0, 0, 8137, 0, 0, 0, 0, 10, 3929, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 1, 0, 0, 4, 0, 2, 0, 0, 0, 5160, 0, 0, 0, 0, 2561, 0, 0, 0, 0, 0, 0, 1261, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 7957, 0, 3, 833, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4005, 2, 0, 0, 0, 3, 0, 1, 0, 0, 250, 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 782, 0, 0, 0, 4186, 0, 0, 0, 0, 0, 65, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 2214, 0, 0, 0, 4071, 0, 0, 0, 4272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 1541, 0, 0, 3505, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 0, 142, 0, 0, 0, 0, 0, 7879, 0, 0, 7885, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 155, 0, 258, 0, 10, 3, 0, 0, 0, 4762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 5767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1659, 0, 0, 0, 0, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 1435, 0, 1394, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4775, 5272, 10581, 5771, 19542, 5484, 5771, 8340] avg=8192.0000 max=19542 max_rank=4 min=4775 min_rank=0 imbalance_ratio=2.385498 max_load_reduction=45994 max_load_reduction_pct=0.701813 moved_to_echo=39137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 580, 0, 0, 0, 30, 0, 0, 21, 0, 0, 0, 0, 0, 0, 2, 7613, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 48, 0, 0, 0, 0, 177, 4, 0, 0, 0, 0, 1, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 240, 23, 0, 0, 3418, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 6, 2951, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 37, 0, 688, 0, 8140, 0, 0, 0, 0, 118, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 58, 0, 0, 7058, 0, 2, 3, 0, 3229, 531, 0, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 233, 0, 0, 0, 82, 3, 5872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 113, 0, 418, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7999, 1424, 95, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8683, 9014, 3685, 3473, 9231, 14568, 6769, 10113] avg=8192.0000 max=14568 max_rank=5 min=3473 min_rank=3 imbalance_ratio=1.778320 max_load_reduction=50968 max_load_reduction_pct=0.777710 moved_to_echo=40681 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 476, 0, 0, 0, 0, 1, 1267, 0, 1, 131, 0, 3, 0, 6376, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 47, 0, 8, 0, 7630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4, 1, 23, 0, 0, 0, 142, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 52, 121, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 92, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 4, 0, 0, 0, 0, 0, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8035, 0, 12, 0, 516, 39, 0, 0, 0, 0, 0, 0, 27, 0, 1562, 1297, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 16665, 186, 9277, 3248, 6353, 10004, 11531] avg=8192.0000 max=16665 max_rank=1 min=186 min_rank=2 imbalance_ratio=2.034302 max_load_reduction=48871 max_load_reduction_pct=0.745712 moved_to_echo=31136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 2237, 0, 0, 1, 148, 0, 6, 8, 171, 0, 3966, 0, 0, 0, 1, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 4052, 0, 12, 0, 4231, 0, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 206, 255, 1, 5823, 13, 0, 3488, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 10, 0, 0, 7245, 0, 0, 0, 6252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 7279, 0, 0, 0, 2, 0, 19, 0, 0, 0, 1, 6515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7053, 4249, 9066, 10650, 2945, 13556, 13820, 4197] avg=8192.0000 max=13820 max_rank=6 min=2945 min_rank=4 imbalance_ratio=1.687012 max_load_reduction=51716 max_load_reduction_pct=0.789124 moved_to_echo=34518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4710, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 33, 6306, 0, 0, 0, 7109, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 1029, 18, 0, 0, 6827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3854, 40, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7861, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8, 0, 3317, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 870, 477, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3273, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 144, 0, 0, 109, 6579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 976, 287, 0, 0, 0, 0, 0, 0, 5, 0, 67, 125, 0, 0, 0, 0, 1590, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20516, 8014, 3934, 12054, 1351, 9735, 6835, 3097] avg=8192.0000 max=20516 max_rank=0 min=1351 min_rank=4 imbalance_ratio=2.504395 max_load_reduction=45020 max_load_reduction_pct=0.686951 moved_to_echo=21018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 476, 0, 0, 0, 0, 1, 1267, 0, 1, 131, 0, 3, 0, 0, 7686, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 47, 0, 8, 0, 7630, 3, 3, 0, 4, 1, 23, 0, 0, 0, 280, 0, 0, 0, 0, 0, 10, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 52, 4468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 6417, 3, 2, 92, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 3535, 4, 0, 0, 0, 0, 0, 8027, 0, 0, 0, 2, 8035, 0, 12, 0, 516, 39, 0, 0, 0, 0, 0, 0, 27, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5986, 0, 0, 0, 777, 0, 8, 76, 0, 7, 69, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 8, 2, 0, 0, 4, 0, 0, 8, 0, 534, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7579, 0, 0, 1, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5619, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4742, 0, 0, 1481, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 0, 2165, 0, 0, 0, 7631, 0, 0, 0, 11, 0, 0, 3461, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 0, 0, 155, 0, 0, 6, 0, 43, 145, 0, 0, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 5, 0, 0, 0, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1593, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7140, 2810, 9303, 12280, 15665, 6318, 6596, 5424] avg=8192.0000 max=15665 max_rank=4 min=2810 min_rank=1 imbalance_ratio=1.912231 max_load_reduction=49871 max_load_reduction_pct=0.760971 moved_to_echo=34003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4710, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 33, 6306, 0, 0, 0, 7109, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 1029, 18, 0, 0, 6827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3854, 40, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7861, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8, 0, 3317, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 870, 477, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3273, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 144, 0, 0, 109, 6579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 976, 287, 0, 0, 0, 0, 0, 0, 5, 0, 67, 125, 0, 0, 0, 0, 1590, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20516, 8014, 3934, 12054, 1351, 9735, 6835, 3097] avg=8192.0000 max=20516 max_rank=0 min=1351 min_rank=4 imbalance_ratio=2.504395 max_load_reduction=45020 max_load_reduction_pct=0.686951 moved_to_echo=21018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 2237, 0, 0, 1, 148, 0, 6, 8, 171, 0, 3966, 0, 0, 0, 1, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 4052, 0, 12, 0, 4231, 0, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 206, 255, 1, 5823, 13, 0, 3488, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 10, 0, 0, 7245, 0, 0, 0, 6252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 7279, 0, 0, 0, 2, 0, 19, 0, 0, 0, 1, 6515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7053, 4249, 9066, 10650, 2945, 13556, 13820, 4197] avg=8192.0000 max=13820 max_rank=6 min=2945 min_rank=4 imbalance_ratio=1.687012 max_load_reduction=51716 max_load_reduction_pct=0.789124 moved_to_echo=34518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 24, 5392, 0, 9, 0, 0, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7258, 0, 0, 5, 4117, 0, 0, 0, 6268, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 253, 0, 0, 3544, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 0, 5, 0, 124, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2134, 0, 0, 0, 572, 0, 3, 0, 0, 3, 0, 36, 0, 3, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2475, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 6458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 7777, 7, 0, 0, 0, 0, 0, 450, 0, 120, 1561, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5944, 8145, 17719, 3817, 5827, 2874, 8962, 12248] avg=8192.0000 max=17719 max_rank=2 min=2874 min_rank=5 imbalance_ratio=2.162964 max_load_reduction=47817 max_load_reduction_pct=0.729630 moved_to_echo=29911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 580, 0, 0, 0, 30, 0, 0, 21, 0, 0, 0, 0, 0, 0, 2, 7613, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 48, 0, 0, 0, 0, 177, 4, 0, 0, 0, 0, 1, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 240, 23, 0, 0, 3418, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 6, 2951, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 37, 0, 688, 0, 8140, 0, 0, 0, 0, 118, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 58, 0, 0, 7058, 0, 2, 3, 0, 3229, 531, 0, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 233, 0, 0, 0, 82, 3, 5872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 113, 0, 418, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7999, 1424, 95, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8683, 9014, 3685, 3473, 9231, 14568, 6769, 10113] avg=8192.0000 max=14568 max_rank=5 min=3473 min_rank=3 imbalance_ratio=1.778320 max_load_reduction=50968 max_load_reduction_pct=0.777710 moved_to_echo=40681 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 476, 0, 0, 0, 0, 1, 1267, 0, 1, 131, 0, 3, 0, 6376, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 47, 0, 8, 0, 7630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4, 1, 23, 0, 0, 0, 142, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 52, 121, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 92, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 4, 0, 0, 0, 0, 0, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8035, 0, 12, 0, 516, 39, 0, 0, 0, 0, 0, 0, 27, 0, 1562, 1297, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 16665, 186, 9277, 3248, 6353, 10004, 11531] avg=8192.0000 max=16665 max_rank=1 min=186 min_rank=2 imbalance_ratio=2.034302 max_load_reduction=48871 max_load_reduction_pct=0.745712 moved_to_echo=31136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4005, 2, 0, 0, 0, 3, 0, 1, 0, 0, 250, 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 782, 0, 0, 0, 4186, 0, 0, 0, 0, 0, 65, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 2214, 0, 0, 0, 4071, 0, 0, 0, 4272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 1541, 0, 0, 3505, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 0, 142, 0, 0, 0, 0, 0, 7879, 0, 0, 7885, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 155, 0, 258, 0, 10, 3, 0, 0, 0, 4762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 5767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1659, 0, 0, 0, 0, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 1435, 0, 1394, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4775, 5272, 10581, 5771, 19542, 5484, 5771, 8340] avg=8192.0000 max=19542 max_rank=4 min=4775 min_rank=0 imbalance_ratio=2.385498 max_load_reduction=45994 max_load_reduction_pct=0.701813 moved_to_echo=39137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 249, 1788, 0, 0, 0, 0, 2, 0, 0, 13, 96, 0, 0, 0, 636, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 0, 345, 6718, 0, 0, 0, 0, 0, 14, 1033, 0, 212, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3929, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 1, 2518, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 5160, 0, 0, 0, 0, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 5230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 6408, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 1, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3153, 11367, 11062, 8214, 7727, 6648, 6779, 10586] avg=8192.0000 max=11367 max_rank=1 min=3153 min_rank=0 imbalance_ratio=1.387573 max_load_reduction=54169 max_load_reduction_pct=0.826553 moved_to_echo=31740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3840, 384, 10880, 0, 512, 0, 13312, 128, 128, 128, 128, 8320, 0, 512, 1536, 8192, 12288, 6272, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[15104, 0, 5376, 0, 768, 0, 15872, 128, 14080, 512, 128, 8064, 0, 6144, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 17408, 128, 0, 128, 24320, 128, 128, 0, 22912, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3840, 384, 10880, 0, 512, 0, 13312, 128, 128, 128, 128, 8320, 0, 512, 1536, 8192, 12288, 6272, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 128, 128, 0, 15360, 0, 256, 512, 0, 3584, 640, 128, 7296, 256, 384, 0, 35456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 17664, 128, 128, 0, 128, 128, 128, 256, 256, 128, 20608, 1024, 128, 20992, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 128, 128, 0, 15360, 0, 256, 512, 0, 3584, 640, 128, 7296, 256, 384, 0, 35456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 17792, 3712, 512, 128, 1024, 128, 128, 5504, 1536, 128, 4480, 6528, 0, 128, 640, 21120, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 5888, 128, 128, 384, 25856, 128, 256, 128, 128, 128, 128, 1280, 128, 896, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 9856, 128, 0, 640, 3584, 640, 128, 256, 0, 128, 128, 50688, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 7, 41, 0, 0, 0, 0, 0, 0, 3202, 0, 4705, 0, 0, 0, 1819, 0, 0, 0, 0, 5289, 0, 0, 0, 0, 7639, 0, 1195, 0, 0, 3, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 5205, 0, 6934, 0, 2, 8109, 0, 0, 0, 60, 37, 2139, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 2435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 4103, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 58, 0, 3956] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[11, 8, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 20, 0, 4, 0, 0, 4262, 0, 166, 0, 0, 0, 8074, 0, 10, 0, 0, 7419, 0, 0, 1130, 0, 0, 2, 0, 650, 1, 12, 0, 0, 7776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8131, 28, 0, 0, 1, 6575, 0, 0, 0, 0, 77, 0, 67, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 7992, 0, 8, 0, 0, 1126, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 775, 0, 0, 0, 0, 0, 731, 3, 0, 0, 0, 0, 212, 1, 0, 0, 0, 0, 0, 254, 0, 1493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 216, 15, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 8192, 1016, 0, 0, 1, 0, 8191, 0, 389, 0, 8158, 0, 0, 4, 33, 22, 0, 0, 0, 0, 17, 0, 12, 0, 6443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 42, 7964, 8172, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 79, 7826] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 98, 0, 0, 0, 2889, 0, 5635, 0, 0, 0, 8052, 0, 0, 8073, 0, 7844, 0, 0, 8115, 0, 30, 0, 0, 0, 2, 0, 0, 89, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 0, 150, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1323, 0, 0, 0, 24, 0, 0, 5, 1130, 0, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 0, 0, 1177, 0, 2109] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[6, 8082, 0, 0, 0, 58, 0, 0, 44, 13, 34, 75, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 1, 17, 0, 0, 0, 1, 0, 0, 7441, 0, 8135, 0, 0, 22, 0, 0, 0, 0, 64, 161, 0, 0, 0, 0, 3, 0, 0, 0, 0, 191, 6082, 0, 0, 7970, 0, 0, 0, 100, 8032, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 72, 33, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 667, 593, 0, 0, 0, 19, 2144, 0, 7090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 17, 0, 79, 6034, 0, 0, 0, 0, 3021, 0, 0, 37, 0, 8192, 2, 41, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 621, 0, 0, 0, 8154, 0, 0, 35, 3574, 8159, 0, 0, 0, 0, 651, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 4636, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 31, 42, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 80, 0, 0, 0, 536, 0, 7947, 0, 0, 0, 8174, 0, 0, 0, 65, 8084, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 8080, 0, 2, 5, 0, 1, 0, 0, 7211, 134, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 12, 0, 1, 0, 0, 1, 0, 1992, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8188, 0, 2, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 0, 4950] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[11, 8, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 20, 0, 4, 0, 0, 4262, 0, 166, 0, 0, 0, 8074, 0, 10, 0, 0, 7419, 0, 0, 1130, 0, 0, 2, 0, 650, 1, 12, 0, 0, 7776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8131, 28, 0, 0, 1, 6575, 0, 0, 0, 0, 77, 0, 67, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 7992, 0, 8, 0, 0, 1126, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 775, 0, 0, 0, 0, 0, 731, 3, 0, 0, 0, 0, 212, 1, 0, 0, 0, 0, 0, 254, 0, 1493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 17, 0, 79, 6034, 0, 0, 0, 0, 3021, 0, 0, 37, 0, 8192, 2, 41, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 621, 0, 0, 0, 8154, 0, 0, 35, 3574, 8159, 0, 0, 0, 0, 651, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 4636, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 7, 41, 0, 0, 0, 0, 0, 0, 3202, 0, 4705, 0, 0, 0, 1819, 0, 0, 0, 0, 5289, 0, 0, 0, 0, 7639, 0, 1195, 0, 0, 3, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 5205, 0, 6934, 0, 2, 8109, 0, 0, 0, 60, 37, 2139, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 2435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 4103, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 58, 0, 3956] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7393, 0, 0, 0, 0, 688, 0, 0, 140, 0, 8081, 0, 0, 0, 0, 759, 0, 11, 0, 0, 0, 0, 0, 0, 6607, 0, 215, 0, 1393, 0, 0, 0, 7975, 0, 0, 100, 0, 0, 0, 83, 0, 0, 0, 0, 0, 3, 0, 8102, 0, 0, 12, 0, 0, 0, 721, 0, 0, 0, 3, 0, 0, 8, 0, 1, 1, 0, 101, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 7906, 4, 0, 0, 0, 6486, 1, 0, 0, 145, 0, 0, 0, 0, 15, 3, 0, 8136, 0, 0, 77, 0, 0, 14, 0, 0, 0, 0, 304, 0, 0, 0, 0, 14, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 7, 41, 0, 0, 0, 0, 25, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3202, 0, 4465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5619, 0, 1195, 0, 0, 3, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5205, 0, 6934, 0, 2, 0, 0, 0, 0, 60, 7, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8109, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 2435, 2020, 454, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 4103, 0, 228, 0, 0, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 58, 0, 3956, 1919, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4577, 7667, 6848, 14347, 9756, 8745, 7543, 6053] avg=8192.0000 max=14347 max_rank=3 min=4577 min_rank=0 imbalance_ratio=1.751343 max_load_reduction=51189 max_load_reduction_pct=0.781082 moved_to_echo=32097 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 216, 15, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 34, 0, 5516, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1016, 0, 0, 1, 0, 0, 0, 389, 0, 8158, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 22, 0, 0, 0, 0, 17, 0, 12, 0, 6443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 42, 1168, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 4562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 79, 7826, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9412, 17760, 6527, 9384, 5167, 8088, 1, 9197] avg=8192.0000 max=17760 max_rank=1 min=1 min_rank=6 imbalance_ratio=2.167969 max_load_reduction=47776 max_load_reduction_pct=0.729004 moved_to_echo=22453 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 8, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 20, 0, 4, 4550, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4262, 0, 158, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1130, 0, 0, 2, 0, 650, 1, 12, 0, 0, 7776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 28, 0, 0, 1, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 67, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 8131, 4498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 7992, 0, 8, 0, 0, 1126, 0, 637, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 775, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 3, 0, 0, 0, 0, 212, 1, 0, 0, 0, 0, 0, 254, 0, 1493, 2694, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 4430, 9571, 9190, 12830, 10025, 4874, 6442] avg=8192.0000 max=12830 max_rank=4 min=4430 min_rank=1 imbalance_ratio=1.566162 max_load_reduction=52706 max_load_reduction_pct=0.804230 moved_to_echo=34171 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7393, 0, 0, 0, 0, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 7668, 0, 0, 0, 0, 759, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4860, 0, 215, 0, 1393, 0, 0, 0, 7975, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 0, 0, 0, 3, 0, 0, 8, 0, 1, 1, 0, 101, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 7906, 4, 0, 0, 0, 6486, 1747, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 145, 0, 0, 0, 0, 15, 3, 0, 8136, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 304, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8578, 14543, 98, 8937, 16589, 8377, 332] avg=8192.0000 max=16589 max_rank=5 min=98 min_rank=3 imbalance_ratio=2.025024 max_load_reduction=48947 max_load_reduction_pct=0.746872 moved_to_echo=34235 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 216, 15, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 8192, 1016, 0, 0, 1, 0, 8191, 0, 389, 0, 8158, 0, 0, 4, 33, 22, 0, 0, 0, 0, 17, 0, 12, 0, 6443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 42, 7964, 8172, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 79, 7826] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 98, 0, 5470, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2889, 0, 5347, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 23, 0, 0, 0, 2, 0, 0, 89, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 4486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1323, 0, 0, 0, 24, 0, 0, 5, 1130, 0, 0, 0, 0, 896, 7, 673, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 4322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 0, 0, 1177, 0, 2109, 2849, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9140, 16309, 8239, 3099, 4636, 4347, 12347, 7419] avg=8192.0000 max=16309 max_rank=1 min=3099 min_rank=3 imbalance_ratio=1.990845 max_load_reduction=49227 max_load_reduction_pct=0.751144 moved_to_echo=28749 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 450, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 17, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 6023, 2, 41, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 8154, 0, 0, 35, 3574, 0, 0, 0, 0, 0, 97, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8159, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 2169, 261, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 0, 8190, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4636, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8036, 1096, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3491, 2577, 6104, 12484, 11521, 2581, 11568, 15210] avg=8192.0000 max=15210 max_rank=7 min=2577 min_rank=1 imbalance_ratio=1.856689 max_load_reduction=50326 max_load_reduction_pct=0.767914 moved_to_echo=40880 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 8, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 20, 0, 4, 4550, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4262, 0, 158, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1130, 0, 0, 2, 0, 650, 1, 12, 0, 0, 7776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 28, 0, 0, 1, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 67, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 8131, 4498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 7992, 0, 8, 0, 0, 1126, 0, 637, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 775, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 3, 0, 0, 0, 0, 212, 1, 0, 0, 0, 0, 0, 254, 0, 1493, 2694, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 4430, 9571, 9190, 12830, 10025, 4874, 6442] avg=8192.0000 max=12830 max_rank=4 min=4430 min_rank=1 imbalance_ratio=1.566162 max_load_reduction=52706 max_load_reduction_pct=0.804230 moved_to_echo=34171 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 31, 42, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 80, 0, 4990, 3621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 7541, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 8080, 0, 2, 0, 0, 1, 0, 0, 1065, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 12, 0, 1, 0, 0, 1, 0, 1992, 0, 0, 0, 0, 8190, 0, 694, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8188, 0, 2, 0, 1, 0, 0, 4456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 0, 4950, 2934, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8788, 8151, 32, 9373, 4562, 12903, 12661, 9066] avg=8192.0000 max=12903 max_rank=5 min=32 min_rank=2 imbalance_ratio=1.575073 max_load_reduction=52633 max_load_reduction_pct=0.803116 moved_to_echo=39192 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 8082, 0, 0, 0, 58, 0, 0, 44, 13, 34, 75, 0, 0, 0, 0, 5559, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7441, 0, 5984, 0, 0, 22, 0, 0, 0, 0, 64, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 191, 6082, 0, 0, 0, 0, 0, 0, 100, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 72, 33, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 667, 593, 0, 0, 0, 19, 2144, 0, 7090, 0, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13872, 64, 13672, 7561, 8127, 2258, 8165, 11817] avg=8192.0000 max=13872 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.693359 max_load_reduction=51664 max_load_reduction_pct=0.788330 moved_to_echo=30367 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 7, 41, 0, 0, 0, 0, 25, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3202, 0, 4465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5619, 0, 1195, 0, 0, 3, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5205, 0, 6934, 0, 2, 0, 0, 0, 0, 60, 7, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8109, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 2435, 2020, 454, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 4103, 0, 228, 0, 0, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 58, 0, 3956, 1919, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4577, 7667, 6848, 14347, 9756, 8745, 7543, 6053] avg=8192.0000 max=14347 max_rank=3 min=4577 min_rank=0 imbalance_ratio=1.751343 max_load_reduction=51189 max_load_reduction_pct=0.781082 moved_to_echo=32097 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 450, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 17, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 6023, 2, 41, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 8154, 0, 0, 35, 3574, 0, 0, 0, 0, 0, 97, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8159, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 2169, 261, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 0, 8190, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4636, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8036, 1096, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3491, 2577, 6104, 12484, 11521, 2581, 11568, 15210] avg=8192.0000 max=15210 max_rank=7 min=2577 min_rank=1 imbalance_ratio=1.856689 max_load_reduction=50326 max_load_reduction_pct=0.767914 moved_to_echo=40880 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 216, 15, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 34, 0, 5516, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1016, 0, 0, 1, 0, 0, 0, 389, 0, 8158, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 22, 0, 0, 0, 0, 17, 0, 12, 0, 6443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 42, 1168, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 4562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 79, 7826, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9412, 17760, 6527, 9384, 5167, 8088, 1, 9197] avg=8192.0000 max=17760 max_rank=1 min=1 min_rank=6 imbalance_ratio=2.167969 max_load_reduction=47776 max_load_reduction_pct=0.729004 moved_to_echo=22453 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7393, 0, 0, 0, 0, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 7668, 0, 0, 0, 0, 759, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4860, 0, 215, 0, 1393, 0, 0, 0, 7975, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 0, 0, 0, 3, 0, 0, 8, 0, 1, 1, 0, 101, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 7906, 4, 0, 0, 0, 6486, 1747, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 145, 0, 0, 0, 0, 15, 3, 0, 8136, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 304, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8578, 14543, 98, 8937, 16589, 8377, 332] avg=8192.0000 max=16589 max_rank=5 min=98 min_rank=3 imbalance_ratio=2.025024 max_load_reduction=48947 max_load_reduction_pct=0.746872 moved_to_echo=34235 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[768, 4736, 1408, 0, 0, 128, 640, 768, 768, 0, 0, 128, 128, 3712, 128, 35584, 11520, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 4352, 384, 0, 128, 128, 0, 128, 128, 21376, 128, 128, 128, 128, 27264, 8192, 2816, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 19328, 1024, 27776, 128, 128, 128, 0, 896, 512, 8192, 8320, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 12160, 128, 0, 128, 128, 128, 0, 128, 128, 7936, 128, 0, 128, 256, 768, 26624, 17920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[640, 128, 128, 0, 23552, 256, 21120, 128, 3584, 0, 128, 128, 128, 256, 5760, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 256, 128, 0, 128, 1792, 128, 32896, 896, 12288, 0, 256, 128, 0, 17536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[16768, 128, 22528, 128, 1536, 768, 1536, 128, 128, 128, 22400, 256, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 256, 128, 0, 128, 1792, 128, 32896, 896, 12288, 0, 256, 128, 0, 17536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 0, 0, 128, 1408, 128, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 40704, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 0, 2651, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 6682, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 7531, 5580, 7374, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2049, 1036, 0, 6404, 48, 7, 7, 6268, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 0, 3696, 75, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 165, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 3, 0, 0, 0, 0, 10, 0, 492, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 539, 37, 0, 0, 0, 2, 0, 0, 17, 37, 0, 0, 0, 0, 4, 0, 0, 0, 44, 0, 4771, 196, 0, 0, 0, 0, 772, 0, 0, 0, 1817, 0, 2, 0, 0, 0, 7849, 0, 0, 1360, 377, 0, 3998, 0, 0, 39, 0, 0, 0, 0, 514, 0, 5123, 7571, 2, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 47, 21, 1, 5, 0, 0, 1, 0, 0, 1208, 0, 136, 0, 0, 3, 101, 0, 0, 16, 0, 0, 6039, 0, 706, 5874, 6966, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 18, 0, 25, 66, 0, 8188, 0, 459, 0, 0, 158, 4, 3, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 1340, 8192, 0, 0, 0, 0, 0, 0, 0, 35, 4, 0, 0, 0, 0, 130, 79, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 4, 0, 0, 0, 207, 294, 0, 0, 0, 0, 2133, 8095, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 12, 7, 7847, 0, 0, 0, 1, 0, 0, 0, 4028, 6945, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 4, 7949, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2556, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 287, 0, 7909, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 1, 0, 0, 6893, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 27, 14, 3573, 0, 0, 7104, 0, 0, 0, 8, 0, 2, 0, 2, 4245, 0, 0, 0, 0, 871, 0, 0, 0, 0, 0, 0, 1, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 3200, 0, 7, 0, 0, 410, 2607, 13, 0, 0, 1442] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 2494, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 4986, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 45, 0, 0, 0, 4539, 0, 0, 1248, 50, 0, 0, 7538, 0, 0, 0, 25, 0, 0, 0, 766, 0, 0, 366, 0, 0, 205, 7773, 264, 289, 0, 0, 0, 0, 0, 186, 0, 0, 0, 34, 0, 0, 3545, 0, 0, 469, 0, 0, 0, 0, 0, 7179, 0, 10, 1, 124, 0, 0, 0, 0, 0, 88, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 681, 469, 0, 0, 0, 0, 39, 0, 1, 0, 7756, 17, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 539, 37, 0, 0, 0, 2, 0, 0, 17, 37, 0, 0, 0, 0, 4, 0, 0, 0, 44, 0, 4771, 196, 0, 0, 0, 0, 772, 0, 0, 0, 1817, 0, 2, 0, 0, 0, 7849, 0, 0, 1360, 377, 0, 3998, 0, 0, 39, 0, 0, 0, 0, 514, 0, 5123, 7571, 2, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 47, 21, 1, 5, 0, 0, 1, 0, 0, 1208, 0, 136, 0, 0, 3, 101, 0, 0, 16, 0, 0, 6039, 0, 706, 5874, 6966, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 0, 0, 3, 0, 0, 0, 1, 0, 0, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 12, 6745, 0, 8192, 406, 0, 0, 0, 0, 7167, 0, 0, 4144, 0, 7357, 0, 0, 381, 0, 8, 7509, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 485, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 65, 4951, 2, 0, 0, 0, 901, 0, 9, 0, 0, 496, 0, 0, 0, 2829, 7978, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 202, 0, 0, 2660, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 634, 0, 0, 0, 0, 0, 1381, 0, 199, 0, 1170, 0, 353, 0, 0, 0, 0, 5, 0, 0, 0, 0, 22, 3310, 0, 0, 0, 12, 346, 7525, 5, 43, 0, 0, 0, 0, 0, 32, 0, 0, 6620, 0, 0, 1076, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 35, 105, 0, 538, 0, 0, 0, 0, 18, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7914, 0, 11, 298, 0, 0, 17, 8148, 2655, 8188, 4251, 57, 0, 0, 6826, 0, 0, 524, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 0, 2651, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 6682, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 7531, 5580, 7374, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2049, 1036, 0, 6404, 48, 7, 7, 6268, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 0, 3696, 75, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 165, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 3, 0, 0, 0, 0, 10, 0, 492, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 2494, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 4986, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 45, 0, 0, 0, 4539, 0, 0, 1248, 50, 0, 0, 7538, 0, 0, 0, 25, 0, 0, 0, 766, 0, 0, 366, 0, 0, 205, 7773, 264, 289, 0, 0, 0, 0, 0, 186, 0, 0, 0, 34, 0, 0, 3545, 0, 0, 469, 0, 0, 0, 0, 0, 7179, 0, 10, 1, 124, 0, 0, 0, 0, 0, 88, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 681, 469, 0, 0, 0, 0, 39, 0, 1, 0, 7756, 17, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 889, 0, 0, 0, 1, 94, 0, 0, 0, 7254, 35, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3591, 0, 2937, 305, 63, 0, 8090, 0, 21, 118, 0, 0, 0, 0, 0, 0, 0, 7845, 61, 34, 0, 0, 7027, 4494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 3003, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 0, 8142, 91, 0, 0, 0, 115, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2556, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 287, 0, 7909, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 1, 0, 0, 6893, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 27, 14, 3573, 0, 0, 7104, 0, 0, 0, 8, 0, 2, 0, 2, 4245, 0, 0, 0, 0, 871, 0, 0, 0, 0, 0, 0, 1, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 3200, 0, 7, 0, 0, 410, 2607, 13, 0, 0, 1442] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 0, 0, 3, 0, 0, 0, 1, 0, 0, 784, 3, 711, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 6745, 0, 5486, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7167, 0, 0, 4144, 0, 3433, 0, 0, 381, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 1, 0, 0, 3213, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7788, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 4951, 2, 0, 0, 0, 901, 0, 9, 0, 0, 6931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 2829, 0, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1715, 1599, 12649, 15133, 8497, 8855, 12861, 4227] avg=8192.0000 max=15133 max_rank=3 min=1599 min_rank=1 imbalance_ratio=1.847290 max_load_reduction=50403 max_load_reduction_pct=0.769089 moved_to_echo=34440 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 539, 37, 0, 0, 0, 2, 0, 0, 17, 37, 0, 0, 0, 7849, 732, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 44, 0, 4771, 196, 0, 0, 0, 0, 772, 0, 0, 293, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1817, 0, 2, 0, 0, 0, 0, 0, 0, 1360, 377, 0, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 514, 0, 5123, 3532, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 3307, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 47, 21, 1, 5, 6800, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1208, 0, 136, 0, 0, 3, 101, 0, 0, 16, 0, 7509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6039, 0, 413, 5874, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9379, 6860, 6235, 9210, 4873, 7503, 8974, 12502] avg=8192.0000 max=12502 max_rank=7 min=4873 min_rank=4 imbalance_ratio=1.526123 max_load_reduction=53034 max_load_reduction_pct=0.809235 moved_to_echo=33852 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 202, 0, 0, 2660, 0, 0, 0, 342, 0, 0, 0, 0, 1381, 320, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 634, 0, 0, 0, 0, 0, 0, 0, 199, 0, 1170, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 22, 1545, 0, 0, 0, 12, 346, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 32, 0, 0, 6620, 0, 0, 1076, 0, 2, 0, 1445, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 35, 105, 0, 538, 0, 0, 0, 0, 7993, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7914, 0, 11, 298, 6947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4760, 2655, 0, 4251, 57, 0, 0, 6826, 0, 0, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5100, 3458, 2244, 1935, 9265, 9254, 15190, 19090] avg=8192.0000 max=19090 max_rank=7 min=1935 min_rank=3 imbalance_ratio=2.330322 max_load_reduction=46446 max_load_reduction_pct=0.708710 moved_to_echo=52799 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 25, 66, 0, 8188, 0, 459, 0, 0, 158, 4, 3, 0, 0, 130, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 1340, 8192, 0, 0, 0, 0, 0, 0, 1, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 5481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 207, 138, 0, 0, 0, 0, 2133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 12, 7, 7847, 0, 0, 0, 128, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4028, 6945, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 3, 7949, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9079, 11134, 5599, 2482, 9105, 11605, 7473, 9059] avg=8192.0000 max=11605 max_rank=5 min=2482 min_rank=3 imbalance_ratio=1.416626 max_load_reduction=53931 max_load_reduction_pct=0.822922 moved_to_echo=37242 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 2494, 0, 0, 0, 0, 0, 0, 0, 476, 4539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 4986, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 1248, 50, 0, 0, 7538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 766, 0, 0, 366, 0, 0, 205, 7773, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 34, 0, 0, 3545, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7179, 0, 10, 1, 124, 0, 0, 0, 0, 0, 88, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 681, 469, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 1, 0, 7756, 17, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15672, 10149, 8881, 9424, 4234, 7423, 1938, 7815] avg=8192.0000 max=15672 max_rank=0 min=1938 min_rank=6 imbalance_ratio=1.913086 max_load_reduction=49864 max_load_reduction_pct=0.760864 moved_to_echo=21410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 202, 0, 0, 2660, 0, 0, 0, 342, 0, 0, 0, 0, 1381, 320, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 634, 0, 0, 0, 0, 0, 0, 0, 199, 0, 1170, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 22, 1545, 0, 0, 0, 12, 346, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 32, 0, 0, 6620, 0, 0, 1076, 0, 2, 0, 1445, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 35, 105, 0, 538, 0, 0, 0, 0, 7993, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7914, 0, 11, 298, 6947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4760, 2655, 0, 4251, 57, 0, 0, 6826, 0, 0, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5100, 3458, 2244, 1935, 9265, 9254, 15190, 19090] avg=8192.0000 max=19090 max_rank=7 min=1935 min_rank=3 imbalance_ratio=2.330322 max_load_reduction=46446 max_load_reduction_pct=0.708710 moved_to_echo=52799 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 0, 2651, 552, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 6682, 686, 0, 0, 0, 0, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 5580, 7374, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2049, 479, 0, 6404, 48, 7, 7, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 0, 3696, 75, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 165, 0, 0, 0, 63, 0, 0, 0, 3, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 18, 0, 0, 0, 0, 0, 0, 0, 5786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 10, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10841, 16516, 12976, 9019, 6156, 714, 7158, 2156] avg=8192.0000 max=16516 max_rank=1 min=714 min_rank=5 imbalance_ratio=2.016113 max_load_reduction=49020 max_load_reduction_pct=0.747986 moved_to_echo=16184 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2556, 0, 0, 0, 0, 0, 15, 0, 0, 3398, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 7909, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 1, 0, 0, 6893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 27, 14, 3573, 0, 0, 7104, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 2, 4245, 0, 0, 0, 0, 871, 0, 0, 0, 0, 0, 3123, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4767, 0, 0, 0, 7, 0, 0, 410, 2607, 13, 0, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7278, 5977, 8205, 7432, 10724, 8881, 7793, 9246] avg=8192.0000 max=10724 max_rank=4 min=5977 min_rank=1 imbalance_ratio=1.309082 max_load_reduction=54812 max_load_reduction_pct=0.836365 moved_to_echo=36644 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 2494, 0, 0, 0, 0, 0, 0, 0, 476, 4539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 4986, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 1248, 50, 0, 0, 7538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 766, 0, 0, 366, 0, 0, 205, 7773, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 34, 0, 0, 3545, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7179, 0, 10, 1, 124, 0, 0, 0, 0, 0, 88, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 681, 469, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 1, 0, 7756, 17, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15672, 10149, 8881, 9424, 4234, 7423, 1938, 7815] avg=8192.0000 max=15672 max_rank=0 min=1938 min_rank=6 imbalance_ratio=1.913086 max_load_reduction=49864 max_load_reduction_pct=0.760864 moved_to_echo=21410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 889, 0, 3591, 758, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 94, 0, 0, 0, 7254, 35, 0, 0, 0, 0, 0, 0, 2, 878, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2937, 305, 63, 0, 5419, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 3661, 61, 34, 0, 0, 7027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 2, 9, 0, 0, 3426, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3003, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 7948, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 0, 0, 91, 0, 0, 0, 115, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6472, 9843, 8747, 10901, 12682, 11301, 4149, 1441] avg=8192.0000 max=12682 max_rank=4 min=1441 min_rank=7 imbalance_ratio=1.548096 max_load_reduction=52854 max_load_reduction_pct=0.806488 moved_to_echo=29573 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 0, 0, 3, 0, 0, 0, 1, 0, 0, 784, 3, 711, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 6745, 0, 5486, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7167, 0, 0, 4144, 0, 3433, 0, 0, 381, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 1, 0, 0, 3213, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7788, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 4951, 2, 0, 0, 0, 901, 0, 9, 0, 0, 6931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 2829, 0, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1715, 1599, 12649, 15133, 8497, 8855, 12861, 4227] avg=8192.0000 max=15133 max_rank=3 min=1599 min_rank=1 imbalance_ratio=1.847290 max_load_reduction=50403 max_load_reduction_pct=0.769089 moved_to_echo=34440 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2556, 0, 0, 0, 0, 0, 15, 0, 0, 3398, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 7909, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 1, 0, 0, 6893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 27, 14, 3573, 0, 0, 7104, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 2, 4245, 0, 0, 0, 0, 871, 0, 0, 0, 0, 0, 3123, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4767, 0, 0, 0, 7, 0, 0, 410, 2607, 13, 0, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7278, 5977, 8205, 7432, 10724, 8881, 7793, 9246] avg=8192.0000 max=10724 max_rank=4 min=5977 min_rank=1 imbalance_ratio=1.309082 max_load_reduction=54812 max_load_reduction_pct=0.836365 moved_to_echo=36644 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 7168, 128, 12928, 19456, 0, 4352, 128, 7808, 128, 8320, 2816, 512, 640, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 8064, 128, 3840, 128, 128, 0, 14592, 8320, 13952, 0, 0, 0, 896, 0, 1536, 9216, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 2560, 128, 128, 0, 128, 0, 0, 5760, 10880, 2944, 16384, 128, 19328, 8064, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 7296, 0, 640, 4992, 7424, 12800, 1024, 6528, 512, 256, 24192, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 640, 384, 128, 20096, 5760, 512, 0, 0, 640, 128, 128, 896, 1280, 32256, 2688, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 7296, 0, 640, 4992, 7424, 12800, 1024, 6528, 512, 256, 24192, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 256, 128, 0, 1280, 4992, 1536, 128, 0, 128, 1024, 8448, 128, 768, 768, 46720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 128, 256, 128, 128, 128, 24576, 128, 128, 23296, 1152, 128, 512, 12032, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 4608, 21376, 128, 128, 256, 256, 896, 640, 128, 128, 128, 128, 33664, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -1755,40 +1755,40 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4850, 0, 3, 0, 0, 0, 33, 0, 4, 30, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7289, 2, 18, 0, 0, 610, 0, 0, 0, 0, 0, 8192, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 0, 0, 0, 0, 6179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 8147, 0, 0, 6, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 458, 1184, 0, 2082, 0, 0, 0, 0, 0, 2012, 8160, 0, 0, 22, 0, 0, 0, 0, 1711, 0, 0, 21, 0, 0, 1, 0, 0, 0, 8101, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6905, 6075, 0, 0, 1, 1360, 0, 0, 0, 0, 0, 7244, 0, 0, 0, 0, 0, 161, 92, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 10, 5262, 322, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1561, 1, 145, 0, 0, 24, 3759, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 8191, 0, 4693, 0, 0, 0, 1096, 0, 5035, 0, 0, 0, 50, 0, 412, 0, 745, 492, 49, 0, 105, 713, 47, 0, 0, 0, 0, 8083, 0, 0, 0, 2799] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 313, 8139, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 3, 6423, 0, 0, 0, 5951, 0, 0, 0, 0, 0, 8120, 1798, 0, 2, 0, 0, 3, 0, 0, 0, 0, 25, 26, 73, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 1, 64, 0, 0, 0, 0, 0, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 7575, 1013, 0, 0, 641, 0, 0, 0, 3775, 91, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 440, 0, 1804, 1283, 0, 0, 0, 0, 0, 2049, 0, 188, 672, 0, 0, 0, 0, 21, 0, 2, 0, 10, 2863, 8192, 5, 0, 28, 29, 0, 0, 0, 0, 0, 7641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6899, 0, 0, 0, 0, 87, 0, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 6546, 0, 9, 1, 0, 3351, 6892, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7320, 47, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3313, 0, 1, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 4672, 0, 693, 0, 1223, 0, 92, 0, 0, 8043, 18, 353, 0, 485, 1049, 0, 14, 1778, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 3943, 2044, 0, 0, 0, 272, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 8188, 0, 0, 0, 0, 0, 249, 0, 0, 2760, 0, 0, 0, 0, 0, 2, 0, 0, 0, 154, 3, 0, 3676, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 25, 0, 6404, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 29, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7845, 0, 2607, 0, 0, 7471, 8, 0, 25, 0, 1, 8050, 0, 0, 0, 0, 0, 2, 2, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, 2, 6, 0, 2, 0, 8057, 1, 0, 1, 0, 0, 1, 0, 20, 486, 0, 0, 0, 0, 0, 0, 2, 6527, 0, 142, 0, 0, 0, 16, 6935, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7, 0, 0, 90, 306, 0, 2, 7954, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6905, 6075, 0, 0, 1, 1360, 0, 0, 0, 0, 0, 7244, 0, 0, 0, 0, 0, 161, 92, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 10, 5262, 322, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1561, 1, 145, 0, 0, 24, 3759, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 8191, 0, 4693, 0, 0, 0, 1096, 0, 5035, 0, 0, 0, 50, 0, 412, 0, 745, 492, 49, 0, 105, 713, 47, 0, 0, 0, 0, 8083, 0, 0, 0, 2799] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4850, 0, 3, 0, 0, 0, 33, 0, 4, 30, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7289, 2, 18, 0, 0, 610, 0, 0, 0, 0, 0, 8192, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 0, 0, 0, 0, 6179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 8147, 0, 0, 6, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 458, 1184, 0, 2082, 0, 0, 0, 0, 0, 2012, 8160, 0, 0, 22, 0, 0, 0, 0, 1711, 0, 0, 21, 0, 0, 1, 0, 0, 0, 8101, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3182, 0, 0, 95, 0, 0, 5, 0, 0, 728, 5, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7667, 0, 0, 0, 0, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 5837, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 0, 163, 6634, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 613, 0, 0, 6, 0, 0, 0, 0, 900, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 0, 0, 0, 1, 8, 8063, 0, 0, 43, 6272] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 587, 0, 0, 0, 0, 0, 0, 0, 2, 32, 349, 0, 0, 0, 0, 8034, 0, 0, 0, 216, 5449, 0, 0, 12, 1, 0, 8065, 85, 0, 5, 0, 473, 0, 11, 0, 0, 0, 0, 0, 0, 0, 34, 239, 37, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7621, 2, 0, 0, 7, 0, 7944, 0, 0, 0, 0, 8, 7330, 0, 5, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8, 0, 2931, 0, 0, 5, 477] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 313, 8139, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 3, 6423, 0, 0, 0, 5951, 0, 0, 0, 0, 0, 8120, 1798, 0, 2, 0, 0, 3, 0, 0, 0, 0, 25, 26, 73, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 1, 64, 0, 0, 0, 0, 0, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 7575, 1013, 0, 0, 641, 0, 0, 0, 3775, 91, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2915, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1360, 0, 0, 0, 0, 0, 3506, 0, 0, 0, 0, 0, 161, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 10, 5262, 322, 0, 0, 1339, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1561, 1, 145, 0, 0, 24, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 2238, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 4693, 0, 0, 0, 1096, 0, 5035, 0, 0, 0, 50, 0, 412, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 492, 49, 0, 105, 713, 47, 0, 0, 0, 0, 7177, 0, 0, 0, 2799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4259, 8733, 5120, 7675, 3253, 4724, 19645, 12127] avg=8192.0000 max=19645 max_rank=6 min=3253 min_rank=4 imbalance_ratio=2.398071 max_load_reduction=45891 max_load_reduction_pct=0.700241 moved_to_echo=39749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3313, 0, 1, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 4672, 0, 693, 0, 1223, 0, 92, 0, 0, 8043, 18, 353, 0, 485, 1049, 0, 14, 1778, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 3943, 2044, 0, 0, 0, 272, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 8188, 0, 0, 0, 0, 0, 249, 0, 0, 2760, 0, 0, 0, 0, 0, 2, 0, 0, 0, 154, 3, 0, 3676, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 25, 0, 6404, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 29, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 3312, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7471, 8, 0, 25, 0, 1, 3895, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6184, 0, 0, 0, 1521, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, 2, 6, 0, 2, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 20, 486, 0, 0, 0, 0, 0, 0, 2, 4798, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6527, 0, 142, 0, 0, 0, 16, 6935, 0, 0, 0, 0, 0, 0, 2, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 90, 306, 0, 2, 7063, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4955, 5619, 11404, 10489, 3839, 7973, 13787, 7470] avg=8192.0000 max=13787 max_rank=6 min=3839 min_rank=4 imbalance_ratio=1.682983 max_load_reduction=51749 max_load_reduction_pct=0.789627 moved_to_echo=33069 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 313, 8139, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 2, 6423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5951, 0, 0, 0, 0, 0, 3929, 1798, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 26, 73, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 1, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 0, 0, 641, 0, 0, 0, 3775, 91, 0, 0, 0, 0, 0, 0, 0, 4511, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7265, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9958, 6718, 11683, 5998, 11014, 12718, 181, 7266] avg=8192.0000 max=12718 max_rank=5 min=181 min_rank=6 imbalance_ratio=1.552490 max_load_reduction=52818 max_load_reduction_pct=0.805939 moved_to_echo=31179 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4850, 0, 3, 0, 0, 0, 33, 0, 4, 3077, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2799, 2, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 3964, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 0, 0, 0, 1413, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 3296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 458, 4851, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 2082, 0, 0, 0, 0, 0, 2012, 8160, 0, 0, 22, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 0, 21, 0, 0, 1, 0, 0, 0, 7192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9484, 2855, 4592, 8254, 9706, 8091, 13629, 8925] avg=8192.0000 max=13629 max_rank=6 min=2855 min_rank=1 imbalance_ratio=1.663696 max_load_reduction=51907 max_load_reduction_pct=0.792038 moved_to_echo=40351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 440, 0, 1804, 1283, 0, 0, 0, 0, 0, 1208, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2049, 0, 188, 672, 0, 0, 0, 0, 21, 0, 2, 0, 10, 1100, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 29, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6899, 0, 0, 0, 0, 87, 0, 7635, 0, 0, 0, 555, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 6546, 0, 9, 1, 0, 3351, 2788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4104, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6500, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 12239, 3755, 15844, 12724, 7108, 1169, 6547] avg=8192.0000 max=15844 max_rank=3 min=1169 min_rank=6 imbalance_ratio=1.934082 max_load_reduction=49692 max_load_reduction_pct=0.758240 moved_to_echo=27548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3182, 0, 0, 95, 0, 0, 5, 0, 0, 728, 5, 0, 0, 0, 0, 0, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7667, 0, 0, 0, 0, 0, 3960, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 5837, 0, 0, 0, 0, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 0, 163, 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 613, 0, 3950, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 900, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 0, 0, 0, 1, 8, 7159, 0, 0, 43, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5531, 8086, 11628, 6652, 2860, 7274, 9103, 14402] avg=8192.0000 max=14402 max_rank=7 min=2860 min_rank=4 imbalance_ratio=1.758057 max_load_reduction=51134 max_load_reduction_pct=0.780243 moved_to_echo=33639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2915, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1360, 0, 0, 0, 0, 0, 3506, 0, 0, 0, 0, 0, 161, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 10, 5262, 322, 0, 0, 1339, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1561, 1, 145, 0, 0, 24, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 2238, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 4693, 0, 0, 0, 1096, 0, 5035, 0, 0, 0, 50, 0, 412, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 492, 49, 0, 105, 713, 47, 0, 0, 0, 0, 7177, 0, 0, 0, 2799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4259, 8733, 5120, 7675, 3253, 4724, 19645, 12127] avg=8192.0000 max=19645 max_rank=6 min=3253 min_rank=4 imbalance_ratio=2.398071 max_load_reduction=45891 max_load_reduction_pct=0.700241 moved_to_echo=39749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3313, 0, 1, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 4672, 0, 693, 0, 1223, 0, 92, 0, 0, 3087, 18, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 1049, 0, 14, 1778, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 3943, 2044, 0, 0, 1560, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 3312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 2760, 0, 0, 0, 0, 0, 2, 0, 4876, 2653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 3, 0, 3676, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 25, 0, 5686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 17827, 7206, 8478, 3597, 10540, 3972, 5713] avg=8192.0000 max=17827 max_rank=1 min=3597 min_rank=4 imbalance_ratio=2.176147 max_load_reduction=47709 max_load_reduction_pct=0.727982 moved_to_echo=23822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4850, 0, 3, 0, 0, 0, 33, 0, 4, 3077, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2799, 2, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 3964, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 0, 0, 0, 1413, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 3296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 458, 4851, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 2082, 0, 0, 0, 0, 0, 2012, 8160, 0, 0, 22, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 0, 21, 0, 0, 1, 0, 0, 0, 7192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9484, 2855, 4592, 8254, 9706, 8091, 13629, 8925] avg=8192.0000 max=13629 max_rank=6 min=2855 min_rank=1 imbalance_ratio=1.663696 max_load_reduction=51907 max_load_reduction_pct=0.792038 moved_to_echo=40351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 587, 0, 3396, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 349, 0, 0, 0, 0, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 5449, 0, 0, 12, 1, 0, 3896, 85, 0, 5, 0, 473, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 239, 37, 0, 0, 0, 0, 2, 0, 1562, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 2, 0, 0, 7, 0, 7944, 0, 0, 0, 0, 8, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4369, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8, 0, 2594, 0, 0, 5, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16996, 3459, 10148, 2146, 18543, 7108, 4050, 3086] avg=8192.0000 max=18543 max_rank=4 min=2146 min_rank=3 imbalance_ratio=2.263550 max_load_reduction=46993 max_load_reduction_pct=0.717056 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 29, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 3312, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7471, 8, 0, 25, 0, 1, 3895, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6184, 0, 0, 0, 1521, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, 2, 6, 0, 2, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 20, 486, 0, 0, 0, 0, 0, 0, 2, 4798, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6527, 0, 142, 0, 0, 0, 16, 6935, 0, 0, 0, 0, 0, 0, 2, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 90, 306, 0, 2, 7063, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4955, 5619, 11404, 10489, 3839, 7973, 13787, 7470] avg=8192.0000 max=13787 max_rank=6 min=3839 min_rank=4 imbalance_ratio=1.682983 max_load_reduction=51749 max_load_reduction_pct=0.789627 moved_to_echo=33069 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3182, 0, 0, 95, 0, 0, 5, 0, 0, 728, 5, 0, 0, 0, 0, 0, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7667, 0, 0, 0, 0, 0, 3960, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 5837, 0, 0, 0, 0, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 0, 163, 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 613, 0, 3950, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 900, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 0, 0, 0, 1, 8, 7159, 0, 0, 43, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5531, 8086, 11628, 6652, 2860, 7274, 9103, 14402] avg=8192.0000 max=14402 max_rank=7 min=2860 min_rank=4 imbalance_ratio=1.758057 max_load_reduction=51134 max_load_reduction_pct=0.780243 moved_to_echo=33639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 313, 8139, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 2, 6423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5951, 0, 0, 0, 0, 0, 3929, 1798, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 26, 73, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 1, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 0, 0, 641, 0, 0, 0, 3775, 91, 0, 0, 0, 0, 0, 0, 0, 4511, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7265, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9958, 6718, 11683, 5998, 11014, 12718, 181, 7266] avg=8192.0000 max=12718 max_rank=5 min=181 min_rank=6 imbalance_ratio=1.552490 max_load_reduction=52818 max_load_reduction_pct=0.805939 moved_to_echo=31179 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1024, 512, 128, 768, 0, 256, 128, 3840, 3456, 0, 0, 0, 128, 0, 640, 512, 33792, 21120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[16000, 128, 7168, 1024, 0, 3840, 2048, 19072, 7168, 8192, 0, 0, 128, 128, 512, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 6528, 128, 7680, 128, 128, 640, 128, 128, 24064, 128, 256, 128, 256, 3584, 22912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[6528, 0, 128, 7808, 128, 128, 384, 13440, 0, 2560, 1408, 128, 3968, 256, 640, 128, 17408, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[17920, 0, 256, 768, 4736, 0, 768, 128, 1664, 128, 128, 0, 384, 15744, 20736, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 768, 29696, 128, 128, 1920, 128, 128, 30848, 1920, 128, 128, 0, 512, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[17920, 0, 256, 768, 4736, 0, 768, 128, 1664, 128, 128, 0, 384, 15744, 20736, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[6528, 0, 128, 7808, 128, 128, 384, 13440, 0, 2560, 1408, 128, 3968, 256, 640, 128, 17408, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 2304, 128, 128, 128, 7040, 0, 128, 256, 128, 384, 128, 40192, 2432, 128, 128, 8064, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[768, 2304, 128, 0, 256, 1664, 128, 128, 384, 128, 128, 50688, 128, 128, 128, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[768, 2304, 128, 0, 256, 1664, 128, 128, 384, 128, 128, 50688, 128, 128, 128, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True @@ -1828,34 +1828,32 @@ WARNING:megatron.core.rerun_state_machine:Result validation enabled warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") - [2026-08-26 16:18:03] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 37726.5 | throughput per GPU (TFLOP/s/GPU): 4.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115992E+01 | loss scale: 1.0 | grad norm: 44.778 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.85 - + [2026-08-26 16:27:11] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38079.8 | throughput per GPU (TFLOP/s/GPU): 4.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115992E+01 | loss scale: 1.0 | grad norm: 44.778 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.85 Number of parameters in embedding layers in billions: 0.41 Total number of parameters in billions: 30.26 Number of parameters in most loaded shard in billions: 30.2590 Theoretical memory footprints: weight and optimizer=216428.96 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 81277.55859375 | max allocated: 96153.076171875 | reserved: 102124.0 | max reserved: 102124.0 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3707, 68, 25, 436, 236, 457, 12, 20, 63, 154, 2215, 412, 1272, 118, 112, 806, 148, 2480, 259, 9, 1318, 50, 172, 1656, 541, 60, 14, 1549, 506, 2068, 127, 270, 461, 1843, 175, 104, 250, 432, 98, 38, 1644, 51, 460, 3108, 25, 64, 94, 596, 204, 70, 4, 118, 8, 414, 53, 24, 170, 94, 41, 392, 1287, 11, 899, 195, 1338, 33, 8, 13, 1242, 13, 120, 1706, 748, 149, 1926, 142, 457, 19, 535, 772, 30, 13, 3, 24, 59, 7, 876, 548, 14, 23, 474, 12, 40, 166, 564, 75, 648, 123, 145, 1877, 177, 0, 103, 72, 549, 72, 509, 3597, 23, 196, 44, 39, 745, 109, 12, 131, 12, 751, 364, 191, 1297, 46, 2120, 408, 108, 237, 3, 3912] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[4024, 230, 10, 431, 54, 540, 53, 20, 50, 374, 1890, 410, 1445, 100, 201, 573, 81, 1542, 170, 15, 827, 27, 112, 1216, 691, 63, 50, 1491, 342, 1660, 30, 261, 641, 1675, 122, 36, 87, 357, 101, 35, 952, 19, 547, 2624, 22, 5, 136, 994, 272, 152, 1, 36, 2, 385, 150, 18, 113, 32, 18, 300, 1277, 68, 434, 128, 2704, 18, 15, 9, 1934, 2, 33, 1459, 980, 69, 1725, 261, 349, 10, 544, 771, 22, 16, 26, 28, 35, 36, 1197, 973, 27, 44, 1112, 16, 38, 103, 253, 67, 1349, 117, 17, 1989, 83, 6, 57, 82, 356, 93, 431, 5244, 27, 126, 53, 47, 923, 26, 14, 151, 1, 583, 147, 325, 1557, 39, 1758, 332, 377, 188, 5, 3435] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4396, 162, 2, 510, 136, 840, 9, 2, 52, 290, 1562, 547, 1308, 107, 203, 866, 144, 2237, 274, 13, 1033, 18, 98, 1615, 424, 66, 42, 1913, 390, 988, 53, 195, 428, 1255, 57, 47, 80, 467, 48, 4, 1189, 16, 719, 3433, 25, 30, 152, 818, 210, 82, 0, 19, 0, 354, 118, 18, 118, 19, 31, 502, 1077, 20, 340, 100, 2324, 26, 14, 19, 2136, 7, 33, 1883, 1048, 130, 1811, 76, 444, 2, 752, 1031, 26, 20, 2, 27, 42, 5, 761, 1036, 35, 24, 613, 3, 15, 185, 262, 79, 926, 115, 50, 1860, 102, 3, 37, 98, 248, 67, 660, 4953, 33, 55, 47, 95, 681, 37, 10, 81, 3, 683, 120, 295, 1193, 37, 1885, 413, 267, 310, 41, 2989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3698, 93, 55, 467, 125, 699, 35, 52, 31, 249, 1681, 253, 1339, 106, 193, 387, 306, 2012, 209, 20, 489, 10, 58, 1558, 678, 60, 34, 1831, 692, 2065, 121, 633, 782, 1658, 100, 68, 166, 189, 183, 67, 1089, 21, 619, 2126, 39, 9, 207, 893, 183, 210, 4, 72, 6, 299, 103, 24, 129, 24, 17, 517, 1055, 16, 562, 287, 1992, 25, 8, 14, 1742, 3, 66, 1572, 865, 103, 1247, 225, 481, 6, 550, 1364, 55, 13, 2, 42, 52, 10, 1289, 676, 59, 48, 934, 2, 29, 336, 272, 66, 1545, 258, 77, 1387, 194, 2, 69, 82, 280, 143, 343, 4992, 30, 134, 114, 56, 813, 145, 9, 153, 8, 211, 254, 443, 1441, 88, 1838, 438, 83, 486, 5, 3304] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[4466, 59, 13, 293, 131, 445, 7, 34, 33, 278, 1154, 284, 2095, 194, 227, 516, 189, 2176, 305, 6, 695, 39, 73, 1400, 589, 25, 53, 2019, 1037, 2078, 27, 378, 682, 1571, 61, 32, 119, 433, 65, 32, 842, 11, 717, 2891, 31, 8, 109, 882, 225, 94, 8, 74, 2, 374, 43, 16, 177, 73, 31, 620, 939, 13, 449, 298, 2401, 27, 0, 6, 1954, 10, 37, 1160, 938, 179, 1729, 158, 516, 6, 494, 798, 62, 4, 0, 43, 105, 12, 770, 432, 18, 61, 1014, 3, 74, 138, 547, 79, 1442, 171, 18, 1677, 123, 5, 21, 115, 567, 81, 905, 4220, 18, 135, 62, 37, 744, 71, 13, 75, 2, 904, 237, 224, 1129, 17, 1387, 413, 255, 210, 6, 3237] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3882, 82, 42, 464, 292, 453, 14, 19, 52, 288, 1082, 1084, 1511, 110, 589, 281, 175, 2286, 193, 34, 1090, 68, 177, 1369, 739, 153, 68, 2335, 776, 1391, 93, 286, 843, 1450, 81, 12, 357, 415, 160, 11, 977, 26, 501, 2988, 25, 33, 152, 624, 223, 223, 7, 48, 7, 243, 40, 41, 94, 77, 57, 725, 1249, 25, 628, 159, 2166, 26, 5, 18, 1431, 79, 125, 1494, 777, 164, 1409, 89, 388, 15, 625, 881, 43, 31, 3, 26, 34, 1, 649, 938, 11, 43, 617, 10, 24, 225, 239, 129, 801, 120, 76, 3116, 157, 3, 91, 254, 566, 106, 762, 4275, 30, 192, 77, 25, 781, 35, 21, 58, 7, 278, 146, 263, 1309, 44, 1269, 339, 147, 249, 8, 3237] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[4024, 230, 10, 431, 54, 540, 53, 20, 50, 374, 1890, 410, 1445, 100, 201, 573, 81, 1542, 170, 15, 827, 27, 112, 1216, 691, 63, 50, 1491, 342, 1660, 30, 261, 641, 1675, 122, 36, 87, 357, 101, 35, 952, 19, 547, 2624, 22, 5, 136, 994, 272, 152, 1, 36, 2, 385, 150, 18, 113, 32, 18, 300, 1277, 68, 434, 128, 2704, 18, 15, 9, 1934, 2, 33, 1459, 980, 69, 1725, 261, 349, 10, 544, 771, 22, 16, 26, 28, 35, 36, 1197, 973, 27, 44, 1112, 16, 38, 103, 253, 67, 1349, 117, 17, 1989, 83, 6, 57, 82, 356, 93, 431, 5244, 27, 126, 53, 47, 923, 26, 14, 151, 1, 583, 147, 325, 1557, 39, 1758, 332, 377, 188, 5, 3435] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3707, 68, 25, 436, 236, 457, 12, 20, 63, 154, 2215, 412, 1272, 118, 112, 806, 148, 2480, 259, 9, 1318, 50, 172, 1656, 541, 60, 14, 1549, 506, 2068, 127, 270, 461, 1843, 175, 104, 250, 432, 98, 38, 1644, 51, 460, 3108, 25, 64, 94, 596, 204, 70, 4, 118, 8, 414, 53, 24, 170, 94, 41, 392, 1287, 11, 899, 195, 1338, 33, 8, 13, 1242, 13, 120, 1706, 748, 149, 1926, 142, 457, 19, 535, 772, 30, 13, 3, 24, 59, 7, 876, 548, 14, 23, 474, 12, 40, 166, 564, 75, 648, 123, 145, 1877, 177, 0, 103, 72, 549, 72, 509, 3597, 23, 196, 44, 39, 745, 109, 12, 131, 12, 751, 364, 191, 1297, 46, 2120, 408, 108, 237, 3, 3912] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3495, 61, 42, 259, 178, 571, 6, 29, 70, 214, 1683, 441, 1991, 113, 353, 267, 114, 2618, 97, 44, 527, 74, 146, 1576, 766, 59, 36, 1579, 332, 1169, 91, 343, 461, 1856, 53, 87, 85, 415, 62, 71, 538, 71, 827, 3238, 39, 8, 326, 849, 260, 214, 4, 37, 6, 132, 79, 8, 33, 39, 51, 652, 2002, 32, 217, 93, 2814, 22, 3, 15, 1553, 16, 61, 1072, 765, 89, 2009, 122, 409, 18, 372, 803, 25, 18, 11, 66, 98, 17, 954, 848, 20, 75, 1080, 9, 12, 261, 275, 96, 1919, 179, 13, 2969, 56, 3, 11, 74, 424, 136, 697, 4726, 25, 86, 69, 75, 483, 25, 31, 22, 4, 555, 133, 408, 1145, 87, 1009, 365, 252, 263, 18, 3077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4396, 162, 2, 510, 136, 840, 9, 2, 52, 290, 1562, 547, 1308, 107, 203, 866, 144, 2237, 274, 13, 1033, 18, 98, 1615, 424, 66, 42, 1913, 390, 988, 53, 195, 428, 1255, 57, 47, 80, 467, 48, 4, 1189, 16, 719, 3433, 25, 30, 152, 818, 210, 82, 0, 19, 0, 354, 118, 18, 118, 19, 31, 502, 1077, 20, 340, 100, 2324, 26, 14, 19, 2136, 7, 33, 1883, 1048, 130, 1811, 76, 444, 2, 752, 1031, 26, 20, 2, 27, 42, 5, 761, 1036, 35, 24, 613, 3, 15, 185, 262, 79, 926, 115, 50, 1860, 102, 3, 37, 98, 248, 67, 660, 4953, 33, 55, 47, 95, 681, 37, 10, 81, 3, 683, 120, 295, 1193, 37, 1885, 413, 267, 310, 41, 2989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3882, 82, 42, 464, 292, 453, 14, 19, 52, 288, 1082, 1084, 1511, 110, 589, 281, 175, 2286, 193, 34, 1090, 68, 177, 1369, 739, 153, 68, 2335, 776, 1391, 93, 286, 843, 1450, 81, 12, 357, 415, 160, 11, 977, 26, 501, 2988, 25, 33, 152, 624, 223, 223, 7, 48, 7, 243, 40, 41, 94, 77, 57, 725, 1249, 25, 628, 159, 2166, 26, 5, 18, 1431, 79, 125, 1494, 777, 164, 1409, 89, 388, 15, 625, 881, 43, 31, 3, 26, 34, 1, 649, 938, 11, 43, 617, 10, 24, 225, 239, 129, 801, 120, 76, 3116, 157, 3, 91, 254, 566, 106, 762, 4275, 30, 192, 77, 25, 781, 35, 21, 58, 7, 278, 146, 263, 1309, 44, 1269, 339, 147, 249, 8, 3237] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3835, 120, 26, 419, 113, 348, 13, 12, 77, 263, 2564, 504, 1737, 90, 230, 435, 132, 2145, 484, 11, 1081, 28, 87, 1274, 508, 40, 76, 2035, 749, 2355, 82, 100, 510, 2885, 142, 30, 227, 427, 48, 69, 733, 63, 561, 2591, 40, 13, 141, 478, 388, 90, 1, 91, 2, 927, 33, 19, 240, 40, 50, 383, 924, 24, 519, 163, 1855, 20, 25, 1, 1356, 14, 73, 1108, 725, 45, 2149, 254, 659, 4, 444, 1094, 38, 9, 11, 16, 41, 5, 457, 562, 20, 31, 758, 15, 49, 100, 374, 112, 1227, 159, 107, 2249, 150, 0, 78, 300, 365, 114, 413, 3833, 25, 221, 41, 45, 674, 48, 18, 92, 11, 239, 190, 417, 1377, 54, 1997, 368, 155, 171, 10, 3139] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1770, 68, 25, 436, 236, 457, 12, 20, 63, 154, 2215, 412, 1272, 118, 112, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 259, 9, 1318, 50, 172, 1656, 541, 60, 14, 1536, 506, 2068, 127, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 1843, 175, 104, 250, 432, 98, 38, 1644, 51, 460, 2523, 25, 64, 94, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 70, 4, 118, 8, 414, 53, 24, 170, 94, 41, 392, 1287, 11, 899, 195, 1285, 1395, 1112, 585, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 33, 8, 13, 1242, 13, 120, 1706, 748, 149, 1926, 142, 457, 19, 535, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 13, 3, 24, 59, 7, 876, 548, 14, 23, 474, 12, 40, 166, 564, 75, 2480, 1265, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 123, 145, 1877, 177, 0, 103, 72, 549, 72, 509, 2312, 23, 196, 44, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 109, 12, 131, 12, 751, 364, 191, 1297, 46, 2120, 408, 108, 237, 3, 2800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8734, 8858, 8374, 7956, 7215, 6889, 9334] avg=8192.0000 max=9334 max_rank=7 min=6889 min_rank=6 imbalance_ratio=1.139404 max_load_reduction=56202 max_load_reduction_pct=0.857574 moved_to_echo=31394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1922, 230, 10, 431, 54, 540, 53, 20, 50, 374, 1890, 410, 1445, 100, 201, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 170, 15, 827, 27, 112, 1216, 691, 63, 50, 1478, 342, 1660, 30, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 1675, 122, 36, 87, 357, 101, 35, 952, 19, 547, 2131, 22, 5, 136, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 152, 1, 36, 2, 385, 150, 18, 113, 32, 18, 300, 1277, 68, 434, 128, 1873, 1514, 976, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 18, 15, 9, 1934, 2, 33, 1459, 980, 69, 1725, 261, 349, 10, 544, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 16, 26, 28, 35, 36, 1197, 973, 27, 44, 1112, 16, 38, 103, 253, 67, 1542, 2557, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 117, 17, 1989, 83, 6, 57, 82, 356, 93, 431, 3371, 27, 126, 53, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 26, 14, 151, 1, 583, 147, 325, 1557, 39, 1758, 332, 377, 188, 5, 2459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 7023, 7860, 8255, 8326, 8680, 8204, 8885] avg=8192.0000 max=8885 max_rank=7 min=7023 min_rank=1 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=34095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2099, 162, 2, 510, 136, 840, 9, 2, 52, 290, 1562, 547, 1308, 107, 203, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 274, 13, 1033, 18, 98, 1615, 424, 66, 42, 1896, 390, 988, 53, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 1255, 57, 47, 80, 467, 48, 4, 1189, 16, 719, 2787, 25, 30, 152, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 82, 0, 19, 0, 354, 118, 18, 118, 19, 31, 502, 1077, 20, 340, 100, 1769, 1654, 849, 646, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 26, 14, 19, 2136, 7, 33, 1883, 1048, 130, 1811, 76, 444, 2, 752, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 20, 2, 27, 42, 5, 761, 1036, 35, 24, 613, 3, 15, 185, 262, 79, 2237, 2198, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 115, 50, 1860, 102, 3, 37, 98, 248, 67, 660, 3184, 33, 55, 47, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, 37, 10, 81, 3, 683, 120, 295, 1193, 37, 1885, 413, 267, 310, 41, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8695, 7249, 8122, 7943, 9538, 8213, 7580, 8196] avg=8192.0000 max=9538 max_rank=4 min=7249 min_rank=1 imbalance_ratio=1.164307 max_load_reduction=55998 max_load_reduction_pct=0.854462 moved_to_echo=33527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1765, 93, 55, 467, 125, 699, 35, 52, 31, 249, 1681, 253, 1339, 106, 193, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 209, 20, 489, 10, 58, 1558, 678, 60, 34, 1815, 692, 2065, 121, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 1658, 100, 68, 166, 189, 183, 67, 1089, 21, 619, 1726, 39, 9, 207, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 210, 4, 72, 6, 299, 103, 24, 129, 24, 17, 517, 1055, 16, 562, 287, 1783, 1392, 939, 400, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 25, 8, 14, 1742, 3, 66, 1572, 865, 103, 1247, 225, 481, 6, 550, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 13, 2, 42, 52, 10, 1289, 676, 59, 48, 934, 2, 29, 336, 272, 66, 2012, 1884, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 258, 77, 1387, 194, 2, 69, 82, 280, 143, 343, 3209, 30, 134, 114, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 145, 9, 153, 8, 211, 254, 443, 1441, 88, 1838, 438, 83, 486, 5, 2365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7530, 8748, 7816, 8038, 8379, 8322, 7923, 8780] avg=8192.0000 max=8780 max_rank=7 min=7530 min_rank=0 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=33404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2124, 59, 13, 293, 131, 445, 7, 34, 33, 278, 1154, 284, 2095, 194, 227, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 305, 6, 695, 39, 73, 1400, 589, 25, 53, 1998, 1037, 2078, 27, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 1571, 61, 32, 119, 433, 65, 32, 842, 11, 717, 2344, 31, 8, 109, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 94, 8, 74, 2, 374, 43, 16, 177, 73, 31, 620, 939, 13, 449, 298, 1513, 1686, 925, 547, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 27, 0, 6, 1954, 10, 37, 1160, 938, 179, 1729, 158, 516, 6, 494, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4, 0, 43, 105, 12, 770, 432, 18, 61, 1014, 3, 74, 138, 547, 79, 2176, 2274, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1442, 171, 18, 1677, 123, 5, 21, 115, 567, 81, 905, 2707, 18, 135, 62, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 71, 13, 75, 2, 904, 237, 224, 1129, 17, 1387, 413, 255, 210, 6, 2312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7887, 8892, 7939, 8128, 8139, 8468, 8084, 7999] avg=8192.0000 max=8892 max_rank=1 min=7887 min_rank=0 imbalance_ratio=1.085449 max_load_reduction=56644 max_load_reduction_pct=0.864319 moved_to_echo=32690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1922, 230, 10, 431, 54, 540, 53, 20, 50, 374, 1890, 410, 1445, 100, 201, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 170, 15, 827, 27, 112, 1216, 691, 63, 50, 1478, 342, 1660, 30, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 1675, 122, 36, 87, 357, 101, 35, 952, 19, 547, 2131, 22, 5, 136, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 152, 1, 36, 2, 385, 150, 18, 113, 32, 18, 300, 1277, 68, 434, 128, 1873, 1514, 976, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 18, 15, 9, 1934, 2, 33, 1459, 980, 69, 1725, 261, 349, 10, 544, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 16, 26, 28, 35, 36, 1197, 973, 27, 44, 1112, 16, 38, 103, 253, 67, 1542, 2557, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 117, 17, 1989, 83, 6, 57, 82, 356, 93, 431, 3371, 27, 126, 53, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 26, 14, 151, 1, 583, 147, 325, 1557, 39, 1758, 332, 377, 188, 5, 2459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 7023, 7860, 8255, 8326, 8680, 8204, 8885] avg=8192.0000 max=8885 max_rank=7 min=7023 min_rank=1 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=34095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1770, 68, 25, 436, 236, 457, 12, 20, 63, 154, 2215, 412, 1272, 118, 112, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 259, 9, 1318, 50, 172, 1656, 541, 60, 14, 1536, 506, 2068, 127, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 1843, 175, 104, 250, 432, 98, 38, 1644, 51, 460, 2523, 25, 64, 94, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 70, 4, 118, 8, 414, 53, 24, 170, 94, 41, 392, 1287, 11, 899, 195, 1285, 1395, 1112, 585, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 33, 8, 13, 1242, 13, 120, 1706, 748, 149, 1926, 142, 457, 19, 535, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 13, 3, 24, 59, 7, 876, 548, 14, 23, 474, 12, 40, 166, 564, 75, 2480, 1265, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 123, 145, 1877, 177, 0, 103, 72, 549, 72, 509, 2312, 23, 196, 44, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 109, 12, 131, 12, 751, 364, 191, 1297, 46, 2120, 408, 108, 237, 3, 2800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8734, 8858, 8374, 7956, 7215, 6889, 9334] avg=8192.0000 max=9334 max_rank=7 min=6889 min_rank=6 imbalance_ratio=1.139404 max_load_reduction=56202 max_load_reduction_pct=0.857574 moved_to_echo=31394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1853, 82, 42, 464, 292, 453, 14, 19, 52, 288, 1082, 1084, 1511, 110, 589, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 193, 34, 1090, 68, 177, 1369, 739, 153, 68, 2315, 776, 1391, 93, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 1450, 81, 12, 357, 415, 160, 11, 977, 26, 501, 2426, 25, 33, 152, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 223, 7, 48, 7, 243, 40, 41, 94, 77, 57, 725, 1249, 25, 628, 159, 1527, 1461, 920, 562, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 26, 5, 18, 1431, 79, 125, 1494, 777, 164, 1409, 89, 388, 15, 625, 881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 31, 3, 26, 34, 1, 649, 938, 11, 43, 617, 10, 24, 225, 239, 129, 2286, 2048, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 120, 76, 3116, 157, 3, 91, 254, 566, 106, 762, 2748, 30, 192, 77, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 35, 21, 58, 7, 278, 146, 263, 1309, 44, 1269, 339, 147, 249, 8, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8927, 8093, 8336, 7644, 7925, 9124, 7271] avg=8192.0000 max=9124 max_rank=6 min=7271 min_rank=7 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=31964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2099, 162, 2, 510, 136, 840, 9, 2, 52, 290, 1562, 547, 1308, 107, 203, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 274, 13, 1033, 18, 98, 1615, 424, 66, 42, 1896, 390, 988, 53, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 1255, 57, 47, 80, 467, 48, 4, 1189, 16, 719, 2787, 25, 30, 152, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 82, 0, 19, 0, 354, 118, 18, 118, 19, 31, 502, 1077, 20, 340, 100, 1769, 1654, 849, 646, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 26, 14, 19, 2136, 7, 33, 1883, 1048, 130, 1811, 76, 444, 2, 752, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 20, 2, 27, 42, 5, 761, 1036, 35, 24, 613, 3, 15, 185, 262, 79, 2237, 2198, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 115, 50, 1860, 102, 3, 37, 98, 248, 67, 660, 3184, 33, 55, 47, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, 37, 10, 81, 3, 683, 120, 295, 1193, 37, 1885, 413, 267, 310, 41, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8695, 7249, 8122, 7943, 9538, 8213, 7580, 8196] avg=8192.0000 max=9538 max_rank=4 min=7249 min_rank=1 imbalance_ratio=1.164307 max_load_reduction=55998 max_load_reduction_pct=0.854462 moved_to_echo=33527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1831, 120, 26, 419, 113, 348, 13, 12, 77, 263, 2564, 504, 1737, 90, 230, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 484, 11, 1081, 28, 87, 1274, 508, 40, 76, 2017, 749, 2355, 82, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 2885, 142, 30, 227, 427, 48, 69, 733, 63, 561, 2104, 40, 13, 141, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 90, 1, 91, 2, 927, 33, 19, 240, 40, 50, 383, 924, 24, 519, 163, 1369, 1443, 892, 487, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 20, 25, 1, 1356, 14, 73, 1108, 725, 45, 2149, 254, 659, 4, 444, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 9, 11, 16, 41, 5, 457, 562, 20, 31, 758, 15, 49, 100, 374, 112, 2145, 1754, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 159, 107, 2249, 150, 0, 78, 300, 365, 114, 413, 2464, 25, 221, 41, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 674, 48, 18, 92, 11, 239, 190, 417, 1377, 54, 1997, 368, 155, 171, 10, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8782, 9024, 8471, 8103, 8072, 7058, 7958, 8068] avg=8192.0000 max=9024 max_rank=1 min=7058 min_rank=5 imbalance_ratio=1.101562 max_load_reduction=56512 max_load_reduction_pct=0.862305 moved_to_echo=31156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1831, 120, 26, 419, 113, 348, 13, 12, 77, 263, 2564, 504, 1737, 90, 230, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 484, 11, 1081, 28, 87, 1274, 508, 40, 76, 2017, 749, 2355, 82, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 2885, 142, 30, 227, 427, 48, 69, 733, 63, 561, 2104, 40, 13, 141, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 90, 1, 91, 2, 927, 33, 19, 240, 40, 50, 383, 924, 24, 519, 163, 1369, 1443, 892, 487, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 20, 25, 1, 1356, 14, 73, 1108, 725, 45, 2149, 254, 659, 4, 444, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 9, 11, 16, 41, 5, 457, 562, 20, 31, 758, 15, 49, 100, 374, 112, 2145, 1754, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 159, 107, 2249, 150, 0, 78, 300, 365, 114, 413, 2464, 25, 221, 41, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 674, 48, 18, 92, 11, 239, 190, 417, 1377, 54, 1997, 368, 155, 171, 10, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8782, 9024, 8471, 8103, 8072, 7058, 7958, 8068] avg=8192.0000 max=9024 max_rank=1 min=7058 min_rank=5 imbalance_ratio=1.101562 max_load_reduction=56512 max_load_reduction_pct=0.862305 moved_to_echo=31156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1669, 61, 42, 259, 178, 571, 6, 29, 70, 214, 1683, 441, 1991, 113, 353, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 97, 44, 527, 74, 146, 1576, 766, 59, 36, 1565, 332, 1169, 91, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 1856, 53, 87, 85, 415, 62, 71, 538, 71, 827, 2629, 39, 8, 326, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 214, 4, 37, 6, 132, 79, 8, 33, 39, 51, 652, 2002, 32, 217, 93, 1688, 1315, 874, 609, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 22, 3, 15, 1553, 16, 61, 1072, 765, 89, 2009, 122, 409, 18, 372, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 18, 11, 66, 98, 17, 954, 848, 20, 75, 1080, 9, 12, 261, 275, 96, 2618, 2661, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 179, 13, 2969, 56, 3, 11, 74, 424, 136, 697, 3038, 25, 86, 69, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 25, 31, 22, 4, 555, 133, 408, 1145, 87, 1009, 365, 252, 263, 18, 2203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7947, 6939, 8377, 8359, 7482, 9655, 9774, 7003] avg=8192.0000 max=9774 max_rank=6 min=6939 min_rank=1 imbalance_ratio=1.193115 max_load_reduction=55762 max_load_reduction_pct=0.850861 moved_to_echo=33914 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 @@ -1867,54 +1865,56 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[2048, 1152, 128, 512, 128, 3200, 640, 256, 1152, 512, 384, 4096, 9856, 256, 4096, 1536, 12928, 11904, 7552, 4352, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[384, 128, 128, 384, 512, 128, 7040, 6016, 256, 384, 6656, 128, 384, 1536, 2816, 768, 17536, 16768, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[5888, 512, 128, 768, 128, 4224, 1664, 2688, 10496, 512, 13312, 3200, 1664, 2176, 128, 18944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1408, 0, 2048, 256, 7168, 384, 1024, 11776, 4992, 640, 384, 14720, 4864, 13824, 640, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[15104, 896, 256, 3328, 1280, 4480, 256, 256, 512, 2176, 13952, 3968, 12800, 1024, 2176, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4864, 14208, 896, 512, 1408, 3200, 768, 384, 8064, 384, 4992, 18688, 256, 256, 1408, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1024, 256, 128, 128, 13440, 256, 640, 11520, 6912, 1024, 14080, 1408, 3712, 128, 4352, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4864, 14208, 896, 512, 1408, 3200, 768, 384, 8064, 384, 4992, 18688, 256, 256, 1408, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[9984, 1280, 512, 17152, 1152, 128, 512, 1152, 3456, 896, 4736, 23040, 256, 1152, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[171, 1250, 1, 34, 0, 0, 0, 0, 0, 216, 8019, 4, 22, 155, 0, 0, 40, 5, 1, 2873, 0, 0, 0, 0, 0, 0, 2, 865, 4, 0, 0, 84, 8, 0, 0, 19, 4, 11, 0, 2, 0, 103, 149, 0, 0, 683, 0, 0, 20, 79, 0, 0, 6417, 0, 12, 6010, 0, 928, 4586, 0, 0, 1021, 0, 644, 408, 0, 0, 0, 13, 8103, 22, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 519, 41, 1, 106, 16, 3517, 0, 6692, 0, 277, 6, 1, 0, 677, 0, 27, 1, 0, 1, 0, 2594, 0, 0, 0, 66, 0, 0, 36, 4, 0, 0, 0, 0, 0, 0, 0, 6, 248, 7, 0, 0, 5, 0, 7682, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[83, 4768, 1, 3233, 85, 0, 15, 3, 0, 102, 7615, 0, 0, 575, 3, 12, 1, 0, 2, 456, 0, 0, 3, 0, 0, 0, 360, 1588, 2, 3, 0, 0, 0, 0, 3, 20, 18, 6, 0, 1, 0, 652, 12, 0, 65, 3, 0, 0, 0, 4, 58, 0, 94, 6, 0, 8142, 0, 4635, 2841, 69, 0, 40, 63, 363, 330, 16, 0, 8, 1177, 8013, 0, 0, 0, 0, 0, 5, 2, 0, 67, 0, 0, 0, 0, 1, 0, 1, 64, 0, 1053, 2, 175, 2, 18, 0, 4143, 0, 0, 0, 445, 0, 13, 835, 43, 517, 12, 6, 24, 79, 0, 0, 0, 0, 1, 0, 17, 0, 2, 0, 4548, 1, 9, 9, 200, 0, 7685, 7, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[768, 333, 1, 3185, 1, 5, 1, 0, 47, 2, 7785, 0, 0, 1271, 2, 117, 0, 351, 137, 248, 0, 1, 0, 0, 1, 2, 13, 108, 0, 6, 33, 329, 0, 0, 26, 0, 0, 1, 0, 0, 0, 148, 56, 51, 11, 12, 0, 2, 131, 27, 17, 44, 1548, 2, 8, 2204, 0, 585, 1112, 2055, 0, 1425, 106, 3314, 93, 10, 1, 0, 1175, 7756, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 201, 2096, 742, 106, 1403, 4, 7915, 1, 89, 0, 1885, 47, 2579, 78, 0, 0, 0, 0, 7, 2012, 0, 0, 0, 6, 14, 375, 2, 7, 0, 3, 2, 226, 0, 89, 102, 1, 1385, 10, 0, 0, 21, 7, 7356, 89, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1546, 1823, 5, 431, 32, 0, 0, 0, 1, 1460, 5628, 0, 12, 496, 0, 4, 0, 1, 106, 627, 0, 0, 0, 0, 0, 0, 2, 154, 2, 0, 4, 2, 1, 97, 0, 0, 0, 0, 5, 482, 4, 1949, 6, 0, 223, 5, 0, 0, 86, 1220, 81, 7, 4274, 10, 138, 6892, 0, 5024, 5, 4, 0, 1957, 0, 1017, 2, 0, 0, 35, 351, 8125, 0, 15, 0, 29, 0, 102, 0, 0, 2, 0, 1, 0, 0, 1, 0, 1414, 98, 19, 7473, 0, 137, 0, 6, 0, 5817, 3, 0, 0, 2, 57, 11, 578, 108, 3, 2, 166, 9, 0, 45, 5, 0, 0, 0, 252, 0, 20, 0, 0, 840, 0, 0, 0, 37, 0, 3948, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[134, 3721, 0, 3384, 2, 0, 3, 9, 2, 21, 6991, 0, 0, 6, 0, 0, 0, 0, 3, 1434, 0, 0, 924, 0, 0, 0, 0, 1820, 47, 2, 126, 0, 0, 14, 1, 9, 0, 0, 0, 660, 0, 17, 613, 1, 1, 55, 3, 0, 22, 9, 1, 0, 2338, 0, 0, 5784, 4, 189, 2567, 0, 0, 8, 5, 185, 1, 0, 0, 0, 210, 8022, 0, 0, 1, 0, 0, 0, 3, 13, 1, 1, 0, 0, 4, 0, 0, 0, 95, 1, 7693, 1, 3546, 1, 13, 0, 1231, 168, 0, 3, 30, 0, 0, 2581, 22, 1, 0, 0, 474, 162, 1, 0, 0, 0, 0, 124, 0, 6, 0, 4, 2526, 0, 1, 0, 11, 13, 7456, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[391, 1862, 0, 307, 13, 0, 61, 273, 0, 0, 8065, 0, 9, 4612, 0, 35, 2, 0, 23, 2758, 0, 0, 266, 0, 0, 0, 2043, 27, 0, 0, 3, 0, 3, 2, 0, 0, 3, 591, 0, 0, 0, 39, 7, 0, 2, 51, 0, 0, 76, 1, 2, 1, 735, 68, 6, 6409, 0, 2, 1141, 63, 0, 49, 1, 665, 10, 0, 0, 0, 23, 8188, 20, 1, 0, 0, 0, 389, 0, 0, 0, 11, 0, 100, 0, 156, 0, 0, 0, 0, 6379, 1, 437, 0, 5, 0, 3538, 0, 1, 0, 0, 29, 0, 1449, 0, 5, 1, 0, 0, 0, 29, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 3, 82, 0, 7837, 16, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[83, 4768, 1, 3233, 85, 0, 15, 3, 0, 102, 7615, 0, 0, 575, 3, 12, 1, 0, 2, 456, 0, 0, 3, 0, 0, 0, 360, 1588, 2, 3, 0, 0, 0, 0, 3, 20, 18, 6, 0, 1, 0, 652, 12, 0, 65, 3, 0, 0, 0, 4, 58, 0, 94, 6, 0, 8142, 0, 4635, 2841, 69, 0, 40, 63, 363, 330, 16, 0, 8, 1177, 8013, 0, 0, 0, 0, 0, 5, 2, 0, 67, 0, 0, 0, 0, 1, 0, 1, 64, 0, 1053, 2, 175, 2, 18, 0, 4143, 0, 0, 0, 445, 0, 13, 835, 43, 517, 12, 6, 24, 79, 0, 0, 0, 0, 1, 0, 17, 0, 2, 0, 4548, 1, 9, 9, 200, 0, 7685, 7, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[171, 1250, 1, 34, 0, 0, 0, 0, 0, 216, 8019, 4, 22, 155, 0, 0, 40, 5, 1, 2873, 0, 0, 0, 0, 0, 0, 2, 865, 4, 0, 0, 84, 8, 0, 0, 19, 4, 11, 0, 2, 0, 103, 149, 0, 0, 683, 0, 0, 20, 79, 0, 0, 6417, 0, 12, 6010, 0, 928, 4586, 0, 0, 1021, 0, 644, 408, 0, 0, 0, 13, 8103, 22, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 519, 41, 1, 106, 16, 3517, 0, 6692, 0, 277, 6, 1, 0, 677, 0, 27, 1, 0, 1, 0, 2594, 0, 0, 0, 66, 0, 0, 36, 4, 0, 0, 0, 0, 0, 0, 0, 6, 248, 7, 0, 0, 5, 0, 7682, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 282, 0, 1579, 0, 0, 0, 3, 17, 71, 4450, 0, 0, 2442, 0, 0, 0, 27, 0, 1220, 1, 0, 0, 0, 8, 2, 9, 3, 0, 0, 1236, 9, 0, 115, 0, 0, 7, 24, 0, 528, 0, 785, 542, 0, 6, 8, 48, 20, 0, 0, 7, 0, 809, 0, 99, 5615, 0, 4148, 611, 2819, 0, 1068, 5, 1206, 322, 0, 0, 0, 1325, 8144, 0, 2, 0, 4, 80, 0, 0, 97, 1, 0, 1, 0, 0, 174, 1, 733, 0, 0, 6540, 0, 540, 0, 0, 33, 5121, 0, 25, 54, 0, 95, 8, 3278, 0, 59, 2, 487, 74, 75, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 7099, 59, 0, 2, 367, 1, 901, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[768, 333, 1, 3185, 1, 5, 1, 0, 47, 2, 7785, 0, 0, 1271, 2, 117, 0, 351, 137, 248, 0, 1, 0, 0, 1, 2, 13, 108, 0, 6, 33, 329, 0, 0, 26, 0, 0, 1, 0, 0, 0, 148, 56, 51, 11, 12, 0, 2, 131, 27, 17, 44, 1548, 2, 8, 2204, 0, 585, 1112, 2055, 0, 1425, 106, 3314, 93, 10, 1, 0, 1175, 7756, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 201, 2096, 742, 106, 1403, 4, 7915, 1, 89, 0, 1885, 47, 2579, 78, 0, 0, 0, 0, 7, 2012, 0, 0, 0, 6, 14, 375, 2, 7, 0, 3, 2, 226, 0, 89, 102, 1, 1385, 10, 0, 0, 21, 7, 7356, 89, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[52, 28, 62, 1689, 1, 1, 2, 23, 0, 0, 4889, 0, 27, 33, 3, 122, 0, 2, 15, 581, 0, 0, 11, 0, 0, 10, 4, 816, 1, 0, 35, 737, 2, 2, 3, 0, 0, 0, 1, 3, 0, 2967, 1367, 0, 4, 0, 0, 0, 52, 0, 127, 0, 4617, 0, 3, 7351, 0, 681, 354, 2129, 0, 4203, 74, 1914, 262, 0, 6, 0, 954, 7870, 0, 51, 0, 0, 0, 19, 0, 0, 0, 0, 0, 1, 104, 98, 3, 13, 705, 7, 4136, 0, 9, 0, 4, 7, 5171, 0, 0, 0, 0, 383, 0, 1544, 0, 12, 0, 9, 239, 1296, 0, 28, 0, 0, 0, 22, 0, 0, 0, 0, 13, 0, 51, 0, 4, 0, 7517, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[83, 4768, 1, 3233, 85, 0, 15, 3, 0, 102, 3133, 0, 0, 575, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 456, 0, 0, 3, 0, 0, 0, 360, 1588, 2, 3, 0, 0, 61, 294, 1457, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 20, 18, 6, 0, 1, 0, 652, 12, 0, 65, 3, 0, 0, 8142, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 58, 0, 33, 6, 0, 0, 0, 4635, 2841, 69, 0, 40, 63, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 16, 0, 8, 1177, 7218, 0, 0, 0, 0, 0, 5, 2, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 64, 0, 379, 2, 175, 2, 18, 0, 4143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 13, 835, 43, 517, 12, 6, 24, 79, 0, 0, 0, 0, 3943, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 2, 0, 4548, 1, 9, 9, 200, 0, 6228, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12013, 5022, 9461, 8112, 8823, 4785, 6297, 11023] avg=8192.0000 max=12013 max_rank=0 min=4785 min_rank=5 imbalance_ratio=1.466431 max_load_reduction=53523 max_load_reduction_pct=0.816696 moved_to_echo=30928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[391, 1862, 0, 307, 13, 0, 61, 273, 0, 0, 8065, 0, 9, 4612, 0, 35, 2, 0, 23, 2758, 0, 0, 266, 0, 0, 0, 2043, 27, 0, 0, 3, 0, 3, 2, 0, 0, 3, 591, 0, 0, 0, 39, 7, 0, 2, 51, 0, 0, 76, 1, 2, 1, 735, 68, 6, 6409, 0, 2, 1141, 63, 0, 49, 1, 665, 10, 0, 0, 0, 23, 8188, 20, 1, 0, 0, 0, 389, 0, 0, 0, 11, 0, 100, 0, 156, 0, 0, 0, 0, 6379, 1, 437, 0, 5, 0, 3538, 0, 1, 0, 0, 29, 0, 1449, 0, 5, 1, 0, 0, 0, 29, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 3, 82, 0, 7837, 16, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[134, 3721, 0, 3384, 2, 0, 3, 9, 2, 21, 6991, 0, 0, 6, 0, 0, 0, 0, 3, 1434, 0, 0, 924, 0, 0, 0, 0, 1820, 47, 2, 126, 0, 0, 14, 1, 9, 0, 0, 0, 660, 0, 17, 613, 1, 1, 55, 3, 0, 22, 9, 1, 0, 2338, 0, 0, 5784, 4, 189, 2567, 0, 0, 8, 5, 185, 1, 0, 0, 0, 210, 8022, 0, 0, 1, 0, 0, 0, 3, 13, 1, 1, 0, 0, 4, 0, 0, 0, 95, 1, 7693, 1, 3546, 1, 13, 0, 1231, 168, 0, 3, 30, 0, 0, 2581, 22, 1, 0, 0, 474, 162, 1, 0, 0, 0, 0, 124, 0, 6, 0, 4, 2526, 0, 1, 0, 11, 13, 7456, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1546, 1823, 5, 431, 32, 0, 0, 0, 1, 1460, 2316, 0, 12, 496, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 106, 627, 0, 0, 0, 0, 0, 0, 2, 154, 2, 0, 4, 2, 2791, 2088, 748, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 97, 0, 0, 0, 0, 5, 482, 4, 1949, 6, 0, 223, 5, 0, 0, 6892, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 1220, 81, 7, 1483, 10, 138, 0, 0, 5024, 5, 4, 0, 1957, 0, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 35, 351, 7319, 0, 15, 0, 29, 0, 102, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1414, 98, 19, 2688, 0, 137, 0, 6, 0, 5817, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 57, 11, 578, 108, 3, 2, 166, 9, 0, 45, 5, 0, 0, 2914, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 20, 0, 0, 840, 0, 0, 0, 37, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 7331, 10062, 11032, 7855, 10184, 6597, 4349] avg=8192.0000 max=11032 max_rank=3 min=4349 min_rank=7 imbalance_ratio=1.346680 max_load_reduction=54504 max_load_reduction_pct=0.831665 moved_to_echo=28985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[391, 1862, 0, 307, 13, 0, 61, 273, 0, 0, 3308, 0, 9, 4612, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 23, 2758, 0, 0, 266, 0, 0, 0, 2043, 27, 0, 0, 3, 0, 484, 1786, 1489, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 591, 0, 0, 0, 39, 7, 0, 2, 51, 0, 0, 6409, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 1, 2, 1, 251, 68, 6, 0, 0, 2, 1141, 63, 0, 49, 1, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 23, 7370, 20, 1, 0, 0, 0, 389, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 156, 0, 0, 0, 0, 2287, 1, 437, 0, 5, 0, 3538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 29, 0, 1449, 0, 5, 1, 0, 0, 0, 29, 0, 2, 0, 4182, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 3, 82, 0, 6348, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10871, 9699, 7682, 2326, 7824, 6524, 8004, 12606] avg=8192.0000 max=12606 max_rank=7 min=2326 min_rank=3 imbalance_ratio=1.538818 max_load_reduction=52930 max_load_reduction_pct=0.807648 moved_to_echo=34958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[134, 3721, 0, 3384, 2, 0, 3, 9, 2, 21, 2876, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1434, 0, 0, 924, 0, 0, 0, 0, 1820, 47, 2, 126, 0, 1526, 2149, 1413, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 9, 0, 0, 0, 660, 0, 17, 613, 1, 1, 55, 3, 0, 5784, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 9, 1, 0, 812, 0, 0, 0, 4, 189, 2567, 0, 0, 8, 5, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 210, 7226, 0, 0, 1, 0, 0, 0, 3, 13, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 95, 1, 2767, 1, 3546, 1, 13, 0, 1231, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 30, 0, 0, 2581, 22, 1, 0, 0, 474, 162, 1, 0, 0, 0, 3620, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 6, 0, 4, 2526, 0, 1, 0, 11, 13, 6043, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10158, 10240, 7653, 3802, 7456, 7827, 9671, 8729] avg=8192.0000 max=10240 max_rank=1 min=3802 min_rank=3 imbalance_ratio=1.250000 max_load_reduction=55296 max_load_reduction_pct=0.843750 moved_to_echo=33683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 282, 0, 1579, 0, 0, 0, 3, 17, 71, 1831, 0, 0, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 1220, 1, 0, 0, 0, 8, 2, 9, 3, 0, 0, 1236, 9, 528, 1827, 170, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 7, 24, 0, 528, 0, 785, 542, 0, 6, 8, 48, 20, 5615, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 281, 0, 99, 0, 0, 4148, 611, 2819, 0, 1068, 5, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 1325, 7336, 0, 2, 0, 4, 80, 0, 0, 97, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 174, 1, 733, 0, 0, 2353, 0, 540, 0, 0, 33, 5121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 54, 0, 95, 8, 3278, 0, 59, 2, 487, 74, 75, 0, 0, 0, 1, 2304, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7099, 59, 0, 2, 367, 1, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6225, 5848, 8013, 10244, 9167, 8956, 8822, 8261] avg=8192.0000 max=10244 max_rank=3 min=5848 min_rank=1 imbalance_ratio=1.250488 max_load_reduction=55292 max_load_reduction_pct=0.843689 moved_to_echo=35206 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[52, 28, 62, 1689, 1, 1, 2, 23, 0, 0, 2012, 0, 27, 33, 3, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 581, 0, 0, 11, 0, 0, 10, 4, 816, 1, 0, 35, 737, 3015, 1155, 1425, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3, 0, 0, 0, 1, 3, 0, 2967, 1367, 0, 4, 0, 0, 0, 7351, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 127, 0, 1602, 0, 3, 0, 0, 681, 354, 2129, 0, 4203, 74, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 6, 0, 954, 7089, 0, 51, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 104, 98, 3, 13, 705, 7, 1488, 0, 9, 0, 4, 7, 5171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 1544, 0, 12, 0, 9, 239, 1296, 0, 28, 0, 0, 2531, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 13, 0, 51, 0, 4, 0, 6092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 8588, 12046, 11139, 8381, 7610, 7535, 6182] avg=8192.0000 max=12046 max_rank=2 min=4055 min_rank=0 imbalance_ratio=1.470459 max_load_reduction=53490 max_load_reduction_pct=0.816193 moved_to_echo=29708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[83, 4768, 1, 3233, 85, 0, 15, 3, 0, 102, 3133, 0, 0, 575, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 456, 0, 0, 3, 0, 0, 0, 360, 1588, 2, 3, 0, 0, 61, 294, 1457, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 20, 18, 6, 0, 1, 0, 652, 12, 0, 65, 3, 0, 0, 8142, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 58, 0, 33, 6, 0, 0, 0, 4635, 2841, 69, 0, 40, 63, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 16, 0, 8, 1177, 7218, 0, 0, 0, 0, 0, 5, 2, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 64, 0, 379, 2, 175, 2, 18, 0, 4143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 13, 835, 43, 517, 12, 6, 24, 79, 0, 0, 0, 0, 3943, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 2, 0, 4548, 1, 9, 9, 200, 0, 6228, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12013, 5022, 9461, 8112, 8823, 4785, 6297, 11023] avg=8192.0000 max=12013 max_rank=0 min=4785 min_rank=5 imbalance_ratio=1.466431 max_load_reduction=53523 max_load_reduction_pct=0.816696 moved_to_echo=30928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[171, 1250, 1, 34, 0, 0, 0, 0, 0, 216, 3299, 4, 22, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 5, 1, 2873, 0, 0, 0, 0, 0, 0, 2, 865, 4, 0, 0, 84, 4190, 1870, 1456, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 19, 4, 11, 0, 2, 0, 103, 149, 0, 0, 683, 0, 0, 6010, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 79, 0, 0, 2227, 0, 12, 0, 0, 928, 4586, 0, 0, 1021, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 0, 13, 7299, 22, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 41, 1, 106, 16, 3517, 0, 2407, 0, 277, 6, 1, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1, 0, 1, 0, 2594, 0, 0, 0, 66, 0, 0, 36, 4, 0, 0, 4152, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 248, 7, 0, 0, 5, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5152, 12194, 7557, 9517, 7760, 7568, 9296, 6492] avg=8192.0000 max=12194 max_rank=1 min=5152 min_rank=0 imbalance_ratio=1.488525 max_load_reduction=53342 max_load_reduction_pct=0.813934 moved_to_echo=31116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 282, 0, 1579, 0, 0, 0, 3, 17, 71, 1831, 0, 0, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 1220, 1, 0, 0, 0, 8, 2, 9, 3, 0, 0, 1236, 9, 528, 1827, 170, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 7, 24, 0, 528, 0, 785, 542, 0, 6, 8, 48, 20, 5615, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 281, 0, 99, 0, 0, 4148, 611, 2819, 0, 1068, 5, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 1325, 7336, 0, 2, 0, 4, 80, 0, 0, 97, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 174, 1, 733, 0, 0, 2353, 0, 540, 0, 0, 33, 5121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 54, 0, 95, 8, 3278, 0, 59, 2, 487, 74, 75, 0, 0, 0, 1, 2304, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7099, 59, 0, 2, 367, 1, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6225, 5848, 8013, 10244, 9167, 8956, 8822, 8261] avg=8192.0000 max=10244 max_rank=3 min=5848 min_rank=1 imbalance_ratio=1.250488 max_load_reduction=55292 max_load_reduction_pct=0.843689 moved_to_echo=35206 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[768, 333, 1, 3185, 1, 5, 1, 0, 47, 2, 3203, 0, 0, 1271, 2, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 137, 248, 0, 1, 0, 0, 1, 2, 13, 108, 0, 6, 33, 329, 1010, 2211, 1394, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 1, 0, 0, 0, 148, 56, 51, 11, 12, 0, 2, 2204, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 27, 17, 44, 538, 2, 8, 0, 0, 585, 1112, 2055, 0, 1425, 106, 3314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 10, 1, 0, 1175, 6986, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 201, 2096, 742, 106, 1403, 4, 2847, 1, 89, 0, 1885, 47, 2579, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2012, 0, 0, 0, 6, 14, 375, 2, 7, 0, 3, 4031, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 226, 0, 89, 102, 1, 1385, 10, 0, 0, 21, 7, 5962, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8936, 6614, 3062, 9364, 8270, 12082, 9314, 7894] avg=8192.0000 max=12082 max_rank=5 min=3062 min_rank=2 imbalance_ratio=1.474854 max_load_reduction=53454 max_load_reduction_pct=0.815643 moved_to_echo=37560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[52, 28, 62, 1689, 1, 1, 2, 23, 0, 0, 2012, 0, 27, 33, 3, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 581, 0, 0, 11, 0, 0, 10, 4, 816, 1, 0, 35, 737, 3015, 1155, 1425, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3, 0, 0, 0, 1, 3, 0, 2967, 1367, 0, 4, 0, 0, 0, 7351, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 127, 0, 1602, 0, 3, 0, 0, 681, 354, 2129, 0, 4203, 74, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 6, 0, 954, 7089, 0, 51, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 104, 98, 3, 13, 705, 7, 1488, 0, 9, 0, 4, 7, 5171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 1544, 0, 12, 0, 9, 239, 1296, 0, 28, 0, 0, 2531, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 13, 0, 51, 0, 4, 0, 6092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 8588, 12046, 11139, 8381, 7610, 7535, 6182] avg=8192.0000 max=12046 max_rank=2 min=4055 min_rank=0 imbalance_ratio=1.470459 max_load_reduction=53490 max_load_reduction_pct=0.816193 moved_to_echo=29708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[391, 1862, 0, 307, 13, 0, 61, 273, 0, 0, 3308, 0, 9, 4612, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 23, 2758, 0, 0, 266, 0, 0, 0, 2043, 27, 0, 0, 3, 0, 484, 1786, 1489, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 591, 0, 0, 0, 39, 7, 0, 2, 51, 0, 0, 6409, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 1, 2, 1, 251, 68, 6, 0, 0, 2, 1141, 63, 0, 49, 1, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 23, 7370, 20, 1, 0, 0, 0, 389, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 156, 0, 0, 0, 0, 2287, 1, 437, 0, 5, 0, 3538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 29, 0, 1449, 0, 5, 1, 0, 0, 0, 29, 0, 2, 0, 4182, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 3, 82, 0, 6348, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10871, 9699, 7682, 2326, 7824, 6524, 8004, 12606] avg=8192.0000 max=12606 max_rank=7 min=2326 min_rank=3 imbalance_ratio=1.538818 max_load_reduction=52930 max_load_reduction_pct=0.807648 moved_to_echo=34958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[134, 3721, 0, 3384, 2, 0, 3, 9, 2, 21, 2876, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1434, 0, 0, 924, 0, 0, 0, 0, 1820, 47, 2, 126, 0, 1526, 2149, 1413, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 9, 0, 0, 0, 660, 0, 17, 613, 1, 1, 55, 3, 0, 5784, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 9, 1, 0, 812, 0, 0, 0, 4, 189, 2567, 0, 0, 8, 5, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 210, 7226, 0, 0, 1, 0, 0, 0, 3, 13, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 95, 1, 2767, 1, 3546, 1, 13, 0, 1231, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 30, 0, 0, 2581, 22, 1, 0, 0, 474, 162, 1, 0, 0, 0, 3620, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 6, 0, 4, 2526, 0, 1, 0, 11, 13, 6043, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10158, 10240, 7653, 3802, 7456, 7827, 9671, 8729] avg=8192.0000 max=10240 max_rank=1 min=3802 min_rank=3 imbalance_ratio=1.250000 max_load_reduction=55296 max_load_reduction_pct=0.843750 moved_to_echo=33683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 512, 384, 10240, 128, 128, 1280, 0, 128, 128, 2560, 5504, 128, 128, 1536, 1280, 13696, 13440, 9600, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 512, 640, 128, 14976, 256, 640, 128, 768, 896, 2048, 128, 128, 128, 128, 27776, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3200, 14080, 128, 13952, 256, 128, 128, 384, 128, 1920, 22016, 128, 128, 9600, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -1924,117 +1924,117 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 128, 128, 640, 128, 1792, 128, 6784, 2816, 128, 384, 896, 128, 128, 48512, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1536, 128, 128, 128, 5248, 57856, 128, 128, 128, 128, 128, 640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1536, 128, 128, 128, 5248, 57856, 128, 128, 128, 128, 128, 640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3532, 0, 5, 0, 0, 0, 15, 19, 9, 8, 0, 0, 0, 64, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 7848, 0, 0, 0, 0, 2, 151, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 410, 10, 1889, 1, 0, 0, 0, 8104, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 2, 177, 0, 0, 0, 7238, 76, 0, 0, 0, 7989, 93, 0, 0, 0, 0, 0, 0, 0, 549, 5, 0, 15, 0, 0, 0, 0, 3, 0, 2, 0, 0, 218, 8147, 55, 0, 0, 41, 6381, 169, 15, 4, 91, 0, 14, 0, 0, 570, 7894, 0, 3, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 7860, 0, 4, 0, 0, 12, 10, 0, 0, 0, 4, 9, 0, 0, 2, 58, 0, 37, 0, 0, 0, 1, 20, 1257, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 204, 0, 0, 128, 0, 0, 4, 19, 0, 730, 34, 0, 0, 0, 0, 0, 6296, 0, 6372, 0, 0, 0, 0, 6986, 0, 7819, 0, 0, 10, 0, 0, 1, 0, 3, 2, 3, 0, 0, 0, 6768, 769, 8, 39, 0, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 1, 0, 2152, 2, 0, 0, 0, 16, 2, 1146, 0, 0, 0, 37, 7240, 0, 5, 0, 0, 7, 1, 5, 0, 273, 7826, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 12, 5, 0, 5744, 0, 22, 23, 57, 2, 259, 25, 4, 0, 41, 3, 0, 190, 0, 0, 249, 11, 0, 0, 1144, 0, 495, 1, 673, 278, 0, 0, 2, 0, 0, 0, 277, 0, 7, 0, 0, 2, 3, 0, 0, 0, 147, 0, 0, 2, 0, 0, 1, 0, 4951, 0, 4422, 0, 3, 0, 0, 7698, 0, 241, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 6800, 0, 0, 1, 0, 3116, 2, 743, 0, 0, 1, 3, 20, 0, 4416, 8, 0, 246, 0, 2, 0, 1, 0, 224, 715, 17, 2697, 0, 3997, 0, 0, 2, 3, 7524, 0, 2, 0, 146, 0, 0, 31, 0, 0, 7517, 0, 45, 18, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3532, 0, 5, 0, 0, 0, 15, 19, 9, 8, 0, 0, 0, 64, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 7848, 0, 0, 0, 0, 2, 151, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 410, 10, 1889, 1, 0, 0, 0, 8104, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 2, 177, 0, 0, 0, 7238, 76, 0, 0, 0, 7989, 93, 0, 0, 0, 0, 0, 0, 0, 549, 5, 0, 15, 0, 0, 0, 0, 3, 0, 2, 0, 0, 218, 8147, 55, 0, 0, 41, 6381, 169, 15, 4, 91, 0, 14, 0, 0, 570, 7894, 0, 3, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 7195, 0, 1, 39, 35, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 180, 0, 8, 6, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1, 0, 0, 0, 0, 0, 36, 0, 8074, 0, 167, 1, 0, 4644, 0, 612, 0, 0, 0, 90, 0, 0, 0, 0, 0, 49, 0, 1, 0, 7444, 726, 0, 59, 51, 30, 0, 5, 0, 1, 0, 451, 0, 54, 46, 0, 5, 0, 140, 0, 0, 1958, 0, 36, 0, 0, 0, 6479, 5546, 1, 0, 1, 0, 7698, 393, 1, 3835, 0, 0, 0, 0, 0, 61, 8147, 1, 0, 258, 57, 21, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 7499, 8, 0, 0, 0, 0, 0, 0, 66, 159, 0, 0, 81, 0, 0, 0, 289, 0, 0, 0, 0, 0, 2, 0, 6428, 69, 0, 0, 0, 0, 0, 0, 23, 0, 2, 3, 0, 0, 0, 0, 2, 0, 57, 0, 244, 29, 17, 0, 0, 0, 338, 0, 7926, 0, 0, 0, 0, 7850, 10, 6153, 1, 0, 0, 77, 0, 0, 0, 6, 15, 0, 8, 0, 0, 7987, 0, 0, 0, 15, 26, 0, 1192, 4, 0, 0, 0, 0, 0, 1547, 0, 0, 532, 1, 10, 0, 0, 0, 0, 241, 0, 0, 11, 830, 0, 0, 0, 0, 7576, 5, 0, 13, 8, 6, 0, 1, 1, 1, 8001, 0, 72, 87, 0, 4, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 14, 7680, 66, 0, 0, 33, 0, 2, 0, 38, 42, 0, 0, 0, 0, 0, 24, 33, 0, 0, 0, 0, 4, 0, 0, 976, 25, 891, 0, 0, 0, 0, 0, 0, 11, 13, 1, 113, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 0, 0, 1, 1, 0, 7997, 0, 0, 0, 2, 6866, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 275, 0, 6, 8103, 0, 0, 0, 2, 8149, 0, 0, 0, 0, 0, 0, 7, 0, 275, 0, 0, 0, 0, 0, 1, 20, 25, 1, 4, 0, 3, 0, 471, 1, 0, 0, 0, 8124, 0, 0, 69, 0, 0, 1, 1, 2, 0, 6932, 0, 0, 45, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 12, 5, 0, 5744, 0, 22, 23, 57, 2, 259, 25, 4, 0, 41, 3, 0, 190, 0, 0, 249, 11, 0, 0, 1144, 0, 495, 1, 673, 278, 0, 0, 2, 0, 0, 0, 277, 0, 7, 0, 0, 2, 3, 0, 0, 0, 147, 0, 0, 2, 0, 0, 1, 0, 4951, 0, 4422, 0, 3, 0, 0, 7698, 0, 241, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 6800, 0, 0, 1, 0, 3116, 2, 743, 0, 0, 1, 3, 20, 0, 4416, 8, 0, 246, 0, 2, 0, 1, 0, 224, 715, 17, 2697, 0, 3997, 0, 0, 2, 3, 7524, 0, 2, 0, 146, 0, 0, 31, 0, 0, 7517, 0, 45, 18, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7894, 0, 0, 0, 170, 2, 0, 0, 0, 1, 11, 0, 0, 0, 0, 16, 0, 0, 0, 6, 1, 0, 0, 0, 1340, 0, 19, 2, 0, 0, 8, 0, 0, 10, 31, 8, 411, 0, 0, 0, 0, 250, 0, 214, 0, 0, 0, 0, 0, 0, 520, 0, 4890, 0, 0, 0, 5, 7328, 0, 2037, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 8191, 11, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 9, 0, 985, 1, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 196, 970, 6255, 0, 0, 0, 0, 7676, 0, 16, 72, 110, 0, 14, 1, 0, 15, 7868, 0, 0, 0, 0, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 7195, 0, 1, 39, 35, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 180, 0, 8, 6, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1, 0, 0, 0, 0, 0, 36, 0, 8074, 0, 167, 1, 0, 4644, 0, 612, 0, 0, 0, 90, 0, 0, 0, 0, 0, 49, 0, 1, 0, 7444, 726, 0, 59, 51, 30, 0, 5, 0, 1, 0, 451, 0, 54, 46, 0, 5, 0, 140, 0, 0, 1958, 0, 36, 0, 0, 0, 6479, 5546, 1, 0, 1, 0, 7698, 393, 1, 3835, 0, 0, 0, 0, 0, 61, 8147, 1, 0, 258, 57, 21, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 9, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 0, 0, 0, 0, 8, 0, 0, 15, 5455, 0, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 34, 0, 2, 0, 24, 0, 3, 2, 0, 6036, 0, 3, 0, 0, 8156, 0, 4149, 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 25, 6949, 1067, 0, 8, 0, 7667, 1, 0, 0, 24, 0, 216, 1, 0, 0, 0, 32, 0, 0, 0, 0, 2, 0, 525, 0, 1, 88, 441, 8144, 46, 0, 0, 0, 7764, 0, 114, 5, 3, 27, 0, 0, 229, 16, 6842, 1, 0, 480, 451, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 7499, 8, 0, 0, 0, 0, 0, 0, 66, 159, 0, 0, 81, 0, 0, 0, 289, 0, 0, 0, 0, 0, 2, 0, 6428, 69, 0, 0, 0, 0, 0, 0, 23, 0, 2, 3, 0, 0, 0, 0, 2, 0, 57, 0, 244, 29, 17, 0, 0, 0, 338, 0, 7926, 0, 0, 0, 0, 7850, 10, 6153, 1, 0, 0, 77, 0, 0, 0, 6, 15, 0, 8, 0, 0, 7987, 0, 0, 0, 15, 26, 0, 1192, 4, 0, 0, 0, 0, 0, 1547, 0, 0, 532, 1, 10, 0, 0, 0, 0, 241, 0, 0, 11, 830, 0, 0, 0, 0, 7576, 5, 0, 13, 8, 6, 0, 1, 1, 1, 8001, 0, 72, 87, 0, 4, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3532, 0, 5, 0, 0, 0, 15, 19, 9, 8, 0, 1585, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 7848, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 151, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7568, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 410, 10, 1889, 1, 0, 0, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 2, 177, 0, 0, 0, 3561, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7989, 93, 0, 0, 0, 0, 0, 0, 0, 549, 5, 0, 15, 2092, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 218, 8147, 55, 0, 0, 41, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 15, 4, 91, 0, 14, 0, 0, 570, 7453, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5615, 12092, 8677, 2858, 7273, 10785, 9914, 8322] avg=8192.0000 max=12092 max_rank=1 min=2858 min_rank=3 imbalance_ratio=1.476074 max_load_reduction=53444 max_load_reduction_pct=0.815491 moved_to_echo=36294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 7860, 0, 4, 0, 0, 12, 10, 0, 0, 0, 4, 1482, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 58, 0, 37, 0, 0, 0, 1, 20, 1257, 0, 0, 0, 4472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 204, 0, 0, 128, 0, 0, 4, 19, 0, 6524, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 34, 0, 0, 0, 0, 0, 6296, 0, 6372, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7819, 0, 0, 10, 0, 0, 1, 0, 3, 2, 3, 0, 0, 0, 3330, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 39, 0, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 1956, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2152, 2, 0, 0, 0, 16, 2, 1146, 0, 0, 0, 37, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7, 1, 5, 0, 273, 7389, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9810, 5856, 7964, 13894, 11937, 3394, 4999, 7682] avg=8192.0000 max=13894 max_rank=3 min=3394 min_rank=5 imbalance_ratio=1.696045 max_load_reduction=51642 max_load_reduction_pct=0.787994 moved_to_echo=28012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 7195, 0, 1, 39, 35, 0, 0, 0, 0, 0, 0, 1631, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 180, 0, 8, 6, 1, 0, 0, 6, 0, 0, 4755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 44, 4336, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 36, 0, 8074, 0, 167, 1, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 90, 0, 0, 0, 0, 0, 49, 0, 1, 0, 3662, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 51, 30, 0, 5, 0, 1, 0, 451, 0, 54, 46, 0, 5, 0, 2151, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 1958, 0, 36, 0, 0, 0, 6479, 5546, 1, 0, 1, 0, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 1, 3835, 0, 0, 0, 0, 0, 61, 7692, 1, 0, 258, 57, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9363, 4957, 6358, 8587, 5140, 2904, 15908, 12319] avg=8192.0000 max=15908 max_rank=6 min=2904 min_rank=5 imbalance_ratio=1.941895 max_load_reduction=49628 max_load_reduction_pct=0.757263 moved_to_echo=36271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3532, 0, 5, 0, 0, 0, 15, 19, 9, 8, 0, 1585, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 7848, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 151, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7568, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 410, 10, 1889, 1, 0, 0, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 2, 177, 0, 0, 0, 3561, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7989, 93, 0, 0, 0, 0, 0, 0, 0, 549, 5, 0, 15, 2092, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 218, 8147, 55, 0, 0, 41, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 15, 4, 91, 0, 14, 0, 0, 570, 7453, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5615, 12092, 8677, 2858, 7273, 10785, 9914, 8322] avg=8192.0000 max=12092 max_rank=1 min=2858 min_rank=3 imbalance_ratio=1.476074 max_load_reduction=53444 max_load_reduction_pct=0.815491 moved_to_echo=36294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 12, 5, 0, 5744, 0, 22, 23, 57, 2, 259, 25, 4, 0, 41, 1489, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 190, 0, 0, 249, 11, 0, 0, 1144, 0, 495, 1, 673, 278, 0, 4648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 277, 0, 7, 0, 0, 2, 3, 0, 0, 0, 147, 7188, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4951, 0, 4422, 0, 3, 0, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3116, 2, 743, 0, 0, 1, 3, 20, 0, 4416, 8, 0, 246, 1965, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 224, 715, 17, 2697, 0, 3997, 0, 0, 2, 3, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 146, 0, 0, 31, 0, 0, 7097, 0, 45, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 7692, 8745, 9889, 3826, 10571, 9365, 7342] avg=8192.0000 max=10571 max_rank=5 min=3826 min_rank=4 imbalance_ratio=1.290405 max_load_reduction=54965 max_load_reduction_pct=0.838699 moved_to_echo=31104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 7499, 8, 0, 0, 0, 0, 0, 0, 66, 159, 0, 1750, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 289, 0, 0, 0, 0, 0, 2, 0, 6428, 69, 0, 4680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 2, 3, 0, 0, 0, 0, 2, 0, 57, 7330, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 29, 17, 0, 0, 0, 338, 0, 7926, 0, 0, 0, 0, 520, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 1, 0, 0, 77, 0, 0, 0, 6, 15, 0, 8, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 26, 0, 1192, 4, 0, 0, 0, 0, 0, 1547, 0, 0, 532, 2308, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 0, 0, 241, 0, 0, 11, 830, 0, 0, 0, 0, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 13, 8, 6, 0, 1, 1, 1, 7554, 0, 72, 87, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9931, 11549, 8543, 9084, 10189, 5674, 2813, 7753] avg=8192.0000 max=11549 max_rank=1 min=2813 min_rank=6 imbalance_ratio=1.409790 max_load_reduction=53987 max_load_reduction_pct=0.823776 moved_to_echo=26429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 14, 7680, 66, 0, 0, 33, 0, 2, 0, 38, 42, 0, 1775, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 33, 0, 0, 0, 0, 4, 0, 0, 976, 25, 891, 5018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 13, 1, 113, 0, 0, 0, 0, 0, 0, 6411, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 0, 0, 1, 1, 0, 7997, 0, 0, 0, 2, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 275, 0, 6, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8149, 0, 0, 0, 0, 0, 0, 7, 0, 275, 0, 0, 0, 2342, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 25, 1, 4, 0, 3, 0, 471, 1, 0, 0, 0, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 1, 1, 2, 0, 6545, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10046, 6971, 7757, 8471, 12429, 10829, 2370, 6663] avg=8192.0000 max=12429 max_rank=4 min=2370 min_rank=6 imbalance_ratio=1.517212 max_load_reduction=53107 max_load_reduction_pct=0.810349 moved_to_echo=32291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7894, 0, 0, 0, 170, 2, 0, 0, 0, 1, 11, 1799, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 1, 0, 0, 0, 1340, 0, 19, 4745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 0, 0, 10, 31, 8, 411, 0, 0, 0, 0, 250, 0, 6848, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 520, 0, 4890, 0, 0, 0, 5, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 4019, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 9, 0, 985, 1, 1, 0, 2373, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 196, 970, 6255, 0, 0, 0, 0, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 72, 110, 0, 14, 1, 0, 15, 7423, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10322, 6127, 8713, 6109, 6077, 11370, 9158, 7660] avg=8192.0000 max=11370 max_rank=5 min=6077 min_rank=4 imbalance_ratio=1.387939 max_load_reduction=54166 max_load_reduction_pct=0.826508 moved_to_echo=34265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 7195, 0, 1, 39, 35, 0, 0, 0, 0, 0, 0, 1631, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 180, 0, 8, 6, 1, 0, 0, 6, 0, 0, 4755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 44, 4336, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 36, 0, 8074, 0, 167, 1, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 90, 0, 0, 0, 0, 0, 49, 0, 1, 0, 3662, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 51, 30, 0, 5, 0, 1, 0, 451, 0, 54, 46, 0, 5, 0, 2151, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 1958, 0, 36, 0, 0, 0, 6479, 5546, 1, 0, 1, 0, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 1, 3835, 0, 0, 0, 0, 0, 61, 7692, 1, 0, 258, 57, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9363, 4957, 6358, 8587, 5140, 2904, 15908, 12319] avg=8192.0000 max=15908 max_rank=6 min=2904 min_rank=5 imbalance_ratio=1.941895 max_load_reduction=49628 max_load_reduction_pct=0.757263 moved_to_echo=36271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 9, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1522, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 0, 0, 0, 0, 8, 0, 0, 15, 5455, 0, 5, 4796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 7616, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2, 0, 24, 0, 3, 2, 0, 6036, 0, 3, 0, 0, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 25, 3419, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 7667, 1, 0, 0, 24, 0, 216, 1, 0, 0, 0, 32, 0, 2008, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 525, 0, 1, 88, 441, 8144, 46, 0, 0, 0, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 5, 3, 27, 0, 0, 229, 16, 6460, 1, 0, 480, 451, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2343, 10292, 8779, 6644, 8665, 10009, 11009, 7795] avg=8192.0000 max=11009 max_rank=6 min=2343 min_rank=0 imbalance_ratio=1.343872 max_load_reduction=54527 max_load_reduction_pct=0.832016 moved_to_echo=37478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 7499, 8, 0, 0, 0, 0, 0, 0, 66, 159, 0, 1750, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 289, 0, 0, 0, 0, 0, 2, 0, 6428, 69, 0, 4680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 2, 3, 0, 0, 0, 0, 2, 0, 57, 7330, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 29, 17, 0, 0, 0, 338, 0, 7926, 0, 0, 0, 0, 520, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 1, 0, 0, 77, 0, 0, 0, 6, 15, 0, 8, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 26, 0, 1192, 4, 0, 0, 0, 0, 0, 1547, 0, 0, 532, 2308, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 0, 0, 241, 0, 0, 11, 830, 0, 0, 0, 0, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 13, 8, 6, 0, 1, 1, 1, 7554, 0, 72, 87, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9931, 11549, 8543, 9084, 10189, 5674, 2813, 7753] avg=8192.0000 max=11549 max_rank=1 min=2813 min_rank=6 imbalance_ratio=1.409790 max_load_reduction=53987 max_load_reduction_pct=0.823776 moved_to_echo=26429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 4224, 128, 896, 1024, 128, 3072, 8192, 34560, 128, 0, 128, 128, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 384, 128, 128, 640, 256, 256, 128, 1280, 128, 640, 1280, 22784, 384, 1024, 37120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[32640, 128, 0, 128, 256, 0, 128, 256, 128, 128, 256, 384, 128, 128, 29312, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 384, 128, 128, 640, 256, 256, 128, 1280, 128, 640, 1280, 22784, 384, 1024, 37120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 36224, 128, 2048, 128, 128, 128, 768, 128, 128, 7936, 128, 128, 896, 17280, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 47872, 128, 128, 128, 384, 128, 384, 128, 128, 256, 128, 13056, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[1024, 384, 128, 128, 128, 128, 128, 12672, 128, 47616, 128, 256, 128, 128, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 256, 128, 384, 128, 1152, 128, 640, 256, 128, 128, 128, 384, 256, 53888, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[640, 256, 4096, 384, 128, 128, 128, 256, 1024, 57728, 128, 128, 896, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[640, 256, 4096, 384, 128, 128, 128, 256, 1024, 57728, 128, 128, 896, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[119, 0, 101, 12, 0, 7291, 0, 0, 14, 0, 7, 3150, 0, 0, 1, 0, 0, 2, 1, 0, 1267, 1, 0, 0, 94, 18, 0, 0, 2, 0, 0, 10, 0, 347, 0, 0, 0, 4, 0, 8077, 2, 0, 2, 2, 0, 0, 0, 0, 0, 5215, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2132, 1, 0, 0, 5, 8153, 0, 0, 55, 0, 0, 732, 0, 462, 954, 0, 0, 32, 0, 0, 0, 306, 2, 0, 5099, 0, 1241, 0, 0, 0, 1, 7178, 0, 0, 0, 0, 28, 3447, 86, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 48, 2, 0, 0, 8155, 0, 0, 1643, 12, 0, 0, 2, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[681, 20, 0, 10, 0, 1979, 0, 7, 0, 0, 206, 8168, 0, 11, 0, 0, 0, 123, 0, 6, 8040, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 7321, 0, 0, 0, 0, 0, 120, 0, 0, 1201, 0, 0, 0, 0, 71, 7, 3, 0, 0, 746, 0, 212, 0, 1233, 137, 0, 0, 4, 0, 3045, 6206, 1, 0, 7, 0, 58, 5444, 0, 76, 2, 0, 0, 0, 8172, 7, 0, 0, 3, 0, 0, 3943, 0, 0, 0, 5, 0, 2, 731, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 7482, 0, 0, 0, 0, 0, 19, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 5, 0, 0, 196, 0, 3, 0, 280, 0, 7861, 0, 0, 0, 787, 0, 0, 2, 0, 3348, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 634, 22, 0, 0, 416, 0, 0, 0, 0, 0, 4118, 1, 0, 1, 0, 169, 0, 0, 0, 72, 0, 0, 0, 8094, 0, 0, 0, 0, 50, 0, 0, 0, 2, 0, 2702, 7, 160, 868, 0, 0, 7147, 0, 0, 0, 0, 0, 0, 7966, 1, 145, 0, 1, 1, 26, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 22, 0, 0, 0, 0, 28, 4, 3730, 0, 8188, 0, 40, 4, 8184, 0, 0, 0, 7, 183] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 5, 0, 0, 0, 2, 0, 0, 464, 40, 7323, 0, 8, 0, 0, 0, 0, 0, 263, 757, 0, 0, 1578, 0, 0, 0, 0, 36, 13, 0, 1945, 0, 4, 9, 0, 0, 1, 7, 23, 0, 0, 0, 7962, 0, 0, 4, 0, 0, 0, 283, 31, 0, 53, 0, 0, 0, 10, 0, 0, 0, 0, 169, 144, 0, 0, 0, 5626, 0, 559, 0, 9, 0, 5873, 0, 0, 0, 0, 0, 7370, 0, 0, 0, 0, 0, 40, 6333, 0, 1, 0, 19, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 0, 4, 0, 613, 0, 0, 0, 0, 2, 0, 5, 7665, 0, 140, 283, 1799, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[119, 0, 101, 12, 0, 7291, 0, 0, 14, 0, 7, 3150, 0, 0, 1, 0, 0, 2, 1, 0, 1267, 1, 0, 0, 94, 18, 0, 0, 2, 0, 0, 10, 0, 347, 0, 0, 0, 4, 0, 8077, 2, 0, 2, 2, 0, 0, 0, 0, 0, 5215, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2132, 1, 0, 0, 5, 8153, 0, 0, 55, 0, 0, 732, 0, 462, 954, 0, 0, 32, 0, 0, 0, 306, 2, 0, 5099, 0, 1241, 0, 0, 0, 1, 7178, 0, 0, 0, 0, 28, 3447, 86, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 48, 2, 0, 0, 8155, 0, 0, 1643, 12, 0, 0, 2, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 4, 0, 45, 0, 6011, 0, 35, 0, 0, 5, 0, 0, 44, 0, 1, 3080, 39, 0, 2, 0, 0, 0, 0, 37, 3, 71, 11, 0, 0, 0, 0, 0, 14, 9, 7494, 170, 0, 23, 2370, 3, 0, 10, 10, 0, 616, 0, 43, 0, 0, 0, 128, 3, 0, 0, 0, 0, 0, 3642, 0, 0, 0, 1625, 5603, 53, 670, 0, 129, 419, 5015, 0, 0, 35, 0, 0, 4302, 0, 0, 1, 2, 1, 597, 4368, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 35, 0, 0, 555, 0, 0, 0, 0, 0, 0, 17, 0, 0, 190, 0, 8, 0, 40, 0, 0, 385, 0, 548, 8, 0, 2793, 6235, 0, 1, 0, 0, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 5, 0, 0, 196, 0, 3, 0, 280, 0, 7861, 0, 0, 0, 787, 0, 0, 2, 0, 3348, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 634, 22, 0, 0, 416, 0, 0, 0, 0, 0, 4118, 1, 0, 1, 0, 169, 0, 0, 0, 72, 0, 0, 0, 8094, 0, 0, 0, 0, 50, 0, 0, 0, 2, 0, 2702, 7, 160, 868, 0, 0, 7147, 0, 0, 0, 0, 0, 0, 7966, 1, 145, 0, 1, 1, 26, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 22, 0, 0, 0, 0, 28, 4, 3730, 0, 8188, 0, 40, 4, 8184, 0, 0, 0, 7, 183] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[155, 0, 0, 0, 0, 184, 0, 1372, 0, 27, 0, 8095, 0, 4, 0, 0, 0, 0, 53, 229, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 83, 0, 0, 4, 0, 0, 0, 767, 7463, 0, 0, 0, 48, 0, 0, 0, 0, 0, 7950, 0, 0, 123, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 13, 0, 0, 431, 6936, 2637, 0, 0, 5, 0, 8136, 0, 2, 922, 0, 0, 1918, 2, 0, 0, 0, 0, 0, 6240, 0, 1, 0, 0, 2567, 438, 11, 0, 0, 0, 0, 0, 190, 28, 0, 0, 0, 0, 4, 0, 33, 0, 0, 0, 0, 0, 2, 0, 0, 3739, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[681, 20, 0, 10, 0, 1979, 0, 7, 0, 0, 206, 8168, 0, 11, 0, 0, 0, 123, 0, 6, 8040, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 7321, 0, 0, 0, 0, 0, 120, 0, 0, 1201, 0, 0, 0, 0, 71, 7, 3, 0, 0, 746, 0, 212, 0, 1233, 137, 0, 0, 4, 0, 3045, 6206, 1, 0, 7, 0, 58, 5444, 0, 76, 2, 0, 0, 0, 8172, 7, 0, 0, 3, 0, 0, 3943, 0, 0, 0, 5, 0, 2, 731, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 7482, 0, 0, 0, 0, 0, 19, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2278, 0, 5, 0, 19, 0, 2770, 0, 0, 0, 0, 0, 2936, 0, 0, 7656, 0, 0, 1, 0, 2, 0, 0, 21, 0, 7, 1, 0, 0, 0, 2, 0, 0, 0, 8138, 1, 0, 32, 24, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 46, 0, 0, 0, 6, 8, 227, 6106, 0, 0, 9, 0, 0, 63, 1, 0, 0, 426, 21, 0, 6876, 0, 0, 0, 6, 0, 0, 3292, 0, 1, 0, 0, 2158, 0, 8062, 0, 0, 5, 0, 0, 5356, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 4, 2, 0, 75, 55, 0, 6525, 0, 213, 213, 207, 2, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 1541, 0, 0, 7425, 5180, 0, 0, 0, 0, 0, 0, 864, 0, 5, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 2514, 0, 0, 0, 0, 0, 0, 7, 1463, 3, 299, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 7639, 5, 62, 0, 0, 1, 7493, 0, 0, 0, 0, 238, 6803, 1, 0, 180, 0, 0, 0, 8187, 8, 0, 3, 0, 0, 0, 258, 283, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6573, 2, 150, 65, 0, 0, 0, 32, 0, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 5, 0, 0, 0, 2, 0, 0, 464, 40, 5883, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 757, 0, 0, 1578, 0, 0, 0, 0, 36, 13, 0, 1945, 2442, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 9, 0, 0, 1, 7, 23, 0, 0, 0, 7962, 0, 0, 4, 0, 1440, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 31, 0, 53, 0, 0, 0, 10, 0, 0, 0, 0, 169, 144, 1861, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5626, 0, 559, 0, 9, 0, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 2807, 0, 1, 0, 19, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 0, 4, 0, 613, 0, 7370, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 5590, 0, 140, 283, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6403, 8758, 10080, 3996, 7764, 10859, 9857, 7819] avg=8192.0000 max=10859 max_rank=5 min=3996 min_rank=3 imbalance_ratio=1.325562 max_load_reduction=54677 max_load_reduction_pct=0.834305 moved_to_echo=36299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[155, 0, 0, 0, 0, 184, 0, 1372, 0, 27, 0, 6503, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 229, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 83, 3383, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 767, 7463, 0, 0, 0, 48, 0, 0, 0, 0, 1592, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 123, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 13, 2579, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 6936, 2637, 0, 0, 5, 0, 2174, 0, 2, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2765, 0, 1, 0, 0, 2567, 438, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 28, 0, 0, 0, 0, 4, 0, 33, 0, 0, 0, 0, 1918, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3739, 0, 2144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 7227, 10115, 11224, 13107, 5784, 3949, 5885] avg=8192.0000 max=13107 max_rank=4 min=3949 min_rank=6 imbalance_ratio=1.599976 max_load_reduction=52429 max_load_reduction_pct=0.800003 moved_to_echo=28725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[119, 0, 101, 12, 0, 7291, 0, 0, 14, 0, 7, 2531, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1267, 1, 0, 0, 94, 18, 0, 0, 2, 0, 0, 10, 304, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 4, 0, 8077, 2, 0, 2, 2, 0, 0, 0, 0, 619, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5215, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2132, 1, 232, 1537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8153, 0, 0, 55, 0, 0, 196, 0, 462, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 2, 0, 2260, 0, 1241, 0, 0, 0, 1, 7178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3447, 86, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 0, 32, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 2, 0, 0, 5948, 0, 0, 1643, 12, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10076, 3087, 9723, 9121, 9825, 10988, 5053, 7663] avg=8192.0000 max=10988 max_rank=5 min=3087 min_rank=1 imbalance_ratio=1.341309 max_load_reduction=54548 max_load_reduction_pct=0.832336 moved_to_echo=33529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2278, 0, 5, 0, 19, 0, 2770, 0, 0, 0, 0, 0, 2936, 0, 0, 7656, 0, 0, 1, 0, 2, 0, 0, 21, 0, 7, 1, 0, 0, 0, 2, 0, 0, 0, 8138, 1, 0, 32, 24, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 46, 0, 0, 0, 6, 8, 227, 6106, 0, 0, 9, 0, 0, 63, 1, 0, 0, 426, 21, 0, 6876, 0, 0, 0, 6, 0, 0, 3292, 0, 1, 0, 0, 2158, 0, 8062, 0, 0, 5, 0, 0, 5356, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 4, 2, 0, 75, 55, 0, 6525, 0, 213, 213, 207, 2, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[155, 0, 0, 0, 0, 184, 0, 1372, 0, 27, 0, 8095, 0, 4, 0, 0, 0, 0, 53, 229, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 83, 0, 0, 4, 0, 0, 0, 767, 7463, 0, 0, 0, 48, 0, 0, 0, 0, 0, 7950, 0, 0, 123, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 13, 0, 0, 431, 6936, 2637, 0, 0, 5, 0, 8136, 0, 2, 922, 0, 0, 1918, 2, 0, 0, 0, 0, 0, 6240, 0, 1, 0, 0, 2567, 438, 11, 0, 0, 0, 0, 0, 190, 28, 0, 0, 0, 0, 4, 0, 33, 0, 0, 0, 0, 0, 2, 0, 0, 3739, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[681, 20, 0, 10, 0, 1979, 0, 7, 0, 0, 206, 6562, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 6, 8040, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2581, 2225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 7321, 0, 0, 0, 0, 1606, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 1201, 0, 0, 0, 0, 71, 7, 3, 0, 0, 746, 0, 1967, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 1233, 137, 0, 0, 4, 0, 3045, 1658, 1, 0, 7, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 2, 0, 0, 0, 3621, 7, 0, 0, 3, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 731, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5444, 2326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 5457, 0, 0, 0, 0, 0, 19, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9476, 12976, 9553, 5525, 6355, 7652, 8512, 5487] avg=8192.0000 max=12976 max_rank=1 min=5487 min_rank=7 imbalance_ratio=1.583984 max_load_reduction=52560 max_load_reduction_pct=0.802002 moved_to_echo=28006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 4, 0, 45, 0, 6011, 0, 29, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1, 3080, 39, 0, 2, 0, 0, 0, 0, 37, 3, 71, 11, 2085, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 9, 7494, 170, 0, 23, 2370, 3, 0, 10, 10, 6, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 43, 0, 0, 0, 128, 3, 0, 0, 0, 0, 0, 3642, 0, 1589, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1625, 5603, 53, 670, 0, 129, 419, 1341, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 597, 1936, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 555, 0, 0, 0, 0, 0, 0, 17, 0, 0, 190, 0, 8, 4302, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 385, 0, 400, 8, 0, 2793, 6235, 0, 1, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6096, 6562, 10154, 6124, 9875, 10478, 6350, 9897] avg=8192.0000 max=10478 max_rank=5 min=6096 min_rank=0 imbalance_ratio=1.279053 max_load_reduction=55058 max_load_reduction_pct=0.840118 moved_to_echo=36600 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[119, 0, 101, 12, 0, 7291, 0, 0, 14, 0, 7, 2531, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1267, 1, 0, 0, 94, 18, 0, 0, 2, 0, 0, 10, 304, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 4, 0, 8077, 2, 0, 2, 2, 0, 0, 0, 0, 619, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5215, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2132, 1, 232, 1537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8153, 0, 0, 55, 0, 0, 196, 0, 462, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 2, 0, 2260, 0, 1241, 0, 0, 0, 1, 7178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3447, 86, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 0, 32, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 2, 0, 0, 5948, 0, 0, 1643, 12, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10076, 3087, 9723, 9121, 9825, 10988, 5053, 7663] avg=8192.0000 max=10988 max_rank=5 min=3087 min_rank=1 imbalance_ratio=1.341309 max_load_reduction=54548 max_load_reduction_pct=0.832336 moved_to_echo=33529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 5, 0, 0, 0, 2, 0, 0, 464, 40, 5883, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 757, 0, 0, 1578, 0, 0, 0, 0, 36, 13, 0, 1945, 2442, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 9, 0, 0, 1, 7, 23, 0, 0, 0, 7962, 0, 0, 4, 0, 1440, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 31, 0, 53, 0, 0, 0, 10, 0, 0, 0, 0, 169, 144, 1861, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5626, 0, 559, 0, 9, 0, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 2807, 0, 1, 0, 19, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 0, 4, 0, 613, 0, 7370, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 5590, 0, 140, 283, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6403, 8758, 10080, 3996, 7764, 10859, 9857, 7819] avg=8192.0000 max=10859 max_rank=5 min=3996 min_rank=3 imbalance_ratio=1.325562 max_load_reduction=54677 max_load_reduction_pct=0.834305 moved_to_echo=36299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 5, 0, 0, 196, 0, 3, 0, 280, 0, 6310, 0, 0, 0, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3348, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 23, 1127, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 634, 22, 0, 0, 416, 0, 0, 0, 0, 1551, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4118, 1, 0, 1, 0, 169, 0, 0, 0, 72, 0, 0, 0, 8094, 0, 860, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 2, 0, 715, 7, 160, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 1, 145, 0, 1, 1, 26, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 22, 0, 0, 7147, 2269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4, 3730, 0, 5962, 0, 40, 4, 8184, 0, 0, 0, 7, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7583, 6674, 3304, 14863, 1802, 3726, 9442, 18142] avg=8192.0000 max=18142 max_rank=7 min=1802 min_rank=4 imbalance_ratio=2.214600 max_load_reduction=47394 max_load_reduction_pct=0.723175 moved_to_echo=33112 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2278, 0, 5, 0, 19, 0, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2936, 0, 0, 7656, 0, 0, 1, 0, 2, 0, 0, 21, 0, 7, 1, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8138, 1, 0, 32, 24, 0, 0, 0, 0, 544, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 46, 0, 0, 0, 6, 0, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 227, 6106, 0, 0, 9, 0, 0, 63, 1, 0, 0, 426, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1459, 0, 1, 0, 0, 2158, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5356, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 6876, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 75, 55, 0, 4759, 0, 213, 213, 207, 2, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4528, 11520, 9277, 2939, 6861, 11686, 13186, 5539] avg=8192.0000 max=13186 max_rank=6 min=2939 min_rank=3 imbalance_ratio=1.609619 max_load_reduction=52350 max_load_reduction_pct=0.798798 moved_to_echo=37272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[155, 0, 0, 0, 0, 184, 0, 1372, 0, 27, 0, 6503, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 229, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 83, 3383, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 767, 7463, 0, 0, 0, 48, 0, 0, 0, 0, 1592, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 123, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 13, 2579, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 6936, 2637, 0, 0, 5, 0, 2174, 0, 2, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2765, 0, 1, 0, 0, 2567, 438, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 28, 0, 0, 0, 0, 4, 0, 33, 0, 0, 0, 0, 1918, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3739, 0, 2144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 7227, 10115, 11224, 13107, 5784, 3949, 5885] avg=8192.0000 max=13107 max_rank=4 min=3949 min_rank=6 imbalance_ratio=1.599976 max_load_reduction=52429 max_load_reduction_pct=0.800003 moved_to_echo=28725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 1541, 0, 0, 7425, 4162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 0, 5, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 2514, 3116, 2229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1463, 3, 299, 0, 0, 0, 0, 0, 0, 1018, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 2375, 1239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7639, 5, 62, 0, 0, 1, 2002, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 180, 0, 0, 0, 3628, 8, 0, 3, 0, 0, 0, 258, 283, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 6803, 2330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4794, 2, 150, 65, 0, 0, 0, 32, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13129, 8732, 3330, 11744, 9947, 4363, 9187, 5104] avg=8192.0000 max=13129 max_rank=0 min=3330 min_rank=2 imbalance_ratio=1.602661 max_load_reduction=52407 max_load_reduction_pct=0.799667 moved_to_echo=28601 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2278, 0, 5, 0, 19, 0, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2936, 0, 0, 7656, 0, 0, 1, 0, 2, 0, 0, 21, 0, 7, 1, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8138, 1, 0, 32, 24, 0, 0, 0, 0, 544, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 46, 0, 0, 0, 6, 0, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 227, 6106, 0, 0, 9, 0, 0, 63, 1, 0, 0, 426, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1459, 0, 1, 0, 0, 2158, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5356, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 6876, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 75, 55, 0, 4759, 0, 213, 213, 207, 2, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4528, 11520, 9277, 2939, 6861, 11686, 13186, 5539] avg=8192.0000 max=13186 max_rank=6 min=2939 min_rank=3 imbalance_ratio=1.609619 max_load_reduction=52350 max_load_reduction_pct=0.798798 moved_to_echo=37272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 3200, 1024, 512, 25984, 128, 128, 1664, 128, 128, 128, 128, 128, 128, 128, 4608, 15104, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1664, 18048, 384, 128, 1408, 128, 256, 256, 128, 128, 128, 128, 0, 0, 22912, 256, 11520, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 7936, 128, 35072, 128, 640, 5120, 16512, 128, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 256, 384, 128, 640, 22016, 128, 1408, 128, 128, 4736, 512, 35456, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 9600, 896, 0, 128, 128, 128, 128, 128, 128, 128, 256, 640, 128, 39936, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 256, 384, 128, 640, 22016, 128, 1408, 128, 128, 4736, 512, 35456, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 256, 9472, 34176, 2816, 1408, 128, 256, 3584, 9728, 128, 640, 3328, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1024, 128, 128, 128, 0, 12032, 128, 3072, 128, 6912, 7680, 34304, 0, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 384, 128, 128, 0, 128, 896, 33408, 256, 384, 128, 18176, 128, 0, 128, 128, 8448, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1024, 128, 128, 128, 0, 12032, 128, 3072, 128, 6912, 7680, 34304, 0, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 6, 2, 0, 24, 0, 0, 0, 0, 0, 7860, 0, 47, 0, 725, 78, 0, 5008, 0, 278, 0, 0, 1, 0, 77, 49, 0, 1780, 0, 0, 0, 149, 0, 0, 0, 0, 0, 11, 0, 20, 0, 0, 5, 17, 0, 102, 262, 972, 0, 0, 0, 0, 0, 0, 0, 240, 0, 919, 21, 0, 17, 2156, 178, 0, 0, 4, 8, 0, 0, 44, 0, 1668, 0, 0, 6, 0, 0, 7498, 0, 0, 0, 1842, 0, 7870, 1, 0, 0, 924, 0, 40, 0, 0, 0, 424, 88, 5, 29, 0, 0, 0, 0, 0, 6722, 820, 7490, 0, 2, 1, 13, 0, 0, 45, 0, 8172, 1, 1, 0, 736, 0, 0, 78, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 82, 0, 933, 0, 0, 8, 0, 0, 0, 0, 0, 4912, 8172, 235, 850, 0, 0, 0, 5086, 0, 3, 13, 0, 0, 0, 160, 0, 0, 0, 0, 6, 0, 0, 312, 0, 238, 0, 0, 0, 0, 0, 0, 0, 444, 0, 3313, 0, 0, 0, 11, 529, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 7327, 0, 0, 5317, 0, 0, 0, 0, 0, 29, 8087, 0, 0, 0, 0, 0, 1132, 0, 0, 7641, 0, 0, 2271, 0, 21, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 46, 1, 0, 2, 8, 1, 43, 2, 0, 7, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 18, 0, 8128, 0, 0, 5339, 132, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 598, 0, 8061, 0, 102, 191, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 2, 0, 0, 0, 85, 71, 0, 0, 7468, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 1625, 0, 0, 0, 0, 0, 996, 0, 0, 0, 0, 0, 0, 0, 10, 1754, 1564, 803, 0, 0, 2, 0, 0, 5937, 0, 0, 5740, 0, 0, 0, 0, 7957, 0, 0, 0, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 82, 0, 933, 0, 0, 8, 0, 0, 0, 0, 0, 4912, 8172, 235, 850, 0, 0, 0, 5086, 0, 3, 13, 0, 0, 0, 160, 0, 0, 0, 0, 6, 0, 0, 312, 0, 238, 0, 0, 0, 0, 0, 0, 0, 444, 0, 3313, 0, 0, 0, 11, 529, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 7327, 0, 0, 5317, 0, 0, 0, 0, 0, 29, 8087, 0, 0, 0, 0, 0, 1132, 0, 0, 7641, 0, 0, 2271, 0, 21, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1856, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 0, 971, 7222, 3801, 0, 104, 1084, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1, 0, 187, 0, 1983, 2, 6, 0, 0, 0, 1, 0, 0, 0, 6680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1265, 0, 0, 0, 7545, 8108, 0, 0, 0, 0, 8187, 0, 79, 0, 0, 5, 0, 0, 1, 1, 5622, 0, 0, 1, 0, 0, 13, 0, 0, 7, 0, 2825, 0, 0, 0, 0, 0, 0, 0, 12, 4517, 0, 0, 0, 38, 9, 0, 0, 0, 328] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3289, 0, 0, 7052, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 21, 7793, 0, 96, 2, 0, 0, 1, 0, 0, 1112, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1, 0, 0, 0, 73, 0, 0, 168, 0, 4, 0, 0, 5439, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7935, 0, 0, 15, 4, 0, 22, 16, 0, 0, 7172, 404, 0, 1, 38, 0, 0, 0, 0, 6795, 0, 3814, 0, 0, 0, 0, 0, 5787, 0, 540, 7645, 0, 29, 0, 0, 0, 19, 51, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 1, 74, 0, 1, 0, 0, 0, 0, 3098, 0, 0, 0, 10, 0, 0, 0, 501, 132, 0, 0, 0, 7916, 213, 0, 0, 8157, 0, 0, 0, 0, 190, 1321, 0, 325, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 565, 0, 0, 0, 323, 0, 0, 0, 12, 0, 7629, 0, 0, 5, 23, 6774, 0, 0, 0, 111, 2, 0, 0, 657, 22, 0, 2994, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 8168, 0, 234, 0, 268, 0, 13, 0, 1, 0, 13, 22, 1, 0, 0, 147, 2, 0, 0, 0, 64, 7069, 1, 0, 32, 0, 0, 311, 0, 5, 4523, 0, 166, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 1074, 0, 0, 2, 0, 0, 0, 0, 29, 0, 0, 0, 0, 2319, 6249, 12, 0, 8163, 0, 12, 7, 6760, 1, 0, 0, 0, 30, 8151, 26, 3, 0, 740, 0, 11, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 0, 0, 5427, 0, 3, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 2660, 0, 7877, 0, 0, 160, 698, 0, 1, 0, 2, 0, 0, 106, 7, 0, 0, 0, 0, 0, 0, 3, 0, 552, 1090, 232, 0, 9, 0, 206, 1, 0, 8040, 0, 231, 0, 0, 969, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 12, 1676, 0, 33, 24, 0, 4, 0, 56, 0, 1260, 0, 137, 0, 0, 0, 0, 37, 0, 0, 0, 0, 4326, 0, 0, 6478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6, 2994, 0, 0, 0, 0, 27, 0, 0, 0, 0, 437, 0, 0, 2, 57, 0, 8045, 0, 0, 0, 0, 717, 0, 0, 0, 1, 5989, 2456, 0, 0, 9, 235, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 27, 1, 0, 0, 0, 7546, 8186, 2, 2, 0, 7, 9, 110, 780, 421, 7650, 0, 0, 0, 1, 0, 0, 0, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1856, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 0, 971, 7222, 3801, 0, 104, 1084, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1, 0, 187, 0, 1983, 2, 6, 0, 0, 0, 1, 0, 0, 0, 6680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1265, 0, 0, 0, 7545, 8108, 0, 0, 0, 0, 8187, 0, 79, 0, 0, 5, 0, 0, 1, 1, 5622, 0, 0, 1, 0, 0, 13, 0, 0, 7, 0, 2825, 0, 0, 0, 0, 0, 0, 0, 12, 4517, 0, 0, 0, 38, 9, 0, 0, 0, 328] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 1, 74, 0, 1, 0, 0, 0, 0, 3098, 0, 0, 0, 10, 0, 0, 0, 501, 132, 0, 0, 0, 7916, 213, 0, 0, 8157, 0, 0, 0, 0, 190, 1321, 0, 325, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 565, 0, 0, 0, 323, 0, 0, 0, 12, 0, 7629, 0, 0, 5, 23, 6774, 0, 0, 0, 111, 2, 0, 0, 657, 22, 0, 2994, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 8168, 0, 234, 0, 268, 0, 13, 0, 1, 0, 13, 22, 1, 0, 0, 147, 2, 0, 0, 0, 64, 7069, 1, 0, 32, 0, 0, 311, 0, 5, 4523, 0, 166, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 46, 1, 0, 2, 8, 1, 43, 2, 0, 7, 1, 0, 4373, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 18, 0, 7905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4922, 132, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 8061, 0, 102, 191, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 2, 0, 0, 0, 85, 71, 0, 0, 7468, 0, 1776, 417, 223, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3668, 0, 0, 0, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 0, 0, 0, 0, 0, 0, 10, 1754, 1564, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5937, 0, 0, 3964, 0, 0, 0, 0, 7957, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4603, 7928, 5445, 9481, 10447, 5295, 4466, 17871] avg=8192.0000 max=17871 max_rank=7 min=4466 min_rank=6 imbalance_ratio=2.181519 max_load_reduction=47665 max_load_reduction_pct=0.727310 moved_to_echo=38079 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4452, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2948, 0, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6658, 3801, 0, 104, 1084, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 187, 0, 1983, 2, 6, 0, 0, 0, 1, 0, 0, 0, 6680, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1265, 0, 1397, 564, 83, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7545, 8108, 0, 0, 0, 0, 3735, 0, 79, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5622, 0, 0, 1, 0, 0, 13, 0, 0, 7, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3120, 0, 0, 0, 38, 9, 0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4875, 5787, 11661, 10707, 3383, 19472, 6144, 3507] avg=8192.0000 max=19472 max_rank=5 min=3383 min_rank=4 imbalance_ratio=2.376953 max_load_reduction=46064 max_load_reduction_pct=0.702881 moved_to_echo=32506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 6, 2, 0, 24, 0, 0, 0, 0, 0, 7860, 4280, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 725, 78, 0, 5008, 0, 278, 0, 0, 1, 0, 75, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 0, 0, 0, 149, 0, 0, 0, 0, 0, 11, 0, 20, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 102, 262, 972, 0, 0, 0, 0, 0, 0, 0, 240, 0, 919, 21, 4899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2156, 178, 0, 0, 4, 8, 0, 0, 44, 0, 1668, 0, 0, 6, 2528, 139, 2, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7498, 0, 0, 0, 1842, 0, 3590, 1, 0, 0, 924, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 88, 5, 29, 0, 0, 0, 0, 0, 6722, 820, 1319, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 45, 0, 5644, 1, 1, 0, 736, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13274, 6261, 1826, 7432, 6920, 13895, 9409, 6519] avg=8192.0000 max=13895 max_rank=5 min=1826 min_rank=2 imbalance_ratio=1.696167 max_load_reduction=51641 max_load_reduction_pct=0.787979 moved_to_echo=36743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 1074, 4283, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 29, 0, 0, 0, 0, 2319, 6078, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 12, 7, 6760, 1, 0, 0, 0, 30, 8151, 26, 3, 0, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5427, 0, 3, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 2, 2488, 638, 171, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2660, 0, 3594, 0, 0, 160, 698, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 106, 7, 0, 0, 0, 0, 0, 0, 3, 0, 98, 1090, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 206, 1, 0, 5552, 0, 231, 0, 0, 969, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5441, 8440, 23255, 3985, 8786, 7114, 1538, 6977] avg=8192.0000 max=23255 max_rank=2 min=1538 min_rank=6 imbalance_ratio=2.838745 max_load_reduction=42281 max_load_reduction_pct=0.645157 moved_to_echo=24415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3289, 0, 0, 7052, 4315, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 21, 7580, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1112, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1, 0, 0, 0, 73, 0, 0, 168, 0, 2494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5439, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2365, 0, 213, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3620, 0, 0, 15, 4, 0, 22, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7172, 404, 0, 1, 38, 0, 0, 0, 0, 6795, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5787, 0, 540, 5280, 0, 29, 0, 0, 0, 19, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15228, 7747, 1164, 2822, 8108, 3678, 15083, 11706] avg=8192.0000 max=15228 max_rank=0 min=1164 min_rank=2 imbalance_ratio=1.858887 max_load_reduction=50308 max_load_reduction_pct=0.767639 moved_to_echo=38575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 1074, 4283, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 29, 0, 0, 0, 0, 2319, 6078, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 12, 7, 6760, 1, 0, 0, 0, 30, 8151, 26, 3, 0, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5427, 0, 3, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 2, 2488, 638, 171, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2660, 0, 3594, 0, 0, 160, 698, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 106, 7, 0, 0, 0, 0, 0, 0, 3, 0, 98, 1090, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 206, 1, 0, 5552, 0, 231, 0, 0, 969, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5441, 8440, 23255, 3985, 8786, 7114, 1538, 6977] avg=8192.0000 max=23255 max_rank=2 min=1538 min_rank=6 imbalance_ratio=2.838745 max_load_reduction=42281 max_load_reduction_pct=0.645157 moved_to_echo=24415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 1, 74, 0, 1, 0, 0, 0, 0, 3098, 4442, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 501, 132, 0, 0, 0, 7916, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 0, 0, 190, 1321, 0, 325, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 323, 0, 0, 0, 12, 0, 7629, 0, 4623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 23, 6774, 0, 0, 0, 111, 2, 0, 0, 657, 22, 0, 2994, 0, 1399, 637, 5, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 3726, 0, 234, 0, 268, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 22, 1, 0, 0, 147, 2, 0, 0, 0, 64, 1246, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 311, 0, 5, 3124, 0, 166, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8673, 8767, 9452, 13152, 12789, 7567, 1497, 3639] avg=8192.0000 max=13152 max_rank=3 min=1497 min_rank=6 imbalance_ratio=1.605469 max_load_reduction=52384 max_load_reduction_pct=0.799316 moved_to_echo=25492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 2825, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 933, 0, 0, 8, 0, 0, 0, 0, 0, 4912, 7944, 235, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5086, 0, 3, 13, 0, 0, 0, 160, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 238, 0, 0, 0, 0, 0, 0, 0, 444, 0, 3313, 0, 5294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 529, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2369, 0, 228, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 7327, 0, 0, 5317, 0, 0, 0, 0, 0, 29, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 0, 0, 5272, 0, 0, 2271, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7164, 14964, 5268, 9601, 3356, 2365, 14122, 8696] avg=8192.0000 max=14964 max_rank=1 min=2365 min_rank=5 imbalance_ratio=1.826660 max_load_reduction=50572 max_load_reduction_pct=0.771667 moved_to_echo=28539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4452, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2948, 0, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6658, 3801, 0, 104, 1084, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 187, 0, 1983, 2, 6, 0, 0, 0, 1, 0, 0, 0, 6680, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1265, 0, 1397, 564, 83, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7545, 8108, 0, 0, 0, 0, 3735, 0, 79, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5622, 0, 0, 1, 0, 0, 13, 0, 0, 7, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3120, 0, 0, 0, 38, 9, 0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4875, 5787, 11661, 10707, 3383, 19472, 6144, 3507] avg=8192.0000 max=19472 max_rank=5 min=3383 min_rank=4 imbalance_ratio=2.376953 max_load_reduction=46064 max_load_reduction_pct=0.702881 moved_to_echo=32506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 12, 1676, 0, 33, 24, 0, 4, 0, 56, 3257, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 0, 137, 0, 0, 0, 0, 37, 0, 0, 0, 0, 4208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6, 2994, 0, 5354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 437, 0, 0, 2, 57, 0, 8045, 0, 2367, 508, 118, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 717, 0, 0, 0, 1, 2732, 2456, 0, 0, 9, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 27, 1, 0, 0, 0, 7546, 1441, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 110, 780, 421, 5283, 0, 0, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 5642, 7465, 8356, 11747, 6150, 13277, 6621] avg=8192.0000 max=13277 max_rank=6 min=5642 min_rank=1 imbalance_ratio=1.620728 max_load_reduction=52259 max_load_reduction_pct=0.797409 moved_to_echo=37795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 1, 74, 0, 1, 0, 0, 0, 0, 3098, 4442, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 501, 132, 0, 0, 0, 7916, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 0, 0, 190, 1321, 0, 325, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 323, 0, 0, 0, 12, 0, 7629, 0, 4623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 23, 6774, 0, 0, 0, 111, 2, 0, 0, 657, 22, 0, 2994, 0, 1399, 637, 5, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 3726, 0, 234, 0, 268, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 22, 1, 0, 0, 147, 2, 0, 0, 0, 64, 1246, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 311, 0, 5, 3124, 0, 166, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8673, 8767, 9452, 13152, 12789, 7567, 1497, 3639] avg=8192.0000 max=13152 max_rank=3 min=1497 min_rank=6 imbalance_ratio=1.605469 max_load_reduction=52384 max_load_reduction_pct=0.799316 moved_to_echo=25492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 18560, 8064, 128, 128, 5376, 256, 128, 0, 128, 15360, 10112, 6912, 1152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[34304, 3968, 128, 5248, 8064, 256, 2560, 0, 384, 128, 8448, 128, 128, 1536, 896, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 15104, 12160, 0, 128, 4608, 128, 27136, 2560, 384, 256, 3584, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -2051,109 +2051,109 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[90, 8158, 0, 37, 0, 0, 5474, 0, 0, 10, 0, 0, 0, 0, 0, 2, 0, 1, 2299, 0, 0, 134, 347, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3016, 18, 0, 0, 0, 301, 509, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 6596, 0, 3138, 0, 0, 0, 27, 0, 1, 0, 0, 0, 0, 5312, 0, 0, 912, 0, 0, 1668, 6930, 0, 0, 0, 7175, 0, 1578, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 13, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 374, 0, 122, 0, 0, 7249, 0, 0, 0, 0, 0, 960, 0, 133, 1, 0, 0, 1, 0, 13, 3, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 39, 2, 6005, 0, 13, 17, 539, 0, 0, 0, 4, 2046, 0, 0, 0, 1570, 146, 0, 0, 0, 2, 0, 0, 0, 3703, 0, 0, 8042, 6193, 0, 0, 0, 0, 293, 0, 8, 0, 5, 15, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8085, 2378, 0, 4, 0, 6270, 12, 1298, 0, 14, 0, 0, 0, 0, 3262, 36, 6606, 0, 0, 3, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 2259, 0, 0, 0, 0, 620, 0, 0, 7910, 0, 0, 0, 0, 0, 0, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 6298, 0, 1685, 0, 0, 1, 637, 43, 0, 264, 0, 0, 0, 0, 1, 12, 0, 0, 0, 67, 28, 1, 0, 0, 0, 5745, 0, 0, 8181, 0, 0, 0, 8, 8189, 0, 5696, 0, 12, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 0, 5, 0, 181, 1010, 0, 0, 1057, 7623, 0, 0, 1, 0, 1, 1, 0, 73, 0, 0, 30, 0, 0, 5715, 0, 0, 0, 0, 0, 0, 77, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 0, 0, 0, 556, 0, 7909, 0, 0, 0, 0, 0, 0, 0, 1, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 7023, 0, 5061, 0, 0, 0, 3115, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 32, 0, 2, 0, 48, 7925, 0, 1, 7241, 0, 0, 1, 0, 7200, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 90, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 0, 5356, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[90, 8158, 0, 37, 0, 0, 5474, 0, 0, 10, 0, 0, 0, 0, 0, 2, 0, 1, 2299, 0, 0, 134, 347, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3016, 18, 0, 0, 0, 301, 509, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 6596, 0, 3138, 0, 0, 0, 27, 0, 1, 0, 0, 0, 0, 5312, 0, 0, 912, 0, 0, 1668, 6930, 0, 0, 0, 7175, 0, 1578, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 13, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 5436, 0, 0, 0, 0, 2, 0, 32, 5260, 0, 3, 0, 0, 0, 0, 0, 0, 8184, 30, 3, 0, 1, 0, 0, 0, 0, 0, 8, 0, 21, 32, 8177, 53, 0, 0, 147, 43, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 8175, 3791, 0, 0, 0, 0, 0, 0, 156, 0, 61, 0, 0, 0, 0, 1, 0, 0, 3, 7628, 0, 510, 0, 0, 1, 0, 0, 1775, 6087, 0, 0, 0, 4039, 0, 0, 0, 682, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 555] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 7122, 0, 0, 0, 1570, 0, 0, 0, 4651, 0, 0, 0, 0, 0, 0, 0, 0, 7100, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 130, 2, 0, 0, 0, 45, 0, 196, 0, 0, 0, 0, 4301, 0, 2, 8151, 1, 5, 0, 0, 0, 0, 0, 0, 162, 21, 0, 0, 0, 0, 75, 0, 0, 0, 0, 13, 0, 7, 0, 0, 8099, 0, 12, 4127, 0, 57, 0, 0, 7055, 1135, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 1, 3, 847, 2, 161, 46, 0, 0, 0, 0, 19, 517, 20, 1689, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 57, 0, 0, 1413, 0, 0, 0, 2721, 0, 10, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 1, 7, 0, 81, 0, 0, 0, 4, 0, 0, 131, 0, 0, 0, 0, 0, 0, 458, 5064, 26, 0, 0, 0, 0, 0, 0, 7996, 0, 5, 144, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 864, 7279, 0, 0, 8181, 0, 0, 0, 873, 6827, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 3, 0, 0, 0, 7267, 2870, 0, 2, 98, 8154, 0, 0, 8, 3443, 0, 0, 0, 6, 0, 0, 127, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 548, 0, 369] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 426, 0, 0, 3, 0, 3452, 10, 8, 5010, 0, 0, 0, 479, 0, 0, 0, 0, 420, 407, 1, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 142, 0, 0, 251, 0, 0, 0, 0, 2, 0, 2, 60, 4362, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 7359, 0, 0, 0, 0, 7230, 7588, 0, 0, 7, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 221, 0, 0, 0, 91, 8165, 0, 0, 0, 6192, 0, 2, 0, 1, 0, 20, 0, 2, 0, 0, 1771, 0, 0, 7999, 0, 0, 0, 0, 0, 0, 0, 709, 0, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 374, 0, 122, 0, 0, 4775, 0, 0, 0, 0, 0, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 1, 0, 0, 1, 0, 13, 3, 0, 0, 0, 1, 0, 0, 0, 3014, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 39, 2, 6005, 0, 487, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 17, 539, 0, 0, 0, 4, 2046, 0, 0, 0, 1570, 146, 0, 0, 0, 3256, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3703, 0, 0, 0, 6193, 0, 0, 0, 0, 293, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 15, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 2300, 0, 4, 0, 0, 12, 1298, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3262, 36, 6606, 0, 0, 3, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6232, 5579, 6615, 7593, 10193, 5711, 11713, 11900] avg=8192.0000 max=11900 max_rank=7 min=5579 min_rank=1 imbalance_ratio=1.452637 max_load_reduction=53636 max_load_reduction_pct=0.818420 moved_to_echo=39517 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 0, 0, 0, 556, 0, 7909, 0, 0, 0, 0, 0, 0, 0, 1, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 7023, 0, 5061, 0, 0, 0, 3115, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 32, 0, 2, 0, 48, 7925, 0, 1, 7241, 0, 0, 1, 0, 7200, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 90, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 0, 5356, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[90, 8158, 0, 37, 0, 0, 5474, 0, 0, 8, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2299, 0, 0, 134, 347, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3449, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3016, 18, 0, 0, 0, 301, 509, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 3726, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3138, 0, 0, 0, 27, 0, 1, 0, 0, 0, 0, 5312, 0, 0, 912, 2456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1668, 6702, 0, 0, 0, 0, 0, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 13, 0, 0, 0, 3325, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13769, 7301, 12207, 6029, 1009, 11846, 9948, 3427] avg=8192.0000 max=13769 max_rank=0 min=1009 min_rank=4 imbalance_ratio=1.680786 max_load_reduction=51767 max_load_reduction_pct=0.789902 moved_to_echo=26230 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 374, 0, 122, 0, 0, 4775, 0, 0, 0, 0, 0, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 1, 0, 0, 1, 0, 13, 3, 0, 0, 0, 1, 0, 0, 0, 3014, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 39, 2, 6005, 0, 487, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 17, 539, 0, 0, 0, 4, 2046, 0, 0, 0, 1570, 146, 0, 0, 0, 3256, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3703, 0, 0, 0, 6193, 0, 0, 0, 0, 293, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 15, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 2300, 0, 4, 0, 0, 12, 1298, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3262, 36, 6606, 0, 0, 3, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6232, 5579, 6615, 7593, 10193, 5711, 11713, 11900] avg=8192.0000 max=11900 max_rank=7 min=5579 min_rank=1 imbalance_ratio=1.452637 max_load_reduction=53636 max_load_reduction_pct=0.818420 moved_to_echo=39517 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5436, 0, 0, 0, 0, 2, 0, 32, 3465, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 30, 3, 0, 1, 0, 0, 0, 0, 0, 8, 0, 21, 32, 1941, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 53, 0, 0, 147, 43, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 353, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2098, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 0, 3791, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 1, 0, 0, 3, 7628, 0, 510, 0, 0, 1, 5723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 5887, 0, 0, 0, 0, 0, 0, 0, 682, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 555, 78, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8938, 12672, 8988, 5377, 5198, 13927, 8359, 2077] avg=8192.0000 max=13927 max_rank=5 min=2077 min_rank=7 imbalance_ratio=1.700073 max_load_reduction=51609 max_load_reduction_pct=0.787491 moved_to_echo=29561 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 57, 0, 0, 1413, 0, 0, 0, 1793, 0, 10, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 1, 7, 0, 3919, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 4, 0, 0, 131, 0, 0, 0, 0, 0, 0, 458, 5064, 182, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 7996, 0, 5, 144, 0, 0, 0, 0, 0, 4235, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 864, 0, 0, 0, 8181, 0, 0, 0, 873, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 3, 0, 5095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7267, 2776, 0, 2, 98, 0, 0, 0, 8, 3443, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 548, 0, 369, 3442, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3296, 6248, 6014, 14759, 10971, 5411, 13600, 5237] avg=8192.0000 max=14759 max_rank=3 min=3296 min_rank=0 imbalance_ratio=1.801636 max_load_reduction=50777 max_load_reduction_pct=0.774796 moved_to_echo=35219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2259, 0, 0, 0, 0, 620, 0, 0, 5210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 3665, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 6298, 0, 531, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 0, 1, 637, 43, 0, 264, 0, 0, 0, 0, 1, 12, 0, 0, 3958, 2823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 28, 1, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 8, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5696, 0, 12, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 181, 977, 0, 0, 1057, 0, 0, 0, 1, 0, 1, 1, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 5715, 0, 0, 0, 0, 0, 0, 77, 8, 0, 0, 4129, 2169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 7464, 6867, 9424, 9522, 9746, 2296, 12128] avg=8192.0000 max=12128 max_rank=7 min=2296 min_rank=6 imbalance_ratio=1.480469 max_load_reduction=53408 max_load_reduction_pct=0.814941 moved_to_echo=33692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7122, 0, 0, 0, 1570, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7100, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 130, 408, 2431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 45, 0, 196, 0, 0, 0, 0, 4301, 0, 2, 8151, 1, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 162, 21, 0, 0, 0, 0, 75, 0, 0, 439, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 7, 0, 0, 0, 0, 12, 4127, 0, 57, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 5668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 1, 3, 0, 2, 161, 46, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 20, 1689, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3557, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11756, 10072, 13010, 3134, 5282, 6813, 8263, 7206] avg=8192.0000 max=13010 max_rank=2 min=3134 min_rank=3 imbalance_ratio=1.588135 max_load_reduction=52526 max_load_reduction_pct=0.801483 moved_to_echo=27564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 0, 0, 0, 556, 0, 5199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 7023, 0, 537, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 0, 0, 0, 3115, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 2, 0, 48, 0, 0, 1, 7241, 0, 0, 1, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 0, 5547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 0, 5356, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 3633, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 10163, 7744, 11635, 8407, 6788, 3345, 11677] avg=8192.0000 max=11677 max_rank=7 min=3345 min_rank=6 imbalance_ratio=1.425415 max_load_reduction=53859 max_load_reduction_pct=0.821823 moved_to_echo=30217 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 57, 0, 0, 1413, 0, 0, 0, 1793, 0, 10, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 1, 7, 0, 3919, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 4, 0, 0, 131, 0, 0, 0, 0, 0, 0, 458, 5064, 182, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 7996, 0, 5, 144, 0, 0, 0, 0, 0, 4235, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 864, 0, 0, 0, 8181, 0, 0, 0, 873, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 3, 0, 5095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7267, 2776, 0, 2, 98, 0, 0, 0, 8, 3443, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 548, 0, 369, 3442, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3296, 6248, 6014, 14759, 10971, 5411, 13600, 5237] avg=8192.0000 max=14759 max_rank=3 min=3296 min_rank=0 imbalance_ratio=1.801636 max_load_reduction=50777 max_load_reduction_pct=0.774796 moved_to_echo=35219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5436, 0, 0, 0, 0, 2, 0, 32, 3465, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 30, 3, 0, 1, 0, 0, 0, 0, 0, 8, 0, 21, 32, 1941, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 53, 0, 0, 147, 43, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 353, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2098, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 0, 3791, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 1, 0, 0, 3, 7628, 0, 510, 0, 0, 1, 5723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 5887, 0, 0, 0, 0, 0, 0, 0, 682, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 555, 78, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8938, 12672, 8988, 5377, 5198, 13927, 8359, 2077] avg=8192.0000 max=13927 max_rank=5 min=2077 min_rank=7 imbalance_ratio=1.700073 max_load_reduction=51609 max_load_reduction_pct=0.787491 moved_to_echo=29561 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 426, 0, 0, 3, 0, 3452, 10, 8, 3301, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 407, 1, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2977, 2170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 142, 0, 0, 251, 0, 0, 0, 0, 2, 0, 2, 60, 336, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4362, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 2, 0, 3215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7359, 0, 0, 0, 0, 0, 7588, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 221, 0, 5060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 7896, 0, 0, 0, 0, 0, 2, 0, 1, 0, 20, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 0, 0, 7999, 0, 0, 0, 0, 0, 0, 0, 709, 0, 32, 0, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7679, 6037, 4091, 7585, 14954, 5294, 8012, 11884] avg=8192.0000 max=14954 max_rank=4 min=4091 min_rank=2 imbalance_ratio=1.825439 max_load_reduction=50582 max_load_reduction_pct=0.771820 moved_to_echo=40144 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7122, 0, 0, 0, 1570, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7100, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 130, 408, 2431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 45, 0, 196, 0, 0, 0, 0, 4301, 0, 2, 8151, 1, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 162, 21, 0, 0, 0, 0, 75, 0, 0, 439, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 7, 0, 0, 0, 0, 12, 4127, 0, 57, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 5668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 1, 3, 0, 2, 161, 46, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 20, 1689, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3557, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11756, 10072, 13010, 3134, 5282, 6813, 8263, 7206] avg=8192.0000 max=13010 max_rank=2 min=3134 min_rank=3 imbalance_ratio=1.588135 max_load_reduction=52526 max_load_reduction_pct=0.801483 moved_to_echo=27564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[14464, 128, 640, 128, 3840, 128, 640, 10496, 128, 128, 256, 1664, 256, 128, 128, 512, 20992, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3840, 128, 10240, 128, 0, 19200, 256, 128, 0, 0, 128, 128, 128, 1792, 128, 1024, 18176, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 256, 27904, 512, 128, 256, 640, 128, 128, 0, 0, 128, 128, 128, 128, 256, 19456, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 7552, 128, 3840, 1408, 1024, 0, 17664, 128, 27776, 128, 128, 384, 896, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 23424, 128, 128, 384, 3072, 9728, 640, 128, 26880, 0, 128, 128, 512, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 27264, 29824, 0, 128, 1280, 0, 128, 3072, 128, 4224, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 23424, 128, 128, 384, 3072, 9728, 640, 128, 26880, 0, 128, 128, 512, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[14336, 128, 0, 128, 384, 384, 768, 512, 0, 128, 0, 4352, 128, 128, 36096, 5248, 2816, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[1152, 9216, 128, 128, 128, 128, 128, 128, 128, 128, 8960, 128, 5888, 128, 256, 1024, 39296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4, 0, 0, 0, 1346, 0, 0, 0, 0, 20, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5, 470, 1, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 8165, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 89, 3, 0, 51, 83, 2, 0, 0, 0, 0, 0, 0, 1, 0, 5, 7447, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 7055, 1783, 51, 0, 0, 0, 0, 268, 0, 756, 124, 0, 3885, 7, 0, 0, 0, 0, 0, 8192, 176, 0, 0, 0, 7486, 0, 0, 0, 5094, 4862] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[11, 0, 2, 0, 417, 0, 0, 29, 1584, 0, 0, 19, 29, 0, 0, 6, 0, 0, 0, 128, 0, 7204, 0, 38, 0, 7458, 0, 0, 0, 0, 0, 0, 0, 27, 20, 0, 0, 0, 282, 80, 0, 1, 0, 0, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 465, 0, 0, 0, 0, 5971, 0, 0, 0, 0, 10, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1888, 0, 0, 0, 42, 0, 129, 0, 30, 1, 811, 2072, 1, 0, 0, 7714, 30, 0, 356, 0, 1, 0, 0, 0, 7873, 0, 0, 7597, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 884, 4825] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 2139, 0, 0, 0, 0, 0, 9, 32, 0, 305, 0, 0, 649, 97, 0, 0, 0, 0, 0, 0, 981, 0, 0, 0, 2636, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 1, 0, 2, 0, 1, 23, 0, 7297, 0, 2, 0, 7200, 4433, 0, 6081, 0, 0, 0, 0, 2, 0, 403, 0, 0, 0, 0, 1, 0, 48, 0, 0, 0, 112, 0, 40, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 959, 80, 863, 8109, 0, 12, 0, 0, 0, 0, 0, 75, 0, 0, 1, 0, 0, 1095, 5744, 52, 1, 0, 4356, 8100] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[136, 731, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 17, 0, 4, 0, 0, 0, 21, 0, 0, 249, 0, 0, 0, 4120, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6078, 0, 433, 0, 1, 0, 0, 0, 0, 1, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 5, 0, 0, 0, 0, 139, 0, 0, 7588, 0, 0, 0, 1, 0, 0, 511, 0, 0, 0, 2288, 0, 0, 5038, 0, 0, 2324, 8175, 0, 7446, 2, 0, 0, 1, 0, 1, 0, 4, 0, 6, 13, 0, 103, 0, 3374, 0, 0, 0, 7787, 8148] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 538, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1127, 0, 4, 320, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 2, 0, 0, 7759, 0, 0, 0, 0, 0, 0, 0, 7221, 0, 0, 9, 0, 0, 8, 0, 8121, 0, 0, 0, 604, 124, 5, 0, 0, 6462, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 156, 0, 0, 20, 0, 0, 889, 0, 34, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 789, 7062, 0, 0, 5298, 0, 0, 0, 0, 0, 0, 6750, 8119, 0, 9, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5106, 0, 6576, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 3679, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 8180, 0, 0, 0, 0, 0, 0, 188, 0, 413, 0, 460, 1175, 0, 131, 0, 0, 4723, 48, 0, 1433, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 16, 0, 309, 66] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[11, 0, 2, 0, 417, 0, 0, 29, 1584, 0, 0, 19, 29, 0, 0, 6, 0, 0, 0, 128, 0, 7204, 0, 38, 0, 7458, 0, 0, 0, 0, 0, 0, 0, 27, 20, 0, 0, 0, 282, 80, 0, 1, 0, 0, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 465, 0, 0, 0, 0, 5971, 0, 0, 0, 0, 10, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1888, 0, 0, 0, 42, 0, 129, 0, 30, 1, 811, 2072, 1, 0, 0, 7714, 30, 0, 356, 0, 1, 0, 0, 0, 7873, 0, 0, 7597, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 884, 4825] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 3, 0, 10, 0, 0, 1724, 0, 7240, 0, 6038, 1, 0, 0, 0, 2, 6466, 579, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 296, 0, 1, 0, 0, 0, 2734, 0, 14, 0, 0, 0, 0, 0, 43, 0, 1, 0, 0, 0, 0, 0, 18, 0, 422, 0, 0, 216, 0, 0, 0, 0, 0, 778, 0, 7604, 0, 0, 0, 0, 11, 34, 2, 5, 2790, 0, 0, 0, 11, 0, 3, 0, 0, 0, 13, 0, 0, 0, 0, 6903, 0, 1, 2, 0, 0, 2, 0, 0, 0, 0, 85, 0, 0, 7099, 0, 0, 1773, 3, 0, 5995, 1, 0, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 78, 0, 12, 4460] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[136, 731, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 17, 0, 4, 0, 0, 0, 21, 0, 0, 249, 0, 0, 0, 4120, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6078, 0, 433, 0, 1, 0, 0, 0, 0, 1, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 5, 0, 0, 0, 0, 139, 0, 0, 7588, 0, 0, 0, 1, 0, 0, 511, 0, 0, 0, 2288, 0, 0, 5038, 0, 0, 2324, 8175, 0, 7446, 2, 0, 0, 1, 0, 1, 0, 4, 0, 6, 13, 0, 103, 0, 3374, 0, 0, 0, 7787, 8148] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 2144, 0, 5, 0, 0, 0, 16, 539, 0, 0, 2, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 34, 0, 0, 0, 100, 0, 0, 0, 2, 0, 12, 0, 0, 1, 8066, 0, 0, 0, 3865, 2, 8189, 2, 0, 5441, 0, 0, 0, 0, 0, 3, 0, 0, 29, 0, 0, 3, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 2, 5141, 3, 517, 18, 0, 0, 0, 0, 0, 0, 0, 6802, 0, 0, 0, 23, 0, 0, 9, 9, 6036, 0, 91, 0, 0, 0, 13, 60, 0, 0, 0, 7654, 0, 0, 1077, 0, 10, 0, 0, 7985, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4, 0, 0, 0, 1346, 0, 0, 0, 0, 20, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5, 470, 1, 0, 178, 0, 0, 4947, 1777, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 4471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 2880, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 89, 3, 0, 51, 83, 2, 0, 0, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 7447, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 2108, 1783, 51, 0, 0, 0, 0, 268, 0, 0, 124, 0, 3885, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 176, 0, 0, 0, 7486, 0, 0, 0, 2758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1625, 7519, 12342, 2915, 987, 13297, 8239, 18612] avg=8192.0000 max=18612 max_rank=7 min=987 min_rank=4 imbalance_ratio=2.271973 max_load_reduction=46924 max_load_reduction_pct=0.716003 moved_to_echo=41135 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 2139, 0, 0, 0, 0, 0, 9, 32, 0, 305, 0, 0, 649, 97, 0, 0, 0, 0, 0, 0, 981, 0, 0, 0, 2636, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 1, 0, 2, 0, 1, 23, 0, 7297, 0, 2, 0, 7200, 4433, 0, 6081, 0, 0, 0, 0, 2, 0, 403, 0, 0, 0, 0, 1, 0, 48, 0, 0, 0, 112, 0, 40, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 959, 80, 863, 8109, 0, 12, 0, 0, 0, 0, 0, 75, 0, 0, 1, 0, 0, 1095, 5744, 52, 1, 0, 4356, 8100] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5106, 0, 6576, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 3679, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 8180, 0, 0, 0, 0, 0, 0, 188, 0, 413, 0, 460, 1175, 0, 131, 0, 0, 4723, 48, 0, 1433, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 16, 0, 309, 66] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4, 0, 0, 0, 1346, 0, 0, 0, 0, 20, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5, 470, 1, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 8165, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 89, 3, 0, 51, 83, 2, 0, 0, 0, 0, 0, 0, 1, 0, 5, 7447, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 7055, 1783, 51, 0, 0, 0, 0, 268, 0, 756, 124, 0, 3885, 7, 0, 0, 0, 0, 0, 8192, 176, 0, 0, 0, 7486, 0, 0, 0, 5094, 4862] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 538, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 4, 5626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 0, 4, 320, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 2, 0, 0, 7759, 0, 0, 0, 0, 0, 0, 0, 7221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8, 0, 8121, 0, 0, 0, 604, 124, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7062, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 20, 0, 0, 889, 0, 34, 0, 0, 0, 0, 67, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 789, 0, 0, 0, 5298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6750, 8119, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10028, 1452, 15056, 8868, 7996, 1168, 6089, 14879] avg=8192.0000 max=15056 max_rank=2 min=1168 min_rank=5 imbalance_ratio=1.837891 max_load_reduction=50480 max_load_reduction_pct=0.770264 moved_to_echo=30132 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[136, 731, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 17, 0, 4, 0, 1, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 249, 0, 0, 0, 4120, 0, 0, 143, 0, 0, 0, 1604, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 368, 0, 0, 0, 0, 0, 0, 7489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6078, 0, 433, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 5, 7446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 7588, 0, 0, 0, 1, 0, 0, 511, 0, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 5038, 0, 0, 2324, 8175, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 6, 13, 0, 103, 0, 3374, 0, 0, 0, 4216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1385, 6373, 7914, 6513, 7600, 11810, 16224, 7717] avg=8192.0000 max=16224 max_rank=6 min=1385 min_rank=0 imbalance_ratio=1.980469 max_load_reduction=49312 max_load_reduction_pct=0.752441 moved_to_echo=43351 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 2, 0, 417, 0, 0, 29, 1584, 0, 0, 19, 29, 0, 0, 6, 5198, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 7204, 0, 38, 0, 7458, 0, 0, 0, 0, 0, 0, 5409, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 20, 0, 0, 0, 282, 80, 0, 1, 0, 0, 0, 2600, 0, 0, 4435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 7873, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1888, 0, 0, 0, 42, 0, 129, 0, 30, 1, 811, 2072, 1, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 30, 0, 356, 0, 1, 0, 0, 0, 0, 0, 0, 7597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7546, 20376, 7445, 4988, 8721, 5394, 10289, 777] avg=8192.0000 max=20376 max_rank=1 min=777 min_rank=7 imbalance_ratio=2.487305 max_load_reduction=45160 max_load_reduction_pct=0.689087 moved_to_echo=25181 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2139, 0, 0, 0, 0, 0, 9, 32, 0, 305, 0, 0, 649, 97, 0, 5294, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 981, 0, 0, 0, 2636, 0, 8, 0, 0, 0, 0, 0, 1567, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 7446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 23, 0, 7297, 0, 2, 0, 2540, 4433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 403, 0, 0, 0, 0, 1, 0, 48, 0, 0, 8109, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 40, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 6, 0, 1997, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 959, 80, 863, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 1, 0, 0, 1095, 5744, 52, 1, 0, 2359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8946, 5425, 7456, 14299, 9350, 5548, 5185, 9327] avg=8192.0000 max=14299 max_rank=3 min=5185 min_rank=6 imbalance_ratio=1.745483 max_load_reduction=51237 max_load_reduction_pct=0.781815 moved_to_echo=29410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 10, 0, 0, 1724, 0, 7240, 0, 6038, 1, 0, 0, 0, 2, 6621, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6466, 579, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 296, 0, 1, 0, 59, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 0, 14, 0, 0, 0, 0, 0, 43, 0, 1, 0, 0, 0, 4099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 422, 0, 0, 216, 0, 0, 0, 0, 0, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 34, 2, 5, 2790, 0, 0, 0, 11, 0, 3, 0, 5995, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 6903, 0, 1, 2, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 7099, 0, 0, 1773, 3, 0, 0, 1, 0, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 78, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21872, 7721, 6891, 1434, 9834, 6926, 10364, 494] avg=8192.0000 max=21872 max_rank=0 min=494 min_rank=7 imbalance_ratio=2.669922 max_load_reduction=43664 max_load_reduction_pct=0.666260 moved_to_echo=27618 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 538, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 4, 5626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 0, 4, 320, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 2, 0, 0, 7759, 0, 0, 0, 0, 0, 0, 0, 7221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8, 0, 8121, 0, 0, 0, 604, 124, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7062, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 20, 0, 0, 889, 0, 34, 0, 0, 0, 0, 67, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 789, 0, 0, 0, 5298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6750, 8119, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10028, 1452, 15056, 8868, 7996, 1168, 6089, 14879] avg=8192.0000 max=15056 max_rank=2 min=1168 min_rank=5 imbalance_ratio=1.837891 max_load_reduction=50480 max_load_reduction_pct=0.770264 moved_to_echo=30132 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2139, 0, 0, 0, 0, 0, 9, 32, 0, 305, 0, 0, 649, 97, 0, 5294, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 981, 0, 0, 0, 2636, 0, 8, 0, 0, 0, 0, 0, 1567, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 7446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 23, 0, 7297, 0, 2, 0, 2540, 4433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 403, 0, 0, 0, 0, 1, 0, 48, 0, 0, 8109, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 40, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 6, 0, 1997, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 959, 80, 863, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 1, 0, 0, 1095, 5744, 52, 1, 0, 2359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8946, 5425, 7456, 14299, 9350, 5548, 5185, 9327] avg=8192.0000 max=14299 max_rank=3 min=5185 min_rank=6 imbalance_ratio=1.745483 max_load_reduction=51237 max_load_reduction_pct=0.781815 moved_to_echo=29410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4, 0, 0, 0, 1346, 0, 0, 0, 0, 20, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5, 470, 1, 0, 178, 0, 0, 4947, 1777, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 4471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 2880, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 89, 3, 0, 51, 83, 2, 0, 0, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 7447, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 2108, 1783, 51, 0, 0, 0, 0, 268, 0, 0, 124, 0, 3885, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 176, 0, 0, 0, 7486, 0, 0, 0, 2758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1625, 7519, 12342, 2915, 987, 13297, 8239, 18612] avg=8192.0000 max=18612 max_rank=7 min=987 min_rank=4 imbalance_ratio=2.271973 max_load_reduction=46924 max_load_reduction_pct=0.716003 moved_to_echo=41135 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6667, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 3, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5106, 0, 6576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 3679, 0, 0, 0, 0, 1433, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 8180, 0, 0, 0, 0, 0, 0, 188, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 135, 1175, 0, 131, 0, 0, 4723, 48, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 16, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6679, 8514, 8297, 11682, 14274, 8516, 6981, 593] avg=8192.0000 max=14274 max_rank=4 min=593 min_rank=7 imbalance_ratio=1.742432 max_load_reduction=51262 max_load_reduction_pct=0.782196 moved_to_echo=30364 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[136, 731, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 17, 0, 4, 0, 1, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 249, 0, 0, 0, 4120, 0, 0, 143, 0, 0, 0, 1604, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 368, 0, 0, 0, 0, 0, 0, 7489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6078, 0, 433, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 5, 7446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 7588, 0, 0, 0, 1, 0, 0, 511, 0, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 5038, 0, 0, 2324, 8175, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 6, 13, 0, 103, 0, 3374, 0, 0, 0, 4216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1385, 6373, 7914, 6513, 7600, 11810, 16224, 7717] avg=8192.0000 max=16224 max_rank=6 min=1385 min_rank=0 imbalance_ratio=1.980469 max_load_reduction=49312 max_load_reduction_pct=0.752441 moved_to_echo=43351 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 2144, 0, 4738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 16, 539, 0, 0, 2, 0, 1042, 0, 0, 0, 0, 0, 4769, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 34, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 0, 1, 8066, 0, 0, 0, 3865, 2, 2887, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 29, 0, 0, 3, 0, 0, 0, 0, 6036, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 2, 5141, 3, 517, 18, 0, 0, 0, 0, 0, 3662, 3519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 0, 0, 0, 23, 0, 0, 9, 9, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 60, 0, 0, 0, 7654, 0, 0, 1077, 0, 10, 0, 0, 4323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7455, 8156, 135, 14837, 6774, 12877, 2165, 13137] avg=8192.0000 max=14837 max_rank=3 min=135 min_rank=2 imbalance_ratio=1.811157 max_load_reduction=50699 max_load_reduction_pct=0.773605 moved_to_echo=34953 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 14976, 16000, 0, 128, 2176, 17792, 128, 128, 0, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[256, 2944, 128, 128, 1024, 128, 1792, 128, 8960, 5888, 6400, 128, 128, 768, 2304, 128, 34176, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[14720, 640, 128, 256, 128, 10112, 128, 128, 384, 14336, 1792, 128, 512, 256, 128, 0, 17152, 5248, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 512, 128, 22400, 256, 17408, 0, 11520, 128, 8320, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[256, 2944, 128, 128, 1024, 128, 1792, 128, 8960, 5888, 6400, 128, 128, 768, 2304, 128, 34176, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[0, 384, 128, 2048, 256, 128, 128, 36608, 128, 768, 128, 128, 128, 896, 2944, 128, 12160, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[512, 128, 7296, 3072, 128, 16000, 128, 128, 9856, 8704, 1664, 0, 128, 128, 18688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 2816, 128, 128, 0, 8192, 128, 128, 384, 128, 0, 128, 2688, 23296, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 512, 128, 11264, 128, 0, 3840, 128, 256, 128, 128, 44800, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 2816, 128, 128, 0, 8192, 128, 128, 384, 128, 0, 128, 2688, 23296, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 3, 3770, 0, 0, 23, 1, 0, 0, 0, 0, 5882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 7635, 0, 1, 0, 0, 0, 152, 0, 0, 2, 0, 206, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4160, 62, 0, 0, 1227, 0, 541, 0, 8189, 0, 58, 0, 0, 10, 0, 0, 0, 7887, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 6393, 0, 0, 44, 1430, 0, 4, 9, 0, 0, 3093, 0, 0, 1, 1, 0, 0, 0, 0, 1480, 0, 0, 1740, 168, 6, 0, 949, 1, 0, 0, 0, 1, 0, 231, 0, 0, 6747, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 1294, 0, 1, 0, 3, 0, 0, 0, 0, 3, 7729, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3012, 62, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 6559, 0, 7188, 0, 0, 0, 0, 0, 0, 1, 33, 0, 0, 412, 0, 0, 0, 0, 0, 0, 677, 21, 0, 3, 0, 4, 0, 775, 0, 63, 3559, 0, 244, 0, 0, 186, 0, 0, 0, 0, 0, 1, 7280, 10, 0, 0, 8121, 0, 0, 7, 0, 46, 0, 8, 0, 0, 0, 0, 0, 0, 1, 502, 0, 140, 38, 11, 0, 5, 2, 0, 0, 0, 0, 247, 1, 0, 1596, 0, 1, 0, 0, 0, 7535, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 28, 0, 0, 95, 0, 0, 0, 0, 0, 0, 7619, 0, 7798, 0, 0, 220, 0, 0, 0, 0, 37, 0, 352, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4232, 591, 0, 0, 0, 0, 1, 0, 0, 0, 7381, 0, 0, 0, 10, 12, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 1436, 3, 1622, 0, 1679, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 752, 112, 0, 0, 0, 0, 34, 0, 6968, 0, 303, 0, 0, 0, 4571, 0, 0, 0, 0, 1317, 7, 0, 0, 0, 0, 0, 0, 0, 0, 722, 43, 8025, 2, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 2, 7370, 2, 0, 0, 0, 1, 386, 1, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 1, 9, 7941, 0, 0, 7932, 7405, 1, 3, 0, 0, 4, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 9, 0, 0, 7984, 0, 0, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 231, 0, 0, 50, 0, 123, 2, 170, 972, 1, 0, 5057, 0, 0, 0, 0, 0, 0, 0, 192, 0, 7, 0, 87, 4, 0, 0, 0, 0, 0, 0, 57, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 11, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7884, 37, 0, 0, 2, 243, 257, 0, 0, 83, 8187, 0, 0, 0, 3, 0, 3499, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 1, 7988, 0, 0, 1, 0, 0, 0, 7567, 0, 240, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 0, 0, 0, 0, 1, 0, 939, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3062, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 3257, 597, 22, 0, 0, 0, 7262, 7352, 0, 0, 0, 0, 5, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 82, 0, 0, 3, 0, 0, 0, 0, 8133, 0, 55, 0, 0, 0, 90, 0, 7, 6857, 0, 0, 0, 4803, 0, 31, 0, 0, 0, 33, 5, 0, 0, 0, 0, 7665, 0, 0, 11, 7261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 20, 0, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 31] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 3632, 0, 11, 0, 34, 0, 0, 9, 0, 0, 50, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 31, 695, 1977, 39, 0, 0, 7202, 0, 91, 1633, 245, 0, 0, 0, 6135, 0, 8123, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 1968, 6217, 0, 14, 64, 0, 0, 259, 0, 0, 7541, 0, 0, 0, 4, 0, 21, 708, 0, 0, 0, 0, 0, 0, 0, 0, 122, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 7667, 5021, 28, 171, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 1294, 0, 1, 0, 3, 0, 0, 0, 0, 3, 7729, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3012, 62, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 6559, 0, 7188, 0, 0, 0, 0, 0, 0, 1, 33, 0, 0, 412, 0, 0, 0, 0, 0, 0, 677, 21, 0, 3, 0, 4, 0, 775, 0, 63, 3559, 0, 244, 0, 0, 186, 0, 0, 0, 0, 0, 1, 7280, 10, 0, 0, 8121, 0, 0, 7, 0, 46, 0, 8, 0, 0, 0, 0, 0, 0, 1, 502, 0, 140, 38, 11, 0, 5, 2, 0, 0, 0, 0, 247, 1, 0, 1596, 0, 1, 0, 0, 0, 7535, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 3, 3770, 0, 0, 23, 1, 0, 0, 0, 0, 5882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 7635, 0, 1, 0, 0, 0, 152, 0, 0, 2, 0, 206, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4160, 62, 0, 0, 1227, 0, 541, 0, 8189, 0, 58, 0, 0, 10, 0, 0, 0, 7887, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 6393, 0, 0, 44, 1430, 0, 4, 9, 0, 0, 3093, 0, 0, 1, 1, 0, 0, 0, 0, 1480, 0, 0, 1740, 168, 6, 0, 949, 1, 0, 0, 0, 1, 0, 231, 0, 0, 6747, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 11, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7884, 37, 0, 0, 2, 243, 257, 0, 0, 83, 8187, 0, 0, 0, 3, 0, 3499, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 1, 7988, 0, 0, 1, 0, 0, 0, 7567, 0, 240, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 0, 0, 0, 0, 1, 0, 939, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 2, 7370, 2, 0, 0, 0, 1, 386, 1, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 1, 9, 7941, 0, 0, 7932, 7405, 1, 3, 0, 0, 4, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 9, 0, 0, 7984, 0, 0, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 231, 0, 0, 50, 0, 123, 2, 170, 972, 1, 0, 5057, 0, 0, 0, 0, 0, 0, 0, 192, 0, 7, 0, 87, 4, 0, 0, 0, 0, 0, 0, 57, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 4151, 0, 3, 1, 0, 0, 306, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1856, 1, 0, 2, 0, 0, 0, 0, 1141, 3984, 0, 0, 438, 0, 0, 4016, 7105, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 8023, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 1023, 0, 0, 8185, 1, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 7, 4, 0, 0, 1661, 8192, 0, 0, 0, 0, 3, 0, 0, 7972, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 34, 0, 102] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3770, 0, 0, 23, 1, 0, 0, 0, 0, 5882, 0, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 2566, 0, 0, 0, 7037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 152, 0, 0, 2, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4160, 62, 0, 0, 1227, 0, 541, 0, 8189, 0, 298, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 10, 0, 0, 0, 5231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 6393, 0, 0, 44, 1403, 0, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 1, 1, 0, 0, 0, 0, 1480, 0, 0, 1740, 168, 6, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 1, 0, 0, 0, 1, 0, 231, 0, 0, 6747, 1, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11096, 10452, 361, 14512, 5299, 7858, 7087, 8871] avg=8192.0000 max=14512 max_rank=3 min=361 min_rank=2 imbalance_ratio=1.771484 max_load_reduction=51024 max_load_reduction_pct=0.778564 moved_to_echo=29115 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 0, 0, 95, 0, 0, 0, 0, 0, 0, 7619, 0, 7798, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 37, 0, 352, 0, 0, 0, 0, 0, 0, 2, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 1, 0, 0, 0, 941, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 632, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5272, 0, 1436, 3, 1622, 0, 1679, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 752, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 6968, 0, 303, 0, 0, 0, 4571, 0, 0, 0, 4264, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 7, 0, 0, 0, 0, 0, 0, 0, 0, 722, 43, 8025, 2, 970, 2176, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16969, 4512, 1543, 952, 10013, 865, 16471, 14211] avg=8192.0000 max=16969 max_rank=0 min=865 min_rank=5 imbalance_ratio=2.071411 max_load_reduction=48567 max_load_reduction_pct=0.741074 moved_to_echo=41560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3770, 0, 0, 23, 1, 0, 0, 0, 0, 5882, 0, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 2566, 0, 0, 0, 7037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 152, 0, 0, 2, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4160, 62, 0, 0, 1227, 0, 541, 0, 8189, 0, 298, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 10, 0, 0, 0, 5231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 6393, 0, 0, 44, 1403, 0, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 1, 1, 0, 0, 0, 0, 1480, 0, 0, 1740, 168, 6, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 1, 0, 0, 0, 1, 0, 231, 0, 0, 6747, 1, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11096, 10452, 361, 14512, 5299, 7858, 7087, 8871] avg=8192.0000 max=14512 max_rank=3 min=361 min_rank=2 imbalance_ratio=1.771484 max_load_reduction=51024 max_load_reduction_pct=0.778564 moved_to_echo=29115 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1294, 0, 1, 0, 3, 0, 0, 0, 0, 3, 7729, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3012, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 6559, 0, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 33, 0, 0, 412, 0, 0, 0, 0, 0, 0, 677, 21, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 775, 0, 63, 2362, 0, 244, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7280, 10, 0, 0, 8121, 0, 0, 7, 0, 46, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 502, 0, 140, 38, 11, 0, 5, 2, 0, 0, 4153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 1, 0, 1596, 0, 1, 0, 0, 0, 7535, 0, 0, 0, 0, 2119, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9670, 3077, 15626, 1278, 3637, 15473, 4852, 11923] avg=8192.0000 max=15626 max_rank=2 min=1278 min_rank=3 imbalance_ratio=1.907471 max_load_reduction=49910 max_load_reduction_pct=0.761566 moved_to_echo=35885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 2, 7370, 2, 0, 0, 0, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 386, 1, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 1, 7321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 7932, 7405, 1, 3, 0, 0, 4, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 5294, 0, 0, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 123, 2, 170, 972, 1, 0, 5057, 0, 0, 0, 0, 0, 255, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 7, 0, 87, 4, 0, 0, 0, 0, 0, 0, 57, 0, 130, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8965, 8080, 15411, 15962, 8172, 266, 7250, 1430] avg=8192.0000 max=15962 max_rank=3 min=266 min_rank=5 imbalance_ratio=1.948486 max_load_reduction=49574 max_load_reduction_pct=0.756439 moved_to_echo=17118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7884, 37, 0, 0, 2, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 83, 8187, 0, 0, 0, 3, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 0, 0, 12, 308, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 5283, 0, 0, 1, 0, 0, 0, 7567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 0, 0, 0, 2023, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 939, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 1034, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4799, 8161, 8959, 511, 12853, 16378, 2761, 11114] avg=8192.0000 max=16378 max_rank=5 min=511 min_rank=3 imbalance_ratio=1.999268 max_load_reduction=49158 max_load_reduction_pct=0.750092 moved_to_echo=43106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3062, 0, 0, 0, 0, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 3257, 597, 22, 0, 0, 6694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7352, 0, 0, 0, 0, 5, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 82, 0, 0, 3, 0, 0, 0, 0, 8133, 0, 259, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 90, 0, 7, 4548, 0, 0, 0, 4803, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 5, 0, 0, 0, 0, 7665, 0, 0, 11, 7122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 20, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 31, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4297, 10656, 7367, 8617, 9534, 14836, 8694, 1535] avg=8192.0000 max=14836 max_rank=5 min=1535 min_rank=7 imbalance_ratio=1.811035 max_load_reduction=50700 max_load_reduction_pct=0.773621 moved_to_echo=34599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3632, 0, 11, 0, 34, 0, 0, 9, 0, 0, 50, 0, 0, 0, 0, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 31, 695, 1977, 39, 0, 6639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 1633, 245, 0, 0, 0, 6135, 0, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 1968, 6217, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 64, 0, 0, 259, 0, 0, 5001, 0, 0, 0, 4, 0, 21, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 4693, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 7667, 5021, 28, 171, 0, 0, 2395, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5091, 9474, 9139, 9303, 6071, 148, 10120, 16190] avg=8192.0000 max=16190 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.976318 max_load_reduction=49346 max_load_reduction_pct=0.752960 moved_to_echo=32529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 2, 7370, 2, 0, 0, 0, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 386, 1, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 1, 7321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 7932, 7405, 1, 3, 0, 0, 4, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 5294, 0, 0, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 123, 2, 170, 972, 1, 0, 5057, 0, 0, 0, 0, 0, 255, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 7, 0, 87, 4, 0, 0, 0, 0, 0, 0, 57, 0, 130, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8965, 8080, 15411, 15962, 8172, 266, 7250, 1430] avg=8192.0000 max=15962 max_rank=3 min=266 min_rank=5 imbalance_ratio=1.948486 max_load_reduction=49574 max_load_reduction_pct=0.756439 moved_to_echo=17118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7884, 37, 0, 0, 2, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 83, 8187, 0, 0, 0, 3, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 0, 0, 12, 308, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 5283, 0, 0, 1, 0, 0, 0, 7567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 0, 0, 0, 2023, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 939, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 1034, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4799, 8161, 8959, 511, 12853, 16378, 2761, 11114] avg=8192.0000 max=16378 max_rank=5 min=511 min_rank=3 imbalance_ratio=1.999268 max_load_reduction=49158 max_load_reduction_pct=0.750092 moved_to_echo=43106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 4151, 0, 3, 1, 0, 0, 306, 4, 0, 0, 0, 0, 0, 1, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 1, 0, 2, 0, 0, 0, 0, 1141, 3984, 0, 0, 438, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7105, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 8023, 7, 0, 0, 0, 0, 38, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 680, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 7, 4, 0, 0, 1661, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7972, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 34, 0, 102, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4649, 11124, 7130, 14401, 9957, 9712, 8301, 262] avg=8192.0000 max=14401 max_rank=3 min=262 min_rank=7 imbalance_ratio=1.757935 max_load_reduction=51135 max_load_reduction_pct=0.780258 moved_to_echo=28232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 7296, 128, 0, 8192, 22400, 128, 0, 1792, 24576, 128, 128, 896, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 3072, 512, 1024, 128, 1664, 128, 128, 256, 0, 7680, 28288, 128, 8320, 128, 1152, 7936, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -2171,110 +2171,110 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2452, 0, 0, 0, 0, 0, 0, 847, 12, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 2475, 0, 1, 0, 0, 0, 9, 0, 2, 125, 15, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 8, 5, 7982, 0, 0, 200, 21, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 533, 290, 2946, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 710, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 1, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 94, 0, 0, 0, 0, 6307, 0, 0, 0, 0, 0, 2, 8084, 0, 0, 209, 0, 0, 0, 6126, 0, 0, 5365, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 11, 512, 0, 0, 2, 0, 2, 28, 0, 0, 0, 1, 1, 0, 0, 0, 1886, 0, 2494, 0, 0, 0, 0, 0, 7468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 282, 0, 6548, 0, 0, 0, 0, 6839, 683, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 94, 6, 2114, 31, 194, 0, 0, 7, 0, 2944, 0, 8192, 0, 0, 1, 0, 0, 1380, 0, 2231, 19, 0, 0, 0, 1169, 0, 0, 69, 0, 0, 6730, 0, 0, 0, 0, 0, 4, 535, 354, 0, 0, 8119, 567, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 75, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 126, 7, 54, 0, 0, 0, 0, 8, 0, 0, 8186, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 19, 0, 0, 6, 0, 0, 25, 1, 1, 2, 0, 0, 0, 0, 5, 50, 0, 0, 0, 1, 0, 3, 0, 0, 1816, 0, 0, 1, 0, 701, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 39, 18, 0, 3, 0, 0, 0, 408, 22, 17, 0, 0, 0, 0, 0, 7666, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3228, 0, 0, 0, 0, 0, 0, 0, 7800, 8188, 11, 0, 0, 0, 0, 8169, 1, 4, 0, 0, 6491, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4182, 97] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2, 5779, 0, 254, 11, 0, 0, 0, 42, 20, 4, 0, 0, 0, 0, 0, 0, 927, 0, 1505, 0, 3, 42, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 6364, 1, 0, 0, 0, 0, 0, 0, 0, 6495, 0, 7905, 53, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7129, 0, 0, 0, 0, 1452, 0, 0, 1, 0, 1, 130, 5, 0, 50, 0, 0, 0, 2173] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 7784, 0, 0, 6866, 0, 6, 3266, 0, 0, 12, 0, 0, 0, 0, 0, 7280, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 26, 0, 0, 3, 0, 0, 0, 0, 58, 4, 0, 0, 0, 0, 257, 1423, 0, 0, 0, 0, 0, 4, 0, 0, 8130, 0, 0, 283, 2, 0, 3, 0, 0, 7514, 16, 11, 0, 0, 0, 0, 194, 0, 0, 15, 0, 0, 0, 0, 0, 30, 0, 73, 0, 0, 17, 1308, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 9, 0, 287, 0, 0, 0, 3, 0, 0, 0, 0, 114, 0, 0, 411, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 7078, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 6888, 960, 0, 0, 0, 3711, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6659, 0, 0, 0, 130, 0, 0, 0, 86, 4, 1, 0, 10, 0, 0, 0, 0, 0, 19, 0, 0, 2, 113, 15, 991, 0, 60, 53, 0, 0, 0, 14, 0, 0, 0, 1064, 2033, 11, 6194, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 1, 5354, 0, 0, 0, 2246, 291, 0, 0, 0, 0, 8070, 47, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 3176] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 126, 7, 54, 0, 0, 0, 0, 8, 0, 0, 8186, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 19, 0, 0, 6, 0, 0, 25, 1, 1, 2, 0, 0, 0, 0, 5, 50, 0, 0, 0, 1, 0, 3, 0, 0, 1816, 0, 0, 1, 0, 701, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 39, 18, 0, 3, 0, 0, 0, 408, 22, 17, 0, 0, 0, 0, 0, 7666, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3228, 0, 0, 0, 0, 0, 0, 0, 7800, 8188, 11, 0, 0, 0, 0, 8169, 1, 4, 0, 0, 6491, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4182, 97] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4919, 19, 0, 0, 0, 0, 0, 0, 11, 21, 0, 1, 0, 0, 1, 0, 7608, 0, 144, 85, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 0, 450, 1, 18, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 1, 0, 385, 0, 0, 411, 0, 0, 1, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 31, 39, 0, 5870, 0, 0, 0, 0, 211, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 2, 0, 0, 1179, 0, 0, 0, 336, 0, 0, 38, 0, 10, 0, 0, 0, 0, 0, 7938, 3257, 4, 4100, 0, 42, 0, 0, 0, 16, 0, 200, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2452, 0, 0, 0, 0, 0, 0, 847, 12, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 2475, 0, 1, 0, 0, 0, 9, 0, 2, 125, 15, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 8, 5, 7982, 0, 0, 200, 21, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 533, 290, 2946, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 710, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 1, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 94, 0, 0, 0, 0, 6307, 0, 0, 0, 0, 0, 2, 8084, 0, 0, 209, 0, 0, 0, 6126, 0, 0, 5365, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2, 5779, 0, 254, 11, 0, 0, 0, 42, 20, 4, 0, 0, 0, 0, 0, 0, 927, 0, 1505, 0, 3, 42, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 6364, 1, 0, 0, 0, 0, 0, 0, 0, 6495, 0, 7905, 53, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7129, 0, 0, 0, 0, 1452, 0, 0, 1, 0, 1, 130, 5, 0, 50, 0, 0, 0, 2173] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 7588, 2, 0, 43, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4173, 1, 0, 4336, 1396, 0, 1, 0, 7222, 0, 0, 0, 1, 0, 1851, 0, 0, 0, 2209, 0, 5998, 0, 0, 2, 8076, 0, 0, 0, 0, 1, 11, 137, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 938, 0, 0, 0, 0, 8030, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 409, 3249, 297, 275, 0, 64, 0, 2, 0, 0, 0, 7847, 0, 0, 0, 0, 2, 0, 0, 149, 9, 0, 0, 0, 0, 9, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2452, 0, 0, 0, 0, 0, 0, 847, 12, 0, 0, 0, 0, 0, 0, 138, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2475, 0, 1, 0, 0, 0, 9, 0, 2, 125, 15, 0, 5576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4398, 0, 8, 5, 7982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 21, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 533, 290, 2946, 0, 3905, 275, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 1, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 94, 0, 0, 0, 0, 2467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8084, 0, 0, 209, 0, 0, 0, 6126, 0, 0, 5365, 0, 2939, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4237, 8203, 12393, 8257, 601, 4113, 3179, 24553] avg=8192.0000 max=24553 max_rank=7 min=601 min_rank=4 imbalance_ratio=2.997192 max_load_reduction=40983 max_load_reduction_pct=0.625351 moved_to_echo=32446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 11, 512, 0, 0, 2, 0, 2, 28, 0, 0, 0, 1, 1, 0, 0, 0, 1886, 0, 2494, 0, 0, 0, 0, 0, 7468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 282, 0, 6548, 0, 0, 0, 0, 6839, 683, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 94, 6, 2114, 31, 194, 0, 0, 7, 0, 2944, 0, 8192, 0, 0, 1, 0, 0, 1380, 0, 2231, 19, 0, 0, 0, 1169, 0, 0, 69, 0, 0, 6730, 0, 0, 0, 0, 0, 4, 535, 354, 0, 0, 8119, 567, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 75, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 7784, 0, 0, 6866, 0, 6, 3266, 0, 0, 12, 0, 0, 0, 0, 0, 7280, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 26, 0, 0, 3, 0, 0, 0, 0, 58, 4, 0, 0, 0, 0, 257, 1423, 0, 0, 0, 0, 0, 4, 0, 0, 8130, 0, 0, 283, 2, 0, 3, 0, 0, 7514, 16, 11, 0, 0, 0, 0, 194, 0, 0, 15, 0, 0, 0, 0, 0, 30, 0, 73, 0, 0, 17, 1308, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 9, 0, 287, 0, 0, 0, 3, 0, 0, 0, 0, 114, 0, 0, 411, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7078, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6888, 960, 0, 0, 0, 3711, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6659, 0, 0, 0, 76, 0, 0, 0, 86, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 19, 0, 0, 2, 113, 15, 991, 0, 60, 235, 4, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 14, 0, 0, 0, 1064, 2033, 11, 6194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5354, 0, 0, 0, 2246, 291, 0, 0, 0, 0, 3157, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 3176, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7482, 11560, 6826, 1536, 9369, 14340, 11097, 3326] avg=8192.0000 max=14340 max_rank=5 min=1536 min_rank=3 imbalance_ratio=1.750488 max_load_reduction=51196 max_load_reduction_pct=0.781189 moved_to_echo=38132 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 126, 7, 54, 0, 0, 0, 0, 8, 0, 0, 8186, 0, 0, 0, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 6, 0, 0, 25, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 50, 0, 0, 0, 1, 0, 3, 0, 0, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 701, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8169, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 18, 0, 3, 0, 0, 0, 408, 22, 17, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3228, 0, 4885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7800, 8188, 11, 0, 0, 0, 0, 3196, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6491, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4182, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9880, 54, 1875, 8964, 6674, 8115, 19200, 10774] avg=8192.0000 max=19200 max_rank=6 min=54 min_rank=1 imbalance_ratio=2.343750 max_load_reduction=46336 max_load_reduction_pct=0.707031 moved_to_echo=44763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 512, 0, 0, 2, 0, 2, 28, 0, 0, 0, 1, 1, 0, 0, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2494, 0, 0, 0, 0, 0, 7468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 282, 0, 3783, 0, 0, 0, 0, 6839, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1886, 237, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 94, 6, 2114, 31, 194, 0, 0, 7, 0, 2944, 0, 6591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1380, 0, 2231, 19, 0, 0, 0, 1169, 0, 0, 69, 4855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6730, 0, 0, 0, 0, 0, 4, 535, 354, 0, 0, 3176, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 75, 0, 1, 0, 0, 0, 0, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2158, 9963, 13801, 3861, 12057, 9724, 11366, 2606] avg=8192.0000 max=13801 max_rank=2 min=2158 min_rank=0 imbalance_ratio=1.684692 max_load_reduction=51735 max_load_reduction_pct=0.789413 moved_to_echo=35753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2452, 0, 0, 0, 0, 0, 0, 847, 12, 0, 0, 0, 0, 0, 0, 138, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2475, 0, 1, 0, 0, 0, 9, 0, 2, 125, 15, 0, 5576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4398, 0, 8, 5, 7982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 21, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 533, 290, 2946, 0, 3905, 275, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 1, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 94, 0, 0, 0, 0, 2467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8084, 0, 0, 209, 0, 0, 0, 6126, 0, 0, 5365, 0, 2939, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4237, 8203, 12393, 8257, 601, 4113, 3179, 24553] avg=8192.0000 max=24553 max_rank=7 min=601 min_rank=4 imbalance_ratio=2.997192 max_load_reduction=40983 max_load_reduction_pct=0.625351 moved_to_echo=32446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4919, 19, 0, 0, 0, 0, 0, 0, 11, 21, 0, 1, 0, 0, 1, 1594, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 85, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 1, 18, 0, 0, 0, 4688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 385, 0, 0, 411, 0, 0, 1, 0, 0, 0, 4029, 0, 0, 7608, 293, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 39, 0, 5870, 0, 0, 0, 0, 211, 0, 6560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 2, 0, 0, 1179, 0, 4747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 0, 38, 0, 10, 0, 0, 0, 0, 0, 3105, 3257, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4100, 0, 42, 0, 0, 0, 16, 0, 200, 0, 0, 0, 0, 0, 0, 0, 3133, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7319, 5614, 5157, 12814, 12711, 5934, 6750, 9237] avg=8192.0000 max=12814 max_rank=3 min=5157 min_rank=2 imbalance_ratio=1.564209 max_load_reduction=52722 max_load_reduction_pct=0.804474 moved_to_echo=34632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 5779, 0, 254, 11, 0, 0, 0, 42, 20, 4, 0, 0, 0, 0, 0, 131, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1505, 0, 3, 42, 0, 0, 0, 0, 0, 0, 0, 0, 5506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 4588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 6364, 1, 0, 927, 287, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6495, 0, 7905, 53, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1452, 0, 0, 1, 0, 1, 130, 5, 0, 50, 0, 0, 0, 2173, 3066, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7021, 7056, 12514, 8209, 14992, 4265, 2796, 8683] avg=8192.0000 max=14992 max_rank=4 min=2796 min_rank=6 imbalance_ratio=1.830078 max_load_reduction=50544 max_load_reduction_pct=0.771240 moved_to_echo=30736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 512, 0, 0, 2, 0, 2, 28, 0, 0, 0, 1, 1, 0, 0, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2494, 0, 0, 0, 0, 0, 7468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 282, 0, 3783, 0, 0, 0, 0, 6839, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1886, 237, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 94, 6, 2114, 31, 194, 0, 0, 7, 0, 2944, 0, 6591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1380, 0, 2231, 19, 0, 0, 0, 1169, 0, 0, 69, 4855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6730, 0, 0, 0, 0, 0, 4, 535, 354, 0, 0, 3176, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 75, 0, 1, 0, 0, 0, 0, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2158, 9963, 13801, 3861, 12057, 9724, 11366, 2606] avg=8192.0000 max=13801 max_rank=2 min=2158 min_rank=0 imbalance_ratio=1.684692 max_load_reduction=51735 max_load_reduction_pct=0.789413 moved_to_echo=35753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7784, 0, 0, 6866, 0, 6, 3266, 0, 0, 12, 0, 0, 0, 0, 0, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3, 0, 0, 0, 0, 58, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 1423, 0, 0, 0, 0, 0, 4, 0, 0, 8130, 0, 0, 7280, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 2, 0, 3, 0, 0, 7514, 16, 11, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 30, 0, 73, 0, 0, 17, 1308, 0, 0, 0, 4843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 287, 0, 0, 0, 3, 0, 0, 0, 0, 114, 0, 0, 411, 7, 0, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18720, 8869, 91, 17186, 8023, 6286, 3711, 2650] avg=8192.0000 max=18720 max_rank=0 min=91 min_rank=2 imbalance_ratio=2.285156 max_load_reduction=46816 max_load_reduction_pct=0.714355 moved_to_echo=20670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7078, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6888, 960, 0, 0, 0, 3711, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6659, 0, 0, 0, 76, 0, 0, 0, 86, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 19, 0, 0, 2, 113, 15, 991, 0, 60, 235, 4, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 14, 0, 0, 0, 1064, 2033, 11, 6194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5354, 0, 0, 0, 2246, 291, 0, 0, 0, 0, 3157, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 3176, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7482, 11560, 6826, 1536, 9369, 14340, 11097, 3326] avg=8192.0000 max=14340 max_rank=5 min=1536 min_rank=3 imbalance_ratio=1.750488 max_load_reduction=51196 max_load_reduction_pct=0.781189 moved_to_echo=38132 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7588, 2, 0, 43, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4336, 1396, 0, 1, 0, 7222, 0, 0, 0, 1, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2209, 0, 5998, 0, 0, 2, 4658, 0, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 5, 4173, 295, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 409, 3249, 297, 275, 0, 64, 0, 2, 0, 0, 0, 3069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 149, 9, 0, 0, 0, 0, 9, 2, 0, 0, 0, 3123, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8719, 14217, 12879, 4709, 1109, 12759, 7437, 3707] avg=8192.0000 max=14217 max_rank=1 min=1109 min_rank=4 imbalance_ratio=1.735474 max_load_reduction=51319 max_load_reduction_pct=0.783066 moved_to_echo=25012 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4919, 19, 0, 0, 0, 0, 0, 0, 11, 21, 0, 1, 0, 0, 1, 1594, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 85, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 1, 18, 0, 0, 0, 4688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 385, 0, 0, 411, 0, 0, 1, 0, 0, 0, 4029, 0, 0, 7608, 293, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 39, 0, 5870, 0, 0, 0, 0, 211, 0, 6560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 2, 0, 0, 1179, 0, 4747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 0, 38, 0, 10, 0, 0, 0, 0, 0, 3105, 3257, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4100, 0, 42, 0, 0, 0, 16, 0, 200, 0, 0, 0, 0, 0, 0, 0, 3133, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7319, 5614, 5157, 12814, 12711, 5934, 6750, 9237] avg=8192.0000 max=12814 max_rank=3 min=5157 min_rank=2 imbalance_ratio=1.564209 max_load_reduction=52722 max_load_reduction_pct=0.804474 moved_to_echo=34632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 0, 8320, 128, 8192, 256, 128, 384, 384, 128, 128, 6400, 128, 0, 9984, 5504, 14848, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 15616, 3456, 128, 2304, 128, 0, 128, 384, 2560, 128, 4480, 128, 36992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 8064, 9344, 384, 384, 7808, 10624, 512, 640, 384, 0, 0, 24192, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 128, 128, 128, 128, 128, 16256, 512, 15104, 2176, 128, 6272, 0, 3456, 384, 21248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 2688, 128, 22912, 0, 384, 128, 22272, 0, 128, 128, 8192, 8704, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[640, 128, 0, 3072, 1536, 0, 512, 384, 0, 128, 128, 128, 640, 19840, 3072, 128, 34304, 1408, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 128, 128, 128, 128, 128, 16256, 512, 15104, 2176, 128, 6272, 0, 3456, 384, 21248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 8064, 9344, 384, 384, 7808, 10624, 512, 640, 384, 0, 0, 24192, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 6656, 11392, 5760, 128, 128, 0, 18432, 0, 128, 256, 128, 128, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 27904, 8448, 384, 128, 8064, 0, 128, 4224, 128, 128, 128, 8192, 128, 0, 128, 5248, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[640, 128, 0, 3072, 1536, 0, 512, 384, 0, 128, 128, 128, 640, 19840, 3072, 128, 34304, 1408, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7241, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6526, 380, 0, 0, 0, 6050, 0, 0, 0, 55, 8056, 0, 0, 0, 0, 0, 86, 0, 8173, 0, 0, 5, 0, 0, 15, 0, 0, 0, 8135, 0, 118, 0, 0, 6992, 0, 0, 167, 0, 30, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 23, 74, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 1393, 0, 0, 946, 0, 0, 0, 63, 0, 0, 0, 0, 64, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 7903] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 39, 0, 171, 0, 6, 133, 0, 0, 0, 0, 0, 0, 6602, 1, 88, 0, 50, 0, 0, 2031, 0, 5, 0, 84, 2, 0, 4, 12, 0, 48, 0, 1184, 0, 0, 7702, 1, 1, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 0, 781, 7224, 137, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 4519, 5684, 57, 0, 0, 0, 0, 41, 0, 6, 0, 0, 0, 0, 0, 0, 0, 96, 0, 229, 0, 0, 0, 3, 0, 0, 0, 303, 0, 0, 0, 16, 7, 0, 0, 0, 6694, 0, 0, 1, 13, 1674, 730, 0, 0, 205, 4, 8002, 0, 0, 0, 0, 347, 0, 147, 3, 36] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[2746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 1, 0, 0, 1793, 44, 0, 0, 19, 134, 0, 0, 0, 8191, 0, 0, 0, 23, 1055, 0, 0, 2, 0, 1282, 0, 30, 0, 0, 2523, 0, 0, 6832, 0, 0, 0, 3829, 1, 0, 0, 0, 0, 343, 0, 5841, 2319, 0, 3978, 0, 0, 0, 9, 8130, 4, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 36, 0, 0, 8131, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 258, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1378, 0, 0, 0, 3343, 0, 1, 0, 0, 0, 4, 0, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7241, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6526, 380, 0, 0, 0, 6050, 0, 0, 0, 55, 8056, 0, 0, 0, 0, 0, 86, 0, 8173, 0, 0, 5, 0, 0, 15, 0, 0, 0, 8135, 0, 118, 0, 0, 6992, 0, 0, 167, 0, 30, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 23, 74, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 1393, 0, 0, 946, 0, 0, 0, 63, 0, 0, 0, 0, 64, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 7903] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[8161, 7670, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 46, 8037, 0, 0, 0, 394, 1, 7, 0, 692, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 6463, 0, 0, 0, 0, 88, 0, 0, 10, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2, 0, 161, 0, 0, 28, 796, 0, 0, 0, 0, 0, 0, 0, 1, 1, 10, 0, 0, 2433, 0, 0, 0, 0, 5, 85, 0, 0, 27, 0, 0, 81, 5047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 2664, 0, 0, 0, 0, 0, 5076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 9, 0, 776, 0, 0, 0, 1, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4911, 0, 966, 0, 8171, 0, 0, 0, 0, 6477, 0, 0, 7, 8, 2783, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4451, 0, 3, 0, 0, 37, 0, 3, 0, 0, 7448, 451, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 3, 0, 0, 73, 0, 0, 0, 0, 2688, 29, 8140, 0, 0, 0, 652, 0, 7750, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 61, 0, 0, 243, 3799, 0, 577, 0, 0, 0, 0, 0, 0, 1, 6117, 0, 0, 0, 7512, 0, 0, 8052, 11, 0, 0, 0, 6773, 0, 0, 2, 0, 0, 8, 1, 3, 0, 7140, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2917, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 60, 0, 7524, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 23, 0, 0, 0, 0, 1941, 588, 0, 0, 247, 0, 0, 0, 0, 0, 3099, 0, 0, 0, 19, 4573] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 8063, 0, 0, 0, 0, 0, 0, 1, 0, 0, 387, 780, 65, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 7823, 0, 127, 0, 0, 1582, 7, 0, 5439, 1, 0, 0, 2, 0, 0, 0, 0, 0, 55, 0, 0, 331, 0, 7382, 0, 0, 0, 0, 7550, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 74, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4211, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 33, 1, 6328, 0, 0, 0, 0, 0, 338, 0, 0, 42, 0, 8036, 0, 601, 0, 0, 0, 0, 0, 0, 6132] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 806, 0, 0, 3, 0, 913, 0, 0, 0, 5846, 198, 0, 7782, 0, 5, 0, 0, 0, 0, 2123, 0, 0, 0, 0, 0, 0, 16, 7800, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 1660, 0, 0, 8012, 0, 0, 0, 0, 3666, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 129, 3, 0, 0, 0, 108, 0, 6459, 1250, 0, 0, 0, 5173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 198, 8047, 0, 0, 0, 517, 6, 13, 0, 0, 0, 0, 0, 0, 7, 3395, 0, 73, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 8063, 0, 0, 0, 0, 0, 0, 1, 0, 0, 387, 780, 65, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 7823, 0, 127, 0, 0, 1582, 7, 0, 5439, 1, 0, 0, 2, 0, 0, 0, 0, 0, 55, 0, 0, 331, 0, 7382, 0, 0, 0, 0, 7550, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 74, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4211, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 33, 1, 6328, 0, 0, 0, 0, 0, 338, 0, 0, 42, 0, 8036, 0, 601, 0, 0, 0, 0, 0, 0, 6132] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 9, 0, 776, 0, 0, 0, 1, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4911, 0, 966, 0, 8171, 0, 0, 0, 0, 6477, 0, 0, 7, 8, 2783, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4451, 0, 3, 0, 0, 37, 0, 3, 0, 0, 7448, 451, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 3, 0, 0, 73, 0, 0, 0, 0, 2688, 29, 8140, 0, 0, 0, 652, 0, 7750, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 39, 0, 171, 0, 6, 133, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6602, 1, 88, 0, 8, 0, 0, 2031, 0, 5, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 12, 0, 48, 0, 1184, 0, 0, 1844, 1, 1, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 0, 781, 364, 137, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4519, 5684, 57, 0, 0, 0, 0, 41, 0, 6860, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 96, 0, 229, 0, 0, 0, 3, 0, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 16, 7, 0, 0, 0, 6694, 0, 0, 1, 13, 3744, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 730, 0, 0, 205, 4, 8002, 0, 0, 0, 0, 347, 0, 147, 3, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[379, 8819, 11269, 3520, 19275, 354, 10780, 11140] avg=8192.0000 max=19275 max_rank=4 min=354 min_rank=5 imbalance_ratio=2.352905 max_load_reduction=46261 max_load_reduction_pct=0.705887 moved_to_echo=41549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7241, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 380, 0, 0, 0, 6050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 8056, 0, 0, 0, 0, 0, 86, 0, 1957, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 8135, 0, 118, 0, 0, 352, 0, 0, 167, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 23, 74, 0, 0, 0, 0, 19, 3, 6640, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 1393, 0, 1694, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 0, 0, 63, 0, 0, 0, 0, 64, 0, 0, 0, 2, 0, 3973, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 5942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11003, 7185, 10159, 8817, 10404, 5064, 5385, 7519] avg=8192.0000 max=11003 max_rank=0 min=5064 min_rank=5 imbalance_ratio=1.343140 max_load_reduction=54533 max_load_reduction_pct=0.832108 moved_to_echo=28372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8161, 7670, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 46, 8037, 0, 0, 0, 394, 1, 7, 0, 167, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 326, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2, 0, 161, 0, 0, 6137, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 796, 0, 0, 0, 0, 0, 0, 0, 1, 1, 10, 0, 0, 2433, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 85, 0, 0, 27, 0, 0, 81, 5047, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 2664, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15832, 8144, 8663, 5768, 6506, 4529, 5581, 10513] avg=8192.0000 max=15832 max_rank=0 min=4529 min_rank=5 imbalance_ratio=1.932617 max_load_reduction=49704 max_load_reduction_pct=0.758423 moved_to_echo=27129 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 1, 0, 0, 1793, 4694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 19, 134, 0, 0, 0, 948, 0, 0, 0, 23, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1282, 0, 30, 0, 0, 2523, 0, 0, 1636, 0, 0, 0, 3829, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 5841, 2319, 0, 201, 0, 0, 0, 9, 8130, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 36, 0, 0, 3777, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 258, 5, 0, 0, 2126, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1378, 0, 0, 0, 3343, 0, 1, 0, 0, 3321, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10640, 2223, 9303, 16847, 5818, 10730, 8470, 1505] avg=8192.0000 max=16847 max_rank=3 min=1505 min_rank=7 imbalance_ratio=2.056519 max_load_reduction=48689 max_load_reduction_pct=0.742935 moved_to_echo=26523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 61, 0, 0, 243, 3799, 0, 577, 0, 0, 0, 4305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6117, 0, 0, 0, 869, 0, 0, 8052, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6773, 0, 0, 2, 0, 0, 8, 1, 3, 0, 1709, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2917, 0, 0, 0, 0, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 60, 0, 7524, 0, 0, 0, 1, 3, 1950, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 23, 0, 0, 0, 0, 3471, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 588, 0, 0, 247, 0, 0, 0, 0, 0, 3099, 0, 0, 0, 19, 3436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8985, 15050, 8497, 4104, 4906, 10772, 3892, 9330] avg=8192.0000 max=15050 max_rank=1 min=3892 min_rank=6 imbalance_ratio=1.837158 max_load_reduction=50486 max_load_reduction_pct=0.770355 moved_to_echo=28900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8161, 7670, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 46, 8037, 0, 0, 0, 394, 1, 7, 0, 167, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 326, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2, 0, 161, 0, 0, 6137, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 796, 0, 0, 0, 0, 0, 0, 0, 1, 1, 10, 0, 0, 2433, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 85, 0, 0, 27, 0, 0, 81, 5047, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 2664, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15832, 8144, 8663, 5768, 6506, 4529, 5581, 10513] avg=8192.0000 max=15832 max_rank=0 min=4529 min_rank=5 imbalance_ratio=1.932617 max_load_reduction=49704 max_load_reduction_pct=0.758423 moved_to_echo=27129 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7241, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 380, 0, 0, 0, 6050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 8056, 0, 0, 0, 0, 0, 86, 0, 1957, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 8135, 0, 118, 0, 0, 352, 0, 0, 167, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 23, 74, 0, 0, 0, 0, 19, 3, 6640, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 1393, 0, 1694, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 0, 0, 63, 0, 0, 0, 0, 64, 0, 0, 0, 2, 0, 3973, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 5942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11003, 7185, 10159, 8817, 10404, 5064, 5385, 7519] avg=8192.0000 max=11003 max_rank=0 min=5064 min_rank=5 imbalance_ratio=1.343140 max_load_reduction=54533 max_load_reduction_pct=0.832108 moved_to_echo=28372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 387, 780, 65, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 127, 0, 0, 1582, 7, 0, 1302, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 331, 0, 372, 0, 0, 0, 0, 7550, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 74, 0, 0, 2, 0, 2, 7010, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4211, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 33, 1, 6328, 0, 0, 0, 0, 2644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 42, 0, 8036, 0, 601, 0, 0, 0, 0, 0, 0, 4610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 1245, 10844, 8311, 8588, 5737, 9118, 13627] avg=8192.0000 max=13627 max_rank=7 min=1245 min_rank=1 imbalance_ratio=1.663452 max_load_reduction=51909 max_load_reduction_pct=0.792068 moved_to_echo=37070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 806, 3353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 913, 0, 0, 0, 669, 198, 0, 7782, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2123, 0, 0, 0, 0, 0, 0, 16, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 1660, 0, 0, 399, 0, 0, 0, 0, 3666, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 7613, 2144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 129, 3, 0, 0, 0, 108, 0, 6459, 1250, 0, 0, 0, 5173, 1520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 198, 8047, 0, 0, 0, 517, 3795, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 13, 0, 0, 0, 0, 0, 0, 7, 3395, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 9570, 4000, 5811, 9760, 14644, 14055, 3494] avg=8192.0000 max=14644 max_rank=5 min=3494 min_rank=7 imbalance_ratio=1.787598 max_load_reduction=50892 max_load_reduction_pct=0.776550 moved_to_echo=41953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 776, 0, 0, 0, 1, 0, 961, 0, 0, 0, 0, 0, 0, 4682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4911, 0, 966, 0, 946, 0, 0, 0, 0, 6477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 2783, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4451, 0, 3, 0, 0, 2, 0, 3, 0, 0, 7448, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 73, 0, 0, 0, 0, 2688, 29, 8140, 0, 0, 0, 652, 0, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7750, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6429, 13300, 2801, 12358, 279, 13706, 8255, 8408] avg=8192.0000 max=13706 max_rank=5 min=279 min_rank=4 imbalance_ratio=1.673096 max_load_reduction=51830 max_load_reduction_pct=0.790863 moved_to_echo=30648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 387, 780, 65, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 127, 0, 0, 1582, 7, 0, 1302, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 331, 0, 372, 0, 0, 0, 0, 7550, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 74, 0, 0, 2, 0, 2, 7010, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4211, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 33, 1, 6328, 0, 0, 0, 0, 2644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 42, 0, 8036, 0, 601, 0, 0, 0, 0, 0, 0, 4610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 1245, 10844, 8311, 8588, 5737, 9118, 13627] avg=8192.0000 max=13627 max_rank=7 min=1245 min_rank=1 imbalance_ratio=1.663452 max_load_reduction=51909 max_load_reduction_pct=0.792068 moved_to_echo=37070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[8192, 896, 128, 256, 128, 0, 128, 4224, 3072, 128, 22400, 1280, 384, 128, 4608, 5248, 9472, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 1408, 128, 0, 4736, 5760, 256, 128, 128, 3200, 256, 128, 128, 38144, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[18176, 7680, 0, 896, 0, 0, 128, 128, 128, 1280, 13568, 128, 640, 256, 0, 2688, 20864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 1408, 128, 0, 4736, 5760, 256, 128, 128, 3200, 256, 128, 128, 38144, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 27264, 896, 1152, 0, 4224, 640, 0, 17920, 128, 13696, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 13056, 128, 18816, 2688, 896, 2048, 256, 128, 256, 128, 27520, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[6912, 128, 30208, 128, 256, 128, 512, 5376, 128, 128, 10496, 128, 128, 8192, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[7808, 1152, 384, 128, 128, 128, 128, 1536, 0, 1280, 384, 29568, 0, 128, 128, 640, 21376, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3712, 1792, 0, 0, 896, 896, 24192, 4096, 640, 6144, 3200, 2048, 0, 256, 128, 18560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[6912, 128, 30208, 128, 256, 128, 512, 5376, 128, 128, 10496, 128, 128, 8192, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6470, 0, 0, 0, 0, 0, 0, 0, 1406, 14, 0, 0, 0, 0, 0, 0, 1, 690, 0, 0, 0, 457, 0, 0, 78, 0, 0, 54, 5793, 1, 501, 348, 196, 0, 0, 0, 0, 0, 0, 0, 78, 38, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1588, 0, 0, 211, 5, 0, 0, 11, 8109, 0, 0, 6267, 0, 0, 7120, 0, 0, 25, 0, 0, 0, 0, 7033, 0, 0, 0, 2, 2100, 0, 0, 0, 0, 106, 0, 0, 0, 0, 2, 456, 0, 295, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 1, 54, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 0, 0, 2, 0, 132, 33, 0, 0, 0, 0, 0, 31, 0, 0, 1, 0, 0, 61, 0, 5489, 0, 8081, 0, 0, 0, 0, 0, 1102, 0, 8081, 6, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 15, 0, 1099, 0, 0, 0, 0, 5, 0, 43, 0, 0, 0, 0, 6, 0, 0, 0, 6600, 0, 0, 0, 2342, 4261, 3, 0, 0, 0, 0, 0, 0, 3, 318, 8, 0, 0, 2, 0, 0, 36, 0, 5178, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 8151, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 0, 1051, 237, 0, 0, 2006, 0, 519, 0, 0, 0, 69, 0, 0, 1238, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 167, 1174, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2392, 0, 7318, 8108, 0, 0, 1, 0, 1, 2, 0, 0, 6382, 0, 0, 7803, 7, 7594, 0, 0, 0, 231, 144, 7549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 398, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6363, 1, 1, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1425, 0, 0, 7649, 0, 0, 0, 0, 0, 0, 30, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 5364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 283, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8118, 0, 0, 0, 709, 0, 1, 6, 8178, 0, 69, 0, 362, 0, 0, 0, 8, 101, 0, 1, 0, 1, 0, 0, 0, 4386, 18, 0, 0, 0, 356, 0, 2, 0, 0, 1929, 0, 0, 1, 0, 0, 0, 4907, 0, 0, 736, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 736, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 0, 0, 0, 0, 0, 8169, 0, 1762, 0, 0, 0, 0, 7283, 7, 3302, 0, 0, 979, 0, 0, 0, 0, 0, 0, 1245] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 967, 3, 0, 0, 4646, 0, 0, 0, 0, 0, 1, 0, 0, 22, 0, 427, 0, 0, 0, 0, 56, 67, 0, 0, 0, 0, 0, 0, 0, 1030, 1, 5035, 0, 0, 0, 0, 1, 7235, 0, 8182, 0, 0, 0, 0, 238, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7718, 3015, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6056, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 56, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4002, 1, 0, 0, 0, 0, 7, 282, 0, 4, 0, 1, 0, 0, 14, 4, 0, 8046, 0, 0, 0, 0, 0, 1, 392, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8039, 337, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 119, 8190, 0, 0, 0, 0, 2, 0, 0, 1, 368, 8094, 0, 1, 0, 1, 0, 0, 3, 2, 7731, 6844, 0, 1587, 0, 450, 0, 0, 0, 2704, 0, 0, 0, 30, 261, 0, 0, 0, 0, 0, 2392, 0, 0, 40, 0, 0, 0, 0, 13, 9, 26, 5409, 0, 0, 807, 286, 0, 0, 0, 0, 0, 0, 0, 1443, 0, 4405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8, 88, 262, 0, 0, 0, 0, 0, 0, 0, 0, 5319, 0, 0, 0, 0, 102, 15, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 2895, 0, 8160, 0, 7343, 0, 0, 0, 0, 28, 8192, 0, 0, 0, 2135, 5799, 9, 0, 0, 0, 7975, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 237, 1, 0, 3395, 239, 0, 0, 6533, 0, 0, 1699, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 128, 0, 166, 0, 0, 0, 7789, 0, 0, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 496, 7868, 0, 0, 0, 0, 2, 0, 0, 7394, 0, 307, 0, 0, 0, 8114, 0, 0, 9, 0, 28, 8173, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7829, 6832, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 172, 0, 5, 2, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 3, 4, 0, 0, 0, 0, 2, 15, 0, 0, 1, 3, 0, 0, 454, 0, 22, 1, 0, 0, 275, 33, 0, 0, 0, 0, 0, 7265, 0, 0, 364, 1047, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 0, 0, 2, 0, 132, 33, 0, 0, 0, 0, 0, 31, 7022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 61, 0, 5489, 0, 8081, 0, 0, 0, 0, 0, 1102, 4059, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 15, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 43, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2342, 4261, 3, 0, 0, 0, 0, 0, 0, 3, 318, 8, 0, 0, 2357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 36, 0, 5178, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 2541, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 237, 0, 0, 2006, 0, 519, 0, 0, 0, 69, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7312, 19852, 4984, 54, 9292, 5231, 13691, 5120] avg=8192.0000 max=19852 max_rank=1 min=54 min_rank=3 imbalance_ratio=2.423340 max_load_reduction=45684 max_load_reduction_pct=0.697083 moved_to_echo=33334 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 167, 1174, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2392, 0, 7318, 8108, 0, 0, 1, 0, 1, 2, 0, 0, 6382, 0, 0, 7803, 7, 7594, 0, 0, 0, 231, 144, 7549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 398, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6363, 1, 1, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1425, 0, 0, 7649, 0, 0, 0, 0, 0, 0, 30, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 2895, 0, 8160, 0, 7343, 0, 0, 0, 0, 28, 8192, 0, 0, 0, 2135, 5799, 9, 0, 0, 0, 7975, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 237, 1, 0, 3395, 239, 0, 0, 6533, 0, 0, 1699, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 967, 3, 0, 0, 4646, 0, 0, 0, 0, 0, 1, 0, 0, 22, 0, 427, 0, 0, 0, 0, 56, 67, 0, 0, 0, 0, 0, 0, 0, 1030, 1, 5035, 0, 0, 0, 0, 1, 7235, 0, 8182, 0, 0, 0, 0, 238, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7718, 3015, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6056, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 56, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4002, 1, 0, 0, 0, 0, 7, 282, 0, 4, 0, 1, 0, 0, 14, 4, 0, 8046, 0, 0, 0, 0, 0, 1, 392, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6470, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 14, 0, 0, 0, 0, 0, 0, 1, 690, 0, 0, 0, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 78, 0, 0, 54, 0, 1, 238, 348, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 38, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 5, 0, 0, 11, 8109, 0, 0, 6267, 0, 0, 7120, 0, 0, 25, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7033, 0, 0, 0, 2, 2100, 0, 0, 0, 0, 106, 0, 5343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 456, 0, 295, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 612, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6922, 3087, 1372, 126, 21892, 14584, 9431, 8122] avg=8192.0000 max=21892 max_rank=4 min=126 min_rank=3 imbalance_ratio=2.672363 max_load_reduction=43644 max_load_reduction_pct=0.665955 moved_to_echo=54029 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8039, 337, 0, 0, 0, 145, 0, 0, 0, 0, 7033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 119, 8190, 0, 0, 0, 0, 2, 0, 0, 1, 1471, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 1, 0, 1, 0, 0, 3, 2, 7731, 6844, 0, 1587, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2704, 0, 0, 0, 30, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 13, 9, 26, 5409, 0, 0, 807, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1443, 0, 4405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8, 88, 262, 0, 0, 0, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5319, 0, 0, 0, 0, 102, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15554, 10845, 16987, 2995, 6590, 5848, 1281, 5436] avg=8192.0000 max=16987 max_rank=2 min=1281 min_rank=6 imbalance_ratio=2.073608 max_load_reduction=48549 max_load_reduction_pct=0.740799 moved_to_echo=19155 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 0, 0, 2, 0, 132, 33, 0, 0, 0, 0, 0, 31, 7022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 61, 0, 5489, 0, 8081, 0, 0, 0, 0, 0, 1102, 4059, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 15, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 43, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2342, 4261, 3, 0, 0, 0, 0, 0, 0, 3, 318, 8, 0, 0, 2357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 36, 0, 5178, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 2541, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 237, 0, 0, 2006, 0, 519, 0, 0, 0, 69, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7312, 19852, 4984, 54, 9292, 5231, 13691, 5120] avg=8192.0000 max=19852 max_rank=1 min=54 min_rank=3 imbalance_ratio=2.423340 max_load_reduction=45684 max_load_reduction_pct=0.697083 moved_to_echo=33334 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 1, 6, 0, 0, 34, 0, 362, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 1, 0, 1, 0, 0, 0, 4386, 18, 0, 0, 0, 292, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 0, 1, 0, 0, 0, 4907, 0, 0, 736, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 736, 0, 0, 34, 0, 0, 0, 0, 7544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 0, 0, 0, 0, 0, 8169, 0, 1762, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7283, 7, 3302, 0, 0, 979, 0, 0, 0, 0, 0, 0, 1245, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6186, 8464, 1120, 4801, 7593, 8327, 16166, 12879] avg=8192.0000 max=16166 max_rank=6 min=1120 min_rank=2 imbalance_ratio=1.973389 max_load_reduction=49370 max_load_reduction_pct=0.753326 moved_to_echo=44965 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 167, 1174, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 552, 0, 0, 0, 4644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 8108, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6382, 0, 0, 7803, 7, 7594, 0, 0, 0, 231, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 398, 0, 0, 0, 0, 0, 2118, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6363, 1, 1, 0, 0, 0, 0, 61, 0, 0, 0, 0, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1425, 0, 0, 7649, 0, 0, 0, 2905, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1605, 5203, 11565, 22133, 2581, 8655, 13732, 62] avg=8192.0000 max=22133 max_rank=3 min=62 min_rank=7 imbalance_ratio=2.701782 max_load_reduction=43403 max_load_reduction_pct=0.662277 moved_to_echo=25030 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 1012, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 0, 8160, 0, 7343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 8192, 0, 0, 0, 2135, 5799, 9, 0, 0, 0, 6529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 0, 11, 0, 0, 0, 835, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 237, 1, 0, 3395, 239, 0, 0, 6533, 0, 0, 1699, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1655, 409, 16874, 22692, 928, 8570, 699, 13709] avg=8192.0000 max=22692 max_rank=3 min=409 min_rank=1 imbalance_ratio=2.770020 max_load_reduction=42844 max_load_reduction_pct=0.653748 moved_to_echo=23906 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8039, 337, 0, 0, 0, 145, 0, 0, 0, 0, 7033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 119, 8190, 0, 0, 0, 0, 2, 0, 0, 1, 1471, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 1, 0, 1, 0, 0, 3, 2, 7731, 6844, 0, 1587, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2704, 0, 0, 0, 30, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 13, 9, 26, 5409, 0, 0, 807, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1443, 0, 4405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8, 88, 262, 0, 0, 0, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5319, 0, 0, 0, 0, 102, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15554, 10845, 16987, 2995, 6590, 5848, 1281, 5436] avg=8192.0000 max=16987 max_rank=2 min=1281 min_rank=6 imbalance_ratio=2.073608 max_load_reduction=48549 max_load_reduction_pct=0.740799 moved_to_echo=19155 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 167, 1174, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 552, 0, 0, 0, 4644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 8108, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6382, 0, 0, 7803, 7, 7594, 0, 0, 0, 231, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 398, 0, 0, 0, 0, 0, 2118, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6363, 1, 1, 0, 0, 0, 0, 61, 0, 0, 0, 0, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1425, 0, 0, 7649, 0, 0, 0, 2905, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1605, 5203, 11565, 22133, 2581, 8655, 13732, 62] avg=8192.0000 max=22133 max_rank=3 min=62 min_rank=7 imbalance_ratio=2.701782 max_load_reduction=43403 max_load_reduction_pct=0.662277 moved_to_echo=25030 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 128, 0, 166, 0, 0, 0, 7789, 0, 0, 0, 0, 0, 586, 0, 0, 6426, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 496, 7868, 0, 0, 0, 0, 2, 0, 4202, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 0, 0, 0, 9, 0, 28, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 172, 0, 5, 2, 0, 0, 1, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 3, 7484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2, 15, 0, 0, 1, 3, 0, 0, 454, 0, 22, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 275, 33, 0, 0, 0, 0, 0, 7265, 0, 0, 364, 1047, 0, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15725, 13582, 8517, 6414, 220, 7571, 3131, 10376] avg=8192.0000 max=15725 max_rank=0 min=220 min_rank=4 imbalance_ratio=1.919556 max_load_reduction=49811 max_load_reduction_pct=0.760056 moved_to_echo=21298 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 967, 3, 0, 0, 4646, 0, 0, 0, 0, 0, 1, 0, 0, 22, 4376, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 56, 67, 0, 0, 0, 0, 0, 0, 0, 1030, 1855, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 3872, 0, 0, 0, 0, 238, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6056, 0, 0, 7990, 0, 0, 0, 2362, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 6673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 1, 0, 0, 0, 0, 7, 282, 0, 4, 0, 1, 0, 0, 1160, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4, 0, 8046, 0, 0, 0, 0, 0, 1, 392, 4, 0, 0, 0, 0, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10577, 4094, 4117, 6321, 16440, 6750, 7405, 9832] avg=8192.0000 max=16440 max_rank=4 min=4094 min_rank=1 imbalance_ratio=2.006836 max_load_reduction=49096 max_load_reduction_pct=0.749146 moved_to_echo=40427 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 17408, 128, 128, 7808, 2176, 18048, 128, 128, 128, 256, 19712, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 256, 4352, 4352, 128, 128, 8192, 128, 128, 23040, 512, 128, 17024, 384, 0, 128, 7936, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[896, 0, 128, 1152, 0, 128, 128, 0, 128, 12928, 16256, 15616, 128, 17152, 1408, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -2282,4876 +2282,4876 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[384, 512, 128, 1408, 128, 512, 256, 13824, 512, 16000, 0, 128, 1280, 0, 128, 10368, 17280, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 1152, 5632, 0, 128, 12800, 8192, 256, 128, 256, 7808, 1024, 640, 256, 128, 24960, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 1152, 256, 15744, 384, 10752, 0, 3968, 1280, 128, 15872, 6656, 128, 512, 4096, 1280, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 13440, 7808, 128, 4480, 768, 2176, 0, 128, 128, 0, 128, 128, 36864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 13440, 7808, 128, 4480, 768, 2176, 0, 128, 128, 0, 128, 128, 36864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[22, 0, 183, 0, 1, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 0, 0, 0, 0, 7257, 5667, 0, 151, 45, 4605, 0, 0, 7648, 537, 22, 5811, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 25, 7948, 13, 5, 343, 0, 0, 0, 0, 0, 29, 0, 85, 0, 21, 0, 0, 0, 0, 0, 64, 1171, 0, 0, 0, 12, 0, 7816, 0, 0, 0, 0, 0, 16, 0, 0, 33, 0, 0, 0, 0, 0, 5, 520, 0, 0, 0, 0, 14, 3976, 56, 655, 0, 0, 0, 0, 12, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[7384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 108, 0, 0, 0, 3528, 0, 1, 0, 1871, 552, 0, 0, 0, 0, 4856, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 1, 0, 581, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 14, 3074, 0, 122, 0, 0, 107, 0, 0, 0, 0, 0, 7905, 0, 0, 0, 3972, 6738, 0, 2, 0, 186, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 0, 1, 5, 6, 645, 0, 1, 484, 0, 0, 0, 3902, 0, 4788, 0, 4118] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[6148, 0, 0, 0, 0, 0, 0, 64, 0, 1641, 0, 0, 0, 1, 0, 0, 57, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 262, 0, 0, 0, 1220, 0, 0, 4786, 0, 134, 0, 7825, 0, 0, 7984, 4, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6807, 0, 0, 0, 2, 0, 7885, 95, 0, 0, 1, 0, 0, 1260, 2944, 757, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1212, 0, 1, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 211, 0, 0, 0, 0, 195, 0, 8192, 0, 78, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 140, 0, 0, 0, 0, 0, 0, 1, 0, 1058, 0, 0, 0, 0, 1833, 0, 43, 0, 6072, 7549, 0, 0, 0, 0, 0, 6454, 0, 0, 0, 933, 0, 2308, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 0, 0, 1, 0, 0, 0, 159, 0, 0, 7415, 0, 0, 0, 0, 0, 3744, 5, 0, 0, 5144, 0, 0, 0, 0, 6826, 0, 2982, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 1, 0, 935, 2, 0, 0, 222, 23, 0, 0, 0, 0, 0, 0, 0, 1, 0, 270, 0, 0, 0, 8083, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 175, 0, 0, 0, 0, 47, 0, 0, 0, 0, 2, 0, 0, 0, 5613, 234, 0, 0, 7839, 0, 0, 0, 0, 1, 8040, 0, 3, 0, 2, 0, 0, 7, 12, 0, 23, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 1084, 2, 4813, 117, 0, 0, 0, 12, 0, 0, 8116, 0, 0, 0, 0, 0, 2227, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 0, 8009, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7016, 0, 0, 0, 56] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[22, 0, 183, 0, 1, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 0, 0, 0, 0, 7257, 5667, 0, 151, 45, 4605, 0, 0, 7648, 537, 22, 5811, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 25, 7948, 13, 5, 343, 0, 0, 0, 0, 0, 29, 0, 85, 0, 21, 0, 0, 0, 0, 0, 64, 1171, 0, 0, 0, 12, 0, 7816, 0, 0, 0, 0, 0, 16, 0, 0, 33, 0, 0, 0, 0, 0, 5, 520, 0, 0, 0, 0, 14, 3976, 56, 655, 0, 0, 0, 0, 12, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[22, 28, 0, 0, 1, 0, 0, 39, 0, 0, 0, 0, 0, 439, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 9, 637, 0, 7832, 0, 0, 38, 0, 707, 34, 0, 8188, 0, 76, 8108, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 7429, 0, 0, 0, 0, 819, 0, 0, 0, 0, 0, 0, 7417, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 156, 0, 0, 0, 0, 0, 0, 0, 0, 29, 323, 0, 0, 0, 0, 0, 0, 4098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 7053, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[8168, 8142, 0, 0, 4, 0, 0, 0, 2, 0, 0, 2, 0, 0, 323, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 40, 0, 108, 0, 7252, 0, 0, 0, 0, 0, 3, 0, 228, 0, 0, 8067, 0, 1632, 0, 0, 9, 0, 0, 0, 0, 8, 1, 0, 0, 4, 0, 0, 0, 0, 4, 7290, 0, 0, 0, 153, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4948, 4, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 396, 0, 0, 3233, 0, 0, 1253, 0, 0, 0, 0, 1, 0, 2, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7854, 0, 6158, 0, 0, 0, 93] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 175, 0, 0, 0, 0, 47, 0, 0, 0, 0, 2, 0, 0, 0, 5613, 234, 0, 0, 7839, 0, 0, 0, 0, 1, 8040, 0, 3, 0, 2, 0, 0, 7, 12, 0, 23, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 1084, 2, 4813, 117, 0, 0, 0, 12, 0, 0, 8116, 0, 0, 0, 0, 0, 2227, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 0, 8009, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7016, 0, 0, 0, 56] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[6848, 0, 0, 5, 0, 0, 215, 0, 239, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 38, 0, 0, 0, 6865, 7787, 0, 0, 0, 0, 23, 0, 43, 0, 5, 6118, 0, 0, 0, 0, 0, 904, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5945, 0, 0, 7, 0, 0, 8, 12, 1, 0, 2209, 34, 0, 0, 0, 0, 0, 0, 0, 7698, 0, 373, 0, 0, 0, 2838, 0, 0, 0, 0, 5134, 0, 0, 0, 0, 1, 161, 0, 0, 0, 0, 0, 0, 937, 470, 0, 0, 0, 0, 0, 0, 1687, 0, 0, 113, 0, 0, 38, 163, 0, 0, 558, 0, 3815, 0, 11, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 183, 0, 1, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1656, 1977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 0, 0, 0, 0, 3755, 5667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 45, 4605, 0, 0, 7648, 537, 22, 378, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 7948, 13, 5, 343, 0, 0, 0, 0, 0, 29, 0, 3777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 21, 0, 0, 0, 0, 0, 64, 1171, 0, 0, 0, 12, 0, 4544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 33, 0, 0, 0, 0, 0, 5, 520, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3976, 56, 655, 0, 0, 0, 0, 12, 0, 9, 0, 3502, 1295, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3849, 11499, 14027, 12140, 5897, 2448, 9873, 5803] avg=8192.0000 max=14027 max_rank=2 min=2448 min_rank=5 imbalance_ratio=1.712280 max_load_reduction=51509 max_load_reduction_pct=0.785965 moved_to_echo=24021 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 140, 0, 0, 0, 0, 0, 0, 1, 0, 1058, 0, 0, 0, 0, 1833, 0, 43, 0, 6072, 7549, 0, 0, 0, 0, 0, 6454, 0, 0, 0, 933, 0, 2308, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 0, 0, 1, 0, 0, 0, 159, 0, 0, 7415, 0, 0, 0, 0, 0, 3744, 5, 0, 0, 5144, 0, 0, 0, 0, 6826, 0, 2982, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 1, 0, 935, 2, 0, 0, 222, 23, 0, 0, 0, 0, 0, 0, 0, 1, 0, 270, 0, 0, 0, 8083, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[8168, 8142, 0, 0, 4, 0, 0, 0, 2, 0, 0, 2, 0, 0, 323, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 40, 0, 108, 0, 7252, 0, 0, 0, 0, 0, 3, 0, 228, 0, 0, 8067, 0, 1632, 0, 0, 9, 0, 0, 0, 0, 8, 1, 0, 0, 4, 0, 0, 0, 0, 4, 7290, 0, 0, 0, 153, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4948, 4, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 396, 0, 0, 3233, 0, 0, 1253, 0, 0, 0, 0, 1, 0, 2, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7854, 0, 6158, 0, 0, 0, 93] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[7384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 108, 0, 0, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1871, 552, 0, 0, 0, 0, 317, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 581, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 14, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 107, 0, 0, 0, 0, 0, 7905, 0, 0, 0, 3972, 6738, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 186, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 0, 509, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 6, 645, 0, 1, 484, 0, 0, 0, 2820, 0, 4788, 0, 4118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9544, 3642, 5229, 3807, 1804, 19754, 8888, 12868] avg=8192.0000 max=19754 max_rank=5 min=1804 min_rank=4 imbalance_ratio=2.411377 max_load_reduction=45782 max_load_reduction_pct=0.698578 moved_to_echo=43314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 175, 0, 0, 0, 0, 47, 0, 0, 0, 0, 6, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5613, 234, 0, 0, 7839, 0, 0, 0, 0, 1, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 7, 12, 0, 3, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 1084, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4813, 117, 0, 0, 0, 12, 0, 0, 8116, 0, 0, 0, 0, 0, 1295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 369, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5070, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[802, 21729, 1777, 1581, 14355, 3288, 16876, 5128] avg=8192.0000 max=21729 max_rank=1 min=802 min_rank=0 imbalance_ratio=2.652466 max_load_reduction=43807 max_load_reduction_pct=0.668442 moved_to_echo=39647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[22, 28, 0, 0, 1, 0, 0, 39, 0, 0, 0, 0, 0, 439, 3, 8, 2310, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 9, 637, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 707, 34, 0, 8188, 0, 76, 528, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3899, 0, 0, 0, 5270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7429, 0, 0, 0, 0, 819, 0, 0, 0, 0, 0, 0, 4312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 156, 0, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 323, 0, 0, 0, 0, 0, 0, 4098, 3780, 1229, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 5096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4726, 4712, 9572, 9189, 12560, 1817, 9770, 13190] avg=8192.0000 max=13190 max_rank=7 min=1817 min_rank=5 imbalance_ratio=1.610107 max_load_reduction=52346 max_load_reduction_pct=0.798737 moved_to_echo=37337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 140, 1839, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1058, 0, 0, 0, 0, 949, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6072, 7549, 0, 0, 0, 0, 0, 420, 0, 0, 0, 933, 0, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 4195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 159, 0, 0, 7415, 0, 0, 0, 0, 0, 2177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5144, 0, 0, 0, 0, 6826, 0, 2982, 0, 0, 0, 0, 0, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 1, 0, 935, 2, 0, 0, 222, 23, 0, 0, 884, 620, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 270, 0, 0, 0, 5841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2928, 2051, 17282, 7259, 9752, 16843, 3309, 6112] avg=8192.0000 max=17282 max_rank=2 min=2051 min_rank=1 imbalance_ratio=2.109619 max_load_reduction=48254 max_load_reduction_pct=0.736298 moved_to_echo=36016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 183, 0, 1, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1656, 1977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 0, 0, 0, 0, 3755, 5667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 45, 4605, 0, 0, 7648, 537, 22, 378, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 7948, 13, 5, 343, 0, 0, 0, 0, 0, 29, 0, 3777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 21, 0, 0, 0, 0, 0, 64, 1171, 0, 0, 0, 12, 0, 4544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 33, 0, 0, 0, 0, 0, 5, 520, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3976, 56, 655, 0, 0, 0, 0, 12, 0, 9, 0, 3502, 1295, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3849, 11499, 14027, 12140, 5897, 2448, 9873, 5803] avg=8192.0000 max=14027 max_rank=2 min=2448 min_rank=5 imbalance_ratio=1.712280 max_load_reduction=51509 max_load_reduction_pct=0.785965 moved_to_echo=24021 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6148, 0, 0, 0, 0, 0, 0, 64, 0, 1641, 0, 0, 0, 1, 0, 0, 2279, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 262, 0, 0, 0, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4786, 0, 134, 0, 7825, 0, 0, 512, 4, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 0, 5193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6807, 0, 0, 0, 2, 0, 4577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 1, 0, 0, 1260, 2944, 757, 0, 50, 0, 0, 0, 0, 0, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1212, 0, 1, 0, 0, 0, 1, 0, 22, 0, 1310, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0, 195, 0, 5913, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12131, 2731, 13878, 9066, 11392, 7021, 2920, 6397] avg=8192.0000 max=13878 max_rank=2 min=2731 min_rank=1 imbalance_ratio=1.694092 max_load_reduction=51658 max_load_reduction_pct=0.788239 moved_to_echo=27730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 175, 0, 0, 0, 0, 47, 0, 0, 0, 0, 6, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5613, 234, 0, 0, 7839, 0, 0, 0, 0, 1, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 7, 12, 0, 3, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 1084, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4813, 117, 0, 0, 0, 12, 0, 0, 8116, 0, 0, 0, 0, 0, 1295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 369, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5070, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[802, 21729, 1777, 1581, 14355, 3288, 16876, 5128] avg=8192.0000 max=21729 max_rank=1 min=802 min_rank=0 imbalance_ratio=2.652466 max_load_reduction=43807 max_load_reduction_pct=0.668442 moved_to_echo=39647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6848, 0, 0, 5, 0, 0, 215, 0, 239, 0, 368, 0, 0, 0, 0, 0, 1743, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 38, 0, 0, 0, 3552, 7787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 43, 0, 5, 399, 0, 0, 0, 0, 0, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5945, 0, 0, 7, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 0, 2209, 34, 0, 0, 0, 0, 0, 0, 0, 4476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 2838, 0, 0, 0, 0, 5134, 0, 0, 0, 0, 1, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 937, 470, 0, 0, 0, 0, 0, 0, 1687, 3313, 1275, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 38, 163, 0, 0, 558, 0, 2757, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11365, 15226, 1374, 9944, 6740, 9236, 8011, 3640] avg=8192.0000 max=15226 max_rank=1 min=1374 min_rank=2 imbalance_ratio=1.858643 max_load_reduction=50310 max_load_reduction_pct=0.767670 moved_to_echo=27627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 140, 1839, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1058, 0, 0, 0, 0, 949, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6072, 7549, 0, 0, 0, 0, 0, 420, 0, 0, 0, 933, 0, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 4195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 159, 0, 0, 7415, 0, 0, 0, 0, 0, 2177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5144, 0, 0, 0, 0, 6826, 0, 2982, 0, 0, 0, 0, 0, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 1, 0, 935, 2, 0, 0, 222, 23, 0, 0, 884, 620, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 270, 0, 0, 0, 5841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2928, 2051, 17282, 7259, 9752, 16843, 3309, 6112] avg=8192.0000 max=17282 max_rank=2 min=2051 min_rank=1 imbalance_ratio=2.109619 max_load_reduction=48254 max_load_reduction_pct=0.736298 moved_to_echo=36016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[8168, 8142, 0, 0, 4, 0, 0, 0, 2, 0, 0, 2, 0, 0, 323, 0, 2299, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 40, 0, 108, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 228, 0, 0, 525, 0, 1632, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 4, 0, 0, 0, 0, 4, 7290, 0, 0, 0, 5243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 396, 0, 0, 3233, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 0, 0, 0, 1, 0, 2, 38, 0, 0, 0, 0, 0, 3504, 820, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7854, 0, 4450, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20191, 3946, 2397, 12550, 3036, 5129, 5889, 12398] avg=8192.0000 max=20191 max_rank=0 min=2397 min_rank=2 imbalance_ratio=2.464722 max_load_reduction=45345 max_load_reduction_pct=0.691910 moved_to_echo=26452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 512, 0, 5248, 256, 2944, 1408, 3072, 7680, 0, 16128, 0, 512, 128, 4224, 12032, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 6912, 256, 128, 0, 14976, 128, 384, 768, 0, 16128, 26368, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[512, 128, 9344, 256, 128, 4096, 128, 11136, 1792, 128, 128, 0, 256, 128, 128, 13824, 15104, 7552, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 6912, 256, 128, 0, 14976, 128, 384, 768, 0, 16128, 26368, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[28672, 8192, 256, 128, 128, 128, 512, 128, 256, 1664, 384, 128, 128, 512, 384, 256, 13568, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 11008, 14848, 768, 0, 24064, 640, 128, 3200, 128, 3456, 0, 3456, 640, 3328, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 8576, 128, 128, 384, 512, 0, 128, 24064, 128, 128, 1152, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 4864, 256, 7552, 128, 0, 2432, 128, 896, 23552, 128, 0, 0, 128, 128, 26112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 896, 128, 256, 768, 0, 16768, 0, 37760, 0, 4992, 0, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True - [2026-08-26 16:18:04] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 720.8 | throughput per GPU (TFLOP/s/GPU): 250.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117604E+01 | loss scale: 1.0 | grad norm: 44.306 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3826, 65, 2, 129, 144, 359, 60, 0, 15, 405, 1514, 114, 1630, 146, 155, 460, 168, 1662, 28, 12, 992, 25, 20, 610, 637, 36, 1, 3994, 1770, 3399, 114, 266, 929, 1801, 161, 18, 92, 1123, 23, 127, 1225, 2, 541, 2684, 0, 44, 30, 4061, 8, 41, 4, 14, 1, 189, 16, 8, 239, 17, 10, 203, 297, 21, 157, 143, 2478, 3, 4, 1, 1560, 2, 24, 762, 1443, 86, 2115, 420, 421, 71, 594, 75, 23, 17, 0, 22, 30, 1, 148, 1607, 4, 79, 201, 40, 108, 68, 164, 3, 163, 270, 14, 410, 58, 9, 318, 87, 165, 444, 236, 4015, 2, 277, 383, 108, 216, 764, 0, 38, 0, 364, 374, 62, 70, 26, 4780, 322, 86, 48, 4, 1862] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3519, 69, 4, 229, 223, 623, 116, 3, 28, 506, 1268, 206, 1283, 110, 324, 214, 111, 1663, 1, 29, 872, 12, 51, 1711, 1413, 82, 10, 4576, 2359, 3651, 110, 631, 512, 1044, 183, 4, 84, 939, 12, 63, 1077, 9, 637, 2844, 4, 75, 20, 3604, 45, 150, 2, 22, 9, 141, 15, 5, 191, 12, 40, 342, 267, 131, 243, 227, 2796, 6, 3, 0, 1204, 0, 30, 472, 939, 25, 1277, 614, 373, 25, 499, 224, 26, 5, 9, 12, 43, 1, 76, 1217, 4, 117, 248, 3, 34, 164, 267, 13, 337, 190, 16, 946, 41, 4, 70, 102, 159, 324, 329, 3366, 2, 298, 488, 99, 374, 958, 1, 126, 0, 244, 549, 142, 30, 27, 4686, 414, 85, 47, 3, 1393] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[4037, 22, 1, 94, 173, 587, 204, 0, 1, 533, 1536, 253, 1299, 147, 216, 741, 116, 1905, 4, 7, 1281, 0, 41, 664, 1291, 25, 1, 4145, 2223, 3246, 176, 371, 744, 1057, 150, 2, 40, 1197, 24, 39, 530, 4, 387, 2282, 0, 26, 37, 4179, 8, 115, 4, 10, 4, 90, 5, 3, 200, 11, 13, 193, 258, 7, 258, 320, 2532, 5, 0, 0, 981, 2, 28, 974, 966, 9, 2401, 401, 597, 0, 507, 278, 7, 0, 0, 14, 25, 0, 68, 1511, 11, 49, 74, 1, 17, 218, 205, 7, 124, 366, 36, 417, 144, 2, 68, 5, 221, 146, 398, 4197, 3, 139, 367, 64, 91, 817, 0, 34, 1, 165, 912, 137, 47, 24, 5016, 398, 50, 59, 9, 1654] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[5012, 33, 5, 428, 61, 703, 86, 0, 36, 226, 1108, 67, 1561, 58, 226, 436, 114, 2030, 7, 25, 1005, 12, 31, 831, 1070, 43, 0, 4034, 2096, 2771, 32, 411, 1019, 1304, 146, 19, 124, 401, 6, 15, 545, 7, 690, 3851, 1, 19, 30, 3543, 11, 57, 12, 18, 0, 244, 22, 4, 97, 10, 31, 546, 223, 43, 128, 130, 2823, 11, 2, 2, 1490, 0, 41, 608, 1277, 2, 2346, 314, 513, 3, 319, 234, 8, 19, 1, 14, 16, 10, 85, 614, 3, 96, 255, 20, 26, 35, 325, 13, 307, 392, 12, 813, 38, 3, 43, 29, 152, 81, 667, 4038, 20, 108, 305, 35, 193, 916, 1, 119, 0, 202, 352, 229, 104, 26, 4685, 371, 59, 82, 24, 1751] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3740, 16, 1, 91, 109, 317, 121, 1, 18, 395, 1076, 180, 1542, 93, 337, 390, 65, 2325, 5, 21, 792, 9, 26, 808, 546, 37, 12, 3916, 2773, 3673, 22, 474, 1087, 2406, 142, 1, 149, 1035, 4, 73, 423, 23, 215, 2404, 3, 19, 16, 3429, 62, 83, 3, 5, 8, 373, 3, 9, 166, 16, 25, 185, 250, 11, 223, 59, 2953, 11, 1, 0, 1117, 1, 21, 706, 1030, 12, 1788, 843, 461, 7, 372, 572, 3, 4, 0, 101, 46, 1, 68, 1638, 8, 67, 154, 4, 13, 71, 303, 19, 237, 388, 54, 892, 26, 10, 48, 20, 194, 112, 366, 4055, 11, 179, 438, 44, 63, 981, 0, 52, 0, 124, 478, 404, 119, 5, 4655, 301, 27, 52, 2, 1963] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4195, 30, 0, 254, 90, 691, 65, 0, 39, 416, 591, 183, 1491, 61, 215, 301, 47, 2029, 7, 81, 1109, 7, 31, 1438, 1151, 53, 6, 4775, 2363, 3590, 85, 617, 826, 1375, 240, 43, 137, 520, 8, 70, 1081, 13, 886, 3667, 0, 18, 39, 3522, 17, 46, 1, 6, 16, 257, 5, 11, 170, 46, 35, 205, 266, 111, 332, 188, 2541, 7, 0, 1, 1222, 10, 53, 395, 951, 16, 2706, 152, 521, 15, 358, 220, 27, 2, 0, 59, 32, 0, 113, 484, 8, 74, 378, 21, 137, 74, 97, 19, 160, 116, 21, 457, 35, 14, 140, 31, 125, 191, 422, 3271, 10, 159, 352, 78, 190, 917, 2, 83, 1, 201, 415, 92, 106, 61, 4841, 388, 85, 99, 15, 1597] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3768, 4, 1, 198, 73, 464, 131, 0, 30, 380, 2159, 194, 1498, 104, 320, 382, 43, 2078, 12, 37, 757, 3, 51, 1375, 938, 49, 2, 4478, 2114, 2728, 36, 622, 969, 1360, 224, 9, 45, 900, 7, 95, 710, 12, 953, 3249, 10, 80, 77, 3329, 15, 77, 0, 22, 1, 286, 23, 9, 47, 9, 21, 200, 381, 7, 261, 396, 2823, 3, 2, 0, 889, 4, 31, 824, 1073, 25, 2338, 277, 436, 4, 394, 150, 11, 12, 0, 28, 44, 0, 108, 1360, 9, 64, 164, 6, 99, 194, 148, 32, 276, 201, 11, 735, 54, 10, 113, 38, 142, 164, 357, 3446, 10, 169, 376, 111, 132, 891, 4, 25, 1, 223, 516, 229, 72, 8, 4536, 489, 47, 58, 11, 1716] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4192, 69, 3, 130, 119, 479, 86, 0, 8, 333, 1220, 119, 1743, 282, 95, 426, 59, 1352, 33, 27, 829, 3, 14, 1769, 1431, 35, 3, 4097, 2313, 3506, 51, 492, 542, 1573, 172, 51, 27, 362, 11, 129, 1103, 2, 796, 2249, 95, 31, 17, 3454, 15, 90, 5, 11, 2, 312, 46, 7, 182, 17, 15, 457, 369, 13, 97, 279, 2972, 27, 1, 0, 807, 0, 9, 1092, 1515, 32, 1922, 225, 624, 11, 172, 219, 30, 1, 4, 21, 15, 0, 161, 1396, 3, 55, 193, 7, 51, 148, 376, 6, 499, 233, 13, 321, 102, 4, 51, 26, 411, 188, 408, 3554, 19, 147, 690, 97, 147, 599, 0, 95, 0, 124, 706, 292, 48, 18, 4627, 297, 58, 43, 32, 1743] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2859, 65, 2, 129, 144, 359, 60, 0, 15, 405, 1514, 114, 1630, 146, 155, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 1662, 28, 12, 992, 25, 20, 610, 637, 36, 1, 0, 1770, 111, 114, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 1801, 161, 18, 92, 1123, 23, 127, 1225, 2, 541, 2684, 0, 44, 30, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 41, 4, 14, 1, 189, 16, 8, 239, 17, 10, 203, 297, 21, 157, 143, 3994, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1276, 3, 4, 1, 1560, 2, 24, 762, 1443, 86, 2115, 420, 421, 71, 594, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 17, 0, 22, 30, 1, 148, 1607, 4, 79, 201, 40, 108, 68, 164, 3, 3288, 1202, 1255, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 270, 14, 410, 58, 9, 318, 87, 165, 444, 236, 4015, 2, 277, 383, 108, 908, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 764, 0, 38, 0, 364, 374, 62, 70, 26, 3872, 322, 86, 48, 4, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8057, 6452, 9045, 7923, 8857, 8606, 8488, 8108] avg=8192.0000 max=9045 max_rank=2 min=6452 min_rank=1 imbalance_ratio=1.104126 max_load_reduction=56491 max_load_reduction_pct=0.861984 moved_to_echo=34059 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2630, 69, 4, 229, 223, 623, 116, 3, 28, 506, 1268, 206, 1283, 110, 324, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 1663, 1, 29, 872, 12, 51, 1711, 1413, 82, 10, 0, 2359, 119, 110, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 1044, 183, 4, 84, 939, 12, 63, 1077, 9, 637, 2844, 4, 75, 20, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 150, 2, 22, 9, 141, 15, 5, 191, 12, 40, 342, 267, 131, 243, 227, 4576, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 6, 3, 0, 1204, 0, 30, 472, 939, 25, 1277, 614, 373, 25, 499, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 5, 9, 12, 43, 1, 76, 1217, 4, 117, 248, 3, 34, 164, 267, 13, 3532, 1357, 1113, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 190, 16, 946, 41, 4, 70, 102, 159, 324, 329, 3366, 2, 298, 488, 99, 891, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 958, 1, 126, 0, 244, 549, 142, 30, 27, 3795, 414, 85, 47, 3, 1393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7836, 9174, 7725, 8691, 7130, 8559, 8233, 8188] avg=8192.0000 max=9174 max_rank=1 min=7130 min_rank=4 imbalance_ratio=1.119873 max_load_reduction=56362 max_load_reduction_pct=0.860016 moved_to_echo=32110 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3017, 22, 1, 94, 173, 587, 204, 0, 1, 533, 1536, 253, 1299, 147, 216, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 1905, 4, 7, 1281, 0, 41, 664, 1291, 25, 1, 0, 2223, 106, 176, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 1057, 150, 2, 40, 1197, 24, 39, 530, 4, 387, 2282, 0, 26, 37, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 115, 4, 10, 4, 90, 5, 3, 200, 11, 13, 193, 258, 7, 258, 320, 4145, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1303, 5, 0, 0, 981, 2, 28, 974, 966, 9, 2401, 401, 597, 0, 507, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 14, 25, 0, 68, 1511, 11, 49, 74, 1, 17, 218, 205, 7, 3140, 1229, 1291, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 366, 36, 417, 144, 2, 68, 5, 221, 146, 398, 4197, 3, 139, 367, 64, 953, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 817, 0, 34, 1, 165, 912, 137, 47, 24, 4063, 398, 50, 59, 9, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8824, 8211, 6771, 8280, 8452, 8232, 8305, 8461] avg=8192.0000 max=8824 max_rank=0 min=6771 min_rank=2 imbalance_ratio=1.077148 max_load_reduction=56712 max_load_reduction_pct=0.865356 moved_to_echo=33450 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3136, 30, 0, 254, 90, 691, 65, 0, 39, 416, 591, 183, 1491, 61, 215, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2029, 7, 81, 1109, 7, 31, 1438, 1151, 53, 6, 0, 2363, 117, 85, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 1375, 240, 43, 137, 520, 8, 70, 1081, 13, 886, 3667, 0, 18, 39, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 46, 1, 6, 16, 257, 5, 11, 170, 46, 35, 205, 266, 111, 332, 188, 4775, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 7, 0, 1, 1222, 10, 53, 395, 951, 16, 2706, 152, 521, 15, 358, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2, 0, 59, 32, 0, 113, 484, 8, 74, 378, 21, 137, 74, 97, 19, 3473, 1233, 1088, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 116, 21, 457, 35, 14, 140, 31, 125, 191, 422, 3271, 10, 159, 352, 78, 920, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 917, 2, 83, 1, 201, 415, 92, 106, 61, 3921, 388, 85, 99, 15, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7563, 9141, 9136, 8708, 7935, 7698, 7182, 8173] avg=8192.0000 max=9141 max_rank=1 min=7182 min_rank=6 imbalance_ratio=1.115845 max_load_reduction=56395 max_load_reduction_pct=0.860519 moved_to_echo=30988 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3740, 33, 5, 428, 61, 703, 86, 0, 36, 226, 1108, 67, 1561, 58, 226, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 2030, 7, 25, 1005, 12, 31, 831, 1070, 43, 0, 0, 2096, 85, 32, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1304, 146, 19, 124, 401, 6, 15, 545, 7, 690, 3851, 1, 19, 30, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 57, 12, 18, 0, 244, 22, 4, 97, 10, 31, 546, 223, 43, 128, 130, 4034, 2239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1449, 11, 2, 2, 1490, 0, 41, 608, 1277, 2, 2346, 314, 513, 3, 319, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 19, 1, 14, 16, 10, 85, 614, 3, 96, 255, 20, 26, 35, 325, 13, 2686, 1374, 1099, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 392, 12, 813, 38, 3, 43, 29, 152, 81, 667, 4038, 20, 108, 305, 35, 895, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 916, 1, 119, 0, 202, 352, 229, 104, 26, 3790, 371, 59, 82, 24, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8774, 7792, 8382, 7849, 8611, 7155, 8754, 8219] avg=8192.0000 max=8774 max_rank=0 min=7155 min_rank=5 imbalance_ratio=1.071045 max_load_reduction=56762 max_load_reduction_pct=0.866119 moved_to_echo=32739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2817, 4, 1, 198, 73, 464, 131, 0, 30, 380, 2159, 194, 1498, 104, 320, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 2078, 12, 37, 757, 3, 51, 1375, 938, 49, 2, 0, 2114, 89, 36, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1360, 224, 9, 45, 900, 7, 95, 710, 12, 953, 3249, 10, 80, 77, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 77, 0, 22, 1, 286, 23, 9, 47, 9, 21, 200, 381, 7, 261, 396, 4478, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1453, 3, 2, 0, 889, 4, 31, 824, 1073, 25, 2338, 277, 436, 4, 394, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 12, 0, 28, 44, 0, 108, 1360, 9, 64, 164, 6, 99, 194, 148, 32, 2639, 1370, 1028, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 201, 11, 735, 54, 10, 113, 38, 142, 164, 357, 3446, 10, 169, 376, 111, 862, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 891, 4, 25, 1, 223, 516, 229, 72, 8, 3674, 489, 47, 58, 11, 1716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8755, 8206, 8901, 8333, 7903, 7656, 7686, 8096] avg=8192.0000 max=8901 max_rank=2 min=7656 min_rank=5 imbalance_ratio=1.086548 max_load_reduction=56635 max_load_reduction_pct=0.864182 moved_to_echo=31341 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2795, 16, 1, 91, 109, 317, 121, 1, 18, 395, 1076, 180, 1542, 93, 337, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 2325, 5, 21, 792, 9, 26, 808, 546, 37, 12, 0, 2773, 120, 22, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 2406, 142, 1, 149, 1035, 4, 73, 423, 23, 215, 2404, 3, 19, 16, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 83, 3, 5, 8, 373, 3, 9, 166, 16, 25, 185, 250, 11, 223, 59, 3916, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 11, 1, 0, 1117, 1, 21, 706, 1030, 12, 1788, 843, 461, 7, 372, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 101, 46, 1, 68, 1638, 8, 67, 154, 4, 13, 71, 303, 19, 3553, 1433, 1059, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 388, 54, 892, 26, 10, 48, 20, 194, 112, 366, 4055, 11, 179, 438, 44, 885, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 981, 0, 52, 0, 124, 478, 404, 119, 5, 3770, 301, 27, 52, 2, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7482, 8035, 8207, 7560, 8462, 8883, 8566, 8341] avg=8192.0000 max=8883 max_rank=5 min=7482 min_rank=0 imbalance_ratio=1.084351 max_load_reduction=56653 max_load_reduction_pct=0.864456 moved_to_echo=34252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3133, 69, 3, 130, 119, 479, 86, 0, 8, 333, 1220, 119, 1743, 282, 95, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 1352, 33, 27, 829, 3, 14, 1769, 1431, 35, 3, 0, 2313, 114, 51, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 1573, 172, 51, 27, 362, 11, 129, 1103, 2, 796, 2249, 95, 31, 17, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 90, 5, 11, 2, 312, 46, 7, 182, 17, 15, 457, 369, 13, 97, 279, 4097, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 27, 1, 0, 807, 0, 9, 1092, 1515, 32, 1922, 225, 624, 11, 172, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1, 4, 21, 15, 0, 161, 1396, 3, 55, 193, 7, 51, 148, 376, 6, 3392, 1442, 1067, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 233, 13, 321, 102, 4, 51, 26, 411, 188, 408, 3554, 19, 147, 690, 97, 879, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 599, 0, 95, 0, 124, 706, 292, 48, 18, 3748, 297, 58, 43, 32, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8525, 7369, 8192, 8186, 8747, 8322, 7950] avg=8192.0000 max=8747 max_rank=5 min=7369 min_rank=2 imbalance_ratio=1.067749 max_load_reduction=56789 max_load_reduction_pct=0.866531 moved_to_echo=33205 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 + [2026-08-26 16:27:12] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 663.0 | throughput per GPU (TFLOP/s/GPU): 272.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117604E+01 | loss scale: 1.0 | grad norm: 44.306 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3816, 65, 1, 126, 142, 356, 58, 0, 15, 407, 1517, 118, 1618, 144, 157, 461, 166, 1657, 29, 12, 993, 24, 18, 605, 637, 38, 1, 4004, 1766, 3406, 111, 266, 925, 1804, 164, 16, 90, 1115, 23, 131, 1220, 2, 531, 2694, 0, 45, 29, 4081, 8, 42, 4, 14, 1, 188, 16, 8, 238, 17, 11, 204, 290, 21, 155, 141, 2481, 3, 4, 1, 1571, 2, 23, 764, 1429, 89, 2128, 419, 415, 67, 591, 73, 22, 17, 0, 23, 31, 1, 149, 1612, 4, 82, 204, 39, 107, 67, 163, 2, 163, 274, 14, 406, 60, 9, 317, 87, 163, 451, 235, 4029, 2, 276, 384, 104, 217, 763, 0, 38, 0, 364, 379, 64, 69, 24, 4778, 320, 86, 46, 4, 1865] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3519, 71, 4, 228, 224, 623, 117, 3, 27, 503, 1272, 205, 1281, 110, 323, 213, 109, 1668, 1, 28, 869, 13, 51, 1713, 1403, 82, 10, 4586, 2360, 3649, 108, 628, 517, 1043, 187, 4, 86, 926, 11, 63, 1079, 9, 636, 2853, 4, 74, 20, 3612, 46, 147, 2, 22, 9, 142, 15, 5, 187, 12, 39, 344, 264, 131, 243, 227, 2790, 5, 3, 0, 1212, 0, 30, 474, 937, 24, 1289, 613, 373, 27, 503, 221, 26, 5, 9, 13, 43, 1, 70, 1219, 4, 119, 248, 3, 35, 163, 263, 13, 336, 185, 16, 944, 39, 4, 69, 103, 161, 324, 332, 3371, 2, 300, 483, 100, 375, 953, 1, 125, 0, 244, 548, 142, 31, 25, 4691, 413, 83, 45, 3, 1393] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[4996, 32, 5, 426, 62, 713, 86, 0, 36, 224, 1103, 67, 1549, 60, 227, 435, 115, 2028, 8, 25, 1006, 12, 30, 829, 1061, 43, 0, 4035, 2105, 2764, 32, 418, 1018, 1305, 149, 19, 123, 399, 6, 15, 541, 7, 701, 3853, 1, 19, 32, 3553, 11, 59, 13, 18, 0, 242, 22, 4, 99, 11, 31, 545, 222, 43, 126, 127, 2808, 11, 2, 2, 1493, 0, 41, 608, 1269, 2, 2358, 318, 513, 3, 322, 236, 8, 19, 0, 13, 19, 10, 83, 616, 3, 97, 262, 18, 26, 34, 327, 14, 306, 392, 12, 813, 39, 3, 43, 29, 157, 84, 670, 4038, 21, 105, 308, 34, 191, 910, 1, 118, 0, 202, 351, 228, 104, 25, 4674, 373, 59, 80, 25, 1760] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3759, 4, 1, 196, 77, 464, 133, 0, 30, 378, 2171, 193, 1488, 103, 320, 383, 43, 2078, 11, 36, 757, 3, 51, 1368, 930, 47, 1, 4478, 2108, 2723, 38, 623, 963, 1361, 229, 9, 45, 886, 7, 95, 714, 12, 951, 3250, 10, 77, 77, 3331, 14, 80, 0, 20, 1, 284, 24, 10, 49, 9, 23, 201, 378, 7, 258, 394, 2818, 3, 2, 0, 890, 5, 32, 822, 1064, 23, 2356, 275, 440, 4, 394, 151, 12, 12, 0, 27, 45, 0, 107, 1370, 8, 65, 163, 6, 101, 197, 146, 33, 278, 202, 11, 734, 53, 11, 113, 40, 144, 164, 358, 3457, 9, 169, 380, 111, 137, 890, 4, 26, 1, 223, 516, 230, 71, 9, 4532, 490, 47, 58, 11, 1722] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[4025, 21, 1, 95, 178, 591, 209, 0, 1, 527, 1527, 252, 1294, 143, 216, 742, 119, 1902, 4, 7, 1277, 0, 40, 662, 1283, 24, 1, 4142, 2231, 3235, 176, 369, 750, 1060, 149, 2, 41, 1191, 24, 38, 528, 4, 387, 2287, 0, 26, 36, 4190, 8, 115, 3, 10, 3, 89, 5, 3, 200, 11, 12, 190, 266, 7, 258, 318, 2534, 5, 0, 0, 986, 2, 30, 980, 956, 8, 2401, 395, 597, 0, 502, 281, 8, 0, 0, 15, 26, 0, 68, 1518, 10, 48, 75, 1, 18, 219, 203, 7, 123, 370, 35, 418, 146, 2, 67, 6, 226, 155, 403, 4197, 3, 140, 366, 63, 95, 816, 0, 35, 1, 169, 905, 137, 47, 26, 5014, 398, 52, 59, 9, 1655] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4187, 30, 0, 255, 87, 687, 65, 0, 40, 413, 593, 188, 1496, 63, 213, 301, 48, 2025, 7, 84, 1107, 7, 31, 1434, 1149, 53, 6, 4758, 2374, 3597, 86, 620, 824, 1371, 244, 42, 135, 519, 9, 71, 1082, 12, 884, 3670, 0, 18, 39, 3525, 18, 46, 1, 7, 15, 257, 6, 11, 164, 46, 34, 204, 266, 111, 336, 186, 2544, 7, 0, 1, 1215, 10, 53, 395, 943, 16, 2712, 152, 524, 15, 362, 214, 26, 2, 0, 58, 31, 0, 115, 486, 8, 72, 385, 20, 134, 73, 96, 19, 161, 118, 22, 458, 33, 14, 141, 30, 127, 195, 424, 3282, 10, 158, 349, 74, 189, 915, 2, 83, 1, 198, 418, 91, 107, 61, 4842, 388, 83, 99, 16, 1602] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4174, 70, 2, 130, 117, 482, 83, 0, 8, 333, 1214, 120, 1743, 284, 95, 428, 58, 1350, 34, 27, 831, 3, 14, 1755, 1430, 35, 3, 4102, 2334, 3505, 48, 491, 544, 1568, 173, 52, 29, 359, 10, 129, 1101, 2, 803, 2255, 95, 31, 18, 3474, 14, 91, 6, 11, 1, 309, 46, 7, 186, 17, 16, 458, 368, 13, 95, 278, 2966, 28, 1, 0, 807, 0, 9, 1092, 1502, 32, 1924, 223, 622, 11, 176, 217, 32, 1, 4, 20, 14, 0, 160, 1401, 3, 52, 189, 7, 51, 145, 377, 7, 497, 233, 13, 320, 101, 4, 51, 26, 413, 190, 414, 3566, 20, 148, 694, 98, 144, 600, 0, 92, 0, 126, 700, 294, 48, 20, 4621, 297, 59, 44, 33, 1735] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3729, 18, 1, 92, 108, 315, 119, 1, 19, 391, 1075, 175, 1540, 91, 338, 398, 66, 2331, 5, 20, 792, 9, 27, 804, 540, 39, 12, 3918, 2773, 3666, 22, 472, 1102, 2402, 145, 1, 149, 1038, 5, 74, 422, 23, 216, 2406, 4, 20, 16, 3433, 61, 81, 3, 4, 8, 374, 3, 8, 166, 16, 25, 185, 249, 11, 223, 54, 2940, 11, 1, 0, 1134, 1, 21, 708, 1024, 13, 1792, 837, 465, 7, 373, 574, 3, 2, 0, 104, 48, 1, 68, 1646, 7, 66, 157, 4, 12, 70, 308, 19, 235, 390, 50, 885, 26, 10, 46, 20, 198, 113, 374, 4060, 11, 177, 438, 44, 64, 969, 0, 55, 0, 125, 465, 401, 116, 4, 4661, 301, 27, 52, 2, 1968] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2864, 65, 1, 126, 142, 356, 58, 0, 15, 407, 1517, 118, 1618, 144, 157, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 1657, 29, 12, 993, 24, 18, 605, 637, 38, 1, 0, 1766, 118, 111, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 925, 1804, 164, 16, 90, 1115, 23, 131, 1220, 2, 531, 2694, 0, 45, 29, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 42, 4, 14, 1, 188, 16, 8, 238, 17, 11, 204, 290, 21, 155, 141, 4004, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 3, 4, 1, 1571, 2, 23, 764, 1429, 89, 2128, 419, 415, 67, 591, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 17, 0, 23, 31, 1, 149, 1612, 4, 82, 204, 39, 107, 67, 163, 2, 3288, 1206, 1267, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 274, 14, 406, 60, 9, 317, 87, 163, 451, 235, 4029, 2, 276, 384, 104, 904, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 763, 0, 38, 0, 364, 379, 64, 69, 24, 3874, 320, 86, 46, 4, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 6441, 9032, 7933, 8854, 8624, 8490, 8113] avg=8192.0000 max=9032 max_rank=2 min=6441 min_rank=1 imbalance_ratio=1.102539 max_load_reduction=56504 max_load_reduction_pct=0.862183 moved_to_echo=34081 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2641, 71, 4, 228, 224, 623, 117, 3, 27, 503, 1272, 205, 1281, 110, 323, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1668, 1, 28, 869, 13, 51, 1713, 1403, 82, 10, 0, 2360, 126, 108, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 1043, 187, 4, 86, 926, 11, 63, 1079, 9, 636, 2853, 4, 74, 20, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 147, 2, 22, 9, 142, 15, 5, 187, 12, 39, 344, 264, 131, 243, 227, 4586, 2276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1434, 5, 3, 0, 1212, 0, 30, 474, 937, 24, 1289, 613, 373, 27, 503, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 5, 9, 13, 43, 1, 70, 1219, 4, 119, 248, 3, 35, 163, 263, 13, 3523, 1356, 1122, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 185, 16, 944, 39, 4, 69, 103, 161, 324, 332, 3371, 2, 300, 483, 100, 887, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 953, 1, 125, 0, 244, 548, 142, 31, 25, 3804, 413, 83, 45, 3, 1393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7845, 9169, 7726, 8697, 7145, 8549, 8220, 8185] avg=8192.0000 max=9169 max_rank=1 min=7145 min_rank=4 imbalance_ratio=1.119263 max_load_reduction=56367 max_load_reduction_pct=0.860092 moved_to_echo=32099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3021, 21, 1, 95, 178, 591, 209, 0, 1, 527, 1527, 252, 1294, 143, 216, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 1902, 4, 7, 1277, 0, 40, 662, 1283, 24, 1, 0, 2231, 112, 176, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 750, 1060, 149, 2, 41, 1191, 24, 38, 528, 4, 387, 2287, 0, 26, 36, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 115, 3, 10, 3, 89, 5, 3, 200, 11, 12, 190, 266, 7, 258, 318, 4142, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 5, 0, 0, 986, 2, 30, 980, 956, 8, 2401, 395, 597, 0, 502, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 15, 26, 0, 68, 1518, 10, 48, 75, 1, 18, 219, 203, 7, 3123, 1232, 1301, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 370, 35, 418, 146, 2, 67, 6, 226, 155, 403, 4197, 3, 140, 366, 63, 949, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 816, 0, 35, 1, 169, 905, 137, 47, 26, 4065, 398, 52, 59, 9, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8818, 8207, 6772, 8280, 8445, 8231, 8314, 8469] avg=8192.0000 max=8818 max_rank=0 min=6772 min_rank=2 imbalance_ratio=1.076416 max_load_reduction=56718 max_load_reduction_pct=0.865448 moved_to_echo=33459 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2821, 4, 1, 196, 77, 464, 133, 0, 30, 378, 2171, 193, 1488, 103, 320, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 2078, 11, 36, 757, 3, 51, 1368, 930, 47, 1, 0, 2108, 94, 38, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1361, 229, 9, 45, 886, 7, 95, 714, 12, 951, 3250, 10, 77, 77, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 80, 0, 20, 1, 284, 24, 10, 49, 9, 23, 201, 378, 7, 258, 394, 4478, 2099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1448, 3, 2, 0, 890, 5, 32, 822, 1064, 23, 2356, 275, 440, 4, 394, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 27, 45, 0, 107, 1370, 8, 65, 163, 6, 101, 197, 146, 33, 2629, 1370, 1034, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 202, 11, 734, 53, 11, 113, 40, 144, 164, 358, 3457, 9, 169, 380, 111, 857, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 890, 4, 26, 1, 223, 516, 230, 71, 9, 3675, 490, 47, 58, 11, 1722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8762, 8188, 8884, 8329, 7909, 7660, 7694, 8110] avg=8192.0000 max=8884 max_rank=2 min=7660 min_rank=5 imbalance_ratio=1.084473 max_load_reduction=56652 max_load_reduction_pct=0.864441 moved_to_echo=31373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3741, 32, 5, 426, 62, 713, 86, 0, 36, 224, 1103, 67, 1549, 60, 227, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 2028, 8, 25, 1006, 12, 30, 829, 1061, 43, 0, 0, 2105, 91, 32, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1305, 149, 19, 123, 399, 6, 15, 541, 7, 701, 3853, 1, 19, 32, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 59, 13, 18, 0, 242, 22, 4, 99, 11, 31, 545, 222, 43, 126, 127, 4035, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 11, 2, 2, 1493, 0, 41, 608, 1269, 2, 2358, 318, 513, 3, 322, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 19, 0, 13, 19, 10, 83, 616, 3, 97, 262, 18, 26, 34, 327, 14, 2673, 1369, 1107, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 392, 12, 813, 39, 3, 43, 29, 157, 84, 670, 4038, 21, 105, 308, 34, 888, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 910, 1, 118, 0, 202, 351, 228, 104, 25, 3786, 373, 59, 80, 25, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8766, 7803, 8391, 7851, 8617, 7148, 8747, 8213] avg=8192.0000 max=8766 max_rank=0 min=7148 min_rank=5 imbalance_ratio=1.070068 max_load_reduction=56770 max_load_reduction_pct=0.866241 moved_to_echo=32725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3133, 70, 2, 130, 117, 482, 83, 0, 8, 333, 1214, 120, 1743, 284, 95, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1350, 34, 27, 831, 3, 14, 1755, 1430, 35, 3, 0, 2334, 121, 48, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 1568, 173, 52, 29, 359, 10, 129, 1101, 2, 803, 2255, 95, 31, 18, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 91, 6, 11, 1, 309, 46, 7, 186, 17, 16, 458, 368, 13, 95, 278, 4102, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1524, 28, 1, 0, 807, 0, 9, 1092, 1502, 32, 1924, 223, 622, 11, 176, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1, 4, 20, 14, 0, 160, 1401, 3, 52, 189, 7, 51, 145, 377, 7, 3384, 1442, 1079, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 233, 13, 320, 101, 4, 51, 26, 413, 190, 414, 3566, 20, 148, 694, 98, 874, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 600, 0, 92, 0, 126, 700, 294, 48, 20, 3747, 297, 59, 44, 33, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8534, 7375, 8207, 8168, 8740, 8331, 7939] avg=8192.0000 max=8740 max_rank=5 min=7375 min_rank=2 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=33178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3143, 30, 0, 255, 87, 687, 65, 0, 40, 413, 593, 188, 1496, 63, 213, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 2025, 7, 84, 1107, 7, 31, 1434, 1149, 53, 6, 0, 2374, 124, 86, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 1371, 244, 42, 135, 519, 9, 71, 1082, 12, 884, 3670, 0, 18, 39, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 46, 1, 7, 15, 257, 6, 11, 164, 46, 34, 204, 266, 111, 336, 186, 4758, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1307, 7, 0, 1, 1215, 10, 53, 395, 943, 16, 2712, 152, 524, 15, 362, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2, 0, 58, 31, 0, 115, 486, 8, 72, 385, 20, 134, 73, 96, 19, 3473, 1237, 1095, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 118, 22, 458, 33, 14, 141, 30, 127, 195, 424, 3282, 10, 158, 349, 74, 916, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 915, 2, 83, 1, 198, 418, 91, 107, 61, 3926, 388, 83, 99, 16, 1602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7574, 9155, 9129, 8687, 7926, 7703, 7183, 8179] avg=8192.0000 max=9155 max_rank=1 min=7183 min_rank=6 imbalance_ratio=1.117554 max_load_reduction=56381 max_load_reduction_pct=0.860306 moved_to_echo=30991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2799, 18, 1, 92, 108, 315, 119, 1, 19, 391, 1075, 175, 1540, 91, 338, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 2331, 5, 20, 792, 9, 27, 804, 540, 39, 12, 0, 2773, 127, 22, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 2402, 145, 1, 149, 1038, 5, 74, 422, 23, 216, 2406, 4, 20, 16, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 81, 3, 4, 8, 374, 3, 8, 166, 16, 25, 185, 249, 11, 223, 54, 3918, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 11, 1, 0, 1134, 1, 21, 708, 1024, 13, 1792, 837, 465, 7, 373, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 104, 48, 1, 68, 1646, 7, 66, 157, 4, 12, 70, 308, 19, 3539, 1429, 1066, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 390, 50, 885, 26, 10, 46, 20, 198, 113, 374, 4060, 11, 177, 438, 44, 882, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 969, 0, 55, 0, 125, 465, 401, 116, 4, 3779, 301, 27, 52, 2, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7480, 8039, 8227, 7552, 8472, 8881, 8557, 8328] avg=8192.0000 max=8881 max_rank=5 min=7480 min_rank=0 imbalance_ratio=1.084106 max_load_reduction=56655 max_load_reduction_pct=0.864487 moved_to_echo=34238 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 15104, 128, 256, 7680, 128, 384, 9216, 8576, 384, 128, 0, 18048, 896, 640, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 384, 256, 128, 896, 9856, 128, 640, 1792, 128, 512, 1024, 1920, 128, 25728, 10752, 9088, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[11392, 128, 128, 128, 9344, 128, 256, 5888, 9216, 256, 16896, 3328, 3968, 256, 3328, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[24192, 384, 128, 1664, 1024, 4224, 896, 128, 256, 3200, 10496, 1408, 12160, 1024, 1920, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 2176, 256, 4992, 512, 128, 896, 384, 1664, 1664, 3200, 29952, 128, 1536, 3456, 640, 7296, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[256, 768, 128, 128, 128, 1920, 256, 128, 1408, 256, 256, 2432, 2432, 384, 1792, 1792, 34048, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[256, 128, 128, 384, 384, 128, 896, 9984, 128, 640, 1792, 128, 512, 1024, 1920, 128, 25728, 10752, 9088, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[24192, 384, 128, 1664, 1024, 4352, 896, 128, 256, 3200, 10496, 1408, 12032, 1024, 1920, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[11264, 128, 128, 128, 9344, 128, 256, 5888, 9216, 256, 17024, 3328, 3968, 256, 3328, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[6656, 12032, 1536, 256, 768, 6528, 128, 640, 6784, 128, 5120, 23296, 128, 384, 384, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[1408, 6912, 128, 640, 128, 1664, 4352, 1664, 640, 256, 30720, 3072, 512, 512, 128, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[2176, 2176, 256, 4992, 512, 128, 896, 384, 1664, 1792, 3328, 30080, 128, 1536, 3456, 640, 7168, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[768, 15104, 128, 256, 7680, 128, 384, 9216, 8448, 384, 128, 0, 18176, 1024, 640, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 768, 128, 128, 128, 1920, 256, 128, 1408, 256, 256, 2432, 2304, 384, 1792, 1792, 34048, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1536, 6912, 128, 640, 128, 1664, 4352, 1664, 640, 256, 30720, 3072, 512, 512, 128, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 0, 4, 3, 1, 0, 21, 0, 0, 0, 0, 1, 0, 7988, 0, 0, 0, 0, 0, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4, 0, 0, 0, 7, 7, 0, 0, 3, 0, 2, 1, 102, 0, 15, 0, 1, 0, 0, 34, 0, 1, 0, 7496, 1, 0, 0, 0, 11, 0, 16, 0, 0, 0, 0, 1, 0, 784, 0, 0, 25, 8151, 4, 0, 1, 0, 7747, 0, 8174, 1, 0, 504, 0, 0, 0, 517, 114, 8144, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4678, 0, 0, 0, 8173, 32, 61, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 20, 1, 0, 0, 3, 1, 0, 0, 7, 0, 0, 8143, 0, 0, 1, 6, 0, 7122, 0, 22, 15, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 102, 0, 0, 0, 0, 0, 0, 5, 0, 0, 236, 34, 4, 0, 0, 0, 0, 5, 0, 434, 0, 83, 366, 5390, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 3, 0, 1, 402, 0, 0, 4, 7978, 0, 0, 0, 0, 7519, 12, 8192, 0, 3, 27, 0, 236, 0, 2432, 0, 7783, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 56, 4, 2, 639, 0, 0, 0, 8177, 2, 28, 11, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[11, 0, 0, 1, 0, 24, 2, 0, 7, 13, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 6, 3293, 2, 0, 0, 0, 0, 0, 7, 0, 14, 0, 6, 0, 0, 0, 7, 0, 1, 9, 0, 0, 0, 0, 61, 0, 1, 0, 0, 0, 1, 178, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 6124, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 6, 0, 784, 0, 0, 1395, 8074, 0, 0, 0, 0, 8155, 0, 8170, 0, 1, 112, 18, 0, 0, 1084, 2, 6785, 0, 1, 0, 0, 0, 11, 0, 0, 1, 2, 187, 25, 0, 15, 0, 2, 0, 238, 0, 197, 4153, 0, 0, 0, 8190, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 6, 0, 5, 7933, 14, 0, 0, 0, 0, 8034, 0, 701, 17, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 1, 2, 7, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 6, 57, 9, 0, 0, 0, 0, 0, 4, 0, 1, 33, 0, 3705, 9, 0, 0, 0, 0, 1, 28, 1, 0, 0, 0, 0, 1, 11, 1, 0, 9, 7892, 2, 21, 120, 0, 8167, 0, 8179, 0, 11, 33, 9, 44, 3, 4567, 10, 6663, 0, 1, 0, 2, 0, 4, 0, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 629, 1, 0, 1438, 0, 0, 1, 7108, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[7, 1, 0, 0, 0, 1, 0, 0, 78, 177, 0, 0, 0, 280, 10, 0, 7848, 3, 0, 0, 0, 0, 7324, 0, 2, 0, 0, 0, 5, 0, 31, 0, 0, 0, 13, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 37, 110, 1, 0, 787, 0, 26, 243, 0, 1, 0, 157, 0, 1, 0, 1, 0, 2, 5625, 0, 1, 0, 0, 3, 0, 32, 0, 0, 0, 20, 0, 0, 0, 0, 3, 454, 5641, 1, 3, 1, 1, 8107, 0, 8121, 0, 5, 1, 1, 4, 0, 12, 0, 5144, 2, 0, 0, 23, 0, 0, 0, 1, 0, 0, 38, 0, 0, 0, 2, 0, 3, 33, 0, 0, 8138, 0, 1, 0, 6956, 2, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 5, 7932, 1, 0, 0, 0, 0, 4709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 2, 3, 333, 0, 2, 1, 66, 0, 0, 0, 0, 18, 37, 8, 1, 14, 0, 0, 0, 0, 6150, 6, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1299, 0, 0, 83, 8088, 0, 0, 1, 2, 8086, 0, 8158, 0, 0, 31, 0, 0, 0, 2043, 0, 8189, 0, 2, 9, 4, 0, 7, 0, 0, 0, 2, 3, 4, 0, 0, 0, 3, 0, 0, 2, 78, 1998, 11, 0, 0, 8051, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 0, 225, 1, 2, 2, 0, 0, 0, 0, 0, 7885, 166, 0, 11, 0, 0, 7350, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 1, 70, 0, 0, 0, 91, 0, 0, 2, 0, 0, 1, 8, 0, 9, 0, 0, 0, 0, 4450, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 29, 0, 0, 578, 8034, 0, 0, 1, 0, 8151, 149, 8190, 0, 1, 70, 0, 12, 0, 563, 0, 3615, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 1, 0, 1, 7638, 0, 0, 0, 8185, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 16, 23, 7, 0, 0, 11, 0, 0, 0, 0, 0, 7613, 0, 0, 1, 0, 0, 7327, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 1, 13, 2, 0, 14, 25, 0, 0, 124, 0, 0, 1, 339, 0, 69, 0, 0, 0, 0, 7809, 0, 0, 0, 15, 0, 0, 0, 3, 0, 0, 24, 17, 0, 3, 0, 0, 75, 7987, 0, 0, 0, 0, 5341, 10, 8160, 0, 1172, 0, 0, 0, 0, 83, 0, 8048, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 1, 3133, 0, 0, 0, 7671, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 0, 4, 3, 1, 0, 21, 0, 0, 0, 0, 1, 0, 8151, 238, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2102, 0, 0, 0, 0, 0, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4, 0, 0, 0, 7, 7, 0, 0, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 1, 102, 0, 15, 0, 1, 0, 0, 34, 0, 1, 0, 7496, 1756, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 16, 0, 0, 0, 0, 1, 0, 784, 0, 0, 5648, 2234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4, 0, 1, 0, 6195, 0, 0, 1, 0, 504, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 8144, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4678, 0, 0, 0, 4183, 32, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8521, 4740, 8116, 9624, 8695, 7247, 9639, 8954] avg=8192.0000 max=9639 max_rank=6 min=4740 min_rank=1 imbalance_ratio=1.176636 max_load_reduction=55897 max_load_reduction_pct=0.852921 moved_to_echo=34535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 20, 1, 0, 0, 3, 1, 0, 0, 7, 0, 0, 7978, 243, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2142, 0, 0, 1, 6, 0, 7122, 0, 22, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 102, 0, 0, 0, 0, 0, 0, 5, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 34, 4, 0, 0, 0, 0, 5, 0, 434, 0, 83, 366, 5390, 1757, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 3, 0, 1, 402, 0, 0, 5758, 2235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6012, 12, 0, 0, 3, 27, 0, 236, 0, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7783, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 56, 4, 2, 639, 0, 0, 0, 4185, 2, 28, 11, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 9309, 8215, 8516, 8403, 8726, 9088, 4930] avg=8192.0000 max=9309 max_rank=1 min=4930 min_rank=7 imbalance_ratio=1.136353 max_load_reduction=56227 max_load_reduction_pct=0.857956 moved_to_echo=31147 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[7, 1, 0, 0, 0, 1, 0, 0, 78, 177, 0, 0, 0, 280, 10, 0, 5641, 234, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 3, 0, 0, 0, 0, 7324, 0, 2, 0, 0, 0, 5, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 37, 110, 1, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 26, 243, 0, 1, 0, 157, 0, 1, 0, 1, 0, 2, 5625, 1495, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 32, 0, 0, 0, 20, 0, 0, 0, 0, 3, 5549, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 1, 3, 1, 1, 6482, 0, 0, 0, 5, 1, 1, 4, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5144, 2, 0, 0, 23, 0, 0, 0, 1, 0, 0, 38, 0, 0, 0, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 33, 0, 0, 8138, 0, 1, 0, 3560, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6521, 9430, 8200, 8561, 7509, 6965, 6610, 11740] avg=8192.0000 max=11740 max_rank=7 min=6521 min_rank=0 imbalance_ratio=1.433105 max_load_reduction=53796 max_load_reduction_pct=0.820862 moved_to_echo=32824 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 6, 0, 5, 7892, 239, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2080, 14, 0, 0, 0, 0, 8034, 0, 701, 17, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 2, 7, 1, 1, 0, 0, 0, 0, 0, 2, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 57, 9, 0, 0, 0, 0, 0, 4, 0, 1, 33, 0, 3705, 1531, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 28, 1, 0, 0, 0, 0, 1, 11, 1, 0, 5614, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 21, 120, 0, 6521, 0, 0, 0, 11, 33, 9, 44, 3, 4567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 6663, 0, 1, 0, 2, 0, 4, 0, 0, 0, 0, 8, 1, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 1, 0, 1438, 0, 0, 1, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 10849, 8102, 5575, 7613, 11340, 8106, 5699] avg=8192.0000 max=11340 max_rank=5 min=5575 min_rank=3 imbalance_ratio=1.384277 max_load_reduction=54196 max_load_reduction_pct=0.826965 moved_to_echo=32758 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 1, 0, 24, 2, 0, 7, 13, 0, 0, 0, 0, 0, 0, 8074, 243, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 0, 0, 0, 0, 6, 3293, 2, 0, 0, 0, 0, 0, 7, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 7, 0, 1, 9, 0, 0, 0, 0, 61, 0, 1, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 178, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 6124, 1760, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 6, 0, 784, 0, 0, 5762, 2239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1395, 0, 0, 0, 0, 0, 6520, 0, 0, 0, 1, 112, 18, 0, 0, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6785, 0, 1, 0, 0, 0, 11, 0, 0, 1, 2, 187, 25, 0, 15, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 238, 0, 197, 4153, 0, 0, 0, 4191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8467, 5466, 8163, 8292, 8798, 9130, 8439, 8781] avg=8192.0000 max=9130 max_rank=5 min=5466 min_rank=1 imbalance_ratio=1.114502 max_load_reduction=56406 max_load_reduction_pct=0.860687 moved_to_echo=35148 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 0, 225, 1, 2, 2, 0, 0, 0, 0, 0, 8034, 235, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2075, 166, 0, 11, 0, 0, 7350, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 1, 70, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 2, 0, 0, 1, 8, 0, 9, 0, 0, 0, 0, 4450, 1759, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 29, 0, 0, 5575, 2237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 1, 0, 6518, 149, 0, 0, 1, 70, 0, 12, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3615, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7638, 0, 0, 0, 4189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8614, 9603, 8177, 6544, 7845, 7892, 5032, 11829] avg=8192.0000 max=11829 max_rank=7 min=5032 min_rank=6 imbalance_ratio=1.443970 max_load_reduction=53707 max_load_reduction_pct=0.819504 moved_to_echo=32598 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 5, 8088, 236, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2087, 1, 0, 0, 0, 0, 4709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 2, 3, 333, 0, 2, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 66, 0, 0, 0, 0, 18, 37, 8, 1, 14, 0, 0, 0, 0, 6150, 1730, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1299, 0, 0, 5609, 2201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 1, 2, 6465, 0, 0, 0, 0, 31, 0, 0, 0, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 2, 9, 4, 0, 7, 0, 0, 0, 2, 3, 4, 0, 0, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 78, 1998, 11, 0, 0, 4120, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8430, 6797, 8469, 8248, 9131, 8625, 9618, 6218] avg=8192.0000 max=9618 max_rank=6 min=6218 min_rank=7 imbalance_ratio=1.174072 max_load_reduction=55918 max_load_reduction_pct=0.853241 moved_to_echo=33592 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 16, 23, 7, 0, 0, 11, 0, 0, 0, 0, 0, 7987, 227, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 0, 1, 0, 0, 7327, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 1, 13, 2, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 25, 0, 0, 124, 0, 0, 1, 339, 0, 69, 0, 0, 0, 0, 7809, 1648, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 3, 0, 0, 24, 17, 0, 3, 0, 0, 5383, 2097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 4271, 10, 0, 0, 1172, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 1, 3133, 0, 0, 0, 3926, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8382, 9342, 8094, 10176, 7542, 5611, 9004, 7385] avg=8192.0000 max=10176 max_rank=3 min=5611 min_rank=5 imbalance_ratio=1.242188 max_load_reduction=55360 max_load_reduction_pct=0.844727 moved_to_echo=29542 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 20, 1, 0, 0, 3, 1, 0, 0, 8, 0, 0, 8149, 0, 0, 2, 6, 0, 7112, 0, 21, 15, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 104, 0, 0, 0, 0, 0, 0, 4, 0, 0, 228, 35, 4, 0, 0, 0, 0, 5, 0, 416, 0, 85, 363, 5353, 0, 0, 0, 0, 1, 0, 1, 3, 1, 0, 4, 0, 0, 394, 0, 0, 8, 7980, 0, 0, 0, 0, 7542, 12, 8191, 0, 3, 29, 0, 228, 0, 2449, 0, 7795, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 53, 4, 1, 658, 0, 0, 0, 8180, 2, 29, 11, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[11, 0, 0, 1, 0, 24, 2, 0, 6, 12, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 7, 3267, 2, 0, 0, 0, 0, 0, 7, 0, 13, 0, 7, 0, 0, 0, 11, 0, 1, 8, 0, 0, 0, 0, 64, 0, 1, 0, 0, 0, 0, 169, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3, 6099, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7, 0, 784, 0, 0, 1413, 8077, 0, 0, 0, 0, 8154, 0, 8170, 0, 2, 122, 14, 0, 0, 1101, 2, 6788, 0, 1, 0, 0, 0, 10, 0, 0, 1, 3, 188, 26, 0, 14, 0, 2, 0, 243, 0, 205, 4151, 0, 0, 0, 8189, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 24, 0, 227, 1, 2, 2, 0, 0, 0, 0, 0, 7874, 160, 0, 11, 0, 0, 7362, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6, 0, 1, 70, 0, 0, 0, 85, 0, 0, 2, 0, 0, 1, 8, 0, 10, 0, 0, 0, 0, 4416, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 28, 0, 0, 589, 8035, 0, 0, 1, 0, 8149, 151, 8190, 0, 1, 73, 0, 11, 0, 586, 0, 3615, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 1, 7643, 0, 0, 0, 8186, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 19, 0, 15, 22, 6, 0, 0, 12, 0, 0, 0, 0, 0, 7600, 0, 0, 1, 0, 0, 7329, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 1, 13, 2, 0, 14, 27, 0, 0, 127, 0, 0, 0, 335, 0, 73, 0, 0, 0, 0, 7805, 0, 0, 0, 14, 0, 0, 0, 3, 0, 0, 22, 17, 0, 3, 0, 0, 72, 7984, 0, 1, 0, 0, 5350, 11, 8160, 0, 1168, 0, 0, 0, 0, 73, 0, 8046, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 1, 3150, 0, 0, 0, 7674, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 0, 4, 4, 1, 0, 17, 0, 0, 0, 0, 1, 0, 7982, 0, 0, 0, 0, 0, 2716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 5, 0, 0, 0, 8, 7, 0, 0, 3, 0, 2, 1, 112, 0, 15, 0, 1, 0, 0, 35, 0, 1, 0, 7483, 1, 0, 0, 0, 12, 0, 15, 0, 0, 0, 0, 1, 0, 791, 0, 0, 23, 8150, 3, 0, 1, 0, 7752, 0, 8175, 1, 0, 505, 0, 0, 0, 523, 113, 8144, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 1, 4591, 0, 0, 0, 8174, 33, 59, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8, 1, 0, 0, 0, 1, 0, 0, 77, 182, 0, 0, 0, 271, 10, 0, 7849, 3, 0, 0, 0, 0, 7315, 0, 2, 0, 0, 0, 5, 0, 32, 0, 0, 0, 14, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 37, 111, 1, 0, 782, 0, 28, 249, 0, 0, 0, 156, 0, 1, 0, 1, 1, 2, 5552, 0, 1, 0, 0, 3, 0, 32, 0, 0, 0, 18, 0, 0, 0, 0, 2, 455, 5679, 1, 3, 1, 1, 8112, 0, 8113, 0, 5, 1, 1, 3, 0, 11, 0, 5178, 2, 0, 0, 23, 0, 0, 0, 2, 0, 0, 42, 0, 0, 0, 2, 0, 3, 33, 0, 0, 8140, 0, 1, 0, 6967, 2, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 1, 0, 0, 5, 7935, 0, 0, 0, 0, 0, 4707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 2, 3, 330, 0, 2, 0, 68, 0, 0, 1, 0, 19, 35, 8, 1, 15, 0, 0, 0, 0, 6128, 6, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1291, 0, 0, 82, 8090, 0, 0, 2, 2, 8094, 0, 8161, 0, 0, 33, 0, 0, 0, 2051, 0, 8189, 0, 1, 9, 4, 0, 4, 0, 0, 0, 2, 3, 4, 0, 0, 0, 3, 0, 0, 1, 74, 2018, 9, 0, 0, 8055, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 5, 0, 5, 7939, 15, 0, 0, 0, 0, 8039, 0, 683, 18, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 1, 3, 7, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 6, 56, 12, 0, 0, 0, 0, 0, 4, 0, 0, 32, 0, 3647, 9, 0, 0, 0, 0, 1, 25, 1, 0, 0, 0, 0, 1, 12, 1, 0, 9, 7898, 2, 21, 115, 0, 8167, 0, 8178, 0, 13, 34, 7, 45, 3, 4568, 10, 6700, 0, 1, 0, 2, 0, 5, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 651, 1, 0, 1463, 0, 0, 1, 7088, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 1, 0, 24, 2, 0, 6, 12, 0, 0, 0, 0, 0, 0, 8077, 238, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 0, 0, 0, 0, 7, 3267, 2, 0, 0, 0, 0, 0, 7, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 11, 0, 1, 8, 0, 0, 0, 0, 64, 0, 1, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3, 6099, 1768, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7, 0, 784, 0, 0, 5767, 2234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1413, 0, 0, 0, 0, 0, 6503, 0, 0, 0, 2, 122, 14, 0, 0, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6788, 0, 1, 0, 0, 0, 10, 0, 0, 1, 3, 188, 26, 0, 14, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 243, 0, 205, 4151, 0, 0, 0, 4187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8463, 5436, 8170, 8295, 8799, 9155, 8430, 8788] avg=8192.0000 max=9155 max_rank=5 min=5436 min_rank=1 imbalance_ratio=1.117554 max_load_reduction=56381 max_load_reduction_pct=0.860306 moved_to_echo=35172 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 24, 0, 227, 1, 2, 2, 0, 0, 0, 0, 0, 8035, 230, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2069, 160, 0, 11, 0, 0, 7362, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6, 0, 1, 70, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 2, 0, 0, 1, 8, 0, 10, 0, 0, 0, 0, 4416, 1767, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 28, 0, 0, 5575, 2233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 1, 0, 6499, 151, 0, 0, 1, 73, 0, 11, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3615, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7643, 0, 0, 0, 4186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8614, 9603, 8176, 6543, 7840, 7911, 5018, 11831] avg=8192.0000 max=11831 max_rank=7 min=5018 min_rank=6 imbalance_ratio=1.444214 max_load_reduction=53705 max_load_reduction_pct=0.819473 moved_to_echo=32600 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 20, 1, 0, 0, 3, 1, 0, 0, 8, 0, 0, 7980, 238, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2141, 0, 0, 2, 6, 0, 7112, 0, 21, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 104, 0, 0, 0, 0, 0, 0, 4, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 35, 4, 0, 0, 0, 0, 5, 0, 416, 0, 85, 363, 5353, 1766, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3, 1, 0, 4, 0, 0, 394, 0, 0, 5770, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 6015, 12, 0, 0, 3, 29, 0, 228, 0, 2449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7795, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 53, 4, 1, 658, 0, 0, 0, 4182, 2, 29, 11, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8347, 9298, 8215, 8490, 8406, 8744, 9093, 4943] avg=8192.0000 max=9298 max_rank=1 min=4943 min_rank=7 imbalance_ratio=1.135010 max_load_reduction=56238 max_load_reduction_pct=0.858124 moved_to_echo=31186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 0, 4, 4, 1, 0, 17, 0, 0, 0, 0, 1, 0, 8150, 233, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 0, 0, 0, 0, 0, 2716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 5, 0, 0, 0, 8, 7, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 1, 112, 0, 15, 0, 1, 0, 0, 35, 0, 1, 0, 7483, 1764, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 12, 0, 15, 0, 0, 0, 0, 1, 0, 791, 0, 0, 5652, 2230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3, 0, 1, 0, 6182, 0, 0, 1, 0, 505, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 8144, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4591, 0, 0, 0, 4180, 33, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8512, 4813, 8121, 9659, 8702, 7238, 9627, 8864] avg=8192.0000 max=9659 max_rank=3 min=4813 min_rank=1 imbalance_ratio=1.179077 max_load_reduction=55877 max_load_reduction_pct=0.852615 moved_to_echo=34431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 0, 0, 0, 1, 0, 0, 77, 182, 0, 0, 0, 271, 10, 0, 5679, 229, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 3, 0, 0, 0, 0, 7315, 0, 2, 0, 0, 0, 5, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 37, 111, 1, 8020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 28, 249, 0, 0, 0, 156, 0, 1, 0, 1, 1, 2, 5552, 1504, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 32, 0, 0, 0, 18, 0, 0, 0, 0, 2, 5557, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 1, 3, 1, 1, 6469, 0, 0, 0, 5, 1, 1, 3, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5178, 2, 0, 0, 23, 0, 0, 0, 2, 0, 0, 42, 0, 0, 0, 1390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 33, 0, 0, 8140, 0, 1, 0, 3562, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6551, 9420, 8190, 8529, 7514, 6951, 6637, 11744] avg=8192.0000 max=11744 max_rank=7 min=6551 min_rank=0 imbalance_ratio=1.433594 max_load_reduction=53792 max_load_reduction_pct=0.820801 moved_to_echo=32846 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 19, 0, 15, 22, 6, 0, 0, 12, 0, 0, 0, 0, 0, 7984, 222, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1997, 0, 0, 1, 0, 0, 7329, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 1, 13, 2, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 27, 0, 0, 127, 0, 0, 0, 335, 0, 73, 0, 0, 0, 0, 7805, 1656, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 3, 0, 0, 22, 17, 0, 3, 0, 0, 5381, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 1, 0, 0, 4266, 11, 0, 0, 1168, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 1, 3150, 0, 0, 0, 3924, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8373, 9338, 8095, 10204, 7534, 5591, 8996, 7405] avg=8192.0000 max=10204 max_rank=3 min=5591 min_rank=5 imbalance_ratio=1.245605 max_load_reduction=55332 max_load_reduction_pct=0.844299 moved_to_echo=29526 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 5, 0, 5, 7898, 236, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2078, 15, 0, 0, 0, 0, 8039, 0, 683, 18, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 3, 7, 1, 1, 0, 0, 0, 0, 0, 2, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 56, 12, 0, 0, 0, 0, 0, 4, 0, 0, 32, 0, 3647, 1534, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 25, 1, 0, 0, 0, 0, 1, 12, 1, 0, 5625, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 21, 115, 0, 6504, 0, 0, 0, 13, 34, 7, 45, 3, 4568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 6700, 0, 1, 0, 2, 0, 5, 0, 0, 0, 0, 9, 1, 0, 0, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 1, 0, 1463, 0, 0, 1, 3616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 10836, 8102, 5550, 7613, 11321, 8132, 5732] avg=8192.0000 max=11321 max_rank=5 min=5550 min_rank=3 imbalance_ratio=1.381958 max_load_reduction=54215 max_load_reduction_pct=0.827255 moved_to_echo=32798 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 1, 0, 0, 5, 8090, 232, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2085, 0, 0, 0, 0, 0, 4707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 2, 3, 330, 0, 2, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 1, 0, 19, 35, 8, 1, 15, 0, 0, 0, 0, 6128, 1739, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1291, 0, 0, 5618, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 2, 2, 6455, 0, 0, 0, 0, 33, 0, 0, 0, 2051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 9, 4, 0, 4, 0, 0, 0, 2, 3, 4, 0, 0, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 74, 2018, 9, 0, 0, 4118, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8426, 6792, 8467, 8266, 9128, 8625, 9603, 6229] avg=8192.0000 max=9603 max_rank=6 min=6229 min_rank=7 imbalance_ratio=1.172241 max_load_reduction=55933 max_load_reduction_pct=0.853470 moved_to_echo=33585 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 3328, 128, 128, 44928, 17152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 3328, 128, 128, 45056, 17152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 1408, 128, 384, 31872, 128, 128, 128, 32000, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[16768, 256, 0, 128, 128, 128, 47872, 128, 768, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 1024, 256, 128, 768, 0, 128, 128, 640, 128, 128, 512, 128, 128, 384, 46848, 13440, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[2688, 0, 128, 128, 128, 128, 49024, 256, 0, 128, 1280, 896, 128, 384, 128, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 54400, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 256, 128, 0, 128, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 1024, 256, 128, 768, 0, 128, 128, 512, 128, 128, 512, 128, 128, 384, 46592, 13568, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[2688, 0, 128, 128, 128, 128, 48896, 256, 0, 128, 1280, 896, 128, 384, 128, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 54528, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 256, 128, 0, 128, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 256, 128, 256, 128, 0, 128, 384, 128, 128, 61952, 1920, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 128, 256, 128, 128, 128, 128, 640, 128, 128, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 365, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 33, 7681, 0, 0, 6864, 8145, 0, 3, 0, 3926, 0, 0, 2, 0, 157, 0, 0, 0, 0, 466, 0, 0, 0, 38, 2, 105, 0, 0, 0, 2141, 0, 0, 3, 0, 0, 10, 0, 32, 4, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 7589, 213, 245, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 3, 0, 0, 0, 92, 1, 1, 1, 0, 0, 0, 1, 0, 1, 14, 2875, 1, 0, 0, 4, 2, 4, 0, 1, 1, 0, 0, 0, 318, 8006, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 1, 24, 0, 1878, 0, 1, 0, 0, 0, 0, 0, 8, 2, 0, 0, 29, 0, 0, 0, 0, 0, 0, 7916, 0, 0, 2612, 8162, 0, 0, 0, 6345, 0, 0, 0, 58, 0, 0, 12, 3, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 4773, 0, 0, 14, 0, 0, 0, 0, 348, 0, 2, 0, 0, 0, 0, 8027, 0, 0, 8, 9, 0, 0, 7981, 362, 1, 0, 0, 0, 8160, 0, 0, 327, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 25, 0, 18, 0, 48, 0, 568, 123, 0, 5, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 7460, 183, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 4148, 8181, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 8, 0, 0, 639, 0, 0, 0, 0, 0, 23, 0, 0, 0, 5185, 0, 0, 0, 0, 0, 0, 2, 515, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 7504, 2, 0, 0, 0, 0, 2841, 0, 0, 2180, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 56, 0, 0, 0, 7, 0, 8, 13, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 8172, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 98, 1, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2, 3, 0, 0, 2, 11, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 7956, 0, 0, 0, 5571, 0, 0, 0, 0, 0, 0, 0, 0, 7, 163, 0, 0, 3, 0, 0, 2, 0, 0, 0, 6406, 0, 0, 0, 0, 0, 1, 0, 461, 0, 0, 3, 0, 0, 0, 7838, 0, 0, 0, 0, 1, 0, 6756, 0, 0, 0, 1, 0, 8092, 0, 4, 2964, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 569, 0, 1305, 0, 0, 232, 27, 0, 0, 0, 279, 0, 0, 0, 0, 0, 67, 0, 8187, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4319, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8133, 0, 0, 1645, 7857, 0, 0, 0, 7032, 0, 2, 0, 98, 38, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2776, 1, 0, 1, 0, 0, 0, 0, 8166, 8, 3, 0, 1, 0, 0, 7526, 0, 1, 3, 0, 5, 0, 7196, 414, 13, 0, 0, 0, 775, 0, 0, 1012, 38, 0, 0, 0, 0, 0, 0, 0, 123, 13, 0, 0, 0, 1, 1, 18, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 2, 0, 7992, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 0, 0, 313, 0, 18, 1, 0, 0, 0, 0, 149, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8128, 0, 1, 1494, 8066, 0, 0, 0, 7832, 10, 0, 0, 0, 0, 0, 9, 0, 0, 895, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 2325, 0, 3, 0, 0, 0, 0, 8111, 2, 0, 0, 5, 0, 0, 7150, 1, 0, 3, 0, 2, 7847, 0, 0, 397, 0, 0, 1, 0, 30, 0, 3, 2, 2, 16, 1, 44, 6, 0, 3, 39, 32, 123, 295, 0, 25, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 1, 0, 1975, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 34, 1, 54, 8182, 0, 0, 0, 7669, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 3, 429, 0, 0, 0, 4, 0, 3, 17, 6536, 0, 0, 0, 0, 0, 0, 8137, 10, 0, 0, 0, 0, 2, 8140, 2, 0, 0, 0, 0, 7664, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 16, 129, 0, 17, 2, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 61, 0, 0, 2, 7972, 0, 1, 0, 0, 0, 1, 29, 0, 2, 0, 1, 0, 3, 0, 0, 0, 77, 0, 8128, 168, 0, 38, 8058, 0, 0, 22, 7378, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 48, 50, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 4, 0, 0, 1, 9, 5615, 0, 4, 4, 0, 0, 22, 7629, 35, 0, 0, 0, 0, 4440, 0, 11, 2458, 0, 0, 55, 0, 0, 0, 0, 0, 0, 2, 182, 891, 0, 98, 0, 5, 15, 1, 2, 0, 19, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 8179, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 1, 24, 0, 1878, 5407, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 2, 0, 0, 29, 0, 0, 0, 0, 8162, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2612, 0, 0, 0, 0, 5663, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 4773, 0, 2385, 1667, 682, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 348, 0, 2, 0, 0, 0, 0, 5642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 0, 0, 907, 362, 1, 0, 0, 0, 8160, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 25, 0, 18, 0, 48, 0, 568, 7082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 5, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 7370, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8056, 8300, 8333, 9620, 6006, 9774, 7744, 7703] avg=8192.0000 max=9774 max_rank=5 min=6006 min_rank=4 imbalance_ratio=1.193115 max_load_reduction=55762 max_load_reduction_pct=0.850861 moved_to_echo=31227 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 365, 5142, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 33, 0, 0, 0, 6864, 0, 0, 3, 0, 3504, 0, 0, 2, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 38, 2, 105, 0, 0, 0, 2141, 0, 2412, 1585, 422, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 10, 0, 32, 4, 0, 0, 0, 0, 0, 5708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 213, 245, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 92, 1, 1, 1, 0, 0, 0, 1, 0, 1, 14, 2875, 6870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 2, 4, 0, 1, 1, 0, 0, 0, 318, 7909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6225, 8244, 10564, 7268, 5757, 9379, 9859, 8240] avg=8192.0000 max=10564 max_rank=2 min=5757 min_rank=4 imbalance_ratio=1.289551 max_load_reduction=54972 max_load_reduction_pct=0.838806 moved_to_echo=33235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1671, 5084, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4148, 0, 0, 0, 0, 7132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 639, 0, 0, 0, 0, 0, 23, 0, 0, 0, 5185, 0, 2434, 1568, 860, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 515, 0, 0, 0, 0, 0, 1, 5757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 852, 2, 0, 0, 0, 0, 2841, 0, 0, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 56, 0, 0, 0, 7, 0, 8, 7319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 8073, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7517, 8283, 11280, 10816, 6275, 5875, 7392, 8098] avg=8192.0000 max=11280 max_rank=2 min=5875 min_rank=5 imbalance_ratio=1.376953 max_load_reduction=54256 max_load_reduction_pct=0.827881 moved_to_echo=27640 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 98, 1, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4577, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2, 3, 0, 0, 2, 11, 0, 0, 0, 7956, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 163, 0, 0, 3, 0, 0, 2, 0, 0, 0, 6406, 0, 2329, 1411, 599, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 461, 0, 0, 3, 0, 0, 0, 5509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 768, 0, 0, 0, 1, 0, 8092, 0, 4, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 569, 0, 1305, 0, 0, 7318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 27, 0, 0, 0, 279, 0, 0, 0, 0, 0, 67, 0, 8088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5453, 8089, 4973, 11019, 5974, 11830, 9504, 8694] avg=8192.0000 max=11830 max_rank=5 min=4973 min_rank=2 imbalance_ratio=1.444092 max_load_reduction=53706 max_load_reduction_pct=0.819489 moved_to_echo=36002 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4319, 4881, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1645, 0, 0, 0, 0, 6272, 0, 2, 0, 98, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2776, 1, 2239, 1507, 760, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8166, 8, 3, 0, 1, 0, 0, 5287, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 0, 808, 414, 13, 0, 0, 0, 775, 0, 0, 1012, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 13, 0, 0, 0, 1, 1, 18, 0, 173, 7274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 2, 0, 7889, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9968, 7962, 8057, 7389, 13467, 3068, 7603, 8022] avg=8192.0000 max=13467 max_rank=4 min=3068 min_rank=5 imbalance_ratio=1.643921 max_load_reduction=52069 max_load_reduction_pct=0.794510 moved_to_echo=32160 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 1, 0, 1975, 5515, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1, 54, 0, 0, 0, 0, 6844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 3, 429, 0, 2418, 1700, 825, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 17, 6536, 0, 0, 0, 0, 0, 0, 5719, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 925, 2, 0, 0, 0, 0, 7664, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 16, 7316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 17, 2, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8288, 6933, 5479, 12289, 8707, 7343, 8240] avg=8192.0000 max=12289 max_rank=4 min=5479 min_rank=3 imbalance_ratio=1.500122 max_load_reduction=53247 max_load_reduction_pct=0.812485 moved_to_echo=36579 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 0, 0, 313, 0, 18, 1, 0, 0, 0, 0, 149, 4844, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 8066, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1494, 0, 0, 0, 0, 6989, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 895, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 2410, 1494, 843, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2325, 0, 3, 0, 0, 0, 0, 5701, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 812, 1, 0, 3, 0, 2, 7847, 0, 0, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 30, 0, 3, 2, 2, 16, 1, 44, 6, 0, 3, 39, 32, 123, 7270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 25, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 8173, 8495, 9687, 8031, 9067, 7572, 8416] avg=8192.0000 max=9687 max_rank=3 min=6095 min_rank=0 imbalance_ratio=1.182495 max_load_reduction=55849 max_load_reduction_pct=0.852188 moved_to_echo=33086 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 61, 0, 0, 2, 7972, 5169, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 29, 0, 2, 0, 1, 0, 3, 0, 0, 0, 8058, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 168, 0, 38, 0, 0, 0, 22, 6584, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 48, 50, 0, 1668, 1594, 794, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 4, 0, 0, 1, 9, 3947, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 22, 866, 35, 0, 0, 0, 0, 4440, 0, 11, 2458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 2, 182, 891, 0, 98, 0, 5, 15, 1, 7270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13965, 8197, 6901, 4258, 7737, 7836, 8519, 8123] avg=8192.0000 max=13965 max_rank=0 min=4258 min_rank=3 imbalance_ratio=1.704712 max_load_reduction=51571 max_load_reduction_pct=0.786911 moved_to_echo=32215 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1, 4, 0, 1, 0, 1, 0, 1985, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 34, 1, 58, 8182, 0, 0, 0, 7678, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2, 431, 0, 0, 0, 2, 0, 3, 19, 6499, 0, 0, 0, 0, 0, 0, 8130, 10, 0, 0, 0, 0, 2, 8149, 2, 0, 0, 0, 0, 7681, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 16, 122, 0, 18, 3, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 368, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 31, 7696, 0, 0, 6894, 8145, 0, 3, 0, 3963, 0, 0, 1, 0, 156, 0, 0, 0, 0, 476, 0, 0, 0, 39, 2, 109, 0, 0, 0, 2052, 0, 0, 3, 0, 0, 9, 0, 27, 3, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 7582, 183, 234, 0, 0, 0, 8057, 0, 0, 0, 0, 0, 2, 0, 0, 0, 93, 1, 2, 2, 0, 0, 0, 1, 0, 1, 14, 2939, 1, 0, 0, 4, 2, 4, 0, 0, 1, 0, 0, 0, 309, 8002, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 2, 24, 0, 1886, 0, 1, 0, 0, 0, 0, 0, 8, 5, 0, 0, 27, 0, 0, 0, 0, 0, 0, 7916, 0, 0, 2667, 8163, 0, 0, 0, 6385, 0, 0, 0, 62, 0, 0, 9, 3, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 4624, 0, 0, 14, 0, 0, 0, 0, 367, 0, 2, 0, 0, 0, 0, 8030, 0, 0, 12, 6, 0, 0, 7981, 356, 1, 0, 0, 0, 8160, 0, 0, 324, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 26, 0, 17, 0, 49, 0, 604, 126, 0, 4, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 7454, 181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 4168, 8181, 0, 0, 0, 7987, 0, 0, 0, 0, 0, 0, 9, 0, 0, 643, 0, 0, 0, 0, 0, 24, 0, 0, 0, 5200, 0, 0, 0, 0, 0, 0, 2, 511, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 7505, 2, 0, 0, 0, 0, 2871, 0, 0, 2101, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 59, 0, 0, 0, 8, 0, 7, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8170, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 60, 0, 0, 2, 7968, 0, 1, 0, 0, 0, 1, 25, 0, 3, 0, 1, 0, 3, 0, 0, 0, 73, 0, 8130, 171, 0, 43, 8056, 0, 0, 21, 7383, 0, 0, 0, 14, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 45, 53, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 4, 0, 0, 1, 11, 5613, 0, 4, 5, 0, 0, 22, 7644, 29, 0, 0, 0, 0, 4380, 0, 11, 2447, 0, 0, 57, 0, 0, 0, 0, 0, 0, 2, 174, 870, 0, 102, 0, 7, 15, 1, 3, 0, 21, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 8179, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 94, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2, 4, 0, 0, 2, 10, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 7964, 0, 0, 0, 5591, 0, 0, 0, 0, 0, 0, 0, 0, 8, 150, 0, 0, 3, 0, 0, 2, 0, 1, 0, 6343, 0, 0, 0, 0, 0, 1, 0, 463, 0, 0, 3, 0, 0, 0, 7824, 0, 0, 0, 0, 1, 0, 6752, 0, 0, 0, 1, 0, 8098, 0, 5, 3027, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 558, 0, 1340, 0, 0, 220, 27, 0, 0, 0, 273, 0, 0, 0, 0, 0, 71, 0, 8187, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4371, 0, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8131, 0, 0, 1633, 7848, 0, 0, 0, 7037, 0, 2, 0, 97, 38, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 2709, 1, 0, 1, 0, 0, 0, 0, 8167, 8, 1, 0, 0, 0, 0, 7535, 0, 1, 2, 0, 5, 0, 7191, 409, 11, 0, 0, 0, 781, 0, 0, 1038, 38, 0, 0, 0, 0, 0, 0, 0, 120, 12, 0, 0, 0, 1, 1, 20, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 2, 0, 7998, 0, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 0, 0, 316, 0, 19, 1, 0, 0, 0, 0, 144, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 8127, 0, 1, 1534, 8070, 0, 0, 0, 7828, 10, 0, 0, 0, 0, 1, 8, 0, 0, 902, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 2291, 0, 2, 0, 0, 0, 0, 8110, 3, 0, 0, 4, 0, 0, 7155, 2, 0, 3, 0, 2, 7849, 0, 0, 396, 0, 0, 1, 0, 31, 0, 3, 1, 3, 18, 1, 43, 6, 0, 3, 35, 32, 126, 298, 0, 24, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1, 4, 0, 1, 0, 1, 0, 1985, 5494, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1, 58, 0, 0, 0, 0, 6818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2, 431, 0, 2416, 1721, 860, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 19, 6499, 0, 0, 0, 0, 0, 0, 5714, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 934, 2, 0, 0, 0, 0, 7681, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 16, 7301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 18, 3, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8288, 6911, 5530, 12247, 8733, 7326, 8240] avg=8192.0000 max=12247 max_rank=4 min=5530 min_rank=3 imbalance_ratio=1.494995 max_load_reduction=53289 max_load_reduction_pct=0.813126 moved_to_echo=36546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 368, 5112, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8145, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 31, 0, 0, 0, 6894, 0, 0, 3, 0, 3519, 0, 0, 1, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 39, 2, 109, 0, 0, 0, 2052, 0, 2412, 1601, 444, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 9, 0, 27, 3, 0, 0, 0, 0, 0, 5703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 183, 234, 0, 0, 0, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 93, 1, 2, 2, 0, 0, 0, 1, 0, 1, 14, 2939, 6871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 2, 4, 0, 0, 1, 0, 0, 0, 309, 7909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 8244, 10606, 7228, 5745, 9343, 9926, 8230] avg=8192.0000 max=10606 max_rank=2 min=5745 min_rank=4 imbalance_ratio=1.294678 max_load_reduction=54930 max_load_reduction_pct=0.838165 moved_to_echo=33244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 2, 24, 0, 1886, 5381, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 5, 0, 0, 27, 0, 0, 0, 0, 8163, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2667, 0, 0, 0, 0, 5670, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 4624, 0, 2387, 1685, 715, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 367, 0, 2, 0, 0, 0, 0, 5643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 6, 0, 0, 915, 356, 1, 0, 0, 0, 8160, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 26, 0, 17, 0, 49, 0, 604, 7069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 4, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 7367, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8051, 8302, 8399, 9516, 6026, 9774, 7768, 7700] avg=8192.0000 max=9774 max_rank=5 min=6026 min_rank=4 imbalance_ratio=1.193115 max_load_reduction=55762 max_load_reduction_pct=0.850861 moved_to_echo=31268 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1689, 5060, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4168, 0, 0, 0, 0, 7093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 643, 0, 0, 0, 0, 0, 24, 0, 0, 0, 5200, 0, 2434, 1585, 894, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 511, 0, 0, 0, 0, 0, 1, 5757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 2, 0, 0, 0, 0, 2871, 0, 0, 2101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 59, 0, 0, 0, 8, 0, 7, 7305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8075, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7525, 8282, 11261, 10884, 6271, 5834, 7381, 8098] avg=8192.0000 max=11261 max_rank=2 min=5834 min_rank=5 imbalance_ratio=1.374634 max_load_reduction=54275 max_load_reduction_pct=0.828171 moved_to_echo=27584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 60, 0, 0, 2, 7968, 5154, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 25, 0, 3, 0, 1, 0, 3, 0, 0, 0, 8056, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 171, 0, 43, 0, 0, 0, 21, 6556, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 45, 53, 0, 1668, 1614, 827, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 4, 0, 0, 1, 11, 3945, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 22, 876, 29, 0, 0, 0, 0, 4380, 0, 11, 2447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 2, 174, 870, 0, 102, 0, 7, 15, 1, 7258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 21, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13957, 8192, 6878, 4307, 7816, 7770, 8486, 8130] avg=8192.0000 max=13957 max_rank=0 min=4307 min_rank=3 imbalance_ratio=1.703735 max_load_reduction=51579 max_load_reduction_pct=0.787033 moved_to_echo=32202 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 94, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4552, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2, 4, 0, 0, 2, 10, 0, 0, 0, 7964, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 150, 0, 0, 3, 0, 0, 2, 0, 1, 0, 6343, 0, 2325, 1426, 626, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 463, 0, 0, 3, 0, 0, 0, 5499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 774, 0, 0, 0, 1, 0, 8098, 0, 5, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 558, 0, 1340, 0, 0, 7305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 27, 0, 0, 0, 273, 0, 0, 0, 0, 0, 71, 0, 8092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5438, 8096, 4966, 10979, 5966, 11906, 9501, 8684] avg=8192.0000 max=11906 max_rank=5 min=4966 min_rank=2 imbalance_ratio=1.453369 max_load_reduction=53630 max_load_reduction_pct=0.818329 moved_to_echo=36057 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4371, 4852, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7848, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1633, 0, 0, 0, 0, 6246, 0, 2, 0, 97, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 2709, 1, 2244, 1523, 791, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8167, 8, 1, 0, 0, 0, 0, 5291, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 816, 409, 11, 0, 0, 0, 781, 0, 0, 1038, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 12, 0, 0, 0, 1, 1, 20, 0, 176, 7259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 2, 0, 7900, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10005, 7954, 8018, 7369, 13469, 3100, 7589, 8032] avg=8192.0000 max=13469 max_rank=4 min=3100 min_rank=5 imbalance_ratio=1.644165 max_load_reduction=52067 max_load_reduction_pct=0.794479 moved_to_echo=32190 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 0, 0, 316, 0, 19, 1, 0, 0, 0, 0, 144, 4824, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 8070, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1534, 0, 0, 0, 0, 6951, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 902, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 2410, 1511, 877, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 0, 2, 0, 0, 0, 0, 5700, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 820, 2, 0, 3, 0, 2, 7849, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 31, 0, 3, 1, 3, 18, 1, 43, 6, 0, 3, 35, 32, 126, 7256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 24, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 8178, 8497, 9723, 7996, 9076, 7559, 8422] avg=8192.0000 max=9723 max_rank=3 min=6085 min_rank=0 imbalance_ratio=1.186890 max_load_reduction=55813 max_load_reduction_pct=0.851639 moved_to_echo=33053 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 2176, 128, 128, 128, 128, 128, 256, 0, 0, 128, 25728, 128, 18304, 12544, 5888, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 0, 0, 128, 128, 128, 128, 384, 128, 128, 128, 128, 128, 128, 128, 18432, 40704, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 0, 128, 128, 22272, 128, 128, 128, 128, 128, 128, 43392, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 6912, 1152, 384, 128, 128, 128, 48000, 0, 128, 9472, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 256, 128, 16896, 0, 0, 128, 128, 48000, 128, 128, 128, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 256, 1408, 128, 768, 128, 1536, 128, 3840, 57728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 0, 0, 128, 64640, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[896, 128, 128, 128, 128, 384, 128, 128, 256, 128, 0, 128, 384, 63616, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 2176, 128, 128, 128, 128, 128, 256, 128, 128, 128, 25344, 128, 18304, 12672, 6144, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 384, 128, 128, 128, 128, 128, 128, 128, 18432, 40448, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 128, 128, 22272, 128, 128, 128, 0, 128, 128, 43264, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 6912, 1024, 256, 128, 128, 128, 48000, 0, 128, 9472, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 256, 128, 17024, 0, 0, 128, 128, 47872, 128, 128, 128, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 256, 1280, 128, 768, 128, 1536, 128, 3968, 57728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 0, 128, 128, 64640, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[896, 128, 128, 128, 128, 384, 128, 128, 256, 128, 0, 128, 384, 63616, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 7922, 4, 0, 2, 0, 0, 0, 0, 48, 0, 1, 0, 0, 0, 15, 7983, 0, 0, 58, 0, 0, 3, 0, 0, 0, 343, 0, 0, 0, 634, 0, 7590, 0, 0, 0, 25, 0, 0, 0, 0, 0, 165, 0, 8142, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7864, 1, 5, 7, 68, 0, 292, 0, 0, 0, 0, 1, 5, 11, 3256, 0, 1, 0, 0, 1, 0, 5, 0, 21, 0, 0, 0, 0, 8017, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 29, 0, 0, 0, 8080, 0, 0, 41, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 50, 0, 0, 0, 794, 0, 0, 0, 6063, 5789, 0, 2, 0, 0, 0, 198, 0, 4647, 0, 0, 0, 0, 0, 0, 7961, 0, 0, 0, 0, 53, 0, 0, 0, 77, 802, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 45, 0, 2, 58, 0, 0, 0, 27, 28, 0, 0, 0, 0, 9, 0, 6965, 2, 0, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 1, 3, 0, 1, 0, 1, 0, 0, 0, 0, 2, 7, 33, 4600, 0, 0, 0, 0, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 0, 0, 8180, 0, 0, 2, 7, 0, 3, 0, 30, 0, 8, 42, 2, 0, 0, 0, 0, 8, 0, 28, 0, 2, 6882, 7423, 17, 0, 0, 0, 0, 218, 2, 5487, 1, 0, 11, 0, 14, 0, 8105, 0, 0, 0, 0, 221, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8101, 0, 10, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 1, 59, 0, 7923, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 3, 0, 0, 0, 0, 1, 0, 562, 0, 0, 0, 0, 0, 3899, 0, 0, 2, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 6336, 0, 200, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 8173, 0, 0, 4, 2, 0, 7, 0, 3678, 5, 0, 2, 0, 1, 0, 2821, 0, 0, 1, 0, 18, 0, 0, 0, 2846, 0, 18, 0, 0, 102, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8140, 0, 0, 0, 56, 0, 0, 0, 4, 0, 0, 0, 0, 0, 956, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 8001, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 0, 0, 7891, 0, 10, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 7276, 1315, 0, 0, 0, 0, 0, 1006, 0, 7668, 21, 0, 0, 0, 4, 0, 7753, 0, 0, 22, 0, 376, 0, 0, 5, 0, 352, 0, 0, 6, 19, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8142, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7179, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 0, 151, 0, 29, 0, 2341, 0, 1554, 0, 0, 0, 0, 411, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 9, 6081, 6110, 0, 3, 26, 0, 0, 6, 0, 8112, 0, 0, 1, 0, 0, 1, 7658, 5, 0, 0, 0, 0, 20, 0, 0, 155, 0, 0, 0, 2, 11, 0, 8181, 0, 0, 0, 0, 0, 6, 0, 7, 0, 1, 9, 6075, 0, 0, 10, 2, 0, 0, 0, 0, 0, 2, 2, 19, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 2705, 0, 0, 0, 215, 0, 0, 0, 0, 0, 5291, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 3, 0, 177, 1236, 0, 0, 0, 0, 454, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1, 0, 0, 8076, 0, 52, 0, 0, 0, 0, 74, 0, 3, 8, 0, 0, 0, 0, 1, 0, 7, 10, 0, 0, 0, 8031, 8103, 0, 0, 0, 0, 0, 83, 0, 8031, 2, 0, 0, 0, 0, 2, 7984, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 1, 0, 1, 3, 8004, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 20, 0, 0, 39, 0, 0, 0, 0, 2, 0, 17, 0, 0, 470, 0, 0, 0, 1, 0, 0, 0, 143, 0, 0, 0, 0, 0, 27, 0, 2, 0, 0, 30, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7460, 0, 0, 0, 0, 0, 0, 2, 41, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 114, 8184, 0, 0, 0, 0, 0, 11, 0, 7385, 19, 0, 8, 0, 0, 0, 8189, 0, 0, 0, 0, 174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8119, 25, 0, 0, 0, 0, 0, 0, 19, 0, 1, 0, 8184, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8133, 0, 0, 0, 183, 0, 0, 0, 8, 0, 0, 0, 0, 0, 2663, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6507, 13, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 29, 0, 0, 0, 8080, 0, 0, 41, 3, 0, 0, 0, 0, 875, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 50, 0, 0, 0, 794, 0, 0, 0, 6063, 1562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 198, 0, 4647, 0, 0, 0, 0, 0, 0, 2555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 77, 802, 0, 0, 0, 0, 0, 7445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 45, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 58, 0, 0, 0, 27, 28, 0, 0, 0, 0, 9, 0, 6965, 2, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 1, 3, 0, 1, 5406, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 7, 33, 4600, 0, 0, 0, 0, 0, 10, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9241, 8471, 7402, 8377, 8499, 7309, 8963, 7274] avg=8192.0000 max=9241 max_rank=0 min=7274 min_rank=7 imbalance_ratio=1.128052 max_load_reduction=56295 max_load_reduction_pct=0.858994 moved_to_echo=32045 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 0, 0, 7891, 0, 10, 0, 0, 0, 0, 1, 0, 197, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 7276, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 0, 7668, 21, 0, 0, 0, 4, 0, 2494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 376, 0, 0, 5, 0, 352, 0, 0, 6, 19, 0, 7509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8142, 0, 0, 0, 0, 7, 1, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 5259, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7179, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 7654, 11193, 8289, 8406, 8393, 5505, 7776] avg=8192.0000 max=11193 max_rank=2 min=5505 min_rank=6 imbalance_ratio=1.366333 max_load_reduction=54343 max_load_reduction_pct=0.829208 moved_to_echo=30080 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 0, 0, 8180, 0, 0, 2, 7, 0, 3, 0, 30, 1117, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 42, 2, 0, 0, 0, 0, 8, 0, 28, 0, 2, 6882, 2017, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 2, 5487, 1, 0, 11, 0, 14, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8101, 0, 10, 0, 0, 0, 24, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 1, 59, 0, 7923, 0, 9, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 3, 0, 0, 0, 5498, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 562, 0, 0, 0, 0, 0, 3899, 0, 0, 2, 2, 0, 0, 3356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9556, 9006, 8340, 7714, 8391, 8220, 6487, 7822] avg=8192.0000 max=9556 max_rank=0 min=6487 min_rank=6 imbalance_ratio=1.166504 max_load_reduction=55980 max_load_reduction_pct=0.854187 moved_to_echo=30920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 6336, 0, 200, 0, 1, 0, 0, 1, 0, 1230, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 7, 0, 3678, 5, 0, 2, 0, 1, 0, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 2846, 0, 18, 0, 0, 102, 0, 7460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8140, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 4, 0, 0, 0, 0, 0, 956, 0, 0, 0, 0, 1913, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 8001, 0, 1, 1, 0, 0, 0, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7983, 9638, 4607, 10445, 8428, 8356, 3956, 12123] avg=8192.0000 max=12123 max_rank=7 min=3956 min_rank=6 imbalance_ratio=1.479858 max_load_reduction=53413 max_load_reduction_pct=0.815018 moved_to_echo=32863 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1, 0, 0, 8076, 0, 52, 0, 0, 0, 0, 74, 0, 1219, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 0, 0, 1, 0, 7, 10, 0, 0, 0, 8031, 2203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 8031, 2, 0, 0, 0, 0, 2, 2568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 1, 0, 1, 3, 7347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 20, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 2, 0, 17, 0, 0, 470, 0, 0, 0, 1, 5416, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 27, 0, 2, 0, 0, 30, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9630, 10263, 10686, 7357, 8371, 8401, 6963, 3865] avg=8192.0000 max=10686 max_rank=2 min=3865 min_rank=7 imbalance_ratio=1.304443 max_load_reduction=54850 max_load_reduction_pct=0.836945 moved_to_echo=27600 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 151, 0, 29, 0, 2341, 0, 1554, 0, 0, 0, 0, 411, 0, 919, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 9, 6081, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 26, 0, 0, 6, 0, 8112, 0, 0, 1, 0, 0, 1, 2463, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 155, 0, 0, 0, 2, 11, 0, 7509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 0, 1, 9, 6075, 0, 0, 10, 2, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 19, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 0, 0, 215, 0, 0, 0, 0, 0, 5291, 0, 0, 0, 0, 5195, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 3, 0, 177, 1236, 0, 0, 0, 0, 454, 0, 2762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5618, 7813, 10617, 7697, 6358, 8430, 14173, 4830] avg=8192.0000 max=14173 max_rank=6 min=4830 min_rank=7 imbalance_ratio=1.730103 max_load_reduction=51363 max_load_reduction_pct=0.783737 moved_to_echo=33791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7460, 0, 0, 0, 0, 0, 0, 2, 41, 1231, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 114, 2225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 7385, 19, 0, 8, 0, 0, 0, 2634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7452, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1, 0, 8184, 0, 0, 0, 53, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8133, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 8, 0, 0, 0, 0, 0, 2663, 0, 1, 0, 0, 5555, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6507, 13, 0, 0, 0, 0, 0, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8942, 2365, 10057, 7652, 8503, 8358, 9438, 10221] avg=8192.0000 max=10221 max_rank=7 min=2365 min_rank=1 imbalance_ratio=1.247681 max_load_reduction=55315 max_load_reduction_pct=0.844040 moved_to_echo=36520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4860, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 2154, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 48, 0, 1, 0, 0, 0, 15, 2568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 3, 0, 0, 0, 343, 0, 0, 0, 634, 0, 6967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 165, 0, 8142, 0, 0, 18, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7864, 1, 5, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 68, 0, 292, 0, 0, 0, 0, 1, 5, 11, 3256, 0, 1, 0, 0, 5415, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 21, 0, 0, 0, 0, 8017, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6246, 10326, 2634, 8005, 8580, 8069, 10051, 11625] avg=8192.0000 max=11625 max_rank=7 min=2634 min_rank=2 imbalance_ratio=1.419067 max_load_reduction=53911 max_load_reduction_pct=0.822617 moved_to_echo=38325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7470, 0, 0, 0, 0, 0, 0, 1, 41, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 120, 8184, 0, 0, 0, 0, 0, 13, 0, 7432, 21, 0, 7, 0, 0, 0, 8189, 0, 0, 0, 0, 159, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8114, 23, 0, 0, 0, 0, 0, 0, 21, 0, 2, 0, 8181, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8128, 0, 0, 0, 191, 0, 0, 0, 9, 0, 0, 0, 0, 0, 2651, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6469, 11, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1, 0, 0, 8067, 0, 54, 0, 0, 0, 0, 73, 0, 2, 8, 0, 0, 0, 0, 1, 0, 6, 9, 0, 0, 0, 8026, 8105, 0, 0, 0, 0, 1, 84, 0, 8045, 2, 0, 0, 0, 0, 2, 7964, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 1, 0, 0, 3, 8005, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 17, 0, 0, 41, 0, 0, 0, 0, 2, 0, 18, 0, 0, 491, 0, 0, 0, 1, 0, 0, 0, 140, 0, 0, 0, 0, 0, 23, 0, 2, 0, 0, 30, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 0, 0, 8180, 0, 0, 2, 8, 0, 2, 0, 30, 0, 7, 40, 3, 0, 0, 0, 0, 6, 0, 28, 0, 2, 6866, 7416, 17, 0, 0, 1, 0, 233, 2, 5490, 0, 0, 11, 0, 22, 0, 8098, 0, 0, 0, 0, 218, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8102, 0, 10, 0, 0, 0, 25, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 1, 57, 0, 7938, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3, 0, 0, 0, 0, 1, 0, 572, 0, 0, 0, 0, 0, 3888, 0, 0, 4, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 6336, 0, 191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7425, 8164, 0, 0, 3, 2, 0, 8, 0, 3753, 7, 0, 2, 0, 1, 0, 2829, 0, 0, 0, 0, 18, 0, 0, 0, 2816, 0, 19, 0, 0, 98, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 54, 0, 0, 0, 5, 0, 0, 0, 0, 0, 898, 0, 0, 0, 0, 1, 0, 0, 436, 0, 0, 0, 0, 0, 8006, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8168, 7926, 5, 0, 3, 1, 0, 0, 0, 63, 0, 1, 0, 0, 0, 15, 7993, 0, 0, 61, 0, 0, 4, 0, 0, 0, 351, 0, 0, 0, 619, 0, 7597, 0, 0, 0, 24, 0, 0, 0, 0, 0, 164, 0, 8141, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7865, 0, 5, 8, 61, 0, 285, 0, 0, 0, 0, 1, 5, 11, 3140, 0, 1, 0, 0, 1, 0, 5, 0, 17, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 15, 0, 0, 0, 7896, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 7264, 1383, 0, 0, 0, 0, 0, 1003, 0, 7646, 21, 0, 0, 0, 3, 0, 7739, 0, 0, 22, 0, 374, 0, 0, 5, 0, 352, 0, 0, 8, 20, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8143, 0, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7158, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[5, 0, 0, 161, 0, 26, 0, 2436, 0, 1548, 0, 0, 0, 0, 410, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 9, 6062, 6137, 0, 3, 22, 0, 0, 12, 0, 8113, 0, 0, 0, 0, 0, 1, 7660, 5, 0, 0, 0, 0, 23, 0, 0, 159, 0, 0, 0, 2, 11, 0, 8181, 0, 0, 0, 0, 0, 6, 0, 5, 0, 1, 8, 6028, 0, 0, 9, 2, 0, 1, 0, 0, 0, 2, 2, 18, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 2755, 0, 0, 0, 214, 0, 0, 0, 0, 0, 5205, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 3, 0, 171, 1228, 0, 0, 0, 0, 441, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 33, 0, 0, 0, 8077, 0, 0, 38, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 51, 0, 0, 0, 797, 0, 0, 0, 6032, 5799, 0, 1, 0, 0, 0, 190, 0, 4639, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 0, 0, 51, 0, 0, 0, 75, 803, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 47, 0, 2, 57, 0, 0, 0, 22, 29, 0, 0, 0, 0, 10, 0, 6980, 3, 0, 0, 2792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1, 3, 0, 1, 0, 1, 0, 0, 0, 0, 2, 7, 34, 4560, 0, 0, 0, 0, 0, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7470, 0, 0, 0, 0, 0, 0, 1, 41, 1210, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 120, 2231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 7432, 21, 0, 7, 0, 0, 0, 2614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 7453, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 2, 0, 8181, 0, 0, 0, 54, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8128, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 9, 0, 0, 0, 0, 0, 2651, 0, 1, 0, 0, 5575, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6469, 11, 0, 0, 0, 0, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8921, 2377, 10087, 7638, 8509, 8352, 9458, 10194] avg=8192.0000 max=10194 max_rank=7 min=2377 min_rank=1 imbalance_ratio=1.244385 max_load_reduction=55342 max_load_reduction_pct=0.844452 moved_to_echo=36513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 6336, 0, 191, 0, 0, 0, 0, 1, 1, 1207, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7425, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 8, 0, 3753, 7, 0, 2, 0, 1, 0, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 2816, 0, 19, 0, 0, 98, 0, 7467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 5, 0, 0, 0, 0, 0, 898, 0, 0, 0, 0, 1926, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 436, 0, 0, 0, 0, 0, 8006, 0, 1, 1, 0, 0, 0, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7942, 9652, 4679, 10418, 8433, 8353, 3911, 12148] avg=8192.0000 max=12148 max_rank=7 min=3911 min_rank=6 imbalance_ratio=1.482910 max_load_reduction=53388 max_load_reduction_pct=0.814636 moved_to_echo=32845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 15, 0, 0, 0, 7896, 0, 8, 0, 0, 0, 0, 0, 0, 204, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 7264, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 0, 7646, 21, 0, 0, 0, 3, 0, 2471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 374, 0, 0, 5, 0, 352, 0, 0, 8, 20, 0, 7515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8143, 0, 0, 0, 0, 8, 1, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 5268, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7158, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 7662, 11144, 8296, 8413, 8391, 5520, 7787] avg=8192.0000 max=11144 max_rank=2 min=5520 min_rank=6 imbalance_ratio=1.360352 max_load_reduction=54392 max_load_reduction_pct=0.829956 moved_to_echo=30111 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 0, 0, 8180, 0, 0, 2, 8, 0, 2, 0, 30, 1097, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 40, 3, 0, 0, 0, 0, 6, 0, 28, 0, 2, 6866, 2021, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 233, 2, 5490, 0, 0, 11, 0, 22, 0, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8102, 0, 10, 0, 0, 0, 25, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 1, 57, 0, 7938, 0, 9, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3, 0, 0, 0, 5513, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 572, 0, 0, 0, 0, 0, 3888, 0, 0, 4, 2, 0, 0, 3364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9527, 8990, 8344, 7716, 8400, 8232, 6496, 7831] avg=8192.0000 max=9527 max_rank=0 min=6496 min_rank=6 imbalance_ratio=1.162964 max_load_reduction=56009 max_load_reduction_pct=0.854630 moved_to_echo=30959 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4962, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8168, 2161, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 63, 0, 1, 0, 0, 0, 15, 2552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 4, 0, 0, 0, 351, 0, 0, 0, 619, 0, 6978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 164, 0, 8141, 0, 0, 18, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7865, 0, 5, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 61, 0, 285, 0, 0, 0, 0, 1, 5, 11, 3140, 0, 1, 0, 0, 5441, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 17, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6320, 10336, 2635, 8013, 8582, 8068, 9951, 11631] avg=8192.0000 max=11631 max_rank=7 min=2635 min_rank=2 imbalance_ratio=1.419800 max_load_reduction=53905 max_load_reduction_pct=0.822525 moved_to_echo=38232 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 161, 0, 26, 0, 2436, 0, 1548, 0, 0, 0, 0, 410, 0, 907, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 9, 6062, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 22, 0, 0, 12, 0, 8113, 0, 0, 0, 0, 0, 1, 2445, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 159, 0, 0, 0, 2, 11, 0, 7515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 0, 1, 8, 6028, 0, 0, 9, 2, 0, 1, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 18, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 0, 0, 0, 214, 0, 0, 0, 0, 0, 5205, 0, 0, 0, 0, 5215, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 3, 0, 171, 1228, 0, 0, 0, 0, 441, 0, 2784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5693, 7808, 10601, 7710, 6313, 8427, 14162, 4822] avg=8192.0000 max=14162 max_rank=6 min=4822 min_rank=7 imbalance_ratio=1.728760 max_load_reduction=51374 max_load_reduction_pct=0.783905 moved_to_echo=33724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1, 0, 0, 8067, 0, 54, 0, 0, 0, 0, 73, 0, 1199, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 1, 0, 6, 9, 0, 0, 0, 8026, 2209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 84, 0, 8045, 2, 0, 0, 0, 0, 2, 2543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 1, 0, 0, 3, 7354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 17, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 2, 0, 18, 0, 0, 491, 0, 0, 0, 1, 5421, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 23, 0, 2, 0, 0, 30, 0, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9593, 10261, 10677, 7362, 8383, 8394, 6995, 3871] avg=8192.0000 max=10677 max_rank=2 min=3871 min_rank=7 imbalance_ratio=1.303345 max_load_reduction=54859 max_load_reduction_pct=0.837082 moved_to_echo=27643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 33, 0, 0, 0, 8077, 0, 0, 38, 4, 0, 0, 0, 0, 862, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 51, 0, 0, 0, 797, 0, 0, 0, 6032, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 190, 0, 4639, 0, 0, 0, 0, 0, 0, 2539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 75, 803, 0, 0, 0, 0, 0, 7454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 47, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 57, 0, 0, 0, 22, 29, 0, 0, 0, 0, 10, 0, 6980, 3, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1, 3, 0, 1, 5425, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 7, 34, 4560, 0, 0, 0, 0, 0, 13, 2635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9217, 8450, 7369, 8383, 8503, 7319, 9043, 7252] avg=8192.0000 max=9217 max_rank=0 min=7252 min_rank=7 imbalance_ratio=1.125122 max_load_reduction=56319 max_load_reduction_pct=0.859360 moved_to_echo=32117 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 5888, 0, 384, 0, 256, 128, 0, 128, 128, 128, 12928, 128, 128, 0, 128, 39680, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 1536, 128, 45056, 128, 128, 128, 0, 128, 128, 18816, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[256, 128, 128, 1152, 128, 0, 128, 128, 256, 39552, 128, 128, 128, 128, 512, 128, 24064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 0, 128, 896, 128, 0, 128, 50048, 14464, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 256, 128, 128, 0, 53248, 0, 1920, 128, 128, 0, 128, 512, 128, 8064, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 6016, 0, 384, 0, 256, 128, 0, 128, 128, 128, 12672, 128, 128, 0, 128, 39808, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 1664, 128, 45184, 128, 128, 128, 0, 128, 128, 18688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[256, 128, 128, 1152, 128, 0, 128, 128, 256, 39424, 128, 128, 128, 128, 512, 128, 24192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 128, 896, 128, 0, 128, 50048, 14592, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 256, 128, 128, 0, 53504, 0, 1920, 128, 128, 0, 128, 512, 128, 7936, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 256, 128, 63232, 128, 128, 128, 128, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 896, 128, 0, 128, 3072, 1536, 128, 128, 128, 768, 128, 59264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 0, 63616, 128, 128, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 896, 128, 0, 128, 3200, 1536, 128, 128, 128, 768, 128, 59264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 256, 128, 63104, 128, 128, 128, 128, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 589, 0, 0, 0, 0, 0, 7124, 0, 0, 0, 0, 0, 4, 1533, 1, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 490, 0, 0, 0, 8188, 8110, 0, 0, 0, 306, 0, 0, 0, 0, 0, 20, 0, 0, 0, 7909, 4145, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 1, 7620, 0, 0, 0, 0, 96, 0, 0, 0, 8112, 4, 0, 0, 13, 0, 6, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 10, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6975, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 105] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 333, 78, 38, 0, 21, 0, 0, 708, 0, 0, 0, 1628, 0, 0, 0, 24, 0, 0, 0, 5342, 11, 0, 0, 8174, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5028, 8116, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 494, 8104, 0, 0, 0, 0, 722, 0, 0, 0, 7977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 0, 0, 0, 0, 740, 0, 14, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 914, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 101, 0, 0, 0, 0, 7, 0, 299, 0, 0, 0, 0, 0, 0, 5, 0, 6346, 0, 0, 0, 2764, 0, 2, 2, 3, 0, 0, 0, 4989, 0, 0, 0, 8190, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1589, 7990, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8042, 0, 0, 0, 0, 31, 0, 0, 0, 7493, 0, 15, 0, 1, 0, 0, 0, 0, 4, 0, 1, 6, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 0, 7, 0, 8142, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 456, 0, 0, 0, 0, 0, 0, 758, 0, 2, 0, 24, 0, 0, 0, 0, 702, 0, 0, 0, 157, 0, 232, 0, 0, 1, 0, 0, 7170, 0, 0, 0, 7975, 8108, 0, 0, 0, 0, 1, 0, 0, 71, 0, 1, 0, 0, 0, 1987, 7754, 0, 3, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7938, 0, 0, 1, 0, 2, 4, 2, 1, 8148, 0, 14, 33, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 18, 8, 4082, 0, 21, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6303, 0, 0, 0, 0, 2, 0, 5, 0, 6, 2, 3167, 8, 0, 119, 0, 0, 0, 0, 8077, 0, 0, 0, 8023, 8188, 0, 0, 2, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 7389, 1952, 0, 5, 1, 0, 2, 0, 0, 0, 0, 0, 73, 8158, 0, 0, 0, 1, 0, 0, 0, 0, 7961, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 65, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 40, 10, 0, 0, 0, 30, 4, 0, 0, 0, 3214, 7, 0, 1, 0, 0, 0, 0, 0, 1986, 0, 0, 281, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 0, 0, 4, 1, 1390, 0, 0, 0, 0, 1, 4, 1089, 0, 0, 0, 3455, 7, 2, 2, 0, 1, 0, 417, 6194, 0, 0, 0, 8171, 8094, 0, 0, 0, 12, 0, 0, 0, 0, 0, 3, 6, 0, 3, 468, 7156, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 2, 7845, 0, 0, 0, 0, 32, 0, 0, 0, 7606, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 2367, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 202, 0, 0, 0, 0, 3, 13, 1, 0, 0, 3657, 0, 0, 0, 0, 0, 0, 6, 0, 7105, 0, 16, 18, 0, 4, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 855, 0, 0, 3, 6, 0, 1, 0, 0, 0, 0, 0, 0, 7388, 0, 254, 72, 0, 0, 0, 0, 7721, 0, 0, 0, 8146, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7986, 7251, 0, 0, 0, 22, 32, 0, 0, 0, 0, 0, 0, 7999, 0, 0, 0, 0, 9, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 155, 2, 0, 0, 0, 0, 0, 0, 0, 0, 180, 23, 0, 0, 0, 12, 0, 0, 23, 7405, 3, 621, 39, 0, 0, 0, 0, 25, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 173, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 6826, 0, 0, 0, 2337, 1, 0, 719, 280, 27, 0, 475, 469, 0, 0, 1, 8043, 8130, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 1, 7712, 7240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7119, 0, 0, 0, 3, 329, 0, 0, 9, 3238, 0, 133, 0, 45, 0, 0, 0, 6, 1, 12, 464, 2, 0, 24, 8, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 38, 278, 0, 60, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 7, 0, 3, 7060, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 589, 0, 0, 0, 0, 0, 7124, 0, 0, 0, 0, 2157, 1646, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1533, 1, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 11, 0, 0, 0, 306, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7909, 1988, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 8112, 4, 0, 0, 13, 0, 6, 0, 0, 112, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 10, 0, 7544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5974, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6597, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11597, 1916, 8525, 9901, 8721, 7577, 6532, 10767] avg=8192.0000 max=11597 max_rank=0 min=1916 min_rank=1 imbalance_ratio=1.415649 max_load_reduction=53939 max_load_reduction_pct=0.823044 moved_to_echo=33597 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 333, 78, 38, 0, 21, 4223, 1750, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 0, 0, 1628, 0, 0, 0, 24, 0, 0, 0, 3259, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5028, 3893, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 0, 0, 0, 7977, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 0, 0, 0, 0, 740, 0, 14, 0, 0, 0, 0, 0, 0, 7593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 914, 0, 0, 0, 6354, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7608, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7305, 5630, 8186, 9416, 10359, 9154, 7876, 7610] avg=8192.0000 max=10359 max_rank=4 min=5630 min_rank=1 imbalance_ratio=1.264526 max_load_reduction=55177 max_load_reduction_pct=0.841934 moved_to_echo=34999 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 456, 0, 0, 0, 0, 0, 0, 758, 0, 2, 0, 24, 0, 0, 4038, 1716, 1158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 702, 0, 0, 0, 157, 0, 232, 0, 0, 1, 0, 0, 4365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7975, 2, 0, 0, 0, 0, 1, 0, 0, 71, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 3716, 0, 3, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4, 2, 1, 8148, 0, 14, 33, 0, 0, 2, 0, 0, 1647, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 7546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 1538, 0, 0, 0, 6222, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7641, 18, 8, 4082, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 5457, 8050, 5715, 10296, 7610, 8486, 11770] avg=8192.0000 max=11770 max_rank=7 min=5457 min_rank=1 imbalance_ratio=1.436768 max_load_reduction=53766 max_load_reduction_pct=0.820404 moved_to_echo=38162 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 101, 0, 0, 0, 0, 7, 0, 299, 0, 0, 0, 0, 0, 0, 4158, 1737, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6346, 0, 0, 0, 2764, 0, 2, 2, 3, 0, 0, 0, 3044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 3832, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 7493, 0, 15, 0, 1, 0, 0, 0, 0, 1143, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 6, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 7574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 6305, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7701, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7104, 12166, 8203, 5430, 9124, 7837, 7715, 7957] avg=8192.0000 max=12166 max_rank=1 min=5430 min_rank=3 imbalance_ratio=1.485107 max_load_reduction=53370 max_load_reduction_pct=0.814362 moved_to_echo=32633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 855, 0, 0, 3, 6, 0, 1, 3773, 1729, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7388, 0, 254, 72, 0, 0, 0, 0, 4709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7986, 3478, 0, 0, 0, 22, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 35, 0, 1770, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 2, 0, 0, 0, 0, 0, 0, 0, 0, 180, 23, 0, 0, 6270, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 23, 7004, 3, 621, 39, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7610, 12423, 8160, 11518, 3326, 7584, 7188, 7727] avg=8192.0000 max=12423 max_rank=1 min=3326 min_rank=4 imbalance_ratio=1.516479 max_load_reduction=53113 max_load_reduction_pct=0.810440 moved_to_echo=25825 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 0, 0, 4, 1, 1390, 0, 0, 0, 0, 3724, 1694, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1089, 0, 0, 0, 3455, 7, 2, 2, 0, 1, 0, 417, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 11, 0, 0, 0, 12, 0, 0, 0, 0, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 468, 3432, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 7606, 0, 92, 0, 0, 0, 0, 0, 0, 1420, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 2367, 0, 0, 0, 1, 0, 0, 0, 0, 7529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 202, 0, 0, 0, 0, 3, 13, 1, 0, 0, 3657, 0, 0, 0, 6151, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6720, 0, 16, 18, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7816, 8756, 8203, 3963, 9535, 9917, 10582, 6764] avg=8192.0000 max=10582 max_rank=6 min=3963 min_rank=3 imbalance_ratio=1.291748 max_load_reduction=54954 max_load_reduction_pct=0.838531 moved_to_echo=36798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 173, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 3767, 1539, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6826, 0, 0, 0, 2337, 1, 0, 719, 280, 27, 0, 475, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8043, 12, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7712, 3473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 329, 0, 0, 9, 3238, 0, 133, 0, 45, 0, 0, 0, 6, 107, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 464, 2, 0, 24, 8, 51, 0, 0, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 38, 278, 0, 60, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5580, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 3, 6678, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5572, 10952, 8060, 11186, 4252, 8124, 6610, 10780] avg=8192.0000 max=11186 max_rank=3 min=4252 min_rank=4 imbalance_ratio=1.365479 max_load_reduction=54350 max_load_reduction_pct=0.829315 moved_to_echo=29766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6303, 0, 0, 0, 0, 1015, 1763, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 6, 2, 3167, 8, 0, 119, 0, 0, 0, 0, 4927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 12, 0, 0, 2, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7389, 937, 0, 5, 1, 0, 2, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7961, 0, 0, 0, 0, 0, 3, 0, 0, 1851, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 65, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 7616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 40, 10, 0, 0, 0, 30, 4, 0, 0, 0, 3214, 7, 0, 1, 6395, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1879, 0, 0, 281, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10380, 8236, 8149, 8407, 9923, 7733, 10547, 2161] avg=8192.0000 max=10547 max_rank=6 min=2161 min_rank=7 imbalance_ratio=1.287476 max_load_reduction=54989 max_load_reduction_pct=0.839066 moved_to_echo=30364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6300, 0, 0, 0, 0, 1, 0, 4, 0, 5, 1, 3209, 8, 0, 125, 0, 0, 0, 0, 8075, 0, 0, 0, 8023, 8188, 0, 0, 2, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 7490, 1760, 0, 2, 1, 0, 1, 0, 0, 0, 0, 0, 55, 8160, 0, 0, 0, 1, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 71, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 47, 8, 0, 0, 0, 29, 5, 0, 0, 0, 3268, 7, 0, 2, 0, 0, 0, 0, 0, 1986, 0, 0, 268, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 483, 0, 0, 0, 0, 0, 0, 769, 0, 2, 0, 29, 0, 0, 0, 0, 676, 0, 0, 0, 130, 0, 228, 0, 0, 0, 0, 0, 7178, 0, 0, 0, 7977, 8108, 0, 0, 0, 0, 2, 0, 0, 73, 0, 1, 0, 0, 0, 2018, 7739, 0, 2, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7935, 0, 0, 1, 0, 2, 5, 1, 1, 8149, 0, 14, 32, 0, 0, 2, 0, 0, 7, 0, 0, 0, 1, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 16, 7, 4076, 0, 19, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 858, 0, 0, 3, 6, 0, 1, 0, 0, 0, 0, 0, 0, 7371, 0, 222, 71, 0, 0, 0, 0, 7788, 0, 0, 0, 8145, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 8003, 7236, 0, 0, 0, 23, 32, 0, 0, 0, 0, 0, 0, 8007, 0, 0, 0, 0, 10, 0, 0, 0, 1064, 0, 0, 0, 0, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 25, 0, 0, 0, 14, 0, 0, 25, 7400, 4, 661, 37, 0, 0, 0, 0, 24, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 337, 79, 32, 0, 23, 1, 0, 716, 0, 0, 0, 1653, 0, 0, 0, 32, 0, 0, 0, 5386, 10, 0, 0, 8173, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4941, 8115, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 429, 8113, 0, 0, 0, 0, 702, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 0, 0, 0, 851, 0, 12, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 887, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 573, 0, 0, 0, 0, 0, 7151, 0, 0, 0, 0, 0, 5, 1574, 1, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 492, 0, 0, 0, 8189, 8111, 0, 0, 0, 301, 0, 0, 0, 0, 0, 21, 0, 0, 0, 7896, 4151, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 1, 7623, 0, 0, 0, 0, 78, 0, 0, 0, 8114, 4, 0, 0, 14, 0, 6, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6971, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 103] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 4, 1, 1439, 0, 0, 0, 0, 0, 4, 1112, 0, 0, 0, 3369, 7, 1, 2, 0, 3, 0, 399, 6129, 0, 0, 0, 8171, 8092, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4, 6, 0, 3, 406, 7139, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 1, 7863, 0, 0, 0, 0, 35, 0, 0, 0, 7656, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 2470, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 202, 0, 0, 0, 0, 3, 13, 1, 0, 0, 3661, 0, 0, 0, 0, 0, 0, 6, 0, 7107, 0, 17, 10, 0, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 99, 0, 0, 0, 0, 6, 0, 298, 0, 0, 0, 0, 0, 0, 4, 0, 6423, 0, 0, 0, 2700, 0, 3, 2, 0, 0, 0, 0, 5011, 0, 0, 0, 8190, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1564, 7985, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8038, 0, 0, 0, 0, 33, 0, 0, 0, 7478, 0, 16, 0, 1, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 0, 0, 0, 0, 0, 5, 0, 8151, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 176, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 6847, 0, 0, 0, 2282, 1, 0, 722, 280, 27, 0, 448, 479, 0, 0, 0, 8043, 8134, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 7716, 7238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7138, 0, 0, 0, 2, 317, 0, 0, 8, 3289, 0, 143, 0, 50, 0, 0, 0, 8, 1, 12, 463, 2, 0, 24, 10, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 37, 279, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 3, 7044, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6300, 0, 0, 0, 0, 907, 1802, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 5, 1, 3209, 8, 0, 125, 0, 0, 0, 0, 4937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 10, 0, 0, 2, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7490, 853, 0, 2, 1, 0, 1, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 3, 0, 1, 1852, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 71, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 7585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 47, 8, 0, 0, 0, 29, 5, 0, 0, 0, 3268, 7, 0, 2, 6358, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1880, 0, 0, 268, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10295, 8290, 8157, 8402, 9913, 7716, 10614, 2149] avg=8192.0000 max=10614 max_rank=6 min=2149 min_rank=7 imbalance_ratio=1.295654 max_load_reduction=54922 max_load_reduction_pct=0.838043 moved_to_echo=30392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 858, 0, 0, 3, 6, 0, 1, 3732, 1768, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7371, 0, 222, 71, 0, 0, 0, 0, 4762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 3504, 0, 0, 0, 23, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1064, 0, 0, 0, 0, 0, 0, 32, 0, 1786, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 25, 0, 0, 6239, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 25, 7003, 4, 661, 37, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7610, 12426, 8158, 11562, 3289, 7553, 7170, 7768] avg=8192.0000 max=12426 max_rank=1 min=3289 min_rank=4 imbalance_ratio=1.516846 max_load_reduction=53110 max_load_reduction_pct=0.810394 moved_to_echo=25780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 337, 79, 32, 0, 23, 4185, 1791, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 716, 0, 0, 0, 1653, 0, 0, 0, 32, 0, 0, 0, 3294, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4941, 3930, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 702, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 0, 0, 0, 851, 0, 12, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 887, 0, 0, 0, 6322, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7637, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7307, 5706, 8184, 9302, 10345, 9203, 7847, 7642] avg=8192.0000 max=10345 max_rank=4 min=5706 min_rank=1 imbalance_ratio=1.262817 max_load_reduction=55191 max_load_reduction_pct=0.842148 moved_to_echo=35037 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 4, 1, 1439, 0, 0, 0, 0, 3682, 1735, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1112, 0, 0, 0, 3369, 7, 1, 2, 0, 3, 0, 399, 3747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 10, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 406, 3457, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 7656, 0, 96, 0, 0, 0, 0, 0, 0, 1406, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 2470, 0, 0, 0, 1, 0, 0, 0, 0, 7496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 202, 0, 0, 0, 0, 3, 13, 1, 0, 0, 3661, 0, 0, 0, 6128, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6726, 0, 17, 10, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7845, 8644, 8204, 3926, 9574, 9984, 10595, 6764] avg=8192.0000 max=10595 max_rank=6 min=3926 min_rank=3 imbalance_ratio=1.293335 max_load_reduction=54941 max_load_reduction_pct=0.838333 moved_to_echo=36917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 573, 0, 0, 0, 0, 0, 7151, 0, 0, 0, 0, 2141, 1683, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1574, 1, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 9, 0, 0, 0, 301, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7896, 2010, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 8114, 4, 0, 0, 14, 0, 6, 0, 0, 112, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4, 0, 6, 0, 7514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5940, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6597, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11628, 1959, 8520, 9910, 8702, 7547, 6530, 10740] avg=8192.0000 max=11628 max_rank=0 min=1959 min_rank=1 imbalance_ratio=1.419434 max_load_reduction=53908 max_load_reduction_pct=0.822571 moved_to_echo=33519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 483, 0, 0, 0, 0, 0, 0, 769, 0, 2, 0, 29, 0, 0, 3996, 1752, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 130, 0, 228, 0, 0, 0, 0, 0, 4380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7977, 2, 0, 0, 0, 0, 2, 0, 0, 73, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2018, 3743, 0, 2, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 5, 1, 1, 8149, 0, 14, 32, 0, 0, 2, 0, 0, 1652, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 7514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 1519, 0, 0, 0, 6183, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7649, 16, 7, 4076, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 5414, 8055, 5772, 10296, 7587, 8468, 11767] avg=8192.0000 max=11767 max_rank=7 min=5414 min_rank=1 imbalance_ratio=1.436401 max_load_reduction=53769 max_load_reduction_pct=0.820450 moved_to_echo=38118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 99, 0, 0, 0, 0, 6, 0, 298, 0, 0, 0, 0, 0, 0, 4118, 1774, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6423, 0, 0, 0, 2700, 0, 3, 2, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 3867, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 7478, 0, 16, 0, 1, 0, 0, 0, 0, 1149, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 7541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 0, 0, 6264, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7714, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7093, 12196, 8201, 5440, 9114, 7846, 7693, 7953] avg=8192.0000 max=12196 max_rank=1 min=5440 min_rank=3 imbalance_ratio=1.488770 max_load_reduction=53340 max_load_reduction_pct=0.813904 moved_to_echo=32606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 176, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 3733, 1576, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6847, 0, 0, 0, 2282, 1, 0, 722, 280, 27, 0, 448, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 10, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7716, 3505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 317, 0, 0, 8, 3289, 0, 143, 0, 50, 0, 0, 0, 8, 109, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 463, 2, 0, 24, 10, 53, 0, 0, 0, 0, 0, 0, 0, 0, 7535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 37, 279, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5562, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 3, 6667, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5581, 10901, 8057, 11222, 4303, 8100, 6619, 10753] avg=8192.0000 max=11222 max_rank=3 min=4303 min_rank=4 imbalance_ratio=1.369873 max_load_reduction=54314 max_load_reduction_pct=0.828766 moved_to_echo=29775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 640, 128, 128, 768, 0, 128, 0, 1920, 128, 15232, 128, 128, 0, 128, 26880, 13696, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 40192, 24832, 0, 128, 128, 128, 128, 0, 128, 0, 0, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 17280, 128, 128, 128, 20992, 128, 512, 1024, 512, 128, 0, 896, 24704, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 512, 512, 384, 0, 128, 0, 128, 128, 128, 256, 128, 10368, 128, 0, 128, 49280, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 1280, 128, 128, 128, 51712, 128, 256, 128, 128, 0, 128, 128, 128, 9344, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 51840, 128, 768, 12928, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 1408, 128, 0, 128, 128, 3584, 0, 128, 128, 128, 128, 0, 128, 0, 60672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 65024, 128, 0, 0, 128, 384, 128, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 640, 128, 128, 768, 128, 128, 0, 2048, 128, 15360, 128, 128, 0, 128, 26496, 13952, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 40064, 24960, 0, 128, 128, 128, 128, 0, 128, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 512, 512, 384, 0, 128, 0, 128, 128, 128, 256, 128, 10368, 128, 0, 128, 49024, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 17408, 128, 128, 128, 20736, 128, 512, 1024, 512, 128, 0, 896, 24832, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 1280, 128, 128, 128, 51712, 128, 384, 128, 128, 0, 128, 128, 128, 9344, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 51968, 128, 768, 12800, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 1408, 128, 128, 128, 128, 3840, 0, 128, 128, 128, 128, 0, 128, 0, 60416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 65024, 128, 0, 0, 128, 384, 128, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 156, 0, 0, 0, 0, 4297, 0, 0, 8183, 0, 0, 2, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 60, 0, 2071, 0, 76, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 7961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 7912, 0, 0, 0, 0, 0, 0, 8190, 0, 5, 0, 0, 0, 8192, 0, 8102, 0, 118, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 179, 60, 0, 2, 0, 0, 8, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8183, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 72, 0, 13, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 5, 7345, 0, 7938, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 4037, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 19, 4188, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 1, 0, 0, 9, 0, 1, 0, 3704, 46, 163, 94, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 0, 0, 0, 0, 7942, 3, 0, 157, 0, 189, 3244, 0, 7995, 0, 1, 0, 0, 0, 0, 328, 0, 0, 0, 0, 47, 0, 5414, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7718, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 4807, 88, 3, 8158, 0, 0, 0, 8, 2, 0, 0, 3, 1, 4258, 0, 0, 0, 0, 0, 11, 7769, 0, 0, 2, 725, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 323, 0, 1, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4625, 0, 1, 1, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 1549, 0, 7834, 0, 2941, 0, 0, 3, 0, 1, 0, 0, 0, 6, 0, 0, 6379, 0, 0, 0, 0, 0, 1, 0, 3, 3, 0, 0, 0, 0, 22, 0, 0, 0, 3, 0, 0, 0, 1, 7771, 0, 4, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4582, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 13, 0, 76, 8151, 1, 4, 0, 0, 1, 21, 2566, 0, 0, 0, 205, 0, 0, 42, 0, 0, 0, 0, 0, 5, 0, 76, 0, 0, 0, 0, 0, 1620, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 7507, 0, 6, 0, 0, 0, 0, 8078, 0, 0, 0, 19, 0, 3709, 0, 7663, 6, 1265, 0, 0, 0, 0, 3623, 0, 0, 0, 0, 0, 0, 1534, 0, 0, 0, 4, 530, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 5551, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 7710, 0, 0, 8116, 0, 9, 0, 0, 4, 0, 164, 0, 0, 6963, 0, 0, 0, 0, 0, 70, 8141, 0, 27, 13, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 233, 0, 1765, 0, 0, 0, 14, 0, 4, 0, 0, 0, 1, 1, 5922, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 766, 0, 7750, 0, 0, 39, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 34, 50, 1, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8174, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 586, 0, 0, 0, 0, 3, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 96, 0, 0, 6, 0, 3, 4, 0, 13, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 7571, 0, 0, 0, 0, 0, 0, 8048, 3, 0, 0, 0, 36, 8129, 0, 7884, 0, 109, 0, 0, 0, 0, 25, 0, 15, 0, 1, 0, 0, 8168, 0, 0, 7, 0, 52, 0, 0, 0, 0, 3, 1, 8, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 15, 0, 81, 3, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 7, 0, 0, 4055, 0, 0, 8072, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5776, 0, 0, 0, 3, 0, 0, 4180, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 8009, 0, 141, 0, 0, 2, 0, 8181, 0, 0, 0, 0, 0, 5815, 0, 8165, 0, 3074, 4, 18, 0, 0, 84, 0, 14, 5, 4, 0, 0, 3590, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 63, 0, 2111, 0, 0, 0, 0, 69, 0, 12, 0, 164, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 4037, 3, 0, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 19, 4188, 0, 0, 0, 0, 0, 0, 4817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 1, 0, 0, 9, 0, 1, 0, 3704, 5760, 241, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 163, 94, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 157, 0, 189, 3244, 0, 2350, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 0, 47, 0, 5414, 0, 0, 0, 0, 42, 1620, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7718, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6478, 9028, 9838, 5777, 5944, 7741, 7889, 12841] avg=8192.0000 max=12841 max_rank=7 min=5777 min_rank=3 imbalance_ratio=1.567505 max_load_reduction=52695 max_load_reduction_pct=0.804062 moved_to_echo=34415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8183, 0, 0, 2430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 72, 5755, 239, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8174, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7345, 0, 2334, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 7, 118, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10613, 8755, 6131, 8312, 9886, 8527, 8138, 5174] avg=8192.0000 max=10613 max_rank=0 min=5174 min_rank=7 imbalance_ratio=1.295532 max_load_reduction=54923 max_load_reduction_pct=0.838058 moved_to_echo=31725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 156, 0, 0, 0, 0, 4297, 0, 0, 2429, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 60, 0, 1226, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 7961, 5754, 244, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 7912, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8192, 0, 2382, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 412, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 179, 60, 0, 2, 0, 0, 8, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6884, 2003, 14031, 8437, 10697, 9442, 8506, 5536] avg=8192.0000 max=14031 max_rank=2 min=2003 min_rank=1 imbalance_ratio=1.712769 max_load_reduction=51505 max_load_reduction_pct=0.785904 moved_to_echo=34181 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4582, 0, 0, 2420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 76, 8151, 1, 4, 0, 0, 1, 21, 1520, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 5, 0, 76, 0, 0, 0, 0, 0, 1620, 5730, 231, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 7507, 0, 6, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3709, 0, 2253, 6, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 0, 0, 0, 0, 0, 1534, 0, 0, 0, 4, 530, 510, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 8021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5551, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7027, 9992, 7761, 8430, 7252, 6479, 8106, 10489] avg=8192.0000 max=10489 max_rank=7 min=6479 min_rank=5 imbalance_ratio=1.280396 max_load_reduction=55047 max_load_reduction_pct=0.839951 moved_to_echo=32326 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 4807, 88, 3, 2422, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4258, 0, 0, 0, 0, 0, 11, 4597, 0, 0, 2, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 5736, 236, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 1, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4625, 0, 1, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 0, 2303, 0, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 6, 0, 0, 6379, 0, 0, 0, 0, 0, 1547, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 3, 0, 0, 0, 0, 22, 0, 0, 0, 3, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7771, 0, 4, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 5011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7336, 9597, 6040, 6621, 6794, 8220, 8133, 12795] avg=8192.0000 max=12795 max_rank=7 min=6040 min_rank=2 imbalance_ratio=1.561890 max_load_reduction=52741 max_load_reduction_pct=0.804764 moved_to_echo=35942 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8174, 0, 0, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 586, 0, 0, 0, 0, 3, 0, 4840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 96, 0, 0, 6, 0, 3, 4, 0, 13, 0, 5760, 237, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 7571, 0, 0, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 36, 8129, 0, 2318, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 15, 0, 1, 0, 0, 8168, 0, 0, 7, 0, 52, 1628, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 8, 0, 0, 0, 0, 0, 2, 0, 0, 1, 7991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 81, 3, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 5043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10606, 5430, 6177, 9387, 10595, 10182, 8006, 5153] avg=8192.0000 max=10606 max_rank=0 min=5153 min_rank=7 imbalance_ratio=1.294678 max_load_reduction=54930 max_load_reduction_pct=0.838165 moved_to_echo=33936 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 7710, 0, 0, 2410, 0, 9, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 6963, 0, 0, 0, 0, 0, 70, 4817, 0, 27, 13, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 233, 5706, 233, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1765, 0, 0, 0, 14, 0, 4, 0, 0, 0, 1, 1, 5922, 0, 0, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 0, 2279, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 1621, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 50, 1, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10136, 12484, 6232, 9410, 3084, 10093, 8130, 5967] avg=8192.0000 max=12484 max_rank=1 min=3084 min_rank=4 imbalance_ratio=1.523926 max_load_reduction=53052 max_load_reduction_pct=0.809509 moved_to_echo=27274 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 7, 0, 0, 4055, 0, 0, 2392, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5776, 0, 0, 0, 3, 0, 0, 2467, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 5680, 250, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 8009, 0, 141, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5815, 0, 2389, 0, 3074, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 84, 0, 14, 5, 4, 0, 0, 3590, 0, 0, 0, 0, 0, 836, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 63, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 0, 0, 0, 0, 69, 0, 12, 0, 164, 0, 0, 0, 0, 0, 5225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6456, 8247, 9326, 9162, 11284, 4852, 8628, 7581] avg=8192.0000 max=11284 max_rank=4 min=4852 min_rank=5 imbalance_ratio=1.377441 max_load_reduction=54252 max_load_reduction_pct=0.827820 moved_to_echo=32345 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8182, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 94, 0, 14, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 5, 7330, 0, 7920, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4603, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 12, 0, 72, 8142, 1, 4, 0, 0, 1, 23, 2774, 0, 0, 0, 208, 0, 0, 47, 0, 0, 0, 0, 0, 6, 0, 78, 0, 0, 0, 0, 0, 1603, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 7522, 0, 6, 0, 0, 0, 0, 8077, 0, 0, 0, 19, 0, 3626, 0, 7658, 6, 1293, 0, 0, 0, 0, 3562, 0, 0, 0, 0, 0, 0, 1405, 0, 0, 0, 4, 527, 0, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 5569, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 151, 0, 0, 0, 0, 4075, 0, 0, 8183, 0, 0, 2, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 67, 0, 2257, 0, 76, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 7958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 7925, 0, 0, 0, 0, 0, 0, 8190, 0, 5, 0, 0, 0, 8192, 0, 8105, 0, 127, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 142, 160, 61, 0, 2, 0, 0, 8, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 4031, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 20, 4079, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 1, 0, 0, 12, 0, 1, 0, 3773, 37, 192, 92, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 7935, 3, 0, 144, 0, 203, 3198, 0, 7992, 0, 1, 0, 0, 0, 0, 323, 0, 0, 0, 0, 48, 0, 5368, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 7713, 0, 0, 8116, 1, 9, 0, 0, 4, 0, 161, 0, 0, 7029, 0, 0, 0, 0, 0, 73, 8146, 0, 27, 13, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 237, 0, 1738, 0, 0, 0, 14, 0, 5, 0, 0, 0, 1, 0, 5974, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 661, 0, 7749, 0, 0, 29, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 35, 64, 1, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8176, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 579, 0, 0, 0, 0, 3, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 94, 0, 0, 5, 0, 3, 5, 0, 17, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 8051, 3, 0, 0, 0, 36, 8123, 0, 7894, 0, 111, 0, 0, 0, 0, 25, 0, 15, 0, 2, 0, 0, 8168, 0, 0, 7, 0, 58, 0, 0, 0, 0, 4, 1, 9, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 15, 0, 81, 5, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 3929, 0, 0, 8072, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5713, 0, 0, 0, 3, 0, 0, 4318, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 8007, 0, 143, 0, 0, 2, 0, 8181, 0, 0, 0, 0, 0, 6028, 0, 8168, 0, 3151, 2, 17, 0, 0, 91, 0, 20, 2, 3, 0, 0, 3415, 0, 0, 0, 0, 0, 439, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 64, 0, 2007, 0, 0, 0, 0, 70, 0, 6, 0, 167, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 5, 0, 4709, 88, 3, 8156, 0, 0, 0, 9, 2, 0, 0, 1, 1, 4244, 0, 0, 0, 0, 0, 15, 7793, 0, 0, 2, 748, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 316, 0, 1, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4581, 0, 1, 2, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 1549, 0, 7832, 0, 3109, 0, 0, 2, 0, 2, 0, 0, 0, 6, 0, 0, 6327, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 22, 0, 0, 0, 1, 0, 0, 0, 2, 7777, 0, 5, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 4031, 2, 0, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 20, 4079, 0, 0, 0, 0, 0, 0, 4783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 1, 0, 0, 12, 0, 1, 0, 3773, 5706, 283, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 192, 92, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 0, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 144, 0, 203, 3198, 0, 2320, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 48, 0, 5368, 0, 0, 0, 0, 46, 1695, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6525, 8886, 9903, 5848, 5869, 7755, 7883, 12867] avg=8192.0000 max=12867 max_rank=7 min=5848 min_rank=3 imbalance_ratio=1.570679 max_load_reduction=52669 max_load_reduction_pct=0.803665 moved_to_echo=34374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8182, 0, 0, 2484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 377, 0, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 94, 5700, 280, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8174, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7330, 0, 2300, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 9, 133, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10666, 8742, 6135, 8319, 9842, 8523, 8137, 5172] avg=8192.0000 max=10666 max_rank=0 min=5172 min_rank=7 imbalance_ratio=1.302002 max_load_reduction=54870 max_load_reduction_pct=0.837250 moved_to_echo=31674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 5, 0, 4709, 88, 3, 2475, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4244, 0, 0, 0, 0, 0, 15, 4580, 0, 0, 2, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 5681, 277, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 1, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4581, 0, 1, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 0, 2274, 0, 3109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 6, 0, 0, 6327, 0, 0, 0, 0, 0, 1624, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 22, 0, 0, 0, 1, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7777, 0, 5, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 5012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7292, 9591, 6021, 6533, 6934, 8230, 8129, 12806] avg=8192.0000 max=12806 max_rank=7 min=6021 min_rank=2 imbalance_ratio=1.563232 max_load_reduction=52730 max_load_reduction_pct=0.804596 moved_to_echo=36099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 151, 0, 0, 0, 0, 4075, 0, 0, 2484, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 67, 0, 1327, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 7958, 5699, 287, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 7925, 0, 0, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8192, 0, 2352, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 1, 470, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 160, 61, 0, 2, 0, 0, 8, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6712, 2068, 14015, 8456, 10676, 9545, 8508, 5556] avg=8192.0000 max=14015 max_rank=2 min=2068 min_rank=1 imbalance_ratio=1.710815 max_load_reduction=51521 max_load_reduction_pct=0.786148 moved_to_echo=34285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4603, 0, 0, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 72, 8142, 1, 4, 0, 0, 1, 23, 1631, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 6, 0, 78, 0, 0, 0, 0, 0, 1603, 5676, 271, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 7522, 0, 6, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3626, 0, 2224, 6, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3562, 0, 0, 0, 0, 0, 0, 1405, 0, 0, 0, 4, 527, 578, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5569, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7101, 10094, 7736, 8487, 7168, 6339, 8101, 10510] avg=8192.0000 max=10510 max_rank=7 min=6339 min_rank=5 imbalance_ratio=1.282959 max_load_reduction=55026 max_load_reduction_pct=0.839630 moved_to_echo=32118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8176, 0, 0, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 579, 0, 0, 0, 0, 3, 0, 4806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 94, 0, 0, 5, 0, 3, 5, 0, 17, 0, 5705, 279, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 7564, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 36, 8123, 0, 2292, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 15, 0, 2, 0, 0, 8168, 0, 0, 7, 0, 58, 1704, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 9, 0, 0, 0, 0, 0, 2, 0, 0, 1, 7996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 81, 5, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10662, 5389, 6165, 9330, 10565, 10251, 8013, 5161] avg=8192.0000 max=10662 max_rank=0 min=5161 min_rank=7 imbalance_ratio=1.301514 max_load_reduction=54874 max_load_reduction_pct=0.837311 moved_to_echo=33990 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 3929, 0, 0, 2446, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5713, 0, 0, 0, 3, 0, 0, 2532, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 5626, 292, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 8007, 0, 143, 883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6028, 0, 2360, 0, 3151, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 91, 0, 20, 2, 3, 0, 0, 3415, 0, 0, 0, 0, 0, 903, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 64, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2007, 0, 0, 0, 0, 70, 0, 6, 0, 167, 0, 0, 0, 0, 0, 5231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6384, 8249, 9340, 9170, 11543, 4736, 8633, 7481] avg=8192.0000 max=11543 max_rank=4 min=4736 min_rank=5 imbalance_ratio=1.409058 max_load_reduction=53993 max_load_reduction_pct=0.823868 moved_to_echo=32393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 7713, 0, 0, 2463, 1, 9, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 7029, 0, 0, 0, 0, 0, 73, 4788, 0, 27, 13, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 237, 5653, 274, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 0, 0, 14, 0, 5, 0, 0, 0, 1, 0, 5974, 0, 0, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 2249, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 1697, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 64, 1, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10194, 12517, 6221, 9393, 2939, 10157, 8132, 5983] avg=8192.0000 max=12517 max_rank=1 min=2939 min_rank=4 imbalance_ratio=1.527954 max_load_reduction=53019 max_load_reduction_pct=0.809006 moved_to_echo=27211 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 128, 128, 0, 45952, 128, 128, 19456, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 256, 128, 256, 38784, 0, 18688, 128, 7808, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 0, 256, 128, 128, 128, 0, 128, 128, 128, 128, 17024, 45952, 1920, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 128, 128, 0, 45440, 128, 128, 19840, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 256, 128, 256, 38784, 0, 18432, 128, 8064, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 0, 256, 128, 128, 128, 0, 128, 128, 128, 128, 17152, 45568, 2304, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 23680, 128, 1024, 128, 0, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 40576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 4864, 0, 128, 128, 128, 128, 0, 49536, 0, 0, 128, 128, 640, 8320, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 128, 128, 38784, 128, 128, 0, 128, 128, 128, 24704, 0, 128, 128, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 2816, 128, 128, 128, 128, 0, 128, 128, 0, 384, 128, 128, 53504, 0, 256, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 38528, 128, 128, 0, 128, 128, 128, 24832, 0, 128, 128, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 4864, 0, 128, 128, 128, 128, 0, 49152, 0, 0, 128, 128, 768, 8832, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 2816, 128, 128, 128, 128, 0, 128, 128, 0, 384, 128, 0, 53632, 0, 256, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[512, 128, 128, 128, 128, 128, 256, 256, 128, 128, 128, 0, 128, 128, 128, 128, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[512, 0, 128, 128, 128, 128, 256, 256, 128, 128, 128, 0, 128, 128, 128, 128, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7937, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7165, 0, 0, 258, 0, 0, 0, 0, 0, 8192, 148, 0, 753, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 8187, 8030, 0, 507, 0, 0, 0, 0, 7848, 1418, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[184, 2511, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 249, 0, 648, 0, 0, 0, 54, 0, 0, 0, 32, 0, 0, 0, 0, 7976, 0, 98, 16, 0, 0, 0, 0, 1, 1, 0, 0, 3, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 3561, 0, 0, 8, 0, 12, 0, 0, 7, 0, 0, 8061, 0, 0, 0, 0, 2534, 0, 0, 0, 8130, 14, 46, 0, 0, 0, 113, 0, 0, 0, 13, 0, 0, 0, 8147, 8019, 0, 0, 1298, 0, 3, 0, 5475, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 7881, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 183, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 800, 22, 0, 5135, 0, 0, 0, 8192, 56, 0, 1438, 0, 0, 0, 0, 0, 0, 2464, 0, 0, 0, 8187, 7428, 0, 0, 9, 1, 0, 0, 8048, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 27, 11, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4509, 0, 68, 0, 4, 0, 103, 0, 90, 0, 0, 1, 0, 0, 0, 0, 457, 0, 0, 50, 1, 37, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6813, 13, 0, 0, 3, 0, 0, 0, 0, 3, 0, 8142, 0, 0, 1444, 0, 531, 0, 0, 0, 8155, 0, 0, 260, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 7733, 1859, 0, 49, 495, 147, 0, 0, 8108, 3, 0, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 390, 0, 0, 0, 0, 1, 0, 31, 38, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 524, 486, 0, 1, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 236, 4726, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 767, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 126, 1, 4, 0, 58, 0, 0, 6536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2171, 0, 38, 143, 0, 41, 0, 20, 0, 7606, 149, 0, 7243, 0, 1, 0, 1, 0, 0, 1384, 0, 0, 0, 8103, 7825, 0, 0, 619, 47, 0, 0, 8103, 2, 0, 0, 306, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[5532, 577, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 34, 3, 0, 0, 0, 0, 0, 0, 29, 8130, 0, 0, 0, 0, 0, 49, 4, 1, 43, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4728, 97, 0, 0, 1, 6144, 0, 22, 0, 7232, 5363, 0, 114, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8063, 8078, 0, 0, 0, 0, 0, 2, 3026, 1654, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[435, 380, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 6580, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 154, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 283, 0, 1, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 5, 8038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 0, 518, 3240, 7, 6822, 2, 12, 111, 8192, 48, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 7485, 0, 0, 0, 43, 0, 0, 3027, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[5036, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 18, 23, 20, 0, 0, 0, 0, 0, 0, 2242, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 231, 0, 0, 15, 561, 7511, 1, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 29, 15, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4, 0, 3, 0, 0, 0, 0, 3508, 0, 32, 0, 0, 0, 0, 4, 0, 4, 0, 705, 130, 0, 0, 0, 7790, 0, 7, 0, 8068, 5331, 0, 2, 0, 0, 0, 0, 0, 0, 3338, 3, 0, 0, 8147, 8153, 0, 91, 0, 1, 0, 0, 4370, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 4, 3027, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4, 0, 0, 0, 5822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4119, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7594, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7165, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 148, 0, 753, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 0, 507, 0, 0, 0, 0, 7848, 1418, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11956, 5844, 9263, 6188, 8470, 10404, 2106, 11305] avg=8192.0000 max=11956 max_rank=0 min=2106 min_rank=6 imbalance_ratio=1.459473 max_load_reduction=53580 max_load_reduction_pct=0.817566 moved_to_echo=32285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[184, 2511, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 1358, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 648, 0, 0, 0, 54, 0, 0, 0, 32, 5794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 0, 98, 16, 0, 0, 0, 0, 1, 1, 0, 0, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4114, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7536, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1336, 0, 0, 8, 0, 12, 0, 0, 7, 0, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2534, 0, 0, 0, 594, 14, 46, 0, 0, 0, 113, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1527, 0, 0, 1298, 0, 3, 0, 5475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6688, 6777, 8959, 13665, 8406, 9424, 3314, 8303] avg=8192.0000 max=13665 max_rank=3 min=3314 min_rank=6 imbalance_ratio=1.668091 max_load_reduction=51871 max_load_reduction_pct=0.791489 moved_to_echo=29447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2086, 2202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7881, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 5822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7594, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 800, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5135, 0, 0, 0, 598, 56, 0, 1438, 0, 0, 0, 0, 0, 0, 2464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1416, 0, 0, 9, 1, 0, 0, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4316, 13799, 9682, 5305, 8467, 4802, 9691, 9474] avg=8192.0000 max=13799 max_rank=1 min=4316 min_rank=0 imbalance_ratio=1.684448 max_load_reduction=51737 max_load_reduction_pct=0.789444 moved_to_echo=32434 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 27, 11, 1, 8192, 0, 0, 0, 0, 0, 0, 2598, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4509, 0, 68, 0, 4, 0, 103, 0, 90, 0, 0, 1, 5499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 50, 1, 37, 0, 0, 0, 0, 0, 0, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7559, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2556, 13, 0, 0, 3, 0, 0, 0, 0, 3, 0, 8142, 0, 0, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 596, 0, 0, 260, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 49, 495, 147, 0, 0, 8108, 3, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11380, 10274, 2204, 10429, 8388, 12161, 1538, 9162] avg=8192.0000 max=12161 max_rank=5 min=1538 min_rank=6 imbalance_ratio=1.484497 max_load_reduction=53375 max_load_reduction_pct=0.814438 moved_to_echo=31249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 390, 0, 0, 0, 0, 1, 0, 31, 38, 1, 1, 0, 0, 0, 0, 2496, 2324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 486, 0, 1, 0, 0, 142, 0, 0, 0, 0, 0, 5764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 4726, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 767, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 126, 1, 4, 0, 58, 0, 0, 7053, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2171, 0, 38, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 20, 0, 553, 149, 0, 7243, 0, 1, 0, 1, 0, 0, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1482, 0, 0, 619, 47, 0, 0, 8103, 2, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5282, 6917, 6563, 13918, 8109, 4796, 9392, 10559] avg=8192.0000 max=13918 max_rank=3 min=4796 min_rank=5 imbalance_ratio=1.698975 max_load_reduction=51618 max_load_reduction_pct=0.787628 moved_to_echo=32856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5532, 577, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 2404, 2395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 34, 3, 0, 0, 0, 5734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 8130, 0, 0, 0, 0, 0, 49, 4, 1, 43, 0, 0, 1535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 4144, 1469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6704, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4728, 97, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 22, 0, 528, 5363, 0, 114, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 0, 0, 0, 0, 0, 2, 3026, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10914, 5780, 9791, 5889, 7577, 7192, 12172, 6221] avg=8192.0000 max=12172 max_rank=6 min=5780 min_rank=1 imbalance_ratio=1.485840 max_load_reduction=53364 max_load_reduction_pct=0.814270 moved_to_echo=33162 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[435, 380, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 3065, 2219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 6580, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 154, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 1, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 3840, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 5, 7594, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 0, 518, 3240, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6822, 2, 12, 111, 598, 48, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1426, 0, 0, 0, 43, 0, 0, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 7762, 10120, 4425, 7757, 14566, 10263, 4496] avg=8192.0000 max=14566 max_rank=5 min=4425 min_rank=3 imbalance_ratio=1.778076 max_load_reduction=50970 max_load_reduction_pct=0.777740 moved_to_echo=37082 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5036, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 18, 23, 20, 0, 0, 1338, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2242, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 231, 5795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 561, 7511, 1, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 15, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4182, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4, 0, 3, 0, 0, 0, 0, 7479, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 0, 32, 0, 0, 0, 0, 4, 0, 4, 0, 705, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7790, 0, 7, 0, 589, 5331, 0, 2, 0, 0, 0, 0, 0, 0, 3338, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1554, 0, 91, 0, 1, 0, 0, 4370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8853, 8383, 8954, 5717, 8362, 2191, 17060, 6016] avg=8192.0000 max=17060 max_rank=6 min=2191 min_rank=5 imbalance_ratio=2.082520 max_load_reduction=48476 max_load_reduction_pct=0.739685 moved_to_echo=33629 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[158, 2494, 0, 0, 63, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 270, 0, 628, 0, 0, 1, 53, 0, 0, 0, 34, 0, 1, 0, 0, 7962, 0, 95, 20, 0, 0, 0, 0, 1, 1, 0, 0, 3, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3613, 0, 0, 9, 0, 11, 0, 0, 6, 0, 0, 8066, 0, 0, 0, 0, 2578, 0, 0, 0, 8126, 16, 47, 0, 0, 0, 114, 0, 0, 0, 9, 0, 0, 0, 8146, 8020, 0, 0, 1281, 0, 3, 0, 5451, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[6284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7168, 0, 0, 277, 0, 0, 0, 0, 0, 8192, 155, 0, 640, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 8187, 8030, 0, 530, 0, 0, 0, 0, 7842, 1472, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[5392, 521, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 7, 0, 0, 0, 34, 1, 0, 0, 0, 0, 0, 0, 39, 8129, 0, 0, 0, 0, 0, 48, 3, 1, 45, 0, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4855, 100, 0, 0, 0, 6125, 0, 20, 0, 7227, 5361, 0, 127, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8072, 8044, 0, 0, 0, 0, 0, 3, 3034, 1661, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7881, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 181, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5460, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1873, 0, 803, 14, 0, 5169, 0, 0, 0, 8192, 50, 0, 1437, 0, 0, 0, 0, 0, 0, 2461, 0, 0, 0, 8186, 7426, 0, 0, 9, 0, 0, 0, 8091, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[5142, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 18, 23, 20, 0, 0, 0, 0, 0, 0, 2298, 0, 0, 0, 0, 0, 124, 0, 0, 0, 1, 238, 0, 0, 15, 566, 7572, 1, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 52, 17, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4, 0, 3, 0, 0, 0, 0, 3527, 0, 35, 0, 0, 0, 0, 4, 0, 4, 0, 764, 131, 0, 0, 0, 7799, 0, 7, 0, 8068, 5203, 0, 0, 0, 0, 0, 0, 0, 0, 3119, 3, 0, 0, 8138, 8155, 0, 92, 0, 2, 0, 0, 4364, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 28, 10, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4508, 0, 63, 0, 6, 0, 107, 0, 91, 0, 0, 1, 0, 0, 0, 0, 393, 0, 0, 41, 1, 36, 0, 0, 0, 0, 0, 0, 1, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6772, 17, 0, 0, 3, 0, 0, 0, 0, 4, 0, 8131, 0, 0, 1520, 0, 556, 0, 0, 0, 8154, 0, 0, 259, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 7739, 1921, 0, 52, 465, 155, 0, 0, 8105, 0, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 381, 0, 0, 0, 0, 3, 0, 33, 43, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 456, 492, 0, 3, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 239, 4796, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 744, 0, 7666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 125, 1, 4, 0, 56, 0, 0, 6591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 0, 39, 149, 0, 39, 0, 22, 0, 7605, 164, 0, 7254, 0, 1, 0, 0, 0, 0, 1292, 0, 0, 0, 8102, 7824, 0, 0, 620, 49, 0, 0, 8101, 2, 0, 0, 298, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[427, 358, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 6300, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 152, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 280, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 8042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7812, 0, 526, 3228, 3, 6907, 2, 17, 112, 8192, 51, 0, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 7470, 0, 0, 0, 48, 0, 0, 3061, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5392, 521, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 2485, 2337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 7, 0, 0, 0, 34, 1, 0, 0, 0, 5781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 8129, 0, 0, 0, 0, 0, 48, 3, 1, 45, 0, 0, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 4187, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6681, 887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4855, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6125, 0, 20, 0, 546, 5361, 0, 127, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 0, 0, 0, 0, 3, 3034, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10743, 5828, 9811, 5865, 7583, 7308, 12180, 6218] avg=8192.0000 max=12180 max_rank=6 min=5828 min_rank=1 imbalance_ratio=1.486816 max_load_reduction=53356 max_load_reduction_pct=0.814148 moved_to_echo=33289 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 3, 3073, 2333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 2, 0, 0, 0, 5863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4180, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7573, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7168, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 155, 0, 640, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1517, 0, 530, 0, 0, 0, 0, 7842, 1472, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12011, 5882, 9225, 6231, 8475, 10357, 1992, 11363] avg=8192.0000 max=12011 max_rank=0 min=1992 min_rank=6 imbalance_ratio=1.466187 max_load_reduction=53525 max_load_reduction_pct=0.816727 moved_to_echo=32187 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[158, 2494, 0, 0, 63, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 199, 1406, 2331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 628, 0, 0, 1, 53, 0, 0, 0, 34, 5834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7962, 0, 95, 20, 0, 0, 0, 0, 1, 1, 0, 0, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4175, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7512, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 0, 0, 9, 0, 11, 0, 0, 6, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2578, 0, 0, 0, 614, 16, 47, 0, 0, 0, 114, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 0, 0, 1281, 0, 3, 0, 5451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6652, 6820, 8955, 13652, 8406, 9424, 3378, 8249] avg=8192.0000 max=13652 max_rank=3 min=3378 min_rank=6 imbalance_ratio=1.666504 max_load_reduction=51884 max_load_reduction_pct=0.791687 moved_to_echo=29457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2126, 2158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7881, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 5863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7573, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1873, 0, 803, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5169, 0, 0, 0, 619, 50, 0, 1437, 0, 0, 0, 0, 0, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 0, 0, 9, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4308, 13849, 9671, 5295, 8471, 4703, 9736, 9503] avg=8192.0000 max=13849 max_rank=1 min=4308 min_rank=0 imbalance_ratio=1.690552 max_load_reduction=51687 max_load_reduction_pct=0.788681 moved_to_echo=32413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 28, 10, 2, 8192, 0, 0, 0, 0, 0, 0, 2637, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4508, 0, 63, 0, 6, 0, 107, 0, 91, 0, 0, 1, 5544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 41, 1, 36, 0, 0, 0, 0, 0, 0, 1640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1000, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7538, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2495, 17, 0, 0, 3, 0, 0, 0, 0, 4, 0, 8131, 0, 0, 1520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 616, 0, 0, 259, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 52, 465, 155, 0, 0, 8105, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11427, 10320, 2111, 10414, 8392, 12170, 1558, 9144] avg=8192.0000 max=12170 max_rank=5 min=1558 min_rank=6 imbalance_ratio=1.485596 max_load_reduction=53366 max_load_reduction_pct=0.814301 moved_to_echo=31264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[427, 358, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 3131, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 6300, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 152, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 3888, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 7573, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7812, 0, 526, 3228, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6907, 2, 17, 112, 619, 51, 0, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1411, 0, 0, 0, 48, 0, 0, 3061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 7453, 10105, 4448, 7731, 14533, 10608, 4520] avg=8192.0000 max=14533 max_rank=5 min=4448 min_rank=3 imbalance_ratio=1.774048 max_load_reduction=51003 max_load_reduction_pct=0.778244 moved_to_echo=37392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 381, 0, 0, 0, 0, 3, 0, 33, 43, 1, 2, 0, 0, 0, 0, 2570, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 492, 0, 3, 0, 0, 138, 0, 0, 0, 0, 0, 5804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 4796, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 0, 7666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 125, 1, 4, 0, 56, 0, 0, 7034, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 0, 39, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 22, 0, 571, 164, 0, 7254, 0, 1, 0, 0, 0, 0, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 0, 0, 620, 49, 0, 0, 8101, 2, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5310, 6893, 6638, 13895, 8113, 4803, 9343, 10541] avg=8192.0000 max=13895 max_rank=3 min=4803 min_rank=5 imbalance_ratio=1.696167 max_load_reduction=51641 max_load_reduction_pct=0.787979 moved_to_echo=32800 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5142, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 18, 23, 20, 0, 0, 1373, 2370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2298, 0, 0, 0, 0, 0, 124, 0, 0, 0, 1, 238, 5830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 566, 7572, 1, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 17, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4245, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4, 0, 3, 0, 0, 0, 0, 7458, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 0, 35, 0, 0, 0, 0, 4, 0, 4, 0, 764, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7799, 0, 7, 0, 610, 5203, 0, 0, 0, 0, 0, 0, 0, 0, 3119, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1540, 0, 92, 0, 2, 0, 0, 4364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8947, 8491, 9020, 5736, 8365, 2238, 16741, 5998] avg=8192.0000 max=16741 max_rank=6 min=2238 min_rank=5 imbalance_ratio=2.043579 max_load_reduction=48795 max_load_reduction_pct=0.744553 moved_to_echo=33342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[17536, 3968, 0, 128, 128, 0, 128, 128, 128, 8320, 128, 384, 128, 128, 0, 256, 18432, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[29056, 128, 128, 128, 4736, 11136, 128, 12544, 0, 128, 128, 128, 128, 0, 8064, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[896, 0, 24960, 128, 128, 0, 128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 29184, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[18176, 128, 128, 128, 128, 128, 0, 128, 128, 128, 0, 40704, 256, 1408, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 21760, 768, 128, 768, 128, 128, 512, 128, 128, 0, 0, 512, 41344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 10880, 0, 768, 2432, 256, 128, 128, 48128, 3200, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 1024, 52480, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[17536, 3840, 0, 128, 128, 0, 128, 128, 128, 8320, 128, 384, 128, 128, 0, 256, 18816, 16640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[29184, 128, 128, 128, 4864, 11008, 128, 12672, 0, 128, 128, 0, 128, 0, 7680, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 0, 24960, 128, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 29696, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[17792, 128, 128, 128, 128, 128, 0, 128, 128, 128, 0, 40960, 256, 1408, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 21504, 768, 128, 640, 128, 128, 512, 128, 128, 0, 128, 512, 41600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 10752, 0, 768, 2432, 256, 128, 128, 48128, 3200, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 1152, 52608, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 0, 128, 256, 128, 128, 0, 128, 0, 128, 59136, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 0, 128, 256, 128, 128, 0, 128, 0, 128, 59008, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 7880, 0, 0, 6332, 0, 0, 0, 0, 0, 0, 8034, 7977, 285, 0, 0, 0, 0, 0, 0, 0, 6817, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 215, 0, 52, 121, 0, 19, 0, 0, 0, 0, 1650, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 1, 8189, 691, 6, 0, 6375, 0, 4, 18, 919, 0, 0, 0, 0, 0, 33, 0, 0, 0, 7, 2, 0, 0, 0, 7610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 260] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 8188, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8169, 8139, 0, 0, 0, 2, 0, 0, 0, 0, 840, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2924, 41, 4, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7921, 4, 4, 0, 6513, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7083, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 0, 0, 3, 0, 0, 7103] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8142, 8190, 0, 0, 0, 1388, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3496, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8192, 5, 0, 0, 4331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8156, 0, 0, 8174, 0, 0, 0, 246, 0, 4, 7962, 8180, 359, 0, 0, 1, 0, 4, 0, 1, 0, 15, 0, 0, 3, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 6, 0, 7, 0, 7648, 2, 0, 0, 0, 0, 0, 0, 20, 0, 17, 0, 59, 0, 0, 0, 0, 4, 0, 8184, 9, 0, 0, 11, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6326, 0, 0, 0, 0, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 7682] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8180, 0, 0, 8185, 0, 0, 0, 1, 0, 0, 8152, 8142, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 1385, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 21, 0, 0, 8180, 579, 0, 0, 7814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7866, 0, 0, 36, 0, 6, 5811, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 207, 0, 0, 2, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 21, 20, 0, 0, 0, 0, 0, 60, 743] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8163, 0, 0, 7391, 0, 0, 0, 24, 0, 0, 8192, 8178, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 404, 0, 0, 0, 6866, 0, 0, 0, 2, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 7790, 2250, 6, 0, 120, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 797, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8109, 0, 0, 7946, 0, 8, 0, 0, 0, 0, 4941, 8048, 3, 2, 98, 3040, 0, 0, 0, 0, 17, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 1, 0, 6, 0, 1, 0, 0, 0, 1, 0, 0, 0, 32, 0, 9, 0, 8104, 0, 0, 0, 7531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 9, 8128, 0, 664, 56, 0, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 15, 0, 0, 4, 0, 0, 0, 155, 175, 157, 0, 3, 0, 0, 0, 0, 0, 78, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8181, 0, 0, 8117, 74, 0, 0, 1, 0, 0, 8080, 8150, 0, 0, 0, 1996, 2, 0, 3, 0, 0, 0, 0, 1, 2, 0, 0, 14, 36, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7882, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 9, 0, 7038, 565, 0, 3, 4502, 1, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 7618, 0, 1, 0, 0, 0, 223, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 24, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 472, 233, 2112] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, 8034, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6817, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6065, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 52, 121, 0, 19, 0, 0, 0, 0, 1650, 0, 0, 0, 1, 1486, 646, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 1, 7543, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6375, 0, 4, 18, 919, 0, 0, 0, 0, 0, 33, 0, 0, 0, 2756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 7006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 7770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 260, 1809, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8600, 12995, 4794, 10034, 10111, 7015, 8003, 3984] avg=8192.0000 max=12995 max_rank=1 min=3984 min_rank=7 imbalance_ratio=1.586304 max_load_reduction=52541 max_load_reduction_pct=0.801712 moved_to_echo=29113 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 840, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 6188, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2924, 41, 4, 0, 0, 1921, 625, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7296, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6513, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6520, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 0, 0, 3, 0, 0, 7103, 2340, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8533, 7172, 6084, 7626, 10093, 6533, 8086, 11409] avg=8192.0000 max=11409 max_rank=7 min=6084 min_rank=2 imbalance_ratio=1.392700 max_load_reduction=54127 max_load_reduction_pct=0.825912 moved_to_echo=36121 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1388, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 850, 0, 0, 0, 0, 6227, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3496, 3, 0, 0, 0, 1920, 646, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 7546, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6489, 2338, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8505, 8580, 6714, 7564, 7893, 7411, 8078, 10791] avg=8192.0000 max=10791 max_rank=7 min=6714 min_rank=2 imbalance_ratio=1.317261 max_load_reduction=54745 max_load_reduction_pct=0.835342 moved_to_echo=34173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 362, 0, 0, 0, 246, 0, 4, 7962, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 1, 0, 15, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6221, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 6, 0, 7, 0, 7648, 2, 0, 0, 0, 1918, 646, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 17, 0, 59, 0, 0, 0, 0, 4, 0, 7538, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5824, 0, 0, 0, 0, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 609, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 7682, 2336, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8933, 6360, 10854, 7647, 3570, 7445, 8651, 12076] avg=8192.0000 max=12076 max_rank=7 min=3570 min_rank=4 imbalance_ratio=1.474121 max_load_reduction=53460 max_load_reduction_pct=0.815735 moved_to_echo=31742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 339, 0, 8, 0, 0, 0, 0, 4941, 0, 3, 2, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3040, 0, 0, 0, 0, 17, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6120, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 1, 0, 1870, 644, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 1, 0, 0, 0, 32, 0, 9, 0, 7460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 7479, 0, 664, 56, 0, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 15, 0, 0, 4, 7995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 175, 157, 0, 3, 0, 0, 0, 0, 0, 78, 0, 0, 2275, 1928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5391, 9292, 11269, 7509, 10997, 8220, 8087, 4771] avg=8192.0000 max=11269 max_rank=2 min=4771 min_rank=7 imbalance_ratio=1.375610 max_load_reduction=54267 max_load_reduction_pct=0.828049 moved_to_echo=32075 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 362, 0, 0, 0, 1, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 6191, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 1385, 0, 0, 0, 0, 1921, 645, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 21, 0, 0, 7535, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7241, 0, 0, 36, 0, 6, 5811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 207, 0, 0, 2, 0, 0, 0, 0, 0, 38, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 20, 0, 0, 0, 0, 0, 60, 743, 2339, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8515, 6356, 4591, 8141, 11377, 13094, 8328, 5134] avg=8192.0000 max=13094 max_rank=5 min=4591 min_rank=2 imbalance_ratio=1.598389 max_load_reduction=52442 max_load_reduction_pct=0.800201 moved_to_echo=37933 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 328, 0, 0, 0, 24, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 404, 0, 0, 0, 6866, 0, 0, 0, 2, 1734, 615, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 7175, 2250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 120, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 1, 7540, 0, 0, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 2112, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8544, 6411, 10283, 9944, 3344, 8567, 8235, 10208] avg=8192.0000 max=10283 max_rank=2 min=3344 min_rank=4 imbalance_ratio=1.255249 max_load_reduction=55253 max_load_reduction_pct=0.843094 moved_to_echo=30354 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 360, 74, 0, 0, 1, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1996, 2, 0, 3, 0, 0, 0, 0, 1, 2, 0, 0, 14, 36, 3, 0, 6198, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7882, 0, 0, 0, 0, 1905, 555, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 9, 0, 6483, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4502, 1, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 20, 3533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7013, 0, 1, 0, 0, 0, 223, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 472, 233, 2112, 2319, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8515, 8370, 10947, 7071, 8151, 7251, 8068, 7163] avg=8192.0000 max=10947 max_rank=2 min=7071 min_rank=3 imbalance_ratio=1.336304 max_load_reduction=54589 max_load_reduction_pct=0.832962 moved_to_echo=30633 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 7894, 0, 0, 6318, 0, 0, 0, 0, 0, 0, 8032, 7979, 263, 0, 0, 0, 0, 0, 0, 0, 6898, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 218, 0, 41, 116, 0, 21, 0, 0, 0, 0, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1750, 1, 8190, 713, 7, 0, 6343, 0, 5, 25, 916, 0, 0, 0, 0, 0, 28, 0, 0, 0, 7, 3, 0, 0, 0, 7607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 272] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8157, 8188, 1, 0, 0, 1387, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3382, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8192, 3, 0, 0, 4329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6587] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8180, 0, 0, 8184, 0, 0, 0, 1, 0, 0, 8153, 8141, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 1484, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 22, 0, 0, 8183, 587, 0, 0, 7823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7861, 0, 0, 36, 0, 5, 5564, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 207, 0, 0, 3, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 17, 21, 0, 0, 0, 0, 0, 62, 877] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 8188, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8169, 8139, 0, 0, 0, 2, 0, 0, 0, 0, 845, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2899, 35, 5, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7939, 3, 3, 0, 6434, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7051, 0, 0, 0, 0, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2, 0, 0, 7234] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8165, 0, 0, 7396, 0, 0, 0, 22, 0, 0, 8192, 8179, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 11, 0, 0, 408, 0, 0, 0, 6879, 0, 0, 0, 2, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 7791, 2223, 5, 0, 139, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 884, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8108, 0, 0, 7946, 0, 8, 0, 0, 0, 0, 5007, 8050, 4, 2, 123, 2958, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 0, 0, 1, 1, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 33, 0, 7, 0, 8108, 0, 0, 0, 7528, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 10, 8132, 0, 665, 38, 0, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 20, 0, 0, 4, 0, 0, 0, 157, 157, 158, 0, 2, 0, 0, 0, 0, 0, 80, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8181, 0, 0, 8118, 73, 0, 0, 0, 0, 0, 8066, 8150, 0, 0, 0, 1961, 2, 0, 3, 0, 0, 0, 0, 2, 2, 0, 0, 17, 48, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7887, 2, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 10, 0, 6938, 577, 0, 2, 4399, 1, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 7616, 0, 1, 0, 0, 0, 213, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 21, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 455, 229, 2372] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8159, 0, 0, 8174, 0, 0, 1, 250, 0, 5, 7966, 8182, 319, 0, 0, 3, 0, 4, 0, 0, 0, 16, 0, 0, 3, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 7653, 4, 0, 0, 0, 0, 0, 0, 20, 0, 15, 0, 60, 0, 0, 0, 0, 7, 0, 8183, 9, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6369, 0, 1, 0, 0, 1616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 7668] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 276, 0, 0, 0, 0, 0, 0, 8032, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6898, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6068, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 41, 116, 0, 21, 0, 0, 0, 0, 1655, 0, 0, 0, 0, 1515, 630, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1750, 1, 7560, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 6343, 0, 5, 25, 916, 0, 0, 0, 0, 0, 28, 0, 0, 0, 2770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 0, 0, 7025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 7784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 272, 1757, 1911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8571, 13078, 4778, 10024, 10094, 7035, 8013, 3943] avg=8192.0000 max=13078 max_rank=1 min=3943 min_rank=7 imbalance_ratio=1.596436 max_load_reduction=52458 max_load_reduction_pct=0.800446 moved_to_echo=29085 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 358, 0, 0, 0, 0, 0, 0, 8157, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 855, 0, 0, 0, 0, 6227, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3382, 4, 0, 0, 0, 1962, 630, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 7562, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6587, 2276, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8516, 8583, 6603, 7578, 7917, 7435, 8079, 10825] avg=8192.0000 max=10825 max_rank=7 min=6603 min_rank=2 imbalance_ratio=1.321411 max_load_reduction=54711 max_load_reduction_pct=0.834824 moved_to_echo=34256 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 358, 0, 0, 0, 1, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 6191, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 1484, 0, 0, 0, 0, 1962, 629, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 22, 0, 0, 7554, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7260, 0, 0, 36, 0, 5, 5564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 207, 0, 0, 3, 0, 0, 0, 0, 0, 33, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 21, 0, 0, 0, 0, 0, 62, 877, 2276, 1950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8512, 6358, 4690, 8170, 11411, 12865, 8327, 5203] avg=8192.0000 max=12865 max_rank=5 min=4690 min_rank=2 imbalance_ratio=1.570435 max_load_reduction=52671 max_load_reduction_pct=0.803696 moved_to_echo=37806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 358, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 845, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 6189, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2899, 35, 5, 0, 0, 1963, 611, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7328, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6434, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6512, 0, 0, 0, 0, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2, 0, 0, 7234, 2277, 1950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8528, 7170, 6058, 7643, 10045, 6528, 8088, 11476] avg=8192.0000 max=11476 max_rank=7 min=6058 min_rank=2 imbalance_ratio=1.400879 max_load_reduction=54060 max_load_reduction_pct=0.824890 moved_to_echo=36137 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 324, 0, 0, 0, 22, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6220, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 11, 0, 0, 408, 0, 0, 0, 6879, 0, 0, 0, 2, 1773, 599, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 7192, 2223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 139, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 1, 7565, 0, 0, 0, 0, 0, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6046, 2057, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8538, 6416, 10300, 9921, 3387, 8678, 8233, 10063] avg=8192.0000 max=10300 max_rank=2 min=3387 min_rank=4 imbalance_ratio=1.257324 max_load_reduction=55236 max_load_reduction_pct=0.842834 moved_to_echo=30361 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 358, 0, 0, 1, 250, 0, 5, 7966, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 16, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6223, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 7653, 4, 0, 0, 0, 1960, 629, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 15, 0, 60, 0, 0, 0, 0, 7, 0, 7554, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5882, 0, 1, 0, 0, 1616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 597, 0, 0, 0, 8045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 7668, 2273, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8899, 6363, 10878, 7665, 3592, 7499, 8643, 11997] avg=8192.0000 max=11997 max_rank=7 min=3592 min_rank=4 imbalance_ratio=1.464478 max_load_reduction=53539 max_load_reduction_pct=0.816940 moved_to_echo=31731 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 334, 0, 8, 0, 0, 0, 0, 5007, 0, 4, 2, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2958, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6123, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 0, 0, 1, 1, 1909, 628, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 33, 0, 7, 0, 7480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7528, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 3488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7507, 0, 665, 38, 0, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 20, 0, 0, 4, 7995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 157, 158, 0, 2, 0, 0, 0, 0, 0, 80, 0, 0, 2215, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5478, 9217, 11279, 7527, 11021, 8234, 8084, 4696] avg=8192.0000 max=11279 max_rank=2 min=4696 min_rank=7 imbalance_ratio=1.376831 max_load_reduction=54257 max_load_reduction_pct=0.827896 moved_to_echo=32035 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 355, 73, 0, 0, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1961, 2, 0, 3, 0, 0, 0, 0, 2, 2, 0, 0, 17, 48, 3, 0, 6199, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7887, 2, 0, 0, 0, 1946, 533, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 10, 0, 6405, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4399, 1, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 21, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7034, 0, 1, 0, 0, 0, 213, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 455, 229, 2372, 2258, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8494, 8351, 10950, 7008, 8069, 7262, 8069, 7333] avg=8192.0000 max=10950 max_rank=2 min=7008 min_rank=3 imbalance_ratio=1.336670 max_load_reduction=54586 max_load_reduction_pct=0.832916 moved_to_echo=30733 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 256, 256, 256, 0, 256, 128, 0, 0, 0, 128, 640, 384, 30592, 17920, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 256, 128, 128, 256, 0, 128, 512, 128, 128, 0, 39936, 128, 128, 128, 128, 14720, 5120, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 37248, 128, 128, 128, 1024, 0, 0, 128, 128, 0, 128, 0, 128, 128, 27264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[6528, 128, 128, 128, 128, 7680, 128, 0, 128, 128, 128, 896, 128, 128, 128, 0, 49536, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 256, 256, 256, 0, 256, 128, 0, 0, 0, 128, 640, 384, 31104, 17408, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 256, 128, 128, 128, 0, 128, 512, 128, 128, 0, 39936, 128, 128, 128, 128, 15104, 4992, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 37120, 128, 128, 128, 1024, 0, 0, 128, 128, 0, 128, 0, 128, 128, 27520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[6400, 128, 128, 128, 0, 7808, 128, 0, 128, 128, 128, 896, 128, 128, 128, 0, 49536, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 0, 0, 128, 56064, 0, 768, 128, 0, 1664, 6912, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 2816, 128, 128, 0, 384, 0, 128, 61696, 0, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 0, 0, 128, 56320, 0, 768, 128, 0, 1664, 6784, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 2816, 128, 128, 128, 384, 0, 128, 61824, 0, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 640, 128, 128, 128, 128, 128, 128, 0, 128, 128, 2048, 128, 58752, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 768, 128, 128, 128, 128, 128, 128, 0, 128, 128, 2048, 128, 58624, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 384, 0, 256, 128, 0, 256, 640, 128, 0, 128, 64256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8134, 6268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1814, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 8170, 0, 0, 0, 0, 7985, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[7905, 0, 0, 0, 0, 0, 0, 7610, 0, 0, 85, 0, 0, 0, 0, 32, 0, 0, 10, 0, 102, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2, 69, 233, 0, 0, 0, 0, 0, 0, 7988, 0, 0, 0, 0, 5658, 7576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 168, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 4, 8100, 0, 0, 0, 596, 0, 0, 0, 8, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 22, 0, 0, 3, 0, 6, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8161, 7930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 44, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7273, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 0, 0, 0, 8114, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 779, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[8161, 0, 0, 0, 11, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6954, 0, 4, 0, 0, 363, 0, 2, 8160, 0, 0, 0, 39, 4526, 3675, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1553, 0, 0, 0, 4, 0, 5, 0, 0, 0, 8150, 0, 2, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 5, 7169, 0, 0, 0, 8151, 0, 0, 87, 0, 8077, 5, 0, 3, 0, 0, 0, 0, 0, 7, 0, 0, 0, 16, 0, 0, 0, 0, 6, 5, 3, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[8177, 0, 0, 0, 0, 0, 0, 4776, 23, 46, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 7, 0, 552, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 8176, 7632, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 88, 1131, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8057, 0, 0, 0, 1380, 15, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 1, 0, 569, 2, 102, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[8172, 0, 0, 0, 1, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 7, 0, 0, 6, 0, 0, 0, 0, 0, 70, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8131, 0, 0, 0, 40, 8036, 8173, 1, 5, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 28, 22, 0, 11, 0, 0, 0, 0, 0, 0, 59, 8154, 0, 22, 0, 0, 2, 2, 0, 0, 0, 3, 0, 0, 85, 0, 8107, 0, 18, 0, 8012, 0, 0, 0, 0, 8086, 0, 0, 0, 10, 0, 0, 0, 0, 127, 0, 0, 0, 35, 0, 2, 0, 0, 0, 0, 18, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[8025, 0, 0, 163, 0, 0, 0, 0, 0, 3108, 0, 0, 3, 0, 88, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 5, 0, 4, 0, 0, 0, 47, 0, 37, 22, 0, 0, 8169, 0, 12, 2, 9, 8039, 8173, 0, 0, 0, 0, 0, 0, 55, 0, 19, 0, 0, 0, 0, 0, 0, 0, 514, 120, 0, 253, 0, 0, 0, 0, 1, 0, 0, 3, 7564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7679, 0, 0, 2, 5125, 0, 0, 95, 0, 7914, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 197, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 6, 0, 1129, 0, 0, 5, 3, 4, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 0, 8163, 7166, 2, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 43, 142, 0, 0, 0, 0, 0, 230, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 4, 0, 0, 7642, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5991, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7213, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 7930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4139, 2201, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 2288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 779, 0, 3, 0, 0, 6805, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6397, 7363, 7253, 9144, 7581, 8048, 10402, 9348] avg=8192.0000 max=10402 max_rank=6 min=6397 min_rank=0 imbalance_ratio=1.269775 max_load_reduction=55134 max_load_reduction_pct=0.841278 moved_to_echo=35379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5781, 0, 0, 0, 0, 0, 0, 7610, 0, 0, 85, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 102, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 9, 8123, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 69, 233, 0, 0, 0, 0, 0, 0, 7072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 837, 7576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 168, 4135, 2124, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 4, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 0, 0, 0, 8, 2286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 22, 0, 0, 3, 0, 6, 0, 0, 0, 0, 1, 0, 4717, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13508, 8351, 7376, 8413, 8499, 9695, 2890, 6804] avg=8192.0000 max=13508 max_rank=0 min=2890 min_rank=6 imbalance_ratio=1.648926 max_load_reduction=52028 max_load_reduction_pct=0.793884 moved_to_echo=27888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5968, 0, 0, 0, 11, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6954, 0, 4, 0, 0, 363, 0, 2, 7224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 669, 3675, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1553, 0, 0, 0, 4, 0, 5, 0, 0, 0, 4083, 2193, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 2, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 5, 7169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 87, 0, 2257, 5, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 16, 0, 0, 0, 0, 6, 5, 3, 0, 0, 3, 3774, 1737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5982, 8445, 14553, 4395, 8826, 7281, 10503, 5551] avg=8192.0000 max=14553 max_rank=2 min=4395 min_rank=3 imbalance_ratio=1.776489 max_load_reduction=50983 max_load_reduction_pct=0.777939 moved_to_echo=32161 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5980, 0, 0, 0, 0, 0, 0, 4776, 23, 46, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 7, 0, 552, 8121, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 7229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 7632, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 88, 1131, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4130, 2197, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1380, 15, 0, 0, 0, 2282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 1, 0, 569, 2, 102, 0, 0, 6817, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10831, 8837, 7234, 8843, 8626, 8144, 3677, 9344] avg=8192.0000 max=10831 max_rank=0 min=3677 min_rank=6 imbalance_ratio=1.322144 max_load_reduction=54705 max_load_reduction_pct=0.834732 moved_to_echo=29791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5991, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8123, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 7253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1203, 6268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1814, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4037, 2201, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 2231, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 6782, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6280, 8273, 7310, 7471, 8995, 8185, 10405, 8617] avg=8192.0000 max=10405 max_rank=6 min=6280 min_rank=0 imbalance_ratio=1.270142 max_load_reduction=55131 max_load_reduction_pct=0.841232 moved_to_echo=36202 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5866, 0, 0, 163, 0, 0, 0, 0, 0, 3108, 0, 0, 3, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 7505, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 47, 0, 37, 22, 0, 0, 7226, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 1182, 8173, 0, 0, 0, 0, 0, 0, 55, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 120, 0, 253, 0, 0, 0, 0, 1, 0, 0, 3, 4003, 2159, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5125, 0, 0, 95, 0, 2206, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 197, 0, 6707, 1705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9228, 7713, 7353, 9440, 7996, 7738, 7437, 8631] avg=8192.0000 max=9440 max_rank=3 min=7353 min_rank=2 imbalance_ratio=1.152344 max_load_reduction=56096 max_load_reduction_pct=0.855957 moved_to_echo=31802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5990, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 6, 0, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 4, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8060, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1207, 7166, 2, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 142, 0, 0, 0, 0, 0, 230, 0, 0, 0, 4120, 2201, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 7642, 0, 0, 0, 0, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6806, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7302, 8228, 7250, 8382, 7674, 8157, 9923, 8620] avg=8192.0000 max=9923 max_rank=6 min=7250 min_rank=2 imbalance_ratio=1.211304 max_load_reduction=55613 max_load_reduction_pct=0.848587 moved_to_echo=34374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5976, 0, 0, 0, 1, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 6, 0, 0, 0, 0, 0, 70, 0, 2, 0, 0, 7, 0, 8087, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 1189, 8173, 1, 5, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 28, 22, 0, 11, 0, 0, 0, 0, 0, 0, 59, 4088, 2196, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 22, 0, 0, 2, 2, 0, 0, 0, 3, 0, 0, 85, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8012, 0, 0, 0, 0, 2259, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 35, 0, 2, 0, 0, 0, 0, 18, 0, 0, 0, 6700, 1739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6008, 8326, 7207, 9448, 7339, 8288, 10299, 8621] avg=8192.0000 max=10299 max_rank=6 min=6008 min_rank=0 imbalance_ratio=1.257202 max_load_reduction=55237 max_load_reduction_pct=0.842850 moved_to_echo=34547 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[7908, 0, 0, 0, 0, 0, 0, 7598, 0, 0, 92, 0, 0, 0, 0, 24, 0, 0, 10, 0, 100, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 71, 232, 0, 0, 0, 0, 0, 0, 7984, 0, 0, 0, 0, 5691, 7550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 181, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1528, 3, 8101, 0, 0, 0, 587, 0, 0, 0, 9, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 20, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[8178, 0, 0, 0, 0, 0, 0, 4687, 20, 46, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 0, 0, 0, 0, 0, 6, 0, 507, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 8174, 7623, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 89, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 1, 8054, 0, 0, 0, 1453, 15, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 3, 0, 586, 0, 119, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 1, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8163, 7948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 45, 0, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7264, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 8125, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 780, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 5, 0, 1187, 0, 0, 6, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 8149, 7119, 2, 3, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 39, 124, 0, 0, 0, 0, 0, 232, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 4, 0, 0, 7691, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 2, 0, 0, 52, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[8161, 0, 0, 0, 10, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7062, 0, 5, 0, 0, 355, 0, 2, 8160, 0, 0, 0, 38, 4622, 3448, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 1572, 0, 0, 0, 0, 1, 4, 0, 0, 0, 8148, 0, 6, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 6, 7160, 0, 0, 0, 8154, 0, 0, 87, 0, 8075, 5, 0, 4, 0, 0, 0, 0, 0, 9, 0, 0, 0, 14, 0, 0, 0, 0, 6, 5, 3, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8122, 6275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1826, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 8159, 0, 0, 0, 0, 7989, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[8171, 0, 0, 0, 1, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4, 0, 0, 0, 0, 0, 66, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8131, 0, 0, 0, 42, 8033, 8173, 1, 6, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 22, 21, 0, 11, 0, 0, 0, 0, 0, 0, 59, 8154, 0, 22, 0, 0, 1, 1, 0, 0, 0, 2, 0, 0, 77, 0, 8109, 0, 18, 0, 8032, 0, 0, 0, 0, 8088, 0, 0, 0, 9, 0, 0, 0, 0, 129, 0, 0, 0, 34, 0, 2, 0, 0, 0, 0, 18, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[8029, 0, 0, 156, 0, 0, 0, 0, 0, 3038, 0, 0, 3, 0, 94, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 5, 0, 2, 0, 0, 0, 44, 0, 36, 15, 0, 0, 8169, 0, 12, 2, 9, 8042, 8172, 0, 0, 0, 0, 0, 0, 56, 0, 19, 0, 0, 0, 0, 0, 0, 0, 516, 118, 0, 251, 0, 0, 0, 0, 1, 0, 0, 3, 7560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7704, 0, 0, 2, 5183, 0, 0, 88, 0, 7922, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 196, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5998, 0, 0, 0, 0, 0, 0, 4687, 20, 46, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 0, 0, 0, 0, 0, 6, 0, 507, 8123, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 7218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1246, 7623, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 89, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4130, 2180, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 1, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1453, 15, 0, 0, 0, 2258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 3, 0, 586, 0, 119, 0, 0, 6772, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10757, 8800, 7224, 8874, 8649, 8141, 3726, 9365] avg=8192.0000 max=10757 max_rank=0 min=3726 min_rank=6 imbalance_ratio=1.313110 max_load_reduction=54779 max_load_reduction_pct=0.835861 moved_to_echo=29881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5800, 0, 0, 0, 0, 0, 0, 7598, 0, 0, 92, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 100, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 8124, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 71, 232, 0, 0, 0, 0, 0, 0, 7058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 867, 7550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 181, 4136, 2108, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1528, 3, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 587, 0, 0, 0, 9, 2260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 20, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 4715, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13514, 8350, 7362, 8417, 8526, 9698, 2856, 6813] avg=8192.0000 max=13514 max_rank=0 min=2856 min_rank=6 imbalance_ratio=1.649658 max_load_reduction=52022 max_load_reduction_pct=0.793793 moved_to_echo=27893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6009, 0, 0, 0, 0, 0, 0, 1, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7205, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 7948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4140, 2183, 950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 2263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 780, 0, 3, 0, 0, 6763, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6406, 7361, 7242, 9200, 7562, 8043, 10388, 9334] avg=8192.0000 max=10388 max_rank=6 min=6406 min_rank=0 imbalance_ratio=1.268066 max_load_reduction=55148 max_load_reduction_pct=0.841492 moved_to_echo=35327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6008, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 5, 0, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8060, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1241, 7119, 2, 3, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 124, 0, 0, 0, 0, 0, 232, 0, 0, 0, 4119, 2183, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 7691, 0, 0, 0, 0, 2252, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6752, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7358, 8232, 7240, 8371, 7646, 8152, 9949, 8588] avg=8192.0000 max=9949 max_rank=6 min=7240 min_rank=2 imbalance_ratio=1.214478 max_load_reduction=55587 max_load_reduction_pct=0.848190 moved_to_echo=34335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5986, 0, 0, 0, 10, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7062, 0, 5, 0, 0, 355, 0, 2, 7214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 705, 3448, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 1572, 0, 0, 0, 0, 1, 4, 0, 0, 0, 4082, 2175, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 6, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 6, 7160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 87, 0, 2232, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 14, 0, 0, 0, 0, 6, 5, 3, 0, 0, 3, 3829, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6000, 8473, 14644, 4202, 8830, 7275, 10482, 5630] avg=8192.0000 max=14644 max_rank=2 min=4202 min_rank=3 imbalance_ratio=1.787598 max_load_reduction=50892 max_load_reduction_pct=0.776550 moved_to_echo=32217 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6009, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8125, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 7242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 6275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1826, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4039, 2183, 950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 2207, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 6729, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 8281, 7299, 7513, 9001, 8186, 10374, 8585] avg=8192.0000 max=10374 max_rank=6 min=6297 min_rank=0 imbalance_ratio=1.266357 max_load_reduction=55162 max_load_reduction_pct=0.841705 moved_to_echo=36146 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5885, 0, 0, 156, 0, 0, 0, 0, 0, 3038, 0, 0, 3, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 7501, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 44, 0, 36, 15, 0, 0, 7216, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 1217, 8172, 0, 0, 0, 0, 0, 0, 56, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 118, 0, 251, 0, 0, 0, 0, 1, 0, 0, 3, 4010, 2144, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5183, 0, 0, 88, 0, 2180, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 196, 0, 6668, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9176, 7710, 7330, 9475, 7996, 7763, 7467, 8619] avg=8192.0000 max=9475 max_rank=3 min=7330 min_rank=2 imbalance_ratio=1.156616 max_load_reduction=56061 max_load_reduction_pct=0.855423 moved_to_echo=31845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5993, 0, 0, 0, 1, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4, 0, 0, 0, 0, 0, 66, 0, 2, 0, 0, 8, 0, 8088, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1225, 8173, 1, 6, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 22, 21, 0, 11, 0, 0, 0, 0, 0, 0, 59, 4089, 2178, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 22, 0, 0, 1, 1, 0, 0, 0, 2, 0, 0, 77, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8032, 0, 0, 0, 0, 2235, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 34, 0, 2, 0, 0, 0, 0, 18, 0, 0, 0, 6655, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6028, 8329, 7195, 9484, 7326, 8278, 10294, 8602] avg=8192.0000 max=10294 max_rank=6 min=6028 min_rank=0 imbalance_ratio=1.256592 max_load_reduction=55242 max_load_reduction_pct=0.842926 moved_to_echo=34500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 1408, 640, 640, 4736, 256, 128, 0, 128, 0, 256, 0, 0, 256, 32768, 17536, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 1408, 640, 640, 4736, 256, 128, 0, 0, 128, 256, 0, 0, 256, 32768, 17408, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 256, 0, 384, 128, 128, 128, 128, 128, 128, 128, 1536, 128, 128, 256, 128, 49152, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 47232, 128, 0, 256, 128, 18176, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[47616, 0, 0, 256, 128, 0, 128, 12416, 128, 4096, 128, 0, 128, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 8704, 56704, 128, 128, 0, 0, 128, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 7168, 256, 128, 0, 128, 512, 128, 128, 57728, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 256, 0, 384, 128, 128, 128, 128, 128, 128, 128, 1536, 128, 128, 256, 128, 48896, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 47488, 128, 0, 256, 128, 17920, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[47744, 0, 0, 256, 128, 0, 128, 12288, 128, 3968, 128, 0, 128, 128, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 9088, 56320, 128, 128, 0, 0, 128, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 7296, 256, 128, 0, 128, 384, 0, 128, 57728, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[640, 0, 128, 0, 0, 128, 128, 0, 0, 0, 128, 128, 0, 1664, 128, 63360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 384, 128, 128, 128, 0, 128, 0, 128, 128, 128, 128, 128, 128, 640, 63360, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 3627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4852, 0, 19, 0, 0, 388, 1, 0, 0, 0, 0, 575, 0, 0, 3, 0, 133, 0, 91, 0, 0, 8192, 0, 0, 363, 0, 0, 81, 3008, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7983, 0, 8190, 0, 20, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 12, 0, 9, 0, 0, 0, 0, 0, 5, 0, 0, 29, 0, 0, 0, 0, 0, 99, 0, 264, 0, 0, 0, 0, 0, 5, 192, 0, 0, 7095, 0, 0, 26, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 75, 0, 0, 10, 0, 0, 0, 0, 0, 5, 0, 4654, 0, 13, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2365, 0, 0, 0, 0, 8013, 0, 0, 2304, 0, 11, 1, 7449, 1, 0, 0, 0, 0, 208, 335, 0, 0, 0, 0, 0, 0, 8192, 0, 8156, 1, 0, 7989, 0, 2, 0, 0, 0, 0, 10, 0, 0, 0, 4, 0, 105, 0, 0, 0, 7800, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 973, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4, 0, 7878, 0, 8, 0, 0, 0, 0, 6536, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 277, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 71, 108, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 8192, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 21, 0, 0, 0, 58, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3390, 0, 0, 3, 0, 0, 0, 0, 11, 0, 5200, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 109, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 12, 0, 2, 0, 0, 0, 7595, 0, 0, 7, 1, 0, 0, 1, 0, 130, 8105, 21, 0, 0, 0, 0, 113, 7413, 0, 61, 0, 0, 15, 0, 3, 0, 0, 4, 0, 0, 0, 8140, 0, 8174, 3, 0, 14, 18, 0, 0, 19, 0, 0, 0, 0, 0, 11, 322, 0, 0, 0, 0, 0, 8120, 31, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 668, 0, 0, 0, 1, 166, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3, 25, 0, 0, 8068, 0, 15, 0, 0, 0, 0, 58, 0, 0, 7876, 0, 0, 13, 0, 0, 0, 0, 28, 0, 0, 10, 0, 278, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 366, 0, 1, 0, 0, 0, 1, 0, 3048, 0, 0, 0, 0, 0, 2, 1, 0, 125, 0, 0, 0, 7, 5, 70, 0, 0, 8054, 0, 0, 92, 0, 0, 12, 7581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4666, 48, 8054, 0, 0, 8105, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 1076, 0, 0, 195, 0, 0, 0, 6838, 2, 1, 7528, 0, 0, 0, 50, 2, 0, 0, 0, 0, 0, 0, 9, 3314, 0, 0, 0, 0, 0, 0, 19, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 19, 0, 7997, 0, 0, 0, 0, 0, 5312, 21, 0, 0, 25, 26, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8054, 0, 8107, 0, 0, 7682, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1662, 0, 0, 0, 12, 0, 0, 0, 0, 8, 0, 20, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 5, 1581, 0, 152, 7668, 0, 0, 2, 0, 0, 0, 0, 0, 9, 31, 0, 0, 0, 0, 0, 0, 0, 74, 0, 14, 1, 0, 42, 0, 0, 1, 0, 0, 115, 0, 0, 186, 0, 540, 1, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8175, 4, 0, 4716, 0, 2, 3, 506, 0, 0, 0, 0, 0, 1633, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 8185, 0, 0, 5646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 3, 0, 0, 0, 0, 8040, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 46, 0, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6960, 2, 0, 2, 0, 0, 0, 4876, 0, 0, 6585, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8141, 0, 1, 0, 0, 0, 0, 11, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 8184, 13, 0, 5146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 7298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1163, 0, 0, 185, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4852, 0, 19, 5687, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 1, 0, 0, 0, 0, 575, 0, 0, 3, 0, 133, 0, 91, 4179, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2970, 0, 0, 363, 0, 0, 81, 3008, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1843, 0, 0, 0, 20, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10976, 3125, 9546, 11450, 5823, 6425, 10020, 8171] avg=8192.0000 max=11450 max_rank=3 min=3125 min_rank=1 imbalance_ratio=1.397705 max_load_reduction=54086 max_load_reduction_pct=0.825287 moved_to_echo=30439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 12, 0, 9, 0, 0, 0, 0, 0, 5, 0, 0, 29, 0, 7269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 264, 0, 0, 0, 0, 0, 5, 192, 0, 0, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7095, 0, 0, 26, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 181, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 75, 0, 0, 10, 0, 0, 0, 0, 0, 5, 0, 5836, 887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4654, 0, 13, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2365, 0, 0, 4088, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2905, 0, 0, 2304, 0, 11, 1, 7449, 1, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 1891, 0, 0, 1, 0, 7989, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4, 0, 105, 0, 0, 0, 7782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7325, 1399, 7413, 6815, 11586, 12879, 10218, 7901] avg=8192.0000 max=12879 max_rank=5 min=1399 min_rank=1 imbalance_ratio=1.572144 max_load_reduction=52657 max_load_reduction_pct=0.803482 moved_to_echo=42584 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 973, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7878, 0, 8, 0, 0, 0, 0, 6536, 0, 0, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 5774, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 4179, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2970, 0, 0, 0, 0, 0, 71, 108, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1870, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 15284, 8295, 7050, 5050, 3202, 10043, 8334] avg=8192.0000 max=15284 max_rank=1 min=3202 min_rank=5 imbalance_ratio=1.865723 max_load_reduction=50252 max_load_reduction_pct=0.766785 moved_to_echo=26629 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 0, 0, 0, 58, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 7285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3390, 0, 0, 3, 0, 0, 0, 0, 11, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5200, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 109, 183, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 5799, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2, 0, 0, 0, 7595, 0, 0, 7, 1, 0, 0, 1, 4135, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 2938, 21, 0, 0, 0, 0, 113, 7413, 0, 61, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 0, 0, 0, 1879, 0, 0, 3, 0, 14, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 11, 322, 0, 0, 0, 0, 0, 8102, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7373, 4253, 5714, 14884, 12215, 10691, 1921, 8485] avg=8192.0000 max=14884 max_rank=3 min=1921 min_rank=6 imbalance_ratio=1.816895 max_load_reduction=50652 max_load_reduction_pct=0.772888 moved_to_echo=33312 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 0, 0, 0, 0, 0, 7225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 1076, 0, 0, 195, 0, 0, 0, 6838, 2, 1, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7528, 0, 0, 0, 50, 2, 0, 0, 0, 0, 0, 0, 9, 3314, 0, 0, 186, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5743, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 4082, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2887, 0, 0, 0, 0, 0, 5312, 21, 0, 0, 25, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1850, 0, 0, 0, 0, 7682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7242, 9034, 11111, 7105, 4841, 8290, 9535, 8378] avg=8192.0000 max=11111 max_rank=2 min=4841 min_rank=4 imbalance_ratio=1.356323 max_load_reduction=54425 max_load_reduction_pct=0.830460 moved_to_echo=31044 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1662, 0, 0, 0, 12, 0, 0, 0, 0, 8, 0, 7294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 5, 1581, 0, 152, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7668, 0, 0, 2, 0, 0, 0, 0, 0, 9, 31, 0, 0, 0, 0, 0, 185, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 14, 1, 0, 42, 0, 0, 1, 0, 0, 115, 0, 0, 5779, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 540, 1, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4170, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2964, 4, 0, 4716, 0, 2, 3, 506, 0, 0, 0, 0, 0, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1873, 0, 0, 0, 0, 5646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 3, 0, 0, 0, 0, 8022, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8977, 10731, 7913, 6917, 5590, 9831, 7519, 8058] avg=8192.0000 max=10731 max_rank=1 min=5590 min_rank=4 imbalance_ratio=1.309937 max_load_reduction=54805 max_load_reduction_pct=0.836258 moved_to_echo=30998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 668, 0, 0, 0, 1, 166, 0, 0, 0, 0, 4, 0, 0, 0, 0, 7178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 25, 0, 0, 8068, 0, 15, 0, 0, 0, 0, 58, 0, 0, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7876, 0, 0, 13, 0, 0, 0, 0, 28, 0, 0, 10, 0, 278, 0, 0, 182, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 366, 0, 1, 0, 0, 0, 1, 0, 3324, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3048, 0, 0, 0, 0, 0, 2, 1, 0, 125, 0, 0, 0, 7, 5, 70, 4109, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2920, 0, 0, 92, 0, 0, 12, 7581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1077, 48, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8017, 9012, 8405, 4583, 7632, 10605, 9237, 8045] avg=8192.0000 max=10605 max_rank=5 min=4583 min_rank=3 imbalance_ratio=1.294556 max_load_reduction=54931 max_load_reduction_pct=0.838181 moved_to_echo=35519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 46, 0, 5, 0, 0, 0, 0, 0, 0, 3, 7294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6960, 2, 0, 2, 0, 0, 0, 4876, 0, 0, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6585, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 346, 185, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 5815, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8141, 0, 1, 0, 0, 0, 0, 11, 4179, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2970, 0, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1884, 0, 0, 13, 0, 5146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7348, 12698, 7139, 6732, 12799, 3613, 7043, 8164] avg=8192.0000 max=12799 max_rank=4 min=3613 min_rank=5 imbalance_ratio=1.562378 max_load_reduction=52737 max_load_reduction_pct=0.804703 moved_to_echo=31619 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 50, 0, 5, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 7012, 1, 0, 2, 0, 0, 0, 4953, 0, 0, 6696, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8138, 0, 1, 0, 0, 0, 0, 11, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 8186, 11, 0, 5086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 7877, 0, 5, 0, 0, 0, 0, 6505, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 271, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 62, 105, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 8192, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 7, 0, 5, 0, 0, 0, 0, 0, 4, 0, 0, 18, 0, 0, 0, 0, 0, 90, 0, 315, 0, 0, 0, 0, 0, 3, 187, 0, 0, 7071, 0, 0, 27, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 92, 0, 0, 9, 0, 0, 0, 0, 0, 3, 0, 4692, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2347, 0, 0, 0, 0, 8016, 0, 0, 2309, 0, 14, 1, 7417, 1, 0, 0, 0, 0, 223, 325, 0, 0, 0, 0, 0, 0, 8192, 0, 8155, 1, 0, 7983, 0, 2, 0, 0, 0, 0, 10, 0, 0, 0, 5, 0, 107, 0, 0, 0, 7797, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1738, 0, 0, 0, 10, 0, 0, 0, 0, 7, 0, 18, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 7, 1642, 0, 124, 7644, 0, 0, 2, 0, 0, 0, 0, 0, 9, 29, 0, 0, 0, 0, 0, 0, 0, 82, 0, 12, 0, 0, 43, 0, 0, 1, 0, 0, 112, 0, 0, 188, 0, 536, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8174, 2, 0, 4800, 0, 2, 4, 445, 0, 0, 0, 0, 0, 1587, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 8185, 0, 0, 5570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3, 0, 0, 0, 0, 8036, 13, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 2202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1220, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4792, 0, 18, 0, 0, 389, 0, 0, 0, 0, 0, 601, 0, 0, 4, 0, 153, 0, 95, 0, 0, 8192, 0, 0, 408, 0, 0, 83, 2827, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 8190, 0, 18, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 0, 67, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 0, 0, 2, 0, 0, 0, 0, 9, 0, 5274, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 105, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 9, 0, 2, 0, 0, 0, 7576, 0, 0, 5, 2, 0, 0, 2, 0, 130, 8104, 21, 0, 0, 0, 0, 112, 7413, 0, 63, 0, 0, 8, 0, 3, 0, 0, 6, 0, 0, 0, 8120, 0, 8176, 1, 0, 13, 21, 0, 0, 17, 0, 0, 0, 0, 0, 10, 321, 0, 0, 0, 0, 0, 8121, 30, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 697, 0, 0, 0, 0, 165, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3, 24, 0, 0, 8033, 0, 17, 0, 0, 0, 0, 63, 0, 0, 7890, 0, 0, 13, 0, 0, 0, 2, 28, 0, 0, 6, 0, 336, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 1, 0, 2874, 0, 0, 0, 1, 0, 2, 1, 0, 124, 0, 0, 0, 7, 4, 67, 0, 0, 8057, 0, 0, 113, 0, 0, 12, 7588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4751, 47, 8060, 0, 0, 8105, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 1026, 0, 0, 183, 0, 0, 0, 6862, 2, 0, 7488, 0, 0, 0, 50, 2, 0, 0, 0, 0, 0, 0, 11, 3374, 0, 0, 0, 0, 0, 0, 24, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 0, 19, 0, 8000, 0, 0, 0, 0, 0, 5154, 21, 0, 0, 20, 31, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8054, 0, 8108, 0, 0, 7688, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1738, 0, 0, 0, 10, 0, 0, 0, 0, 7, 0, 7252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 7, 1642, 0, 124, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7644, 0, 0, 2, 0, 0, 0, 0, 0, 9, 29, 0, 0, 0, 0, 0, 150, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 12, 0, 0, 43, 0, 0, 1, 0, 0, 112, 0, 0, 5716, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 536, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3021, 2, 0, 4800, 0, 2, 4, 445, 0, 0, 0, 0, 0, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1891, 0, 0, 0, 0, 5570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3, 0, 0, 0, 0, 8015, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9008, 10776, 7855, 6899, 5605, 9865, 7461, 8067] avg=8192.0000 max=10776 max_rank=1 min=5605 min_rank=4 imbalance_ratio=1.315430 max_load_reduction=54760 max_load_reduction_pct=0.835571 moved_to_echo=30998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7877, 0, 5, 0, 0, 0, 0, 6505, 0, 0, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 5717, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 4158, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3028, 0, 0, 0, 0, 0, 62, 105, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1892, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 15249, 8256, 7038, 5081, 3241, 10064, 8337] avg=8192.0000 max=15249 max_rank=1 min=3241 min_rank=5 imbalance_ratio=1.861450 max_load_reduction=50287 max_load_reduction_pct=0.767319 moved_to_echo=26723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 50, 0, 5, 0, 0, 0, 0, 0, 1, 3, 7253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7012, 1, 0, 2, 0, 0, 0, 4953, 0, 0, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6696, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 325, 150, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 5745, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8138, 0, 1, 0, 0, 0, 0, 11, 4158, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3028, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1901, 0, 0, 11, 0, 5086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7312, 12824, 7196, 6699, 12817, 3527, 6998, 8163] avg=8192.0000 max=12824 max_rank=1 min=3527 min_rank=5 imbalance_ratio=1.565430 max_load_reduction=52712 max_load_reduction_pct=0.804321 moved_to_echo=31505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 7, 0, 5, 0, 0, 0, 0, 0, 4, 0, 0, 18, 0, 7225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 315, 0, 0, 0, 0, 0, 3, 187, 0, 0, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7071, 0, 0, 27, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 147, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 92, 0, 0, 9, 0, 0, 0, 0, 0, 3, 0, 5774, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4692, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2347, 0, 0, 4069, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2963, 0, 0, 2309, 0, 14, 1, 7417, 1, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 1911, 0, 0, 1, 0, 7983, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 5, 0, 107, 0, 0, 0, 7777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7260, 1432, 7360, 6810, 11625, 12928, 10222, 7899] avg=8192.0000 max=12928 max_rank=5 min=1432 min_rank=1 imbalance_ratio=1.578125 max_load_reduction=52608 max_load_reduction_pct=0.802734 moved_to_echo=42674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 7256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1220, 0, 1, 150, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4792, 0, 18, 5590, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 0, 0, 0, 0, 0, 601, 0, 0, 4, 0, 153, 0, 95, 4158, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3028, 0, 0, 408, 0, 0, 83, 2827, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1849, 0, 0, 0, 18, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11137, 3058, 9576, 11334, 5891, 6348, 10024, 8168] avg=8192.0000 max=11334 max_rank=3 min=3058 min_rank=1 imbalance_ratio=1.383545 max_load_reduction=54202 max_load_reduction_pct=0.827057 moved_to_echo=30431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 0, 67, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 7245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 0, 0, 2, 0, 0, 0, 0, 9, 0, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5274, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 105, 149, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 5724, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 0, 0, 0, 7576, 0, 0, 5, 2, 0, 0, 2, 4113, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 2996, 21, 0, 0, 0, 0, 112, 7413, 0, 63, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6, 0, 0, 0, 1893, 0, 0, 1, 0, 13, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 10, 321, 0, 0, 0, 0, 0, 8100, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7341, 4225, 5750, 14850, 12212, 10743, 1937, 8478] avg=8192.0000 max=14850 max_rank=3 min=1937 min_rank=6 imbalance_ratio=1.812744 max_load_reduction=50686 max_load_reduction_pct=0.773407 moved_to_echo=33370 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 7185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 1026, 0, 0, 183, 0, 0, 0, 6862, 2, 0, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7488, 0, 0, 0, 50, 2, 0, 0, 0, 0, 0, 0, 11, 3374, 0, 0, 150, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5681, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 0, 4064, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2947, 0, 0, 0, 0, 0, 5154, 21, 0, 0, 20, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1871, 0, 0, 0, 0, 7688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7200, 8991, 11099, 7264, 4842, 8192, 9563, 8385] avg=8192.0000 max=11099 max_rank=2 min=4842 min_rank=4 imbalance_ratio=1.354858 max_load_reduction=54437 max_load_reduction_pct=0.830643 moved_to_echo=30982 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 697, 0, 0, 0, 0, 165, 0, 0, 0, 0, 4, 0, 0, 0, 0, 7142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 24, 0, 0, 8033, 0, 17, 0, 0, 0, 0, 63, 0, 0, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7890, 0, 0, 13, 0, 0, 0, 2, 28, 0, 0, 6, 0, 336, 0, 0, 148, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 1, 0, 3349, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2874, 0, 0, 0, 1, 0, 2, 1, 0, 124, 0, 0, 0, 7, 4, 67, 4089, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2979, 0, 0, 113, 0, 0, 12, 7588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1108, 47, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8008, 8981, 8444, 4642, 7463, 10692, 9267, 8039] avg=8192.0000 max=10692 max_rank=5 min=4642 min_rank=3 imbalance_ratio=1.305176 max_load_reduction=54844 max_load_reduction_pct=0.836853 moved_to_echo=35461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 256, 23552, 128, 0, 7552, 0, 128, 5632, 26368, 128, 128, 128, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 128, 38144, 128, 128, 256, 0, 0, 128, 20096, 128, 256, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[7936, 0, 1280, 128, 128, 256, 128, 128, 16512, 128, 128, 128, 128, 2816, 128, 256, 33152, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 0, 128, 128, 0, 0, 128, 14208, 128, 0, 128, 128, 50944, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 9088, 0, 128, 384, 0, 128, 0, 0, 5120, 128, 128, 43776, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 4352, 0, 128, 0, 2816, 256, 128, 128, 128, 128, 128, 0, 0, 128, 128, 58240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[58240, 0, 0, 128, 128, 128, 0, 0, 128, 128, 128, 128, 128, 4992, 0, 512, 1536, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 256, 24064, 128, 0, 7680, 0, 128, 5504, 25984, 128, 128, 128, 128, 128, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 128, 38016, 128, 128, 256, 0, 0, 128, 20224, 128, 128, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[7808, 0, 1280, 0, 128, 256, 128, 128, 16512, 128, 128, 128, 128, 2816, 128, 256, 33024, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 0, 128, 128, 0, 0, 128, 14336, 128, 0, 128, 128, 50816, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 9344, 0, 128, 384, 0, 128, 0, 0, 4992, 128, 128, 43392, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 4608, 0, 128, 0, 2944, 256, 128, 128, 128, 128, 128, 0, 0, 128, 128, 57856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[58368, 0, 0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 5248, 0, 512, 1280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 0, 256, 128, 512, 128, 128, 0, 0, 0, 64640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 0, 256, 128, 640, 128, 128, 0, 0, 0, 64640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7483, 0, 0, 0, 0, 0, 7141, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8160, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5449, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 300, 3232, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7777, 0, 0, 0, 0, 0, 7751, 0, 2, 0, 0, 1780, 155, 0, 7, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 106, 3, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 17, 8030, 0, 391, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7744, 0, 0, 0, 0, 0, 0, 8, 7, 1, 0, 10, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6788, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 7745, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 497, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 32, 8171, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 89, 8045, 119, 0, 0, 39, 0, 5949, 0, 2362, 25, 1, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 69, 0, 0, 0, 1, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5873, 1, 0, 4, 3, 37, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 10, 6, 0, 0, 0, 4, 0, 0, 8164, 0, 5, 13, 0, 218, 0, 0, 0, 10, 0, 0, 0, 8043, 1708, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 16, 0, 0, 0, 0, 7690, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 22, 0, 0, 0, 60, 0, 0, 0, 0, 7992, 301, 0, 0, 1, 0, 7381, 0, 0, 95, 0, 0, 0, 0, 8, 0, 0, 0, 0, 181, 1, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 574, 32, 0, 0, 0, 0, 326, 1, 0, 0, 7688, 0, 0, 1, 0, 2618, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 6769, 0, 0, 0, 12, 0, 0, 76, 0, 1, 0, 0, 8143, 0, 0, 0, 4, 1502, 0, 0, 0, 0, 71, 0, 0, 15, 5667, 0, 0, 0, 130] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 44, 0, 0, 32, 6449, 0, 2, 0, 0, 0, 5838, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 1097, 0, 0, 0, 0, 0, 0, 0, 34, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 87, 0, 0, 322, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 8172, 0, 0, 0, 0, 16, 0, 6, 13, 0, 0, 0, 8176, 5, 18, 14, 0, 30, 0, 0, 0, 0, 0, 0, 0, 3243, 7027, 0, 0, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 19, 0, 0, 0, 6, 5, 0, 0, 0, 25, 0, 8185, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 1, 0, 0, 0, 0, 7601, 0, 0, 35, 0, 0, 6607, 0, 6, 0, 0, 506, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7458, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 4, 0, 544, 6, 0, 0, 15, 0, 0, 8169, 0, 0, 0, 6, 76, 0, 0, 0, 0, 0, 0, 0, 0, 301, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 10, 0, 8165, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 5, 0, 5370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 7, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 8192, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 147, 0, 0, 1, 0, 0, 0, 0, 6133, 3364, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7674, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 7141, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 208, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5449, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 300, 3232, 0, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8136, 8298, 8044, 8485, 13554, 5965, 4807] avg=8192.0000 max=13554 max_rank=5 min=4807 min_rank=7 imbalance_ratio=1.654541 max_load_reduction=51982 max_load_reduction_pct=0.793182 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8159, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7678, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 0, 0, 0, 0, 7751, 0, 2, 0, 0, 1780, 155, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 8021, 0, 0, 106, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 17, 8030, 216, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7744, 0, 0, 0, 0, 0, 0, 8, 7, 1, 0, 10, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6788, 0, 0, 0, 0, 0, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8187, 10071, 8229, 8649, 5949, 8284, 7946] avg=8192.0000 max=10071 max_rank=2 min=5949 min_rank=5 imbalance_ratio=1.229370 max_load_reduction=55465 max_load_reduction_pct=0.846329 moved_to_echo=30828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 7678, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 215, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 32, 8171, 0, 0, 0, 0, 1150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8219, 8580, 8048, 8492, 6056, 8578, 9372] avg=8192.0000 max=9372 max_rank=7 min=6056 min_rank=5 imbalance_ratio=1.144043 max_load_reduction=56164 max_load_reduction_pct=0.856995 moved_to_echo=32498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 16, 0, 0, 0, 0, 7690, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 22, 0, 0, 0, 60, 0, 0, 0, 0, 7632, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 301, 0, 0, 1, 0, 7381, 0, 0, 95, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 1, 0, 0, 0, 0, 0, 7935, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 32, 0, 0, 0, 0, 326, 1, 0, 0, 7688, 222, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2618, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 5708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6769, 0, 0, 0, 12, 0, 0, 76, 0, 1, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1502, 0, 0, 0, 0, 71, 0, 0, 15, 5667, 0, 0, 0, 130, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7768, 8240, 8172, 8123, 8928, 8360, 7369, 8576] avg=8192.0000 max=8928 max_rank=4 min=7369 min_rank=6 imbalance_ratio=1.089844 max_load_reduction=56608 max_load_reduction_pct=0.863770 moved_to_echo=33233 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 121, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 89, 7657, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 119, 0, 0, 39, 0, 5949, 0, 2362, 25, 1, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5873, 225, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 3, 37, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 10, 6, 0, 0, 0, 4, 0, 0, 507, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 218, 0, 0, 0, 10, 0, 0, 0, 8043, 1708, 0, 0, 0, 0, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 8245, 9007, 8078, 6250, 5811, 8606, 11190] avg=8192.0000 max=11190 max_rank=7 min=5811 min_rank=5 imbalance_ratio=1.365967 max_load_reduction=54346 max_load_reduction_pct=0.829254 moved_to_echo=31857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 1, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 44, 0, 0, 32, 7663, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 2, 0, 0, 0, 5838, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8166, 179, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 322, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 16, 0, 6, 13, 0, 0, 0, 513, 5, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 30, 0, 0, 0, 0, 0, 0, 0, 3243, 7027, 0, 0, 0, 8, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8153, 6583, 9065, 8468, 5020, 8743, 11280] avg=8192.0000 max=11280 max_rank=7 min=5020 min_rank=5 imbalance_ratio=1.376953 max_load_reduction=54256 max_load_reduction_pct=0.827881 moved_to_echo=33511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 10, 0, 8165, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7677, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 0, 5, 0, 5370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 0, 7, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 8189, 225, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 147, 0, 0, 1, 0, 0, 0, 0, 6133, 3364, 0, 0, 0, 2, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8174, 5767, 8083, 8503, 7189, 8731, 10852] avg=8192.0000 max=10852 max_rank=7 min=5767 min_rank=2 imbalance_ratio=1.324707 max_load_reduction=54684 max_load_reduction_pct=0.834412 moved_to_echo=35275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 0, 0, 0, 6, 5, 0, 0, 0, 25, 0, 8185, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 1, 0, 0, 0, 0, 7657, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 35, 0, 0, 6607, 0, 6, 0, 0, 506, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7458, 211, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 4, 0, 544, 6, 0, 0, 15, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 76, 0, 0, 0, 0, 0, 0, 0, 0, 301, 1, 0, 0, 0, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8299, 8182, 9058, 7866, 7761, 13597, 9260, 1513] avg=8192.0000 max=13597 max_rank=5 min=1513 min_rank=7 imbalance_ratio=1.659790 max_load_reduction=51939 max_load_reduction_pct=0.792526 moved_to_echo=32131 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 7777, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 465, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 27, 8171, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7408, 0, 0, 0, 0, 0, 7208, 0, 0, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8153, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 173, 0, 0, 0, 0, 0, 0, 0, 285, 3360, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7774, 0, 0, 0, 0, 0, 7782, 0, 0, 0, 0, 1806, 160, 0, 7, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 114, 3, 0, 0, 0, 0, 283, 0, 2, 0, 0, 0, 0, 0, 14, 0, 0, 17, 8002, 0, 394, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 0, 0, 0, 0, 10, 4, 1, 0, 9, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 6763, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 43, 0, 0, 32, 6452, 0, 2, 0, 0, 0, 5815, 0, 0, 0, 0, 402, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 1090, 0, 0, 0, 0, 0, 0, 0, 38, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 85, 0, 0, 308, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 14, 0, 5, 15, 0, 0, 0, 8175, 6, 17, 14, 0, 32, 0, 0, 0, 0, 0, 0, 0, 3319, 7015, 0, 0, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 11, 0, 8164, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 5, 0, 5357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 6, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 8192, 1, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 142, 0, 0, 0, 0, 0, 0, 0, 6010, 3486, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 17, 0, 0, 0, 0, 7719, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 22, 0, 0, 0, 59, 0, 0, 0, 0, 8016, 294, 0, 0, 1, 0, 7389, 0, 0, 88, 0, 0, 0, 0, 6, 0, 0, 0, 3, 192, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 583, 21, 0, 0, 0, 0, 288, 0, 0, 0, 7688, 0, 0, 4, 0, 2286, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 6785, 0, 0, 0, 12, 0, 0, 91, 0, 1, 0, 0, 8143, 0, 0, 0, 2, 1461, 0, 0, 0, 0, 72, 0, 0, 18, 5992, 0, 0, 0, 128] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 84, 8045, 117, 2, 0, 41, 0, 6089, 0, 2338, 34, 1, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 72, 0, 0, 0, 3, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5815, 1, 0, 4, 1, 26, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 11, 8, 0, 0, 0, 4, 0, 0, 8164, 0, 5, 10, 0, 213, 0, 0, 0, 16, 0, 0, 0, 8041, 1657, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 19, 0, 0, 0, 7, 4, 0, 0, 0, 25, 0, 8186, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 1, 0, 0, 0, 0, 7602, 0, 0, 35, 0, 0, 6592, 0, 5, 0, 0, 467, 0, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8007, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7443, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8176, 0, 0, 4, 0, 526, 6, 0, 0, 16, 0, 0, 8170, 0, 0, 0, 7, 113, 0, 0, 0, 0, 0, 0, 0, 0, 340, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 7666, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 231, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 27, 8171, 0, 0, 0, 0, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8225, 8565, 8045, 8512, 6097, 8591, 9309] avg=8192.0000 max=9309 max_rank=7 min=6097 min_rank=5 imbalance_ratio=1.136353 max_load_reduction=56227 max_load_reduction_pct=0.857956 moved_to_echo=32509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7660, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 0, 7208, 0, 0, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 220, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 0, 0, 0, 285, 3360, 0, 0, 0, 0, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8128, 8340, 8041, 8502, 13536, 5883, 4862] avg=8192.0000 max=13536 max_rank=5 min=4862 min_rank=7 imbalance_ratio=1.652344 max_load_reduction=52000 max_load_reduction_pct=0.793457 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8143, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7666, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 0, 0, 0, 0, 7782, 0, 0, 0, 0, 1806, 160, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 114, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 2, 0, 0, 0, 0, 0, 14, 0, 0, 17, 8002, 231, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 0, 0, 0, 0, 10, 4, 1, 0, 9, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 6763, 0, 0, 0, 0, 0, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8208, 10113, 8232, 8639, 6000, 8278, 7863] avg=8192.0000 max=10113 max_rank=2 min=6000 min_rank=5 imbalance_ratio=1.234497 max_load_reduction=55423 max_load_reduction_pct=0.845688 moved_to_echo=30780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 43, 0, 0, 32, 7650, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 2, 0, 0, 0, 5815, 0, 0, 0, 0, 402, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8165, 191, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 308, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 14, 0, 5, 15, 0, 0, 0, 525, 6, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 32, 0, 0, 0, 0, 0, 0, 0, 3319, 7015, 0, 0, 0, 16, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8152, 6518, 9054, 8485, 5046, 8753, 11306] avg=8192.0000 max=11306 max_rank=7 min=5046 min_rank=5 imbalance_ratio=1.380127 max_load_reduction=54230 max_load_reduction_pct=0.827484 moved_to_echo=33590 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 17, 0, 0, 0, 0, 7719, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 22, 0, 0, 0, 59, 0, 0, 0, 0, 7620, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 294, 0, 0, 1, 0, 7389, 0, 0, 88, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 192, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 21, 0, 0, 0, 0, 288, 0, 0, 0, 7688, 238, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2286, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 5776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6785, 0, 0, 0, 12, 0, 0, 91, 0, 1, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1461, 0, 0, 0, 0, 72, 0, 0, 18, 5992, 0, 0, 0, 128, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7796, 8233, 8148, 8138, 8907, 8099, 7412, 8803] avg=8192.0000 max=8907 max_rank=4 min=7412 min_rank=6 imbalance_ratio=1.087280 max_load_reduction=56629 max_load_reduction_pct=0.864090 moved_to_echo=33221 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 11, 0, 8164, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7664, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 0, 5, 0, 5357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 0, 6, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 8189, 240, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 1, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 142, 0, 0, 0, 0, 0, 0, 0, 6010, 3486, 0, 0, 0, 2, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8173, 5736, 8080, 8523, 7262, 8742, 10784] avg=8192.0000 max=10784 max_rank=7 min=5736 min_rank=2 imbalance_ratio=1.316406 max_load_reduction=54752 max_load_reduction_pct=0.835449 moved_to_echo=35311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 120, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 84, 7641, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 117, 2, 0, 41, 0, 6089, 0, 2338, 34, 1, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8012, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5815, 242, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 1, 26, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 11, 8, 0, 0, 0, 4, 0, 0, 523, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 213, 0, 0, 0, 16, 0, 0, 0, 8041, 1657, 0, 0, 0, 0, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8345, 8238, 9115, 8084, 6205, 5849, 8624, 11076] avg=8192.0000 max=11076 max_rank=7 min=5849 min_rank=5 imbalance_ratio=1.352051 max_load_reduction=54460 max_load_reduction_pct=0.830994 moved_to_echo=31754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 0, 0, 0, 7, 4, 0, 0, 0, 25, 0, 8186, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 1, 0, 0, 0, 0, 7645, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 35, 0, 0, 6592, 0, 5, 0, 0, 467, 0, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7443, 225, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 4, 0, 526, 6, 0, 0, 16, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 113, 0, 0, 0, 0, 0, 0, 0, 0, 340, 1, 0, 0, 0, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8179, 9001, 7862, 7763, 13647, 9253, 1533] avg=8192.0000 max=13647 max_rank=5 min=1533 min_rank=7 imbalance_ratio=1.665894 max_load_reduction=51889 max_load_reduction_pct=0.791763 moved_to_echo=32196 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 2176, 0, 0, 128, 128, 128, 0, 0, 24576, 29568, 128, 0, 0, 256, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 512, 128, 128, 21248, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 0, 43776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3072, 512, 128, 128, 128, 0, 54272, 0, 2432, 128, 128, 2816, 256, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 60672, 0, 0, 128, 128, 640, 128, 128, 128, 128, 0, 128, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 128, 128, 0, 128, 128, 0, 0, 128, 61440, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 384, 128, 640, 128, 128, 0, 128, 0, 384, 128, 128, 128, 61824, 1792, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 64000, 1152, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 2176, 0, 0, 0, 128, 128, 0, 0, 24576, 30080, 128, 0, 0, 256, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 512, 128, 128, 20864, 128, 128, 0, 128, 0, 0, 128, 128, 128, 0, 0, 44160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 60544, 128, 0, 128, 128, 640, 128, 128, 128, 128, 0, 128, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[2816, 512, 128, 128, 128, 0, 54528, 0, 2432, 128, 128, 2816, 256, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 128, 128, 0, 128, 128, 0, 0, 128, 61312, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 384, 128, 640, 128, 128, 0, 128, 0, 384, 0, 128, 128, 61696, 1920, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 64000, 1152, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 64896, 128, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 92, 0, 642, 0, 0, 0, 0, 843, 0, 0, 312, 0, 1, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7834, 0, 0, 0, 7869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 7490, 0, 0, 0, 0, 62, 7969, 8004, 0, 0, 3405, 38, 0, 0, 0, 134, 0, 0, 0, 17, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4160, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 11, 0, 0, 28, 0, 0, 8158, 0, 0, 21, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 0, 0, 7204, 0, 3, 0, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 1, 0, 0, 816, 8091, 8, 0, 10, 259, 41, 0, 0, 0, 102, 0, 0, 0, 0, 7269, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 676, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 3613, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 5691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 8192, 3523, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 3, 0, 0, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 816, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2858, 0, 0, 0, 0, 0, 0, 4, 0, 7064, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 67, 8174, 1365, 0, 0, 0, 0, 0, 0, 0, 3440, 0, 237, 0, 0, 8139, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6619, 101, 0, 0, 0, 0, 7, 0, 0, 0, 0, 60, 0, 5401, 0, 0, 7, 8156, 0, 0, 0, 38, 0, 73, 0, 0, 0, 0, 8015, 4663, 0, 21, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 16, 0, 0, 0, 0, 0, 0, 6008, 0, 639, 0, 0, 0, 0, 0, 23, 7991, 0, 0, 15, 7349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 75, 0, 0, 61, 0, 0, 0, 5961, 8171, 1, 0, 0, 0, 1, 0, 0, 0, 0, 13, 278, 0, 0, 8188, 0, 0, 5, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2079, 1634, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 6, 0, 1, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7960, 37, 14, 0, 85] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1246, 0, 1, 0, 0, 0, 55, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8128, 2, 0, 1, 0, 6, 0, 0, 0, 8178, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 4, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 8002, 5502, 0, 0, 774, 54, 0, 0, 0, 742, 0, 0, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 136, 0, 149, 0, 7828, 0, 107, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 8150, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 7610, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 12, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 16, 4, 0, 0, 0, 0, 68, 0, 0, 7284, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 3, 3, 2, 0, 133, 3, 0, 0, 0, 0, 0, 0, 0, 7534, 8110, 1819, 0, 0, 0, 0, 0, 45, 0, 2532, 0, 291, 0, 0, 4531, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 30, 0, 1, 0, 8, 0, 5, 0, 4, 7576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8166, 969, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 1, 0, 0, 0, 8122, 0, 2, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 14, 0, 0, 87, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 2, 0, 0, 0, 1, 8182, 0, 29, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 16, 0, 8076, 19, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7661, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 533, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 7519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 0, 0, 3195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 1, 0, 0, 816, 2814, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 259, 41, 0, 0, 0, 102, 0, 0, 0, 0, 6718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 676, 0, 8174, 689, 405, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 3613, 0, 0, 0, 5277, 749, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7480, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8581, 11393, 6302, 3741, 7130, 10584, 10322, 7483] avg=8192.0000 max=11393 max_rank=1 min=3741 min_rank=3 imbalance_ratio=1.390747 max_load_reduction=54143 max_load_reduction_pct=0.826157 moved_to_echo=35519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 92, 0, 642, 0, 0, 0, 0, 843, 0, 0, 312, 0, 7415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 7490, 0, 0, 0, 0, 62, 2771, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 38, 0, 0, 0, 134, 0, 0, 0, 17, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4160, 688, 440, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5198, 814, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 28, 0, 0, 7470, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9309, 3493, 10974, 18407, 3870, 5313, 6640, 7530] avg=8192.0000 max=18407 max_rank=3 min=3493 min_rank=1 imbalance_ratio=2.246948 max_load_reduction=47129 max_load_reduction_pct=0.719131 moved_to_echo=23353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 93, 0, 0, 0, 7554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 2849, 3523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 3, 0, 0, 2438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7892, 687, 457, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 5343, 845, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7463, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7650, 7603, 8914, 6575, 10626, 9239, 7464, 7465] avg=8192.0000 max=10626 max_rank=4 min=6575 min_rank=3 imbalance_ratio=1.297119 max_load_reduction=54910 max_load_reduction_pct=0.837860 moved_to_echo=34794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1246, 0, 1, 0, 0, 0, 55, 0, 0, 0, 0, 19, 0, 0, 0, 7517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8128, 2, 0, 1, 0, 6, 0, 0, 0, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 4, 0, 0, 0, 72, 0, 0, 3242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 2783, 5502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 54, 0, 0, 0, 742, 0, 0, 0, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 136, 0, 149, 0, 7828, 642, 460, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 5219, 850, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 6968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8838, 11773, 11490, 8331, 2036, 9255, 6829, 6984] avg=8192.0000 max=11773 max_rank=1 min=2036 min_rank=4 imbalance_ratio=1.437134 max_load_reduction=53763 max_load_reduction_pct=0.820358 moved_to_echo=25104 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 816, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2858, 0, 0, 0, 0, 0, 0, 4, 0, 3120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 67, 2839, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 0, 237, 0, 0, 7517, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6619, 680, 401, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 7, 0, 0, 0, 0, 60, 0, 5401, 0, 0, 7, 5335, 738, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 73, 0, 0, 0, 0, 7335, 4663, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8343, 5982, 2987, 4273, 11215, 8324, 12282, 12130] avg=8192.0000 max=12282 max_rank=6 min=2987 min_rank=2 imbalance_ratio=1.499268 max_load_reduction=53254 max_load_reduction_pct=0.812592 moved_to_echo=43951 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 12, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 7548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 16, 4, 0, 0, 0, 0, 68, 0, 0, 3231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 3, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 133, 3, 0, 0, 0, 0, 0, 0, 0, 7534, 2820, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 2532, 0, 291, 0, 0, 4188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 30, 0, 1, 0, 8, 0, 5, 0, 4, 7576, 689, 409, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5290, 757, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7477, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7569, 3320, 11069, 12314, 7056, 9066, 6684, 8458] avg=8192.0000 max=12314 max_rank=3 min=3320 min_rank=1 imbalance_ratio=1.503174 max_load_reduction=53222 max_load_reduction_pct=0.812103 moved_to_echo=31264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 16, 7492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6008, 0, 639, 0, 0, 0, 0, 0, 23, 3544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 75, 0, 0, 61, 0, 0, 0, 5961, 2841, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 13, 278, 0, 0, 7567, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2079, 671, 449, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1634, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 6, 0, 1, 0, 0, 5330, 831, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7289, 37, 14, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 10214, 10531, 8942, 7864, 3874, 8510, 7425] avg=8192.0000 max=10531 max_rank=2 min=3874 min_rank=5 imbalance_ratio=1.285522 max_load_reduction=55005 max_load_reduction_pct=0.839310 moved_to_echo=27673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 1, 0, 0, 0, 3602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 14, 0, 0, 87, 0, 0, 0, 0, 0, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 8182, 0, 29, 0, 0, 7525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 16, 0, 8076, 688, 457, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5341, 844, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7470, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7070, 11758, 3269, 2953, 15739, 9881, 6805, 8061] avg=8192.0000 max=15739 max_rank=4 min=2953 min_rank=3 imbalance_ratio=1.921265 max_load_reduction=49797 max_load_reduction_pct=0.759842 moved_to_echo=40486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 99, 0, 715, 0, 0, 0, 0, 874, 0, 0, 309, 0, 1, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7799, 0, 0, 0, 7806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 7476, 0, 0, 0, 0, 77, 7981, 8024, 0, 0, 3502, 45, 0, 0, 0, 104, 0, 0, 0, 13, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4003, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 11, 0, 0, 28, 0, 0, 8158, 0, 0, 22, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 1, 0, 0, 0, 8126, 0, 4, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 14, 0, 0, 87, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 1, 8178, 0, 23, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 12, 0, 8070, 20, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7681, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 522, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3600, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 5761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 8192, 3637, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 3, 0, 0, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 4, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 0, 0, 7214, 0, 3, 0, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 1, 0, 0, 770, 8092, 8, 0, 10, 266, 38, 0, 0, 0, 113, 0, 0, 0, 0, 7333, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 682, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 3203, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 9, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 68, 0, 0, 7249, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 3, 5, 1, 0, 132, 10, 0, 0, 0, 0, 0, 0, 0, 7505, 8111, 1884, 0, 0, 0, 0, 0, 44, 0, 2823, 0, 299, 0, 0, 4338, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 28, 0, 1, 0, 7, 0, 6, 0, 4, 7538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8166, 906, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 819, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2826, 0, 0, 0, 0, 0, 0, 3, 0, 7034, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 112, 8174, 1529, 0, 0, 0, 0, 0, 0, 0, 3644, 0, 225, 0, 0, 8135, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6684, 102, 0, 0, 0, 0, 6, 0, 1, 0, 0, 60, 0, 5096, 0, 0, 4, 8157, 0, 0, 0, 38, 0, 73, 0, 0, 1, 1, 8016, 4554, 0, 19, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1308, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8127, 1, 0, 1, 0, 5, 0, 0, 0, 8179, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 3, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 7998, 5324, 0, 0, 741, 54, 0, 0, 0, 911, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 137, 0, 120, 0, 7850, 0, 114, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 9, 0, 0, 10, 8149, 0, 1, 0, 0, 0, 0, 0, 10, 0, 0, 7611, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 15, 0, 0, 0, 0, 0, 0, 5961, 0, 585, 0, 0, 0, 0, 0, 23, 7986, 0, 0, 12, 7318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 67, 0, 0, 63, 0, 0, 0, 5906, 8171, 0, 0, 0, 0, 1, 0, 0, 0, 0, 17, 264, 0, 0, 8188, 0, 0, 4, 0, 0, 44, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2167, 1616, 0, 0, 0, 0, 0, 0, 1, 102, 0, 1, 8, 0, 1, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 30, 13, 0, 78] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 99, 0, 715, 0, 0, 0, 0, 874, 0, 0, 309, 0, 7351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 7476, 0, 0, 0, 0, 77, 2767, 8024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 45, 0, 0, 0, 104, 0, 0, 0, 13, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4003, 663, 38, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5214, 862, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 28, 0, 0, 7495, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9352, 3549, 10855, 18410, 4016, 5065, 6733, 7556] avg=8192.0000 max=18410 max_rank=3 min=3549 min_rank=1 imbalance_ratio=2.247314 max_load_reduction=47126 max_load_reduction_pct=0.719086 moved_to_echo=23370 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 88, 0, 0, 0, 7520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3600, 0, 0, 0, 0, 0, 0, 0, 0, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 2841, 3637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 3, 0, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7887, 662, 283, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 5351, 898, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7487, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7615, 7279, 8937, 6673, 10796, 9206, 7539, 7491] avg=8192.0000 max=10796 max_rank=4 min=6673 min_rank=3 imbalance_ratio=1.317871 max_load_reduction=54740 max_load_reduction_pct=0.835266 moved_to_echo=35032 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 0, 0, 0, 7486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 0, 0, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 1, 0, 0, 770, 2806, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 266, 38, 0, 0, 0, 113, 0, 0, 0, 0, 6614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 682, 0, 8173, 664, 719, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 3203, 0, 0, 0, 5286, 797, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7504, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8525, 11465, 6658, 3672, 7041, 10665, 10001, 7509] avg=8192.0000 max=11465 max_rank=1 min=3672 min_rank=3 imbalance_ratio=1.399536 max_load_reduction=54071 max_load_reduction_pct=0.825058 moved_to_echo=35216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 9, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4, 7515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 68, 0, 0, 3283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 3, 2834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 132, 10, 0, 0, 0, 0, 0, 0, 0, 7505, 2812, 1884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 2823, 0, 299, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 28, 0, 1, 0, 7, 0, 6, 0, 4, 7538, 664, 425, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5299, 801, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 7502, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7536, 3362, 11018, 12349, 7079, 9005, 6769, 8418] avg=8192.0000 max=12349 max_rank=3 min=3362 min_rank=1 imbalance_ratio=1.507446 max_load_reduction=53187 max_load_reduction_pct=0.811569 moved_to_echo=31271 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 1, 0, 0, 0, 3680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 14, 0, 0, 87, 0, 0, 0, 0, 0, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8178, 0, 23, 0, 0, 7343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 12, 0, 8070, 663, 798, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5348, 898, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7496, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7061, 11842, 3231, 2945, 15545, 9939, 6895, 8078] avg=8192.0000 max=15545 max_rank=4 min=2945 min_rank=3 imbalance_ratio=1.897583 max_load_reduction=49991 max_load_reduction_pct=0.762802 moved_to_echo=40457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1308, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 20, 0, 0, 0, 7485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8127, 1, 0, 1, 0, 5, 0, 0, 0, 3704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 3, 0, 0, 0, 74, 0, 0, 3198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 2773, 5324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 741, 54, 0, 0, 0, 911, 0, 0, 0, 0, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 137, 0, 120, 0, 7850, 618, 49, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 9, 0, 0, 10, 5225, 904, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 0, 6993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8856, 11851, 11441, 8142, 2165, 9149, 6928, 7004] avg=8192.0000 max=11851 max_rank=1 min=2165 min_rank=4 imbalance_ratio=1.446655 max_load_reduction=53685 max_load_reduction_pct=0.819168 moved_to_echo=25246 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 819, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2826, 0, 0, 0, 0, 0, 0, 3, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 112, 2831, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 0, 225, 0, 0, 7332, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6684, 655, 803, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 6, 0, 1, 0, 0, 60, 0, 5096, 0, 0, 4, 5343, 782, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 73, 0, 0, 1, 1, 7361, 4554, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8318, 6002, 2944, 4474, 11223, 8469, 12059, 12047] avg=8192.0000 max=12059 max_rank=6 min=2944 min_rank=2 imbalance_ratio=1.472046 max_load_reduction=53477 max_load_reduction_pct=0.815994 moved_to_echo=43798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 15, 7461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5961, 0, 585, 0, 0, 0, 0, 0, 23, 3617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 7318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 67, 0, 0, 63, 0, 0, 0, 5906, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 17, 264, 0, 0, 7385, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2167, 647, 803, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 0, 0, 0, 0, 0, 0, 1, 102, 0, 1, 8, 0, 1, 0, 0, 5338, 883, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7312, 30, 13, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 10186, 10452, 8871, 7671, 4038, 8612, 7433] avg=8192.0000 max=10452 max_rank=2 min=4038 min_rank=5 imbalance_ratio=1.275879 max_load_reduction=55084 max_load_reduction_pct=0.840515 moved_to_echo=27754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 256, 256, 128, 0, 7680, 128, 128, 0, 0, 14720, 22656, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 0, 128, 37376, 128, 768, 128, 0, 128, 128, 128, 128, 27520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 40832, 0, 256, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 24576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1792, 128, 128, 0, 0, 128, 0, 0, 128, 0, 128, 128, 9728, 128, 0, 128, 42368, 6528, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 4480, 256, 0, 128, 128, 23424, 128, 896, 0, 128, 36736, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 256, 128, 896, 128, 52480, 5504, 3584, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 256, 256, 128, 0, 7680, 128, 128, 0, 0, 14592, 22528, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 0, 128, 36992, 128, 640, 128, 0, 128, 128, 128, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1792, 128, 128, 0, 0, 128, 0, 128, 128, 0, 128, 128, 8960, 128, 0, 128, 42496, 6912, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 41216, 0, 256, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 24192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 4608, 256, 0, 128, 128, 24064, 128, 896, 0, 128, 36096, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 256, 128, 896, 128, 52480, 5248, 3968, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 2176, 128, 0, 128, 128, 768, 128, 0, 128, 0, 896, 2048, 0, 384, 128, 59392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 2176, 128, 128, 128, 128, 768, 128, 0, 128, 0, 896, 1920, 0, 384, 128, 59648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 0, 128, 128, 0, 59008, 6272, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 128, 128, 128, 59264, 6016, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True - [2026-08-26 16:18:05] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 743.7 | throughput per GPU (TFLOP/s/GPU): 243.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.693989E+00 | loss scale: 1.0 | grad norm: 66.368 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[342, 0, 0, 85, 118, 368, 658, 0, 0, 887, 203, 32, 221, 79, 137, 261, 11, 1931, 0, 14, 1151, 0, 5, 1292, 1922, 3, 0, 7596, 5028, 5979, 33, 577, 106, 374, 307, 2, 20, 790, 0, 5, 186, 1, 308, 1134, 0, 156, 3, 7920, 0, 15, 17, 0, 0, 102, 0, 6, 211, 0, 18, 227, 1, 3, 0, 54, 2641, 0, 2, 1, 504, 0, 7, 59, 811, 0, 1116, 1065, 135, 10, 254, 21, 0, 3, 0, 2, 71, 1, 1, 722, 1, 329, 9, 22, 61, 6, 55, 2, 6, 283, 4, 15, 35, 8, 52, 1, 277, 341, 427, 636, 0, 103, 1961, 16, 7, 2908, 0, 18, 0, 94, 806, 6, 2, 10, 8069, 524, 7, 16, 0, 94] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[167, 2, 0, 66, 94, 309, 441, 0, 4, 554, 280, 10, 148, 91, 95, 162, 8, 592, 0, 36, 818, 0, 15, 1440, 3242, 5, 0, 7439, 4425, 6065, 26, 533, 69, 358, 178, 0, 5, 960, 0, 50, 142, 1, 468, 885, 0, 134, 15, 7848, 0, 15, 5, 0, 0, 25, 1, 11, 105, 0, 36, 303, 1, 1, 3, 47, 1774, 0, 0, 0, 615, 0, 11, 165, 883, 0, 1845, 1906, 57, 20, 290, 1, 1, 1, 1, 4, 68, 0, 0, 680, 1, 107, 12, 6, 43, 9, 64, 0, 11, 133, 0, 3, 20, 2, 15, 7, 108, 688, 412, 713, 0, 142, 2205, 9, 3, 3183, 0, 4, 1, 95, 1695, 2, 1, 64, 8117, 485, 6, 11, 6, 98] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[95, 1, 0, 80, 99, 331, 556, 0, 1, 1023, 547, 46, 226, 58, 85, 166, 2, 1075, 0, 20, 1159, 0, 1, 1227, 2166, 7, 0, 7833, 5100, 6176, 13, 902, 109, 340, 199, 1, 21, 1173, 0, 7, 70, 0, 388, 1221, 0, 231, 4, 7966, 1, 12, 7, 2, 0, 78, 0, 6, 187, 0, 20, 356, 0, 0, 8, 33, 1622, 1, 0, 0, 807, 0, 13, 127, 524, 1, 1273, 980, 51, 2, 197, 6, 0, 2, 0, 0, 29, 0, 0, 586, 0, 135, 8, 35, 13, 15, 29, 6, 33, 137, 3, 9, 12, 5, 23, 6, 271, 420, 203, 521, 0, 54, 2073, 11, 1, 4078, 0, 5, 0, 44, 1133, 6, 0, 23, 8168, 307, 2, 12, 2, 77] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[267, 1, 0, 65, 122, 248, 489, 1, 2, 668, 352, 16, 206, 87, 163, 129, 11, 1262, 0, 7, 829, 1, 0, 1180, 1858, 15, 0, 7577, 5332, 6572, 2, 493, 244, 562, 204, 1, 25, 798, 0, 37, 118, 1, 423, 1056, 0, 161, 3, 7865, 0, 17, 18, 3, 0, 81, 4, 14, 43, 1, 21, 291, 16, 4, 5, 71, 2045, 1, 0, 0, 447, 0, 30, 196, 701, 0, 1207, 1764, 59, 16, 177, 10, 0, 6, 0, 22, 93, 1, 0, 776, 0, 87, 30, 6, 1, 4, 27, 5, 9, 102, 7, 19, 17, 10, 34, 12, 222, 496, 301, 611, 4, 73, 2566, 4, 0, 3552, 0, 20, 0, 30, 1236, 3, 1, 41, 8005, 338, 9, 5, 4, 52] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[318, 5, 0, 55, 31, 226, 669, 0, 0, 572, 663, 22, 439, 73, 183, 212, 30, 1087, 0, 14, 951, 8, 84, 1209, 1535, 12, 1, 7682, 4703, 5626, 7, 715, 94, 480, 201, 3, 11, 1524, 0, 111, 180, 4, 533, 1152, 0, 122, 21, 7540, 0, 18, 5, 6, 1, 82, 0, 0, 93, 4, 15, 184, 2, 4, 15, 38, 2038, 1, 14, 0, 608, 1, 7, 160, 648, 1, 1523, 1318, 64, 11, 211, 11, 0, 23, 2, 22, 126, 0, 1, 966, 0, 212, 50, 110, 34, 9, 65, 9, 11, 115, 3, 10, 106, 35, 42, 14, 211, 840, 278, 875, 3, 77, 1904, 30, 2, 3615, 0, 17, 1, 83, 865, 20, 7, 27, 8024, 378, 36, 10, 13, 64] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[328, 2, 0, 95, 129, 293, 604, 1, 3, 1186, 198, 41, 145, 36, 91, 170, 6, 853, 3, 8, 1114, 11, 2, 1706, 2076, 9, 0, 7404, 4851, 6533, 3, 825, 69, 404, 195, 1, 4, 884, 2, 12, 162, 8, 686, 1442, 0, 67, 3, 7886, 14, 8, 24, 3, 0, 98, 0, 13, 256, 7, 43, 243, 5, 1, 3, 39, 2120, 9, 0, 0, 482, 0, 18, 40, 849, 0, 1263, 1002, 90, 2, 174, 7, 0, 2, 0, 1, 32, 0, 7, 543, 1, 110, 24, 19, 15, 7, 141, 6, 28, 114, 10, 47, 30, 3, 43, 3, 235, 298, 424, 679, 2, 81, 1665, 20, 4, 3187, 0, 8, 1, 120, 1313, 23, 0, 22, 8085, 750, 10, 13, 8, 38] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[290, 1, 0, 171, 62, 167, 479, 0, 0, 848, 257, 18, 141, 42, 108, 65, 5, 860, 1, 23, 815, 3, 1, 1146, 1742, 12, 0, 7508, 5114, 6567, 10, 1063, 138, 250, 351, 1, 16, 659, 0, 49, 177, 0, 598, 1290, 0, 75, 7, 7760, 1, 51, 10, 1, 0, 31, 2, 6, 148, 4, 26, 233, 1, 4, 38, 222, 2154, 1, 6, 0, 502, 0, 44, 17, 651, 0, 1320, 1150, 56, 9, 112, 4, 2, 7, 1, 4, 32, 0, 4, 636, 1, 79, 21, 33, 15, 13, 18, 0, 0, 111, 0, 7, 33, 11, 74, 13, 160, 1092, 214, 607, 0, 48, 2966, 6, 14, 3817, 0, 6, 0, 72, 1106, 3, 3, 36, 8026, 409, 9, 5, 7, 121] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[245, 0, 0, 124, 106, 167, 590, 0, 0, 579, 286, 27, 159, 70, 269, 133, 13, 1020, 0, 14, 1033, 0, 0, 1017, 1376, 33, 0, 7473, 4657, 5886, 6, 1231, 294, 375, 326, 2, 52, 1534, 0, 1, 108, 0, 448, 1219, 0, 236, 35, 7777, 0, 48, 3, 0, 1, 53, 1, 6, 209, 1, 13, 133, 2, 1, 6, 26, 1955, 1, 1, 0, 597, 3, 16, 90, 714, 0, 2105, 1291, 95, 2, 206, 6, 0, 1, 0, 10, 32, 0, 1, 681, 3, 149, 17, 4, 21, 4, 57, 3, 2, 279, 0, 14, 24, 11, 69, 17, 474, 258, 316, 838, 0, 151, 2558, 18, 0, 2661, 0, 7, 0, 139, 1627, 5, 1, 2, 8128, 369, 9, 10, 1, 59] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[342, 0, 0, 85, 118, 368, 658, 0, 0, 887, 203, 32, 221, 79, 137, 261, 4218, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1931, 0, 14, 1151, 0, 5, 1292, 1922, 3, 0, 0, 2047, 0, 33, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 374, 307, 2, 20, 790, 0, 5, 186, 1, 308, 1134, 0, 156, 3, 4415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 17, 0, 0, 102, 0, 6, 211, 0, 18, 227, 1, 3, 0, 54, 7596, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 2, 1, 504, 0, 7, 59, 811, 0, 1116, 1065, 135, 10, 254, 21, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 71, 1, 1, 722, 1, 329, 9, 22, 61, 6, 55, 2, 5977, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 283, 4, 15, 35, 8, 52, 1, 277, 341, 427, 636, 0, 103, 1961, 16, 2542, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2908, 0, 18, 0, 94, 806, 6, 2, 10, 2907, 524, 7, 16, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8572, 8986, 7807, 8252, 8332, 8206, 7982, 7399] avg=8192.0000 max=8986 max_rank=1 min=7399 min_rank=7 imbalance_ratio=1.096924 max_load_reduction=56550 max_load_reduction_pct=0.862885 moved_to_echo=31919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[167, 2, 0, 66, 94, 309, 441, 0, 4, 554, 280, 10, 148, 91, 95, 162, 4243, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 592, 0, 36, 818, 0, 15, 1440, 3242, 5, 0, 0, 1802, 0, 26, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 358, 178, 0, 5, 960, 0, 50, 142, 1, 468, 885, 0, 134, 15, 4374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 5, 0, 0, 25, 1, 11, 105, 0, 36, 303, 1, 1, 3, 47, 7439, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1774, 0, 0, 0, 615, 0, 11, 165, 883, 0, 1845, 1906, 57, 20, 290, 1, 1501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4, 68, 0, 0, 680, 1, 107, 12, 6, 43, 9, 64, 0, 6064, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 133, 0, 3, 20, 2, 15, 7, 108, 688, 412, 713, 0, 142, 2205, 9, 2519, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3183, 0, 4, 1, 95, 1695, 2, 1, 64, 2925, 485, 6, 11, 6, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7621, 8517, 7639, 7993, 9068, 8010, 8109, 8579] avg=8192.0000 max=9068 max_rank=4 min=7621 min_rank=0 imbalance_ratio=1.106934 max_load_reduction=56468 max_load_reduction_pct=0.861633 moved_to_echo=33766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[267, 1, 0, 65, 122, 248, 489, 1, 2, 668, 352, 16, 206, 87, 163, 129, 4185, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1262, 0, 7, 829, 1, 0, 1180, 1858, 15, 0, 0, 2171, 0, 2, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 562, 204, 1, 25, 798, 0, 37, 118, 1, 423, 1056, 0, 161, 3, 4384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 18, 3, 0, 81, 4, 14, 43, 1, 21, 291, 16, 4, 5, 71, 7577, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2045, 1, 0, 0, 447, 0, 30, 196, 701, 0, 1207, 1764, 59, 16, 177, 10, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 22, 93, 1, 0, 776, 0, 87, 30, 6, 1, 4, 27, 5, 6571, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 102, 7, 19, 17, 10, 34, 12, 222, 496, 301, 611, 4, 73, 2566, 4, 2524, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3552, 0, 20, 0, 30, 1236, 3, 1, 41, 2884, 338, 9, 5, 4, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7958, 7829, 8017, 8167, 8462, 8565, 8363, 8175] avg=8192.0000 max=8565 max_rank=5 min=7829 min_rank=1 imbalance_ratio=1.045532 max_load_reduction=56971 max_load_reduction_pct=0.869308 moved_to_echo=33565 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[318, 5, 0, 55, 31, 226, 669, 0, 0, 572, 663, 22, 439, 73, 183, 212, 4195, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1087, 0, 14, 951, 8, 84, 1209, 1535, 12, 1, 0, 1916, 0, 7, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 480, 201, 3, 11, 1524, 0, 111, 180, 4, 533, 1152, 0, 122, 21, 4203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 5, 6, 1, 82, 0, 0, 93, 4, 15, 184, 2, 4, 15, 38, 7682, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2038, 1, 14, 0, 608, 1, 7, 160, 648, 1, 1523, 1318, 64, 11, 211, 11, 1595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 22, 126, 0, 1, 966, 0, 212, 50, 110, 34, 9, 65, 9, 5625, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 115, 3, 10, 106, 35, 42, 14, 211, 840, 278, 875, 3, 77, 1904, 30, 2420, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3615, 0, 17, 1, 83, 865, 20, 7, 27, 2891, 378, 36, 10, 13, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8580, 7569, 8639, 8150, 8211, 8192, 8166, 8029] avg=8192.0000 max=8639 max_rank=2 min=7569 min_rank=1 imbalance_ratio=1.054565 max_load_reduction=56897 max_load_reduction_pct=0.868179 moved_to_echo=32598 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[290, 1, 0, 171, 62, 167, 479, 0, 0, 848, 257, 18, 141, 42, 108, 65, 4196, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 860, 1, 23, 815, 3, 1, 1146, 1742, 12, 0, 0, 2082, 0, 10, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 250, 351, 1, 16, 659, 0, 49, 177, 0, 598, 1290, 0, 75, 7, 4325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 51, 10, 1, 0, 31, 2, 6, 148, 4, 26, 233, 1, 4, 38, 222, 7508, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2154, 1, 6, 0, 502, 0, 44, 17, 651, 0, 1320, 1150, 56, 9, 112, 4, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 1, 4, 32, 0, 4, 636, 1, 79, 21, 33, 15, 13, 18, 0, 6565, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 7, 33, 11, 74, 13, 160, 1092, 214, 607, 0, 48, 2966, 6, 2491, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3817, 0, 6, 0, 72, 1106, 3, 3, 36, 2891, 409, 9, 5, 7, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7789, 7763, 7936, 8288, 7761, 8370, 9130, 8499] avg=8192.0000 max=9130 max_rank=6 min=7761 min_rank=4 imbalance_ratio=1.114502 max_load_reduction=56406 max_load_reduction_pct=0.860687 moved_to_echo=33760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[328, 2, 0, 95, 129, 293, 604, 1, 3, 1186, 198, 41, 145, 36, 91, 170, 4229, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 853, 3, 8, 1114, 11, 2, 1706, 2076, 9, 0, 0, 1968, 0, 3, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 404, 195, 1, 4, 884, 2, 12, 162, 8, 686, 1442, 0, 67, 3, 4390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8, 24, 3, 0, 98, 0, 13, 256, 7, 43, 243, 5, 1, 3, 39, 7404, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2120, 9, 0, 0, 482, 0, 18, 40, 849, 0, 1263, 1002, 90, 2, 174, 7, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 32, 0, 7, 543, 1, 110, 24, 19, 15, 7, 141, 6, 6531, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 114, 10, 47, 30, 3, 43, 3, 235, 298, 424, 679, 2, 81, 1665, 20, 2534, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3187, 0, 8, 1, 120, 1313, 23, 0, 22, 2907, 750, 10, 13, 8, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8513, 8584, 8329, 8163, 7706, 8388, 7449, 8404] avg=8192.0000 max=8584 max_rank=1 min=7449 min_rank=6 imbalance_ratio=1.047852 max_load_reduction=56952 max_load_reduction_pct=0.869019 moved_to_echo=31947 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[95, 1, 0, 80, 99, 331, 556, 0, 1, 1023, 547, 46, 226, 58, 85, 166, 4270, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1075, 0, 20, 1159, 0, 1, 1227, 2166, 7, 0, 0, 2077, 0, 13, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 340, 199, 1, 21, 1173, 0, 7, 70, 0, 388, 1221, 0, 231, 4, 4440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 7, 2, 0, 78, 0, 6, 187, 0, 20, 356, 0, 0, 8, 33, 7833, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1622, 1, 0, 0, 807, 0, 13, 127, 524, 1, 1273, 980, 51, 2, 197, 6, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 29, 0, 0, 586, 0, 135, 8, 35, 13, 15, 29, 6, 6174, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 137, 3, 9, 12, 5, 23, 6, 271, 420, 203, 521, 0, 54, 2073, 11, 2557, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 5, 0, 44, 1133, 6, 0, 23, 2943, 307, 2, 12, 2, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8553, 8649, 8204, 8545, 7334, 7987, 7631, 8633] avg=8192.0000 max=8649 max_rank=1 min=7334 min_rank=4 imbalance_ratio=1.055786 max_load_reduction=56887 max_load_reduction_pct=0.868027 moved_to_echo=31585 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[245, 0, 0, 124, 106, 167, 590, 0, 0, 579, 286, 27, 159, 70, 269, 133, 4249, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1020, 0, 14, 1033, 0, 0, 1017, 1376, 33, 0, 0, 1896, 0, 6, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 375, 326, 2, 52, 1534, 0, 1, 108, 0, 448, 1219, 0, 236, 35, 4335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 3, 0, 1, 53, 1, 6, 209, 1, 13, 133, 2, 1, 6, 26, 7473, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1955, 1, 1, 0, 597, 3, 16, 90, 714, 0, 2105, 1291, 95, 2, 206, 6, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 32, 0, 1, 681, 3, 149, 17, 4, 21, 4, 57, 3, 5884, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 279, 0, 14, 24, 11, 69, 17, 474, 258, 316, 838, 0, 151, 2558, 18, 2496, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 0, 7, 0, 139, 1627, 5, 1, 2, 2928, 369, 9, 10, 1, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7950, 7639, 8965, 7978, 8662, 7818, 8706, 7818] avg=8192.0000 max=8965 max_rank=2 min=7639 min_rank=1 imbalance_ratio=1.094360 max_load_reduction=56571 max_load_reduction_pct=0.863205 moved_to_echo=33004 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True + [2026-08-26 16:27:13] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 914.7 | throughput per GPU (TFLOP/s/GPU): 197.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.693684E+00 | loss scale: 1.0 | grad norm: 66.334 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[249, 0, 0, 127, 110, 171, 594, 0, 0, 584, 285, 29, 160, 68, 272, 134, 13, 1016, 0, 15, 1034, 0, 0, 999, 1378, 33, 0, 7470, 4639, 5887, 6, 1221, 294, 376, 328, 2, 52, 1521, 0, 1, 109, 0, 451, 1224, 0, 234, 36, 7780, 0, 48, 3, 0, 1, 53, 1, 6, 211, 1, 14, 129, 2, 1, 6, 25, 1942, 1, 1, 0, 593, 3, 16, 89, 717, 0, 2111, 1270, 103, 2, 203, 6, 0, 1, 0, 11, 32, 0, 1, 677, 3, 149, 17, 4, 21, 4, 59, 3, 2, 278, 0, 14, 23, 12, 67, 16, 479, 263, 310, 834, 0, 149, 2589, 18, 0, 2669, 0, 7, 0, 141, 1627, 5, 1, 2, 8132, 377, 9, 10, 1, 59] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[315, 5, 0, 59, 31, 227, 674, 0, 0, 575, 659, 23, 431, 68, 184, 215, 30, 1095, 0, 13, 946, 8, 85, 1187, 1527, 12, 1, 7680, 4699, 5623, 8, 711, 93, 480, 212, 2, 11, 1503, 0, 105, 184, 4, 535, 1165, 0, 125, 21, 7545, 0, 19, 5, 7, 1, 83, 0, 0, 100, 4, 15, 185, 2, 4, 15, 43, 2027, 1, 15, 0, 601, 1, 7, 155, 645, 1, 1524, 1296, 68, 10, 210, 11, 0, 24, 2, 21, 124, 0, 1, 955, 0, 210, 50, 113, 35, 9, 65, 9, 11, 112, 3, 10, 106, 35, 43, 14, 216, 838, 282, 886, 2, 78, 1921, 29, 2, 3659, 0, 17, 1, 82, 858, 20, 6, 27, 8023, 383, 35, 11, 12, 65] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[290, 1, 0, 172, 61, 168, 480, 0, 0, 849, 253, 18, 141, 42, 104, 68, 6, 860, 1, 23, 815, 3, 1, 1119, 1737, 12, 0, 7504, 5104, 6568, 10, 1050, 136, 252, 353, 0, 16, 652, 0, 46, 181, 0, 600, 1300, 0, 75, 7, 7760, 1, 51, 10, 1, 0, 29, 2, 6, 148, 4, 27, 234, 1, 4, 37, 227, 2140, 1, 5, 0, 506, 0, 44, 19, 650, 0, 1329, 1135, 57, 9, 111, 4, 2, 6, 1, 3, 31, 0, 4, 628, 1, 80, 21, 32, 16, 13, 18, 0, 0, 110, 0, 7, 32, 11, 75, 13, 162, 1094, 215, 601, 0, 45, 2997, 7, 13, 3853, 0, 7, 0, 71, 1111, 4, 3, 36, 8028, 413, 9, 5, 7, 121] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[347, 0, 0, 91, 117, 372, 661, 0, 0, 892, 199, 32, 225, 77, 132, 264, 12, 1933, 0, 15, 1152, 0, 5, 1286, 1911, 4, 0, 7604, 5004, 5972, 35, 574, 108, 373, 308, 1, 19, 788, 0, 5, 183, 1, 310, 1135, 0, 155, 3, 7921, 0, 15, 17, 0, 0, 104, 0, 6, 216, 0, 20, 225, 1, 3, 0, 54, 2632, 0, 2, 1, 502, 0, 7, 62, 807, 0, 1114, 1047, 133, 10, 257, 20, 0, 3, 0, 2, 67, 1, 1, 725, 1, 337, 8, 21, 65, 6, 56, 2, 7, 285, 4, 16, 34, 8, 49, 1, 276, 341, 422, 636, 0, 98, 1976, 16, 7, 2938, 0, 15, 0, 94, 804, 5, 2, 12, 8070, 523, 6, 18, 0, 97] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[331, 2, 0, 94, 126, 300, 605, 1, 3, 1192, 191, 42, 140, 34, 87, 168, 6, 850, 3, 7, 1117, 10, 1, 1684, 2067, 10, 0, 7404, 4839, 6532, 4, 818, 67, 407, 193, 1, 4, 886, 2, 11, 168, 8, 683, 1450, 0, 69, 3, 7885, 15, 9, 25, 3, 0, 99, 0, 13, 259, 7, 42, 243, 6, 1, 3, 38, 2108, 10, 0, 0, 478, 0, 18, 40, 855, 0, 1265, 984, 95, 2, 180, 7, 0, 2, 0, 1, 33, 0, 5, 542, 1, 110, 24, 20, 16, 7, 144, 6, 29, 114, 10, 46, 29, 3, 43, 3, 238, 307, 412, 689, 2, 76, 1673, 22, 4, 3212, 0, 8, 1, 120, 1318, 22, 0, 22, 8085, 756, 9, 13, 9, 40] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[265, 1, 0, 65, 123, 251, 493, 1, 2, 666, 358, 16, 202, 81, 164, 131, 11, 1257, 0, 7, 837, 1, 0, 1159, 1837, 14, 0, 7581, 5315, 6580, 2, 480, 249, 565, 206, 1, 27, 792, 0, 35, 119, 1, 420, 1066, 0, 165, 3, 7872, 0, 16, 19, 2, 0, 82, 4, 14, 44, 1, 21, 287, 15, 4, 5, 73, 2029, 2, 0, 0, 447, 0, 29, 193, 697, 0, 1216, 1742, 61, 18, 174, 10, 0, 6, 0, 22, 96, 1, 0, 765, 0, 89, 31, 6, 1, 4, 28, 5, 8, 102, 7, 19, 17, 10, 33, 14, 226, 497, 305, 614, 4, 74, 2576, 4, 0, 3584, 0, 22, 0, 29, 1243, 3, 1, 43, 8007, 341, 9, 6, 4, 54] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[168, 2, 0, 67, 90, 304, 447, 0, 5, 556, 276, 9, 150, 92, 96, 163, 8, 589, 0, 35, 819, 0, 14, 1420, 3231, 5, 0, 7436, 4409, 6053, 26, 515, 71, 362, 179, 0, 5, 958, 0, 48, 142, 1, 473, 884, 0, 144, 16, 7854, 0, 14, 6, 0, 0, 24, 1, 10, 104, 0, 37, 298, 1, 1, 3, 46, 1766, 0, 0, 0, 617, 0, 11, 164, 892, 0, 1855, 1883, 59, 20, 294, 1, 1, 0, 1, 4, 68, 0, 0, 672, 1, 106, 11, 6, 42, 10, 64, 0, 13, 129, 0, 3, 20, 2, 14, 7, 109, 698, 413, 723, 0, 141, 2221, 9, 2, 3212, 0, 4, 1, 99, 1709, 2, 1, 65, 8118, 486, 6, 12, 5, 97] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[103, 1, 0, 87, 102, 332, 555, 0, 1, 1033, 536, 44, 231, 53, 82, 169, 2, 1074, 0, 19, 1159, 0, 1, 1204, 2168, 7, 0, 7828, 5088, 6170, 13, 884, 108, 344, 201, 1, 22, 1165, 0, 7, 71, 0, 385, 1226, 0, 231, 4, 7973, 1, 13, 6, 2, 0, 77, 0, 6, 190, 0, 20, 356, 0, 0, 9, 31, 1609, 1, 0, 0, 807, 0, 13, 126, 520, 1, 1275, 966, 53, 2, 196, 7, 0, 2, 0, 0, 32, 0, 0, 584, 0, 141, 8, 35, 14, 17, 31, 5, 32, 140, 3, 9, 11, 5, 23, 5, 269, 426, 204, 526, 0, 54, 2098, 13, 1, 4098, 0, 6, 0, 45, 1126, 7, 0, 22, 8169, 313, 2, 12, 2, 75] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[249, 0, 0, 127, 110, 171, 594, 0, 0, 584, 285, 29, 160, 68, 272, 134, 4285, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1016, 0, 15, 1034, 0, 0, 999, 1378, 33, 0, 0, 1930, 0, 6, 1221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 376, 328, 2, 52, 1521, 0, 1, 109, 0, 451, 1224, 0, 234, 36, 4328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 3, 0, 1, 53, 1, 6, 211, 1, 14, 129, 2, 1, 6, 25, 7470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1942, 1, 1, 0, 593, 3, 16, 89, 717, 0, 2111, 1270, 103, 2, 203, 6, 1602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 32, 0, 1, 677, 3, 149, 17, 4, 21, 4, 59, 3, 5887, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 278, 0, 14, 23, 12, 67, 16, 479, 263, 310, 834, 0, 149, 2589, 18, 2548, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 7, 0, 141, 1627, 5, 1, 2, 2894, 377, 9, 10, 1, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7972, 7645, 8956, 7971, 8659, 7822, 8709, 7802] avg=8192.0000 max=8956 max_rank=2 min=7645 min_rank=1 imbalance_ratio=1.093262 max_load_reduction=56580 max_load_reduction_pct=0.863342 moved_to_echo=32992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[315, 5, 0, 59, 31, 227, 674, 0, 0, 575, 659, 23, 431, 68, 184, 215, 4228, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1095, 0, 13, 946, 8, 85, 1187, 1527, 12, 1, 0, 1955, 0, 8, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 480, 212, 2, 11, 1503, 0, 105, 184, 4, 535, 1165, 0, 125, 21, 4197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 5, 7, 1, 83, 0, 0, 100, 4, 15, 185, 2, 4, 15, 43, 7679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2027, 1, 15, 0, 601, 1, 7, 155, 645, 1, 1524, 1296, 68, 10, 210, 11, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 2, 21, 124, 0, 1, 955, 0, 210, 50, 113, 35, 9, 65, 9, 5623, 940, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 112, 3, 10, 106, 35, 43, 14, 216, 838, 282, 886, 2, 78, 1921, 29, 2471, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3659, 0, 17, 1, 82, 858, 20, 6, 27, 2855, 383, 35, 11, 12, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8571, 7578, 8637, 8162, 8195, 8182, 8178, 8033] avg=8192.0000 max=8637 max_rank=2 min=7578 min_rank=1 imbalance_ratio=1.054321 max_load_reduction=56899 max_load_reduction_pct=0.868210 moved_to_echo=32588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[265, 1, 0, 65, 123, 251, 493, 1, 2, 666, 358, 16, 202, 81, 164, 131, 4219, 915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1257, 0, 7, 837, 1, 0, 1159, 1837, 14, 0, 0, 2212, 0, 2, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 565, 206, 1, 27, 792, 0, 35, 119, 1, 420, 1066, 0, 165, 3, 4379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 19, 2, 0, 82, 4, 14, 44, 1, 21, 287, 15, 4, 5, 73, 7580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 2, 0, 0, 447, 0, 29, 193, 697, 0, 1216, 1742, 61, 18, 174, 10, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 22, 96, 1, 0, 765, 0, 89, 31, 6, 1, 4, 28, 5, 6580, 938, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 102, 7, 19, 17, 10, 33, 14, 226, 497, 305, 614, 4, 74, 2576, 4, 2578, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3584, 0, 22, 0, 29, 1243, 3, 1, 43, 2850, 341, 9, 6, 4, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7953, 7817, 8028, 8167, 8453, 8573, 8356, 8189] avg=8192.0000 max=8573 max_rank=5 min=7817 min_rank=1 imbalance_ratio=1.046509 max_load_reduction=56963 max_load_reduction_pct=0.869186 moved_to_echo=33571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[290, 1, 0, 172, 61, 168, 480, 0, 0, 849, 253, 18, 141, 42, 104, 68, 4230, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 860, 1, 23, 815, 3, 1, 1119, 1737, 12, 0, 0, 2123, 0, 10, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 252, 353, 0, 16, 652, 0, 46, 181, 0, 600, 1300, 0, 75, 7, 4317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 51, 10, 1, 0, 29, 2, 6, 148, 4, 27, 234, 1, 4, 37, 227, 7504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 1, 5, 0, 506, 0, 44, 19, 650, 0, 1329, 1135, 57, 9, 111, 4, 1763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 1, 3, 31, 0, 4, 628, 1, 80, 21, 32, 16, 13, 18, 0, 6568, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 7, 32, 11, 75, 13, 162, 1094, 215, 601, 0, 45, 2997, 7, 2541, 1218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3853, 0, 7, 0, 71, 1111, 4, 3, 36, 2857, 413, 9, 5, 7, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7779, 7760, 7935, 8286, 7773, 8365, 9128, 8510] avg=8192.0000 max=9128 max_rank=6 min=7760 min_rank=1 imbalance_ratio=1.114258 max_load_reduction=56408 max_load_reduction_pct=0.860718 moved_to_echo=33776 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[331, 2, 0, 94, 126, 300, 605, 1, 3, 1192, 191, 42, 140, 34, 87, 168, 4265, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 850, 3, 7, 1117, 10, 1, 1684, 2067, 10, 0, 0, 2005, 0, 4, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 407, 193, 1, 4, 886, 2, 11, 168, 8, 683, 1450, 0, 69, 3, 4381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 9, 25, 3, 0, 99, 0, 13, 259, 7, 42, 243, 6, 1, 3, 38, 7404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2108, 10, 0, 0, 478, 0, 18, 40, 855, 0, 1265, 984, 95, 2, 180, 7, 1675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 33, 0, 5, 542, 1, 110, 24, 20, 16, 7, 144, 6, 6532, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 114, 10, 46, 29, 3, 43, 3, 238, 307, 412, 689, 2, 76, 1673, 22, 2584, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3212, 0, 8, 1, 120, 1318, 22, 0, 22, 2869, 756, 9, 13, 9, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8501, 8582, 8333, 8167, 7717, 8394, 7439, 8403] avg=8192.0000 max=8582 max_rank=1 min=7439 min_rank=6 imbalance_ratio=1.047607 max_load_reduction=56954 max_load_reduction_pct=0.869049 moved_to_echo=31953 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[347, 0, 0, 91, 117, 372, 661, 0, 0, 892, 199, 32, 225, 77, 132, 264, 4253, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1933, 0, 15, 1152, 0, 5, 1286, 1911, 4, 0, 0, 2082, 0, 35, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 373, 308, 1, 19, 788, 0, 5, 183, 1, 310, 1135, 0, 155, 3, 4406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 17, 0, 0, 104, 0, 6, 216, 0, 20, 225, 1, 3, 0, 54, 7603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 0, 2, 1, 502, 0, 7, 62, 807, 0, 1114, 1047, 133, 10, 257, 20, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 67, 1, 1, 725, 1, 337, 8, 21, 65, 6, 56, 2, 5972, 945, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 285, 4, 16, 34, 8, 49, 1, 276, 341, 422, 636, 0, 98, 1976, 16, 2594, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2938, 0, 15, 0, 94, 804, 5, 2, 12, 2872, 523, 6, 18, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8583, 9009, 7795, 8264, 8322, 8213, 7957, 7393] avg=8192.0000 max=9009 max_rank=1 min=7393 min_rank=7 imbalance_ratio=1.099731 max_load_reduction=56527 max_load_reduction_pct=0.862534 moved_to_echo=31885 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[168, 2, 0, 67, 90, 304, 447, 0, 5, 556, 276, 9, 150, 92, 96, 163, 4278, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 589, 0, 35, 819, 0, 14, 1420, 3231, 5, 0, 0, 1835, 0, 26, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 362, 179, 0, 5, 958, 0, 48, 142, 1, 473, 884, 0, 144, 16, 4369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 6, 0, 0, 24, 1, 10, 104, 0, 37, 298, 1, 1, 3, 46, 7435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1766, 0, 0, 0, 617, 0, 11, 164, 892, 0, 1855, 1883, 59, 20, 294, 1, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4, 68, 0, 0, 672, 1, 106, 11, 6, 42, 10, 64, 0, 6053, 951, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 129, 0, 3, 20, 2, 14, 7, 109, 698, 413, 723, 0, 141, 2221, 9, 2572, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3212, 0, 4, 1, 99, 1709, 2, 1, 65, 2889, 486, 6, 12, 5, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7616, 8497, 7652, 7980, 9084, 7991, 8126, 8590] avg=8192.0000 max=9084 max_rank=4 min=7616 min_rank=0 imbalance_ratio=1.108887 max_load_reduction=56452 max_load_reduction_pct=0.861389 moved_to_echo=33791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[103, 1, 0, 87, 102, 332, 555, 0, 1, 1033, 536, 44, 231, 53, 82, 169, 4305, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1074, 0, 19, 1159, 0, 1, 1204, 2168, 7, 0, 0, 2117, 0, 13, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 344, 201, 1, 22, 1165, 0, 7, 71, 0, 385, 1226, 0, 231, 4, 4435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 6, 2, 0, 77, 0, 6, 190, 0, 20, 356, 0, 0, 9, 31, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1609, 1, 0, 0, 807, 0, 13, 126, 520, 1, 1275, 966, 53, 2, 196, 7, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 32, 0, 0, 584, 0, 141, 8, 35, 14, 17, 31, 5, 6170, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 140, 3, 9, 11, 5, 23, 5, 269, 426, 204, 526, 0, 54, 2098, 13, 2611, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4098, 0, 6, 0, 45, 1126, 7, 0, 22, 2907, 313, 2, 12, 2, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8561, 8648, 8200, 8539, 7333, 7996, 7643, 8616] avg=8192.0000 max=8648 max_rank=1 min=7333 min_rank=4 imbalance_ratio=1.055664 max_load_reduction=56888 max_load_reduction_pct=0.868042 moved_to_echo=31588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[2176, 128, 0, 768, 768, 2176, 4608, 128, 128, 6400, 2816, 256, 1792, 640, 1152, 1408, 33792, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 1280, 128, 128, 384, 128, 384, 128, 2048, 4480, 2688, 5504, 128, 768, 17920, 128, 20096, 9984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 8704, 128, 256, 7936, 128, 128, 10240, 16000, 128, 128, 0, 16000, 0, 128, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[16384, 128, 128, 128, 4608, 128, 256, 896, 5888, 128, 11776, 10496, 640, 128, 1664, 128, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1152, 3200, 2048, 128, 256, 8448, 128, 384, 1152, 128, 3968, 9472, 0, 1280, 128, 34944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 27008, 0, 128, 128, 768, 9856, 128, 128, 256, 23296, 3584, 128, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 512, 128, 128, 5632, 128, 1280, 256, 256, 256, 128, 512, 128, 49408, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[2176, 128, 0, 768, 768, 2176, 4608, 128, 128, 6400, 2816, 256, 1792, 640, 1152, 1408, 34176, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 8704, 128, 256, 7936, 128, 128, 10112, 15872, 128, 128, 0, 16384, 0, 128, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[16256, 128, 128, 128, 4608, 128, 256, 896, 5888, 128, 11776, 10368, 640, 128, 1664, 128, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 1280, 128, 128, 384, 128, 384, 128, 2048, 4480, 2688, 5632, 128, 768, 18176, 128, 20608, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1152, 3200, 2048, 128, 256, 8320, 128, 384, 1280, 128, 3968, 9472, 0, 1280, 128, 34816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 27264, 0, 128, 128, 768, 9856, 128, 128, 256, 23040, 3712, 128, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 512, 128, 128, 5632, 128, 1280, 256, 256, 256, 128, 512, 128, 49408, 7680, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 128, 128, 640, 128, 128, 1280, 128, 256, 2048, 128, 128, 128, 640, 60544, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 7976, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 1, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 8055, 4, 0, 0, 0, 0, 2, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 6, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 161, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 296, 0, 0, 2, 0, 0, 0, 0, 0, 8087, 1, 0, 0, 1, 0, 29, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8, 0, 0, 325, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8177, 0, 0, 0, 0, 0, 21, 0, 10, 0, 7596, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 73, 0, 8187, 0, 0, 6, 0, 0, 0, 1, 0, 117, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 45, 0, 1, 0, 0, 1, 0, 0, 0, 7373, 8, 1, 0, 0, 0, 913, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8173, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 6, 0, 0, 218, 7751, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 6, 0, 0, 29, 0, 0, 0, 2, 7990, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 6, 0, 9, 0, 8192, 0, 0, 47, 0, 0, 0, 9, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 7504, 0, 0, 0, 0, 0, 815, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 7983, 0, 0, 17, 16, 1, 0, 0, 0, 0, 0, 8187, 5, 0, 0, 5, 0, 0, 81, 7906, 0, 0, 0, 0, 0, 0, 7, 3, 1, 0, 0, 0, 16, 0, 11, 4, 6, 2, 2, 2, 7, 1, 0, 0, 21, 7980, 4, 2, 1, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 9, 1, 0, 9, 8160, 0, 0, 2, 0, 140, 0, 8169, 2, 10, 222, 1, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 1, 0, 0, 12, 0, 1689, 0, 0, 4, 0, 0, 1, 0, 0, 6664, 0, 3, 4, 0, 3, 189, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8165, 0, 0, 17, 0, 0, 0, 0, 5, 0, 0, 8178, 0, 0, 0, 15, 0, 0, 4, 7239, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 2, 0, 4615, 28, 0, 0, 0, 1, 72, 2, 0, 0, 7790, 0, 0, 13, 0, 11, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4, 0, 0, 1, 8152, 0, 2, 0, 0, 49, 0, 8192, 0, 0, 20, 0, 3, 0, 36, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 0, 0, 4, 0, 0, 0, 0, 6, 8149, 0, 0, 0, 0, 1, 17, 0, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8185, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 0, 37, 8100, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8184, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 1, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 47, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 11, 0, 0, 0, 0, 0, 8037, 1, 0, 0, 2, 0, 206, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7, 5550, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7779, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 6, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3413, 0, 0, 2, 0, 0, 0, 0, 0, 7781, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[17, 0, 0, 0, 0, 8182, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 9, 5753, 10, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8151, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 16, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 923, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 3, 0, 0, 0, 0, 467, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 0, 1, 0, 0, 7976, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 1, 0, 0, 2, 0, 1, 0, 0, 0, 7592, 712, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 4, 0, 0, 0, 0, 2, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8114, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 296, 0, 0, 2, 6563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 1, 0, 0, 1, 0, 29, 0, 0, 0, 0, 116, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8786, 8380, 8623, 8189, 8123, 7023, 8273] avg=8192.0000 max=8786 max_rank=1 min=7023 min_rank=6 imbalance_ratio=1.072510 max_load_reduction=56750 max_load_reduction_pct=0.865936 moved_to_echo=31608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 8, 0, 0, 325, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7072, 712, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8177, 0, 0, 0, 0, 0, 21, 0, 10, 0, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 1, 0, 73, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 45, 0, 1, 0, 6565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7373, 8, 1, 0, 0, 0, 913, 0, 0, 0, 0, 116, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 9227, 7787, 8733, 8190, 8195, 6729, 8451] avg=8192.0000 max=9227 max_rank=1 min=6729 min_rank=6 imbalance_ratio=1.126343 max_load_reduction=56309 max_load_reduction_pct=0.859207 moved_to_echo=31565 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8173, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 6, 0, 0, 218, 7751, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 6, 0, 0, 29, 0, 0, 7562, 712, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7990, 0, 0, 0, 0, 0, 0, 0, 1, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 6, 0, 9, 0, 0, 0, 0, 47, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 6565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7504, 0, 0, 0, 0, 0, 815, 0, 0, 0, 0, 116, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8775, 8330, 8553, 8193, 8187, 6814, 8474] avg=8192.0000 max=8775 max_rank=1 min=6814 min_rank=6 imbalance_ratio=1.071167 max_load_reduction=56761 max_load_reduction_pct=0.866104 moved_to_echo=31668 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 7983, 0, 0, 17, 16, 1, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 5, 0, 0, 5, 0, 0, 81, 7906, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 16, 0, 11, 4, 6, 2, 2, 2, 7, 1, 0, 7408, 716, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7980, 4, 2, 1, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 9, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8080, 0, 0, 2, 0, 140, 0, 0, 2, 10, 222, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 1, 0, 0, 12, 0, 1689, 0, 0, 4, 6567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6664, 0, 3, 4, 0, 3, 189, 1, 0, 0, 0, 121, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8058, 8787, 8181, 8550, 8184, 8466, 8283, 7027] avg=8192.0000 max=8787 max_rank=1 min=7027 min_rank=7 imbalance_ratio=1.072632 max_load_reduction=56749 max_load_reduction_pct=0.865921 moved_to_echo=31960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8165, 0, 0, 17, 0, 0, 0, 0, 5, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 0, 0, 0, 15, 0, 0, 4, 7239, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 7253, 711, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4615, 28, 0, 0, 0, 1, 72, 2, 0, 0, 537, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8079, 0, 2, 0, 0, 49, 0, 0, 0, 0, 20, 0, 3, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 0, 0, 4, 6555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8149, 0, 0, 0, 0, 1, 17, 0, 0, 1, 1, 116, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8075, 7981, 5270, 8207, 8190, 11261, 8329] avg=8192.0000 max=11261 max_rank=6 min=5270 min_rank=3 imbalance_ratio=1.374634 max_load_reduction=54275 max_load_reduction_pct=0.828171 moved_to_echo=35987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8185, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 1, 0, 0, 37, 8100, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7546, 712, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8184, 0, 0, 0, 0, 0, 3, 0, 0, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 11, 6563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 1, 0, 0, 2, 0, 206, 0, 0, 0, 0, 116, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8959, 8261, 8748, 8193, 8120, 6631, 8401] avg=8192.0000 max=8959 max_rank=1 min=6631 min_rank=6 imbalance_ratio=1.093628 max_load_reduction=56577 max_load_reduction_pct=0.863297 moved_to_echo=31345 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 7, 5550, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7616, 712, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7779, 0, 0, 0, 0, 0, 4, 0, 0, 0, 564, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3413, 0, 0, 2, 6567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 116, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 6356, 8343, 8348, 8192, 8126, 10012, 7939] avg=8192.0000 max=10012 max_rank=6 min=6356 min_rank=1 imbalance_ratio=1.222168 max_load_reduction=55524 max_load_reduction_pct=0.847229 moved_to_echo=34269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 0, 0, 0, 8182, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 9, 5753, 10, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7532, 712, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 1, 0, 0, 0, 1, 0, 0, 0, 0, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 6565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 3, 0, 0, 0, 0, 467, 1, 0, 0, 0, 116, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 6571, 8273, 8711, 8188, 8129, 8783, 8642] avg=8192.0000 max=8783 max_rank=6 min=6571 min_rank=1 imbalance_ratio=1.072144 max_load_reduction=56753 max_load_reduction_pct=0.865982 moved_to_echo=33742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 256, 128, 128, 128, 640, 128, 128, 1280, 128, 256, 2048, 128, 128, 128, 640, 60544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7, 0, 0, 328, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 21, 0, 11, 0, 7601, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 72, 0, 8187, 0, 0, 6, 0, 0, 0, 1, 0, 116, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 44, 0, 1, 0, 0, 1, 0, 0, 0, 7369, 9, 1, 0, 0, 0, 903, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 8183, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 2, 0, 0, 39, 8099, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8184, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 1, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 44, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 12, 0, 0, 0, 0, 0, 8034, 1, 0, 0, 1, 0, 209, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 7983, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 8056, 4, 0, 0, 0, 0, 2, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 6, 0, 8190, 0, 0, 0, 0, 0, 0, 3, 0, 142, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 304, 0, 0, 2, 0, 0, 0, 0, 0, 8079, 1, 0, 0, 1, 0, 32, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 7989, 0, 0, 19, 16, 2, 0, 0, 0, 0, 0, 8187, 5, 0, 0, 5, 0, 0, 72, 7894, 0, 0, 0, 0, 0, 0, 7, 3, 1, 0, 0, 0, 15, 0, 12, 4, 6, 2, 2, 1, 6, 1, 0, 0, 21, 7980, 4, 2, 1, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8, 1, 0, 9, 8160, 0, 1, 2, 0, 136, 0, 8169, 2, 10, 223, 1, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 1, 0, 0, 11, 0, 1856, 0, 0, 3, 0, 0, 0, 0, 0, 6519, 0, 1, 4, 0, 3, 182, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8171, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 6, 0, 0, 217, 7766, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 7, 0, 0, 29, 0, 0, 0, 3, 7994, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 5, 0, 9, 0, 8192, 0, 0, 46, 0, 0, 0, 10, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 7497, 0, 0, 0, 0, 0, 812, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 8182, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 12, 5757, 11, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8148, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 18, 0, 8190, 0, 1, 1, 0, 0, 0, 0, 0, 847, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 3, 0, 0, 0, 0, 452, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 9, 5526, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7757, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8179, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 7, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 0, 0, 2, 0, 0, 0, 0, 0, 7735, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8162, 0, 0, 15, 0, 0, 0, 0, 4, 0, 0, 8178, 0, 0, 0, 18, 0, 0, 4, 7286, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 2, 0, 4652, 29, 0, 0, 0, 1, 73, 2, 0, 0, 7785, 0, 0, 13, 0, 11, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8151, 0, 1, 0, 0, 48, 0, 8192, 0, 0, 21, 0, 3, 0, 32, 0, 3853, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 780, 0, 0, 4, 0, 0, 0, 0, 6, 8149, 0, 0, 0, 0, 1, 16, 0, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 0, 0, 0, 7, 0, 0, 328, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7080, 708, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 21, 0, 11, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 1, 0, 72, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 44, 0, 1, 0, 6541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7369, 9, 1, 0, 0, 0, 903, 0, 0, 0, 0, 149, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 9231, 7791, 8731, 8190, 8195, 6704, 8470] avg=8192.0000 max=9231 max_rank=1 min=6704 min_rank=6 imbalance_ratio=1.126831 max_load_reduction=56305 max_load_reduction_pct=0.859146 moved_to_echo=31559 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 8183, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 0, 0, 0, 2, 0, 0, 39, 8099, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7550, 708, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8184, 0, 0, 0, 0, 0, 1, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 12, 6539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8034, 1, 0, 0, 1, 0, 209, 0, 0, 0, 0, 149, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8956, 8261, 8743, 8193, 8121, 6608, 8432] avg=8192.0000 max=8956 max_rank=1 min=6608 min_rank=6 imbalance_ratio=1.093262 max_load_reduction=56580 max_load_reduction_pct=0.863342 moved_to_echo=31354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8171, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 0, 0, 0, 6, 0, 0, 217, 7766, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 7, 0, 0, 29, 0, 0, 7564, 708, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7994, 0, 0, 0, 0, 0, 0, 0, 1, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 5, 0, 9, 0, 0, 0, 0, 46, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 6541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7497, 0, 0, 0, 0, 0, 812, 0, 0, 0, 0, 149, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8784, 8328, 8555, 8194, 8187, 6784, 8496] avg=8192.0000 max=8784 max_rank=1 min=6784 min_rank=6 imbalance_ratio=1.072266 max_load_reduction=56752 max_load_reduction_pct=0.865967 moved_to_echo=31661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 0, 0, 0, 0, 1, 0, 0, 7983, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1, 0, 0, 2, 0, 1, 0, 0, 0, 7598, 708, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 4, 0, 0, 0, 0, 2, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8115, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 304, 0, 0, 2, 6539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 1, 0, 0, 1, 0, 32, 0, 0, 0, 0, 149, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8785, 8385, 8622, 8189, 8125, 6988, 8300] avg=8192.0000 max=8785 max_rank=1 min=6988 min_rank=6 imbalance_ratio=1.072388 max_load_reduction=56751 max_load_reduction_pct=0.865952 moved_to_echo=31602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 8182, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 0, 0, 0, 0, 0, 0, 12, 5757, 11, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7539, 708, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 1, 0, 0, 0, 1, 0, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 18, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1386, 0, 0, 0, 6541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 3, 0, 0, 0, 0, 452, 1, 0, 0, 0, 149, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 6574, 8276, 8705, 8188, 8133, 8775, 8648] avg=8192.0000 max=8775 max_rank=6 min=6574 min_rank=1 imbalance_ratio=1.071167 max_load_reduction=56761 max_load_reduction_pct=0.866104 moved_to_echo=33744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 7989, 0, 0, 19, 16, 2, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 5, 0, 0, 5, 0, 0, 72, 7894, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 15, 0, 12, 4, 6, 2, 2, 1, 6, 1, 0, 7413, 714, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7980, 4, 2, 1, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8079, 0, 1, 2, 0, 136, 0, 0, 2, 10, 223, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 1, 0, 0, 11, 0, 1856, 0, 0, 3, 6543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6519, 0, 1, 4, 0, 3, 182, 1, 0, 0, 0, 156, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8757, 8182, 8550, 8185, 8463, 8424, 6905] avg=8192.0000 max=8757 max_rank=1 min=6905 min_rank=7 imbalance_ratio=1.068970 max_load_reduction=56779 max_load_reduction_pct=0.866379 moved_to_echo=31977 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8162, 0, 0, 15, 0, 0, 0, 0, 4, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 791, 0, 0, 0, 18, 0, 0, 4, 7286, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 7251, 707, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4652, 29, 0, 0, 0, 1, 73, 2, 0, 0, 534, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8079, 0, 1, 0, 0, 48, 0, 0, 0, 0, 21, 0, 3, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 780, 0, 0, 4, 6531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8149, 0, 0, 0, 0, 1, 16, 0, 0, 1, 1, 149, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8119, 7974, 5306, 8206, 8185, 11169, 8360] avg=8192.0000 max=11169 max_rank=6 min=5306 min_rank=3 imbalance_ratio=1.363403 max_load_reduction=54367 max_load_reduction_pct=0.829575 moved_to_echo=35920 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 0, 0, 0, 0, 0, 0, 9, 5526, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7618, 708, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7757, 0, 0, 0, 0, 0, 5, 0, 0, 0, 561, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 0, 0, 2, 6542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 149, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 6330, 8339, 8324, 8191, 8127, 10084, 7925] avg=8192.0000 max=10084 max_rank=6 min=6330 min_rank=1 imbalance_ratio=1.230957 max_load_reduction=55452 max_load_reduction_pct=0.846130 moved_to_echo=34327 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 5504, 128, 128, 0, 0, 128, 128, 0, 0, 128, 0, 7680, 0, 128, 128, 52608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 5248, 128, 128, 0, 0, 128, 128, 0, 0, 128, 0, 8064, 0, 128, 128, 52352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[6400, 128, 0, 0, 128, 128, 0, 768, 58496, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 0, 256, 0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 59648, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 61056, 128, 128, 128, 0, 128, 128, 128, 128, 0, 4480, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 0, 128, 61696, 128, 128, 128, 128, 128, 2688, 128, 0, 128, 128, 1024, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 0, 128, 61440, 128, 128, 128, 128, 128, 2688, 128, 0, 128, 128, 1280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 64896, 0, 128, 128, 0, 384, 0, 0, 128, 128, 384, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 128, 65280, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 128, 0, 128, 128, 128, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 65280, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[8186, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 5009, 95, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 23, 0, 0, 8085, 0, 0, 0, 1, 8130, 0, 0, 0, 0, 0, 2882, 0, 8149, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8191, 0, 181] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[7954, 6, 0, 13, 233, 0, 0, 0, 0, 0, 0, 6519, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8153, 274, 0, 0, 8079, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 384, 0, 8047, 4, 2, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 0, 24, 3, 0, 4, 0, 1, 0, 0, 0, 208, 0, 7, 0, 18, 0, 0, 0, 0, 0, 131, 0, 0, 2, 0, 8162, 0, 0, 0, 0, 0, 0, 21, 45, 0, 1, 0, 0, 0, 4, 0, 7, 0, 11, 0, 0, 0, 0, 0, 814, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 16, 8175, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 12, 0, 0, 23, 0, 0, 0, 7687, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 8165, 1, 0, 0, 0, 7987, 0, 0, 0, 0, 0, 148, 0, 8076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 46] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 8, 0, 0, 117, 1, 1, 0, 3279, 21, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 8189, 0, 0, 0, 3, 8162, 0, 0, 0, 1, 0, 3568, 0, 8175, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 0, 1, 0, 0, 7, 0, 0, 234, 0, 0, 0, 0, 8183, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1042, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4809, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 8191, 19, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 1592, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1655, 0, 0, 6, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8189, 0, 0, 1, 4, 0, 0, 17, 0, 0, 0, 7837, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8189, 4, 0, 0, 7993, 0, 0, 0, 3, 8101, 2, 0, 0, 0, 0, 250, 0, 8074, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4, 12, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 396, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[8189, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6242, 37, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 49, 0, 0, 8115, 103, 0, 0, 0, 8184, 0, 0, 0, 7, 0, 1718, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3, 93, 4, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 96, 0, 0, 45, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[8179, 0, 0, 0, 69, 0, 0, 1202, 0, 0, 0, 5284, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 8186, 0, 0, 0, 0, 7950, 0, 1, 0, 0, 0, 636, 0, 6998, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 203, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1, 0, 0, 0, 5, 0, 2070, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 25, 54, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2120, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 5009, 95, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 23, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2882, 0, 6930, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8085, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8097, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 5994, 1219, 166, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8135, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7238, 8210, 9815, 8119, 8172, 8030, 7626, 8326] avg=8192.0000 max=9815 max_rank=2 min=7238 min_rank=0 imbalance_ratio=1.198120 max_load_reduction=55721 max_load_reduction_pct=0.850235 moved_to_echo=32154 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2121, 0, 0, 0, 12, 0, 0, 23, 0, 0, 0, 7687, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 6868, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 8026, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 5999, 1208, 166, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9851, 8203, 7018, 8197, 8153, 8429, 7497, 8188] avg=8192.0000 max=9851 max_rank=0 min=7018 min_rank=2 imbalance_ratio=1.202515 max_load_reduction=55685 max_load_reduction_pct=0.849686 moved_to_echo=32267 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2121, 0, 0, 0, 8, 0, 0, 117, 1, 1, 0, 3279, 21, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 3568, 0, 6952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 0, 1, 0, 0, 7, 0, 0, 8129, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 8017, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1042, 0, 0, 5998, 1223, 166, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5568, 8109, 10524, 8223, 8227, 8257, 8477, 8151] avg=8192.0000 max=10524 max_rank=2 min=5568 min_rank=0 imbalance_ratio=1.284668 max_load_reduction=55012 max_load_reduction_pct=0.839417 moved_to_echo=33112 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2052, 6, 0, 13, 233, 0, 0, 0, 0, 0, 0, 6519, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8153, 274, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 6839, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 0, 24, 3, 0, 8079, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 208, 0, 7, 0, 18, 0, 0, 0, 0, 0, 8087, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 2, 0, 7995, 0, 0, 0, 0, 0, 0, 21, 45, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 0, 11, 0, 0, 0, 0, 0, 814, 4, 0, 5829, 1208, 167, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 16, 8109, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8876, 8440, 7227, 8158, 8398, 8195, 8098, 8144] avg=8192.0000 max=8876 max_rank=0 min=7227 min_rank=2 imbalance_ratio=1.083496 max_load_reduction=56660 max_load_reduction_pct=0.864563 moved_to_echo=32835 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2121, 0, 0, 1, 4, 0, 0, 17, 0, 0, 0, 7837, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8189, 4, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 250, 0, 6866, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7993, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8068, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 0, 1, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 396, 0, 3, 5996, 1208, 166, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9983, 8204, 7123, 8028, 8159, 8061, 7820, 8158] avg=8192.0000 max=9983 max_rank=0 min=7123 min_rank=2 imbalance_ratio=1.218628 max_load_reduction=55553 max_load_reduction_pct=0.847672 moved_to_echo=32198 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2121, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4809, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1592, 0, 6920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 8026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1655, 0, 0, 5999, 1217, 166, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6940, 8089, 8531, 8225, 8209, 8307, 9084, 8151] avg=8192.0000 max=9084 max_rank=6 min=6940 min_rank=0 imbalance_ratio=1.108887 max_load_reduction=56452 max_load_reduction_pct=0.861389 moved_to_echo=33751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2118, 0, 0, 0, 69, 0, 0, 1202, 0, 0, 0, 5284, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 636, 0, 5951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 203, 3, 9, 0, 0, 0, 8186, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7918, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 1, 0, 0, 8024, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1, 0, 0, 0, 5, 0, 2070, 2, 0, 5989, 1047, 166, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 25, 54, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8675, 8074, 6588, 8438, 7990, 8136, 9416, 8219] avg=8192.0000 max=9416 max_rank=6 min=6588 min_rank=2 imbalance_ratio=1.149414 max_load_reduction=56120 max_load_reduction_pct=0.856323 moved_to_echo=33761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2121, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6242, 37, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 49, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1718, 0, 6882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3, 8151, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 4, 0, 0, 1, 8021, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 96, 0, 0, 5996, 1211, 166, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8405, 8207, 8710, 8148, 8228, 8121, 7518, 8199] avg=8192.0000 max=8710 max_rank=2 min=7518 min_rank=6 imbalance_ratio=1.063232 max_load_reduction=56826 max_load_reduction_pct=0.867096 moved_to_echo=32066 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 13, 0, 0, 18, 0, 0, 0, 7651, 7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 8163, 1, 0, 0, 0, 7990, 0, 0, 0, 0, 0, 177, 0, 8072, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 45] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 9, 0, 0, 115, 1, 1, 0, 3126, 22, 1, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 8189, 1, 0, 0, 3, 8157, 0, 0, 0, 1, 0, 3754, 0, 8174, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 1, 0, 0, 7, 0, 0, 209, 0, 0, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1041, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6048, 33, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 42, 0, 0, 8111, 100, 0, 0, 0, 8184, 0, 0, 0, 8, 0, 1931, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4, 92, 5, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 101, 0, 0, 44, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 19] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[8186, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 4837, 104, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 24, 0, 0, 8093, 0, 0, 0, 1, 8128, 0, 0, 0, 0, 0, 3008, 0, 8153, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8191, 0, 176] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4469, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 8191, 20, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 1869, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1730, 0, 0, 4, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8189, 0, 0, 1, 2, 0, 0, 16, 0, 0, 0, 7795, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 5, 0, 0, 7993, 0, 0, 0, 3, 8100, 2, 0, 0, 0, 0, 276, 0, 8086, 2, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 408, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[7952, 5, 0, 13, 234, 0, 0, 0, 0, 0, 0, 6311, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8154, 284, 0, 0, 8083, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 526, 0, 8036, 4, 2, 0, 0, 0, 0, 1, 0, 0, 16, 0, 0, 0, 0, 25, 3, 0, 3, 0, 1, 0, 0, 0, 196, 0, 6, 0, 19, 0, 0, 0, 0, 0, 130, 0, 0, 2, 0, 8162, 0, 0, 0, 0, 0, 0, 21, 47, 0, 1, 0, 0, 0, 3, 0, 7, 0, 9, 0, 0, 0, 0, 0, 893, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 15, 8175, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[8180, 0, 0, 0, 58, 0, 0, 1192, 0, 0, 0, 5147, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 0, 8187, 0, 0, 0, 0, 7954, 0, 1, 0, 0, 0, 743, 0, 6998, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 209, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 1, 0, 0, 0, 5, 0, 2126, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 26, 52, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2284, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 4837, 104, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 24, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3008, 0, 6793, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8093, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8097, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8029, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 5831, 1360, 162, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8137, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7239, 8211, 9803, 8125, 8171, 8032, 7633, 8322] avg=8192.0000 max=9803 max_rank=2 min=7239 min_rank=0 imbalance_ratio=1.196655 max_load_reduction=55733 max_load_reduction_pct=0.850418 moved_to_echo=32158 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2286, 0, 0, 0, 13, 0, 0, 18, 0, 0, 0, 7651, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 6725, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7960, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 8030, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 5835, 1347, 162, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9975, 8205, 6905, 8193, 8161, 8435, 7474, 8188] avg=8192.0000 max=9975 max_rank=0 min=6905 min_rank=2 imbalance_ratio=1.217651 max_load_reduction=55561 max_load_reduction_pct=0.847794 moved_to_echo=32258 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2285, 0, 0, 0, 9, 0, 0, 115, 1, 1, 0, 3126, 22, 1, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 1, 0, 3754, 0, 6810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 1, 0, 0, 7, 0, 0, 8125, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 0, 8023, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1041, 0, 0, 5835, 1364, 161, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5580, 8110, 10569, 8222, 8220, 8237, 8446, 8152] avg=8192.0000 max=10569 max_rank=2 min=5580 min_rank=0 imbalance_ratio=1.290161 max_load_reduction=54967 max_load_reduction_pct=0.838730 moved_to_echo=33055 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2284, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6048, 33, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 42, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1931, 0, 6739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4, 8152, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 5, 0, 0, 1, 8025, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 101, 0, 0, 5833, 1349, 162, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8369, 8200, 8778, 8143, 8229, 8124, 7491, 8202] avg=8192.0000 max=8778 max_rank=2 min=7491 min_rank=6 imbalance_ratio=1.071533 max_load_reduction=56758 max_load_reduction_pct=0.866058 moved_to_echo=32046 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2285, 0, 0, 1, 2, 0, 0, 16, 0, 0, 0, 7795, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 5, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 276, 0, 6737, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7993, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8069, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 1, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 408, 0, 3, 5833, 1349, 162, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10102, 8205, 7020, 8027, 8159, 8061, 7802, 8160] avg=8192.0000 max=10102 max_rank=0 min=7020 min_rank=2 imbalance_ratio=1.233154 max_load_reduction=55434 max_load_reduction_pct=0.845856 moved_to_echo=32182 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2282, 0, 0, 0, 58, 0, 0, 1192, 0, 0, 0, 5147, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 743, 0, 5831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 209, 3, 6, 0, 0, 0, 8187, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 1, 0, 0, 8028, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 1, 0, 0, 0, 5, 0, 2126, 1, 0, 5827, 1167, 162, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 26, 52, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8682, 8072, 6575, 8441, 7995, 8132, 9418, 8221] avg=8192.0000 max=9418 max_rank=6 min=6575 min_rank=2 imbalance_ratio=1.149658 max_load_reduction=56118 max_load_reduction_pct=0.856293 moved_to_echo=33766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2209, 5, 0, 13, 234, 0, 0, 0, 0, 0, 0, 6311, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8154, 284, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 6692, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 16, 0, 0, 0, 0, 25, 3, 0, 8083, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 196, 0, 6, 0, 19, 0, 0, 0, 0, 0, 8092, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 2, 0, 7999, 0, 0, 0, 0, 0, 0, 21, 47, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 9, 0, 0, 0, 0, 0, 893, 5, 0, 5669, 1344, 163, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 15, 8107, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8821, 8456, 7222, 8162, 8391, 8200, 8144, 8140] avg=8192.0000 max=8821 max_rank=0 min=7222 min_rank=2 imbalance_ratio=1.076782 max_load_reduction=56715 max_load_reduction_pct=0.865402 moved_to_echo=32875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2286, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4469, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1869, 0, 6775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 0, 0, 8030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1730, 0, 0, 5835, 1357, 162, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6768, 8077, 8664, 8223, 8210, 8315, 9128, 8151] avg=8192.0000 max=9128 max_rank=6 min=6768 min_rank=0 imbalance_ratio=1.114258 max_load_reduction=56408 max_load_reduction_pct=0.860718 moved_to_echo=33804 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 128, 128, 0, 6400, 128, 128, 47872, 9600, 1408, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[16896, 128, 0, 128, 384, 128, 0, 1408, 128, 128, 0, 46720, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 128, 128, 0, 128, 0, 11264, 0, 54272, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 128, 128, 0, 6656, 128, 128, 46592, 10752, 1408, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[18304, 128, 0, 128, 384, 128, 0, 1408, 128, 128, 0, 45440, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 128, 128, 0, 128, 0, 12288, 0, 53120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1280, 128, 128, 128, 128, 64256, 0, 128, 0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 65152, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 0, 128, 0, 128, 128, 128, 256, 128, 128, 128, 128, 0, 65024, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 128, 0, 0, 128, 0, 128, 128, 128, 65152, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 0, 128, 512, 0, 128, 128, 128, 0, 0, 128, 0, 128, 64640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 65152, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 0, 128, 0, 128, 128, 128, 256, 128, 128, 128, 128, 0, 65024, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 0, 128, 384, 0, 128, 128, 128, 0, 0, 128, 0, 128, 64640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8097, 8187, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 3, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 92, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 8192, 7185, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 8192, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 6595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2470, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8021, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 295, 8192, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 8192, 7466, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 766, 0, 0, 0, 0, 0, 0, 0, 0, 7583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 6, 0, 0, 0, 0, 212, 0, 284, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 6, 8, 0, 3, 0, 0, 0, 0, 0, 34, 8158, 3, 0, 14, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 1, 0, 8171, 8147, 6819, 0, 1, 1, 6, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 8151, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 6861, 3, 0, 0, 0, 0, 0, 0, 0, 3, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 61, 24, 0, 0, 0, 0, 12, 7, 7, 0, 0, 15, 0, 0, 39, 0, 0, 0, 0, 1216, 0, 1344, 14, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 205, 3, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8182, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 8186, 6195, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2, 1948, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8192, 8187, 7298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8187, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 839, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8189, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8191, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 750, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 293, 54, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8171, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 92, 1, 0, 7054, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8140, 8938, 8186, 8540, 8191, 8185, 7165] avg=8192.0000 max=8938 max_rank=2 min=7165 min_rank=7 imbalance_ratio=1.091064 max_load_reduction=56598 max_load_reduction_pct=0.863617 moved_to_echo=32081 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 752, 0, 7185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 6595, 0, 0, 0, 0, 0, 294, 54, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2470, 0, 0, 7078, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8211, 7937, 8254, 7032, 8187, 8181, 9568] avg=8192.0000 max=9568 max_rank=7 min=7032 min_rank=4 imbalance_ratio=1.167969 max_load_reduction=55968 max_load_reduction_pct=0.854004 moved_to_echo=32968 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 758, 0, 7298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 297, 54, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 0, 8171, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 839, 0, 0, 7137, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8157, 8058, 8184, 8546, 8216, 8188, 7994] avg=8192.0000 max=8546 max_rank=4 min=7994 min_rank=7 imbalance_ratio=1.043213 max_load_reduction=56990 max_load_reduction_pct=0.869598 moved_to_echo=32944 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 295, 8138, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 0, 7466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 0, 0, 0, 0, 0, 0, 0, 0, 7583, 0, 0, 0, 0, 0, 292, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8177, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 6, 0, 0, 0, 0, 212, 0, 284, 0, 0, 7034, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8021, 8450, 8214, 8192, 8698, 8191, 8187, 7583] avg=8192.0000 max=8698 max_rank=4 min=7583 min_rank=7 imbalance_ratio=1.061768 max_load_reduction=56838 max_load_reduction_pct=0.867279 moved_to_echo=32659 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 205, 3, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8128, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 756, 0, 6195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 296, 54, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8170, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2, 1948, 0, 0, 7116, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8402, 8136, 6951, 8181, 8388, 8187, 8183, 9108] avg=8192.0000 max=9108 max_rank=7 min=6951 min_rank=2 imbalance_ratio=1.111816 max_load_reduction=56428 max_load_reduction_pct=0.861023 moved_to_echo=33866 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 0, 3, 0, 0, 0, 0, 0, 34, 8099, 3, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 1, 0, 748, 0, 6819, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 6861, 3, 0, 0, 0, 0, 300, 59, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 61, 24, 0, 0, 0, 0, 12, 7, 8132, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 15, 0, 0, 39, 0, 0, 0, 0, 1216, 0, 1344, 14, 4, 7123, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8167, 7578, 8162, 7245, 8187, 8246, 9777] avg=8192.0000 max=9777 max_rank=7 min=7245 min_rank=4 imbalance_ratio=1.193481 max_load_reduction=55759 max_load_reduction_pct=0.850815 moved_to_echo=33455 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 297, 54, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 7135, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8138, 8929, 8189, 8541, 8187, 8186, 7166] avg=8192.0000 max=8929 max_rank=2 min=7166 min_rank=7 imbalance_ratio=1.089966 max_load_reduction=56607 max_load_reduction_pct=0.863754 moved_to_echo=32080 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8133, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 296, 54, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8173, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 7, 0, 0, 7131, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8137, 8931, 8188, 8546, 8190, 8180, 7175] avg=8192.0000 max=8931 max_rank=2 min=7175 min_rank=7 imbalance_ratio=1.090210 max_load_reduction=56605 max_load_reduction_pct=0.863724 moved_to_echo=32091 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8187, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8189, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8190, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 286, 8192, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8068, 8192, 7459, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 761, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 8, 0, 0, 0, 0, 210, 0, 314, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 8189, 7291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8187, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 4, 7, 0, 3, 0, 0, 0, 0, 0, 33, 8158, 4, 0, 17, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 1, 0, 8171, 8147, 6815, 0, 1, 1, 6, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 8152, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8, 6773, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 56, 24, 0, 0, 0, 0, 14, 7, 7, 0, 0, 13, 0, 0, 41, 0, 0, 0, 0, 1207, 0, 1444, 16, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 8192, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 8192, 7125, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 8192, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 6521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2601, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8095, 8187, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 3, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 93, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 214, 2, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8184, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 8186, 6158, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2, 1976, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 286, 8138, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 0, 7459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 0, 0, 0, 0, 0, 313, 54, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8177, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 8, 0, 0, 0, 0, 210, 0, 314, 0, 0, 6994, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8024, 8439, 8220, 8194, 8714, 8190, 8185, 7570] avg=8192.0000 max=8714 max_rank=4 min=7570 min_rank=7 imbalance_ratio=1.063721 max_load_reduction=56822 max_load_reduction_pct=0.867035 moved_to_echo=32659 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 0, 7125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 6521, 0, 0, 0, 0, 0, 315, 54, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2601, 0, 0, 7045, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8212, 7891, 8255, 6978, 8187, 8182, 9667] avg=8192.0000 max=9667 max_rank=7 min=6978 min_rank=4 imbalance_ratio=1.180054 max_load_reduction=55869 max_load_reduction_pct=0.852493 moved_to_echo=33014 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 763, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 314, 54, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8172, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 93, 1, 0, 7018, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8140, 8951, 8185, 8562, 8191, 8188, 7129] avg=8192.0000 max=8951 max_rank=2 min=7129 min_rank=7 imbalance_ratio=1.092651 max_load_reduction=56585 max_load_reduction_pct=0.863419 moved_to_echo=32070 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 3, 0, 0, 0, 0, 0, 33, 8103, 4, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 1, 0, 763, 0, 6815, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8, 6773, 3, 0, 0, 0, 0, 320, 55, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 56, 24, 0, 0, 0, 0, 14, 7, 8132, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 13, 0, 0, 41, 0, 0, 0, 0, 1207, 0, 1444, 16, 4, 7088, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8171, 7587, 8162, 7179, 8187, 8242, 9835] avg=8192.0000 max=9835 max_rank=7 min=7179 min_rank=4 imbalance_ratio=1.200562 max_load_reduction=55701 max_load_reduction_pct=0.849930 moved_to_echo=33443 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 214, 2, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8131, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 317, 53, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8170, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2, 1976, 0, 0, 7083, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8409, 8139, 6928, 8180, 8407, 8187, 8183, 9103] avg=8192.0000 max=9103 max_rank=7 min=6928 min_rank=2 imbalance_ratio=1.111206 max_load_reduction=56433 max_load_reduction_pct=0.861099 moved_to_echo=33880 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 772, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 318, 54, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 7101, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8137, 8945, 8189, 8564, 8188, 8185, 7132] avg=8192.0000 max=8945 max_rank=2 min=7132 min_rank=7 imbalance_ratio=1.091919 max_load_reduction=56591 max_load_reduction_pct=0.863510 moved_to_echo=32069 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8133, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 772, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 317, 54, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8174, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 5, 0, 0, 7096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8138, 8949, 8187, 8568, 8190, 8181, 7135] avg=8192.0000 max=8949 max_rank=2 min=7135 min_rank=7 imbalance_ratio=1.092407 max_load_reduction=56587 max_load_reduction_pct=0.863449 moved_to_echo=32074 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 772, 0, 7291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 318, 54, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3, 0, 8173, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 7101, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8160, 8065, 8184, 8564, 8216, 8190, 7965] avg=8192.0000 max=8564 max_rank=4 min=7965 min_rank=7 imbalance_ratio=1.045410 max_load_reduction=56972 max_load_reduction_pct=0.869324 moved_to_echo=32935 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 1536, 128, 7040, 128, 128, 56832, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 128, 0, 128, 0, 6144, 0, 59520, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 896, 128, 0, 0, 0, 0, 0, 0, 128, 61824, 128, 0, 0, 0, 0, 2432, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 1536, 128, 7296, 128, 128, 56576, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 128, 0, 128, 0, 6144, 0, 59392, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 896, 128, 0, 0, 0, 0, 0, 0, 128, 61696, 128, 0, 0, 0, 0, 2560, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 256, 128, 128, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 0, 512, 65152, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 1, 20, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8177, 3, 0, 0, 0, 0, 1, 0, 8105, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 6, 1, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8161, 0, 1, 0, 4, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 270, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 5, 0, 0, 0, 0, 8192, 0, 7699, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 4242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 13, 0, 0, 8145, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7885, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 51, 3, 0, 0, 8182, 0, 7857, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 98, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2805, 4791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 16, 995, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 7996, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 8162, 0, 131, 0, 0, 0, 0, 0, 0, 8162, 0, 8169, 5, 0, 0, 21, 0, 2, 18, 26, 8010, 0, 0, 0, 0, 3, 0, 0, 0, 77, 0, 6, 1, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 740, 1, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8172, 3, 2, 14, 0, 1, 9, 4, 7928, 0, 18, 0, 1, 0, 0, 0, 0, 37, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7565, 0, 0, 1, 29, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 7223, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 1368, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 7787, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 28, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 14, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8191, 0, 0, 0, 4, 0, 0, 0, 0, 8191, 0, 7254, 0, 3, 0, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 0, 1, 0, 0, 0, 0, 11, 0, 0, 0, 112, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3459, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 13, 8185, 0, 0, 0, 0, 1, 3, 0, 4599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 76, 0, 0, 95, 0, 0, 8022, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 7836, 365, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2168, 3, 0, 0, 0, 0, 1, 0, 8105, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 6, 1, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8161, 0, 1, 0, 4, 2, 171, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8541, 8234, 5559, 10280, 8186, 8197, 8369] avg=8192.0000 max=10280 max_rank=4 min=5559 min_rank=3 imbalance_ratio=1.254883 max_load_reduction=55256 max_load_reduction_pct=0.843140 moved_to_echo=35032 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7699, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7796, 365, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 4242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2173, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 13, 0, 0, 8136, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7885, 0, 0, 0, 0, 0, 171, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8051, 8166, 11041, 5570, 8187, 8259, 8085] avg=8192.0000 max=11041 max_rank=3 min=5570 min_rank=4 imbalance_ratio=1.347778 max_load_reduction=54495 max_load_reduction_pct=0.831528 moved_to_echo=30101 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 2, 0, 0, 0, 0, 7832, 365, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 270, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2173, 0, 0, 0, 0, 0, 0, 0, 7215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 4, 0, 0, 0, 0, 171, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8475, 8214, 6561, 9388, 8186, 8177, 8366] avg=8192.0000 max=9388 max_rank=4 min=6561 min_rank=3 imbalance_ratio=1.145996 max_load_reduction=56148 max_load_reduction_pct=0.856750 moved_to_echo=34117 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 51, 3, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7762, 365, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2805, 4791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2172, 0, 0, 0, 0, 0, 0, 16, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7988, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 7996, 0, 0, 0, 0, 0, 171, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8207, 8234, 13078, 3183, 8189, 8211, 8212] avg=8192.0000 max=13078 max_rank=3 min=3183 min_rank=4 imbalance_ratio=1.596436 max_load_reduction=52458 max_load_reduction_pct=0.800446 moved_to_echo=27795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 5, 0, 0, 21, 0, 2, 18, 26, 342, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 6, 1, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 7668, 370, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 740, 1, 0, 0, 0, 0, 0, 0, 0, 13, 0, 5473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2156, 3, 2, 14, 0, 1, 9, 4, 7928, 0, 18, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 22, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7565, 0, 0, 1, 29, 0, 173, 22, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8586, 8138, 6228, 10136, 8222, 8146, 7806] avg=8192.0000 max=10136 max_rank=4 min=6228 min_rank=3 imbalance_ratio=1.237305 max_load_reduction=55400 max_load_reduction_pct=0.845337 moved_to_echo=34310 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7254, 0, 3, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 112, 0, 5, 0, 0, 0, 0, 0, 0, 7630, 365, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3459, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2171, 0, 0, 0, 0, 1, 3, 0, 4599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 95, 0, 0, 8013, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 0, 0, 171, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 7602, 8128, 10548, 6787, 8193, 8185, 7917] avg=8192.0000 max=10548 max_rank=3 min=6787 min_rank=4 imbalance_ratio=1.287598 max_load_reduction=54988 max_load_reduction_pct=0.839050 moved_to_echo=31082 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7834, 365, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 14, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2173, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 171, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8497, 8209, 5667, 10233, 8186, 8183, 8388] avg=8192.0000 max=10233 max_rank=4 min=5667 min_rank=3 imbalance_ratio=1.249146 max_load_reduction=55303 max_load_reduction_pct=0.843857 moved_to_echo=34990 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7223, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7834, 365, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1368, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2171, 0, 0, 0, 0, 0, 0, 0, 7787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 28, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 171, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 7577, 8213, 6854, 9959, 8187, 8178, 8393] avg=8192.0000 max=9959 max_rank=4 min=6854 min_rank=3 imbalance_ratio=1.215698 max_load_reduction=55577 max_load_reduction_pct=0.848038 moved_to_echo=34717 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 269, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 24, 0, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8176, 3, 0, 0, 0, 0, 1, 0, 8106, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 5, 1, 2, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 4, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8191, 0, 0, 0, 4, 0, 0, 0, 0, 8191, 0, 7218, 0, 3, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 2, 0, 0, 0, 0, 10, 0, 0, 0, 113, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3371, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 18, 8184, 0, 0, 0, 0, 1, 3, 0, 4766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 74, 0, 0, 94, 0, 0, 8012, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7682, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 7165, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 1456, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 7758, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 25, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 7688, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 15, 0, 0, 8148, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 0, 8162, 0, 125, 0, 0, 0, 0, 0, 1, 8161, 0, 8170, 2, 0, 0, 21, 0, 0, 18, 27, 8008, 0, 0, 0, 0, 3, 0, 0, 0, 79, 0, 6, 1, 0, 1, 0, 7, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 736, 1, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 8171, 3, 2, 13, 0, 1, 10, 3, 7927, 0, 18, 0, 2, 0, 0, 0, 0, 36, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7577, 0, 0, 0, 29, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 51, 3, 0, 0, 8183, 0, 7846, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 100, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2962, 4688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 16, 946, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7986, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 16, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 7821, 380, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2137, 3, 0, 0, 0, 0, 1, 0, 8106, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 5, 1, 2, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 4, 1, 175, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8556, 8238, 5567, 10250, 8187, 8201, 8366] avg=8192.0000 max=10250 max_rank=4 min=5567 min_rank=3 imbalance_ratio=1.251221 max_load_reduction=55286 max_load_reduction_pct=0.843597 moved_to_echo=35004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 2, 0, 0, 0, 0, 7816, 380, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 269, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2141, 0, 0, 0, 0, 0, 0, 0, 7195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 4, 0, 0, 0, 0, 175, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8488, 8212, 6600, 9336, 8186, 8180, 8364] avg=8192.0000 max=9336 max_rank=4 min=6600 min_rank=3 imbalance_ratio=1.139648 max_load_reduction=56200 max_load_reduction_pct=0.857544 moved_to_echo=34066 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7688, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7777, 381, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2141, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 15, 0, 0, 8141, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 175, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8055, 8163, 10901, 5701, 8187, 8258, 8094] avg=8192.0000 max=10901 max_rank=3 min=5701 min_rank=4 imbalance_ratio=1.330688 max_load_reduction=54635 max_load_reduction_pct=0.833664 moved_to_echo=30240 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7218, 0, 3, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 113, 0, 5, 0, 0, 0, 0, 0, 0, 7612, 380, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3371, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2139, 0, 0, 0, 0, 1, 3, 0, 4766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 94, 0, 0, 8005, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7682, 0, 0, 0, 0, 0, 175, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 7582, 8125, 10472, 6927, 8196, 8174, 7883] avg=8192.0000 max=10472 max_rank=3 min=6927 min_rank=4 imbalance_ratio=1.278320 max_load_reduction=55064 max_load_reduction_pct=0.840210 moved_to_echo=31180 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7165, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7819, 380, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1456, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2140, 0, 0, 0, 0, 0, 0, 0, 7758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 25, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 175, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 7534, 8213, 6954, 9899, 8187, 8179, 8394] avg=8192.0000 max=9899 max_rank=4 min=6954 min_rank=3 imbalance_ratio=1.208374 max_load_reduction=55637 max_load_reduction_pct=0.848953 moved_to_echo=34659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 1, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 2, 0, 0, 21, 0, 0, 18, 27, 355, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 6, 1, 0, 1, 0, 7, 0, 0, 3, 0, 0, 0, 7653, 384, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 736, 1, 0, 0, 0, 0, 0, 0, 0, 15, 0, 5485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2122, 3, 2, 13, 0, 1, 10, 3, 7927, 0, 18, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 20, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7577, 0, 0, 0, 29, 0, 180, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8596, 8140, 6239, 10101, 8218, 8155, 7819] avg=8192.0000 max=10101 max_rank=4 min=6239 min_rank=3 imbalance_ratio=1.233032 max_load_reduction=55435 max_load_reduction_pct=0.845871 moved_to_echo=34293 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 51, 3, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7846, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 380, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2962, 4688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2141, 0, 0, 0, 0, 0, 0, 16, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7979, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 0, 175, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8211, 8234, 13145, 3103, 8189, 8204, 8227] avg=8192.0000 max=13145 max_rank=3 min=3103 min_rank=4 imbalance_ratio=1.604614 max_load_reduction=52391 max_load_reduction_pct=0.799423 moved_to_echo=27723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7818, 381, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 16, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2141, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 175, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8514, 8211, 5658, 10219, 8186, 8185, 8389] avg=8192.0000 max=10219 max_rank=4 min=5658 min_rank=3 imbalance_ratio=1.247437 max_load_reduction=55317 max_load_reduction_pct=0.844070 moved_to_echo=34979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 9600, 12288, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 43904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 17408, 128, 128, 128, 0, 128, 128, 128, 48128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 256, 128, 128, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 62208, 2944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[62720, 128, 128, 0, 128, 0, 128, 128, 128, 2816, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 9728, 12032, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 44032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 17152, 128, 128, 128, 0, 128, 128, 128, 48384, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 256, 128, 128, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 62080, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 63872, 128, 128, 128, 128, 128, 1408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 256, 128, 0, 65024, 0, 128, 384, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[62592, 128, 128, 0, 128, 0, 0, 128, 128, 2944, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 63872, 128, 0, 128, 128, 128, 1408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 256, 128, 128, 65024, 0, 128, 384, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 256, 0, 128, 128, 128, 0, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 8190, 5513, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 2, 0, 8117, 0, 0, 0, 0, 8180, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 0, 0, 0, 23, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 4, 1, 0, 0, 821, 0, 1, 0, 0, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8192, 0, 26, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 104, 0, 0, 1, 0, 8022, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1215, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8183, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 404, 0, 0, 8, 0, 6404, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 134, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 0, 2, 1, 0, 8097, 0, 2, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 113, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4408, 1, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3645, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 2021, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 150, 0, 0, 0, 0, 0, 0, 7780, 0, 0, 0, 0, 7964, 0, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 677, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 975, 0, 0, 7, 0, 57, 0, 0, 21, 0, 0, 0, 0, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8156, 45, 0, 0, 6, 21, 0, 0, 8166, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 6, 0, 0, 0, 8160, 0, 15, 14, 0, 8172, 0, 4910, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 20, 22, 0, 0, 0, 0, 0, 8155, 0, 0, 2, 4, 1, 0, 0, 0, 0, 0, 1, 0, 0, 4, 8152, 0, 0, 0, 12, 20, 1455, 0, 24, 0, 0, 271, 0, 0, 0, 7, 0, 0, 0, 307, 0, 0, 0, 0, 15, 0, 0, 11, 0, 0, 0, 0, 2, 0, 0, 0, 1178, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6794, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8188, 0, 23, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 885, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 416, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 8183, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 194, 0, 0, 1, 0, 6531, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7106, 0, 26, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7684, 0, 0, 0, 0, 0, 104, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8022, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2008, 1086, 707, 507, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 7420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 8192, 8240, 7159, 8001, 7789, 12523, 7456] avg=8192.0000 max=12523 max_rank=6 min=6176 min_rank=0 imbalance_ratio=1.528687 max_load_reduction=53013 max_load_reduction_pct=0.808914 moved_to_echo=35769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 6179, 5513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 2, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7096, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7998, 0, 0, 0, 0, 23, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7684, 0, 4, 1, 0, 0, 821, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2011, 1084, 707, 507, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11704, 8192, 8173, 7558, 8024, 8511, 5666, 7708] avg=8192.0000 max=11704 max_rank=0 min=5666 min_rank=6 imbalance_ratio=1.428711 max_load_reduction=53832 max_load_reduction_pct=0.821411 moved_to_echo=29909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7098, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 404, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6404, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2011, 1085, 708, 507, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1, 0, 0, 0, 0, 7430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7396, 8192, 8244, 7103, 8069, 8097, 10918, 7517] avg=8192.0000 max=10918 max_rank=6 min=7103 min_rank=3 imbalance_ratio=1.332764 max_load_reduction=54618 max_load_reduction_pct=0.833405 moved_to_echo=34601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6159, 2021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 150, 0, 0, 0, 0, 0, 0, 7780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6908, 0, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 0, 0, 0, 0, 677, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7664, 0, 0, 0, 0, 0, 975, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 21, 0, 0, 0, 0, 1427, 0, 0, 0, 0, 0, 0, 0, 2004, 1056, 708, 506, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 7430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8186, 7984, 10440, 8686, 8646, 5971, 7443] avg=8192.0000 max=10440 max_rank=3 min=5971 min_rank=6 imbalance_ratio=1.274414 max_load_reduction=55096 max_load_reduction_pct=0.840698 moved_to_echo=30746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7024, 0, 2, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 7684, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4408, 1, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2003, 1073, 708, 507, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3645, 0, 0, 0, 0, 0, 0, 7428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6290, 8194, 8243, 7032, 8112, 7696, 8896, 11073] avg=8192.0000 max=11073 max_rank=7 min=6290 min_rank=0 imbalance_ratio=1.351685 max_load_reduction=54463 max_load_reduction_pct=0.831039 moved_to_echo=35777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 6148, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 21, 0, 0, 8162, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 6, 0, 0, 0, 8160, 0, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7084, 0, 4910, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 7959, 0, 0, 2, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 7641, 0, 0, 0, 12, 20, 1455, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 7, 0, 0, 0, 307, 0, 0, 0, 0, 15, 0, 0, 2008, 1088, 710, 511, 196, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 2, 0, 0, 0, 1178, 0, 0, 0, 0, 0, 1, 7411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6218, 8196, 8239, 12018, 7988, 9157, 5117, 8603] avg=8192.0000 max=12018 max_rank=3 min=5117 min_rank=6 imbalance_ratio=1.467041 max_load_reduction=53518 max_load_reduction_pct=0.816620 moved_to_echo=30865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 6794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7103, 0, 23, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 0, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 885, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2011, 1085, 708, 507, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12975, 8192, 8246, 7127, 8406, 7762, 5390, 7438] avg=8192.0000 max=12975 max_rank=0 min=5390 min_rank=6 imbalance_ratio=1.583862 max_load_reduction=52561 max_load_reduction_pct=0.802017 moved_to_echo=28996 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6180, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7098, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7683, 0, 0, 0, 0, 0, 194, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6531, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2011, 1085, 707, 507, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 0, 0, 0, 0, 0, 0, 7425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6597, 8192, 8167, 7099, 8250, 7878, 11055, 8298] avg=8192.0000 max=11055 max_rank=6 min=6597 min_rank=0 imbalance_ratio=1.349487 max_load_reduction=54481 max_load_reduction_pct=0.831314 moved_to_echo=35481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 8190, 5579, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 2, 0, 8112, 0, 0, 0, 0, 8179, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 0, 0, 0, 22, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 4, 1, 0, 0, 821, 0, 1, 0, 0, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1204, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8183, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 486, 0, 0, 9, 0, 6347, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 3, 0, 0, 1, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8192, 0, 39, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 110, 0, 0, 2, 0, 8008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 137, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 0, 1, 1, 0, 8098, 0, 19, 0, 2, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 89, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 4536, 1, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3516, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 492, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 8182, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 205, 0, 0, 1, 0, 6523, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6701, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8187, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8163, 1974, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 165, 0, 0, 0, 0, 0, 0, 7793, 0, 0, 0, 0, 7961, 0, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 626, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 894, 1, 0, 7, 0, 43, 0, 0, 18, 0, 0, 0, 0, 1312, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8156, 37, 0, 0, 6, 20, 0, 0, 8166, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 6, 0, 0, 0, 8160, 0, 14, 16, 0, 8170, 0, 5184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 20, 21, 0, 0, 0, 0, 0, 8154, 0, 0, 2, 5, 1, 0, 0, 0, 0, 0, 1, 0, 0, 5, 8152, 0, 0, 0, 12, 21, 1385, 0, 24, 0, 0, 209, 0, 0, 0, 9, 0, 0, 0, 277, 0, 0, 0, 0, 18, 0, 0, 10, 0, 0, 0, 0, 2, 0, 0, 0, 1072, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 6180, 5579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 2, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7009, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8009, 0, 0, 0, 0, 22, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7692, 0, 4, 1, 0, 0, 821, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 1170, 663, 499, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 7470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11769, 8192, 8166, 7549, 8034, 8519, 5594, 7713] avg=8192.0000 max=11769 max_rank=0 min=5594 min_rank=6 imbalance_ratio=1.436646 max_load_reduction=53767 max_load_reduction_pct=0.820419 moved_to_echo=29860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6182, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7012, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7692, 0, 0, 0, 0, 0, 486, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6347, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 1171, 664, 500, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 1, 0, 0, 0, 0, 7477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7386, 8192, 8241, 7018, 8069, 8187, 10881, 7562] avg=8192.0000 max=10881 max_rank=6 min=7018 min_rank=3 imbalance_ratio=1.328247 max_load_reduction=54655 max_load_reduction_pct=0.833969 moved_to_echo=34699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6159, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6939, 0, 19, 0, 2, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 7692, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4536, 1, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2003, 1159, 664, 499, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3516, 0, 0, 0, 0, 0, 0, 7475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6296, 8193, 8239, 6965, 8099, 7706, 9047, 10991] avg=8192.0000 max=10991 max_rank=7 min=6296 min_rank=0 imbalance_ratio=1.341675 max_load_reduction=54545 max_load_reduction_pct=0.832291 moved_to_echo=35843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7011, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7691, 0, 0, 0, 0, 0, 205, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6523, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 1171, 663, 499, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 0, 0, 0, 0, 0, 0, 7471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6674, 8192, 8163, 7013, 8264, 7897, 11069, 8264] avg=8192.0000 max=11069 max_rank=6 min=6674 min_rank=0 imbalance_ratio=1.351196 max_load_reduction=54467 max_load_reduction_pct=0.831100 moved_to_echo=35494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 6160, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 165, 0, 0, 0, 0, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6822, 0, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 0, 0, 0, 0, 626, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7672, 0, 0, 0, 0, 0, 894, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 18, 0, 0, 0, 0, 1312, 0, 0, 0, 0, 0, 0, 0, 2003, 1139, 664, 498, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8186, 8009, 10643, 8645, 8574, 5858, 7485] avg=8192.0000 max=10643 max_rank=3 min=5858 min_rank=6 imbalance_ratio=1.299194 max_load_reduction=54893 max_load_reduction_pct=0.837601 moved_to_echo=30562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 6152, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 20, 0, 0, 8165, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 6, 0, 0, 0, 8160, 0, 14, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6997, 0, 5184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 7971, 0, 0, 2, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 7650, 0, 0, 0, 12, 21, 1385, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 9, 0, 0, 0, 277, 0, 0, 0, 0, 18, 0, 0, 2004, 1173, 666, 502, 183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 2, 0, 0, 0, 1072, 0, 0, 0, 0, 0, 1, 7456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 8197, 8239, 12205, 8000, 9098, 5042, 8541] avg=8192.0000 max=12205 max_rank=3 min=5042 min_rank=6 imbalance_ratio=1.489868 max_load_reduction=53331 max_load_reduction_pct=0.813766 moved_to_echo=30681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 3, 0, 0, 1, 0, 6172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7020, 0, 39, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7692, 0, 0, 0, 0, 0, 110, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2007, 1172, 663, 499, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 1, 0, 0, 0, 7467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 8192, 8236, 7086, 8013, 7804, 12530, 7497] avg=8192.0000 max=12530 max_rank=6 min=6178 min_rank=0 imbalance_ratio=1.529541 max_load_reduction=53006 max_load_reduction_pct=0.808807 moved_to_echo=35844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6182, 6701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7015, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7692, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 1172, 664, 500, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12883, 8192, 8243, 7057, 8412, 7751, 5515, 7483] avg=8192.0000 max=12883 max_rank=0 min=5515 min_rank=6 imbalance_ratio=1.572632 max_load_reduction=52653 max_load_reduction_pct=0.803421 moved_to_echo=29161 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[27776, 128, 128, 128, 128, 0, 0, 0, 1792, 0, 0, 0, 0, 128, 0, 0, 16128, 8704, 5760, 4096, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[27776, 128, 128, 128, 128, 0, 0, 0, 1664, 0, 0, 0, 0, 128, 0, 0, 16128, 9344, 5376, 4096, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 49408, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 128, 128, 128, 128, 128, 0, 49408, 16256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 56576, 0, 8960, 0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 0, 128, 0, 0, 0, 6144, 0, 128, 128, 0, 0, 128, 59520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 128, 61440, 0, 128, 128, 128, 128, 4096, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 128, 0, 64000, 0, 0, 128, 128, 1664, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 512, 0, 256, 0, 0, 128, 0, 128, 0, 65024, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 55936, 0, 9728, 0, 128, 0, 0, 128, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 0, 128, 0, 0, 0, 5760, 0, 128, 128, 0, 0, 128, 59776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 0, 128, 61568, 0, 128, 128, 128, 128, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 128, 0, 64128, 0, 0, 128, 128, 1536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 512, 0, 256, 0, 0, 128, 0, 128, 0, 65024, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8167, 4, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 11, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4712, 8190, 0, 8155, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 25, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 29, 0, 8191, 0, 8182, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 153, 8190, 0, 8191, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8117, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 8183, 0, 0, 0, 0, 16, 0, 0, 0, 0, 3, 0, 0, 0, 10, 0, 0, 7, 0, 8190, 0, 7394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 112, 78, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1334, 8191, 0, 7708, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 7525, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7954, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 6043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 8192, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 0, 8181, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5285, 8192, 0, 8028, 0, 0, 0, 55, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 44, 6336, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 40, 3, 8138, 0, 8164, 0, 14, 0, 1, 25, 0, 0, 0, 0, 8, 0, 4, 0, 0, 0, 0, 0, 861, 0, 0, 0, 0, 0, 0, 41, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 99, 8124, 0, 8171, 0, 0, 0, 0, 8137, 10, 0, 26, 0, 0, 0, 0, 0, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 298, 0, 7888, 0, 0, 0, 3, 16, 0, 0, 0, 0, 0, 0, 19, 8163, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 381, 846, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7490, 8191, 0, 7165, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 7671, 0, 0, 431, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 22, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8192, 0, 7223, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 11, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 955, 8192, 0, 8176, 0, 1, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 29, 0, 2757, 0, 8182, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 5434, 2165, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 153, 5619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8117, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 10970, 7660, 5790, 8398, 7940, 8245, 8225] avg=8192.0000 max=10970 max_rank=1 min=5790 min_rank=3 imbalance_ratio=1.339111 max_load_reduction=54566 max_load_reduction_pct=0.832611 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 2748, 4, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 11, 7, 0, 0, 0, 5419, 2165, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4712, 5619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 207, 0, 1, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 7933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 38, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 25, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8426, 6183, 7655, 10332, 8374, 7955, 8352, 8259] avg=8192.0000 max=10332 max_rank=3 min=6183 min_rank=1 imbalance_ratio=1.261230 max_load_reduction=55204 max_load_reduction_pct=0.842346 moved_to_echo=32940 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 0, 8181, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 47, 0, 0, 0, 5435, 2165, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5285, 5620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 55, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 7937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 1, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8434, 5662, 7793, 10905, 8290, 7954, 8314, 8184] avg=8192.0000 max=10905 max_rank=3 min=5662 min_rank=1 imbalance_ratio=1.331177 max_load_reduction=54631 max_load_reduction_pct=0.833603 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 44, 6336, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 40, 3, 2734, 0, 8164, 0, 14, 0, 1, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4, 0, 0, 0, 0, 0, 861, 0, 0, 0, 5404, 2152, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 99, 5558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 199, 10, 0, 26, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 843, 0, 0, 0, 0, 0, 0, 7890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 298, 0, 7888, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 16, 0, 0, 0, 0, 0, 0, 19, 8163, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6625, 11005, 8477, 5707, 8408, 8748, 8322, 8244] avg=8192.0000 max=11005 max_rank=1 min=5707 min_rank=3 imbalance_ratio=1.343384 max_load_reduction=54531 max_load_reduction_pct=0.832077 moved_to_echo=33722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 8183, 0, 0, 0, 0, 16, 0, 0, 0, 0, 3, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 7, 0, 2756, 0, 7394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 112, 78, 0, 0, 0, 5434, 2165, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1334, 5620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7708, 0, 0, 0, 0, 207, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 7525, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7954, 1, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8445, 10167, 7838, 6955, 7916, 8563, 7653, 7999] avg=8192.0000 max=10167 max_rank=1 min=6955 min_rank=3 imbalance_ratio=1.241089 max_load_reduction=55369 max_load_reduction_pct=0.844864 moved_to_echo=32131 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2757, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 381, 846, 0, 0, 0, 5435, 2165, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7490, 5620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7165, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 7671, 0, 0, 431, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8433, 2760, 9150, 13117, 7372, 8498, 8052, 8154] avg=8192.0000 max=13117 max_rank=3 min=2760 min_rank=1 imbalance_ratio=1.601196 max_load_reduction=52419 max_load_reduction_pct=0.799850 moved_to_echo=32076 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 6043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1619, 0, 0, 0, 5434, 2165, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 5620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8446, 8800, 9271, 6155, 8395, 7939, 8295, 8235] avg=8192.0000 max=9271 max_rank=2 min=6155 min_rank=3 imbalance_ratio=1.131714 max_load_reduction=56265 max_load_reduction_pct=0.858536 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 22, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2757, 0, 7223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 11, 32, 0, 0, 0, 5435, 2165, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 955, 5620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 1, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 0, 0, 7933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8180, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 2, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8419, 9989, 7692, 6575, 8383, 7939, 8303, 8236] avg=8192.0000 max=9989 max_rank=1 min=6575 min_rank=3 imbalance_ratio=1.219360 max_load_reduction=55547 max_load_reduction_pct=0.847580 moved_to_echo=32861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 32, 0, 8191, 0, 8183, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 160, 8190, 0, 8191, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8109, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 8182, 0, 0, 0, 0, 14, 0, 0, 0, 0, 3, 0, 0, 0, 10, 0, 0, 7, 0, 8190, 0, 7418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 107, 83, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 8191, 0, 7711, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 7530, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7965, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 20, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8192, 0, 7090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1075, 8192, 0, 8176, 0, 1, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 8181, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5473, 8192, 0, 8032, 0, 0, 0, 52, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8168, 4, 3409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 12, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4734, 8190, 0, 8153, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 24, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 45, 6429, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 41, 4, 8139, 0, 8161, 0, 13, 0, 1, 26, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 0, 804, 0, 0, 0, 0, 0, 0, 42, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 91, 8123, 0, 8171, 0, 0, 0, 0, 8137, 13, 1, 26, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 0, 0, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 297, 0, 7887, 0, 0, 0, 4, 16, 0, 0, 0, 0, 0, 0, 19, 8164, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 5982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 8192, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 435, 754, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7556, 8192, 0, 7175, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7929, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 7671, 0, 0, 422, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 8182, 0, 0, 0, 0, 14, 0, 0, 0, 0, 3, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 7, 0, 2804, 0, 7418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 107, 83, 0, 0, 0, 5386, 2215, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 5582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7711, 0, 0, 0, 0, 205, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 0, 0, 0, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 7530, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7965, 1, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8431, 10239, 7829, 6905, 7918, 8546, 7659, 8009] avg=8192.0000 max=10239 max_rank=1 min=6905 min_rank=3 imbalance_ratio=1.249878 max_load_reduction=55297 max_load_reduction_pct=0.843765 moved_to_echo=32132 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 32, 0, 2804, 0, 8183, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 5387, 2215, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 160, 5581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8109, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 11021, 7652, 5759, 8396, 7953, 8237, 8224] avg=8192.0000 max=11021 max_rank=1 min=5759 min_rank=3 imbalance_ratio=1.345337 max_load_reduction=54515 max_load_reduction_pct=0.831833 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 8181, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2805, 0, 2704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 62, 0, 0, 0, 5387, 2216, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5473, 5582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 52, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 1, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8424, 5509, 7791, 11055, 8289, 7968, 8314, 8186] avg=8192.0000 max=11055 max_rank=3 min=5509 min_rank=1 imbalance_ratio=1.349487 max_load_reduction=54481 max_load_reduction_pct=0.831314 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 20, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 2805, 0, 7090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8, 38, 0, 0, 0, 5387, 2216, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1075, 5582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 1, 0, 0, 205, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8183, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 3, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8408, 9906, 7688, 6658, 8383, 7952, 8306, 8235] avg=8192.0000 max=9906 max_rank=1 min=6658 min_rank=3 imbalance_ratio=1.209229 max_load_reduction=55630 max_load_reduction_pct=0.848846 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 2796, 4, 3409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 12, 7, 0, 0, 0, 5372, 2215, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4734, 5581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 205, 0, 1, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 36, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 24, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8416, 6215, 7648, 10316, 8372, 7963, 8350, 8256] avg=8192.0000 max=10316 max_rank=3 min=6215 min_rank=1 imbalance_ratio=1.259277 max_load_reduction=55220 max_load_reduction_pct=0.842590 moved_to_echo=32941 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 45, 6429, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 41, 4, 2782, 0, 8161, 0, 13, 0, 1, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 0, 804, 0, 0, 0, 5357, 2199, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 91, 5519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 193, 13, 1, 26, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 816, 0, 0, 0, 0, 0, 0, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 297, 0, 7887, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 16, 0, 0, 0, 0, 0, 0, 19, 8164, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6706, 11052, 8414, 5658, 8405, 8730, 8322, 8249] avg=8192.0000 max=11052 max_rank=1 min=5658 min_rank=3 imbalance_ratio=1.349121 max_load_reduction=54484 max_load_reduction_pct=0.831360 moved_to_echo=33706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2805, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 435, 754, 0, 0, 0, 5387, 2216, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7556, 5582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7175, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7929, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 7671, 0, 0, 422, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8422, 2808, 9116, 13144, 7380, 8472, 8051, 8143] avg=8192.0000 max=13144 max_rank=3 min=2808 min_rank=1 imbalance_ratio=1.604492 max_load_reduction=52392 max_load_reduction_pct=0.799438 moved_to_echo=32046 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2804, 0, 5982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1753, 0, 0, 0, 5387, 2216, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 5582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8435, 8786, 9398, 6041, 8393, 7952, 8297, 8234] avg=8192.0000 max=9398 max_rank=2 min=6041 min_rank=3 imbalance_ratio=1.147217 max_load_reduction=56138 max_load_reduction_pct=0.856598 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 384, 128, 0, 128, 0, 0, 128, 128, 640, 3584, 0, 0, 0, 43520, 17408, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 22144, 128, 43392, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 20608, 44928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 384, 128, 0, 128, 0, 0, 128, 128, 768, 3584, 0, 0, 0, 43136, 17792, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 22528, 128, 43008, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 0, 0, 128, 128, 0, 0, 0, 20992, 44672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 63616, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 63616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 2176, 128, 0, 0, 0, 0, 0, 63488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 63488, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 63872, 0, 128, 0, 128, 1664, 128, 128, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 128, 384, 128, 64256, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 63872, 0, 128, 0, 128, 1664, 128, 128, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 0, 128, 0, 128, 64768, 128, 0, 512, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2, 0, 0, 0, 8186, 0, 0, 41, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 7740, 0, 0, 0, 5, 0, 0, 0, 0, 2, 13, 0, 0, 48, 0, 0, 8008, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 0, 8177, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8185, 0, 1, 0, 8190, 0, 1, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 1, 0, 0, 1, 37, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 7930, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1, 0, 7988, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 8122, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 29, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8142, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8191, 1, 0, 7, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 40, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1480, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4, 0, 115, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 27, 11, 0, 0, 0, 0, 0, 341, 0, 0, 4, 88, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 33, 0, 5, 0, 0, 0, 0, 3, 27, 0, 0, 0, 0, 0, 0, 6553, 0, 0, 0, 1, 0, 0, 0, 23, 0, 0, 0, 0, 1, 0, 0, 7982, 7956, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 504, 0, 8088, 0, 0, 141, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 17, 8056, 0, 0, 2, 8145, 0, 42, 0, 0, 0, 8143, 29, 0, 0, 0, 0, 0, 0, 23, 34, 972, 116, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6012, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2649, 0, 0, 7675, 8192, 0, 0, 14, 0, 0, 0, 0, 1, 0, 0, 0, 0, 561, 0, 7148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 12, 0, 0, 8036, 12, 0, 0, 3, 0, 0, 0, 0, 0, 554, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 0, 8156, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8164, 13, 0, 0, 0, 1, 0, 0, 38, 0, 553, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8186, 0, 0, 41, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 7740, 0, 0, 0, 5, 0, 0, 0, 327, 280, 269, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 0, 48, 0, 0, 7728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 0, 7908, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8185, 0, 1, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7855, 0, 0, 0, 0, 0, 1, 0, 0, 1, 37, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8243, 8191, 8652, 7791, 8412, 8196, 7902] avg=8192.0000 max=8652 max_rank=3 min=7791 min_rank=4 imbalance_ratio=1.056152 max_load_reduction=56884 max_load_reduction_pct=0.867981 moved_to_echo=32301 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 0, 0, 0, 0, 0, 0, 0, 326, 278, 269, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 7652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8192, 8194, 8894, 8087, 7924, 8199, 7889] avg=8192.0000 max=8894 max_rank=3 min=7889 min_rank=7 imbalance_ratio=1.085693 max_load_reduction=56642 max_load_reduction_pct=0.864288 moved_to_echo=32099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 29, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8142, 0, 0, 3, 0, 0, 1, 0, 327, 283, 269, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7864, 2, 0, 0, 0, 0, 0, 0, 0, 0, 40, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8221, 8191, 9054, 7800, 8000, 8200, 7919] avg=8192.0000 max=9054 max_rank=3 min=7800 min_rank=4 imbalance_ratio=1.105225 max_load_reduction=56482 max_load_reduction_pct=0.861847 moved_to_echo=31919 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8012, 0, 0, 0, 0, 0, 0, 0, 327, 280, 267, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1, 0, 7708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 7855, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8192, 8193, 8918, 7933, 7892, 8198, 8054] avg=8192.0000 max=8918 max_rank=3 min=7892 min_rank=5 imbalance_ratio=1.088623 max_load_reduction=56618 max_load_reduction_pct=0.863922 moved_to_echo=32077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6012, 0, 0, 17, 0, 0, 0, 0, 321, 269, 235, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2649, 0, 0, 7406, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 561, 0, 6913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 7715, 12, 0, 0, 3, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8200, 8192, 6880, 10069, 7475, 8214, 8296] avg=8192.0000 max=10069 max_rank=4 min=6880 min_rank=3 imbalance_ratio=1.229126 max_load_reduction=55467 max_load_reduction_pct=0.846359 moved_to_echo=34054 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 11, 0, 0, 0, 0, 0, 341, 0, 0, 4, 88, 0, 0, 0, 7924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 33, 0, 5, 0, 0, 0, 0, 8140, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 27, 0, 0, 0, 0, 0, 0, 6553, 0, 0, 0, 1, 0, 0, 0, 331, 282, 269, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 1, 0, 0, 7700, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 0, 7812, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 17, 8056, 0, 0, 2, 5, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7812, 29, 0, 0, 0, 0, 0, 0, 23, 34, 972, 116, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8395, 8102, 8189, 7491, 7725, 8457, 8130, 9047] avg=8192.0000 max=9047 max_rank=7 min=7491 min_rank=3 imbalance_ratio=1.104370 max_load_reduction=56489 max_load_reduction_pct=0.861954 moved_to_echo=33359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6594, 0, 0, 0, 0, 0, 0, 0, 328, 287, 269, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1480, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7864, 0, 0, 0, 0, 0, 0, 0, 4, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8192, 8192, 7504, 7904, 9403, 8199, 7983] avg=8192.0000 max=9403 max_rank=5 min=7504 min_rank=3 imbalance_ratio=1.147827 max_load_reduction=56133 max_load_reduction_pct=0.856522 moved_to_echo=33489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7235, 0, 0, 0, 0, 0, 0, 0, 326, 286, 269, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 0, 7870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7838, 13, 0, 0, 0, 1, 0, 0, 38, 0, 553, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8194, 8194, 8143, 8227, 7973, 8200, 8446] avg=8192.0000 max=8446 max_rank=7 min=7973 min_rank=5 imbalance_ratio=1.031006 max_load_reduction=57090 max_load_reduction_pct=0.871124 moved_to_echo=32846 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2, 0, 0, 0, 8186, 0, 0, 46, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7736, 0, 0, 0, 5, 0, 0, 0, 0, 2, 13, 0, 0, 44, 0, 0, 8022, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 0, 8178, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8184, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 1, 0, 0, 3, 35, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 8155, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8163, 13, 0, 3, 0, 3, 0, 0, 44, 0, 569, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 27, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8144, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8084, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8191, 2, 0, 9, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 1, 44, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 7910, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 1, 0, 7977, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 8124, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6033, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2563, 0, 0, 7669, 8192, 0, 0, 15, 0, 0, 0, 0, 1, 0, 0, 0, 0, 553, 0, 7149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 12, 0, 0, 8021, 13, 0, 0, 3, 0, 0, 0, 0, 0, 648, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 28, 12, 0, 0, 0, 0, 0, 339, 0, 0, 3, 87, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 33, 0, 6, 0, 0, 0, 0, 3, 27, 0, 0, 0, 0, 0, 0, 6586, 0, 0, 0, 1, 0, 0, 0, 23, 0, 0, 0, 0, 0, 2, 0, 7974, 7953, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 475, 0, 8090, 0, 0, 138, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 17, 8056, 0, 0, 3, 8145, 0, 39, 0, 0, 0, 8143, 25, 0, 0, 0, 0, 0, 0, 21, 35, 991, 115, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1419, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 126, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8186, 0, 0, 46, 14, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7736, 0, 0, 0, 5, 0, 0, 0, 347, 266, 254, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 0, 44, 0, 0, 7756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 0, 7923, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8184, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7835, 0, 0, 0, 0, 0, 1, 0, 0, 3, 35, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8249, 8192, 8636, 7815, 8414, 8195, 7885] avg=8192.0000 max=8636 max_rank=3 min=7815 min_rank=4 imbalance_ratio=1.054199 max_load_reduction=56900 max_load_reduction_pct=0.868225 moved_to_echo=32309 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 27, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8144, 0, 0, 3, 0, 0, 1, 0, 347, 268, 254, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 7937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 7844, 1, 0, 0, 0, 0, 0, 0, 0, 1, 44, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8219, 8191, 9044, 7817, 8006, 8202, 7906] avg=8192.0000 max=9044 max_rank=3 min=7817 min_rank=4 imbalance_ratio=1.104004 max_load_reduction=56492 max_load_reduction_pct=0.862000 moved_to_echo=31931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7231, 0, 0, 0, 0, 0, 0, 0, 346, 270, 254, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 7932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7817, 13, 0, 3, 0, 3, 0, 0, 44, 0, 569, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8193, 8194, 8126, 8228, 7985, 8198, 8452] avg=8192.0000 max=8452 max_rank=7 min=7985 min_rank=5 imbalance_ratio=1.031738 max_load_reduction=57084 max_load_reduction_pct=0.871033 moved_to_echo=32863 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 0, 0, 0, 347, 265, 252, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 1, 0, 7712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 7871, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8193, 8192, 8899, 7942, 7909, 8199, 8044] avg=8192.0000 max=8899 max_rank=3 min=7909 min_rank=5 imbalance_ratio=1.086304 max_load_reduction=56637 max_load_reduction_pct=0.864212 moved_to_echo=32094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7996, 0, 0, 0, 0, 0, 0, 0, 346, 262, 254, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 7648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8193, 8194, 8882, 8091, 7939, 8200, 7879] avg=8192.0000 max=8882 max_rank=3 min=7879 min_rank=7 imbalance_ratio=1.084229 max_load_reduction=56654 max_load_reduction_pct=0.864471 moved_to_echo=32109 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6033, 0, 0, 18, 0, 0, 0, 0, 340, 254, 222, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2563, 0, 0, 7415, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 553, 0, 6926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 7681, 13, 0, 0, 3, 0, 0, 0, 0, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8199, 8193, 6892, 9993, 7480, 8214, 8357] avg=8192.0000 max=9993 max_rank=4 min=6892 min_rank=3 imbalance_ratio=1.219849 max_load_reduction=55543 max_load_reduction_pct=0.847519 moved_to_echo=34044 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 12, 0, 0, 0, 0, 0, 339, 0, 0, 3, 87, 0, 0, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 33, 0, 6, 0, 0, 0, 0, 8138, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 27, 0, 0, 0, 0, 0, 0, 6586, 0, 0, 0, 1, 0, 0, 0, 351, 269, 254, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 2, 0, 7705, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 7834, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 17, 8056, 0, 0, 3, 7, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7792, 25, 0, 0, 0, 0, 0, 0, 21, 35, 991, 115, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8391, 8097, 8188, 7514, 7731, 8447, 8128, 9040] avg=8192.0000 max=9040 max_rank=7 min=7514 min_rank=3 imbalance_ratio=1.103516 max_load_reduction=56496 max_load_reduction_pct=0.862061 moved_to_echo=33346 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6646, 0, 0, 0, 0, 0, 0, 0, 347, 272, 254, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1419, 0, 7937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7845, 0, 0, 0, 0, 0, 0, 0, 2, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8193, 8192, 7543, 7919, 9356, 8200, 7973] avg=8192.0000 max=9356 max_rank=5 min=7543 min_rank=3 imbalance_ratio=1.142090 max_load_reduction=56180 max_load_reduction_pct=0.857239 moved_to_echo=33448 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 0, 58368, 0, 0, 128, 128, 0, 128, 0, 2688, 2304, 2176, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 62720, 128, 0, 0, 128, 128, 128, 0, 128, 128, 2560, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 3328, 0, 62208, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 0, 3840, 128, 0, 61824, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 0, 128, 0, 0, 128, 65408, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 0, 58496, 0, 0, 128, 128, 0, 128, 0, 2816, 2176, 2048, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 62592, 128, 0, 128, 128, 128, 128, 0, 128, 128, 2688, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 0, 3712, 128, 0, 61952, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 3200, 0, 62336, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 65536, 128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 384, 0, 0, 128, 128, 0, 128, 0, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 0, 65536, 128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 128, 65408, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8190, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 37, 0, 0, 0, 0, 0, 0, 15, 10, 0, 0, 0, 0, 8155, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 29, 0, 8187, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7879, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8185, 0, 0, 0, 0, 0, 1, 8189, 523, 0, 0, 0, 1, 0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 8160, 0, 7838, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 14, 0, 8180, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 31, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 3, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8145, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8184, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[22, 0, 0, 8140, 0, 0, 0, 0, 0, 4, 7402, 866, 11, 0, 0, 0, 0, 0, 0, 0, 7812, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 1, 8101, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 13, 8057, 0, 8151, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 25, 38, 0, 0, 0, 3, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 13, 0, 262, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 88, 0, 0, 15, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 31, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 14, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 6, 0, 0, 8185, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 47, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 32, 8189, 0, 0, 0, 0, 2, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8190, 0, 7608, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 91, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 15, 10, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 93, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8072, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8322, 8200, 8328, 8200, 8081, 8112, 8187] avg=8192.0000 max=8328 max_rank=3 min=8081 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8098, 523, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 91, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7838, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 14, 0, 8064, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 31, 0, 0, 0, 1, 8091, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8623, 8078, 8220, 8018, 8197, 8089, 8131, 8180] avg=8192.0000 max=8623 max_rank=0 min=8018 min_rank=3 imbalance_ratio=1.052612 max_load_reduction=56913 max_load_reduction_pct=0.868423 moved_to_echo=32597 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 8109, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 7981, 8173, 8327, 8200, 8325, 8230, 8194] avg=8192.0000 max=8327 max_rank=3 min=7981 min_rank=1 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32949 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7879, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, 0, 0, 0, 8103, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8314, 8338, 8054, 8175, 7757, 8576, 8192] avg=8192.0000 max=8576 max_rank=6 min=7757 min_rank=5 imbalance_ratio=1.046875 max_load_reduction=56960 max_load_reduction_pct=0.869141 moved_to_echo=32700 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7318, 866, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7812, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 1, 84, 24, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 13, 8030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 25, 38, 0, 97, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 262, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 8018, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 31, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 14, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 7964, 8072, 8399, 8164, 8384, 8100, 8232] avg=8192.0000 max=8399 max_rank=3 min=7964 min_rank=1 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32880 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 32, 0, 0, 0, 0, 0, 2, 0, 8101, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7608, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8108, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8286, 8175, 7787, 8200, 8630, 8129, 8183] avg=8192.0000 max=8630 max_rank=5 min=7787 min_rank=3 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=33142 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8315, 8175, 8343, 8200, 8078, 8142, 8184] avg=8192.0000 max=8343 max_rank=3 min=8078 min_rank=5 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32604 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 6, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 47, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8276, 8183, 8280, 8200, 8192, 8116, 8184] avg=8192.0000 max=8280 max_rank=3 min=8105 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 33, 8189, 0, 0, 0, 0, 2, 0, 8191, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8190, 0, 7625, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8189, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 39, 0, 0, 0, 0, 0, 0, 15, 7, 0, 0, 0, 0, 8155, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 3, 0, 0, 0, 0, 0, 0, 7857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8148, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8185, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8185, 0, 0, 0, 0, 0, 1, 8189, 528, 0, 0, 0, 2, 0, 0, 0, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 8160, 0, 7827, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 13, 0, 8182, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 32, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 16, 0, 8188, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7900, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[22, 0, 0, 8141, 0, 0, 0, 0, 0, 4, 7408, 852, 11, 0, 0, 0, 0, 0, 0, 1, 7826, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 2, 8100, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 13, 8055, 0, 8149, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 27, 36, 0, 0, 0, 4, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 255, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 95, 0, 0, 15, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 1, 0, 0, 0, 0, 9, 0, 26, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 3, 0, 0, 8186, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 47, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 33, 0, 0, 0, 0, 0, 2, 0, 8102, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7625, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 90, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8107, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8283, 8175, 7802, 8201, 8616, 8127, 8184] avg=8192.0000 max=8616 max_rank=5 min=7802 min_rank=3 imbalance_ratio=1.051758 max_load_reduction=56920 max_load_reduction_pct=0.868530 moved_to_echo=33128 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 8108, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 7979, 8172, 8327, 8201, 8329, 8225, 8194] avg=8192.0000 max=8329 max_rank=5 min=7979 min_rank=1 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32949 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 15, 7, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 90, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8074, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8318, 8198, 8325, 8202, 8085, 8110, 8189] avg=8192.0000 max=8325 max_rank=3 min=8085 min_rank=5 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8100, 528, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 89, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7827, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 13, 0, 8068, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 32, 0, 0, 0, 1, 8089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8631, 8075, 8225, 8005, 8198, 8092, 8129, 8181] avg=8192.0000 max=8631 max_rank=0 min=8005 min_rank=3 imbalance_ratio=1.053589 max_load_reduction=56905 max_load_reduction_pct=0.868301 moved_to_echo=32600 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7900, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 8102, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8313, 8340, 8073, 8171, 7742, 8587, 8193] avg=8192.0000 max=8587 max_rank=6 min=7742 min_rank=5 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32693 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7324, 852, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7826, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 2, 84, 25, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 13, 8031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 27, 36, 0, 93, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 255, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 1, 0, 0, 0, 8016, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 26, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 11, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 7980, 8072, 8393, 8161, 8391, 8101, 8225] avg=8192.0000 max=8393 max_rank=3 min=7980 min_rank=1 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32878 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8314, 8174, 8338, 8201, 8080, 8143, 8185] avg=8192.0000 max=8338 max_rank=3 min=8080 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32609 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 3, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 47, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8274, 8180, 8273, 8201, 8201, 8114, 8185] avg=8192.0000 max=8274 max_rank=1 min=8108 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32701 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[64128, 0, 0, 128, 0, 128, 128, 0, 0, 128, 0, 0, 128, 128, 128, 0, 768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 0, 0, 0, 0, 128, 128, 64128, 1408, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 1024, 384, 128, 0, 0, 128, 128, 0, 128, 0, 128, 128, 64256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 0, 0, 128, 128, 0, 128, 0, 128, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 128, 128, 512, 0, 128, 0, 0, 0, 128, 64768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 128, 64640, 0, 128, 0, 0, 0, 0, 128, 128, 0, 128, 128, 768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 256, 128, 0, 128, 128, 128, 0, 128, 0, 128, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 64640, 0, 128, 0, 128, 0, 0, 128, 128, 0, 128, 128, 768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 896, 384, 128, 0, 0, 128, 128, 0, 128, 0, 128, 128, 64256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 0, 0, 128, 128, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 0, 0, 0, 0, 128, 128, 64128, 1408, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 0, 0, 0, 128, 0, 128, 128, 512, 0, 128, 128, 0, 0, 128, 64768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7226, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 23, 0, 0, 0, 0, 0, 0, 7908, 0, 392, 1, 0, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 1, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7197, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 8190, 0, 0, 0, 0, 0, 1, 0, 8041, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 10, 0, 1, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7765, 2, 390, 0, 0, 0, 3, 0, 0, 0, 0, 1428, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8184] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7464, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7702, 0, 522, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[22, 0, 0, 2151, 25, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 9, 0, 0, 0, 0, 14, 10, 0, 0, 2, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 2, 0, 23, 7567, 0, 0, 50, 0, 0, 0, 0, 7882, 0, 0, 1, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 1930, 0, 26, 0, 8145, 36, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 19, 7413, 27, 12, 0, 0, 57, 0, 0, 37, 10, 23, 5316, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 8106] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6659, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6755, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 2626, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 2911, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6502, 0, 844, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6875, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 1, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 5, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7315, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6839, 260, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6, 0, 1, 0, 0, 0, 0, 7881, 0, 602, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 6, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6359, 0, 4, 213, 0, 0, 2, 0, 0, 0, 0, 2361, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7226, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 710, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7337, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 23, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7908, 0, 392, 1, 0, 0, 0, 0, 136, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5895, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9698, 7412, 8235, 8440, 8204, 7916, 7427, 8204] avg=8192.0000 max=9698 max_rank=0 min=7412 min_rank=1 imbalance_ratio=1.183838 max_load_reduction=55838 max_load_reduction_pct=0.852020 moved_to_echo=31751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1404, 698, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7210, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8190, 0, 0, 0, 0, 0, 1, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 1, 7981, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6361, 2, 390, 0, 0, 0, 3, 0, 0, 0, 0, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8184, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9227, 7288, 8113, 8326, 8197, 7996, 8184, 8205] avg=8192.0000 max=9227 max_rank=0 min=7288 min_rank=1 imbalance_ratio=1.126343 max_load_reduction=56309 max_load_reduction_pct=0.859207 moved_to_echo=32582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7464, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 711, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7345, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7702, 0, 522, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6640, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9914, 7919, 8205, 8360, 8200, 7985, 6745, 8208] avg=8192.0000 max=9914 max_rank=0 min=6745 min_rank=6 imbalance_ratio=1.210205 max_load_reduction=55622 max_load_reduction_pct=0.848724 moved_to_echo=31138 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2911, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1190, 711, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7345, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6502, 0, 844, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 5, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4999, 13088, 8338, 7482, 8194, 7741, 7452, 8242] avg=8192.0000 max=13088 max_rank=1 min=4999 min_rank=0 imbalance_ratio=1.597656 max_load_reduction=52448 max_load_reduction_pct=0.800293 moved_to_echo=31629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 2151, 25, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 1344, 687, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 14, 10, 0, 0, 2, 0, 0, 39, 0, 0, 7072, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 2, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 7567, 0, 0, 50, 0, 0, 0, 0, 123, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 26, 0, 7932, 36, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 6069, 27, 12, 0, 0, 57, 0, 0, 37, 10, 23, 5316, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 8106, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4460, 7218, 8310, 7764, 8138, 9937, 11580, 8129] avg=8192.0000 max=11580 max_rank=6 min=4460 min_rank=0 imbalance_ratio=1.413574 max_load_reduction=53956 max_load_reduction_pct=0.823303 moved_to_echo=37784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6839, 260, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1150, 710, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7340, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6, 0, 1, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7881, 0, 602, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7985, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5209, 0, 4, 213, 0, 0, 2, 0, 0, 0, 0, 2361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9152, 7459, 8114, 8619, 8201, 7992, 7789, 8210] avg=8192.0000 max=9152 max_rank=0 min=7459 min_rank=1 imbalance_ratio=1.117188 max_load_reduction=56384 max_load_reduction_pct=0.860352 moved_to_echo=32192 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6659, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 711, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7345, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5533, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8798, 7736, 8112, 8304, 8200, 7985, 8192, 8209] avg=8192.0000 max=8798 max_rank=0 min=7736 min_rank=1 imbalance_ratio=1.073975 max_load_reduction=56738 max_load_reduction_pct=0.865753 moved_to_echo=32586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6875, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1323, 711, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7344, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 1, 0, 0, 0, 0, 0, 135, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 5, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5992, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9288, 7416, 8109, 8241, 8202, 7984, 8167, 8129] avg=8192.0000 max=9288 max_rank=0 min=7416 min_rank=1 imbalance_ratio=1.133789 max_load_reduction=56248 max_load_reduction_pct=0.858276 moved_to_echo=32482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6826, 268, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 1, 0, 0, 0, 0, 7879, 0, 604, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 6, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6291, 0, 4, 213, 0, 0, 2, 0, 0, 0, 0, 2421, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6929, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 5, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7234, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 2187, 0, 0, 42, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8082] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 3006, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6553, 0, 842, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2062, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7235, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 107, 18, 0, 0, 0, 0, 0, 0, 7927, 0, 393, 1, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 1, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7213, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 1, 0, 8051, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 11, 0, 1, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7732, 2, 381, 0, 0, 0, 4, 0, 0, 0, 0, 1296, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8185] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7452, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7697, 0, 548, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6775, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6685, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 2593, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[21, 0, 0, 2139, 22, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 9, 0, 0, 0, 0, 12, 7, 0, 0, 3, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 2, 0, 22, 7612, 0, 0, 50, 0, 0, 0, 0, 7849, 0, 0, 1, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 2025, 0, 25, 0, 8144, 36, 0, 0, 0, 0, 0, 13, 0, 2, 0, 0, 0, 0, 21, 7386, 28, 11, 0, 0, 59, 0, 0, 35, 10, 26, 5257, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 8106] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6826, 268, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1094, 705, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7361, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 1, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7879, 0, 604, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7976, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5197, 0, 4, 213, 0, 0, 2, 0, 0, 0, 0, 2421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9098, 7494, 8112, 8602, 8202, 7984, 7837, 8207] avg=8192.0000 max=9098 max_rank=0 min=7494 min_rank=1 imbalance_ratio=1.110596 max_load_reduction=56438 max_load_reduction_pct=0.861176 moved_to_echo=32230 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6929, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1258, 705, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7366, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 1, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 5, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5976, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 2187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9276, 7440, 8106, 8225, 8203, 7975, 8173, 8138] avg=8192.0000 max=9276 max_rank=0 min=7440 min_rank=1 imbalance_ratio=1.132324 max_load_reduction=56260 max_load_reduction_pct=0.858459 moved_to_echo=32489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7235, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 705, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7360, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 107, 18, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7927, 0, 393, 1, 0, 0, 0, 0, 119, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5959, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9673, 7436, 8233, 8443, 8205, 7901, 7443, 8202] avg=8192.0000 max=9673 max_rank=0 min=7436 min_rank=1 imbalance_ratio=1.180786 max_load_reduction=55863 max_load_reduction_pct=0.852402 moved_to_echo=31751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3006, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1146, 706, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7367, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6553, 0, 842, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 5, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5055, 12955, 8356, 7514, 8192, 7715, 7509, 8240] avg=8192.0000 max=12955 max_rank=1 min=5055 min_rank=0 imbalance_ratio=1.581421 max_load_reduction=52581 max_load_reduction_pct=0.802322 moved_to_echo=31656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1344, 693, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7241, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 1, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 11, 0, 1, 7972, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6388, 2, 381, 0, 0, 0, 4, 0, 0, 0, 0, 1296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8185, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9332, 7321, 8110, 8309, 8198, 7989, 8071, 8206] avg=8192.0000 max=9332 max_rank=0 min=7321 min_rank=1 imbalance_ratio=1.139160 max_load_reduction=56204 max_load_reduction_pct=0.857605 moved_to_echo=32464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7452, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 706, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7367, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7697, 0, 548, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6699, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9851, 7939, 8203, 8364, 8201, 7976, 6797, 8205] avg=8192.0000 max=9851 max_rank=0 min=6797 min_rank=6 imbalance_ratio=1.202515 max_load_reduction=55685 max_load_reduction_pct=0.849686 moved_to_echo=31179 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 2139, 22, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1288, 679, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 12, 7, 0, 0, 3, 0, 0, 36, 0, 0, 7064, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 2, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 7612, 0, 0, 50, 0, 0, 0, 0, 106, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 0, 25, 0, 7919, 36, 0, 0, 0, 0, 0, 13, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 6098, 28, 11, 0, 0, 59, 0, 0, 35, 10, 26, 5257, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 8106, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4389, 7209, 8305, 7791, 8136, 10020, 11556, 8130] avg=8192.0000 max=11556 max_rank=6 min=4389 min_rank=0 imbalance_ratio=1.410645 max_load_reduction=53980 max_load_reduction_pct=0.823669 moved_to_echo=37842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6775, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1162, 706, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7367, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5523, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8862, 7742, 8111, 8288, 8199, 7976, 8150, 8208] avg=8192.0000 max=8862 max_rank=0 min=7742 min_rank=1 imbalance_ratio=1.081787 max_load_reduction=56674 max_load_reduction_pct=0.864777 moved_to_echo=32533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 47104, 896, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 10496, 5760, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 6656, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 0, 0, 58368, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 47104, 128, 512, 384, 0, 128, 128, 0, 128, 128, 128, 17664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 62080, 0, 2432, 128, 0, 0, 128, 0, 1152, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[2048, 128, 128, 128, 63616, 128, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 384, 0, 0, 128, 0, 128, 384, 0, 128, 0, 128, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 47488, 896, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 9984, 5632, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 47360, 128, 512, 384, 0, 128, 128, 0, 128, 128, 128, 17408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 6400, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 0, 0, 58496, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 62208, 0, 2432, 128, 0, 0, 128, 0, 1024, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[2048, 128, 128, 128, 63488, 128, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 384, 0, 0, 128, 0, 128, 384, 0, 128, 0, 128, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 4, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 45, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8173, 0, 0, 0, 8190, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 1, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 17, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 8187, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 372, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 8183, 0, 0, 0, 8190, 0, 0, 0, 80, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 86, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 336, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 7943, 0, 0, 40, 0, 11, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 49, 8142, 0, 2, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 2, 0, 8158, 2, 0, 0, 8097, 0, 0, 0, 8109, 0, 0, 3, 0, 23, 0, 43, 0, 0, 0, 0, 0, 8, 0, 0, 7849, 0, 25, 0, 0, 0, 0, 0, 21, 0, 2, 36, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 0, 0, 0, 0, 0, 100, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8094, 0, 0, 0, 8189, 0, 0, 9, 8125, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8184, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 8191, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 4, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 8041, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 5, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8173, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8226, 8114, 8202, 8199, 8213, 8164, 8225] avg=8192.0000 max=8226 max_rank=1 min=8114 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8238, 8081, 8213, 8171, 8231, 8195, 8228] avg=8192.0000 max=8238 max_rank=1 min=8081 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 1, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 8038, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 17, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8241, 8067, 8224, 8210, 8215, 8194, 8197] avg=8192.0000 max=8241 max_rank=1 min=8067 min_rank=2 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8034, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 80, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 7925, 8444, 8223, 8132, 8132, 8274, 8228] avg=8192.0000 max=8444 max_rank=2 min=7925 min_rank=1 imbalance_ratio=1.030762 max_load_reduction=57092 max_load_reduction_pct=0.871155 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 86, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7954, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 7943, 0, 0, 40, 0, 11, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 49, 3, 0, 2, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8158, 2, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 3, 0, 23, 0, 43, 0, 0, 0, 0, 0, 8, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7849, 0, 25, 0, 0, 0, 0, 0, 21, 0, 2, 36, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8187, 8458, 8038, 8241, 8183, 8192, 7973] avg=8192.0000 max=8458 max_rank=2 min=7973 min_rank=7 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=32589 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 8040, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8094, 0, 0, 0, 39, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8232, 8217, 8209, 8196, 8143, 8131, 8229] avg=8192.0000 max=8232 max_rank=1 min=8131 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8042, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8244, 8054, 8213, 8195, 8230, 8195, 8228] avg=8192.0000 max=8244 max_rank=1 min=8054 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8042, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8243, 8101, 8214, 8192, 8189, 8191, 8228] avg=8192.0000 max=8243 max_rank=1 min=8101 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 0, 0, 99, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8092, 0, 0, 0, 8189, 0, 0, 13, 8125, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7821, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 376, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 8181, 0, 0, 0, 8187, 0, 0, 0, 81, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 4, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 51, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8171, 0, 0, 0, 8190, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 20, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 8184, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8184, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 8191, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 85, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 334, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7948, 0, 0, 42, 0, 11, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 52, 8142, 0, 2, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 2, 0, 8159, 2, 0, 0, 8097, 0, 0, 0, 8106, 0, 0, 2, 0, 25, 0, 44, 0, 0, 0, 0, 0, 8, 0, 0, 7850, 0, 21, 0, 0, 0, 0, 0, 20, 0, 3, 34, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 8037, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8092, 0, 0, 0, 37, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8238, 8213, 8207, 8196, 8143, 8132, 8229] avg=8192.0000 max=8238 max_rank=1 min=8132 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 4, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 8038, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 5, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8171, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8219, 8118, 8201, 8198, 8209, 8166, 8226] avg=8192.0000 max=8226 max_rank=7 min=8118 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8242, 8084, 8212, 8167, 8229, 8195, 8229] avg=8192.0000 max=8242 max_rank=1 min=8084 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7821, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 81, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 7906, 8448, 8223, 8140, 8141, 8272, 8229] avg=8192.0000 max=8448 max_rank=2 min=7906 min_rank=1 imbalance_ratio=1.031250 max_load_reduction=57088 max_load_reduction_pct=0.871094 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8032, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 12, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 20, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8247, 8061, 8224, 8212, 8214, 8194, 8200] avg=8192.0000 max=8247 max_rank=1 min=8061 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8248, 8103, 8213, 8190, 8186, 8190, 8229] avg=8192.0000 max=8248 max_rank=1 min=8103 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 85, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7950, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7948, 0, 0, 42, 0, 11, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 52, 0, 0, 2, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8159, 2, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 2, 0, 25, 0, 44, 0, 0, 0, 0, 0, 8, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7850, 0, 21, 0, 0, 0, 0, 0, 20, 0, 3, 34, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8186, 8455, 8044, 8241, 8186, 8192, 7965] avg=8192.0000 max=8455 max_rank=2 min=7965 min_rank=7 imbalance_ratio=1.032104 max_load_reduction=57081 max_load_reduction_pct=0.870987 moved_to_echo=32584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8250, 8054, 8212, 8192, 8228, 8195, 8229] avg=8192.0000 max=8250 max_rank=1 min=8054 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 768, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 64256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[65280, 0, 128, 128, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 64256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 65152, 0, 128, 0, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[65408, 0, 0, 128, 128, 128, 0, 128, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 65408, 128, 0, 128, 128, 128, 0, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 65280, 128, 0, 128, 0, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[65280, 0, 128, 128, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 65408, 128, 0, 128, 128, 128, 0, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 0, 65280, 128, 0, 0, 384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[65408, 0, 0, 128, 128, 128, 0, 128, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 8187, 0, 16, 0, 0, 0, 8191, 0, 0, 38, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8187, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 0, 0, 2, 0, 0, 0, 0, 0, 7524, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 6216, 0, 8192, 0, 1242, 0, 0, 0, 8192, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7899, 1932, 0, 0, 0, 0, 0, 0, 0, 6987, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 5767, 0, 8188, 0, 2044, 0, 0, 0, 8183, 1, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 3, 8190, 0, 17, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8112, 284, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 10, 0, 0, 8188, 0, 8176, 0, 0, 0, 8110, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7607, 1169, 0, 0, 0, 1, 0, 0, 1, 7020, 0, 7, 0, 0, 0, 0, 0, 0, 8192, 0, 187, 0, 0, 0, 84, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 7760, 0, 8190, 0, 505, 0, 0, 0, 8161, 0, 0, 0, 194, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 7946, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 1, 0, 0, 5, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8185, 0, 0, 0, 17, 8135, 0, 8192, 0, 50, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7637, 417, 0, 0, 0, 3, 0, 0, 0, 8188, 0, 0, 4, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 6, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4, 0, 0, 0, 25, 0, 0, 7737, 0, 27, 18, 0, 8180, 0, 7785, 0, 0, 0, 0, 0, 7858, 0, 0, 111, 1982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 165, 0, 8136, 0, 334, 0, 0, 0, 0, 0, 0, 0, 17, 0, 16, 29, 0, 0, 8002, 0, 0, 0, 0, 0, 0, 0, 0, 6659, 17, 0, 0, 0, 28, 1, 2, 0, 8065, 0, 0, 0, 15, 193, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8113, 0, 8192, 0, 106, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 29, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7798, 370, 92, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8139, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7822, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 38, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8119, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 2, 0, 0, 0, 0, 0, 7524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8241, 8243, 8664, 8239, 8145, 7625, 8097] avg=8192.0000 max=8664 max_rank=3 min=7625 min_rank=6 imbalance_ratio=1.057617 max_load_reduction=56872 max_load_reduction_pct=0.867798 moved_to_echo=32106 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7750, 350, 92, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 4, 0, 0, 1, 0, 0, 0, 8109, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7410, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8190, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 99, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 1, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8214, 8244, 8109, 8246, 8118, 8293, 8104] avg=8192.0000 max=8293 max_rank=6 min=8104 min_rank=7 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7704, 280, 92, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8140, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5936, 0, 0, 0, 1242, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 1932, 0, 0, 0, 0, 0, 0, 0, 6987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8237, 8236, 7224, 8244, 8127, 9018, 8359] avg=8192.0000 max=9018 max_rank=6 min=7224 min_rank=3 imbalance_ratio=1.100830 max_load_reduction=56518 max_load_reduction_pct=0.862396 moved_to_echo=33748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 0, 7419, 0, 92, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 95, 1169, 0, 0, 0, 1, 0, 0, 1, 7020, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 187, 0, 0, 0, 84, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7933, 8153, 8226, 8189, 8239, 8124, 8298, 8374] avg=8192.0000 max=8374 max_rank=7 min=7933 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=57162 max_load_reduction_pct=0.872223 moved_to_echo=33035 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7912, 260, 92, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5507, 0, 0, 0, 2044, 0, 0, 0, 0, 1, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 8188, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8122, 0, 17, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 284, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8230, 8240, 7717, 8246, 8143, 8572, 8102] avg=8192.0000 max=8572 max_rank=6 min=7717 min_rank=3 imbalance_ratio=1.046387 max_load_reduction=56964 max_load_reduction_pct=0.869202 moved_to_echo=33063 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 6, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7808, 374, 97, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7809, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4, 0, 0, 0, 25, 0, 0, 7737, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 18, 0, 7806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 1982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7785, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 8067, 0, 334, 0, 0, 0, 0, 0, 0, 0, 17, 0, 16, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 6659, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1, 2, 0, 7968, 0, 0, 0, 15, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8350, 7946, 7857, 9944, 7836, 8628, 6768, 8207] avg=8192.0000 max=9944 max_rank=3 min=6768 min_rank=6 imbalance_ratio=1.213867 max_load_reduction=55592 max_load_reduction_pct=0.848267 moved_to_echo=31439 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7990, 366, 92, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7747, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 29, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8463, 8241, 8245, 7853, 8243, 8129, 8255, 8107] avg=8192.0000 max=8463 max_rank=0 min=7853 min_rank=3 imbalance_ratio=1.033081 max_load_reduction=57073 max_load_reduction_pct=0.870865 moved_to_echo=32734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7449, 367, 92, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 8136, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8185, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7768, 0, 0, 0, 50, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 417, 0, 0, 0, 3, 0, 0, 0, 8188, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7923, 8274, 8245, 7836, 8243, 8122, 8707, 8186] avg=8192.0000 max=8707 max_rank=6 min=7836 min_rank=3 imbalance_ratio=1.062866 max_load_reduction=56829 max_load_reduction_pct=0.867142 moved_to_echo=33258 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 5724, 0, 8188, 0, 2101, 0, 0, 0, 8184, 1, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 4, 8190, 0, 14, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8118, 224, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 7741, 0, 8190, 0, 516, 0, 0, 0, 8161, 0, 0, 0, 179, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 7962, 7, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 8188, 0, 22, 0, 0, 0, 8191, 0, 0, 37, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 8186, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7994, 0, 0, 2, 0, 0, 0, 0, 0, 7496, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 6187, 0, 8192, 0, 1245, 0, 0, 0, 8192, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7907, 1952, 0, 0, 0, 0, 0, 0, 0, 6994, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8185, 0, 0, 0, 18, 8135, 0, 8192, 0, 32, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7642, 429, 0, 0, 0, 2, 0, 0, 0, 8188, 0, 0, 4, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8111, 0, 8192, 0, 103, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 23, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 11, 0, 0, 8188, 0, 8178, 0, 0, 0, 8103, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7591, 1243, 0, 0, 0, 1, 0, 0, 1, 6959, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 187, 0, 0, 0, 88, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 10, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 0, 0, 0, 24, 0, 0, 7752, 0, 26, 20, 0, 8179, 0, 7787, 0, 0, 0, 0, 0, 7857, 0, 0, 108, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 178, 0, 8136, 0, 307, 0, 0, 0, 0, 0, 0, 0, 21, 0, 19, 28, 0, 0, 7984, 0, 0, 0, 0, 0, 0, 0, 0, 6671, 12, 0, 0, 0, 29, 0, 3, 0, 8062, 0, 0, 0, 18, 196, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7915, 259, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5465, 0, 0, 0, 2101, 0, 0, 0, 0, 1, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 8188, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8124, 0, 14, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 224, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8231, 8235, 7778, 8245, 8143, 8514, 8100] avg=8192.0000 max=8514 max_rank=6 min=7778 min_rank=3 imbalance_ratio=1.039307 max_load_reduction=57022 max_load_reduction_pct=0.870087 moved_to_echo=33002 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7763, 350, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 4, 0, 0, 1, 0, 0, 0, 8109, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7391, 0, 0, 0, 516, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 102, 7, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8216, 8242, 8086, 8245, 8120, 8302, 8102] avg=8192.0000 max=8302 max_rank=6 min=8086 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 371, 93, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8139, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7821, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 37, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 8120, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 2, 0, 0, 0, 0, 0, 7496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8241, 8241, 8677, 8240, 8148, 7600, 8109] avg=8192.0000 max=8677 max_rank=3 min=7600 min_rank=6 imbalance_ratio=1.059204 max_load_reduction=56859 max_load_reduction_pct=0.867599 moved_to_echo=32097 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7709, 280, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8140, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5907, 0, 0, 0, 1245, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 1952, 0, 0, 0, 0, 0, 0, 0, 6994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8240, 8235, 7195, 8244, 8128, 9047, 8349] avg=8192.0000 max=9047 max_rank=6 min=7195 min_rank=3 imbalance_ratio=1.104370 max_load_reduction=56489 max_load_reduction_pct=0.861954 moved_to_echo=33768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7451, 368, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 8136, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8185, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 7767, 0, 0, 0, 32, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 429, 0, 0, 0, 2, 0, 0, 0, 8188, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7928, 8277, 8244, 7818, 8243, 8123, 8720, 8183] avg=8192.0000 max=8720 max_rank=6 min=7818 min_rank=3 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=33269 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 0, 0, 0, 0, 7401, 0, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 97, 1243, 0, 0, 0, 1, 0, 0, 1, 6959, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 187, 0, 0, 0, 88, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7924, 8146, 8223, 8191, 8238, 8126, 8312, 8376] avg=8192.0000 max=8376 max_rank=7 min=7924 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=33052 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 10, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7788, 375, 95, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7808, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 0, 0, 0, 24, 0, 0, 7752, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 20, 0, 7804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7787, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 8064, 0, 307, 0, 0, 0, 0, 0, 0, 0, 21, 0, 19, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 6671, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 3, 0, 7967, 0, 0, 0, 18, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8330, 7944, 7873, 9944, 7838, 8617, 6777, 8213] avg=8192.0000 max=9944 max_rank=3 min=6777 min_rank=6 imbalance_ratio=1.213867 max_load_reduction=55592 max_load_reduction_pct=0.848267 moved_to_echo=31445 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7987, 367, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7744, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 23, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8463, 8241, 8243, 7847, 8243, 8131, 8264, 8104] avg=8192.0000 max=8463 max_rank=0 min=7847 min_rank=3 imbalance_ratio=1.033081 max_load_reduction=57073 max_load_reduction_pct=0.870865 moved_to_echo=32742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 896, 3840, 0, 128, 0, 128, 0, 0, 128, 60928, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 50048, 0, 0, 0, 12160, 0, 0, 0, 0, 128, 0, 256, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 0, 0, 128, 0, 512, 0, 0, 0, 0, 61952, 2432, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 65152, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 64640, 128, 512, 0, 128, 384, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 49920, 0, 0, 0, 12288, 0, 0, 0, 0, 128, 0, 256, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 896, 3968, 0, 128, 0, 128, 0, 0, 128, 60928, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 0, 0, 128, 0, 512, 0, 0, 0, 0, 61824, 2432, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 64768, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 128, 0, 65152, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 65152, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 64640, 128, 512, 0, 128, 384, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 65024, 0, 384, 128, 0, 0, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True - [2026-08-26 16:18:05] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 678.1 | throughput per GPU (TFLOP/s/GPU): 266.7 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338541E+01 | loss scale: 1.0 | grad norm: 19.946 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 43, 154, 220, 1640, 0, 0, 3240, 13, 0, 3, 5, 126, 22, 1, 204, 0, 1, 757, 0, 3, 1791, 4360, 0, 0, 8189, 1903, 7840, 0, 1121, 2, 20, 268, 0, 6, 317, 0, 17, 84, 0, 659, 469, 0, 626, 0, 8174, 0, 1, 44, 0, 0, 4, 0, 1, 316, 0, 34, 83, 0, 0, 0, 9, 621, 0, 0, 0, 680, 0, 2, 4, 774, 0, 712, 775, 5, 5, 265, 0, 0, 1, 0, 3, 62, 0, 0, 859, 0, 220, 0, 4, 4, 0, 7, 0, 0, 193, 0, 0, 18, 5, 0, 0, 222, 527, 647, 3, 1, 63, 3593, 1, 0, 4930, 0, 0, 0, 13, 664, 0, 0, 0, 6512, 371, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[5, 0, 0, 25, 155, 230, 1855, 0, 0, 3097, 3, 3, 19, 2, 98, 2, 0, 150, 0, 26, 719, 2, 0, 2506, 3461, 0, 0, 8169, 1832, 7361, 6, 996, 5, 60, 235, 2, 2, 283, 0, 26, 70, 0, 466, 148, 0, 848, 9, 8173, 0, 1, 18, 0, 0, 5, 0, 0, 189, 1, 29, 287, 0, 0, 0, 26, 1222, 5, 0, 0, 716, 0, 3, 2, 813, 0, 822, 1014, 30, 4, 124, 0, 0, 0, 0, 1, 26, 0, 0, 922, 0, 262, 1, 3, 15, 2, 0, 8, 0, 89, 0, 0, 18, 4, 9, 1, 204, 969, 370, 1, 4, 96, 4343, 2, 1, 4208, 0, 3, 0, 12, 1012, 0, 1, 8, 5905, 672, 0, 3, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 0, 9, 160, 351, 1638, 0, 0, 2594, 2, 3, 5, 5, 146, 13, 0, 134, 0, 21, 631, 0, 5, 1786, 4300, 0, 3, 8163, 1654, 7825, 11, 1112, 16, 61, 100, 0, 0, 190, 0, 11, 29, 2, 1096, 341, 0, 640, 2, 8159, 2, 1, 59, 0, 4, 1, 0, 2, 102, 3, 58, 248, 0, 1, 0, 13, 1072, 0, 0, 0, 398, 1, 17, 1, 916, 1, 1159, 1224, 15, 8, 140, 0, 2, 3, 0, 0, 103, 0, 0, 885, 6, 165, 0, 5, 2, 0, 0, 5, 0, 160, 8, 3, 10, 6, 0, 0, 162, 1114, 543, 9, 1, 73, 3361, 2, 2, 4152, 0, 2, 0, 9, 1196, 0, 2, 3, 6335, 535, 0, 1, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 201, 229, 1850, 0, 8, 3753, 1, 0, 26, 2, 37, 3, 0, 127, 0, 7, 950, 0, 0, 2092, 4318, 1, 0, 8175, 1301, 7510, 2, 218, 23, 49, 143, 0, 13, 433, 0, 6, 146, 2, 1009, 150, 0, 680, 39, 8179, 0, 1, 48, 1, 0, 5, 0, 3, 90, 1, 35, 252, 0, 2, 0, 9, 782, 0, 0, 0, 566, 0, 45, 2, 951, 0, 914, 846, 3, 9, 322, 0, 1, 1, 0, 3, 39, 0, 0, 565, 2, 311, 0, 6, 16, 2, 0, 3, 0, 196, 3, 0, 14, 9, 8, 4, 40, 946, 425, 21, 0, 95, 4033, 0, 0, 4195, 2, 0, 1, 36, 1635, 0, 2, 1, 5754, 582, 0, 2, 2, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 181, 212, 1950, 0, 0, 2559, 2, 0, 7, 2, 78, 16, 0, 59, 0, 3, 1245, 0, 11, 2399, 3143, 0, 0, 8159, 1876, 7339, 3, 1412, 10, 40, 324, 0, 9, 178, 0, 3, 211, 0, 630, 109, 0, 629, 6, 8173, 0, 6, 22, 0, 0, 2, 0, 0, 238, 0, 19, 122, 0, 0, 4, 11, 835, 0, 0, 0, 564, 0, 4, 3, 971, 0, 873, 497, 24, 2, 239, 0, 0, 1, 0, 0, 40, 0, 2, 531, 0, 111, 0, 16, 4, 1, 2, 0, 0, 248, 1, 0, 13, 2, 2, 0, 123, 293, 456, 19, 0, 132, 4735, 1, 2, 5270, 0, 0, 0, 16, 728, 0, 0, 3, 7001, 354, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4, 1, 0, 6, 92, 194, 1812, 0, 0, 2101, 1, 0, 4, 2, 63, 5, 0, 183, 0, 10, 931, 0, 2, 2008, 3733, 1, 0, 8176, 1528, 7520, 1, 1194, 4, 36, 226, 0, 1, 87, 0, 4, 68, 0, 665, 185, 0, 676, 4, 8142, 0, 0, 28, 1, 0, 1, 0, 0, 200, 0, 39, 175, 1, 0, 0, 29, 608, 0, 0, 0, 570, 0, 13, 1, 605, 0, 1440, 732, 16, 16, 207, 0, 0, 0, 2, 1, 75, 0, 1, 760, 1, 283, 0, 1, 10, 0, 3, 0, 0, 305, 1, 0, 13, 2, 2, 1, 228, 813, 619, 1, 0, 117, 4343, 0, 1, 5360, 0, 2, 0, 9, 1364, 0, 0, 0, 6348, 517, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 185, 97, 1609, 0, 0, 3008, 5, 2, 21, 5, 122, 34, 1, 288, 0, 18, 801, 0, 4, 1567, 4077, 2, 1, 8172, 1307, 7413, 0, 777, 32, 31, 155, 0, 6, 307, 0, 14, 84, 1, 1382, 185, 1, 664, 32, 8170, 0, 1, 59, 0, 0, 1, 0, 1, 147, 1, 19, 371, 0, 1, 0, 18, 1022, 0, 0, 0, 577, 0, 8, 1, 784, 0, 1243, 1244, 19, 4, 152, 3, 17, 7, 2, 0, 53, 0, 3, 785, 6, 346, 0, 3, 35, 4, 12, 0, 2, 299, 4, 1, 21, 6, 1, 0, 212, 899, 685, 17, 1, 166, 3680, 0, 1, 4555, 0, 0, 0, 22, 839, 0, 0, 13, 6179, 370, 2, 18, 2, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 237, 201, 1412, 4, 7, 2025, 1, 2, 13, 0, 114, 11, 1, 120, 0, 13, 1141, 0, 0, 1942, 4047, 3, 0, 8166, 1448, 7548, 6, 791, 11, 34, 227, 0, 1, 213, 1, 17, 56, 3, 566, 136, 2, 1509, 4, 8169, 0, 1, 20, 2, 0, 6, 0, 6, 223, 1, 12, 232, 0, 2, 0, 25, 666, 0, 0, 0, 480, 0, 6, 11, 1046, 0, 780, 1299, 39, 19, 439, 0, 0, 0, 0, 0, 86, 0, 0, 1330, 0, 197, 1, 30, 1, 4, 0, 6, 1, 568, 0, 4, 12, 2, 0, 4, 133, 1138, 428, 12, 4, 165, 3506, 0, 0, 4797, 3, 0, 0, 7, 1187, 0, 2, 2, 6059, 316, 1, 0, 4, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 43, 154, 220, 1640, 0, 0, 3240, 13, 0, 3, 5, 126, 22, 1703, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 204, 0, 1, 757, 0, 3, 1791, 2825, 0, 0, 0, 1903, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 268, 0, 6, 317, 0, 17, 84, 0, 659, 469, 0, 626, 0, 5750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 44, 0, 0, 4, 0, 1, 316, 0, 34, 83, 0, 0, 0, 9, 7704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 680, 0, 2, 4, 774, 0, 712, 775, 5, 5, 265, 0, 2794, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 62, 0, 0, 859, 0, 220, 0, 4, 4, 0, 7, 0, 6803, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 18, 5, 0, 0, 222, 527, 647, 3, 1, 63, 3593, 1, 1535, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4930, 0, 0, 0, 13, 664, 0, 0, 0, 2015, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8669, 8606, 8218, 8196, 7674, 8448, 7732, 7993] avg=8192.0000 max=8669 max_rank=0 min=7674 min_rank=4 imbalance_ratio=1.058228 max_load_reduction=56867 max_load_reduction_pct=0.867722 moved_to_echo=31847 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 9, 160, 351, 1638, 0, 0, 2594, 2, 3, 5, 5, 146, 13, 1656, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 21, 631, 0, 5, 1786, 2786, 0, 3, 0, 1654, 0, 11, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 61, 100, 0, 0, 190, 0, 11, 29, 2, 1096, 341, 0, 640, 2, 5739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 59, 0, 4, 1, 0, 2, 102, 3, 58, 248, 0, 1, 0, 13, 7679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 0, 0, 0, 398, 1, 17, 1, 916, 1, 1159, 1224, 15, 8, 140, 0, 2718, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 103, 0, 0, 885, 6, 165, 0, 5, 2, 0, 0, 5, 6790, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 8, 3, 10, 6, 0, 0, 162, 1114, 543, 9, 1, 73, 3361, 2, 1514, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4152, 0, 2, 0, 9, 1196, 0, 2, 3, 1961, 535, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8143, 8227, 8173, 8705, 8450, 7889, 7867] avg=8192.0000 max=8705 max_rank=4 min=7867 min_rank=7 imbalance_ratio=1.062622 max_load_reduction=56831 max_load_reduction_pct=0.867172 moved_to_echo=32911 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 201, 229, 1850, 0, 8, 3753, 1, 0, 26, 2, 37, 3, 1504, 1501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 7, 950, 0, 0, 2092, 2798, 1, 0, 0, 1301, 0, 2, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 49, 143, 0, 13, 433, 0, 6, 146, 2, 1009, 150, 0, 680, 39, 5753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 48, 1, 0, 5, 0, 3, 90, 1, 35, 252, 0, 2, 0, 9, 7692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 0, 0, 566, 0, 45, 2, 951, 0, 914, 846, 3, 9, 322, 0, 2469, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3, 39, 0, 0, 565, 2, 311, 0, 6, 16, 2, 0, 3, 6516, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 3, 0, 14, 9, 8, 4, 40, 946, 425, 21, 0, 95, 4033, 0, 1520, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4195, 2, 0, 1, 36, 1635, 0, 2, 1, 1781, 582, 0, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9124, 7496, 8446, 8139, 7903, 7948, 8239, 8241] avg=8192.0000 max=9124 max_rank=0 min=7496 min_rank=1 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=32331 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 181, 212, 1950, 0, 0, 2559, 2, 0, 7, 2, 78, 16, 1834, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 3, 1245, 0, 11, 2399, 2031, 0, 0, 0, 1876, 0, 3, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 40, 324, 0, 9, 178, 0, 3, 211, 0, 630, 109, 0, 629, 6, 5743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 22, 0, 0, 2, 0, 0, 238, 0, 19, 122, 0, 0, 4, 11, 7677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 0, 564, 0, 4, 3, 971, 0, 873, 497, 24, 2, 239, 0, 3007, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 40, 0, 2, 531, 0, 111, 0, 16, 4, 1, 2, 0, 6367, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 1, 0, 13, 2, 2, 0, 123, 293, 456, 19, 0, 132, 4735, 1, 1112, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5270, 0, 0, 0, 16, 728, 0, 0, 3, 2160, 354, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8354, 9039, 7892, 8101, 7991, 7557, 8065, 8537] avg=8192.0000 max=9039 max_rank=1 min=7557 min_rank=5 imbalance_ratio=1.103394 max_load_reduction=56497 max_load_reduction_pct=0.862076 moved_to_echo=32150 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 25, 155, 230, 1855, 0, 0, 3097, 3, 3, 19, 2, 98, 2, 1544, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 26, 719, 2, 0, 2506, 2243, 0, 0, 0, 1832, 0, 6, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 60, 235, 2, 2, 283, 0, 26, 70, 0, 466, 148, 0, 848, 9, 5749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 18, 0, 0, 5, 0, 0, 189, 1, 29, 287, 0, 0, 0, 26, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 5, 0, 0, 716, 0, 3, 2, 813, 0, 822, 1014, 30, 4, 124, 0, 2534, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 26, 0, 0, 922, 0, 262, 1, 3, 15, 2, 0, 8, 6387, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 18, 4, 9, 1, 204, 969, 370, 1, 4, 96, 4343, 2, 1218, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4208, 0, 3, 0, 12, 1012, 0, 1, 8, 1827, 672, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8538, 8480, 7903, 8241, 8263, 8111, 8252, 7748] avg=8192.0000 max=8538 max_rank=0 min=7748 min_rank=7 imbalance_ratio=1.042236 max_load_reduction=56998 max_load_reduction_pct=0.869720 moved_to_echo=32374 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 237, 201, 1412, 4, 7, 2025, 1, 2, 13, 0, 114, 11, 1584, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 120, 0, 13, 1141, 0, 0, 1942, 2622, 3, 0, 0, 1448, 0, 6, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 34, 227, 0, 1, 213, 1, 17, 56, 3, 566, 136, 2, 1509, 4, 5746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 2, 0, 6, 0, 6, 223, 1, 12, 232, 0, 2, 0, 25, 7682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 666, 0, 0, 0, 480, 0, 6, 11, 1046, 0, 780, 1299, 39, 19, 439, 0, 2600, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 1330, 0, 197, 1, 30, 1, 4, 0, 6, 6550, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 568, 0, 4, 12, 2, 0, 4, 133, 1138, 428, 12, 4, 165, 3506, 0, 1425, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4797, 3, 0, 0, 7, 1187, 0, 2, 2, 1875, 316, 1, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7118, 8087, 8526, 8212, 8383, 8689, 8326, 8195] avg=8192.0000 max=8689 max_rank=5 min=7118 min_rank=0 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=33593 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 185, 97, 1609, 0, 0, 3008, 5, 2, 21, 5, 122, 34, 1616, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 288, 0, 18, 801, 0, 4, 1567, 2642, 2, 1, 0, 1307, 0, 0, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 31, 155, 0, 6, 307, 0, 14, 84, 1, 1382, 185, 1, 664, 32, 5747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 59, 0, 0, 1, 0, 1, 147, 1, 19, 371, 0, 1, 0, 18, 7689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 0, 0, 0, 577, 0, 8, 1, 784, 0, 1243, 1244, 19, 4, 152, 3, 2651, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 2, 0, 53, 0, 3, 785, 6, 346, 0, 3, 35, 4, 12, 0, 6432, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 299, 4, 1, 21, 6, 1, 0, 212, 899, 685, 17, 1, 166, 3680, 0, 1435, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4555, 0, 0, 0, 22, 839, 0, 0, 13, 1912, 370, 2, 18, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 7408, 8641, 8308, 8689, 8188, 8353, 7737] avg=8192.0000 max=8689 max_rank=4 min=7408 min_rank=1 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=32967 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 1, 0, 6, 92, 194, 1812, 0, 0, 2101, 1, 0, 4, 2, 63, 5, 1660, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 10, 931, 0, 2, 2008, 2419, 1, 0, 0, 1528, 0, 1, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 36, 226, 0, 1, 87, 0, 4, 68, 0, 665, 185, 0, 676, 4, 5727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1, 0, 1, 0, 0, 200, 0, 39, 175, 1, 0, 0, 29, 7692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 0, 570, 0, 13, 1, 605, 0, 1440, 732, 16, 16, 207, 0, 2724, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 75, 0, 1, 760, 1, 283, 0, 1, 10, 0, 3, 0, 6524, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 1, 0, 13, 2, 2, 1, 228, 813, 619, 1, 0, 117, 4343, 0, 1314, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5360, 0, 2, 0, 9, 1364, 0, 0, 0, 1964, 517, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7439, 8277, 7683, 8166, 7928, 8145, 8680, 9218] avg=8192.0000 max=9218 max_rank=7 min=7439 min_rank=0 imbalance_ratio=1.125244 max_load_reduction=56318 max_load_reduction_pct=0.859344 moved_to_echo=33971 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 + [2026-08-26 16:27:13] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 438.6 | throughput per GPU (TFLOP/s/GPU): 412.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338525E+01 | loss scale: 1.0 | grad norm: 19.949 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 44, 147, 217, 1624, 0, 0, 3222, 12, 0, 3, 5, 123, 22, 1, 207, 0, 0, 752, 0, 4, 1788, 4380, 0, 0, 8189, 1887, 7845, 0, 1127, 2, 19, 262, 0, 5, 323, 0, 17, 88, 0, 669, 488, 0, 635, 0, 8174, 0, 2, 43, 0, 0, 4, 0, 1, 327, 0, 35, 87, 0, 0, 0, 10, 618, 0, 0, 0, 672, 0, 1, 4, 772, 0, 696, 746, 6, 6, 263, 0, 0, 1, 0, 3, 60, 0, 0, 845, 0, 218, 0, 4, 5, 0, 7, 0, 0, 192, 0, 0, 19, 5, 0, 0, 212, 533, 645, 3, 1, 64, 3661, 1, 0, 4936, 0, 0, 0, 11, 646, 0, 0, 0, 6514, 376, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 205, 228, 1831, 0, 8, 3736, 1, 0, 27, 1, 37, 3, 0, 124, 0, 7, 944, 1, 0, 2064, 4353, 1, 0, 8175, 1294, 7499, 1, 211, 23, 47, 144, 0, 13, 449, 0, 6, 141, 2, 1027, 156, 0, 688, 39, 8178, 0, 1, 46, 1, 0, 4, 0, 3, 98, 0, 34, 255, 0, 2, 0, 10, 775, 0, 0, 0, 557, 0, 45, 2, 957, 0, 908, 832, 3, 10, 325, 0, 1, 1, 0, 4, 38, 0, 0, 556, 1, 304, 0, 6, 17, 2, 0, 3, 0, 203, 3, 0, 14, 8, 8, 4, 39, 937, 430, 21, 0, 95, 4111, 0, 0, 4197, 2, 0, 1, 36, 1614, 0, 2, 1, 5742, 589, 0, 1, 2, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[4, 0, 0, 27, 153, 230, 1837, 0, 0, 3084, 3, 3, 19, 2, 103, 2, 0, 149, 0, 27, 727, 2, 1, 2508, 3486, 0, 0, 8171, 1823, 7355, 6, 1006, 4, 57, 229, 2, 1, 290, 0, 25, 68, 0, 475, 152, 0, 856, 10, 8172, 0, 1, 19, 0, 0, 5, 0, 0, 192, 1, 28, 292, 0, 0, 0, 28, 1214, 4, 0, 0, 700, 0, 3, 2, 813, 0, 815, 973, 29, 6, 127, 0, 0, 0, 0, 1, 22, 0, 0, 922, 0, 259, 1, 4, 17, 2, 0, 7, 0, 89, 0, 0, 21, 4, 9, 2, 193, 980, 369, 1, 4, 96, 4398, 2, 1, 4216, 0, 4, 0, 13, 992, 0, 1, 8, 5900, 673, 0, 3, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 0, 12, 161, 355, 1601, 0, 0, 2574, 2, 3, 5, 3, 141, 14, 0, 135, 0, 21, 631, 0, 4, 1777, 4317, 0, 2, 8163, 1643, 7823, 11, 1113, 16, 61, 95, 0, 0, 195, 0, 12, 29, 2, 1101, 341, 0, 641, 2, 8158, 2, 2, 62, 0, 4, 1, 0, 1, 103, 3, 60, 252, 0, 1, 0, 15, 1073, 0, 1, 0, 392, 1, 17, 1, 922, 1, 1149, 1194, 16, 8, 138, 0, 2, 3, 0, 0, 104, 0, 0, 889, 6, 164, 0, 5, 2, 0, 0, 6, 0, 162, 7, 3, 11, 6, 0, 0, 160, 1110, 541, 9, 1, 74, 3429, 2, 2, 4161, 0, 2, 0, 10, 1195, 0, 2, 3, 6331, 544, 0, 1, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 233, 204, 1395, 4, 7, 1993, 1, 2, 13, 0, 111, 12, 1, 120, 0, 13, 1131, 0, 1, 1925, 4054, 3, 0, 8165, 1447, 7541, 6, 797, 10, 35, 231, 0, 2, 223, 1, 16, 54, 3, 569, 132, 2, 1529, 5, 8168, 0, 1, 21, 2, 0, 6, 0, 6, 225, 1, 14, 242, 0, 2, 0, 27, 665, 0, 0, 0, 482, 0, 6, 12, 1041, 0, 772, 1253, 39, 22, 447, 0, 0, 0, 0, 0, 86, 0, 0, 1328, 0, 196, 1, 30, 1, 5, 0, 6, 1, 571, 0, 4, 14, 2, 0, 4, 128, 1137, 429, 12, 4, 163, 3590, 0, 0, 4777, 3, 0, 0, 7, 1183, 0, 2, 2, 6061, 329, 1, 0, 5, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 13, 183, 212, 1923, 0, 0, 2546, 2, 0, 7, 2, 77, 16, 0, 62, 0, 3, 1244, 0, 12, 2414, 3161, 0, 0, 8158, 1878, 7330, 3, 1405, 11, 41, 320, 0, 9, 187, 0, 2, 214, 0, 634, 108, 0, 623, 5, 8174, 0, 6, 23, 0, 0, 2, 0, 0, 241, 0, 19, 125, 0, 0, 4, 12, 841, 0, 0, 0, 558, 0, 4, 2, 965, 0, 864, 473, 23, 2, 240, 0, 0, 1, 0, 0, 42, 0, 2, 527, 0, 109, 0, 16, 4, 1, 2, 0, 0, 251, 1, 0, 15, 2, 2, 0, 119, 294, 453, 20, 0, 139, 4781, 1, 2, 5273, 0, 0, 0, 17, 714, 0, 0, 3, 6994, 364, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 189, 97, 1584, 0, 0, 3001, 4, 2, 22, 5, 125, 31, 0, 289, 0, 16, 808, 0, 3, 1549, 4107, 2, 1, 8173, 1289, 7400, 0, 777, 33, 30, 153, 0, 6, 312, 0, 14, 84, 1, 1392, 188, 1, 667, 33, 8171, 0, 1, 64, 0, 0, 0, 0, 1, 151, 1, 19, 368, 0, 1, 0, 18, 1007, 0, 0, 0, 569, 0, 8, 1, 784, 0, 1222, 1207, 20, 4, 152, 3, 17, 8, 2, 0, 56, 0, 3, 782, 5, 344, 0, 3, 35, 4, 12, 0, 2, 298, 4, 1, 21, 6, 1, 0, 208, 908, 692, 16, 2, 169, 3740, 0, 1, 4589, 0, 0, 0, 22, 833, 0, 0, 13, 6173, 372, 2, 18, 2, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4, 1, 0, 7, 91, 195, 1793, 0, 0, 2085, 1, 0, 5, 2, 61, 5, 0, 181, 0, 10, 926, 0, 2, 2002, 3756, 1, 0, 8176, 1527, 7514, 1, 1185, 3, 35, 221, 0, 1, 97, 0, 4, 68, 0, 678, 194, 0, 683, 4, 8142, 0, 0, 32, 1, 0, 1, 0, 0, 210, 0, 41, 175, 1, 0, 0, 31, 607, 0, 0, 0, 570, 0, 12, 1, 611, 0, 1419, 707, 13, 17, 209, 0, 0, 0, 2, 1, 77, 0, 1, 753, 1, 276, 0, 1, 9, 0, 3, 0, 0, 311, 1, 0, 12, 1, 2, 1, 215, 818, 623, 1, 0, 124, 4396, 0, 1, 5355, 0, 2, 0, 10, 1364, 0, 0, 0, 6339, 516, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 44, 147, 217, 1624, 0, 0, 3222, 12, 0, 3, 5, 123, 22, 1595, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 207, 0, 0, 752, 0, 4, 1788, 2841, 0, 0, 0, 1887, 0, 0, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 19, 262, 0, 5, 323, 0, 17, 88, 0, 669, 488, 0, 635, 0, 5726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 43, 0, 0, 4, 0, 1, 327, 0, 35, 87, 0, 0, 0, 10, 7692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 672, 0, 1, 4, 772, 0, 696, 746, 6, 6, 263, 0, 2850, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 60, 0, 0, 845, 0, 218, 0, 4, 5, 0, 7, 0, 6808, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 19, 5, 0, 0, 212, 533, 645, 3, 1, 64, 3661, 1, 1539, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4936, 0, 0, 0, 11, 646, 0, 0, 0, 2018, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8660, 8607, 8234, 8201, 7671, 8448, 7728, 7987] avg=8192.0000 max=8660 max_rank=0 min=7671 min_rank=4 imbalance_ratio=1.057129 max_load_reduction=56876 max_load_reduction_pct=0.867859 moved_to_echo=31834 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 205, 228, 1831, 0, 8, 3736, 1, 0, 27, 1, 37, 3, 1596, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 7, 944, 1, 0, 2064, 2824, 1, 0, 0, 1294, 0, 1, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 47, 144, 0, 13, 449, 0, 6, 141, 2, 1027, 156, 0, 688, 39, 5728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 46, 1, 0, 4, 0, 3, 98, 0, 34, 255, 0, 2, 0, 10, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 0, 557, 0, 45, 2, 957, 0, 908, 832, 3, 10, 325, 0, 2512, 993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4, 38, 0, 0, 556, 1, 304, 0, 6, 17, 2, 0, 3, 6506, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 3, 0, 14, 8, 8, 4, 39, 937, 430, 21, 0, 95, 4111, 0, 1529, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4197, 2, 0, 1, 36, 1614, 0, 2, 1, 1779, 589, 0, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9133, 7471, 8463, 8134, 7919, 7934, 8256, 8226] avg=8192.0000 max=9133 max_rank=0 min=7471 min_rank=1 imbalance_ratio=1.114868 max_load_reduction=56403 max_load_reduction_pct=0.860641 moved_to_echo=32335 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 27, 153, 230, 1837, 0, 0, 3084, 3, 3, 19, 2, 103, 2, 1595, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 27, 727, 2, 1, 2508, 2261, 0, 0, 0, 1823, 0, 6, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 57, 229, 2, 1, 290, 0, 25, 68, 0, 475, 152, 0, 856, 10, 5724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 19, 0, 0, 5, 0, 0, 192, 1, 28, 292, 0, 0, 0, 28, 7675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 4, 0, 0, 700, 0, 3, 2, 813, 0, 815, 973, 29, 6, 127, 0, 2582, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 22, 0, 0, 922, 0, 259, 1, 4, 17, 2, 0, 7, 6382, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 21, 4, 9, 2, 193, 980, 369, 1, 4, 96, 4398, 2, 1225, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4216, 0, 4, 0, 13, 992, 0, 1, 8, 1827, 673, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8553, 8510, 7893, 8241, 8241, 8113, 8246, 7739] avg=8192.0000 max=8553 max_rank=0 min=7739 min_rank=7 imbalance_ratio=1.044067 max_load_reduction=56983 max_load_reduction_pct=0.869492 moved_to_echo=32339 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 233, 204, 1395, 4, 7, 1993, 1, 2, 13, 0, 111, 12, 1594, 1532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 120, 0, 13, 1131, 0, 1, 1925, 2630, 3, 0, 0, 1447, 0, 6, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 35, 231, 0, 2, 223, 1, 16, 54, 3, 569, 132, 2, 1529, 5, 5722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 21, 2, 0, 6, 0, 6, 225, 1, 14, 242, 0, 2, 0, 27, 7669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 0, 0, 0, 482, 0, 6, 12, 1041, 0, 772, 1253, 39, 22, 447, 0, 2652, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 1328, 0, 196, 1, 30, 1, 5, 0, 6, 6544, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 571, 0, 4, 14, 2, 0, 4, 128, 1137, 429, 12, 4, 163, 3590, 0, 1424, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4777, 3, 0, 0, 7, 1183, 0, 2, 2, 1877, 329, 1, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7109, 8074, 8534, 8216, 8388, 8693, 8335, 8187] avg=8192.0000 max=8693 max_rank=5 min=7109 min_rank=0 imbalance_ratio=1.061157 max_load_reduction=56843 max_load_reduction_pct=0.867355 moved_to_echo=33603 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 12, 161, 355, 1601, 0, 0, 2574, 2, 3, 5, 3, 141, 14, 1592, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 21, 631, 0, 4, 1777, 2800, 0, 2, 0, 1643, 0, 11, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 61, 95, 0, 0, 195, 0, 12, 29, 2, 1101, 341, 0, 641, 2, 5715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 62, 0, 4, 1, 0, 1, 103, 3, 60, 252, 0, 1, 0, 15, 7668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 0, 1, 0, 392, 1, 17, 1, 922, 1, 1149, 1194, 16, 8, 138, 0, 2770, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 104, 0, 0, 889, 6, 164, 0, 5, 2, 0, 0, 6, 6788, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 7, 3, 11, 6, 0, 0, 160, 1110, 541, 9, 1, 74, 3429, 2, 1517, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4161, 0, 2, 0, 10, 1195, 0, 2, 3, 1961, 544, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8137, 8210, 8174, 8718, 8464, 7883, 7884] avg=8192.0000 max=8718 max_rank=4 min=7883 min_rank=6 imbalance_ratio=1.064209 max_load_reduction=56818 max_load_reduction_pct=0.866974 moved_to_echo=32949 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 13, 183, 212, 1923, 0, 0, 2546, 2, 0, 7, 2, 77, 16, 1598, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 3, 1244, 0, 12, 2414, 2047, 0, 0, 0, 1878, 0, 3, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 41, 320, 0, 9, 187, 0, 2, 214, 0, 634, 108, 0, 623, 5, 5719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 23, 0, 0, 2, 0, 0, 241, 0, 19, 125, 0, 0, 4, 12, 7664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 0, 0, 0, 558, 0, 4, 2, 965, 0, 864, 473, 23, 2, 240, 0, 3065, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 42, 0, 2, 527, 0, 109, 0, 16, 4, 1, 2, 0, 6360, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1, 0, 15, 2, 2, 0, 119, 294, 453, 20, 0, 139, 4781, 1, 1114, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5273, 0, 0, 0, 17, 714, 0, 0, 3, 2157, 364, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8351, 9068, 7873, 8096, 8007, 7558, 8049, 8534] avg=8192.0000 max=9068 max_rank=1 min=7558 min_rank=5 imbalance_ratio=1.106934 max_load_reduction=56468 max_load_reduction_pct=0.861633 moved_to_echo=32148 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 1, 0, 7, 91, 195, 1793, 0, 0, 2085, 1, 0, 5, 2, 61, 5, 1589, 1602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 10, 926, 0, 2, 2002, 2436, 1, 0, 0, 1527, 0, 1, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 35, 221, 0, 1, 97, 0, 4, 68, 0, 678, 194, 0, 683, 4, 5703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1, 0, 1, 0, 0, 210, 0, 41, 175, 1, 0, 0, 31, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 570, 0, 12, 1, 611, 0, 1419, 707, 13, 17, 209, 0, 2774, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 77, 0, 1, 753, 1, 276, 0, 1, 9, 0, 3, 0, 6520, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 1, 0, 12, 1, 2, 1, 215, 818, 623, 1, 0, 124, 4396, 0, 1320, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5355, 0, 2, 0, 10, 1364, 0, 0, 0, 1963, 516, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7441, 8271, 7691, 8172, 7934, 8140, 8675, 9212] avg=8192.0000 max=9212 max_rank=7 min=7441 min_rank=0 imbalance_ratio=1.124512 max_load_reduction=56324 max_load_reduction_pct=0.859436 moved_to_echo=33961 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 189, 97, 1584, 0, 0, 3001, 4, 2, 22, 5, 125, 31, 1594, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 16, 808, 0, 3, 1549, 2664, 2, 1, 0, 1289, 0, 0, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 30, 153, 0, 6, 312, 0, 14, 84, 1, 1392, 188, 1, 667, 33, 5724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 64, 0, 0, 0, 0, 1, 151, 1, 19, 368, 0, 1, 0, 18, 7678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 0, 0, 569, 0, 8, 1, 784, 0, 1222, 1207, 20, 4, 152, 3, 2701, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8, 2, 0, 56, 0, 3, 782, 5, 344, 0, 3, 35, 4, 12, 0, 6420, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 298, 4, 1, 21, 6, 1, 0, 208, 908, 692, 16, 2, 169, 3740, 0, 1443, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4589, 0, 0, 0, 22, 833, 0, 0, 13, 1912, 372, 2, 18, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 7398, 8638, 8302, 8658, 8186, 8364, 7767] avg=8192.0000 max=8658 max_rank=4 min=7398 min_rank=1 imbalance_ratio=1.056885 max_load_reduction=56878 max_load_reduction_pct=0.867889 moved_to_echo=32975 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 1280, 0, 128, 7296, 128, 128, 16128, 20480, 128, 128, 0, 12928, 0, 128, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 1408, 1792, 13824, 128, 128, 22400, 128, 128, 128, 128, 896, 128, 13184, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[6912, 128, 0, 0, 4608, 128, 128, 128, 6912, 128, 8064, 7680, 256, 128, 1920, 128, 21504, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 2176, 128, 128, 128, 128, 128, 128, 1408, 6784, 4224, 128, 128, 1024, 31616, 128, 11136, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 37504, 128, 128, 128, 128, 8704, 0, 128, 128, 15616, 3840, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 384, 1792, 128, 128, 2048, 128, 128, 768, 128, 6528, 1792, 128, 6272, 128, 46080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 512, 0, 128, 6656, 128, 1920, 128, 128, 128, 128, 128, 128, 52480, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 1408, 1792, 13696, 128, 128, 22272, 128, 128, 128, 128, 896, 128, 12800, 12672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 1280, 0, 128, 7168, 128, 128, 16128, 20608, 128, 128, 0, 12800, 0, 128, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[6912, 128, 128, 0, 4608, 128, 128, 128, 6912, 128, 7936, 7424, 256, 128, 1920, 128, 22016, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 2176, 128, 128, 128, 128, 128, 128, 1280, 6784, 4224, 128, 128, 1024, 32128, 128, 11136, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 37504, 128, 128, 128, 128, 8576, 0, 128, 128, 15616, 3840, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 384, 1664, 128, 128, 2176, 128, 128, 768, 128, 6656, 1792, 128, 6400, 128, 45824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 512, 0, 128, 6656, 128, 1920, 128, 128, 128, 128, 128, 128, 52352, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 384, 128, 128, 128, 0, 128, 1536, 128, 256, 1792, 128, 128, 128, 256, 61568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 384, 128, 128, 128, 0, 128, 1664, 128, 256, 1920, 128, 128, 128, 256, 61440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8180, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 8189, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8188, 0, 3817, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 3357, 0, 0, 8150, 0, 2, 0, 1, 0, 14, 1, 1, 0, 0, 0, 1010, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8184, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 8183, 18, 7, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 4, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8149, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 1, 2, 0, 0, 4, 8192, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1986, 0, 0, 8085, 0, 1, 0, 0, 0, 330, 0, 0, 0, 1, 0, 2840, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 120, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 2904, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 8192, 0, 1, 0, 0, 0, 131, 0, 0, 0, 0, 0, 1231, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8140, 0, 15, 0, 1, 0, 0, 1, 2, 5, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 13, 0, 8171, 0, 0, 2, 3, 0, 0, 0, 2, 1, 0, 1, 0, 0, 36, 0, 0, 7, 0, 0, 0, 2, 0, 6, 1, 4, 2, 103, 0, 0, 0, 0, 0, 9, 2, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 1, 4, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 8175, 0, 1, 3, 0, 0, 0, 8179, 0, 4521, 0, 0, 4, 0, 0, 0, 64, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2430, 0, 3, 7949, 2, 0, 3, 0, 0, 73, 0, 2, 3, 0, 0, 1226, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8165, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4, 72, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 10, 0, 1, 0, 8186, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 718, 0, 0, 8138, 0, 22, 0, 0, 0, 393, 0, 0, 0, 0, 0, 6436, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8147, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 2, 0, 18, 0, 0, 0, 8191, 0, 0, 8182, 3, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 8, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 12, 0, 0, 0, 8186, 0, 369, 21, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1596, 0, 0, 8184, 0, 15, 0, 0, 0, 52, 1, 2, 0, 0, 0, 6064, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8104, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 3, 1, 17, 0, 0, 0, 8191, 0, 0, 8181, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 38, 0, 14, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 8190, 0, 1373, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2174, 0, 0, 8146, 0, 4, 0, 0, 0, 54, 0, 2, 0, 0, 0, 4631, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 9, 0, 2, 0, 8192, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3307, 0, 0, 8082, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 4398, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8180, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8164, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6046, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 3357, 0, 0, 5788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 14, 1, 1, 0, 0, 0, 1010, 0, 0, 0, 0, 2362, 2141, 34, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8200, 8181, 8151, 8192, 9863, 9156, 5591] avg=8192.0000 max=9863 max_rank=5 min=5591 min_rank=7 imbalance_ratio=1.203979 max_load_reduction=55673 max_load_reduction_pct=0.849503 moved_to_echo=32802 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8184, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8115, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6047, 0, 1, 2, 0, 0, 4, 0, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1986, 0, 0, 5742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 330, 0, 0, 0, 1, 0, 2840, 0, 0, 0, 0, 2343, 2142, 34, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8183, 8182, 8131, 8189, 9199, 7728, 7715] avg=8192.0000 max=9199 max_rank=5 min=7715 min_rank=7 imbalance_ratio=1.122925 max_load_reduction=56337 max_load_reduction_pct=0.859634 moved_to_echo=32831 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8104, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 3, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 8156, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 38, 0, 14, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6040, 0, 0, 0, 0, 0, 0, 0, 0, 1373, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2174, 0, 0, 5785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 54, 0, 2, 0, 0, 0, 4631, 0, 0, 0, 0, 2361, 2139, 34, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8197, 8234, 8160, 8191, 7417, 7959, 9250] avg=8192.0000 max=9250 max_rank=7 min=7417 min_rank=5 imbalance_ratio=1.129150 max_load_reduction=56286 max_load_reduction_pct=0.858856 moved_to_echo=32817 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8165, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 72, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6037, 0, 0, 10, 0, 1, 0, 0, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 718, 0, 0, 5779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 393, 0, 0, 0, 0, 0, 6436, 0, 0, 0, 0, 2359, 2138, 34, 15, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8168, 8171, 8261, 8193, 6653, 6500, 11405] avg=8192.0000 max=11405 max_rank=7 min=6500 min_rank=6 imbalance_ratio=1.392212 max_load_reduction=54131 max_load_reduction_pct=0.825974 moved_to_echo=32751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8147, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 2, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 8148, 3, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6042, 0, 0, 12, 0, 0, 0, 0, 0, 369, 21, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1596, 0, 0, 5807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 52, 1, 2, 0, 0, 0, 6064, 1, 0, 0, 0, 2377, 2145, 39, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8172, 8183, 8203, 8192, 6446, 7404, 10724] avg=8192.0000 max=10724 max_rank=7 min=6446 min_rank=5 imbalance_ratio=1.309082 max_load_reduction=54812 max_load_reduction_pct=0.836365 moved_to_echo=32766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8140, 0, 15, 0, 1, 0, 0, 1, 2, 5, 0, 11, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 0, 8146, 0, 0, 2, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 36, 0, 0, 7, 0, 0, 0, 2, 0, 6, 1, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 103, 0, 0, 0, 0, 0, 9, 2, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6037, 0, 1, 3, 0, 0, 0, 0, 0, 4521, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2430, 0, 3, 5645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 73, 0, 2, 3, 0, 0, 1226, 0, 0, 0, 4, 2304, 2138, 34, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8167, 8216, 8243, 8192, 10567, 8145, 5814] avg=8192.0000 max=10567 max_rank=5 min=5814 min_rank=7 imbalance_ratio=1.289917 max_load_reduction=54969 max_load_reduction_pct=0.838760 moved_to_echo=32718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 120, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6049, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 5818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 131, 0, 0, 0, 0, 0, 1231, 0, 0, 0, 0, 2374, 2142, 34, 15, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8282, 8193, 8175, 8194, 8963, 9598, 5936] avg=8192.0000 max=9598 max_rank=6 min=5936 min_rank=7 imbalance_ratio=1.171631 max_load_reduction=55938 max_load_reduction_pct=0.853546 moved_to_echo=32691 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6045, 0, 0, 9, 0, 2, 0, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3307, 0, 0, 5739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 4398, 0, 0, 0, 0, 2343, 2141, 34, 15, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8167, 8176, 8212, 8193, 6428, 9046, 9101] avg=8192.0000 max=9101 max_rank=7 min=6428 min_rank=5 imbalance_ratio=1.110962 max_load_reduction=56435 max_load_reduction_pct=0.861130 moved_to_echo=32768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8184, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8184, 20, 7, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 5, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8148, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 1, 2, 0, 0, 4, 8192, 0, 2434, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2922, 0, 0, 8080, 0, 0, 0, 0, 0, 257, 0, 0, 0, 1, 0, 2691, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8147, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 8191, 0, 0, 8181, 11, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 7, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 13, 0, 0, 0, 8186, 0, 213, 20, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2174, 0, 0, 8187, 0, 14, 0, 0, 0, 42, 1, 2, 0, 0, 0, 5652, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8102, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 3, 0, 14, 0, 0, 0, 8191, 0, 0, 8181, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 40, 0, 12, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 8190, 0, 1014, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2923, 0, 0, 8146, 0, 4, 0, 0, 0, 24, 0, 2, 0, 0, 0, 4274, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8181, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8189, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8188, 0, 2663, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 4541, 0, 0, 8149, 0, 2, 0, 2, 0, 7, 1, 0, 0, 0, 0, 982, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 139, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 2074, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4705, 0, 0, 8192, 0, 1, 0, 0, 0, 87, 0, 0, 0, 0, 0, 1164, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8138, 0, 15, 0, 1, 0, 0, 1, 1, 5, 0, 10, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 15, 0, 8171, 0, 0, 2, 3, 0, 0, 0, 2, 1, 0, 0, 0, 0, 30, 0, 0, 8, 0, 0, 0, 2, 0, 6, 1, 4, 2, 101, 0, 0, 0, 0, 0, 8, 2, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 1, 4, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 8175, 0, 1, 3, 0, 0, 1, 8179, 0, 3472, 0, 0, 4, 0, 1, 0, 61, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 3543, 0, 3, 7944, 2, 0, 3, 0, 0, 68, 0, 1, 3, 0, 0, 1184, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8164, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4, 72, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 10, 0, 1, 0, 8185, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 1104, 0, 0, 8143, 0, 22, 0, 0, 0, 337, 0, 0, 0, 0, 0, 6361, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 9, 0, 2, 0, 8192, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4103, 0, 0, 8072, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 3773, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8184, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8114, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6631, 0, 1, 2, 0, 0, 4, 0, 0, 2434, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2922, 0, 0, 4910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 1, 0, 2691, 0, 0, 0, 0, 3170, 1558, 34, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8182, 8183, 8131, 8189, 9073, 7832, 7737] avg=8192.0000 max=9073 max_rank=5 min=7737 min_rank=7 imbalance_ratio=1.107544 max_load_reduction=56463 max_load_reduction_pct=0.861557 moved_to_echo=32831 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8102, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 3, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8152, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 40, 0, 12, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6623, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2923, 0, 0, 4950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 24, 0, 2, 0, 0, 0, 4274, 0, 0, 0, 0, 3196, 1556, 34, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8193, 8233, 8160, 8193, 7641, 7873, 9116] avg=8192.0000 max=9116 max_rank=7 min=7641 min_rank=5 imbalance_ratio=1.112793 max_load_reduction=56420 max_load_reduction_pct=0.860901 moved_to_echo=32823 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8147, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 2, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 8144, 11, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6625, 0, 0, 13, 0, 0, 0, 0, 0, 213, 20, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2174, 0, 0, 4969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 42, 1, 2, 0, 0, 0, 5652, 0, 0, 0, 0, 3218, 1562, 37, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8174, 8183, 8199, 8192, 6874, 7144, 10559] avg=8192.0000 max=10559 max_rank=7 min=6874 min_rank=5 imbalance_ratio=1.288940 max_load_reduction=54977 max_load_reduction_pct=0.838882 moved_to_echo=32769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8181, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6630, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 4541, 0, 0, 4952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 7, 1, 0, 0, 0, 0, 982, 0, 0, 0, 0, 3197, 1557, 34, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8198, 8182, 8153, 8193, 9293, 9504, 5809] avg=8192.0000 max=9504 max_rank=6 min=5809 min_rank=7 imbalance_ratio=1.160156 max_load_reduction=56032 max_load_reduction_pct=0.854980 moved_to_echo=32799 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8164, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 72, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6620, 0, 0, 10, 0, 1, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 1104, 0, 0, 4948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 337, 0, 0, 0, 0, 0, 6361, 0, 0, 0, 0, 3195, 1555, 34, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8164, 8172, 8267, 8191, 6972, 6055, 11530] avg=8192.0000 max=11530 max_rank=7 min=6055 min_rank=6 imbalance_ratio=1.407471 max_load_reduction=54006 max_load_reduction_pct=0.824066 moved_to_echo=32748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 139, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6633, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4705, 0, 0, 4978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 87, 0, 0, 0, 0, 0, 1164, 0, 0, 0, 0, 3214, 1558, 34, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8297, 8195, 8170, 8194, 8717, 9683, 6084] avg=8192.0000 max=9683 max_rank=6 min=6084 min_rank=7 imbalance_ratio=1.182007 max_load_reduction=55853 max_load_reduction_pct=0.852249 moved_to_echo=32678 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8138, 0, 15, 0, 1, 0, 0, 1, 1, 5, 0, 10, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 0, 8142, 0, 0, 2, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 30, 0, 0, 8, 0, 0, 0, 2, 0, 6, 1, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 101, 0, 0, 0, 0, 0, 8, 2, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6620, 0, 1, 3, 0, 0, 1, 0, 0, 3472, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 3543, 0, 3, 4827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 68, 0, 1, 3, 0, 0, 1184, 0, 0, 0, 4, 3117, 1555, 34, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8165, 8211, 8241, 8192, 10103, 8437, 5998] avg=8192.0000 max=10103 max_rank=5 min=5998 min_rank=7 imbalance_ratio=1.233276 max_load_reduction=55433 max_load_reduction_pct=0.845840 moved_to_echo=32730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6629, 0, 0, 9, 0, 2, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4103, 0, 0, 4905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 3773, 0, 0, 0, 0, 3167, 1557, 34, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8163, 8177, 8215, 8192, 6863, 9008, 8703] avg=8192.0000 max=9008 max_rank=6 min=6863 min_rank=5 imbalance_ratio=1.099609 max_load_reduction=56528 max_load_reduction_pct=0.862549 moved_to_echo=32766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 0, 1280, 128, 128, 128, 128, 0, 27904, 128, 0, 0, 128, 18944, 17152, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 128, 0, 19456, 0, 128, 46208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 48384, 0, 128, 128, 0, 128, 128, 0, 0, 17152, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 1024, 128, 128, 128, 128, 0, 26112, 0, 0, 0, 128, 25600, 12544, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 128, 0, 26112, 0, 128, 39552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 53120, 0, 128, 128, 0, 128, 128, 0, 0, 12544, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[65408, 128, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 0, 0, 128, 0, 65280, 256, 128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 256, 256, 128, 0, 128, 0, 0, 128, 128, 0, 0, 65280, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 0, 128, 0, 65280, 256, 128, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[65408, 128, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 1, 0, 0, 0, 0, 0, 0, 0, 8182, 8191, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 3, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8189, 0, 0, 7953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8190, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8188, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 1, 0, 1, 1, 0, 0, 0, 8181, 8191, 0, 1, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 5, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[8163, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 134, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 8182, 0, 0, 0, 1, 0, 0, 8147, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 0, 4, 0, 8184, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8141, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8189, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 1, 0, 1, 1, 0, 0, 0, 8179, 8191, 0, 1, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[8162, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 132, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 8181, 0, 0, 0, 1, 0, 0, 8139, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 0, 4, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8149, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[8156, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 2, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[8076, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 8130, 8172, 0, 1, 1, 0, 0, 4, 8163, 0, 0, 0, 4, 0, 0, 0, 0, 0, 38, 10, 0, 0, 0, 12, 0, 0, 0, 0, 1, 17, 0, 8171, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 8166, 6, 1, 8158, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4, 0, 0, 0, 0, 110, 0, 11, 0, 7, 0, 1, 0, 0, 0, 0, 20, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8192, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[8138, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 184, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 8185, 0, 0, 0, 0, 0, 3, 8181, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8190, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 1, 8033, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7, 0, 4, 0, 1, 0, 1, 0, 1, 4, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[8073, 3, 0, 0, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 8, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 8133, 8172, 0, 0, 1, 0, 0, 4, 8163, 0, 0, 0, 2, 0, 0, 0, 0, 0, 36, 10, 0, 0, 0, 12, 0, 0, 0, 0, 1, 18, 0, 8171, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 8166, 7, 1, 8157, 0, 0, 0, 0, 10, 0, 0, 0, 0, 5, 0, 0, 0, 0, 112, 0, 11, 0, 6, 0, 1, 0, 0, 0, 0, 21, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[8158, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 1, 0, 1, 1, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 5, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8047, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8199, 8204, 8189, 8192, 8227, 8127, 8184] avg=8192.0000 max=8227 max_rank=5 min=8127 min_rank=6 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 1, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8169, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 7953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 8045, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8325, 8192, 8173, 8198, 8096, 8158, 8189] avg=8192.0000 max=8325 max_rank=1 min=8096 min_rank=5 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32641 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8163, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 134, 0, 0, 0, 0, 0, 0, 0, 0, 8007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8147, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 0, 4, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 8141, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 8048, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8142, 8175, 8194, 8189, 8300, 8156, 8190] avg=8192.0000 max=8300 max_rank=5 min=8142 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32835 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8192, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8178, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 143, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8048, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8191, 8145, 8200, 8194, 8228, 8157, 8195] avg=8192.0000 max=8228 max_rank=5 min=8145 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8138, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 23, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 184, 0, 1, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8181, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8172, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 1, 8033, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7, 0, 0, 8041, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 1, 0, 1, 0, 1, 4, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8264, 8204, 8188, 8193, 8175, 8136, 8196] avg=8192.0000 max=8264 max_rank=1 min=8136 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32700 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8073, 3, 0, 0, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 3, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 8038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 8163, 0, 0, 0, 2, 0, 0, 0, 0, 0, 36, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 12, 0, 0, 0, 0, 1, 18, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8172, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 143, 7, 1, 8157, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 112, 0, 11, 0, 6, 0, 1, 0, 0, 8022, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8051, 8217, 8200, 8183, 8322, 8236, 8212] avg=8192.0000 max=8322 max_rank=5 min=8051 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8158, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8046, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8181, 8203, 8208, 8194, 8243, 8133, 8185] avg=8192.0000 max=8243 max_rank=5 min=8133 min_rank=6 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 7802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 0, 8048, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8183, 8196, 8184, 8193, 7945, 8433, 8185] avg=8192.0000 max=8433 max_rank=6 min=7945 min_rank=5 imbalance_ratio=1.029419 max_load_reduction=57103 max_load_reduction_pct=0.871323 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[8138, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 176, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 8185, 0, 0, 0, 0, 0, 3, 8176, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8190, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 1, 8044, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7, 0, 4, 0, 1, 0, 2, 0, 1, 4, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 1, 0, 0, 0, 0, 0, 0, 0, 8181, 8191, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 3, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8187, 0, 0, 7960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8189, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 1, 0, 1, 1, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 1, 0, 1, 0, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8050, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8201, 8206, 8187, 8192, 8225, 8125, 8184] avg=8192.0000 max=8225 max_rank=5 min=8125 min_rank=6 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8162, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 132, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8139, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 0, 4, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 8149, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 26, 0, 0, 8051, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8144, 8169, 8196, 8190, 8306, 8151, 8191] avg=8192.0000 max=8306 max_rank=5 min=8144 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8156, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8049, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8183, 8203, 8205, 8194, 8244, 8134, 8185] avg=8192.0000 max=8244 max_rank=5 min=8134 min_rank=6 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8192, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8178, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 140, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8051, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8195, 8146, 8200, 8194, 8225, 8155, 8195] avg=8192.0000 max=8226 max_rank=0 min=8146 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8076, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8, 0, 0, 3, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4, 8163, 0, 0, 0, 4, 0, 0, 0, 0, 0, 38, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 12, 0, 0, 0, 0, 1, 17, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8172, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 140, 6, 1, 8158, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 110, 0, 11, 0, 7, 0, 1, 0, 0, 8025, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8050, 8222, 8200, 8182, 8321, 8232, 8211] avg=8192.0000 max=8321 max_rank=5 min=8050 min_rank=1 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32946 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8138, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 21, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 176, 0, 1, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8176, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8172, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 1, 8044, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 8047, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 1, 0, 2, 0, 1, 4, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8264, 8201, 8190, 8190, 8183, 8135, 8197] avg=8192.0000 max=8264 max_rank=1 min=8135 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32705 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 7792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 8051, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8176, 8197, 8184, 8193, 7932, 8452, 8185] avg=8192.0000 max=8452 max_rank=6 min=7932 min_rank=5 imbalance_ratio=1.031738 max_load_reduction=57084 max_load_reduction_pct=0.871033 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 1, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8170, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 7960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 8046, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8323, 8192, 8174, 8201, 8100, 8152, 8188] avg=8192.0000 max=8323 max_rank=1 min=8100 min_rank=5 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32641 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 128, 0, 128, 0, 128, 128, 512, 0, 0, 64384, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 1152, 128, 128, 64384, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[65408, 128, 128, 128, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 0, 128, 128, 128, 65408, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 1024, 128, 128, 128, 128, 128, 0, 0, 0, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 65408, 128, 0, 0, 128, 0, 128, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 1152, 128, 128, 64384, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[65408, 128, 128, 128, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 0, 128, 128, 128, 65408, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 65408, 128, 0, 0, 128, 0, 128, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 1, 8031, 0, 0, 0, 0, 8001, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 11, 0, 0, 305, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 8191, 181, 1, 0, 0, 0, 0, 3, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 7, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8180, 0, 0, 0, 8175, 11, 0, 0, 0, 7799, 0, 8136, 0, 0, 1, 10, 3, 0, 0, 0, 0, 0, 13, 0, 4, 0, 0, 12, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 8122, 0, 0, 0, 0, 16, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 1, 0, 0, 1, 0, 70, 8170, 0, 0, 0, 1, 2, 0, 57, 8161, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7915, 0, 0, 0, 8151, 0, 1, 0, 0, 7379, 0, 8186, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 802, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8180, 0, 0, 0, 0, 4, 0, 1, 282, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 2, 8188, 56, 0, 0, 1, 0, 0, 3, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7832, 0, 0, 0, 7720, 0, 0, 0, 0, 7326, 0, 8192, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 16, 0, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 8192, 7, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7739, 0, 0, 0, 8174, 0, 0, 0, 0, 8174, 0, 8185, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 90, 0, 0, 556, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 7896, 0, 0, 0, 0, 0, 0, 8191, 93, 0, 0, 0, 0, 0, 0, 8182, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8120, 0, 0, 0, 8145, 0, 0, 0, 0, 8132, 0, 8181, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 73, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 8191, 9, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 8188, 0, 0, 0, 8157, 0, 0, 0, 0, 7877, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8085, 0, 1, 0, 0, 64, 0, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 1, 0, 3, 8191, 1, 0, 0, 0, 0, 0, 2, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7916, 0, 0, 0, 7976, 0, 0, 0, 0, 8173, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 12, 10, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 184, 0, 0, 75, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 8190, 287, 0, 0, 0, 0, 9, 0, 8183, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7915, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 1, 0, 0, 7379, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 802, 0, 0, 0, 0, 7601, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8180, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 282, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7884, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 8134, 0, 64, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 56, 0, 0, 1, 0, 0, 3, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 7670, 8706, 8217, 8177, 8186, 8295, 8155] avg=8192.0000 max=8706 max_rank=2 min=7670 min_rank=1 imbalance_ratio=1.062744 max_load_reduction=56830 max_load_reduction_pct=0.867157 moved_to_echo=32813 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8180, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 11, 0, 0, 0, 7799, 0, 0, 0, 0, 1, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4, 0, 0, 12, 0, 488, 0, 0, 0, 0, 7619, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 8122, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 7836, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 1, 61, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 70, 0, 0, 0, 0, 1, 2, 0, 57, 8070, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8431, 8124, 8436, 8163, 7964, 8191, 8017, 8210] avg=8192.0000 max=8436 max_rank=2 min=7964 min_rank=4 imbalance_ratio=1.029785 max_load_reduction=57100 max_load_reduction_pct=0.871277 moved_to_echo=32382 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7739, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 0, 8174, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 7619, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 556, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7883, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 7896, 0, 62, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7947, 8493, 7956, 8223, 8532, 8190, 8012, 8183] avg=8192.0000 max=8532 max_rank=4 min=7947 min_rank=0 imbalance_ratio=1.041504 max_load_reduction=57004 max_load_reduction_pct=0.869812 moved_to_echo=32917 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 8188, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 0, 7877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 5, 0, 0, 0, 7603, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8085, 0, 1, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7886, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 62, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 2, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8399, 8176, 8289, 8122, 7971, 8191, 8283, 8105] avg=8192.0000 max=8399 max_rank=0 min=7971 min_rank=4 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7832, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 0, 0, 0, 7326, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 3, 0, 0, 0, 7195, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 62, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8028, 7847, 8442, 8223, 8508, 8190, 8192, 8106] avg=8192.0000 max=8508 max_rank=4 min=7847 min_rank=1 imbalance_ratio=1.038574 max_load_reduction=57028 max_load_reduction_pct=0.870178 moved_to_echo=32996 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8120, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 0, 8132, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 1, 0, 0, 0, 7592, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7879, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8142, 0, 62, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8460, 7979, 8196, 8020, 8191, 8254, 8108] avg=8192.0000 max=8460 max_rank=1 min=7979 min_rank=2 imbalance_ratio=1.032715 max_load_reduction=57076 max_load_reduction_pct=0.870911 moved_to_echo=32573 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7916, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 0, 0, 8173, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 12, 10, 0, 0, 0, 7434, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 75, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7883, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 8052, 0, 62, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 9, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8468, 7760, 8164, 8145, 8207, 8278, 8387] avg=8192.0000 max=8468 max_rank=1 min=7760 min_rank=2 imbalance_ratio=1.033691 max_load_reduction=57068 max_load_reduction_pct=0.870789 moved_to_echo=33017 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 295, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 7485, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 305, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 7887, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8088, 0, 62, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 1, 0, 0, 0, 0, 3, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8298, 7968, 8228, 8219, 8190, 8205, 8282] avg=8192.0000 max=8298 max_rank=1 min=7968 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32896 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 6, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8181, 0, 0, 0, 8175, 12, 0, 0, 0, 7908, 0, 8143, 0, 0, 1, 10, 3, 0, 0, 0, 0, 0, 14, 0, 4, 0, 0, 11, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 3, 8123, 0, 0, 0, 0, 16, 0, 0, 94, 1, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 0, 0, 0, 0, 0, 73, 8169, 0, 0, 0, 1, 3, 0, 56, 8159, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 1, 8029, 0, 0, 0, 0, 8057, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 13, 0, 0, 310, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 8191, 175, 1, 0, 0, 0, 0, 3, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7915, 0, 0, 0, 7968, 0, 0, 0, 0, 8179, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 11, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 191, 0, 0, 77, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 1, 8189, 289, 0, 0, 0, 0, 7, 0, 8182, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 8188, 0, 0, 0, 8153, 0, 0, 0, 0, 8064, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8052, 0, 0, 0, 0, 67, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 0, 3, 8191, 2, 0, 0, 0, 0, 0, 2, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8121, 0, 0, 0, 8142, 0, 0, 0, 0, 8171, 0, 8181, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 71, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 8191, 9, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7836, 0, 0, 0, 7681, 0, 0, 0, 0, 7777, 0, 8192, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 18, 0, 0, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7917, 0, 0, 0, 8146, 0, 1, 0, 0, 7682, 0, 8185, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 504, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8180, 0, 0, 0, 0, 5, 0, 1, 284, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 8188, 50, 0, 0, 1, 0, 0, 3, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 0, 0, 8173, 0, 0, 0, 0, 8176, 0, 8186, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 90, 0, 0, 541, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 7914, 0, 0, 0, 0, 0, 0, 8191, 92, 0, 0, 0, 0, 0, 0, 8181, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8181, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 12, 0, 0, 0, 7908, 0, 0, 0, 0, 1, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4, 0, 0, 11, 0, 375, 0, 0, 0, 0, 7918, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 3, 8123, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 94, 1, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 7765, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 0, 56, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1, 3, 0, 56, 8069, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8428, 8087, 8323, 8170, 8114, 8190, 8014, 8210] avg=8192.0000 max=8428 max_rank=0 min=8014 min_rank=6 imbalance_ratio=1.028809 max_load_reduction=57108 max_load_reduction_pct=0.871399 moved_to_echo=32528 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7915, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 8179, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 11, 0, 0, 0, 7957, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 77, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7568, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 19, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 8052, 0, 56, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 289, 0, 0, 0, 0, 7, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8333, 7979, 8166, 8063, 8207, 8277, 8389] avg=8192.0000 max=8389 max_rank=7 min=7979 min_rank=2 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32936 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 8188, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 6, 0, 0, 0, 7960, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8052, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7744, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 56, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 2, 0, 0, 0, 0, 0, 2, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8396, 8217, 8187, 8092, 8060, 8192, 8283, 8109] avg=8192.0000 max=8396 max_rank=0 min=8060 min_rank=4 imbalance_ratio=1.024902 max_load_reduction=57140 max_load_reduction_pct=0.871887 moved_to_echo=32644 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 151, 0, 0, 0, 0, 8057, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 7962, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 310, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 7626, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8074, 0, 56, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 1, 0, 0, 0, 0, 3, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8210, 8090, 8231, 8190, 8190, 8189, 8279] avg=8192.0000 max=8279 max_rank=7 min=8090 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32848 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8121, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 8171, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 0, 7956, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7733, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8146, 0, 56, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8355, 7996, 8206, 8095, 8191, 8256, 8111] avg=8192.0000 max=8355 max_rank=1 min=7996 min_rank=2 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32653 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7917, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 1, 0, 0, 7682, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 504, 1, 0, 0, 0, 7960, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8180, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 284, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7741, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 8139, 0, 61, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 1, 0, 0, 3, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 7829, 8468, 8218, 8260, 8186, 8299, 8148] avg=8192.0000 max=8468 max_rank=2 min=7829 min_rank=1 imbalance_ratio=1.033691 max_load_reduction=57068 max_load_reduction_pct=0.870789 moved_to_echo=32893 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7836, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 7777, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 4, 0, 0, 0, 7966, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7296, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 56, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8028, 8158, 8505, 8226, 8133, 8190, 8189, 8107] avg=8192.0000 max=8505 max_rank=2 min=8028 min_rank=0 imbalance_ratio=1.038208 max_load_reduction=57031 max_load_reduction_pct=0.870224 moved_to_echo=32619 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 8176, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 7960, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 541, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7763, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 7914, 0, 56, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7951, 8347, 7988, 8227, 8621, 8190, 8029, 8183] avg=8192.0000 max=8621 max_rank=4 min=7951 min_rank=0 imbalance_ratio=1.052368 max_load_reduction=56915 max_load_reduction_pct=0.868454 moved_to_echo=33023 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 128, 0, 128, 0, 2944, 128, 0, 0, 0, 60160, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 2432, 128, 128, 0, 0, 62976, 0, 0, 384, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 512, 0, 128, 2048, 128, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 63104, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 256, 128, 0, 0, 0, 64512, 128, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 63872, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 128, 128, 65280, 0, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 1280, 128, 128, 0, 0, 64128, 0, 0, 384, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 0, 0, 128, 0, 0, 128, 128, 128, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 512, 0, 128, 2048, 128, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 61312, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 128, 0, 640, 128, 0, 128, 128, 128, 128, 64768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 128, 0, 128, 0, 1920, 128, 0, 0, 0, 63744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 128, 128, 128, 128, 65280, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 256, 128, 0, 0, 0, 64512, 0, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1, 8183, 0, 0, 0, 0, 0, 0, 0, 7393, 8191, 0, 8191, 8128, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 88, 0, 259, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8190, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7986, 8191, 0, 8191, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 27, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 42, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 1, 7113, 8187, 0, 8192, 8168, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4625, 8192, 0, 8192, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 6, 1, 6, 0, 8146, 0, 16, 0, 23, 0, 0, 0, 2186, 8154, 0, 8166, 8012, 0, 10, 0, 0, 5, 0, 6, 17, 0, 0, 0, 0, 0, 0, 0, 0, 98, 17, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 14, 0, 0, 8171, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 21, 8127, 0, 0, 0, 0, 0, 0, 1732, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 4410, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 2515, 8178, 0, 8190, 8191, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 29, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 49, 0, 413, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5192, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 14, 2, 0, 0, 8190, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 2, 0, 8027, 8156, 0, 8190, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 39, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 8109, 8192, 0, 8191, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 34, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1, 2193, 0, 0, 0, 0, 0, 0, 0, 7393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8128, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 8161, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7706, 0, 0, 0, 0, 88, 0, 259, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 0, 5975, 466, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9589, 8154, 8189, 8190, 8055, 8204, 6960, 8195] avg=8192.0000 max=9589 max_rank=0 min=6960 min_rank=6 imbalance_ratio=1.170532 max_load_reduction=55947 max_load_reduction_pct=0.853683 moved_to_echo=31414 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 2195, 0, 0, 0, 0, 0, 0, 0, 7986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7718, 0, 0, 0, 0, 27, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8191, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 5982, 467, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10182, 8211, 8176, 8192, 7752, 8202, 6634, 8187] avg=8192.0000 max=10182 max_rank=0 min=6634 min_rank=6 imbalance_ratio=1.242920 max_load_reduction=55354 max_load_reduction_pct=0.844635 moved_to_echo=30775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2193, 0, 0, 0, 0, 0, 0, 1, 7113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8168, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 0, 1, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 0, 0, 3, 0, 0, 0, 0, 0, 5976, 468, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9307, 8251, 8179, 8192, 7873, 8196, 7351, 8187] avg=8192.0000 max=9307 max_rank=0 min=7351 min_rank=6 imbalance_ratio=1.136108 max_load_reduction=56229 max_load_reduction_pct=0.857986 moved_to_echo=31607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2189, 0, 0, 0, 0, 0, 2, 0, 8027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7723, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5963, 467, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 39, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10218, 8202, 8175, 8210, 7885, 8165, 6445, 8236] avg=8192.0000 max=10218 max_rank=0 min=6445 min_rank=6 imbalance_ratio=1.247314 max_load_reduction=55318 max_load_reduction_pct=0.844086 moved_to_echo=30731 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 1, 6, 0, 2183, 0, 16, 0, 23, 0, 0, 0, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8012, 0, 10, 0, 0, 5, 0, 6, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 17, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 8136, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8169, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7663, 0, 0, 0, 0, 0, 0, 1732, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8154, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 4410, 0, 0, 0, 4, 0, 0, 0, 0, 5948, 464, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4421, 8071, 8275, 8186, 9424, 8167, 10845, 8147] avg=8192.0000 max=10845 max_rank=6 min=4421 min_rank=0 imbalance_ratio=1.323853 max_load_reduction=54691 max_load_reduction_pct=0.834518 moved_to_echo=36583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2179, 0, 0, 0, 0, 0, 0, 0, 2515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8191, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 8164, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7718, 0, 0, 0, 0, 49, 0, 413, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8178, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5192, 2, 2, 0, 0, 0, 0, 0, 0, 5967, 473, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 14, 2, 0, 0, 8179, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4694, 8233, 8188, 8186, 8182, 8196, 11650, 8207] avg=8192.0000 max=11650 max_rank=6 min=4694 min_rank=0 imbalance_ratio=1.422119 max_load_reduction=53886 max_load_reduction_pct=0.822235 moved_to_echo=36235 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2195, 0, 0, 0, 0, 0, 1, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7723, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5981, 467, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10305, 8204, 8177, 8190, 7776, 8205, 6491, 8188] avg=8192.0000 max=10305 max_rank=0 min=6491 min_rank=6 imbalance_ratio=1.257935 max_load_reduction=55231 max_load_reduction_pct=0.842758 moved_to_echo=30660 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2195, 0, 0, 0, 0, 0, 0, 0, 4625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7723, 0, 0, 0, 0, 2, 0, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 5982, 467, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6820, 8210, 8177, 8190, 8589, 8201, 9160, 8189] avg=8192.0000 max=9160 max_rank=6 min=6820 min_rank=0 imbalance_ratio=1.118164 max_load_reduction=56376 max_load_reduction_pct=0.860229 moved_to_echo=34139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8037, 8191, 0, 8190, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 23, 0, 15, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 40, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1, 8183, 0, 0, 0, 0, 0, 0, 1, 7604, 8191, 0, 8191, 8125, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 85, 0, 268, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8190, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 2971, 8178, 0, 8190, 8190, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 30, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 55, 0, 442, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4700, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 12, 2, 0, 0, 8190, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 2, 0, 8034, 8156, 0, 8188, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 40, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 8114, 8192, 0, 8191, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 7501, 8186, 0, 8192, 8167, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 483, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 6, 1, 8, 0, 8151, 0, 16, 0, 22, 0, 0, 0, 2670, 8154, 0, 8166, 8011, 0, 9, 0, 0, 5, 0, 6, 20, 1, 0, 0, 0, 0, 0, 0, 0, 93, 17, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 14, 0, 0, 8170, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 21, 8127, 0, 0, 0, 0, 0, 0, 2074, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 3585, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4891, 8192, 0, 8192, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1, 1959, 0, 0, 0, 0, 0, 0, 1, 7604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8125, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 8161, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 0, 0, 85, 0, 268, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 6208, 542, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9567, 8150, 8189, 8191, 7982, 8204, 7059, 8194] avg=8192.0000 max=9567 max_rank=0 min=7059 min_rank=6 imbalance_ratio=1.167847 max_load_reduction=55969 max_load_reduction_pct=0.854019 moved_to_echo=31439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 1961, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7642, 0, 0, 0, 0, 23, 0, 15, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 6215, 543, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9999, 8210, 8177, 8192, 7681, 8204, 6886, 8187] avg=8192.0000 max=9999 max_rank=0 min=6886 min_rank=6 imbalance_ratio=1.220581 max_load_reduction=55537 max_load_reduction_pct=0.847427 moved_to_echo=30958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1955, 0, 0, 0, 0, 0, 2, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7647, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8156, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6193, 543, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 40, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9991, 8201, 8174, 8212, 7806, 8167, 6748, 8237] avg=8192.0000 max=9991 max_rank=0 min=6748 min_rank=6 imbalance_ratio=1.219604 max_load_reduction=55545 max_load_reduction_pct=0.847549 moved_to_echo=30958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 1, 8, 0, 1952, 0, 16, 0, 22, 0, 0, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8011, 0, 9, 0, 0, 5, 0, 6, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 17, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 8136, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8169, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7588, 0, 0, 0, 0, 0, 0, 2074, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8154, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 3585, 0, 0, 0, 4, 0, 0, 0, 0, 6183, 539, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4675, 8073, 8270, 8186, 9690, 8167, 10329, 8146] avg=8192.0000 max=10329 max_rank=6 min=4675 min_rank=0 imbalance_ratio=1.260864 max_load_reduction=55207 max_load_reduction_pct=0.842392 moved_to_echo=36332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1945, 0, 0, 0, 0, 0, 0, 0, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8190, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 8163, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7644, 0, 0, 1, 0, 55, 0, 442, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8178, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4700, 2, 4, 0, 0, 0, 0, 0, 0, 6199, 547, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 12, 2, 0, 0, 8182, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4916, 8239, 8189, 8181, 8144, 8191, 11468, 8208] avg=8192.0000 max=11468 max_rank=6 min=4916 min_rank=0 imbalance_ratio=1.399902 max_load_reduction=54068 max_load_reduction_pct=0.825012 moved_to_echo=36011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1961, 0, 0, 0, 0, 0, 1, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7647, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6214, 543, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10076, 8204, 8178, 8191, 7696, 8205, 6798, 8188] avg=8192.0000 max=10076 max_rank=0 min=6798 min_rank=6 imbalance_ratio=1.229980 max_load_reduction=55460 max_load_reduction_pct=0.846252 moved_to_echo=30887 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1959, 0, 0, 0, 0, 0, 0, 0, 7501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8167, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7649, 0, 0, 0, 0, 1, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 483, 0, 0, 3, 0, 0, 0, 0, 0, 6208, 543, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9460, 8250, 8181, 8192, 7826, 8196, 7244, 8187] avg=8192.0000 max=9460 max_rank=0 min=7244 min_rank=6 imbalance_ratio=1.154785 max_load_reduction=56076 max_load_reduction_pct=0.855652 moved_to_echo=31453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1961, 0, 0, 0, 0, 0, 0, 0, 4891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7648, 0, 0, 0, 0, 2, 0, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 6215, 543, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6852, 8209, 8178, 8191, 8711, 8202, 9004, 8189] avg=8192.0000 max=9004 max_rank=6 min=6852 min_rank=0 imbalance_ratio=1.099121 max_load_reduction=56532 max_load_reduction_pct=0.862610 moved_to_echo=34106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 0, 256, 0, 128, 13824, 128, 128, 128, 128, 0, 0, 0, 0, 47872, 3840, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 17536, 0, 128, 0, 128, 0, 128, 128, 48000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 61824, 0, 0, 0, 0, 384, 0, 3456, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 65280, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 128, 15744, 0, 128, 0, 128, 0, 128, 128, 49920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 256, 0, 128, 11392, 128, 128, 128, 128, 0, 0, 0, 0, 49664, 4352, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 61184, 0, 0, 128, 0, 384, 0, 4096, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 65280, 0, 128, 0, 128, 128, 0, 128, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 128, 0, 0, 65536, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 65536, 128, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 8129, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8189, 0, 40, 8029, 0, 0, 0, 1, 0, 2, 0, 84, 0, 0, 0, 4, 0, 77, 0, 1, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 0, 0, 1, 0, 8173, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 8115, 81, 37, 8033, 0, 0, 0, 0, 0, 2, 0, 165, 0, 0, 0, 0, 0, 141, 0, 68, 0, 0, 5, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8141, 0, 13, 8006, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 1, 0, 3, 2, 0, 0, 98, 0, 0, 36, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 62, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 58, 0, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7645, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8164, 0, 4, 0, 15, 0, 0, 0, 8167, 0, 0, 11, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 6, 1, 8138, 1, 0, 0, 0, 0, 0, 8162, 0, 0, 7909, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2, 0, 39, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 37, 44, 0, 3, 13, 0, 0, 8031, 0, 2, 0, 16, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 15, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 6, 3, 0, 8177, 0, 7, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 1, 0, 0, 71, 432, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 8187, 0, 1, 8170, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 8167, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 13, 8006, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 2, 0, 0, 98, 0, 0, 36, 0, 0, 7898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 8096, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8207, 8288, 8038, 8197, 8220, 8190, 8213] avg=8192.0000 max=8288 max_rank=2 min=8038 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 1, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 145, 81, 37, 8033, 0, 0, 0, 0, 0, 2, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 68, 0, 0, 5, 0, 0, 0, 54, 0, 0, 7872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8060, 8466, 8140, 8193, 8129, 8197, 8187] avg=8192.0000 max=8466 max_rank=2 min=8060 min_rank=1 imbalance_ratio=1.033447 max_load_reduction=57070 max_load_reduction_pct=0.870819 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 53, 0, 0, 2, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 40, 8029, 0, 0, 0, 1, 0, 2, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 77, 0, 1, 0, 0, 20, 2, 0, 0, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 0, 0, 1, 0, 8173, 0, 0, 1, 1, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8164, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8100, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8210, 8285, 8053, 8197, 8208, 8202, 8197] avg=8192.0000 max=8285 max_rank=2 min=8053 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 143, 0, 7, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 1, 0, 0, 71, 432, 0, 0, 1, 0, 0, 7933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8093, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8188, 7987, 8481, 8194, 8211, 8195, 8191] avg=8192.0000 max=8481 max_rank=3 min=7987 min_rank=2 imbalance_ratio=1.035278 max_load_reduction=57055 max_load_reduction_pct=0.870590 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 58, 0, 4, 0, 15, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 6, 1, 30, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 7909, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 39, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 21, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 37, 44, 0, 3, 13, 0, 0, 8031, 0, 2, 0, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8108, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 15, 0, 0, 0, 0, 0, 0, 8071, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8225, 8063, 8323, 8175, 8162, 8159, 8185] avg=8192.0000 max=8323 max_rank=3 min=8063 min_rank=2 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 144, 0, 1, 8170, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 8167, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 8096, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8222, 8329, 7978, 8194, 8181, 8199, 8189] avg=8192.0000 max=8329 max_rank=2 min=7978 min_rank=3 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 1, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 62, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8202, 8324, 8029, 8193, 8203, 8197, 8187] avg=8192.0000 max=8324 max_rank=2 min=8029 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 7645, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 546, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8222, 7794, 8494, 8193, 8222, 8197, 8187] avg=8192.0000 max=8494 max_rank=3 min=7794 min_rank=2 imbalance_ratio=1.036865 max_load_reduction=57042 max_load_reduction_pct=0.870392 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8137, 0, 12, 8004, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 1, 0, 4, 2, 0, 0, 98, 0, 0, 39, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 8187, 0, 0, 8171, 1, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 37, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8164, 0, 4, 0, 14, 0, 0, 0, 8165, 0, 0, 10, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 7, 1, 8140, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 7893, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 3, 0, 40, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 37, 39, 0, 3, 14, 0, 0, 8036, 0, 2, 0, 16, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 15, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 8109, 82, 37, 8030, 0, 0, 0, 0, 0, 3, 0, 172, 0, 0, 0, 0, 0, 144, 0, 69, 0, 0, 5, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 64, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 57, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 5, 4, 0, 8177, 0, 7, 7800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 77, 456, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 8128, 0, 0, 1, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8189, 0, 40, 8022, 0, 0, 0, 1, 0, 2, 0, 99, 0, 0, 0, 4, 0, 79, 0, 1, 0, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 12, 0, 0, 0, 1, 0, 8174, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7626, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 12, 8004, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 2, 0, 0, 98, 0, 0, 39, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 8095, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8208, 8293, 8029, 8197, 8224, 8189, 8213] avg=8192.0000 max=8293 max_rank=2 min=8029 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 4, 0, 14, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 7, 1, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 7893, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 40, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 21, 7909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 37, 39, 0, 3, 14, 0, 0, 8036, 0, 2, 0, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8109, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 15, 0, 0, 0, 0, 0, 0, 8069, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8225, 8053, 8332, 8176, 8164, 8161, 8183] avg=8192.0000 max=8332 max_rank=3 min=8053 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 150, 0, 0, 8171, 1, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 8095, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8224, 8335, 7974, 8193, 8177, 8198, 8189] avg=8192.0000 max=8335 max_rank=2 min=7974 min_rank=3 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 2, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 149, 82, 37, 8030, 0, 0, 0, 0, 0, 3, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 69, 0, 0, 5, 0, 0, 0, 57, 0, 0, 7858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8060, 8476, 8133, 8193, 8125, 8196, 8188] avg=8192.0000 max=8476 max_rank=2 min=8060 min_rank=1 imbalance_ratio=1.034668 max_load_reduction=57060 max_load_reduction_pct=0.870667 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 1, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 64, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8204, 8331, 8020, 8193, 8203, 8196, 8188] avg=8192.0000 max=8331 max_rank=2 min=8020 min_rank=3 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 53, 0, 0, 2, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 40, 8022, 0, 0, 0, 1, 0, 2, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 79, 0, 1, 0, 0, 18, 2, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 12, 0, 0, 0, 1, 0, 8174, 0, 0, 1, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8165, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8097, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8206, 8300, 8044, 8195, 8207, 8206, 8195] avg=8192.0000 max=8300 max_rank=2 min=8044 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 150, 0, 7, 7800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 77, 456, 0, 0, 1, 0, 0, 7924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8092, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8185, 7966, 8501, 8196, 8212, 8194, 8192] avg=8192.0000 max=8501 max_rank=3 min=7966 min_rank=2 imbalance_ratio=1.037720 max_load_reduction=57035 max_load_reduction_pct=0.870285 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 7626, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 0, 0, 7938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8224, 7782, 8503, 8193, 8224, 8196, 8188] avg=8192.0000 max=8503 max_rank=3 min=7782 min_rank=2 imbalance_ratio=1.037964 max_load_reduction=57033 max_load_reduction_pct=0.870255 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 384, 0, 128, 128, 0, 128, 1536, 0, 0, 128, 0, 128, 63488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 1152, 256, 128, 63744, 0, 0, 0, 128, 0, 128, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 65280, 0, 0, 128, 0, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 384, 0, 128, 128, 0, 128, 1536, 0, 0, 128, 0, 128, 63360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 64768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 65280, 0, 0, 128, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 512, 0, 128, 128, 128, 0, 0, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 0, 128, 128, 0, 0, 0, 64768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 1280, 256, 128, 63744, 128, 0, 0, 128, 0, 128, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 128, 256, 0, 128, 128, 128, 0, 65280, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 82, 5, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 6857, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1152, 0, 120, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 6770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 0, 27, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 18, 2, 0, 0, 1, 8190, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 152, 6094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2092, 0, 30, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8071, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 31] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 72, 7597, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 12, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 7222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 30, 7164, 0, 0, 0, 0, 3, 0, 0, 2, 0, 1, 0, 0, 927, 0, 20, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 143] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8145, 1, 0, 0, 29, 0, 0, 0, 0, 41, 0, 0, 6744, 0, 35, 0, 295, 0, 0, 8, 0, 0, 15, 0, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7, 22, 8, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 15, 2, 0, 0, 0, 8102, 0, 0, 0, 8157, 0, 5, 0, 0, 0, 0, 0, 0, 82, 8170, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 7796, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 320, 0, 22, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 18, 2, 0, 0, 1, 8171, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 6094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2092, 0, 30, 0, 0, 0, 0, 0, 7139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 3, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 44, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 31, 8119, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7267, 9261, 8194, 8133, 8183, 8147, 8157] avg=8192.0000 max=9261 max_rank=2 min=7267 min_rank=1 imbalance_ratio=1.130493 max_load_reduction=56275 max_load_reduction_pct=0.858688 moved_to_echo=32620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 72, 7597, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1007, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 12, 0, 0, 0, 0, 0, 7141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 2, 0, 0, 0, 44, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8118, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8693, 7698, 8187, 8225, 8187, 8231, 8142] avg=8192.0000 max=8693 max_rank=1 min=7698 min_rank=2 imbalance_ratio=1.061157 max_load_reduction=56843 max_load_reduction_pct=0.867355 moved_to_echo=32785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 82, 5, 0, 4, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 6857, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1152, 0, 120, 0, 0, 0, 0, 0, 7141, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 44, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8119, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8097, 8417, 8187, 8110, 8177, 8153, 8129] avg=8192.0000 max=8417 max_rank=2 min=8097 min_rank=1 imbalance_ratio=1.027466 max_load_reduction=57119 max_load_reduction_pct=0.871567 moved_to_echo=32569 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 6770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 0, 27, 0, 0, 0, 0, 0, 7140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 44, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 8119, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 7834, 8590, 8185, 8158, 8184, 8188, 8171] avg=8192.0000 max=8590 max_rank=2 min=7834 min_rank=1 imbalance_ratio=1.048584 max_load_reduction=56946 max_load_reduction_pct=0.868927 moved_to_echo=32701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 30, 7164, 0, 0, 0, 0, 3, 0, 0, 2, 0, 1012, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 927, 0, 20, 0, 0, 0, 0, 0, 7127, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8181, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 51, 23, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 143, 8117, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8234, 8076, 8189, 8244, 8185, 8171, 8268] avg=8192.0000 max=8268 max_rank=7 min=8076 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32868 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8126, 1, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 6744, 0, 35, 0, 295, 0, 0, 8, 0, 0, 1003, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 7106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7, 22, 8, 0, 5, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 15, 2, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 82, 8163, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 26, 44, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 8030, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8140, 8163, 8211, 8243, 8253, 8236, 8133] avg=8192.0000 max=8253 max_rank=5 min=8133 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 7796, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1007, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 22, 0, 0, 0, 0, 0, 7140, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 44, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 8119, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8873, 7484, 8193, 8224, 8183, 8254, 8152] avg=8192.0000 max=8873 max_rank=1 min=7484 min_rank=2 imbalance_ratio=1.083130 max_load_reduction=56663 max_load_reduction_pct=0.864609 moved_to_echo=32813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 7222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 0, 1, 0, 0, 0, 0, 0, 7141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 44, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 8119, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8398, 7847, 8190, 8199, 8184, 8156, 8384] avg=8192.0000 max=8398 max_rank=1 min=7847 min_rank=2 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 7795, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 329, 0, 20, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 27] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 2, 0, 0, 1, 8190, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 156, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2014, 0, 34, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8071, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8145, 1, 0, 0, 28, 0, 0, 0, 0, 42, 0, 0, 6703, 0, 35, 0, 291, 0, 0, 9, 0, 0, 14, 0, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7, 22, 8, 0, 8, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 17, 2, 0, 0, 0, 8105, 0, 0, 0, 8157, 0, 5, 0, 0, 0, 0, 0, 0, 80, 8170, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 85, 6, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 6841, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1150, 0, 123, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 8050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 79, 7693, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 12, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 37, 7173, 0, 0, 0, 0, 3, 0, 0, 3, 0, 1, 0, 0, 918, 0, 21, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 147] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 6831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 0, 26, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 7247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8126, 1, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 6703, 0, 35, 0, 291, 0, 0, 9, 0, 0, 970, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 7135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7, 22, 8, 0, 8, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 17, 2, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 80, 8164, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 25, 47, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 8033, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8064, 8243, 8214, 8244, 8251, 8234, 8129] avg=8192.0000 max=8251 max_rank=5 min=8064 min_rank=1 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 2, 0, 0, 1, 8171, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2014, 0, 34, 0, 0, 0, 0, 0, 7167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 3, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 48, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 32, 8119, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7317, 9215, 8192, 8132, 8184, 8147, 8159] avg=8192.0000 max=9215 max_rank=2 min=7317 min_rank=1 imbalance_ratio=1.124878 max_load_reduction=56321 max_load_reduction_pct=0.859390 moved_to_echo=32622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 7795, 0, 0, 0, 0, 0, 0, 0, 1, 0, 975, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 20, 0, 0, 0, 0, 0, 7168, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 48, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 27, 8119, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8836, 7518, 8193, 8223, 8184, 8257, 8151] avg=8192.0000 max=8836 max_rank=1 min=7518 min_rank=2 imbalance_ratio=1.078613 max_load_reduction=56700 max_load_reduction_pct=0.865173 moved_to_echo=32815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 85, 6, 0, 5, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 6841, 16, 0, 0, 0, 0, 0, 0, 0, 0, 975, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1150, 0, 123, 0, 0, 0, 0, 0, 7169, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 48, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8119, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8059, 8446, 8190, 8108, 8174, 8158, 8130] avg=8192.0000 max=8446 max_rank=2 min=8059 min_rank=1 imbalance_ratio=1.031006 max_load_reduction=57090 max_load_reduction_pct=0.871124 moved_to_echo=32570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 79, 7693, 0, 0, 0, 1, 0, 0, 0, 0, 0, 975, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 12, 0, 0, 0, 0, 0, 7169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 3, 0, 0, 0, 48, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8118, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8766, 7625, 8185, 8229, 8187, 8233, 8138] avg=8192.0000 max=8766 max_rank=1 min=7625 min_rank=2 imbalance_ratio=1.070068 max_load_reduction=56770 max_load_reduction_pct=0.866241 moved_to_echo=32787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 37, 7173, 0, 0, 0, 0, 3, 0, 0, 3, 0, 977, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 918, 0, 21, 0, 0, 0, 0, 0, 7163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8182, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 50, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 147, 8120, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8213, 8103, 8188, 8244, 8186, 8158, 8277] avg=8192.0000 max=8277 max_rank=7 min=8103 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 6831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 0, 26, 0, 0, 0, 0, 0, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 48, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 8119, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 7866, 8556, 8185, 8156, 8185, 8194, 8168] avg=8192.0000 max=8556 max_rank=2 min=7866 min_rank=1 imbalance_ratio=1.044434 max_load_reduction=56980 max_load_reduction_pct=0.869446 moved_to_echo=32703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 7247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 1, 0, 0, 0, 0, 0, 7169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 48, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 8119, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8415, 7830, 8189, 8200, 8185, 8155, 8384] avg=8192.0000 max=8415 max_rank=1 min=7830 min_rank=2 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=32924 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 8320, 0, 256, 0, 0, 0, 0, 0, 57088, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 768, 64896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 0, 768, 56320, 128, 128, 0, 384, 128, 0, 128, 128, 0, 8064, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 0, 128, 256, 128, 0, 128, 128, 65408, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 65024, 0, 0, 0, 0, 128, 0, 0, 128, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 8064, 0, 256, 0, 0, 0, 0, 0, 57344, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 0, 768, 56576, 128, 128, 0, 384, 128, 0, 128, 128, 0, 7808, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 640, 64896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65152, 0, 0, 128, 0, 128, 128, 128, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 65024, 0, 0, 0, 0, 128, 0, 0, 128, 512, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 65536, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 128, 128, 256, 128, 0, 128, 128, 65408, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 8190, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 1377, 8192, 0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8191, 1, 0, 0, 6498, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 8189, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 7882, 4, 0, 0, 1396, 8192, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 5535, 0, 0, 8183, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8192, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7953, 1, 0, 0, 532, 8186, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 4499, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 8192, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 161, 8191, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6895, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 8190, 0, 60, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 1, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 1, 29, 8190, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8187, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7564, 0, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 8192, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 160, 8192, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8192, 0, 0, 0, 6606, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 11, 5, 0, 40, 0, 12, 0, 4, 0, 9, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7791, 0, 0, 15, 0, 8168, 0, 0, 8093, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 7308, 21, 0, 0, 0, 834, 0, 0, 0, 0, 0, 422, 1, 0, 0, 8142, 0, 9, 0, 8178, 0, 0, 8163, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 757, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6872, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 8025, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 6619, 415, 165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7742, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1557, 1, 0, 0, 6498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8069, 7511, 8202, 8194, 9120, 8191, 8060] avg=8192.0000 max=9120 max_rank=5 min=7511 min_rank=2 imbalance_ratio=1.113281 max_load_reduction=56416 max_load_reduction_pct=0.860840 moved_to_echo=33565 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 8024, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 6619, 401, 165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7481, 4, 0, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 0, 0, 0, 5535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8559, 8278, 8147, 8207, 8881, 8192, 7092] avg=8192.0000 max=8881 max_rank=5 min=7092 min_rank=7 imbalance_ratio=1.084106 max_load_reduction=56655 max_load_reduction_pct=0.864487 moved_to_echo=32372 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8027, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 3161, 0, 0, 0, 0, 0, 0, 6620, 405, 165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7548, 1, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1557, 0, 0, 0, 4499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8274, 10371, 8185, 8188, 8081, 8191, 6057] avg=8192.0000 max=10371 max_rank=2 min=6057 min_rank=7 imbalance_ratio=1.265991 max_load_reduction=55165 max_load_reduction_pct=0.841751 moved_to_echo=30517 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 8027, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1073, 0, 0, 0, 0, 0, 0, 6620, 415, 165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7742, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 0, 0, 0, 6895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8143, 8276, 8191, 8191, 7903, 8191, 8452] avg=8192.0000 max=8452 max_rank=7 min=7903 min_rank=5 imbalance_ratio=1.031738 max_load_reduction=57084 max_load_reduction_pct=0.871033 moved_to_echo=32737 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 8023, 0, 60, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 1, 1, 0, 0, 0, 6623, 418, 167, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 8189, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7700, 0, 0, 1, 29, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8184, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1548, 0, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8262, 7632, 8214, 8193, 7737, 8197, 9112] avg=8192.0000 max=9112 max_rank=7 min=7632 min_rank=2 imbalance_ratio=1.112305 max_load_reduction=56424 max_load_reduction_pct=0.860962 moved_to_echo=33239 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 60, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 6580, 397, 164, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 5, 0, 40, 0, 12, 0, 4, 0, 9, 0, 3, 1, 0, 0, 0, 8093, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7394, 0, 0, 15, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 7307, 21, 0, 0, 0, 834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 1, 0, 0, 1547, 0, 9, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 7998, 7159, 8194, 8178, 7436, 8193, 10157] avg=8192.0000 max=10157 max_rank=7 min=7159 min_rank=2 imbalance_ratio=1.239868 max_load_reduction=55379 max_load_reduction_pct=0.845016 moved_to_echo=33964 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 8027, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 0, 0, 0, 0, 0, 0, 6620, 417, 165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7770, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1557, 0, 0, 0, 6606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8159, 8504, 8194, 8192, 7930, 8190, 8177] avg=8192.0000 max=8504 max_rank=2 min=7930 min_rank=5 imbalance_ratio=1.038086 max_load_reduction=57032 max_load_reduction_pct=0.870239 moved_to_echo=32489 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 8027, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 6620, 413, 165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7691, 0, 0, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 0, 0, 0, 6872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8072, 7805, 8209, 8193, 8448, 8191, 8429] avg=8192.0000 max=8448 max_rank=5 min=7805 min_rank=2 imbalance_ratio=1.031250 max_load_reduction=57088 max_load_reduction_pct=0.871094 moved_to_echo=33261 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 23, 8190, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 1412, 8192, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8191, 1, 0, 0, 6429, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 12, 4, 0, 40, 0, 12, 0, 4, 0, 7, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 15, 1, 8168, 0, 0, 8092, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 7317, 22, 0, 0, 0, 819, 0, 0, 0, 0, 0, 435, 1, 0, 0, 8143, 0, 9, 0, 8178, 0, 0, 8163, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8192, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 603, 8183, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 3880, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 8192, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 733, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6795, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 8189, 1, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 7882, 5, 0, 0, 1380, 8192, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 5326, 0, 0, 8183, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 8190, 0, 72, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 1, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 1, 27, 8189, 0, 0, 8188, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8187, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7522, 0, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8192, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 170, 8191, 0, 0, 8173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6777, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 157, 8192, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8192, 0, 0, 0, 6422, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 6417, 403, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4, 0, 40, 0, 12, 0, 4, 0, 7, 0, 3, 1, 0, 0, 0, 8092, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7382, 0, 0, 15, 1, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 7317, 22, 0, 0, 0, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 1, 0, 0, 1710, 0, 9, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 7998, 7001, 8192, 8178, 7425, 8189, 10333] avg=8192.0000 max=10333 max_rank=7 min=7001 min_rank=2 imbalance_ratio=1.261353 max_load_reduction=55203 max_load_reduction_pct=0.842331 moved_to_echo=34125 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8027, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 0, 0, 0, 6456, 411, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 0, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1720, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8289, 10762, 8184, 8185, 8134, 8192, 5601] avg=8192.0000 max=10762 max_rank=2 min=5601 min_rank=7 imbalance_ratio=1.313721 max_load_reduction=54774 max_load_reduction_pct=0.835785 moved_to_echo=30112 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 23, 8025, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 0, 6455, 422, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7732, 0, 0, 0, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1720, 1, 0, 0, 6429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8076, 7386, 8201, 8194, 9145, 8191, 8154] avg=8192.0000 max=9145 max_rank=5 min=7386 min_rank=2 imbalance_ratio=1.116333 max_load_reduction=56391 max_load_reduction_pct=0.860458 moved_to_echo=33684 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 8024, 1, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 0, 0, 6455, 408, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7474, 5, 0, 0, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 0, 0, 0, 5326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8558, 8345, 8148, 8207, 8859, 8192, 7046] avg=8192.0000 max=8859 max_rank=5 min=7046 min_rank=7 imbalance_ratio=1.081421 max_load_reduction=56677 max_load_reduction_pct=0.864822 moved_to_echo=32304 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 8027, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 0, 0, 0, 0, 6456, 419, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7686, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 0, 0, 0, 6795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8074, 7744, 8209, 8194, 8419, 8192, 8515] avg=8192.0000 max=8515 max_rank=7 min=7744 min_rank=2 imbalance_ratio=1.039429 max_load_reduction=57021 max_load_reduction_pct=0.870071 moved_to_echo=33320 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 8024, 0, 72, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 1, 1, 0, 0, 0, 6456, 423, 166, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 8188, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7692, 0, 0, 1, 27, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8185, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1712, 0, 0, 0, 7522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8276, 7505, 8216, 8193, 7727, 8197, 9234] avg=8192.0000 max=9234 max_rank=7 min=7505 min_rank=2 imbalance_ratio=1.127197 max_load_reduction=56302 max_load_reduction_pct=0.859100 moved_to_echo=33351 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8027, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1199, 0, 0, 0, 0, 0, 0, 6456, 422, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 0, 0, 0, 170, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 0, 0, 0, 6777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8125, 8244, 8190, 8192, 7906, 8192, 8497] avg=8192.0000 max=8497 max_rank=7 min=7906 min_rank=5 imbalance_ratio=1.037231 max_load_reduction=57039 max_load_reduction_pct=0.870346 moved_to_echo=32787 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 8027, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1505, 0, 0, 0, 0, 0, 0, 6456, 423, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7764, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1720, 0, 0, 0, 6422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8140, 8549, 8196, 8193, 7921, 8191, 8156] avg=8192.0000 max=8549 max_rank=2 min=7921 min_rank=5 imbalance_ratio=1.043579 max_load_reduction=56987 max_load_reduction_pct=0.869553 moved_to_echo=32461 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 0, 0, 8064, 0, 128, 128, 0, 0, 128, 52992, 3328, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 512, 128, 0, 0, 12544, 128, 128, 0, 52736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 61184, 128, 0, 128, 4480, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 896, 64256, 128, 512, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 128, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 0, 64640, 128, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 0, 0, 9344, 0, 128, 128, 0, 0, 128, 51712, 3456, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 512, 128, 0, 0, 13824, 128, 128, 0, 51456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 61056, 128, 0, 128, 4608, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 128, 65408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 64768, 128, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 896, 64256, 128, 512, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 18, 8172, 0, 0, 1594, 0, 0, 0, 0, 0, 61, 0, 0, 0, 8166, 0, 0, 0, 0, 62, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 420, 0, 0, 0, 0, 0, 0, 12, 0, 7780, 125, 0, 7210, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8106, 0, 6, 0, 16, 0, 0, 0, 5402, 0, 0, 0, 0, 0, 4, 0, 0, 8137, 0, 0, 0, 17, 0, 0, 0, 4, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 25, 0, 3, 8136, 0, 0, 0, 0, 1396, 0, 0, 20, 12, 0, 0, 6, 0, 0, 546, 0, 0, 0, 17, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[38, 0, 8057, 0, 0, 4239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 7700, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 535, 0, 224, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[33, 0, 8139, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7664, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 18, 8192, 0, 0, 0, 0, 0, 0, 0, 5714, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 198, 0, 360, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4435, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[45, 1, 8175, 0, 0, 6456, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8130, 0, 1, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 5216, 0, 0, 0, 1, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 77, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[7, 0, 8190, 0, 0, 4383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 0, 0, 0, 7186, 0, 0, 0, 0, 0, 0, 0, 0, 3373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 593, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[5, 0, 8191, 1, 0, 4764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 8, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 5047, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 297, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 1, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 8168, 0, 0, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6971, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 34, 0, 0, 3, 0, 0, 0, 0, 7894, 5, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 1, 6946, 0, 0, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 513, 0, 199, 0, 0, 0, 0, 0, 0, 0, 8, 1, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7846, 5, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 642, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5748, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[38, 0, 0, 0, 0, 4239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8057, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 7700, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 0, 0, 0, 0, 0, 0, 0, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1795, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7536, 0, 0, 535, 0, 224, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6210, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9984, 8079, 7979, 8339, 10014, 4720, 8301, 8120] avg=8192.0000 max=10014 max_rank=4 min=4720 min_rank=5 imbalance_ratio=1.222412 max_load_reduction=55522 max_load_reduction_pct=0.847198 moved_to_echo=31155 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[45, 1, 0, 0, 0, 6456, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8130, 0, 1, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 0, 0, 0, 0, 0, 0, 0, 5216, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7536, 0, 0, 77, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6209, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9992, 8197, 8353, 8305, 7177, 8490, 7638, 7384] avg=8192.0000 max=9992 max_rank=0 min=7177 min_rank=4 imbalance_ratio=1.219727 max_load_reduction=55544 max_load_reduction_pct=0.847534 moved_to_echo=30689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 1, 0, 4764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 8, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5047, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1550, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7536, 0, 0, 297, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 1, 0, 0, 0, 1, 0, 0, 6209, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9700, 8213, 8259, 8330, 9994, 6890, 7900, 6250] avg=8192.0000 max=9994 max_rank=4 min=6250 min_rank=7 imbalance_ratio=1.219971 max_load_reduction=55542 max_load_reduction_pct=0.847504 moved_to_echo=31034 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7664, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 18, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 0, 0, 0, 0, 0, 0, 0, 5714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7536, 0, 0, 198, 0, 360, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6210, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4880, 8161, 7882, 8357, 7674, 9840, 8095, 10647] avg=8192.0000 max=10647 max_rank=7 min=4880 min_rank=0 imbalance_ratio=1.299683 max_load_reduction=54889 max_load_reduction_pct=0.837540 moved_to_echo=36256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 0, 1594, 0, 0, 0, 0, 0, 61, 0, 0, 0, 6210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 420, 0, 0, 0, 0, 0, 0, 12, 0, 7780, 125, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7210, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1940, 0, 6, 0, 16, 0, 0, 0, 5402, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 17, 0, 0, 0, 4, 0, 0, 0, 0, 10, 0, 1953, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 3, 7485, 0, 0, 0, 0, 1396, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6, 0, 0, 546, 0, 0, 0, 17, 0, 0, 0, 0, 0, 6145, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7883, 8266, 8554, 7397, 7368, 10410, 8929, 6729] avg=8192.0000 max=10410 max_rank=5 min=6729 min_rank=7 imbalance_ratio=1.270752 max_load_reduction=55126 max_load_reduction_pct=0.841156 moved_to_echo=33436 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8168, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 34, 0, 0, 3, 0, 0, 0, 0, 7894, 5, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1953, 0, 1, 0, 0, 0, 0, 1, 6946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1670, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 0, 0, 0, 0, 7525, 0, 0, 513, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6212, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7457, 8194, 8156, 8240, 8901, 9595, 8246, 6747] avg=8192.0000 max=9595 max_rank=5 min=6747 min_rank=7 imbalance_ratio=1.171265 max_load_reduction=55941 max_load_reduction_pct=0.853592 moved_to_echo=33489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 4383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 0, 0, 0, 0, 0, 0, 0, 7186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1859, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7536, 0, 0, 593, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6210, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10300, 8212, 8283, 8233, 9146, 5523, 8140, 7699] avg=8192.0000 max=10300 max_rank=0 min=5523 min_rank=5 imbalance_ratio=1.257324 max_load_reduction=55236 max_load_reduction_pct=0.842834 moved_to_echo=30508 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7846, 5, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 0, 0, 0, 0, 0, 0, 0, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7536, 0, 0, 642, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6210, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5340, 8214, 8070, 8335, 5262, 10068, 8287, 11960] avg=8192.0000 max=11960 max_rank=7 min=5262 min_rank=4 imbalance_ratio=1.459961 max_load_reduction=53576 max_load_reduction_pct=0.817505 moved_to_echo=35577 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[44, 1, 8176, 0, 0, 5937, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8126, 0, 2, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 5436, 0, 0, 0, 1, 0, 0, 0, 0, 7046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 79, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[36, 0, 8046, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 1, 0, 0, 7663, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 2398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 517, 0, 252, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[5, 0, 8191, 0, 0, 4534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1, 0, 0, 0, 0, 0, 0, 8007, 9, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 4600, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 296, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 1, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 16, 8172, 0, 0, 1365, 0, 0, 0, 0, 0, 60, 0, 0, 0, 8167, 0, 0, 0, 0, 64, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 12, 0, 7764, 118, 0, 7176, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8103, 0, 6, 0, 16, 0, 0, 0, 5514, 0, 0, 0, 0, 0, 4, 0, 0, 8133, 0, 0, 0, 17, 0, 0, 0, 3, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 29, 0, 3, 8137, 0, 0, 0, 0, 1432, 0, 0, 19, 12, 0, 0, 7, 0, 0, 726, 0, 0, 0, 19, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[35, 0, 8134, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7615, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 19, 8192, 0, 0, 0, 0, 0, 0, 0, 5540, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 192, 0, 433, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5091, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 8167, 0, 0, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7369, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 33, 0, 0, 3, 0, 0, 0, 0, 7865, 6, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 1, 7155, 0, 0, 0, 0, 0, 0, 0, 0, 7538, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 517, 0, 210, 0, 0, 0, 0, 0, 0, 0, 8, 1, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 3, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3617, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 613, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5789, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[8, 0, 8190, 0, 0, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 7372, 0, 0, 0, 0, 0, 0, 0, 0, 3418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 562, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[36, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 1, 0, 0, 7663, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 468, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7519, 0, 0, 517, 0, 252, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9995, 8069, 7981, 8344, 9916, 4505, 8295, 8431] avg=8192.0000 max=9995 max_rank=0 min=4505 min_rank=5 imbalance_ratio=1.220093 max_load_reduction=55541 max_load_reduction_pct=0.847488 moved_to_echo=31147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[44, 1, 0, 0, 0, 5937, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8126, 0, 2, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 5436, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 468, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 79, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9565, 8199, 8385, 8310, 7273, 8610, 7624, 7570] avg=8192.0000 max=9565 max_rank=0 min=7273 min_rank=4 imbalance_ratio=1.167603 max_load_reduction=55971 max_load_reduction_pct=0.854050 moved_to_echo=31077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 4534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1, 0, 0, 0, 0, 0, 0, 8007, 9, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4600, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1290, 468, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 296, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10113, 8214, 8287, 8335, 9972, 6627, 7937, 6051] avg=8192.0000 max=10113 max_rank=0 min=6051 min_rank=7 imbalance_ratio=1.234497 max_load_reduction=55423 max_load_reduction_pct=0.845688 moved_to_echo=30587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 0, 1365, 0, 0, 0, 0, 0, 60, 0, 0, 0, 6632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 12, 0, 7764, 118, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7176, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1816, 0, 6, 0, 16, 0, 0, 0, 5514, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 17, 0, 0, 0, 3, 0, 0, 0, 0, 10, 0, 1535, 463, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 3, 7469, 0, 0, 0, 0, 1432, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 7, 0, 0, 726, 0, 0, 0, 19, 0, 0, 0, 0, 0, 5801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8270, 8526, 7367, 7356, 10427, 8952, 6565] avg=8192.0000 max=10427 max_rank=5 min=6565 min_rank=7 imbalance_ratio=1.272827 max_load_reduction=55109 max_load_reduction_pct=0.840897 moved_to_echo=33300 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7615, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 19, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 5540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 468, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 192, 0, 433, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4738, 8157, 7868, 8363, 7376, 9932, 8146, 10956] avg=8192.0000 max=10956 max_rank=7 min=4738 min_rank=0 imbalance_ratio=1.337402 max_load_reduction=54580 max_load_reduction_pct=0.832825 moved_to_echo=36410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8167, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 33, 0, 0, 3, 0, 0, 0, 0, 7865, 6, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1825, 0, 1, 0, 0, 0, 0, 1, 7155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7538, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1388, 471, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 7506, 0, 0, 517, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7629, 8199, 8164, 8241, 8982, 9673, 8241, 6407] avg=8192.0000 max=9673 max_rank=5 min=6407 min_rank=7 imbalance_ratio=1.180786 max_load_reduction=55863 max_load_reduction_pct=0.852402 moved_to_echo=33303 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 7372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 468, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 562, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9988, 8213, 8288, 8236, 9208, 5614, 8087, 7902] avg=8192.0000 max=9988 max_rank=0 min=5614 min_rank=5 imbalance_ratio=1.219238 max_load_reduction=55548 max_load_reduction_pct=0.847595 moved_to_echo=30811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 3, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 3617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 468, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 613, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5435, 8215, 8037, 8340, 5453, 10148, 8254, 11654] avg=8192.0000 max=11654 max_rank=7 min=5435 min_rank=0 imbalance_ratio=1.422607 max_load_reduction=53882 max_load_reduction_pct=0.822174 moved_to_echo=35509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 128, 128, 15872, 0, 0, 128, 128, 0, 0, 128, 0, 0, 49664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 0, 24320, 0, 0, 128, 0, 0, 128, 0, 0, 0, 41088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[15744, 0, 128, 0, 128, 0, 0, 128, 49920, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 50304, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 12928, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 60288, 0, 0, 2944, 0, 2432, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[64384, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 128, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 512, 128, 0, 0, 128, 0, 128, 128, 0, 63104, 256, 128, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 128, 128, 18688, 0, 0, 128, 128, 0, 0, 128, 0, 0, 46976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 21504, 0, 0, 128, 0, 0, 128, 0, 0, 0, 44032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 49536, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 10240, 3840, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[14720, 0, 128, 0, 128, 0, 0, 128, 50944, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 60160, 0, 0, 2816, 0, 2688, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 512, 128, 128, 0, 128, 0, 128, 128, 0, 62976, 256, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[64256, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 128, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8142, 0, 0, 0, 8189, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7803, 0, 0, 6729, 0, 0, 0, 8113, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 6663, 0, 0, 0, 8162, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 7343, 3, 0, 8138, 199, 0, 0, 8164, 0, 0, 48, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 8159, 0, 10, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 8083, 2, 0, 0, 0, 16, 8017, 6, 0, 0, 0, 654, 3, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 22, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 111, 71, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7642, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 550, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 6842, 0, 0, 0, 8170, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1461, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7460, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 7415, 2, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8163, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 799, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 779, 0, 0, 6729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7024, 82, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7509, 8119, 8183, 8234, 8207, 8118, 8318, 8848] avg=8192.0000 max=8848 max_rank=7 min=7509 min_rank=0 imbalance_ratio=1.080078 max_load_reduction=56688 max_load_reduction_pct=0.864990 moved_to_echo=33491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7374, 82, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8960, 8180, 8208, 8186, 8209, 8110, 8170, 7513] avg=8192.0000 max=8960 max_rank=0 min=7513 min_rank=7 imbalance_ratio=1.093750 max_load_reduction=56576 max_load_reduction_pct=0.863281 moved_to_echo=32002 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 805, 0, 0, 6842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7256, 82, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7648, 8161, 8203, 8189, 8193, 8149, 8183, 8810] avg=8192.0000 max=8810 max_rank=7 min=7648 min_rank=0 imbalance_ratio=1.075439 max_load_reduction=56726 max_load_reduction_pct=0.865570 moved_to_echo=33335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 6663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7370, 82, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7481, 8155, 8203, 8193, 8204, 8110, 8216, 8974] avg=8192.0000 max=8974 max_rank=7 min=7481 min_rank=0 imbalance_ratio=1.095459 max_load_reduction=56562 max_load_reduction_pct=0.863068 moved_to_echo=33504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 733, 3, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 8151, 0, 0, 48, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 23, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 10, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 17, 2, 0, 0, 0, 16, 8017, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 3, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 22, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 71, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 18, 6610, 82, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8971, 8409, 8147, 8172, 8115, 8701, 8114, 6907] avg=8192.0000 max=8971 max_rank=0 min=6907 min_rank=7 imbalance_ratio=1.095093 max_load_reduction=56565 max_load_reduction_pct=0.863113 moved_to_echo=31837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 812, 0, 0, 7415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8168, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8163, 0, 0, 0, 0, 1, 0, 0, 0, 12, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 12, 0, 0, 1, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 799, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 7360, 88, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8171, 8185, 8184, 8200, 8128, 8176, 8263] avg=8192.0000 max=8263 max_rank=7 min=8128 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 7642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7374, 82, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8460, 8163, 8203, 8189, 8206, 8110, 8187, 8018] avg=8192.0000 max=8460 max_rank=0 min=8018 min_rank=7 imbalance_ratio=1.032715 max_load_reduction=57076 max_load_reduction_pct=0.870911 moved_to_echo=32521 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 7460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7374, 82, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8178, 8204, 8189, 8202, 8110, 8172, 8203] avg=8192.0000 max=8278 max_rank=0 min=8110 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7799, 0, 0, 6626, 0, 0, 0, 8106, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 1827, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 7311, 1, 0, 8137, 193, 0, 0, 8164, 0, 0, 50, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 8159, 0, 10, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 8082, 2, 0, 0, 0, 14, 8015, 6, 0, 0, 0, 684, 4, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 27, 22, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 120, 71, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8149, 0, 0, 0, 8190, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 6812, 0, 0, 0, 8172, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1479, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 7375, 2, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8164, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7627, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 565, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 6634, 0, 0, 0, 8164, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7436, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 808, 0, 0, 6626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6991, 86, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7435, 8116, 8180, 8236, 8209, 8112, 8326, 8922] avg=8192.0000 max=8922 max_rank=7 min=7435 min_rank=0 imbalance_ratio=1.089111 max_load_reduction=56614 max_load_reduction_pct=0.863861 moved_to_echo=33569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 757, 1, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 8151, 0, 0, 50, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 23, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 10, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 19, 2, 0, 0, 0, 14, 8015, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 4, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 27, 22, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 71, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 18, 6554, 85, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8993, 8406, 8150, 8172, 8113, 8727, 8111, 6864] avg=8192.0000 max=8993 max_rank=0 min=6864 min_rank=7 imbalance_ratio=1.097778 max_load_reduction=56543 max_load_reduction_pct=0.862778 moved_to_echo=31815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7344, 86, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8997, 8182, 8208, 8185, 8210, 8106, 8166, 7482] avg=8192.0000 max=8997 max_rank=0 min=7482 min_rank=7 imbalance_ratio=1.098267 max_load_reduction=56539 max_load_reduction_pct=0.862717 moved_to_echo=31964 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 0, 6812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7237, 86, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7649, 8163, 8203, 8189, 8193, 8144, 8180, 8815] avg=8192.0000 max=8815 max_rank=7 min=7649 min_rank=0 imbalance_ratio=1.076050 max_load_reduction=56721 max_load_reduction_pct=0.865494 moved_to_echo=33332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 844, 0, 0, 7375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8166, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8164, 0, 0, 0, 0, 1, 0, 0, 0, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 7327, 87, 8, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8169, 8184, 8186, 8196, 8128, 8178, 8274] avg=8192.0000 max=8274 max_rank=7 min=8128 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 6634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7341, 86, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7482, 8157, 8204, 8192, 8203, 8107, 8223, 8968] avg=8192.0000 max=8968 max_rank=7 min=7482 min_rank=0 imbalance_ratio=1.094727 max_load_reduction=56568 max_load_reduction_pct=0.863159 moved_to_echo=33501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 7627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7344, 86, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8475, 8164, 8203, 8188, 8208, 8106, 8183, 8009] avg=8192.0000 max=8475 max_rank=0 min=8009 min_rank=7 imbalance_ratio=1.034546 max_load_reduction=57061 max_load_reduction_pct=0.870682 moved_to_echo=32506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 7436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7344, 86, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8179, 8204, 8188, 8204, 8106, 8169, 8202] avg=8192.0000 max=8284 max_rank=0 min=8106 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 7040, 128, 0, 0, 128, 128, 0, 0, 128, 0, 128, 128, 0, 128, 57856, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 6528, 128, 0, 59136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 65280, 0, 128, 128, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 0, 256, 128, 0, 128, 0, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 7168, 128, 0, 0, 128, 128, 0, 0, 128, 0, 128, 128, 0, 128, 57600, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 6656, 128, 0, 58880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 256, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[768, 128, 128, 0, 0, 0, 0, 0, 0, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 65536, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 65280, 0, 128, 128, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 65408, 0, 0, 0, 128, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 256, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 65536, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 0, 256, 128, 0, 128, 0, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 23, 0, 0, 0, 8161, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 28, 8192, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 356, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 22, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8192, 0, 0, 0, 7559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 0, 0, 95, 0, 0, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 16, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 8190, 4, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 5, 2, 1, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 8187, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 8190, 0, 0, 0, 2, 0, 112, 0, 0, 0, 9, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 44, 0, 8127, 3, 1160, 13, 7012, 0, 38, 0, 112, 18, 15, 0, 0, 0, 7972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 34, 1, 8152, 0, 1, 0, 0, 139, 0, 12, 8160, 0, 0, 0, 0, 0, 0, 4, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8018, 0, 0, 0, 0, 0, 0, 12, 19, 0, 7, 17, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 5, 8192, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 60, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 17, 0, 0, 0, 8004, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 8, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 23, 0, 0, 0, 8003, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8112, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8268, 8204, 8030, 8186, 8204, 8179, 8195] avg=8192.0000 max=8270 max_rank=0 min=8030 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7794, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 28, 77, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 356, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 8112, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 22, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7876, 8301, 8204, 8335, 8238, 8205, 8184, 8193] avg=8192.0000 max=8335 max_rank=3 min=7876 min_rank=0 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 64, 4, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 3, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 2, 1, 1, 0, 8018, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 8117, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 2, 0, 112, 0, 0, 0, 9, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8150, 8203, 8031, 8228, 8217, 8287, 8199] avg=8192.0000 max=8287 max_rank=6 min=8031 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 77, 0, 0, 0, 7559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 0, 0, 95, 0, 0, 0, 7899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 16, 8112, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 7661, 8198, 8709, 8200, 8205, 8200, 8194] avg=8192.0000 max=8709 max_rank=3 min=7661 min_rank=1 imbalance_ratio=1.063110 max_load_reduction=56827 max_load_reduction_pct=0.867111 moved_to_echo=32799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 44, 0, 76, 3, 1160, 13, 7012, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 18, 15, 0, 0, 0, 7972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 34, 1, 7994, 0, 1, 0, 0, 139, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 22, 0, 0, 0, 0, 0, 0, 0, 8048, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8018, 0, 0, 0, 0, 0, 0, 12, 19, 0, 7, 17, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8415, 8129, 8192, 8143, 8092, 8149, 8172] avg=8192.0000 max=8415 max_rank=1 min=8092 min_rank=5 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=32556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 5, 77, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 60, 0, 0, 0, 8003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8112, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8209, 8204, 8134, 8182, 8205, 8193, 8195] avg=8192.0000 max=8214 max_rank=0 min=8134 min_rank=3 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 17, 0, 0, 0, 7849, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8239, 8204, 8067, 8180, 8205, 8175, 8195] avg=8192.0000 max=8271 max_rank=0 min=8067 min_rank=3 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 0, 62, 8190, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 1, 0, 0, 8186, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 0, 62, 77, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 0, 0, 0, 0, 7787, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8110, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8293, 8190, 8038, 8179, 8203, 8169, 8193] avg=8192.0000 max=8293 max_rank=1 min=8038 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7785, 0, 0, 0, 0, 0, 0, 0, 1, 0, 111, 0, 0, 29, 8192, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 374, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 23, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 62, 8190, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 1, 0, 0, 8187, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 21, 0, 0, 0, 8163, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 5, 8192, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 58, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 46, 0, 8127, 3, 1176, 12, 6995, 0, 39, 0, 113, 18, 17, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 32, 1, 8150, 0, 1, 0, 0, 143, 0, 12, 8160, 0, 0, 0, 0, 0, 0, 6, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8020, 0, 0, 0, 0, 0, 0, 12, 19, 0, 7, 18, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8192, 0, 0, 0, 7572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 0, 0, 94, 0, 0, 0, 8054, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 16, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2, 0, 8190, 4, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 6, 5, 1, 1, 1, 8190, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 8187, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 8190, 0, 0, 0, 2, 0, 115, 0, 0, 0, 8, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 19, 0, 0, 0, 8004, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7785, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 111, 0, 0, 29, 76, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 374, 0, 0, 0, 7939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 8113, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 23, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7868, 8300, 8204, 8342, 8243, 8205, 8182, 8192] avg=8192.0000 max=8342 max_rank=3 min=7868 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 62, 76, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 7796, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8111, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 8187, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8292, 8188, 8038, 8179, 8203, 8170, 8194] avg=8192.0000 max=8292 max_rank=1 min=8038 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 7, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 21, 0, 0, 0, 8005, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8113, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8266, 8205, 8029, 8188, 8205, 8180, 8195] avg=8192.0000 max=8268 max_rank=0 min=8029 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 46, 0, 76, 3, 1176, 12, 6995, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 18, 17, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 1, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 32, 1, 7993, 0, 1, 0, 0, 143, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 21, 0, 0, 0, 0, 0, 0, 0, 8048, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8020, 0, 0, 0, 0, 0, 0, 12, 19, 0, 7, 18, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8411, 8131, 8191, 8142, 8095, 8149, 8170] avg=8192.0000 max=8411 max_rank=1 min=8095 min_rank=5 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 5, 76, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 58, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8205, 8204, 8136, 8181, 8206, 8192, 8195] avg=8192.0000 max=8217 max_rank=0 min=8136 min_rank=3 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2, 0, 67, 4, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 3, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 5, 1, 1, 1, 8023, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 8113, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 2, 0, 115, 0, 0, 0, 8, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8148, 8202, 8039, 8225, 8211, 8292, 8201] avg=8192.0000 max=8292 max_rank=6 min=8039 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8092, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 76, 0, 0, 0, 7572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 0, 0, 94, 0, 0, 0, 7898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 16, 8113, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 7673, 8198, 8696, 8199, 8205, 8197, 8194] avg=8192.0000 max=8696 max_rank=3 min=7673 min_rank=1 imbalance_ratio=1.061523 max_load_reduction=56840 max_load_reduction_pct=0.867310 moved_to_echo=32795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 19, 0, 0, 0, 7850, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8241, 8204, 8065, 8179, 8206, 8174, 8195] avg=8192.0000 max=8272 max_rank=0 min=8065 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 0, 128, 128, 640, 128, 1280, 128, 63360, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 128, 128, 64896, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[65408, 0, 0, 0, 128, 0, 256, 0, 0, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1280, 0, 128, 640, 128, 128, 128, 63616, 0, 128, 0, 0, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 0, 128, 128, 640, 128, 1280, 128, 63360, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 65408, 0, 0, 0, 128, 0, 0, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 0, 65408, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65408, 0, 0, 0, 128, 0, 256, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 64896, 0, 0, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 0, 65408, 0, 128, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 65408, 0, 0, 0, 128, 0, 0, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 474, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 16, 0, 0, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7734, 8191, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 99, 1, 0, 1, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 751, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 79, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 190, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7500, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 269, 0, 0, 0, 0, 0, 0, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 157, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 7, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 118, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 8, 0, 0, 1, 0, 0, 26, 0, 4, 0, 0, 0, 0, 0, 0, 38, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3, 37, 0, 0, 19, 0, 0, 3, 0, 0, 4, 0, 0, 0, 7, 23, 8119, 8118, 0, 0, 16, 0, 14, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 18, 0, 10, 0, 0, 0, 0, 32, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 91, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 100, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 441, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 34, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7758, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 101, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 256, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8090, 1, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 0, 1, 7863, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 205, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 323, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7880, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 123, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 751, 0, 0, 7882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8054, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 79, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 190, 0, 0, 7761, 302, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7500, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8633, 8180, 8126, 8281, 7806, 8319, 7989, 8202] avg=8192.0000 max=8633 max_rank=0 min=7806 min_rank=4 imbalance_ratio=1.053833 max_load_reduction=56903 max_load_reduction_pct=0.868271 moved_to_echo=32316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 474, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 16, 0, 0, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 7761, 303, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7734, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 99, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8359, 8180, 8121, 8240, 8038, 8163, 8241, 8194] avg=8192.0000 max=8359 max_rank=0 min=8038 min_rank=4 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 157, 0, 0, 7889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 1, 0, 0, 7, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 7761, 303, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8178, 8197, 8113, 8369, 8197, 8243, 8193] avg=8192.0000 max=8369 max_rank=4 min=8046 min_rank=0 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=33002 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 441, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 34, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 239, 0, 0, 7761, 303, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7758, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8045, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8181, 8116, 8331, 8063, 8146, 8181, 8192] avg=8192.0000 max=8331 max_rank=3 min=8063 min_rank=4 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 8, 0, 0, 1, 0, 0, 26, 0, 4, 0, 0, 0, 0, 7844, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3, 37, 7691, 301, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3, 0, 0, 4, 0, 0, 0, 7, 23, 8119, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 14, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 18, 0, 10, 0, 0, 0, 0, 32, 0, 0, 0, 9, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7837, 8255, 8304, 8086, 8477, 8081, 8319, 8177] avg=8192.0000 max=8477 max_rank=4 min=7837 min_rank=0 imbalance_ratio=1.034790 max_load_reduction=57059 max_load_reduction_pct=0.870651 moved_to_echo=33054 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 256, 0, 0, 7885, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8055, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 1, 0, 8090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 133, 0, 0, 7766, 305, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 0, 1, 7863, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8057, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8200, 8227, 8253, 8162, 8262, 8095, 8194] avg=8192.0000 max=8262 max_rank=5 min=8095 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 91, 0, 0, 7889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 13, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 7761, 303, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7980, 8181, 8244, 8133, 8436, 8166, 8204, 8192] avg=8192.0000 max=8436 max_rank=4 min=7980 min_rank=0 imbalance_ratio=1.029785 max_load_reduction=57100 max_load_reduction_pct=0.871277 moved_to_echo=32998 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 323, 0, 0, 7889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 7761, 303, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7880, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8181, 8201, 8099, 8185, 8202, 8264, 8192] avg=8192.0000 max=8264 max_rank=6 min=8099 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 164, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 9, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 114, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 456, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 13, 0, 0, 0, 0, 0, 0, 7986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7743, 8191, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 94, 1, 0, 1, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 731, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 84, 0, 0, 0, 0, 0, 0, 7935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 174, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7508, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 267, 0, 0, 0, 0, 0, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 8, 0, 0, 0, 0, 0, 25, 0, 4, 0, 0, 0, 0, 0, 0, 39, 0, 0, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3, 37, 0, 0, 19, 0, 0, 3, 0, 0, 4, 0, 0, 0, 7, 23, 8122, 8118, 0, 0, 16, 0, 14, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 18, 0, 11, 0, 0, 0, 0, 30, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 236, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8095, 1, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 0, 1, 7875, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 211, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 90, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 14, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 98, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 430, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 34, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7773, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 97, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 321, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7879, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 122, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 164, 0, 0, 7898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 9, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 7776, 294, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8062, 8178, 8186, 8118, 8361, 8193, 8245, 8193] avg=8192.0000 max=8361 max_rank=4 min=8062 min_rank=0 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32992 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 456, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 13, 0, 0, 0, 0, 0, 0, 7986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 7775, 294, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7743, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 94, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8350, 8179, 8127, 8243, 8042, 8159, 8241, 8195] avg=8192.0000 max=8350 max_rank=0 min=8042 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 8, 0, 0, 0, 0, 0, 25, 0, 4, 0, 0, 0, 0, 7851, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3, 37, 7706, 293, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3, 0, 0, 4, 0, 0, 0, 7, 23, 8122, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 14, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 18, 0, 11, 0, 0, 0, 0, 30, 0, 0, 0, 10, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7845, 8249, 8298, 8094, 8474, 8082, 8318, 8176] avg=8192.0000 max=8474 max_rank=4 min=7845 min_rank=0 imbalance_ratio=1.034424 max_load_reduction=57062 max_load_reduction_pct=0.870697 moved_to_echo=33050 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 731, 0, 0, 7889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 84, 0, 0, 0, 0, 0, 0, 7935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 174, 0, 0, 7776, 294, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7508, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8620, 8181, 8147, 8272, 7808, 8320, 7985, 8203] avg=8192.0000 max=8620 max_rank=0 min=7808 min_rank=4 imbalance_ratio=1.052246 max_load_reduction=56916 max_load_reduction_pct=0.868469 moved_to_echo=32316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 90, 0, 0, 7898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 14, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 7776, 294, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7988, 8181, 8242, 8135, 8430, 8164, 8204, 8192] avg=8192.0000 max=8430 max_rank=4 min=7988 min_rank=0 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32990 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 430, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 34, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 224, 0, 0, 7776, 294, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7773, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8181, 8119, 8321, 8072, 8144, 8183, 8192] avg=8192.0000 max=8324 max_rank=0 min=8072 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 236, 0, 0, 7890, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8068, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 1, 0, 8095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 129, 0, 0, 7777, 300, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 0, 1, 7875, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8206, 8219, 8251, 8171, 8273, 8095, 8193] avg=8192.0000 max=8273 max_rank=5 min=8095 min_rank=6 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 321, 0, 0, 7898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7776, 294, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7879, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8181, 8198, 8102, 8178, 8201, 8265, 8192] avg=8192.0000 max=8265 max_rank=6 min=8102 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 128, 0, 896, 128, 128, 62080, 2432, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 128, 0, 768, 128, 128, 62208, 2432, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 0, 128, 0, 128, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 64384, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 2432, 0, 0, 63232, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 0, 128, 128, 0, 0, 128, 128, 63104, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 128, 0, 128, 0, 128, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 64256, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 1152, 0, 256, 0, 0, 128, 0, 128, 0, 64256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 2560, 0, 0, 63104, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 64768, 128, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 0, 0, 64512, 0, 1024, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 1024, 0, 256, 0, 0, 128, 0, 128, 0, 64384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 64768, 128, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True - [2026-08-26 16:18:06] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 459.9 | throughput per GPU (TFLOP/s/GPU): 393.2 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.010108E+01 | loss scale: 1.0 | grad norm: 83.678 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 746, 320, 4685, 0, 1, 5756, 6, 0, 0, 0, 626, 0, 2, 5, 0, 9, 356, 0, 4, 2090, 278, 0, 0, 8192, 694, 8175, 3, 3524, 0, 1, 361, 0, 1, 0, 0, 2, 16, 0, 2624, 2, 0, 5168, 2, 184, 0, 6, 857, 0, 0, 0, 0, 0, 370, 4, 53, 1365, 0, 0, 0, 49, 0, 0, 0, 0, 925, 0, 0, 1, 870, 0, 1240, 7, 32, 23, 235, 0, 0, 2, 2, 0, 344, 0, 0, 2720, 1, 146, 0, 45, 71, 1, 5, 0, 0, 1671, 1, 0, 1, 11, 0, 0, 26, 4484, 979, 0, 0, 46, 422, 0, 0, 3394, 0, 0, 0, 0, 57, 0, 0, 3, 48, 1183, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 1072, 261, 5672, 0, 0, 6146, 0, 0, 1, 1, 477, 0, 0, 7, 0, 1, 228, 0, 2, 2625, 234, 0, 0, 8187, 1185, 8181, 2, 3369, 0, 1, 998, 0, 0, 0, 0, 1, 39, 0, 2314, 1, 0, 5733, 0, 131, 0, 7, 1002, 0, 0, 0, 0, 0, 165, 0, 48, 1672, 0, 1, 0, 6, 5, 0, 0, 0, 798, 2, 1, 1, 812, 1, 707, 1, 1, 13, 215, 0, 1, 3, 0, 0, 137, 0, 0, 1887, 1, 134, 1, 23, 9, 0, 0, 1, 0, 2461, 0, 0, 1, 3, 1, 2, 11, 2452, 343, 0, 0, 19, 410, 0, 0, 4235, 0, 0, 0, 6, 80, 2, 0, 2, 7, 973, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1624, 295, 4057, 0, 0, 6374, 0, 1, 1, 0, 243, 0, 1, 10, 0, 6, 332, 0, 1, 2936, 210, 1, 0, 8189, 767, 8175, 1, 3567, 0, 0, 922, 0, 3, 1, 0, 1, 25, 0, 3059, 0, 0, 5263, 0, 69, 0, 1, 701, 1, 9, 0, 0, 2, 317, 1, 66, 1983, 0, 1, 0, 5, 0, 0, 0, 0, 602, 0, 3, 0, 1101, 0, 819, 0, 1, 12, 271, 0, 0, 0, 0, 0, 88, 0, 0, 1621, 2, 80, 0, 41, 66, 1, 0, 0, 0, 2814, 1, 0, 2, 21, 1, 0, 15, 4056, 504, 0, 0, 76, 776, 3, 0, 2359, 0, 2, 0, 3, 60, 0, 0, 0, 4, 906, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 774, 237, 4823, 0, 0, 6616, 0, 0, 0, 0, 424, 2, 0, 16, 0, 0, 336, 0, 0, 2045, 184, 0, 0, 8191, 658, 8187, 0, 3414, 2, 0, 907, 0, 4, 24, 0, 0, 19, 0, 2594, 26, 0, 5686, 1, 63, 0, 0, 984, 0, 0, 4, 0, 1, 349, 0, 100, 1284, 0, 0, 0, 14, 3, 1, 0, 0, 1199, 0, 1, 0, 939, 0, 725, 19, 0, 5, 439, 0, 0, 0, 0, 0, 240, 0, 0, 2685, 0, 236, 0, 4, 20, 2, 0, 0, 0, 3685, 0, 0, 5, 12, 1, 0, 18, 3462, 577, 0, 0, 123, 468, 0, 0, 1936, 0, 0, 0, 4, 64, 0, 0, 0, 7, 675, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 567, 508, 5414, 0, 0, 6708, 8, 0, 0, 0, 369, 2, 0, 23, 0, 11, 404, 0, 2, 2667, 303, 0, 0, 8188, 866, 8172, 0, 2928, 11, 17, 562, 0, 2, 5, 1, 0, 9, 1, 3004, 4, 4, 5113, 9, 107, 0, 0, 717, 0, 0, 4, 0, 0, 401, 2, 53, 1483, 2, 0, 0, 13, 0, 3, 0, 2, 1044, 0, 0, 0, 769, 0, 921, 21, 1, 11, 316, 0, 0, 0, 0, 1, 184, 0, 0, 1574, 0, 194, 4, 11, 86, 0, 0, 0, 0, 2207, 0, 1, 2, 40, 0, 0, 42, 4647, 926, 0, 0, 60, 353, 0, 0, 2219, 0, 5, 0, 17, 80, 0, 0, 2, 20, 1098, 1, 1, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1329, 181, 4350, 0, 0, 7179, 0, 0, 0, 0, 350, 0, 0, 22, 0, 25, 328, 0, 11, 2750, 268, 0, 0, 8192, 903, 8184, 0, 2996, 1, 28, 1033, 1, 7, 14, 0, 4, 47, 0, 2612, 3, 0, 4001, 0, 109, 0, 0, 1352, 0, 0, 0, 0, 0, 369, 0, 81, 1264, 0, 0, 0, 15, 0, 1, 0, 0, 1019, 0, 1, 1, 1002, 0, 861, 10, 6, 49, 176, 0, 0, 2, 1, 1, 237, 0, 0, 2088, 3, 217, 0, 15, 91, 1, 0, 0, 0, 2552, 0, 0, 4, 86, 1, 0, 6, 4743, 676, 0, 0, 43, 500, 0, 0, 1862, 0, 1, 0, 2, 144, 0, 0, 0, 16, 1098, 0, 1, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 973, 330, 5033, 2, 0, 7039, 0, 0, 0, 1, 130, 0, 1, 4, 0, 2, 243, 0, 1, 2196, 180, 0, 0, 8190, 645, 8188, 0, 3992, 0, 2, 1147, 0, 0, 1, 0, 0, 2, 0, 3356, 15, 1, 5617, 0, 87, 0, 0, 649, 1, 0, 1, 0, 2, 421, 0, 53, 1478, 0, 0, 0, 20, 0, 2, 0, 0, 637, 0, 3, 0, 908, 0, 672, 5, 12, 13, 356, 0, 0, 0, 0, 1, 38, 0, 0, 2545, 2, 115, 0, 15, 64, 0, 0, 3, 0, 2048, 2, 0, 4, 38, 3, 2, 5, 2507, 424, 0, 0, 31, 644, 1, 0, 2996, 0, 0, 0, 0, 43, 0, 0, 2, 6, 1377, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 848, 370, 5046, 0, 0, 5862, 0, 5, 3, 0, 468, 2, 0, 4, 0, 29, 96, 0, 3, 2318, 396, 1, 0, 8192, 923, 8162, 2, 4143, 3, 4, 576, 0, 1, 5, 0, 3, 20, 1, 2418, 1, 0, 4902, 3, 73, 0, 0, 671, 0, 4, 1, 0, 0, 257, 0, 66, 1747, 0, 0, 0, 32, 0, 1, 0, 0, 808, 0, 0, 0, 919, 0, 1040, 13, 17, 20, 186, 0, 0, 0, 1, 6, 177, 0, 0, 2169, 0, 148, 0, 39, 16, 0, 3, 0, 0, 2488, 3, 0, 4, 9, 0, 0, 29, 3592, 781, 0, 2, 17, 382, 0, 0, 3938, 0, 0, 1, 0, 46, 0, 0, 0, 12, 993, 4, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 1072, 261, 5672, 0, 0, 1532, 0, 0, 1, 1, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 228, 0, 2, 2625, 234, 0, 0, 0, 1185, 845, 2, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 998, 0, 0, 0, 0, 1, 39, 0, 2314, 1, 0, 4951, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1002, 0, 0, 0, 0, 0, 165, 0, 48, 1672, 0, 1, 0, 6, 2787, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 798, 2, 1, 1, 812, 1, 707, 1, 1, 13, 215, 0, 4549, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 137, 0, 0, 1887, 1, 134, 1, 23, 9, 0, 0, 1, 5606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 0, 0, 1, 3, 1, 2, 11, 2452, 343, 0, 0, 19, 410, 0, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4235, 0, 0, 0, 6, 80, 2, 0, 2, 7, 973, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9020, 8498, 8436, 8269, 7735, 7803, 6485, 9290] avg=8192.0000 max=9290 max_rank=7 min=6485 min_rank=6 imbalance_ratio=1.134033 max_load_reduction=56246 max_load_reduction_pct=0.858246 moved_to_echo=31313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 746, 320, 4685, 0, 1, 1434, 6, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 9, 356, 0, 4, 2090, 278, 0, 0, 0, 694, 844, 3, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 361, 0, 1, 0, 0, 2, 16, 0, 2624, 2, 0, 4463, 2, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 857, 0, 0, 0, 0, 0, 370, 4, 53, 1365, 0, 0, 0, 49, 2785, 2582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 925, 0, 0, 1, 870, 0, 1240, 7, 32, 23, 235, 0, 4546, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 344, 0, 0, 2720, 1, 146, 0, 45, 71, 1, 5, 0, 5610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 1, 0, 1, 11, 0, 0, 26, 4484, 979, 0, 0, 46, 422, 0, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3394, 0, 0, 0, 0, 57, 0, 0, 3, 48, 1183, 0, 0, 2, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7819, 7809, 7656, 8071, 8468, 8947, 8346, 8420] avg=8192.0000 max=8947 max_rank=5 min=7656 min_rank=2 imbalance_ratio=1.092163 max_load_reduction=56589 max_load_reduction_pct=0.863480 moved_to_echo=34181 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1624, 295, 4057, 0, 0, 1588, 0, 1, 1, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 6, 332, 0, 1, 2936, 210, 1, 0, 0, 767, 844, 1, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 922, 0, 3, 1, 0, 1, 25, 0, 3059, 0, 0, 4546, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 701, 1, 9, 0, 0, 2, 317, 1, 66, 1983, 0, 1, 0, 5, 2785, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 3, 0, 1101, 0, 819, 0, 1, 12, 271, 0, 4546, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 1621, 2, 80, 0, 41, 66, 1, 0, 0, 5608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 1, 0, 2, 21, 1, 0, 15, 4056, 504, 0, 0, 76, 776, 3, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2359, 0, 2, 0, 3, 60, 0, 0, 0, 4, 906, 0, 0, 0, 2, 4133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7811, 8676, 8626, 8453, 8008, 7507, 8986, 7469] avg=8192.0000 max=8986 max_rank=6 min=7469 min_rank=7 imbalance_ratio=1.096924 max_load_reduction=56550 max_load_reduction_pct=0.862885 moved_to_echo=31970 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 774, 237, 4823, 0, 0, 1648, 0, 0, 0, 0, 424, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 336, 0, 0, 2045, 184, 0, 0, 0, 658, 845, 0, 3414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 907, 0, 4, 24, 0, 0, 19, 0, 2594, 26, 0, 4911, 1, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 0, 0, 4, 0, 1, 349, 0, 100, 1284, 0, 0, 0, 14, 2789, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 1199, 0, 1, 0, 939, 0, 725, 19, 0, 5, 439, 0, 4553, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 2685, 0, 236, 0, 4, 20, 2, 0, 0, 5608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3685, 0, 0, 5, 12, 1, 0, 18, 3462, 577, 0, 0, 123, 468, 0, 775, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 0, 0, 0, 4, 64, 0, 0, 0, 7, 675, 0, 0, 3, 0, 4290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7917, 7498, 8551, 8108, 8561, 8795, 9127, 6979] avg=8192.0000 max=9127 max_rank=6 min=6979 min_rank=7 imbalance_ratio=1.114136 max_load_reduction=56409 max_load_reduction_pct=0.860733 moved_to_echo=33462 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 567, 508, 5414, 0, 0, 1670, 8, 0, 0, 0, 369, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 11, 404, 0, 2, 2667, 303, 0, 0, 0, 866, 844, 0, 2928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 17, 562, 0, 2, 5, 1, 0, 9, 1, 3004, 4, 4, 4416, 9, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 717, 0, 0, 4, 0, 0, 401, 2, 53, 1483, 2, 0, 0, 13, 2784, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 1044, 0, 0, 0, 769, 0, 921, 21, 1, 11, 316, 0, 4544, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 184, 0, 0, 1574, 0, 194, 4, 11, 86, 0, 0, 0, 5607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2207, 0, 1, 2, 40, 0, 0, 42, 4647, 926, 0, 0, 60, 353, 0, 697, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2219, 0, 5, 0, 17, 80, 0, 0, 2, 20, 1098, 1, 1, 5, 0, 4350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8542, 8048, 8152, 8040, 8319, 7661, 8976, 7798] avg=8192.0000 max=8976 max_rank=6 min=7661 min_rank=5 imbalance_ratio=1.095703 max_load_reduction=56560 max_load_reduction_pct=0.863037 moved_to_echo=32754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1329, 181, 4350, 0, 0, 1775, 0, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 25, 328, 0, 11, 2750, 268, 0, 0, 0, 903, 837, 0, 2996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 1033, 1, 7, 14, 0, 4, 47, 0, 2612, 3, 0, 3452, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 0, 0, 0, 0, 0, 369, 0, 81, 1264, 0, 0, 0, 15, 2792, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1019, 0, 1, 1, 1002, 0, 861, 10, 6, 49, 176, 0, 4555, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 237, 0, 0, 2088, 3, 217, 0, 15, 91, 1, 0, 0, 5609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 4, 86, 1, 0, 6, 4743, 676, 0, 0, 43, 500, 0, 549, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1862, 0, 1, 0, 2, 144, 0, 0, 0, 16, 1098, 0, 1, 4, 0, 4660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7991, 8140, 7311, 8456, 8420, 8265, 9165, 7788] avg=8192.0000 max=9165 max_rank=6 min=7311 min_rank=2 imbalance_ratio=1.118774 max_load_reduction=56371 max_load_reduction_pct=0.860153 moved_to_echo=33638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 848, 370, 5046, 0, 0, 1461, 0, 5, 3, 0, 468, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 29, 96, 0, 3, 2318, 396, 1, 0, 0, 923, 842, 2, 4143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 576, 0, 1, 5, 0, 3, 20, 1, 2418, 1, 0, 4234, 3, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 4, 1, 0, 0, 257, 0, 66, 1747, 0, 0, 0, 32, 2781, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 808, 0, 0, 0, 919, 0, 1040, 13, 17, 20, 186, 0, 4539, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 177, 0, 0, 2169, 0, 148, 0, 39, 16, 0, 3, 0, 5609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 3, 0, 4, 9, 0, 0, 29, 3592, 781, 0, 2, 17, 382, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 0, 1, 0, 46, 0, 0, 0, 12, 993, 4, 0, 8, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8757, 7342, 8142, 8143, 8168, 7975, 8803] avg=8192.0000 max=8803 max_rank=7 min=7342 min_rank=2 imbalance_ratio=1.074585 max_load_reduction=56733 max_load_reduction_pct=0.865677 moved_to_echo=33089 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 973, 330, 5033, 2, 0, 1752, 0, 0, 0, 1, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 2, 243, 0, 1, 2196, 180, 0, 0, 0, 645, 846, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1147, 0, 0, 1, 0, 0, 2, 0, 3356, 15, 1, 4851, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 1, 0, 1, 0, 2, 421, 0, 53, 1478, 0, 0, 0, 20, 2789, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 637, 0, 3, 0, 908, 0, 672, 5, 12, 13, 356, 0, 4553, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 0, 0, 2545, 2, 115, 0, 15, 64, 0, 0, 3, 5607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2048, 2, 0, 4, 38, 3, 2, 5, 2507, 424, 0, 0, 31, 644, 1, 766, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2996, 0, 0, 0, 0, 43, 0, 0, 2, 6, 1377, 0, 0, 0, 0, 4565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8110, 9462, 7997, 7882, 8390, 6476, 8989] avg=8192.0000 max=9462 max_rank=2 min=6476 min_rank=6 imbalance_ratio=1.155029 max_load_reduction=56074 max_load_reduction_pct=0.855621 moved_to_echo=31737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True + [2026-08-26 16:27:14] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 448.6 | throughput per GPU (TFLOP/s/GPU): 403.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.010178E+01 | loss scale: 1.0 | grad norm: 83.811 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1608, 296, 4057, 0, 0, 6270, 0, 1, 1, 0, 245, 0, 1, 9, 0, 5, 338, 0, 1, 2903, 214, 1, 0, 8189, 715, 8175, 1, 3593, 0, 0, 899, 0, 2, 1, 0, 1, 26, 0, 3066, 0, 0, 5391, 0, 70, 0, 1, 705, 2, 9, 0, 0, 2, 350, 1, 67, 2002, 0, 1, 0, 6, 0, 0, 0, 0, 584, 0, 3, 0, 1103, 0, 803, 1, 1, 18, 271, 0, 0, 0, 0, 0, 88, 0, 0, 1630, 2, 70, 0, 42, 81, 1, 0, 0, 0, 2808, 3, 0, 3, 22, 2, 0, 13, 4095, 509, 0, 0, 77, 764, 3, 0, 2320, 0, 2, 0, 3, 59, 0, 0, 0, 5, 921, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 840, 371, 5053, 0, 0, 5745, 0, 5, 3, 0, 478, 2, 0, 4, 0, 28, 99, 0, 3, 2305, 411, 1, 0, 8192, 882, 8162, 2, 4154, 3, 4, 559, 0, 1, 5, 0, 3, 22, 1, 2450, 1, 0, 5011, 3, 71, 0, 0, 689, 0, 4, 1, 0, 0, 283, 0, 70, 1796, 0, 0, 0, 29, 0, 1, 0, 0, 765, 0, 0, 0, 921, 0, 1000, 12, 17, 33, 184, 0, 0, 0, 1, 6, 170, 0, 0, 2189, 0, 130, 0, 37, 16, 0, 3, 0, 0, 2472, 3, 0, 4, 8, 0, 0, 28, 3607, 795, 0, 2, 18, 374, 0, 0, 3913, 0, 0, 1, 0, 44, 0, 0, 0, 15, 1005, 4, 0, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 729, 316, 4666, 0, 1, 5635, 6, 0, 0, 0, 642, 0, 2, 6, 0, 9, 359, 0, 4, 2078, 284, 0, 0, 8192, 639, 8175, 3, 3512, 0, 1, 342, 0, 1, 0, 0, 2, 14, 0, 2635, 2, 0, 5276, 2, 179, 0, 7, 881, 0, 0, 0, 0, 0, 392, 4, 50, 1388, 0, 0, 0, 53, 0, 0, 0, 0, 913, 0, 0, 1, 867, 0, 1224, 9, 32, 32, 236, 0, 0, 2, 2, 0, 344, 0, 0, 2728, 1, 132, 0, 41, 82, 1, 5, 0, 0, 1673, 2, 0, 1, 13, 0, 0, 26, 4545, 991, 0, 0, 46, 428, 0, 0, 3364, 0, 0, 0, 0, 51, 0, 0, 3, 50, 1201, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1303, 181, 4334, 0, 0, 7127, 0, 0, 0, 0, 359, 0, 0, 23, 0, 24, 329, 0, 10, 2724, 273, 0, 0, 8192, 864, 8184, 0, 3002, 1, 28, 1007, 1, 8, 14, 0, 3, 51, 0, 2627, 3, 0, 4094, 0, 101, 0, 0, 1374, 0, 0, 0, 0, 0, 402, 0, 81, 1280, 0, 0, 0, 16, 0, 1, 0, 0, 991, 0, 1, 0, 1002, 0, 854, 15, 7, 67, 173, 0, 0, 2, 1, 1, 238, 0, 0, 2105, 3, 193, 0, 13, 107, 2, 0, 0, 0, 2547, 0, 0, 4, 85, 1, 0, 6, 4752, 684, 0, 0, 40, 491, 0, 0, 1854, 0, 1, 0, 2, 136, 0, 0, 0, 16, 1110, 0, 1, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 9, 952, 331, 5010, 2, 0, 6967, 0, 0, 0, 1, 136, 0, 1, 3, 0, 1, 241, 0, 1, 2149, 181, 0, 0, 8190, 590, 8188, 0, 3994, 0, 1, 1107, 0, 0, 1, 0, 0, 2, 0, 3376, 15, 1, 5771, 0, 84, 0, 0, 663, 1, 0, 1, 0, 3, 460, 0, 54, 1519, 0, 0, 0, 18, 0, 2, 0, 0, 610, 0, 3, 0, 908, 0, 646, 7, 12, 24, 360, 0, 0, 0, 0, 1, 39, 0, 0, 2555, 2, 110, 0, 15, 69, 0, 0, 3, 0, 2054, 2, 0, 4, 41, 3, 2, 5, 2503, 431, 0, 0, 33, 648, 1, 0, 2972, 0, 0, 0, 0, 38, 0, 0, 2, 6, 1400, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 770, 239, 4819, 0, 0, 6501, 0, 0, 0, 0, 445, 2, 0, 16, 0, 0, 335, 0, 0, 2022, 190, 0, 0, 8191, 626, 8187, 0, 3420, 2, 0, 886, 0, 5, 24, 0, 0, 20, 0, 2614, 28, 0, 5800, 1, 63, 0, 0, 1016, 0, 0, 4, 0, 1, 372, 0, 105, 1317, 0, 0, 0, 14, 3, 1, 0, 0, 1165, 0, 1, 0, 931, 0, 698, 21, 0, 5, 432, 0, 0, 0, 0, 0, 245, 0, 0, 2735, 0, 211, 0, 3, 22, 2, 0, 0, 0, 3664, 0, 0, 5, 12, 0, 0, 19, 3491, 576, 0, 0, 129, 457, 0, 0, 1912, 0, 0, 0, 3, 59, 0, 0, 0, 8, 680, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 557, 501, 5413, 0, 0, 6624, 8, 0, 0, 0, 378, 2, 0, 24, 0, 11, 403, 0, 2, 2643, 310, 0, 0, 8188, 826, 8173, 0, 2924, 12, 17, 538, 0, 2, 6, 1, 0, 10, 1, 3034, 4, 4, 5220, 9, 102, 0, 0, 726, 0, 0, 4, 0, 0, 437, 2, 55, 1518, 2, 0, 0, 12, 0, 2, 0, 2, 1014, 0, 0, 0, 753, 0, 895, 20, 1, 19, 314, 0, 0, 0, 0, 1, 176, 0, 0, 1586, 0, 162, 4, 10, 110, 0, 0, 0, 0, 2220, 0, 1, 2, 40, 0, 0, 44, 4664, 929, 0, 0, 63, 343, 0, 0, 2210, 0, 5, 0, 17, 80, 0, 1, 3, 21, 1110, 1, 1, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 1072, 262, 5664, 0, 0, 6031, 0, 0, 1, 1, 496, 0, 0, 7, 0, 1, 228, 0, 2, 2613, 238, 0, 0, 8187, 1103, 8181, 2, 3355, 0, 1, 992, 0, 0, 0, 0, 2, 40, 0, 2340, 1, 0, 5870, 0, 126, 0, 7, 1021, 0, 0, 0, 0, 0, 171, 0, 51, 1711, 0, 1, 0, 6, 5, 0, 0, 0, 772, 2, 1, 1, 828, 0, 677, 2, 1, 18, 212, 0, 1, 3, 0, 0, 135, 0, 0, 1897, 1, 123, 1, 21, 11, 0, 0, 1, 0, 2473, 0, 0, 1, 4, 1, 2, 12, 2473, 350, 0, 0, 21, 406, 0, 1, 4205, 0, 0, 0, 6, 79, 2, 0, 2, 11, 986, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1608, 296, 4057, 0, 0, 1595, 0, 1, 1, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 5, 338, 0, 1, 2903, 214, 1, 0, 0, 715, 908, 1, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, 2, 1, 0, 1, 26, 0, 3066, 0, 0, 4557, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 705, 2, 9, 0, 0, 2, 350, 1, 67, 2002, 0, 1, 0, 6, 2700, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 3, 0, 1103, 0, 803, 1, 1, 18, 271, 0, 4567, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 1630, 2, 70, 0, 42, 81, 1, 0, 0, 5601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2808, 3, 0, 3, 22, 2, 0, 13, 4095, 509, 0, 0, 77, 764, 3, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 0, 2, 0, 3, 59, 0, 0, 0, 5, 921, 0, 0, 0, 2, 4002, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7805, 8689, 8622, 8434, 8024, 7515, 8992, 7455] avg=8192.0000 max=8992 max_rank=6 min=7455 min_rank=7 imbalance_ratio=1.097656 max_load_reduction=56544 max_load_reduction_pct=0.862793 moved_to_echo=31986 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 729, 316, 4666, 0, 1, 1433, 6, 0, 0, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 9, 359, 0, 4, 2078, 284, 0, 0, 0, 639, 908, 3, 3512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 342, 0, 1, 0, 0, 2, 14, 0, 2635, 2, 0, 4459, 2, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 881, 0, 0, 0, 0, 0, 392, 4, 50, 1388, 0, 0, 0, 53, 2700, 2589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 0, 0, 1, 867, 0, 1224, 9, 32, 32, 236, 0, 4567, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 344, 0, 0, 2728, 1, 132, 0, 41, 82, 1, 5, 0, 5603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1673, 2, 0, 1, 13, 0, 0, 26, 4545, 991, 0, 0, 46, 428, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3364, 0, 0, 0, 0, 51, 0, 0, 3, 50, 1201, 0, 0, 2, 0, 3597, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7794, 7804, 7637, 8064, 8486, 8941, 8404, 8406] avg=8192.0000 max=8941 max_rank=5 min=7637 min_rank=2 imbalance_ratio=1.091431 max_load_reduction=56595 max_load_reduction_pct=0.863571 moved_to_echo=34237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 770, 239, 4819, 0, 0, 1653, 0, 0, 0, 0, 445, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 335, 0, 0, 2022, 190, 0, 0, 0, 626, 909, 0, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 886, 0, 5, 24, 0, 0, 20, 0, 2614, 28, 0, 4902, 1, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 0, 0, 4, 0, 1, 372, 0, 105, 1317, 0, 0, 0, 14, 2704, 2589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 1165, 0, 1, 0, 931, 0, 698, 21, 0, 5, 432, 0, 4574, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 2735, 0, 211, 0, 3, 22, 2, 0, 0, 5602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3664, 0, 0, 5, 12, 0, 0, 19, 3491, 576, 0, 0, 129, 457, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1912, 0, 0, 0, 3, 59, 0, 0, 0, 8, 680, 0, 0, 2, 0, 4150, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7937, 7518, 8545, 8122, 8529, 8820, 9099, 6966] avg=8192.0000 max=9099 max_rank=6 min=6966 min_rank=7 imbalance_ratio=1.110718 max_load_reduction=56437 max_load_reduction_pct=0.861160 moved_to_echo=33414 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1303, 181, 4334, 0, 0, 1806, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 24, 329, 0, 10, 2724, 273, 0, 0, 0, 864, 902, 0, 3002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 1007, 1, 8, 14, 0, 3, 51, 0, 2627, 3, 0, 3450, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 402, 0, 81, 1280, 0, 0, 0, 16, 2705, 2590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 991, 0, 1, 0, 1002, 0, 854, 15, 7, 67, 173, 0, 4577, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 238, 0, 0, 2105, 3, 193, 0, 13, 107, 2, 0, 0, 5602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2547, 0, 0, 4, 85, 1, 0, 6, 4752, 684, 0, 0, 40, 491, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1854, 0, 1, 0, 2, 136, 0, 0, 0, 16, 1110, 0, 1, 4, 0, 4552, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7989, 8151, 7294, 8448, 8457, 8267, 9141, 7789] avg=8192.0000 max=9141 max_rank=6 min=7294 min_rank=2 imbalance_ratio=1.115845 max_load_reduction=56395 max_load_reduction_pct=0.860519 moved_to_echo=33654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 840, 371, 5053, 0, 0, 1461, 0, 5, 3, 0, 478, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 28, 99, 0, 3, 2305, 411, 1, 0, 0, 882, 907, 2, 4154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 559, 0, 1, 5, 0, 3, 22, 1, 2450, 1, 0, 4235, 3, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, 0, 4, 1, 0, 0, 283, 0, 70, 1796, 0, 0, 0, 29, 2695, 2590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 765, 0, 0, 0, 921, 0, 1000, 12, 17, 33, 184, 0, 4560, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 170, 0, 0, 2189, 0, 130, 0, 37, 16, 0, 3, 0, 5602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2472, 3, 0, 4, 8, 0, 0, 28, 3607, 795, 0, 2, 18, 374, 0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 1, 0, 44, 0, 0, 0, 15, 1005, 4, 0, 9, 0, 3667, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8796, 7358, 8157, 8110, 8154, 7956, 8789] avg=8192.0000 max=8796 max_rank=1 min=7358 min_rank=2 imbalance_ratio=1.073730 max_load_reduction=56740 max_load_reduction_pct=0.865784 moved_to_echo=33009 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 557, 501, 5413, 0, 0, 1685, 8, 0, 0, 0, 378, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 11, 403, 0, 2, 2643, 310, 0, 0, 0, 826, 908, 0, 2924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 17, 538, 0, 2, 6, 1, 0, 10, 1, 3034, 4, 4, 4412, 9, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 726, 0, 0, 4, 0, 0, 437, 2, 55, 1518, 2, 0, 0, 12, 2699, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 1014, 0, 0, 0, 753, 0, 895, 20, 1, 19, 314, 0, 4566, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 176, 0, 0, 1586, 0, 162, 4, 10, 110, 0, 0, 0, 5601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2220, 0, 1, 2, 40, 0, 0, 44, 4664, 929, 0, 0, 63, 343, 0, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210, 0, 5, 0, 17, 80, 0, 1, 3, 21, 1110, 1, 1, 5, 0, 4228, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8548, 8051, 8152, 8042, 8297, 7650, 8977, 7819] avg=8192.0000 max=8977 max_rank=6 min=7650 min_rank=5 imbalance_ratio=1.095825 max_load_reduction=56559 max_load_reduction_pct=0.863022 moved_to_echo=32743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 9, 952, 331, 5010, 2, 0, 1772, 0, 0, 0, 1, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 1, 241, 0, 1, 2149, 181, 0, 0, 0, 590, 909, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1107, 0, 0, 1, 0, 0, 2, 0, 3376, 15, 1, 4878, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, 1, 0, 1, 0, 3, 460, 0, 54, 1519, 0, 0, 0, 18, 2704, 2589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 610, 0, 3, 0, 908, 0, 646, 7, 12, 24, 360, 0, 4575, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 39, 0, 0, 2555, 2, 110, 0, 15, 69, 0, 0, 3, 5601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2054, 2, 0, 4, 41, 3, 2, 5, 2503, 431, 0, 0, 33, 648, 1, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2972, 0, 0, 0, 0, 38, 0, 0, 2, 6, 1400, 0, 0, 0, 0, 4447, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8070, 9465, 8012, 7895, 8395, 6469, 9016] avg=8192.0000 max=9465 max_rank=2 min=6469 min_rank=6 imbalance_ratio=1.155396 max_load_reduction=56071 max_load_reduction_pct=0.855576 moved_to_echo=31775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 1072, 262, 5664, 0, 0, 1533, 0, 0, 1, 1, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 228, 0, 2, 2613, 238, 0, 0, 0, 1103, 908, 2, 3355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 992, 0, 0, 0, 0, 2, 40, 0, 2340, 1, 0, 4961, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1021, 0, 0, 0, 0, 0, 171, 0, 51, 1711, 0, 1, 0, 6, 2702, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 772, 2, 1, 1, 828, 0, 677, 2, 1, 18, 212, 0, 4571, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 135, 0, 0, 1897, 1, 123, 1, 21, 11, 0, 0, 1, 5600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2473, 0, 0, 1, 4, 1, 2, 12, 2473, 350, 0, 0, 21, 406, 0, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4205, 0, 0, 0, 6, 79, 2, 0, 2, 11, 986, 0, 0, 0, 0, 3850, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9033, 8457, 8463, 8257, 7738, 7794, 6498, 9296] avg=8192.0000 max=9296 max_rank=7 min=6498 min_rank=6 imbalance_ratio=1.134766 max_load_reduction=56240 max_load_reduction_pct=0.858154 moved_to_echo=31326 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 7040, 128, 128, 128, 0, 128, 2688, 128, 640, 12288, 128, 128, 0, 256, 22400, 20736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 2432, 0, 128, 19712, 2176, 128, 0, 0, 6656, 6784, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 23040, 0, 128, 128, 128, 640, 128, 0, 128, 128, 8320, 128, 128, 128, 128, 33536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 7936, 2560, 39168, 128, 128, 12928, 128, 128, 128, 128, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 6528, 128, 128, 128, 128, 128, 256, 128, 22016, 128, 128, 35840, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 7040, 128, 128, 128, 7424, 128, 7040, 128, 128, 256, 2304, 0, 36480, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 19968, 128, 128, 128, 256, 128, 128, 256, 29952, 5248, 0, 128, 512, 3968, 128, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 1536, 0, 0, 17408, 128, 1280, 128, 256, 512, 128, 128, 128, 44928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 7168, 128, 128, 128, 0, 128, 2944, 128, 640, 12544, 128, 128, 0, 256, 21632, 20736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 2432, 0, 128, 19456, 2176, 128, 0, 0, 6272, 7296, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 22784, 0, 128, 128, 128, 640, 128, 128, 128, 256, 8448, 128, 128, 128, 128, 32512, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 7936, 2560, 39040, 128, 128, 13056, 128, 128, 128, 128, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 6400, 128, 128, 128, 128, 128, 256, 128, 22144, 128, 128, 35968, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 19968, 128, 128, 128, 256, 128, 128, 256, 30208, 5376, 0, 128, 512, 3968, 128, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 6912, 128, 128, 128, 7424, 0, 6912, 128, 128, 256, 2304, 0, 36608, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 1536, 0, 0, 17536, 128, 1152, 128, 256, 512, 128, 128, 128, 44928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7871, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8192, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8185, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 945, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8189, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 8156, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 8182, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 12, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 8111, 0, 8185, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 8184, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8170, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 8191, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8184, 0, 7953, 5, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 9, 0, 2, 0, 2, 8186, 8163, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 111, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 1, 0, 3, 0, 1, 8180, 1, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 136, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8189, 0, 7782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 7676, 0, 8188, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8188, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 1, 0, 8191, 0, 657, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7934, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8185, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[293, 0, 7871, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8106, 85, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7814, 240, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8276, 8194, 8203, 8191, 8193, 8203, 8109] avg=8192.0000 max=8276 max_rank=1 min=8109 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[293, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8111, 0, 86, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 7944, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8173, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8099, 85, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7814, 240, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8371, 8203, 8012, 8177, 8193, 8189, 8187, 8204] avg=8192.0000 max=8371 max_rank=0 min=8012 min_rank=2 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[293, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 7942, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8104, 85, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7811, 240, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8461, 8242, 7999, 8197, 8190, 8191, 8189, 8067] avg=8192.0000 max=8461 max_rank=0 min=7999 min_rank=2 imbalance_ratio=1.032837 max_load_reduction=57075 max_load_reduction_pct=0.870895 moved_to_echo=32637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[291, 0, 7197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 945, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7815, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7488, 8268, 8889, 8198, 8192, 8195, 8184, 8122] avg=8192.0000 max=8889 max_rank=2 min=7488 min_rank=0 imbalance_ratio=1.085083 max_load_reduction=56647 max_load_reduction_pct=0.864365 moved_to_echo=32693 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[293, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 7950, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8103, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7814, 240, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8474, 8256, 7966, 8185, 8192, 8196, 8197, 8070] avg=8192.0000 max=8474 max_rank=0 min=7966 min_rank=2 imbalance_ratio=1.034424 max_load_reduction=57062 max_load_reduction_pct=0.870697 moved_to_echo=32655 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[293, 0, 7953, 5, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 8163, 0, 86, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 1, 0, 3, 0, 1, 8180, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8099, 85, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 240, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8256, 8056, 8197, 8195, 8188, 8190, 8189] avg=8192.0000 max=8265 max_rank=0 min=8056 min_rank=2 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[293, 0, 7934, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 85, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7814, 240, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8271, 8212, 8187, 8191, 8192, 8189, 8059] avg=8192.0000 max=8271 max_rank=1 min=8059 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32631 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[293, 0, 7782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7676, 0, 86, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 7948, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8188, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 8102, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 1, 0, 7811, 240, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 7764, 8208, 8192, 8192, 8192, 8197, 8716] avg=8192.0000 max=8716 max_rank=7 min=7764 min_rank=1 imbalance_ratio=1.063965 max_load_reduction=56820 max_load_reduction_pct=0.867004 moved_to_echo=33297 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7891, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 8192, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 8116, 0, 8185, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 8184, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8185, 0, 7955, 5, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 9, 0, 2, 0, 2, 8186, 8159, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 124, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 1, 0, 3, 0, 0, 8180, 1, 0, 0, 0, 1, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 124, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8189, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 8158, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 8182, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 10, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8170, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 8191, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7959, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8189, 0, 7827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 7700, 0, 8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8188, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 0, 8191, 0, 594, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[291, 0, 7891, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8110, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 247, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8272, 8177, 8203, 8191, 8193, 8205, 8110] avg=8192.0000 max=8272 max_rank=1 min=8110 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[291, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 7935, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8108, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7817, 247, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8458, 8240, 7993, 8197, 8190, 8191, 8189, 8078] avg=8192.0000 max=8458 max_rank=0 min=7993 min_rank=2 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=32648 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[291, 0, 7955, 5, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 8159, 0, 82, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 7938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 1, 0, 3, 0, 0, 8180, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8103, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7813, 247, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8248, 8062, 8196, 8195, 8188, 8190, 8191] avg=8192.0000 max=8266 max_rank=0 min=8062 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[291, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8116, 0, 82, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 7937, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8171, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8103, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 247, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8396, 8204, 7989, 8175, 8193, 8188, 8186, 8205] avg=8192.0000 max=8396 max_rank=0 min=7989 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=57140 max_load_reduction_pct=0.871887 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[279, 0, 7110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 0, 0, 7939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7826, 251, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7389, 8267, 8984, 8198, 8192, 8198, 8182, 8126] avg=8192.0000 max=8984 max_rank=2 min=7389 min_rank=0 imbalance_ratio=1.096680 max_load_reduction=56552 max_load_reduction_pct=0.862915 moved_to_echo=32698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[291, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 7943, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8107, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7820, 247, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8466, 8252, 7958, 8188, 8192, 8195, 8197, 8088] avg=8192.0000 max=8466 max_rank=0 min=7958 min_rank=2 imbalance_ratio=1.033447 max_load_reduction=57070 max_load_reduction_pct=0.870819 moved_to_echo=32672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[291, 0, 7959, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 247, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8268, 8178, 8187, 8192, 8191, 8190, 8072] avg=8192.0000 max=8268 max_rank=1 min=8072 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32645 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[291, 0, 7827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7700, 0, 82, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 7942, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8188, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 8106, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 1, 0, 7817, 247, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 7785, 8195, 8192, 8191, 8192, 8197, 8666] avg=8192.0000 max=8666 max_rank=7 min=7785 min_rank=1 imbalance_ratio=1.057861 max_load_reduction=56870 max_load_reduction_pct=0.867767 moved_to_echo=33246 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 1152, 128, 0, 128, 0, 0, 128, 128, 0, 0, 128, 128, 0, 128, 0, 62592, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[2432, 0, 63232, 128, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 1024, 128, 0, 128, 0, 0, 128, 128, 0, 0, 128, 128, 0, 128, 0, 62592, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2432, 0, 63232, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 2048, 0, 0, 63616, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 64896, 0, 768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 0, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 64896, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 128, 128, 128, 0, 128, 65536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 0, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7652, 0, 0, 0, 0, 0, 1047, 0, 7891, 0, 0, 0, 0, 0, 0, 20, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6945, 0, 1, 0, 0, 0, 0, 0, 0, 1018, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 505, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 16, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 14, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6679, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 8170, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6864, 0, 0, 0, 0, 0, 1700, 0, 8000, 0, 0, 0, 0, 0, 0, 31, 0, 8189, 0, 0, 9, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7596, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 8184, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 29, 0, 8064, 0, 0, 0, 0, 0, 0, 3, 0, 8187, 0, 0, 15, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7215, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 0, 8166, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7448, 0, 0, 0, 0, 0, 1095, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7863, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 13, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 8178, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 7, 17, 0, 8183, 0, 0, 0, 0, 0, 5, 0, 0, 8117, 0, 0, 1, 34, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8183, 0, 45, 0, 0, 4, 9, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7202, 0, 1, 3, 8, 0, 0, 4, 0, 1019, 0, 8158, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7474, 0, 0, 0, 0, 0, 438, 0, 8146, 0, 0, 0, 0, 0, 0, 36, 0, 8190, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6818, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7206, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 22, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 7404, 0, 2, 1, 175, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8188, 0, 18, 2, 0, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5228, 0, 0, 0, 3, 1, 0, 2, 0, 4595, 0, 8166, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7878, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7652, 0, 0, 0, 0, 322, 107, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 7673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6838, 0, 1, 0, 0, 0, 0, 0, 0, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7973, 8331, 8095, 8740, 8156, 8178, 8206, 7857] avg=8192.0000 max=8740 max_rank=3 min=7857 min_rank=7 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=32397 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 321, 103, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7661, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6576, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8297, 8596, 8182, 8195, 8191, 8185, 7622] avg=8192.0000 max=8596 max_rank=2 min=7622 min_rank=7 imbalance_ratio=1.049316 max_load_reduction=56940 max_load_reduction_pct=0.868835 moved_to_echo=32193 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 322, 111, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7668, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7104, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8322, 8557, 7715, 8199, 8203, 8179, 8215] avg=8192.0000 max=8557 max_rank=2 min=7715 min_rank=3 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=32796 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7448, 0, 0, 0, 0, 321, 121, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7643, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7742, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8330, 7903, 8745, 8156, 8177, 8204, 7749] avg=8192.0000 max=8745 max_rank=3 min=7749 min_rank=7 imbalance_ratio=1.067505 max_load_reduction=56791 max_load_reduction_pct=0.866562 moved_to_echo=32286 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7988, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6864, 0, 0, 0, 0, 326, 121, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 7664, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7961, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7475, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8084, 8323, 7331, 9404, 8170, 8173, 8196, 7855] avg=8192.0000 max=9404 max_rank=3 min=7331 min_rank=2 imbalance_ratio=1.147949 max_load_reduction=56132 max_load_reduction_pct=0.856506 moved_to_echo=32394 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 13, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 2, 0, 8170, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 319, 111, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 17, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7603, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 7945, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8170, 0, 45, 0, 0, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8158, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7091, 0, 1, 3, 8, 0, 0, 4, 0, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8255, 8635, 7633, 8189, 8235, 8179, 8126] avg=8192.0000 max=8635 max_rank=2 min=7633 min_rank=3 imbalance_ratio=1.054077 max_load_reduction=56901 max_load_reduction_pct=0.868240 moved_to_echo=32729 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7474, 0, 0, 0, 0, 322, 105, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 7671, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6713, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8331, 7914, 8157, 8156, 8183, 8204, 8363] avg=8192.0000 max=8363 max_rank=7 min=7914 min_rank=2 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32906 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 291, 80, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6935, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 7956, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8175, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5148, 0, 0, 0, 3, 1, 0, 2, 0, 4595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 7347, 8505, 6960, 8315, 8196, 8183, 9749] avg=8192.0000 max=9749 max_rank=7 min=6960 min_rank=3 imbalance_ratio=1.190063 max_load_reduction=55787 max_load_reduction_pct=0.851242 moved_to_echo=34443 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7799, 0, 0, 0, 0, 0, 857, 0, 7901, 0, 0, 0, 0, 0, 0, 18, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6905, 0, 1, 0, 0, 0, 0, 0, 0, 1091, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 1, 0, 0, 24, 0, 8068, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 0, 16, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7333, 0, 0, 0, 0, 0, 0, 0, 0, 923, 0, 8165, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 238, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 16, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 14, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7013, 0, 0, 0, 0, 0, 0, 0, 0, 955, 0, 8168, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7110, 0, 0, 0, 0, 0, 1420, 0, 8011, 0, 0, 0, 0, 0, 0, 31, 0, 8190, 0, 0, 8, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7451, 0, 0, 0, 0, 0, 0, 0, 0, 548, 0, 8185, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 13, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 8178, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 7, 8, 0, 8183, 0, 0, 0, 0, 0, 6, 0, 0, 8096, 0, 0, 1, 39, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8182, 0, 49, 0, 0, 4, 9, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7270, 0, 0, 3, 8, 0, 0, 4, 0, 959, 0, 8155, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7826, 0, 0, 0, 0, 0, 398, 0, 8146, 0, 0, 0, 0, 0, 0, 37, 0, 8186, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7137, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7727, 0, 0, 0, 0, 0, 779, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7631, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 10, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 7112, 0, 2, 1, 187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 19, 2, 0, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5005, 0, 0, 0, 3, 1, 0, 2, 0, 4677, 0, 8165, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8054, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 1, 0, 199, 90, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7852, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7243, 0, 0, 0, 0, 0, 0, 0, 0, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8268, 8481, 7894, 8200, 8202, 8179, 8166] avg=8192.0000 max=8481 max_rank=2 min=7894 min_rank=3 imbalance_ratio=1.035278 max_load_reduction=57055 max_load_reduction_pct=0.870590 moved_to_echo=32747 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7799, 0, 0, 0, 0, 199, 85, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 7856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6820, 0, 1, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7981, 8278, 8099, 8731, 8152, 8177, 8206, 7912] avg=8192.0000 max=8731 max_rank=3 min=7912 min_rank=7 imbalance_ratio=1.065796 max_load_reduction=56805 max_load_reduction_pct=0.866776 moved_to_echo=32447 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 199, 86, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7841, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6927, 0, 0, 0, 0, 0, 0, 0, 0, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8247, 8483, 8095, 8192, 8191, 8183, 7882] avg=8192.0000 max=8483 max_rank=2 min=7882 min_rank=7 imbalance_ratio=1.035522 max_load_reduction=57053 max_load_reduction_pct=0.870560 moved_to_echo=32448 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7110, 0, 0, 0, 0, 205, 96, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 7843, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8020, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7355, 0, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8094, 8265, 7427, 9302, 8172, 8176, 8197, 7903] avg=8192.0000 max=9302 max_rank=3 min=7427 min_rank=2 imbalance_ratio=1.135498 max_load_reduction=56234 max_load_reduction_pct=0.858063 moved_to_echo=32448 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 13, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 2, 0, 8169, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 197, 90, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7763, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 8002, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8168, 0, 49, 0, 0, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8155, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7180, 0, 0, 3, 8, 0, 0, 4, 0, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8205, 8504, 7785, 8192, 8237, 8178, 8154] avg=8192.0000 max=8504 max_rank=2 min=7785 min_rank=3 imbalance_ratio=1.038086 max_load_reduction=57032 max_load_reduction_pct=0.870239 moved_to_echo=32761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7727, 0, 0, 0, 0, 198, 98, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7813, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7819, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8277, 8037, 8599, 8152, 8176, 8205, 7821] avg=8192.0000 max=8599 max_rank=3 min=7821 min_rank=7 imbalance_ratio=1.049683 max_load_reduction=56937 max_load_reduction_pct=0.868790 moved_to_echo=32354 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7826, 0, 0, 0, 0, 199, 88, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 7850, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7049, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8278, 8127, 8297, 8152, 8182, 8205, 8071] avg=8192.0000 max=8297 max_rank=3 min=8071 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32610 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 173, 61, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6820, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 8012, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4944, 0, 0, 0, 3, 1, 0, 2, 0, 4677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 7718, 8378, 6833, 8324, 8195, 8183, 9627] avg=8192.0000 max=9627 max_rank=7 min=6833 min_rank=3 imbalance_ratio=1.175171 max_load_reduction=55909 max_load_reduction_pct=0.853104 moved_to_echo=34329 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 62080, 0, 0, 0, 0, 2560, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 4864, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 60544, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[384, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 128, 0, 63744, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 54784, 0, 128, 128, 128, 128, 0, 128, 0, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 64768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 1152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 64512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 3840, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 61696, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 63232, 0, 0, 128, 0, 1664, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 55424, 0, 128, 128, 128, 128, 0, 128, 0, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 128, 0, 64128, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 64896, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 65408, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 7277, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6, 0, 861, 0, 0, 0, 0, 8169, 176, 0, 0, 0, 0, 8053, 0, 41, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 3294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 275, 0, 927, 0, 0, 0, 0, 8183, 162, 0, 0, 0, 0, 8160, 0, 3557, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 29, 0, 0, 0, 0, 0, 13, 0, 9, 8180, 0, 8178, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6902, 0, 0, 0, 0, 6, 3, 3, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 12, 8065, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 538, 0, 734, 0, 1, 0, 0, 8170, 2, 0, 0, 7, 0, 8154, 0, 38, 8180, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 6168, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 42, 0, 1991, 0, 0, 0, 0, 8191, 85, 0, 0, 0, 0, 8116, 0, 1, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 0, 0, 5785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 0, 1448, 0, 0, 0, 0, 8192, 122, 0, 0, 0, 0, 8189, 1, 405, 8186, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 5424, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 1494, 0, 0, 0, 0, 8192, 324, 0, 0, 0, 0, 8121, 0, 902, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 5881, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 683, 0, 1581, 0, 0, 0, 0, 8192, 126, 0, 0, 0, 0, 8105, 0, 3, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 0, 8185, 0, 7992, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 19, 4215, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 8179, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 4, 0, 4198, 0, 0, 0, 0, 8161, 30, 0, 0, 0, 2, 8085, 0, 0, 8189, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8155, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 7277, 0, 0, 0, 2305, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8180, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 5739, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8135, 8181, 8154, 9796, 8253, 8006, 6817] avg=8192.0000 max=9796 max_rank=4 min=6817 min_rank=7 imbalance_ratio=1.195801 max_load_reduction=55740 max_load_reduction_pct=0.850525 moved_to_echo=32872 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8170, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8128, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 3294, 0, 0, 0, 2336, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 5815, 0, 3557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8231, 8187, 8158, 5840, 8196, 8275, 10461] avg=8192.0000 max=10461 max_rank=7 min=5840 min_rank=4 imbalance_ratio=1.276978 max_load_reduction=55075 max_load_reduction_pct=0.840378 moved_to_echo=32772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8178, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8126, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 6168, 0, 0, 0, 2323, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1991, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 5784, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8162, 8186, 8149, 8686, 8255, 8042, 7861] avg=8192.0000 max=8686 max_rank=4 min=7861 min_rank=7 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=32844 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8179, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 8131, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 0, 0, 5785, 0, 0, 0, 2344, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1448, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 5836, 1, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8240, 8186, 8365, 8595, 8149, 8008, 7812] avg=8192.0000 max=8595 max_rank=4 min=7812 min_rank=7 imbalance_ratio=1.049194 max_load_reduction=56941 max_load_reduction_pct=0.868851 moved_to_echo=32564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 29, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 9, 48, 0, 8178, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 71, 0, 0, 0, 8157, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 2, 0, 0, 8132, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6902, 0, 0, 0, 2334, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 3, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 12, 8065, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 7986, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 0, 1, 0, 0, 0, 2, 0, 0, 7, 0, 5811, 0, 38, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8257, 8241, 8163, 9424, 8102, 8532, 6603] avg=8192.0000 max=9424 max_rank=4 min=6603 min_rank=7 imbalance_ratio=1.150391 max_load_reduction=56112 max_load_reduction_pct=0.856201 moved_to_echo=32661 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 8144, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 5424, 0, 0, 0, 2329, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7987, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 5780, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8235, 8185, 8185, 8052, 8205, 7988, 8500] avg=8192.0000 max=8500 max_rank=7 min=7988 min_rank=6 imbalance_ratio=1.037598 max_load_reduction=57036 max_load_reduction_pct=0.870300 moved_to_echo=32745 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 8144, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 5881, 0, 0, 0, 2320, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1581, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 5776, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8216, 8185, 8198, 8404, 8178, 8683, 7486] avg=8192.0000 max=8683 max_rank=6 min=7486 min_rank=7 imbalance_ratio=1.059937 max_load_reduction=56853 max_load_reduction_pct=0.867508 moved_to_echo=32751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 0, 48, 0, 7992, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 22, 0, 0, 0, 8148, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8137, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 19, 4215, 0, 0, 0, 2314, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 8179, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7996, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4198, 0, 0, 0, 0, 0, 30, 0, 0, 0, 2, 5762, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8060, 8185, 8164, 6739, 8198, 8002, 9996] avg=8192.0000 max=9996 max_rank=7 min=6739 min_rank=4 imbalance_ratio=1.220215 max_load_reduction=55540 max_load_reduction_pct=0.847473 moved_to_echo=32935 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 7464, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 722, 0, 0, 0, 0, 8169, 181, 0, 0, 0, 0, 8039, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 0, 6456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1168, 0, 0, 0, 0, 8192, 106, 0, 0, 0, 0, 8189, 1, 44, 8185, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 29, 0, 0, 0, 0, 0, 13, 0, 10, 8180, 0, 8179, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6900, 0, 0, 0, 0, 6, 2, 3, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 12, 8065, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 370, 0, 944, 0, 1, 0, 0, 8169, 1, 0, 0, 7, 0, 8157, 0, 2, 8180, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 6346, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 24, 0, 1831, 0, 0, 0, 0, 8191, 89, 0, 0, 0, 0, 8110, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 5122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 413, 0, 1958, 0, 0, 0, 0, 8183, 169, 0, 0, 0, 0, 8158, 0, 557, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 5949, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 385, 0, 1813, 0, 0, 0, 0, 8192, 126, 0, 0, 0, 0, 8107, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 1, 0, 0, 0, 0, 2, 17, 0, 0, 8185, 0, 7995, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 18, 4147, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 8178, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 1, 0, 4282, 0, 0, 0, 0, 8161, 23, 0, 0, 0, 1, 8070, 0, 0, 8189, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 6282, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 0, 1545, 0, 0, 0, 0, 8192, 281, 0, 0, 0, 0, 8120, 0, 47, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 8156, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 7464, 0, 0, 0, 1881, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8180, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8136, 8180, 8162, 9515, 8257, 8046, 7046] avg=8192.0000 max=9515 max_rank=4 min=7046 min_rank=7 imbalance_ratio=1.161499 max_load_reduction=56021 max_load_reduction_pct=0.854813 moved_to_echo=32864 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8179, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 8134, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 0, 6456, 0, 0, 0, 1916, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 6258, 1, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8238, 8187, 8317, 8847, 8147, 8044, 7577] avg=8192.0000 max=8847 max_rank=4 min=7577 min_rank=7 imbalance_ratio=1.079956 max_load_reduction=56689 max_load_reduction_pct=0.865005 moved_to_echo=32615 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 6346, 0, 0, 0, 1898, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1831, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 6197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8163, 8184, 8158, 8395, 8255, 8068, 8117] avg=8192.0000 max=8395 max_rank=4 min=8068 min_rank=6 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32835 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8171, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8132, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 5122, 0, 0, 0, 1909, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 6234, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8229, 8188, 8165, 7195, 8196, 8457, 8918] avg=8192.0000 max=8918 max_rank=7 min=7195 min_rank=4 imbalance_ratio=1.088623 max_load_reduction=56618 max_load_reduction_pct=0.863922 moved_to_echo=32766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 1, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 17, 0, 0, 46, 0, 7995, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 25, 0, 0, 0, 8149, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8139, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 18, 4147, 0, 0, 0, 1888, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4282, 0, 0, 0, 0, 0, 23, 0, 0, 0, 1, 6167, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8062, 8190, 8171, 6200, 8199, 8045, 10477] avg=8192.0000 max=10477 max_rank=7 min=6200 min_rank=4 imbalance_ratio=1.278931 max_load_reduction=55059 max_load_reduction_pct=0.840134 moved_to_echo=32921 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8147, 19, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 6282, 0, 0, 0, 1905, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 6196, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8234, 8186, 8189, 8428, 8205, 8039, 8069] avg=8192.0000 max=8428 max_rank=4 min=8039 min_rank=6 imbalance_ratio=1.028809 max_load_reduction=57108 max_load_reduction_pct=0.871399 moved_to_echo=32741 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 29, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 10, 46, 0, 8179, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 66, 0, 0, 0, 8156, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 8134, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6900, 0, 0, 0, 1909, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 3, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 12, 8065, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 0, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 0, 1, 0, 0, 0, 1, 0, 0, 7, 0, 6233, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8257, 8235, 8170, 8952, 8101, 8408, 7198] avg=8192.0000 max=8952 max_rank=4 min=7198 min_rank=7 imbalance_ratio=1.092773 max_load_reduction=56584 max_load_reduction_pct=0.863403 moved_to_echo=32659 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 8146, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 5949, 0, 0, 0, 1897, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1813, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 6195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8217, 8186, 8204, 8004, 8176, 8429, 8134] avg=8192.0000 max=8429 max_rank=6 min=8004 min_rank=4 imbalance_ratio=1.028931 max_load_reduction=57107 max_load_reduction_pct=0.871384 moved_to_echo=32743 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 512, 0, 0, 0, 128, 0, 0, 128, 45056, 0, 0, 0, 18688, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[13312, 0, 128, 0, 0, 0, 1152, 0, 0, 128, 128, 46336, 128, 4992, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 64000, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 512, 0, 0, 0, 128, 0, 0, 128, 48768, 0, 0, 0, 15232, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[14336, 0, 128, 0, 0, 0, 1024, 0, 0, 128, 128, 49664, 128, 768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 64384, 0, 0, 0, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 384, 0, 65152, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 384, 0, 65152, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 384, 128, 0, 0, 65152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 256, 128, 0, 0, 65152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 0, 1, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7557, 0, 8192, 0, 0, 0, 8185, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4026, 0, 2, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4182, 0, 8192, 0, 0, 0, 8189, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7973, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 8192, 0, 0, 0, 8187, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 0, 0, 0, 0, 2237, 0, 187, 1465, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 11, 8, 0, 0, 0, 5, 0, 0, 0, 0, 0, 16, 0, 0, 2, 0, 0, 5489, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8151, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8174, 6, 0, 6, 0, 0, 8166, 12, 0, 0, 0, 0, 0, 0, 0, 13, 6972, 3, 8167, 0, 0, 0, 8180, 2, 8, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 999, 0, 0, 397, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 7593, 0, 0, 0, 22, 0, 0, 1, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8192, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7346, 0, 8192, 0, 0, 1, 8163, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4615, 0, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2483, 0, 8192, 0, 0, 0, 8160, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1693, 0, 1, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6049, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 8192, 0, 0, 0, 8178, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 281, 3285, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 20, 1, 0, 0, 4543, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 3, 0, 8186, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 4, 1, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 5, 8184, 4, 8183, 0, 2, 0, 8186, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 0, 5433, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7104, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 7876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7557, 0, 0, 0, 0, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7275, 7210, 8187, 9309, 8191, 8186, 8060, 9118] avg=8192.0000 max=9309 max_rank=3 min=7210 min_rank=1 imbalance_ratio=1.136353 max_load_reduction=56227 max_load_reduction_pct=0.857956 moved_to_echo=33555 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 999, 0, 5419, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 7090, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 7593, 0, 0, 0, 22, 0, 0, 1, 0, 0, 0, 0, 1188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7346, 0, 0, 0, 0, 1, 1556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7404, 7505, 8187, 8837, 8215, 8215, 8270, 8903] avg=8192.0000 max=8903 max_rank=7 min=7404 min_rank=0 imbalance_ratio=1.086792 max_load_reduction=56633 max_load_reduction_pct=0.864151 moved_to_echo=33603 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4026, 0, 5436, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7104, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 7690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4182, 0, 0, 0, 0, 0, 1562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10385, 7841, 8187, 9122, 8191, 8192, 7874, 5744] avg=8192.0000 max=10385 max_rank=0 min=5744 min_rank=7 imbalance_ratio=1.267700 max_load_reduction=55151 max_load_reduction_pct=0.841537 moved_to_echo=30001 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 5435, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7102, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7973, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 1191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6545, 7159, 8187, 9210, 8193, 8192, 8320, 9730] avg=8192.0000 max=9730 max_rank=7 min=6545 min_rank=0 imbalance_ratio=1.187744 max_load_reduction=55806 max_load_reduction_pct=0.851532 moved_to_echo=34435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4615, 0, 5421, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7109, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 7715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2483, 0, 0, 0, 0, 0, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10965, 8762, 8187, 9325, 8189, 8186, 7891, 4031] avg=8192.0000 max=10965 max_rank=0 min=4031 min_rank=7 imbalance_ratio=1.338501 max_load_reduction=54571 max_load_reduction_pct=0.832687 moved_to_echo=28297 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 0, 0, 0, 0, 2237, 0, 5430, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 1465, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7088, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 11, 8, 0, 0, 0, 5, 0, 0, 0, 0, 0, 16, 0, 8157, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5489, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 6, 0, 6, 0, 0, 8166, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6972, 3, 0, 0, 0, 0, 1560, 2, 8, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8583, 8752, 8221, 6683, 8176, 8180, 8374, 8567] avg=8192.0000 max=8752 max_rank=1 min=6683 min_rank=3 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=33297 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1693, 0, 5429, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7104, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6049, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8032, 7637, 8187, 7244, 8191, 8187, 8371, 9687] avg=8192.0000 max=9687 max_rank=7 min=7244 min_rank=3 imbalance_ratio=1.182495 max_load_reduction=55849 max_load_reduction_pct=0.852188 moved_to_echo=34436 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 5434, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 3285, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 7085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 8172, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 0, 0, 4543, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 1191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8185, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 4, 1, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8184, 4, 0, 0, 2, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6347, 10670, 8193, 5806, 8190, 8198, 8376, 9756] avg=8192.0000 max=10670 max_rank=1 min=5806 min_rank=3 imbalance_ratio=1.302490 max_load_reduction=54866 max_load_reduction_pct=0.837189 moved_to_echo=34520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 866, 0, 0, 138, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 7577, 0, 0, 0, 22, 0, 0, 3, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8192, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7726, 0, 8192, 0, 0, 1, 8162, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 904, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7622, 0, 8192, 0, 0, 0, 8185, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 2955, 0, 175, 1129, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 11, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 16, 0, 0, 3, 0, 0, 4811, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8151, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8174, 6, 0, 6, 0, 0, 8168, 12, 0, 0, 0, 0, 0, 0, 0, 13, 7277, 2, 8167, 0, 0, 0, 8180, 2, 8, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4641, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 8192, 0, 0, 0, 8189, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4932, 0, 0, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 0, 8192, 0, 0, 0, 8160, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 326, 2715, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 23, 3, 0, 0, 5053, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 4, 1, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 5, 8184, 4, 8183, 0, 2, 1, 8186, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1728, 0, 1, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5749, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 8192, 0, 0, 0, 8179, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 904, 0, 5441, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7311, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7622, 0, 0, 0, 0, 0, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7042, 7373, 8187, 9379, 8191, 8186, 8078, 9100] avg=8192.0000 max=9379 max_rank=3 min=7042 min_rank=0 imbalance_ratio=1.144897 max_load_reduction=56157 max_load_reduction_pct=0.856888 moved_to_echo=33555 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 866, 0, 5425, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7295, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 7577, 0, 0, 0, 22, 0, 0, 3, 0, 0, 0, 0, 1262, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7726, 0, 0, 0, 0, 1, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7069, 7450, 8186, 8902, 8215, 8217, 8295, 9202] avg=8192.0000 max=9202 max_rank=7 min=7069 min_rank=0 imbalance_ratio=1.123291 max_load_reduction=56334 max_load_reduction_pct=0.859589 moved_to_echo=33929 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 5441, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7310, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 1266, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6415, 7356, 8187, 9214, 8193, 8192, 8331, 9648] avg=8192.0000 max=9648 max_rank=7 min=6415 min_rank=0 imbalance_ratio=1.177734 max_load_reduction=55888 max_load_reduction_pct=0.852783 moved_to_echo=34364 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 2955, 0, 5437, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 1129, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7295, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 11, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 16, 0, 8158, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4811, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1265, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 6, 0, 6, 0, 0, 8168, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7277, 2, 0, 0, 0, 0, 1478, 2, 8, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9102, 8609, 8220, 6083, 8176, 8181, 8376, 8789] avg=8192.0000 max=9102 max_rank=0 min=6083 min_rank=3 imbalance_ratio=1.111084 max_load_reduction=56434 max_load_reduction_pct=0.861115 moved_to_echo=33522 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4641, 0, 5443, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7311, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 7649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 0, 0, 0, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10799, 7721, 8188, 9192, 8191, 8191, 7834, 5420] avg=8192.0000 max=10799 max_rank=0 min=5420 min_rank=7 imbalance_ratio=1.318237 max_load_reduction=54737 max_load_reduction_pct=0.835220 moved_to_echo=29636 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4932, 0, 5428, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7316, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1265, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 7696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 0, 0, 0, 0, 0, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11083, 8615, 8187, 9352, 8190, 8186, 7872, 4051] avg=8192.0000 max=11083 max_rank=0 min=4051 min_rank=7 imbalance_ratio=1.352905 max_load_reduction=54453 max_load_reduction_pct=0.830887 moved_to_echo=28299 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1728, 0, 5437, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7311, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5749, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1265, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7867, 8061, 8187, 7017, 8191, 8187, 8375, 9651] avg=8192.0000 max=9651 max_rank=7 min=7017 min_rank=3 imbalance_ratio=1.178101 max_load_reduction=55885 max_load_reduction_pct=0.852737 moved_to_echo=34404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 5441, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 2715, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 7293, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 8173, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3, 0, 0, 5053, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 1266, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 4, 1, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8184, 4, 0, 0, 2, 1, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 10351, 8194, 6397, 8189, 8196, 8375, 9675] avg=8192.0000 max=10351 max_rank=1 min=6159 min_rank=0 imbalance_ratio=1.263550 max_load_reduction=55185 max_load_reduction_pct=0.842056 moved_to_echo=34435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 14720, 0, 43520, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 128, 53120, 128, 0, 0, 128, 128, 12544, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 55936, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 9600, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 8192, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 56832, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 16384, 0, 43520, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 128, 53760, 128, 0, 0, 128, 128, 11904, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 55296, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 10240, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 6528, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 58496, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 1536, 128, 128, 128, 0, 0, 64128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 16, 8183, 0, 1, 0, 0, 0, 0, 0, 8007, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8192, 0, 185, 0, 0, 8180, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8084, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 108, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8154, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 37, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8162, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 27, 0, 0, 8192, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 8053, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8192, 0, 139, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 6, 8180, 0, 0, 0, 0, 0, 0, 0, 7279, 8183, 21, 5, 0, 0, 0, 3, 0, 0, 5, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 2, 0, 0, 0, 8183, 0, 934, 0, 4, 8170, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 12, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 8180, 8176, 1, 11, 0, 0, 0, 6, 49, 0, 16, 0, 0, 11, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 15, 15, 0, 0, 0, 0, 3, 0, 8169, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 5, 91, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 16, 0, 0, 1, 0, 0, 0, 0, 0, 8007, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 185, 0, 0, 8008, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8004, 155, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8178, 8203, 8205, 8169, 8161, 8195, 8220] avg=8192.0000 max=8220 max_rank=7 min=8161 min_rank=5 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32745 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8020, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8022, 155, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8351, 8183, 8206, 8024, 8207, 8179, 8201, 8185] avg=8192.0000 max=8351 max_rank=0 min=8024 min_rank=3 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 139, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 155, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8183, 8203, 8165, 8191, 8172, 8197, 8198] avg=8192.0000 max=8227 max_rank=0 min=8165 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32758 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8020, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8021, 155, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 8183, 8203, 8057, 8197, 8180, 8195, 8189] avg=8192.0000 max=8332 max_rank=0 min=8057 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 8012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 159, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8177, 8211, 8049, 8209, 8189, 8196, 8185] avg=8192.0000 max=8320 max_rank=0 min=8049 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32779 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 169, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 6, 49, 0, 16, 0, 0, 11, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 0, 0, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8169, 0, 0, 0, 0, 0, 3, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 5, 91, 12, 8007, 153, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 8258, 8125, 7955, 8190, 8292, 8170, 8179] avg=8192.0000 max=8367 max_rank=0 min=7955 min_rank=3 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32831 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 108, 0, 0, 8020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8022, 155, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8183, 8203, 8130, 8204, 8180, 8196, 8186] avg=8192.0000 max=8254 max_rank=0 min=8130 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7279, 169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 5, 0, 0, 2, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 934, 0, 4, 7998, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 5, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 8014, 155, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7480, 8191, 8182, 8951, 8169, 8183, 8186, 8194] avg=8192.0000 max=8951 max_rank=3 min=7480 min_rank=0 imbalance_ratio=1.092651 max_load_reduction=56585 max_load_reduction_pct=0.863419 moved_to_echo=32732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 11, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 8180, 8176, 1, 11, 0, 0, 0, 6, 43, 0, 16, 0, 0, 11, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 16, 15, 0, 0, 0, 0, 2, 0, 8169, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 6, 92, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 8051, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8192, 0, 141, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 16, 8183, 0, 1, 0, 0, 0, 0, 0, 8003, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8192, 0, 189, 0, 0, 8180, 0, 0, 4, 1, 0, 0, 0, 0, 0, 2, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8161, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 28, 0, 0, 8192, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8154, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 38, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8083, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 109, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 4, 8180, 0, 0, 0, 0, 0, 0, 0, 7234, 8183, 19, 5, 0, 0, 0, 3, 0, 0, 5, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 6, 0, 3, 0, 0, 0, 2, 0, 0, 0, 8184, 0, 981, 0, 4, 8169, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8180, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 6, 43, 0, 16, 0, 0, 11, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7916, 0, 0, 0, 0, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8169, 0, 0, 0, 0, 0, 3, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 6, 92, 12, 7999, 161, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8374, 8253, 8129, 7947, 8189, 8294, 8171, 8179] avg=8192.0000 max=8374 max_rank=0 min=7947 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=57162 max_load_reduction_pct=0.872223 moved_to_echo=32833 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 141, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 162, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8183, 8203, 8162, 8190, 8170, 8196, 8199] avg=8192.0000 max=8233 max_rank=0 min=8162 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 8013, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8013, 162, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8183, 8203, 8052, 8196, 8178, 8196, 8190] avg=8192.0000 max=8338 max_rank=0 min=8052 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 162, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8357, 8183, 8207, 8017, 8207, 8180, 8200, 8185] avg=8192.0000 max=8357 max_rank=0 min=8017 min_rank=3 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8020, 168, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8180, 8207, 8038, 8213, 8192, 8195, 8185] avg=8192.0000 max=8326 max_rank=0 min=8038 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 109, 0, 0, 8013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 162, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8183, 8203, 8125, 8204, 8180, 8196, 8185] avg=8192.0000 max=8260 max_rank=0 min=8125 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 16, 0, 0, 1, 0, 0, 0, 0, 0, 8003, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 189, 0, 0, 8001, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7996, 162, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8179, 8202, 8204, 8167, 8160, 8195, 8220] avg=8192.0000 max=8220 max_rank=7 min=8160 min_rank=5 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7234, 177, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 5, 0, 0, 2, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 6, 0, 3, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 981, 0, 4, 7990, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 5, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 8006, 162, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7439, 8192, 8182, 8991, 8170, 8182, 8187, 8193] avg=8192.0000 max=8991 max_rank=3 min=7439 min_rank=0 imbalance_ratio=1.097534 max_load_reduction=56545 max_load_reduction_pct=0.862808 moved_to_echo=32732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 64128, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 64256, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 64128, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 64128, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 0, 0, 0, 1536, 0, 128, 64128, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 65408, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 0, 0, 1536, 0, 128, 64000, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 128, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8184, 0, 7, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 8192, 0, 0, 24, 0, 0, 8173, 0, 0, 80, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 52, 0, 11, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 201, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 316, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 7738, 0, 0, 0, 8174, 12, 0, 0, 24, 0, 0, 0, 0, 8102, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 3, 8168, 0, 0, 0, 0, 7, 8134, 13, 0, 101, 8173, 0, 0, 0, 0, 11, 0, 15, 13, 0, 0, 0, 0, 0, 0, 118, 2, 0, 3, 0, 0, 0, 1522, 8, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 9, 0, 8178, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 14, 0, 0, 0, 0, 0, 6681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 739, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 1275, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 1308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 8192, 0, 0, 234, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 2172, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 8180, 0, 0, 0, 8179, 3, 4, 0, 0, 0, 0, 9, 0, 7884, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 14, 0, 0, 0, 0, 0, 8183, 0, 0, 42, 0, 4, 8150, 0, 0, 506, 8182, 0, 0, 0, 0, 5, 5, 3, 3, 0, 0, 0, 0, 0, 0, 677, 0, 0, 0, 0, 0, 0, 42, 0, 11, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 6, 0, 8166, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6958, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 52, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8005, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7020, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8252, 8239, 8018, 7971, 8226, 8182, 8402] avg=8192.0000 max=8402 max_rank=7 min=7971 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 52, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 7906, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, 8004, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 11, 0, 0, 0, 7, 0, 0, 0, 0, 7013, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7924, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1119, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8193, 8010, 8277, 7271, 7993, 8235, 9307] avg=8192.0000 max=9307 max_rank=7 min=7271 min_rank=4 imbalance_ratio=1.136108 max_load_reduction=56229 max_load_reduction_pct=0.857986 moved_to_echo=32806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 8005, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 7020, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8236, 8124, 8172, 7319, 8121, 8203, 9117] avg=8192.0000 max=9117 max_rank=7 min=7319 min_rank=4 imbalance_ratio=1.112915 max_load_reduction=56419 max_load_reduction_pct=0.860886 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8005, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7025, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8256, 7914, 8032, 8520, 8226, 8181, 8172] avg=8192.0000 max=8520 max_rank=4 min=7914 min_rank=2 imbalance_ratio=1.040039 max_load_reduction=57016 max_load_reduction_pct=0.869995 moved_to_echo=33099 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 52, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7923, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 8005, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2172, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 7020, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8246, 8158, 8100, 9458, 8244, 8188, 6896] avg=8192.0000 max=9458 max_rank=4 min=6896 min_rank=7 imbalance_ratio=1.154541 max_load_reduction=56078 max_load_reduction_pct=0.855682 moved_to_echo=32786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 7738, 0, 0, 0, 52, 12, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7, 7867, 13, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 15, 13, 0, 0, 0, 0, 0, 0, 118, 2, 7981, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1522, 8, 0, 0, 0, 0, 267, 0, 0, 0, 0, 7005, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 9, 0, 8178, 0, 0, 10, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 14, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7828, 8173, 7994, 8150, 8992, 8274, 8189, 7936] avg=8192.0000 max=8992 max_rank=4 min=7828 min_rank=0 imbalance_ratio=1.097656 max_load_reduction=56544 max_load_reduction_pct=0.862793 moved_to_echo=33391 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 8005, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7020, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8210, 8662, 8087, 8484, 8189, 8183, 7477] avg=8192.0000 max=8662 max_rank=2 min=7477 min_rank=7 imbalance_ratio=1.057373 max_load_reduction=56874 max_load_reduction_pct=0.867828 moved_to_echo=32333 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 8180, 0, 0, 0, 52, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7884, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 14, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 4, 7883, 0, 0, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 3, 3, 0, 0, 0, 0, 0, 0, 677, 0, 7997, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 11, 0, 0, 0, 273, 0, 0, 0, 0, 7009, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 6, 0, 8166, 0, 0, 8, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6958, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1118, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 7970, 8435, 8700, 7521, 8263, 8175, 8229] avg=8192.0000 max=8700 max_rank=3 min=7521 min_rank=4 imbalance_ratio=1.062012 max_load_reduction=56836 max_load_reduction_pct=0.867249 moved_to_echo=32188 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 7658, 0, 0, 0, 8174, 12, 0, 0, 25, 0, 0, 0, 0, 8094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 3, 8168, 0, 0, 0, 0, 7, 8132, 13, 0, 101, 8173, 0, 0, 0, 0, 11, 0, 15, 13, 0, 0, 0, 0, 0, 0, 138, 2, 0, 3, 0, 0, 0, 1548, 8, 0, 0, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 9, 0, 8178, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 14, 0, 0, 0, 0, 0, 6700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 234, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 177, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 779, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 1632, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8184, 0, 6, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 8192, 0, 0, 26, 0, 0, 8173, 0, 0, 64, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 46, 0, 11, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 8192, 0, 0, 192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 1532, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 8180, 0, 0, 0, 8179, 3, 4, 0, 0, 0, 0, 9, 0, 7842, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 0, 0, 0, 0, 8183, 0, 0, 46, 0, 4, 8150, 0, 0, 613, 8182, 0, 0, 0, 0, 5, 5, 4, 3, 0, 0, 0, 0, 0, 0, 782, 0, 0, 0, 0, 0, 0, 46, 0, 11, 0, 0, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 4, 0, 8166, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6742, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 7658, 0, 0, 0, 62, 12, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7, 7867, 13, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 15, 13, 0, 0, 0, 0, 0, 0, 138, 2, 7956, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1548, 8, 0, 0, 0, 0, 292, 0, 0, 0, 0, 7001, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 9, 0, 8178, 0, 0, 10, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 14, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7759, 8171, 7994, 8145, 9064, 8278, 8189, 7936] avg=8192.0000 max=9064 max_rank=4 min=7759 min_rank=0 imbalance_ratio=1.106445 max_load_reduction=56472 max_load_reduction_pct=0.861694 moved_to_echo=33467 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 7980, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 7017, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8244, 8102, 8159, 7324, 8113, 8204, 9137] avg=8192.0000 max=9137 max_rank=7 min=7324 min_rank=4 imbalance_ratio=1.115356 max_load_reduction=56399 max_load_reduction_pct=0.860580 moved_to_echo=32778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 61, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7980, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7017, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8259, 8159, 7993, 7956, 8234, 8182, 8498] avg=8192.0000 max=8498 max_rank=7 min=7956 min_rank=4 imbalance_ratio=1.037354 max_load_reduction=57038 max_load_reduction_pct=0.870331 moved_to_echo=32870 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 7980, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1632, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 7017, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8218, 8704, 8071, 8864, 8191, 8184, 7051] avg=8192.0000 max=8864 max_rank=4 min=7051 min_rank=7 imbalance_ratio=1.082031 max_load_reduction=56672 max_load_reduction_pct=0.864746 moved_to_echo=32290 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 61, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7925, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 7980, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1532, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 7017, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8253, 8118, 8084, 8874, 8245, 8187, 7520] avg=8192.0000 max=8874 max_rank=4 min=7520 min_rank=7 imbalance_ratio=1.083252 max_load_reduction=56662 max_load_reduction_pct=0.864594 moved_to_echo=32826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7912, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 7981, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7019, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8260, 7914, 8013, 8582, 8232, 8181, 8105] avg=8192.0000 max=8582 max_rank=4 min=7914 min_rank=2 imbalance_ratio=1.047607 max_load_reduction=56954 max_load_reduction_pct=0.869049 moved_to_echo=33100 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 62, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 7907, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 7980, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 11, 0, 0, 0, 6, 0, 0, 0, 0, 7010, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7903, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8198, 7997, 8289, 7285, 7977, 8235, 9296] avg=8192.0000 max=9296 max_rank=7 min=7285 min_rank=4 imbalance_ratio=1.134766 max_load_reduction=56240 max_load_reduction_pct=0.858154 moved_to_echo=32793 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 8180, 0, 0, 0, 61, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7842, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4, 7885, 0, 0, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4, 3, 0, 0, 0, 0, 0, 0, 782, 0, 7972, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 11, 0, 0, 0, 313, 0, 0, 0, 0, 7006, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 4, 0, 8166, 0, 0, 8, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6742, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1112, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 7933, 8548, 8782, 7587, 8266, 8174, 7993] avg=8192.0000 max=8782 max_rank=3 min=7587 min_rank=4 imbalance_ratio=1.072021 max_load_reduction=56754 max_load_reduction_pct=0.865997 moved_to_echo=32020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 55552, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 8960, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 7296, 128, 128, 0, 0, 0, 640, 0, 0, 0, 0, 56192, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 63360, 128, 0, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 1408, 128, 64128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 55680, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 8960, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 7040, 128, 128, 0, 0, 0, 768, 0, 0, 0, 0, 56192, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 63360, 128, 0, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 1664, 128, 63872, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 65024, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 65024, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 0, 65024, 0, 0, 128, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 0, 65152, 0, 0, 0, 512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 0, 64896, 0, 0, 128, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 65024, 0, 0, 0, 512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2462, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 8187, 0, 8175, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 79, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8108, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 17, 8183, 0, 8177, 0, 0, 2, 0, 0, 0, 8183, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 5, 0, 0, 0, 8171, 0, 0, 11, 32, 0, 0, 4, 0, 11, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8059, 0, 6, 8178, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 33, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5376, 8192, 0, 8163, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 4617, 8173, 0, 8129, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 16, 0, 0, 33, 7966, 0, 32, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 1, 8167, 0, 0, 0, 11, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 3537, 4, 0, 0, 0, 17, 0, 14, 0, 0, 0, 3, 0, 0, 0, 0, 0, 12, 0, 197, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6351, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 5520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 10, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 41, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7903, 0, 0, 0, 0, 0, 0, 0, 2640, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8011, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 5800, 8257, 8208, 8196, 8155, 10673, 8056] avg=8192.0000 max=10673 max_rank=6 min=5800 min_rank=1 imbalance_ratio=1.302856 max_load_reduction=54863 max_load_reduction_pct=0.837143 moved_to_echo=35080 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5376, 0, 0, 5512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2820, 0, 0, 0, 0, 0, 0, 0, 2636, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 8020, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 10888, 8158, 8160, 8195, 8236, 5586, 8122] avg=8192.0000 max=10888 max_rank=1 min=5586 min_rank=6 imbalance_ratio=1.329102 max_load_reduction=54648 max_load_reduction_pct=0.833862 moved_to_echo=30139 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6985, 0, 0, 0, 0, 0, 0, 0, 2646, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8020, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6735, 8203, 8204, 8186, 8197, 9761, 8058] avg=8192.0000 max=9761 max_rank=6 min=6735 min_rank=1 imbalance_ratio=1.191528 max_load_reduction=55775 max_load_reduction_pct=0.851059 moved_to_echo=34202 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2462, 0, 0, 5521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5154, 0, 0, 0, 0, 0, 0, 0, 2649, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 8026, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7983, 8161, 8205, 8199, 8221, 7940, 8635] avg=8192.0000 max=8635 max_rank=7 min=7940 min_rank=6 imbalance_ratio=1.054077 max_load_reduction=56901 max_load_reduction_pct=0.868240 moved_to_echo=32995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6351, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1601, 0, 0, 0, 0, 0, 0, 0, 2646, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 8020, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 11882, 8170, 8195, 8193, 8209, 4377, 8319] avg=8192.0000 max=11882 max_rank=1 min=4377 min_rank=6 imbalance_ratio=1.450439 max_load_reduction=53654 max_load_reduction_pct=0.818695 moved_to_echo=29098 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4617, 0, 0, 5489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 8138, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 33, 7966, 0, 32, 1, 54, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 1, 41, 0, 0, 0, 11, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 3537, 4, 0, 0, 0, 17, 0, 14, 2625, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 12, 0, 197, 0, 0, 0, 0, 7996, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 10106, 8177, 8114, 8178, 8177, 6343, 8243] avg=8192.0000 max=10106 max_rank=1 min=6343 min_rank=6 imbalance_ratio=1.233643 max_load_reduction=55430 max_load_reduction_pct=0.845795 moved_to_echo=30941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7124, 0, 0, 0, 0, 0, 0, 0, 2646, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8020, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6602, 8162, 8204, 8194, 8228, 9900, 8055] avg=8192.0000 max=9900 max_rank=6 min=6602 min_rank=1 imbalance_ratio=1.208496 max_load_reduction=55636 max_load_reduction_pct=0.848938 moved_to_echo=34377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 5521, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 8149, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8171, 0, 0, 11, 32, 0, 0, 4, 0, 11, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 6, 41, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 2641, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8007, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 5540, 8248, 8246, 8195, 8113, 10956, 8048] avg=8192.0000 max=10956 max_rank=6 min=5540 min_rank=1 imbalance_ratio=1.337402 max_load_reduction=54580 max_load_reduction_pct=0.832825 moved_to_echo=35312 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 8187, 0, 8172, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 81, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8108, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4621, 8192, 0, 8165, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4169, 8173, 0, 8132, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 16, 0, 0, 34, 7949, 0, 28, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 1, 8167, 1, 0, 0, 10, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 3961, 5, 0, 0, 0, 18, 0, 14, 0, 0, 0, 3, 0, 0, 0, 0, 0, 12, 0, 232, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6122, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5, 8183, 0, 8177, 0, 0, 2, 0, 0, 0, 8183, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 5, 0, 0, 0, 8170, 0, 0, 11, 32, 0, 0, 4, 0, 11, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8050, 0, 6, 8177, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 37, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4169, 0, 0, 5868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 8135, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 34, 7949, 0, 28, 1, 52, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 1, 42, 1, 0, 0, 10, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 3961, 5, 0, 0, 0, 18, 0, 14, 2247, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 12, 0, 232, 0, 0, 0, 0, 7917, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 10037, 8181, 8094, 8178, 8178, 6469, 8202] avg=8192.0000 max=10037 max_rank=1 min=6469 min_rank=6 imbalance_ratio=1.225220 max_load_reduction=55499 max_load_reduction_pct=0.846848 moved_to_echo=31027 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 5897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 10, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 42, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 2258, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7931, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6028, 8256, 8211, 8196, 8157, 10518, 7979] avg=8192.0000 max=10518 max_rank=6 min=6028 min_rank=1 imbalance_ratio=1.283936 max_load_reduction=55018 max_load_reduction_pct=0.839508 moved_to_echo=34850 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4621, 0, 0, 5892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 2256, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 7941, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 10513, 8155, 8161, 8195, 8236, 6012, 8073] avg=8192.0000 max=10513 max_rank=1 min=6012 min_rank=6 imbalance_ratio=1.283325 max_load_reduction=55023 max_load_reduction_pct=0.839584 moved_to_echo=30516 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6122, 0, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 0, 0, 0, 0, 0, 0, 2263, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 0, 0, 7941, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 12033, 8167, 8198, 8194, 8209, 4210, 8334] avg=8192.0000 max=12033 max_rank=1 min=4210 min_rank=6 imbalance_ratio=1.468872 max_load_reduction=53503 max_load_reduction_pct=0.816391 moved_to_echo=28947 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7577, 0, 0, 0, 0, 0, 0, 0, 2263, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6527, 8159, 8207, 8194, 8230, 10049, 7979] avg=8192.0000 max=10049 max_rank=6 min=6527 min_rank=1 imbalance_ratio=1.226685 max_load_reduction=55487 max_load_reduction_pct=0.846664 moved_to_echo=34452 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 0, 0, 5897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5124, 0, 0, 0, 0, 0, 0, 0, 2269, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 0, 0, 0, 0, 7946, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7929, 8158, 8210, 8200, 8226, 7605, 9016] avg=8192.0000 max=9016 max_rank=7 min=7605 min_rank=6 imbalance_ratio=1.100586 max_load_reduction=56520 max_load_reduction_pct=0.862427 moved_to_echo=33047 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 3, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5900, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8170, 0, 0, 11, 32, 0, 0, 4, 0, 11, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 6, 41, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 2259, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7927, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 5907, 8257, 8248, 8195, 8105, 10662, 7971] avg=8192.0000 max=10662 max_rank=6 min=5907 min_rank=1 imbalance_ratio=1.301514 max_load_reduction=54874 max_load_reduction_pct=0.837311 moved_to_echo=34933 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7539, 0, 0, 0, 0, 0, 0, 0, 2263, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7941, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6562, 8203, 8207, 8184, 8195, 10011, 7982] avg=8192.0000 max=10011 max_rank=6 min=6562 min_rank=1 imbalance_ratio=1.222046 max_load_reduction=55525 max_load_reduction_pct=0.847244 moved_to_echo=34372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 43392, 128, 0, 0, 0, 128, 0, 128, 21248, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 21504, 0, 0, 44160, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 1152, 0, 0, 0, 0, 64128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 45824, 128, 0, 0, 0, 128, 0, 128, 18176, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 18432, 0, 0, 47232, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 1792, 0, 0, 0, 0, 63488, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 384, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 65280, 0, 128, 128, 128, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 65280, 0, 128, 384, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 256, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 65280, 0, 128, 128, 128, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3074, 8192, 0, 0, 27, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 5183, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5542, 8192, 0, 0, 122, 0, 0, 0, 0, 0, 52, 0, 0, 9, 0, 0, 0, 0, 2826, 0, 0, 8160, 0, 0, 0, 8191, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 7845, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 8192, 0, 0, 12, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1656, 8174, 0, 0, 96, 12, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7114, 0, 0, 8147, 0, 0, 0, 8186, 0, 0, 2, 5, 0, 0, 0, 28, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8173, 0, 5, 1, 4, 0, 0, 0, 7594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 0, 0, 0, 0, 8184, 5, 0, 32, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 3, 29, 0, 0, 8026, 0, 0, 0, 0, 0, 19, 0, 0, 0, 7565, 8143, 0, 0, 0, 40, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 763, 0, 0, 8167, 0, 0, 0, 8176, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8161, 0, 0, 5, 6, 0, 0, 0, 7961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 21, 0, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 153, 2, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 16, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4222, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7949, 234, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 5526, 2571, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10615, 8151, 5625, 8189, 8256, 8186, 8327, 8187] avg=8192.0000 max=10615 max_rank=0 min=5625 min_rank=2 imbalance_ratio=1.295776 max_load_reduction=54921 max_load_reduction_pct=0.838028 moved_to_echo=32956 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 122, 0, 0, 0, 0, 0, 52, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2826, 0, 0, 5599, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7941, 233, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 7845, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 5495, 2561, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7960, 8334, 8473, 8180, 8089, 8186, 8139, 8175] avg=8192.0000 max=8473 max_rank=2 min=7960 min_rank=0 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=32589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 27, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5183, 0, 0, 5620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7949, 233, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 5510, 2570, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5500, 8189, 10803, 8188, 8333, 8187, 8151, 8185] avg=8192.0000 max=10803 max_rank=2 min=5500 min_rank=0 imbalance_ratio=1.318726 max_load_reduction=54733 max_load_reduction_pct=0.835159 moved_to_echo=32856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 12, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 5621, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7949, 234, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 5521, 2571, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10213, 8167, 6038, 8189, 8392, 8186, 8166, 8185] avg=8192.0000 max=10213 max_rank=0 min=6038 min_rank=2 imbalance_ratio=1.246704 max_load_reduction=55323 max_load_reduction_pct=0.844162 moved_to_echo=32929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 5619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 239, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 5529, 2573, 47, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10550, 8145, 5795, 8200, 8207, 8186, 8268, 8185] avg=8192.0000 max=10550 max_rank=0 min=5795 min_rank=2 imbalance_ratio=1.287842 max_load_reduction=54986 max_load_reduction_pct=0.839020 moved_to_echo=32846 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 29, 0, 0, 2383, 0, 0, 0, 0, 0, 19, 0, 0, 0, 7565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 40, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 0, 0, 5604, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7919, 229, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 5, 6, 0, 0, 0, 7961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 21, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 153, 2, 0, 0, 0, 0, 5414, 2563, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9999, 8148, 6368, 8187, 8214, 8226, 8196, 8198] avg=8192.0000 max=9999 max_rank=0 min=6368 min_rank=2 imbalance_ratio=1.220581 max_load_reduction=55537 max_load_reduction_pct=0.847427 moved_to_echo=32834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 5621, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7949, 234, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 5525, 2571, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6653, 8154, 9722, 8189, 8199, 8186, 8246, 8187] avg=8192.0000 max=9722 max_rank=2 min=6653 min_rank=0 imbalance_ratio=1.186768 max_load_reduction=55814 max_load_reduction_pct=0.851654 moved_to_echo=32818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 96, 12, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7114, 0, 0, 5591, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7931, 230, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 5, 1, 4, 0, 0, 0, 7594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5432, 2556, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 0, 0, 0, 0, 8179, 5, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 8248, 12712, 8214, 7846, 8193, 8043, 8234] avg=8192.0000 max=12712 max_rank=2 min=4046 min_rank=0 imbalance_ratio=1.551758 max_load_reduction=52824 max_load_reduction_pct=0.806030 moved_to_echo=32316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4224, 8192, 0, 0, 123, 0, 0, 0, 0, 0, 51, 0, 0, 8, 0, 0, 0, 0, 4012, 0, 0, 8160, 0, 0, 0, 8191, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 7989, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 8192, 0, 0, 13, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3267, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4931, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2196, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5988, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 3, 27, 0, 0, 8024, 0, 0, 0, 0, 0, 19, 0, 0, 0, 6642, 8141, 0, 0, 0, 42, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1621, 0, 0, 8167, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8161, 0, 0, 5, 6, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 21, 0, 15, 0, 1, 0, 0, 0, 0, 0, 0, 0, 125, 4, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 16, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 8173, 0, 0, 97, 12, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7490, 0, 0, 8148, 0, 0, 0, 8186, 0, 0, 2, 1, 0, 0, 0, 27, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8173, 0, 5, 2, 3, 0, 0, 0, 7833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 0, 0, 0, 0, 8184, 5, 0, 29, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 123, 0, 0, 0, 0, 0, 51, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 5062, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8022, 151, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 7989, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4960, 3098, 41, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7257, 8333, 9109, 8179, 8152, 8185, 8144, 8177] avg=8192.0000 max=9109 max_rank=2 min=7257 min_rank=0 imbalance_ratio=1.111938 max_load_reduction=56427 max_load_reduction_pct=0.861008 moved_to_echo=32658 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 5082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 152, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 4989, 3110, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11166, 8151, 5153, 8189, 8187, 8186, 8317, 8187] avg=8192.0000 max=11166 max_rank=0 min=5153 min_rank=2 imbalance_ratio=1.363037 max_load_reduction=54370 max_load_reduction_pct=0.829620 moved_to_echo=32877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 13, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 5082, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 152, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4984, 3110, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10665, 8168, 5663, 8189, 8317, 8186, 8163, 8185] avg=8192.0000 max=10665 max_rank=0 min=5663 min_rank=2 imbalance_ratio=1.301880 max_load_reduction=54871 max_load_reduction_pct=0.837265 moved_to_echo=32851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 28, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4931, 0, 0, 5080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 151, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4975, 3110, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6310, 8192, 10011, 8189, 8309, 8186, 8154, 8185] avg=8192.0000 max=10011 max_rank=2 min=6310 min_rank=0 imbalance_ratio=1.222046 max_load_reduction=55525 max_load_reduction_pct=0.847244 moved_to_echo=32834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 27, 0, 0, 2988, 0, 0, 0, 0, 0, 19, 0, 0, 0, 6642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 42, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1621, 0, 0, 5066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8001, 149, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 5, 6, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 21, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 0, 0, 0, 0, 0, 125, 4, 0, 0, 0, 0, 4887, 3101, 41, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9679, 8146, 6687, 8190, 8229, 8228, 8178, 8199] avg=8192.0000 max=9679 max_rank=0 min=6687 min_rank=2 imbalance_ratio=1.181519 max_load_reduction=55857 max_load_reduction_pct=0.852310 moved_to_echo=32834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5988, 0, 0, 5082, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 152, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 4988, 3110, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5246, 8153, 11138, 8189, 8179, 8186, 8258, 8187] avg=8192.0000 max=11138 max_rank=2 min=5246 min_rank=0 imbalance_ratio=1.359619 max_load_reduction=54398 max_load_reduction_pct=0.830048 moved_to_echo=32810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 5077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 154, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 8003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 4992, 3115, 46, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11161, 8146, 5228, 8197, 8159, 8185, 8276, 8184] avg=8192.0000 max=11161 max_rank=0 min=5228 min_rank=2 imbalance_ratio=1.362427 max_load_reduction=54375 max_load_reduction_pct=0.829697 moved_to_echo=32804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 97, 12, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7490, 0, 0, 5054, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8012, 149, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 5, 2, 3, 0, 0, 0, 7833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4898, 3094, 41, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 0, 0, 0, 0, 8180, 5, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 8247, 12547, 8214, 8004, 8194, 8046, 8232] avg=8192.0000 max=12547 max_rank=2 min=4052 min_rank=0 imbalance_ratio=1.531616 max_load_reduction=52989 max_load_reduction_pct=0.808548 moved_to_echo=32476 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 0, 128, 128, 0, 0, 0, 0, 0, 256, 256, 128, 128, 0, 0, 0, 44032, 20608, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 19456, 0, 0, 0, 0, 0, 128, 0, 0, 0, 46208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 20480, 0, 0, 44928, 0, 0, 0, 0, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[384, 0, 128, 128, 0, 0, 0, 0, 0, 256, 128, 128, 128, 0, 0, 0, 39680, 24960, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 24320, 0, 0, 0, 0, 0, 128, 0, 0, 0, 41344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 24960, 0, 0, 40704, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2048, 0, 128, 128, 128, 0, 0, 0, 63616, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 128, 0, 63616, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 64256, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1408, 0, 128, 128, 128, 0, 0, 0, 64256, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65152, 0, 0, 384, 128, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 44, 8183, 0, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 87, 0, 0, 0, 0, 0, 1022, 0, 0, 0, 0, 0, 0, 6850, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5535, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 2, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 34, 0, 0, 0, 0, 0, 825, 0, 0, 0, 0, 0, 0, 4984, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 4202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 3899, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 8176, 9, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 30, 8178, 0, 0, 2350, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8156, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 51, 0, 0, 0, 7770, 10, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8106, 0, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 2249, 5, 0, 0, 0, 0, 3, 3943, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 30, 0, 0, 0, 0, 0, 4, 14, 0, 0, 0, 0, 5, 0, 0, 0, 14, 0, 6, 8155, 0, 8168, 0, 0, 6460, 0, 0, 0, 0, 0, 0, 0, 13, 3, 8138, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 8034, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 18, 0, 0, 10, 4, 0, 0, 7, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 15, 0, 0, 12, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 77, 14, 0, 0, 0, 0, 0, 1628, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 5, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 7322, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 0, 983, 0, 0, 0, 0, 0, 0, 4236, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5334, 44, 0, 0, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 1022, 0, 0, 0, 0, 0, 0, 6850, 0, 0, 2669, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8185, 6107, 8181, 7817, 8192, 8254, 10638] avg=8192.0000 max=10638 max_rank=7 min=6107 min_rank=2 imbalance_ratio=1.298584 max_load_reduction=54898 max_load_reduction_pct=0.837677 moved_to_echo=34901 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5334, 0, 0, 0, 0, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5535, 0, 0, 2669, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8184, 7991, 8202, 8323, 8193, 8240, 8216] avg=8192.0000 max=8323 max_rank=4 min=7991 min_rank=2 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5334, 0, 0, 0, 0, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 825, 0, 0, 0, 0, 0, 0, 4984, 0, 0, 2669, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8208, 7883, 8193, 8260, 8195, 8104, 8522] avg=8192.0000 max=8522 max_rank=7 min=7883 min_rank=2 imbalance_ratio=1.040283 max_load_reduction=57014 max_load_reduction_pct=0.869965 moved_to_echo=33081 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5334, 0, 0, 0, 0, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 7322, 0, 0, 2669, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8185, 6119, 8199, 8298, 8193, 8176, 10184] avg=8192.0000 max=10184 max_rank=7 min=6119 min_rank=2 imbalance_ratio=1.243164 max_load_reduction=55352 max_load_reduction_pct=0.844604 moved_to_echo=34851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5318, 0, 0, 0, 0, 4202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 2670, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8183, 9520, 8213, 8322, 8197, 8190, 6734] avg=8192.0000 max=9520 max_rank=2 min=6734 min_rank=7 imbalance_ratio=1.162109 max_load_reduction=56016 max_load_reduction_pct=0.854736 moved_to_echo=31443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4, 14, 0, 0, 0, 0, 5, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 6, 5310, 0, 0, 0, 0, 6460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3, 8138, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 8034, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 10, 4, 0, 0, 7, 0, 8153, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 15, 0, 0, 12, 0, 0, 2, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 77, 14, 0, 0, 0, 0, 0, 1628, 0, 2, 2657, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8211, 11790, 8165, 8250, 8193, 8244, 4401] avg=8192.0000 max=11790 max_rank=2 min=4401 min_rank=7 imbalance_ratio=1.439209 max_load_reduction=53746 max_load_reduction_pct=0.820099 moved_to_echo=29088 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5334, 0, 0, 0, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 983, 0, 0, 0, 0, 0, 0, 4236, 0, 0, 2669, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8191, 8427, 8201, 8286, 8193, 8155, 7899] avg=8192.0000 max=8427 max_rank=2 min=7899 min_rank=7 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=32533 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5319, 30, 0, 0, 0, 2350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8156, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 51, 0, 0, 0, 7770, 10, 0, 0, 0, 0, 0, 15, 0, 1, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8106, 0, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 2249, 5, 0, 0, 0, 0, 3, 3943, 0, 0, 2661, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8189, 7699, 8182, 7980, 8180, 8173, 8942] avg=8192.0000 max=8942 max_rank=7 min=7699 min_rank=2 imbalance_ratio=1.091553 max_load_reduction=56594 max_load_reduction_pct=0.863556 moved_to_echo=33275 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 31, 0, 0, 0, 0, 0, 6, 13, 0, 0, 0, 0, 4, 0, 0, 0, 14, 0, 6, 8156, 0, 8168, 0, 0, 6906, 0, 1, 0, 0, 0, 0, 0, 13, 2, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 8032, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 18, 0, 0, 8, 6, 0, 0, 9, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 15, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 118, 14, 0, 0, 0, 0, 0, 1141, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 44, 8183, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 87, 0, 0, 0, 0, 0, 1156, 0, 0, 0, 0, 0, 0, 6263, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 4431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3761, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 8176, 9, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 29, 8178, 0, 0, 2838, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8155, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 47, 0, 0, 0, 7700, 10, 0, 0, 0, 0, 0, 15, 0, 2, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 8109, 0, 0, 0, 4, 0, 1, 2, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 2429, 5, 0, 0, 0, 0, 2, 3346, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 4187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 3899, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 2, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 37, 0, 0, 0, 0, 0, 1052, 0, 0, 0, 0, 0, 0, 4370, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 5, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 7702, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 8192, 0, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1, 0, 0, 0, 0, 0, 1328, 0, 0, 0, 0, 0, 0, 3432, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4875, 44, 0, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 1156, 0, 0, 0, 0, 0, 0, 6263, 0, 0, 3106, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8184, 6155, 8182, 7790, 8192, 8249, 10623] avg=8192.0000 max=10623 max_rank=7 min=6155 min_rank=2 imbalance_ratio=1.296753 max_load_reduction=54913 max_load_reduction_pct=0.837906 moved_to_echo=34854 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 6, 13, 0, 0, 0, 0, 4, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 6, 4854, 0, 0, 0, 0, 6906, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 8032, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 8, 6, 0, 0, 9, 0, 8152, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 15, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 118, 14, 0, 0, 0, 0, 0, 1141, 0, 2, 3093, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8211, 11781, 8164, 8271, 8194, 8239, 4391] avg=8192.0000 max=11781 max_rank=2 min=4391 min_rank=7 imbalance_ratio=1.438110 max_load_reduction=53755 max_load_reduction_pct=0.820236 moved_to_echo=29095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4875, 0, 0, 0, 0, 4431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 3106, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8183, 9306, 8203, 8347, 8193, 8238, 6880] avg=8192.0000 max=9306 max_rank=2 min=6880 min_rank=7 imbalance_ratio=1.135986 max_load_reduction=56230 max_load_reduction_pct=0.858002 moved_to_echo=31658 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4875, 0, 0, 0, 0, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 1052, 0, 0, 0, 0, 0, 0, 4370, 0, 0, 3106, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8205, 7842, 8194, 8240, 8195, 8117, 8576] avg=8192.0000 max=8576 max_rank=7 min=7842 min_rank=2 imbalance_ratio=1.046875 max_load_reduction=56960 max_load_reduction_pct=0.869141 moved_to_echo=33128 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4859, 0, 0, 0, 0, 4187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 3112, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8182, 9046, 8209, 8341, 8196, 8183, 7197] avg=8192.0000 max=9046 max_rank=2 min=7197 min_rank=7 imbalance_ratio=1.104248 max_load_reduction=56490 max_load_reduction_pct=0.861969 moved_to_echo=31917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4875, 0, 0, 0, 0, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 7702, 0, 0, 3106, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8185, 5255, 8200, 8319, 8193, 8173, 11030] avg=8192.0000 max=11030 max_rank=7 min=5255 min_rank=2 imbalance_ratio=1.346436 max_load_reduction=54506 max_load_reduction_pct=0.831696 moved_to_echo=35715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4875, 1, 0, 0, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1328, 0, 0, 0, 0, 0, 0, 3432, 0, 0, 3106, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8198, 8423, 8202, 8298, 8193, 8161, 7877] avg=8192.0000 max=8423 max_rank=2 min=7877 min_rank=7 imbalance_ratio=1.028198 max_load_reduction=57113 max_load_reduction_pct=0.871475 moved_to_echo=32529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4861, 29, 0, 0, 0, 2838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8155, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 47, 0, 0, 0, 7700, 10, 0, 0, 0, 0, 0, 15, 0, 2, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 8109, 0, 0, 0, 4, 0, 1, 2, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 2429, 5, 0, 0, 0, 0, 2, 3346, 0, 0, 3098, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8188, 7728, 8182, 7930, 8180, 8176, 8962] avg=8192.0000 max=8962 max_rank=7 min=7728 min_rank=2 imbalance_ratio=1.093994 max_load_reduction=56574 max_load_reduction_pct=0.863251 moved_to_echo=33248 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 128, 0, 0, 5504, 128, 0, 0, 0, 0, 128, 38400, 0, 128, 21376, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 42624, 128, 0, 0, 0, 22912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 0, 0, 128, 0, 0, 6528, 128, 0, 0, 0, 0, 128, 33920, 0, 128, 24960, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 39040, 128, 0, 0, 0, 26496, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 0, 64384, 128, 0, 0, 0, 128, 0, 128, 128, 128, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 65536, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 64896, 128, 0, 0, 128, 0, 0, 128, 0, 256, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 64128, 128, 0, 0, 0, 128, 0, 128, 0, 128, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 64896, 128, 0, 0, 128, 0, 128, 128, 0, 256, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 65536, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 0, 65536, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 37, 0, 4, 0, 27, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 8192, 0, 11, 0, 0, 0, 0, 0, 4932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 8144, 0, 0, 13, 20, 19, 4, 94, 0, 25, 0, 0, 0, 0, 0, 8152, 0, 0, 11, 0, 0, 34, 0, 0, 0, 0, 0, 8163, 0, 0, 16, 0, 12, 29, 19, 0, 10, 0, 0, 5, 737, 0, 0, 0, 0, 18, 0, 11, 9, 0, 0, 0, 0, 0, 0, 0, 5015, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 11, 0, 0, 0, 4, 8168, 0, 8132, 0, 0, 0, 0, 0, 6, 0, 2519, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 8122, 0, 0, 0, 9, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 62, 9, 10, 0, 39, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 9, 0, 0, 0, 0, 7, 0, 0, 0, 8154, 0, 0, 0, 4, 0, 0, 0, 0, 20, 0, 3, 2, 0, 0, 0, 0, 8171, 0, 8155, 0, 29, 0, 0, 0, 3, 0, 2, 0, 0, 4, 0, 3, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 37, 0, 4, 0, 27, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3260, 3868, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4267, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8192, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8193, 8169, 8133, 7212, 8198, 9210, 8221] avg=8192.0000 max=9210 max_rank=6 min=7212 min_rank=4 imbalance_ratio=1.124268 max_load_reduction=56326 max_load_reduction_pct=0.859467 moved_to_echo=32841 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 3884, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8169, 8184, 8073, 4663, 8204, 11831, 8212] avg=8192.0000 max=11831 max_rank=6 min=4663 min_rank=4 imbalance_ratio=1.444214 max_load_reduction=53705 max_load_reduction_pct=0.819473 moved_to_echo=32910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6810, 3879, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8199, 8201, 8078, 10773, 8199, 5666, 8220] avg=8192.0000 max=10773 max_rank=4 min=5666 min_rank=6 imbalance_ratio=1.315063 max_load_reduction=54763 max_load_reduction_pct=0.835617 moved_to_echo=32858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 3882, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8199, 8197, 8073, 5064, 8200, 11385, 8218] avg=8192.0000 max=11385 max_rank=6 min=5064 min_rank=4 imbalance_ratio=1.389771 max_load_reduction=54151 max_load_reduction_pct=0.826279 moved_to_echo=32867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 3889, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8112, 8204, 8058, 4682, 8201, 11850, 8224] avg=8192.0000 max=11850 max_rank=6 min=4682 min_rank=4 imbalance_ratio=1.446533 max_load_reduction=53686 max_load_reduction_pct=0.819183 moved_to_echo=32957 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 8132, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 7, 0, 0, 13, 20, 19, 4, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8152, 0, 0, 11, 0, 0, 34, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 16, 0, 12, 29, 19, 0, 10, 0, 0, 5, 737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 11, 9, 0, 0, 0, 0, 0, 0, 0, 5015, 3873, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 11, 0, 0, 0, 4, 8129, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4273, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2519, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8278, 8232, 8872, 9010, 8177, 6806, 8001] avg=8192.0000 max=9010 max_rank=4 min=6806 min_rank=6 imbalance_ratio=1.099854 max_load_reduction=56526 max_load_reduction_pct=0.862518 moved_to_echo=31994 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 3883, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8192, 8181, 8073, 12002, 8199, 4472, 8217] avg=8192.0000 max=12002 max_rank=4 min=4472 min_rank=6 imbalance_ratio=1.465088 max_load_reduction=53534 max_load_reduction_pct=0.816864 moved_to_echo=32890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 7, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 0, 62, 9, 10, 0, 39, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 9, 0, 0, 0, 0, 7, 0, 0, 0, 8154, 3874, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 20, 0, 3, 2, 0, 0, 0, 0, 8107, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4275, 0, 0, 0, 29, 0, 0, 0, 3, 0, 2, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8194, 8168, 8176, 12130, 8158, 4316, 8223] avg=8192.0000 max=12130 max_rank=4 min=4316 min_rank=6 imbalance_ratio=1.480713 max_load_reduction=53406 max_load_reduction_pct=0.814911 moved_to_echo=32827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 36, 0, 4, 0, 27, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 8192, 0, 11, 0, 0, 0, 0, 0, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 950, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1659, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 6539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 8123, 0, 0, 0, 9, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 59, 8, 10, 0, 38, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 9, 0, 0, 0, 0, 7, 0, 0, 0, 8152, 0, 0, 0, 4, 0, 0, 0, 0, 20, 0, 3, 2, 0, 0, 0, 0, 8171, 0, 8155, 0, 31, 0, 0, 0, 3, 0, 1, 0, 0, 4, 0, 3, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 3, 0, 8144, 0, 0, 16, 20, 15, 3, 95, 0, 25, 0, 0, 0, 0, 0, 8154, 0, 0, 11, 0, 0, 36, 0, 0, 0, 0, 0, 8163, 0, 0, 16, 0, 11, 27, 19, 0, 10, 0, 0, 4, 725, 0, 0, 0, 0, 17, 0, 9, 8, 0, 0, 0, 0, 0, 0, 0, 5843, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 11, 0, 0, 0, 4, 8168, 0, 8133, 0, 0, 0, 0, 0, 6, 0, 1729, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7388, 3334, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8200, 8199, 8081, 10804, 8198, 5632, 8222] avg=8192.0000 max=10804 max_rank=4 min=5632 min_rank=6 imbalance_ratio=1.318848 max_load_reduction=54732 max_load_reduction_pct=0.835144 moved_to_echo=32856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 8, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 950, 3339, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8167, 8183, 8075, 4371, 8203, 12123, 8214] avg=8192.0000 max=12123 max_rank=6 min=4371 min_rank=4 imbalance_ratio=1.479858 max_load_reduction=53413 max_load_reduction_pct=0.815018 moved_to_echo=32911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 0, 36, 0, 4, 0, 27, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4875, 3325, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4809, 0, 0, 0, 11, 0, 0, 0, 0, 0, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8195, 8169, 8134, 8282, 8197, 8137, 8222] avg=8192.0000 max=8282 max_rank=4 min=8134 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1659, 3337, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8200, 8197, 8075, 5078, 8199, 11367, 8220] avg=8192.0000 max=11367 max_rank=6 min=5078 min_rank=4 imbalance_ratio=1.387573 max_load_reduction=54169 max_load_reduction_pct=0.826553 moved_to_echo=32864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 3339, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8197, 8184, 8075, 11571, 8198, 4892, 8219] avg=8192.0000 max=11571 max_rank=4 min=4892 min_rank=6 imbalance_ratio=1.412476 max_load_reduction=53965 max_load_reduction_pct=0.823441 moved_to_echo=32880 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 8133, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 3, 0, 8, 0, 0, 16, 20, 15, 3, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8154, 0, 0, 11, 0, 0, 36, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 16, 0, 11, 27, 19, 0, 10, 0, 0, 4, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 9, 8, 0, 0, 0, 0, 0, 0, 0, 5843, 3330, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 11, 0, 0, 0, 4, 8128, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4816, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1729, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8270, 8234, 8858, 9289, 8174, 6559, 7991] avg=8192.0000 max=9289 max_rank=4 min=6559 min_rank=6 imbalance_ratio=1.133911 max_load_reduction=56247 max_load_reduction_pct=0.858261 moved_to_echo=32013 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 3342, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8115, 8200, 8064, 4558, 8206, 11966, 8223] avg=8192.0000 max=11966 max_rank=6 min=4558 min_rank=4 imbalance_ratio=1.460693 max_load_reduction=53570 max_load_reduction_pct=0.817413 moved_to_echo=32953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 5, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 0, 59, 8, 10, 0, 38, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 9, 0, 0, 0, 0, 7, 0, 0, 0, 8152, 3331, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 20, 0, 3, 2, 0, 0, 0, 0, 8110, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4818, 0, 0, 0, 31, 0, 0, 0, 3, 0, 1, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8192, 8170, 8174, 11583, 8161, 4860, 8225] avg=8192.0000 max=11583 max_rank=4 min=4860 min_rank=6 imbalance_ratio=1.413940 max_load_reduction=53953 max_load_reduction_pct=0.823257 moved_to_echo=32829 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[34304, 0, 0, 0, 128, 0, 0, 0, 128, 0, 31360, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 33792, 31104, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 38272, 26752, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[38656, 0, 0, 0, 128, 0, 0, 0, 128, 0, 27008, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 64512, 0, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 64640, 0, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 128, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 65408, 0, 0, 0, 0, 128, 0, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 65408, 0, 0, 128, 0, 128, 0, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8192, 0, 170, 0, 0, 71, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 8192, 0, 8185, 0, 0, 8128, 5, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 8190, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 6792, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 26, 0, 8192, 0, 17, 0, 0, 81, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 8192, 0, 8166, 0, 0, 8176, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 8158, 0, 0, 0, 0, 30, 0, 0, 978, 0, 7727, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8080, 0, 126, 0, 0, 8161, 4, 7083, 0, 0, 8135, 0, 0, 0, 16, 7938, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 6, 18, 0, 4, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 12, 8170, 0, 519, 21, 0, 11, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 53, 0, 8192, 0, 9, 0, 0, 11, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 8192, 0, 8138, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 52, 0, 8192, 0, 2, 0, 0, 30, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 8192, 0, 8140, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 8179, 0, 0, 0, 0, 0, 0, 0, 220, 226, 8173, 0, 19, 0, 0, 0, 4, 0, 2, 0, 0, 7830, 0, 1, 0, 0, 8182, 0, 8161, 0, 0, 8101, 0, 8, 0, 0, 8112, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 1, 0, 0, 6, 0, 0, 8178, 0, 0, 10, 0, 9, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 6, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 303, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 8192, 0, 7271, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 7823, 0, 170, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8124, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8080, 8617, 8215, 8192, 7890, 8153, 8197] avg=8192.0000 max=8617 max_rank=2 min=7890 min_rank=5 imbalance_ratio=1.051880 max_load_reduction=56919 max_load_reduction_pct=0.868515 moved_to_echo=32432 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 7821, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 6792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7697, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8915, 7228, 8224, 8190, 8373, 8216, 8195] avg=8192.0000 max=8915 max_rank=1 min=7228 min_rank=2 imbalance_ratio=1.088257 max_load_reduction=56621 max_load_reduction_pct=0.863968 moved_to_echo=32974 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 7823, 0, 17, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7947, 8598, 8223, 8190, 7989, 8200, 8195] avg=8192.0000 max=8598 max_rank=2 min=7947 min_rank=1 imbalance_ratio=1.049561 max_load_reduction=56938 max_load_reduction_pct=0.868805 moved_to_echo=32574 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 7823, 0, 9, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7683, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7896, 8574, 8223, 8190, 8053, 8215, 8195] avg=8192.0000 max=8574 max_rank=2 min=7896 min_rank=1 imbalance_ratio=1.046631 max_load_reduction=56962 max_load_reduction_pct=0.869171 moved_to_echo=32653 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 8158, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 978, 0, 7380, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 0, 126, 0, 0, 0, 4, 7083, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 6, 18, 7912, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 12, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 21, 0, 11, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 7592, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8161, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8396, 7659, 8031, 8206, 8731, 8166, 8170] avg=8192.0000 max=8731 max_rank=5 min=7659 min_rank=2 imbalance_ratio=1.065796 max_load_reduction=56805 max_load_reduction_pct=0.866776 moved_to_echo=33273 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 7816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 0, 0, 7271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 7693, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8119, 7698, 8229, 8184, 8694, 8215, 8195] avg=8192.0000 max=8694 max_rank=5 min=7698 min_rank=2 imbalance_ratio=1.061279 max_load_reduction=56842 max_load_reduction_pct=0.867340 moved_to_echo=33288 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 7823, 0, 2, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7669, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7907, 8575, 8223, 8190, 8037, 8216, 8195] avg=8192.0000 max=8575 max_rank=2 min=7907 min_rank=1 imbalance_ratio=1.046753 max_load_reduction=56961 max_load_reduction_pct=0.869156 moved_to_echo=32638 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 8179, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 226, 7805, 0, 19, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 1, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8086, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 1, 0, 0, 6, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 9, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 7357, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 6, 0, 17, 0, 0, 8098, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8276, 8587, 8168, 8194, 7769, 8155, 8194] avg=8192.0000 max=8587 max_rank=2 min=7769 min_rank=5 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32312 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 634, 0, 8190, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 7428, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 2, 0, 0, 30, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 179, 0, 0, 67, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 8192, 0, 8191, 0, 0, 8129, 5, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 0, 18, 0, 0, 83, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 8192, 0, 8186, 0, 0, 8175, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 308, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8192, 0, 7717, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 11, 0, 0, 10, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 8192, 0, 8188, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 8179, 0, 0, 0, 0, 0, 0, 0, 214, 241, 8172, 0, 19, 0, 0, 0, 4, 0, 2, 0, 0, 7819, 0, 1, 0, 0, 8182, 0, 8156, 0, 0, 8101, 0, 9, 0, 0, 8114, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8178, 0, 0, 10, 0, 9, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 6, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 8159, 0, 0, 0, 0, 30, 0, 0, 927, 0, 7729, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8076, 0, 126, 0, 0, 8160, 3, 7124, 0, 0, 8137, 0, 0, 0, 15, 7941, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 6, 19, 0, 3, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 12, 8170, 0, 529, 21, 0, 11, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 0, 7898, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 7428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7855, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8534, 7707, 8223, 8190, 8276, 8216, 8195] avg=8192.0000 max=8534 max_rank=1 min=7707 min_rank=2 imbalance_ratio=1.041748 max_load_reduction=57002 max_load_reduction_pct=0.869781 moved_to_echo=32877 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7900, 0, 2, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7825, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7934, 8468, 8223, 8190, 8117, 8216, 8195] avg=8192.0000 max=8468 max_rank=2 min=7934 min_rank=1 imbalance_ratio=1.033691 max_load_reduction=57068 max_load_reduction_pct=0.870789 moved_to_echo=32718 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7900, 0, 18, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7776, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8007, 8462, 8222, 8190, 8067, 8199, 8195] avg=8192.0000 max=8462 max_rank=2 min=8007 min_rank=1 imbalance_ratio=1.032959 max_load_reduction=57074 max_load_reduction_pct=0.870880 moved_to_echo=32651 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7900, 0, 179, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7677, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8125, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8150, 8469, 8214, 8193, 7968, 8154, 8197] avg=8192.0000 max=8469 max_rank=2 min=7968 min_rank=5 imbalance_ratio=1.033813 max_load_reduction=57067 max_load_reduction_pct=0.870773 moved_to_echo=32512 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 0, 7896, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7852, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8205, 7989, 8227, 8184, 8319, 8215, 8195] avg=8192.0000 max=8319 max_rank=5 min=7989 min_rank=2 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32913 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7900, 0, 11, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7840, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7925, 8466, 8223, 8190, 8132, 8215, 8195] avg=8192.0000 max=8466 max_rank=2 min=7925 min_rank=1 imbalance_ratio=1.033447 max_load_reduction=57070 max_load_reduction_pct=0.870819 moved_to_echo=32732 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 8159, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 927, 0, 7455, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 126, 0, 0, 0, 3, 7124, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 6, 19, 7915, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 12, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 21, 0, 11, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 1, 7744, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8160, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8420, 7543, 8034, 8206, 8818, 8167, 8170] avg=8192.0000 max=8818 max_rank=5 min=7543 min_rank=2 imbalance_ratio=1.076416 max_load_reduction=56718 max_load_reduction_pct=0.865448 moved_to_echo=33361 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 8179, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 241, 7881, 0, 19, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 1, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 8088, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 9, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 7498, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 6, 0, 16, 0, 0, 8098, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8361, 8432, 8170, 8193, 7839, 8154, 8194] avg=8192.0000 max=8432 max_rank=2 min=7839 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32380 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3456, 0, 128, 0, 0, 0, 128, 61952, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 1536, 128, 0, 128, 0, 0, 0, 0, 384, 0, 128, 0, 0, 0, 0, 60928, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 2816, 256, 62208, 0, 256, 0, 0, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 128, 65024, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2304, 0, 128, 0, 0, 0, 128, 63232, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 896, 128, 0, 128, 0, 0, 0, 0, 384, 0, 128, 0, 0, 0, 128, 62080, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 2176, 256, 62848, 0, 256, 0, 0, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 128, 65024, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True - [2026-08-26 16:18:06] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 455.3 | throughput per GPU (TFLOP/s/GPU): 397.2 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.353138E+01 | loss scale: 1.0 | grad norm: 121.410 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 32, 512, 39, 0, 0, 8013, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 440, 0, 0, 3944, 7, 1, 0, 8185, 4, 8184, 0, 2, 1, 3, 3, 0, 1, 0, 0, 0, 0, 1, 4, 0, 0, 0, 1, 0, 0, 5, 20, 0, 0, 0, 4, 0, 4, 5, 0, 8044, 0, 2, 1, 0, 2, 0, 0, 1, 3500, 0, 0, 0, 3548, 2, 6730, 7, 0, 3, 1518, 0, 0, 2, 0, 2, 11, 0, 1, 5547, 1, 1, 0, 0, 4, 0, 12, 0, 1, 39, 3, 0, 0, 0, 0, 0, 1, 2, 6595, 1, 0, 8, 3, 2, 0, 131, 0, 2, 0, 7, 1, 0, 0, 0, 0, 382, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 78, 572, 6, 0, 0, 8101, 0, 0, 0, 2, 1, 3, 0, 1, 0, 0, 599, 0, 0, 2078, 10, 0, 0, 8191, 5, 8190, 0, 11, 1, 0, 4, 0, 1, 2, 0, 1, 1, 0, 7, 1, 0, 1, 0, 0, 0, 3, 4, 1, 0, 0, 1, 0, 2, 0, 0, 7844, 0, 0, 0, 1, 0, 0, 0, 0, 3937, 0, 0, 0, 4293, 0, 6816, 6, 0, 0, 2224, 0, 1, 0, 0, 0, 0, 0, 0, 5601, 0, 14, 0, 0, 0, 0, 7, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 17, 6459, 0, 0, 3, 0, 0, 0, 20, 0, 2, 0, 0, 0, 0, 0, 1, 0, 390, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 20, 855, 20, 0, 0, 8088, 0, 0, 1, 0, 0, 0, 2, 1, 1, 0, 548, 0, 0, 2482, 7, 0, 0, 8190, 3, 8190, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 7910, 1, 0, 0, 0, 0, 1, 0, 0, 2982, 0, 0, 0, 4086, 0, 6677, 1, 0, 0, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 11, 0, 0, 10, 0, 3, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 4, 6867, 0, 0, 2, 0, 0, 1, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 36, 1032, 27, 0, 0, 8087, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 488, 0, 0, 3520, 5, 2, 0, 8192, 3, 8190, 0, 9, 0, 0, 4, 0, 0, 4, 0, 1, 1, 0, 13, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 1, 0, 0, 4, 3, 0, 7910, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 2, 3573, 0, 6573, 2, 2, 0, 1837, 0, 0, 0, 0, 0, 1, 0, 0, 6213, 0, 1, 0, 0, 2, 2, 19, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 11, 5594, 0, 0, 5, 7, 0, 0, 199, 0, 0, 0, 0, 1, 0, 0, 1, 0, 376, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 38, 712, 29, 0, 0, 8156, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 564, 0, 0, 3318, 1, 0, 0, 8191, 0, 8191, 3, 2, 0, 0, 10, 0, 0, 0, 0, 0, 10, 0, 17, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 3, 0, 7937, 0, 0, 0, 2, 1, 0, 0, 0, 3854, 0, 0, 0, 3739, 0, 6944, 0, 0, 0, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 5775, 0, 13, 0, 0, 6, 0, 39, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 5708, 0, 0, 7, 0, 0, 0, 60, 0, 1, 0, 1, 2, 0, 0, 0, 1, 387, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 53, 713, 38, 0, 0, 8078, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 535, 0, 0, 2819, 29, 0, 0, 8192, 1, 8192, 0, 23, 0, 1, 5, 1, 1, 1, 0, 0, 3, 0, 50, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 7844, 0, 0, 0, 1, 0, 0, 0, 0, 3773, 0, 0, 0, 4146, 0, 6743, 0, 0, 0, 2019, 0, 0, 0, 0, 0, 4, 0, 0, 5878, 0, 4, 0, 1, 9, 0, 9, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 16, 5801, 0, 0, 3, 3, 0, 0, 109, 0, 0, 0, 2, 7, 0, 0, 1, 0, 393, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 0, 2, 25, 593, 20, 0, 0, 8071, 0, 0, 0, 1, 0, 5, 0, 0, 0, 1, 476, 1, 3, 2912, 9, 0, 0, 8192, 2, 8191, 0, 24, 0, 0, 10, 1, 1, 0, 0, 6, 1, 0, 24, 2, 0, 0, 1, 1, 0, 1, 3, 0, 0, 3, 3, 0, 0, 0, 0, 7854, 0, 0, 0, 3, 3, 0, 0, 0, 3694, 0, 2, 0, 3765, 1, 6331, 3, 1, 0, 2659, 0, 0, 2, 0, 0, 1, 0, 0, 6241, 0, 27, 0, 1, 3, 2, 2, 0, 0, 12, 0, 0, 0, 2, 0, 0, 0, 39, 5912, 0, 0, 3, 1, 0, 0, 27, 0, 0, 0, 1, 1, 0, 0, 0, 2, 353, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 91, 566, 44, 0, 1, 8103, 0, 0, 0, 0, 5, 4, 0, 1, 0, 0, 561, 0, 0, 3679, 3, 1, 0, 8191, 14, 8190, 0, 9, 0, 0, 7, 0, 4, 1, 0, 4, 0, 0, 85, 3, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 3, 0, 3214, 0, 7202, 6, 0, 0, 1688, 0, 0, 1, 0, 0, 5, 0, 0, 5668, 0, 60, 0, 0, 29, 0, 70, 0, 0, 55, 0, 0, 0, 1, 0, 0, 0, 19, 5804, 0, 0, 9, 0, 0, 0, 25, 0, 0, 0, 9, 0, 1, 0, 0, 0, 353, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 78, 572, 6, 0, 0, 7429, 0, 0, 0, 2, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 599, 0, 0, 2078, 10, 0, 0, 0, 5, 4547, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 1, 2, 0, 1, 1, 0, 7, 1, 0, 1, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 1, 0, 0, 1, 0, 2, 0, 0, 7844, 0, 0, 0, 1, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 2941, 0, 0, 6, 0, 0, 2224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5601, 0, 14, 0, 0, 0, 0, 7, 0, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 17, 6459, 0, 0, 3, 0, 0, 1352, 446, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2, 0, 0, 0, 0, 0, 1, 0, 390, 0, 0, 0, 0, 6816, 940, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 7251, 8168, 8113, 9108, 7880, 8700, 8211] avg=8192.0000 max=9108 max_rank=4 min=7251 min_rank=1 imbalance_ratio=1.111816 max_load_reduction=56428 max_load_reduction_pct=0.861023 moved_to_echo=33899 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 20, 855, 20, 0, 0, 7416, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 548, 0, 0, 2482, 7, 0, 0, 0, 3, 4547, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 7910, 1, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2982, 0, 0, 0, 2799, 0, 0, 1, 0, 0, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 11, 0, 0, 10, 0, 3, 0, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 4, 6867, 0, 0, 2, 0, 0, 1287, 446, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 6677, 940, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8312, 7593, 8170, 8171, 7746, 8442, 9035, 8067] avg=8192.0000 max=9035 max_rank=6 min=7593 min_rank=1 imbalance_ratio=1.102905 max_load_reduction=56501 max_load_reduction_pct=0.862137 moved_to_echo=33290 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 32, 512, 39, 0, 0, 7348, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 3944, 7, 1, 0, 0, 4, 4545, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3, 0, 1, 0, 0, 0, 0, 1, 4, 0, 0, 0, 1, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 20, 0, 0, 0, 4, 0, 4, 5, 0, 8044, 0, 2, 1, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3500, 0, 0, 0, 2430, 2, 0, 7, 0, 3, 1518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 11, 0, 1, 5547, 1, 1, 0, 0, 4, 0, 12, 0, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 39, 3, 0, 0, 0, 0, 0, 1, 2, 6595, 1, 0, 8, 3, 2, 1118, 445, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 2, 0, 7, 1, 0, 0, 0, 0, 382, 0, 0, 0, 0, 6730, 939, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7935, 8943, 8157, 8339, 7463, 7836, 8629, 8234] avg=8192.0000 max=8943 max_rank=1 min=7463 min_rank=4 imbalance_ratio=1.091675 max_load_reduction=56593 max_load_reduction_pct=0.863541 moved_to_echo=32162 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 38, 712, 29, 0, 0, 7479, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 564, 0, 0, 3318, 1, 0, 0, 0, 0, 4548, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 10, 0, 17, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 3, 0, 7937, 0, 0, 0, 2, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3854, 0, 0, 0, 2561, 0, 0, 0, 0, 0, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5775, 0, 13, 0, 0, 6, 0, 39, 2, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 5708, 0, 0, 7, 0, 0, 1178, 446, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 1, 0, 1, 2, 0, 0, 0, 1, 387, 0, 0, 0, 0, 6944, 940, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8437, 8186, 8213, 8199, 8092, 7769, 8378] avg=8192.0000 max=8437 max_rank=1 min=7769 min_rank=6 imbalance_ratio=1.029907 max_load_reduction=57099 max_load_reduction_pct=0.871262 moved_to_echo=32438 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 36, 1032, 27, 0, 0, 7406, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 0, 0, 3520, 5, 2, 0, 0, 3, 4537, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 1, 1, 0, 13, 0, 0, 0, 2, 4, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 3, 0, 7910, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 2, 2444, 0, 0, 2, 2, 0, 1837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6213, 0, 1, 0, 0, 2, 2, 19, 0, 2260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 11, 5594, 0, 0, 5, 7, 0, 1129, 447, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 1, 0, 0, 1, 0, 376, 0, 0, 0, 0, 6573, 946, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8509, 8564, 8179, 8180, 7847, 8498, 7621, 8138] avg=8192.0000 max=8564 max_rank=1 min=7621 min_rank=6 imbalance_ratio=1.045410 max_load_reduction=56972 max_load_reduction_pct=0.869324 moved_to_echo=32104 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 53, 713, 38, 0, 0, 7408, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 535, 0, 0, 2819, 29, 0, 0, 0, 1, 4549, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 1, 1, 1, 0, 0, 3, 0, 50, 2, 0, 0, 1, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 7844, 0, 0, 0, 1, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 0, 0, 0, 2840, 0, 0, 0, 0, 0, 2019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5878, 0, 4, 0, 1, 9, 0, 9, 0, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 16, 5801, 0, 0, 3, 3, 0, 1306, 446, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 2, 7, 0, 0, 1, 0, 393, 0, 0, 0, 0, 6743, 940, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 7957, 8214, 8107, 8632, 8162, 8005, 8238] avg=8192.0000 max=8632 max_rank=4 min=7957 min_rank=1 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=33037 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 2, 25, 593, 20, 0, 0, 7401, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 476, 1, 3, 2912, 9, 0, 0, 0, 2, 4548, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 1, 0, 0, 6, 1, 0, 24, 2, 0, 0, 1, 1, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 3, 3, 0, 0, 0, 0, 7854, 0, 0, 0, 3, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3694, 0, 2, 0, 2579, 1, 0, 3, 1, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 6241, 0, 27, 0, 1, 3, 2, 2, 0, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 2, 0, 0, 0, 39, 5912, 0, 0, 3, 1, 0, 1186, 446, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 1, 1, 0, 0, 0, 2, 353, 0, 0, 0, 0, 6331, 940, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8050, 7976, 8197, 8123, 8942, 8536, 8015, 7697] avg=8192.0000 max=8942 max_rank=4 min=7697 min_rank=7 imbalance_ratio=1.091553 max_load_reduction=56594 max_load_reduction_pct=0.863556 moved_to_echo=33190 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 91, 566, 44, 0, 1, 7430, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 561, 0, 0, 3679, 3, 1, 0, 0, 14, 4547, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 1, 0, 4, 0, 0, 85, 3, 0, 0, 11, 1, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 8029, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 3, 0, 2202, 0, 0, 6, 0, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 5668, 0, 60, 0, 0, 29, 0, 70, 0, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 1, 0, 0, 0, 19, 5804, 0, 0, 9, 0, 0, 1012, 446, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 9, 0, 1, 0, 0, 0, 353, 1, 0, 0, 0, 7202, 940, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8815, 8265, 8290, 7599, 8090, 7762, 8573] avg=8192.0000 max=8815 max_rank=1 min=7599 min_rank=4 imbalance_ratio=1.076050 max_load_reduction=56721 max_load_reduction_pct=0.865494 moved_to_echo=32024 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] + [2026-08-26 16:27:14] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 452.4 | throughput per GPU (TFLOP/s/GPU): 399.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.355710E+01 | loss scale: 1.0 | grad norm: 121.570 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 41, 1265, 46, 0, 0, 8096, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 513, 0, 0, 3695, 6, 2, 0, 8192, 4, 8190, 0, 4, 0, 0, 8, 0, 0, 4, 0, 1, 1, 0, 14, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 1, 0, 0, 4, 3, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 0, 0, 2, 3725, 0, 6614, 2, 2, 0, 2113, 0, 0, 0, 0, 0, 1, 0, 0, 5060, 0, 2, 0, 0, 2, 2, 20, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 11, 5695, 0, 0, 5, 7, 0, 0, 222, 0, 0, 0, 0, 1, 0, 0, 1, 0, 408, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 91, 731, 7, 0, 0, 8135, 0, 0, 0, 2, 1, 6, 0, 1, 0, 1, 706, 0, 0, 2183, 14, 0, 0, 8191, 5, 8190, 1, 5, 1, 0, 7, 0, 2, 2, 0, 1, 2, 0, 7, 1, 0, 1, 0, 0, 0, 3, 4, 1, 0, 0, 1, 0, 2, 0, 0, 7912, 0, 0, 0, 2, 0, 0, 0, 0, 3974, 0, 0, 0, 4496, 0, 6872, 12, 0, 0, 2637, 0, 1, 0, 0, 0, 0, 0, 0, 4270, 0, 31, 0, 0, 0, 0, 6, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 18, 6506, 0, 0, 4, 0, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 1, 0, 439, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 0, 2, 31, 734, 33, 0, 0, 8109, 0, 0, 0, 1, 0, 5, 0, 0, 0, 1, 500, 1, 3, 3049, 10, 0, 0, 8192, 2, 8191, 1, 17, 0, 0, 20, 1, 1, 0, 0, 6, 1, 0, 23, 3, 0, 0, 2, 1, 0, 1, 3, 0, 0, 3, 3, 0, 0, 0, 0, 7934, 0, 0, 0, 4, 3, 0, 0, 0, 3679, 0, 2, 0, 3944, 1, 6478, 7, 1, 0, 3021, 0, 0, 2, 0, 0, 1, 0, 0, 4944, 0, 76, 0, 1, 3, 2, 1, 0, 0, 11, 0, 0, 0, 2, 0, 0, 0, 44, 6001, 0, 0, 3, 1, 0, 0, 36, 0, 0, 0, 1, 1, 0, 0, 0, 2, 377, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 42, 891, 46, 0, 0, 8158, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 654, 0, 0, 3521, 3, 0, 0, 8191, 0, 8191, 3, 2, 0, 0, 19, 0, 0, 0, 0, 0, 9, 0, 15, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 3, 0, 8003, 0, 0, 0, 2, 1, 0, 0, 0, 3877, 0, 0, 0, 3976, 0, 7014, 0, 0, 0, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 4259, 0, 33, 0, 0, 10, 0, 41, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 9, 5838, 0, 0, 7, 0, 0, 0, 64, 0, 1, 0, 1, 2, 0, 0, 0, 1, 439, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 60, 911, 76, 0, 0, 8105, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 568, 0, 0, 2961, 33, 0, 0, 8192, 1, 8192, 0, 15, 0, 1, 12, 1, 1, 1, 0, 0, 3, 0, 51, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 7913, 0, 0, 0, 2, 0, 0, 0, 0, 3803, 0, 0, 0, 4336, 0, 6817, 1, 0, 0, 2434, 0, 0, 0, 0, 0, 4, 0, 0, 4477, 0, 15, 0, 1, 9, 0, 9, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 17, 5883, 0, 0, 3, 3, 0, 0, 117, 0, 0, 0, 3, 7, 0, 0, 1, 0, 459, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 27, 1031, 28, 0, 0, 8117, 0, 0, 1, 0, 0, 0, 2, 1, 1, 2, 594, 0, 0, 2656, 8, 0, 0, 8190, 4, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 7972, 1, 0, 0, 0, 0, 1, 0, 0, 3000, 0, 0, 0, 4270, 0, 6748, 2, 0, 0, 2377, 0, 0, 0, 0, 0, 0, 0, 0, 4903, 0, 25, 0, 0, 11, 0, 3, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4, 6879, 0, 0, 1, 0, 0, 1, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 41, 639, 56, 0, 0, 8037, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 466, 0, 0, 4258, 7, 1, 0, 8185, 4, 8184, 0, 0, 1, 3, 11, 0, 1, 0, 0, 0, 0, 2, 7, 0, 0, 0, 1, 0, 0, 5, 23, 0, 0, 0, 3, 0, 4, 5, 0, 8070, 0, 2, 1, 0, 2, 0, 0, 1, 3536, 0, 0, 0, 3783, 2, 6847, 9, 0, 3, 1888, 0, 0, 2, 0, 2, 10, 0, 1, 4087, 1, 12, 0, 0, 4, 0, 11, 0, 1, 43, 3, 0, 0, 0, 0, 0, 1, 5, 6677, 1, 0, 9, 3, 2, 0, 141, 0, 3, 0, 6, 1, 0, 0, 0, 0, 417, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 111, 729, 54, 0, 1, 8112, 0, 0, 0, 0, 5, 5, 0, 1, 0, 1, 612, 0, 0, 3927, 3, 1, 1, 8191, 14, 8190, 0, 1, 0, 0, 16, 0, 4, 1, 0, 4, 2, 0, 77, 6, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 0, 3, 0, 3354, 0, 7221, 11, 0, 0, 2080, 0, 0, 1, 0, 0, 4, 0, 0, 4330, 0, 102, 0, 0, 29, 0, 70, 0, 0, 52, 0, 0, 0, 1, 0, 0, 0, 20, 5916, 0, 0, 11, 0, 0, 0, 31, 0, 1, 0, 8, 1, 1, 0, 0, 0, 414, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 91, 731, 7, 0, 0, 7242, 0, 0, 0, 2, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 706, 0, 0, 2183, 14, 0, 0, 0, 5, 4311, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 2, 2, 0, 1, 2, 0, 7, 1, 0, 1, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 1, 0, 0, 1, 0, 2, 0, 0, 7912, 0, 0, 0, 2, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 2478, 0, 0, 12, 0, 0, 2637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4270, 0, 31, 0, 0, 0, 0, 6, 0, 3073, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 18, 6506, 0, 0, 4, 0, 0, 1446, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 1, 0, 439, 0, 0, 0, 0, 6872, 806, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8094, 7227, 8166, 8126, 9101, 7953, 8676, 8193] avg=8192.0000 max=9101 max_rank=4 min=7227 min_rank=1 imbalance_ratio=1.110962 max_load_reduction=56435 max_load_reduction_pct=0.861130 moved_to_echo=33923 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 41, 1265, 46, 0, 0, 7200, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 0, 0, 3695, 6, 2, 0, 0, 4, 4305, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4, 0, 1, 1, 0, 14, 0, 0, 0, 2, 4, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 3, 0, 7952, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 0, 0, 2, 2046, 0, 0, 2, 2, 0, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5060, 0, 2, 0, 0, 2, 2, 20, 0, 3076, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 11, 5695, 0, 0, 5, 7, 0, 1201, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 0, 0, 1, 0, 0, 1, 0, 408, 0, 0, 0, 0, 6614, 809, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8560, 8529, 8177, 8164, 7740, 8641, 7621, 8104] avg=8192.0000 max=8641 max_rank=5 min=7621 min_rank=6 imbalance_ratio=1.054810 max_load_reduction=56895 max_load_reduction_pct=0.868149 moved_to_echo=32106 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 42, 891, 46, 0, 0, 7262, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 654, 0, 0, 3521, 3, 0, 0, 0, 0, 4312, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 9, 0, 15, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 3, 0, 8003, 0, 0, 0, 2, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3877, 0, 0, 0, 2191, 0, 0, 0, 0, 0, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4259, 0, 33, 0, 0, 10, 0, 41, 1, 3073, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 9, 5838, 0, 0, 7, 0, 0, 1279, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 1, 0, 1, 2, 0, 0, 0, 1, 439, 0, 0, 0, 0, 7014, 806, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8496, 8185, 8223, 8253, 7923, 7833, 8377] avg=8192.0000 max=8496 max_rank=1 min=7833 min_rank=6 imbalance_ratio=1.037109 max_load_reduction=57040 max_load_reduction_pct=0.870361 moved_to_echo=32386 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 60, 911, 76, 0, 0, 7216, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 568, 0, 0, 2961, 33, 0, 0, 0, 1, 4312, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 1, 1, 1, 0, 0, 3, 0, 51, 2, 0, 0, 1, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 7913, 0, 0, 0, 2, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 0, 0, 2390, 0, 0, 1, 0, 0, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4477, 0, 15, 0, 1, 9, 0, 9, 0, 3074, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 17, 5883, 0, 0, 3, 3, 0, 1395, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 3, 7, 0, 0, 1, 0, 459, 0, 0, 0, 0, 6817, 806, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 7892, 8215, 8123, 8628, 8140, 8006, 8260] avg=8192.0000 max=8628 max_rank=4 min=7892 min_rank=1 imbalance_ratio=1.053223 max_load_reduction=56908 max_load_reduction_pct=0.868347 moved_to_echo=33034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 41, 639, 56, 0, 0, 7154, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 4258, 7, 1, 0, 0, 4, 4308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 11, 0, 1, 0, 0, 0, 0, 2, 7, 0, 0, 0, 1, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 23, 0, 0, 0, 3, 0, 4, 5, 0, 8070, 0, 2, 1, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3536, 0, 0, 0, 2085, 2, 0, 9, 0, 3, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 10, 0, 1, 4087, 1, 12, 0, 0, 4, 0, 11, 0, 3071, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 43, 3, 0, 0, 0, 0, 0, 1, 5, 6677, 1, 0, 9, 3, 2, 1217, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 3, 0, 6, 1, 0, 0, 0, 0, 417, 0, 0, 0, 0, 6847, 805, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7895, 9044, 8163, 8312, 7526, 7682, 8646, 8268] avg=8192.0000 max=9044 max_rank=1 min=7526 min_rank=4 imbalance_ratio=1.104004 max_load_reduction=56492 max_load_reduction_pct=0.862000 moved_to_echo=32122 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 27, 1031, 28, 0, 0, 7226, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 2, 594, 0, 0, 2656, 8, 0, 0, 0, 4, 4311, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 1, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 7972, 1, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3000, 0, 0, 0, 2353, 0, 0, 2, 0, 0, 2377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4903, 0, 25, 0, 0, 11, 0, 3, 0, 3073, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4, 6879, 0, 0, 1, 0, 0, 1374, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 0, 6748, 806, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 7580, 8162, 8177, 7733, 8558, 8964, 8049] avg=8192.0000 max=8964 max_rank=6 min=7580 min_rank=1 imbalance_ratio=1.094238 max_load_reduction=56572 max_load_reduction_pct=0.863220 moved_to_echo=33304 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 111, 729, 54, 0, 1, 7222, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 612, 0, 0, 3927, 3, 1, 1, 0, 14, 4311, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4, 1, 0, 4, 2, 0, 77, 6, 0, 0, 14, 1, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 8059, 0, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 0, 3, 0, 1849, 0, 0, 11, 0, 0, 2080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 4330, 0, 102, 0, 0, 29, 0, 70, 0, 3073, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 1, 0, 0, 0, 20, 5916, 0, 0, 11, 0, 0, 1079, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 1, 0, 8, 1, 1, 0, 0, 0, 414, 3, 0, 0, 0, 7221, 806, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8872, 8267, 8263, 7667, 8035, 7769, 8535] avg=8192.0000 max=8872 max_rank=1 min=7667 min_rank=4 imbalance_ratio=1.083008 max_load_reduction=56664 max_load_reduction_pct=0.864624 moved_to_echo=32006 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 2, 31, 734, 33, 0, 0, 7219, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 500, 1, 3, 3049, 10, 0, 0, 0, 2, 4312, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 1, 0, 0, 6, 1, 0, 23, 3, 0, 0, 2, 1, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 3, 3, 0, 0, 0, 0, 7934, 0, 0, 0, 4, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3679, 0, 2, 0, 2174, 1, 0, 7, 1, 0, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 4944, 0, 76, 0, 1, 3, 2, 1, 0, 3073, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 2, 0, 0, 0, 44, 6001, 0, 0, 3, 1, 0, 1269, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 1, 1, 0, 0, 0, 2, 377, 0, 0, 0, 0, 6478, 806, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8028, 7896, 8201, 8148, 8888, 8604, 8021, 7750] avg=8192.0000 max=8888 max_rank=4 min=7750 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=56648 max_load_reduction_pct=0.864380 moved_to_echo=33263 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 4224, 128, 128, 24832, 128, 128, 0, 0, 128, 36480, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 29056, 0, 128, 128, 20864, 128, 0, 128, 128, 128, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 47104, 128, 256, 0, 128, 128, 128, 256, 128, 18176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 256, 128, 0, 0, 128, 0, 0, 128, 128, 48768, 128, 0, 128, 128, 128, 9600, 3584, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 768, 0, 128, 0, 128, 128, 128, 0, 128, 128, 3072, 128, 0, 0, 0, 54016, 7552, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 384, 5632, 256, 0, 128, 59392, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 63488, 128, 128, 128, 128, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 256, 128, 0, 128, 128, 128, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 36352, 128, 384, 0, 128, 128, 128, 256, 128, 24704, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 4736, 128, 128, 26368, 128, 128, 128, 0, 128, 34560, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 29184, 0, 128, 128, 17664, 128, 0, 128, 128, 128, 18816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 256, 128, 0, 0, 128, 0, 0, 128, 128, 49408, 128, 0, 128, 128, 128, 10368, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 768, 0, 128, 0, 128, 128, 128, 0, 128, 128, 3328, 128, 0, 0, 0, 54656, 6528, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 512, 7040, 384, 0, 128, 57856, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 63872, 128, 128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 256, 128, 0, 128, 128, 128, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8185, 1, 0, 1, 0, 0, 3, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 6, 2, 8187, 8184, 2, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 8186, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8185, 1, 0, 1, 0, 0, 3, 3, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 6, 2, 8187, 8184, 2, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 8186, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8189, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8186, 8193, 8197, 8192, 8192, 8193] avg=8192.0000 max=8197 max_rank=4 min=8186 min_rank=2 imbalance_ratio=1.000610 max_load_reduction=57339 max_load_reduction_pct=0.874924 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8187, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8195, 8192, 8193, 8191, 8192, 8193] avg=8192.0000 max=8195 max_rank=2 min=8188 min_rank=0 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8182, 1, 0, 1, 0, 0, 3, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 8184, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 8182, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8195, 8191, 8192, 8192, 8193, 8188, 8192] avg=8192.0000 max=8195 max_rank=1 min=8188 min_rank=6 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8192, 8188, 8193, 8191, 8191, 8192, 8193] avg=8192.0000 max=8196 max_rank=0 min=8188 min_rank=2 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8196, 8193, 8191, 8192, 8192, 8191] avg=8192.0000 max=8196 max_rank=2 min=8189 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8178, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8188, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8192, 8191, 8194, 8199, 8192, 8196, 8191] avg=8192.0000 max=8199 max_rank=4 min=8181 min_rank=0 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8175, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8192, 8196, 8186, 8180, 8193, 8192, 8192] avg=8192.0000 max=8205 max_rank=0 min=8180 min_rank=4 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 8193, 8193, 8193, 8192, 8192, 8191] avg=8192.0000 max=8193 max_rank=0 min=8189 min_rank=1 imbalance_ratio=1.000122 max_load_reduction=57343 max_load_reduction_pct=0.874985 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8185, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8187, 8193, 8197, 8192, 8192, 8193] avg=8192.0000 max=8197 max_rank=4 min=8187 min_rank=2 imbalance_ratio=1.000610 max_load_reduction=57339 max_load_reduction_pct=0.874924 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8192, 8187, 8193, 8192, 8191, 8192, 8193] avg=8192.0000 max=8196 max_rank=0 min=8187 min_rank=2 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8182, 1, 0, 1, 0, 0, 3, 3, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 8184, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8195, 8191, 8192, 8191, 8192, 8188, 8193] avg=8192.0000 max=8195 max_rank=1 min=8188 min_rank=6 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8196, 8193, 8192, 8192, 8192, 8191] avg=8192.0000 max=8196 max_rank=2 min=8188 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8187, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8192, 8195, 8192, 8194, 8192, 8192, 8192] avg=8192.0000 max=8195 max_rank=2 min=8187 min_rank=0 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 8193, 8193, 8194, 8192, 8192, 8191] avg=8192.0000 max=8194 max_rank=4 min=8189 min_rank=1 imbalance_ratio=1.000244 max_load_reduction=57342 max_load_reduction_pct=0.874969 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8181, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8188, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8192, 8191, 8194, 8196, 8192, 8196, 8191] avg=8192.0000 max=8196 max_rank=4 min=8184 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8186, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8174, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8192, 8196, 8186, 8180, 8193, 8192, 8192] avg=8192.0000 max=8205 max_rank=0 min=8180 min_rank=4 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[65536, 128, 128, 128, 0, 0, 128, 128, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 65536, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 128, 128, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 65536, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[65536, 128, 128, 128, 0, 0, 128, 128, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 128, 128, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 8176, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1991, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6218, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 8178, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8120, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 5, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 1878, 0, 0, 0, 5, 8182, 0, 0, 8, 0, 0, 0, 0, 0, 8169, 0, 8180, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 6, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6305, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2, 6, 0, 0, 0, 0, 0, 0, 8183, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4517, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3455, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6032, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2145, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2975, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 5414, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 1, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1991, 0, 0, 0, 4492, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6218, 0, 0, 0, 0, 1, 0, 0, 3662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8176, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8188, 6489, 8216, 8190, 9881, 8178, 8196] avg=8192.0000 max=9881 max_rank=5 min=6489 min_rank=2 imbalance_ratio=1.206177 max_load_reduction=55655 max_load_reduction_pct=0.849228 moved_to_echo=34445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3663, 0, 0, 0, 4492, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 36, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4517, 0, 1, 0, 0, 0, 0, 0, 3662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8177, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8203, 8159, 8230, 8190, 8180, 8192, 8191] avg=8192.0000 max=8230 max_rank=3 min=8159 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 8178, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 5, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 1878, 0, 0, 0, 4487, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8169, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 6, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6305, 0, 0, 0, 0, 0, 0, 0, 3658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2, 6, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8136, 6445, 8217, 8201, 9967, 8176, 8203] avg=8192.0000 max=9967 max_rank=5 min=6445 min_rank=2 imbalance_ratio=1.216675 max_load_reduction=55569 max_load_reduction_pct=0.847916 moved_to_echo=34547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3455, 0, 0, 0, 4493, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8203, 7953, 8099, 8191, 8527, 8182, 8193] avg=8192.0000 max=8527 max_rank=5 min=7953 min_rank=2 imbalance_ratio=1.040894 max_load_reduction=57009 max_load_reduction_pct=0.869888 moved_to_echo=33093 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4493, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4122, 0, 0, 0, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8203, 8562, 8179, 8191, 7785, 8232, 8189] avg=8192.0000 max=8562 max_rank=2 min=7785 min_rank=5 imbalance_ratio=1.045166 max_load_reduction=56974 max_load_reduction_pct=0.869354 moved_to_echo=32397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2975, 0, 0, 0, 4496, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5219, 0, 0, 0, 0, 0, 0, 0, 3666, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8197, 7477, 8216, 8191, 8888, 8194, 8186] avg=8192.0000 max=8888 max_rank=5 min=7477 min_rank=2 imbalance_ratio=1.084961 max_load_reduction=56648 max_load_reduction_pct=0.864380 moved_to_echo=33459 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 5414, 0, 0, 0, 4488, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 36, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 9, 0, 0, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8203, 9923, 8149, 8191, 6499, 8191, 8189] avg=8192.0000 max=9923 max_rank=2 min=6499 min_rank=5 imbalance_ratio=1.211304 max_load_reduction=55613 max_load_reduction_pct=0.848587 moved_to_echo=31070 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6032, 0, 0, 0, 4492, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 36, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2145, 0, 2, 0, 0, 0, 0, 0, 3662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8203, 10528, 8230, 8191, 5809, 8191, 8189] avg=8192.0000 max=10528 max_rank=2 min=5809 min_rank=5 imbalance_ratio=1.285156 max_load_reduction=55008 max_load_reduction_pct=0.839355 moved_to_echo=30380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 8178, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8123, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 5, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8182, 0, 0, 8, 0, 0, 0, 0, 0, 8176, 0, 8180, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 6, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 6, 0, 0, 0, 0, 0, 0, 8183, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 8178, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 21, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 8171, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 8149, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8203, 8178, 8196, 8191, 8197, 8189, 8191] avg=8192.0000 max=8203 max_rank=1 min=8178 min_rank=2 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 8178, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 5, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8176, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 6, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 6, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8138, 8234, 8191, 8201, 8217, 8165, 8199] avg=8192.0000 max=8234 max_rank=2 min=8138 min_rank=1 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 1, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8171, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8149, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8189, 8184, 8186, 8190, 8225, 8167, 8198] avg=8192.0000 max=8225 max_rank=5 min=8167 min_rank=6 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8172, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 8149, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8203, 8175, 8194, 8191, 8161, 8229, 8189] avg=8192.0000 max=8229 max_rank=6 min=8161 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 21, 0, 0, 0, 8171, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 1, 0, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 8149, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8203, 8194, 8196, 8191, 8146, 8223, 8189] avg=8192.0000 max=8223 max_rank=6 min=8146 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 8172, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8149, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8203, 8195, 8194, 8191, 8191, 8178, 8194] avg=8192.0000 max=8203 max_rank=1 min=8178 min_rank=6 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 8174, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 8155, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8194, 8196, 8192, 8190, 8182, 8207, 8187] avg=8192.0000 max=8207 max_rank=6 min=8182 min_rank=5 imbalance_ratio=1.001831 max_load_reduction=57329 max_load_reduction_pct=0.874771 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 8164, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 9, 0, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8149, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8203, 8180, 8187, 8191, 8217, 8178, 8189] avg=8192.0000 max=8217 max_rank=5 min=8178 min_rank=6 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 36224, 0, 128, 0, 0, 128, 0, 0, 29312, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 29568, 0, 0, 0, 35968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65280, 0, 384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 65280, 0, 128, 0, 0, 128, 0, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 65536, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 256, 0, 0, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 5, 0, 0, 0, 44, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8182, 0, 0, 0, 0, 1, 5394, 0, 0, 0, 0, 0, 0, 0, 8164, 8176, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6787, 0, 0, 0, 0, 0, 20, 0, 0, 0, 2, 4133, 8177, 0, 0, 0, 8, 1, 8182, 8172, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 8192, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 8192, 0, 0, 0, 0, 0, 8192, 8180, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 20, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 1, 7778, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7604, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 979, 8191, 0, 0, 0, 1, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 8192, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 7206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8183, 8192, 8648, 8197, 8188, 8569, 7368] avg=8192.0000 max=8648 max_rank=3 min=7368 min_rank=7 imbalance_ratio=1.055664 max_load_reduction=56888 max_load_reduction_pct=0.868042 moved_to_echo=32322 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 5, 0, 8175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7725, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 457, 0, 0, 0, 0, 1, 5394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8176, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7, 0, 8157, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6787, 0, 0, 0, 0, 0, 20, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4133, 0, 0, 0, 0, 8, 1, 0, 7189, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8225, 8196, 5863, 8190, 8183, 7349, 11340] avg=8192.0000 max=11340 max_rank=7 min=5863 min_rank=3 imbalance_ratio=1.384277 max_load_reduction=54196 max_load_reduction_pct=0.826965 moved_to_echo=35062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7738, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 7198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8191, 8493, 8194, 8192, 8680, 7404] avg=8192.0000 max=8680 max_rank=6 min=7404 min_rank=7 imbalance_ratio=1.059570 max_load_reduction=56856 max_load_reduction_pct=0.867554 moved_to_echo=32470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 20, 0, 0, 0, 1, 0, 0, 7734, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 1, 1, 7778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7604, 0, 0, 0, 0, 0, 1, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 0, 0, 0, 0, 1, 0, 0, 7206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8204, 8192, 8237, 8198, 8186, 8143, 8186] avg=8192.0000 max=8237 max_rank=3 min=8143 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32713 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8183, 8192, 8648, 8198, 8188, 8730, 7207] avg=8192.0000 max=8730 max_rank=6 min=7207 min_rank=7 imbalance_ratio=1.065674 max_load_reduction=56806 max_load_reduction_pct=0.866791 moved_to_echo=32323 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 7206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8183, 8191, 8611, 8199, 8187, 8632, 7342] avg=8192.0000 max=8632 max_rank=6 min=7342 min_rank=7 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=32360 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7735, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 7195, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8184, 8191, 8544, 8163, 8223, 8728, 7303] avg=8192.0000 max=8728 max_rank=6 min=7303 min_rank=7 imbalance_ratio=1.065430 max_load_reduction=56808 max_load_reduction_pct=0.866821 moved_to_echo=32417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6168, 0, 0, 0, 0, 0, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 0, 0, 0, 0, 0, 7205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8183, 8191, 8492, 8197, 8189, 6705, 9386] avg=8192.0000 max=9386 max_rank=7 min=6705 min_rank=6 imbalance_ratio=1.145752 max_load_reduction=56150 max_load_reduction_pct=0.856781 moved_to_echo=32477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 5, 0, 0, 0, 43, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8182, 0, 0, 0, 0, 1, 6662, 0, 0, 0, 0, 0, 0, 0, 8165, 8177, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3597, 0, 0, 0, 0, 0, 19, 0, 0, 0, 3, 6055, 8178, 0, 0, 0, 8, 1, 8182, 8172, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 25, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 1, 7975, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5254, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3127, 8191, 0, 0, 0, 1, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5372, 8192, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1421, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 8192, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 8192, 0, 0, 0, 0, 0, 8192, 8180, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7937, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8182, 8192, 8447, 8198, 8189, 10343, 5796] avg=8192.0000 max=10343 max_rank=6 min=5796 min_rank=7 imbalance_ratio=1.262573 max_load_reduction=55193 max_load_reduction_pct=0.842178 moved_to_echo=32526 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 5, 0, 8176, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7927, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 255, 0, 0, 0, 0, 1, 6662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8177, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7, 0, 8158, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3597, 0, 0, 0, 0, 0, 19, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6055, 0, 0, 0, 0, 8, 1, 0, 5777, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8224, 8196, 6928, 8191, 8183, 5772, 11851] avg=8192.0000 max=11851 max_rank=7 min=5772 min_rank=6 imbalance_ratio=1.446655 max_load_reduction=53685 max_load_reduction_pct=0.819168 moved_to_echo=33997 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 25, 0, 0, 0, 1, 0, 0, 7936, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 1, 1, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5254, 0, 0, 0, 0, 0, 1, 2155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3127, 0, 0, 0, 0, 1, 0, 0, 5791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8208, 8192, 8232, 8198, 8186, 7411, 8919] avg=8192.0000 max=8919 max_rank=7 min=7411 min_rank=6 imbalance_ratio=1.088745 max_load_reduction=56617 max_load_reduction_pct=0.863907 moved_to_echo=32714 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7937, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2852, 0, 0, 0, 0, 0, 0, 2154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5372, 0, 0, 0, 0, 0, 0, 0, 5791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8182, 8191, 8415, 8197, 8189, 5006, 11163] avg=8192.0000 max=11163 max_rank=7 min=5006 min_rank=6 imbalance_ratio=1.362671 max_load_reduction=54373 max_load_reduction_pct=0.829666 moved_to_echo=32555 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7937, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6762, 0, 0, 0, 0, 0, 0, 2155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 0, 0, 0, 0, 0, 0, 0, 5792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8183, 8191, 8432, 8199, 8187, 8917, 7236] avg=8192.0000 max=8917 max_rank=6 min=7236 min_rank=7 imbalance_ratio=1.088501 max_load_reduction=56619 max_load_reduction_pct=0.863937 moved_to_echo=32539 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 249, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6892, 0, 0, 0, 0, 0, 0, 2158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 0, 5785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8191, 8322, 8193, 8192, 9050, 7206] avg=8192.0000 max=9050 max_rank=6 min=7206 min_rank=7 imbalance_ratio=1.104736 max_load_reduction=56486 max_load_reduction_pct=0.861908 moved_to_echo=32641 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7937, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6542, 0, 0, 0, 0, 0, 0, 2155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 5792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8183, 8192, 8447, 8196, 8188, 8697, 7442] avg=8192.0000 max=8697 max_rank=6 min=7442 min_rank=7 imbalance_ratio=1.061646 max_load_reduction=56839 max_load_reduction_pct=0.867294 moved_to_echo=32523 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7937, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 5783, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8183, 8191, 8313, 8164, 8222, 10340, 5923] avg=8192.0000 max=10340 max_rank=6 min=5923 min_rank=7 imbalance_ratio=1.262207 max_load_reduction=55196 max_load_reduction_pct=0.842224 moved_to_echo=32649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 7936, 0, 0, 0, 0, 128, 128, 0, 57728, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 61952, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 61312, 0, 0, 0, 0, 0, 128, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 3712, 0, 0, 0, 128, 128, 61952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 48384, 0, 0, 0, 0, 0, 128, 17280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 19328, 0, 0, 0, 0, 128, 128, 0, 46336, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 63616, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 2048, 0, 0, 0, 128, 128, 63488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 65536, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 8174, 0, 0, 0, 455, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 1, 0, 0, 5021, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8179, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4148, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 70, 0, 0, 8107, 0, 0, 0, 42, 0, 0, 0, 0, 8178, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 13, 10, 0, 0, 6852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8125, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 7, 0, 5, 0, 27, 0, 9, 0, 0, 0, 0, 1120, 0, 34, 8, 0, 4, 8158, 0, 5, 0, 0, 0, 0, 0, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4245, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1510, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8181, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7498, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7505, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 8117, 0, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 1, 0, 0, 5021, 0, 0, 0, 1, 0, 0, 0, 0, 4579, 57, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8171, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 0, 0, 0, 0, 0, 3609, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8576, 8195, 9686, 8194, 8178, 8191, 8190, 6326] avg=8192.0000 max=9686 max_rank=2 min=6326 min_rank=7 imbalance_ratio=1.182373 max_load_reduction=55850 max_load_reduction_pct=0.852203 moved_to_echo=30885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8135, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 4582, 57, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4148, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8193, 8689, 8193, 8184, 8191, 8192, 7758] avg=8192.0000 max=8689 max_rank=2 min=7758 min_rank=7 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=32325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 4582, 57, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4245, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8195, 8596, 8189, 8184, 8190, 8192, 7855] avg=8192.0000 max=8596 max_rank=2 min=7855 min_rank=7 imbalance_ratio=1.049316 max_load_reduction=56940 max_load_reduction_pct=0.868835 moved_to_echo=32421 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 4582, 57, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8195, 4973, 8189, 8184, 8191, 8192, 11477] avg=8192.0000 max=11477 max_rank=7 min=4973 min_rank=2 imbalance_ratio=1.401001 max_load_reduction=54059 max_load_reduction_pct=0.824875 moved_to_echo=36044 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 70, 0, 0, 8050, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 13, 10, 0, 0, 6852, 0, 0, 0, 0, 0, 0, 0, 0, 4563, 57, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8117, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 7, 0, 5, 0, 27, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 0, 34, 8, 0, 4, 3595, 0, 5, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8182, 11572, 8198, 8258, 8197, 8191, 4776] avg=8192.0000 max=11572 max_rank=2 min=4776 min_rank=7 imbalance_ratio=1.412598 max_load_reduction=53964 max_load_reduction_pct=0.823425 moved_to_echo=29422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 4583, 63, 12, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7505, 0, 2, 0, 0, 0, 3609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8194, 5349, 8184, 8180, 8190, 8194, 11116] avg=8192.0000 max=11116 max_rank=7 min=5349 min_rank=2 imbalance_ratio=1.356934 max_load_reduction=54420 max_load_reduction_pct=0.830383 moved_to_echo=35680 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8124, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 693, 0, 0, 0, 0, 0, 0, 0, 0, 4582, 57, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7498, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8189, 5342, 8200, 8184, 8194, 8193, 11108] avg=8192.0000 max=11108 max_rank=7 min=5342 min_rank=2 imbalance_ratio=1.355957 max_load_reduction=54428 max_load_reduction_pct=0.830505 moved_to_echo=35679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8135, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6680, 0, 0, 0, 0, 0, 0, 0, 0, 4582, 57, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1510, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8193, 11329, 8189, 8184, 8192, 8192, 5120] avg=8192.0000 max=11329 max_rank=2 min=5120 min_rank=7 imbalance_ratio=1.382935 max_load_reduction=54207 max_load_reduction_pct=0.827133 moved_to_echo=29688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7674, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4456, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4126, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 8174, 0, 0, 0, 480, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 1, 0, 0, 4798, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8179, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2914, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 67, 0, 0, 8108, 0, 0, 0, 54, 0, 0, 0, 0, 8179, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 13, 10, 0, 0, 6936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8124, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 7, 0, 5, 0, 26, 0, 9, 0, 0, 0, 0, 1030, 0, 34, 8, 0, 3, 8159, 0, 5, 0, 0, 0, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8182, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7416, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7374, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 8111, 0, 0, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 1, 0, 0, 4798, 0, 0, 0, 1, 0, 0, 0, 0, 4539, 63, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8172, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2914, 0, 0, 0, 0, 0, 3649, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8594, 8194, 9427, 8195, 8180, 8191, 8190, 6565] avg=8192.0000 max=9427 max_rank=2 min=6565 min_rank=7 imbalance_ratio=1.150757 max_load_reduction=56109 max_load_reduction_pct=0.856155 moved_to_echo=31126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8129, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 0, 0, 0, 0, 0, 0, 0, 0, 4541, 63, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7674, 0, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8194, 5128, 8190, 8185, 8190, 8192, 11325] avg=8192.0000 max=11325 max_rank=7 min=5128 min_rank=2 imbalance_ratio=1.382446 max_load_reduction=54211 max_load_reduction_pct=0.827194 moved_to_echo=35892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8129, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 4541, 63, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4126, 0, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8192, 8674, 8193, 8185, 8191, 8192, 7777] avg=8192.0000 max=8674 max_rank=2 min=7777 min_rank=7 imbalance_ratio=1.058838 max_load_reduction=56862 max_load_reduction_pct=0.867645 moved_to_echo=32345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8129, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 0, 0, 0, 0, 0, 0, 0, 4541, 63, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4456, 0, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8194, 8347, 8190, 8185, 8190, 8192, 8107] avg=8192.0000 max=8347 max_rank=2 min=8107 min_rank=7 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 67, 0, 0, 8045, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 13, 10, 0, 0, 6936, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 63, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8117, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 7, 0, 5, 0, 26, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 0, 34, 8, 0, 3, 3636, 0, 5, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8182, 11621, 8199, 8252, 8198, 8193, 4725] avg=8192.0000 max=11621 max_rank=2 min=4725 min_rank=7 imbalance_ratio=1.418579 max_load_reduction=53915 max_load_reduction_pct=0.822678 moved_to_echo=29368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8129, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6891, 0, 0, 0, 0, 0, 0, 0, 0, 4541, 63, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8192, 11504, 8190, 8185, 8192, 8192, 4945] avg=8192.0000 max=11504 max_rank=2 min=4945 min_rank=7 imbalance_ratio=1.404297 max_load_reduction=54032 max_load_reduction_pct=0.824463 moved_to_echo=29514 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8124, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 4543, 68, 13, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7374, 0, 2, 0, 0, 0, 3649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8199, 5447, 8179, 8179, 8190, 8192, 11025] avg=8192.0000 max=11025 max_rank=7 min=5447 min_rank=2 imbalance_ratio=1.345825 max_load_reduction=54511 max_load_reduction_pct=0.831772 moved_to_echo=35586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8119, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 0, 0, 0, 0, 0, 0, 4541, 63, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 0, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8189, 5388, 8200, 8185, 8194, 8193, 11067] avg=8192.0000 max=11067 max_rank=7 min=5388 min_rank=2 imbalance_ratio=1.350952 max_load_reduction=54469 max_load_reduction_pct=0.831131 moved_to_echo=35639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[36608, 0, 128, 128, 0, 128, 28928, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 28288, 0, 0, 0, 128, 0, 0, 0, 0, 36736, 512, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 0, 65024, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[36352, 0, 128, 128, 0, 128, 29312, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 28544, 0, 0, 0, 128, 0, 0, 0, 0, 36352, 512, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 65024, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 65408, 0, 256, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 65536, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65536, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 65408, 0, 256, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 65536, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8183, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 33, 8185, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7756, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8172, 2, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 190, 8081, 0, 4, 0, 0, 0, 9, 11, 0, 8, 0, 0, 8, 0, 8157, 0, 0, 0, 0, 0, 0, 1993, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 5999, 0, 0, 0, 6, 0, 8179, 0, 0, 0, 1, 0, 4, 0, 0, 0, 8178, 0, 0, 0, 5, 0, 0, 0, 12, 0, 6, 0, 0, 10, 0, 11, 1, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 102, 0, 0, 0, 0, 0, 0, 7965, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8179, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 8092, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8176, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 127, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8180, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7780, 48, 13, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8191, 7882, 8178, 8539, 8189, 8183, 8195] avg=8192.0000 max=8539 max_rank=4 min=7882 min_rank=2 imbalance_ratio=1.042358 max_load_reduction=56997 max_load_reduction_pct=0.869705 moved_to_echo=33106 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 7780, 48, 13, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7756, 1, 0, 0, 0, 0, 411, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8196, 8256, 8136, 8176, 8190, 8183, 8194] avg=8192.0000 max=8256 max_rank=2 min=8136 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32743 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 9, 11, 0, 8, 0, 0, 8, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1993, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 7769, 48, 13, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5999, 0, 0, 0, 6, 0, 410, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 5, 0, 0, 0, 12, 0, 6, 0, 0, 10, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8197, 9846, 8156, 6420, 8219, 8258, 8180] avg=8192.0000 max=9846 max_rank=2 min=6420 min_rank=4 imbalance_ratio=1.201904 max_load_reduction=55690 max_load_reduction_pct=0.849762 moved_to_echo=31077 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 49, 13, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 2, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8192, 7920, 8175, 8505, 8189, 8183, 8193] avg=8192.0000 max=8505 max_rank=4 min=7920 min_rank=2 imbalance_ratio=1.038208 max_load_reduction=57031 max_load_reduction_pct=0.870224 moved_to_echo=33070 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7788, 50, 15, 12, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8185, 7942, 8178, 8500, 8182, 8180, 8193] avg=8192.0000 max=8500 max_rank=4 min=7942 min_rank=2 imbalance_ratio=1.037598 max_load_reduction=57036 max_load_reduction_pct=0.870300 moved_to_echo=33055 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 48, 13, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8191, 7855, 8204, 8540, 8189, 8183, 8195] avg=8192.0000 max=8540 max_rank=4 min=7855 min_rank=2 imbalance_ratio=1.042480 max_load_reduction=56996 max_load_reduction_pct=0.869690 moved_to_echo=33107 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 48, 13, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 102, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7965, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8192, 7979, 8245, 8376, 8189, 8183, 8193] avg=8192.0000 max=8376 max_rank=4 min=7979 min_rank=2 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7780, 50, 13, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 127, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8192, 7856, 8264, 8480, 8189, 8183, 8193] avg=8192.0000 max=8480 max_rank=4 min=7856 min_rank=2 imbalance_ratio=1.035156 max_load_reduction=57056 max_load_reduction_pct=0.870605 moved_to_echo=33045 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 35, 8181, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 7826, 2, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8172, 2, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 197, 8058, 0, 3, 0, 0, 0, 9, 12, 0, 9, 0, 0, 8, 0, 8157, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 13, 6173, 0, 0, 0, 6, 0, 8179, 0, 0, 0, 1, 0, 4, 0, 0, 0, 8179, 0, 0, 0, 5, 0, 0, 0, 11, 0, 5, 0, 0, 10, 0, 11, 1, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 287, 0, 0, 0, 0, 0, 0, 7884, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8180, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 7833, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8176, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 441, 0, 0, 0, 0, 0, 0, 7755, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8182, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7690, 196, 10, 10, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 7994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8190, 7911, 8150, 8538, 8189, 8182, 8194] avg=8192.0000 max=8538 max_rank=4 min=7911 min_rank=2 imbalance_ratio=1.042236 max_load_reduction=56998 max_load_reduction_pct=0.869720 moved_to_echo=33103 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 9, 12, 0, 9, 0, 0, 8, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 7678, 195, 10, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 13, 7974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6173, 0, 0, 0, 6, 0, 501, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 5, 0, 0, 0, 11, 0, 5, 0, 0, 10, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8198, 9735, 8011, 6685, 8218, 8263, 8179] avg=8192.0000 max=9735 max_rank=2 min=6685 min_rank=4 imbalance_ratio=1.188354 max_load_reduction=55801 max_load_reduction_pct=0.851456 moved_to_echo=31345 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 7689, 195, 10, 10, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 7987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7826, 2, 0, 0, 0, 0, 502, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8195, 8229, 7998, 8340, 8190, 8182, 8196] avg=8192.0000 max=8340 max_rank=4 min=7998 min_rank=3 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32908 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7690, 196, 10, 10, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 7983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7833, 1, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8191, 7944, 8320, 8336, 8189, 8182, 8193] avg=8192.0000 max=8336 max_rank=4 min=7944 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7690, 196, 10, 10, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 287, 0, 0, 0, 0, 0, 0, 7993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7884, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8191, 7931, 8282, 8386, 8189, 8182, 8193] avg=8192.0000 max=8386 max_rank=4 min=7931 min_rank=2 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32950 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7693, 197, 17, 11, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 7984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8190, 7958, 8214, 8443, 8183, 8181, 8193] avg=8192.0000 max=8443 max_rank=4 min=7958 min_rank=2 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=33000 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 195, 10, 10, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 7989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8190, 7917, 8130, 8551, 8189, 8182, 8195] avg=8192.0000 max=8551 max_rank=4 min=7917 min_rank=2 imbalance_ratio=1.043823 max_load_reduction=56985 max_load_reduction_pct=0.869522 moved_to_echo=33117 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 197, 10, 10, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 441, 0, 0, 0, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7755, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8191, 7911, 8431, 8257, 8189, 8182, 8193] avg=8192.0000 max=8431 max_rank=3 min=7911 min_rank=2 imbalance_ratio=1.029175 max_load_reduction=57105 max_load_reduction_pct=0.871353 moved_to_echo=32821 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 2688, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 62336, 512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[62336, 128, 0, 0, 128, 0, 3328, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[61568, 128, 0, 0, 128, 0, 4096, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 2304, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 61568, 1664, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 0, 128, 0, 128, 1664, 128, 0, 0, 0, 0, 128, 64000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 0, 128, 0, 128, 512, 128, 0, 0, 0, 0, 128, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 1, 0, 8136, 603, 0, 0, 0, 0, 8191, 1, 0, 2, 0, 8182, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 375, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 83, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8173, 12, 0, 7568, 3933, 0, 0, 0, 0, 8171, 9, 0, 127, 0, 8056, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 10, 8028, 0, 0, 464, 0, 5, 0, 0, 0, 12, 3, 13, 12, 0, 1008, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 8171, 0, 0, 9, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3382, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8164] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 87, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7983, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7343, 489, 209, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7703, 8193, 8091, 8058, 8189, 8173, 8193, 8936] avg=8192.0000 max=8936 max_rank=7 min=7703 min_rank=0 imbalance_ratio=1.090820 max_load_reduction=56600 max_load_reduction_pct=0.863647 moved_to_echo=33491 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7651, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 2, 0, 8182, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 0, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 83, 0, 0, 1, 0, 0, 0, 7339, 485, 209, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8191, 8357, 8137, 8185, 8179, 8192, 8039] avg=8192.0000 max=8357 max_rank=2 min=8039 min_rank=7 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32595 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 12, 0, 7117, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 9, 0, 127, 0, 8056, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 10, 7823, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 12, 3, 13, 12, 0, 1008, 0, 0, 0, 0, 0, 0, 7317, 451, 205, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 8155, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3382, 0, 0, 0, 0, 0, 11, 0, 0, 0, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11064, 8207, 8304, 9042, 8183, 8317, 8179, 4240] avg=8192.0000 max=11064 max_rank=0 min=4240 min_rank=7 imbalance_ratio=1.350586 max_load_reduction=54472 max_load_reduction_pct=0.831177 moved_to_echo=28919 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7983, 0, 0, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7343, 489, 209, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7703, 8193, 8327, 8058, 8192, 8175, 8197, 8691] avg=8192.0000 max=8691 max_rank=7 min=7703 min_rank=0 imbalance_ratio=1.060913 max_load_reduction=56845 max_load_reduction_pct=0.867386 moved_to_echo=33255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7979, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7345, 491, 213, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7701, 8190, 8029, 8068, 8208, 8173, 8193, 8974] avg=8192.0000 max=8974 max_rank=7 min=7701 min_rank=0 imbalance_ratio=1.095459 max_load_reduction=56562 max_load_reduction_pct=0.863068 moved_to_echo=33548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7983, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7343, 489, 209, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7703, 8196, 8196, 8058, 8188, 8175, 8193, 8827] avg=8192.0000 max=8827 max_rank=7 min=7703 min_rank=0 imbalance_ratio=1.077515 max_load_reduction=56709 max_load_reduction_pct=0.865311 moved_to_echo=33383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7983, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7343, 489, 209, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7703, 8194, 8148, 8058, 8189, 8175, 8193, 8876] avg=8192.0000 max=8876 max_rank=7 min=7703 min_rank=0 imbalance_ratio=1.083496 max_load_reduction=56660 max_load_reduction_pct=0.864563 moved_to_echo=33433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7983, 0, 0, 87, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7343, 489, 209, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7703, 8172, 8084, 8057, 8202, 8169, 8196, 8953] avg=8192.0000 max=8953 max_rank=7 min=7703 min_rank=0 imbalance_ratio=1.092896 max_load_reduction=56583 max_load_reduction_pct=0.863388 moved_to_echo=33520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 1, 0, 8142, 508, 0, 0, 0, 0, 8191, 0, 0, 2, 0, 8180, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 194, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 73, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8173, 12, 0, 7569, 3826, 0, 0, 0, 0, 8171, 9, 0, 130, 0, 8047, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 10, 8031, 0, 0, 334, 0, 5, 0, 0, 0, 12, 3, 13, 11, 0, 904, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 8171, 0, 0, 9, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3723, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8164] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 32, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7681, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 8180, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 73, 0, 0, 1, 0, 0, 0, 7476, 461, 111, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8189, 8274, 8142, 8185, 8178, 8193, 8184] avg=8192.0000 max=8274 max_rank=2 min=8142 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7478, 464, 111, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7728, 8194, 8138, 8070, 8189, 8173, 8194, 8850] avg=8192.0000 max=8850 max_rank=7 min=7728 min_rank=0 imbalance_ratio=1.080322 max_load_reduction=56686 max_load_reduction_pct=0.864960 moved_to_echo=33406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 12, 0, 7140, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 0, 130, 0, 8047, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 10, 7922, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 12, 3, 13, 11, 0, 904, 0, 0, 0, 0, 0, 0, 7453, 429, 109, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 8154, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3723, 0, 0, 0, 0, 0, 11, 0, 0, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10982, 8202, 8273, 8956, 8182, 8319, 8178, 4444] avg=8192.0000 max=10982 max_rank=0 min=4444 min_rank=7 imbalance_ratio=1.340576 max_load_reduction=54554 max_load_reduction_pct=0.832428 moved_to_echo=29123 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7478, 464, 111, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7728, 8196, 8224, 8070, 8188, 8175, 8194, 8761] avg=8192.0000 max=8761 max_rank=7 min=7728 min_rank=0 imbalance_ratio=1.069458 max_load_reduction=56775 max_load_reduction_pct=0.866318 moved_to_echo=33318 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7478, 464, 111, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7728, 8197, 8286, 8070, 8193, 8175, 8194, 8693] avg=8192.0000 max=8693 max_rank=7 min=7728 min_rank=0 imbalance_ratio=1.061157 max_load_reduction=56843 max_load_reduction_pct=0.867355 moved_to_echo=33255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7478, 464, 111, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7728, 8194, 8121, 8070, 8188, 8175, 8194, 8866] avg=8192.0000 max=8866 max_rank=7 min=7728 min_rank=0 imbalance_ratio=1.082275 max_load_reduction=56670 max_load_reduction_pct=0.864716 moved_to_echo=33423 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7484, 469, 116, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7723, 8190, 8094, 8088, 8208, 8173, 8194, 8866] avg=8192.0000 max=8866 max_rank=7 min=7723 min_rank=0 imbalance_ratio=1.082275 max_load_reduction=56670 max_load_reduction_pct=0.864716 moved_to_echo=33441 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 32, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7478, 464, 111, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7728, 8174, 8126, 8070, 8203, 8168, 8195, 8872] avg=8192.0000 max=8872 max_rank=7 min=7728 min_rank=0 imbalance_ratio=1.083008 max_load_reduction=56664 max_load_reduction_pct=0.864624 moved_to_echo=33438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 58752, 0, 0, 0, 0, 0, 128, 0, 0, 0, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 1152, 0, 0, 128, 0, 0, 0, 58752, 3968, 1792, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 61056, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 63744, 0, 0, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 256, 0, 65408, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 59904, 0, 0, 0, 0, 0, 128, 0, 0, 0, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 1024, 0, 0, 128, 0, 0, 0, 59904, 3712, 896, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 61184, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 64512, 0, 0, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 256, 65408, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 256, 0, 65408, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 114, 3, 0, 0, 0, 0, 4, 7130, 0, 0, 0, 0, 1, 27, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 19, 0, 0, 1560, 0, 0, 9, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 14, 0, 7508, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 8168, 0, 0, 0, 9, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 16, 0, 8169, 15, 0, 8173, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8189, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 19, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 91, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8197, 8191, 7982, 8285, 8192, 8192, 8191] avg=8192.0000 max=8306 max_rank=0 min=7982 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32860 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 114, 3, 0, 0, 0, 0, 4, 7130, 0, 0, 0, 0, 1, 27, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1560, 0, 0, 9, 14, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 14, 0, 7508, 0, 0, 0, 0, 0, 90, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 9, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 16, 0, 8166, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7399, 8148, 8193, 9550, 7635, 8198, 8201, 8212] avg=8192.0000 max=9550 max_rank=3 min=7399 min_rank=0 imbalance_ratio=1.165771 max_load_reduction=55986 max_load_reduction_pct=0.854279 moved_to_echo=32246 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 1, 2, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7978, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 90, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8184, 8192, 8087, 8209, 8191, 8192, 8190] avg=8192.0000 max=8291 max_rank=0 min=8087 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32782 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 89, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 8211, 8192, 7984, 8265, 8192, 8190, 8189] avg=8192.0000 max=8313 max_rank=0 min=7984 min_rank=3 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32836 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 91, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8189, 8192, 7982, 8291, 8192, 8191, 8189] avg=8192.0000 max=8310 max_rank=0 min=7982 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32863 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 89, 11, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8314, 8200, 8192, 7983, 8283, 8187, 8190, 8187] avg=8192.0000 max=8314 max_rank=0 min=7983 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32847 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7983, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 89, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8194, 8192, 7985, 8286, 8192, 8190, 8189] avg=8192.0000 max=8308 max_rank=0 min=7985 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32857 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 89, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8213, 8192, 7983, 8282, 8192, 8190, 8189] avg=8192.0000 max=8295 max_rank=0 min=7983 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32853 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 96, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8189, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 104, 3, 0, 0, 0, 0, 4, 7125, 0, 0, 0, 0, 1, 26, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 19, 0, 0, 1550, 0, 0, 21, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 15, 0, 7516, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 8168, 0, 0, 0, 10, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 16, 0, 8169, 15, 0, 8173, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8192, 0, 19, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 89, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8196, 8192, 7983, 8284, 8192, 8191, 8191] avg=8192.0000 max=8307 max_rank=0 min=7983 min_rank=3 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32858 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 1, 2, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 96, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7980, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 88, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8183, 8191, 8080, 8208, 8192, 8191, 8190] avg=8192.0000 max=8301 max_rank=0 min=8080 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 88, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8314, 8211, 8192, 7984, 8264, 8192, 8190, 8189] avg=8192.0000 max=8314 max_rank=0 min=7984 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32835 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 104, 3, 0, 0, 0, 0, 4, 7125, 0, 0, 0, 0, 1, 26, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1550, 0, 0, 21, 14, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 15, 0, 7516, 0, 0, 0, 0, 0, 90, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 10, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 16, 0, 8166, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7383, 8148, 8193, 9553, 7644, 8199, 8204, 8212] avg=8192.0000 max=9553 max_rank=3 min=7383 min_rank=0 imbalance_ratio=1.166138 max_load_reduction=55983 max_load_reduction_pct=0.854233 moved_to_echo=32259 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 88, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8312, 8191, 8192, 7984, 8286, 8192, 8190, 8189] avg=8192.0000 max=8312 max_rank=0 min=7984 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32857 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 89, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8189, 8192, 7984, 8290, 8192, 8190, 8189] avg=8192.0000 max=8310 max_rank=0 min=7984 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32861 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 88, 7, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8314, 8205, 8192, 7984, 8279, 8185, 8190, 8187] avg=8192.0000 max=8314 max_rank=0 min=7984 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32841 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 88, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8213, 8192, 7984, 8281, 8192, 8190, 8189] avg=8192.0000 max=8295 max_rank=0 min=7984 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 64512, 0, 0, 0, 0, 128, 128, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 1792, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 63872, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 64768, 0, 0, 0, 0, 0, 768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 1664, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 63872, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 64768, 0, 0, 0, 0, 0, 768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 7916, 2, 0, 0, 8150, 0, 0, 4, 0, 0, 41, 0, 0, 0, 8169, 0, 0, 0, 0, 51, 0, 17, 78, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 8, 0, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8018, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 68, 0, 0, 12, 0, 0, 0, 0, 5081, 0, 22, 0, 1755, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 13, 0, 0, 14, 0, 0, 5, 0, 0, 8162, 15, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8037, 1, 1, 0, 150, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 0, 0, 0, 0, 0, 0, 0, 7662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 235, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8177, 8000, 8193, 8213, 8561, 7981, 8190] avg=8192.0000 max=8561 max_rank=5 min=7981 min_rank=6 imbalance_ratio=1.045044 max_load_reduction=56975 max_load_reduction_pct=0.869370 moved_to_echo=32945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 29, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8037, 1, 1, 0, 235, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8178, 8000, 8194, 8208, 8453, 8092, 8189] avg=8192.0000 max=8453 max_rank=5 min=8000 min_rank=2 imbalance_ratio=1.031860 max_load_reduction=57083 max_load_reduction_pct=0.871017 moved_to_echo=32942 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7916, 2, 0, 0, 30, 0, 0, 4, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 17, 78, 10, 0, 0, 0, 0, 0, 0, 0, 8120, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7974, 0, 8, 0, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8018, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 68, 0, 0, 12, 0, 0, 0, 0, 5081, 0, 22, 0, 234, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1755, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 14, 0, 0, 5, 0, 0, 8160, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7993, 8291, 9545, 8183, 8045, 5605, 9666, 8208] avg=8192.0000 max=9666 max_rank=6 min=5605 min_rank=5 imbalance_ratio=1.179932 max_load_reduction=55870 max_load_reduction_pct=0.852509 moved_to_echo=31524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 235, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8177, 8000, 8193, 8213, 8588, 7954, 8190] avg=8192.0000 max=8588 max_rank=5 min=7954 min_rank=6 imbalance_ratio=1.048340 max_load_reduction=56948 max_load_reduction_pct=0.868958 moved_to_echo=32945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 241, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8182, 7989, 8194, 8218, 8569, 7979, 8189] avg=8192.0000 max=8569 max_rank=5 min=7979 min_rank=6 imbalance_ratio=1.046021 max_load_reduction=56967 max_load_reduction_pct=0.869247 moved_to_echo=32955 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 235, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8177, 8000, 8193, 8213, 8599, 7943, 8190] avg=8192.0000 max=8599 max_rank=5 min=7943 min_rank=6 imbalance_ratio=1.049683 max_load_reduction=56937 max_load_reduction_pct=0.868790 moved_to_echo=32945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 235, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8177, 8000, 8193, 8213, 8563, 7979, 8190] avg=8192.0000 max=8563 max_rank=5 min=7979 min_rank=6 imbalance_ratio=1.045288 max_load_reduction=56973 max_load_reduction_pct=0.869339 moved_to_echo=32945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 0, 0, 0, 0, 0, 0, 0, 7662, 0, 0, 0, 235, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8177, 8002, 8193, 8213, 8598, 7942, 8190] avg=8192.0000 max=8598 max_rank=5 min=7942 min_rank=6 imbalance_ratio=1.049561 max_load_reduction=56938 max_load_reduction_pct=0.868805 moved_to_echo=32943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 7912, 2, 0, 0, 8149, 0, 0, 3, 0, 0, 42, 0, 0, 0, 8170, 0, 0, 0, 0, 51, 0, 23, 79, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 7, 0, 2372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8014, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 52, 0, 0, 12, 0, 0, 0, 0, 4464, 0, 21, 0, 1585, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8152, 0, 0, 1, 13, 0, 0, 14, 0, 0, 7, 0, 0, 8162, 15, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8052, 0, 1, 0, 140, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 0, 0, 0, 0, 0, 0, 7478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 280, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8175, 7898, 8193, 8213, 8688, 7957, 8190] avg=8192.0000 max=8688 max_rank=5 min=7898 min_rank=2 imbalance_ratio=1.060547 max_load_reduction=56848 max_load_reduction_pct=0.867432 moved_to_echo=33048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 30, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8052, 0, 1, 0, 280, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8177, 7898, 8193, 8208, 8551, 8097, 8189] avg=8192.0000 max=8551 max_rank=5 min=7898 min_rank=2 imbalance_ratio=1.043823 max_load_reduction=56985 max_load_reduction_pct=0.869522 moved_to_echo=33045 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7912, 2, 0, 0, 30, 0, 0, 3, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 23, 79, 10, 0, 0, 0, 0, 0, 0, 0, 8119, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7873, 0, 7, 0, 2372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8014, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 52, 0, 0, 12, 0, 0, 0, 0, 4464, 0, 21, 0, 279, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1585, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7917, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 14, 0, 0, 7, 0, 0, 8160, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7989, 8296, 10252, 8184, 8041, 5059, 9506, 8209] avg=8192.0000 max=10252 max_rank=2 min=5059 min_rank=5 imbalance_ratio=1.251465 max_load_reduction=55284 max_load_reduction_pct=0.843567 moved_to_echo=30815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 280, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8175, 7898, 8193, 8213, 8670, 7975, 8190] avg=8192.0000 max=8670 max_rank=5 min=7898 min_rank=2 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=33048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 280, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8175, 7898, 8193, 8213, 8617, 8028, 8190] avg=8192.0000 max=8617 max_rank=5 min=7898 min_rank=2 imbalance_ratio=1.051880 max_load_reduction=56919 max_load_reduction_pct=0.868515 moved_to_echo=33048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 282, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8188, 7890, 8194, 8222, 8637, 8003, 8188] avg=8192.0000 max=8637 max_rank=5 min=7890 min_rank=2 imbalance_ratio=1.054321 max_load_reduction=56899 max_load_reduction_pct=0.868210 moved_to_echo=33050 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7898, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 0, 0, 0, 0, 0, 0, 7478, 0, 0, 0, 280, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8175, 7904, 8193, 8213, 8683, 7956, 8190] avg=8192.0000 max=8683 max_rank=5 min=7904 min_rank=2 imbalance_ratio=1.059937 max_load_reduction=56853 max_load_reduction_pct=0.867508 moved_to_echo=33042 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 280, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8175, 7898, 8193, 8213, 8631, 8014, 8190] avg=8192.0000 max=8631 max_rank=5 min=7898 min_rank=2 imbalance_ratio=1.053589 max_load_reduction=56905 max_load_reduction_pct=0.868301 moved_to_echo=33048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 640, 0, 0, 128, 0, 0, 0, 0, 61696, 128, 128, 0, 1920, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 64000, 0, 128, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[2048, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 63616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 896, 0, 0, 128, 0, 0, 0, 0, 60800, 0, 128, 0, 2304, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 63232, 0, 128, 0, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65408, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2048, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 63616, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65280, 128, 0, 0, 256, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65408, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 72, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 7, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8191, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 271, 8148, 0, 0, 0, 0, 0, 0, 0, 14, 0, 8164, 33, 1, 0, 85, 0, 0, 6, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 18, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 0, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6958, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 12, 8168, 13, 0, 0, 19, 0, 2, 11, 0, 0, 0, 1, 0, 0, 0, 8175, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 28, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8183, 8202, 8195, 8076, 8339, 8187, 8200] avg=8192.0000 max=8339 max_rank=5 min=8076 min_rank=4 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32802 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 72, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 27, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8186, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8175, 8200, 8187, 8070, 8287, 8192, 8200] avg=8192.0000 max=8287 max_rank=5 min=8070 min_rank=4 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 271, 8111, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1, 0, 85, 0, 0, 6, 13, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 0, 0, 0, 7789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6958, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 27, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 12, 8164, 13, 0, 0, 19, 0, 2, 11, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8396, 8275, 8112, 8180, 9013, 7132, 8228, 8200] avg=8192.0000 max=9013 max_rank=4 min=7132 min_rank=5 imbalance_ratio=1.100220 max_load_reduction=56523 max_load_reduction_pct=0.862473 moved_to_echo=32573 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 27, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8165, 8200, 8195, 8076, 8358, 8187, 8200] avg=8192.0000 max=8358 max_rank=5 min=8076 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 27, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8166, 8205, 8196, 8073, 8363, 8181, 8208] avg=8192.0000 max=8363 max_rank=5 min=8073 min_rank=4 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32825 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 28, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8178, 8200, 8195, 8076, 8359, 8187, 8187] avg=8192.0000 max=8359 max_rank=5 min=8076 min_rank=4 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 116, 27, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8221, 8216, 8193, 8076, 8344, 8187, 8145] avg=8192.0000 max=8344 max_rank=5 min=8076 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 27, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8173, 8201, 8195, 8076, 8354, 8187, 8196] avg=8192.0000 max=8354 max_rank=5 min=8076 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 282, 8148, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8165, 33, 1, 0, 83, 0, 0, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 18, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 0, 0, 7891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6928, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 12, 8169, 14, 0, 0, 19, 0, 2, 11, 0, 0, 0, 1, 0, 0, 0, 8176, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 55, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 18, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 28, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8179, 8200, 8195, 8073, 8362, 8187, 8185] avg=8192.0000 max=8362 max_rank=5 min=8073 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 282, 8111, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1, 0, 83, 0, 0, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 0, 0, 7777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6928, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 28, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 12, 8165, 14, 0, 0, 19, 0, 2, 11, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8406, 8276, 8115, 8179, 9030, 7104, 8230, 8196] avg=8192.0000 max=9030 max_rank=4 min=7104 min_rank=5 imbalance_ratio=1.102295 max_load_reduction=56506 max_load_reduction_pct=0.862213 moved_to_echo=32560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 55, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 28, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8188, 8200, 8188, 8070, 8297, 8188, 8196] avg=8192.0000 max=8297 max_rank=5 min=8070 min_rank=4 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 31, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8167, 8202, 8196, 8071, 8362, 8183, 8210] avg=8192.0000 max=8362 max_rank=5 min=8071 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 28, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8167, 8200, 8195, 8073, 8362, 8187, 8196] avg=8192.0000 max=8362 max_rank=5 min=8073 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 28, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8171, 8201, 8195, 8073, 8359, 8187, 8195] avg=8192.0000 max=8359 max_rank=5 min=8073 min_rank=4 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 28, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8186, 8202, 8195, 8073, 8342, 8187, 8196] avg=8192.0000 max=8342 max_rank=5 min=8073 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 119, 28, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8202, 8216, 8193, 8073, 8348, 8187, 8162] avg=8192.0000 max=8348 max_rank=5 min=8073 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[64256, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 1024, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 1280, 0, 0, 0, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 384, 65280, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 65280, 0, 0, 0, 128, 0, 0, 0, 0, 0, 256, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[64256, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 1024, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 65536, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 384, 65280, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 65152, 0, 0, 0, 128, 0, 0, 0, 0, 0, 384, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 58, 0, 0, 9, 19, 0, 8004, 8166, 0, 0, 66, 0, 3387, 0, 0, 4, 0, 0, 0, 1, 0, 0, 31, 1, 0, 0, 8116, 0, 8044, 0, 0, 17, 0, 15, 0, 14, 0, 0, 8165, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 18, 5, 0, 0, 8, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 180, 0, 0, 18, 19, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4792, 0, 0, 11, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4, 0, 8185, 8191, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8188, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8052, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8183, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 8022, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4, 0, 8187, 8191, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8188, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8103, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 7, 19, 0, 8002, 8166, 0, 0, 65, 0, 2375, 0, 0, 4, 0, 0, 0, 0, 0, 0, 28, 1, 0, 0, 7796, 0, 8038, 0, 0, 17, 0, 21, 0, 13, 0, 0, 8165, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 19, 5, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 178, 0, 0, 18, 18, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 12, 0, 6133, 0, 0, 11, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8183, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4, 0, 8185, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 26, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8150, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8052, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8237, 7824, 8223, 8190, 8180, 8494, 8195] avg=8192.0000 max=8494 max_rank=6 min=7824 min_rank=2 imbalance_ratio=1.036865 max_load_reduction=57042 max_load_reduction_pct=0.870392 moved_to_echo=33059 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8162, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 9, 19, 0, 8004, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 3387, 0, 0, 4, 0, 0, 0, 1, 0, 0, 31, 1, 0, 0, 7692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 27, 0, 0, 17, 0, 15, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 5, 0, 0, 8, 0, 0, 2, 0, 8153, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 180, 0, 0, 18, 19, 0, 0, 0, 0, 8006, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4792, 0, 0, 11, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8099, 11182, 8189, 8197, 8255, 5262, 8176] avg=8192.0000 max=11182 max_rank=2 min=5262 min_rank=6 imbalance_ratio=1.364990 max_load_reduction=54354 max_load_reduction_pct=0.829376 moved_to_echo=29890 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8202, 7726, 8218, 8190, 8182, 8632, 8193] avg=8192.0000 max=8632 max_rank=6 min=7726 min_rank=2 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=33197 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8192, 7729, 8214, 8196, 8191, 8620, 8194] avg=8192.0000 max=8620 max_rank=6 min=7729 min_rank=2 imbalance_ratio=1.052246 max_load_reduction=56916 max_load_reduction_pct=0.868469 moved_to_echo=33201 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8202, 7724, 8218, 8190, 8182, 8632, 8194] avg=8192.0000 max=8632 max_rank=6 min=7724 min_rank=2 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=33198 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8202, 7725, 8217, 8190, 8182, 8632, 8195] avg=8192.0000 max=8632 max_rank=6 min=7725 min_rank=2 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=33199 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8202, 7733, 8209, 8190, 8182, 8632, 8195] avg=8192.0000 max=8632 max_rank=6 min=7733 min_rank=2 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=33199 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8183, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8022, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8200, 7893, 8048, 8193, 8182, 8632, 8194] avg=8192.0000 max=8632 max_rank=6 min=7893 min_rank=2 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=33201 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8184, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 8005, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8200, 7857, 8259, 8191, 8183, 8458, 8195] avg=8192.0000 max=8458 max_rank=6 min=7857 min_rank=2 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=33027 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4, 0, 8187, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 68, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8109, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8103, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8245, 7897, 8263, 8190, 8181, 8371, 8195] avg=8192.0000 max=8371 max_rank=6 min=7897 min_rank=2 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32937 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 7, 19, 0, 8002, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 2375, 0, 0, 4, 0, 0, 0, 0, 0, 0, 28, 1, 0, 0, 7825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7796, 0, 68, 0, 0, 17, 0, 21, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 5, 0, 0, 8, 0, 0, 1, 0, 8154, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 178, 0, 0, 18, 18, 0, 0, 0, 0, 7960, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0, 12, 0, 6133, 0, 0, 11, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8098, 10298, 7915, 8197, 8248, 6429, 8176] avg=8192.0000 max=10298 max_rank=2 min=6429 min_rank=6 imbalance_ratio=1.257080 max_load_reduction=55238 max_load_reduction_pct=0.842865 moved_to_echo=31050 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 7856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8200, 7865, 8251, 8191, 8183, 8458, 8195] avg=8192.0000 max=8458 max_rank=6 min=7865 min_rank=2 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=33027 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8200, 7858, 8260, 8191, 8183, 8458, 8193] avg=8192.0000 max=8458 max_rank=6 min=7858 min_rank=2 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=33025 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8200, 7856, 8260, 8191, 8183, 8458, 8194] avg=8192.0000 max=8458 max_rank=6 min=7856 min_rank=2 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=33026 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8192, 7863, 8255, 8194, 8192, 8446, 8194] avg=8192.0000 max=8446 max_rank=6 min=7863 min_rank=2 imbalance_ratio=1.031006 max_load_reduction=57090 max_load_reduction_pct=0.871124 moved_to_echo=33026 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8184, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 7856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8201, 8042, 8073, 8191, 8183, 8458, 8194] avg=8192.0000 max=8458 max_rank=6 min=8042 min_rank=2 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=33026 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 3584, 0, 0, 128, 0, 0, 0, 256, 0, 0, 128, 128, 0, 0, 61824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3584, 0, 0, 0, 0, 0, 0, 0, 128, 0, 62080, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2176, 0, 0, 0, 0, 0, 0, 0, 128, 0, 63488, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 2432, 0, 0, 128, 0, 0, 0, 256, 0, 0, 128, 128, 0, 0, 62848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65024, 0, 640, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 256, 0, 0, 128, 128, 0, 0, 0, 0, 65152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[65280, 0, 256, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 256, 0, 0, 128, 128, 0, 0, 0, 0, 64768, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 8191, 0, 4847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 1, 0, 8173, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 110, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 26, 0, 0, 0, 8174, 0, 0, 8169, 0, 0, 0, 0, 0, 14, 0, 10, 5, 14, 0, 0, 8134, 0, 8172, 0, 1692, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 7172, 0, 0, 0, 5306, 0, 0, 0, 7, 0, 6, 5, 0, 9, 0, 0, 0, 0, 7, 8, 0, 7650, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 511, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 40, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 6000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 7798, 1, 0, 0, 0, 0, 0, 0, 0, 0, 430, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 9, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 6434, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1819, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 39, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 7549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 5, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 8150, 77, 0, 0, 0, 0, 0, 0, 0, 0, 87, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 92, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 6497, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1673, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 4847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7551, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 1, 0, 8173, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 8164, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7587, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8219, 8751, 7588, 8243, 8195, 8186, 8163] avg=8192.0000 max=8751 max_rank=2 min=7588 min_rank=3 imbalance_ratio=1.068237 max_load_reduction=56785 max_load_reduction_pct=0.866470 moved_to_echo=32787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 9, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 6434, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1819, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7589, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8154, 8847, 7656, 8257, 8192, 8183, 8060] avg=8192.0000 max=8847 max_rank=2 min=7656 min_rank=3 imbalance_ratio=1.079956 max_load_reduction=56689 max_load_reduction_pct=0.865005 moved_to_echo=32692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 26, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8169, 0, 0, 0, 0, 0, 14, 0, 10, 5, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 0, 0, 0, 1692, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 6702, 0, 0, 0, 5306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 6, 5, 0, 9, 0, 0, 0, 0, 7, 8, 0, 7650, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 69, 0, 8147, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7535, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8239, 3871, 12031, 7749, 8190, 8249, 8997] avg=8192.0000 max=12031 max_rank=3 min=3871 min_rank=2 imbalance_ratio=1.468628 max_load_reduction=53505 max_load_reduction_pct=0.816422 moved_to_echo=33185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 40, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 6000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7589, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8254, 8757, 7645, 8253, 8192, 8184, 8061] avg=8192.0000 max=8757 max_rank=2 min=7645 min_rank=3 imbalance_ratio=1.068970 max_load_reduction=56779 max_load_reduction_pct=0.866379 moved_to_echo=32690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 39, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 7549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 71, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7593, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8173, 8813, 7639, 8263, 8191, 8192, 8075] avg=8192.0000 max=8813 max_rank=2 min=7639 min_rank=3 imbalance_ratio=1.075806 max_load_reduction=56723 max_load_reduction_pct=0.865524 moved_to_echo=32721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 7, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 7798, 1, 0, 0, 0, 0, 0, 0, 0, 0, 430, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7589, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8178, 8819, 7656, 8257, 8192, 8184, 8060] avg=8192.0000 max=8819 max_rank=2 min=7656 min_rank=3 imbalance_ratio=1.076538 max_load_reduction=56717 max_load_reduction_pct=0.865433 moved_to_echo=32693 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 92, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 6497, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1673, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7589, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8226, 8763, 7665, 8257, 8192, 8183, 8060] avg=8192.0000 max=8763 max_rank=2 min=7665 min_rank=3 imbalance_ratio=1.069702 max_load_reduction=56773 max_load_reduction_pct=0.866287 moved_to_echo=32692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 5, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 8150, 77, 0, 0, 0, 0, 0, 0, 0, 0, 87, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7589, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8093, 8915, 7656, 8257, 8192, 8175, 8060] avg=8192.0000 max=8915 max_rank=2 min=7656 min_rank=3 imbalance_ratio=1.088257 max_load_reduction=56621 max_load_reduction_pct=0.863968 moved_to_echo=32684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 21, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 3301, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4948, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 56, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 6036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 54, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 6985, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1282, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 26, 0, 0, 0, 8174, 0, 0, 8169, 0, 0, 0, 0, 0, 14, 0, 10, 5, 14, 0, 0, 8136, 0, 8173, 0, 1110, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3121, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 7144, 0, 0, 0, 4522, 0, 0, 0, 8, 0, 6, 5, 0, 9, 0, 0, 0, 0, 7, 8, 0, 7658, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 440, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 8191, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 8184, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 78, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 171, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 5157, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 13, 8002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8189, 0, 7976, 79, 0, 0, 0, 0, 0, 0, 0, 0, 309, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 7, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 6985, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1282, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7711, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8146, 8728, 7760, 8255, 8192, 8185, 8080] avg=8192.0000 max=8728 max_rank=2 min=7760 min_rank=3 imbalance_ratio=1.065430 max_load_reduction=56808 max_load_reduction_pct=0.866821 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 54, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7711, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8265, 8624, 7749, 8251, 8192, 8185, 8080] avg=8192.0000 max=8624 max_rank=2 min=7749 min_rank=3 imbalance_ratio=1.052734 max_load_reduction=56912 max_load_reduction_pct=0.868408 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 26, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 8169, 0, 0, 0, 0, 0, 14, 0, 10, 5, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 1110, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 6767, 0, 0, 0, 4522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 5, 0, 9, 0, 0, 0, 0, 7, 8, 0, 7658, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 59, 0, 8137, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7658, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8249, 4690, 11312, 7756, 8190, 8241, 8889] avg=8192.0000 max=11312 max_rank=3 min=4690 min_rank=2 imbalance_ratio=1.380859 max_load_reduction=54224 max_load_reduction_pct=0.827393 moved_to_echo=33076 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7685, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 8184, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 8154, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 7709, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8228, 8638, 7690, 8251, 8194, 8190, 8154] avg=8192.0000 max=8638 max_rank=2 min=7690 min_rank=3 imbalance_ratio=1.054443 max_load_reduction=56898 max_load_reduction_pct=0.868195 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 56, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 0, 6036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7714, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8155, 8711, 7744, 8258, 8192, 8192, 8093] avg=8192.0000 max=8711 max_rank=2 min=7744 min_rank=3 imbalance_ratio=1.063354 max_load_reduction=56825 max_load_reduction_pct=0.867081 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 21, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 3301, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4948, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7711, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8168, 8710, 7760, 8255, 8192, 8183, 8080] avg=8192.0000 max=8710 max_rank=2 min=7760 min_rank=3 imbalance_ratio=1.063232 max_load_reduction=56826 max_load_reduction_pct=0.867096 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 171, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 5157, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7760, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7711, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8273, 8602, 7761, 8255, 8192, 8183, 8080] avg=8192.0000 max=8602 max_rank=2 min=7761 min_rank=3 imbalance_ratio=1.050049 max_load_reduction=56934 max_load_reduction_pct=0.868744 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 13, 8002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 7976, 79, 0, 0, 0, 0, 0, 0, 0, 0, 309, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7711, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8052, 8833, 7760, 8255, 8192, 8177, 8080] avg=8192.0000 max=8833 max_rank=2 min=7760 min_rank=3 imbalance_ratio=1.078247 max_load_reduction=56703 max_load_reduction_pct=0.865219 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 640, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 60672, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4736, 0, 0, 0, 49024, 128, 128, 0, 0, 0, 0, 0, 0, 0, 11776, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 60288, 0, 0, 0, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 256, 65152, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3712, 0, 0, 0, 36096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 25856, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 640, 512, 128, 0, 0, 0, 0, 128, 0, 0, 0, 61696, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 61056, 0, 0, 0, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 0, 384, 64896, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65280, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65536, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 128, 0, 128, 128, 0, 65024, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 128, 0, 128, 128, 0, 65024, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True - [2026-08-26 16:18:07] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 446.9 | throughput per GPU (TFLOP/s/GPU): 404.7 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.031589E+01 | loss scale: 1.0 | grad norm: 6.300 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 738, 0, 0, 0, 138, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 840, 0, 0, 2618, 3, 0, 0, 8192, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 5041, 0, 0, 0, 6906, 0, 7872, 1, 0, 0, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 7212, 0, 2, 0, 0, 1, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7802, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 438, 0, 0, 0, 61, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 717, 0, 2, 2101, 1, 0, 0, 8192, 0, 8190, 1, 2, 0, 0, 0, 0, 1, 0, 0, 2, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 1, 0, 0, 1, 0, 0, 0, 5411, 0, 1, 0, 6685, 0, 7768, 0, 4, 0, 1887, 0, 0, 1, 1, 3, 3, 0, 0, 7852, 0, 0, 0, 0, 6, 0, 27, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7959, 0, 0, 4, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 2, 1, 1, 9, 616, 1, 0, 0, 348, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1101, 0, 0, 2337, 1, 0, 0, 8191, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4, 0, 5, 4, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 6848, 0, 7742, 31, 1, 0, 2168, 0, 0, 0, 0, 0, 0, 0, 0, 7848, 0, 0, 1, 2, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7931, 0, 0, 3, 0, 0, 0, 1, 0, 2, 1, 1, 3, 2, 0, 0, 0, 24, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1021, 1, 0, 0, 309, 0, 2, 0, 2, 0, 13, 0, 1, 1, 0, 1087, 0, 0, 2767, 0, 0, 0, 8189, 0, 8188, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 22, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 1, 0, 0, 0, 0, 4351, 0, 0, 0, 6462, 0, 7498, 7, 0, 0, 2067, 0, 0, 2, 0, 0, 0, 0, 2, 7138, 0, 1, 0, 2, 9, 3, 38, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4, 4, 8014, 0, 0, 4, 1, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 144, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 20, 475, 0, 0, 0, 392, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 912, 0, 1, 2945, 0, 0, 0, 8192, 0, 8192, 3, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 1, 0, 0, 0, 0, 4372, 0, 0, 0, 6790, 0, 7437, 2, 0, 0, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 7594, 0, 1, 0, 0, 4, 0, 16, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 7856, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 460, 0, 0, 0, 110, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 966, 0, 0, 3045, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 4781, 0, 0, 0, 6694, 0, 7938, 4, 0, 0, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 7488, 0, 1, 0, 0, 7, 0, 63, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 7955, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 550, 0, 0, 0, 186, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 935, 0, 0, 3168, 0, 0, 0, 8192, 1, 8192, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 8, 0, 0, 0, 4836, 0, 0, 0, 6484, 0, 7924, 3, 0, 0, 1136, 0, 0, 0, 0, 0, 14, 0, 0, 7723, 0, 2, 0, 0, 1, 0, 16, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 5, 346, 0, 0, 0, 177, 0, 0, 0, 0, 0, 7, 0, 1, 0, 0, 700, 0, 1, 3015, 4, 0, 0, 8191, 1, 8191, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 4859, 0, 0, 0, 6843, 0, 7831, 8, 1, 0, 1577, 0, 0, 0, 0, 0, 0, 0, 0, 7503, 0, 0, 0, 0, 2, 0, 19, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 738, 0, 0, 0, 138, 1, 0, 0, 0, 0, 0, 4723, 2790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 840, 0, 0, 2618, 3, 0, 0, 0, 0, 4533, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5041, 0, 0, 0, 1796, 0, 0, 1, 0, 0, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7212, 0, 2, 0, 0, 1, 0, 9, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7802, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7872, 387, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8396, 7995, 8182, 8202, 8596, 7836, 8048, 8281] avg=8192.0000 max=8596 max_rank=4 min=7836 min_rank=5 imbalance_ratio=1.049316 max_load_reduction=56940 max_load_reduction_pct=0.868835 moved_to_echo=32761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1021, 1, 0, 0, 309, 0, 2, 0, 2, 0, 13, 4420, 2788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1087, 0, 0, 2767, 0, 0, 0, 0, 0, 4532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 22, 3, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4351, 0, 0, 0, 1680, 0, 0, 7, 0, 0, 2067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 7138, 0, 1, 0, 2, 9, 3, 38, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4, 4, 8014, 0, 0, 4, 1, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 144, 0, 1, 0, 0, 7498, 362, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8557, 8388, 8209, 8175, 8105, 7807, 8276, 8019] avg=8192.0000 max=8557 max_rank=0 min=7807 min_rank=5 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=32207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 20, 475, 0, 0, 0, 392, 0, 0, 0, 0, 4, 2, 4644, 2790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 0, 1, 2945, 0, 0, 0, 0, 0, 4533, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4372, 0, 0, 0, 1766, 0, 0, 2, 0, 0, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7594, 0, 1, 0, 0, 4, 0, 16, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 7856, 0, 0, 2, 2, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 7437, 380, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8395, 8188, 8203, 8079, 8227, 8108, 8008] avg=8192.0000 max=8395 max_rank=1 min=8008 min_rank=7 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32422 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 438, 0, 0, 0, 61, 0, 0, 0, 0, 0, 1, 4572, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 717, 0, 2, 2101, 1, 0, 0, 0, 0, 4533, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 3, 0, 0, 4, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5411, 0, 1, 0, 1739, 0, 0, 0, 4, 0, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 3, 0, 0, 7852, 0, 0, 0, 0, 6, 0, 27, 1, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7959, 0, 0, 4, 1, 2, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 7768, 374, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7862, 7358, 8192, 8197, 9043, 8506, 8211, 8167] avg=8192.0000 max=9043 max_rank=4 min=7358 min_rank=1 imbalance_ratio=1.103882 max_load_reduction=56493 max_load_reduction_pct=0.862015 moved_to_echo=33927 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1, 1, 9, 616, 1, 0, 0, 348, 0, 0, 0, 0, 0, 3, 4689, 2793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 0, 0, 2337, 1, 0, 0, 0, 0, 4521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4, 0, 5, 4, 0, 2, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 1771, 0, 0, 31, 1, 0, 2168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7848, 0, 0, 1, 2, 0, 0, 17, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7931, 0, 0, 3, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 1, 3, 2, 0, 0, 0, 24, 0, 0, 0, 0, 7742, 388, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8463, 7960, 8199, 8181, 7892, 8485, 8183, 8173] avg=8192.0000 max=8485 max_rank=5 min=7892 min_rank=4 imbalance_ratio=1.035767 max_load_reduction=57051 max_load_reduction_pct=0.870529 moved_to_echo=32733 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 550, 0, 0, 0, 186, 0, 0, 1, 0, 0, 0, 4435, 2790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 0, 0, 3168, 0, 0, 0, 0, 1, 4533, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 5, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4836, 0, 0, 0, 1686, 0, 0, 3, 0, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 7723, 0, 2, 0, 0, 1, 0, 16, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 0, 0, 2, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7924, 363, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7963, 8640, 8191, 8192, 7669, 8368, 8208, 8305] avg=8192.0000 max=8640 max_rank=1 min=7669 min_rank=4 imbalance_ratio=1.054688 max_load_reduction=56896 max_load_reduction_pct=0.868164 moved_to_echo=32550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 460, 0, 0, 0, 110, 0, 0, 0, 0, 0, 21, 4578, 2790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 3045, 0, 0, 0, 0, 0, 4533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4781, 0, 0, 0, 1741, 0, 0, 4, 0, 0, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7488, 0, 1, 0, 0, 7, 0, 63, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 7955, 0, 0, 2, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 7938, 375, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7960, 8544, 8189, 8190, 7927, 8171, 8207, 8348] avg=8192.0000 max=8544 max_rank=1 min=7927 min_rank=4 imbalance_ratio=1.042969 max_load_reduction=56992 max_load_reduction_pct=0.869629 moved_to_echo=32653 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 5, 346, 0, 0, 0, 177, 0, 0, 0, 0, 0, 7, 4680, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 700, 0, 1, 3015, 4, 0, 0, 0, 1, 4533, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4859, 0, 0, 0, 1780, 0, 0, 8, 1, 0, 1577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7503, 0, 0, 0, 0, 2, 0, 19, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 1, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 7831, 383, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8007, 8256, 8186, 8196, 8225, 8136, 8295, 8235] avg=8192.0000 max=8295 max_rank=6 min=8007 min_rank=0 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32891 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True + [2026-08-26 16:27:14] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 441.2 | throughput per GPU (TFLOP/s/GPU): 409.8 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.031645E+01 | loss scale: 1.0 | grad norm: 6.373 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1062, 0, 0, 0, 222, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1180, 0, 0, 2905, 4, 0, 0, 8192, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 4904, 0, 0, 0, 7138, 0, 7947, 3, 0, 0, 2375, 0, 0, 0, 0, 0, 0, 0, 0, 5364, 0, 4, 0, 0, 1, 0, 13, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7766, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 755, 0, 0, 0, 290, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1185, 0, 0, 3470, 0, 0, 0, 8192, 1, 8192, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 9, 0, 0, 0, 4620, 0, 0, 0, 6704, 0, 8001, 9, 0, 0, 1593, 0, 0, 0, 0, 0, 13, 0, 0, 6298, 0, 2, 0, 0, 1, 0, 21, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 634, 0, 0, 0, 117, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 854, 0, 2, 2395, 1, 0, 0, 8192, 0, 8190, 1, 0, 0, 0, 0, 0, 1, 0, 0, 2, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 1, 0, 0, 1, 0, 0, 0, 5173, 0, 1, 0, 6906, 0, 7883, 0, 4, 0, 2524, 0, 0, 1, 1, 3, 4, 0, 0, 6336, 0, 1, 0, 0, 6, 0, 28, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7936, 0, 0, 4, 1, 2, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 131, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 19, 648, 0, 0, 0, 559, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 1128, 0, 1, 3059, 0, 0, 0, 8192, 0, 8192, 3, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 4185, 0, 0, 0, 6993, 0, 7551, 3, 0, 0, 2430, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 9, 0, 0, 4, 0, 22, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7823, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 632, 0, 0, 0, 144, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 1236, 0, 0, 3249, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 4627, 0, 0, 0, 6962, 0, 7969, 6, 0, 0, 1863, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 1, 0, 0, 7, 0, 71, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 7938, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 2, 1, 2, 9, 814, 1, 0, 0, 532, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1231, 0, 0, 2427, 1, 0, 0, 8191, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4, 0, 2, 4, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 0, 0, 6995, 0, 7801, 40, 2, 0, 2678, 0, 0, 0, 0, 0, 0, 0, 0, 6572, 0, 1, 1, 3, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7890, 0, 0, 4, 0, 0, 0, 1, 0, 2, 1, 1, 3, 2, 0, 0, 0, 246, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1353, 1, 0, 0, 466, 0, 2, 0, 2, 0, 13, 0, 1, 1, 0, 1298, 0, 0, 2946, 0, 0, 0, 8189, 0, 8188, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 22, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 1, 0, 0, 0, 0, 4179, 0, 0, 0, 6686, 0, 7606, 11, 0, 0, 2705, 0, 0, 2, 0, 0, 0, 0, 1, 5326, 0, 4, 0, 2, 7, 3, 44, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4, 3, 8020, 0, 0, 6, 1, 0, 0, 3, 0, 0, 1, 1, 0, 0, 0, 0, 0, 260, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 5, 508, 0, 0, 0, 277, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 961, 0, 1, 3278, 5, 0, 0, 8191, 1, 8191, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 4719, 0, 0, 0, 7096, 0, 7910, 9, 1, 0, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 5961, 0, 0, 0, 0, 2, 0, 33, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1062, 0, 0, 0, 222, 1, 0, 0, 0, 0, 0, 5510, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1180, 0, 0, 2905, 4, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4904, 0, 0, 0, 1427, 0, 0, 3, 0, 0, 2375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5364, 0, 4, 0, 0, 1, 0, 13, 0, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7766, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 7947, 201, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8497, 8181, 8182, 8201, 8709, 7509, 8034, 8223] avg=8192.0000 max=8709 max_rank=4 min=7509 min_rank=5 imbalance_ratio=1.063110 max_load_reduction=56827 max_load_reduction_pct=0.867111 moved_to_echo=32475 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 755, 0, 0, 0, 290, 0, 0, 1, 0, 0, 0, 5175, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 0, 0, 3470, 0, 0, 0, 0, 1, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4620, 0, 0, 0, 1340, 0, 0, 9, 0, 0, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 6298, 0, 2, 0, 0, 1, 0, 21, 0, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 1, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 8001, 189, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7918, 8752, 8189, 8193, 7571, 8462, 8196, 8255] avg=8192.0000 max=8752 max_rank=1 min=7571 min_rank=4 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=32484 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 19, 648, 0, 0, 0, 559, 0, 0, 0, 0, 3, 2, 5398, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 1, 3059, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8189, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4185, 0, 0, 0, 1398, 0, 0, 3, 0, 0, 2430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 9, 0, 0, 4, 0, 22, 0, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7823, 0, 0, 2, 2, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 7551, 197, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8282, 8190, 8203, 8016, 8351, 8096, 8072] avg=8192.0000 max=8351 max_rank=5 min=8016 min_rank=4 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 634, 0, 0, 0, 117, 0, 0, 0, 0, 0, 1, 5331, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 854, 0, 2, 2395, 1, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 3, 0, 0, 3, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5173, 0, 1, 0, 1380, 0, 0, 0, 4, 0, 2524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 4, 0, 0, 6336, 0, 1, 0, 0, 6, 0, 28, 1, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7936, 0, 0, 4, 1, 2, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 131, 0, 0, 0, 0, 7883, 195, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7781, 7343, 8191, 8196, 9083, 8508, 8211, 8223] avg=8192.0000 max=9083 max_rank=4 min=7343 min_rank=1 imbalance_ratio=1.108765 max_load_reduction=56453 max_load_reduction_pct=0.861404 moved_to_echo=34025 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 632, 0, 0, 0, 144, 0, 0, 0, 0, 0, 26, 5374, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 0, 0, 3249, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4627, 0, 0, 0, 1392, 0, 0, 6, 0, 0, 1863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 1, 0, 0, 7, 0, 71, 0, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 7938, 0, 0, 2, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1, 0, 0, 0, 7969, 196, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7873, 8576, 8191, 8190, 7888, 8337, 8212, 8269] avg=8192.0000 max=8576 max_rank=1 min=7873 min_rank=0 imbalance_ratio=1.046875 max_load_reduction=56960 max_load_reduction_pct=0.869141 moved_to_echo=32706 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1, 2, 9, 814, 1, 0, 0, 532, 0, 0, 0, 0, 0, 3, 5406, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 0, 0, 2427, 1, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4, 0, 2, 4, 0, 2, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 0, 0, 1387, 0, 0, 40, 2, 0, 2678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6572, 0, 1, 1, 3, 0, 0, 22, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7890, 0, 0, 4, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 1, 3, 2, 0, 0, 0, 246, 0, 0, 0, 0, 7801, 202, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8472, 7731, 8196, 8185, 7788, 8731, 8165, 8268] avg=8192.0000 max=8731 max_rank=5 min=7731 min_rank=1 imbalance_ratio=1.065796 max_load_reduction=56805 max_load_reduction_pct=0.866776 moved_to_echo=32952 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1353, 1, 0, 0, 466, 0, 2, 0, 2, 0, 13, 5161, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1298, 0, 0, 2946, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 22, 2, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4179, 0, 0, 0, 1337, 0, 0, 11, 0, 0, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 5326, 0, 4, 0, 2, 7, 3, 44, 0, 2126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4, 3, 8020, 0, 0, 6, 1, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 1, 0, 0, 0, 0, 0, 260, 0, 1, 0, 0, 7606, 188, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8695, 8334, 8211, 8173, 8232, 7515, 8306, 8070] avg=8192.0000 max=8695 max_rank=0 min=7515 min_rank=5 imbalance_ratio=1.061401 max_load_reduction=56841 max_load_reduction_pct=0.867325 moved_to_echo=32123 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 5, 508, 0, 0, 0, 277, 0, 0, 0, 0, 0, 8, 5478, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 961, 0, 1, 3278, 5, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4719, 0, 0, 0, 1418, 0, 0, 9, 1, 0, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5961, 0, 0, 0, 0, 2, 0, 33, 0, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 1, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 7910, 200, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7974, 8337, 8186, 8195, 8249, 8123, 8316, 8156] avg=8192.0000 max=8337 max_rank=1 min=7974 min_rank=0 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32844 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 4736, 128, 0, 0, 1792, 128, 128, 128, 128, 128, 128, 36864, 22400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 7296, 0, 128, 22016, 128, 0, 0, 0, 128, 36352, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 37632, 0, 128, 0, 14080, 0, 0, 128, 128, 0, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 6528, 128, 0, 0, 2688, 128, 128, 128, 128, 128, 128, 42880, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 9088, 0, 128, 23808, 128, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 36096, 0, 128, 0, 11136, 0, 0, 128, 128, 0, 18304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 48256, 0, 128, 128, 128, 128, 128, 256, 128, 17024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 60416, 0, 128, 128, 128, 128, 128, 256, 128, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 512, 128, 128, 0, 0, 62080, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 128, 128, 63616, 0, 0, 128, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 1280, 128, 128, 0, 0, 62720, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 128, 128, 63360, 0, 0, 128, 128, 128, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 128, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 8189, 0, 8186, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 8188, 0, 0, 0, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6972, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 890, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7060, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 8191, 1, 8182, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7674, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8186, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 10, 0, 8186, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 7500, 0, 0, 0, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 1, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6972, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8179, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8200, 8146, 8777, 8181, 8192, 7643, 8193] avg=8192.0000 max=8777 max_rank=3 min=7643 min_rank=6 imbalance_ratio=1.071411 max_load_reduction=56759 max_load_reduction_pct=0.866074 moved_to_echo=32209 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 7504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7842, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8175, 8200, 7880, 8196, 8194, 8512, 8190] avg=8192.0000 max=8512 max_rank=6 min=7880 min_rank=3 imbalance_ratio=1.039062 max_load_reduction=57024 max_load_reduction_pct=0.870117 moved_to_echo=33092 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 7504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7060, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8201, 8200, 8637, 8182, 8192, 7730, 8192] avg=8192.0000 max=8637 max_rank=3 min=7730 min_rank=6 imbalance_ratio=1.054321 max_load_reduction=56899 max_load_reduction_pct=0.868210 moved_to_echo=32296 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 890, 7504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7327, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8177, 8199, 8394, 8198, 8191, 7997, 8191] avg=8192.0000 max=8394 max_rank=3 min=7997 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=57142 max_load_reduction_pct=0.871918 moved_to_echo=32577 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 7504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8197, 8195, 8069, 8181, 8192, 8324, 8190] avg=8192.0000 max=8324 max_rank=6 min=8069 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32887 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8183, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 1, 8182, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 7487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7674, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8187, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8189, 8204, 8014, 8204, 8191, 8349, 8200] avg=8192.0000 max=8349 max_rank=6 min=8014 min_rank=3 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 7504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7842, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8196, 8200, 7859, 8196, 8192, 8512, 8190] avg=8192.0000 max=8512 max_rank=6 min=7859 min_rank=3 imbalance_ratio=1.039062 max_load_reduction=57024 max_load_reduction_pct=0.870117 moved_to_echo=33090 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 7504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7799, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8201, 8192, 7906, 8198, 8192, 8469, 8190] avg=8192.0000 max=8469 max_rank=6 min=7906 min_rank=3 imbalance_ratio=1.033813 max_load_reduction=57067 max_load_reduction_pct=0.870773 moved_to_echo=33049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 8189, 0, 8174, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7981, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 8188, 0, 0, 0, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 8191, 1, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8206, 8220, 8149, 8180, 8192, 8192, 8192] avg=8192.0000 max=8220 max_rank=2 min=8149 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8149, 8219, 8208, 8198, 8192, 8191, 8190] avg=8192.0000 max=8219 max_rank=2 min=8149 min_rank=1 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8182, 8220, 8175, 8197, 8193, 8191, 8190] avg=8192.0000 max=8220 max_rank=2 min=8175 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8186, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 15, 0, 8174, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 8145, 0, 0, 0, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 1, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8174, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8193, 8010, 8373, 8181, 8192, 8190, 8193] avg=8192.0000 max=8373 max_rank=3 min=8010 min_rank=2 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8181, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 10, 1, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8181, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8198, 8222, 8140, 8205, 8191, 8196, 8201] avg=8192.0000 max=8222 max_rank=2 min=8140 min_rank=3 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8203, 8209, 8164, 8198, 8192, 8192, 8190] avg=8192.0000 max=8209 max_rank=2 min=8164 min_rank=3 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8204, 8216, 8173, 8181, 8192, 8192, 8190] avg=8192.0000 max=8216 max_rank=2 min=8173 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8201, 8220, 8154, 8196, 8192, 8192, 8190] avg=8192.0000 max=8220 max_rank=2 min=8154 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 60288, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5632, 60032, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65408, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[65536, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 65280, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 128, 128, 65536, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 67, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 3, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6815, 0, 1, 1348, 0, 8133, 0, 8188, 0, 3, 0, 0, 1, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 75, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7242, 0, 0, 950, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 29, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7450, 0, 0, 712, 0, 8185, 0, 8191, 0, 1, 0, 0, 1, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7379, 0, 0, 813, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6991, 0, 0, 1201, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 92, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7933, 0, 0, 261, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6815, 0, 1, 669, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 0, 7464, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8189, 8240, 8193, 8195, 8203, 7506, 8816] avg=8192.0000 max=8816 max_rank=7 min=7506 min_rank=6 imbalance_ratio=1.076172 max_load_reduction=56720 max_load_reduction_pct=0.865479 moved_to_echo=32720 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 674, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8182, 8192, 8192, 8191, 8804, 7593] avg=8192.0000 max=8804 max_rank=6 min=7593 min_rank=7 imbalance_ratio=1.074707 max_load_reduction=56732 max_load_reduction_pct=0.865662 moved_to_echo=32780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7242, 0, 0, 674, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 950, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8194, 8182, 8192, 8192, 8189, 7929, 8468] avg=8192.0000 max=8468 max_rank=7 min=7929 min_rank=6 imbalance_ratio=1.033691 max_load_reduction=57068 max_load_reduction_pct=0.870789 moved_to_echo=32778 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 674, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8182, 8192, 8191, 8189, 8787, 7610] avg=8192.0000 max=8787 max_rank=6 min=7610 min_rank=7 imbalance_ratio=1.072632 max_load_reduction=56749 max_load_reduction_pct=0.865921 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7450, 0, 0, 677, 14, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 0, 7508, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8191, 8204, 8191, 8193, 8191, 8147, 8222] avg=8192.0000 max=8222 max_rank=7 min=8147 min_rank=6 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32753 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6991, 0, 0, 674, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8182, 8192, 8192, 8191, 7678, 8719] avg=8192.0000 max=8719 max_rank=7 min=7678 min_rank=6 imbalance_ratio=1.064331 max_load_reduction=56817 max_load_reduction_pct=0.866959 moved_to_echo=32780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7379, 0, 0, 674, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8182, 8192, 8191, 8191, 8066, 8331] avg=8192.0000 max=8331 max_rank=7 min=8066 min_rank=6 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7933, 0, 0, 674, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 7516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8182, 8192, 8190, 8191, 8619, 7777] avg=8192.0000 max=8619 max_rank=6 min=7777 min_rank=7 imbalance_ratio=1.052124 max_load_reduction=56917 max_load_reduction_pct=0.868484 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 73, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 3, 8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7225, 0, 1, 938, 0, 8125, 0, 8188, 0, 3, 0, 0, 1, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7851, 0, 0, 340, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7418, 0, 0, 774, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7853, 0, 0, 339, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 45, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 35, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7928, 0, 0, 227, 0, 8183, 0, 8191, 0, 1, 0, 0, 1, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 141, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7225, 0, 1, 339, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 938, 0, 7786, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 8244, 8194, 8195, 8204, 7589, 8728] avg=8192.0000 max=8728 max_rank=7 min=7589 min_rank=6 imbalance_ratio=1.065430 max_load_reduction=56808 max_load_reduction_pct=0.866821 moved_to_echo=32716 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7851, 0, 0, 342, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 7850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8195, 8181, 8192, 8192, 8189, 8207, 8190] avg=8192.0000 max=8207 max_rank=6 min=8181 min_rank=2 imbalance_ratio=1.001831 max_load_reduction=57329 max_load_reduction_pct=0.874771 moved_to_echo=32778 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 342, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8180, 8192, 8191, 8191, 8548, 7851] avg=8192.0000 max=8548 max_rank=6 min=7851 min_rank=7 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32781 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7418, 0, 0, 342, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 7850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8181, 8192, 8192, 8191, 7774, 8624] avg=8192.0000 max=8624 max_rank=7 min=7774 min_rank=6 imbalance_ratio=1.052734 max_load_reduction=56912 max_load_reduction_pct=0.868408 moved_to_echo=32781 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7853, 0, 0, 342, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 7850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8181, 8192, 8191, 8191, 8209, 8189] avg=8192.0000 max=8209 max_rank=6 min=8181 min_rank=2 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 342, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 7850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8194, 8181, 8192, 8191, 8190, 8502, 7895] avg=8192.0000 max=8502 max_rank=6 min=7895 min_rank=7 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=32778 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7928, 0, 0, 343, 16, 3, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 7840, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8189, 8207, 8190, 8193, 8190, 8299, 8069] avg=8192.0000 max=8299 max_rank=6 min=8069 min_rank=7 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32751 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 341, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 7849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8193, 8181, 8192, 8191, 8190, 8408, 7990] avg=8192.0000 max=8408 max_rank=6 min=7990 min_rank=7 imbalance_ratio=1.026367 max_load_reduction=57128 max_load_reduction_pct=0.871704 moved_to_echo=32779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 60032, 0, 128, 5504, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[5504, 0, 60160, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 62720, 0, 128, 2816, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2816, 0, 62848, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 19, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7390, 0, 8017, 0, 4, 0, 0, 0, 0, 5240, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 0, 0, 2, 1995, 8187, 8163, 0, 0, 0, 3, 0, 8186, 8166, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 1873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6317, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8186, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7879, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7105, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6687, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7624, 4, 7917, 0, 1, 0, 0, 0, 0, 4490, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 0, 3214, 8191, 8190, 0, 0, 0, 1, 0, 8190, 8186, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 2861, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5267, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5720, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5505, 2124, 123, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 1, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5266, 0, 0, 0, 4, 0, 0, 0, 0, 5240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4, 0, 8163, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1995, 0, 0, 0, 0, 0, 3, 0, 0, 2661, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8803, 8194, 8205, 10510, 8177, 8791, 4665] avg=8192.0000 max=10510 max_rank=4 min=4665 min_rank=7 imbalance_ratio=1.282959 max_load_reduction=55026 max_load_reduction_pct=0.839630 moved_to_echo=32143 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5523, 2354, 125, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5838, 0, 0, 0, 0, 0, 0, 0, 0, 1873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6317, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8006, 8192, 8190, 7711, 8194, 8065, 8986] avg=8192.0000 max=8986 max_rank=7 min=7711 min_rank=4 imbalance_ratio=1.096924 max_load_reduction=56550 max_load_reduction_pct=0.862885 moved_to_echo=32956 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 5523, 2354, 127, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5838, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7879, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8034, 8192, 8194, 6120, 8194, 8062, 10548] avg=8192.0000 max=10548 max_rank=7 min=6120 min_rank=4 imbalance_ratio=1.287598 max_load_reduction=54988 max_load_reduction_pct=0.839050 moved_to_echo=32924 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5523, 2354, 125, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6687, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8004, 8191, 8190, 7342, 8194, 8067, 9356] avg=8192.0000 max=9356 max_rank=7 min=7342 min_rank=4 imbalance_ratio=1.142090 max_load_reduction=56180 max_load_reduction_pct=0.857239 moved_to_echo=32959 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5520, 2197, 122, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5427, 4, 0, 0, 1, 0, 0, 0, 0, 4490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 7794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 0, 0, 0, 0, 1, 0, 0, 2666, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8608, 8191, 8187, 9922, 8194, 8359, 5882] avg=8192.0000 max=9922 max_rank=4 min=5882 min_rank=7 imbalance_ratio=1.211182 max_load_reduction=55614 max_load_reduction_pct=0.848602 moved_to_echo=32357 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5523, 2354, 127, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5838, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7105, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8008, 8192, 8190, 6923, 8193, 8064, 9774] avg=8192.0000 max=9774 max_rank=7 min=6923 min_rank=4 imbalance_ratio=1.193115 max_load_reduction=55762 max_load_reduction_pct=0.850861 moved_to_echo=32954 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 5523, 2354, 125, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5838, 0, 0, 0, 0, 0, 0, 0, 0, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5267, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8068, 8192, 8190, 8699, 8195, 8064, 7936] avg=8192.0000 max=8699 max_rank=4 min=7936 min_rank=7 imbalance_ratio=1.061890 max_load_reduction=56837 max_load_reduction_pct=0.867264 moved_to_echo=32894 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5523, 2354, 125, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5720, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8005, 8192, 8190, 8309, 8195, 8064, 8389] avg=8192.0000 max=8389 max_rank=7 min=8005 min_rank=1 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32957 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7333, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 21, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7549, 0, 8097, 0, 4, 0, 0, 0, 0, 3893, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 0, 2, 3701, 8185, 8168, 0, 0, 0, 3, 0, 8186, 8160, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6864, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7523, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7756, 4, 8040, 0, 1, 0, 0, 0, 0, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 750, 0, 0, 0, 5191, 8191, 8190, 0, 0, 0, 1, 0, 8190, 8186, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8185, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7578, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7018, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6763, 1173, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7523, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8154, 8192, 8190, 7680, 8194, 7982, 8952] avg=8192.0000 max=8952 max_rank=7 min=7680 min_rank=4 imbalance_ratio=1.092773 max_load_reduction=56584 max_load_reduction_pct=0.863403 moved_to_echo=32808 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6644, 4, 0, 0, 1, 0, 0, 0, 0, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 750, 6761, 1112, 42, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5191, 0, 0, 0, 0, 0, 1, 0, 0, 1425, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8134, 8191, 8186, 9350, 8193, 8671, 6618] avg=8192.0000 max=9350 max_rank=4 min=6618 min_rank=7 imbalance_ratio=1.141357 max_load_reduction=56186 max_load_reduction_pct=0.857330 moved_to_echo=32832 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7019, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 6763, 1173, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8201, 8192, 8193, 7063, 8192, 7985, 9518] avg=8192.0000 max=9518 max_rank=7 min=7063 min_rank=4 imbalance_ratio=1.161865 max_load_reduction=56018 max_load_reduction_pct=0.854767 moved_to_echo=32758 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7019, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6763, 1173, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7578, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8270, 8192, 8190, 7508, 8196, 7981, 9007] avg=8192.0000 max=9007 max_rank=7 min=7508 min_rank=4 imbalance_ratio=1.099487 max_load_reduction=56529 max_load_reduction_pct=0.862564 moved_to_echo=32692 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7019, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6763, 1173, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7333, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8176, 8192, 8190, 7849, 8194, 7981, 8762] avg=8192.0000 max=8762 max_rank=7 min=7849 min_rank=4 imbalance_ratio=1.069580 max_load_reduction=56774 max_load_reduction_pct=0.866302 moved_to_echo=32786 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7019, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6763, 1173, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8152, 8191, 8190, 7384, 8192, 7982, 9252] avg=8192.0000 max=9252 max_rank=7 min=7384 min_rank=4 imbalance_ratio=1.129395 max_load_reduction=56284 max_load_reduction_pct=0.858826 moved_to_echo=32810 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7018, 0, 0, 0, 0, 0, 0, 0, 0, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6763, 1173, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6864, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8158, 8192, 8190, 8337, 8193, 7981, 8293] avg=8192.0000 max=8337 max_rank=4 min=7981 min_rank=6 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32804 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 1, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6468, 0, 0, 0, 4, 0, 0, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4, 0, 8168, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 6736, 1081, 44, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3701, 0, 0, 0, 0, 0, 3, 0, 0, 1424, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8291, 8194, 8207, 10365, 8182, 8973, 5134] avg=8192.0000 max=10365 max_rank=4 min=5134 min_rank=7 imbalance_ratio=1.265259 max_load_reduction=55171 max_load_reduction_pct=0.841843 moved_to_echo=32654 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 1920, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 44288, 18560, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 44288, 0, 0, 0, 0, 0, 128, 0, 0, 21376, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 45824, 128, 0, 0, 128, 0, 0, 0, 0, 19840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 1536, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 1920, 54144, 9344, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 54144, 0, 0, 0, 0, 0, 128, 0, 0, 11520, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 55296, 128, 0, 0, 128, 0, 0, 0, 0, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 384, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 128, 0, 0, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 7874, 0, 0, 0, 0, 0, 0, 4689, 0, 0, 0, 1385, 8187, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4, 3, 0, 0, 0, 0, 8096, 0, 0, 0, 5, 0, 0, 0, 26, 0, 8162, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4380, 4360, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2178, 0, 4, 0, 0, 0, 0, 0, 0, 13, 4, 0, 3, 7864, 0, 4, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 6897, 0, 0, 0, 0, 0, 0, 4020, 1, 0, 0, 1786, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1, 1, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8189, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 5317, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3649, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8075, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 335, 0, 0, 0, 0, 0, 0, 6240, 0, 0, 0, 3633, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6314, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 55, 0, 0, 0, 0, 0, 0, 5803, 0, 0, 0, 3429, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 97, 7965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 603, 0, 0, 0, 0, 0, 0, 7089, 0, 0, 0, 3708, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4810, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 214, 0, 0, 0, 0, 0, 0, 5518, 0, 0, 0, 5897, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 8042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4826, 0, 0, 0, 0, 0, 0, 5, 0, 39, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 234, 0, 0, 0, 0, 0, 0, 5052, 0, 0, 0, 5739, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5350, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 114, 0, 0, 0, 0, 0, 0, 6461, 0, 0, 0, 3976, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5676, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7874, 0, 0, 0, 0, 0, 0, 2054, 0, 0, 0, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4, 3, 0, 0, 0, 0, 8096, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 8162, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4380, 4326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2178, 0, 4, 0, 0, 0, 0, 2635, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4, 0, 3, 7864, 0, 4, 0, 0, 0, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11313, 8194, 8198, 8190, 8706, 8192, 4842, 7901] avg=8192.0000 max=11313 max_rank=0 min=4842 min_rank=6 imbalance_ratio=1.380981 max_load_reduction=54223 max_load_reduction_pct=0.827377 moved_to_echo=29641 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 335, 0, 0, 0, 0, 0, 0, 2734, 0, 0, 0, 3633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6314, 0, 0, 0, 0, 0, 0, 3506, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6702, 8191, 8194, 8193, 7988, 8192, 9859, 8217] avg=8192.0000 max=9859 max_rank=6 min=6702 min_rank=0 imbalance_ratio=1.203491 max_load_reduction=55677 max_load_reduction_pct=0.849564 moved_to_echo=34256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 55, 0, 0, 0, 0, 0, 0, 2542, 0, 0, 0, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 97, 7903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7220, 0, 0, 0, 0, 0, 0, 3261, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6026, 8191, 8194, 8193, 8007, 8192, 10519, 8214] avg=8192.0000 max=10519 max_rank=6 min=6026 min_rank=0 imbalance_ratio=1.284058 max_load_reduction=55017 max_load_reduction_pct=0.839493 moved_to_echo=34932 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6897, 0, 0, 0, 0, 0, 0, 1757, 1, 0, 0, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1, 1, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 5260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3649, 0, 1, 0, 0, 0, 0, 2263, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8075, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10441, 8193, 8174, 8191, 8303, 8193, 5944, 8097] avg=8192.0000 max=10441 max_rank=0 min=5944 min_rank=6 imbalance_ratio=1.274536 max_load_reduction=55095 max_load_reduction_pct=0.840683 moved_to_echo=30537 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 603, 0, 0, 0, 0, 0, 0, 3106, 0, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4810, 0, 0, 0, 0, 0, 0, 3983, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7417, 8191, 8194, 8193, 8252, 8192, 8832, 8265] avg=8192.0000 max=8832 max_rank=6 min=7417 min_rank=0 imbalance_ratio=1.078125 max_load_reduction=56704 max_load_reduction_pct=0.865234 moved_to_echo=33541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 214, 0, 0, 0, 0, 0, 0, 2418, 0, 0, 0, 5897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 7979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4826, 0, 0, 0, 0, 0, 0, 3100, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 39, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8529, 8191, 8194, 8192, 8014, 8192, 7966, 8258] avg=8192.0000 max=8529 max_rank=0 min=7966 min_rank=6 imbalance_ratio=1.041138 max_load_reduction=57007 max_load_reduction_pct=0.869858 moved_to_echo=32430 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 114, 0, 0, 0, 0, 0, 0, 2831, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5676, 0, 0, 0, 0, 0, 0, 3630, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6921, 8192, 8194, 8192, 8171, 8192, 9346, 8328] avg=8192.0000 max=9346 max_rank=6 min=6921 min_rank=0 imbalance_ratio=1.140869 max_load_reduction=56190 max_load_reduction_pct=0.857391 moved_to_echo=34037 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 234, 0, 0, 0, 0, 0, 0, 2214, 0, 0, 0, 5739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5350, 0, 0, 0, 0, 0, 0, 2838, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8193, 8194, 8192, 8095, 8191, 8228, 8256] avg=8192.0000 max=8256 max_rank=7 min=8095 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 50, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 8192, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 40, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 5816, 0, 0, 0, 0, 0, 0, 5946, 0, 0, 0, 130, 8187, 0, 0, 84, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4, 3, 0, 0, 0, 0, 8097, 0, 0, 0, 6, 0, 0, 0, 28, 0, 8160, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4794, 4674, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3314, 0, 4, 0, 0, 0, 0, 0, 0, 50, 4, 0, 3, 7931, 0, 4, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 8192, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 4, 0, 41, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3675, 0, 0, 0, 0, 0, 0, 5522, 2, 0, 0, 50, 8190, 0, 0, 44, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 1, 1, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8189, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3555, 5423, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6365, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8123, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 8192, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7787, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 8192, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 40, 7818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 604, 360, 41, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7543, 8196, 8190, 8192, 7869, 8191, 9187, 8168] avg=8192.0000 max=9187 max_rank=6 min=7543 min_rank=0 imbalance_ratio=1.121460 max_load_reduction=56349 max_load_reduction_pct=0.859818 moved_to_echo=33415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 7496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 7801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 600, 359, 41, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7500, 8162, 8190, 8192, 7957, 8191, 9169, 8175] avg=8192.0000 max=9169 max_rank=6 min=7500 min_rank=0 imbalance_ratio=1.119263 max_load_reduction=56367 max_load_reduction_pct=0.860092 moved_to_echo=33492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 50, 0, 0, 0, 0, 0, 0, 7472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 7780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 598, 358, 41, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7522, 8154, 8190, 8192, 8042, 8191, 8980, 8265] avg=8192.0000 max=8980 max_rank=6 min=7522 min_rank=0 imbalance_ratio=1.096191 max_load_reduction=56556 max_load_reduction_pct=0.862976 moved_to_echo=33478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 7830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 605, 360, 40, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 41, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7557, 8153, 8190, 8192, 7874, 8191, 9166, 8213] avg=8192.0000 max=9166 max_rank=6 min=7557 min_rank=0 imbalance_ratio=1.118896 max_load_reduction=56370 max_load_reduction_pct=0.860138 moved_to_echo=33444 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5816, 0, 0, 0, 0, 0, 0, 5505, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8140, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4, 3, 0, 0, 0, 0, 8096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 8160, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4794, 4469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3314, 0, 4, 0, 0, 0, 0, 441, 205, 40, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4, 0, 3, 7907, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11451, 8229, 8203, 8190, 9263, 8198, 4033, 7969] avg=8192.0000 max=11451 max_rank=0 min=4033 min_rank=6 imbalance_ratio=1.397827 max_load_reduction=54085 max_load_reduction_pct=0.825272 moved_to_echo=29463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 7556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 7821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7787, 0, 0, 0, 0, 0, 0, 605, 360, 40, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7559, 8237, 8191, 8192, 7916, 8191, 8818, 8432] avg=8192.0000 max=8818 max_rank=6 min=7559 min_rank=0 imbalance_ratio=1.076416 max_load_reduction=56718 max_load_reduction_pct=0.865448 moved_to_echo=33357 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3675, 0, 0, 0, 0, 0, 0, 5109, 2, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8148, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 1, 1, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3555, 5181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6365, 0, 1, 0, 0, 0, 0, 413, 242, 40, 30, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8836, 8195, 8192, 8194, 8736, 8192, 7096, 8095] avg=8192.0000 max=8836 max_rank=0 min=7096 min_rank=6 imbalance_ratio=1.078613 max_load_reduction=56700 max_load_reduction_pct=0.865173 moved_to_echo=32119 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 7567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 7831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 0, 0, 0, 0, 0, 0, 606, 360, 40, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7568, 8210, 8190, 8192, 7879, 8191, 9087, 8219] avg=8192.0000 max=9087 max_rank=6 min=7568 min_rank=0 imbalance_ratio=1.109253 max_load_reduction=56449 max_load_reduction_pct=0.861343 moved_to_echo=33376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 40064, 0, 128, 0, 0, 0, 0, 25216, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 16384, 0, 0, 0, 0, 0, 0, 19712, 128, 0, 0, 29568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 8064, 57600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 256, 128, 128, 0, 0, 0, 0, 65408, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 384, 128, 0, 128, 65152, 0, 128, 0, 0, 0, 0, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 0, 57984, 0, 128, 0, 0, 0, 0, 4480, 2688, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 9600, 0, 0, 0, 0, 0, 0, 55808, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 9088, 56576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 384, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 512, 128, 0, 128, 65024, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 256, 128, 128, 0, 0, 128, 0, 65408, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 80, 0, 7998, 0, 3, 0, 0, 0, 0, 0, 261, 0, 0, 3, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3774, 0, 8107, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1758, 0, 5, 0, 8168, 0, 0, 3, 0, 8187, 0, 0, 0, 4, 8185, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2657, 0, 0, 0, 177, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8186, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2992, 0, 8124, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3427, 0, 1, 0, 8188, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8191, 0, 0, 0, 1, 7998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 0, 0, 0, 62, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 8098, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8183, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8186, 0, 0, 870, 535, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8191, 7327, 9620, 8195, 8247, 7640, 8162] avg=8192.0000 max=9620 max_rank=3 min=7327 min_rank=2 imbalance_ratio=1.174316 max_load_reduction=55916 max_load_reduction_pct=0.853210 moved_to_echo=32244 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 80, 0, 7961, 0, 3, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 7246, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1758, 0, 5, 861, 535, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 3, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 37, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8196, 11027, 3204, 8178, 7995, 10293, 8338] avg=8192.0000 max=11027 max_rank=2 min=3204 min_rank=3 imbalance_ratio=1.346069 max_load_reduction=54509 max_load_reduction_pct=0.831741 moved_to_echo=34804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 7322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 870, 535, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8191, 7403, 9542, 8195, 8247, 7640, 8162] avg=8192.0000 max=9542 max_rank=3 min=7403 min_rank=2 imbalance_ratio=1.164795 max_load_reduction=55994 max_load_reduction_pct=0.854401 moved_to_echo=32244 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 7322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 8098, 0, 0, 870, 535, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8191, 7408, 9540, 8195, 8247, 7640, 8162] avg=8192.0000 max=9540 max_rank=3 min=7408 min_rank=2 imbalance_ratio=1.164551 max_load_reduction=55996 max_load_reduction_pct=0.854431 moved_to_echo=32244 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 7322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 870, 535, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8191, 7456, 9492, 8195, 8247, 7640, 8162] avg=8192.0000 max=9492 max_rank=3 min=7456 min_rank=2 imbalance_ratio=1.158691 max_load_reduction=56044 max_load_reduction_pct=0.855164 moved_to_echo=32244 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2992, 0, 7260, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3427, 0, 1, 864, 541, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 42, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8193, 10258, 4897, 8191, 8060, 9403, 8224] avg=8192.0000 max=10258 max_rank=2 min=4897 min_rank=3 imbalance_ratio=1.252197 max_load_reduction=55278 max_load_reduction_pct=0.843475 moved_to_echo=33878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 870, 535, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8191, 7327, 9623, 8195, 8247, 7640, 8162] avg=8192.0000 max=9623 max_rank=3 min=7327 min_rank=2 imbalance_ratio=1.174683 max_load_reduction=55913 max_load_reduction_pct=0.853165 moved_to_echo=32244 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 7322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8183, 0, 0, 870, 535, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8192, 7330, 9618, 8192, 8246, 7640, 8164] avg=8192.0000 max=9618 max_rank=3 min=7330 min_rank=2 imbalance_ratio=1.174072 max_load_reduction=55918 max_load_reduction_pct=0.853241 moved_to_echo=32242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 79, 0, 7982, 0, 3, 0, 0, 0, 0, 0, 249, 0, 0, 4, 0, 0, 0, 1, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4235, 0, 8104, 0, 0, 20, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1291, 0, 5, 0, 8166, 0, 0, 3, 0, 8187, 0, 0, 0, 4, 8184, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 0, 0, 174, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8185, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 8179, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2690, 0, 8121, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 1, 0, 8187, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8191, 0, 0, 0, 1, 7997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 0, 0, 63, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 7328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 864, 541, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8191, 7368, 9584, 8196, 8245, 7632, 8162] avg=8192.0000 max=9584 max_rank=3 min=7368 min_rank=2 imbalance_ratio=1.169922 max_load_reduction=55952 max_load_reduction_pct=0.853760 moved_to_echo=32235 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 7328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 864, 541, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8191, 7367, 9586, 8196, 8245, 7632, 8162] avg=8192.0000 max=9586 max_rank=3 min=7367 min_rank=2 imbalance_ratio=1.170166 max_load_reduction=55950 max_load_reduction_pct=0.853729 moved_to_echo=32235 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 8179, 0, 0, 864, 541, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8191, 7332, 9621, 8193, 8245, 7632, 8165] avg=8192.0000 max=9621 max_rank=3 min=7332 min_rank=2 imbalance_ratio=1.174438 max_load_reduction=55915 max_load_reduction_pct=0.853195 moved_to_echo=32235 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 864, 541, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8191, 7329, 9623, 8196, 8245, 7632, 8162] avg=8192.0000 max=9623 max_rank=3 min=7329 min_rank=2 imbalance_ratio=1.174683 max_load_reduction=55913 max_load_reduction_pct=0.853165 moved_to_echo=32235 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2690, 0, 7262, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 1, 859, 546, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 38, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 8192, 9962, 5141, 8190, 8059, 9454, 8225] avg=8192.0000 max=9962 max_rank=2 min=5141 min_rank=3 imbalance_ratio=1.216064 max_load_reduction=55574 max_load_reduction_pct=0.847992 moved_to_echo=33928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 864, 541, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8191, 7335, 9620, 8196, 8245, 7632, 8162] avg=8192.0000 max=9620 max_rank=3 min=7335 min_rank=2 imbalance_ratio=1.174316 max_load_reduction=55916 max_load_reduction_pct=0.853210 moved_to_echo=32235 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 79, 0, 7949, 0, 3, 0, 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4235, 0, 7250, 0, 0, 20, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1291, 0, 5, 854, 541, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 3, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 33, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8198, 11509, 2740, 8177, 8008, 10290, 8334] avg=8192.0000 max=11509 max_rank=2 min=2740 min_rank=3 imbalance_ratio=1.404907 max_load_reduction=54027 max_load_reduction_pct=0.824387 moved_to_echo=34809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8185, 0, 0, 864, 541, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8191, 7334, 9621, 8192, 8244, 7632, 8164] avg=8192.0000 max=9621 max_rank=3 min=7334 min_rank=2 imbalance_ratio=1.174438 max_load_reduction=55915 max_load_reduction_pct=0.853195 moved_to_echo=32232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 0, 54016, 0, 128, 7040, 4352, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 7168, 0, 58496, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 61184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 0, 54016, 0, 128, 6912, 4352, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 7040, 0, 58496, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 61056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 256, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 65024, 0, 128, 0, 0, 0, 0, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 65024, 0, 128, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 2, 0, 0, 8187, 0, 0, 0, 0, 8186, 4, 0, 8187, 0, 8185, 0, 1, 0, 4, 0, 7815, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 3, 6359, 0, 1, 0, 0, 0, 5, 5, 4, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 0, 0, 0, 4, 0, 0, 1, 0, 3, 2, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 302] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8179, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 8191, 0, 0, 0, 0, 8191, 1, 0, 8191, 0, 8189, 0, 0, 0, 1, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7217, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 974, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 101] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 8192, 0, 0, 0, 0, 8192, 0, 0, 8171, 0, 8192, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8189, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 340, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7793, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8109, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8250, 8533, 8191, 8193, 7844, 8159, 8142] avg=8192.0000 max=8533 max_rank=2 min=7844 min_rank=5 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=32338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 2, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 58, 0, 0, 0, 1, 0, 4, 0, 7815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 3, 6359, 0, 340, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 5, 4, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 0, 0, 0, 4, 0, 0, 1, 7789, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 302, 8104, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8050, 7882, 6709, 8201, 8187, 9674, 8392, 8441] avg=8192.0000 max=9674 max_rank=5 min=6709 min_rank=2 imbalance_ratio=1.180908 max_load_reduction=55862 max_load_reduction_pct=0.852386 moved_to_echo=34694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 8192, 0, 340, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8250, 8548, 8191, 8192, 7832, 8160, 8141] avg=8192.0000 max=8548 max_rank=2 min=7832 min_rank=5 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32325 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 8192, 0, 340, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7774, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8109, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8238, 8566, 8191, 8193, 7825, 8159, 8142] avg=8192.0000 max=8566 max_rank=2 min=7825 min_rank=5 imbalance_ratio=1.045654 max_load_reduction=56970 max_load_reduction_pct=0.869293 moved_to_echo=32319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8192, 0, 340, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8246, 8537, 8191, 8192, 7845, 8160, 8141] avg=8192.0000 max=8537 max_rank=2 min=7845 min_rank=5 imbalance_ratio=1.042114 max_load_reduction=56999 max_load_reduction_pct=0.869736 moved_to_echo=32338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 49, 0, 0, 0, 0, 0, 1, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7217, 0, 346, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 974, 0, 0, 0, 1, 0, 0, 0, 7796, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 101, 8112, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8174, 7570, 8189, 8193, 8829, 8188, 8246] avg=8192.0000 max=8829 max_rank=5 min=7570 min_rank=2 imbalance_ratio=1.077759 max_load_reduction=56707 max_load_reduction_pct=0.865280 moved_to_echo=33456 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8192, 0, 340, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8247, 8536, 8191, 8193, 7845, 8159, 8141] avg=8192.0000 max=8536 max_rank=2 min=7845 min_rank=5 imbalance_ratio=1.041992 max_load_reduction=57000 max_load_reduction_pct=0.869751 moved_to_echo=32338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8190, 0, 340, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7791, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8109, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8249, 8537, 8191, 8193, 7842, 8159, 8142] avg=8192.0000 max=8537 max_rank=2 min=7842 min_rank=5 imbalance_ratio=1.042114 max_load_reduction=56999 max_load_reduction_pct=0.869736 moved_to_echo=32336 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6834, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1319] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7706, 2, 0, 0, 8187, 0, 0, 0, 0, 8186, 4, 0, 8187, 0, 8185, 0, 2, 0, 4, 0, 7906, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 3, 6412, 0, 1, 0, 0, 0, 5, 5, 5, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1780, 0, 0, 0, 4, 0, 0, 1, 0, 3, 2, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 420] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7390, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7203, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8185, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6357, 0, 0, 1, 8192, 0, 0, 0, 0, 8192, 0, 0, 8177, 0, 8192, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 977, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7904, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 1, 0, 8191, 0, 8189, 0, 0, 0, 1, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7347, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 153] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5999, 0, 0, 1, 8192, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1334] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7390, 0, 0, 0, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 7116, 346, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8466, 8233, 8330, 8191, 8193, 7848, 8147, 8128] avg=8192.0000 max=8466 max_rank=0 min=7848 min_rank=5 imbalance_ratio=1.033447 max_load_reduction=57070 max_load_reduction_pct=0.870819 moved_to_echo=32316 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7706, 2, 0, 0, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 40, 0, 0, 0, 2, 0, 4, 0, 7906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 3, 6374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 5, 5, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1780, 0, 0, 0, 4, 0, 0, 1, 7801, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 420, 7111, 346, 38, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8784, 7956, 6382, 8201, 8188, 9630, 8475, 7920] avg=8192.0000 max=9630 max_rank=5 min=6382 min_rank=2 imbalance_ratio=1.175537 max_load_reduction=55906 max_load_reduction_pct=0.853058 moved_to_echo=34213 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7203, 0, 0, 0, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7799, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 7116, 346, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8232, 8547, 8191, 8193, 7843, 8147, 8104] avg=8192.0000 max=8547 max_rank=2 min=7843 min_rank=5 imbalance_ratio=1.043335 max_load_reduction=56989 max_load_reduction_pct=0.869583 moved_to_echo=32287 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 0, 0, 0, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7805, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 7116, 346, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8607, 8233, 8669, 8191, 8193, 7849, 8147, 7647] avg=8192.0000 max=8669 max_rank=2 min=7647 min_rank=7 imbalance_ratio=1.058228 max_load_reduction=56867 max_load_reduction_pct=0.867722 moved_to_echo=31836 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6357, 0, 0, 1, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 977, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7791, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 7116, 346, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7434, 8230, 9120, 8191, 8193, 7835, 8147, 8386] avg=8192.0000 max=9120 max_rank=2 min=7434 min_rank=0 imbalance_ratio=1.113281 max_load_reduction=56416 max_load_reduction_pct=0.860840 moved_to_echo=32561 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6834, 0, 0, 0, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7805, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1319, 7116, 346, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7910, 8233, 8182, 8191, 8192, 7850, 8147, 8831] avg=8192.0000 max=8831 max_rank=7 min=7850 min_rank=5 imbalance_ratio=1.078003 max_load_reduction=56705 max_load_reduction_pct=0.865250 moved_to_echo=33020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7904, 0, 0, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 29, 0, 0, 0, 0, 0, 1, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 0, 0, 1, 0, 0, 0, 7810, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 153, 7116, 352, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8980, 8186, 7304, 8189, 8191, 8833, 8179, 7674] avg=8192.0000 max=8980 max_rank=0 min=7304 min_rank=2 imbalance_ratio=1.096191 max_load_reduction=56556 max_load_reduction_pct=0.862976 moved_to_echo=32877 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5999, 0, 0, 1, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1334, 7116, 346, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7076, 8233, 9002, 8191, 8193, 7848, 8147, 8846] avg=8192.0000 max=9002 max_rank=2 min=7076 min_rank=0 imbalance_ratio=1.098877 max_load_reduction=56534 max_load_reduction_pct=0.862640 moved_to_echo=33034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 128, 62848, 0, 2816, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 0, 0, 128, 0, 0, 128, 62336, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 512, 64896, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 56960, 128, 0, 128, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 5504, 56960, 2816, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 3072, 0, 128, 62592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 0, 0, 128, 0, 0, 128, 62464, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 512, 0, 0, 0, 128, 0, 128, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 384, 0, 0, 0, 128, 0, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8186, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 43, 145, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 5, 5, 0, 0, 8187, 0, 0, 3222, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 8181, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 6, 0, 4823, 60, 0, 8185, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 20, 52, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 8191, 0, 0, 1505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 6632, 6, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7484, 676, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7514, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8203, 8191, 7658, 8198, 8192, 8193, 8740] avg=8192.0000 max=8740 max_rank=7 min=7658 min_rank=3 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=33323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8155, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 43, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 5, 7477, 675, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7512, 0, 0, 3222, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 6, 0, 4823, 60, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8345, 8138, 8194, 10748, 8151, 8191, 8183, 5586] avg=8192.0000 max=10748 max_rank=3 min=5586 min_rank=7 imbalance_ratio=1.312012 max_load_reduction=54788 max_load_reduction_pct=0.835999 moved_to_echo=30111 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7484, 676, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7514, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8203, 8191, 7592, 8198, 8192, 8193, 8808] avg=8192.0000 max=8808 max_rank=7 min=7592 min_rank=3 imbalance_ratio=1.075195 max_load_reduction=56728 max_load_reduction_pct=0.865601 moved_to_echo=33391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7484, 673, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7490, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8203, 8188, 7675, 8198, 8194, 8193, 8727] avg=8192.0000 max=8727 max_rank=7 min=7675 min_rank=3 imbalance_ratio=1.065308 max_load_reduction=56809 max_load_reduction_pct=0.866837 moved_to_echo=33312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8157, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 20, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7484, 679, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7512, 0, 0, 1505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 6632, 6, 0, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8180, 8199, 9020, 8197, 8190, 8195, 7324] avg=8192.0000 max=9020 max_rank=3 min=7324 min_rank=7 imbalance_ratio=1.101074 max_load_reduction=56516 max_load_reduction_pct=0.862366 moved_to_echo=31906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7484, 676, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7516, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8203, 8191, 7682, 8198, 8192, 8193, 8716] avg=8192.0000 max=8716 max_rank=7 min=7682 min_rank=3 imbalance_ratio=1.063965 max_load_reduction=56820 max_load_reduction_pct=0.867004 moved_to_echo=33299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7484, 676, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7516, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8203, 8191, 7639, 8198, 8192, 8193, 8759] avg=8192.0000 max=8759 max_rank=7 min=7639 min_rank=3 imbalance_ratio=1.069214 max_load_reduction=56777 max_load_reduction_pct=0.866348 moved_to_echo=33342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7484, 676, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7514, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8203, 8191, 7522, 8198, 8193, 8193, 8876] avg=8192.0000 max=8876 max_rank=7 min=7522 min_rank=3 imbalance_ratio=1.083496 max_load_reduction=56660 max_load_reduction_pct=0.864563 moved_to_echo=33460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8186, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 49, 135, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 5, 5, 0, 0, 8187, 0, 0, 2779, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 8181, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 6, 0, 5279, 59, 0, 8185, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 26, 39, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 8191, 0, 0, 1120, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 7026, 6, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7650, 510, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8203, 8189, 7713, 8200, 8192, 8193, 8683] avg=8192.0000 max=8683 max_rank=7 min=7713 min_rank=3 imbalance_ratio=1.059937 max_load_reduction=56853 max_load_reduction_pct=0.867508 moved_to_echo=33268 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7650, 510, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8203, 8189, 7705, 8200, 8192, 8193, 8693] avg=8192.0000 max=8693 max_rank=7 min=7705 min_rank=3 imbalance_ratio=1.061157 max_load_reduction=56843 max_load_reduction_pct=0.867355 moved_to_echo=33278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7650, 510, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7682, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8203, 8189, 7709, 8200, 8192, 8193, 8687] avg=8192.0000 max=8687 max_rank=7 min=7709 min_rank=3 imbalance_ratio=1.060425 max_load_reduction=56849 max_load_reduction_pct=0.867447 moved_to_echo=33272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8157, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 49, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 5, 7644, 510, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7677, 0, 0, 2779, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 6, 0, 5279, 59, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8343, 8135, 8194, 10470, 8147, 8191, 8183, 5873] avg=8192.0000 max=10470 max_rank=3 min=5873 min_rank=7 imbalance_ratio=1.278076 max_load_reduction=55066 max_load_reduction_pct=0.840240 moved_to_echo=30394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7650, 510, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7682, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8203, 8189, 7745, 8200, 8192, 8193, 8651] avg=8192.0000 max=8651 max_rank=7 min=7745 min_rank=3 imbalance_ratio=1.056030 max_load_reduction=56885 max_load_reduction_pct=0.867996 moved_to_echo=33236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7650, 508, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7651, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8203, 8187, 7713, 8200, 8194, 8193, 8686] avg=8192.0000 max=8686 max_rank=7 min=7713 min_rank=3 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=33273 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 26, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7654, 517, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7674, 0, 0, 1120, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 7026, 6, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8183, 8210, 8797, 8189, 8190, 8195, 7551] avg=8192.0000 max=8797 max_rank=3 min=7551 min_rank=7 imbalance_ratio=1.073853 max_load_reduction=56739 max_load_reduction_pct=0.865768 moved_to_echo=32125 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7650, 510, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8203, 8189, 7684, 8200, 8193, 8193, 8712] avg=8192.0000 max=8712 max_rank=7 min=7684 min_rank=3 imbalance_ratio=1.063477 max_load_reduction=56824 max_load_reduction_pct=0.867065 moved_to_echo=33298 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 59904, 5504, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 60032, 128, 0, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 60160, 0, 0, 5504, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65280, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 61312, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 61312, 128, 0, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 61440, 0, 0, 4224, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65408, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 7078, 0, 0, 0, 0, 1, 0, 8185, 0, 0, 23, 0, 0, 0, 8171, 0, 0, 0, 0, 35, 0, 0, 1108, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 5, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8159, 6, 0, 7956, 0, 1, 5, 0, 0, 0, 0, 0, 0, 7, 0, 8183, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 6, 0, 0, 241, 0, 0, 0, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 7944, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 7, 0, 0, 0, 8179, 0, 0, 0, 0, 5, 0, 3, 243, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 1, 0, 1, 1, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 6694, 0, 0, 0, 0, 1, 0, 8185, 0, 0, 25, 0, 0, 0, 8169, 0, 0, 0, 0, 38, 0, 1, 1491, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 6, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8154, 6, 0, 7961, 0, 1, 5, 0, 0, 0, 0, 0, 0, 8, 0, 8183, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 6, 0, 0, 283, 0, 0, 0, 4, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 8021, 8192, 8192, 8191, 8220, 8210, 8150] avg=8192.0000 max=8360 max_rank=0 min=8021 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7078, 0, 0, 0, 0, 1, 0, 0, 0, 0, 23, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1108, 4, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 5, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8163, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 6, 0, 7956, 0, 1, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 241, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8112, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7270, 9145, 8191, 8196, 8194, 8003, 8168, 8369] avg=8192.0000 max=9145 max_rank=1 min=7270 min_rank=0 imbalance_ratio=1.116333 max_load_reduction=56391 max_load_reduction_pct=0.860458 moved_to_echo=32734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 8027, 8192, 8186, 8191, 8220, 8210, 8150] avg=8192.0000 max=8360 max_rank=0 min=8027 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 8023, 8192, 8185, 8191, 8220, 8210, 8155] avg=8192.0000 max=8360 max_rank=0 min=8023 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 243, 1, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8262, 8193, 8196, 8196, 8213, 8108, 8262] avg=8192.0000 max=8262 max_rank=1 min=8106 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 8019, 8192, 8194, 8191, 8220, 8210, 8150] avg=8192.0000 max=8360 max_rank=0 min=8019 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 8019, 8192, 8194, 8191, 8220, 8210, 8150] avg=8192.0000 max=8360 max_rank=0 min=8019 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 8020, 8192, 8193, 8191, 8220, 8210, 8150] avg=8192.0000 max=8360 max_rank=0 min=8020 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 7633, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 6, 0, 0, 0, 8180, 0, 0, 0, 0, 6, 0, 3, 532, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 1, 0, 1, 1, 0, 0, 180, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8447, 7943, 8192, 8185, 8191, 8219, 8223, 8136] avg=8192.0000 max=8447 max_rank=0 min=7943 min_rank=1 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8447, 7934, 8192, 8194, 8191, 8219, 8223, 8136] avg=8192.0000 max=8447 max_rank=0 min=7934 min_rank=1 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8447, 7937, 8192, 8191, 8191, 8219, 8223, 8136] avg=8192.0000 max=8447 max_rank=0 min=7937 min_rank=1 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6694, 0, 0, 0, 0, 1, 0, 0, 0, 0, 25, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 1, 1491, 4, 0, 0, 0, 0, 0, 0, 0, 7912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 6, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8150, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 6, 0, 7961, 0, 1, 5, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 283, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6974, 9446, 8192, 8197, 8193, 8009, 8134, 8391] avg=8192.0000 max=9446 max_rank=1 min=6974 min_rank=0 imbalance_ratio=1.153076 max_load_reduction=56090 max_load_reduction_pct=0.855865 moved_to_echo=32727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8447, 7934, 8192, 8194, 8191, 8219, 8223, 8136] avg=8192.0000 max=8447 max_rank=0 min=7934 min_rank=1 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 532, 1, 0, 0, 0, 0, 0, 0, 0, 7928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 180, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8129, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7880, 8470, 8192, 8198, 8196, 8213, 8065, 8322] avg=8192.0000 max=8470 max_rank=1 min=7880 min_rank=0 imbalance_ratio=1.033936 max_load_reduction=57066 max_load_reduction_pct=0.870758 moved_to_echo=32796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8447, 7938, 8192, 8184, 8191, 8219, 8222, 8143] avg=8192.0000 max=8447 max_rank=0 min=7938 min_rank=1 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8447, 7934, 8192, 8193, 8192, 8219, 8223, 8136] avg=8192.0000 max=8447 max_rank=0 min=7934 min_rank=1 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 64256, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 63488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 63488, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 0, 512, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 256, 128, 0, 65408, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 0, 384, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3639, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3261, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5572, 8168, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8179, 19, 4, 0, 8185, 0, 8170, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, 4, 0, 3, 0, 8157, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 8184, 0, 0, 0, 7, 0, 2, 2, 1, 0, 0, 4, 0, 0, 0, 1143, 0, 0, 0, 7, 3, 0, 0, 0, 0, 7953, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2169, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 531, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1788, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2716, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 1, 0, 8190, 0, 8184, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 975, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8190, 0, 0, 0, 2, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 409, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8146, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3315, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3639, 5321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 160, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8960, 8192, 8024, 6423, 9524, 8192, 8189, 8032] avg=8192.0000 max=9524 max_rank=4 min=6423 min_rank=3 imbalance_ratio=1.162598 max_load_reduction=56012 max_load_reduction_pct=0.854675 moved_to_echo=33937 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5572, 5305, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4, 0, 0, 0, 8170, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, 4, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8157, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1109, 156, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1754, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 8181, 0, 0, 0, 7, 0, 2, 2, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 0, 0, 7, 3, 0, 0, 0, 0, 7797, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10882, 8198, 8313, 9431, 3360, 8191, 8209, 8952] avg=8192.0000 max=10882 max_rank=0 min=3360 min_rank=4 imbalance_ratio=1.328369 max_load_reduction=54654 max_load_reduction_pct=0.833954 moved_to_echo=28712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 531, 5321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 160, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5852, 8192, 8065, 9207, 9807, 8192, 8189, 8032] avg=8192.0000 max=9807 max_rank=4 min=5852 min_rank=0 imbalance_ratio=1.197144 max_load_reduction=55729 max_load_reduction_pct=0.850357 moved_to_echo=34220 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2169, 5321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 160, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7490, 8192, 8046, 7736, 9659, 8192, 8189, 8032] avg=8192.0000 max=9659 max_rank=4 min=7490 min_rank=0 imbalance_ratio=1.179077 max_load_reduction=55877 max_load_reduction_pct=0.852615 moved_to_echo=34072 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2716, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 8180, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 975, 0, 0, 0, 0, 0, 1, 0, 8024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1115, 165, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8186, 0, 0, 0, 2, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8186, 9001, 9342, 6206, 8191, 8193, 8392] avg=8192.0000 max=9342 max_rank=3 min=6206 min_rank=4 imbalance_ratio=1.140381 max_load_reduction=56194 max_load_reduction_pct=0.857452 moved_to_echo=30982 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3315, 5321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 160, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8636, 8192, 8024, 9146, 7123, 8194, 8189, 8032] avg=8192.0000 max=9146 max_rank=3 min=7123 min_rank=4 imbalance_ratio=1.116455 max_load_reduction=56390 max_load_reduction_pct=0.860443 moved_to_echo=31538 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1788, 5321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 160, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7109, 8192, 8033, 7834, 9955, 8192, 8189, 8032] avg=8192.0000 max=9955 max_rank=4 min=7109 min_rank=0 imbalance_ratio=1.215210 max_load_reduction=55581 max_load_reduction_pct=0.848099 moved_to_echo=34368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3261, 5321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 160, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8582, 8192, 8030, 6417, 9902, 8192, 8189, 8032] avg=8192.0000 max=9902 max_rank=4 min=6417 min_rank=3 imbalance_ratio=1.208740 max_load_reduction=55634 max_load_reduction_pct=0.848907 moved_to_echo=34315 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7106, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7196, 8166, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8178, 21, 4, 0, 8185, 0, 8164, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 8168, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 8184, 0, 0, 0, 7, 0, 3, 2, 1, 0, 0, 4, 0, 0, 0, 1156, 0, 0, 0, 7, 3, 0, 0, 0, 0, 7976, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6378, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8152, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4290, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6262, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7367, 8186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 5, 1, 0, 8190, 0, 8184, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 0, 1, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 2, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 500, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8145, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6690, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6378, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 115, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7891, 8192, 8199, 8379, 8477, 8193, 8189, 8016] avg=8192.0000 max=8477 max_rank=4 min=7891 min_rank=0 imbalance_ratio=1.034790 max_load_reduction=57059 max_load_reduction_pct=0.870651 moved_to_echo=32875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7106, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 115, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8619, 8192, 8191, 8224, 7912, 8193, 8189, 8016] avg=8192.0000 max=8619 max_rank=0 min=7912 min_rank=4 imbalance_ratio=1.052124 max_load_reduction=56917 max_load_reduction_pct=0.868484 moved_to_echo=32310 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7196, 1508, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4, 0, 0, 0, 8164, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8168, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 171, 115, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 8181, 0, 0, 0, 7, 0, 3, 2, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 0, 0, 0, 7, 3, 0, 0, 0, 0, 7805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8709, 8194, 8181, 8464, 6615, 8190, 8212, 8971] avg=8192.0000 max=8971 max_rank=7 min=6615 min_rank=4 imbalance_ratio=1.095093 max_load_reduction=56565 max_load_reduction_pct=0.863113 moved_to_echo=31988 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8152, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 115, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9665, 8192, 8191, 8445, 6645, 8193, 8189, 8016] avg=8192.0000 max=9665 max_rank=0 min=6645 min_rank=4 imbalance_ratio=1.179810 max_load_reduction=55871 max_load_reduction_pct=0.852524 moved_to_echo=31043 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4290, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 115, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5803, 8192, 8192, 7911, 11041, 8192, 8189, 8016] avg=8192.0000 max=11041 max_rank=4 min=5803 min_rank=0 imbalance_ratio=1.347778 max_load_reduction=54495 max_load_reduction_pct=0.831528 moved_to_echo=35438 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6690, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 115, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8192, 8192, 8149, 8403, 8192, 8189, 8016] avg=8192.0000 max=8403 max_rank=4 min=8016 min_rank=7 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32800 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7367, 1503, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 8183, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 121, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 0, 2, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8871, 8190, 8197, 8425, 7003, 8191, 8190, 8469] avg=8192.0000 max=8871 max_rank=0 min=7003 min_rank=4 imbalance_ratio=1.082886 max_load_reduction=56665 max_load_reduction_pct=0.864639 moved_to_echo=31853 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6262, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 115, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7775, 8192, 8193, 7539, 9440, 8192, 8189, 8016] avg=8192.0000 max=9440 max_rank=4 min=7539 min_rank=3 imbalance_ratio=1.152344 max_load_reduction=56096 max_load_reduction_pct=0.855957 moved_to_echo=33837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 23040, 42624, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 50176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14080, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 55424, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 8960, 1408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 1408, 0, 0, 0, 0, 0, 128, 0, 64256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 1664, 0, 0, 0, 128, 128, 0, 0, 0, 0, 64000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 53504, 12160, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 1664, 0, 0, 0, 128, 128, 0, 0, 0, 0, 63872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 63232, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 1408, 1024, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 65536, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 4157, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8184, 0, 0, 0, 8177, 0, 0, 32, 0, 8182, 0, 0, 2, 0, 0, 0, 10, 0, 0, 0, 0, 7504, 0, 8155, 0, 0, 0, 2, 5, 0, 11, 0, 4, 0, 0, 8171, 0, 0, 1017, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 7836, 0, 0, 3802, 0, 0, 5, 4, 0, 0, 0, 0, 224, 0, 0, 0, 0, 7, 0, 0, 4, 0, 0, 0, 0, 5, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 4371, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8183, 0, 0, 0, 8177, 0, 0, 36, 0, 8181, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 0, 7357, 0, 8151, 0, 0, 0, 2, 5, 0, 10, 0, 5, 0, 0, 8171, 0, 0, 2413, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 7728, 0, 0, 2451, 0, 0, 6, 4, 0, 0, 0, 0, 224, 0, 0, 0, 1, 7, 0, 0, 4, 0, 0, 0, 0, 5, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 5869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 8183, 0, 0, 4, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 8188, 0, 0, 0, 1, 1, 0, 7, 0, 0, 0, 0, 8190, 0, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8179, 0, 0, 698, 0, 0, 2, 1, 0, 0, 0, 0, 54, 0, 0, 0, 1, 4, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 5832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 8183, 0, 0, 3, 0, 8145, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7982, 0, 8188, 0, 0, 0, 1, 1, 0, 7, 0, 0, 0, 0, 8190, 0, 0, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 1580, 0, 0, 1, 1, 0, 0, 0, 0, 48, 0, 0, 0, 1, 5, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 4157, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 531, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 104, 0, 0, 2, 0, 0, 0, 10, 0, 0, 0, 0, 7504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7908, 0, 0, 0, 2, 5, 0, 11, 0, 4, 0, 0, 0, 0, 0, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 0, 0, 0, 0, 8179, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7232, 0, 0, 3802, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 7, 0, 0, 4, 0, 0, 0, 0, 5, 0, 5, 8074, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4945, 8185, 7652, 8947, 8193, 11047, 8392, 8175] avg=8192.0000 max=11047 max_rank=5 min=4945 min_rank=0 imbalance_ratio=1.348511 max_load_reduction=54489 max_load_reduction_pct=0.831436 moved_to_echo=35807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8984, 8195, 8306, 7944, 8191, 7561, 8161, 8194] avg=8192.0000 max=8984 max_rank=0 min=7561 min_rank=5 imbalance_ratio=1.096680 max_load_reduction=56552 max_load_reduction_pct=0.862915 moved_to_echo=32107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8995, 8194, 8296, 7944, 8191, 7561, 8160, 8195] avg=8192.0000 max=8995 max_rank=0 min=7561 min_rank=5 imbalance_ratio=1.098022 max_load_reduction=56541 max_load_reduction_pct=0.862747 moved_to_echo=32107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8992, 8194, 8300, 7944, 8191, 7561, 8160, 8194] avg=8192.0000 max=8992 max_rank=0 min=7561 min_rank=5 imbalance_ratio=1.097656 max_load_reduction=56544 max_load_reduction_pct=0.862793 moved_to_echo=32106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8993, 8195, 8297, 7944, 8191, 7561, 8161, 8194] avg=8192.0000 max=8993 max_rank=0 min=7561 min_rank=5 imbalance_ratio=1.097778 max_load_reduction=56543 max_load_reduction_pct=0.862778 moved_to_echo=32107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 5832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 560, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 98, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7939, 0, 0, 0, 1, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7540, 0, 0, 1580, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 1, 5, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8039, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6643, 8186, 8090, 8925, 8197, 9123, 8179, 8193] avg=8192.0000 max=9123 max_rank=5 min=6643 min_rank=0 imbalance_ratio=1.113647 max_load_reduction=56413 max_load_reduction_pct=0.860794 moved_to_echo=33692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8990, 8195, 8297, 7944, 8191, 7562, 8163, 8194] avg=8192.0000 max=8990 max_rank=0 min=7562 min_rank=5 imbalance_ratio=1.097412 max_load_reduction=56546 max_load_reduction_pct=0.862823 moved_to_echo=32110 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8994, 8192, 8298, 7944, 8191, 7560, 8160, 8197] avg=8192.0000 max=8994 max_rank=0 min=7560 min_rank=5 imbalance_ratio=1.097900 max_load_reduction=56542 max_load_reduction_pct=0.862762 moved_to_echo=32108 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8957, 8195, 8325, 7661, 8191, 7854, 8159, 8194] avg=8192.0000 max=8957 max_rank=0 min=7661 min_rank=3 imbalance_ratio=1.093384 max_load_reduction=56579 max_load_reduction_pct=0.863327 moved_to_echo=32398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8962, 8192, 8320, 7661, 8191, 7854, 8159, 8197] avg=8192.0000 max=8962 max_rank=0 min=7661 min_rank=3 imbalance_ratio=1.093994 max_load_reduction=56574 max_load_reduction_pct=0.863251 moved_to_echo=32401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8961, 8195, 8321, 7661, 8191, 7854, 8159, 8194] avg=8192.0000 max=8961 max_rank=0 min=7661 min_rank=3 imbalance_ratio=1.093872 max_load_reduction=56575 max_load_reduction_pct=0.863266 moved_to_echo=32398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7854, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8957, 8195, 8321, 7663, 8191, 7855, 8160, 8194] avg=8192.0000 max=8957 max_rank=0 min=7663 min_rank=3 imbalance_ratio=1.093384 max_load_reduction=56579 max_load_reduction_pct=0.863327 moved_to_echo=32400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 4371, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 430, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 127, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 0, 7357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7622, 0, 0, 0, 2, 5, 0, 10, 0, 5, 0, 0, 0, 0, 0, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 0, 0, 0, 0, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7409, 0, 0, 2451, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 1, 7, 0, 0, 4, 0, 0, 0, 0, 5, 0, 5, 8050, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5129, 8186, 7528, 10057, 8192, 9874, 8395, 8175] avg=8192.0000 max=10057 max_rank=3 min=5129 min_rank=0 imbalance_ratio=1.227661 max_load_reduction=55479 max_load_reduction_pct=0.846542 moved_to_echo=34636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 5869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 435, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7650, 0, 0, 0, 1, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7836, 0, 0, 698, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 1, 4, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8020, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6649, 8186, 8078, 9511, 8196, 8538, 8185, 8193] avg=8192.0000 max=9511 max_rank=3 min=6649 min_rank=0 imbalance_ratio=1.161011 max_load_reduction=56025 max_load_reduction_pct=0.854874 moved_to_echo=33112 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8960, 8195, 8321, 7661, 8191, 7854, 8160, 8194] avg=8192.0000 max=8960 max_rank=0 min=7661 min_rank=3 imbalance_ratio=1.093750 max_load_reduction=56576 max_load_reduction_pct=0.863281 moved_to_echo=32399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8961, 8192, 8322, 7661, 8193, 7853, 8159, 8195] avg=8192.0000 max=8961 max_rank=0 min=7661 min_rank=3 imbalance_ratio=1.093872 max_load_reduction=56575 max_load_reduction_pct=0.863266 moved_to_echo=32400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 59136, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 4480, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 60160, 0, 0, 5504, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[63616, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 896, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 59392, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 3584, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[61312, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 1024, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 64512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 62464, 0, 0, 3200, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[384, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64512, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 5, 14, 26, 0, 0, 0, 8185, 0, 8093, 3, 0, 0, 0, 0, 0, 2, 0, 4, 2, 6, 0, 0, 8169, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 5, 0, 0, 14, 0, 0, 0, 0, 0, 0, 31, 66, 0, 0, 0, 0, 0, 4, 5, 0, 3, 0, 0, 8059, 0, 5, 1, 6, 0, 0, 8081, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 8189, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8187, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 3, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 8180, 0, 1, 0, 0, 0, 0, 8070, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 5, 14, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8093, 3, 0, 0, 0, 0, 0, 2, 0, 4, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 14, 0, 0, 0, 0, 0, 0, 31, 66, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 3, 0, 0, 8059, 0, 5, 1, 6, 0, 29, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8181, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8131, 8348, 8251, 8123, 8114, 8186, 8181] avg=8192.0000 max=8348 max_rank=2 min=8114 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 29, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8213, 8153, 8189, 8185, 8220, 8193, 8192] avg=8192.0000 max=8220 max_rank=5 min=8153 min_rank=2 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 29, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8213, 8153, 8190, 8184, 8220, 8193, 8192] avg=8192.0000 max=8220 max_rank=5 min=8153 min_rank=2 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 29, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8213, 8153, 8239, 8135, 8220, 8193, 8192] avg=8192.0000 max=8239 max_rank=3 min=8135 min_rank=4 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 29, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8213, 8153, 8146, 8228, 8220, 8193, 8192] avg=8192.0000 max=8228 max_rank=4 min=8146 min_rank=3 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 3, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 8180, 0, 1, 0, 0, 0, 30, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8127, 8270, 8228, 8226, 8102, 8192, 8203] avg=8192.0000 max=8270 max_rank=2 min=8102 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 29, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8213, 8153, 8149, 8225, 8220, 8193, 8192] avg=8192.0000 max=8225 max_rank=4 min=8149 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 29, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8213, 8153, 8144, 8230, 8220, 8193, 8192] avg=8192.0000 max=8230 max_rank=4 min=8144 min_rank=3 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 5, 15, 29, 0, 0, 0, 8184, 0, 8087, 3, 0, 0, 0, 0, 0, 2, 0, 4, 2, 6, 0, 0, 8169, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 5, 0, 0, 14, 0, 0, 0, 0, 0, 0, 5, 143, 0, 0, 0, 0, 0, 5, 5, 0, 3, 0, 0, 7980, 0, 5, 1, 5, 0, 0, 8106, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8188, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8187, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 11, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 8173, 0, 1, 0, 0, 0, 0, 8081, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 49, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8214, 8155, 8145, 8231, 8215, 8193, 8193] avg=8192.0000 max=8231 max_rank=4 min=8145 min_rank=3 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 49, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8214, 8155, 8235, 8141, 8215, 8193, 8193] avg=8192.0000 max=8235 max_rank=3 min=8141 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 5, 15, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8087, 3, 0, 0, 0, 0, 0, 2, 0, 4, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 14, 0, 0, 0, 0, 0, 0, 5, 143, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 3, 0, 0, 7980, 0, 5, 1, 5, 0, 49, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8182, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8126, 8356, 8280, 8067, 8136, 8187, 8181] avg=8192.0000 max=8356 max_rank=2 min=8067 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32571 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 49, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8214, 8155, 8261, 8115, 8215, 8193, 8193] avg=8192.0000 max=8261 max_rank=3 min=8115 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 49, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8214, 8155, 8135, 8241, 8215, 8193, 8193] avg=8192.0000 max=8241 max_rank=4 min=8135 min_rank=3 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 49, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8214, 8155, 8144, 8232, 8215, 8193, 8193] avg=8192.0000 max=8232 max_rank=4 min=8144 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 11, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 8173, 0, 1, 0, 0, 0, 55, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8188, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8126, 8250, 8215, 8254, 8110, 8191, 8197] avg=8192.0000 max=8254 max_rank=4 min=8110 min_rank=5 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 49, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8214, 8155, 8121, 8255, 8215, 8193, 8193] avg=8192.0000 max=8255 max_rank=4 min=8121 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[65280, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 512, 0, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65408, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 384, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 0, 65408, 128, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 65280, 0, 128, 128, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65408, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[65280, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 65024, 0, 128, 128, 128, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True - [2026-08-26 16:18:07] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 439.8 | throughput per GPU (TFLOP/s/GPU): 411.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.881598E+00 | loss scale: 1.0 | grad norm: 5.595 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1012, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2437, 1, 0, 2409, 0, 1, 1, 8190, 0, 8190, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 1, 0, 2, 4169, 0, 0, 2, 5738, 0, 6686, 2, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 7927, 0, 0, 0, 1, 4, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7655, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 668, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1977, 0, 0, 3166, 3, 0, 0, 8191, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 5418, 0, 6858, 3, 0, 0, 3362, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 0, 0, 0, 0, 0, 0, 48, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7794, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 1135, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 2085, 0, 0, 3175, 0, 0, 0, 8191, 0, 8191, 1, 0, 1, 0, 5, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 5153, 0, 6978, 1, 0, 1, 2753, 0, 0, 0, 0, 1, 0, 0, 0, 7890, 0, 1, 0, 0, 0, 2, 117, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7669, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 34, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1448, 0, 0, 0, 3, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1851, 0, 0, 3172, 0, 0, 0, 8192, 2, 8192, 1, 0, 0, 0, 5, 0, 0, 1, 0, 0, 5, 0, 1, 2, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 0, 0, 0, 0, 4068, 0, 0, 0, 5362, 0, 6782, 4, 0, 1, 2548, 0, 0, 0, 0, 0, 0, 0, 0, 7928, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7604, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 3, 0, 0, 0, 0, 18, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 778, 0, 1, 0, 1, 0, 0, 0, 1, 2, 2, 0, 0, 0, 0, 2314, 0, 0, 2412, 0, 0, 0, 8191, 0, 8189, 0, 0, 1, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 5, 0, 0, 0, 4002, 0, 0, 0, 5921, 0, 6851, 18, 0, 0, 3018, 0, 0, 0, 1, 4, 1, 0, 0, 7663, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7739, 0, 0, 16, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 42, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 3, 0, 2, 1, 737, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 1, 3, 0, 0, 2154, 0, 0, 3413, 6, 0, 0, 8191, 0, 8189, 1, 4, 0, 4, 0, 0, 1, 0, 0, 1, 2, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 4011, 0, 0, 0, 5447, 0, 6779, 3, 0, 1, 2507, 0, 0, 1, 0, 0, 3, 0, 0, 8067, 0, 2, 0, 1, 0, 0, 110, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 38, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1039, 1, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2210, 0, 0, 2831, 0, 0, 0, 8192, 2, 8192, 3, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 0, 0, 0, 0, 0, 3615, 0, 0, 0, 5875, 0, 6994, 24, 1, 0, 2454, 0, 0, 0, 0, 0, 6, 0, 0, 7872, 0, 4, 0, 0, 3, 0, 280, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7669, 0, 0, 2, 0, 0, 0, 2, 0, 3, 0, 6, 0, 0, 0, 0, 0, 49, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1599, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2195, 0, 0, 2708, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 0, 5423, 0, 6529, 0, 0, 0, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1012, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3737, 3512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2437, 1, 0, 2409, 0, 1, 1, 0, 0, 3123, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4169, 0, 0, 2, 1499, 0, 0, 2, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7927, 0, 0, 0, 1, 4, 0, 39, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7655, 0, 0, 4, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 1, 6686, 1330, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 7983, 8182, 8200, 8481, 8156, 8192, 8080] avg=8192.0000 max=8481 max_rank=4 min=7983 min_rank=1 imbalance_ratio=1.035278 max_load_reduction=57055 max_load_reduction_pct=0.870590 moved_to_echo=32909 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 1135, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 3737, 3153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2085, 0, 0, 3175, 0, 0, 0, 0, 0, 3124, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8171, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 1347, 0, 0, 1, 0, 1, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7890, 0, 1, 0, 0, 0, 2, 117, 1, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7669, 0, 0, 0, 0, 6, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 34, 0, 0, 0, 0, 6978, 1330, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8031, 8389, 8193, 8180, 8056, 8178, 8155, 8354] avg=8192.0000 max=8389 max_rank=1 min=8031 min_rank=0 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32743 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 778, 0, 1, 0, 1, 0, 0, 0, 1, 2, 2, 3736, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2314, 0, 0, 2412, 0, 0, 0, 0, 0, 3123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4002, 0, 0, 0, 1546, 0, 0, 18, 0, 0, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 0, 0, 7663, 0, 0, 0, 0, 0, 0, 166, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7739, 0, 0, 16, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 42, 0, 0, 0, 0, 6851, 1330, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 7849, 8194, 8189, 8589, 8026, 8306, 8237] avg=8192.0000 max=8589 max_rank=4 min=7849 min_rank=1 imbalance_ratio=1.048462 max_load_reduction=56947 max_load_reduction_pct=0.868942 moved_to_echo=33158 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 668, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 3737, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1977, 0, 0, 3166, 3, 0, 0, 0, 0, 3124, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 1, 0, 0, 2, 1, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 1416, 0, 0, 3, 0, 0, 3362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 0, 0, 0, 0, 0, 0, 48, 1, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7794, 0, 0, 0, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 6858, 1330, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7731, 8272, 8190, 8195, 8582, 8007, 8300, 8259] avg=8192.0000 max=8582 max_rank=4 min=7731 min_rank=0 imbalance_ratio=1.047607 max_load_reduction=56954 max_load_reduction_pct=0.869049 moved_to_echo=33148 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1448, 0, 0, 0, 3, 0, 0, 0, 0, 0, 9, 3742, 3285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 0, 0, 3172, 0, 0, 0, 0, 2, 3115, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 5, 0, 1, 2, 0, 0, 2, 1, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 1387, 0, 0, 4, 0, 1, 2548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7928, 0, 0, 0, 0, 0, 0, 149, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7604, 0, 0, 2, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 3, 0, 0, 0, 0, 18, 0, 0, 0, 0, 6782, 1335, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8489, 8141, 8200, 8187, 8008, 8253, 8109, 8149] avg=8192.0000 max=8489 max_rank=0 min=8008 min_rank=4 imbalance_ratio=1.036255 max_load_reduction=57047 max_load_reduction_pct=0.870468 moved_to_echo=32519 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1039, 1, 0, 0, 4, 0, 0, 0, 0, 0, 1, 3737, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210, 0, 0, 2831, 0, 0, 0, 0, 2, 3124, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 1, 0, 0, 8, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3615, 0, 0, 0, 1536, 0, 0, 24, 1, 0, 2454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7872, 0, 4, 0, 0, 3, 0, 280, 2, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7669, 0, 0, 2, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 6, 0, 0, 0, 0, 0, 49, 0, 1, 1, 0, 6994, 1331, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8378, 8170, 8198, 8191, 7630, 8356, 8217, 8396] avg=8192.0000 max=8396 max_rank=7 min=7630 min_rank=4 imbalance_ratio=1.024902 max_load_reduction=57140 max_load_reduction_pct=0.871887 moved_to_echo=32599 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 2, 1, 737, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 3736, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 2154, 0, 0, 3413, 6, 0, 0, 0, 0, 3123, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 1, 2, 0, 5, 0, 0, 1, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4011, 0, 0, 0, 1425, 0, 0, 3, 0, 1, 2507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 8067, 0, 2, 0, 1, 0, 0, 110, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 1, 0, 1, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 38, 0, 0, 0, 0, 6779, 1330, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7835, 8705, 8196, 8194, 7948, 8359, 8136, 8163] avg=8192.0000 max=8705 max_rank=1 min=7835 min_rank=0 imbalance_ratio=1.062622 max_load_reduction=56831 max_load_reduction_pct=0.867172 moved_to_echo=32606 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1599, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 3737, 3319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 0, 0, 2708, 0, 0, 0, 0, 0, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 85, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 6529, 1331, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8664, 8027, 8183, 8200, 8242, 8201, 8121, 7898] avg=8192.0000 max=8664 max_rank=0 min=7898 min_rank=7 imbalance_ratio=1.057617 max_load_reduction=56872 max_load_reduction_pct=0.867798 moved_to_echo=32462 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True + [2026-08-26 16:27:15] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 433.9 | throughput per GPU (TFLOP/s/GPU): 416.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.885499E+00 | loss scale: 1.0 | grad norm: 5.600 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1308, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2465, 1, 0, 2412, 0, 1, 1, 8190, 0, 8190, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 1, 0, 2, 3959, 0, 0, 2, 5826, 0, 6931, 3, 0, 0, 3306, 0, 0, 0, 0, 0, 0, 0, 0, 6800, 0, 1, 0, 1, 4, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 930, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2118, 0, 0, 3192, 3, 0, 0, 8191, 0, 8191, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 4, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 0, 5522, 0, 7063, 7, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 6487, 0, 2, 0, 0, 0, 0, 64, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7795, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 1524, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 2225, 0, 0, 3149, 0, 0, 0, 8191, 0, 8191, 1, 0, 1, 0, 6, 0, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 5322, 0, 7165, 3, 0, 1, 3264, 0, 0, 0, 0, 2, 0, 0, 0, 6454, 0, 1, 0, 0, 0, 2, 151, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7648, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 285, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 4, 0, 2, 1, 1010, 0, 0, 0, 24, 0, 0, 0, 0, 1, 0, 1, 3, 0, 0, 2199, 0, 0, 3352, 6, 0, 0, 8191, 0, 8189, 1, 4, 0, 3, 0, 0, 1, 0, 0, 1, 3, 0, 5, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 3785, 0, 0, 0, 5564, 0, 6965, 6, 0, 0, 2925, 0, 0, 1, 0, 0, 3, 0, 0, 7108, 0, 6, 0, 1, 0, 0, 154, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7609, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 208, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 2062, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2234, 0, 0, 2612, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 0, 0, 5518, 0, 6761, 0, 0, 0, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 6713, 0, 5, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1317, 1, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2313, 0, 0, 2782, 0, 0, 0, 8192, 3, 8192, 2, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 2, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 0, 6064, 0, 7192, 29, 2, 0, 2889, 0, 0, 0, 0, 0, 7, 0, 0, 6655, 0, 12, 0, 0, 3, 0, 362, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 2, 0, 0, 0, 2, 0, 3, 0, 5, 0, 0, 0, 0, 0, 245, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1841, 0, 0, 0, 2, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1951, 0, 0, 3126, 0, 0, 0, 8192, 2, 8192, 1, 0, 0, 0, 15, 0, 0, 1, 0, 0, 5, 0, 0, 3, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 0, 0, 0, 0, 3853, 0, 0, 0, 5454, 0, 6962, 6, 0, 0, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 6703, 0, 1, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7568, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 2, 4, 0, 0, 0, 0, 314, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1053, 0, 1, 0, 9, 0, 0, 0, 1, 2, 2, 0, 0, 0, 0, 2434, 0, 0, 2404, 0, 0, 0, 8191, 0, 8189, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 5, 0, 0, 0, 3815, 0, 0, 0, 6101, 0, 7111, 27, 0, 0, 3541, 0, 0, 0, 1, 4, 1, 0, 0, 6308, 0, 1, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7742, 0, 0, 17, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 156, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 1524, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 4212, 2427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2225, 0, 0, 3149, 0, 0, 0, 0, 0, 3066, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 3, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 1109, 0, 0, 3, 0, 1, 3264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6454, 0, 1, 0, 0, 0, 2, 151, 1, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7648, 0, 0, 0, 0, 6, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 285, 0, 0, 0, 0, 7165, 913, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8445, 8193, 8178, 8127, 7892, 8154, 8377] avg=8192.0000 max=8445 max_rank=1 min=7892 min_rank=5 imbalance_ratio=1.030884 max_load_reduction=57091 max_load_reduction_pct=0.871140 moved_to_echo=32550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1308, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4211, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2465, 1, 0, 2412, 0, 1, 1, 0, 0, 3066, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 3959, 0, 0, 2, 1214, 0, 0, 3, 0, 0, 3306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6800, 0, 1, 0, 1, 4, 0, 49, 0, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 0, 4, 0, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 1, 6931, 913, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 7957, 8178, 8200, 8487, 8257, 8189, 8091] avg=8192.0000 max=8487 max_rank=4 min=7957 min_rank=1 imbalance_ratio=1.036011 max_load_reduction=57049 max_load_reduction_pct=0.870499 moved_to_echo=33024 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 930, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4212, 2519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2118, 0, 0, 3192, 3, 0, 0, 0, 0, 3066, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 4, 1, 0, 0, 2, 1, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 0, 1149, 0, 0, 7, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6487, 0, 2, 0, 0, 0, 0, 64, 1, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7795, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 7063, 913, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7674, 8381, 8190, 8196, 8618, 7883, 8314, 8280] avg=8192.0000 max=8618 max_rank=4 min=7674 min_rank=0 imbalance_ratio=1.052002 max_load_reduction=56918 max_load_reduction_pct=0.868500 moved_to_echo=33095 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 2062, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 4212, 2517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2234, 0, 0, 2612, 0, 0, 0, 0, 0, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 0, 0, 1149, 0, 0, 0, 0, 0, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6713, 0, 5, 0, 0, 0, 0, 119, 0, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 6761, 914, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8800, 7913, 8184, 8200, 8269, 8165, 8106, 7899] avg=8192.0000 max=8800 max_rank=0 min=7899 min_rank=7 imbalance_ratio=1.074219 max_load_reduction=56736 max_load_reduction_pct=0.865723 moved_to_echo=32439 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1053, 0, 1, 0, 9, 0, 0, 0, 1, 2, 2, 4210, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2434, 0, 0, 2404, 0, 0, 0, 0, 0, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 7, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3815, 0, 0, 0, 1270, 0, 0, 27, 0, 0, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 0, 0, 6308, 0, 1, 0, 0, 0, 0, 219, 0, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7742, 0, 0, 17, 0, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 156, 0, 0, 0, 0, 7111, 913, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8062, 7904, 8193, 8190, 8658, 8002, 8330, 8197] avg=8192.0000 max=8658 max_rank=4 min=7904 min_rank=1 imbalance_ratio=1.056885 max_load_reduction=56878 max_load_reduction_pct=0.867889 moved_to_echo=33187 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1317, 1, 0, 0, 4, 0, 0, 0, 0, 0, 1, 4212, 2766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2313, 0, 0, 2782, 0, 0, 0, 0, 3, 3066, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 2, 0, 0, 10, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 0, 1263, 0, 0, 29, 2, 0, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 6655, 0, 12, 0, 0, 3, 0, 362, 2, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 2, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 5, 0, 0, 0, 0, 0, 245, 0, 1, 1, 0, 7192, 914, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8302, 8166, 8197, 8190, 7580, 8500, 8226, 8375] avg=8192.0000 max=8500 max_rank=5 min=7580 min_rank=4 imbalance_ratio=1.037598 max_load_reduction=57036 max_load_reduction_pct=0.870300 moved_to_echo=32681 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1841, 0, 0, 0, 2, 0, 0, 0, 0, 0, 9, 4216, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1951, 0, 0, 3126, 0, 0, 0, 0, 2, 3058, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 5, 0, 0, 3, 0, 0, 2, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 1122, 0, 0, 6, 0, 0, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6703, 0, 1, 0, 0, 0, 0, 205, 0, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7568, 0, 0, 1, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 4, 0, 0, 0, 0, 314, 0, 0, 0, 0, 6962, 918, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8561, 8138, 8208, 8187, 7922, 8225, 8083, 8212] avg=8192.0000 max=8561 max_rank=0 min=7922 min_rank=4 imbalance_ratio=1.045044 max_load_reduction=56975 max_load_reduction_pct=0.869370 moved_to_echo=32442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 2, 1, 1010, 0, 0, 0, 24, 0, 0, 0, 0, 1, 0, 4210, 2538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 2199, 0, 0, 3352, 6, 0, 0, 0, 0, 3066, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 3, 0, 5, 0, 0, 1, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3785, 0, 0, 0, 1158, 0, 0, 6, 0, 0, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 7108, 0, 6, 0, 1, 0, 0, 154, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7609, 0, 0, 1, 0, 2, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 208, 0, 0, 0, 0, 6965, 913, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7790, 8632, 8193, 8195, 7875, 8612, 8134, 8105] avg=8192.0000 max=8632 max_rank=1 min=7790 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=32726 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 8448, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 29952, 27264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 17280, 128, 0, 23296, 128, 128, 128, 0, 128, 25088, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 31616, 0, 0, 128, 11648, 0, 0, 128, 128, 128, 22400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 11136, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 33792, 20736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 18048, 128, 0, 23040, 128, 128, 128, 0, 128, 24576, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 29952, 0, 0, 128, 9472, 0, 0, 128, 128, 128, 26240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 128, 0, 128, 384, 0, 128, 128, 128, 54528, 10752, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 61440, 0, 0, 128, 0, 128, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 63104, 0, 128, 0, 128, 128, 128, 1024, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 53248, 0, 128, 0, 128, 128, 128, 1408, 128, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 128, 0, 128, 2048, 0, 128, 128, 128, 56192, 7424, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 61312, 0, 0, 128, 0, 128, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 5, 2, 0, 2, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 5, 1, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 63, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 1, 0, 0, 0, 8181, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 1, 0, 0, 8186, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 8191, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 14, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8188, 0, 111, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8184, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 21, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 64, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 2, 0, 2, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8191, 8191, 8193, 8198, 8190, 8189, 8190] avg=8192.0000 max=8198 max_rank=4 min=8189 min_rank=6 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 8181, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 7, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8193, 8188, 8192, 8199, 8192, 8191, 8197] avg=8192.0000 max=8199 max_rank=4 min=8184 min_rank=0 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32779 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 8186, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8185, 8192, 8198, 8192, 8195, 8189] avg=8192.0000 max=8198 max_rank=4 min=8185 min_rank=2 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 63, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8184, 8203, 8192, 8201, 8193, 8191, 8189] avg=8192.0000 max=8203 max_rank=2 min=8183 min_rank=0 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 1, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8192, 8194, 8196, 8190, 8190, 8188] avg=8192.0000 max=8196 max_rank=4 min=8188 min_rank=7 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32764 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 14, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8200, 8192, 8193, 8182, 8193, 8191, 8190] avg=8192.0000 max=8200 max_rank=1 min=8182 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 111, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8191, 8193, 8187, 8168, 8195, 8200, 8204] avg=8192.0000 max=8204 max_rank=7 min=8168 min_rank=4 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 64, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8193, 8192, 8193, 8194, 8191, 8189, 8189] avg=8192.0000 max=8195 max_rank=0 min=8189 min_rank=6 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 2, 0, 0, 0, 8177, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 6, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 10, 2, 0, 2, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 14, 1, 0, 1, 8177, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 14, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 157, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8189, 0, 209, 0, 0, 0, 7959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 3, 1, 0, 0, 8164, 0, 4, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 8191, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8184, 0, 182, 0, 0, 0, 7973, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8183, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 82, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 2, 0, 0, 0, 8177, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8166, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 6, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8192, 8191, 8193, 8201, 8193, 8171, 8196] avg=8192.0000 max=8201 max_rank=4 min=8171 min_rank=6 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 14, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8167, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8199, 8193, 8192, 8184, 8193, 8193, 8192] avg=8192.0000 max=8199 max_rank=1 min=8184 min_rank=4 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 10, 2, 0, 2, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8166, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8191, 8192, 8192, 8200, 8190, 8168, 8191] avg=8192.0000 max=8212 max_rank=0 min=8168 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=57324 max_load_reduction_pct=0.874695 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 209, 0, 0, 0, 7959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 8165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8193, 8193, 8194, 8197, 8191, 8193, 8190] avg=8192.0000 max=8197 max_rank=4 min=8185 min_rank=0 imbalance_ratio=1.000610 max_load_reduction=57339 max_load_reduction_pct=0.874924 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 14, 1, 0, 1, 8177, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8192, 8193, 8195, 8198, 8190, 8168, 8189] avg=8192.0000 max=8211 max_rank=0 min=8168 min_rank=6 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 3, 1, 0, 0, 8164, 0, 4, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8187, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 17, 8167, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8186, 8192, 8200, 8192, 8192, 8191] avg=8192.0000 max=8200 max_rank=4 min=8186 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 157, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 8175, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8187, 8194, 8192, 8205, 8192, 8204, 8191] avg=8192.0000 max=8205 max_rank=4 min=8171 min_rank=0 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 182, 0, 0, 0, 7973, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 82, 8160, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8191, 8194, 8186, 8151, 8195, 8247, 8196] avg=8192.0000 max=8247 max_rank=6 min=8151 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 640, 128, 0, 128, 64896, 0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 65536, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 256, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 128, 0, 0, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 384, 128, 0, 128, 65280, 0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 128, 0, 0, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 193, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 8189, 2, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 7997, 0, 8190, 0, 2, 0, 0, 2, 0, 0, 0, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8190, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 178, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8013, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 103, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 8088, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8192, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8181, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8129, 58, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 7997, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8328, 8186, 8192, 8196, 8189, 8062] avg=8192.0000 max=8328 max_rank=2 min=8062 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32639 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 58, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8134, 8193, 8191, 8192, 8191, 8252] avg=8192.0000 max=8252 max_rank=7 min=8134 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32826 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8130, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 8013, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8311, 8192, 8193, 8190, 8190, 8075] avg=8192.0000 max=8311 max_rank=2 min=8075 min_rank=7 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32648 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 61, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8185, 8131, 8196, 8200, 8191, 8191, 8250] avg=8192.0000 max=8250 max_rank=7 min=8131 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32832 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8128, 58, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 8088, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8236, 8192, 8189, 8193, 8191, 8150] avg=8192.0000 max=8236 max_rank=2 min=8150 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32723 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 58, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8133, 8192, 8191, 8192, 8195, 8247] avg=8192.0000 max=8247 max_rank=7 min=8133 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32825 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8198, 8129, 8193, 8190, 8191, 8196, 8247] avg=8192.0000 max=8247 max_rank=7 min=8129 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32824 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 58, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8134, 8192, 8190, 8191, 8193, 8253] avg=8192.0000 max=8253 max_rank=7 min=8134 min_rank=2 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32827 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 106, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 8083, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 202, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 8185, 2, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 7985, 0, 8190, 0, 2, 0, 0, 2, 0, 0, 0, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8191, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8190, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 183, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8008, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8180, 0, 8185, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 60, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 8132, 8192, 8191, 8192, 8192, 8256] avg=8192.0000 max=8256 max_rank=7 min=8132 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32831 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8125, 60, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 8083, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8236, 8192, 8189, 8195, 8191, 8148] avg=8192.0000 max=8236 max_rank=2 min=8148 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8125, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8126, 60, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 7985, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8330, 8189, 8192, 8200, 8190, 8053] avg=8192.0000 max=8330 max_rank=2 min=8053 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32635 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8127, 60, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 8008, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8311, 8192, 8192, 8191, 8190, 8073] avg=8192.0000 max=8311 max_rank=2 min=8073 min_rank=7 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 60, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8132, 8192, 8191, 8190, 8195, 8250] avg=8192.0000 max=8250 max_rank=7 min=8132 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 60, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8132, 8194, 8190, 8191, 8191, 8255] avg=8192.0000 max=8255 max_rank=7 min=8132 min_rank=2 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32827 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 61, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8187, 8131, 8192, 8201, 8190, 8191, 8252] avg=8192.0000 max=8252 max_rank=7 min=8131 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32834 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 60, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8197, 8132, 8193, 8190, 8187, 8196, 8249] avg=8192.0000 max=8249 max_rank=7 min=8132 min_rank=2 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32822 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 0, 0, 0, 0, 65152, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 0, 65152, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 65152, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 65024, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[512, 0, 65024, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 65536, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 963, 0, 2, 0, 0, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 113, 8190, 8167, 0, 0, 0, 2, 0, 8183, 8190, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8180, 0, 0, 0, 0, 1, 0, 8101, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 5, 8191, 8189, 0, 0, 0, 0, 0, 8192, 8191, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1798, 0, 1, 0, 0, 0, 0, 8063, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 71, 8191, 8187, 0, 0, 0, 1, 0, 8190, 8190, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 7989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 45, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 0, 1, 0, 0, 0, 0, 8124, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 62, 8191, 8176, 0, 0, 0, 1, 0, 8190, 8183, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8157, 0, 0, 0, 0, 0, 0, 7488, 0, 0, 0, 0, 0, 18, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 16, 8192, 8188, 0, 0, 0, 0, 0, 8192, 8187, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7825, 0, 0, 0, 0, 0, 0, 8012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 29, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5575, 13, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 963, 0, 2, 0, 0, 0, 0, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 8131, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 12845, 8186, 8193, 3469, 8191, 8215, 8248] avg=8192.0000 max=12845 max_rank=1 min=3469 min_rank=4 imbalance_ratio=1.567993 max_load_reduction=52691 max_load_reduction_pct=0.804001 moved_to_echo=28123 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5590, 13, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8180, 0, 0, 0, 0, 1, 0, 2511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8178, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 5699, 8194, 8191, 10701, 8194, 8206, 8159] avg=8192.0000 max=10701 max_rank=4 min=5699 min_rank=1 imbalance_ratio=1.306274 max_load_reduction=54835 max_load_reduction_pct=0.836716 moved_to_echo=35260 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5564, 13, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1798, 0, 1, 0, 0, 0, 0, 2499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 8151, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 12034, 8191, 8192, 4298, 8191, 8215, 8224] avg=8192.0000 max=12034 max_rank=1 min=4298 min_rank=4 imbalance_ratio=1.468994 max_load_reduction=53502 max_load_reduction_pct=0.816376 moved_to_echo=28928 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5606, 13, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 0, 1, 0, 0, 0, 0, 2518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8180, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 8140, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 11452, 8191, 8195, 4903, 8186, 8215, 8204] avg=8192.0000 max=11452 max_rank=1 min=4903 min_rank=4 imbalance_ratio=1.397949 max_load_reduction=54084 max_load_reduction_pct=0.825256 moved_to_echo=29508 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 5515, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 29, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 5765, 8191, 8188, 10662, 8189, 8157, 8192] avg=8192.0000 max=10662 max_rank=4 min=5765 min_rank=1 imbalance_ratio=1.301514 max_load_reduction=54874 max_load_reduction_pct=0.837311 moved_to_echo=35200 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 5529, 13, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7825, 0, 0, 0, 0, 0, 0, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8137, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 6107, 8194, 8192, 10313, 8190, 8162, 8185] avg=8192.0000 max=10313 max_rank=4 min=6107 min_rank=1 imbalance_ratio=1.258911 max_load_reduction=55223 max_load_reduction_pct=0.842636 moved_to_echo=34850 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 5167, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8157, 0, 0, 0, 0, 0, 0, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8138, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 5933, 8198, 8193, 10479, 8204, 8163, 8168] avg=8192.0000 max=10479 max_rank=4 min=5933 min_rank=1 imbalance_ratio=1.279175 max_load_reduction=55057 max_load_reduction_pct=0.840103 moved_to_echo=35014 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5642, 13, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 2534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 5701, 8191, 8192, 10711, 8191, 8203, 8156] avg=8192.0000 max=10711 max_rank=4 min=5701 min_rank=1 imbalance_ratio=1.307495 max_load_reduction=54825 max_load_reduction_pct=0.836563 moved_to_echo=35261 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8187, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 3, 8191, 8189, 1, 0, 0, 0, 0, 8191, 8191, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 12, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4461, 0, 1, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 16, 8191, 8183, 0, 0, 0, 1, 0, 8190, 8190, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 0, 2, 0, 0, 0, 0, 8187, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 20, 8190, 8166, 0, 0, 0, 2, 0, 8179, 8189, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5138, 0, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 7, 8191, 8181, 0, 0, 0, 1, 0, 8190, 8178, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8172, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 19, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 5, 8192, 8188, 0, 0, 0, 0, 0, 8192, 8186, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8189, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 1, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6752, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8187, 0, 0, 0, 0, 1, 0, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8181, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6754, 8192, 8191, 9616, 8195, 8203, 8193] avg=8192.0000 max=9616 max_rank=4 min=6754 min_rank=1 imbalance_ratio=1.173828 max_load_reduction=55920 max_load_reduction_pct=0.853271 moved_to_echo=34207 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6749, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4461, 0, 1, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8181, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 10480, 8192, 8192, 5883, 8189, 8210, 8199] avg=8192.0000 max=10480 max_rank=1 min=5883 min_rank=4 imbalance_ratio=1.279297 max_load_reduction=55056 max_load_reduction_pct=0.840088 moved_to_echo=30481 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 6758, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 22, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6801, 8192, 8191, 9603, 8190, 8171, 8196] avg=8192.0000 max=9603 max_rank=4 min=6801 min_rank=1 imbalance_ratio=1.172241 max_load_reduction=55933 max_load_reduction_pct=0.853470 moved_to_echo=34160 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6750, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 0, 2, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 8164, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 11323, 8183, 8193, 5057, 8192, 8210, 8188] avg=8192.0000 max=11323 max_rank=1 min=5057 min_rank=4 imbalance_ratio=1.382202 max_load_reduction=54213 max_load_reduction_pct=0.827225 moved_to_echo=29647 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6752, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5138, 0, 1, 0, 0, 0, 0, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8179, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 9821, 8192, 8193, 6561, 8181, 8210, 8188] avg=8192.0000 max=9821 max_rank=1 min=6561 min_rank=4 imbalance_ratio=1.198853 max_load_reduction=55715 max_load_reduction_pct=0.850143 moved_to_echo=31140 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 6742, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8172, 0, 0, 0, 0, 0, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8141, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 6798, 8199, 8192, 9592, 8205, 8161, 8191] avg=8192.0000 max=9592 max_rank=4 min=6798 min_rank=1 imbalance_ratio=1.170898 max_load_reduction=55944 max_load_reduction_pct=0.853638 moved_to_echo=34149 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6754, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6757, 8192, 8192, 9612, 8192, 8210, 8190] avg=8192.0000 max=9612 max_rank=4 min=6757 min_rank=1 imbalance_ratio=1.173340 max_load_reduction=55924 max_load_reduction_pct=0.853333 moved_to_echo=34204 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 6752, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8189, 0, 0, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8141, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6802, 8194, 8192, 9612, 8192, 8161, 8191] avg=8192.0000 max=9612 max_rank=4 min=6802 min_rank=1 imbalance_ratio=1.173340 max_load_reduction=55924 max_load_reduction_pct=0.853333 moved_to_echo=34156 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 21248, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 44288, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 45696, 0, 128, 0, 0, 128, 0, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 11392, 0, 128, 0, 0, 0, 0, 256, 0, 0, 0, 128, 0, 0, 54016, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 54272, 0, 128, 0, 0, 128, 0, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 0, 65536, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65408, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 384, 0, 65280, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2094, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 2, 2, 0, 0, 3, 0, 7851, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8187, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 6230, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 138, 0, 2, 0, 0, 0, 0, 0, 0, 8037, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8180, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 8190, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8192, 7542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 1, 1, 0, 0, 1, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 5107, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 45, 0, 1, 0, 0, 0, 0, 0, 0, 8012, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1830, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 8183, 6385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8157, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 1, 1, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 6576, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 52, 0, 1, 0, 0, 0, 0, 0, 0, 8037, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8189, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8192, 7254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6839, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 2, 2, 0, 0, 3, 0, 7851, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 9, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1307, 6230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8178, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 138, 0, 2, 0, 0, 0, 0, 6876, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8935, 8193, 8091, 8178, 7537, 8204, 8306, 8092] avg=8192.0000 max=8935 max_rank=0 min=7537 min_rank=4 imbalance_ratio=1.090698 max_load_reduction=56601 max_load_reduction_pct=0.863663 moved_to_echo=32139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 6835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1306, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6867, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6994, 8192, 8203, 8201, 9351, 8189, 8155, 8251] avg=8192.0000 max=9351 max_rank=4 min=6994 min_rank=0 imbalance_ratio=1.141479 max_load_reduction=56185 max_load_reduction_pct=0.857315 moved_to_echo=33946 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 1, 1, 0, 0, 1, 0, 8089, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1307, 5107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8177, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 45, 0, 1, 0, 0, 0, 0, 6877, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8012, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10046, 8192, 8218, 8200, 6414, 8188, 8212, 8066] avg=8192.0000 max=10046 max_rank=0 min=6414 min_rank=4 imbalance_ratio=1.226318 max_load_reduction=55490 max_load_reduction_pct=0.846710 moved_to_echo=30880 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1298, 7542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6881, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7467, 8192, 8208, 8195, 8849, 8202, 8174, 8249] avg=8192.0000 max=8849 max_rank=4 min=7467 min_rank=0 imbalance_ratio=1.080200 max_load_reduction=56687 max_load_reduction_pct=0.864975 moved_to_echo=33474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 1, 1, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 9, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1307, 6576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8180, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 52, 0, 1, 0, 0, 0, 0, 6877, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8600, 8192, 8209, 8153, 7883, 8189, 8219, 8091] avg=8192.0000 max=8600 max_rank=0 min=7883 min_rank=4 imbalance_ratio=1.049805 max_load_reduction=56936 max_load_reduction_pct=0.868774 moved_to_echo=32382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1307, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6875, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7116, 8192, 8203, 8201, 9206, 8189, 8159, 8270] avg=8192.0000 max=9206 max_rank=4 min=7116 min_rank=0 imbalance_ratio=1.123779 max_load_reduction=56330 max_load_reduction_pct=0.859528 moved_to_echo=33824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8192, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 1306, 6385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6870, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8157, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8604, 8192, 8200, 8207, 7731, 8184, 8146, 8272] avg=8192.0000 max=8604 max_rank=0 min=7731 min_rank=4 imbalance_ratio=1.050293 max_load_reduction=56932 max_load_reduction_pct=0.868713 moved_to_echo=32333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8191, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1308, 7254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6877, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7774, 8191, 8204, 8201, 8565, 8191, 8165, 8245] avg=8192.0000 max=8565 max_rank=4 min=7774 min_rank=0 imbalance_ratio=1.045532 max_load_reduction=56971 max_load_reduction_pct=0.869308 moved_to_echo=33166 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 1, 1, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 7691, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 581, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 46, 0, 1, 0, 0, 0, 0, 0, 0, 8059, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 2, 2, 0, 0, 0, 0, 7782, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8186, 7824, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 538, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 110, 0, 2, 0, 0, 0, 0, 0, 0, 8072, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8192, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8178, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8187, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8191, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8175, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8148, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 1, 1, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 224, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 35, 0, 1, 0, 0, 0, 0, 0, 0, 8015, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8191, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8191, 8219, 8309, 8014, 8191, 8169, 8241] avg=8192.0000 max=8309 max_rank=3 min=8014 min_rank=4 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 2, 2, 0, 0, 0, 0, 7782, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 118, 7824, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8012, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8179, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 110, 0, 2, 0, 0, 0, 0, 8061, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8194, 8044, 7947, 8550, 8199, 8277, 8125] avg=8192.0000 max=8550 max_rank=4 min=7947 min_rank=3 imbalance_ratio=1.043701 max_load_reduction=56986 max_load_reduction_pct=0.869537 moved_to_echo=33151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 1, 1, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 117, 7691, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8013, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8181, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 46, 0, 1, 0, 0, 0, 0, 8064, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8191, 8219, 7814, 8594, 8192, 8214, 8111] avg=8192.0000 max=8594 max_rank=4 min=7814 min_rank=3 imbalance_ratio=1.049072 max_load_reduction=56942 max_load_reduction_pct=0.868866 moved_to_echo=33111 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8000, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8192, 8207, 8308, 8026, 8192, 8168, 8244] avg=8192.0000 max=8308 max_rank=3 min=8026 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8221, 8300, 8010, 8197, 8179, 8247] avg=8192.0000 max=8300 max_rank=3 min=8010 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 8192, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 7997, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8148, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8192, 8199, 8314, 8082, 8186, 8162, 8244] avg=8192.0000 max=8314 max_rank=3 min=8082 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8009, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8191, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8192, 8212, 8309, 8023, 8188, 8168, 8257] avg=8192.0000 max=8309 max_rank=3 min=8023 min_rank=4 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 1, 1, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 118, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8013, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8180, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 35, 0, 1, 0, 0, 0, 0, 8060, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8192, 8215, 8235, 8237, 8191, 8199, 8067] avg=8192.0000 max=8237 max_rank=4 min=8067 min_rank=7 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 1024, 64640, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 256, 0, 128, 0, 0, 0, 0, 64640, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 64128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 65152, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 65536, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 512, 128, 128, 0, 0, 128, 0, 64896, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7504, 0, 778, 0, 0, 3269, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 3648, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8118, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1145, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7298, 0, 1577, 0, 0, 2426, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 3077, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 8190, 0, 0, 0, 1, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1921, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 8162, 0, 498, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 7211, 0, 0, 43, 0, 0, 0, 8192, 0, 0, 0, 7, 8185, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7019, 0, 1171, 0, 0, 3545, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2228, 0, 0, 0, 0, 1, 0, 8171, 0, 0, 2, 0, 8190, 0, 0, 0, 2, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2409, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 56, 0, 0, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7630, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 55, 0, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 7844, 0, 0, 13, 0, 0, 0, 8191, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 379, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7520, 0, 0, 8, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 87, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7847, 0, 0, 18, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5918, 0, 778, 0, 0, 3269, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 3648, 0, 0, 0, 0, 0, 1586, 625, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8118, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8191, 9966, 5889, 8156, 8195, 8655, 8249] avg=8192.0000 max=9966 max_rank=2 min=5889 min_rank=3 imbalance_ratio=1.216553 max_load_reduction=55570 max_load_reduction_pct=0.847931 moved_to_echo=33255 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6455, 0, 55, 0, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 7844, 0, 0, 13, 0, 0, 1730, 625, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8190, 6824, 10225, 8227, 8189, 7511, 8218] avg=8192.0000 max=10225 max_rank=3 min=6824 min_rank=2 imbalance_ratio=1.248169 max_load_reduction=55311 max_load_reduction_pct=0.843979 moved_to_echo=32145 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5755, 0, 1577, 0, 0, 2426, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 3077, 0, 0, 0, 0, 0, 1543, 625, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8193, 9759, 5301, 8156, 8192, 9431, 8249] avg=8192.0000 max=9759 max_rank=2 min=5301 min_rank=3 imbalance_ratio=1.191284 max_load_reduction=55777 max_load_reduction_pct=0.851089 moved_to_echo=34028 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6460, 0, 56, 0, 0, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7630, 0, 0, 0, 0, 0, 1732, 625, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8191, 7063, 9994, 8227, 8194, 7511, 8206] avg=8192.0000 max=9994 max_rank=3 min=7063 min_rank=2 imbalance_ratio=1.219971 max_load_reduction=55542 max_load_reduction_pct=0.847504 moved_to_echo=32138 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5535, 0, 1171, 0, 0, 3545, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2228, 0, 0, 0, 0, 1, 1484, 625, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 2, 0, 0, 0, 0, 0, 2, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8194, 10253, 4347, 8093, 8193, 9918, 8250] avg=8192.0000 max=10253 max_rank=2 min=4347 min_rank=3 imbalance_ratio=1.251587 max_load_reduction=55283 max_load_reduction_pct=0.843552 moved_to_echo=34454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6445, 0, 379, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7520, 0, 0, 8, 0, 0, 1733, 629, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8192, 7322, 9907, 8218, 8194, 7501, 8048] avg=8192.0000 max=9907 max_rank=3 min=7322 min_rank=2 imbalance_ratio=1.209351 max_load_reduction=55629 max_load_reduction_pct=0.848831 moved_to_echo=31961 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6437, 0, 498, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 7211, 0, 0, 43, 0, 0, 1725, 625, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8193, 7483, 9640, 8232, 8186, 7505, 8145] avg=8192.0000 max=9640 max_rank=3 min=7483 min_rank=2 imbalance_ratio=1.176758 max_load_reduction=55896 max_load_reduction_pct=0.852905 moved_to_echo=32068 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6444, 0, 87, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7847, 0, 0, 18, 0, 0, 1728, 625, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8192, 6866, 10233, 8227, 8193, 7504, 8171] avg=8192.0000 max=10233 max_rank=3 min=6866 min_rank=2 imbalance_ratio=1.249146 max_load_reduction=55303 max_load_reduction_pct=0.843857 moved_to_echo=32095 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 256, 0, 128, 0, 0, 0, 128, 55040, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 10496, 55040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 54656, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 65024, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 512, 128, 128, 0, 0, 128, 0, 65024, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 67, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 7817, 0, 0, 11, 0, 0, 0, 8192, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7103, 0, 1195, 0, 0, 1925, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1824, 0, 0, 0, 0, 1, 0, 8171, 0, 0, 2, 0, 8190, 0, 0, 0, 2, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4328, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 464, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7545, 0, 0, 12, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 85, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7869, 0, 0, 25, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7426, 0, 1296, 0, 0, 1390, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 3333, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8118, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2900, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7299, 0, 1015, 0, 0, 1164, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 3289, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3524, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 125, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7727, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 8160, 0, 740, 0, 0, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 6755, 0, 0, 47, 0, 0, 0, 8192, 0, 0, 0, 7, 8185, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6113, 0, 1195, 0, 0, 1925, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1824, 0, 0, 0, 0, 1, 1342, 947, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 2, 0, 0, 0, 0, 0, 2, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8193, 9235, 4127, 8095, 8193, 11176, 8237] avg=8192.0000 max=11176 max_rank=6 min=4127 min_rank=3 imbalance_ratio=1.364258 max_load_reduction=54360 max_load_reduction_pct=0.829468 moved_to_echo=35701 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7044, 0, 67, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 7817, 0, 0, 11, 0, 0, 1342, 1092, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8191, 7449, 10279, 8226, 8187, 6849, 8205] avg=8192.0000 max=10279 max_rank=3 min=6849 min_rank=6 imbalance_ratio=1.254761 max_load_reduction=55257 max_load_reduction_pct=0.843155 moved_to_echo=31467 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6281, 0, 1015, 0, 0, 1164, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 3289, 0, 0, 0, 0, 0, 1342, 974, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8193, 8461, 5653, 8167, 8192, 10373, 8237] avg=8192.0000 max=10373 max_rank=6 min=5653 min_rank=3 imbalance_ratio=1.266235 max_load_reduction=55163 max_load_reduction_pct=0.841721 moved_to_echo=34969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7028, 0, 464, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7545, 0, 0, 12, 0, 0, 1344, 1094, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8192, 7850, 10010, 8213, 8199, 6848, 8071] avg=8192.0000 max=10010 max_rank=3 min=6848 min_rank=6 imbalance_ratio=1.221924 max_load_reduction=55526 max_load_reduction_pct=0.847260 moved_to_echo=31331 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6390, 0, 1296, 0, 0, 1390, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 3333, 0, 0, 0, 0, 0, 1342, 991, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8118, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8191, 9077, 5692, 8154, 8194, 9749, 8238] avg=8192.0000 max=9749 max_rank=6 min=5692 min_rank=3 imbalance_ratio=1.190063 max_load_reduction=55787 max_load_reduction_pct=0.851242 moved_to_echo=34335 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7027, 0, 85, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7869, 0, 0, 25, 0, 0, 1341, 1089, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8192, 7417, 10339, 8226, 8193, 6845, 8175] avg=8192.0000 max=10339 max_rank=3 min=6845 min_rank=6 imbalance_ratio=1.262085 max_load_reduction=55197 max_load_reduction_pct=0.842239 moved_to_echo=31439 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7022, 0, 740, 0, 0, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 6755, 0, 0, 47, 0, 0, 1341, 1089, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8193, 8505, 9264, 8229, 8185, 6845, 8161] avg=8192.0000 max=9264 max_rank=3 min=6845 min_rank=6 imbalance_ratio=1.130859 max_load_reduction=56272 max_load_reduction_pct=0.858643 moved_to_echo=31420 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7049, 0, 125, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7727, 0, 0, 0, 0, 0, 1342, 1093, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8191, 7542, 10172, 8226, 8193, 6851, 8212] avg=8192.0000 max=10172 max_rank=3 min=6851 min_rank=6 imbalance_ratio=1.241699 max_load_reduction=55364 max_load_reduction_pct=0.844788 moved_to_echo=31482 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 46208, 0, 0, 128, 0, 128, 10752, 8448, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 54016, 0, 4992, 0, 0, 6656, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 256, 0, 0, 128, 128, 0, 0, 0, 0, 128, 65280, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 384, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 47104, 0, 0, 128, 0, 128, 13312, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 49536, 0, 4608, 0, 0, 11520, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[5504, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 60160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65152, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 384, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 65152, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 384, 0, 0, 128, 128, 0, 0, 0, 0, 128, 65280, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 353, 2, 2, 8190, 0, 8186, 0, 0, 0, 2, 0, 7972, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7718, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 335] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 1, 0, 0, 0, 15, 0, 0, 8183, 0, 8191, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 232, 1, 5, 8191, 0, 8185, 0, 0, 0, 1, 0, 7999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7918, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 233] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 15, 0, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 6, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 266, 1, 0, 8191, 0, 8190, 0, 0, 0, 1, 0, 7994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7987, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 391, 2, 2, 8190, 0, 8186, 0, 0, 0, 2, 0, 7984, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7743, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 2, 2, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 260] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 1, 0, 0, 0, 20, 0, 0, 8184, 0, 8191, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8188, 2, 0, 0, 0, 8, 0, 0, 8172, 0, 8191, 0, 0, 0, 0, 0, 8188, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 23, 0, 0, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 18, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 2, 2, 0, 0, 0, 0, 0, 0, 2, 0, 7938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7718, 0, 2, 0, 0, 0, 0, 0, 97, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 335, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8297, 7834, 8197, 8189, 8205, 8198, 8427] avg=8192.0000 max=8427 max_rank=7 min=7834 min_rank=2 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=33019 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 5, 0, 0, 0, 1, 96, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 9, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 12, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8171, 8303, 8192, 8193, 8191, 8190, 8108] avg=8192.0000 max=8303 max_rank=2 min=8108 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32682 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 1, 5, 0, 0, 0, 0, 0, 0, 1, 0, 7964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7918, 0, 1, 0, 0, 0, 0, 0, 98, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 233, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8203, 8033, 8194, 8187, 8205, 8198, 8325] avg=8192.0000 max=8325 max_rank=7 min=8033 min_rank=2 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32915 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7987, 0, 1, 0, 0, 0, 0, 0, 96, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8227, 8100, 8194, 8192, 8205, 8198, 8228] avg=8192.0000 max=8228 max_rank=7 min=8100 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32823 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 98, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 4, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 1, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8148, 8308, 8188, 8194, 8167, 8197, 8135] avg=8192.0000 max=8308 max_rank=2 min=8135 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32693 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 6, 0, 0, 3, 0, 97, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 9, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8172, 8300, 8192, 8194, 8175, 8186, 8125] avg=8192.0000 max=8300 max_rank=2 min=8125 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32680 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 23, 0, 0, 14, 6, 97, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 20, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 18, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8161, 8354, 8187, 8193, 8183, 8171, 8094] avg=8192.0000 max=8354 max_rank=2 min=8094 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32641 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 96, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8157, 8304, 8192, 8194, 8205, 8198, 8094] avg=8192.0000 max=8304 max_rank=2 min=8094 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32691 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 272, 1, 0, 8191, 0, 8190, 0, 0, 0, 1, 0, 7987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 243, 1, 5, 8191, 0, 8186, 0, 0, 0, 1, 0, 8014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7946, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 11, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8188, 2, 0, 0, 0, 13, 0, 0, 8172, 0, 8191, 0, 0, 0, 0, 0, 8187, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 25, 0, 0, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 2, 2, 0, 0, 0, 0, 0, 0, 2, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7743, 0, 2, 0, 0, 0, 0, 0, 74, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 2, 2, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 260, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8338, 7843, 8197, 8188, 8206, 8199, 8376] avg=8192.0000 max=8376 max_rank=7 min=7843 min_rank=2 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 74, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8147, 8289, 8192, 8194, 8207, 8199, 8116] avg=8192.0000 max=8289 max_rank=2 min=8116 min_rank=7 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32716 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 0, 1, 74, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 7, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8164, 8296, 8192, 8192, 8186, 8193, 8126] avg=8192.0000 max=8296 max_rank=2 min=8126 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32697 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 1, 0, 0, 0, 0, 0, 74, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8217, 8127, 8194, 8192, 8207, 8199, 8208] avg=8192.0000 max=8217 max_rank=1 min=8127 min_rank=2 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 1, 5, 0, 0, 0, 0, 0, 0, 1, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7946, 0, 1, 0, 0, 0, 0, 0, 74, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8220, 8043, 8195, 8189, 8207, 8199, 8292] avg=8192.0000 max=8292 max_rank=7 min=8043 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32887 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 6, 0, 0, 6, 0, 74, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 8, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8147, 8301, 8192, 8194, 8176, 8186, 8148] avg=8192.0000 max=8301 max_rank=2 min=8147 min_rank=1 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 75, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 5, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8148, 8292, 8186, 8194, 8169, 8191, 8156] avg=8192.0000 max=8292 max_rank=2 min=8148 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32710 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 25, 0, 0, 18, 6, 75, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 19, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8155, 8345, 8188, 8193, 8178, 8170, 8114] avg=8192.0000 max=8345 max_rank=2 min=8114 min_rank=7 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32655 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 64640, 0, 128, 128, 0, 0, 128, 128, 896, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 640, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 64768, 0, 128, 128, 0, 0, 128, 128, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 768, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 896, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 1024, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7961, 0, 0, 8192, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 8189, 0, 0, 8077, 6107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 3, 0, 0, 2085, 0, 0, 8159, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8189, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 39, 0, 8189, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8185, 0, 0, 8153, 6471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1721, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 20, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 8191, 0, 0, 8155, 6525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1666, 0, 0, 8174, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 31, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7811, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7469, 0, 0, 8192, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7753, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 6107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 7414, 676, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2085, 0, 0, 7483, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 39, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8534, 6531, 8106, 9575, 8189, 8192, 8189, 8220] avg=8192.0000 max=9575 max_rank=3 min=6531 min_rank=1 imbalance_ratio=1.168823 max_load_reduction=55961 max_load_reduction_pct=0.853897 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 679, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7948, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8849, 8213, 7512, 8192, 8192, 8192, 8183] avg=8192.0000 max=8849 max_rank=1 min=7512 min_rank=3 imbalance_ratio=1.080200 max_load_reduction=56687 max_load_reduction_pct=0.864975 moved_to_echo=32759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 6471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7484, 679, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1721, 0, 0, 7510, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7949, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 20, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8447, 6899, 8177, 9234, 8191, 8191, 8194, 8203] avg=8192.0000 max=9234 max_rank=3 min=6899 min_rank=1 imbalance_ratio=1.127197 max_load_reduction=56302 max_load_reduction_pct=0.859100 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 6525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7486, 677, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1666, 0, 0, 7497, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7949, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 31, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8455, 6953, 8177, 9166, 8191, 8191, 8190, 8213] avg=8192.0000 max=9166 max_rank=3 min=6953 min_rank=1 imbalance_ratio=1.118896 max_load_reduction=56370 max_load_reduction_pct=0.860138 moved_to_echo=32785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 682, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8045, 8994, 8227, 7510, 8201, 8187, 8195, 8177] avg=8192.0000 max=8994 max_rank=1 min=7510 min_rank=3 imbalance_ratio=1.097900 max_load_reduction=56542 max_load_reduction_pct=0.862762 moved_to_echo=32760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 679, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8909, 8212, 7513, 8190, 8194, 8192, 8180] avg=8192.0000 max=8909 max_rank=1 min=7513 min_rank=3 imbalance_ratio=1.087524 max_load_reduction=56627 max_load_reduction_pct=0.864059 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 679, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7943, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7711, 9341, 8212, 7513, 8189, 8198, 8192, 8180] avg=8192.0000 max=9341 max_rank=1 min=7513 min_rank=3 imbalance_ratio=1.140259 max_load_reduction=56195 max_load_reduction_pct=0.857468 moved_to_echo=32759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 679, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7995, 9060, 8212, 7513, 8193, 8191, 8192, 8180] avg=8192.0000 max=9060 max_rank=1 min=7513 min_rank=3 imbalance_ratio=1.105957 max_load_reduction=56476 max_load_reduction_pct=0.861755 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 8191, 0, 0, 8146, 6471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1720, 0, 0, 8174, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 31, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7610, 0, 0, 8192, 8186, 0, 0, 1, 0, 0, 0, 0, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6850, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 8189, 0, 0, 8060, 6076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 3, 0, 0, 2116, 0, 0, 8161, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8189, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 39, 0, 8189, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7344, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6805, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8185, 0, 0, 8147, 6393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1798, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 24, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6804, 0, 0, 8192, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7478, 698, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1720, 0, 0, 7476, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 31, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8896, 6484, 8191, 9199, 8173, 8191, 8190, 8212] avg=8192.0000 max=9199 max_rank=3 min=6484 min_rank=1 imbalance_ratio=1.122925 max_load_reduction=56337 max_load_reduction_pct=0.859634 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8186, 0, 0, 1, 0, 0, 0, 0, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7477, 699, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8179, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8782, 8190, 7492, 8219, 8190, 8193, 8186] avg=8192.0000 max=8782 max_rank=1 min=7492 min_rank=3 imbalance_ratio=1.072021 max_load_reduction=56754 max_load_reduction_pct=0.865997 moved_to_echo=32788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 7477, 697, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2116, 0, 0, 7464, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 39, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8981, 6089, 8191, 9587, 8088, 8192, 8189, 8219] avg=8192.0000 max=9587 max_rank=3 min=6089 min_rank=1 imbalance_ratio=1.170288 max_load_reduction=55949 max_load_reduction_pct=0.853714 moved_to_echo=32688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7479, 699, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7524, 9547, 8191, 7493, 8218, 8191, 8192, 8180] avg=8192.0000 max=9547 max_rank=1 min=7493 min_rank=3 imbalance_ratio=1.165405 max_load_reduction=55989 max_load_reduction_pct=0.854324 moved_to_echo=32781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7476, 699, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8018, 9053, 8188, 7493, 8218, 8194, 8192, 8180] avg=8192.0000 max=9053 max_rank=1 min=7493 min_rank=3 imbalance_ratio=1.105103 max_load_reduction=56483 max_load_reduction_pct=0.861862 moved_to_echo=32784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7483, 704, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7472, 9586, 8208, 7488, 8224, 8190, 8195, 8173] avg=8192.0000 max=9586 max_rank=1 min=7472 min_rank=0 imbalance_ratio=1.170166 max_load_reduction=55950 max_load_reduction_pct=0.853729 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7478, 699, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1798, 0, 0, 7490, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 24, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8884, 6406, 8192, 9291, 8174, 8190, 8193, 8206] avg=8192.0000 max=9291 max_rank=3 min=6406 min_rank=1 imbalance_ratio=1.134155 max_load_reduction=56245 max_load_reduction_pct=0.858231 moved_to_echo=32763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7473, 699, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8179, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7477, 9589, 8185, 7493, 8222, 8198, 8192, 8180] avg=8192.0000 max=9589 max_rank=1 min=7477 min_rank=0 imbalance_ratio=1.170532 max_load_reduction=55947 max_load_reduction_pct=0.853683 moved_to_echo=32792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 60032, 5504, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 3456, 60160, 0, 0, 128, 0, 0, 0, 0, 0, 0, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 5504, 0, 0, 60160, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 63488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 63616, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 60032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 59904, 0, 0, 128, 0, 0, 0, 0, 0, 0, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 59904, 5632, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 5760, 0, 0, 59904, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 7788, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 6869, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 1, 2, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 1, 1, 7092, 0, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 7854, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7381, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 3, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 1, 6332, 0, 2880, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 7343, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 7835, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 6, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2, 2, 5256, 0, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8153, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 8120, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8169, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7343, 0, 0, 0, 0, 0, 0, 194, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 7835, 329, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 6, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 7664, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4902, 0, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7549, 8182, 8187, 8299, 8196, 8191, 8203, 8729] avg=8192.0000 max=8729 max_rank=7 min=7549 min_rank=0 imbalance_ratio=1.065552 max_load_reduction=56807 max_load_reduction_pct=0.866806 moved_to_echo=33319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7854, 0, 0, 0, 0, 0, 0, 194, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7381, 329, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 3, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 7665, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5905, 0, 2880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8052, 7712, 8189, 8205, 8196, 8191, 8204, 8787] avg=8192.0000 max=8787 max_rank=7 min=7712 min_rank=1 imbalance_ratio=1.072632 max_load_reduction=56749 max_load_reduction_pct=0.865921 moved_to_echo=33378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1, 7556, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8389, 8478, 8191, 8114, 8192, 8193, 8201, 7778] avg=8192.0000 max=8478 max_rank=1 min=7778 min_rank=7 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=32364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7788, 0, 0, 0, 0, 0, 0, 194, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 6869, 329, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 1, 2, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 7664, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 6613, 0, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7987, 7209, 8189, 8354, 8196, 8189, 8208, 9204] avg=8192.0000 max=9204 max_rank=7 min=7209 min_rank=1 imbalance_ratio=1.123535 max_load_reduction=56332 max_load_reduction_pct=0.859558 moved_to_echo=33797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 7602, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7628, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8382, 8506, 8189, 8176, 8202, 8191, 8208, 7682] avg=8192.0000 max=8506 max_rank=1 min=7682 min_rank=7 imbalance_ratio=1.038330 max_load_reduction=57030 max_load_reduction_pct=0.870209 moved_to_echo=32283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 7540, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8387, 8493, 8193, 8113, 8191, 8192, 8198, 7769] avg=8192.0000 max=8493 max_rank=1 min=7769 min_rank=7 imbalance_ratio=1.036743 max_load_reduction=57043 max_load_reduction_pct=0.870407 moved_to_echo=32350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 7524, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8404, 8444, 8207, 8105, 8169, 8197, 8111, 7899] avg=8192.0000 max=8444 max_rank=1 min=7899 min_rank=7 imbalance_ratio=1.030762 max_load_reduction=57092 max_load_reduction_pct=0.871155 moved_to_echo=32376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7628, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7639, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8386, 8512, 8191, 8170, 8194, 8192, 8203, 7688] avg=8192.0000 max=8512 max_rank=1 min=7688 min_rank=7 imbalance_ratio=1.039062 max_load_reduction=57024 max_load_reduction_pct=0.870117 moved_to_echo=32277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 6034, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7885, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 1, 3, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 1, 1, 7293, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8172, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8059, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8151, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 5659, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8142, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 5, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 2, 5639, 0, 4760, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 6251, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8065, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 4, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 6632, 0, 3500, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8169, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8175, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 8082, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6034, 0, 0, 0, 0, 0, 0, 836, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7885, 90, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 1, 3, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 7185, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 6345, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6875, 7988, 8190, 8275, 8273, 8187, 8266, 9482] avg=8192.0000 max=9482 max_rank=7 min=6875 min_rank=0 imbalance_ratio=1.157471 max_load_reduction=56054 max_load_reduction_pct=0.855316 moved_to_echo=34208 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8172, 0, 0, 0, 0, 0, 0, 836, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1, 7185, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7125, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9009, 8148, 8192, 8187, 8138, 8193, 8269, 7400] avg=8192.0000 max=9009 max_rank=0 min=7400 min_rank=7 imbalance_ratio=1.099731 max_load_reduction=56527 max_load_reduction_pct=0.862534 moved_to_echo=32000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6251, 0, 0, 0, 0, 0, 0, 836, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8065, 90, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 4, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 7185, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5769, 0, 3500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7091, 8159, 8190, 8106, 8273, 8190, 8256, 9271] avg=8192.0000 max=9271 max_rank=7 min=7091 min_rank=0 imbalance_ratio=1.131714 max_load_reduction=56265 max_load_reduction_pct=0.858536 moved_to_echo=33990 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7186, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7126, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9025, 8277, 8192, 8173, 8186, 8192, 8266, 7225] avg=8192.0000 max=9025 max_rank=0 min=7225 min_rank=7 imbalance_ratio=1.101685 max_load_reduction=56511 max_load_reduction_pct=0.862289 moved_to_echo=31869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5659, 0, 0, 0, 0, 0, 0, 836, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8142, 90, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 5, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 7183, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4906, 0, 4760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6508, 8255, 8188, 8208, 8274, 8186, 8247, 9670] avg=8192.0000 max=9670 max_rank=7 min=6508 min_rank=0 imbalance_ratio=1.180420 max_load_reduction=55866 max_load_reduction_pct=0.852448 moved_to_echo=34377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8151, 0, 0, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 7189, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7117, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8978, 8275, 8185, 8199, 8204, 8202, 8050, 7443] avg=8192.0000 max=8978 max_rank=0 min=7443 min_rank=7 imbalance_ratio=1.095947 max_load_reduction=56558 max_load_reduction_pct=0.863007 moved_to_echo=31899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8169, 0, 0, 0, 0, 0, 0, 836, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 7185, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7979, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7126, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9006, 8264, 8193, 8196, 8149, 8192, 8251, 7285] avg=8192.0000 max=9006 max_rank=0 min=7285 min_rank=7 imbalance_ratio=1.099365 max_load_reduction=56530 max_load_reduction_pct=0.862579 moved_to_echo=31877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 834, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 2, 7166, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7126, 0, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9044, 8170, 8206, 8192, 8039, 8194, 7931, 7760] avg=8192.0000 max=9044 max_rank=0 min=7760 min_rank=7 imbalance_ratio=1.104004 max_load_reduction=56492 max_load_reduction_pct=0.862000 moved_to_echo=31924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 512, 128, 128, 0, 0, 0, 0, 0, 128, 256, 0, 0, 0, 0, 128, 60928, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 62976, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64000, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 55680, 0, 9984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 64000, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 58880, 0, 0, 0, 0, 0, 0, 6784, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 512, 128, 128, 0, 0, 0, 0, 0, 128, 256, 0, 0, 0, 0, 128, 57472, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 52736, 0, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64256, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 64896, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[65024, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65536, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 65536, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7926, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8183, 7, 2, 0, 8190, 0, 8179, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 1, 0, 0, 2, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 3, 2, 8189, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8092, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7923, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 4, 1, 0, 8191, 0, 8187, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 0, 1, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8171, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8151, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 1, 0, 8191, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 1, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8165, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8182, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8191, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 8177, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 1, 0, 0, 2, 0, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 8108, 64, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 3, 2, 8098, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8092, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8002, 8190, 8135, 8194, 8225, 8191, 8487, 8112] avg=8192.0000 max=8487 max_rank=6 min=8002 min_rank=0 imbalance_ratio=1.036011 max_load_reduction=57049 max_load_reduction_pct=0.870499 moved_to_echo=33015 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7923, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 8185, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 1, 0, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 0, 1, 8099, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8171, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7999, 8192, 8191, 8193, 8188, 8192, 8392, 8189] avg=8192.0000 max=8392 max_rank=6 min=7999 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=57144 max_load_reduction_pct=0.871948 moved_to_echo=32961 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 69, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8189, 8210, 8191, 8191, 8192, 8087, 8213] avg=8192.0000 max=8263 max_rank=0 min=8087 min_rank=6 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32683 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 1, 0, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8109, 64, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 1, 8098, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8165, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8191, 8180, 8200, 8188, 8192, 8169, 8190] avg=8192.0000 max=8226 max_rank=0 min=8169 min_rank=6 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32739 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 64, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8190, 8205, 8189, 8184, 8192, 8101, 8208] avg=8192.0000 max=8267 max_rank=0 min=8101 min_rank=6 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32685 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8195, 8205, 8191, 8192, 8192, 8096, 8208] avg=8192.0000 max=8257 max_rank=0 min=8096 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32688 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8190, 8205, 8189, 8182, 8194, 8101, 8208] avg=8192.0000 max=8267 max_rank=0 min=8101 min_rank=6 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32685 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8180, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 8192, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 64, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8199, 8205, 8189, 8186, 8191, 8103, 8208] avg=8192.0000 max=8255 max_rank=0 min=8103 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32688 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4028, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8183, 7, 2, 0, 8190, 0, 8178, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 1, 0, 0, 2, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4205, 0, 2, 2, 8189, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8135, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4484, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 1, 0, 8191, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 1, 1, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8170, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6646, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1544, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6514, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5802, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 8192, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2380, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5235, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4166, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 4, 1, 0, 8191, 0, 8186, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 1, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8174, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7307, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 885, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 5516, 2654, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 1, 1, 5510, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8170, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7151, 8191, 8179, 8202, 8189, 8193, 9241, 8190] avg=8192.0000 max=9241 max_rank=6 min=7151 min_rank=0 imbalance_ratio=1.128052 max_load_reduction=56295 max_load_reduction_pct=0.858994 moved_to_echo=33813 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 8176, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 1, 0, 0, 2, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 5516, 2653, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4205, 0, 2, 2, 5510, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8135, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6696, 8189, 8102, 8195, 8236, 8191, 9779, 8148] avg=8192.0000 max=9779 max_rank=6 min=6696 min_rank=0 imbalance_ratio=1.193726 max_load_reduction=55757 max_load_reduction_pct=0.850784 moved_to_echo=34354 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5521, 2654, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2380, 0, 0, 0, 5510, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8472, 8200, 8211, 8190, 8180, 8191, 7891, 8201] avg=8192.0000 max=8472 max_rank=0 min=7891 min_rank=6 imbalance_ratio=1.034180 max_load_reduction=57064 max_load_reduction_pct=0.870728 moved_to_echo=32463 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5522, 2654, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 0, 0, 0, 5512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9184, 8190, 8211, 8189, 8178, 8193, 7190, 8201] avg=8192.0000 max=9184 max_rank=0 min=7190 min_rank=6 imbalance_ratio=1.121094 max_load_reduction=56352 max_load_reduction_pct=0.859863 moved_to_echo=31762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5522, 2654, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1544, 0, 0, 0, 5512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9316, 8192, 8211, 8189, 8179, 8192, 7056, 8201] avg=8192.0000 max=9316 max_rank=0 min=7056 min_rank=6 imbalance_ratio=1.137207 max_load_reduction=56220 max_load_reduction_pct=0.857849 moved_to_echo=31628 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4166, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 8184, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5520, 2654, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 1, 5510, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8174, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6836, 8191, 8193, 8192, 8187, 8192, 9560, 8185] avg=8192.0000 max=9560 max_rank=6 min=6836 min_rank=0 imbalance_ratio=1.166992 max_load_reduction=55976 max_load_reduction_pct=0.854126 moved_to_echo=34124 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5523, 2659, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 885, 0, 0, 0, 5495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9976, 8187, 8218, 8190, 8184, 8192, 6380, 8209] avg=8192.0000 max=9976 max_rank=0 min=6380 min_rank=6 imbalance_ratio=1.217773 max_load_reduction=55560 max_load_reduction_pct=0.847778 moved_to_echo=30965 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5522, 2652, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 0, 0, 0, 5508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7905, 8196, 8211, 8189, 8203, 8192, 8439, 8201] avg=8192.0000 max=8439 max_rank=6 min=7905 min_rank=0 imbalance_ratio=1.030151 max_load_reduction=57097 max_load_reduction_pct=0.871231 moved_to_echo=33035 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 65024, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65024, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[768, 0, 128, 128, 64896, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65408, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 44288, 21248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 44288, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 21376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[21504, 0, 128, 128, 44160, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 0, 0, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 1, 0, 0, 3, 0, 0, 9, 0, 6117, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 1804, 0, 8187, 0, 0, 0, 1, 1, 0, 8189, 0, 0, 0, 0, 8183, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8180, 0, 0, 8189, 0, 0, 3, 1, 0, 0, 0, 0, 131, 0, 0, 0, 0, 29, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8, 0, 7098, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 996, 0, 8187, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2845, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5353, 0, 8192, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 7091, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 938, 0, 8190, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8191, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 407, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7788, 0, 8192, 0, 0, 1, 0, 0, 0, 8183, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7582, 0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 381, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7861, 0, 8192, 0, 0, 6, 0, 0, 0, 8135, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 414, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7778, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8187, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 6117, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8180, 0, 0, 4, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 29, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 7976, 49, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8193, 7986, 8407, 8185, 8190, 8193, 8191] avg=8192.0000 max=8407 max_rank=3 min=7986 min_rank=2 imbalance_ratio=1.026245 max_load_reduction=57129 max_load_reduction_pct=0.871719 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7091, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8190, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8032, 49, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8086, 8256, 8196, 8188, 8233, 8192] avg=8192.0000 max=8256 max_rank=3 min=8086 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 1, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 7788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 49, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8194, 8253, 8135, 8191, 8196, 8181, 8193] avg=8192.0000 max=8253 max_rank=2 min=8135 min_rank=3 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 7098, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8037, 49, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 8159, 8232, 8192, 8188, 8195, 8189] avg=8192.0000 max=8232 max_rank=3 min=8159 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32764 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 7582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 52, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8186, 8248, 8136, 8193, 8192, 8191, 8192] avg=8192.0000 max=8248 max_rank=2 min=8136 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2845, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 5353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 49, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8193, 8255, 8135, 8194, 8194, 8181, 8193] avg=8192.0000 max=8255 max_rank=2 min=8135 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 1, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 7861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 49, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8198, 8300, 8092, 8193, 8192, 8181, 8193] avg=8192.0000 max=8300 max_rank=2 min=8092 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 7778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 49, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 8249, 8143, 8192, 8196, 8181, 8193] avg=8192.0000 max=8249 max_rank=2 min=8143 min_rank=3 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8123, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 52, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8175, 0, 8192, 0, 0, 6, 0, 0, 0, 8148, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 3, 0, 0, 9, 0, 5364, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 2723, 0, 8187, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8157, 0, 8191, 0, 0, 1, 0, 0, 0, 8182, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8152, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8189, 0, 0, 0, 15, 0, 0, 9, 0, 3321, 0, 0, 0, 0, 0, 0, 8017, 0, 0, 0, 0, 4582, 0, 8186, 0, 0, 0, 1, 1, 0, 8189, 0, 0, 0, 0, 8184, 0, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8180, 0, 0, 8189, 0, 0, 3, 1, 0, 0, 0, 0, 146, 0, 0, 0, 0, 28, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8163, 0, 8192, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 4519, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 3496, 0, 8190, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8191, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8256, 8138, 8192, 8196, 8181, 8193] avg=8192.0000 max=8256 max_rank=2 min=8138 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 5364, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 2723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8025, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 8159, 8231, 8189, 8187, 8196, 8192] avg=8192.0000 max=8231 max_rank=3 min=8159 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32764 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8189, 8261, 8129, 8192, 8196, 8181, 8192] avg=8192.0000 max=8261 max_rank=2 min=8129 min_rank=3 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8185, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3321, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 4582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8180, 0, 0, 4, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 28, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 7950, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8203, 7974, 8413, 8188, 8190, 8188, 8189] avg=8192.0000 max=8413 max_rank=3 min=7974 min_rank=2 imbalance_ratio=1.026978 max_load_reduction=57123 max_load_reduction_pct=0.871628 moved_to_echo=32755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4519, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8016, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8078, 8263, 8193, 8187, 8238, 8193] avg=8192.0000 max=8263 max_rank=3 min=8078 min_rank=2 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32811 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 56, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8187, 8252, 8133, 8193, 8192, 8190, 8192] avg=8192.0000 max=8252 max_rank=2 min=8133 min_rank=3 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8263, 8129, 8192, 8196, 8181, 8193] avg=8192.0000 max=8263 max_rank=2 min=8129 min_rank=3 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 2, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8196, 8293, 8100, 8197, 8192, 8181, 8192] avg=8192.0000 max=8293 max_rank=2 min=8100 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 13568, 128, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 51584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64640, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 25088, 128, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 40192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64768, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 29, 0, 0, 2, 14, 201, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 8163, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 8188, 7696, 0, 0, 277, 1, 0, 0, 2, 0, 2, 0, 0, 8188, 0, 2, 1, 0, 0, 0, 8047, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8161, 0, 14, 28, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8189, 0, 0, 116, 0, 0, 1, 2, 89, 0, 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 8190, 7862, 0, 1, 113, 0, 0, 0, 1, 0, 1, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 8091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8176, 0, 4, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 63, 0, 0, 1, 1, 74, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8158, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 8190, 7916, 0, 71, 27, 0, 0, 0, 1, 0, 1, 0, 0, 8188, 0, 1, 0, 1, 0, 0, 8124, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8176, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8167, 0, 12, 16, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8185, 0, 0, 41, 0, 0, 2, 7, 192, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 8161, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 8188, 7894, 0, 0, 90, 1, 0, 0, 2, 0, 2, 0, 0, 8188, 0, 2, 1, 0, 0, 0, 8094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8175, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8116, 0, 0, 29, 0, 0, 2, 14, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 8087, 73, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 101, 7696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 1, 0, 0, 2, 0, 2, 0, 0, 8115, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8182, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8362, 8187, 8173, 7945, 8400, 8089, 8188, 8192] avg=8192.0000 max=8400 max_rank=4 min=7945 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=57136 max_load_reduction_pct=0.871826 moved_to_echo=32869 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 73, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8200, 8198, 8277, 8129, 8223, 8194, 8192] avg=8192.0000 max=8277 max_rank=3 min=8123 min_rank=0 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32738 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8119, 0, 0, 116, 0, 0, 1, 2, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8089, 73, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 101, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 113, 0, 0, 0, 1, 0, 1, 0, 0, 8118, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8327, 8209, 8192, 8061, 8235, 8132, 8187, 8193] avg=8192.0000 max=8327 max_rank=0 min=8061 min_rank=3 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32747 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 80, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8200, 8204, 8275, 8128, 8235, 8201, 8192] avg=8192.0000 max=8275 max_rank=3 min=8101 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32756 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8116, 0, 0, 63, 0, 0, 1, 1, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8089, 73, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 101, 7916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 27, 0, 0, 0, 1, 0, 1, 0, 0, 8115, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8188, 8171, 8155, 8217, 8165, 8194, 8191] avg=8192.0000 max=8255 max_rank=0 min=8155 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32767 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 73, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 16, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8184, 8198, 8268, 8147, 8232, 8193, 8192] avg=8192.0000 max=8268 max_rank=3 min=8122 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32764 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 73, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 28, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8184, 8202, 8262, 8161, 8228, 8183, 8192] avg=8192.0000 max=8262 max_rank=3 min=8124 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32764 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 73, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8184, 8198, 8293, 8119, 8232, 8196, 8192] avg=8192.0000 max=8293 max_rank=3 min=8119 min_rank=4 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32739 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8169, 0, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8175, 0, 5, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8165, 0, 16, 23, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8189, 0, 0, 142, 0, 0, 1, 1, 73, 0, 0, 0, 1, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 8190, 7955, 0, 0, 15, 0, 0, 0, 1, 0, 1, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 8117, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 78, 0, 0, 1, 0, 66, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8147, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 8190, 7981, 0, 31, 3, 0, 0, 0, 1, 0, 1, 0, 0, 8188, 0, 1, 0, 1, 0, 0, 8145, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8113, 0, 0, 41, 0, 0, 2, 7, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 8119, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 69, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 1, 0, 0, 2, 0, 2, 0, 0, 8160, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8355, 8190, 8174, 8055, 8258, 8122, 8190, 8192] avg=8192.0000 max=8355 max_rank=0 min=8055 min_rank=3 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32762 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8184, 8201, 8261, 8164, 8219, 8196, 8192] avg=8192.0000 max=8261 max_rank=3 min=8119 min_rank=0 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32771 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8201, 8201, 8244, 8172, 8212, 8194, 8192] avg=8192.0000 max=8244 max_rank=3 min=8120 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32770 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8184, 8201, 8238, 8190, 8219, 8193, 8192] avg=8192.0000 max=8238 max_rank=3 min=8119 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32794 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8117, 0, 0, 142, 0, 0, 1, 1, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8121, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 69, 7955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1, 0, 1, 0, 0, 8163, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8334, 8213, 8194, 8089, 8181, 8144, 8188, 8193] avg=8192.0000 max=8334 max_rank=0 min=8089 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32706 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8193, 8202, 8242, 8170, 8229, 8199, 8192] avg=8192.0000 max=8242 max_rank=3 min=8109 min_rank=0 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32790 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8114, 0, 0, 78, 0, 0, 1, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8121, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 69, 7981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3, 0, 0, 0, 1, 0, 1, 0, 0, 8160, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8187, 8162, 8173, 8198, 8172, 8194, 8191] avg=8192.0000 max=8259 max_rank=0 min=8162 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32755 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 23, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8184, 8201, 8234, 8203, 8219, 8182, 8192] avg=8192.0000 max=8234 max_rank=3 min=8121 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32796 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 896, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 65024, 0, 0, 256, 0, 0, 128, 128, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 64768, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 512, 128, 0, 0, 128, 0, 128, 0, 0, 65024, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 65024, 0, 0, 384, 0, 0, 128, 128, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 65024, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 256, 128, 0, 0, 128, 0, 128, 0, 0, 65408, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 640, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True - [2026-08-26 16:18:07] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 449.3 | throughput per GPU (TFLOP/s/GPU): 402.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.636795E+00 | loss scale: 1.0 | grad norm: 5.491 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1283, 0, 0, 0, 0, 0, 3, 0, 0, 7, 11, 1, 0, 0, 0, 3197, 0, 0, 3394, 3, 0, 0, 8190, 1, 8191, 4, 0, 1, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 1, 1, 0, 5095, 0, 6185, 0, 1, 0, 3219, 0, 0, 0, 0, 0, 1, 0, 0, 7839, 0, 2, 0, 0, 3, 0, 304, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7559, 0, 1, 12, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 35, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 13, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2865, 0, 5, 2959, 8, 1, 0, 8192, 0, 8191, 1, 0, 1, 0, 0, 0, 5, 2, 0, 0, 2, 1, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 3, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 5001, 0, 5557, 5, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 1, 0, 7554, 0, 0, 0, 1, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 32, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 1015, 0, 0, 0, 9, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 2606, 0, 0, 2769, 3, 0, 0, 8192, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 1, 0, 1, 18, 0, 1, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 3587, 0, 0, 0, 5482, 0, 6005, 2, 0, 0, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 7902, 0, 8, 0, 0, 3, 0, 235, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7583, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 157, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3, 2319, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 1, 1, 3, 2090, 0, 0, 3716, 7, 0, 0, 8186, 0, 8185, 0, 0, 1, 0, 0, 2, 0, 0, 0, 3, 2, 0, 1, 0, 1, 0, 6, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8186, 0, 1, 0, 0, 0, 1, 0, 0, 3291, 0, 0, 0, 4703, 0, 5900, 8, 3, 3, 2884, 0, 1, 4, 0, 2, 0, 1, 0, 7806, 0, 0, 1, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7619, 1, 0, 3, 0, 3, 0, 1, 0, 1, 1, 8, 1, 0, 0, 0, 1, 23, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 1174, 0, 1, 1, 3, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 2992, 1, 0, 3226, 0, 0, 0, 8190, 0, 8190, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 2, 0, 3598, 0, 0, 0, 5159, 0, 5919, 11, 1, 0, 2821, 0, 0, 0, 0, 0, 0, 0, 0, 7815, 0, 3, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7618, 0, 0, 1, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 49, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 2742, 0, 0, 2347, 0, 0, 0, 8192, 0, 8190, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 4, 0, 0, 0, 0, 2823, 0, 0, 0, 5763, 0, 6354, 3, 1, 1, 3708, 0, 0, 0, 0, 1, 1, 0, 0, 7934, 0, 1, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7488, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 162, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1240, 1, 1, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 2767, 1, 0, 2748, 0, 0, 0, 8189, 0, 8188, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 0, 3058, 0, 2, 0, 5181, 0, 5977, 5, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 2, 1, 0, 0, 0, 161, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7609, 0, 0, 12, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 3, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1727, 0, 0, 0, 3, 0, 0, 3, 0, 3, 1, 5, 0, 0, 0, 2877, 0, 0, 2884, 0, 0, 0, 8191, 0, 8188, 2, 0, 0, 1, 0, 2, 1, 0, 0, 0, 2, 0, 0, 1, 0, 1, 5, 1, 2, 1, 1, 2, 0, 1, 0, 0, 0, 2, 2, 8178, 0, 0, 0, 0, 1, 0, 0, 0, 2967, 0, 1, 0, 5319, 0, 5912, 7, 0, 0, 3137, 0, 0, 0, 0, 0, 0, 1, 0, 8009, 0, 0, 0, 1, 0, 0, 339, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1283, 0, 0, 0, 0, 0, 3, 0, 0, 7, 11, 3666, 3054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3197, 0, 0, 3394, 3, 0, 0, 0, 1, 2413, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 28, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8184, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 1, 1, 0, 1513, 0, 0, 0, 1, 0, 3219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7799, 0, 2, 0, 0, 3, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7559, 0, 1, 12, 0, 1, 528, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 35, 3, 0, 0, 0, 6185, 2112, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8030, 9013, 8207, 8189, 7494, 8109, 8140, 8354] avg=8192.0000 max=9013 max_rank=1 min=7494 min_rank=4 imbalance_ratio=1.100220 max_load_reduction=56523 max_load_reduction_pct=0.862473 moved_to_echo=32097 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 13, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3666, 2997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2865, 0, 5, 2959, 8, 1, 0, 0, 0, 2413, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 2, 0, 0, 2, 1, 0, 3, 0, 0, 4, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 1486, 0, 0, 5, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7515, 0, 0, 0, 1, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7650, 0, 0, 0, 0, 0, 518, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 32, 0, 3, 0, 0, 5557, 2112, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7851, 8252, 8194, 8190, 9189, 7933, 8206, 7721] avg=8192.0000 max=9189 max_rank=4 min=7721 min_rank=7 imbalance_ratio=1.121704 max_load_reduction=56347 max_load_reduction_pct=0.859787 moved_to_echo=33049 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 1015, 0, 0, 0, 9, 0, 0, 0, 0, 2, 5, 3667, 3286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 0, 0, 2769, 3, 0, 0, 0, 0, 2412, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 18, 0, 1, 0, 0, 0, 11, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3587, 0, 0, 0, 1628, 0, 0, 2, 0, 0, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7862, 0, 8, 0, 0, 3, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7583, 0, 0, 6, 0, 0, 568, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 157, 0, 0, 0, 0, 6005, 2113, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7995, 7800, 8209, 8191, 8744, 8108, 8198, 8291] avg=8192.0000 max=8744 max_rank=4 min=7800 min_rank=1 imbalance_ratio=1.067383 max_load_reduction=56792 max_load_reduction_pct=0.866577 moved_to_echo=33341 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 1174, 0, 1, 1, 3, 0, 0, 0, 0, 0, 4, 3671, 3095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2992, 1, 0, 3226, 0, 0, 0, 0, 0, 2404, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3598, 0, 0, 0, 1523, 0, 0, 11, 1, 0, 2821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7770, 0, 3, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7618, 0, 0, 1, 0, 0, 541, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 49, 0, 0, 0, 2, 5919, 2115, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7954, 8628, 8181, 8196, 7957, 8312, 8203, 8105] avg=8192.0000 max=8628 max_rank=1 min=7954 min_rank=0 imbalance_ratio=1.053223 max_load_reduction=56908 max_load_reduction_pct=0.868347 moved_to_echo=32577 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3, 2319, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 3664, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 2090, 0, 0, 3716, 7, 0, 0, 0, 0, 2410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 3, 2, 0, 1, 0, 1, 0, 6, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8186, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3291, 0, 0, 0, 1397, 0, 0, 8, 3, 3, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 2, 0, 1, 0, 7766, 0, 0, 1, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7619, 1, 0, 3, 0, 3, 487, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 8, 1, 0, 0, 0, 1, 23, 0, 0, 0, 0, 5900, 2111, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8812, 8228, 8186, 8192, 7587, 8316, 8152, 8063] avg=8192.0000 max=8812 max_rank=0 min=7587 min_rank=4 imbalance_ratio=1.075684 max_load_reduction=56724 max_load_reduction_pct=0.865540 moved_to_echo=32118 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3666, 3454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2742, 0, 0, 2347, 0, 0, 0, 0, 0, 2412, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2823, 0, 0, 0, 1712, 0, 0, 3, 1, 1, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 7893, 0, 1, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7488, 0, 0, 2, 1, 0, 597, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 162, 1, 0, 0, 0, 6354, 2112, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 7506, 8183, 8193, 8248, 8346, 8128, 8647] avg=8192.0000 max=8647 max_rank=7 min=7506 min_rank=1 imbalance_ratio=1.055542 max_load_reduction=56889 max_load_reduction_pct=0.868057 moved_to_echo=33369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1240, 1, 1, 3, 0, 0, 0, 1, 0, 0, 0, 3665, 3105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2767, 1, 0, 2748, 0, 0, 0, 0, 0, 2411, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 9, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 8185, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3058, 0, 2, 0, 1539, 0, 0, 5, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7939, 0, 2, 1, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7609, 0, 0, 12, 0, 0, 537, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 3, 0, 1, 5977, 2112, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8016, 7930, 8187, 8194, 8624, 8103, 8200, 8282] avg=8192.0000 max=8624 max_rank=4 min=7930 min_rank=1 imbalance_ratio=1.052734 max_load_reduction=56912 max_load_reduction_pct=0.868408 moved_to_echo=33209 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1727, 0, 0, 0, 3, 0, 0, 3, 0, 3, 1, 3665, 3188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2877, 0, 0, 2884, 0, 0, 0, 0, 0, 2411, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 0, 0, 0, 2, 0, 0, 1, 0, 1, 5, 1, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 2, 0, 1, 0, 0, 0, 2, 2, 8178, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2967, 0, 1, 0, 1580, 0, 0, 7, 0, 0, 3137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7968, 0, 0, 0, 1, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7718, 0, 0, 0, 0, 0, 551, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 5912, 2112, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8593, 8179, 8189, 8191, 7693, 8309, 8309, 8073] avg=8192.0000 max=8593 max_rank=0 min=7693 min_rank=4 imbalance_ratio=1.048950 max_load_reduction=56943 max_load_reduction_pct=0.868881 moved_to_echo=32384 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 + [2026-08-26 16:27:15] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 434.4 | throughput per GPU (TFLOP/s/GPU): 416.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.641932E+00 | loss scale: 1.0 | grad norm: 5.495 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3, 2823, 0, 0, 0, 13, 0, 1, 0, 0, 0, 0, 0, 1, 1, 3, 2223, 0, 0, 3584, 7, 0, 0, 8186, 0, 8185, 0, 0, 1, 0, 1, 2, 0, 0, 0, 3, 2, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8186, 0, 1, 0, 0, 0, 1, 0, 0, 3129, 0, 0, 0, 4683, 0, 6156, 10, 3, 4, 3289, 0, 1, 4, 0, 2, 0, 0, 0, 6609, 0, 3, 1, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7591, 1, 0, 3, 0, 3, 0, 1, 0, 1, 1, 8, 1, 0, 0, 0, 1, 224, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 16, 1529, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 0, 5, 2865, 8, 1, 0, 8192, 0, 8191, 1, 0, 1, 0, 0, 0, 5, 2, 0, 0, 2, 1, 0, 3, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 3, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 5054, 0, 5910, 14, 0, 0, 4113, 0, 0, 0, 0, 0, 0, 1, 0, 6188, 0, 2, 0, 1, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 229, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2158, 0, 0, 0, 3, 0, 0, 4, 0, 3, 1, 5, 0, 0, 0, 2946, 0, 0, 2688, 0, 0, 0, 8191, 0, 8188, 2, 0, 0, 1, 1, 2, 1, 0, 0, 0, 2, 0, 0, 1, 0, 1, 6, 1, 2, 1, 1, 2, 0, 0, 0, 0, 0, 2, 2, 8177, 0, 0, 0, 0, 1, 0, 0, 0, 2761, 0, 1, 0, 5297, 0, 6180, 11, 0, 0, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 7113, 0, 1, 0, 1, 0, 0, 393, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 2845, 0, 0, 2301, 0, 0, 0, 8192, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 4, 0, 0, 0, 0, 2674, 0, 0, 0, 5809, 0, 6556, 6, 1, 1, 4247, 0, 0, 0, 0, 1, 1, 0, 0, 6685, 0, 2, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7463, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 302, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1697, 0, 0, 0, 0, 0, 3, 0, 0, 6, 11, 1, 0, 0, 0, 3266, 0, 1, 3291, 3, 0, 0, 8190, 1, 8191, 5, 0, 1, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 2593, 1, 1, 0, 5113, 0, 6439, 2, 1, 0, 3650, 0, 0, 0, 0, 0, 1, 0, 0, 6643, 0, 5, 0, 0, 3, 0, 366, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7545, 0, 1, 10, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 259, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1531, 1, 1, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 2810, 1, 0, 2660, 0, 0, 0, 8189, 0, 8188, 2, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 3, 0, 0, 0, 0, 2899, 0, 1, 1, 5160, 0, 6235, 6, 0, 0, 4458, 0, 0, 0, 0, 0, 0, 0, 0, 7110, 0, 3, 1, 0, 0, 0, 198, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7592, 0, 0, 12, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 3, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1531, 0, 1, 1, 3, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 3097, 1, 0, 3180, 0, 0, 0, 8190, 0, 8190, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 3389, 0, 0, 0, 5198, 0, 6206, 17, 1, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 6535, 0, 6, 0, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7590, 0, 0, 1, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 311, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 1336, 0, 0, 0, 8, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 2733, 0, 0, 2711, 3, 0, 0, 8192, 0, 8192, 9, 0, 0, 0, 1, 0, 0, 1, 0, 2, 19, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 3407, 0, 0, 0, 5507, 0, 6261, 3, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 6759, 0, 16, 0, 0, 2, 0, 280, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7556, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 294, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3, 2823, 0, 0, 0, 13, 0, 1, 0, 0, 0, 0, 4108, 2058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 2223, 0, 0, 3584, 7, 0, 0, 0, 0, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 0, 0, 0, 3, 2, 0, 0, 0, 1, 0, 6, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8186, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3129, 0, 0, 0, 1138, 0, 0, 10, 3, 4, 3289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 2, 0, 0, 0, 6609, 0, 3, 1, 0, 0, 0, 568, 0, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7591, 1, 0, 3, 0, 3, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 8, 1, 0, 0, 0, 1, 224, 0, 0, 0, 0, 6156, 1665, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9007, 8231, 8185, 8192, 7574, 8129, 8143, 8075] avg=8192.0000 max=9007 max_rank=0 min=7574 min_rank=4 imbalance_ratio=1.099487 max_load_reduction=56529 max_load_reduction_pct=0.862564 moved_to_echo=31921 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 16, 1529, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4111, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 0, 5, 2865, 8, 1, 0, 0, 0, 2414, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 2, 0, 0, 2, 1, 0, 3, 0, 0, 5, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 1229, 0, 0, 14, 0, 0, 4113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6188, 0, 2, 0, 1, 0, 0, 461, 0, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7632, 0, 0, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 229, 0, 3, 0, 0, 5910, 1666, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7879, 8261, 8193, 8189, 9299, 7668, 8220, 7827] avg=8192.0000 max=9299 max_rank=4 min=7668 min_rank=5 imbalance_ratio=1.135132 max_load_reduction=56237 max_load_reduction_pct=0.858109 moved_to_echo=33014 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2158, 0, 0, 0, 3, 0, 0, 4, 0, 3, 1, 4110, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2946, 0, 0, 2688, 0, 0, 0, 0, 0, 2413, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 1, 0, 0, 0, 2, 0, 0, 1, 0, 1, 6, 1, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 2, 0, 0, 0, 0, 0, 2, 2, 8177, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2761, 0, 1, 0, 1288, 0, 0, 11, 0, 0, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7113, 0, 1, 0, 1, 0, 0, 393, 0, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0, 6180, 1665, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8610, 8054, 8189, 8189, 7575, 8572, 8273, 8074] avg=8192.0000 max=8610 max_rank=0 min=7575 min_rank=4 imbalance_ratio=1.051025 max_load_reduction=56926 max_load_reduction_pct=0.868622 moved_to_echo=32494 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1697, 0, 0, 0, 0, 0, 3, 0, 0, 6, 11, 4111, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3266, 0, 1, 3291, 3, 0, 0, 0, 1, 2414, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 31, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8185, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2593, 1, 1, 0, 1243, 0, 0, 2, 1, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6643, 0, 5, 0, 0, 3, 0, 366, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7545, 0, 1, 10, 0, 1, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 259, 3, 0, 0, 0, 6439, 1666, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 8982, 8208, 8190, 7491, 8045, 8152, 8387] avg=8192.0000 max=8982 max_rank=1 min=7491 min_rank=4 imbalance_ratio=1.096436 max_load_reduction=56554 max_load_reduction_pct=0.862946 moved_to_echo=32075 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1531, 1, 1, 3, 0, 0, 0, 1, 0, 0, 0, 4110, 2268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2810, 1, 0, 2660, 0, 0, 0, 0, 0, 2413, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 9, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2899, 0, 1, 1, 1254, 0, 0, 6, 0, 0, 4458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7110, 0, 3, 1, 0, 0, 0, 198, 0, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7592, 0, 0, 12, 0, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 3, 0, 1, 6235, 1665, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7915, 7888, 8186, 8194, 8619, 8349, 8206, 8179] avg=8192.0000 max=8619 max_rank=4 min=7888 min_rank=1 imbalance_ratio=1.052124 max_load_reduction=56917 max_load_reduction_pct=0.868484 moved_to_echo=33353 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1531, 0, 1, 1, 3, 0, 0, 0, 0, 0, 4, 4116, 2288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3097, 1, 0, 3180, 0, 0, 0, 0, 0, 2404, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3389, 0, 0, 0, 1249, 0, 0, 17, 1, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6535, 0, 6, 0, 0, 0, 0, 646, 0, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7590, 0, 0, 1, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 311, 0, 0, 0, 2, 6206, 1670, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7950, 8687, 8181, 8198, 7875, 8236, 8199, 8210] avg=8192.0000 max=8687 max_rank=1 min=7875 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=56849 max_load_reduction_pct=0.867447 moved_to_echo=32520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 1336, 0, 0, 0, 8, 0, 0, 0, 0, 2, 5, 4112, 2420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2733, 0, 0, 2711, 3, 0, 0, 0, 0, 2414, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 2, 19, 0, 1, 0, 0, 0, 14, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3407, 0, 0, 0, 1340, 0, 0, 3, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6759, 0, 16, 0, 0, 2, 0, 280, 0, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7556, 0, 0, 3, 0, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 294, 0, 0, 0, 0, 6261, 1666, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7897, 7870, 8213, 8191, 8762, 8163, 8201, 8239] avg=8192.0000 max=8762 max_rank=4 min=7870 min_rank=1 imbalance_ratio=1.069580 max_load_reduction=56774 max_load_reduction_pct=0.866302 moved_to_echo=33365 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4111, 2553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2845, 0, 0, 2301, 0, 0, 0, 0, 0, 2413, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2674, 0, 0, 0, 1412, 0, 0, 6, 1, 1, 4247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6685, 0, 2, 0, 0, 0, 0, 518, 0, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7463, 0, 0, 2, 1, 0, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 302, 1, 0, 0, 0, 6556, 1666, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7563, 8181, 8193, 8341, 8374, 8142, 8545] avg=8192.0000 max=8545 max_rank=7 min=7563 min_rank=1 imbalance_ratio=1.043091 max_load_reduction=56991 max_load_reduction_pct=0.869614 moved_to_echo=33402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 128, 128, 11136, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 29440, 25088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 22144, 128, 128, 24064, 128, 128, 0, 0, 128, 19328, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 26112, 128, 128, 0, 12416, 0, 0, 128, 128, 128, 27008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 0, 0, 128, 128, 768, 128, 128, 0, 128, 47872, 17024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 60928, 128, 128, 128, 128, 128, 4352, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 128, 0, 62592, 0, 128, 128, 128, 128, 0, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 0, 128, 128, 128, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 128, 128, 14208, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 32896, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 22912, 128, 128, 23296, 128, 128, 0, 0, 128, 19328, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 24832, 128, 128, 128, 10240, 0, 0, 128, 128, 128, 30592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 0, 0, 0, 128, 2176, 128, 128, 0, 128, 50048, 13440, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 128, 0, 53760, 0, 128, 128, 128, 128, 0, 3456, 0, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 0, 128, 128, 60672, 128, 128, 128, 128, 128, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 0, 0, 128, 128, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8159, 0, 0, 0, 0, 0, 8188, 1, 74, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8182, 0, 0, 0, 0, 0, 8189, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8183, 4, 176, 2, 0, 0, 8012, 0, 0, 0, 0, 0, 0, 1, 0, 5, 1, 0, 2, 0, 8187, 8186, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8184, 0, 0, 0, 0, 0, 5, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8164, 0, 0, 0, 0, 0, 8189, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8181, 0, 4, 2, 0, 2, 8186, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 8190, 2, 0, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 20, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8150, 0, 64, 0, 0, 0, 8154, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 8192, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8184, 0, 5, 1, 1, 0, 8187, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 8188, 8189, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8189, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8171, 1, 74, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 4, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 8159, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8187, 8136, 8192, 8199, 8190, 8192, 8193] avg=8192.0000 max=8247 max_rank=0 min=8136 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32774 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8172, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 4, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 8182, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8192, 8193, 8191, 8200, 8190, 8194, 8198] avg=8192.0000 max=8200 max_rank=4 min=8178 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32782 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8172, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 4, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 8164, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8196, 8203, 8191, 8191, 8197, 8191, 8191] avg=8192.0000 max=8203 max_rank=2 min=8176 min_rank=0 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32770 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 64, 0, 0, 0, 8137, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8150, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8189, 8199, 8191, 8199, 8191, 8194, 8166] avg=8192.0000 max=8207 max_rank=0 min=8166 min_rank=7 imbalance_ratio=1.001831 max_load_reduction=57329 max_load_reduction_pct=0.874771 moved_to_echo=32750 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 176, 2, 0, 0, 7995, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 8183, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 5, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 4, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8195, 8199, 8195, 8176, 8192, 8195, 8201] avg=8192.0000 max=8201 max_rank=7 min=8176 min_rank=4 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32764 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 2, 0, 2, 8162, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 2, 0, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 7, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 8181, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8197, 8205, 8189, 8195, 8194, 8189, 8195] avg=8192.0000 max=8205 max_rank=2 min=8172 min_rank=0 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32773 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 1, 1, 0, 8170, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 8184, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8189, 8198, 8196, 8195, 8190, 8190, 8195] avg=8192.0000 max=8198 max_rank=2 min=8183 min_rank=0 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32770 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 20, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 4, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 8203, 8191, 8181, 8192, 8191, 8197] avg=8192.0000 max=8203 max_rank=2 min=8181 min_rank=4 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32761 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8181, 4, 223, 2, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 1, 0, 5, 1, 0, 2, 0, 8186, 8186, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8184, 0, 0, 0, 0, 0, 6, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 1, 0, 0, 7, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8152, 0, 0, 0, 0, 0, 8184, 1, 99, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8162, 0, 0, 0, 0, 0, 8188, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8124, 0, 177, 0, 0, 0, 8066, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8180, 0, 0, 0, 0, 0, 8189, 0, 6, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8189, 8191, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8183, 0, 6, 1, 1, 0, 8187, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 8188, 8189, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8189, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8179, 0, 7, 2, 0, 2, 8185, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 8190, 2, 0, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 24, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 223, 2, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 8184, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 6, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 4, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 1, 0, 0, 7, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8181, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8191, 8200, 8195, 8176, 8192, 8198, 8204] avg=8192.0000 max=8204 max_rank=7 min=8176 min_rank=4 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32770 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 177, 0, 0, 0, 8043, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8124, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8190, 8200, 8192, 8197, 8191, 8194, 8145] avg=8192.0000 max=8227 max_rank=0 min=8145 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32727 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8165, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 8162, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8193, 8204, 8192, 8191, 8197, 8190, 8197] avg=8192.0000 max=8204 max_rank=2 min=8172 min_rank=0 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32775 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8161, 1, 99, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 8152, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8189, 8126, 8193, 8197, 8190, 8191, 8188] avg=8192.0000 max=8262 max_rank=0 min=8126 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32766 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8166, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8187, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 8180, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8194, 8194, 8191, 8199, 8191, 8191, 8203] avg=8192.0000 max=8203 max_rank=7 min=8173 min_rank=0 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32784 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 1, 1, 0, 8164, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 8183, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8192, 8198, 8195, 8194, 8190, 8190, 8199] avg=8192.0000 max=8199 max_rank=7 min=8178 min_rank=0 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32773 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 2, 0, 2, 8150, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8180, 0, 2, 0, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 5, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 8179, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8194, 8210, 8186, 8203, 8193, 8189, 8198] avg=8192.0000 max=8210 max_rank=2 min=8163 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32783 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8193, 8204, 8192, 8179, 8192, 8193, 8202] avg=8192.0000 max=8204 max_rank=2 min=8179 min_rank=4 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32766 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 65536, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 0, 128, 128, 128, 128, 128, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 65536, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 384, 128, 128, 128, 65152, 128, 128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 0, 128, 128, 128, 128, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 65536, 128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[65536, 0, 128, 0, 128, 0, 0, 128, 128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 512, 128, 128, 128, 65024, 128, 128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 71, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 4062, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8192, 0, 4656, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 549, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 8189, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 30, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 7067, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 562, 0, 8190, 0, 2, 0, 0, 1, 0, 0, 0, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 50, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 149, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2519, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 121, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 2308, 0, 8190, 0, 1, 0, 1, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3015, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8183, 0, 192, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8183, 1, 0, 0, 6, 8150, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6338, 0, 0, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8157, 0, 1706, 0, 8184, 1, 6, 0, 0, 1, 0, 2, 0, 8185, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 3866, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3834, 0, 2833, 117, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5358, 0, 4062, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8143, 8194, 8312, 6892, 8191, 9422] avg=8192.0000 max=9422 max_rank=7 min=6892 min_rank=5 imbalance_ratio=1.150146 max_load_reduction=56114 max_load_reduction_pct=0.856232 moved_to_echo=32817 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 2834, 117, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5358, 0, 4656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8072, 8195, 8261, 6414, 8197, 10014] avg=8192.0000 max=10014 max_rank=7 min=6414 min_rank=5 imbalance_ratio=1.222412 max_load_reduction=55522 max_load_reduction_pct=0.847198 moved_to_echo=32886 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 149, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8084, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5472, 0, 2833, 117, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5358, 0, 2519, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8121, 8196, 8239, 8530, 8190, 7878] avg=8192.0000 max=8530 max_rank=5 min=7878 min_rank=7 imbalance_ratio=1.041260 max_load_reduction=57006 max_load_reduction_pct=0.869843 moved_to_echo=32837 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5081, 0, 2834, 117, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5358, 0, 3015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8192, 8072, 8195, 8177, 8139, 8191, 8373] avg=8192.0000 max=8373 max_rank=7 min=8072 min_rank=2 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32880 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 30, 0, 0, 0, 0, 0, 1, 0, 2, 0, 8084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 7067, 0, 2831, 119, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5341, 0, 562, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8197, 8607, 8197, 8121, 10127, 8189, 5906] avg=8192.0000 max=10127 max_rank=5 min=5906 min_rank=7 imbalance_ratio=1.236206 max_load_reduction=55409 max_load_reduction_pct=0.845474 moved_to_echo=32343 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8150, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 7, 0, 8077, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6338, 0, 2822, 116, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5335, 0, 1706, 0, 0, 1, 6, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8185, 8257, 8167, 8098, 9388, 8196, 7051] avg=8192.0000 max=9388 max_rank=5 min=7051 min_rank=7 imbalance_ratio=1.145996 max_load_reduction=56148 max_load_reduction_pct=0.856750 moved_to_echo=32733 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5742, 0, 2833, 116, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5358, 0, 2308, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8196, 8192, 8197, 8104, 8800, 8191, 7668] avg=8192.0000 max=8800 max_rank=5 min=7668 min_rank=7 imbalance_ratio=1.074219 max_load_reduction=56736 max_load_reduction_pct=0.865723 moved_to_echo=32763 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4189, 0, 2834, 117, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5358, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8072, 8195, 8224, 7246, 8191, 9224] avg=8192.0000 max=9224 max_rank=7 min=7246 min_rank=5 imbalance_ratio=1.125977 max_load_reduction=56312 max_load_reduction_pct=0.859253 moved_to_echo=32885 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 61, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 460, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 554, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 64, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 746, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8192, 0, 1024, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 172, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8178, 1, 0, 0, 6, 8184, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8153, 0, 309, 0, 8183, 1, 7, 0, 0, 1, 0, 2, 0, 8185, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 107, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7762, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 316, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 476, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 8187, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 108, 0, 8190, 0, 2, 0, 0, 1, 0, 0, 0, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 1009, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8122, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7615, 0, 560, 107, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7631, 0, 460, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8144, 8193, 8181, 8352, 8191, 8092] avg=8192.0000 max=8352 max_rank=5 min=8092 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32816 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8184, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 7, 0, 8115, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 0, 557, 107, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7596, 0, 309, 0, 0, 1, 7, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8184, 8244, 8198, 8133, 8472, 8195, 7916] avg=8192.0000 max=8472 max_rank=5 min=7916 min_rank=7 imbalance_ratio=1.034180 max_load_reduction=57064 max_load_reduction_pct=0.870728 moved_to_echo=32716 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7243, 0, 560, 107, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7631, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8193, 8149, 8191, 8266, 7978, 8190, 8378] avg=8192.0000 max=8378 max_rank=7 min=7978 min_rank=5 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32812 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6974, 0, 560, 107, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8085, 8192, 8311, 7709, 8199, 8656] avg=8192.0000 max=8656 max_rank=7 min=7709 min_rank=5 imbalance_ratio=1.056641 max_load_reduction=56880 max_load_reduction_pct=0.867920 moved_to_echo=32875 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7596, 0, 560, 107, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8192, 8085, 8192, 8164, 8331, 8190, 8186] avg=8192.0000 max=8331 max_rank=5 min=8085 min_rank=2 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32871 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8122, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7762, 0, 560, 107, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7631, 0, 316, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8196, 8190, 8194, 8130, 8499, 8189, 7948] avg=8192.0000 max=8499 max_rank=5 min=7948 min_rank=7 imbalance_ratio=1.037476 max_load_reduction=57037 max_load_reduction_pct=0.870316 moved_to_echo=32766 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 2, 0, 8122, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7617, 0, 563, 111, 68, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7608, 0, 108, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8195, 8554, 8184, 8138, 8365, 8190, 7719] avg=8192.0000 max=8554 max_rank=2 min=7719 min_rank=7 imbalance_ratio=1.044189 max_load_reduction=56982 max_load_reduction_pct=0.869476 moved_to_echo=32412 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7095, 0, 560, 107, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8085, 8192, 8213, 7830, 8192, 8641] avg=8192.0000 max=8641 max_rank=7 min=7830 min_rank=5 imbalance_ratio=1.054810 max_load_reduction=56895 max_load_reduction_pct=0.868149 moved_to_echo=32876 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 128, 41088, 0, 22656, 1024, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[42880, 0, 22784, 128, 0, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 59648, 0, 4480, 896, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[61056, 0, 4608, 0, 0, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 128, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 128, 512, 128, 0, 0, 0, 0, 128, 0, 128, 0, 65024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[896, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 64768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 0, 128, 896, 128, 0, 0, 128, 0, 128, 0, 128, 0, 64768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 64640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 5111, 0, 0, 0, 6456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 56, 8191, 8155, 0, 0, 0, 1, 0, 8191, 8191, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 2, 1, 0, 0, 0, 6723, 0, 0, 0, 2783, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 90, 8190, 8140, 0, 0, 0, 2, 0, 8185, 8181, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 165, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 7136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 4, 8192, 8192, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6769, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7, 0, 7, 6, 0, 0, 0, 6092, 0, 0, 0, 3507, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 4, 178, 8181, 7997, 0, 0, 0, 7, 0, 8182, 8117, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 306, 0, 0, 0, 0, 0, 0, 7599, 0, 0, 0, 5773, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 8181, 1, 0, 0, 55, 8191, 8132, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 7464, 0, 0, 0, 7858, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7274, 0, 0, 0, 16, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 7826, 0, 0, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7756, 0, 0, 0, 81, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 0, 0, 0, 0, 7488, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7896, 0, 0, 0, 58, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5321, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 165, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 170, 27, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 6368, 8191, 8184, 8920, 9119, 8392, 8169] avg=8192.0000 max=9119 max_rank=5 min=6368 min_rank=1 imbalance_ratio=1.113159 max_load_reduction=56417 max_load_reduction_pct=0.860855 moved_to_echo=34600 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 5321, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 5111, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 172, 27, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 8128, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 10117, 8191, 8181, 5835, 8438, 8396, 8186] avg=8192.0000 max=10117 max_rank=1 min=5835 min_rank=4 imbalance_ratio=1.234985 max_load_reduction=55419 max_load_reduction_pct=0.845627 moved_to_echo=30855 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 5322, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 7826, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7756, 171, 27, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 5628, 8199, 8182, 9086, 10037, 7965, 8246] avg=8192.0000 max=10037 max_rank=5 min=5628 min_rank=1 imbalance_ratio=1.225220 max_load_reduction=55499 max_load_reduction_pct=0.846848 moved_to_echo=35334 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 5324, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 0, 0, 0, 0, 7488, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7896, 170, 27, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 5368, 8189, 8182, 9178, 10097, 8106, 8223] avg=8192.0000 max=10097 max_rank=5 min=5368 min_rank=1 imbalance_ratio=1.232544 max_load_reduction=55439 max_load_reduction_pct=0.845932 moved_to_echo=35604 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6769, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 5273, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 6, 0, 0, 0, 6092, 0, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3507, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 171, 26, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 178, 0, 7971, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 12052, 8193, 8241, 6822, 5481, 8396, 8167] avg=8192.0000 max=12052 max_rank=1 min=5481 min_rank=5 imbalance_ratio=1.471191 max_load_reduction=53484 max_load_reduction_pct=0.816101 moved_to_echo=28866 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 5324, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 7464, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7274, 170, 27, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 5863, 8197, 8182, 9598, 9840, 7482, 8181] avg=8192.0000 max=9840 max_rank=5 min=5863 min_rank=1 imbalance_ratio=1.201172 max_load_reduction=55696 max_load_reduction_pct=0.849854 moved_to_echo=35101 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5317, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 2, 1, 0, 0, 0, 6723, 0, 0, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2783, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 170, 31, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 8109, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 12108, 8186, 8203, 7473, 4765, 8405, 8203] avg=8192.0000 max=12108 max_rank=1 min=4765 min_rank=5 imbalance_ratio=1.478027 max_load_reduction=53428 max_load_reduction_pct=0.815247 moved_to_echo=28846 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5321, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 306, 0, 0, 0, 0, 0, 0, 7599, 0, 0, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5773, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 8181, 170, 27, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 55, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8032, 8190, 8181, 8624, 7759, 8394, 8161] avg=8192.0000 max=8624 max_rank=4 min=7759 min_rank=5 imbalance_ratio=1.052734 max_load_reduction=56912 max_load_reduction_pct=0.868408 moved_to_echo=32938 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 267, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 7873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 1, 8192, 8192, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2, 1, 0, 0, 0, 8182, 0, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 22, 8190, 8140, 0, 0, 0, 2, 0, 8184, 8171, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7703, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7019, 0, 0, 0, 5, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 29, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7446, 0, 0, 0, 22, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5721, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7, 0, 7, 6, 0, 0, 0, 8122, 0, 0, 0, 2855, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 4, 83, 8168, 7799, 0, 0, 0, 7, 0, 8183, 8090, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 5547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 32, 8191, 8164, 0, 0, 0, 1, 0, 8191, 8191, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 1, 0, 0, 566, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 7535, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 8033, 1, 0, 0, 20, 8187, 8081, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2, 1, 0, 0, 0, 7700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1627, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 315, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8140, 0, 0, 0, 2, 0, 0, 31, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 10971, 8198, 8203, 7730, 5531, 8519, 8197] avg=8192.0000 max=10971 max_rank=1 min=5531 min_rank=5 imbalance_ratio=1.339233 max_load_reduction=54565 max_load_reduction_pct=0.832596 moved_to_echo=29977 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 6520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 7719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 312, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 8192, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 6626, 8199, 8182, 8242, 9789, 8039, 8266] avg=8192.0000 max=9789 max_rank=5 min=6626 min_rank=1 imbalance_ratio=1.194946 max_load_reduction=55747 max_load_reduction_pct=0.850632 moved_to_echo=34336 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5721, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 6439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 6, 0, 0, 0, 7653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2855, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 1, 0, 1607, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 309, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 83, 0, 7799, 0, 0, 0, 7, 0, 0, 44, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 12168, 8181, 8240, 7673, 4637, 8507, 7944] avg=8192.0000 max=12168 max_rank=1 min=4637 min_rank=5 imbalance_ratio=1.485352 max_load_reduction=53368 max_load_reduction_pct=0.814331 moved_to_echo=28761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 6520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 0, 0, 0, 0, 0, 0, 7719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7446, 312, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6551, 8190, 8182, 8628, 9763, 7769, 8259] avg=8192.0000 max=9763 max_rank=5 min=6551 min_rank=1 imbalance_ratio=1.191772 max_load_reduction=55773 max_load_reduction_pct=0.851028 moved_to_echo=34419 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 6519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 7717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1627, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 312, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 8164, 0, 0, 0, 1, 0, 0, 45, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 9163, 8192, 8181, 7719, 7336, 8510, 8243] avg=8192.0000 max=9163 max_rank=1 min=7336 min_rank=5 imbalance_ratio=1.118530 max_load_reduction=56373 max_load_reduction_pct=0.860184 moved_to_echo=31808 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 6520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 0, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7703, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7019, 312, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6606, 8195, 8182, 9283, 9492, 7342, 8242] avg=8192.0000 max=9492 max_rank=5 min=6606 min_rank=1 imbalance_ratio=1.158691 max_load_reduction=56044 max_load_reduction_pct=0.855164 moved_to_echo=34359 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 566, 0, 0, 0, 0, 0, 0, 7707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7535, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 8033, 312, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 20, 0, 8081, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 6860, 8188, 8183, 8275, 9327, 8359, 8147] avg=8192.0000 max=9327 max_rank=5 min=6860 min_rank=1 imbalance_ratio=1.138550 max_load_reduction=56209 max_load_reduction_pct=0.857681 moved_to_echo=34108 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 267, 0, 0, 0, 0, 0, 0, 7718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 312, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 6591, 8193, 8183, 7986, 9661, 8491, 8238] avg=8192.0000 max=9661 max_rank=5 min=6591 min_rank=1 imbalance_ratio=1.179321 max_load_reduction=55875 max_load_reduction_pct=0.852585 moved_to_echo=34376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 22912, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 128, 128, 0, 42624, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 49792, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 128, 3456, 0, 128, 128, 0, 0, 0, 56448, 0, 0, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 63872, 1408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 51328, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 13056, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 13312, 0, 128, 0, 0, 0, 0, 256, 0, 0, 128, 128, 128, 0, 52096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 128, 3968, 0, 128, 128, 0, 0, 0, 61696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 62976, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 640, 0, 65024, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 256, 0, 65024, 0, 0, 0, 128, 0, 0, 384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7868, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 1, 1, 0, 0, 94, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 6850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 6389, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7987, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 8191, 7309, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 8188, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8153, 8184, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 15, 0, 0, 0, 465, 3, 0, 0, 0, 309, 0, 7750, 0, 0, 0, 5, 0, 0, 0, 41, 0, 7928, 5995, 0, 0, 0, 5, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2137, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 1, 0, 194, 0, 7, 0, 0, 0, 0, 0, 0, 7843, 7, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 8192, 7855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 36, 0, 0, 0, 553, 2, 1, 0, 0, 398, 0, 7660, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8165, 6934, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2006, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 182, 0, 2, 0, 0, 0, 0, 0, 0, 6836, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 8191, 7672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 519, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 1, 160, 0, 0, 0, 1, 27, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8191, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 8189, 6989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 4, 0, 1106, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 109, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 2509, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7499, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5006, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8165, 8083, 8776, 10570, 8188, 8172, 5384] avg=8192.0000 max=10570 max_rank=4 min=5384 min_rank=7 imbalance_ratio=1.290283 max_load_reduction=54966 max_load_reduction_pct=0.838715 moved_to_echo=32314 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 109, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2511, 7309, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7499, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5007, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8189, 8072, 8777, 9824, 8187, 8172, 6212] avg=8192.0000 max=9824 max_rank=4 min=6212 min_rank=7 imbalance_ratio=1.199219 max_load_reduction=55712 max_load_reduction_pct=0.850098 moved_to_echo=32395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 109, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 2511, 7855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7499, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5008, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8180, 8071, 8776, 10395, 8188, 8172, 5566] avg=8192.0000 max=10395 max_rank=4 min=5566 min_rank=7 imbalance_ratio=1.268921 max_load_reduction=55141 max_load_reduction_pct=0.841385 moved_to_echo=32321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8076, 109, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 1, 1, 0, 0, 94, 0, 8020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 6850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2511, 3331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 7490, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6389, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5007, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7984, 8189, 8211, 7433, 5842, 8189, 8167, 11521] avg=8192.0000 max=11521 max_rank=7 min=5842 min_rank=4 imbalance_ratio=1.406372 max_load_reduction=54015 max_load_reduction_pct=0.824203 moved_to_echo=33719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 36, 0, 8080, 115, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 2, 1, 0, 0, 398, 0, 7540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 571, 6934, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2501, 2006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 182, 0, 2, 0, 0, 0, 0, 7479, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6836, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5012, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8239, 8494, 7540, 4507, 8208, 8343, 11972] avg=8192.0000 max=11972 max_rank=7 min=4507 min_rank=4 imbalance_ratio=1.461426 max_load_reduction=53564 max_load_reduction_pct=0.817322 moved_to_echo=33030 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 115, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 15, 0, 8070, 106, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 3, 0, 0, 0, 309, 0, 7633, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 564, 5995, 0, 0, 0, 5, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2508, 2137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 1, 0, 194, 0, 7, 0, 0, 0, 0, 7258, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7843, 7, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 5003, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8201, 8415, 6683, 4645, 8199, 8145, 12979] avg=8192.0000 max=12979 max_rank=7 min=4645 min_rank=4 imbalance_ratio=1.584351 max_load_reduction=52557 max_load_reduction_pct=0.801956 moved_to_echo=33968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 109, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 2511, 7672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7498, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 519, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5007, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8166, 8071, 8775, 10251, 8188, 8171, 5656] avg=8192.0000 max=10251 max_rank=4 min=5656 min_rank=7 imbalance_ratio=1.251343 max_load_reduction=55285 max_load_reduction_pct=0.843582 moved_to_echo=32266 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 8073, 109, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 160, 0, 0, 0, 1, 27, 0, 7930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 8191, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2510, 6989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 7496, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1106, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5006, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8207, 8119, 8776, 9502, 8189, 8194, 6246] avg=8192.0000 max=9502 max_rank=4 min=6246 min_rank=7 imbalance_ratio=1.159912 max_load_reduction=56034 max_load_reduction_pct=0.855011 moved_to_echo=32131 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8179, 7919, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8173, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8183, 8183, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 24, 0, 0, 0, 436, 3, 0, 0, 0, 36, 0, 7601, 0, 0, 0, 5, 0, 0, 0, 48, 0, 7908, 1026, 0, 0, 0, 5, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 7439, 0, 0, 0, 0, 0, 0, 0, 6, 8, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 1, 0, 184, 0, 7, 0, 0, 0, 0, 0, 0, 7945, 7, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8174, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8186, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 1, 1, 0, 0, 23, 0, 8008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 7755, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7039, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 141, 0, 0, 0, 1, 3, 0, 7931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8191, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 8188, 7739, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 0, 539, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 40, 0, 0, 0, 505, 2, 1, 0, 0, 45, 0, 7460, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8164, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 7413, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 143, 0, 2, 0, 0, 0, 0, 0, 0, 7284, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 8190, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8143, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 373, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7800, 330, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8182, 8238, 10715, 8450, 8189, 8137, 5433] avg=8192.0000 max=10715 max_rank=3 min=5433 min_rank=7 imbalance_ratio=1.307983 max_load_reduction=54821 max_load_reduction_pct=0.836502 moved_to_echo=30209 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 373, 7919, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 330, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8181, 8092, 10716, 8297, 8188, 8143, 5721] avg=8192.0000 max=10716 max_rank=3 min=5721 min_rank=7 imbalance_ratio=1.308105 max_load_reduction=54820 max_load_reduction_pct=0.836487 moved_to_echo=30349 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 1, 1, 0, 0, 23, 0, 8008, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2522, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 7755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7818, 329, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7039, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8177, 8226, 4174, 8128, 8192, 8157, 12284] avg=8192.0000 max=12284 max_rank=7 min=4174 min_rank=3 imbalance_ratio=1.499512 max_load_reduction=53252 max_load_reduction_pct=0.812561 moved_to_echo=36761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 24, 0, 8170, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 3, 0, 0, 0, 36, 0, 7601, 0, 0, 0, 5, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 2436, 1026, 0, 0, 0, 5, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 7439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 1, 0, 184, 0, 7, 0, 0, 0, 0, 7811, 318, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 7, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 5066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8204, 8169, 3596, 7812, 8196, 8340, 13027] avg=8192.0000 max=13027 max_rank=7 min=3596 min_rank=3 imbalance_ratio=1.590210 max_load_reduction=52509 max_load_reduction_pct=0.801224 moved_to_echo=37375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 373, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7801, 330, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8183, 8267, 10715, 8518, 8188, 8138, 5333] avg=8192.0000 max=10715 max_rank=3 min=5333 min_rank=7 imbalance_ratio=1.307983 max_load_reduction=54821 max_load_reduction_pct=0.836502 moved_to_echo=30177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 8177, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 1, 3, 0, 7931, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 8191, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 373, 7739, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7815, 329, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 539, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8196, 8167, 10716, 8116, 8189, 8159, 5798] avg=8192.0000 max=10716 max_rank=3 min=5798 min_rank=7 imbalance_ratio=1.308105 max_load_reduction=54820 max_load_reduction_pct=0.836487 moved_to_echo=30262 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 40, 0, 8181, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 2, 1, 0, 0, 45, 0, 7460, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2503, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 7413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 143, 0, 2, 0, 0, 0, 0, 7820, 332, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7284, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8229, 8109, 4191, 7783, 8206, 8307, 12521] avg=8192.0000 max=12521 max_rank=7 min=4191 min_rank=3 imbalance_ratio=1.528442 max_load_reduction=53015 max_load_reduction_pct=0.808945 moved_to_echo=36817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2521, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 373, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7817, 329, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8184, 8268, 10713, 8432, 8188, 8155, 5419] avg=8192.0000 max=10713 max_rank=3 min=5419 min_rank=7 imbalance_ratio=1.307739 max_load_reduction=54823 max_load_reduction_pct=0.836533 moved_to_echo=30194 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 24576, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 40064, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 20096, 45440, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 512, 0, 128, 0, 0, 0, 0, 59776, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 4736, 60800, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 23808, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 41856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 20096, 45312, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 3072, 62592, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 384, 0, 128, 0, 0, 0, 0, 62592, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 1280, 128, 128, 0, 128, 128, 0, 63616, 0, 0, 0, 128, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 64640, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 1024, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 1280, 128, 128, 0, 128, 896, 0, 63488, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7524, 0, 0, 0, 0, 1352, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 8180, 0, 0, 0, 1, 7346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 8082, 0, 23, 0, 0, 73, 0, 0, 0, 0, 0, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 9, 0, 0, 0, 8192, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 73, 0, 0, 0, 1, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5410, 0, 0, 0, 0, 2921, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 2, 0, 8174, 0, 0, 0, 2, 7738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8190, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 44, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 50, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7975, 0, 31, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 12, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 154, 0, 0, 0, 6, 0, 0, 0, 0, 0, 175, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4388, 0, 0, 0, 0, 5919, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 872, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 8080, 0, 0, 7, 0, 8098, 0, 0, 0, 7, 5044, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8, 8182, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8184, 2, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 1, 0, 0, 162, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 7084, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 697, 0, 0, 0, 0, 0, 3, 3, 0, 0, 3, 0, 0, 0, 8191, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8091, 0, 20, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 36, 0, 0, 38, 0, 0, 0, 8192, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8183, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7790, 0, 23, 0, 0, 73, 0, 0, 0, 0, 0, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 9, 0, 0, 7636, 272, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8192, 7896, 8000, 8740, 8191, 8197, 8212] avg=8192.0000 max=8740 max_rank=4 min=7896 min_rank=2 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=33340 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8183, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7252, 0, 0, 0, 0, 1352, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7632, 253, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 1, 0, 0, 0, 0, 0, 1, 7346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8191, 8608, 8134, 7904, 8193, 8199, 8210] avg=8192.0000 max=8608 max_rank=2 min=7904 min_rank=4 imbalance_ratio=1.050781 max_load_reduction=56928 max_load_reduction_pct=0.868652 moved_to_echo=32506 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7687, 0, 31, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 12, 0, 0, 7636, 268, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8190, 7872, 8068, 8748, 8192, 8193, 8169] avg=8192.0000 max=8748 max_rank=4 min=7872 min_rank=2 imbalance_ratio=1.067871 max_load_reduction=56788 max_load_reduction_pct=0.866516 moved_to_echo=33302 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7808, 0, 44, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 50, 0, 0, 3, 0, 0, 7636, 272, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8192, 7886, 8048, 8748, 8192, 8188, 8178] avg=8192.0000 max=8748 max_rank=4 min=7886 min_rank=2 imbalance_ratio=1.067871 max_load_reduction=56788 max_load_reduction_pct=0.866516 moved_to_echo=33306 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 154, 0, 0, 0, 6, 0, 0, 0, 0, 0, 175, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 8176, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4230, 0, 0, 0, 0, 5919, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 7532, 147, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 7, 0, 0, 0, 0, 0, 7, 5044, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8346, 8194, 10155, 8637, 5613, 8195, 8190, 8206] avg=8192.0000 max=10155 max_rank=2 min=5613 min_rank=4 imbalance_ratio=1.239624 max_load_reduction=55381 max_load_reduction_pct=0.845047 moved_to_echo=30204 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 73, 0, 0, 0, 1, 0, 0, 0, 0, 0, 247, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8185, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5207, 0, 0, 0, 0, 2921, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7593, 187, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 0, 0, 2, 0, 0, 0, 0, 0, 2, 7738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8409, 8194, 8130, 7918, 8289, 8190, 8195, 8211] avg=8192.0000 max=8409 max_rank=0 min=7918 min_rank=3 imbalance_ratio=1.026489 max_load_reduction=57127 max_load_reduction_pct=0.871689 moved_to_echo=32885 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 1, 0, 0, 162, 3, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8182, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6828, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 697, 0, 0, 0, 0, 0, 3, 3, 0, 0, 3, 0, 0, 7635, 238, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8192, 7076, 8662, 8746, 8192, 8192, 8207] avg=8192.0000 max=8746 max_rank=4 min=7076 min_rank=2 imbalance_ratio=1.067627 max_load_reduction=56790 max_load_reduction_pct=0.866547 moved_to_echo=33337 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8181, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7799, 0, 20, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 36, 0, 0, 38, 0, 0, 7636, 272, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8191, 7913, 8069, 8748, 8191, 8182, 8143] avg=8192.0000 max=8748 max_rank=4 min=7913 min_rank=2 imbalance_ratio=1.067871 max_load_reduction=56788 max_load_reduction_pct=0.866516 moved_to_echo=33264 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7572, 0, 0, 0, 0, 1173, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 8179, 0, 0, 0, 1, 7414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 8170, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8, 0, 0, 0, 8192, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 26, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 73, 0, 0, 19, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 173, 0, 0, 0, 6, 0, 0, 0, 0, 0, 177, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 5673, 0, 0, 0, 0, 3682, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 8070, 0, 0, 7, 0, 8091, 0, 0, 0, 7, 5931, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 8, 8180, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8184, 2, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 38, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 50, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 85, 0, 0, 0, 1, 0, 0, 0, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 2077, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 2, 0, 8175, 0, 0, 0, 2, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8189, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 2, 7450, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 580, 0, 0, 0, 0, 0, 4, 3, 0, 0, 5, 0, 0, 0, 8191, 0, 0, 0, 2, 8187, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8157, 0, 20, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 40, 0, 0, 29, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7988, 0, 26, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 73, 0, 0, 19, 0, 0, 7768, 113, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8191, 8023, 8057, 8613, 8192, 8191, 8168] avg=8192.0000 max=8613 max_rank=4 min=8023 min_rank=2 imbalance_ratio=1.051392 max_load_reduction=56923 max_load_reduction_pct=0.868576 moved_to_echo=33164 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8180, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7450, 0, 0, 0, 0, 1173, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7765, 105, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 1, 0, 0, 0, 0, 0, 1, 7414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8191, 8627, 8174, 7840, 8193, 8202, 8208] avg=8192.0000 max=8627 max_rank=2 min=7840 min_rank=4 imbalance_ratio=1.053101 max_load_reduction=56909 max_load_reduction_pct=0.868362 moved_to_echo=32443 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 173, 0, 0, 0, 6, 0, 0, 0, 0, 0, 177, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 8173, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5582, 0, 0, 0, 0, 3682, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 7652, 79, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 0, 7, 0, 0, 0, 0, 0, 7, 5931, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 8, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8357, 8194, 9270, 8748, 6372, 8196, 8191, 8208] avg=8192.0000 max=9270 max_rank=2 min=6372 min_rank=4 imbalance_ratio=1.131592 max_load_reduction=56266 max_load_reduction_pct=0.858551 moved_to_echo=30967 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8020, 0, 38, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 50, 0, 0, 2, 0, 0, 7768, 113, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8192, 8059, 8023, 8616, 8191, 8177, 8177] avg=8192.0000 max=8616 max_rank=4 min=8023 min_rank=3 imbalance_ratio=1.051758 max_load_reduction=56920 max_load_reduction_pct=0.868530 moved_to_echo=33161 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 85, 0, 0, 0, 1, 0, 0, 0, 0, 0, 250, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8182, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6028, 0, 0, 0, 0, 2077, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7718, 89, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 2, 0, 0, 0, 0, 0, 2, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8421, 8195, 8107, 7964, 8249, 8190, 8197, 8213] avg=8192.0000 max=8421 max_rank=0 min=7964 min_rank=3 imbalance_ratio=1.027954 max_load_reduction=57115 max_load_reduction_pct=0.871506 moved_to_echo=32849 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 2, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8178, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7331, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 580, 0, 0, 0, 0, 0, 4, 3, 0, 0, 5, 0, 0, 7767, 103, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8191, 7336, 8545, 8617, 8192, 8196, 8202] avg=8192.0000 max=8617 max_rank=4 min=7336 min_rank=2 imbalance_ratio=1.051880 max_load_reduction=56919 max_load_reduction_pct=0.868515 moved_to_echo=33207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8180, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8, 0, 0, 7768, 113, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8192, 8046, 7983, 8615, 8191, 8199, 8210] avg=8192.0000 max=8615 max_rank=4 min=7983 min_rank=3 imbalance_ratio=1.051636 max_load_reduction=56921 max_load_reduction_pct=0.868546 moved_to_echo=33215 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8178, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 20, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 40, 0, 0, 29, 0, 0, 7768, 113, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8190, 8068, 8042, 8614, 8191, 8183, 8150] avg=8192.0000 max=8614 max_rank=4 min=8042 min_rank=3 imbalance_ratio=1.051514 max_load_reduction=56922 max_load_reduction_pct=0.868561 moved_to_echo=33138 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 1792, 0, 0, 0, 0, 128, 128, 256, 0, 0, 128, 0, 128, 61056, 1920, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 4480, 0, 0, 128, 128, 0, 0, 0, 0, 128, 61184, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 54656, 0, 128, 0, 0, 10880, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 256, 0, 128, 128, 128, 0, 0, 128, 0, 0, 640, 128, 0, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65408, 128, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 3456, 0, 0, 128, 128, 0, 0, 0, 0, 128, 62208, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 1920, 0, 0, 0, 0, 128, 128, 256, 0, 0, 128, 0, 128, 62080, 896, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 58496, 0, 128, 0, 0, 7040, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 384, 0, 128, 0, 128, 0, 0, 0, 0, 0, 640, 128, 0, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 128, 0, 128, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65408, 128, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 1, 0, 0, 0, 4, 0, 0, 8151, 0, 8191, 0, 0, 0, 0, 0, 8138, 0, 0, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 0, 0, 104] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 8192, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 10, 1, 0, 8191, 0, 8188, 0, 0, 0, 1, 0, 8056, 0, 0, 0, 0, 491, 0, 0, 0, 0, 3267, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4421, 3, 0, 0, 0, 128] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8106, 4, 0, 0, 0, 40, 0, 1, 8187, 0, 8188, 0, 0, 0, 0, 0, 7865, 0, 0, 5, 0, 0, 0, 0, 0, 0, 203, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 178] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 177, 7, 0, 8184, 1, 8181, 0, 0, 0, 8, 0, 4059, 0, 0, 0, 0, 1357, 0, 0, 0, 0, 4888, 0, 7, 0, 0, 4, 0, 0, 2, 0, 0, 0, 8, 7, 8, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8, 0, 0, 1, 0, 4, 4, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4813, 0, 0, 0, 0, 709] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 132, 2, 60, 8190, 0, 8186, 0, 0, 0, 2, 0, 7213, 0, 0, 0, 0, 71, 0, 0, 0, 0, 3293, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 2, 0, 0, 0, 560] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 649, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3, 0, 0, 2, 0, 6282, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 0, 0, 104, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8791, 6473, 8190, 8024, 8194, 8194, 9488] avg=8192.0000 max=9488 max_rank=7 min=6473 min_rank=2 imbalance_ratio=1.158203 max_load_reduction=56048 max_load_reduction_pct=0.855225 moved_to_echo=33900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 651, 0, 0, 0, 0, 0, 1, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 0, 0, 0, 0, 3267, 0, 1, 0, 0, 0, 0, 0, 6313, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4421, 3, 0, 0, 0, 128, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8719, 10244, 8192, 8027, 8201, 8190, 5759] avg=8192.0000 max=10244 max_rank=2 min=5759 min_rank=7 imbalance_ratio=1.250488 max_load_reduction=55292 max_load_reduction_pct=0.843689 moved_to_echo=30177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 2, 0, 0, 0, 0, 6296, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 27, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8839, 6511, 8190, 8023, 8176, 8193, 9400] avg=8192.0000 max=9400 max_rank=7 min=6511 min_rank=2 imbalance_ratio=1.147461 max_load_reduction=56136 max_load_reduction_pct=0.856567 moved_to_echo=33792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 6305, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 33, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8842, 6488, 8190, 8023, 8198, 8194, 9397] avg=8192.0000 max=9397 max_rank=7 min=6488 min_rank=2 imbalance_ratio=1.147095 max_load_reduction=56139 max_load_reduction_pct=0.856613 moved_to_echo=33812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 7, 0, 651, 1, 0, 0, 0, 0, 8, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 0, 0, 0, 0, 4888, 0, 7, 0, 0, 4, 0, 0, 6307, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8, 7, 8, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 8009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8, 0, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4813, 0, 0, 0, 0, 709, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 4903, 12735, 8206, 8364, 8202, 8193, 6727] avg=8192.0000 max=12735 max_rank=2 min=4903 min_rank=1 imbalance_ratio=1.554565 max_load_reduction=52801 max_load_reduction_pct=0.805679 moved_to_echo=31486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8106, 4, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 1, 651, 0, 0, 0, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 203, 0, 0, 5, 0, 0, 0, 2, 6309, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 4, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 178, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8557, 6696, 8189, 8016, 8193, 8188, 9572] avg=8192.0000 max=9572 max_rank=7 min=6696 min_rank=2 imbalance_ratio=1.168457 max_load_reduction=55964 max_load_reduction_pct=0.853943 moved_to_echo=33969 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 6, 0, 0, 0, 0, 6299, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 24, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8843, 6534, 8190, 8019, 8168, 8182, 9396] avg=8192.0000 max=9396 max_rank=7 min=6534 min_rank=2 imbalance_ratio=1.146973 max_load_reduction=56140 max_load_reduction_pct=0.856628 moved_to_echo=33765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 2, 60, 633, 0, 0, 0, 0, 0, 2, 0, 7213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 3293, 0, 2, 0, 0, 0, 0, 0, 6316, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 8013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 2, 0, 0, 0, 560, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8042, 9855, 8189, 9040, 8204, 8202, 5797] avg=8192.0000 max=9855 max_rank=2 min=5797 min_rank=7 imbalance_ratio=1.203003 max_load_reduction=55681 max_load_reduction_pct=0.849625 moved_to_echo=31243 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 9, 1, 0, 8191, 0, 8188, 0, 0, 0, 1, 0, 8072, 0, 0, 0, 0, 507, 0, 0, 0, 0, 4644, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3030, 3, 0, 0, 0, 116] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 8192, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 190, 7, 6, 8184, 2, 8180, 0, 0, 0, 8, 0, 5227, 0, 0, 0, 0, 1349, 0, 0, 0, 0, 5725, 0, 8, 0, 0, 4, 0, 0, 2, 0, 0, 0, 8, 8, 8, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8, 0, 0, 1, 0, 4, 4, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 0, 0, 0, 0, 568] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 2, 0, 0, 8156, 0, 8191, 0, 0, 0, 0, 0, 8172, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 0, 0, 5, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 68] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 124, 2, 94, 8190, 0, 8186, 0, 0, 0, 2, 0, 7523, 0, 0, 0, 0, 134, 0, 0, 0, 1, 4435, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2818, 2, 0, 0, 0, 383] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8108, 4, 0, 0, 0, 35, 0, 0, 8187, 0, 8188, 0, 0, 0, 0, 0, 7883, 0, 0, 5, 0, 0, 0, 0, 0, 0, 307, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 99] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 454, 0, 0, 0, 0, 0, 1, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 0, 0, 0, 4644, 0, 1, 0, 0, 0, 0, 0, 5906, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3030, 3, 0, 0, 0, 116, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8537, 11186, 8192, 8064, 8205, 8192, 4955] avg=8192.0000 max=11186 max_rank=2 min=4955 min_rank=7 imbalance_ratio=1.365479 max_load_reduction=54350 max_load_reduction_pct=0.829315 moved_to_echo=29416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 7, 6, 454, 2, 0, 0, 0, 0, 8, 0, 5227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 0, 0, 0, 0, 5725, 0, 8, 0, 0, 4, 0, 0, 5901, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8, 8, 8, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 178, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8, 0, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 0, 0, 0, 0, 568, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 5894, 13115, 8207, 8232, 8206, 8193, 5481] avg=8192.0000 max=13115 max_rank=2 min=5481 min_rank=7 imbalance_ratio=1.600952 max_load_reduction=52421 max_load_reduction_pct=0.799881 moved_to_echo=30112 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 3, 0, 0, 0, 0, 5891, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 25, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8641, 6095, 8190, 8066, 8154, 8193, 9992] avg=8192.0000 max=9992 max_rank=7 min=6095 min_rank=2 imbalance_ratio=1.219727 max_load_reduction=55544 max_load_reduction_pct=0.847534 moved_to_echo=34405 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 452, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 0, 0, 5, 0, 0, 8, 0, 5881, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 1, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 68, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8626, 6036, 8190, 8063, 8198, 8193, 10055] avg=8192.0000 max=10055 max_rank=7 min=6036 min_rank=2 imbalance_ratio=1.227417 max_load_reduction=55481 max_load_reduction_pct=0.846573 moved_to_echo=34509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 5899, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 32, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8645, 6041, 8190, 8064, 8202, 8194, 9995] avg=8192.0000 max=9995 max_rank=7 min=6041 min_rank=2 imbalance_ratio=1.220093 max_load_reduction=55541 max_load_reduction_pct=0.847488 moved_to_echo=34455 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 2, 94, 432, 0, 0, 0, 0, 0, 2, 0, 7523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 1, 4435, 0, 2, 0, 0, 0, 0, 0, 5910, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2818, 2, 0, 0, 0, 383, 1808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8177, 10611, 8188, 8925, 8211, 8208, 5011] avg=8192.0000 max=10611 max_rank=2 min=5011 min_rank=7 imbalance_ratio=1.295288 max_load_reduction=54925 max_load_reduction_pct=0.838089 moved_to_echo=30355 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8108, 4, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 453, 0, 0, 0, 0, 0, 0, 0, 7883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 307, 0, 0, 6, 0, 0, 0, 2, 5904, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 4, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 99, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8371, 6353, 8189, 8059, 8196, 8188, 10052] avg=8192.0000 max=10052 max_rank=7 min=6353 min_rank=2 imbalance_ratio=1.227051 max_load_reduction=55484 max_load_reduction_pct=0.846619 moved_to_echo=34495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 6, 0, 0, 1, 0, 5899, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 23, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8645, 6099, 8190, 8063, 8164, 8175, 9995] avg=8192.0000 max=9995 max_rank=7 min=6099 min_rank=2 imbalance_ratio=1.220093 max_load_reduction=55541 max_load_reduction_pct=0.847488 moved_to_echo=34397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 1920, 0, 0, 0, 0, 11776, 0, 128, 128, 0, 128, 128, 128, 50432, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 54272, 128, 0, 0, 0, 1792, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 384, 128, 128, 5248, 128, 0, 0, 0, 0, 128, 0, 59904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 64256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 2048, 0, 0, 0, 128, 15360, 0, 128, 128, 0, 128, 128, 128, 47232, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 49920, 128, 0, 0, 0, 1280, 14464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 384, 128, 128, 3712, 128, 0, 0, 0, 0, 128, 0, 61568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 64512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 8188, 0, 0, 8000, 7115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 944, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8187, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 9, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7238, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 940, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7805, 0, 0, 8192, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8183, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 332, 8183, 0, 0, 7765, 3915, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 0, 0, 0, 0, 0, 7, 7, 0, 0, 8, 5, 0, 0, 2413, 0, 0, 8048, 9, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 8181, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 12, 0, 0, 0, 3, 0, 0, 169, 0, 8181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 8190, 0, 0, 7962, 6401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 1, 0, 0, 1686, 0, 0, 8121, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 89, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 0, 0, 8192, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8188, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7953, 0, 0, 8192, 8072, 0, 0, 2, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7124, 609, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7429, 9082, 8710, 7580, 8193, 8192, 8192, 8158] avg=8192.0000 max=9082 max_rank=1 min=7429 min_rank=0 imbalance_ratio=1.108643 max_load_reduction=56454 max_load_reduction_pct=0.861420 moved_to_echo=32735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 854, 7115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 6957, 609, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 944, 0, 0, 7577, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 9, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8413, 7969, 7887, 8524, 8189, 8194, 8192, 8168] avg=8192.0000 max=8524 max_rank=3 min=7887 min_rank=2 imbalance_ratio=1.040527 max_load_reduction=57012 max_load_reduction_pct=0.869934 moved_to_echo=32743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7124, 609, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7996, 9239, 7977, 7583, 8194, 8200, 8190, 8157] avg=8192.0000 max=9239 max_rank=1 min=7583 min_rank=3 imbalance_ratio=1.127808 max_load_reduction=56297 max_load_reduction_pct=0.859024 moved_to_echo=32741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 191, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 332, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 3915, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 0, 0, 0, 0, 0, 7, 7, 0, 0, 8, 6753, 598, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2413, 0, 0, 7450, 9, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7991, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 12, 0, 0, 0, 3, 0, 0, 169, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8710, 4745, 9317, 9885, 8186, 8171, 8190, 8332] avg=8192.0000 max=9885 max_rank=3 min=4745 min_rank=1 imbalance_ratio=1.206665 max_load_reduction=55651 max_load_reduction_pct=0.849167 moved_to_echo=32879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7123, 609, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 9066, 7988, 7583, 8194, 8200, 8193, 8157] avg=8192.0000 max=9066 max_rank=1 min=7583 min_rank=3 imbalance_ratio=1.106689 max_load_reduction=56470 max_load_reduction_pct=0.861664 moved_to_echo=32744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 6401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 6929, 608, 38, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1686, 0, 0, 7513, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8001, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 89, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8534, 7244, 7732, 9204, 8194, 8186, 8197, 8245] avg=8192.0000 max=9204 max_rank=3 min=7244 min_rank=1 imbalance_ratio=1.123535 max_load_reduction=56332 max_load_reduction_pct=0.859558 moved_to_echo=32822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 8072, 0, 0, 2, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7124, 609, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 7582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7996, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 9064, 7994, 7594, 8192, 8195, 8190, 8162] avg=8192.0000 max=9064 max_rank=1 min=7594 min_rank=3 imbalance_ratio=1.106445 max_load_reduction=56472 max_load_reduction_pct=0.861694 moved_to_echo=32739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7124, 609, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 9127, 7931, 7583, 8194, 8198, 8192, 8157] avg=8192.0000 max=9127 max_rank=1 min=7583 min_rank=3 imbalance_ratio=1.114136 max_load_reduction=56409 max_load_reduction_pct=0.860733 moved_to_echo=32741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8183, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 373, 8183, 0, 0, 7719, 3022, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2417, 0, 0, 0, 0, 0, 7, 8, 0, 0, 8, 5, 0, 0, 2807, 0, 0, 8044, 9, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 8180, 0, 0, 4, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 12, 0, 0, 0, 3, 0, 0, 168, 0, 8181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7259, 0, 0, 8192, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 8188, 0, 0, 7970, 6954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1105, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8187, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 10, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8188, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7879, 0, 0, 8192, 8104, 0, 0, 2, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 8190, 0, 0, 7943, 6010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 1, 0, 0, 1938, 0, 0, 8121, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 80, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7069, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7670, 0, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 331, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 373, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 966, 3022, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2417, 0, 0, 0, 0, 0, 7, 8, 0, 0, 8, 6436, 696, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2807, 0, 0, 7348, 9, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7850, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 4, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 12, 0, 0, 0, 3, 0, 0, 168, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8891, 3989, 9606, 10177, 8178, 8174, 8189, 8332] avg=8192.0000 max=10177 max_rank=3 min=3989 min_rank=1 imbalance_ratio=1.242310 max_load_reduction=55359 max_load_reduction_pct=0.844711 moved_to_echo=32873 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6831, 709, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7590, 9461, 8258, 7483, 8196, 8199, 8191, 8158] avg=8192.0000 max=9461 max_rank=1 min=7483 min_rank=3 imbalance_ratio=1.154907 max_load_reduction=56075 max_load_reduction_pct=0.855637 moved_to_echo=32744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6830, 709, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 9221, 7935, 7483, 8196, 8198, 8194, 8158] avg=8192.0000 max=9221 max_rank=1 min=7483 min_rank=3 imbalance_ratio=1.125610 max_load_reduction=56315 max_load_reduction_pct=0.859299 moved_to_echo=32746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 331, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 996, 6954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 6646, 709, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1105, 0, 0, 7476, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 10, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8571, 7950, 7686, 8584, 8187, 8195, 8193, 8170] avg=8192.0000 max=8584 max_rank=3 min=7686 min_rank=2 imbalance_ratio=1.047852 max_load_reduction=56952 max_load_reduction_pct=0.869019 moved_to_echo=32745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 331, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 8104, 0, 0, 2, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6831, 709, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7855, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 9283, 7806, 7489, 8194, 8196, 8190, 8167] avg=8192.0000 max=9283 max_rank=1 min=7489 min_rank=3 imbalance_ratio=1.133179 max_load_reduction=56253 max_load_reduction_pct=0.858353 moved_to_echo=32747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6831, 709, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7400, 9323, 8592, 7480, 8195, 8194, 8193, 8159] avg=8192.0000 max=9323 max_rank=1 min=7400 min_rank=0 imbalance_ratio=1.138062 max_load_reduction=56213 max_load_reduction_pct=0.857742 moved_to_echo=32741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 988, 6010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 6625, 707, 40, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1938, 0, 0, 7414, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7861, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 80, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8721, 6998, 7655, 9357, 8194, 8183, 8194, 8234] avg=8192.0000 max=9357 max_rank=3 min=6998 min_rank=1 imbalance_ratio=1.142212 max_load_reduction=56179 max_load_reduction_pct=0.857224 moved_to_echo=32805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6831, 709, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8001, 9311, 7998, 7483, 8196, 8197, 8192, 8158] avg=8192.0000 max=9311 max_rank=1 min=7483 min_rank=3 imbalance_ratio=1.136597 max_load_reduction=56225 max_load_reduction_pct=0.857925 moved_to_echo=32743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 0, 4096, 128, 0, 0, 0, 128, 128, 128, 0, 0, 128, 56320, 4864, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 6912, 58368, 0, 0, 128, 0, 0, 128, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 5120, 0, 0, 60544, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 1536, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 640, 63488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 64000, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 0, 5760, 0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 53888, 5760, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 8192, 56960, 0, 0, 128, 0, 0, 128, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 2688, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 640, 62336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 5888, 0, 0, 59776, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 62976, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 128, 384, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 65536, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 65536, 0, 0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7852, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6659, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2573, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8011, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2914, 5626, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 2, 8, 0, 0, 0, 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, 0, 0, 1, 7405, 0, 7982, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 7462, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 11, 0, 1232, 7343, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 665, 1, 3, 0, 0, 0, 0, 0, 0, 1226, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 2, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 6893, 0, 7847, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5504, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7207, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2862, 6033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 6969, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7766, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 6763, 0, 0, 0, 0, 2, 0, 8182, 0, 0, 35, 0, 0, 0, 0, 0, 3, 0, 0, 80, 0, 1105, 7701, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 1366, 0, 64, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7979, 9, 0, 0, 0, 1, 5, 46, 0, 0, 0, 0, 1, 7, 0, 8170, 0, 0, 0, 0, 0, 4, 0, 0, 0, 57, 6, 15, 0, 0, 0, 4, 0, 5, 9, 5902, 0, 7638, 0, 0, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6659, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2573, 0, 0, 0, 0, 0, 2, 4488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7902, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8476, 9261, 8200, 7063, 8204, 8203, 7360, 8769] avg=8192.0000 max=9261 max_rank=1 min=7063 min_rank=3 imbalance_ratio=1.130493 max_load_reduction=56275 max_load_reduction_pct=0.858688 moved_to_echo=32536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 217, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7843, 0, 0, 0, 0, 0, 0, 4354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7905, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8479, 4751, 8191, 12197, 8197, 8202, 6768, 8751] avg=8192.0000 max=12197 max_rank=3 min=4751 min_rank=1 imbalance_ratio=1.488892 max_load_reduction=53339 max_load_reduction_pct=0.813889 moved_to_echo=31918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8011, 0, 0, 0, 0, 0, 0, 287, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2914, 5626, 1, 0, 0, 0, 0, 0, 0, 0, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 2, 8, 0, 0, 0, 0, 0, 0, 561, 0, 0, 0, 0, 0, 4426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 7405, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 9631, 8189, 5292, 8203, 8165, 9779, 7976] avg=8192.0000 max=9779 max_rank=6 min=5292 min_rank=3 imbalance_ratio=1.193726 max_load_reduction=55757 max_load_reduction_pct=0.850784 moved_to_echo=34123 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5504, 841, 0, 0, 0, 0, 0, 0, 0, 0, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5563, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7905, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8479, 7329, 8191, 9559, 8173, 8201, 6898, 8706] avg=8192.0000 max=9559 max_rank=3 min=6898 min_rank=6 imbalance_ratio=1.166870 max_load_reduction=55977 max_load_reduction_pct=0.854141 moved_to_echo=31978 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7462, 0, 0, 0, 0, 0, 0, 282, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 0, 1232, 7343, 2, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 1, 3, 0, 0, 0, 0, 0, 0, 1226, 0, 0, 0, 0, 0, 4355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7907, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 2, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 6893, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7756, 9664, 8186, 6250, 8210, 8304, 9732, 7434] avg=8192.0000 max=9732 max_rank=6 min=6250 min_rank=3 imbalance_ratio=1.187988 max_load_reduction=55804 max_load_reduction_pct=0.851501 moved_to_echo=33680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6763, 0, 0, 0, 0, 2, 0, 286, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 80, 0, 1105, 7701, 7, 0, 0, 0, 0, 0, 0, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1366, 0, 64, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 0, 0, 0, 4235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7896, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7979, 9, 0, 0, 0, 1, 5, 46, 0, 0, 0, 0, 1, 7, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 4, 0, 0, 0, 57, 6, 15, 0, 0, 0, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 9, 5902, 0, 546, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7086, 9938, 8185, 7831, 8196, 8057, 9775, 6468] avg=8192.0000 max=9938 max_rank=1 min=6468 min_rank=7 imbalance_ratio=1.213135 max_load_reduction=55598 max_load_reduction_pct=0.848358 moved_to_echo=32496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2862, 6033, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 1, 0, 0, 0, 0, 4, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7900, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8480, 9846, 8198, 6588, 8159, 8202, 7376, 8687] avg=8192.0000 max=9846 max_rank=1 min=6588 min_rank=3 imbalance_ratio=1.201904 max_load_reduction=55690 max_load_reduction_pct=0.849762 moved_to_echo=32424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 284, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6450, 0, 0, 0, 0, 0, 0, 4306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7905, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8479, 5116, 8196, 10756, 8194, 8202, 7848, 8745] avg=8192.0000 max=10756 max_rank=3 min=5116 min_rank=1 imbalance_ratio=1.312988 max_load_reduction=54780 max_load_reduction_pct=0.835876 moved_to_echo=32989 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5579, 4191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7674, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6772, 4656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 7875, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3360, 6231, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 2, 8, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 0, 1, 0, 1, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 1, 7239, 0, 7472, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 7090, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 13, 0, 0, 0, 0, 0, 1, 0, 0, 16, 0, 1501, 7566, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 853, 1, 3, 0, 0, 0, 0, 0, 0, 1268, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 2, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 6916, 0, 7535, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 6257, 0, 0, 0, 0, 2, 0, 8182, 0, 0, 37, 0, 0, 0, 0, 0, 3, 0, 0, 85, 0, 1098, 7921, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 0, 59, 0, 0, 0, 0, 0, 0, 2246, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 9, 0, 0, 0, 1, 4, 73, 0, 0, 0, 0, 1, 7, 0, 8063, 0, 0, 0, 0, 0, 3, 0, 0, 0, 82, 18, 0, 0, 0, 0, 6, 0, 6, 9, 6122, 0, 7629, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4717, 2203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7966, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3538, 6981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1593, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7689, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5729, 2644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6772, 3639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 0, 0, 0, 0, 0, 2, 6248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7775, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8605, 10411, 8202, 6711, 8205, 8215, 6436, 8751] avg=8192.0000 max=10411 max_rank=1 min=6436 min_rank=6 imbalance_ratio=1.270874 max_load_reduction=55125 max_load_reduction_pct=0.841141 moved_to_echo=31607 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7875, 0, 0, 0, 0, 0, 0, 413, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3360, 4870, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 2, 8, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 5775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7778, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 0, 1, 0, 1, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1334, 783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 7239, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8232, 8188, 6835, 8178, 8080, 9975, 7757] avg=8192.0000 max=9975 max_rank=6 min=6835 min_rank=3 imbalance_ratio=1.217651 max_load_reduction=55561 max_load_reduction_pct=0.847794 moved_to_echo=33990 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5579, 3276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2273, 0, 0, 0, 0, 0, 0, 5931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7779, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8605, 8855, 8190, 8204, 8188, 8212, 6560, 8722] avg=8192.0000 max=8855 max_rank=1 min=6560 min_rank=6 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=31682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6257, 0, 0, 0, 0, 2, 0, 413, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 85, 0, 1098, 6190, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 0, 59, 0, 0, 0, 0, 0, 0, 2246, 0, 0, 0, 0, 0, 5896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7769, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 9, 0, 0, 0, 1, 4, 73, 0, 0, 0, 0, 1, 7, 0, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 3, 0, 0, 0, 82, 18, 0, 0, 0, 0, 1696, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 9, 6122, 0, 527, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6709, 7383, 8185, 9670, 8191, 8065, 10662, 6671] avg=8192.0000 max=10662 max_rank=6 min=6671 min_rank=7 imbalance_ratio=1.301514 max_load_reduction=54874 max_load_reduction_pct=0.837311 moved_to_echo=33589 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7090, 0, 0, 0, 0, 0, 0, 411, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 0, 1501, 5905, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 853, 1, 3, 0, 0, 0, 0, 0, 0, 1268, 0, 0, 0, 0, 0, 5828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7778, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 2, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 38, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1623, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 6916, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7514, 7425, 8186, 7953, 8187, 8329, 10513, 7429] avg=8192.0000 max=10513 max_rank=6 min=7425 min_rank=1 imbalance_ratio=1.283325 max_load_reduction=55023 max_load_reduction_pct=0.839584 moved_to_echo=34458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4717, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4243, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7779, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8605, 6440, 8190, 10400, 8203, 8203, 6753, 8742] avg=8192.0000 max=10400 max_rank=3 min=6440 min_rank=1 imbalance_ratio=1.269531 max_load_reduction=55136 max_load_reduction_pct=0.841309 moved_to_echo=31901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3538, 5456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1593, 1, 0, 0, 0, 0, 4, 5943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7772, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 31, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8602, 8994, 8199, 7541, 8182, 8227, 7068, 8723] avg=8192.0000 max=8994 max_rank=1 min=7068 min_rank=6 imbalance_ratio=1.097900 max_load_reduction=56542 max_load_reduction_pct=0.862762 moved_to_echo=32200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5729, 2067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2085, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7779, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8605, 7796, 8196, 8222, 8202, 8205, 7569, 8741] avg=8192.0000 max=8741 max_rank=7 min=7569 min_rank=6 imbalance_ratio=1.067017 max_load_reduction=56795 max_load_reduction_pct=0.866623 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 2432, 128, 128, 0, 0, 0, 0, 0, 25216, 3968, 0, 0, 0, 0, 128, 34048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 0, 27520, 29568, 128, 0, 0, 0, 0, 0, 0, 0, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[53248, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 61184, 0, 4352, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 63232, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 63232, 0, 0, 0, 0, 128, 0, 2304, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 65280, 128, 0, 128, 0, 128, 128, 256, 0, 0, 0, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 2816, 128, 128, 0, 0, 0, 0, 0, 10752, 4224, 0, 0, 0, 0, 128, 48000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[49920, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 9088, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 0, 32384, 33152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 61312, 0, 4352, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 62208, 0, 0, 0, 0, 128, 0, 3328, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 62336, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 65152, 128, 0, 128, 0, 128, 128, 256, 0, 0, 0, 0, 128, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8111, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 254, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8173, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6015, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 8191, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 0, 0, 0, 1, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2390, 0, 1, 1, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8159, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6484, 8165, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8147, 29, 7, 0, 8182, 0, 8136, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7706, 0, 0, 0, 0, 0, 7, 0, 4, 0, 9, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 1, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2198, 0, 4, 10, 8179, 0, 0, 0, 10, 0, 2, 2, 14, 0, 0, 74, 0, 0, 0, 37, 0, 0, 0, 10, 6, 0, 0, 0, 0, 7966, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8157, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7169, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8188, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6050, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 8, 2, 0, 8190, 0, 8183, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2191, 0, 0, 2, 8189, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8083, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7291, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 7145, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9011, 8194, 8149, 8193, 8165, 8194, 7400, 8230] avg=8192.0000 max=9011 max_rank=0 min=7400 min_rank=6 imbalance_ratio=1.099976 max_load_reduction=56525 max_load_reduction_pct=0.862503 moved_to_echo=31989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7292, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 7146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9086, 8192, 8254, 8191, 8166, 8193, 7224, 8230] avg=8192.0000 max=9086 max_rank=0 min=7224 min_rank=6 imbalance_ratio=1.109131 max_load_reduction=56450 max_load_reduction_pct=0.861359 moved_to_echo=31813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7292, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 7146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9073, 8192, 8316, 8191, 8166, 8193, 7175, 8230] avg=8192.0000 max=9073 max_rank=0 min=7175 min_rank=6 imbalance_ratio=1.107544 max_load_reduction=56463 max_load_reduction_pct=0.861557 moved_to_echo=31764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 0, 0, 0, 1, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7287, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2390, 0, 1, 1, 7145, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8159, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6914, 8188, 8115, 8196, 8165, 8192, 9568, 8198] avg=8192.0000 max=9568 max_rank=6 min=6914 min_rank=0 imbalance_ratio=1.167969 max_load_reduction=55968 max_load_reduction_pct=0.854004 moved_to_echo=34123 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6484, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 7, 0, 1, 0, 8136, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7706, 0, 0, 0, 0, 0, 7, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 9, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 1, 0, 0, 38, 0, 0, 0, 0, 0, 0, 7252, 872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2198, 0, 4, 10, 7136, 0, 0, 0, 10, 0, 2, 2, 14, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 10, 6, 0, 0, 0, 0, 7966, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7384, 8182, 7846, 8182, 8245, 8190, 9450, 8057] avg=8192.0000 max=9450 max_rank=6 min=7384 min_rank=0 imbalance_ratio=1.153564 max_load_reduction=56086 max_load_reduction_pct=0.855804 moved_to_echo=33942 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6050, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 8183, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 2, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8185, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 7286, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2191, 0, 0, 2, 7133, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8083, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6942, 8196, 8234, 8194, 8299, 8192, 9348, 8131] avg=8192.0000 max=9348 max_rank=6 min=6942 min_rank=0 imbalance_ratio=1.141113 max_load_reduction=56188 max_load_reduction_pct=0.857361 moved_to_echo=33970 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7290, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 0, 0, 0, 7140, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8069, 8200, 8317, 8198, 8164, 8189, 8169, 8230] avg=8192.0000 max=8317 max_rank=2 min=8069 min_rank=0 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7292, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 7146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9057, 8192, 8305, 8191, 8166, 8193, 7202, 8230] avg=8192.0000 max=9057 max_rank=0 min=7202 min_rank=6 imbalance_ratio=1.105591 max_load_reduction=56479 max_load_reduction_pct=0.861801 moved_to_echo=31791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 144, 8162, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8150, 33, 7, 0, 8181, 0, 8143, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 0, 0, 0, 0, 7, 0, 4, 0, 18, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 1, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7999, 0, 4, 10, 8179, 0, 0, 0, 11, 0, 2, 2, 22, 0, 0, 235, 0, 0, 0, 57, 0, 0, 0, 10, 6, 0, 0, 0, 0, 8039, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 959, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7233, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 436, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8187, 2, 1, 0, 8191, 0, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 0, 1, 1, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8165, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 764, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7429, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 285, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8188, 0, 8191, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7899, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1379, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6809, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2793, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 8191, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 5395, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 269, 8184, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8180, 10, 2, 0, 8190, 0, 8183, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8057, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7909, 0, 0, 2, 8189, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8143, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 7, 0, 0, 0, 8141, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 0, 0, 0, 0, 7, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 18, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 1, 0, 0, 95, 0, 0, 0, 0, 0, 0, 7356, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 0, 4, 10, 823, 0, 0, 0, 11, 0, 2, 2, 22, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 10, 6, 0, 0, 0, 0, 8039, 1, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7433, 8189, 7969, 8188, 8325, 8193, 9108, 8131] avg=8192.0000 max=9108 max_rank=6 min=7433 min_rank=0 imbalance_ratio=1.111816 max_load_reduction=56428 max_load_reduction_pct=0.861023 moved_to_echo=33757 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7368, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7233, 0, 0, 0, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8190, 8266, 8191, 8150, 8193, 8057, 8210] avg=8192.0000 max=8279 max_rank=0 min=8057 min_rank=6 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32610 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 436, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 0, 0, 1, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7366, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 0, 1, 1, 824, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8165, 0, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7752, 8189, 8105, 8196, 8154, 8193, 8763, 8184] avg=8192.0000 max=8763 max_rank=6 min=7752 min_rank=0 imbalance_ratio=1.069702 max_load_reduction=56773 max_load_reduction_pct=0.866287 moved_to_echo=33294 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7368, 782, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7429, 0, 0, 0, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8084, 8190, 8265, 8191, 8151, 8192, 8253, 8210] avg=8192.0000 max=8265 max_rank=2 min=8084 min_rank=0 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7361, 782, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7899, 0, 0, 0, 824, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7605, 8198, 8265, 8196, 8146, 8188, 8728, 8210] avg=8192.0000 max=8728 max_rank=6 min=7605 min_rank=0 imbalance_ratio=1.065430 max_load_reduction=56808 max_load_reduction_pct=0.866821 moved_to_echo=33272 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7367, 782, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5395, 0, 0, 0, 824, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10113, 8193, 8265, 8192, 8150, 8193, 6220, 8210] avg=8192.0000 max=10113 max_rank=0 min=6220 min_rank=6 imbalance_ratio=1.234497 max_load_reduction=55423 max_load_reduction_pct=0.845688 moved_to_echo=30773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 0, 0, 0, 8177, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8057, 0, 0, 0, 0, 0, 2, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 7367, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7909, 0, 0, 2, 822, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8143, 0, 0, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7571, 8193, 8135, 8191, 8309, 8192, 8774, 8171] avg=8192.0000 max=8774 max_rank=6 min=7571 min_rank=0 imbalance_ratio=1.071045 max_load_reduction=56762 max_load_reduction_pct=0.866119 moved_to_echo=33446 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7368, 782, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6809, 0, 0, 0, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8699, 8194, 8266, 8191, 8151, 8192, 7633, 8210] avg=8192.0000 max=8699 max_rank=0 min=7633 min_rank=6 imbalance_ratio=1.061890 max_load_reduction=56837 max_load_reduction_pct=0.867264 moved_to_echo=32186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 58368, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 0, 256, 0, 0, 128, 0, 0, 0, 58368, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 0, 128, 128, 57216, 0, 0, 0, 128, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 64512, 0, 0, 0, 0, 0, 128, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[58624, 0, 128, 128, 6656, 0, 0, 0, 128, 0, 128, 128, 128, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 7040, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 58624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 256, 0, 0, 0, 128, 128, 0, 0, 256, 0, 0, 128, 0, 0, 0, 59008, 6272, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65408, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65024, 0, 0, 0, 0, 0, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65280, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8178, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8189, 0, 8192, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8186, 0, 8192, 0, 0, 4, 0, 0, 0, 8181, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 20, 0, 0, 8179, 0, 0, 0, 4, 0, 0, 87, 0, 2011, 0, 0, 2, 0, 0, 0, 7944, 0, 0, 0, 0, 5666, 0, 8131, 0, 0, 0, 5, 5, 0, 8178, 0, 5, 0, 0, 8145, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 7, 8116, 0, 10, 8181, 0, 0, 5, 2, 2, 0, 0, 0, 489, 0, 0, 0, 3, 73, 0, 0, 7, 0, 0, 0, 0, 8, 0, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8178, 0, 8192, 0, 0, 1, 0, 0, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 12, 0, 180, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 7775, 0, 8184, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 8185, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 25, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8179, 0, 0, 8189, 0, 0, 2, 1, 0, 0, 0, 0, 214, 0, 0, 0, 0, 63, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 1, 0, 29, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 7936, 0, 8190, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8190, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8194, 8301, 8141, 8187, 8199, 8125, 8194] avg=8192.0000 max=8301 max_rank=2 min=8125 min_rank=6 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32705 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8193, 8300, 8139, 8188, 8199, 8126, 8196] avg=8192.0000 max=8300 max_rank=2 min=8126 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32709 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8141, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 8176, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 2011, 0, 0, 2, 0, 0, 0, 109, 0, 0, 0, 0, 5666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 5, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 7, 7, 0, 0, 0, 0, 8167, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8116, 0, 10, 7, 0, 0, 5, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 3, 73, 0, 0, 7, 0, 0, 0, 0, 8, 0, 8, 7825, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8200, 7875, 8292, 8197, 8149, 8469, 8190] avg=8192.0000 max=8469 max_rank=6 min=7875 min_rank=2 imbalance_ratio=1.033813 max_load_reduction=57067 max_load_reduction_pct=0.870773 moved_to_echo=33005 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8189, 8306, 8139, 8188, 8199, 8126, 8193] avg=8192.0000 max=8306 max_rank=2 min=8126 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32706 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8186, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 29, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8177, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 7976, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8193, 8077, 8362, 8188, 8198, 8130, 8193] avg=8192.0000 max=8362 max_rank=3 min=8077 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32709 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 180, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 7775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 25, 2, 0, 0, 0, 0, 8177, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8179, 0, 0, 3, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 63, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 7956, 60, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8183, 8073, 8180, 8217, 8187, 8308, 8195] avg=8192.0000 max=8308 max_rank=6 min=8073 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32907 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8193, 8298, 8143, 8184, 8206, 8126, 8184] avg=8192.0000 max=8298 max_rank=2 min=8126 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32700 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8191, 8306, 8140, 8187, 8199, 8126, 8191] avg=8192.0000 max=8306 max_rank=2 min=8126 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8186, 0, 8192, 0, 0, 4, 0, 0, 0, 8181, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8184, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 20, 0, 0, 8179, 0, 0, 0, 8, 0, 0, 99, 0, 438, 0, 0, 2, 0, 0, 0, 7937, 0, 0, 0, 0, 7184, 0, 8129, 0, 0, 0, 6, 5, 0, 8178, 0, 5, 0, 0, 8142, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 21, 7, 0, 0, 0, 0, 0, 0, 0, 7, 8113, 0, 11, 8181, 0, 0, 5, 2, 0, 0, 0, 0, 536, 0, 0, 0, 4, 80, 0, 0, 8, 0, 0, 0, 0, 9, 0, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 7965, 0, 8190, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8189, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 2, 0, 0, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 13, 0, 8, 0, 0, 0, 0, 0, 0, 8068, 0, 0, 0, 0, 7923, 0, 8185, 0, 0, 0, 1, 1, 0, 8189, 0, 0, 0, 0, 8184, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 49, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8178, 0, 0, 8189, 0, 0, 2, 1, 0, 0, 0, 0, 221, 0, 0, 0, 0, 62, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8181, 0, 8192, 0, 0, 1, 0, 0, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8308, 8141, 8183, 8201, 8120, 8196] avg=8192.0000 max=8308 max_rank=2 min=8120 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32700 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8138, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 8175, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 438, 0, 0, 2, 0, 0, 0, 116, 0, 0, 0, 0, 7184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 6, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 21, 7, 0, 0, 0, 0, 8165, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8113, 0, 11, 9, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 0, 0, 4, 80, 0, 0, 8, 0, 0, 0, 0, 9, 0, 8, 7814, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8203, 7839, 8277, 8205, 8147, 8514, 8190] avg=8192.0000 max=8514 max_rank=6 min=7839 min_rank=2 imbalance_ratio=1.039307 max_load_reduction=57022 max_load_reduction_pct=0.870087 moved_to_echo=33056 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8195, 8307, 8145, 8181, 8202, 8120, 8184] avg=8192.0000 max=8307 max_rank=2 min=8120 min_rank=6 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32687 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8187, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 7965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8175, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 7963, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8087, 8363, 8183, 8200, 8123, 8194] avg=8192.0000 max=8363 max_rank=3 min=8087 min_rank=2 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32700 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8193, 8310, 8143, 8182, 8201, 8120, 8192] avg=8192.0000 max=8310 max_rank=2 min=8120 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32695 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 49, 2, 0, 0, 0, 0, 8175, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8178, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 62, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 7944, 57, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8183, 8058, 8184, 8237, 8183, 8299, 8195] avg=8192.0000 max=8299 max_rank=6 min=8058 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32914 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8188, 8313, 8141, 8183, 8201, 8120, 8194] avg=8192.0000 max=8313 max_rank=2 min=8120 min_rank=6 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32698 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8190, 8314, 8142, 8182, 8201, 8120, 8191] avg=8192.0000 max=8314 max_rank=2 min=8120 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 2304, 0, 0, 128, 0, 0, 0, 896, 0, 0, 0, 0, 62336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[65152, 0, 0, 128, 128, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 512, 128, 0, 128, 0, 0, 0, 1024, 0, 0, 0, 0, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[896, 0, 0, 0, 128, 256, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65152, 0, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7446, 756, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5048, 3180, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 233, 0, 0, 2, 11, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 0, 7218, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 0, 0, 8189, 12, 0, 1076, 6896, 1, 0, 0, 2, 0, 2, 0, 0, 8188, 0, 2, 0, 0, 0, 0, 7795, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 8142, 0, 0, 579, 0, 0, 9, 79, 504, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 7, 0, 9, 0, 0, 5123, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 8176, 102, 0, 2951, 5163, 1, 0, 4, 8, 0, 1, 0, 0, 8167, 0, 8, 0, 6, 0, 0, 7679, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 7722, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 74, 0, 0, 1, 11, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 6599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 4331, 4326, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 8119, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5365, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 6506, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4882, 3364, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 611, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5001, 3180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 45, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7941, 8191, 8885, 7581, 8181, 8319, 8244, 8194] avg=8192.0000 max=8885 max_rank=2 min=7581 min_rank=3 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=32938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7940, 0, 0, 74, 0, 0, 1, 11, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 611, 81, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 0, 0, 0, 0, 0, 0, 7579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4290, 4326, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 39, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8192, 7294, 8579, 8618, 8254, 8248, 8194] avg=8192.0000 max=8618 max_rank=4 min=7294 min_rank=2 imbalance_ratio=1.052002 max_load_reduction=56918 max_load_reduction_pct=0.868500 moved_to_echo=33314 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 611, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7376, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 67, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7942, 8192, 8891, 7581, 8132, 8311, 8293, 8194] avg=8192.0000 max=8891 max_rank=2 min=7581 min_rank=3 imbalance_ratio=1.085327 max_load_reduction=56645 max_load_reduction_pct=0.864334 moved_to_echo=32930 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 611, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6445, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 58, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7941, 8191, 8885, 7581, 8137, 8320, 8286, 8195] avg=8192.0000 max=8885 max_rank=2 min=7581 min_rank=3 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=32938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 7893, 0, 0, 579, 0, 0, 9, 79, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 7, 0, 9, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5123, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 610, 80, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 7566, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2924, 5163, 1, 0, 4, 8, 0, 1, 0, 0, 0, 0, 8, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7679, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 7722, 0, 26, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9069, 8198, 5879, 10479, 8115, 7817, 7802, 8177] avg=8192.0000 max=10479 max_rank=3 min=5879 min_rank=2 imbalance_ratio=1.279175 max_load_reduction=55057 max_load_reduction_pct=0.840103 moved_to_echo=31911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7923, 0, 0, 233, 0, 0, 2, 11, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7218, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 613, 85, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 0, 0, 7576, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 6896, 1, 0, 0, 2, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7795, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8104, 0, 14, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8589, 8192, 7932, 8573, 7961, 7932, 8162, 8195] avg=8192.0000 max=8589 max_rank=0 min=7932 min_rank=2 imbalance_ratio=1.048462 max_load_reduction=56947 max_load_reduction_pct=0.868942 moved_to_echo=32250 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7941, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 611, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4836, 3364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 44, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7956, 8189, 8885, 7581, 8200, 8263, 8269, 8193] avg=8192.0000 max=8885 max_rank=2 min=7581 min_rank=3 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=32925 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 611, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5315, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 48, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7941, 8191, 8885, 7581, 8192, 8320, 8232, 8194] avg=8192.0000 max=8885 max_rank=2 min=7581 min_rank=3 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=32938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7402, 826, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5662, 2542, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 87, 0, 0, 1, 6, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 6936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 3855, 4440, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 8169, 1, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3258, 4954, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 287, 0, 0, 2, 5, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 0, 7415, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 8189, 16, 0, 973, 6696, 1, 0, 0, 2, 0, 2, 0, 0, 8188, 0, 2, 0, 0, 0, 0, 8027, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 8141, 0, 0, 625, 0, 0, 9, 29, 461, 0, 0, 0, 11, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 7, 0, 9, 0, 0, 5446, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2841, 0, 0, 0, 0, 0, 0, 8176, 103, 0, 2189, 5362, 1, 0, 5, 9, 0, 1, 0, 0, 8166, 0, 8, 0, 6, 0, 0, 8057, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 7592, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4588, 3607, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5519, 2712, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7402, 826, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7945, 8190, 8777, 7588, 8343, 8232, 8267, 8194] avg=8192.0000 max=8777 max_rank=2 min=7588 min_rank=3 imbalance_ratio=1.071411 max_load_reduction=56759 max_load_reduction_pct=0.866074 moved_to_echo=33036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 491, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5662, 2542, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7946, 8190, 8779, 7588, 8319, 8230, 8290, 8194] avg=8192.0000 max=8779 max_rank=2 min=7588 min_rank=3 imbalance_ratio=1.071655 max_load_reduction=56757 max_load_reduction_pct=0.866043 moved_to_echo=33033 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7944, 0, 0, 87, 0, 0, 1, 6, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8076, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 491, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 0, 0, 0, 0, 0, 0, 7586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 4440, 0, 0, 0, 1, 0, 0, 0, 0, 115, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 1, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8192, 7522, 8598, 8412, 8217, 8233, 8194] avg=8192.0000 max=8598 max_rank=3 min=7522 min_rank=2 imbalance_ratio=1.049561 max_load_reduction=56938 max_load_reduction_pct=0.868805 moved_to_echo=33056 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 7895, 0, 0, 625, 0, 0, 9, 29, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 7, 0, 9, 0, 0, 8052, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5446, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 490, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2841, 0, 0, 0, 0, 0, 0, 7574, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 5362, 1, 0, 5, 9, 0, 1, 0, 0, 114, 0, 8, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8057, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 7592, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9024, 8197, 6098, 10523, 7695, 8108, 7715, 8176] avg=8192.0000 max=10523 max_rank=3 min=6098 min_rank=2 imbalance_ratio=1.284546 max_load_reduction=55013 max_load_reduction_pct=0.839432 moved_to_echo=31694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7945, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3258, 4954, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7959, 8188, 8777, 7588, 8327, 8210, 8293, 8194] avg=8192.0000 max=8777 max_rank=2 min=7588 min_rank=3 imbalance_ratio=1.071411 max_load_reduction=56759 max_load_reduction_pct=0.866074 moved_to_echo=33024 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4588, 3607, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7945, 8190, 8777, 7588, 8310, 8232, 8300, 8194] avg=8192.0000 max=8777 max_rank=2 min=7588 min_rank=3 imbalance_ratio=1.071411 max_load_reduction=56759 max_load_reduction_pct=0.866074 moved_to_echo=33036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7921, 0, 0, 287, 0, 0, 2, 5, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 0, 8080, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7415, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 498, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 7577, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 973, 6696, 1, 0, 0, 2, 0, 2, 0, 0, 108, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8027, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8059, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8604, 8199, 8029, 8475, 7784, 8075, 8174, 8196] avg=8192.0000 max=8604 max_rank=0 min=7784 min_rank=4 imbalance_ratio=1.050293 max_load_reduction=56932 max_load_reduction_pct=0.868713 moved_to_echo=32229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5519, 2712, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7945, 8190, 8777, 7588, 8346, 8232, 8264, 8194] avg=8192.0000 max=8777 max_rank=2 min=7588 min_rank=3 imbalance_ratio=1.071411 max_load_reduction=56759 max_load_reduction_pct=0.866074 moved_to_echo=33036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[59904, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 4992, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 60672, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 63488, 0, 0, 1024, 0, 0, 128, 128, 1152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[37248, 28288, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 64512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 64896, 0, 384, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 0, 0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[60800, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 3968, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 4736, 0, 0, 0, 0, 0, 0, 60800, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[33536, 31232, 128, 0, 128, 128, 0, 128, 0, 0, 1024, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 64640, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 63488, 0, 0, 1024, 0, 0, 128, 128, 1024, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 64640, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65280, 128, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True - [2026-08-26 16:18:08] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 444.2 | throughput per GPU (TFLOP/s/GPU): 407.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.543447E+00 | loss scale: 1.0 | grad norm: 5.356 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 2984, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 2100, 0, 1, 2411, 0, 0, 0, 8191, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 4127, 0, 4385, 5, 0, 0, 4464, 1, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 1, 0, 0, 0, 0, 874, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 7790, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 121, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1623, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2776, 0, 0, 2854, 2, 0, 0, 8192, 0, 8192, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2832, 0, 0, 0, 4821, 0, 5193, 6, 1, 0, 4176, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 2, 1, 1, 2, 0, 912, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7675, 0, 0, 5, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2691, 0, 0, 2176, 2, 0, 0, 8192, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 0, 0, 0, 4457, 0, 5282, 6, 0, 0, 4522, 0, 0, 0, 0, 0, 1, 0, 0, 7920, 0, 1, 0, 0, 3, 0, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2679, 0, 0, 3067, 0, 0, 0, 8192, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3241, 0, 0, 0, 4243, 0, 3886, 10, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 7912, 0, 2, 0, 0, 1, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7649, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2157, 0, 0, 0, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2651, 0, 0, 3021, 2, 0, 0, 8192, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2747, 0, 6, 0, 4418, 0, 5071, 3, 0, 0, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 10, 0, 0, 4, 0, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7807, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 203, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 7, 2788, 0, 0, 1, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2670, 0, 0, 2709, 2, 1, 0, 8191, 0, 8192, 2, 10, 0, 2, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 3407, 0, 0, 0, 4570, 0, 4380, 19, 0, 1, 3776, 0, 0, 0, 0, 0, 1, 0, 0, 7849, 0, 3, 0, 6, 1, 0, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 6, 0, 1, 0, 17, 0, 2, 0, 11, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 18, 2263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 2384, 0, 1, 2635, 0, 0, 0, 8192, 0, 8190, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 0, 4272, 0, 4400, 2, 2, 0, 4810, 0, 0, 0, 0, 0, 1, 0, 0, 8022, 0, 1, 0, 2, 0, 0, 901, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 7654, 0, 0, 11, 0, 0, 0, 1, 0, 3, 0, 0, 4, 0, 0, 0, 0, 77, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 2, 1786, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3044, 0, 1, 3784, 6, 0, 0, 8183, 0, 8184, 1, 0, 0, 2, 1, 0, 0, 2, 0, 3, 2, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 2515, 1, 1, 0, 4739, 0, 4347, 1, 0, 0, 3666, 2, 0, 0, 1, 0, 1, 0, 1, 8094, 1, 6, 0, 0, 0, 1, 981, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 7949, 0, 0, 1, 0, 0, 0, 7, 0, 3, 0, 0, 2, 0, 0, 0, 1, 21, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 2343, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2691, 0, 0, 2176, 2, 0, 0, 0, 0, 2727, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 0, 0, 0, 910, 0, 0, 6, 0, 0, 4522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7175, 0, 1, 0, 0, 3, 0, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 0, 4, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 5465, 2939, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 7610, 8188, 8191, 8501, 8105, 8154, 8665] avg=8192.0000 max=8665 max_rank=7 min=7610 min_rank=1 imbalance_ratio=1.057739 max_load_reduction=56871 max_load_reduction_pct=0.867783 moved_to_echo=33425 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 2984, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 3284, 1945, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2100, 0, 1, 2411, 0, 0, 0, 0, 0, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 843, 0, 0, 5, 0, 0, 4464, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7181, 0, 1, 0, 0, 0, 0, 874, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 7790, 0, 0, 4, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 121, 0, 0, 0, 0, 5463, 2440, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8534, 7239, 8184, 8192, 9067, 8058, 8225, 8037] avg=8192.0000 max=9067 max_rank=4 min=7239 min_rank=1 imbalance_ratio=1.106812 max_load_reduction=56469 max_load_reduction_pct=0.861649 moved_to_echo=33387 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1623, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3836, 2304, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2776, 0, 0, 2854, 2, 0, 0, 0, 0, 2727, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2832, 0, 0, 0, 985, 0, 0, 6, 1, 0, 4176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7198, 0, 2, 1, 1, 2, 0, 912, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7675, 0, 0, 5, 1, 1, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 5465, 2889, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 8360, 8188, 8193, 8000, 8118, 8113, 8483] avg=8192.0000 max=8483 max_rank=7 min=8000 min_rank=4 imbalance_ratio=1.035522 max_load_reduction=57053 max_load_reduction_pct=0.870560 moved_to_echo=32714 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3376, 1725, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2679, 0, 0, 3067, 0, 0, 0, 0, 0, 2727, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 6, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3241, 0, 0, 0, 867, 0, 0, 10, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7169, 0, 2, 0, 0, 1, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7649, 0, 0, 3, 0, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 5465, 2161, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 8482, 8198, 8194, 8133, 8497, 8080, 7874] avg=8192.0000 max=8497 max_rank=5 min=7874 min_rank=7 imbalance_ratio=1.037231 max_load_reduction=57039 max_load_reduction_pct=0.870346 moved_to_echo=32584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 7, 2788, 0, 0, 1, 1, 0, 0, 0, 0, 3, 0, 3637, 1943, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 0, 0, 2709, 2, 1, 0, 0, 0, 2727, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 0, 0, 933, 0, 0, 19, 0, 1, 3776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7111, 0, 3, 0, 6, 1, 0, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 6, 0, 1, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 2, 0, 11, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 5465, 2437, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8697, 8121, 8190, 8195, 8136, 7928, 8217, 8052] avg=8192.0000 max=8697 max_rank=0 min=7928 min_rank=5 imbalance_ratio=1.061646 max_load_reduction=56839 max_load_reduction_pct=0.867294 moved_to_echo=32333 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2157, 0, 0, 0, 2, 0, 0, 0, 1, 1, 0, 3520, 2250, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2651, 0, 0, 3021, 2, 0, 0, 0, 0, 2723, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 19, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2747, 0, 6, 0, 898, 0, 0, 3, 0, 0, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7269, 0, 10, 0, 0, 4, 0, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7807, 0, 0, 4, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 203, 0, 0, 0, 0, 5469, 2821, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8411, 8208, 8197, 7376, 8330, 8256, 8502] avg=8192.0000 max=8502 max_rank=7 min=7376 min_rank=4 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=32464 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 2, 1786, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3771, 1929, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3044, 0, 1, 3784, 6, 0, 0, 0, 0, 2725, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 2, 0, 3, 2, 0, 0, 0, 1, 1, 0, 1, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 8179, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2515, 1, 1, 0, 968, 0, 0, 1, 0, 0, 3666, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 7334, 1, 6, 0, 0, 0, 1, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 7949, 0, 0, 1, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 2, 0, 0, 0, 1, 21, 0, 0, 0, 0, 5459, 2418, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7814, 9561, 8187, 8184, 7154, 8326, 8390, 7920] avg=8192.0000 max=9561 max_rank=1 min=7154 min_rank=4 imbalance_ratio=1.167114 max_load_reduction=55975 max_load_reduction_pct=0.854111 moved_to_echo=31790 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 18, 2263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3399, 1953, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 2384, 0, 1, 2635, 0, 0, 0, 0, 0, 2727, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 4, 0, 0, 2, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 0, 873, 0, 0, 2, 2, 0, 4810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7268, 0, 1, 0, 2, 0, 0, 901, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 7654, 0, 0, 11, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4, 0, 0, 0, 0, 77, 0, 0, 0, 0, 5463, 2447, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7954, 7752, 8193, 8190, 9169, 8174, 8101, 8003] avg=8192.0000 max=9169 max_rank=4 min=7752 min_rank=1 imbalance_ratio=1.119263 max_load_reduction=56367 max_load_reduction_pct=0.860092 moved_to_echo=33447 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True + [2026-08-26 16:27:16] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 434.7 | throughput per GPU (TFLOP/s/GPU): 416.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.547873E+00 | loss scale: 1.0 | grad norm: 5.350 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 7, 7, 3163, 0, 0, 1, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 2729, 0, 0, 2634, 2, 1, 0, 8191, 0, 8192, 2, 4, 0, 3, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 3309, 0, 0, 0, 4576, 0, 4696, 42, 0, 1, 4127, 0, 0, 0, 0, 0, 1, 0, 0, 6736, 0, 6, 0, 6, 1, 0, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7693, 0, 0, 0, 0, 1, 0, 19, 0, 2, 0, 13, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 2, 2079, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 0, 1, 3576, 9, 0, 0, 8183, 0, 8184, 1, 0, 0, 2, 1, 0, 0, 2, 0, 3, 2, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 2415, 1, 1, 0, 4742, 0, 4636, 3, 0, 0, 3909, 2, 0, 0, 1, 0, 1, 0, 1, 7347, 1, 12, 0, 0, 0, 1, 989, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 7849, 0, 0, 1, 0, 0, 0, 7, 0, 3, 0, 0, 2, 0, 0, 0, 1, 294, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 3350, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 2189, 0, 1, 2365, 0, 0, 0, 8191, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 4117, 0, 4688, 29, 0, 0, 4866, 1, 0, 0, 0, 0, 0, 0, 0, 6770, 0, 5, 0, 0, 0, 0, 938, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 7681, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 243, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1961, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 0, 2745, 2, 0, 0, 8192, 0, 8192, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 0, 4883, 0, 5523, 20, 1, 0, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 6613, 0, 11, 1, 1, 1, 0, 991, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7613, 0, 0, 4, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 306, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2802, 0, 0, 2150, 2, 0, 0, 8192, 0, 8192, 17, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2995, 0, 0, 0, 4449, 0, 5586, 15, 0, 0, 4931, 0, 0, 0, 0, 0, 1, 0, 0, 6739, 0, 17, 0, 0, 4, 0, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7659, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2586, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2711, 0, 0, 2950, 2, 0, 0, 8192, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 0, 6, 0, 4356, 0, 5338, 6, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 7085, 0, 31, 0, 0, 4, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7723, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 344, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3118, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2792, 0, 0, 2970, 0, 0, 0, 8192, 0, 8192, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3139, 0, 0, 0, 4232, 0, 4173, 21, 0, 0, 4336, 0, 0, 0, 0, 0, 0, 0, 0, 6863, 0, 12, 0, 0, 1, 0, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7575, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 19, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 2449, 0, 1, 2513, 0, 0, 0, 8192, 0, 8190, 1, 0, 0, 0, 8, 0, 0, 0, 0, 1, 1, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 3389, 0, 0, 0, 4242, 0, 4705, 6, 2, 0, 5253, 0, 0, 0, 0, 0, 1, 0, 0, 7017, 0, 1, 0, 1, 0, 0, 946, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 7537, 0, 0, 11, 0, 0, 0, 1, 0, 4, 0, 0, 4, 0, 0, 0, 0, 242, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 7, 7, 3163, 0, 0, 1, 2, 0, 0, 0, 0, 2, 0, 2788, 2382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2729, 0, 0, 2634, 2, 1, 0, 0, 0, 2742, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3309, 0, 0, 0, 4576, 0, 0, 42, 0, 1, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6736, 0, 6, 0, 6, 1, 0, 842, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7693, 0, 0, 0, 0, 1, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2, 0, 13, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 5450, 2314, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8353, 8116, 8188, 8195, 8548, 7834, 8169, 8133] avg=8192.0000 max=8548 max_rank=4 min=7834 min_rank=5 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 3350, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 3288, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2189, 0, 1, 2365, 0, 0, 0, 0, 0, 2741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 4117, 0, 0, 29, 0, 0, 730, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6770, 0, 5, 0, 0, 0, 0, 938, 2, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 7681, 0, 0, 4, 0, 0, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 243, 0, 0, 0, 0, 5449, 2310, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9023, 7297, 8183, 8193, 8575, 8000, 8248, 8017] avg=8192.0000 max=9023 max_rank=0 min=7297 min_rank=1 imbalance_ratio=1.101440 max_load_reduction=56513 max_load_reduction_pct=0.862320 moved_to_echo=32840 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1961, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3121, 2802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 0, 2745, 2, 0, 0, 0, 0, 2742, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 0, 4883, 0, 0, 20, 1, 0, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6613, 0, 11, 1, 1, 1, 0, 991, 2, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7613, 0, 0, 4, 1, 1, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 306, 0, 0, 0, 0, 5450, 2721, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7888, 8363, 8187, 8193, 8372, 7891, 8152, 8490] avg=8192.0000 max=8490 max_rank=7 min=7888 min_rank=0 imbalance_ratio=1.036377 max_load_reduction=57046 max_load_reduction_pct=0.870453 moved_to_echo=32905 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 2, 2079, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2641, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 0, 1, 3576, 9, 0, 0, 0, 0, 2739, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 2, 0, 3, 2, 0, 0, 0, 1, 1, 0, 1, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 8179, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2415, 1, 1, 0, 4742, 0, 0, 3, 0, 0, 587, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 7347, 1, 12, 0, 0, 0, 1, 989, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 7849, 0, 0, 1, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 2, 0, 0, 0, 1, 294, 0, 0, 0, 0, 5445, 2284, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7079, 9406, 8186, 8184, 7751, 8582, 8302, 8046] avg=8192.0000 max=9406 max_rank=1 min=7079 min_rank=0 imbalance_ratio=1.148193 max_load_reduction=56130 max_load_reduction_pct=0.856476 moved_to_echo=32681 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3332, 2834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2802, 0, 0, 2150, 2, 0, 0, 0, 0, 2742, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2995, 0, 0, 0, 4449, 0, 0, 15, 0, 0, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6739, 0, 17, 0, 0, 4, 0, 954, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7659, 0, 0, 4, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0, 5450, 2752, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8449, 7713, 8188, 8191, 8199, 8004, 8231, 8561] avg=8192.0000 max=8561 max_rank=7 min=7713 min_rank=1 imbalance_ratio=1.045044 max_load_reduction=56975 max_load_reduction_pct=0.869370 moved_to_echo=32995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2586, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 2720, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2711, 0, 0, 2950, 2, 0, 0, 0, 0, 2739, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 21, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 0, 6, 0, 4356, 0, 0, 6, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7085, 0, 31, 0, 0, 4, 0, 1075, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7723, 0, 0, 3, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 344, 0, 0, 0, 0, 5453, 2629, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8021, 8417, 8208, 8196, 7620, 8435, 8201, 8438] avg=8192.0000 max=8438 max_rank=7 min=7620 min_rank=4 imbalance_ratio=1.030029 max_load_reduction=57098 max_load_reduction_pct=0.871246 moved_to_echo=32694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3118, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2930, 2118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2792, 0, 0, 2970, 0, 0, 0, 0, 0, 2742, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 7, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3139, 0, 0, 0, 4232, 0, 0, 21, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6863, 0, 12, 0, 0, 1, 0, 1385, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7575, 0, 0, 4, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 0, 0, 5450, 2055, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8514, 8198, 8194, 8043, 8515, 8078, 7824] avg=8192.0000 max=8515 max_rank=5 min=7824 min_rank=7 imbalance_ratio=1.039429 max_load_reduction=57021 max_load_reduction_pct=0.870071 moved_to_echo=32460 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 19, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3549, 2388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 2449, 0, 1, 2513, 0, 0, 0, 0, 0, 2741, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 1, 0, 0, 4, 0, 0, 2, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3389, 0, 0, 0, 4242, 0, 0, 6, 2, 0, 789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7017, 0, 1, 0, 1, 0, 0, 946, 1, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 7537, 0, 0, 11, 0, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 4, 0, 0, 0, 0, 242, 0, 0, 0, 0, 5449, 2317, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8553, 7710, 8198, 8190, 8428, 8275, 8155, 8027] avg=8192.0000 max=8553 max_rank=0 min=7710 min_rank=1 imbalance_ratio=1.044067 max_load_reduction=56983 max_load_reduction_pct=0.869492 moved_to_echo=32885 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 0, 128, 128, 18304, 128, 128, 128, 128, 0, 0, 128, 128, 128, 128, 28416, 16512, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 21120, 0, 128, 22784, 128, 128, 0, 0, 0, 21888, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 25088, 128, 128, 0, 7296, 0, 0, 128, 128, 128, 33152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 0, 0, 128, 128, 1152, 0, 0, 0, 0, 43776, 20608, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 57728, 128, 128, 128, 128, 128, 128, 7808, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 128, 0, 128, 62080, 0, 0, 128, 128, 128, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 21248, 128, 128, 128, 128, 0, 0, 128, 0, 128, 128, 24448, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 21632, 0, 128, 22016, 128, 128, 0, 0, 0, 22016, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 24448, 128, 128, 0, 35712, 0, 0, 256, 128, 128, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 0, 0, 128, 128, 2432, 0, 0, 0, 0, 43648, 19456, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 55296, 128, 128, 128, 128, 128, 128, 8192, 128, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 128, 0, 128, 61440, 0, 0, 128, 128, 128, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 128, 128, 0, 65536, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 30, 0, 0, 0, 8113, 0, 3813, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 4904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7070, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 0, 0, 8107, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 558, 0, 0, 0, 0, 0, 0, 0, 0, 6721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4307, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 8153, 1, 4787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 8190, 1, 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 1, 4805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5954, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 8140, 0, 7019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8188, 0, 6337, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 8170, 8192, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 2683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6491, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[5, 0, 189, 2, 0, 0, 7894, 0, 481, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 8190, 0, 0, 0, 479, 0, 2, 0, 0, 1, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8188, 0, 0, 0, 0, 0, 2, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 2, 0, 0, 5356, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 208, 0, 0, 0, 8077, 0, 3616, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 6993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4665, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 8059, 0, 2833, 0, 0, 9, 0, 0, 3, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 7032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5998, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 30, 0, 0, 0, 4063, 0, 3813, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7851, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2177, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 0, 7478, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7070, 0, 0, 0, 0, 1873, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7907, 8247, 7793, 8194, 8195, 7738, 8190, 9272] avg=8192.0000 max=9272 max_rank=7 min=7738 min_rank=5 imbalance_ratio=1.131836 max_load_reduction=56264 max_load_reduction_pct=0.858521 moved_to_echo=33395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 0, 0, 4059, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 0, 0, 0, 0, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2176, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1113, 0, 0, 0, 7478, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4307, 0, 0, 0, 0, 1872, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7831, 8415, 9609, 8188, 8183, 8598, 8196, 6516] avg=8192.0000 max=9609 max_rank=2 min=6516 min_rank=7 imbalance_ratio=1.172974 max_load_reduction=55927 max_load_reduction_pct=0.853378 moved_to_echo=31493 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 4082, 1, 4787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7855, 0, 1, 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 7477, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5954, 0, 0, 1, 0, 1883, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8871, 8469, 7706, 8195, 8195, 7742, 8189, 8169] avg=8192.0000 max=8871 max_rank=0 min=7706 min_rank=2 imbalance_ratio=1.082886 max_load_reduction=56665 max_load_reduction_pct=0.864639 moved_to_echo=32295 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 4076, 0, 7019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7853, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2184, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 7479, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 0, 0, 1880, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11101, 7880, 4107, 8192, 8197, 7607, 8189, 10263] avg=8192.0000 max=11101 max_rank=0 min=4107 min_rank=2 imbalance_ratio=1.355103 max_load_reduction=54435 max_load_reduction_pct=0.830612 moved_to_echo=34256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4092, 0, 6337, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7828, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2201, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 0, 0, 0, 7478, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6491, 0, 0, 0, 0, 1895, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10441, 7884, 5598, 8192, 8200, 8315, 8189, 8717] avg=8192.0000 max=10441 max_rank=0 min=5598 min_rank=2 imbalance_ratio=1.274536 max_load_reduction=55095 max_load_reduction_pct=0.840683 moved_to_echo=33421 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 208, 0, 0, 0, 4044, 0, 3616, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 0, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2168, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 687, 0, 0, 0, 7476, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4665, 0, 0, 0, 0, 1865, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7870, 8158, 9873, 8190, 8185, 8167, 8204, 6889] avg=8192.0000 max=9873 max_rank=2 min=6889 min_rank=7 imbalance_ratio=1.205200 max_load_reduction=55663 max_load_reduction_pct=0.849350 moved_to_echo=31445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 189, 2, 0, 0, 3953, 0, 481, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7855, 0, 0, 0, 0, 479, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2118, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 2049, 0, 0, 0, 7478, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 2, 0, 0, 5356, 1, 0, 0, 0, 1823, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4632, 8339, 10941, 8193, 8190, 9533, 8187, 7521] avg=8192.0000 max=10941 max_rank=2 min=4632 min_rank=0 imbalance_ratio=1.335571 max_load_reduction=54595 max_load_reduction_pct=0.833054 moved_to_echo=33431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 4035, 0, 2833, 0, 0, 9, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7852, 0, 0, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2163, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 0, 7478, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5998, 0, 0, 0, 0, 1861, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6883, 8144, 9909, 8192, 8191, 7836, 8192, 8189] avg=8192.0000 max=9909 max_rank=2 min=6883 min_rank=0 imbalance_ratio=1.209595 max_load_reduction=55627 max_load_reduction_pct=0.848801 moved_to_echo=32408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 63, 0, 0, 0, 6422, 0, 5289, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 6836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5494, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 40, 0, 0, 0, 6982, 0, 4861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 7282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3391, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 0, 0, 7700, 1, 6209, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 8190, 1, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 1, 4503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5417, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 319, 0, 0, 0, 7311, 0, 5231, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 3, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2946, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 7574, 0, 6968, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 8126, 8192, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 309, 2, 0, 0, 6655, 0, 1349, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 8190, 0, 0, 0, 241, 0, 2, 0, 0, 1, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8188, 0, 0, 0, 0, 0, 2, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 2, 2, 0, 0, 5500, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 0, 8000, 0, 7148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7750, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 0, 0, 6420, 0, 4298, 0, 0, 6, 0, 0, 3, 0, 0, 0, 0, 0, 8186, 8192, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 7307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 967, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5454, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 0, 0, 3158, 1, 6209, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8066, 0, 1, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2256, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8189, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 0, 0, 0, 7111, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5417, 0, 0, 1, 0, 2286, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9371, 8227, 6878, 8195, 8197, 7699, 8186, 8783] avg=8192.0000 max=9371 max_rank=0 min=6878 min_rank=2 imbalance_ratio=1.143921 max_load_reduction=56165 max_load_reduction_pct=0.857010 moved_to_echo=32865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 319, 0, 0, 0, 2998, 0, 5231, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2142, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 865, 0, 0, 0, 7109, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 3, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2946, 0, 0, 0, 0, 2171, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8551, 8219, 9976, 8190, 8182, 7980, 8214, 6224] avg=8192.0000 max=9976 max_rank=2 min=6224 min_rank=7 imbalance_ratio=1.217773 max_load_reduction=55560 max_load_reduction_pct=0.847778 moved_to_echo=30600 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 63, 0, 0, 0, 2634, 0, 5289, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 7112, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5494, 0, 0, 0, 0, 1907, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7987, 8302, 8835, 8194, 8195, 7356, 8188, 8479] avg=8192.0000 max=8835 max_rank=2 min=7356 min_rank=5 imbalance_ratio=1.078491 max_load_reduction=56701 max_load_reduction_pct=0.865189 moved_to_echo=32218 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 40, 0, 0, 0, 2864, 0, 4861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2045, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1809, 0, 0, 0, 7111, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3391, 0, 0, 0, 0, 2073, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7765, 8279, 9445, 8188, 8183, 8927, 8199, 6550] avg=8192.0000 max=9445 max_rank=2 min=6550 min_rank=7 imbalance_ratio=1.152954 max_load_reduction=56091 max_load_reduction_pct=0.855881 moved_to_echo=31859 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 309, 2, 0, 0, 2729, 0, 1349, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8067, 0, 0, 0, 0, 241, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1950, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 2469, 0, 0, 0, 7111, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 2, 2, 0, 0, 5500, 1, 0, 0, 0, 1976, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4391, 8313, 10107, 8193, 8191, 9588, 8185, 8568] avg=8192.0000 max=10107 max_rank=2 min=4391 min_rank=0 imbalance_ratio=1.233765 max_load_reduction=55429 max_load_reduction_pct=0.845779 moved_to_echo=34532 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 0, 3281, 0, 7148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2344, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 7113, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7750, 0, 0, 0, 0, 2375, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10437, 7983, 3924, 8192, 8197, 7412, 8187, 11204] avg=8192.0000 max=11204 max_rank=7 min=3924 min_rank=2 imbalance_ratio=1.367676 max_load_reduction=54332 max_load_reduction_pct=0.829041 moved_to_echo=35000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 3099, 0, 6968, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2222, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 0, 0, 0, 7114, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 0, 2253, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10077, 8038, 7065, 8192, 8199, 8488, 8187, 7290] avg=8192.0000 max=10077 max_rank=0 min=7065 min_rank=2 imbalance_ratio=1.230103 max_load_reduction=55459 max_load_reduction_pct=0.846237 moved_to_echo=32164 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 0, 0, 2633, 0, 4298, 0, 0, 6, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 967, 0, 0, 0, 7114, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5454, 0, 0, 0, 0, 1906, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6957, 8175, 9306, 8192, 8192, 8086, 8190, 8438] avg=8192.0000 max=9306 max_rank=2 min=6957 min_rank=0 imbalance_ratio=1.135986 max_load_reduction=56230 max_load_reduction_pct=0.858002 moved_to_echo=32906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 42496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17408, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 512, 128, 0, 0, 32512, 128, 32640, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 48000, 128, 0, 128, 0, 14976, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 5760, 0, 0, 0, 59904, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 39936, 128, 0, 128, 0, 17024, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 768, 128, 0, 0, 23424, 128, 41472, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 47872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16768, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 8704, 0, 0, 0, 56960, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 64384, 0, 128, 0, 0, 1280, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 62848, 0, 128, 0, 0, 2816, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 65536, 128, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8185, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 700, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 1, 1, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7210, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 1, 0, 0, 1, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8184, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 333, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 3, 7978, 0, 3, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 6563, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 8189, 0, 2, 1, 0, 0, 0, 5, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7953, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8193, 8063, 8219, 8529, 7962, 8192, 8189] avg=8192.0000 max=8529 max_rank=4 min=7962 min_rank=5 imbalance_ratio=1.041138 max_load_reduction=57007 max_load_reduction_pct=0.869858 moved_to_echo=32872 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7953, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8192, 8065, 8218, 8524, 7959, 8193, 8198] avg=8192.0000 max=8524 max_rank=4 min=7959 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=57012 max_load_reduction_pct=0.869934 moved_to_echo=32874 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 0, 0, 8062, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7210, 0, 0, 0, 0, 0, 0, 0, 1, 0, 236, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 7952, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 3, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8764, 8206, 7549, 8246, 8191, 8196] avg=8192.0000 max=8764 max_rank=2 min=7549 min_rank=4 imbalance_ratio=1.069824 max_load_reduction=56772 max_load_reduction_pct=0.866272 moved_to_echo=32182 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7956, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8064, 8219, 8513, 7974, 8192, 8194] avg=8192.0000 max=8513 max_rank=4 min=7974 min_rank=5 imbalance_ratio=1.039185 max_load_reduction=57023 max_load_reduction_pct=0.870102 moved_to_echo=32873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 7953, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8193, 8060, 8221, 8472, 8002, 8192, 8192] avg=8192.0000 max=8472 max_rank=4 min=8002 min_rank=5 imbalance_ratio=1.034180 max_load_reduction=57064 max_load_reduction_pct=0.870728 moved_to_echo=32858 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7951, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8193, 8062, 8219, 8528, 7965, 8193, 8189] avg=8192.0000 max=8528 max_rank=4 min=7965 min_rank=5 imbalance_ratio=1.041016 max_load_reduction=57008 max_load_reduction_pct=0.869873 moved_to_echo=32875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7978, 0, 3, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6563, 0, 0, 0, 0, 0, 0, 0, 2, 0, 236, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 0, 7953, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8189, 8394, 8015, 6905, 9469, 8191, 8185] avg=8192.0000 max=9469 max_rank=5 min=6905 min_rank=4 imbalance_ratio=1.155884 max_load_reduction=56067 max_load_reduction_pct=0.855515 moved_to_echo=32750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7953, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8192, 8064, 8219, 8516, 7959, 8192, 8193] avg=8192.0000 max=8516 max_rank=4 min=7959 min_rank=5 imbalance_ratio=1.039551 max_load_reduction=57020 max_load_reduction_pct=0.870056 moved_to_echo=32860 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 331, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 3, 8186, 0, 3, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 6741, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 8189, 0, 2, 1, 0, 0, 0, 5, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 668, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7372, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 1, 0, 0, 1, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8019, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8193, 8069, 8192, 8480, 8032, 8193, 8191] avg=8192.0000 max=8480 max_rank=4 min=8032 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=57056 max_load_reduction_pct=0.870605 moved_to_echo=32896 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8021, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8070, 8192, 8477, 8026, 8193, 8198] avg=8192.0000 max=8477 max_rank=4 min=8026 min_rank=5 imbalance_ratio=1.034790 max_load_reduction=57059 max_load_reduction_pct=0.870651 moved_to_echo=32894 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8020, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8193, 8069, 8192, 8482, 8029, 8192, 8191] avg=8192.0000 max=8482 max_rank=4 min=8029 min_rank=5 imbalance_ratio=1.035400 max_load_reduction=57054 max_load_reduction_pct=0.870575 moved_to_echo=32894 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8023, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8069, 8192, 8475, 8033, 8192, 8194] avg=8192.0000 max=8475 max_rank=4 min=8033 min_rank=5 imbalance_ratio=1.034546 max_load_reduction=57061 max_load_reduction_pct=0.870682 moved_to_echo=32894 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7372, 0, 0, 0, 0, 0, 0, 0, 1, 0, 167, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 8020, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 4, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8736, 8190, 7664, 8176, 8191, 8197] avg=8192.0000 max=8736 max_rank=2 min=7664 min_rank=4 imbalance_ratio=1.066406 max_load_reduction=56800 max_load_reduction_pct=0.866699 moved_to_echo=32228 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 130, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 8021, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8193, 8062, 8189, 8454, 8048, 8192, 8192] avg=8192.0000 max=8454 max_rank=4 min=8048 min_rank=5 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=32886 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8186, 0, 3, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6741, 0, 0, 0, 0, 0, 0, 0, 2, 0, 167, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 0, 8018, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8189, 8392, 8197, 7036, 9164, 8191, 8179] avg=8192.0000 max=9164 max_rank=5 min=7036 min_rank=4 imbalance_ratio=1.118652 max_load_reduction=56372 max_load_reduction_pct=0.860168 moved_to_echo=32570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8023, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8192, 8069, 8192, 8468, 8028, 8192, 8194] avg=8192.0000 max=8468 max_rank=4 min=8028 min_rank=5 imbalance_ratio=1.033691 max_load_reduction=57068 max_load_reduction_pct=0.870789 moved_to_echo=32882 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1920, 0, 63744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1152, 0, 0, 0, 0, 64512, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 62848, 0, 0, 0, 0, 0, 0, 0, 128, 0, 1920, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 63232, 0, 0, 0, 0, 0, 0, 0, 128, 0, 1408, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 0, 64256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64640, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[65536, 0, 128, 0, 128, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 65408, 0, 128, 128, 0, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 65536, 0, 128, 128, 0, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 22, 0, 1, 0, 0, 0, 0, 5180, 0, 0, 0, 7410, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 101, 8191, 8055, 0, 0, 0, 1, 0, 8160, 8188, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 0, 0, 0, 4169, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 0, 8079, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 0, 0, 0, 3800, 8192, 8190, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2248, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 7813, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2133, 0, 0, 0, 3634, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 7599, 0, 0, 0, 8179, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 0, 0, 6106, 8192, 8168, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 0, 0, 0, 0, 7791, 0, 0, 0, 8176, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1772, 0, 0, 0, 5251, 8192, 8186, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 1668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 741, 0, 0, 0, 0, 0, 0, 7899, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 0, 0, 4948, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6980, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 55, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 3074, 0, 0, 0, 6535, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 217, 8188, 7692, 0, 0, 0, 2, 0, 8188, 8167, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 0, 0, 5820, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 8182, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 22, 0, 1, 0, 0, 0, 0, 5180, 0, 0, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7410, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 307, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2340, 2260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 4634, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 9642, 8188, 8200, 6058, 7732, 12787, 4737] avg=8192.0000 max=12787 max_rank=6 min=4737 min_rank=7 imbalance_ratio=1.560913 max_load_reduction=52749 max_load_reduction_pct=0.804886 moved_to_echo=31314 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 0, 0, 0, 0, 0, 5844, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 2348, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4169, 0, 4700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7253, 8189, 8178, 9313, 8506, 7036, 8869] avg=8192.0000 max=9313 max_rank=4 min=7036 min_rank=6 imbalance_ratio=1.136841 max_load_reduction=56223 max_load_reduction_pct=0.857895 moved_to_echo=33724 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 1668, 0, 0, 0, 0, 0, 0, 5843, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 741, 0, 0, 0, 0, 0, 0, 7899, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 2349, 2299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4948, 0, 4712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7554, 8190, 8184, 9509, 8497, 5750, 9660] avg=8192.0000 max=9660 max_rank=7 min=5750 min_rank=6 imbalance_ratio=1.179199 max_load_reduction=55876 max_load_reduction_pct=0.852600 moved_to_echo=33416 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2248, 0, 0, 0, 0, 0, 0, 5844, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 7813, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2133, 2348, 2299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 0, 4712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8095, 8206, 8184, 9296, 8437, 6780, 8346] avg=8192.0000 max=9296 max_rank=4 min=6780 min_rank=6 imbalance_ratio=1.134766 max_load_reduction=56240 max_load_reduction_pct=0.858154 moved_to_echo=32859 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 1094, 0, 0, 0, 0, 0, 0, 5843, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 7599, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 2349, 2292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6106, 0, 4699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7001, 8207, 8184, 8551, 8517, 6077, 10805] avg=8192.0000 max=10805 max_rank=7 min=6077 min_rank=6 imbalance_ratio=1.318970 max_load_reduction=54731 max_load_reduction_pct=0.835129 moved_to_echo=33950 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 1020, 0, 0, 0, 0, 0, 0, 5843, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 0, 0, 0, 0, 7791, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1772, 2349, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5251, 0, 4709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 6968, 8196, 8184, 9114, 8500, 6421, 9960] avg=8192.0000 max=9960 max_rank=7 min=6421 min_rank=6 imbalance_ratio=1.215820 max_load_reduction=55576 max_load_reduction_pct=0.848022 moved_to_echo=33995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6980, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 3, 0, 0, 5841, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 3074, 0, 0, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6535, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 0, 2, 0, 293, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2347, 2158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 217, 0, 4425, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 12844, 8171, 8238, 3894, 6857, 12695, 4648] avg=8192.0000 max=12844 max_rank=1 min=3894 min_rank=4 imbalance_ratio=1.567871 max_load_reduction=52692 max_load_reduction_pct=0.804016 moved_to_echo=28094 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 5843, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 0, 8079, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 2348, 2298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 4711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6179, 8189, 8184, 9801, 8490, 7990, 8511] avg=8192.0000 max=9801 max_rank=4 min=6179 min_rank=1 imbalance_ratio=1.196411 max_load_reduction=55735 max_load_reduction_pct=0.850449 moved_to_echo=34792 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 10, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 7714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 0, 6, 8192, 8192, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4511, 0, 0, 0, 0, 0, 0, 420, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 16, 0, 1, 0, 0, 0, 0, 8187, 0, 0, 0, 4199, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7585, 0, 0, 0, 48, 8191, 7828, 0, 0, 0, 1, 0, 8148, 8185, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 11, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 55, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6863, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 49, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 8154, 0, 0, 0, 1540, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 1, 120, 8169, 7712, 0, 0, 0, 3, 0, 8187, 8158, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8175, 0, 17, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 23, 8192, 8168, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 7734, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 8157, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 366, 0, 0, 0, 19, 8192, 8187, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4511, 0, 0, 0, 0, 0, 0, 420, 0, 0, 1, 1, 0, 0, 653, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8183, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 16, 0, 1, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4199, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7585, 6082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 7828, 0, 0, 0, 1, 0, 73, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 5613, 8187, 8195, 8185, 5542, 13669, 7951] avg=8192.0000 max=13669 max_rank=6 min=5542 min_rank=5 imbalance_ratio=1.668579 max_load_reduction=51867 max_load_reduction_pct=0.791428 moved_to_echo=35347 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 7714, 0, 0, 0, 0, 0, 0, 656, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 6114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8422, 8191, 8185, 8272, 9505, 6494, 8272] avg=8192.0000 max=9505 max_rank=5 min=6494 min_rank=6 imbalance_ratio=1.160278 max_load_reduction=56031 max_load_reduction_pct=0.854965 moved_to_echo=32543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6863, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 3, 0, 0, 656, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 0, 8156, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1540, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 2, 0, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 120, 0, 7712, 0, 0, 0, 3, 0, 74, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 7800, 8165, 8233, 8140, 2905, 14208, 7913] avg=8192.0000 max=14208 max_rank=6 min=2905 min_rank=5 imbalance_ratio=1.734375 max_load_reduction=51328 max_load_reduction_pct=0.783203 moved_to_echo=33166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 0, 0, 0, 0, 0, 662, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 6121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 8192, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8753, 8196, 8183, 8215, 9523, 6169, 8302] avg=8192.0000 max=9523 max_rank=5 min=6169 min_rank=6 imbalance_ratio=1.162476 max_load_reduction=56013 max_load_reduction_pct=0.854691 moved_to_echo=32209 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 7734, 0, 0, 0, 0, 0, 0, 656, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 366, 6115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 8187, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8521, 8198, 8185, 8167, 9506, 6484, 8280] avg=8192.0000 max=9506 max_rank=5 min=6484 min_rank=6 imbalance_ratio=1.160400 max_load_reduction=56030 max_load_reduction_pct=0.854950 moved_to_echo=32437 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 656, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 17, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 6115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8168, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8811, 8202, 8185, 8165, 9538, 6174, 8265] avg=8192.0000 max=9538 max_rank=5 min=6174 min_rank=6 imbalance_ratio=1.164307 max_load_reduction=55998 max_load_reduction_pct=0.854462 moved_to_echo=32142 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 656, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 6115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8192, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8808, 8192, 8185, 8200, 9518, 6163, 8276] avg=8192.0000 max=9518 max_rank=5 min=6163 min_rank=6 imbalance_ratio=1.161865 max_load_reduction=56018 max_load_reduction_pct=0.854767 moved_to_echo=32157 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 656, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 6115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8192, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8808, 8205, 8185, 8192, 9499, 6175, 8277] avg=8192.0000 max=9499 max_rank=5 min=6175 min_rank=6 imbalance_ratio=1.159546 max_load_reduction=56037 max_load_reduction_pct=0.855057 moved_to_echo=32143 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 28672, 18816, 18304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 11136, 0, 0, 0, 0, 0, 0, 7808, 0, 0, 128, 128, 0, 0, 46848, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 28288, 0, 37376, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 3200, 0, 128, 0, 0, 0, 0, 55680, 0, 0, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 62976, 128, 128, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 128, 0, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 16768, 48896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 54784, 128, 128, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 128, 0, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 11776, 0, 0, 0, 0, 0, 0, 48384, 0, 0, 128, 128, 0, 0, 5376, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 384, 0, 64768, 0, 0, 0, 128, 0, 640, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 256, 0, 128, 0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4317, 0, 0, 0, 0, 5301, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 8190, 6355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 7129, 1, 0, 0, 0, 7950, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8084, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 35, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4273, 0, 0, 0, 0, 5960, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 8189, 5749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6838, 0, 0, 0, 0, 4906, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 8180, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 5326, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 8136, 6441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1951, 0, 0, 0, 0, 7084, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 8192, 6920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5047, 0, 0, 0, 0, 4197, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 8171, 6992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 71, 0, 0, 0, 7013, 4, 0, 0, 0, 8129, 0, 5001, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7360, 2829, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 2098, 0, 0, 0, 0, 0, 0, 0, 3, 21, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 142, 0, 3, 0, 0, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6845, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4317, 0, 0, 0, 0, 1585, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 2862, 6355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8282, 6001, 9515, 9561, 8188, 7607, 8196] avg=8192.0000 max=9561 max_rank=4 min=6001 min_rank=2 imbalance_ratio=1.167114 max_load_reduction=55975 max_load_reduction_pct=0.854111 moved_to_echo=33552 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 6762, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7129, 1, 0, 0, 0, 2377, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1312, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2862, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 35, 0, 1, 0, 0, 0, 0, 5573, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 8172, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8214, 13389, 4348, 5487, 8199, 9505, 8191] avg=8192.0000 max=13389 max_rank=2 min=4348 min_rank=3 imbalance_ratio=1.634399 max_load_reduction=52147 max_load_reduction_pct=0.795700 moved_to_echo=31382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6853, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4273, 0, 0, 0, 0, 1782, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1328, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 2862, 5749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4178, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8289, 6239, 9520, 8853, 8187, 8069, 8174] avg=8192.0000 max=9520 max_rank=3 min=6239 min_rank=2 imbalance_ratio=1.162109 max_load_reduction=56016 max_load_reduction_pct=0.854736 moved_to_echo=33283 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6855, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 1590, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 2834, 6441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3736, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8286, 6203, 9513, 9350, 8188, 7607, 8190] avg=8192.0000 max=9513 max_rank=3 min=6203 min_rank=2 imbalance_ratio=1.161255 max_load_reduction=56023 max_load_reduction_pct=0.854843 moved_to_echo=33335 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6849, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1951, 0, 0, 0, 0, 2118, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1328, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 2863, 6920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4966, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8286, 4469, 9520, 9820, 8188, 8858, 8190] avg=8192.0000 max=9820 max_rank=4 min=4469 min_rank=2 imbalance_ratio=1.198730 max_load_reduction=55716 max_load_reduction_pct=0.850159 moved_to_echo=35056 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6791, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6838, 0, 0, 0, 0, 1467, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 2858, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8226, 8391, 9505, 7564, 8187, 7326, 8201] avg=8192.0000 max=9505 max_rank=3 min=7326 min_rank=6 imbalance_ratio=1.160278 max_load_reduction=56031 max_load_reduction_pct=0.854965 moved_to_echo=31278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 71, 0, 6157, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7013, 4, 0, 0, 0, 2431, 0, 5001, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1194, 2829, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2862, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 21, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 142, 0, 3, 0, 0, 0, 0, 5698, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 8170, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 7667, 14450, 4095, 4962, 8211, 9740, 8209] avg=8192.0000 max=14450 max_rank=2 min=4095 min_rank=3 imbalance_ratio=1.763916 max_load_reduction=51086 max_load_reduction_pct=0.779510 moved_to_echo=31122 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6853, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5047, 0, 0, 0, 0, 1255, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1328, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 2856, 6992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2942, 3882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8286, 6394, 9520, 9939, 8188, 6824, 8185] avg=8192.0000 max=9939 max_rank=4 min=6394 min_rank=2 imbalance_ratio=1.213257 max_load_reduction=55597 max_load_reduction_pct=0.848343 moved_to_echo=33136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 5718, 1, 0, 0, 0, 6624, 0, 2736, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8137, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 7867, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 26, 0, 1, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5766, 0, 0, 0, 0, 1332, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 0, 0, 8144, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3193, 0, 0, 0, 0, 3125, 0, 1716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 8124, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 105, 0, 0, 0, 5344, 4, 0, 0, 0, 8027, 0, 3097, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7349, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 7397, 0, 0, 0, 0, 0, 0, 0, 3, 23, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 80, 0, 3, 0, 0, 0, 0, 0, 0, 45, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5254, 0, 0, 0, 0, 1810, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 0, 0, 8190, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4866, 0, 0, 0, 0, 2276, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 0, 0, 8186, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6242, 0, 0, 0, 0, 1408, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 8164, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 960, 0, 0, 0, 0, 6611, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 8192, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6469, 1760, 228, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 0, 0, 0, 1810, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 0, 0, 0, 7867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8461, 5600, 9895, 8820, 8187, 8194, 8192] avg=8192.0000 max=9895 max_rank=3 min=5600 min_rank=2 imbalance_ratio=1.207886 max_load_reduction=55641 max_load_reduction_pct=0.849014 moved_to_echo=33393 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 105, 0, 5811, 1790, 208, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3554, 4, 0, 0, 0, 8027, 0, 3097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1538, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 23, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 80, 0, 3, 0, 0, 0, 0, 8185, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 8172, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7921, 14682, 2825, 7190, 8212, 8282, 8229] avg=8192.0000 max=14682 max_rank=2 min=2825 min_rank=3 imbalance_ratio=1.792236 max_load_reduction=50854 max_load_reduction_pct=0.775970 moved_to_echo=31913 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6477, 1630, 228, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3236, 0, 0, 0, 0, 2276, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 0, 0, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8339, 6161, 9901, 8377, 8187, 8190, 8183] avg=8192.0000 max=9901 max_rank=3 min=6161 min_rank=2 imbalance_ratio=1.208618 max_load_reduction=55635 max_load_reduction_pct=0.848923 moved_to_echo=32937 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 6434, 1916, 221, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3802, 1, 0, 0, 0, 6624, 0, 2736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1703, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 26, 0, 1, 0, 0, 0, 0, 8187, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 8174, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8595, 13164, 3324, 7646, 8199, 8226, 8186] avg=8192.0000 max=13164 max_rank=2 min=3324 min_rank=3 imbalance_ratio=1.606934 max_load_reduction=52372 max_load_reduction_pct=0.799133 moved_to_echo=32257 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6481, 1073, 234, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2120, 0, 0, 0, 0, 3125, 0, 1716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7793, 6961, 9901, 8185, 8188, 8133, 8183] avg=8192.0000 max=9901 max_rank=3 min=6961 min_rank=2 imbalance_ratio=1.208618 max_load_reduction=55635 max_load_reduction_pct=0.848923 moved_to_echo=32689 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6430, 1932, 228, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 0, 0, 0, 0, 1332, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1702, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8594, 5339, 9885, 9013, 8187, 8148, 8191] avg=8192.0000 max=9885 max_rank=3 min=5339 min_rank=2 imbalance_ratio=1.206665 max_load_reduction=55651 max_load_reduction_pct=0.849167 moved_to_echo=33539 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6477, 2091, 230, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4151, 0, 0, 0, 0, 1408, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 0, 7934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8803, 5781, 9899, 8316, 8188, 8168, 8189] avg=8192.0000 max=9899 max_rank=3 min=5781 min_rank=2 imbalance_ratio=1.208374 max_load_reduction=55637 max_load_reduction_pct=0.848953 moved_to_echo=32861 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6474, 321, 230, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 0, 6611, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7030, 7848, 9906, 7989, 8188, 8195, 8183] avg=8192.0000 max=9906 max_rank=3 min=7030 min_rank=1 imbalance_ratio=1.209229 max_load_reduction=55630 max_load_reduction_pct=0.848846 moved_to_echo=32555 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 0, 128, 128, 128, 128, 256, 0, 128, 0, 0, 0, 0, 34304, 31104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 40704, 128, 0, 0, 0, 14720, 0, 10368, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 1536, 0, 0, 22912, 41344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 54016, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 10496, 55040, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 128, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 51072, 12544, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 24832, 128, 0, 0, 0, 31232, 0, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 13568, 51968, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 3328, 0, 0, 0, 62336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 7605, 0, 0, 0, 0, 0, 7891, 0, 0, 0, 0, 0, 0, 5, 0, 0, 594, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 7435, 0, 0, 0, 0, 0, 6879, 0, 0, 0, 0, 0, 11, 3, 0, 0, 1799, 0, 0, 0, 8192, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 51, 0, 1, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1255, 0, 0, 0, 0, 2297, 0, 1, 0, 1, 0, 0, 5008, 0, 0, 0, 0, 0, 7596, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 8182, 0, 0, 1, 0, 8185, 0, 0, 0, 1, 31, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 8190, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 22, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 7557, 0, 0, 0, 0, 0, 2, 33, 0, 0, 441, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 57, 0, 3, 0, 0, 25, 0, 0, 0, 0, 0, 0, 7199, 0, 0, 0, 0, 0, 7376, 0, 0, 0, 0, 0, 51, 10, 0, 0, 1442, 0, 0, 0, 8192, 0, 0, 0, 14, 8191, 0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 8018, 0, 0, 0, 0, 0, 7680, 0, 0, 0, 0, 0, 0, 1, 0, 0, 319, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 19, 0, 15, 0, 0, 43, 0, 0, 0, 0, 0, 0, 7804, 0, 0, 0, 0, 0, 7623, 0, 0, 0, 0, 0, 0, 27, 0, 0, 755, 0, 0, 0, 8192, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 127, 0, 0, 0, 2, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 6909, 0, 1, 0, 3, 0, 0, 888, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 8097, 0, 0, 2, 0, 8127, 0, 0, 0, 3, 76, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 8188, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 7605, 0, 0, 136, 61, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7815, 0, 0, 0, 0, 0, 0, 5, 0, 0, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8191, 7922, 8414, 8274, 8192, 8195, 8204] avg=8192.0000 max=8414 max_rank=3 min=7922 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=57122 max_load_reduction_pct=0.871613 moved_to_echo=32865 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 51, 0, 1, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 0, 0, 0, 0, 2297, 0, 1, 0, 1, 0, 0, 5008, 0, 0, 136, 59, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 1, 0, 0, 0, 0, 0, 1, 31, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8337, 8193, 8803, 7526, 8083, 8192, 8193, 8209] avg=8192.0000 max=8803 max_rank=2 min=7526 min_rank=3 imbalance_ratio=1.074585 max_load_reduction=56733 max_load_reduction_pct=0.865677 moved_to_echo=32677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 7435, 0, 0, 136, 53, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6813, 0, 0, 0, 0, 0, 11, 3, 0, 0, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 7, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8186, 7697, 8626, 8314, 8192, 8189, 8187] avg=8192.0000 max=8626 max_rank=3 min=7697 min_rank=2 imbalance_ratio=1.052979 max_load_reduction=56910 max_load_reduction_pct=0.868378 moved_to_echo=32882 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 57, 0, 3, 0, 0, 25, 0, 0, 0, 0, 0, 0, 7199, 0, 0, 136, 57, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7305, 0, 0, 0, 0, 0, 51, 10, 0, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 14, 0, 0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8190, 7524, 8808, 8333, 8192, 8189, 8155] avg=8192.0000 max=8808 max_rank=3 min=7524 min_rank=2 imbalance_ratio=1.075195 max_load_reduction=56728 max_load_reduction_pct=0.865601 moved_to_echo=32869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 8018, 0, 0, 136, 59, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 0, 0, 0, 0, 0, 0, 1, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8191, 8557, 7926, 8129, 8192, 8195, 8202] avg=8192.0000 max=8557 max_rank=2 min=7926 min_rank=3 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=32718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 15, 0, 0, 43, 0, 0, 0, 0, 0, 0, 7804, 0, 0, 136, 59, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7549, 0, 0, 0, 0, 0, 0, 27, 0, 0, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8195, 8120, 8331, 8186, 8192, 8192, 8178] avg=8192.0000 max=8331 max_rank=3 min=8120 min_rank=2 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 127, 0, 0, 0, 2, 0, 0, 0, 0, 0, 125, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 8186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 6909, 0, 1, 0, 3, 0, 0, 888, 0, 0, 135, 62, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7962, 0, 0, 2, 0, 0, 0, 0, 0, 3, 76, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8333, 8196, 8421, 7953, 8045, 8193, 8191, 8204] avg=8192.0000 max=8421 max_rank=2 min=7953 min_rank=3 imbalance_ratio=1.027954 max_load_reduction=57115 max_load_reduction_pct=0.871506 moved_to_echo=32633 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 22, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8013, 0, 0, 141, 63, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7476, 0, 0, 0, 0, 0, 2, 33, 0, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8194, 8492, 7952, 8172, 8191, 8192, 8197] avg=8192.0000 max=8492 max_rank=2 min=7952 min_rank=3 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=32752 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 166, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7160, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 58, 6, 0, 0, 886, 0, 0, 0, 8192, 0, 0, 0, 14, 8191, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7674, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 0, 0, 14, 3, 0, 0, 368, 0, 0, 0, 8192, 0, 0, 0, 8, 8192, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 60, 0, 1, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1541, 0, 0, 0, 0, 540, 0, 1, 0, 1, 0, 0, 5952, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 8177, 0, 0, 1, 0, 8186, 0, 0, 0, 1, 20, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 8190, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 7903, 0, 0, 0, 0, 0, 6, 34, 0, 0, 106, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 7985, 0, 0, 0, 0, 0, 0, 1, 0, 0, 68, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 138, 0, 0, 0, 2, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1483, 0, 0, 0, 0, 2720, 0, 0, 0, 3, 0, 0, 3732, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 4, 0, 8094, 0, 0, 2, 0, 8137, 0, 0, 0, 3, 137, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 8188, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7708, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 5, 0, 0, 249, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7898, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 26, 0, 0, 226, 0, 0, 0, 8192, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 166, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7160, 0, 0, 132, 86, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7866, 0, 0, 0, 0, 0, 58, 6, 0, 0, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 14, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8190, 7601, 8816, 8259, 8192, 8189, 8150] avg=8192.0000 max=8816 max_rank=3 min=7601 min_rank=2 imbalance_ratio=1.076172 max_load_reduction=56720 max_load_reduction_pct=0.865479 moved_to_echo=32790 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 60, 0, 1, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 8187, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 0, 0, 0, 0, 540, 0, 1, 0, 1, 0, 0, 5952, 0, 0, 134, 86, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7979, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 1, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8357, 8194, 8307, 7982, 8101, 8192, 8193, 8210] avg=8192.0000 max=8357 max_rank=0 min=7982 min_rank=3 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32696 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 132, 86, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7853, 0, 0, 0, 0, 0, 0, 1, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8191, 8563, 7922, 8133, 8192, 8195, 8203] avg=8192.0000 max=8563 max_rank=2 min=7922 min_rank=3 imbalance_ratio=1.045288 max_load_reduction=56973 max_load_reduction_pct=0.869339 moved_to_echo=32723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 138, 0, 0, 0, 2, 0, 0, 0, 0, 0, 136, 0, 0, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 8186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1483, 0, 0, 0, 0, 2720, 0, 0, 0, 3, 0, 0, 3732, 0, 0, 135, 85, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7993, 0, 0, 2, 0, 0, 0, 0, 0, 3, 137, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8358, 8196, 8209, 8047, 8138, 8193, 8191, 8204] avg=8192.0000 max=8358 max_rank=0 min=8047 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32726 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 134, 88, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7769, 0, 0, 0, 0, 0, 6, 34, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8194, 8590, 7915, 8116, 8191, 8192, 8201] avg=8192.0000 max=8590 max_rank=2 min=7915 min_rank=3 imbalance_ratio=1.048584 max_load_reduction=56946 max_load_reduction_pct=0.868927 moved_to_echo=32700 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7708, 0, 0, 135, 86, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 0, 0, 0, 5, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8191, 8069, 8283, 8266, 8192, 8195, 8203] avg=8192.0000 max=8283 max_rank=3 min=8069 min_rank=2 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32856 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7674, 0, 0, 133, 86, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 14, 3, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 8, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8186, 8003, 8334, 8306, 8192, 8187, 8190] avg=8192.0000 max=8334 max_rank=3 min=8003 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7898, 0, 0, 134, 86, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 0, 0, 0, 0, 0, 26, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8194, 8194, 8237, 8217, 8192, 8194, 8175] avg=8192.0000 max=8237 max_rank=3 min=8133 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 2304, 0, 128, 0, 0, 9472, 0, 128, 0, 128, 0, 0, 52096, 0, 0, 1152, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 60160, 0, 0, 0, 0, 128, 128, 128, 0, 0, 5376, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 64384, 0, 0, 128, 128, 0, 0, 0, 0, 128, 1280, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 65408, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 256, 0, 128, 0, 128, 0, 0, 0, 0, 0, 384, 128, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 3840, 0, 128, 0, 0, 3328, 0, 128, 0, 128, 0, 0, 56320, 0, 0, 1152, 768, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 64640, 0, 0, 128, 128, 0, 0, 0, 0, 128, 1024, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 65408, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 63488, 0, 0, 0, 0, 128, 128, 128, 0, 0, 1920, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 256, 0, 128, 0, 128, 0, 0, 0, 0, 0, 384, 0, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 8192, 0, 0, 0, 0, 0, 463, 0, 0, 3, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7789, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 8192, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8020, 0, 8192, 0, 0, 0, 0, 0, 1432, 0, 0, 3, 0, 0, 0, 0, 0, 0, 32, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 7887, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7048, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 8, 1, 117, 8190, 0, 8185, 1, 0, 0, 2, 0, 1090, 0, 0, 0, 0, 4120, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 2, 2, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 2584, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 115] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 8192, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 8192, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 8192, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7170, 0, 0, 0, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 31, 2, 33, 8188, 0, 8183, 0, 2, 0, 3, 0, 216, 0, 0, 0, 0, 4721, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 0, 0, 0, 3, 3, 3, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 2055, 3, 0, 0, 1, 0, 2, 1, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1066, 8, 0, 0, 0, 116] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7513, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 2, 7079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7789, 0, 0, 0, 0, 0, 1113, 641, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 7976, 7106, 8191, 8207, 7610, 8182, 10075] avg=8192.0000 max=10075 max_rank=7 min=7106 min_rank=2 imbalance_ratio=1.229858 max_load_reduction=55461 max_load_reduction_pct=0.846268 moved_to_echo=34074 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7542, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7611, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 4, 1114, 643, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 7795, 7078, 8191, 8210, 7638, 8195, 10251] avg=8192.0000 max=10251 max_rank=7 min=7078 min_rank=2 imbalance_ratio=1.251343 max_load_reduction=55285 max_load_reduction_pct=0.843582 moved_to_echo=34294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 117, 7547, 0, 0, 1, 0, 0, 2, 0, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4120, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 2, 2, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7621, 0, 0, 2584, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 115, 1112, 643, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8766, 11194, 8197, 8125, 10207, 8193, 2648] avg=8192.0000 max=11194 max_rank=2 min=2648 min_rank=7 imbalance_ratio=1.366455 max_load_reduction=54342 max_load_reduction_pct=0.829193 moved_to_echo=29173 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7535, 0, 0, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7598, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 0, 0, 0, 1113, 643, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7951, 7082, 8191, 8210, 7639, 8195, 10075] avg=8192.0000 max=10075 max_rank=7 min=7082 min_rank=2 imbalance_ratio=1.229858 max_load_reduction=55461 max_load_reduction_pct=0.846268 moved_to_echo=34119 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7390, 0, 0, 0, 0, 0, 0, 0, 1432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 32, 0, 0, 1, 0, 0, 0, 2, 7079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7339, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7048, 0, 0, 0, 0, 0, 1113, 630, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8822, 7117, 8191, 8207, 7508, 8180, 9307] avg=8192.0000 max=9307 max_rank=7 min=7117 min_rank=2 imbalance_ratio=1.136108 max_load_reduction=56229 max_load_reduction_pct=0.857986 moved_to_echo=33202 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7507, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7535, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 1113, 645, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 7785, 7079, 8186, 8212, 7641, 8205, 10242] avg=8192.0000 max=10242 max_rank=7 min=7079 min_rank=2 imbalance_ratio=1.250244 max_load_reduction=55294 max_load_reduction_pct=0.843719 moved_to_echo=34300 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2, 33, 7545, 0, 0, 0, 2, 0, 3, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4721, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 7072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 3, 3, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7620, 0, 0, 2055, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1066, 8, 0, 0, 0, 116, 1111, 643, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 7832, 11797, 8198, 8156, 9680, 8191, 3481] avg=8192.0000 max=11797 max_rank=2 min=3481 min_rank=7 imbalance_ratio=1.440063 max_load_reduction=53739 max_load_reduction_pct=0.819992 moved_to_echo=29508 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7488, 0, 0, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7534, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7170, 0, 0, 0, 0, 6, 1113, 638, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8609, 7083, 8191, 8209, 7613, 8195, 9457] avg=8192.0000 max=9457 max_rank=7 min=7083 min_rank=2 imbalance_ratio=1.154419 max_load_reduction=56079 max_load_reduction_pct=0.855698 moved_to_echo=33474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 39, 2, 74, 8188, 0, 8183, 0, 3, 0, 3, 0, 859, 0, 0, 0, 0, 4842, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 3, 4, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 1114, 3, 0, 0, 1, 0, 2, 1, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1269, 8, 0, 0, 0, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8192, 0, 0, 0, 0, 0, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7508, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 8192, 0, 0, 0, 0, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7571, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8045, 0, 8192, 0, 0, 0, 0, 0, 1236, 0, 0, 4, 0, 0, 0, 0, 0, 0, 48, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 7786, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7314, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 8, 1, 153, 8190, 0, 8184, 1, 0, 0, 2, 0, 3146, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 2, 2, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 658, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 191, 0, 0, 0, 0, 90] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 8192, 0, 0, 0, 0, 0, 1321, 0, 0, 3, 0, 0, 0, 0, 0, 0, 23, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6934, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 8192, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 8192, 0, 0, 0, 0, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5665, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 2, 74, 6837, 0, 0, 0, 3, 0, 3, 0, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4842, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 3, 4, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8008, 0, 0, 1114, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1269, 8, 0, 0, 0, 42, 1351, 1119, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7817, 11909, 8199, 8157, 9127, 8190, 3939] avg=8192.0000 max=11909 max_rank=2 min=3939 min_rank=7 imbalance_ratio=1.453735 max_load_reduction=53627 max_load_reduction_pct=0.818283 moved_to_echo=29413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 153, 6839, 0, 0, 1, 0, 0, 2, 0, 3146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 2, 2, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8009, 0, 0, 658, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 191, 0, 0, 0, 0, 90, 1351, 1119, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 10150, 11103, 8197, 8120, 8669, 8193, 2901] avg=8192.0000 max=11103 max_rank=2 min=2901 min_rank=7 imbalance_ratio=1.355347 max_load_reduction=54433 max_load_reduction_pct=0.830582 moved_to_echo=27883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6830, 0, 0, 0, 0, 0, 0, 0, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7983, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7508, 0, 0, 0, 0, 0, 1350, 1120, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7527, 7075, 8191, 8212, 8015, 8195, 10126] avg=8192.0000 max=10126 max_rank=7 min=7075 min_rank=2 imbalance_ratio=1.236084 max_load_reduction=55410 max_load_reduction_pct=0.845490 moved_to_echo=34548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6838, 0, 0, 0, 0, 0, 0, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7571, 0, 0, 0, 0, 2, 1351, 1121, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 7482, 7071, 8191, 8212, 8012, 8195, 10193] avg=8192.0000 max=10193 max_rank=7 min=7071 min_rank=2 imbalance_ratio=1.244263 max_load_reduction=55343 max_load_reduction_pct=0.844467 moved_to_echo=34612 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6811, 0, 0, 0, 0, 0, 0, 0, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 23, 0, 0, 1, 0, 0, 0, 2, 7072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7935, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6934, 0, 0, 0, 0, 0, 1346, 1120, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8132, 7101, 8191, 8207, 7983, 8182, 9547] avg=8192.0000 max=9547 max_rank=7 min=7101 min_rank=2 imbalance_ratio=1.165405 max_load_reduction=55989 max_load_reduction_pct=0.854324 moved_to_echo=33919 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6803, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0, 0, 1349, 1120, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 7021, 7072, 8185, 8214, 7999, 8208, 10651] avg=8192.0000 max=10651 max_rank=7 min=7021 min_rank=1 imbalance_ratio=1.300171 max_load_reduction=54885 max_load_reduction_pct=0.837479 moved_to_echo=35072 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6718, 0, 0, 0, 0, 0, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 48, 0, 0, 2, 0, 0, 0, 2, 7071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7615, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7314, 0, 0, 0, 0, 0, 1327, 1121, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 7954, 7127, 8191, 8203, 7779, 8178, 9903] avg=8192.0000 max=9903 max_rank=7 min=7127 min_rank=2 imbalance_ratio=1.208862 max_load_reduction=55633 max_load_reduction_pct=0.848892 moved_to_echo=34063 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6799, 0, 0, 0, 0, 0, 0, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 7072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7873, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5665, 0, 0, 0, 0, 1, 1344, 1120, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 9453, 7078, 8191, 8211, 7952, 8195, 8276] avg=8192.0000 max=9453 max_rank=1 min=7078 min_rank=2 imbalance_ratio=1.153931 max_load_reduction=56083 max_load_reduction_pct=0.855759 moved_to_echo=32634 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 47104, 128, 0, 0, 0, 256, 8960, 5248, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 8960, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 0, 128, 56704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 60544, 0, 0, 5120, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 256, 60160, 0, 0, 128, 128, 0, 128, 0, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 44544, 128, 0, 0, 0, 256, 10880, 8960, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 8960, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 128, 56576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 256, 54528, 0, 0, 128, 128, 0, 128, 0, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 63360, 0, 0, 2304, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8189, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8168, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8192, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 8189, 0, 0, 62, 5242, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 0, 0, 0, 908, 0, 0, 8098, 2, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1982, 8189, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 96, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8192, 47, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8188, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 127, 8183, 0, 0, 106, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 3, 3, 0, 0, 3, 2, 0, 0, 1225, 0, 0, 8074, 4, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3297, 8106, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4, 0, 0, 159, 0, 8187, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 8157, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7530, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5881, 2078, 661, 242, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 10503, 8211, 7950, 8192, 7541, 8867, 8160] avg=8192.0000 max=10503 max_rank=1 min=6112 min_rank=0 imbalance_ratio=1.282104 max_load_reduction=55033 max_load_reduction_pct=0.839737 moved_to_echo=32760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5890, 2079, 661, 242, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 10493, 8218, 7950, 8192, 7531, 8877, 8160] avg=8192.0000 max=10493 max_rank=1 min=6115 min_rank=0 imbalance_ratio=1.280884 max_load_reduction=55043 max_load_reduction_pct=0.839890 moved_to_echo=32760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 1473, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 0, 0, 7859, 2, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1982, 7528, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3769, 2078, 661, 239, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 96, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14381, 1536, 8132, 8772, 8192, 9512, 6756, 8255] avg=8192.0000 max=14381 max_rank=0 min=1536 min_rank=1 imbalance_ratio=1.755493 max_load_reduction=51155 max_load_reduction_pct=0.780563 moved_to_echo=32715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7527, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5895, 2081, 665, 248, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 10465, 8223, 7944, 8192, 7539, 8896, 8154] avg=8192.0000 max=10465 max_rank=1 min=6123 min_rank=0 imbalance_ratio=1.277466 max_load_reduction=55071 max_load_reduction_pct=0.840317 moved_to_echo=32781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7528, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5855, 2078, 661, 242, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 10488, 8213, 7950, 8194, 7577, 8842, 8160] avg=8192.0000 max=10488 max_rank=1 min=6112 min_rank=0 imbalance_ratio=1.280273 max_load_reduction=55048 max_load_reduction_pct=0.839966 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5879, 2079, 661, 242, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 10486, 8217, 7950, 8191, 7553, 8866, 8160] avg=8192.0000 max=10486 max_rank=1 min=6113 min_rank=0 imbalance_ratio=1.280029 max_load_reduction=55050 max_load_reduction_pct=0.839996 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6110, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 127, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 3, 3, 0, 0, 3, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1225, 0, 0, 7835, 4, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3297, 7452, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2560, 2078, 654, 239, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4, 0, 0, 159, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14432, 1107, 8103, 9070, 8192, 10752, 5553, 8327] avg=8192.0000 max=14432 max_rank=0 min=1107 min_rank=1 imbalance_ratio=1.761719 max_load_reduction=51104 max_load_reduction_pct=0.779785 moved_to_echo=32824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5891, 2079, 661, 242, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 10458, 8219, 7950, 8191, 7531, 8879, 8160] avg=8192.0000 max=10458 max_rank=1 min=6148 min_rank=0 imbalance_ratio=1.276611 max_load_reduction=55078 max_load_reduction_pct=0.840424 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8192, 37, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 8190, 0, 0, 175, 5118, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 0, 0, 0, 979, 0, 0, 8111, 2, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1918, 8188, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 101, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8189, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8178, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8186, 0, 0, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 155, 8183, 0, 0, 426, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 2, 0, 0, 1299, 0, 0, 8105, 4, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3003, 8074, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 4, 0, 1, 159, 0, 8187, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7579, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5914, 2076, 610, 266, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 10439, 8219, 7926, 8194, 7618, 8867, 8159] avg=8192.0000 max=10439 max_rank=1 min=6114 min_rank=0 imbalance_ratio=1.274292 max_load_reduction=55097 max_load_reduction_pct=0.840714 moved_to_echo=32838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 1400, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 0, 0, 7848, 2, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 7578, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3718, 2076, 610, 263, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 101, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14385, 1576, 8123, 8831, 8192, 9498, 6671, 8260] avg=8192.0000 max=14385 max_rank=0 min=1576 min_rank=1 imbalance_ratio=1.755981 max_load_reduction=51151 max_load_reduction_pct=0.780502 moved_to_echo=32621 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5940, 2076, 610, 266, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 10444, 8217, 7926, 8192, 7592, 8893, 8159] avg=8192.0000 max=10444 max_rank=1 min=6113 min_rank=0 imbalance_ratio=1.274902 max_load_reduction=55092 max_load_reduction_pct=0.840637 moved_to_echo=32836 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5951, 2077, 610, 266, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6117, 10431, 8225, 7926, 8191, 7582, 8905, 8159] avg=8192.0000 max=10431 max_rank=1 min=6117 min_rank=0 imbalance_ratio=1.273315 max_load_reduction=55105 max_load_reduction_pct=0.840836 moved_to_echo=32837 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6111, 0, 0, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 155, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1299, 0, 0, 7842, 4, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3003, 7472, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2513, 2075, 602, 263, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 4, 0, 1, 159, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14461, 1373, 8077, 9152, 8191, 10478, 5476, 8328] avg=8192.0000 max=14461 max_rank=0 min=1373 min_rank=1 imbalance_ratio=1.765259 max_load_reduction=51075 max_load_reduction_pct=0.779343 moved_to_echo=32473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5951, 2077, 610, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 10433, 8225, 7926, 8192, 7582, 8904, 8159] avg=8192.0000 max=10433 max_rank=1 min=6115 min_rank=0 imbalance_ratio=1.273560 max_load_reduction=55103 max_load_reduction_pct=0.840805 moved_to_echo=32837 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 2235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7576, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5956, 2082, 616, 269, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6116, 10413, 8227, 7923, 8192, 7585, 8927, 8153] avg=8192.0000 max=10413 max_rank=1 min=6116 min_rank=0 imbalance_ratio=1.271118 max_load_reduction=55123 max_load_reduction_pct=0.841110 moved_to_echo=32857 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7582, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5940, 2077, 610, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 10427, 8223, 7926, 8192, 7601, 8893, 8159] avg=8192.0000 max=10427 max_rank=1 min=6115 min_rank=0 imbalance_ratio=1.272827 max_load_reduction=55109 max_load_reduction_pct=0.840897 moved_to_echo=32845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 41728, 16640, 5376, 2048, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 49280, 16256, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 48896, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 256, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 41984, 16640, 4992, 2176, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 49792, 15872, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 49024, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 256, 16384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 5376, 60160, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 2176, 0, 0, 63488, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 4992, 60544, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 65280, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 256, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 384, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 2304, 0, 0, 63360, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5430, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 2772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4825, 0, 0, 0, 0, 7973, 0, 0, 0, 0, 0, 0, 0, 3748, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4481, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5848, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 2426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 7959, 1, 0, 0, 0, 1, 0, 8190, 0, 0, 9, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 217, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6895, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 4426, 0, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5932, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4951, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 3331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 7752, 1, 0, 0, 0, 1, 0, 8188, 0, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 19, 0, 0, 433, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 2, 0, 0, 0, 0, 0, 0, 6477, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5742, 4, 0, 0, 0, 0, 2, 8172, 0, 0, 0, 0, 0, 2, 0, 6963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 3, 0, 2, 3, 2852, 0, 2506, 0, 0, 0, 1, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7639, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 0, 0, 0, 5430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4610, 0, 0, 0, 0, 0, 0, 0, 2772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3431, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8142, 9688, 7382, 8190, 8665, 6343, 8853] avg=8192.0000 max=9688 max_rank=2 min=6343 min_rank=6 imbalance_ratio=1.182617 max_load_reduction=55848 max_load_reduction_pct=0.852173 moved_to_echo=32051 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7552, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 0, 0, 0, 4481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4586, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 8106, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3431, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8055, 8739, 8459, 8192, 8652, 6314, 8849] avg=8192.0000 max=8849 max_rank=7 min=6314 min_rank=6 imbalance_ratio=1.080200 max_load_reduction=56687 max_load_reduction_pct=0.864975 moved_to_echo=32007 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7959, 1, 0, 0, 0, 1, 0, 81, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 217, 1, 0, 0, 0, 0, 0, 0, 0, 7638, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4615, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6895, 2, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 4426, 0, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8051, 8366, 4265, 8594, 8190, 7371, 14288, 6411] avg=8192.0000 max=14288 max_rank=6 min=4265 min_rank=2 imbalance_ratio=1.744141 max_load_reduction=51248 max_load_reduction_pct=0.781982 moved_to_echo=36260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7624, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 0, 0, 0, 5932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4621, 0, 0, 0, 0, 0, 0, 0, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3431, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8128, 10190, 6874, 8189, 8665, 6363, 8854] avg=8192.0000 max=10190 max_rank=2 min=6363 min_rank=6 imbalance_ratio=1.243896 max_load_reduction=55346 max_load_reduction_pct=0.844513 moved_to_echo=32071 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7485, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 0, 0, 0, 4825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4501, 0, 0, 0, 0, 0, 0, 0, 3748, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8106, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3431, 2829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 7988, 9083, 8252, 8195, 8635, 6260, 8847] avg=8192.0000 max=9083 max_rank=2 min=6260 min_rank=6 imbalance_ratio=1.108765 max_load_reduction=56453 max_load_reduction_pct=0.861404 moved_to_echo=31937 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7639, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4254, 0, 0, 0, 0, 0, 0, 0, 5848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4564, 0, 0, 0, 0, 0, 0, 0, 2426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3433, 2880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8144, 10102, 6990, 8200, 8659, 6324, 8850] avg=8192.0000 max=10102 max_rank=2 min=6324 min_rank=6 imbalance_ratio=1.233154 max_load_reduction=55434 max_load_reduction_pct=0.845856 moved_to_echo=32033 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7752, 1, 0, 0, 0, 1, 0, 81, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 19, 0, 0, 433, 3, 0, 0, 0, 0, 0, 0, 0, 7621, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4255, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4604, 0, 2, 0, 0, 0, 0, 0, 0, 6477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5742, 4, 0, 0, 0, 0, 2, 473, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 3428, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 3, 2852, 0, 2506, 0, 0, 0, 1, 2, 0, 0, 0, 0, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7847, 8580, 4260, 11083, 8191, 6223, 13325, 6027] avg=8192.0000 max=13325 max_rank=6 min=4260 min_rank=2 imbalance_ratio=1.626587 max_load_reduction=52211 max_load_reduction_pct=0.796677 moved_to_echo=33766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7626, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 0, 0, 0, 4951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4571, 0, 0, 0, 0, 0, 0, 0, 3331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3431, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8133, 9209, 7902, 8189, 8666, 6319, 8845] avg=8192.0000 max=9209 max_rank=2 min=6319 min_rank=6 imbalance_ratio=1.124146 max_load_reduction=56327 max_load_reduction_pct=0.859482 moved_to_echo=32019 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 7814, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 7879, 1, 0, 0, 0, 1, 0, 8190, 0, 0, 9, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 299, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 1959, 4217, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6040, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 5772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 4434, 0, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 7768, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 7739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 7734, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7969, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 7822, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 7800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 707, 0, 0, 0, 0, 7493, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 7475, 1, 0, 0, 0, 1, 0, 8188, 0, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 24, 0, 0, 712, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 2, 0, 0, 0, 0, 0, 460, 6575, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4790, 4, 0, 0, 0, 0, 2, 8170, 0, 0, 0, 0, 0, 3, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 3, 0, 2, 4, 3067, 0, 3467, 0, 0, 0, 1, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7879, 1, 0, 0, 0, 1, 0, 125, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 299, 1, 0, 0, 0, 0, 0, 0, 0, 7641, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 0, 0, 0, 0, 1959, 4217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6040, 2, 0, 0, 0, 0, 0, 712, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6767, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 4434, 0, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8015, 8660, 7740, 6589, 8192, 6755, 12587, 6998] avg=8192.0000 max=12587 max_rank=6 min=6589 min_rank=3 imbalance_ratio=1.536499 max_load_reduction=52949 max_load_reduction_pct=0.807938 moved_to_echo=34532 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7300, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7741, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6703, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 8004, 8188, 8585, 8187, 8898, 6762, 8595] avg=8192.0000 max=8898 max_rank=5 min=6762 min_rank=6 imbalance_ratio=1.086182 max_load_reduction=56638 max_load_reduction_pct=0.864227 moved_to_echo=32442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7257, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7741, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 8062, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6394, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 7931, 8610, 8587, 8208, 8828, 6442, 8610] avg=8192.0000 max=8828 max_rank=5 min=6442 min_rank=6 imbalance_ratio=1.077637 max_load_reduction=56708 max_load_reduction_pct=0.865295 moved_to_echo=32088 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7308, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7741, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 8061, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6444, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8321, 7986, 8500, 8587, 8191, 8861, 6492, 8598] avg=8192.0000 max=8861 max_rank=5 min=6492 min_rank=6 imbalance_ratio=1.081665 max_load_reduction=56675 max_load_reduction_pct=0.864792 moved_to_echo=32142 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7510, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7741, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6739, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 8218, 7920, 8598, 8190, 8902, 6795, 8596] avg=8192.0000 max=8902 max_rank=5 min=6795 min_rank=6 imbalance_ratio=1.086670 max_load_reduction=56634 max_load_reduction_pct=0.864166 moved_to_echo=32483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7006, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7734, 0, 0, 0, 0, 0, 0, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6765, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 7719, 8441, 8558, 8188, 8897, 6821, 8597] avg=8192.0000 max=8897 max_rank=5 min=6821 min_rank=6 imbalance_ratio=1.086060 max_load_reduction=56639 max_load_reduction_pct=0.864243 moved_to_echo=32503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7475, 1, 0, 0, 0, 1, 0, 125, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 24, 0, 0, 712, 3, 0, 0, 0, 0, 0, 0, 0, 7639, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7734, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 2, 0, 0, 0, 0, 0, 460, 6575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4790, 4, 0, 0, 0, 0, 2, 711, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 6750, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4, 3067, 0, 3467, 0, 0, 0, 1, 1, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7614, 9089, 7739, 7450, 8194, 5510, 12993, 6947] avg=8192.0000 max=12993 max_rank=6 min=5510 min_rank=5 imbalance_ratio=1.586060 max_load_reduction=52543 max_load_reduction_pct=0.801743 moved_to_echo=33644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7225, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7741, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6584, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 7929, 8398, 8582, 8186, 8885, 6644, 8595] avg=8192.0000 max=8885 max_rank=5 min=6644 min_rank=6 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=32310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[15104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 27520, 23168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 34176, 0, 128, 0, 128, 0, 0, 0, 31488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 36736, 0, 128, 0, 0, 0, 0, 0, 18432, 10496, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 768, 128, 0, 0, 0, 0, 0, 0, 0, 60928, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 61824, 128, 0, 0, 0, 0, 128, 3840, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 56448, 0, 3968, 0, 0, 0, 128, 128, 0, 0, 0, 0, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 64896, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 64896, 128, 0, 0, 0, 128, 0, 768, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8153, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7681, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 5, 1, 0, 8190, 0, 8069, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 2, 0, 1, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 2, 8190, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8036, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8151, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8176, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8087, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8180, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8089, 8183, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8187, 7, 2, 0, 8188, 0, 8155, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 4, 8185, 0, 0, 0, 4, 0, 2, 0, 3, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 7947, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 36, 0, 8069, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 2, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 1, 0, 0, 0, 8154, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8036, 0, 0, 36, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7766, 8112, 8546, 8196, 8278, 8197, 8354, 8087] avg=8192.0000 max=8546 max_rank=2 min=7766 min_rank=0 imbalance_ratio=1.043213 max_load_reduction=56990 max_load_reduction_pct=0.869598 moved_to_echo=32916 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 36, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8227, 8144, 8196, 8158, 8186, 8147, 8240] avg=8192.0000 max=8240 max_rank=7 min=8144 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32731 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 36, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8223, 8099, 8190, 8157, 8194, 8156, 8240] avg=8192.0000 max=8277 max_rank=0 min=8099 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32747 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 36, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8226, 8096, 8191, 8158, 8192, 8156, 8240] avg=8192.0000 max=8277 max_rank=0 min=8096 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32746 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 8151, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 39, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8192, 8135, 8191, 8161, 8192, 8153, 8248] avg=8192.0000 max=8264 max_rank=0 min=8135 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 8087, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8156, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 36, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8125, 8245, 8193, 8157, 8166, 8149, 8240] avg=8192.0000 max=8261 max_rank=0 min=8125 min_rank=1 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 8180, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 36, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8227, 8111, 8192, 8158, 8189, 8142, 8240] avg=8192.0000 max=8277 max_rank=0 min=8111 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32729 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 36, 0, 8155, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 3, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 1, 0, 0, 0, 8152, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 4, 0, 0, 0, 0, 4, 0, 2, 0, 3, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 7947, 0, 0, 36, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8204, 8160, 8187, 8309, 8220, 8279, 8001] avg=8192.0000 max=8309 max_rank=4 min=8001 min_rank=7 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 896, 0, 0, 0, 0, 0, 128, 0, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 128, 0, 0, 0, 65280, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 384, 0, 65280, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[65280, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65152, 0, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 53248, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 60032, 128, 0, 0, 0, 0, 128, 5632, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 59008, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 56704, 0, 5760, 0, 0, 0, 128, 128, 0, 0, 0, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 3200, 0, 128, 0, 0, 0, 0, 0, 51584, 10880, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 64512, 128, 0, 0, 0, 128, 0, 1024, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 61952, 0, 128, 0, 128, 0, 0, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 64640, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8183, 0, 8192, 0, 0, 20, 0, 0, 0, 8176, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 8142, 0, 8192, 0, 0, 50, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 3, 0, 193, 0, 0, 0, 0, 0, 0, 7996, 0, 0, 0, 0, 5809, 0, 8186, 0, 0, 0, 1, 0, 0, 8189, 0, 1, 0, 0, 8167, 0, 0, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2103, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 8190, 0, 0, 2, 0, 1, 0, 0, 0, 170, 0, 0, 0, 1, 110, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8139, 0, 8192, 0, 0, 78, 0, 0, 0, 8157, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8192, 0, 0, 0, 0, 8134, 0, 8192, 0, 0, 58, 0, 0, 0, 8182, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8188, 6, 0, 0, 0, 0, 0, 4, 0, 887, 0, 0, 0, 0, 0, 1, 7926, 0, 0, 0, 0, 2873, 0, 8174, 2, 0, 0, 1, 1, 0, 8150, 0, 1, 0, 0, 8153, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4379, 3, 0, 0, 0, 0, 0, 0, 0, 3, 8183, 0, 9, 8187, 0, 0, 5, 2, 1, 0, 0, 0, 255, 0, 0, 0, 2, 85, 0, 0, 3, 0, 0, 0, 0, 4, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8182, 0, 8192, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 854, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7338, 69, 20, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8195, 9038, 8176, 7431, 8191, 8120, 8191] avg=8192.0000 max=9038 max_rank=2 min=7431 min_rank=4 imbalance_ratio=1.103271 max_load_reduction=56498 max_load_reduction_pct=0.862091 moved_to_echo=31933 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 854, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7338, 69, 21, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 9045, 8172, 7432, 8191, 8121, 8193] avg=8192.0000 max=9045 max_rank=2 min=7432 min_rank=4 imbalance_ratio=1.104126 max_load_reduction=56491 max_load_reduction_pct=0.861984 moved_to_echo=31937 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 193, 0, 0, 0, 0, 0, 0, 834, 0, 0, 0, 0, 5809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8168, 0, 1, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2103, 1, 0, 0, 0, 0, 7162, 69, 21, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 1, 110, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8189, 6839, 8193, 9361, 8191, 8382, 8192] avg=8192.0000 max=9361 max_rank=4 min=6839 min_rank=2 imbalance_ratio=1.142700 max_load_reduction=56175 max_load_reduction_pct=0.857162 moved_to_echo=34126 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 851, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7341, 69, 24, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8188, 8995, 8216, 7442, 8188, 8120, 8192] avg=8192.0000 max=8995 max_rank=2 min=7442 min_rank=4 imbalance_ratio=1.098022 max_load_reduction=56541 max_load_reduction_pct=0.862747 moved_to_echo=31942 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 854, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7338, 70, 20, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8199, 8995, 8215, 7432, 8191, 8120, 8191] avg=8192.0000 max=8995 max_rank=2 min=7432 min_rank=4 imbalance_ratio=1.098022 max_load_reduction=56541 max_load_reduction_pct=0.862747 moved_to_echo=31934 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 854, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7338, 69, 21, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8189, 9039, 8178, 7432, 8191, 8120, 8192] avg=8192.0000 max=9039 max_rank=2 min=7432 min_rank=4 imbalance_ratio=1.103394 max_load_reduction=56497 max_load_reduction_pct=0.862076 moved_to_echo=31935 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 854, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7338, 69, 21, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8193, 8994, 8219, 7432, 8191, 8120, 8192] avg=8192.0000 max=8994 max_rank=2 min=7432 min_rank=4 imbalance_ratio=1.097900 max_load_reduction=56542 max_load_reduction_pct=0.862762 moved_to_echo=31935 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8173, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8185, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 887, 0, 0, 0, 0, 0, 1, 826, 0, 0, 0, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 0, 8130, 0, 1, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4379, 3, 0, 0, 0, 0, 7100, 68, 20, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8182, 0, 9, 0, 0, 0, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 2, 85, 0, 0, 3, 0, 0, 0, 0, 4, 0, 3, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8194, 4591, 8167, 11574, 8202, 8433, 8193] avg=8192.0000 max=11574 max_rank=4 min=4591 min_rank=2 imbalance_ratio=1.412842 max_load_reduction=53962 max_load_reduction_pct=0.823395 moved_to_echo=36402 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7149, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8189, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3350, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 5, 1, 0, 8190, 0, 8179, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7079, 0, 0, 0, 0, 0, 2, 0, 1, 0, 8, 41, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 104, 0, 0, 2, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5591, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8125, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8062, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2637, 8182, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8187, 7, 2, 0, 8188, 0, 8168, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6193, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 457, 0, 0, 4, 8187, 0, 0, 0, 5, 0, 1, 0, 4, 0, 0, 6913, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 8060, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8155, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 1298, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6611, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8190, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1106, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8171, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8189, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 743, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7850, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8191, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6760, 0, 0, 0, 0, 0, 0, 0, 1973, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 0, 0, 0, 0, 0, 0, 7209, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 6195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8844, 8190, 9068, 8453, 8132, 8179, 6455, 8215] avg=8192.0000 max=9068 max_rank=2 min=6455 min_rank=6 imbalance_ratio=1.106934 max_load_reduction=56468 max_load_reduction_pct=0.861633 moved_to_echo=30981 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 8175, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7079, 0, 0, 0, 0, 0, 2, 0, 1974, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 41, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 1, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 7203, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 2, 6197, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8125, 0, 0, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5043, 8186, 9389, 7187, 8363, 7321, 11896, 8151] avg=8192.0000 max=11896 max_rank=6 min=5043 min_rank=0 imbalance_ratio=1.452148 max_load_reduction=53640 max_load_reduction_pct=0.818481 moved_to_echo=35731 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2637, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 8164, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6193, 0, 0, 0, 0, 0, 3, 0, 1973, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6159, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 2, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 7200, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 4, 6195, 0, 0, 0, 5, 0, 1, 0, 4, 0, 0, 6913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 8060, 0, 0, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4333, 8179, 8504, 7145, 8383, 7324, 13579, 8089] avg=8192.0000 max=13579 max_rank=6 min=4333 min_rank=0 imbalance_ratio=1.657593 max_load_reduction=51957 max_load_reduction_pct=0.792801 moved_to_echo=37375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5294, 0, 0, 0, 0, 0, 0, 0, 1974, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 7209, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1298, 0, 0, 0, 6198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9850, 8188, 7603, 8399, 8132, 7372, 7777, 8215] avg=8192.0000 max=9850 max_rank=0 min=7372 min_rank=5 imbalance_ratio=1.202393 max_load_reduction=55686 max_load_reduction_pct=0.849701 moved_to_echo=31496 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7063, 0, 0, 0, 0, 0, 0, 0, 1972, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1106, 0, 0, 0, 0, 0, 0, 7210, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 6193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8190, 9370, 8484, 8131, 8377, 6463, 8215] avg=8192.0000 max=9370 max_rank=2 min=6463 min_rank=6 imbalance_ratio=1.143799 max_load_reduction=56166 max_load_reduction_pct=0.857025 moved_to_echo=31186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3149, 0, 0, 0, 0, 0, 0, 0, 1974, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 0, 0, 0, 0, 0, 0, 7210, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 6199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9757, 8189, 5458, 8806, 8131, 10380, 6600, 8215] avg=8192.0000 max=10380 max_rank=5 min=5458 min_rank=2 imbalance_ratio=1.267090 max_load_reduction=55156 max_load_reduction_pct=0.841614 moved_to_echo=33326 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5201, 0, 0, 0, 0, 0, 0, 0, 1979, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6168, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 743, 0, 0, 0, 0, 0, 0, 7209, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 6191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9858, 8212, 7517, 9073, 8132, 8012, 6511, 8221] avg=8192.0000 max=9858 max_rank=0 min=6511 min_rank=6 imbalance_ratio=1.203369 max_load_reduction=55678 max_load_reduction_pct=0.849579 moved_to_echo=30876 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6318, 0, 0, 0, 0, 0, 0, 0, 1974, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 0, 0, 0, 0, 0, 0, 7209, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 6199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9545, 8202, 8627, 7989, 8132, 8571, 6255, 8215] avg=8192.0000 max=9545 max_rank=0 min=6255 min_rank=6 imbalance_ratio=1.165161 max_load_reduction=55991 max_load_reduction_pct=0.854355 moved_to_echo=31173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 47104, 0, 0, 0, 0, 0, 128, 0, 15872, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 8448, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49408, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 52096, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3072, 0, 0, 128, 49664, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 7424, 0, 0, 0, 0, 0, 0, 57728, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65408, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 128, 0, 0, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8135, 0, 8192, 0, 0, 77, 0, 0, 0, 8160, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8192, 0, 0, 0, 0, 8137, 0, 8192, 0, 0, 55, 0, 0, 0, 8182, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8183, 0, 8192, 0, 0, 22, 0, 0, 0, 8174, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 7975, 0, 0, 0, 0, 5712, 0, 8186, 0, 0, 0, 1, 0, 0, 8188, 0, 1, 0, 0, 8169, 0, 0, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2392, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 1, 8189, 0, 0, 2, 0, 1, 0, 0, 0, 193, 0, 0, 0, 1, 108, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8187, 6, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 6, 7890, 0, 0, 0, 0, 2960, 0, 8173, 2, 0, 0, 2, 2, 0, 8141, 0, 2, 0, 0, 8152, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5160, 3, 0, 0, 0, 0, 0, 0, 0, 3, 8182, 0, 8, 8187, 0, 0, 5, 2, 0, 0, 0, 0, 291, 0, 0, 0, 2, 97, 0, 0, 3, 0, 0, 0, 0, 4, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 8189, 0, 8192, 0, 0, 3, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8170, 0, 8192, 0, 0, 22, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 8133, 0, 8192, 0, 0, 59, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 993, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7199, 77, 22, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8201, 9130, 8215, 7302, 8191, 8112, 8195] avg=8192.0000 max=9130 max_rank=2 min=7302 min_rank=4 imbalance_ratio=1.114502 max_load_reduction=56406 max_load_reduction_pct=0.860687 moved_to_echo=31800 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 993, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7199, 78, 22, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8194, 9177, 8174, 7303, 8191, 8111, 8194] avg=8192.0000 max=9177 max_rank=2 min=7303 min_rank=4 imbalance_ratio=1.120239 max_load_reduction=56359 max_load_reduction_pct=0.859970 moved_to_echo=31799 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 967, 0, 0, 0, 0, 5712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8166, 0, 1, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2392, 1, 0, 0, 0, 0, 7008, 77, 22, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 1, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 1, 108, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8190, 6685, 8188, 9504, 8192, 8397, 8189] avg=8192.0000 max=9504 max_rank=4 min=6685 min_rank=2 imbalance_ratio=1.160156 max_load_reduction=56032 max_load_reduction_pct=0.854980 moved_to_echo=34282 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8184, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 0, 0, 0, 6, 956, 0, 0, 0, 0, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 2, 0, 8119, 0, 2, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5160, 3, 0, 0, 0, 0, 6934, 78, 22, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8182, 0, 8, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 2, 97, 0, 0, 3, 0, 0, 0, 0, 4, 0, 4, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8172, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8194, 3936, 8162, 12200, 8200, 8472, 8180] avg=8192.0000 max=12200 max_rank=4 min=3936 min_rank=2 imbalance_ratio=1.489258 max_load_reduction=53336 max_load_reduction_pct=0.813843 moved_to_echo=37052 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 993, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7199, 78, 22, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 9163, 8192, 7303, 8191, 8111, 8194] avg=8192.0000 max=9163 max_rank=2 min=7303 min_rank=4 imbalance_ratio=1.118530 max_load_reduction=56373 max_load_reduction_pct=0.860184 moved_to_echo=31799 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 993, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7199, 77, 22, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 9185, 8170, 7302, 8191, 8111, 8196] avg=8192.0000 max=9185 max_rank=2 min=7302 min_rank=4 imbalance_ratio=1.121216 max_load_reduction=56351 max_load_reduction_pct=0.859848 moved_to_echo=31800 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 988, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7204, 78, 28, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8186, 9124, 8220, 7319, 8189, 8111, 8194] avg=8192.0000 max=9124 max_rank=2 min=7319 min_rank=4 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=31813 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 993, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7199, 78, 22, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8193, 9136, 8215, 7303, 8191, 8111, 8194] avg=8192.0000 max=9136 max_rank=2 min=7303 min_rank=4 imbalance_ratio=1.115234 max_load_reduction=56400 max_load_reduction_pct=0.860596 moved_to_echo=31799 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6528, 128, 0, 0, 0, 0, 58368, 640, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 1152, 0, 0, 0, 0, 0, 128, 6784, 0, 0, 0, 0, 57728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 256, 128, 128, 0, 65280, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 0, 0, 0, 128, 256, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 128, 7936, 0, 0, 0, 0, 57728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7552, 128, 0, 0, 0, 0, 57216, 640, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 0, 0, 0, 128, 256, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 256, 128, 128, 0, 65280, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 5713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3337, 0, 3, 0, 0, 0, 0, 0, 0, 6968, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1563, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5518, 0, 0, 0, 0, 0, 0, 0, 0, 7560, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 7508, 0, 0, 2, 11, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 8190, 6, 0, 161, 9, 0, 0, 0, 2, 0, 1, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 7941, 2, 1, 0, 0, 1, 0, 0, 1, 0, 7329, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 6026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 0, 0, 0, 0, 0, 0, 6004, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 6815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4390, 0, 3, 0, 0, 0, 0, 0, 0, 6560, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 979, 21, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 6421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4957, 0, 0, 0, 0, 0, 0, 0, 0, 6605, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8185, 0, 0, 7681, 0, 0, 3, 0, 1685, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 1, 0, 3, 1, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 84, 6, 1, 0, 0, 3, 0, 2, 0, 0, 8186, 0, 3, 1, 0, 0, 0, 7861, 4, 0, 0, 0, 1, 0, 0, 5, 0, 7006, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 259, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 0, 0, 0, 0, 0, 0, 0, 0, 7254, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1441, 0, 0, 5713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7158, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3337, 0, 3, 3974, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6968, 0, 0, 0, 0, 0, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5719, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7154, 8190, 8416, 8002, 8668, 8763, 8151, 8192] avg=8192.0000 max=8763 max_rank=5 min=7154 min_rank=0 imbalance_ratio=1.069702 max_load_reduction=56773 max_load_reduction_pct=0.866287 moved_to_echo=33774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1441, 0, 0, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7158, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5518, 0, 0, 3975, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7560, 0, 0, 0, 0, 0, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5719, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4484, 8190, 10619, 8594, 9182, 8123, 8152, 8192] avg=8192.0000 max=10619 max_rank=2 min=4484 min_rank=0 imbalance_ratio=1.296265 max_load_reduction=54917 max_load_reduction_pct=0.837967 moved_to_echo=33649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1440, 0, 0, 7508, 0, 0, 2, 11, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 7157, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 3853, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 1033, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 9, 0, 0, 0, 2, 0, 1, 0, 0, 7096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1730, 2, 1, 0, 0, 1, 0, 0, 1, 0, 7329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 5715, 2358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10411, 8191, 4949, 9190, 7270, 9064, 8270, 8191] avg=8192.0000 max=10411 max_rank=0 min=4949 min_rank=2 imbalance_ratio=1.270874 max_load_reduction=55125 max_load_reduction_pct=0.841141 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1441, 0, 0, 6421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7158, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4957, 0, 0, 3975, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6605, 0, 0, 0, 0, 0, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5719, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7862, 8190, 10035, 7639, 9273, 6191, 8154, 8192] avg=8192.0000 max=10035 max_rank=2 min=6191 min_rank=5 imbalance_ratio=1.224976 max_load_reduction=55501 max_load_reduction_pct=0.846878 moved_to_echo=31810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1434, 0, 0, 6026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7164, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 3979, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6004, 0, 0, 0, 0, 0, 0, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 5724, 2436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7460, 8198, 10222, 7032, 8258, 7974, 8196, 8196] avg=8192.0000 max=10222 max_rank=2 min=7032 min_rank=3 imbalance_ratio=1.247803 max_load_reduction=55314 max_load_reduction_pct=0.844025 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1441, 0, 0, 6815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7158, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4390, 0, 3, 3974, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6560, 0, 0, 0, 0, 0, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 21, 0, 0, 0, 0, 0, 0, 0, 0, 7098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5719, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8190, 9461, 7594, 8098, 7594, 8151, 8192] avg=8192.0000 max=9461 max_rank=2 min=7594 min_rank=3 imbalance_ratio=1.154907 max_load_reduction=56075 max_load_reduction_pct=0.855637 moved_to_echo=32035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1440, 0, 0, 7681, 0, 0, 3, 0, 1685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 1, 0, 3, 1, 4, 0, 0, 7154, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3814, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 6, 1, 0, 0, 3, 0, 2, 0, 0, 7093, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 4, 0, 0, 0, 1, 0, 0, 5, 0, 7006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 259, 0, 5714, 2334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10809, 8197, 4912, 9197, 7193, 8729, 8310, 8189] avg=8192.0000 max=10809 max_rank=0 min=4912 min_rank=2 imbalance_ratio=1.319458 max_load_reduction=54727 max_load_reduction_pct=0.835068 moved_to_echo=32421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1441, 0, 0, 7659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7158, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 0, 0, 3975, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7254, 0, 0, 0, 0, 0, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5719, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9100, 8190, 6922, 8288, 7594, 9098, 8152, 8192] avg=8192.0000 max=9100 max_rank=0 min=6922 min_rank=2 imbalance_ratio=1.110840 max_load_reduction=56436 max_load_reduction_pct=0.861145 moved_to_echo=33036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 3, 0, 0, 0, 0, 0, 0, 7816, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 819, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8184, 0, 0, 8142, 0, 0, 4, 0, 1574, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3, 1, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 246, 2, 1, 1, 0, 3, 0, 2, 0, 0, 8186, 0, 3, 1, 0, 0, 0, 7991, 4, 0, 0, 0, 1, 0, 0, 6, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 0, 0, 0, 0, 0, 0, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8185, 0, 0, 8152, 0, 0, 2, 9, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 8189, 6, 0, 388, 0, 0, 0, 0, 2, 0, 1, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 8060, 2, 0, 0, 0, 1, 0, 0, 1, 0, 6550, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 3, 0, 0, 0, 0, 0, 0, 7277, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1052, 16, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 7730, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7820, 0, 0, 4, 0, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3, 1, 4, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4950, 1951, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 2, 1, 1, 0, 3, 0, 2, 0, 0, 6235, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2711, 4, 0, 0, 0, 1, 0, 0, 6, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 514, 0, 7761, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9398, 8196, 7228, 8513, 6494, 8909, 8608, 8190] avg=8192.0000 max=9398 max_rank=0 min=6494 min_rank=4 imbalance_ratio=1.147217 max_load_reduction=56138 max_load_reduction_pct=0.856598 moved_to_echo=32201 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 3, 5075, 1952, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7277, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 16, 0, 0, 0, 0, 0, 0, 0, 0, 6240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7767, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7862, 8192, 7809, 7702, 7308, 10366, 8105, 8192] avg=8192.0000 max=10366 max_rank=5 min=7308 min_rank=4 imbalance_ratio=1.265381 max_load_reduction=55170 max_load_reduction_pct=0.841827 moved_to_echo=33971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7830, 0, 0, 2, 9, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4993, 1951, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 425, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 2, 0, 1, 0, 0, 6238, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 2, 0, 0, 0, 1, 0, 0, 1, 0, 6550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 89, 0, 7764, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9170, 8188, 7269, 8608, 6630, 9288, 8192, 8191] avg=8192.0000 max=9288 max_rank=5 min=6630 min_rank=4 imbalance_ratio=1.133789 max_load_reduction=56248 max_load_reduction_pct=0.858276 moved_to_echo=32301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 3, 5075, 1952, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7816, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7767, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7862, 8192, 7678, 8241, 7064, 10199, 8108, 8192] avg=8192.0000 max=10199 max_rank=5 min=7064 min_rank=4 imbalance_ratio=1.244995 max_load_reduction=55337 max_load_reduction_pct=0.844376 moved_to_echo=33563 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 0, 0, 5079, 1956, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 7769, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7813, 8192, 8362, 7862, 10396, 6517, 8199, 8195] avg=8192.0000 max=10396 max_rank=4 min=6517 min_rank=5 imbalance_ratio=1.269043 max_load_reduction=55140 max_load_reduction_pct=0.841370 moved_to_echo=33307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2400, 0, 0, 5075, 1952, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7767, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7853, 8192, 9794, 8082, 11777, 3536, 8110, 8192] avg=8192.0000 max=11777 max_rank=4 min=3536 min_rank=5 imbalance_ratio=1.437622 max_load_reduction=53759 max_load_reduction_pct=0.820297 moved_to_echo=31615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 0, 0, 5075, 1952, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7767, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7754, 8192, 8705, 8373, 7866, 8345, 8109, 8192] avg=8192.0000 max=8705 max_rank=2 min=7754 min_rank=0 imbalance_ratio=1.062622 max_load_reduction=56831 max_load_reduction_pct=0.867172 moved_to_echo=32512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1338, 0, 0, 5075, 1952, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7730, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7767, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7824, 8192, 8691, 8155, 8001, 8376, 8105, 8192] avg=8192.0000 max=8691 max_rank=2 min=7824 min_rank=0 imbalance_ratio=1.060913 max_load_reduction=56845 max_load_reduction_pct=0.867386 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 25216, 0, 128, 31616, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 512, 0, 45824, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 11520, 0, 0, 50944, 0, 0, 128, 128, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 14208, 128, 128, 0, 0, 128, 0, 0, 128, 0, 51456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 57344, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 57344, 0, 0, 0, 0, 0, 0, 8320, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8832, 128, 128, 0, 0, 128, 0, 128, 0, 0, 56832, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 6912, 0, 128, 40448, 15744, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 22144, 128, 0, 0, 0, 128, 0, 0, 128, 0, 43520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[15616, 128, 128, 128, 0, 128, 0, 128, 0, 0, 49920, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 768, 0, 62208, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 62208, 0, 0, 0, 0, 0, 0, 3456, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 62720, 0, 0, 128, 128, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True - [2026-08-26 16:18:08] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 452.5 | throughput per GPU (TFLOP/s/GPU): 399.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.418067E+00 | loss scale: 1.0 | grad norm: 5.080 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3350, 0, 0, 0, 1, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 2869, 0, 0, 3668, 0, 0, 0, 8191, 1, 8191, 13, 0, 0, 0, 3, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 3208, 0, 0, 0, 3268, 0, 2552, 29, 0, 0, 3824, 0, 0, 0, 0, 0, 1, 0, 0, 8012, 0, 0, 0, 0, 1, 0, 1780, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 17, 0, 1, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 18, 2990, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3182, 0, 0, 2948, 14, 0, 0, 8192, 1, 8192, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 1, 0, 0, 0, 3155, 0, 2, 0, 3340, 0, 2966, 14, 0, 0, 4272, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 1, 0, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8074, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 346, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 3031, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 2840, 3, 1, 2639, 0, 0, 0, 8186, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 4, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 2, 0, 3520, 0, 2, 0, 3248, 1, 3120, 28, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 1, 0, 7975, 0, 0, 1, 0, 0, 0, 2291, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 8055, 1, 1, 0, 0, 0, 1, 0, 4, 0, 1, 2, 0, 0, 0, 0, 0, 344, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 3459, 0, 0, 0, 0, 0, 4, 0, 0, 1, 15, 0, 6, 0, 0, 2645, 0, 1, 2609, 0, 1, 2, 8191, 2, 8187, 1, 0, 0, 4, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2, 3, 8178, 2, 0, 0, 0, 0, 0, 0, 0, 2983, 2, 0, 2, 3364, 0, 3161, 33, 0, 0, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 5, 2, 0, 2, 0, 2331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 26, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 2, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2688, 0, 0, 2540, 5, 0, 0, 8192, 0, 8191, 17, 1, 0, 0, 0, 0, 4, 0, 0, 1, 3, 2, 1, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 3101, 0, 0, 1, 4148, 0, 2998, 26, 0, 0, 4045, 0, 0, 3, 0, 0, 2, 0, 0, 8015, 0, 17, 0, 1, 1, 0, 1571, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7979, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 2875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1949, 0, 0, 3164, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 1, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 2656, 0, 0, 0, 3323, 0, 3568, 31, 0, 0, 4479, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 2, 0, 0, 0, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8068, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 11, 3523, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2584, 0, 0, 3334, 0, 0, 0, 8191, 1, 8182, 0, 3, 0, 0, 2, 1, 1, 5, 0, 0, 7, 0, 0, 0, 0, 2, 13, 1, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 3081, 0, 0, 0, 3361, 0, 2165, 12, 3, 0, 3842, 2, 0, 0, 3, 3, 0, 1, 0, 8062, 0, 2, 0, 0, 0, 0, 2450, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 8129, 0, 1, 5, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 2, 0, 0, 339, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 3100, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 3676, 0, 3074, 8, 0, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 6, 0, 0, 0, 0, 2004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3350, 0, 0, 0, 1, 1, 0, 0, 0, 0, 21, 1906, 1750, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2869, 0, 0, 3668, 0, 0, 0, 0, 1, 2489, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 1, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3208, 0, 0, 0, 2047, 0, 2552, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6106, 0, 0, 0, 0, 1, 0, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 17, 0, 1, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 5702, 2074, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 9041, 8171, 8192, 7836, 7888, 8234, 8027] avg=8192.0000 max=9041 max_rank=1 min=7836 min_rank=4 imbalance_ratio=1.103638 max_load_reduction=56495 max_load_reduction_pct=0.862045 moved_to_echo=31985 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 18, 2990, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1923, 1956, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3182, 0, 0, 2948, 14, 0, 0, 0, 1, 2490, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 13, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3155, 0, 2, 0, 2092, 0, 2966, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 1, 0, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8074, 0, 0, 10, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 346, 0, 0, 0, 0, 5702, 2316, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8032, 8657, 8189, 8191, 8230, 7646, 8198, 8393] avg=8192.0000 max=8657 max_rank=1 min=7646 min_rank=5 imbalance_ratio=1.056763 max_load_reduction=56879 max_load_reduction_pct=0.867905 moved_to_echo=32467 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 3031, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1898, 1754, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2840, 3, 1, 2639, 0, 0, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 4, 0, 6, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8186, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3520, 0, 2, 0, 2035, 1, 3120, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6077, 0, 0, 1, 0, 0, 0, 2291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 8055, 1, 1, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 1, 2, 0, 0, 0, 0, 0, 344, 0, 0, 0, 3, 5700, 2076, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7802, 7971, 8173, 8191, 8708, 8370, 8163, 8158] avg=8192.0000 max=8708 max_rank=4 min=7802 min_rank=0 imbalance_ratio=1.062988 max_load_reduction=56828 max_load_reduction_pct=0.867126 moved_to_echo=33399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 3459, 0, 0, 0, 0, 0, 4, 0, 0, 1, 15, 1945, 1727, 1150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2645, 0, 1, 2609, 0, 1, 2, 0, 2, 2488, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 1, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2, 3, 8178, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2983, 2, 0, 2, 2107, 0, 3161, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6231, 0, 5, 2, 0, 2, 0, 2331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 26, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 2, 0, 1, 0, 5699, 2046, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 7755, 8183, 8190, 8288, 8571, 8150, 8095] avg=8192.0000 max=8571 max_rank=5 min=7755 min_rank=1 imbalance_ratio=1.046265 max_load_reduction=56965 max_load_reduction_pct=0.869217 moved_to_echo=33104 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1912, 1852, 1421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2688, 0, 0, 2540, 5, 0, 0, 0, 0, 2484, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 3, 2, 1, 0, 0, 0, 22, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3101, 0, 0, 1, 2588, 0, 2998, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 6103, 0, 17, 0, 1, 1, 0, 1571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7979, 0, 0, 15, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 5707, 2193, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8673, 7735, 8199, 8192, 8714, 7698, 8128, 8197] avg=8192.0000 max=8714 max_rank=4 min=7698 min_rank=5 imbalance_ratio=1.063721 max_load_reduction=56822 max_load_reduction_pct=0.867035 moved_to_echo=32737 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 11, 3523, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 1918, 1758, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 0, 0, 3334, 0, 0, 0, 0, 1, 2487, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 5, 0, 0, 7, 0, 0, 0, 0, 2, 13, 1, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3081, 0, 0, 0, 2106, 0, 2165, 12, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 1, 0, 6144, 0, 2, 0, 0, 0, 0, 2450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 8129, 0, 1, 5, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 2, 0, 0, 339, 0, 0, 0, 2, 5695, 2084, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8366, 8409, 8196, 8196, 7369, 8603, 8242, 8155] avg=8192.0000 max=8603 max_rank=5 min=7369 min_rank=4 imbalance_ratio=1.050171 max_load_reduction=56933 max_load_reduction_pct=0.868729 moved_to_echo=32369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1935, 1719, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 3100, 0, 0, 0, 0, 0, 2490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 30, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 2303, 0, 3074, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6199, 0, 6, 0, 0, 0, 0, 2004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 2, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 0, 0, 0, 0, 5702, 2034, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8365, 8197, 8194, 8054, 8209, 8210, 8109] avg=8192.0000 max=8365 max_rank=1 min=8054 min_rank=4 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32582 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 2875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1942, 2051, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 0, 0, 3164, 0, 0, 0, 0, 0, 2490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 1, 0, 0, 56, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2656, 0, 0, 0, 2082, 0, 3568, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 0, 2, 0, 0, 0, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8068, 0, 0, 40, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 5702, 2428, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8014, 7603, 8228, 8190, 8337, 8551, 8211, 8402] avg=8192.0000 max=8551 max_rank=5 min=7603 min_rank=1 imbalance_ratio=1.043823 max_load_reduction=56985 max_load_reduction_pct=0.869522 moved_to_echo=33501 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True + [2026-08-26 16:27:16] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 434.5 | throughput per GPU (TFLOP/s/GPU): 416.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.422771E+00 | loss scale: 1.0 | grad norm: 5.086 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 3590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2723, 0, 0, 3011, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2616, 0, 0, 0, 3550, 0, 3216, 30, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 0, 44, 0, 0, 0, 0, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2718, 0, 0, 2433, 5, 0, 0, 8192, 0, 8191, 16, 0, 0, 0, 1, 0, 4, 0, 0, 1, 4, 2, 1, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 0, 1, 4055, 0, 3210, 81, 0, 1, 4465, 0, 0, 3, 0, 0, 2, 0, 0, 7131, 0, 49, 0, 1, 1, 0, 1559, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7929, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 356, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1875, 0, 0, 3051, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 1, 0, 0, 2, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 0, 0, 0, 3197, 0, 3735, 69, 0, 0, 4862, 0, 0, 0, 0, 0, 0, 0, 0, 7646, 0, 6, 0, 0, 0, 0, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 3816, 0, 0, 0, 0, 0, 4, 0, 0, 1, 18, 0, 6, 0, 2, 2563, 0, 1, 2452, 0, 1, 2, 8191, 2, 8187, 1, 0, 0, 4, 2, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2, 3, 8178, 2, 0, 0, 0, 0, 0, 0, 0, 2946, 2, 0, 2, 3287, 0, 3385, 64, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 7660, 0, 9, 2, 0, 2, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7991, 0, 0, 19, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 2, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3716, 0, 0, 0, 1, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 2843, 0, 0, 3631, 0, 0, 0, 8191, 1, 8191, 12, 0, 0, 0, 8, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 3184, 0, 0, 0, 3214, 0, 2774, 69, 0, 0, 4220, 0, 0, 0, 0, 0, 1, 0, 0, 6965, 0, 2, 0, 0, 1, 0, 1788, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 12, 0, 1, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 10, 3882, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2575, 0, 0, 3326, 0, 0, 0, 8191, 1, 8182, 0, 2, 0, 0, 3, 1, 1, 5, 0, 0, 7, 0, 0, 0, 0, 2, 16, 1, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 3021, 0, 0, 0, 3297, 0, 2334, 32, 3, 0, 4155, 2, 0, 0, 3, 3, 0, 1, 0, 7335, 0, 26, 0, 0, 0, 0, 2401, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 8123, 0, 1, 2, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 2, 0, 0, 377, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 3368, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3105, 0, 0, 2852, 15, 0, 0, 8192, 1, 8192, 21, 0, 0, 0, 1, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 1, 0, 0, 0, 3116, 0, 2, 0, 3232, 0, 3144, 61, 0, 0, 4663, 0, 0, 0, 0, 0, 0, 0, 0, 7434, 0, 13, 0, 0, 1, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8054, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 362, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2807, 3, 1, 2640, 0, 0, 0, 8186, 0, 8188, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 3, 0, 3, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 3, 0, 3517, 0, 2, 0, 3144, 1, 3390, 94, 0, 0, 4214, 0, 0, 0, 0, 0, 0, 1, 0, 7085, 0, 3, 1, 0, 0, 0, 2223, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 8059, 1, 1, 0, 0, 0, 1, 0, 4, 0, 1, 1, 0, 0, 0, 0, 0, 406, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3716, 0, 0, 0, 1, 1, 0, 0, 0, 0, 27, 2069, 1341, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2843, 0, 0, 3631, 0, 0, 0, 0, 1, 2606, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 3, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3184, 0, 0, 0, 1873, 0, 2774, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5842, 0, 2, 0, 0, 1, 0, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 12, 0, 1, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 5585, 2151, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 9094, 8173, 8193, 7900, 7634, 8252, 8133] avg=8192.0000 max=9094 max_rank=1 min=7634 min_rank=5 imbalance_ratio=1.110107 max_load_reduction=56442 max_load_reduction_pct=0.861237 moved_to_echo=31919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 10, 3882, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3, 2037, 1376, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2575, 0, 0, 3326, 0, 0, 0, 0, 1, 2603, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 5, 0, 0, 7, 0, 0, 0, 0, 2, 16, 1, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3021, 0, 0, 0, 1921, 0, 2334, 32, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 1, 0, 6153, 0, 26, 0, 0, 0, 0, 2401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 8123, 0, 1, 2, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 2, 0, 0, 377, 0, 0, 0, 2, 5579, 2118, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 8507, 8194, 8196, 7313, 8587, 8255, 8117] avg=8192.0000 max=8587 max_rank=5 min=7313 min_rank=4 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32272 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2190, 1695, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2718, 0, 0, 2433, 5, 0, 0, 0, 0, 2600, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 1, 4, 2, 1, 0, 0, 0, 22, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 0, 1, 2360, 0, 3210, 81, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 5972, 0, 49, 0, 1, 1, 0, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7929, 0, 0, 13, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 356, 0, 1, 0, 0, 5591, 2275, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8746, 7772, 8195, 8190, 8720, 7587, 8068, 8258] avg=8192.0000 max=8746 max_rank=0 min=7587 min_rank=5 imbalance_ratio=1.067627 max_load_reduction=56790 max_load_reduction_pct=0.866547 moved_to_echo=32633 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 3590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 1481, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2723, 0, 0, 3011, 0, 0, 0, 0, 0, 2606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 39, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2616, 0, 0, 0, 2069, 0, 3216, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6473, 0, 44, 0, 0, 0, 0, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 1, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 5586, 2056, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8341, 8200, 8194, 7931, 8451, 8221, 8035] avg=8192.0000 max=8451 max_rank=5 min=7931 min_rank=4 imbalance_ratio=1.031616 max_load_reduction=57085 max_load_reduction_pct=0.871048 moved_to_echo=32638 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 3368, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 2288, 1349, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3105, 0, 0, 2852, 15, 0, 0, 0, 1, 2606, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 15, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3116, 0, 2, 0, 1883, 0, 3144, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6235, 0, 13, 0, 0, 1, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8054, 0, 0, 7, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 362, 0, 0, 0, 0, 5586, 2375, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8600, 8187, 8191, 8207, 7696, 8199, 8358] avg=8192.0000 max=8600 max_rank=1 min=7696 min_rank=5 imbalance_ratio=1.049805 max_load_reduction=56936 max_load_reduction_pct=0.868774 moved_to_echo=32460 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 3816, 0, 0, 0, 0, 0, 4, 0, 0, 1, 18, 2007, 1372, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 2563, 0, 1, 2452, 0, 1, 2, 0, 2, 2604, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7, 1, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2, 3, 8178, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2946, 2, 0, 2, 1915, 0, 3385, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6425, 0, 9, 2, 0, 2, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7991, 0, 0, 19, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 2, 0, 1, 0, 5583, 2085, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 7634, 8182, 8190, 8314, 8710, 8142, 8042] avg=8192.0000 max=8710 max_rank=5 min=7634 min_rank=1 imbalance_ratio=1.063232 max_load_reduction=56826 max_load_reduction_pct=0.867096 moved_to_echo=33208 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2385, 1334, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 0, 0, 3051, 0, 0, 0, 0, 0, 2606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 1, 0, 0, 2, 0, 0, 64, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 0, 0, 0, 1863, 0, 3735, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6414, 0, 6, 0, 0, 0, 0, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 0, 0, 25, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0, 5586, 2477, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7943, 7532, 8232, 8191, 8312, 8701, 8211, 8414] avg=8192.0000 max=8701 max_rank=5 min=7532 min_rank=1 imbalance_ratio=1.062134 max_load_reduction=56835 max_load_reduction_pct=0.867233 moved_to_echo=33638 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2067, 1312, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 3, 1, 2640, 0, 0, 0, 0, 0, 2605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 3, 0, 3, 0, 10, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8186, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3517, 0, 2, 0, 1832, 1, 3390, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5942, 0, 3, 1, 0, 0, 0, 2223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 8059, 1, 1, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 1, 1, 0, 0, 0, 0, 0, 406, 0, 0, 0, 3, 5583, 2147, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7740, 8056, 8173, 8191, 8839, 8170, 8188, 8179] avg=8192.0000 max=8839 max_rank=4 min=7740 min_rank=0 imbalance_ratio=1.078979 max_load_reduction=56697 max_load_reduction_pct=0.865128 moved_to_echo=33376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 26112, 0, 0, 128, 128, 128, 128, 0, 0, 128, 128, 15488, 14592, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 0, 21632, 128, 128, 24064, 128, 128, 128, 0, 128, 19968, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 0, 24448, 128, 128, 128, 17408, 128, 23680, 256, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 49280, 0, 128, 128, 128, 128, 0, 16256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 0, 2432, 128, 0, 128, 128, 45696, 17280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 28672, 0, 0, 128, 128, 128, 128, 0, 0, 128, 128, 17024, 11264, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 21248, 128, 128, 23424, 128, 128, 128, 0, 128, 20864, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 0, 24192, 128, 128, 128, 15744, 128, 25216, 512, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 49536, 0, 256, 128, 128, 128, 0, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67456 observed_loss=-1920 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 0, 2944, 128, 128, 128, 128, 44800, 17792, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 128, 128, 128, 64640, 128, 128, 128, 0, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 128, 128, 128, 64512, 128, 128, 128, 0, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 256, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 256, 128, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 65536, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67456 slack_slots=63616 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8135, 0, 6617, 0, 0, 8156, 0, 0, 247, 0, 0, 0, 0, 0, 954, 8192, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 50, 0, 0, 0, 8011, 0, 6652, 0, 0, 8179, 0, 0, 47, 0, 0, 0, 0, 0, 1244, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 21, 0, 0, 0, 7843, 0, 2680, 0, 0, 8191, 0, 0, 12, 0, 0, 0, 0, 0, 1525, 8192, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 0, 0, 7860, 6, 3143, 4, 0, 8186, 0, 0, 16, 0, 1, 0, 0, 0, 2630, 8186, 6, 0, 0, 8183, 0, 2, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8187, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8184, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 0, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 12, 0, 0, 0, 8120, 0, 5054, 0, 0, 8181, 0, 0, 14, 0, 0, 0, 0, 0, 1223, 8192, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 12, 0, 0, 0, 6794, 0, 2766, 0, 0, 8188, 0, 0, 15, 0, 0, 0, 0, 0, 3584, 8192, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 8004, 0, 7201, 0, 0, 8181, 0, 0, 572, 0, 0, 0, 0, 0, 458, 8192, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 8145, 0, 7286, 0, 0, 8188, 0, 0, 288, 0, 0, 0, 0, 0, 468, 8192, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2949, 0, 6617, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 0, 0, 0, 0, 6675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 5186, 1479, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9813, 7629, 8150, 8195, 8190, 7185, 8189, 8185] avg=8192.0000 max=9813 max_rank=0 min=7185 min_rank=5 imbalance_ratio=1.197876 max_load_reduction=55723 max_load_reduction_pct=0.850266 moved_to_echo=31749 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 21, 0, 0, 0, 2843, 0, 2680, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1525, 0, 0, 0, 0, 6684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4218, 0, 0, 0, 5000, 1481, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 8155, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5556, 8209, 8243, 8192, 8174, 10774, 8208, 8180] avg=8192.0000 max=10774 max_rank=5 min=5556 min_rank=0 imbalance_ratio=1.315186 max_load_reduction=54762 max_load_reduction_pct=0.835602 moved_to_echo=35336 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 12, 0, 0, 0, 2944, 0, 5054, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 0, 0, 6685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1955, 0, 0, 0, 5176, 1482, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 1, 8150, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8026, 7908, 8161, 8191, 8186, 8689, 8183, 8192] avg=8192.0000 max=8689 max_rank=5 min=7908 min_rank=1 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=33250 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 0, 0, 2850, 6, 3143, 4, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2630, 0, 6, 0, 0, 6689, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8184, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2700, 0, 0, 0, 5010, 1482, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8152, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 0, 0, 4, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6025, 9328, 8158, 8192, 8202, 9267, 8176, 8188] avg=8192.0000 max=9328 max_rank=1 min=6025 min_rank=0 imbalance_ratio=1.138672 max_load_reduction=56208 max_load_reduction_pct=0.857666 moved_to_echo=33833 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 12, 0, 0, 0, 2463, 0, 2766, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3584, 0, 0, 0, 0, 6687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3016, 0, 0, 0, 4331, 1482, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 69, 0, 0, 0, 8153, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 4, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5258, 10271, 8287, 8192, 8197, 8905, 8238, 8188] avg=8192.0000 max=10271 max_rank=1 min=5258 min_rank=0 imbalance_ratio=1.253784 max_load_reduction=55265 max_load_reduction_pct=0.843277 moved_to_echo=33528 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 2949, 0, 7286, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 6641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 5196, 1478, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8155, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10526, 7109, 8165, 8191, 8204, 6933, 8171, 8237] avg=8192.0000 max=10526 max_rank=0 min=6933 min_rank=5 imbalance_ratio=1.284912 max_load_reduction=55010 max_load_reduction_pct=0.839386 moved_to_echo=31545 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 50, 0, 0, 0, 2904, 0, 6652, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 0, 0, 0, 0, 6691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 0, 5107, 1483, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 8158, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9653, 7935, 8212, 8191, 8184, 6976, 8200, 8185] avg=8192.0000 max=9653 max_rank=0 min=6976 min_rank=5 imbalance_ratio=1.178345 max_load_reduction=55883 max_load_reduction_pct=0.852707 moved_to_echo=31545 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 2902, 0, 7201, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 0, 0, 6689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 5102, 1482, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8155, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10679, 7147, 8160, 8192, 8199, 6807, 8171, 8181] avg=8192.0000 max=10679 max_rank=0 min=6807 min_rank=5 imbalance_ratio=1.303589 max_load_reduction=54857 max_load_reduction_pct=0.837051 moved_to_echo=31358 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 13, 0, 0, 0, 7937, 0, 6354, 0, 0, 8184, 0, 0, 17, 0, 0, 0, 0, 0, 378, 8192, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 12, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 1, 5, 0, 0, 7010, 6, 4468, 4, 0, 8186, 0, 0, 19, 0, 2, 0, 0, 0, 811, 8186, 6, 0, 0, 8181, 0, 2, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8186, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 541, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 16, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 119, 0, 0, 0, 7891, 0, 7213, 0, 0, 8179, 0, 0, 156, 0, 0, 0, 0, 0, 421, 8192, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7874, 0, 7453, 0, 0, 8158, 0, 0, 330, 0, 0, 0, 0, 0, 132, 8192, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 68, 0, 0, 0, 5802, 0, 4801, 0, 0, 8190, 0, 0, 23, 0, 0, 0, 0, 0, 994, 8192, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 8040, 0, 7680, 0, 0, 8188, 0, 0, 353, 0, 0, 0, 0, 0, 33, 8192, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 7904, 0, 7625, 0, 0, 8181, 0, 0, 621, 0, 0, 0, 0, 0, 71, 8192, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 41, 0, 0, 0, 6929, 0, 3502, 0, 0, 8192, 0, 0, 23, 0, 0, 0, 0, 0, 447, 8192, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 5, 0, 0, 1727, 6, 4468, 4, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 811, 0, 6, 0, 0, 7793, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 0, 0, 5283, 377, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 541, 0, 0, 0, 7845, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 16, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6231, 8614, 8144, 8192, 8202, 9496, 8466, 8191] avg=8192.0000 max=9496 max_rank=5 min=6231 min_rank=0 imbalance_ratio=1.159180 max_load_reduction=56040 max_load_reduction_pct=0.855103 moved_to_echo=34355 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 68, 0, 0, 0, 1429, 0, 4801, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 994, 0, 0, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3859, 0, 0, 0, 4373, 377, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 697, 0, 0, 0, 7850, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 4, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6323, 8787, 8279, 8192, 8197, 8950, 8627, 8181] avg=8192.0000 max=8950 max_rank=5 min=6323 min_rank=0 imbalance_ratio=1.092529 max_load_reduction=56586 max_load_reduction_pct=0.863434 moved_to_echo=33955 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 13, 0, 0, 0, 1955, 0, 6354, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 7786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1500, 0, 0, 0, 5982, 377, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 2, 7844, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 12, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8340, 8164, 8154, 8190, 8186, 8200, 8108, 8194] avg=8192.0000 max=8340 max_rank=0 min=8108 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32688 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1940, 0, 7453, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 7777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 5934, 376, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 7819, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9723, 7909, 8194, 8194, 8190, 7180, 7964, 8182] avg=8192.0000 max=9723 max_rank=0 min=7180 min_rank=5 imbalance_ratio=1.186890 max_load_reduction=55813 max_load_reduction_pct=0.851639 moved_to_echo=31516 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 119, 0, 0, 0, 1944, 0, 7213, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, 7794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 0, 5947, 377, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 7838, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9432, 8215, 8213, 8193, 8185, 7075, 8023, 8200] avg=8192.0000 max=9432 max_rank=0 min=7075 min_rank=5 imbalance_ratio=1.151367 max_load_reduction=56104 max_load_reduction_pct=0.856079 moved_to_echo=31483 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 1976, 0, 7680, 0, 0, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 7715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 0, 6064, 378, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 7847, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10015, 7748, 8149, 8191, 8202, 7060, 7966, 8205] avg=8192.0000 max=10015 max_rank=0 min=7060 min_rank=5 imbalance_ratio=1.222534 max_load_reduction=55521 max_load_reduction_pct=0.847183 moved_to_echo=31433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 1947, 0, 7625, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 5957, 377, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 7840, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10199, 7864, 8149, 8192, 8199, 6781, 7941, 8211] avg=8192.0000 max=10199 max_rank=0 min=6781 min_rank=5 imbalance_ratio=1.244995 max_load_reduction=55337 max_load_reduction_pct=0.844376 moved_to_echo=31132 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 41, 0, 0, 0, 1707, 0, 3502, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 7788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4854, 0, 0, 0, 5222, 377, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 7852, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5273, 8235, 8254, 8192, 8175, 10794, 8441, 8172] avg=8192.0000 max=10794 max_rank=5 min=5273 min_rank=0 imbalance_ratio=1.317627 max_load_reduction=54742 max_load_reduction_pct=0.835297 moved_to_echo=35582 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 13056, 0, 0, 0, 40192, 11904, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 0, 0, 22912, 128, 41472, 128, 0, 0, 0, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 12160, 0, 128, 0, 0, 53504, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 15104, 0, 0, 0, 44800, 3072, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 256, 128, 0, 0, 14720, 128, 49152, 128, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 3328, 0, 128, 0, 0, 62336, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 2176, 0, 0, 128, 62848, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 512, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64896, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 128, 128, 0, 0, 256, 0, 0, 128, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 512, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64896, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 256, 0, 0, 128, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 128, 128, 0, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8184, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 16, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 22, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 3828, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 69, 0, 8186, 5, 0, 0, 0, 6785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1648, 0, 0, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 8186, 0, 6, 0, 0, 4, 0, 0, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 786, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 7405, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8192, 7704, 8244, 9156, 7679, 8192, 8189] avg=8192.0000 max=9156 max_rank=4 min=7679 min_rank=5 imbalance_ratio=1.117676 max_load_reduction=56380 max_load_reduction_pct=0.860291 moved_to_echo=33216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 786, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 7406, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8191, 7706, 8395, 8889, 7737, 8191, 8192] avg=8192.0000 max=8889 max_rank=4 min=7706 min_rank=2 imbalance_ratio=1.085083 max_load_reduction=56647 max_load_reduction_pct=0.864365 moved_to_echo=33009 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 202, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 786, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 360, 0, 7402, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8194, 7699, 8389, 8905, 7778, 8196, 8192] avg=8192.0000 max=8905 max_rank=4 min=7699 min_rank=2 imbalance_ratio=1.087036 max_load_reduction=56631 max_load_reduction_pct=0.864120 moved_to_echo=33071 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 7695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 7404, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8188, 7698, 8396, 8984, 7698, 8190, 8192] avg=8192.0000 max=8984 max_rank=4 min=7698 min_rank=2 imbalance_ratio=1.096680 max_load_reduction=56552 max_load_reduction_pct=0.862915 moved_to_echo=33064 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 22, 8173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 0, 0, 0, 0, 7699, 0, 0, 0, 0, 69, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1648, 0, 0, 0, 0, 0, 3, 0, 5, 0, 787, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 7399, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8191, 11601, 6988, 2734, 11441, 8185, 8193] avg=8192.0000 max=11601 max_rank=2 min=2734 min_rank=4 imbalance_ratio=1.416138 max_load_reduction=53935 max_load_reduction_pct=0.822983 moved_to_echo=30553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 786, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 7404, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8198, 7696, 8354, 9224, 7486, 8201, 8194] avg=8192.0000 max=9224 max_rank=4 min=7486 min_rank=5 imbalance_ratio=1.125977 max_load_reduction=56312 max_load_reduction_pct=0.859253 moved_to_echo=33105 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 7405, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8191, 7705, 8375, 9035, 7665, 8191, 8193] avg=8192.0000 max=9035 max_rank=4 min=7665 min_rank=5 imbalance_ratio=1.102905 max_load_reduction=56501 max_load_reduction_pct=0.862137 moved_to_echo=33084 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 7402, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8191, 7727, 8395, 8609, 8052, 8190, 8191] avg=8192.0000 max=8609 max_rank=4 min=7727 min_rank=2 imbalance_ratio=1.050903 max_load_reduction=56927 max_load_reduction_pct=0.868637 moved_to_echo=33042 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 4550, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 30, 0, 8184, 2, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2175, 0, 0, 0, 0, 0, 4, 0, 6, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8186, 0, 6, 0, 0, 5, 0, 0, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8184, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 10, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 7969, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8191, 7622, 8196, 8855, 8046, 8191, 8192] avg=8192.0000 max=8855 max_rank=4 min=7622 min_rank=2 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=33284 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 5, 8173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4550, 0, 0, 0, 0, 7616, 0, 0, 0, 0, 30, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2175, 0, 0, 0, 0, 0, 4, 0, 6, 0, 567, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 0, 7963, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 6, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8191, 12198, 8155, 2981, 9445, 8185, 8195] avg=8192.0000 max=12198 max_rank=2 min=2981 min_rank=4 imbalance_ratio=1.489014 max_load_reduction=53338 max_load_reduction_pct=0.813873 moved_to_echo=28806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 7968, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8192, 7620, 8196, 8965, 8000, 8192, 8189] avg=8192.0000 max=8965 max_rank=4 min=7620 min_rank=2 imbalance_ratio=1.094360 max_load_reduction=56571 max_load_reduction_pct=0.863205 moved_to_echo=33346 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7967, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8198, 7612, 8196, 8967, 7983, 8201, 8194] avg=8192.0000 max=8967 max_rank=4 min=7612 min_rank=2 imbalance_ratio=1.094604 max_load_reduction=56569 max_load_reduction_pct=0.863174 moved_to_echo=33345 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 7969, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8191, 7621, 8196, 8970, 7994, 8191, 8191] avg=8192.0000 max=8970 max_rank=4 min=7621 min_rank=2 imbalance_ratio=1.094971 max_load_reduction=56566 max_load_reduction_pct=0.863129 moved_to_echo=33346 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 7967, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 7615, 8201, 8941, 8017, 8190, 8192] avg=8192.0000 max=8941 max_rank=4 min=7615 min_rank=2 imbalance_ratio=1.091431 max_load_reduction=56595 max_load_reduction_pct=0.863571 moved_to_echo=33340 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7621, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 7966, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8191, 7633, 8196, 8898, 8055, 8190, 8191] avg=8192.0000 max=8898 max_rank=4 min=7633 min_rank=2 imbalance_ratio=1.086182 max_load_reduction=56638 max_load_reduction_pct=0.864227 moved_to_echo=33334 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 16, 0, 7965, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8194, 7615, 8200, 8959, 7996, 8196, 8192] avg=8192.0000 max=8959 max_rank=4 min=7615 min_rank=2 imbalance_ratio=1.093628 max_load_reduction=56577 max_load_reduction_pct=0.863297 moved_to_echo=33343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 56960, 0, 0, 0, 0, 0, 128, 0, 128, 0, 6400, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 6272, 0, 59264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[3840, 0, 0, 0, 0, 61696, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 64000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 0, 128, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 59264, 0, 0, 0, 0, 0, 128, 0, 128, 0, 4608, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4608, 0, 0, 0, 0, 61056, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 1792, 0, 63744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65536, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 0, 0, 0, 0, 0, 0, 4387, 0, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8131, 8192, 688, 0, 0, 0, 0, 0, 8081, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 0, 0, 0, 0, 0, 6049, 0, 0, 0, 8115, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8150, 8192, 221, 0, 0, 0, 0, 0, 8169, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 0, 0, 0, 0, 0, 0, 6405, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 7891, 8192, 324, 0, 0, 0, 0, 0, 8001, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1653, 0, 0, 0, 0, 0, 0, 6243, 0, 0, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 8118, 8192, 395, 0, 0, 0, 0, 0, 8023, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1805, 0, 0, 0, 0, 0, 0, 6393, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 8102, 8192, 260, 0, 0, 0, 0, 0, 7970, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7851, 0, 0, 0, 0, 0, 0, 6567, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 6, 0, 0, 0, 0, 2123, 0, 0, 0, 7664, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 357, 8185, 1072, 0, 0, 0, 6, 0, 6794, 8181, 0, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 0, 0, 0, 0, 0, 0, 6394, 0, 0, 0, 8153, 0, 14, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 8019, 8192, 450, 0, 0, 0, 0, 0, 8046, 8187, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1583, 0, 0, 0, 0, 0, 0, 6495, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 8109, 8192, 188, 0, 0, 0, 0, 0, 8060, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 0, 0, 0, 0, 0, 0, 4387, 0, 0, 740, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 7245, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 688, 0, 0, 0, 0, 0, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7365, 8183, 8165, 8712, 8136, 7319, 9466] avg=8192.0000 max=9466 max_rank=7 min=7319 min_rank=6 imbalance_ratio=1.155518 max_load_reduction=56070 max_load_reduction_pct=0.855560 moved_to_echo=33633 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 7364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1653, 0, 0, 0, 0, 0, 0, 6243, 0, 0, 740, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 29, 7193, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 395, 0, 0, 0, 0, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7423, 8192, 8165, 8823, 8300, 7286, 9156] avg=8192.0000 max=9156 max_rank=7 min=7286 min_rank=6 imbalance_ratio=1.117676 max_load_reduction=56380 max_load_reduction_pct=0.860291 moved_to_echo=33565 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 0, 0, 0, 0, 0, 0, 6405, 0, 0, 745, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 11, 7179, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7891, 0, 324, 0, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7354, 8201, 8161, 9298, 8224, 7259, 8847] avg=8192.0000 max=9298 max_rank=4 min=7259 min_rank=6 imbalance_ratio=1.135010 max_load_reduction=56238 max_load_reduction_pct=0.858124 moved_to_echo=33628 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 7365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 0, 0, 0, 0, 0, 6049, 0, 0, 740, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 87, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 10, 7324, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 221, 0, 0, 0, 0, 0, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 7474, 8211, 8165, 8814, 8235, 7411, 9025] avg=8192.0000 max=9025 max_rank=7 min=7411 min_rank=6 imbalance_ratio=1.101685 max_load_reduction=56511 max_load_reduction_pct=0.862289 moved_to_echo=33485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 0, 0, 0, 0, 0, 0, 6394, 0, 0, 739, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 14, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 87, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 73, 7214, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 450, 0, 0, 0, 0, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7366, 8233, 8165, 8808, 8286, 7368, 9113] avg=8192.0000 max=9113 max_rank=7 min=7366 min_rank=1 imbalance_ratio=1.112427 max_load_reduction=56423 max_load_reduction_pct=0.860947 moved_to_echo=33575 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 7363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1583, 0, 0, 0, 0, 0, 0, 6495, 0, 0, 740, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 7226, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 188, 0, 0, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7428, 8225, 8165, 9006, 8265, 7314, 8943] avg=8192.0000 max=9006 max_rank=4 min=7314 min_rank=6 imbalance_ratio=1.099365 max_load_reduction=56530 max_load_reduction_pct=0.862579 moved_to_echo=33528 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7851, 0, 0, 0, 0, 0, 0, 5904, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 6, 0, 0, 0, 0, 2123, 0, 0, 593, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7664, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 6, 0, 70, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 6091, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 1072, 0, 0, 0, 6, 0, 545, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 13761, 8131, 8385, 2951, 7791, 14346, 1986] avg=8192.0000 max=14346 max_rank=6 min=1986 min_rank=7 imbalance_ratio=1.751221 max_load_reduction=51190 max_load_reduction_pct=0.781097 moved_to_echo=27074 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1805, 0, 0, 0, 0, 0, 0, 6393, 0, 0, 740, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 7146, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 260, 0, 0, 0, 0, 0, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7365, 8160, 8165, 9124, 8299, 7233, 9000] avg=8192.0000 max=9124 max_rank=4 min=7233 min_rank=6 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=33656 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 7062, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 5647, 8192, 2894, 0, 0, 0, 0, 0, 8092, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 8147, 0, 12, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 6772, 8189, 1620, 0, 0, 0, 0, 0, 7908, 8187, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 7799, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 5505, 8192, 2279, 0, 0, 0, 0, 0, 8106, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8164, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 7159, 8191, 847, 0, 0, 0, 0, 0, 7936, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 7983, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 7761, 0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6928, 8182, 719, 0, 0, 0, 0, 0, 8098, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 2410, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 6, 0, 0, 0, 0, 8179, 0, 0, 0, 5279, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 232, 8179, 1802, 0, 0, 0, 6, 0, 6459, 8176, 0, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7647, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 7545, 8192, 1113, 0, 0, 0, 0, 0, 7912, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 6201, 8192, 1518, 0, 0, 0, 0, 0, 8012, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 285, 206, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 7080, 127, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7159, 0, 847, 0, 0, 0, 0, 0, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8223, 8194, 8166, 7968, 8683, 7248, 8862] avg=8192.0000 max=8862 max_rank=7 min=7248 min_rank=6 imbalance_ratio=1.081787 max_load_reduction=56674 max_load_reduction_pct=0.864777 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 6810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 212, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 37, 7223, 129, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5647, 0, 2894, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6896, 8194, 8160, 8626, 8631, 7425, 9410] avg=8192.0000 max=9410 max_rank=7 min=6896 min_rank=1 imbalance_ratio=1.148682 max_load_reduction=56126 max_load_reduction_pct=0.856415 moved_to_echo=34092 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 7524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 206, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 7231, 127, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5505, 0, 2279, 0, 0, 0, 0, 0, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7778, 8198, 8166, 8731, 8356, 7456, 8659] avg=8192.0000 max=8731 max_rank=4 min=7456 min_rank=6 imbalance_ratio=1.065796 max_load_reduction=56805 max_load_reduction_pct=0.866776 moved_to_echo=33202 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 7701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 7841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7761, 0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 282, 206, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 7224, 127, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6928, 0, 719, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8812, 8195, 8166, 7956, 8287, 7396, 8521] avg=8192.0000 max=8812 max_rank=1 min=7396 min_rank=6 imbalance_ratio=1.075684 max_load_reduction=56724 max_load_reduction_pct=0.865540 moved_to_echo=32160 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 2325, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 6, 0, 0, 0, 0, 7846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5279, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6, 0, 85, 206, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 5762, 127, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 1802, 0, 0, 0, 6, 0, 697, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 10505, 8159, 8380, 7871, 5605, 14097, 2743] avg=8192.0000 max=14097 max_rank=6 min=2743 min_rank=7 imbalance_ratio=1.720825 max_load_reduction=51439 max_load_reduction_pct=0.784897 moved_to_echo=30316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 7760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 7842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 12, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 284, 206, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 40, 7055, 127, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6772, 0, 1620, 0, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7843, 8233, 8166, 7904, 8680, 7270, 9245] avg=8192.0000 max=9245 max_rank=7 min=7270 min_rank=6 imbalance_ratio=1.128540 max_load_reduction=56291 max_load_reduction_pct=0.858932 moved_to_echo=33099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0, 0, 0, 7750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 206, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 7147, 127, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6201, 0, 1518, 0, 0, 0, 0, 0, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8098, 8187, 8166, 8292, 8618, 7399, 8584] avg=8192.0000 max=8618 max_rank=5 min=7399 min_rank=6 imbalance_ratio=1.052002 max_load_reduction=56918 max_load_reduction_pct=0.868500 moved_to_echo=32893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 7857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 206, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 7058, 127, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7545, 0, 1113, 0, 0, 0, 0, 0, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7381, 8176, 8166, 8188, 8676, 7245, 9512] avg=8192.0000 max=9512 max_rank=7 min=7245 min_rank=6 imbalance_ratio=1.161133 max_load_reduction=56024 max_load_reduction_pct=0.854858 moved_to_echo=33621 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 56960, 0, 3712, 0, 0, 0, 128, 0, 5120, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 57472, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 13824, 0, 128, 0, 0, 0, 0, 44544, 0, 0, 5888, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 8448, 56704, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 512, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 64640, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 46080, 0, 12800, 0, 0, 0, 128, 0, 6784, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 8448, 55808, 1024, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 10624, 0, 0, 0, 0, 0, 0, 55040, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 2816, 0, 128, 0, 0, 0, 0, 62848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 61696, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 2048, 1664, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 384, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 256, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4392, 0, 0, 0, 84, 0, 0, 0, 0, 8190, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 8192, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 5528, 0, 0, 8163, 6326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3353, 0, 0, 0, 314, 0, 0, 0, 0, 8192, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5854, 0, 0, 8139, 6387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8155, 8186, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 2013, 0, 0, 0, 7985, 7, 3, 0, 0, 8075, 0, 1975, 0, 0, 0, 0, 0, 0, 0, 31, 0, 11, 6583, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 36, 0, 0, 8186, 5462, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 1, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 78, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 56, 0, 0, 0, 0, 8182, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 8186, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 5918, 0, 0, 8162, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2892, 0, 0, 0, 14, 0, 0, 0, 0, 8192, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5444, 0, 0, 8180, 7611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5672, 0, 0, 0, 78, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4429, 0, 0, 8189, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5012, 0, 0, 0, 483, 0, 0, 0, 0, 8192, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 885, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2656, 0, 0, 8158, 7368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 0, 0, 0, 293, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5842, 0, 0, 8191, 2181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3353, 0, 2025, 1416, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 6776, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5854, 0, 0, 0, 4358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 8136, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 6909, 7126, 8549, 10212, 8173, 8188, 8186] avg=8192.0000 max=10212 max_rank=4 min=6909 min_rank=1 imbalance_ratio=1.246582 max_load_reduction=55324 max_load_reduction_pct=0.844177 moved_to_echo=34759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 2013, 0, 1732, 1396, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 7, 3, 0, 0, 6679, 0, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 11, 6490, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 1, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 78, 0, 6, 0, 0, 0, 0, 8173, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 5241, 16649, 6978, 3763, 8269, 8276, 8191] avg=8192.0000 max=16649 max_rank=2 min=3763 min_rank=4 imbalance_ratio=2.032349 max_load_reduction=48887 max_load_reduction_pct=0.745956 moved_to_echo=28499 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4392, 0, 2006, 1416, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 6774, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 8077, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5528, 0, 0, 0, 4316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 8160, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7929, 6894, 8121, 9844, 8179, 8165, 8207] avg=8192.0000 max=9844 max_rank=4 min=6894 min_rank=2 imbalance_ratio=1.201660 max_load_reduction=55692 max_load_reduction_pct=0.849792 moved_to_echo=34395 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 712, 1414, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 6768, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 8071, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5918, 0, 0, 0, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 8159, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 10360, 6851, 8149, 7452, 8188, 8164, 8205] avg=8192.0000 max=10360 max_rank=1 min=6851 min_rank=2 imbalance_ratio=1.264648 max_load_reduction=55176 max_load_reduction_pct=0.841919 moved_to_echo=32009 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2892, 0, 2417, 1420, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 6772, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 8071, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5444, 0, 0, 0, 5186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8178, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 6850, 6816, 8481, 10630, 8187, 8186, 8189] avg=8192.0000 max=10630 max_rank=4 min=6816 min_rank=2 imbalance_ratio=1.297607 max_load_reduction=54906 max_load_reduction_pct=0.837799 moved_to_echo=35192 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 0, 691, 1416, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 6775, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 8075, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5842, 0, 0, 0, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 10217, 7069, 8152, 7331, 8190, 8185, 8188] avg=8192.0000 max=10217 max_rank=1 min=7069 min_rank=2 imbalance_ratio=1.247192 max_load_reduction=55319 max_load_reduction_pct=0.844101 moved_to_echo=31894 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5672, 0, 1948, 1416, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 6774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4429, 0, 0, 0, 4192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 9151, 6852, 8144, 8621, 8190, 8189, 8185] avg=8192.0000 max=9151 max_rank=1 min=6852 min_rank=2 imbalance_ratio=1.117065 max_load_reduction=56385 max_load_reduction_pct=0.860367 moved_to_echo=33185 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5012, 0, 2336, 1416, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 6776, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 885, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2656, 0, 0, 0, 5027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8879, 7279, 8962, 7683, 8160, 8183, 8185] avg=8192.0000 max=8962 max_rank=3 min=7279 min_rank=2 imbalance_ratio=1.093994 max_load_reduction=56574 max_load_reduction_pct=0.863251 moved_to_echo=32211 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7799, 0, 0, 0, 9, 0, 0, 0, 0, 8183, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8124, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 1177, 0, 0, 7810, 7809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6687, 0, 0, 0, 50, 0, 0, 0, 0, 8188, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 8191, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1892, 0, 0, 7694, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6232, 0, 0, 0, 49, 0, 0, 0, 0, 8192, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1794, 0, 0, 7956, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7110, 0, 0, 0, 27, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1651, 0, 0, 7571, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7653, 0, 0, 0, 23, 0, 0, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1029, 0, 0, 8106, 7776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5003, 0, 0, 0, 16, 0, 0, 0, 0, 8192, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 837, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2456, 0, 0, 7983, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6261, 0, 0, 0, 163, 0, 0, 0, 0, 8192, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 0, 0, 7390, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8177, 8186, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 1287, 0, 0, 0, 7636, 7, 3, 0, 0, 8098, 0, 2636, 0, 0, 0, 0, 0, 0, 0, 33, 0, 9, 4560, 0, 0, 0, 0, 0, 0, 0, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 16, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7799, 0, 1207, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 6837, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8124, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 0, 0, 6747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7809, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 9983, 6875, 8226, 7924, 7965, 8189, 8193] avg=8192.0000 max=9983 max_rank=1 min=6875 min_rank=2 imbalance_ratio=1.218628 max_load_reduction=55553 max_load_reduction_pct=0.847672 moved_to_echo=32271 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7110, 0, 1208, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 6845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1651, 0, 0, 6541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 9265, 6872, 8380, 8192, 8251, 8189, 8189] avg=8192.0000 max=9265 max_rank=1 min=6872 min_rank=2 imbalance_ratio=1.130981 max_load_reduction=56271 max_load_reduction_pct=0.858627 moved_to_echo=32821 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6687, 0, 1207, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 6842, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 8191, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1892, 0, 0, 6647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8151, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8857, 6943, 8326, 8539, 8295, 8188, 8203] avg=8192.0000 max=8857 max_rank=1 min=6943 min_rank=2 imbalance_ratio=1.081177 max_load_reduction=56679 max_load_reduction_pct=0.864853 moved_to_echo=33225 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5003, 0, 1211, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 6835, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 837, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2456, 0, 0, 6890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 146, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 7215, 6940, 9121, 9346, 8341, 8189, 8195] avg=8192.0000 max=9346 max_rank=4 min=6940 min_rank=2 imbalance_ratio=1.140869 max_load_reduction=56190 max_load_reduction_pct=0.857391 moved_to_echo=34071 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6232, 0, 1208, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 6845, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1794, 0, 0, 6873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8109, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8435, 6947, 8643, 8667, 8268, 8189, 8189] avg=8192.0000 max=8667 max_rank=4 min=6947 min_rank=2 imbalance_ratio=1.057983 max_load_reduction=56869 max_load_reduction_pct=0.867752 moved_to_echo=33313 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7653, 0, 1208, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 6844, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 0, 0, 7003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7776, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 9875, 6869, 8269, 8032, 7916, 8189, 8189] avg=8192.0000 max=9875 max_rank=1 min=6869 min_rank=2 imbalance_ratio=1.205444 max_load_reduction=55661 max_load_reduction_pct=0.849319 moved_to_echo=32326 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 1287, 0, 1194, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7636, 7, 3, 0, 0, 6766, 0, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 9, 4560, 0, 0, 0, 0, 0, 0, 0, 0, 441, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7950, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 16, 0, 6, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8171, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 3513, 17048, 5133, 7072, 8177, 8214, 8186] avg=8192.0000 max=17048 max_rank=2 min=3513 min_rank=1 imbalance_ratio=2.081055 max_load_reduction=48488 max_load_reduction_pct=0.739868 moved_to_echo=31649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6261, 0, 1208, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 6845, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 0, 0, 6385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8393, 7042, 9438, 7764, 8323, 8189, 8192] avg=8192.0000 max=9438 max_rank=3 min=7042 min_rank=2 imbalance_ratio=1.152100 max_load_reduction=56098 max_load_reduction_pct=0.855988 moved_to_echo=32468 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 39552, 0, 13952, 11392, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 35712, 0, 0, 0, 29952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 9344, 128, 128, 0, 0, 54144, 0, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 48128, 0, 9728, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 8064, 128, 128, 0, 0, 54784, 0, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 11392, 0, 0, 54272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 2688, 61824, 0, 0, 0, 0, 0, 0, 0, 128, 512, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 64384, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 2048, 63104, 0, 0, 0, 0, 0, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 28, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 26, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 0, 0, 0, 0, 0, 455, 0, 0, 0, 0, 0, 3, 1, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 3, 0, 0, 0, 9, 0, 0, 0, 0, 0, 2, 9, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 18, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 33, 9, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 28, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7892, 0, 16, 0, 0, 0, 252, 0, 0, 0, 0, 0, 17, 19, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 59, 8171, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 760, 0, 0, 0, 1, 0, 0, 0, 0, 0, 700, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2881, 0, 0, 0, 6, 0, 0, 5488, 0, 0, 0, 0, 0, 6904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 1, 0, 7987, 0, 0, 5, 0, 7842, 0, 0, 0, 6, 0, 0, 0, 0, 87, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 6, 8184, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8186, 1, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 6, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 28, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 26, 0, 0, 8190, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8033, 8189, 8219, 8304, 8213, 8192, 8186, 8200] avg=8192.0000 max=8304 max_rank=3 min=8033 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8171, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 33, 9, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 28, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 8190, 8218, 8301, 8227, 8192, 8194, 8178] avg=8192.0000 max=8301 max_rank=3 min=8036 min_rank=0 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8179, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 0, 0, 0, 3, 1, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 8, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 8189, 7849, 8654, 8223, 8192, 8196, 8197] avg=8192.0000 max=8654 max_rank=3 min=7849 min_rank=2 imbalance_ratio=1.056396 max_load_reduction=56882 max_load_reduction_pct=0.867950 moved_to_echo=32808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 760, 0, 0, 0, 1, 0, 0, 0, 0, 0, 700, 0, 0, 7832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 8172, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2881, 0, 0, 0, 6, 0, 0, 5488, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 5, 0, 7842, 0, 0, 0, 6, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 6, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9293, 8192, 8486, 7002, 7964, 8194, 8198, 8207] avg=8192.0000 max=9293 max_rank=0 min=7002 min_rank=3 imbalance_ratio=1.134399 max_load_reduction=56243 max_load_reduction_pct=0.858200 moved_to_echo=32563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8184, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 3, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 2, 9, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 18, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 8203, 8294, 8216, 8212, 8191, 8193, 8191] avg=8192.0000 max=8294 max_rank=2 min=8036 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 8166, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7892, 0, 16, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 17, 19, 0, 0, 8189, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 59, 8171, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8033, 8194, 8033, 8480, 8255, 8192, 8181, 8168] avg=8192.0000 max=8480 max_rank=3 min=8033 min_rank=0 imbalance_ratio=1.035156 max_load_reduction=57056 max_load_reduction_pct=0.870605 moved_to_echo=32796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 6, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8033, 8190, 8270, 8238, 8220, 8191, 8198, 8196] avg=8192.0000 max=8270 max_rank=2 min=8033 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 8189, 8167, 8341, 8222, 8192, 8190, 8199] avg=8192.0000 max=8341 max_rank=3 min=8036 min_rank=0 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 39, 10, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 28, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7360, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 6, 1, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 10, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7829, 0, 17, 0, 0, 0, 334, 0, 0, 0, 0, 0, 4, 30, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7474, 0, 9, 0, 0, 0, 660, 0, 0, 0, 0, 0, 31, 18, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 69, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 859, 0, 0, 0, 2, 0, 0, 0, 0, 0, 692, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2567, 0, 0, 0, 5, 0, 0, 5110, 0, 0, 0, 0, 0, 7443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 1, 0, 7974, 0, 0, 5, 0, 7837, 0, 0, 0, 6, 0, 0, 0, 0, 95, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 6, 8184, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8186, 1, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 5, 8, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 20, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 5, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8179, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7360, 0, 0, 278, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 6, 1, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 10, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8026, 8189, 7778, 8740, 8222, 8192, 8190, 8199] avg=8192.0000 max=8740 max_rank=3 min=7778 min_rank=2 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=32803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7829, 0, 17, 278, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 4, 30, 0, 0, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8189, 8263, 8268, 8209, 8192, 8188, 8204] avg=8192.0000 max=8268 max_rank=3 min=8023 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7865, 0, 0, 278, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8027, 8189, 8284, 8229, 8219, 8192, 8194, 8202] avg=8192.0000 max=8284 max_rank=2 min=8027 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 859, 0, 0, 0, 2, 0, 0, 0, 0, 0, 692, 0, 0, 7810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 8172, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2567, 0, 0, 0, 5, 0, 0, 5110, 0, 0, 5, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 5, 0, 7837, 0, 0, 0, 6, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 6, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9363, 8191, 7822, 7606, 7964, 8194, 8198, 8198] avg=8192.0000 max=9363 max_rank=0 min=7606 min_rank=3 imbalance_ratio=1.142944 max_load_reduction=56173 max_load_reduction_pct=0.857132 moved_to_echo=32554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 8167, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7474, 0, 9, 278, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 0, 0, 0, 0, 31, 18, 0, 0, 7898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 69, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8196, 7912, 8607, 8259, 8192, 8182, 8165] avg=8192.0000 max=8607 max_rank=3 min=7912 min_rank=2 imbalance_ratio=1.050659 max_load_reduction=56929 max_load_reduction_pct=0.868668 moved_to_echo=32798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8185, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 282, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 5, 8, 0, 0, 7895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 20, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8028, 8200, 8548, 7961, 8222, 8191, 8192, 8194] avg=8192.0000 max=8548 max_rank=2 min=7961 min_rank=3 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8170, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 0, 0, 278, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 39, 10, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 28, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8192, 8395, 8138, 8225, 8192, 8194, 8177] avg=8192.0000 max=8395 max_rank=2 min=8023 min_rank=0 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 278, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 10, 0, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 5, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8190, 8534, 7987, 8216, 8191, 8198, 8197] avg=8192.0000 max=8534 max_rank=2 min=7987 min_rank=3 imbalance_ratio=1.041748 max_load_reduction=57002 max_load_reduction_pct=0.869781 moved_to_echo=32802 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 8064, 0, 0, 0, 0, 0, 128, 128, 0, 0, 57472, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 768, 128, 0, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 2944, 0, 0, 0, 128, 0, 0, 61696, 0, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 896, 0, 0, 0, 128, 0, 0, 0, 0, 0, 768, 0, 0, 64000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 256, 0, 0, 128, 256, 65152, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 0, 2688, 0, 0, 0, 128, 0, 0, 59904, 0, 128, 2048, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 9984, 0, 0, 0, 0, 0, 128, 128, 0, 0, 55552, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 256, 0, 0, 128, 128, 65152, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 5166, 0, 8192, 0, 0, 0, 0, 0, 5783, 0, 0, 0, 0, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1791, 0, 0, 0, 0, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 0, 0, 6634, 0, 8192, 0, 0, 0, 0, 0, 7658, 0, 0, 1, 0, 1015, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 4006, 0, 8192, 0, 0, 0, 0, 0, 5117, 0, 0, 1, 0, 4935, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1645, 0, 0, 0, 0, 695] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 0, 0, 0, 0, 0, 0, 5830, 0, 8192, 0, 0, 0, 0, 0, 4289, 0, 0, 9, 0, 5630, 0, 0, 0, 0, 33, 0, 0, 11, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 634, 0, 0, 0, 0, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7830, 0, 0, 0, 0, 0, 6, 1230, 6289, 4, 8177, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 7046, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6, 3, 6, 5, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1873, 6, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 231] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 6648, 0, 8192, 0, 0, 0, 0, 0, 4846, 0, 0, 0, 0, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2027, 0, 0, 0, 0, 1689] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 14, 0, 0, 0, 0, 0, 0, 5300, 0, 8192, 0, 0, 0, 0, 0, 7285, 0, 0, 10, 0, 1986, 0, 0, 0, 0, 33, 0, 0, 12, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1925, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 5339, 0, 8192, 0, 0, 0, 0, 0, 7366, 0, 0, 0, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1790, 0, 0, 0, 0, 30] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2508, 0, 0, 0, 0, 0, 0, 0, 5783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 1854, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1791, 0, 0, 0, 0, 15, 5484, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7185, 8291, 8144, 8231, 8206, 8984, 8191, 8304] avg=8192.0000 max=8984 max_rank=5 min=7185 min_rank=0 imbalance_ratio=1.096680 max_load_reduction=56552 max_load_reduction_pct=0.862915 moved_to_echo=33685 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1, 7178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3221, 0, 0, 0, 0, 0, 0, 0, 7658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1015, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 1, 3413, 1846, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 0, 0, 0, 0, 5460, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7205, 10879, 6286, 8182, 8206, 8980, 8207, 7591] avg=8192.0000 max=10879 max_rank=1 min=6286 min_rank=2 imbalance_ratio=1.328003 max_load_reduction=54657 max_load_reduction_pct=0.834000 moved_to_echo=32984 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 7177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2830, 0, 0, 0, 0, 0, 0, 0, 4289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 5630, 0, 0, 0, 0, 33, 0, 0, 11, 0, 0, 0, 8, 3000, 1850, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 789, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 634, 0, 0, 0, 0, 14, 5471, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7191, 7119, 10543, 8191, 8198, 8981, 8174, 7139] avg=8192.0000 max=10543 max_rank=2 min=7119 min_rank=1 imbalance_ratio=1.286987 max_load_reduction=54993 max_load_reduction_pct=0.839127 moved_to_echo=32492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 7178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 0, 0, 0, 0, 0, 5117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4935, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 2065, 1861, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1645, 0, 0, 0, 0, 695, 5496, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7192, 7058, 8881, 8189, 8212, 8976, 8178, 8850] avg=8192.0000 max=8976 max_rank=5 min=7058 min_rank=1 imbalance_ratio=1.095703 max_load_reduction=56560 max_load_reduction_pct=0.863037 moved_to_echo=34216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7830, 0, 7165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1230, 3053, 4, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7046, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 3236, 1867, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6, 3, 6, 5, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 1873, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 231, 5522, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15148, 4300, 12161, 8201, 8097, 2676, 8184, 6769] avg=8192.0000 max=15148 max_rank=0 min=2676 min_rank=5 imbalance_ratio=1.849121 max_load_reduction=50388 max_load_reduction_pct=0.768860 moved_to_echo=25726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 14, 7177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2573, 0, 0, 0, 0, 0, 0, 0, 7285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1986, 0, 0, 0, 0, 33, 0, 0, 12, 0, 0, 0, 8, 2727, 1815, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1925, 0, 0, 0, 0, 0, 5369, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7212, 9858, 6593, 8176, 8205, 8967, 8215, 8310] avg=8192.0000 max=9858 max_rank=1 min=6593 min_rank=2 imbalance_ratio=1.203369 max_load_reduction=55678 max_load_reduction_pct=0.849579 moved_to_echo=33697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 7178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2592, 0, 0, 0, 0, 0, 0, 0, 7366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2747, 1860, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 794, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1790, 0, 0, 0, 0, 30, 5500, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7215, 9958, 6456, 8183, 8206, 8986, 8198, 8334] avg=8192.0000 max=9958 max_rank=1 min=6456 min_rank=2 imbalance_ratio=1.215576 max_load_reduction=55578 max_load_reduction_pct=0.848053 moved_to_echo=33724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 7178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3227, 0, 0, 0, 0, 0, 0, 0, 4846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 1863, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 794, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2027, 0, 0, 0, 0, 1689, 5509, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7188, 8073, 6472, 8183, 8206, 8986, 8189, 10239] avg=8192.0000 max=10239 max_rank=7 min=6472 min_rank=2 imbalance_ratio=1.249878 max_load_reduction=55297 max_load_reduction_pct=0.843765 moved_to_echo=35620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 0, 0, 0, 0, 0, 0, 0, 6420, 0, 8192, 0, 0, 0, 0, 0, 6056, 0, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2598, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 7335, 0, 8192, 0, 0, 0, 0, 0, 6350, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 14, 0, 0, 0, 0, 0, 0, 6165, 0, 8192, 0, 0, 0, 0, 0, 7878, 0, 0, 11, 0, 640, 0, 0, 0, 0, 35, 0, 0, 13, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7848, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1451, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 1, 0, 0, 0, 0, 0, 0, 7743, 0, 8192, 0, 0, 0, 0, 0, 6819, 0, 0, 1, 0, 144, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1500, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7955, 0, 0, 0, 0, 0, 6, 1436, 6348, 3, 8177, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 6941, 0, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 3, 6, 5, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1646, 6, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 177] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 8, 0, 0, 0, 0, 0, 0, 6403, 0, 8192, 0, 0, 0, 0, 0, 6772, 0, 0, 9, 0, 2315, 0, 0, 0, 0, 37, 0, 0, 11, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 661, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 1, 0, 0, 0, 0, 0, 0, 7041, 0, 8192, 0, 0, 0, 0, 0, 6501, 0, 0, 1, 0, 873, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1778, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 7826, 0, 8192, 0, 0, 0, 0, 0, 4166, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3287, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 14, 6750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2180, 0, 0, 0, 0, 0, 0, 0, 7878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 640, 0, 0, 0, 0, 35, 0, 0, 13, 0, 0, 0, 9, 5057, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1451, 0, 0, 0, 0, 0, 3980, 1996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7273, 10058, 7207, 8175, 8206, 8987, 8201, 7429] avg=8192.0000 max=10058 max_rank=1 min=7207 min_rank=2 imbalance_ratio=1.227783 max_load_reduction=55478 max_load_reduction_pct=0.846527 moved_to_echo=32823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 1, 6750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2738, 0, 0, 0, 0, 0, 0, 0, 6819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 144, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 1, 5178, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 814, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1500, 0, 0, 0, 0, 0, 4999, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7068, 9557, 6773, 8183, 8206, 9006, 8201, 8542] avg=8192.0000 max=9557 max_rank=1 min=6773 min_rank=2 imbalance_ratio=1.166626 max_load_reduction=55979 max_load_reduction_pct=0.854172 moved_to_echo=33955 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 0, 6750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 6056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5235, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2598, 0, 0, 0, 0, 2, 4144, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7546, 8327, 7210, 8230, 8206, 9015, 8192, 8810] avg=8192.0000 max=9015 max_rank=5 min=7210 min_rank=2 imbalance_ratio=1.100464 max_load_reduction=56521 max_load_reduction_pct=0.862442 moved_to_echo=34223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7955, 0, 6737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1436, 2245, 3, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6941, 0, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 5275, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 3, 6, 5, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 0, 0, 1646, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 177, 4098, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14860, 3697, 13665, 8202, 8082, 2481, 8189, 6360] avg=8192.0000 max=14860 max_rank=0 min=2481 min_rank=5 imbalance_ratio=1.813965 max_load_reduction=50676 max_load_reduction_pct=0.773254 moved_to_echo=25112 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 6750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2594, 0, 0, 0, 0, 0, 0, 0, 6350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5203, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 0, 0, 0, 0, 0, 4735, 2053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7303, 8944, 6968, 8183, 8207, 9010, 8197, 8724] avg=8192.0000 max=9010 max_rank=5 min=6968 min_rank=2 imbalance_ratio=1.099854 max_load_reduction=56526 max_load_reduction_pct=0.862518 moved_to_echo=34138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 1, 6751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 0, 6501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 873, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 5250, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 16, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 816, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1778, 0, 0, 0, 0, 0, 4550, 2074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6973, 8982, 7581, 8184, 8223, 9008, 8183, 8402] avg=8192.0000 max=9008 max_rank=5 min=6973 min_rank=0 imbalance_ratio=1.099609 max_load_reduction=56528 max_load_reduction_pct=0.862549 moved_to_echo=33816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 8, 6750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2265, 0, 0, 0, 0, 0, 0, 0, 6772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2315, 0, 0, 0, 0, 37, 0, 0, 11, 0, 0, 0, 8, 5218, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 661, 0, 0, 0, 0, 0, 4133, 2059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7015, 9037, 9040, 8194, 8199, 9012, 8181, 6858] avg=8192.0000 max=9040 max_rank=2 min=6858 min_rank=7 imbalance_ratio=1.103516 max_load_reduction=56496 max_load_reduction_pct=0.862061 moved_to_echo=32250 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 6751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2768, 0, 0, 0, 0, 0, 0, 0, 4166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5249, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 825, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3287, 0, 0, 0, 0, 0, 5052, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7498, 6934, 7092, 8185, 8207, 9017, 8192, 10411] avg=8192.0000 max=10411 max_rank=7 min=6934 min_rank=1 imbalance_ratio=1.270874 max_load_reduction=55125 max_load_reduction_pct=0.841141 moved_to_echo=35827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 27392, 0, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 23296, 14848, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 11008, 128, 0, 0, 0, 2688, 43904, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 1280, 22016, 128, 0, 0, 128, 0, 128, 0, 42368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 128, 0, 0, 8064, 128, 57472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 6400, 0, 0, 59264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 1536, 19584, 128, 0, 0, 128, 0, 128, 0, 44544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 13312, 128, 0, 0, 0, 256, 35712, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 12288, 0, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 41728, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 128, 0, 0, 11392, 128, 54016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 6656, 0, 0, 59008, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 8192, 6723, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 8191, 0, 0, 8191, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8185, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 8185, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7684, 0, 0, 0, 0, 0, 6, 5, 0, 0, 6, 2, 473, 0, 1275, 0, 0, 7217, 7, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7988, 5925, 0, 0, 8184, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 5, 0, 0, 985, 0, 8181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 0, 0, 8192, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 8190, 4495, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 8177, 0, 0, 8182, 6269, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 8192, 7117, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 8176, 8178, 0, 0, 8185, 5016, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4424, 1157, 134, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 0, 0, 6723, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7035, 7219, 8256, 8093, 8219, 10461, 8195, 8058] avg=8192.0000 max=10461 max_rank=5 min=7035 min_rank=0 imbalance_ratio=1.276978 max_load_reduction=55075 max_load_reduction_pct=0.840378 moved_to_echo=34933 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4423, 1157, 134, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8185, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 0, 0, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7034, 8055, 8253, 8094, 8220, 9627, 8194, 8059] avg=8192.0000 max=9627 max_rank=5 min=7034 min_rank=0 imbalance_ratio=1.175171 max_load_reduction=55909 max_load_reduction_pct=0.853104 moved_to_echo=34100 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7029, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3200, 1156, 134, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7684, 0, 0, 0, 0, 0, 6, 5, 0, 0, 6, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 473, 0, 1275, 0, 0, 7130, 7, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7986, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2701, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 5, 0, 0, 985, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16312, 4534, 7759, 8895, 8020, 2712, 8190, 9114] avg=8192.0000 max=16312 max_rank=0 min=2712 min_rank=5 imbalance_ratio=1.991211 max_load_reduction=49224 max_load_reduction_pct=0.751099 moved_to_echo=28036 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4428, 1159, 136, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8182, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 4495, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7031, 9464, 8259, 8085, 8219, 8229, 8193, 8056] avg=8192.0000 max=9464 max_rank=1 min=7031 min_rank=0 imbalance_ratio=1.155273 max_load_reduction=56072 max_load_reduction_pct=0.855591 moved_to_echo=32697 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4416, 1155, 134, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 6269, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7026, 7693, 8255, 8094, 8198, 10017, 8185, 8068] avg=8192.0000 max=10017 max_rank=5 min=7026 min_rank=0 imbalance_ratio=1.222778 max_load_reduction=55519 max_load_reduction_pct=0.847153 moved_to_echo=34468 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4423, 1157, 134, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 0, 0, 0, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7035, 12801, 8255, 8090, 8221, 4881, 8195, 8058] avg=8192.0000 max=12801 max_rank=1 min=4881 min_rank=5 imbalance_ratio=1.562622 max_load_reduction=52735 max_load_reduction_pct=0.804672 moved_to_echo=29355 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4424, 1157, 134, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 0, 0, 7117, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7035, 6830, 8254, 8091, 8221, 10852, 8195, 8058] avg=8192.0000 max=10852 max_rank=5 min=6830 min_rank=1 imbalance_ratio=1.324707 max_load_reduction=54684 max_load_reduction_pct=0.834412 moved_to_echo=35326 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4416, 1156, 134, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 8172, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 5016, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7028, 8940, 8245, 8094, 8218, 8757, 8189, 8065] avg=8192.0000 max=8940 max_rank=1 min=7028 min_rank=0 imbalance_ratio=1.091309 max_load_reduction=56596 max_load_reduction_pct=0.863586 moved_to_echo=33229 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 8177, 0, 0, 8182, 5976, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 8176, 8177, 0, 0, 8184, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 8192, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8185, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 8185, 0, 0, 0, 18, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7690, 0, 0, 0, 0, 0, 6, 5, 0, 0, 6, 2, 646, 0, 1272, 0, 0, 7249, 7, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7864, 5653, 0, 0, 8184, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 5, 0, 0, 1087, 0, 8181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 0, 0, 8192, 805, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8187, 8191, 0, 0, 8191, 6116, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8183, 8191, 0, 0, 8190, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 8192, 4689, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3199, 1166, 146, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8173, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4928, 0, 0, 0, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7018, 9909, 8243, 8067, 8234, 7825, 8187, 8053] avg=8192.0000 max=9909 max_rank=1 min=7018 min_rank=0 imbalance_ratio=1.209595 max_load_reduction=55627 max_load_reduction_pct=0.848801 moved_to_echo=32299 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3205, 1168, 146, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4937, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7024, 8793, 8256, 8067, 8235, 8920, 8195, 8046] avg=8192.0000 max=8920 max_rank=5 min=7024 min_rank=0 imbalance_ratio=1.088867 max_load_reduction=56616 max_load_reduction_pct=0.863892 moved_to_echo=33396 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3199, 1166, 146, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4928, 0, 0, 0, 5976, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7015, 6817, 8255, 8067, 8214, 10927, 8185, 8056] avg=8192.0000 max=10927 max_rank=5 min=6817 min_rank=1 imbalance_ratio=1.333862 max_load_reduction=54609 max_load_reduction_pct=0.833267 moved_to_echo=35382 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7018, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2211, 1167, 145, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7690, 0, 0, 0, 0, 0, 6, 5, 0, 0, 6, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 646, 0, 1272, 0, 0, 7138, 7, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7861, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 5, 0, 0, 1087, 0, 8036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 3596, 7765, 9073, 7906, 3418, 8193, 9198] avg=8192.0000 max=16387 max_rank=0 min=3418 min_rank=5 imbalance_ratio=2.000366 max_load_reduction=49149 max_load_reduction_pct=0.749954 moved_to_echo=28715 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3204, 1167, 146, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8184, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4937, 0, 0, 0, 6116, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7024, 6659, 8251, 8067, 8235, 11059, 8194, 8047] avg=8192.0000 max=11059 max_rank=5 min=6659 min_rank=1 imbalance_ratio=1.349976 max_load_reduction=54477 max_load_reduction_pct=0.831253 moved_to_echo=35535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3209, 1170, 148, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8181, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4927, 0, 0, 0, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7020, 9707, 8257, 8061, 8238, 8017, 8192, 8044] avg=8192.0000 max=9707 max_rank=1 min=7020 min_rank=0 imbalance_ratio=1.184937 max_load_reduction=55829 max_load_reduction_pct=0.851883 moved_to_echo=32491 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3205, 1168, 146, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4937, 0, 0, 0, 4689, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7024, 8086, 8254, 8067, 8237, 9627, 8195, 8046] avg=8192.0000 max=9627 max_rank=5 min=7024 min_rank=0 imbalance_ratio=1.175171 max_load_reduction=55909 max_load_reduction_pct=0.853104 moved_to_echo=34105 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3204, 1168, 146, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4936, 0, 0, 0, 805, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7024, 11969, 8255, 8067, 8237, 5743, 8195, 8046] avg=8192.0000 max=11969 max_rank=1 min=5743 min_rank=5 imbalance_ratio=1.461060 max_load_reduction=53567 max_load_reduction_pct=0.817368 moved_to_echo=30221 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 20736, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 34176, 9344, 1152, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 28928, 0, 0, 0, 36736, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 56320, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 29952, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 24704, 9344, 1280, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 38016, 0, 0, 0, 27648, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 56192, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 512, 0, 1280, 0, 0, 63872, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 1024, 0, 64512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 1152, 0, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 768, 0, 1280, 0, 0, 63616, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 65152, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 65024, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2186, 0, 0, 0, 0, 6243, 0, 0, 0, 0, 0, 0, 0, 14, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5824, 0, 0, 0, 0, 4768, 0, 0, 0, 0, 0, 0, 0, 34, 7756, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 6239, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1298, 0, 0, 0, 0, 6920, 0, 0, 0, 0, 0, 0, 0, 18, 8188, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 0, 0, 5521, 0, 0, 0, 0, 0, 0, 0, 48, 7928, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1581, 0, 0, 0, 0, 7045, 0, 0, 0, 0, 0, 0, 0, 13, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 6917, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 66, 0, 0, 0, 0, 0, 2, 0, 0, 81, 0, 0, 179, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 14, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1766, 7, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 3, 2, 0, 6002, 0, 0, 0, 0, 0, 3, 0, 0, 0, 22, 0, 0, 0, 0, 0, 6, 0, 6, 7, 416, 0, 1243, 0, 8010, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 6802, 0, 0, 0, 0, 4355, 0, 0, 0, 0, 0, 0, 0, 54, 7968, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 5429, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2603, 0, 0, 0, 0, 6319, 0, 0, 0, 0, 0, 0, 0, 3, 7989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 7674, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6421, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5177, 0, 0, 0, 0, 0, 0, 0, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6243, 0, 0, 0, 0, 0, 0, 0, 14, 2041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 0, 0, 0, 0, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4360, 3015, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 8189, 7363, 8298, 8187, 9542, 7536, 8077] avg=8192.0000 max=9542 max_rank=5 min=7363 min_rank=2 imbalance_ratio=1.164795 max_load_reduction=55994 max_load_reduction_pct=0.854401 moved_to_echo=33342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6423, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5177, 0, 0, 0, 0, 0, 0, 0, 5824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4768, 0, 0, 0, 0, 0, 0, 0, 34, 1937, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 8038, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6239, 0, 0, 0, 0, 0, 0, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4140, 3015, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7979, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 8102, 11001, 6741, 8193, 7857, 7315, 7983] avg=8192.0000 max=11001 max_rank=2 min=6741 min_rank=3 imbalance_ratio=1.342896 max_load_reduction=54535 max_load_reduction_pct=0.832138 moved_to_echo=31348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6917, 0, 0, 0, 0, 0, 0, 152, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 81, 0, 0, 179, 6, 0, 0, 0, 0, 0, 0, 0, 6419, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5167, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 14, 0, 0, 0, 0, 0, 0, 2029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1766, 7, 0, 0, 0, 0, 0, 1617, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6002, 0, 0, 0, 0, 0, 3, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4334, 3009, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 7, 408, 0, 1243, 0, 8010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7135, 8445, 5168, 10144, 8191, 3395, 13378, 9680] avg=8192.0000 max=13378 max_rank=6 min=3395 min_rank=5 imbalance_ratio=1.633057 max_load_reduction=52158 max_load_reduction_pct=0.795868 moved_to_echo=34644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6408, 1716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5177, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5521, 0, 0, 0, 0, 0, 0, 0, 48, 1981, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 8029, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 1615, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4231, 3015, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 8124, 9013, 7561, 8194, 9060, 7403, 7837] avg=8192.0000 max=9060 max_rank=5 min=7403 min_rank=6 imbalance_ratio=1.105957 max_load_reduction=56476 max_load_reduction_pct=0.861755 moved_to_echo=32494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6428, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5176, 0, 0, 0, 0, 0, 0, 0, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6920, 0, 0, 0, 0, 0, 0, 0, 18, 2038, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8042, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4374, 3016, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8204, 6474, 8980, 8198, 9722, 7555, 8065] avg=8192.0000 max=9722 max_rank=5 min=6474 min_rank=2 imbalance_ratio=1.186768 max_load_reduction=55814 max_load_reduction_pct=0.851654 moved_to_echo=33540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6412, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5177, 0, 0, 0, 0, 0, 0, 0, 6802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4355, 0, 0, 0, 0, 0, 0, 0, 54, 1990, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 8023, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5429, 0, 0, 0, 0, 0, 0, 1615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4253, 3015, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8343, 8146, 11979, 6421, 8190, 7044, 7428, 7985] avg=8192.0000 max=11979 max_rank=2 min=6421 min_rank=3 imbalance_ratio=1.462280 max_load_reduction=53557 max_load_reduction_pct=0.817215 moved_to_echo=30647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6424, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5177, 0, 0, 0, 0, 0, 0, 0, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7045, 0, 0, 0, 0, 0, 0, 0, 13, 2015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8039, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4305, 3015, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 8170, 6758, 9073, 8189, 9624, 7480, 7898] avg=8192.0000 max=9624 max_rank=5 min=6758 min_rank=2 imbalance_ratio=1.174805 max_load_reduction=55912 max_load_reduction_pct=0.853149 moved_to_echo=33191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6427, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5177, 0, 0, 0, 0, 0, 0, 0, 2603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6319, 0, 0, 0, 0, 0, 0, 0, 3, 1996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 8039, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7674, 0, 0, 0, 0, 0, 0, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4264, 3015, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 8156, 7780, 8318, 8194, 9292, 7441, 8011] avg=8192.0000 max=9292 max_rank=5 min=7441 min_rank=6 imbalance_ratio=1.134277 max_load_reduction=56244 max_load_reduction_pct=0.858215 moved_to_echo=32938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4223, 0, 0, 0, 0, 0, 0, 0, 890, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3074, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4826, 0, 0, 0, 0, 0, 0, 0, 2517, 7900, 0, 0, 0, 0, 12, 2, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4636, 0, 0, 0, 0, 0, 0, 0, 1144, 8120, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2536, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6805, 0, 0, 0, 0, 0, 0, 0, 1486, 7947, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 5781, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 69, 0, 0, 0, 0, 0, 2, 0, 0, 86, 0, 0, 201, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 2, 0, 0, 0, 0, 0, 6462, 0, 0, 0, 0, 8075, 0, 17, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 3, 2, 0, 3034, 0, 0, 0, 0, 0, 3, 0, 0, 0, 17, 0, 0, 0, 0, 0, 5, 0, 6, 7, 0, 0, 1094, 0, 7981, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4995, 0, 0, 0, 0, 0, 0, 0, 1955, 7833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1573, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 21, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6515, 0, 0, 0, 0, 0, 0, 0, 1849, 7802, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 1235, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7694, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4223, 0, 0, 0, 0, 0, 0, 0, 890, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7975, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3074, 0, 0, 0, 0, 0, 0, 6959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6566, 1220, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8486, 8163, 8409, 6145, 8179, 10033, 7887, 8234] avg=8192.0000 max=10033 max_rank=5 min=6145 min_rank=3 imbalance_ratio=1.224731 max_load_reduction=55503 max_load_reduction_pct=0.846909 moved_to_echo=34333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7692, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6805, 0, 0, 0, 0, 0, 0, 0, 1486, 1017, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 7975, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 6964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6468, 1221, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8486, 8154, 8409, 9310, 8195, 7101, 7791, 8090] avg=8192.0000 max=9310 max_rank=3 min=7101 min_rank=5 imbalance_ratio=1.136475 max_load_reduction=56226 max_load_reduction_pct=0.857941 moved_to_echo=31177 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7682, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4826, 0, 0, 0, 0, 0, 0, 0, 2517, 1010, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 7975, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 6948, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6430, 1218, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8485, 8142, 8409, 8365, 8202, 8099, 7749, 8085] avg=8192.0000 max=8485 max_rank=0 min=7749 min_rank=6 imbalance_ratio=1.035767 max_load_reduction=57051 max_load_reduction_pct=0.870529 moved_to_echo=32135 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5781, 0, 0, 0, 0, 0, 0, 294, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 86, 0, 0, 201, 6, 0, 0, 0, 0, 0, 0, 0, 7686, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 2, 0, 0, 0, 0, 0, 6462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 17, 0, 0, 0, 0, 0, 0, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7960, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 6963, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3034, 0, 0, 0, 0, 0, 3, 0, 0, 0, 17, 0, 0, 0, 0, 0, 6608, 1221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 7, 0, 0, 1094, 0, 7981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 8453, 6680, 9131, 8177, 6975, 10883, 9093] avg=8192.0000 max=10883 max_rank=6 min=6144 min_rank=0 imbalance_ratio=1.328491 max_load_reduction=54653 max_load_reduction_pct=0.833939 moved_to_echo=35128 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7677, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6515, 0, 0, 0, 0, 0, 0, 0, 1849, 998, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 7975, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 6947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6350, 1218, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8485, 8141, 8409, 9387, 8206, 7142, 7669, 8097] avg=8192.0000 max=9387 max_rank=3 min=7142 min_rank=5 imbalance_ratio=1.145874 max_load_reduction=56149 max_load_reduction_pct=0.856766 moved_to_echo=31114 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7693, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4995, 0, 0, 0, 0, 0, 0, 0, 1955, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 7975, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1573, 0, 0, 0, 0, 0, 0, 6968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6375, 1222, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 1, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8486, 8149, 8409, 7952, 8191, 8541, 7699, 8109] avg=8192.0000 max=8541 max_rank=5 min=7699 min_rank=6 imbalance_ratio=1.042603 max_load_reduction=56995 max_load_reduction_pct=0.869675 moved_to_echo=32540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7693, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4636, 0, 0, 0, 0, 0, 0, 0, 1144, 1031, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 7982, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2536, 0, 0, 0, 0, 0, 0, 6958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6613, 1226, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7990, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8478, 8169, 8402, 6815, 8201, 9494, 7941, 8036] avg=8192.0000 max=9494 max_rank=5 min=6815 min_rank=3 imbalance_ratio=1.158936 max_load_reduction=56042 max_load_reduction_pct=0.855133 moved_to_echo=33672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7693, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 1235, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7975, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 0, 0, 0, 0, 0, 0, 6963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6598, 1221, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8486, 8165, 8409, 8431, 8185, 8151, 7917, 7792] avg=8192.0000 max=8486 max_rank=0 min=7792 min_rank=7 imbalance_ratio=1.035889 max_load_reduction=57050 max_load_reduction_pct=0.870514 moved_to_echo=32045 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[6016, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 34304, 24192, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 51456, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 41472, 0, 128, 0, 0, 0, 0, 0, 24192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 52608, 128, 0, 0, 0, 0, 0, 12928, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 49280, 0, 128, 0, 0, 0, 0, 0, 256, 16128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 56192, 0, 1280, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 64384, 0, 0, 0, 0, 0, 0, 1280, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 64384, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 9856, 128, 0, 0, 0, 0, 0, 55680, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3072, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 52096, 9856, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 46336, 0, 128, 0, 0, 0, 0, 0, 11136, 8192, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 61568, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 63872, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 63232, 0, 0, 0, 0, 0, 0, 2432, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 1792, 0, 128, 0, 0, 0, 0, 0, 63872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 56320, 0, 1152, 0, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8120, 8171, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8029, 45, 5, 0, 8184, 0, 150, 8, 6, 2299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 2, 0, 21, 2908, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 2711, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, 6650, 0, 0, 7, 1960, 1, 0, 0, 8, 0, 1, 0, 84, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 7515, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2312, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8175, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8188, 0, 1, 0, 0, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 4773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 3557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2798, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7222, 0, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6229, 0, 0, 0, 0, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2109, 0, 0, 0, 0, 0, 0, 4325, 1223, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 7449, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3738, 0, 0, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 7972, 8190, 8849, 8379, 7602, 8056, 8275] avg=8192.0000 max=8849 max_rank=3 min=7602 min_rank=5 imbalance_ratio=1.080200 max_load_reduction=56687 max_load_reduction_pct=0.864975 moved_to_echo=32312 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6229, 0, 0, 0, 0, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 0, 0, 0, 0, 0, 0, 4373, 1223, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2312, 0, 0, 0, 0, 0, 0, 7449, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 0, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 7403, 8191, 8849, 7419, 9763, 7423, 8275] avg=8192.0000 max=9763 max_rank=5 min=7403 min_rank=1 imbalance_ratio=1.191772 max_load_reduction=55773 max_load_reduction_pct=0.851028 moved_to_echo=32880 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 5, 0, 6224, 0, 150, 8, 6, 2299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 21, 2908, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 2711, 0, 0, 1, 0, 0, 0, 3567, 1222, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, 7383, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3083, 0, 0, 7, 1960, 1, 0, 0, 8, 0, 1, 0, 84, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 7515, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8052, 8737, 8178, 3591, 8233, 7806, 13329, 7610] avg=8192.0000 max=13329 max_rank=6 min=3591 min_rank=3 imbalance_ratio=1.627075 max_load_reduction=52207 max_load_reduction_pct=0.796616 moved_to_echo=36978 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6227, 0, 1, 0, 0, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3164, 0, 0, 0, 0, 0, 0, 4370, 1222, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 7433, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3777, 0, 0, 0, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8645, 8199, 8849, 9477, 7450, 6429, 8274] avg=8192.0000 max=9477 max_rank=4 min=6429 min_rank=6 imbalance_ratio=1.156860 max_load_reduction=56059 max_load_reduction_pct=0.855392 moved_to_echo=31630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6220, 0, 0, 0, 0, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 0, 0, 0, 0, 0, 0, 4355, 1226, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 7452, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 7992, 8193, 8853, 9268, 7468, 7279, 8277] avg=8192.0000 max=9268 max_rank=4 min=7279 min_rank=6 imbalance_ratio=1.131348 max_load_reduction=56268 max_load_reduction_pct=0.858582 moved_to_echo=32292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6229, 0, 0, 0, 0, 3557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 4349, 1223, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2798, 0, 0, 0, 0, 0, 0, 7449, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 0, 0, 0, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 9786, 8202, 8849, 6654, 10249, 5308, 8275] avg=8192.0000 max=10249 max_rank=5 min=5308 min_rank=6 imbalance_ratio=1.251099 max_load_reduction=55287 max_load_reduction_pct=0.843613 moved_to_echo=30486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6229, 0, 0, 0, 0, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1422, 0, 0, 0, 0, 0, 0, 4354, 1223, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 7449, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 0, 0, 0, 4773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 7433, 8191, 8847, 7721, 7741, 9115, 8275] avg=8192.0000 max=9115 max_rank=6 min=7433 min_rank=1 imbalance_ratio=1.112671 max_load_reduction=56421 max_load_reduction_pct=0.860916 moved_to_echo=32852 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6229, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 0, 0, 3874, 1223, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7449, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3348, 0, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 7568, 8192, 8849, 8385, 7457, 8597, 8275] avg=8192.0000 max=8849 max_rank=3 min=7457 min_rank=5 imbalance_ratio=1.080200 max_load_reduction=56687 max_load_reduction_pct=0.864975 moved_to_echo=32714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 4031, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 294, 8167, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8076, 49, 5, 0, 8184, 0, 7822, 8, 6, 2427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 2, 0, 23, 3064, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 7192, 2, 0, 0, 8, 0, 1, 0, 152, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 7904, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8188, 0, 1484, 0, 0, 7516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 27, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 1920, 0, 0, 7821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 1284, 0, 0, 7266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 133, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 2137, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 66, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 821, 0, 0, 7177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 76, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 960, 0, 0, 7159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 6582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4277, 0, 0, 0, 0, 0, 0, 1189, 525, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 8122, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6948, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 10613, 8200, 8928, 6061, 8392, 6994, 8226] avg=8192.0000 max=10613 max_rank=1 min=6061 min_rank=4 imbalance_ratio=1.295532 max_load_reduction=54923 max_load_reduction_pct=0.838058 moved_to_echo=29673 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1920, 0, 0, 6606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6400, 0, 0, 0, 0, 0, 0, 1186, 526, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 8122, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8526, 8191, 8931, 8182, 8232, 7113, 8226] avg=8192.0000 max=8931 max_rank=3 min=7113 min_rank=6 imbalance_ratio=1.090210 max_load_reduction=56605 max_load_reduction_pct=0.863724 moved_to_echo=31753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 5, 0, 0, 0, 7822, 8, 6, 2051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 23, 3064, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 213, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 0, 1, 0, 0, 0, 1051, 163, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 8098, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 6141, 2, 0, 0, 8, 0, 1, 0, 152, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 7904, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 9941, 8190, 3355, 5120, 8193, 14497, 7950] avg=8192.0000 max=14497 max_rank=6 min=3355 min_rank=3 imbalance_ratio=1.769653 max_load_reduction=51039 max_load_reduction_pct=0.778793 moved_to_echo=35760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 0, 0, 6348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7343, 0, 0, 0, 0, 0, 0, 1186, 506, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 8122, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6927, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 7832, 8193, 8904, 9109, 8143, 7020, 8226] avg=8192.0000 max=9109 max_rank=4 min=7020 min_rank=6 imbalance_ratio=1.111938 max_load_reduction=56427 max_load_reduction_pct=0.861008 moved_to_echo=32498 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7371, 0, 0, 0, 0, 0, 0, 1194, 489, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8122, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 7421, 8190, 8882, 9124, 8129, 7275, 8226] avg=8192.0000 max=9124 max_rank=4 min=7275 min_rank=6 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=32754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2137, 0, 0, 5185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7387, 0, 0, 0, 0, 0, 0, 1186, 413, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 8122, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 7322, 8190, 8781, 9056, 8176, 7544, 8226] avg=8192.0000 max=9056 max_rank=4 min=7322 min_rank=1 imbalance_ratio=1.105469 max_load_reduction=56480 max_load_reduction_pct=0.861816 moved_to_echo=33002 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 960, 0, 0, 6038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7593, 0, 0, 0, 0, 0, 0, 1195, 486, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 8121, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 6998, 8192, 8881, 9345, 8143, 7571, 8230] avg=8192.0000 max=9345 max_rank=4 min=6998 min_rank=1 imbalance_ratio=1.140747 max_load_reduction=56191 max_load_reduction_pct=0.857407 moved_to_echo=33289 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 0, 0, 6062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7797, 0, 0, 0, 0, 0, 0, 1189, 483, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8122, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 6883, 8190, 8874, 9539, 8128, 7522, 8226] avg=8192.0000 max=9539 max_rank=4 min=6883 min_rank=1 imbalance_ratio=1.164429 max_load_reduction=55997 max_load_reduction_pct=0.854446 moved_to_echo=33415 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 16512, 0, 0, 128, 0, 0, 0, 33664, 9856, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[29056, 0, 0, 128, 21376, 128, 0, 0, 128, 0, 128, 0, 128, 128, 0, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 49920, 0, 256, 128, 128, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 6016, 0, 0, 0, 0, 0, 0, 59520, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 60288, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 64896, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 52096, 0, 0, 128, 0, 0, 0, 9472, 3712, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 54784, 128, 0, 0, 128, 0, 128, 0, 256, 128, 0, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 20480, 128, 128, 45056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 60416, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 4736, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 65024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65280, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 28, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 34, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 8181, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 273, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 22, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 1, 0, 0, 4722, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 6783, 5, 2, 0, 6919, 0, 1, 0, 0, 3333, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 8099, 6, 0, 0, 0, 0, 0, 0, 0, 6, 8165, 0, 1484, 8183, 1, 0, 6, 3, 0, 0, 0, 0, 140, 0, 0, 0, 0, 1177, 0, 0, 6, 0, 0, 0, 0, 6, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 30, 0, 8182, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 202, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 8192, 0, 0, 0, 19, 0, 0, 0, 0, 7991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 107, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 18, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 266, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 42, 0, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 276, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 0, 0, 7933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 28, 0, 0, 0, 0, 0, 0, 168, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8186, 8507, 8043, 8202, 8009, 8201, 8194] avg=8192.0000 max=8507 max_rank=2 min=8009 min_rank=5 imbalance_ratio=1.038452 max_load_reduction=57029 max_load_reduction_pct=0.870193 moved_to_echo=32606 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 30, 0, 8175, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 202, 0, 0, 0, 0, 1, 0, 168, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 7857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8211, 8686, 7876, 8176, 7998, 8200, 8195] avg=8192.0000 max=8686 max_rank=2 min=7876 min_rank=3 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=32569 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 34, 0, 0, 0, 0, 0, 0, 171, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8182, 8524, 8013, 8223, 8003, 8203, 8193] avg=8192.0000 max=8524 max_rank=2 min=8003 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=57012 max_load_reduction_pct=0.869934 moved_to_echo=32622 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8165, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 1, 0, 0, 4722, 0, 0, 0, 0, 0, 0, 0, 168, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 6919, 0, 1, 0, 0, 3278, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 8099, 6, 0, 0, 0, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1484, 8002, 1, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 1177, 0, 0, 6, 0, 0, 0, 0, 6, 0, 7, 6780, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 8162, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8183, 4987, 10252, 8129, 9502, 8129, 8178] avg=8192.0000 max=10252 max_rank=3 min=4987 min_rank=2 imbalance_ratio=1.251465 max_load_reduction=55284 max_load_reduction_pct=0.843567 moved_to_echo=33938 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 273, 0, 0, 0, 0, 0, 0, 168, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 7771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8210, 8752, 7793, 8201, 7991, 8200, 8195] avg=8192.0000 max=8752 max_rank=2 min=7793 min_rank=3 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=32587 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 107, 0, 0, 0, 0, 0, 0, 168, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8185, 8583, 7957, 8211, 8011, 8201, 8194] avg=8192.0000 max=8583 max_rank=2 min=7957 min_rank=3 imbalance_ratio=1.047729 max_load_reduction=56953 max_load_reduction_pct=0.869034 moved_to_echo=32617 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 266, 0, 0, 0, 0, 0, 0, 168, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 7753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8185, 8743, 7795, 8213, 8011, 8201, 8193] avg=8192.0000 max=8743 max_rank=2 min=7795 min_rank=3 imbalance_ratio=1.067261 max_load_reduction=56793 max_load_reduction_pct=0.866592 moved_to_echo=32618 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 276, 0, 0, 0, 0, 0, 0, 168, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8194, 8754, 7807, 8181, 8011, 8201, 8194] avg=8192.0000 max=8754 max_rank=2 min=7807 min_rank=3 imbalance_ratio=1.068604 max_load_reduction=56782 max_load_reduction_pct=0.866425 moved_to_echo=32587 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 43, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 18, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 8180, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 44, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 17, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 32, 0, 8181, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8192, 92, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 44, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 34, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 1, 0, 0, 4593, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 6781, 6, 2, 0, 5954, 0, 2, 0, 0, 4285, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 8100, 6, 0, 0, 0, 0, 0, 0, 0, 6, 8164, 0, 1568, 8183, 1, 0, 6, 3, 0, 0, 0, 0, 144, 0, 0, 0, 0, 1231, 0, 0, 6, 0, 0, 0, 0, 6, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 43, 0, 0, 0, 0, 0, 0, 246, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8198, 8656, 7923, 8176, 8001, 8194, 8194] avg=8192.0000 max=8656 max_rank=2 min=7923 min_rank=3 imbalance_ratio=1.056641 max_load_reduction=56880 max_load_reduction_pct=0.867920 moved_to_echo=32565 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 8172, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 44, 0, 0, 0, 0, 0, 0, 246, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8206, 8660, 7903, 8204, 7981, 8194, 8194] avg=8192.0000 max=8660 max_rank=2 min=7903 min_rank=3 imbalance_ratio=1.057129 max_load_reduction=56876 max_load_reduction_pct=0.867859 moved_to_echo=32573 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 17, 0, 0, 0, 0, 0, 0, 247, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8185, 8628, 7936, 8206, 7999, 8194, 8194] avg=8192.0000 max=8628 max_rank=2 min=7936 min_rank=3 imbalance_ratio=1.053223 max_load_reduction=56908 max_load_reduction_pct=0.868347 moved_to_echo=32593 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 32, 0, 8173, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8192, 92, 0, 0, 0, 0, 1, 0, 245, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8212, 8710, 7860, 8183, 7989, 8194, 8195] avg=8192.0000 max=8710 max_rank=2 min=7860 min_rank=3 imbalance_ratio=1.063232 max_load_reduction=56826 max_load_reduction_pct=0.867096 moved_to_echo=32561 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 34, 0, 0, 0, 0, 0, 0, 251, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 22, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8183, 8656, 7906, 8219, 7988, 8197, 8193] avg=8192.0000 max=8656 max_rank=2 min=7906 min_rank=3 imbalance_ratio=1.056641 max_load_reduction=56880 max_load_reduction_pct=0.867920 moved_to_echo=32597 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 44, 0, 0, 0, 0, 0, 0, 247, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8184, 8653, 7925, 8191, 8001, 8194, 8194] avg=8192.0000 max=8653 max_rank=2 min=7925 min_rank=3 imbalance_ratio=1.056274 max_load_reduction=56883 max_load_reduction_pct=0.867966 moved_to_echo=32580 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8168, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 1, 0, 0, 4593, 0, 0, 0, 0, 0, 0, 0, 129, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 5954, 0, 2, 0, 0, 4153, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 8100, 6, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1568, 7992, 1, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 1231, 0, 0, 6, 0, 0, 0, 0, 6, 0, 7, 6777, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 8162, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8184, 4956, 10147, 8141, 9576, 8174, 8179] avg=8192.0000 max=10147 max_rank=3 min=4956 min_rank=2 imbalance_ratio=1.238647 max_load_reduction=55389 max_load_reduction_pct=0.845169 moved_to_echo=34070 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 247, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8184, 8617, 7936, 8216, 8001, 8195, 8193] avg=8192.0000 max=8617 max_rank=2 min=7936 min_rank=3 imbalance_ratio=1.051880 max_load_reduction=56919 max_load_reduction_pct=0.868515 moved_to_echo=32605 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 62080, 1280, 0, 0, 0, 0, 128, 0, 1408, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 7040, 0, 128, 0, 0, 58496, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 1536, 64128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 0, 0, 1280, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 61952, 384, 0, 0, 0, 0, 128, 0, 1920, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 6016, 0, 128, 0, 0, 59648, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 1664, 64000, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 65408, 128, 0, 0, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 0, 0, 1280, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 65408, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 12, 0, 0, 0, 0, 8082, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 38, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 8105, 0, 0, 416, 0, 0, 7, 211, 8186, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6066, 0, 0, 0, 0, 0, 1, 0, 0, 2193, 0, 0, 0, 0, 0, 0, 8180, 10, 0, 0, 0, 2, 0, 2, 6, 0, 6, 0, 0, 8180, 0, 6, 1, 0, 0, 0, 7445, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 7904, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 8082, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 11, 182, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8200, 8460, 7918, 8213, 8277, 8180, 8192] avg=8192.0000 max=8460 max_rank=2 min=7918 min_rank=3 imbalance_ratio=1.032715 max_load_reduction=57076 max_load_reduction_pct=0.870911 moved_to_echo=32862 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7, 182, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8200, 8455, 7918, 8229, 8287, 8180, 8192] avg=8192.0000 max=8455 max_rank=2 min=7918 min_rank=3 imbalance_ratio=1.032104 max_load_reduction=57081 max_load_reduction_pct=0.870987 moved_to_echo=32888 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 416, 0, 0, 7, 211, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4, 7, 0, 0, 8105, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6066, 0, 0, 181, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2193, 0, 0, 0, 0, 0, 0, 7907, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 6, 0, 6, 0, 0, 8106, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7445, 7, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8063, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8821, 8135, 6328, 10111, 8129, 7548, 8274, 8190] avg=8192.0000 max=10111 max_rank=3 min=6328 min_rank=2 imbalance_ratio=1.234253 max_load_reduction=55425 max_load_reduction_pct=0.845718 moved_to_echo=32141 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 182, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8200, 8449, 7918, 8145, 8288, 8180, 8192] avg=8192.0000 max=8449 max_rank=2 min=7918 min_rank=3 imbalance_ratio=1.031372 max_load_reduction=57087 max_load_reduction_pct=0.871078 moved_to_echo=32805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 183, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8201, 8458, 7916, 8155, 8284, 8182, 8194] avg=8192.0000 max=8458 max_rank=2 min=7916 min_rank=3 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=32815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7904, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 24, 182, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7913, 8200, 8473, 7918, 8384, 8276, 8180, 8192] avg=8192.0000 max=8473 max_rank=2 min=7913 min_rank=0 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=33032 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 11, 182, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8200, 8459, 7919, 8148, 8288, 8180, 8192] avg=8192.0000 max=8459 max_rank=2 min=7919 min_rank=3 imbalance_ratio=1.032593 max_load_reduction=57077 max_load_reduction_pct=0.870926 moved_to_echo=32808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 6, 182, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8200, 8454, 7918, 8133, 8288, 8180, 8192] avg=8192.0000 max=8454 max_rank=2 min=7918 min_rank=3 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=32793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 124, 0, 0, 0, 0, 7795, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 68, 0, 0, 0, 0, 8047, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 56, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 18, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 18, 0, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 22, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8107, 0, 0, 234, 0, 0, 7, 54, 8186, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7768, 0, 0, 0, 0, 0, 1, 0, 0, 353, 0, 0, 0, 0, 0, 0, 8180, 10, 0, 0, 0, 2, 0, 2, 6, 0, 6, 0, 0, 8179, 0, 6, 2, 0, 0, 0, 7937, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 39, 0, 0, 124, 0, 0, 0, 0, 7795, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 71, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 60, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7965, 8191, 8298, 8149, 8334, 8219, 8188, 8192] avg=8192.0000 max=8334 max_rank=4 min=7965 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 60, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8191, 8227, 8149, 8164, 8226, 8188, 8192] avg=8192.0000 max=8227 max_rank=2 min=8149 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 39, 0, 0, 56, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 18, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 60, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8191, 8245, 8150, 8160, 8226, 8188, 8192] avg=8192.0000 max=8245 max_rank=2 min=8150 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 39, 0, 0, 18, 0, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 16, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 60, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8191, 8243, 8149, 8232, 8225, 8188, 8192] avg=8192.0000 max=8243 max_rank=2 min=8116 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 39, 0, 0, 68, 0, 0, 0, 0, 8047, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 60, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8191, 8237, 8149, 8205, 8217, 8188, 8192] avg=8192.0000 max=8237 max_rank=2 min=8149 min_rank=3 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 28, 0, 0, 4, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 40, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 65, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8203, 8242, 8140, 8160, 8219, 8203, 8191] avg=8192.0000 max=8242 max_rank=2 min=8140 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 39, 0, 0, 22, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 60, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8191, 8234, 8149, 8138, 8226, 8188, 8192] avg=8192.0000 max=8234 max_rank=2 min=8138 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 38, 0, 0, 234, 0, 0, 7, 54, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4, 7, 0, 0, 8131, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7768, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 353, 0, 0, 0, 0, 0, 0, 8137, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 6, 0, 6, 0, 0, 8119, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7937, 7, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8036, 60, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8519, 8187, 7810, 8501, 8143, 7978, 8205, 8193] avg=8192.0000 max=8519 max_rank=0 min=7810 min_rank=2 imbalance_ratio=1.039917 max_load_reduction=57017 max_load_reduction_pct=0.870010 moved_to_echo=32519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 63488, 0, 128, 1536, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 2304, 0, 0, 0, 0, 0, 0, 63360, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 64768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 64768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 512, 0, 0, 128, 256, 64896, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[640, 128, 128, 0, 128, 128, 0, 128, 0, 0, 65024, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 384, 0, 0, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 384, 0, 0, 640, 0, 0, 128, 128, 64640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 0, 128, 0, 128, 128, 0, 128, 0, 0, 65152, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65152, 0, 256, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 64896, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True - [2026-08-26 16:18:09] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 446.6 | throughput per GPU (TFLOP/s/GPU): 404.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.296318E+00 | loss scale: 1.0 | grad norm: 4.762 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 16:18:09 -W0826 16:18:09.514000 203627 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 16:18:09.515000 203627 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 203721 closing signal SIGTERM -W0826 16:18:09.516000 203627 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 203722 closing signal SIGTERM -W0826 16:18:09.517000 203627 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 203723 closing signal SIGTERM -W0826 16:18:09.519000 203627 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 203724 closing signal SIGTERM -W0826 16:18:09.520000 203627 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 203725 closing signal SIGTERM -W0826 16:18:09.522000 203627 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 203726 closing signal SIGTERM -W0826 16:18:09.523000 203627 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 203727 closing signal SIGTERM -W0826 16:18:09.525000 203627 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 203728 closing signal SIGTERM +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True + [2026-08-26 16:27:17] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 442.5 | throughput per GPU (TFLOP/s/GPU): 408.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.300362E+00 | loss scale: 1.0 | grad norm: 4.767 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:27:17 +W0826 16:27:17.345000 213257 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 16:27:17.346000 213257 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213351 closing signal SIGTERM +W0826 16:27:17.346000 213257 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213352 closing signal SIGTERM +W0826 16:27:17.348000 213257 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213353 closing signal SIGTERM +W0826 16:27:17.350000 213257 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213354 closing signal SIGTERM +W0826 16:27:17.351000 213257 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213355 closing signal SIGTERM +W0826 16:27:17.352000 213257 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213356 closing signal SIGTERM +W0826 16:27:17.354000 213257 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213357 closing signal SIGTERM +W0826 16:27:17.357000 213257 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213358 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -7179,4 +7179,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 203627 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 213257 got signal: 15 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..50be28456028124d7dc3d4efa2851b3e31611a03 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,272 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 877 +1, 899 +2, 895 +3, 899 +4, 885 +5, 897 +6, 897 +7, 905 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 39169 +1, 38995 +2, 39197 +3, 38995 +4, 38995 +5, 39197 +6, 38995 +7, 39197 +0, 44989 +1, 44991 +2, 45001 +3, 44963 +4, 44987 +5, 45001 +6, 44991 +7, 45001 +0, 45895 +1, 45897 +2, 45907 +3, 45869 +4, 45893 +5, 45907 +6, 45897 +7, 45907 +0, 46917 +1, 46919 +2, 46929 +3, 46891 +4, 46915 +5, 46929 +6, 46919 +7, 46929 +0, 47013 +1, 47015 +2, 47025 +3, 46987 +4, 47011 +5, 47025 +6, 47015 +7, 47025 +0, 47193 +1, 47203 +2, 47233 +3, 47171 +4, 47181 +5, 47217 +6, 47209 +7, 47221 +0, 47877 +1, 47879 +2, 47889 +3, 47851 +4, 47875 +5, 47889 +6, 47879 +7, 47889 +0, 48037 +1, 48039 +2, 48049 +3, 48011 +4, 48035 +5, 48049 +6, 48039 +7, 48049 +0, 48737 +1, 48739 +2, 48749 +3, 48711 +4, 48735 +5, 48749 +6, 48699 +7, 48749 +0, 48737 +1, 48739 +2, 48749 +3, 48711 +4, 48735 +5, 48749 +6, 48699 +7, 48749 +0, 92911 +1, 93679 +2, 93683 +3, 92905 +4, 93695 +5, 93683 +6, 93679 +7, 92897 +0, 94411 +1, 94393 +2, 94397 +3, 94405 +4, 94409 +5, 94397 +6, 94393 +7, 94397 +0, 94411 +1, 94393 +2, 94397 +3, 94405 +4, 94409 +5, 94397 +6, 94393 +7, 94397 +0, 94413 +1, 94395 +2, 94399 +3, 94407 +4, 94411 +5, 94399 +6, 94395 +7, 94399 +0, 94413 +1, 94395 +2, 94399 +3, 94407 +4, 94411 +5, 94399 +6, 94395 +7, 94399 +0, 95007 +1, 95015 +2, 94969 +3, 94999 +4, 95003 +5, 94975 +6, 94985 +7, 94977 +0, 95965 +1, 95947 +2, 95759 +3, 95959 +4, 95867 +5, 95951 +6, 95755 +7, 95951 +0, 96663 +1, 96621 +2, 96465 +3, 96637 +4, 96539 +5, 96625 +6, 96425 +7, 96633 +0, 126025 +1, 126025 +2, 125839 +3, 126019 +4, 125965 +5, 126031 +6, 125795 +7, 126031 +0, 126025 +1, 126025 +2, 125839 +3, 126019 +4, 125965 +5, 126031 +6, 125795 +7, 126031 +0, 126025 +1, 126025 +2, 125839 +3, 126019 +4, 125965 +5, 126031 +6, 125795 +7, 126031 +0, 126025 +1, 126025 +2, 125839 +3, 126019 +4, 125965 +5, 126031 +6, 125795 +7, 126031 +0, 126025 +1, 126025 +2, 125839 +3, 126019 +4, 125965 +5, 126031 +6, 125795 +7, 126031 +0, 126025 +1, 126025 +2, 125839 +3, 126019 +4, 125965 +5, 126031 +6, 125795 +7, 126031 +0, 906 +1, 1162 +2, 125023 +3, 2442 +4, 125149 +5, 2442 +6, 124979 +7, 2186 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/run.env b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..57fcd02313ad29f1ec401b552f6c1e30762cb6a6 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c90e01150436c295a3078d5790232724b0c67731 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/train.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f59d8f2658c1bf9973fd1c6a8738fea92ed503364de4076a4c3ab080dda98768 +size 13020943 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..72df7adaa61f7b0fdde9889c6133372248bf8cc8 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv @@ -0,0 +1,272 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 11795 +1, 11815 +2, 11819 +3, 11805 +4, 11803 +5, 11819 +6, 11813 +7, 18257 +0, 44891 +1, 44855 +2, 44895 +3, 44813 +4, 44837 +5, 44819 +6, 44849 +7, 44859 +0, 45895 +1, 45897 +2, 45907 +3, 45869 +4, 45893 +5, 45907 +6, 45897 +7, 45907 +0, 46917 +1, 46919 +2, 46929 +3, 46891 +4, 46915 +5, 46929 +6, 46919 +7, 46929 +0, 47013 +1, 47015 +2, 47025 +3, 46987 +4, 47011 +5, 47025 +6, 47015 +7, 47025 +0, 47153 +1, 47151 +2, 47155 +3, 47119 +4, 47171 +5, 47155 +6, 47163 +7, 47155 +0, 47877 +1, 47879 +2, 47889 +3, 47851 +4, 47875 +5, 47889 +6, 47879 +7, 47889 +0, 48037 +1, 48039 +2, 48049 +3, 48011 +4, 48035 +5, 48049 +6, 48039 +7, 48049 +0, 48513 +1, 48515 +2, 48529 +3, 48487 +4, 48511 +5, 48529 +6, 48475 +7, 48529 +0, 48737 +1, 48739 +2, 48749 +3, 48711 +4, 48735 +5, 48749 +6, 48699 +7, 48749 +0, 92891 +1, 92893 +2, 92897 +3, 92905 +4, 92889 +5, 92897 +6, 92893 +7, 92897 +0, 94391 +1, 94393 +2, 94397 +3, 94405 +4, 94389 +5, 94397 +6, 94393 +7, 94397 +0, 94391 +1, 94393 +2, 94397 +3, 94405 +4, 94389 +5, 94397 +6, 94393 +7, 94397 +0, 94393 +1, 94395 +2, 94399 +3, 94407 +4, 94391 +5, 94399 +6, 94395 +7, 94399 +0, 94393 +1, 94395 +2, 94399 +3, 94407 +4, 94391 +5, 94399 +6, 94395 +7, 94399 +0, 95031 +1, 95031 +2, 94977 +3, 95001 +4, 94983 +5, 94985 +6, 94977 +7, 94997 +0, 96041 +1, 95947 +2, 95759 +3, 96055 +4, 95943 +5, 95759 +6, 95755 +7, 96047 +0, 96685 +1, 96653 +2, 96427 +3, 96719 +4, 96597 +5, 96417 +6, 96419 +7, 96723 +0, 126121 +1, 126025 +2, 125859 +3, 126115 +4, 126061 +5, 125859 +6, 125815 +7, 126147 +0, 126121 +1, 126025 +2, 125859 +3, 126115 +4, 126061 +5, 125859 +6, 125815 +7, 126147 +0, 126121 +1, 126025 +2, 125859 +3, 126115 +4, 126061 +5, 125859 +6, 125815 +7, 126147 +0, 126121 +1, 126025 +2, 125859 +3, 126115 +4, 126061 +5, 125859 +6, 125815 +7, 126147 +0, 126121 +1, 126025 +2, 125859 +3, 126115 +4, 126061 +5, 125859 +6, 125815 +7, 126147 +0, 126121 +1, 126025 +2, 125859 +3, 126115 +4, 126061 +5, 125859 +6, 125815 +7, 126147 +0, 125305 +1, 125209 +2, 125043 +3, 2186 +4, 1162 +5, 125043 +6, 124999 +7, 125331 +0, 125302 +1, 2183 +2, 903 +3, 903 +4, 903 +5, 903 +6, 905 +7, 1162 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..de9f4bbea87ba4f3a95cb0ba4790144960de3fc9 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e141dcb448a92622989b305c573d4a5bdac57e5d --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dd6f58e7eee63ad96bb2cf53ad35b2b7bae74f4e30d7d440a04ee800647c5f1 +size 15159073 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv index fe75e13d7a8f5b339a6b7c5ad68888176c17544a..dabc234f6ae1341b820bae3029b65e302db9fb60 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 2 1, 1 2, 1 3, 1 @@ -39,21 +39,21 @@ 6, 4 7, 4 0, 4 -1, 629 -2, 4 +1, 4 +2, 27 3, 4 -4, 27 +4, 4 5, 4 -6, 4 -7, 4 -0, 821 -1, 821 -2, 821 -3, 821 -4, 821 -5, 821 -6, 821 -7, 821 +6, 41 +7, 629 +0, 895 +1, 903 +2, 917 +3, 929 +4, 925 +5, 899 +6, 915 +7, 913 0, 1831 1, 1831 2, 1831 @@ -70,22 +70,22 @@ 5, 39197 6, 38995 7, 39197 -0, 44953 -1, 44991 -2, 44965 -3, 44963 -4, 44951 -5, 44965 -6, 44991 -7, 45001 -0, 45895 -1, 45897 -2, 45907 -3, 45869 -4, 45893 -5, 45907 -6, 45897 -7, 45907 +0, 44847 +1, 44875 +2, 44877 +3, 44859 +4, 44821 +5, 44837 +6, 44841 +7, 44875 +0, 45791 +1, 45839 +2, 45845 +3, 45775 +4, 45789 +5, 45793 +6, 45835 +7, 45855 0, 46917 1, 46919 2, 46929 @@ -102,14 +102,14 @@ 5, 47025 6, 47015 7, 47025 -0, 47275 -1, 47265 -2, 47281 -3, 47245 -4, 47255 -5, 47251 -6, 47245 -7, 47281 +0, 47191 +1, 47219 +2, 47209 +3, 47163 +4, 47213 +5, 47213 +6, 47191 +7, 47211 0, 47877 1, 47879 2, 47889 @@ -142,14 +142,22 @@ 5, 49145 6, 49117 7, 49145 -0, 100829 -1, 100835 -2, 100849 -3, 100875 -4, 100853 -5, 100861 -6, 100833 -7, 100861 +0, 100699 +1, 100655 +2, 100703 +3, 100647 +4, 100715 +5, 100639 +6, 100639 +7, 100703 +0, 101347 +1, 101369 +2, 101353 +3, 101361 +4, 101365 +5, 101353 +6, 101353 +7, 101353 0, 101347 1, 101369 2, 101353 @@ -174,99 +182,91 @@ 5, 101355 6, 101355 7, 101355 -0, 101349 -1, 101371 -2, 101355 -3, 101363 -4, 101367 -5, 101355 -6, 101355 -7, 101355 -0, 102061 -1, 102851 -2, 102835 -3, 102075 -4, 102847 -5, 102835 -6, 102355 -7, 102067 -0, 102805 -1, 103499 -2, 103483 -3, 102915 -4, 103495 -5, 103483 -6, 103195 +0, 101957 +1, 101953 +2, 101945 +3, 101963 +4, 101903 +5, 101965 +6, 101923 +7, 101951 +0, 102901 +1, 102731 +2, 102907 +3, 102723 +4, 102727 +5, 102907 +6, 102811 7, 102907 -0, 103503 -1, 104195 -2, 104173 -3, 103585 -4, 104153 -5, 104183 -6, 103863 -7, 103593 -0, 132853 -1, 133533 -2, 133539 -3, 132951 -4, 133569 -5, 133539 -6, 133231 +0, 103543 +1, 103383 +2, 103557 +3, 103389 +4, 103383 +5, 103559 +6, 103459 +7, 103559 +0, 132949 +1, 132765 +2, 132963 +3, 132759 +4, 132801 +5, 132963 +6, 132847 7, 132963 -0, 132853 -1, 133533 -2, 133539 -3, 132951 -4, 133569 -5, 133539 -6, 133231 +0, 132949 +1, 132765 +2, 132963 +3, 132759 +4, 132801 +5, 132963 +6, 132847 7, 132963 -0, 132853 -1, 133533 -2, 133539 -3, 132951 -4, 133569 -5, 133539 -6, 133231 +0, 132949 +1, 132765 +2, 132963 +3, 132759 +4, 132801 +5, 132963 +6, 132847 7, 132963 -0, 132853 -1, 133533 -2, 133539 -3, 132951 -4, 133569 -5, 133539 -6, 133231 +0, 132949 +1, 132765 +2, 132963 +3, 132759 +4, 132801 +5, 132963 +6, 132847 7, 132963 -0, 132853 -1, 133533 -2, 133539 -3, 132951 -4, 133569 -5, 133539 -6, 133231 +0, 132949 +1, 132765 +2, 132963 +3, 132759 +4, 132801 +5, 132963 +6, 132847 7, 132963 -0, 132853 -1, 133533 -2, 133539 -3, 132951 -4, 133569 -5, 133539 -6, 133231 +0, 132949 +1, 132765 +2, 132963 +3, 132759 +4, 132801 +5, 132963 +6, 132847 7, 132963 -0, 1162 -1, 132717 -2, 132723 -3, 132135 +0, 132133 +1, 131949 +2, 2442 +3, 131943 4, 2442 -5, 132723 -6, 132415 -7, 2186 -0, 0 -1, 0 -2, 0 -3, 0 -4, 0 -5, 0 -6, 0 -7, 0 +5, 132147 +6, 132031 +7, 132147 +0, 132130 +1, 2183 +2, 903 +3, 903 +4, 903 +5, 903 +6, 905 +7, 1162 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log index 80dcf08915d661c26534bf3482fc2f11b74afe1b..32186913656317ed4ac22eb73bf0059c72813698 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:df595e47799be5bbf50d80509f6ea6b2cba57e19228e6f7973711658afe74dc1 -size 13026229 +oid sha256:01c7900ca48884e0bc2e1036053aa7a996f875bd01f2f8370ebc63610dc4f90b +size 13023349 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index b24bcaae77fc7774c1a221cb529f8f60ce913c40..6cdb4c2470e72fbf698d933fb1f895c9ade0aa2a 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -1,2 +1,8 @@ model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir -QWEN3_235B echo natural 2.00 OK 141943 1 0.0 1.02734375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 +QWEN3_235B echo measure 2.00 OK 141943 1 0.0 1.029296875 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 +QWEN3_235B echo validate 1.25 OK 134713 1 0.0 1.03125 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1 +QWEN3_235B echo forced 1.25 OK 134699 1 0.0 1.025390625 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1 +QWEN3_30B echo measure 2.00 OK 132963 1 0.0 1.05859375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN3_30B echo validate 1.25 OK 126031 1 0.0 1.05859375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1 +QWEN3_30B echo forced 1.25 OK 126147 1 0.0 1.0546875 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1 +DSV2_LITE_L24 echo measure 2.00 OK 140961 1 0.0 1.0234375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index 39d3ade27d5b209764c83daa5db58c8c54a0b215..1d8825870a4aed495017ae223a15bff09b3a34b8 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -1 +1,3 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir +QWEN3_235B echo SELECTED 1.25 1.03125 0.0 134713 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1 +QWEN3_30B echo SELECTED 1.25 1.05859375 0.0 126031 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv index cd78bed429e56840a765b05910a0b79d8efd9e9b..03cb95e563a1f56cbd76c851e0c60c0b3afd6ca5 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv @@ -1,2 +1,8 @@ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir -QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 447.80 439.80 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 +QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 439.09 433.90 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 +QWEN3_235B echo matched 16 8192 8 1.25 none 1 OK 443.50 437.00 134713 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1 +QWEN3_235B echo matched 16 8192 8 1.25 none+forcebal 1 OK 415.86 413.80 134699 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1 +QWEN3_30B echo matched 16 4096 8 2.00 none 1 OK 1002.86 951.60 132963 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN3_30B echo matched 16 4096 8 1.25 none 1 OK 1007.40 966.70 126031 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1 +QWEN3_30B echo matched 16 4096 8 1.25 none+forcebal 1 OK 1020.94 952.60 126147 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1 +DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 1 OK 817.63 810.40 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 diff --git a/tierb_study_20260826_144644/hf_watcher.log b/tierb_study_20260826_144644/hf_watcher.log index d5e5ce9dc7ef5acb3d44a78175f9d393d90506e7..547a71e25f3c281a2344cd360adc8f873e7108ed 100644 --- a/tierb_study_20260826_144644/hf_watcher.log +++ b/tierb_study_20260826_144644/hf_watcher.log @@ -36,3 +36,6 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main Already logged in to Hugging Face as: AbhishekGhosh1998 Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop) [2026-08-26T16:25:52+00:00] backup pass + uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644 +https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main +[2026-08-26T16:35:54+00:00] backup pass diff --git a/tierb_study_20260826_144644/study.log b/tierb_study_20260826_144644/study.log index f03232187232814219bd071d0937382572c1ee62..fc056e3cfbb399248ed75b899467a702bd96cf90 100644 --- a/tierb_study_20260826_144644/study.log +++ b/tierb_study_20260826_144644/study.log @@ -910,3 +910,46 @@ driver_repo: /workspace/blackwell_run_scripts @ 6e4b1c8 === STAGE 7 -- Tier B probe and zero-drop capacity selection === STEP echo__tierbprobe__gbs8 +============================================================== + CAMPAIGN: echo backend=echo phase=tierbprobe + Megatron: /workspace/Megatron-MoE-Echo @ token-drop-analysis + GBS=8 seq=4096 repeats=3 nsys=1 + results -> /workspace/tierb_study_20260826_144644/echo/tierbprobe +============================================================== +-------------------------------------------------------------------------- +[16:26:07] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00 (slots=16) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +[16:27:23] QWEN3_235B echo: measured required=1.029296875 at C=2.00 -> validating C=1.25 +-------------------------------------------------------------------------- +[16:27:23] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25 (slots=16) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[16:28:40] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal (slots=16) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[16:29:56] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00 (slots=16) + [guard] iter 5/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +[16:31:22] QWEN3_30B echo: measured required=1.05859375 at C=2.00 -> validating C=1.25 +-------------------------------------------------------------------------- +[16:31:22] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25 (slots=16) + [guard] iter 5/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[16:32:47] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal (slots=16) + [guard] iter 4/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[16:34:13] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00 (slots=8) + [guard] iter 6/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +[16:35:35] DSV2_LITE_L24 echo: measured required=1.0234375 at C=2.00 -> validating C=1.25 +-------------------------------------------------------------------------- +[16:35:35] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25 (slots=8)